task调度器该用ants线程池

TODO-List

@@ -10,3 +10,4 @@
 3. ListenHandler 的实现遇到问题只能panic, 没有处理错误
 4. 暂时不考虑和区域主控间的同步消息, 假设所有消息都是异步的, 这可能导致无法知道指令是否执行成功
 5. 如果系统停机时间很长, 待执行任务表中的任务过期了怎么办, 目前没有任务过期机制
+6. Task不支持插队

go.mod

@@ -5,21 +5,19 @@ go 1.25
 require (
 	github.com/gin-gonic/gin v1.10.1
 	github.com/go-openapi/errors v0.22.2
-	github.com/go-openapi/loads v0.22.0
 	github.com/go-openapi/runtime v0.28.0
-	github.com/go-openapi/spec v0.21.0
 	github.com/go-openapi/strfmt v0.23.0
 	github.com/go-openapi/swag v0.23.0
 	github.com/go-openapi/validate v0.24.0
 	github.com/golang-jwt/jwt/v5 v5.3.0
-	github.com/jessevdk/go-flags v1.6.1
+	github.com/panjf2000/ants/v2 v2.11.3
 	github.com/stretchr/testify v1.11.1
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.1
 	github.com/swaggo/swag v1.16.6
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.36.0
-	golang.org/x/net v0.38.0
+	google.golang.org/protobuf v1.34.1
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v2 v2.4.0
 	gorm.io/datatypes v1.2.6
@@ -37,12 +35,15 @@ require (
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/go-units v0.5.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/analysis v0.23.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/loads v0.22.0 // indirect
+	github.com/go-openapi/spec v0.21.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.20.0 // indirect
@@ -66,6 +67,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
@@ -73,14 +75,17 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	go.mongodb.org/mongo-driver v1.14.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	golang.org/x/arch v0.8.0 // indirect
 	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/net v0.38.0 // indirect
 	golang.org/x/sync v0.12.0 // indirect
 	golang.org/x/sys v0.31.0 // indirect
 	golang.org/x/text v0.23.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/mysql v1.5.6 // indirect
 )

go.sum

@@ -15,8 +15,6 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
-github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
 github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4=
@@ -25,6 +23,11 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
 github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU=
 github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo=
 github.com/go-openapi/errors v0.22.2 h1:rdxhzcBUazEcGccKqbY1Y7NS8FDcMyIRr0934jrYnZg=
@@ -77,8 +80,6 @@ github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
 github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
-github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4=
-github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -114,6 +115,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
+github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -147,6 +152,14 @@ github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=
 go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
+go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
+go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
+go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
+go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
+go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
+go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
+go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
+go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=

internal/infra/task/task.go

@@ -8,6 +8,7 @@ import (
 
 	"git.huangwc.com/pig/pig-farm-controller/internal/infra/models"
 	"git.huangwc.com/pig/pig-farm-controller/internal/infra/repository"
+	"github.com/panjf2000/ants/v2"
 	"gorm.io/gorm"
 )
 
@@ -30,36 +31,6 @@ func NewProgressTracker() *ProgressTracker {
 	}
 }
 
-// StartTracking 开始跟踪一个新的计划执行
-func (t *ProgressTracker) StartTracking(planLogID uint, total int) {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	t.totalTasks[planLogID] = total
-	t.completedTasks[planLogID] = 0
-}
-
-// Increment 将指定计划的完成计数加一
-func (t *ProgressTracker) Increment(planLogID uint) {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	t.completedTasks[planLogID]++
-}
-
-// IsComplete 检查指定计划是否已完成所有任务
-func (t *ProgressTracker) IsComplete(planLogID uint) bool {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	return t.completedTasks[planLogID] >= t.totalTasks[planLogID]
-}
-
-// StopTracking 停止跟踪一个计划，清理内存
-func (t *ProgressTracker) StopTracking(planLogID uint) {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	delete(t.totalTasks, planLogID)
-	delete(t.completedTasks, planLogID)
-}
-
 // Scheduler 是核心的、持久化的任务调度器
 type Scheduler struct {
 	logger          Logger
@@ -68,10 +39,10 @@ type Scheduler struct {
 	pendingTaskRepo repository.PendingTaskRepository
 	progressTracker *ProgressTracker
 
-	taskChannel chan *models.TaskExecutionLog // 用于向 workers 派发任务的缓冲 channel
+	pool   *ants.Pool // 使用 ants 协程池来管理并发
-	wg          sync.WaitGroup
+	wg     sync.WaitGroup
-	ctx         context.Context
+	ctx    context.Context
-	cancel      context.CancelFunc
+	cancel context.CancelFunc
 }
 
 // NewScheduler 创建一个新的调度器实例
@@ -83,21 +54,23 @@ func NewScheduler(pendingTaskRepo repository.PendingTaskRepository, logger Logge
 		pollingInterval: interval,
 		workers:         numWorkers,
 		progressTracker: NewProgressTracker(),
-		taskChannel:     make(chan *models.TaskExecutionLog, numWorkers), // 缓冲大小与 worker 数量一致
 		ctx:             ctx,
 		cancel:          cancel,
 	}
 }
 
-// Start 启动调度器，包括主轮询循环和所有工作协程
+// Start 启动调度器，包括初始化协程池和启动主轮询循环
 func (s *Scheduler) Start() {
 	s.logger.Printf("任务调度器正在启动，工作协程数: %d...", s.workers)
 
-	// 启动工作协程池
+	// 初始化 ants 协程池
-	s.wg.Add(s.workers)
+	pool, err := ants.NewPool(s.workers, ants.WithPanicHandler(func(err interface{}) {
-	for i := 0; i < s.workers; i++ {
+		s.logger.Printf("[严重] 任务执行时发生 panic: %v", err)
-		go s.worker(i)
+	}))
+	if err != nil {
+		panic("初始化协程池失败: " + err.Error())
 	}
+	s.pool = pool
 
 	// 启动主轮询循环
 	s.wg.Add(1)
@@ -106,15 +79,16 @@ func (s *Scheduler) Start() {
 	s.logger.Printf("任务调度器已成功启动")
 }
 
-// Stop 优雅地停止调度器和所有工作协程
+// Stop 优雅地停止调度器
 func (s *Scheduler) Stop() {
 	s.logger.Printf("正在停止任务调度器...")
-	s.cancel()  // 发出取消信号
+	s.cancel()       // 1. 发出取消信号，停止主循环
-	s.wg.Wait() // 等待所有协程完成
+	s.wg.Wait()      // 2. 等待主循环完成
+	s.pool.Release() // 3. 释放 ants 池 (等待所有已提交的任务执行完毕)
 	s.logger.Printf("任务调度器已安全停止")
 }
 
-// run 是主轮询循环，负责从数据库认领任务并派发
+// run 是主轮询循环，负责从数据库认领任务并提交到协程池
 func (s *Scheduler) run() {
 	defer s.wg.Done()
 	ticker := time.NewTicker(s.pollingInterval)
@@ -123,16 +97,22 @@ func (s *Scheduler) run() {
 	for {
 		select {
 		case <-s.ctx.Done():
-			close(s.taskChannel) // 关闭 channel，让 workers 退出循环
 			return
 		case <-ticker.C:
-			s.claimAndDispatch()
+			s.claimAndSubmit()
 		}
 	}
 }
 
-// claimAndDispatch 认领一个任务并将其发送到派发通道
+// claimAndSubmit 认领一个任务并将其提交到 ants 协程池
-func (s *Scheduler) claimAndDispatch() {
+func (s *Scheduler) claimAndSubmit() {
+	// ants 池的 Running() 数量可以用来提前判断是否繁忙，但这只是一个快照，
+	// 真正的阻塞和背压由 Submit() 方法保证。
+	if s.pool.Running() >= s.workers {
+		// 可选：如果所有 worker 都在忙，可以跳过本次数据库查询，以减轻数据库压力
+		return
+	}
+
 	claimedLog, err := s.pendingTaskRepo.ClaimNextDueTask()
 	if err != nil {
 		if !errors.Is(err, gorm.ErrRecordNotFound) {
@@ -141,39 +121,29 @@ func (s *Scheduler) claimAndDispatch() {
 		return
 	}
 
-	// 将认领到的任务发送到派发通道
+	// 将任务处理逻辑作为一个函数提交给 ants 池。
-	// 如果所有 worker 都在忙，这里会阻塞，从而实现背压，防止队列无限增长
+	// 如果池已满，Submit 方法会阻塞，直到有协程空闲出来，这自然地实现了背压。
-	select {
+	err = s.pool.Submit(func() {
-	case s.taskChannel <- claimedLog:
+		s.processTask(claimedLog)
-		s.logger.Printf("成功认领并派发任务, 日志ID: %d, 任务ID: %d", claimedLog.ID, claimedLog.TaskID)
+	})
-	case <-s.ctx.Done():
+	if err != nil {
-		// 如果在等待派发时调度器被停止，需要处理这个未派发的任务
+		// 如果在调度器停止期间提交任务，可能会发生此错误
-		// 简单的处理方式是忽略它，让清理器进程后续来处理这个 'running' 状态的任务
+		s.logger.Printf("向协程池提交任务失败: %v", err)
-		s.logger.Printf("在派发任务时调度器被停止, 日志ID: %d", claimedLog.ID)
+		// 可以在这里添加逻辑，将任务状态恢复为 pending
 	}
 }
 
-// worker 是工作协程的实现
-func (s *Scheduler) worker(id int) {
-	defer s.wg.Done()
-	s.logger.Printf("工作协程 #%d 已启动", id)
-	for claimedLog := range s.taskChannel {
-		s.processTask(id, claimedLog)
-	}
-	s.logger.Printf("工作协程 #%d 已停止", id)
-}
-
 // processTask 包含了处理单个任务的完整逻辑
-func (s *Scheduler) processTask(workerID int, claimedLog *models.TaskExecutionLog) {
+func (s *Scheduler) processTask(claimedLog *models.TaskExecutionLog) {
-	s.logger.Printf("工作协程 #%d 正在处理任务, 日志ID: %d, 任务ID: %d, 任务名称: %s",
+	s.logger.Printf("开始处理任务, 日志ID: %d, 任务ID: %d, 任务名称: %s",
-		workerID, claimedLog.ID, claimedLog.TaskID, claimedLog.Task.Name)
+		claimedLog.ID, claimedLog.TaskID, claimedLog.Task.Name)
 
 	// 在这里，我们将根据 claimedLog.TaskID 或未来的 Task.Kind 来分发给不同的处理器
 	// 现在，我们只做一个模拟执行
 	time.Sleep(2 * time.Second) // 模拟任务执行耗时
 
 	// 任务执行完毕后，更新日志和进度
-	s.logger.Printf("工作协程 #%d 已完成任务, 日志ID: %d", workerID, claimedLog.ID)
+	s.logger.Printf("完成任务, 日志ID: %d", claimedLog.ID)
 
 	// ----------------------------------------------------
 	// 未来的逻辑将在这里展开：
@@ -186,3 +156,30 @@ func (s *Scheduler) processTask(workerID int, claimedLog *models.TaskExecutionLo
 	//
 	// ----------------------------------------------------
 }
+
+// ProgressTracker 的方法实现
+func (t *ProgressTracker) StartTracking(planLogID uint, total int) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.totalTasks[planLogID] = total
+	t.completedTasks[planLogID] = 0
+}
+
+func (t *ProgressTracker) Increment(planLogID uint) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	t.completedTasks[planLogID]++
+}
+
+func (t *ProgressTracker) IsComplete(planLogID uint) bool {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	return t.completedTasks[planLogID] >= t.totalTasks[planLogID]
+}
+
+func (t *ProgressTracker) StopTracking(planLogID uint) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	delete(t.totalTasks, planLogID)
+	delete(t.completedTasks, planLogID)
+}