mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-06-20 16:10:50 +00:00
Graceful Queues: Issue Indexing and Tasks (#9363)
* Queue: Add generic graceful queues with settings * Queue & Setting: Add worker pool implementation * Queue: Add worker settings * Queue: Make resizing worker pools * Queue: Add name variable to queues * Queue: Add monitoring * Queue: Improve logging * Issues: Gracefulise the issues indexer Remove the old now unused specific queues * Task: Move to generic queue and gracefulise * Issues: Standardise the issues indexer queue settings * Fix test * Queue: Allow Redis to connect to unix * Prevent deadlock during early shutdown of issue indexer * Add MaxWorker settings to queues * Merge branch 'master' into graceful-queues * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/indexer/issues/indexer.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update modules/queue/queue_disk.go * Update modules/queue/queue_disk_channel.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Rename queue.Description to queue.ManagedQueue as per @guillep2k * Cancel pool workers when removed * Remove dependency on queue from setting * Update modules/queue/queue_redis.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * As per @guillep2k add mutex locks on shutdown/terminate * move unlocking out of setInternal * Add warning if number of workers < 0 * Small changes as per @guillep2k * No redis host specified not found * Clean up documentation for queues * Update docs/content/doc/advanced/config-cheat-sheet.en-us.md * Update modules/indexer/issues/indexer_test.go * Ensure that persistable channel queue is added to manager * Rename QUEUE_NAME REDIS_QUEUE_NAME * Revert "Rename QUEUE_NAME REDIS_QUEUE_NAME" This reverts commit 1f83b4fc9b9dabda186257b38c265fe7012f90df. Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: techknowlogick <matti@mdranta.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
parent
f71e1c8e79
commit
62eb1b0f25
35 changed files with 2875 additions and 647 deletions
|
@ -8,14 +8,15 @@ import (
|
|||
"fmt"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/migrations/base"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/queue"
|
||||
"code.gitea.io/gitea/modules/structs"
|
||||
)
|
||||
|
||||
// taskQueue is a global queue of tasks
|
||||
var taskQueue Queue
|
||||
var taskQueue queue.Queue
|
||||
|
||||
// Run a task
|
||||
func Run(t *models.Task) error {
|
||||
|
@ -23,38 +24,32 @@ func Run(t *models.Task) error {
|
|||
case structs.TaskTypeMigrateRepo:
|
||||
return runMigrateTask(t)
|
||||
default:
|
||||
return fmt.Errorf("Unknow task type: %d", t.Type)
|
||||
return fmt.Errorf("Unknown task type: %d", t.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// Init will start the service to get all unfinished tasks and run them
|
||||
func Init() error {
|
||||
switch setting.Task.QueueType {
|
||||
case setting.ChannelQueueType:
|
||||
taskQueue = NewChannelQueue(setting.Task.QueueLength)
|
||||
case setting.RedisQueueType:
|
||||
var err error
|
||||
addrs, pass, idx, err := parseConnStr(setting.Task.QueueConnStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
taskQueue, err = NewRedisQueue(addrs, pass, idx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("Unsupported task queue type: %v", setting.Task.QueueType)
|
||||
taskQueue = queue.CreateQueue("task", handle, &models.Task{})
|
||||
|
||||
if taskQueue == nil {
|
||||
return fmt.Errorf("Unable to create Task Queue")
|
||||
}
|
||||
|
||||
go func() {
|
||||
if err := taskQueue.Run(); err != nil {
|
||||
log.Error("taskQueue.Run end failed: %v", err)
|
||||
}
|
||||
}()
|
||||
go graceful.GetManager().RunWithShutdownFns(taskQueue.Run)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func handle(data ...queue.Data) {
|
||||
for _, datum := range data {
|
||||
task := datum.(*models.Task)
|
||||
if err := Run(task); err != nil {
|
||||
log.Error("Run task failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MigrateRepository add migration repository to task
|
||||
func MigrateRepository(doer, u *models.User, opts base.MigrateOptions) error {
|
||||
task, err := models.CreateMigrateTask(doer, u, opts)
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue