mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-06-01 20:32:11 +00:00
Rewrite queue (#24505)
# ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: 
This commit is contained in:
wxiaoguang
GitHub
parent
cb700aedd1
commit
6f9c278559
100 changed files with 2496 additions and 6858 deletions
|
|
@ -49,7 +49,6 @@ type SearchResult struct {
|
|||
type Indexer interface {
|
||||
Init() (bool, error)
|
||||
Ping() bool
|
||||
SetAvailabilityChangeCallback(callback func(bool))
|
||||
Index(issue []*IndexerData) error
|
||||
Delete(ids ...int64) error
|
||||
Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
|
||||
|
|
@ -94,7 +93,7 @@ func (h *indexerHolder) get() Indexer {
|
|||
|
||||
var (
|
||||
// issueIndexerQueue queue of issue ids to be updated
|
||||
issueIndexerQueue queue.Queue
|
||||
issueIndexerQueue *queue.WorkerPoolQueue[*IndexerData]
|
||||
holder = newIndexerHolder()
|
||||
)
|
||||
|
||||
|
|
@ -108,62 +107,44 @@ func InitIssueIndexer(syncReindex bool) {
|
|||
// Create the Queue
|
||||
switch setting.Indexer.IssueType {
|
||||
case "bleve", "elasticsearch", "meilisearch":
|
||||
handler := func(data ...queue.Data) []queue.Data {
|
||||
handler := func(items ...*IndexerData) (unhandled []*IndexerData) {
|
||||
indexer := holder.get()
|
||||
if indexer == nil {
|
||||
log.Error("Issue indexer handler: unable to get indexer!")
|
||||
return data
|
||||
log.Error("Issue indexer handler: unable to get indexer.")
|
||||
return items
|
||||
}
|
||||
|
||||
iData := make([]*IndexerData, 0, len(data))
|
||||
unhandled := make([]queue.Data, 0, len(data))
|
||||
for _, datum := range data {
|
||||
indexerData, ok := datum.(*IndexerData)
|
||||
if !ok {
|
||||
log.Error("Unable to process provided datum: %v - not possible to cast to IndexerData", datum)
|
||||
continue
|
||||
}
|
||||
toIndex := make([]*IndexerData, 0, len(items))
|
||||
for _, indexerData := range items {
|
||||
log.Trace("IndexerData Process: %d %v %t", indexerData.ID, indexerData.IDs, indexerData.IsDelete)
|
||||
if indexerData.IsDelete {
|
||||
if err := indexer.Delete(indexerData.IDs...); err != nil {
|
||||
log.Error("Error whilst deleting from index: %v Error: %v", indexerData.IDs, err)
|
||||
if indexer.Ping() {
|
||||
continue
|
||||
log.Error("Issue indexer handler: failed to from index: %v Error: %v", indexerData.IDs, err)
|
||||
if !indexer.Ping() {
|
||||
log.Error("Issue indexer handler: indexer is unavailable when deleting")
|
||||
unhandled = append(unhandled, indexerData)
|
||||
}
|
||||
// Add back to queue
|
||||
unhandled = append(unhandled, datum)
|
||||
}
|
||||
continue
|
||||
}
|
||||
iData = append(iData, indexerData)
|
||||
toIndex = append(toIndex, indexerData)
|
||||
}
|
||||
if len(unhandled) > 0 {
|
||||
for _, indexerData := range iData {
|
||||
unhandled = append(unhandled, indexerData)
|
||||
if err := indexer.Index(toIndex); err != nil {
|
||||
log.Error("Error whilst indexing: %v Error: %v", toIndex, err)
|
||||
if !indexer.Ping() {
|
||||
log.Error("Issue indexer handler: indexer is unavailable when indexing")
|
||||
unhandled = append(unhandled, toIndex...)
|
||||
}
|
||||
return unhandled
|
||||
}
|
||||
if err := indexer.Index(iData); err != nil {
|
||||
log.Error("Error whilst indexing: %v Error: %v", iData, err)
|
||||
if indexer.Ping() {
|
||||
return nil
|
||||
}
|
||||
// Add back to queue
|
||||
for _, indexerData := range iData {
|
||||
unhandled = append(unhandled, indexerData)
|
||||
}
|
||||
return unhandled
|
||||
}
|
||||
return nil
|
||||
return unhandled
|
||||
}
|
||||
|
||||
issueIndexerQueue = queue.CreateQueue("issue_indexer", handler, &IndexerData{})
|
||||
issueIndexerQueue = queue.CreateSimpleQueue("issue_indexer", handler)
|
||||
|
||||
if issueIndexerQueue == nil {
|
||||
log.Fatal("Unable to create issue indexer queue")
|
||||
}
|
||||
default:
|
||||
issueIndexerQueue = &queue.DummyQueue{}
|
||||
issueIndexerQueue = queue.CreateSimpleQueue[*IndexerData]("issue_indexer", nil)
|
||||
}
|
||||
|
||||
// Create the Indexer
|
||||
|
|
@ -240,18 +221,6 @@ func InitIssueIndexer(syncReindex bool) {
|
|||
log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType)
|
||||
}
|
||||
|
||||
if queue, ok := issueIndexerQueue.(queue.Pausable); ok {
|
||||
holder.get().SetAvailabilityChangeCallback(func(available bool) {
|
||||
if !available {
|
||||
log.Info("Issue index queue paused")
|
||||
queue.Pause()
|
||||
} else {
|
||||
log.Info("Issue index queue resumed")
|
||||
queue.Resume()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Start processing the queue
|
||||
go graceful.GetManager().RunWithShutdownFns(issueIndexerQueue.Run)
|
||||
|
||||
|
|
@ -285,9 +254,7 @@ func InitIssueIndexer(syncReindex bool) {
|
|||
case <-graceful.GetManager().IsShutdown():
|
||||
log.Warn("Shutdown occurred before issue index initialisation was complete")
|
||||
case <-time.After(timeout):
|
||||
if shutdownable, ok := issueIndexerQueue.(queue.Shutdownable); ok {
|
||||
shutdownable.Terminate()
|
||||
}
|
||||
issueIndexerQueue.ShutdownWait(5 * time.Second)
|
||||
log.Fatal("Issue Indexer Initialization timed-out after: %v", timeout)
|
||||
}
|
||||
}()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue