refactor issue indexer, add some testing and fix a bug (#6131)

* refactor issue indexer, add some testing and fix a bug

* fix error copyright year on comment header

* issues indexer package import keep consistent

modules/indexer/issues/indexer.go

@@ -4,6 +4,15 @@
 
 package issues
 
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+)
+
 // IndexerData data stored in the issue indexer
 type IndexerData struct {
 	ID       int64
@@ -34,3 +43,142 @@ type Indexer interface {
 	Delete(ids ...int64) error
 	Search(kw string, repoID int64, limit, start int) (*SearchResult, error)
 }
+
+var (
+	// issueIndexerUpdateQueue queue of issue ids to be updated
+	issueIndexerUpdateQueue Queue
+	issueIndexer            Indexer
+)
+
+// InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until
+// all issue index done.
+func InitIssueIndexer(syncReindex bool) error {
+	var populate bool
+	switch setting.Indexer.IssueType {
+	case "bleve":
+		issueIndexer = NewBleveIndexer(setting.Indexer.IssuePath)
+		exist, err := issueIndexer.Init()
+		if err != nil {
+			return err
+		}
+		populate = !exist
+	default:
+		return fmt.Errorf("unknow issue indexer type: %s", setting.Indexer.IssueType)
+	}
+
+	var err error
+	switch setting.Indexer.IssueIndexerQueueType {
+	case setting.LevelQueueType:
+		issueIndexerUpdateQueue, err = NewLevelQueue(
+			issueIndexer,
+			setting.Indexer.IssueIndexerQueueDir,
+			setting.Indexer.IssueIndexerQueueBatchNumber)
+		if err != nil {
+			return err
+		}
+	case setting.ChannelQueueType:
+		issueIndexerUpdateQueue = NewChannelQueue(issueIndexer, setting.Indexer.IssueIndexerQueueBatchNumber)
+	default:
+		return fmt.Errorf("Unsupported indexer queue type: %v", setting.Indexer.IssueIndexerQueueType)
+	}
+
+	go issueIndexerUpdateQueue.Run()
+
+	if populate {
+		if syncReindex {
+			populateIssueIndexer()
+		} else {
+			go populateIssueIndexer()
+		}
+	}
+
+	return nil
+}
+
+// populateIssueIndexer populate the issue indexer with issue data
+func populateIssueIndexer() {
+	for page := 1; ; page++ {
+		repos, _, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
+			Page:        page,
+			PageSize:    models.RepositoryListDefaultPageSize,
+			OrderBy:     models.SearchOrderByID,
+			Private:     true,
+			Collaborate: util.OptionalBoolFalse,
+		})
+		if err != nil {
+			log.Error(4, "SearchRepositoryByName: %v", err)
+			continue
+		}
+		if len(repos) == 0 {
+			return
+		}
+
+		for _, repo := range repos {
+			is, err := models.Issues(&models.IssuesOptions{
+				RepoIDs:  []int64{repo.ID},
+				IsClosed: util.OptionalBoolNone,
+				IsPull:   util.OptionalBoolNone,
+			})
+			if err != nil {
+				log.Error(4, "Issues: %v", err)
+				continue
+			}
+			if err = models.IssueList(is).LoadDiscussComments(); err != nil {
+				log.Error(4, "LoadComments: %v", err)
+				continue
+			}
+			for _, issue := range is {
+				UpdateIssueIndexer(issue)
+			}
+		}
+	}
+}
+
+// UpdateIssueIndexer add/update an issue to the issue indexer
+func UpdateIssueIndexer(issue *models.Issue) {
+	var comments []string
+	for _, comment := range issue.Comments {
+		if comment.Type == models.CommentTypeComment {
+			comments = append(comments, comment.Content)
+		}
+	}
+	issueIndexerUpdateQueue.Push(&IndexerData{
+		ID:       issue.ID,
+		RepoID:   issue.RepoID,
+		Title:    issue.Title,
+		Content:  issue.Content,
+		Comments: comments,
+	})
+}
+
+// DeleteRepoIssueIndexer deletes repo's all issues indexes
+func DeleteRepoIssueIndexer(repo *models.Repository) {
+	var ids []int64
+	ids, err := models.GetIssueIDsByRepoID(repo.ID)
+	if err != nil {
+		log.Error(4, "getIssueIDsByRepoID failed: %v", err)
+		return
+	}
+
+	if len(ids) <= 0 {
+		return
+	}
+
+	issueIndexerUpdateQueue.Push(&IndexerData{
+		IDs:      ids,
+		IsDelete: true,
+	})
+}
+
+// SearchIssuesByKeyword search issue ids by keywords and repo id
+func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
+	var issueIDs []int64
+	res, err := issueIndexer.Search(keyword, repoID, 1000, 0)
+	if err != nil {
+		return nil, err
+	}
+	for _, r := range res.Hits {
+		issueIDs = append(issueIDs, r.ID)
+	}
+	return issueIDs, nil
+}