luckyimnotanigger/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2025-06-19 23:50:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Move Bleve and Elastic code indexers to use a common cat-file --batch (#14781)

Browse source 
* Extract out the common cat-file batch calls

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Move bleve and elastic indexers to use a common cat-file --batch when indexing

Signed-off-by: Andrew Thornton <art27@cantab.net>

* move catfilebatch to batch_reader and rename to batch_reader.go

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: Lauris BH <lauris@nix.lv>
This commit is contained in:
zeripath 2021-03-04 02:57:01 +00:00 • committed by GitHub
parent 0044e80491
commit 523efa433b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 91 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

38
modules/indexer/code/elastic_search.go
View file

@ -5,8 +5,11 @@
package code
import (
"bufio"
"context"
"fmt"
"io"
"io/ioutil"
"strconv"
"strings"
"time"
@ -172,7 +175,7 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
return exists, nil
}
func (b *ElasticSearchIndexer) addUpdate(sha string, update fileUpdate, repo *models.Repository) ([]elastic.BulkableRequest, error) {
func (b *ElasticSearchIndexer) addUpdate(batchWriter *io.PipeWriter, batchReader *bufio.Reader, sha string, update fileUpdate, repo *models.Repository) ([]elastic.BulkableRequest, error) {
// Ignore vendored files in code search
if setting.Indexer.ExcludeVendored && enry.IsVendor(update.Filename) {
return nil, nil
@ -195,8 +198,16 @@ func (b *ElasticSearchIndexer) addUpdate(sha string, update fileUpdate, repo *mo
return []elastic.BulkableRequest{b.addDelete(update.Filename, repo)}, nil
}
fileContents, err := git.NewCommand("cat-file", "blob", update.BlobSha).
RunInDirBytes(repo.RepoPath())
if _, err := batchWriter.Write([]byte(update.BlobSha + "\n")); err != nil {
return nil, err
}
_, _, size, err := git.ReadBatchLine(batchReader)
if err != nil {
return nil, err
}
fileContents, err := ioutil.ReadAll(io.LimitReader(batchReader, size))
if err != nil {
return nil, err
} else if !base.IsTextFile(fileContents) {
@ -230,14 +241,21 @@ func (b *ElasticSearchIndexer) addDelete(filename string, repo *models.Repositor
// Index will save the index data
func (b *ElasticSearchIndexer) Index(repo *models.Repository, sha string, changes *repoChanges) error {
reqs := make([]elastic.BulkableRequest, 0)
for _, update := range changes.Updates {
updateReqs, err := b.addUpdate(sha, update, repo)
if err != nil {
return err
}
if len(updateReqs) > 0 {
reqs = append(reqs, updateReqs...)
if len(changes.Updates) > 0 {
batchWriter, batchReader, cancel := git.CatFileBatch(repo.RepoPath())
defer cancel()
for _, update := range changes.Updates {
updateReqs, err := b.addUpdate(batchWriter, batchReader, sha, update, repo)
if err != nil {
return err
}
if len(updateReqs) > 0 {
reqs = append(reqs, updateReqs...)
}
}
cancel()
}
for _, filename := range changes.RemovedFilenames {