luckyimnotanigger/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2025-05-27 04:07:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Optimization of labels handling in issue_search (#4228)

Browse source 
This PR optimizes the SQL query and de-duplicate the labels' ids when generating the query string, on the issue page.

<hr/>

### Background

Some time ago, BingBot and some other crawlers have been putting my instance on its knees with requests containing a lot of label ids, like this one :

```
[07/Aug/2023:11:28:37 +0200] "GET /Dolibarr/sendrecurringinvoicebymail/issues?q=&type=all&sort=&state=closed&labels=1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c2%2c10%2c2%2c1%2c1%2c10%2c10%2c7%2c6%2c10%2c10%2c3%2c2%2c1%2c5%2c10%2c1%2c6%2c2%2c7%2c3%2c7%2c6%2c10%2c1%2c10%2c1%2c1%2c7%2c7%2c1%2c1%2c1%2c1%2c10%2c10%2c1%2c2%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c1%2c2%2c1%2c12%2c6%2c6%2c10&milestone=0&project=-1&poster=0 HTTP/1.1" 499 0 "-" "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36"
```

Since each of the label ids implies a join, it grows exponentially expensive for the database engine (at least on PostgreSQL but SQLite suffers a little too).

Thus, this PR proposes two enhancements:

* rewrite the database query to use only one squashed condition,
* deduplicate the label ids when generating the URL.

### Performance comparison

Here are some timings on Postgresql-backed, Forgejo 7.0.4 instances :
```sh
$ time curl -s -o /dev/null "http://localhost:3000/toto/tata/issues?q=&type=all&sort=&labels=19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25&state=open&milestone=0&project=0&assignee=0&poster=0"

real    0m10,491s
user    0m0,017s
sys     0m0,008s
```
...and with the patch:
```sh
$ time curl -s -o /dev/null "http://localhost:3000/toto/tata/issues?q=&type=all&sort=&labels=19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25%2c19%2c25&state=open&milestone=0&project=0&assignee=0&poster=0"

real    0m0,094s
user    0m0,012s
sys     0m0,013s
```

### Annex

This issue was originally proposed to [Gitea](https://github.com/go-gitea/gitea/pull/26460) but didn't get much attention, and I switched to Forgejo in the meantime :)

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4228
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Chl <chl@xlii.si>
Co-committed-by: Chl <chl@xlii.si>
This commit is contained in:
Chl 2024-06-28 05:11:57 +00:00 • committed by Earl Warren
parent 2121a29f89
commit 544cbc6f01
4 changed files with 80 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

35
models/issues/label.go
View file

@ -7,6 +7,7 @@ package issues
import (
"context"
"fmt"
"slices"
"strconv"
"strings"
@ -142,28 +143,38 @@ func (l *Label) CalOpenOrgIssues(ctx context.Context, repoID, labelID int64) {
// LoadSelectedLabelsAfterClick calculates the set of selected labels when a label is clicked
func (l *Label) LoadSelectedLabelsAfterClick(currentSelectedLabels []int64, currentSelectedExclusiveScopes []string) {
var labelQuerySlice []string
labelQuerySlice := []int64{}
labelSelected := false
labelID := strconv.FormatInt(l.ID, 10)
labelScope := l.ExclusiveScope()
for i, s := range currentSelectedLabels {
if s == l.ID {
exclusiveScope := l.ExclusiveScope()
for i, curSel := range currentSelectedLabels {
if curSel == l.ID {
labelSelected = true
} else if -s == l.ID {
} else if -curSel == l.ID {
labelSelected = true
l.IsExcluded = true
} else if s != 0 {
} else if curSel != 0 {
// Exclude other labels in the same scope from selection
if s < 0 || labelScope == "" || labelScope != currentSelectedExclusiveScopes[i] {
labelQuerySlice = append(labelQuerySlice, strconv.FormatInt(s, 10))
if curSel < 0 || exclusiveScope == "" || exclusiveScope != currentSelectedExclusiveScopes[i] {
labelQuerySlice = append(labelQuerySlice, curSel)
}
}
}
if !labelSelected {
labelQuerySlice = append(labelQuerySlice, labelID)
labelQuerySlice = append(labelQuerySlice, l.ID)
}
l.IsSelected = labelSelected
l.QueryString = strings.Join(labelQuerySlice, ",")
// Sort and deduplicate the ids to avoid the crawlers asking for the
// same thing with simply a different order of parameters
slices.Sort(labelQuerySlice)
labelQuerySlice = slices.Compact(labelQuerySlice)
// Quick conversion (strings.Join() doesn't accept slices of Int64)
labelQuerySliceStrings := make([]string, len(labelQuerySlice))
for i, x := range labelQuerySlice {
labelQuerySliceStrings[i] = strconv.FormatInt(x, 10)
}
l.QueryString = strings.Join(labelQuerySliceStrings, ",")
}
// BelongsToOrg returns true if label is an organization label
@ -176,7 +187,7 @@ func (l *Label) BelongsToRepo() bool {
return l.RepoID > 0
}
// Return scope substring of label name, or empty string if none exists
// ExclusiveScope returns scope substring of label name, or empty string if none exists
func (l *Label) ExclusiveScope() string {
if !l.Exclusive {
return ""