mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-05-31 11:52:10 +00:00
Merge pull request '[v7.0/forgejo] Refactor LFS GC functions' (#3072) from bp-v7.0/forgejo-7ffa7f5 into v7.0/forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/3072 Reviewed-by: Gusted <gusted@noreply.codeberg.org>
This commit is contained in:
Gusted
commit
d2a7905d9d
5 changed files with 132 additions and 56 deletions
|
|
@ -0,0 +1,7 @@
|
|||
-
|
||||
|
||||
id: 1000
|
||||
oid: 9d172e5c64b4f0024b9901ec6afe9ea052f3c9b6ff9f4b07956d8c48c86fca82
|
||||
size: 25
|
||||
repository_id: 1
|
||||
created_unix: 1712309123
|
||||
|
|
@ -337,58 +337,47 @@ func GetRepoLFSSize(ctx context.Context, repoID int64) (int64, error) {
|
|||
func IterateRepositoryIDsWithLFSMetaObjects(ctx context.Context, f func(ctx context.Context, repoID, count int64) error) error {
|
||||
batchSize := setting.Database.IterateBufferSize
|
||||
sess := db.GetEngine(ctx)
|
||||
id := int64(0)
|
||||
var start int
|
||||
type RepositoryCount struct {
|
||||
RepositoryID int64
|
||||
Count int64
|
||||
}
|
||||
for {
|
||||
counts := make([]*RepositoryCount, 0, batchSize)
|
||||
sess.Select("repository_id, COUNT(id) AS count").
|
||||
if err := sess.Select("repository_id, COUNT(id) AS count").
|
||||
Table("lfs_meta_object").
|
||||
Where("repository_id > ?", id).
|
||||
GroupBy("repository_id").
|
||||
OrderBy("repository_id ASC")
|
||||
|
||||
if err := sess.Limit(batchSize, 0).Find(&counts); err != nil {
|
||||
OrderBy("repository_id ASC").Limit(batchSize, start).Find(&counts); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(counts) == 0 {
|
||||
return nil
|
||||
}
|
||||
start += len(counts)
|
||||
|
||||
for _, count := range counts {
|
||||
if err := f(ctx, count.RepositoryID, count.Count); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
id = counts[len(counts)-1].RepositoryID
|
||||
}
|
||||
}
|
||||
|
||||
// IterateLFSMetaObjectsForRepoOptions provides options for IterateLFSMetaObjectsForRepo
|
||||
type IterateLFSMetaObjectsForRepoOptions struct {
|
||||
OlderThan timeutil.TimeStamp
|
||||
UpdatedLessRecentlyThan timeutil.TimeStamp
|
||||
OrderByUpdated bool
|
||||
LoopFunctionAlwaysUpdates bool
|
||||
OlderThan timeutil.TimeStamp
|
||||
UpdatedLessRecentlyThan timeutil.TimeStamp
|
||||
}
|
||||
|
||||
// IterateLFSMetaObjectsForRepo provides a iterator for LFSMetaObjects per Repo
|
||||
func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(context.Context, *LFSMetaObject, int64) error, opts *IterateLFSMetaObjectsForRepoOptions) error {
|
||||
var start int
|
||||
func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(context.Context, *LFSMetaObject) error, opts *IterateLFSMetaObjectsForRepoOptions) error {
|
||||
batchSize := setting.Database.IterateBufferSize
|
||||
engine := db.GetEngine(ctx)
|
||||
type CountLFSMetaObject struct {
|
||||
Count int64
|
||||
LFSMetaObject `xorm:"extends"`
|
||||
}
|
||||
|
||||
id := int64(0)
|
||||
|
||||
for {
|
||||
beans := make([]*CountLFSMetaObject, 0, batchSize)
|
||||
sess := engine.Table("lfs_meta_object").Select("`lfs_meta_object`.*, COUNT(`l1`.oid) AS `count`").
|
||||
beans := make([]*LFSMetaObject, 0, batchSize)
|
||||
sess := engine.Table("lfs_meta_object").Select("`lfs_meta_object`.*").
|
||||
Join("INNER", "`lfs_meta_object` AS l1", "`lfs_meta_object`.oid = `l1`.oid").
|
||||
Where("`lfs_meta_object`.repository_id = ?", repoID)
|
||||
if !opts.OlderThan.IsZero() {
|
||||
|
|
@ -397,25 +386,19 @@ func IterateLFSMetaObjectsForRepo(ctx context.Context, repoID int64, f func(cont
|
|||
if !opts.UpdatedLessRecentlyThan.IsZero() {
|
||||
sess.And("`lfs_meta_object`.updated_unix < ?", opts.UpdatedLessRecentlyThan)
|
||||
}
|
||||
sess.GroupBy("`lfs_meta_object`.id")
|
||||
if opts.OrderByUpdated {
|
||||
sess.OrderBy("`lfs_meta_object`.updated_unix ASC")
|
||||
} else {
|
||||
sess.And("`lfs_meta_object`.id > ?", id)
|
||||
sess.OrderBy("`lfs_meta_object`.id ASC")
|
||||
}
|
||||
if err := sess.Limit(batchSize, start).Find(&beans); err != nil {
|
||||
sess.GroupBy("`lfs_meta_object`.id").
|
||||
And("`lfs_meta_object`.id > ?", id).
|
||||
OrderBy("`lfs_meta_object`.id ASC")
|
||||
|
||||
if err := sess.Limit(batchSize, 0).Find(&beans); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(beans) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !opts.LoopFunctionAlwaysUpdates {
|
||||
start += len(beans)
|
||||
}
|
||||
|
||||
for _, bean := range beans {
|
||||
if err := f(ctx, &bean.LFSMetaObject, bean.Count); err != nil {
|
||||
if err := f(ctx, bean); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
101
models/git/lfs_test.go
Normal file
101
models/git/lfs_test.go
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
// Copyright 2024 The Forgejo Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package git
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/models/unittest"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/test"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIterateRepositoryIDsWithLFSMetaObjects(t *testing.T) {
|
||||
defer unittest.OverrideFixtures(
|
||||
unittest.FixturesOptions{
|
||||
Dir: filepath.Join(setting.AppWorkPath, "models/fixtures/"),
|
||||
Base: setting.AppWorkPath,
|
||||
Dirs: []string{"models/git/TestIterateRepositoryIDsWithLFSMetaObjects/"},
|
||||
},
|
||||
)()
|
||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||
|
||||
type repocount struct {
|
||||
repoid int64
|
||||
count int64
|
||||
}
|
||||
expected := []repocount{{1, 1}, {54, 4}}
|
||||
|
||||
t.Run("Normal batch size", func(t *testing.T) {
|
||||
defer test.MockVariableValue(&setting.Database.IterateBufferSize, 20)()
|
||||
cases := []repocount{}
|
||||
|
||||
err := IterateRepositoryIDsWithLFSMetaObjects(db.DefaultContext, func(ctx context.Context, repoID, count int64) error {
|
||||
cases = append(cases, repocount{repoID, count})
|
||||
return nil
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, expected, cases)
|
||||
})
|
||||
|
||||
t.Run("Low batch size", func(t *testing.T) {
|
||||
defer test.MockVariableValue(&setting.Database.IterateBufferSize, 1)()
|
||||
cases := []repocount{}
|
||||
|
||||
err := IterateRepositoryIDsWithLFSMetaObjects(db.DefaultContext, func(ctx context.Context, repoID, count int64) error {
|
||||
cases = append(cases, repocount{repoID, count})
|
||||
return nil
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, expected, cases)
|
||||
})
|
||||
}
|
||||
|
||||
func TestIterateLFSMetaObjectsForRepo(t *testing.T) {
|
||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||
|
||||
expectedIDs := []int64{1, 2, 3, 4}
|
||||
|
||||
t.Run("Normal batch size", func(t *testing.T) {
|
||||
defer test.MockVariableValue(&setting.Database.IterateBufferSize, 20)()
|
||||
actualIDs := []int64{}
|
||||
|
||||
err := IterateLFSMetaObjectsForRepo(db.DefaultContext, 54, func(ctx context.Context, lo *LFSMetaObject) error {
|
||||
actualIDs = append(actualIDs, lo.ID)
|
||||
return nil
|
||||
}, &IterateLFSMetaObjectsForRepoOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, expectedIDs, actualIDs)
|
||||
})
|
||||
|
||||
t.Run("Low batch size", func(t *testing.T) {
|
||||
defer test.MockVariableValue(&setting.Database.IterateBufferSize, 1)()
|
||||
actualIDs := []int64{}
|
||||
|
||||
err := IterateLFSMetaObjectsForRepo(db.DefaultContext, 54, func(ctx context.Context, lo *LFSMetaObject) error {
|
||||
actualIDs = append(actualIDs, lo.ID)
|
||||
return nil
|
||||
}, &IterateLFSMetaObjectsForRepoOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, expectedIDs, actualIDs)
|
||||
|
||||
t.Run("Batch handles updates", func(t *testing.T) {
|
||||
actualIDs := []int64{}
|
||||
|
||||
err := IterateLFSMetaObjectsForRepo(db.DefaultContext, 54, func(ctx context.Context, lo *LFSMetaObject) error {
|
||||
actualIDs = append(actualIDs, lo.ID)
|
||||
_, err := db.DeleteByID[LFSMetaObject](ctx, lo.ID)
|
||||
assert.NoError(t, err)
|
||||
return nil
|
||||
}, &IterateLFSMetaObjectsForRepoOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, expectedIDs, actualIDs)
|
||||
})
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue