mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-05-14 05:52:43 +00:00
Sync branches into databases (#22743)
Related #14180 Related #25233 Related #22639 Close #19786 Related #12763 This PR will change all the branches retrieve method from reading git data to read database to reduce git read operations. - [x] Sync git branches information into database when push git data - [x] Create a new table `Branch`, merge some columns of `DeletedBranch` into `Branch` table and drop the table `DeletedBranch`. - [x] Read `Branch` table when visit `code` -> `branch` page - [x] Read `Branch` table when list branch names in `code` page dropdown - [x] Read `Branch` table when list git ref compare page - [x] Provide a button in admin page to manually sync all branches. - [x] Sync branches if repository is not empty but database branches are empty when visiting pages with branches list - [x] Use `commit_time desc` as the default FindBranch order by to keep consistent as before and deleted branches will be always at the end. --------- Co-authored-by: Jason Song <i@wolfogre.com>
This commit is contained in:
Lunny Xiao
GitHub
parent
5a871932f0
commit
6e19484f4d
44 changed files with 1416 additions and 724 deletions
|
|
@ -10,13 +10,21 @@ import (
|
|||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
git_model "code.gitea.io/gitea/models/git"
|
||||
issues_model "code.gitea.io/gitea/models/issues"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/graceful"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/notification"
|
||||
"code.gitea.io/gitea/modules/queue"
|
||||
repo_module "code.gitea.io/gitea/modules/repository"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
files_service "code.gitea.io/gitea/services/repository/files"
|
||||
|
||||
"xorm.io/builder"
|
||||
)
|
||||
|
||||
// CreateNewBranch creates a new repository branch
|
||||
|
|
@ -27,7 +35,7 @@ func CreateNewBranch(ctx context.Context, doer *user_model.User, repo *repo_mode
|
|||
}
|
||||
|
||||
if !git.IsBranchExist(ctx, repo.RepoPath(), oldBranchName) {
|
||||
return models.ErrBranchDoesNotExist{
|
||||
return git_model.ErrBranchNotExist{
|
||||
BranchName: oldBranchName,
|
||||
}
|
||||
}
|
||||
|
|
@ -40,16 +48,165 @@ func CreateNewBranch(ctx context.Context, doer *user_model.User, repo *repo_mode
|
|||
if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("Push: %w", err)
|
||||
return fmt.Errorf("push: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBranches returns branches from the repository, skipping skip initial branches and
|
||||
// returning at most limit branches, or all branches if limit is 0.
|
||||
func GetBranches(ctx context.Context, repo *repo_model.Repository, skip, limit int) ([]*git.Branch, int, error) {
|
||||
return git.GetBranchesByPath(ctx, repo.RepoPath(), skip, limit)
|
||||
// Branch contains the branch information
|
||||
type Branch struct {
|
||||
DBBranch *git_model.Branch
|
||||
IsProtected bool
|
||||
IsIncluded bool
|
||||
CommitsAhead int
|
||||
CommitsBehind int
|
||||
LatestPullRequest *issues_model.PullRequest
|
||||
MergeMovedOn bool
|
||||
}
|
||||
|
||||
// LoadBranches loads branches from the repository limited by page & pageSize.
|
||||
func LoadBranches(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, isDeletedBranch util.OptionalBool, page, pageSize int) (*Branch, []*Branch, int64, error) {
|
||||
defaultDBBranch, err := git_model.GetBranch(ctx, repo.ID, repo.DefaultBranch)
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
|
||||
branchOpts := git_model.FindBranchOptions{
|
||||
RepoID: repo.ID,
|
||||
IsDeletedBranch: isDeletedBranch,
|
||||
ListOptions: db.ListOptions{
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
},
|
||||
}
|
||||
|
||||
totalNumOfBranches, err := git_model.CountBranches(ctx, branchOpts)
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
|
||||
branchOpts.ExcludeBranchNames = []string{repo.DefaultBranch}
|
||||
|
||||
dbBranches, err := git_model.FindBranches(ctx, branchOpts)
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
|
||||
if err := dbBranches.LoadDeletedBy(ctx); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
if err := dbBranches.LoadPusher(ctx); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
|
||||
rules, err := git_model.FindRepoProtectedBranchRules(ctx, repo.ID)
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
|
||||
repoIDToRepo := map[int64]*repo_model.Repository{}
|
||||
repoIDToRepo[repo.ID] = repo
|
||||
|
||||
repoIDToGitRepo := map[int64]*git.Repository{}
|
||||
repoIDToGitRepo[repo.ID] = gitRepo
|
||||
|
||||
branches := make([]*Branch, 0, len(dbBranches))
|
||||
for i := range dbBranches {
|
||||
branch, err := loadOneBranch(ctx, repo, dbBranches[i], &rules, repoIDToRepo, repoIDToGitRepo)
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("loadOneBranch: %v", err)
|
||||
}
|
||||
|
||||
branches = append(branches, branch)
|
||||
}
|
||||
|
||||
// Always add the default branch
|
||||
log.Debug("loadOneBranch: load default: '%s'", defaultDBBranch.Name)
|
||||
defaultBranch, err := loadOneBranch(ctx, repo, defaultDBBranch, &rules, repoIDToRepo, repoIDToGitRepo)
|
||||
if err != nil {
|
||||
return nil, nil, 0, fmt.Errorf("loadOneBranch: %v", err)
|
||||
}
|
||||
|
||||
return defaultBranch, branches, totalNumOfBranches, nil
|
||||
}
|
||||
|
||||
func loadOneBranch(ctx context.Context, repo *repo_model.Repository, dbBranch *git_model.Branch, protectedBranches *git_model.ProtectedBranchRules,
|
||||
repoIDToRepo map[int64]*repo_model.Repository,
|
||||
repoIDToGitRepo map[int64]*git.Repository,
|
||||
) (*Branch, error) {
|
||||
log.Trace("loadOneBranch: '%s'", dbBranch.Name)
|
||||
|
||||
branchName := dbBranch.Name
|
||||
p := protectedBranches.GetFirstMatched(branchName)
|
||||
isProtected := p != nil
|
||||
|
||||
divergence := &git.DivergeObject{
|
||||
Ahead: -1,
|
||||
Behind: -1,
|
||||
}
|
||||
|
||||
// it's not default branch
|
||||
if repo.DefaultBranch != dbBranch.Name && !dbBranch.IsDeleted {
|
||||
var err error
|
||||
divergence, err = files_service.CountDivergingCommits(ctx, repo, git.BranchPrefix+branchName)
|
||||
if err != nil {
|
||||
log.Error("CountDivergingCommits: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
pr, err := issues_model.GetLatestPullRequestByHeadInfo(repo.ID, branchName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetLatestPullRequestByHeadInfo: %v", err)
|
||||
}
|
||||
headCommit := dbBranch.CommitID
|
||||
|
||||
mergeMovedOn := false
|
||||
if pr != nil {
|
||||
pr.HeadRepo = repo
|
||||
if err := pr.LoadIssue(ctx); err != nil {
|
||||
return nil, fmt.Errorf("LoadIssue: %v", err)
|
||||
}
|
||||
if repo, ok := repoIDToRepo[pr.BaseRepoID]; ok {
|
||||
pr.BaseRepo = repo
|
||||
} else if err := pr.LoadBaseRepo(ctx); err != nil {
|
||||
return nil, fmt.Errorf("LoadBaseRepo: %v", err)
|
||||
} else {
|
||||
repoIDToRepo[pr.BaseRepoID] = pr.BaseRepo
|
||||
}
|
||||
pr.Issue.Repo = pr.BaseRepo
|
||||
|
||||
if pr.HasMerged {
|
||||
baseGitRepo, ok := repoIDToGitRepo[pr.BaseRepoID]
|
||||
if !ok {
|
||||
baseGitRepo, err = git.OpenRepository(ctx, pr.BaseRepo.RepoPath())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("OpenRepository: %v", err)
|
||||
}
|
||||
defer baseGitRepo.Close()
|
||||
repoIDToGitRepo[pr.BaseRepoID] = baseGitRepo
|
||||
}
|
||||
pullCommit, err := baseGitRepo.GetRefCommitID(pr.GetGitRefName())
|
||||
if err != nil && !git.IsErrNotExist(err) {
|
||||
return nil, fmt.Errorf("GetBranchCommitID: %v", err)
|
||||
}
|
||||
if err == nil && headCommit != pullCommit {
|
||||
// the head has moved on from the merge - we shouldn't delete
|
||||
mergeMovedOn = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
isIncluded := divergence.Ahead == 0 && repo.DefaultBranch != branchName
|
||||
return &Branch{
|
||||
DBBranch: dbBranch,
|
||||
IsProtected: isProtected,
|
||||
IsIncluded: isIncluded,
|
||||
CommitsAhead: divergence.Ahead,
|
||||
CommitsBehind: divergence.Behind,
|
||||
LatestPullRequest: pr,
|
||||
MergeMovedOn: mergeMovedOn,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func GetBranchCommitID(ctx context.Context, repo *repo_model.Repository, branch string) (string, error) {
|
||||
|
|
@ -62,17 +219,17 @@ func checkBranchName(ctx context.Context, repo *repo_model.Repository, name stri
|
|||
branchRefName := strings.TrimPrefix(refName, git.BranchPrefix)
|
||||
switch {
|
||||
case branchRefName == name:
|
||||
return models.ErrBranchAlreadyExists{
|
||||
return git_model.ErrBranchAlreadyExists{
|
||||
BranchName: name,
|
||||
}
|
||||
// If branchRefName like a/b but we want to create a branch named a then we have a conflict
|
||||
case strings.HasPrefix(branchRefName, name+"/"):
|
||||
return models.ErrBranchNameConflict{
|
||||
return git_model.ErrBranchNameConflict{
|
||||
BranchName: branchRefName,
|
||||
}
|
||||
// Conversely if branchRefName like a but we want to create a branch named a/b then we also have a conflict
|
||||
case strings.HasPrefix(name, branchRefName+"/"):
|
||||
return models.ErrBranchNameConflict{
|
||||
return git_model.ErrBranchNameConflict{
|
||||
BranchName: branchRefName,
|
||||
}
|
||||
case refName == git.TagPrefix+name:
|
||||
|
|
@ -101,7 +258,7 @@ func CreateNewBranchFromCommit(ctx context.Context, doer *user_model.User, repo
|
|||
if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("Push: %w", err)
|
||||
return fmt.Errorf("push: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -169,13 +326,28 @@ func DeleteBranch(ctx context.Context, doer *user_model.User, repo *repo_model.R
|
|||
return git_model.ErrBranchIsProtected
|
||||
}
|
||||
|
||||
rawBranch, err := git_model.GetBranch(ctx, repo.ID, branchName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("GetBranch: %vc", err)
|
||||
}
|
||||
|
||||
if rawBranch.IsDeleted {
|
||||
return nil
|
||||
}
|
||||
|
||||
commit, err := gitRepo.GetBranchCommit(branchName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := gitRepo.DeleteBranch(branchName, git.DeleteBranchOptions{
|
||||
Force: true,
|
||||
if err := db.WithTx(ctx, func(ctx context.Context) error {
|
||||
if err := git_model.AddDeletedBranch(ctx, repo.ID, branchName, doer.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return gitRepo.DeleteBranch(branchName, git.DeleteBranchOptions{
|
||||
Force: true,
|
||||
})
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -196,3 +368,45 @@ func DeleteBranch(ctx context.Context, doer *user_model.User, repo *repo_model.R
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
type BranchSyncOptions struct {
|
||||
RepoID int64
|
||||
}
|
||||
|
||||
// branchSyncQueue represents a queue to handle branch sync jobs.
|
||||
var branchSyncQueue *queue.WorkerPoolQueue[*BranchSyncOptions]
|
||||
|
||||
func handlerBranchSync(items ...*BranchSyncOptions) []*BranchSyncOptions {
|
||||
for _, opts := range items {
|
||||
_, err := repo_module.SyncRepoBranches(graceful.GetManager().ShutdownContext(), opts.RepoID, 0)
|
||||
if err != nil {
|
||||
log.Error("syncRepoBranches [%d] failed: %v", opts.RepoID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func addRepoToBranchSyncQueue(repoID, doerID int64) error {
|
||||
return branchSyncQueue.Push(&BranchSyncOptions{
|
||||
RepoID: repoID,
|
||||
})
|
||||
}
|
||||
|
||||
func initBranchSyncQueue(ctx context.Context) error {
|
||||
branchSyncQueue = queue.CreateUniqueQueue(ctx, "branch_sync", handlerBranchSync)
|
||||
if branchSyncQueue == nil {
|
||||
return errors.New("unable to create branch_sync queue")
|
||||
}
|
||||
go graceful.GetManager().RunWithCancel(branchSyncQueue)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func AddAllRepoBranchesToSyncQueue(ctx context.Context, doerID int64) error {
|
||||
if err := db.Iterate(ctx, builder.Eq{"is_empty": false}, func(ctx context.Context, repo *repo_model.Repository) error {
|
||||
return addRepoToBranchSyncQueue(repo.ID, doerID)
|
||||
}); err != nil {
|
||||
return fmt.Errorf("run sync all branches failed: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue