Add push webhook support for mirrored repositories (#4127)

models/repo_mirror.go

@@ -1,4 +1,5 @@
 // Copyright 2016 The Gogs Authors. All rights reserved.
+// Copyright 2018 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
@@ -6,6 +7,7 @@ package models
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"code.gitea.io/git"
@@ -119,8 +121,68 @@ func (m *Mirror) SaveAddress(addr string) error {
 	return cfg.SaveToIndent(configPath, "\t")
 }
 
+// gitShortEmptySha Git short empty SHA
+const gitShortEmptySha = "0000000"
+
+// mirrorSyncResult contains information of a updated reference.
+// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
+// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
+type mirrorSyncResult struct {
+	refName     string
+	oldCommitID string
+	newCommitID string
+}
+
+// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
+func parseRemoteUpdateOutput(output string) []*mirrorSyncResult {
+	results := make([]*mirrorSyncResult, 0, 3)
+	lines := strings.Split(output, "\n")
+	for i := range lines {
+		// Make sure reference name is presented before continue
+		idx := strings.Index(lines[i], "-> ")
+		if idx == -1 {
+			continue
+		}
+
+		refName := lines[i][idx+3:]
+
+		switch {
+		case strings.HasPrefix(lines[i], " * "): // New reference
+			results = append(results, &mirrorSyncResult{
+				refName:     refName,
+				oldCommitID: gitShortEmptySha,
+			})
+		case strings.HasPrefix(lines[i], " - "): // Delete reference
+			results = append(results, &mirrorSyncResult{
+				refName:     refName,
+				newCommitID: gitShortEmptySha,
+			})
+		case strings.HasPrefix(lines[i], "   "): // New commits of a reference
+			delimIdx := strings.Index(lines[i][3:], " ")
+			if delimIdx == -1 {
+				log.Error(2, "SHA delimiter not found: %q", lines[i])
+				continue
+			}
+			shas := strings.Split(lines[i][3:delimIdx+3], "..")
+			if len(shas) != 2 {
+				log.Error(2, "Expect two SHAs but not what found: %q", lines[i])
+				continue
+			}
+			results = append(results, &mirrorSyncResult{
+				refName:     refName,
+				oldCommitID: shas[0],
+				newCommitID: shas[1],
+			})
+
+		default:
+			log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
+		}
+	}
+	return results
+}
+
 // runSync returns true if sync finished without error.
-func (m *Mirror) runSync() bool {
+func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) {
 	repoPath := m.Repo.RepoPath()
 	wikiPath := m.Repo.WikiPath()
 	timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
@@ -130,28 +192,30 @@ func (m *Mirror) runSync() bool {
 		gitArgs = append(gitArgs, "--prune")
 	}
 
-	if _, stderr, err := process.GetManager().ExecDir(
+	_, stderr, err := process.GetManager().ExecDir(
 		timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath),
-		"git", gitArgs...); err != nil {
+		"git", gitArgs...)
+	if err != nil {
 		// sanitize the output, since it may contain the remote address, which may
 		// contain a password
 		message, err := sanitizeOutput(stderr, repoPath)
 		if err != nil {
 			log.Error(4, "sanitizeOutput: %v", err)
-			return false
+			return nil, false
 		}
 		desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message)
 		log.Error(4, desc)
 		if err = CreateRepositoryNotice(desc); err != nil {
 			log.Error(4, "CreateRepositoryNotice: %v", err)
 		}
-		return false
+		return nil, false
 	}
+	output := stderr
 
 	gitRepo, err := git.OpenRepository(repoPath)
 	if err != nil {
 		log.Error(4, "OpenRepository: %v", err)
-		return false
+		return nil, false
 	}
 	if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
 		log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
@@ -170,21 +234,21 @@ func (m *Mirror) runSync() bool {
 			message, err := sanitizeOutput(stderr, wikiPath)
 			if err != nil {
 				log.Error(4, "sanitizeOutput: %v", err)
-				return false
+				return nil, false
 			}
 			desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message)
 			log.Error(4, desc)
 			if err = CreateRepositoryNotice(desc); err != nil {
 				log.Error(4, "CreateRepositoryNotice: %v", err)
 			}
-			return false
+			return nil, false
 		}
 	}
 
 	branches, err := m.Repo.GetBranches()
 	if err != nil {
 		log.Error(4, "GetBranches: %v", err)
-		return false
+		return nil, false
 	}
 
 	for i := range branches {
@@ -192,7 +256,7 @@ func (m *Mirror) runSync() bool {
 	}
 
 	m.UpdatedUnix = util.TimeStampNow()
-	return true
+	return parseRemoteUpdateOutput(output), true
 }
 
 func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) {
@@ -268,7 +332,8 @@ func SyncMirrors() {
 			continue
 		}
 
-		if !m.runSync() {
+		results, ok := m.runSync()
+		if !ok {
 			continue
 		}
 
@@ -278,6 +343,66 @@ func SyncMirrors() {
 			continue
 		}
 
+		var gitRepo *git.Repository
+		if len(results) == 0 {
+			log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID)
+		} else {
+			gitRepo, err = git.OpenRepository(m.Repo.RepoPath())
+			if err != nil {
+				log.Error(2, "OpenRepository [%d]: %v", m.RepoID, err)
+				continue
+			}
+		}
+
+		for _, result := range results {
+			// Discard GitHub pull requests, i.e. refs/pull/*
+			if strings.HasPrefix(result.refName, "refs/pull/") {
+				continue
+			}
+
+			// Create reference
+			if result.oldCommitID == gitShortEmptySha {
+				if err = MirrorSyncCreateAction(m.Repo, result.refName); err != nil {
+					log.Error(2, "MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err)
+				}
+				continue
+			}
+
+			// Delete reference
+			if result.newCommitID == gitShortEmptySha {
+				if err = MirrorSyncDeleteAction(m.Repo, result.refName); err != nil {
+					log.Error(2, "MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err)
+				}
+				continue
+			}
+
+			// Push commits
+			oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID)
+			if err != nil {
+				log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err)
+				continue
+			}
+			newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID)
+			if err != nil {
+				log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err)
+				continue
+			}
+			commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID)
+			if err != nil {
+				log.Error(2, "CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err)
+				continue
+			}
+			if err = MirrorSyncPushAction(m.Repo, MirrorSyncPushActionOptions{
+				RefName:     result.refName,
+				OldCommitID: oldCommitID,
+				NewCommitID: newCommitID,
+				Commits:     ListToPushCommits(commits),
+			}); err != nil {
+				log.Error(2, "MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err)
+				continue
+			}
+		}
+
 		// Get latest commit date and update to current repository updated time
 		commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath())
 		if err != nil {