Add push to remote mirror repository (#15157)

* Added push mirror model.

* Integrated push mirror into queue.

* Moved methods into own file.

* Added basic implementation.

* Mirror wiki too.

* Removed duplicated method.

* Get url for different remotes.

* Added migration.

* Unified remote url access.

* Add/Remove push mirror remotes.

* Prevent hangs with missing credentials.

* Moved code between files.

* Changed sanitizer interface.

* Added push mirror backend methods.

* Only update the mirror remote.

* Limit refs on push.

* Added UI part.

* Added missing table.

* Delete mirror if repository gets removed.

* Changed signature. Handle object errors.

* Added upload method.

* Added "upload" unit tests.

* Added transfer adapter unit tests.

* Send correct headers.

* Added pushing of LFS objects.

* Added more logging.

* Simpler body handling.

* Process files in batches to reduce HTTP calls.

* Added created timestamp.

* Fixed invalid column name.

* Changed name to prevent xorm auto setting.

* Remove table header im empty.

* Strip exit code from error message.

* Added docs page about mirroring.

* Fixed date.

* Fixed merge errors.

* Moved test to integrations.

* Added push mirror test.

* Added test.

models/migrations/migrations.go

@@ -315,6 +315,8 @@ var migrations = []Migration{
 	NewMigration("Always save primary email on email address table", addPrimaryEmail2EmailAddress),
 	// v182 -> v183
 	NewMigration("Add issue resource index table", addIssueResourceIndexTable),
+	// v183 -> v184
+	NewMigration("Create PushMirror table", createPushMirrorTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v180.go

@@ -64,7 +64,7 @@ func removeCredentials(payload string) (string, error) {
 
 	opts.AuthPassword = ""
 	opts.AuthToken = ""
-	opts.CloneAddr = util.SanitizeURLCredentials(opts.CloneAddr, true)
+	opts.CloneAddr = util.NewStringURLSanitizer(opts.CloneAddr, true).Replace(opts.CloneAddr)
 
 	confBytes, err := json.Marshal(opts)
 	if err != nil {

models/migrations/v183.go

@@ -0,0 +1,39 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+	"time"
+
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func createPushMirrorTable(x *xorm.Engine) error {
+	type PushMirror struct {
+		ID         int64 `xorm:"pk autoincr"`
+		RepoID     int64 `xorm:"INDEX"`
+		RemoteName string
+
+		Interval       time.Duration
+		CreatedUnix    timeutil.TimeStamp `xorm:"created"`
+		LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
+		LastError      string             `xorm:"text"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Sync2(new(PushMirror)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+
+	return sess.Commit()
+}

models/models.go

@@ -135,6 +135,7 @@ func init() {
 		new(Session),
 		new(RepoTransfer),
 		new(IssueIndex),
+		new(PushMirror),
 	)
 
 	gonicNames := []string{"SSL", "UID"}

models/repo.go

@@ -216,12 +216,13 @@ type Repository struct {
 	NumClosedProjects   int `xorm:"NOT NULL DEFAULT 0"`
 	NumOpenProjects     int `xorm:"-"`
 
-	IsPrivate  bool `xorm:"INDEX"`
-	IsEmpty    bool `xorm:"INDEX"`
-	IsArchived bool `xorm:"INDEX"`
-	IsMirror   bool `xorm:"INDEX"`
-	*Mirror    `xorm:"-"`
-	Status     RepositoryStatus `xorm:"NOT NULL DEFAULT 0"`
+	IsPrivate   bool `xorm:"INDEX"`
+	IsEmpty     bool `xorm:"INDEX"`
+	IsArchived  bool `xorm:"INDEX"`
+	IsMirror    bool `xorm:"INDEX"`
+	*Mirror     `xorm:"-"`
+	PushMirrors []*PushMirror    `xorm:"-"`
+	Status      RepositoryStatus `xorm:"NOT NULL DEFAULT 0"`
 
 	RenderingMetas         map[string]string `xorm:"-"`
 	DocumentRenderingMetas map[string]string `xorm:"-"`
@@ -255,7 +256,12 @@ func (repo *Repository) SanitizedOriginalURL() string {
 	if repo.OriginalURL == "" {
 		return ""
 	}
-	return util.SanitizeURLCredentials(repo.OriginalURL, false)
+	u, err := url.Parse(repo.OriginalURL)
+	if err != nil {
+		return ""
+	}
+	u.User = nil
+	return u.String()
 }
 
 // ColorFormat returns a colored string to represent this repo
@@ -657,6 +663,12 @@ func (repo *Repository) GetMirror() (err error) {
 	return err
 }
 
+// LoadPushMirrors populates the repository push mirrors.
+func (repo *Repository) LoadPushMirrors() (err error) {
+	repo.PushMirrors, err = GetPushMirrorsByRepoID(repo.ID)
+	return err
+}
+
 // GetBaseRepo populates repo.BaseRepo for a fork repository and
 // returns an error on failure (NOTE: no error is returned for
 // non-fork repositories, and BaseRepo will be left untouched)
@@ -1487,6 +1499,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
 		&Notification{RepoID: repoID},
 		&ProtectedBranch{RepoID: repoID},
 		&PullRequest{BaseRepoID: repoID},
+		&PushMirror{RepoID: repoID},
 		&Release{RepoID: repoID},
 		&RepoIndexerStatus{RepoID: repoID},
 		&RepoRedirect{RedirectRepoID: repoID},

models/repo_mirror.go

@@ -14,6 +14,12 @@ import (
 	"xorm.io/xorm"
 )
 
+// RemoteMirrorer defines base methods for pull/push mirrors.
+type RemoteMirrorer interface {
+	GetRepository() *Repository
+	GetRemoteName() string
+}
+
 // Mirror represents mirror information of a repository.
 type Mirror struct {
 	ID          int64       `xorm:"pk autoincr"`
@@ -52,6 +58,16 @@ func (m *Mirror) AfterLoad(session *xorm.Session) {
 	}
 }
 
+// GetRepository returns the repository.
+func (m *Mirror) GetRepository() *Repository {
+	return m.Repo
+}
+
+// GetRemoteName returns the name of the remote.
+func (m *Mirror) GetRemoteName() string {
+	return "origin"
+}
+
 // ScheduleNextUpdate calculates and sets next update time.
 func (m *Mirror) ScheduleNextUpdate() {
 	if m.Interval != 0 {

models/repo_pushmirror.go

@@ -0,0 +1,106 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import (
+	"errors"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+var (
+	// ErrPushMirrorNotExist mirror does not exist error
+	ErrPushMirrorNotExist = errors.New("PushMirror does not exist")
+)
+
+// PushMirror represents mirror information of a repository.
+type PushMirror struct {
+	ID         int64       `xorm:"pk autoincr"`
+	RepoID     int64       `xorm:"INDEX"`
+	Repo       *Repository `xorm:"-"`
+	RemoteName string
+
+	Interval       time.Duration
+	CreatedUnix    timeutil.TimeStamp `xorm:"created"`
+	LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
+	LastError      string             `xorm:"text"`
+}
+
+// AfterLoad is invoked from XORM after setting the values of all fields of this object.
+func (m *PushMirror) AfterLoad(session *xorm.Session) {
+	if m == nil {
+		return
+	}
+
+	var err error
+	m.Repo, err = getRepositoryByID(session, m.RepoID)
+	if err != nil {
+		log.Error("getRepositoryByID[%d]: %v", m.ID, err)
+	}
+}
+
+// GetRepository returns the path of the repository.
+func (m *PushMirror) GetRepository() *Repository {
+	return m.Repo
+}
+
+// GetRemoteName returns the name of the remote.
+func (m *PushMirror) GetRemoteName() string {
+	return m.RemoteName
+}
+
+// InsertPushMirror inserts a push-mirror to database
+func InsertPushMirror(m *PushMirror) error {
+	_, err := x.Insert(m)
+	return err
+}
+
+// UpdatePushMirror updates the push-mirror
+func UpdatePushMirror(m *PushMirror) error {
+	_, err := x.ID(m.ID).AllCols().Update(m)
+	return err
+}
+
+// DeletePushMirrorByID deletes a push-mirrors by ID
+func DeletePushMirrorByID(ID int64) error {
+	_, err := x.ID(ID).Delete(&PushMirror{})
+	return err
+}
+
+// DeletePushMirrorsByRepoID deletes all push-mirrors by repoID
+func DeletePushMirrorsByRepoID(repoID int64) error {
+	_, err := x.Delete(&PushMirror{RepoID: repoID})
+	return err
+}
+
+// GetPushMirrorByID returns push-mirror information.
+func GetPushMirrorByID(ID int64) (*PushMirror, error) {
+	m := &PushMirror{}
+	has, err := x.ID(ID).Get(m)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, ErrPushMirrorNotExist
+	}
+	return m, nil
+}
+
+// GetPushMirrorsByRepoID returns push-mirror informations of a repository.
+func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
+	mirrors := make([]*PushMirror, 0, 10)
+	return mirrors, x.Where("repo_id=?", repoID).Find(&mirrors)
+}
+
+// PushMirrorsIterate iterates all push-mirror repositories.
+func PushMirrorsIterate(f func(idx int, bean interface{}) error) error {
+	return x.
+		Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
+		And("`interval` != 0").
+		Iterate(new(PushMirror), f)
+}

models/repo_pushmirror_test.go

@@ -0,0 +1,49 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import (
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPushMirrorsIterate(t *testing.T) {
+	assert.NoError(t, PrepareTestDatabase())
+
+	now := timeutil.TimeStampNow()
+
+	InsertPushMirror(&PushMirror{
+		RemoteName:     "test-1",
+		LastUpdateUnix: now,
+		Interval:       1,
+	})
+
+	long, _ := time.ParseDuration("24h")
+	InsertPushMirror(&PushMirror{
+		RemoteName:     "test-2",
+		LastUpdateUnix: now,
+		Interval:       long,
+	})
+
+	InsertPushMirror(&PushMirror{
+		RemoteName:     "test-3",
+		LastUpdateUnix: now,
+		Interval:       0,
+	})
+
+	time.Sleep(1 * time.Millisecond)
+
+	PushMirrorsIterate(func(idx int, bean interface{}) error {
+		m, ok := bean.(*PushMirror)
+		assert.True(t, ok)
+		assert.Equal(t, "test-1", m.RemoteName)
+		assert.Equal(t, m.RemoteName, m.GetRemoteName())
+		return nil
+	})
+}

models/task.go

@@ -234,7 +234,7 @@ func FinishMigrateTask(task *Task) error {
 	}
 	conf.AuthPassword = ""
 	conf.AuthToken = ""
-	conf.CloneAddr = util.SanitizeURLCredentials(conf.CloneAddr, true)
+	conf.CloneAddr = util.NewStringURLSanitizer(conf.CloneAddr, true).Replace(conf.CloneAddr)
 	conf.AuthPasswordEncrypted = ""
 	conf.AuthTokenEncrypted = ""
 	conf.CloneAddrEncrypted = ""