luckyimnotanigger/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2025-05-31 11:52:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Add support to migrate from gogs (#14342)

Browse source 
Add support to migrate gogs:

  *  issues
  *  comments
  *  labels
  *  milestones
  *  wiki


Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
6543 2021-01-21 20:33:58 +01:00 • committed by GitHub
parent b5570d3e68
commit 81c833d92d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
55 changed files with 2782 additions and 353 deletions
Download patch file Download diff file Expand all files Collapse all files

212
modules/migrations/base/downloader.go
View file

@ -7,7 +7,6 @@ package base
import (
"context"
"time"
"code.gitea.io/gitea/modules/structs"
)
@ -24,6 +23,7 @@ type Downloader interface {
GetComments(issueNumber int64) ([]*Comment, error)
GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
GetReviews(pullRequestNumber int64) ([]*Review, error)
FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
}
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
@ -31,213 +31,3 @@ type DownloaderFactory interface {
New(ctx context.Context, opts MigrateOptions) (Downloader, error)
GitServiceType() structs.GitServiceType
}
var (
_ Downloader = &RetryDownloader{}
)
// RetryDownloader retry the downloads
type RetryDownloader struct {
Downloader
ctx context.Context
RetryTimes int // the total execute times
RetryDelay int // time to delay seconds
}
// NewRetryDownloader creates a retry downloader
func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
return &RetryDownloader{
Downloader: downloader,
ctx: ctx,
RetryTimes: retryTimes,
RetryDelay: retryDelay,
}
}
// SetContext set context
func (d *RetryDownloader) SetContext(ctx context.Context) {
d.ctx = ctx
d.Downloader.SetContext(ctx)
}
// GetRepoInfo returns a repository information with retry
func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
var (
times = d.RetryTimes
repo *Repository
err error
)
for ; times > 0; times-- {
if repo, err = d.Downloader.GetRepoInfo(); err == nil {
return repo, nil
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetTopics returns a repository's topics with retry
func (d *RetryDownloader) GetTopics() ([]string, error) {
var (
times = d.RetryTimes
topics []string
err error
)
for ; times > 0; times-- {
if topics, err = d.Downloader.GetTopics(); err == nil {
return topics, nil
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetMilestones returns a repository's milestones with retry
func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
var (
times = d.RetryTimes
milestones []*Milestone
err error
)
for ; times > 0; times-- {
if milestones, err = d.Downloader.GetMilestones(); err == nil {
return milestones, nil
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetReleases returns a repository's releases with retry
func (d *RetryDownloader) GetReleases() ([]*Release, error) {
var (
times = d.RetryTimes
releases []*Release
err error
)
for ; times > 0; times-- {
if releases, err = d.Downloader.GetReleases(); err == nil {
return releases, nil
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetLabels returns a repository's labels with retry
func (d *RetryDownloader) GetLabels() ([]*Label, error) {
var (
times = d.RetryTimes
labels []*Label
err error
)
for ; times > 0; times-- {
if labels, err = d.Downloader.GetLabels(); err == nil {
return labels, nil
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetIssues returns a repository's issues with retry
func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
var (
times = d.RetryTimes
issues []*Issue
isEnd bool
err error
)
for ; times > 0; times-- {
if issues, isEnd, err = d.Downloader.GetIssues(page, perPage); err == nil {
return issues, isEnd, nil
}
select {
case <-d.ctx.Done():
return nil, false, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, false, err
}
// GetComments returns a repository's comments with retry
func (d *RetryDownloader) GetComments(issueNumber int64) ([]*Comment, error) {
var (
times = d.RetryTimes
comments []*Comment
err error
)
for ; times > 0; times-- {
if comments, err = d.Downloader.GetComments(issueNumber); err == nil {
return comments, nil
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetPullRequests returns a repository's pull requests with retry
func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
var (
times = d.RetryTimes
prs []*PullRequest
err error
isEnd bool
)
for ; times > 0; times-- {
if prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage); err == nil {
return prs, isEnd, nil
}
select {
case <-d.ctx.Done():
return nil, false, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, false, err
}
// GetReviews returns pull requests reviews
func (d *RetryDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
var (
times = d.RetryTimes
reviews []*Review
err error
)
for ; times > 0; times-- {
if reviews, err = d.Downloader.GetReviews(pullRequestNumber); err == nil {
return reviews, nil
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}

26
modules/migrations/base/error.go Normal file
View file

@ -0,0 +1,26 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
import "fmt"
// ErrNotSupported represents status if a downloader do not supported something.
type ErrNotSupported struct {
Entity string
}
// IsErrNotSupported checks if an error is an ErrNotSupported
func IsErrNotSupported(err error) bool {
_, ok := err.(ErrNotSupported)
return ok
}
// Error return error message
func (err ErrNotSupported) Error() string {
if len(err.Entity) != 0 {
return fmt.Sprintf("'%s' not supported", err.Entity)
}
return "not supported"
}

2
modules/migrations/base/milestone.go
View file

@ -15,5 +15,5 @@ type Milestone struct {
Created time.Time
Updated *time.Time
Closed *time.Time
State string
State string // open, closed
}

82
modules/migrations/base/null_downloader.go Normal file
View file

@ -0,0 +1,82 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
import (
"context"
"net/url"
)
// NullDownloader implements a blank downloader
type NullDownloader struct {
}
var (
_ Downloader = &NullDownloader{}
)
// SetContext set context
func (n NullDownloader) SetContext(_ context.Context) {}
// GetRepoInfo returns a repository information
func (n NullDownloader) GetRepoInfo() (*Repository, error) {
return nil, &ErrNotSupported{Entity: "RepoInfo"}
}
// GetTopics return repository topics
func (n NullDownloader) GetTopics() ([]string, error) {
return nil, &ErrNotSupported{Entity: "Topics"}
}
// GetMilestones returns milestones
func (n NullDownloader) GetMilestones() ([]*Milestone, error) {
return nil, &ErrNotSupported{Entity: "Milestones"}
}
// GetReleases returns releases
func (n NullDownloader) GetReleases() ([]*Release, error) {
return nil, &ErrNotSupported{Entity: "Releases"}
}
// GetLabels returns labels
func (n NullDownloader) GetLabels() ([]*Label, error) {
return nil, &ErrNotSupported{Entity: "Labels"}
}
// GetIssues returns issues according start and limit
func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
return nil, false, &ErrNotSupported{Entity: "Issues"}
}
// GetComments returns comments according issueNumber
func (n NullDownloader) GetComments(issueNumber int64) ([]*Comment, error) {
return nil, &ErrNotSupported{Entity: "Comments"}
}
// GetPullRequests returns pull requests according page and perPage
func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
return nil, false, &ErrNotSupported{Entity: "PullRequests"}
}
// GetReviews returns pull requests review
func (n NullDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
return nil, &ErrNotSupported{Entity: "Reviews"}
}
// FormatCloneURL add authentification into remote URLs
func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
if len(opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", opts.AuthToken)
}
return u.String(), nil
}
return remoteAddr, nil
}

247
modules/migrations/base/retry_downloader.go Normal file
View file

@ -0,0 +1,247 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
import (
"context"
"time"
)
var (
_ Downloader = &RetryDownloader{}
)
// RetryDownloader retry the downloads
type RetryDownloader struct {
Downloader
ctx context.Context
RetryTimes int // the total execute times
RetryDelay int // time to delay seconds
}
// NewRetryDownloader creates a retry downloader
func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
return &RetryDownloader{
Downloader: downloader,
ctx: ctx,
RetryTimes: retryTimes,
RetryDelay: retryDelay,
}
}
// SetContext set context
func (d *RetryDownloader) SetContext(ctx context.Context) {
d.ctx = ctx
d.Downloader.SetContext(ctx)
}
// GetRepoInfo returns a repository information with retry
func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
var (
times = d.RetryTimes
repo *Repository
err error
)
for ; times > 0; times-- {
if repo, err = d.Downloader.GetRepoInfo(); err == nil {
return repo, nil
}
if IsErrNotSupported(err) {
return nil, err
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetTopics returns a repository's topics with retry
func (d *RetryDownloader) GetTopics() ([]string, error) {
var (
times = d.RetryTimes
topics []string
err error
)
for ; times > 0; times-- {
if topics, err = d.Downloader.GetTopics(); err == nil {
return topics, nil
}
if IsErrNotSupported(err) {
return nil, err
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetMilestones returns a repository's milestones with retry
func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
var (
times = d.RetryTimes
milestones []*Milestone
err error
)
for ; times > 0; times-- {
if milestones, err = d.Downloader.GetMilestones(); err == nil {
return milestones, nil
}
if IsErrNotSupported(err) {
return nil, err
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetReleases returns a repository's releases with retry
func (d *RetryDownloader) GetReleases() ([]*Release, error) {
var (
times = d.RetryTimes
releases []*Release
err error
)
for ; times > 0; times-- {
if releases, err = d.Downloader.GetReleases(); err == nil {
return releases, nil
}
if IsErrNotSupported(err) {
return nil, err
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetLabels returns a repository's labels with retry
func (d *RetryDownloader) GetLabels() ([]*Label, error) {
var (
times = d.RetryTimes
labels []*Label
err error
)
for ; times > 0; times-- {
if labels, err = d.Downloader.GetLabels(); err == nil {
return labels, nil
}
if IsErrNotSupported(err) {
return nil, err
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetIssues returns a repository's issues with retry
func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
var (
times = d.RetryTimes
issues []*Issue
isEnd bool
err error
)
for ; times > 0; times-- {
if issues, isEnd, err = d.Downloader.GetIssues(page, perPage); err == nil {
return issues, isEnd, nil
}
if IsErrNotSupported(err) {
return nil, false, err
}
select {
case <-d.ctx.Done():
return nil, false, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, false, err
}
// GetComments returns a repository's comments with retry
func (d *RetryDownloader) GetComments(issueNumber int64) ([]*Comment, error) {
var (
times = d.RetryTimes
comments []*Comment
err error
)
for ; times > 0; times-- {
if comments, err = d.Downloader.GetComments(issueNumber); err == nil {
return comments, nil
}
if IsErrNotSupported(err) {
return nil, err
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}
// GetPullRequests returns a repository's pull requests with retry
func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
var (
times = d.RetryTimes
prs []*PullRequest
err error
isEnd bool
)
for ; times > 0; times-- {
if prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage); err == nil {
return prs, isEnd, nil
}
if IsErrNotSupported(err) {
return nil, false, err
}
select {
case <-d.ctx.Done():
return nil, false, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, false, err
}
// GetReviews returns pull requests reviews
func (d *RetryDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
var (
times = d.RetryTimes
reviews []*Review
err error
)
for ; times > 0; times-- {
if reviews, err = d.Downloader.GetReviews(pullRequestNumber); err == nil {
return reviews, nil
}
if IsErrNotSupported(err) {
return nil, err
}
select {
case <-d.ctx.Done():
return nil, d.ctx.Err()
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
}
}
return nil, err
}

3
modules/migrations/error.go
View file

@ -12,9 +12,6 @@ import (
)
var (
// ErrNotSupported returns the error not supported
ErrNotSupported = errors.New("not supported")
// ErrRepoNotCreated returns the error that repository not created
ErrRepoNotCreated = errors.New("repository is not created yet")
)

40
modules/migrations/git.go
View file

@ -16,6 +16,7 @@ var (
// PlainGitDownloader implements a Downloader interface to clone git from a http/https URL
type PlainGitDownloader struct {
base.NullDownloader
ownerName string
repoName string
remoteURL string
@ -44,42 +45,7 @@ func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
}, nil
}
// GetTopics returns empty list for plain git repo
func (g *PlainGitDownloader) GetTopics() ([]string, error) {
// GetTopics return empty string slice
func (g PlainGitDownloader) GetTopics() ([]string, error) {
return []string{}, nil
}
// GetMilestones returns milestones
func (g *PlainGitDownloader) GetMilestones() ([]*base.Milestone, error) {
return nil, ErrNotSupported
}
// GetLabels returns labels
func (g *PlainGitDownloader) GetLabels() ([]*base.Label, error) {
return nil, ErrNotSupported
}
// GetReleases returns releases
func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
return nil, ErrNotSupported
}
// GetIssues returns issues according page and perPage
func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
return nil, false, ErrNotSupported
}
// GetComments returns comments according issueNumber
func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
return nil, ErrNotSupported
}
// GetPullRequests returns pull requests according page and perPage
func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, bool, error) {
return nil, false, ErrNotSupported
}
// GetReviews returns reviews according issue number
func (g *PlainGitDownloader) GetReviews(issueNumber int64) ([]*base.Review, error) {
return nil, ErrNotSupported
}

3
modules/migrations/gitea_downloader.go
View file

@ -69,6 +69,7 @@ func (f *GiteaDownloaderFactory) GitServiceType() structs.GitServiceType {
// GiteaDownloader implements a Downloader interface to get repository information's
type GiteaDownloader struct {
base.NullDownloader
ctx context.Context
client *gitea_sdk.Client
repoOwner string
@ -95,7 +96,7 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
path := strings.Split(repoPath, "/")
paginationSupport := true
if err := giteaClient.CheckServerVersionConstraint(">=1.12"); err != nil {
if err = giteaClient.CheckServerVersionConstraint(">=1.12"); err != nil {
paginationSupport = false
}

23
modules/migrations/gitea_uploader.go
View file

@ -10,7 +10,6 @@ import (
"context"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strings"
@ -86,22 +85,6 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
return 10
}
func fullURL(opts base.MigrateOptions, remoteAddr string) (string, error) {
var fullRemoteAddr = remoteAddr
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
if len(opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", opts.AuthToken)
}
fullRemoteAddr = u.String()
}
return fullRemoteAddr, nil
}
// CreateRepo creates a repository
func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
owner, err := models.GetUserByName(g.repoOwner)
@ -109,10 +92,6 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
return err
}
remoteAddr, err := fullURL(opts, repo.CloneURL)
if err != nil {
return err
}
var r *models.Repository
if opts.MigrateToRepoID <= 0 {
r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{
@ -138,7 +117,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
OriginalURL: repo.OriginalURL,
GitServiceType: opts.GitServiceType,
Mirror: repo.IsMirror,
CloneAddr: remoteAddr,
CloneAddr: repo.CloneURL,
Private: repo.IsPrivate,
Wiki: opts.Wiki,
Releases: opts.Releases, // if didn't get releases, then sync them from tags

1
modules/migrations/github.go
View file

@ -65,6 +65,7 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
// GithubDownloaderV3 implements a Downloader interface to get repository informations
// from github via APIv3
type GithubDownloaderV3 struct {
base.NullDownloader
ctx context.Context
client *github.Client
repoOwner string

1
modules/migrations/gitlab.go
View file

@ -63,6 +63,7 @@ func (f *GitlabDownloaderFactory) GitServiceType() structs.GitServiceType {
// - issueSeen, working alongside issueCount, is checked in GetComments() to see whether we
// need to fetch the Issue or PR comments, as Gitlab stores them separately.
type GitlabDownloader struct {
base.NullDownloader
ctx context.Context
client *gitlab.Client
repoID int

312
modules/migrations/gogs.go Normal file
View file

@ -0,0 +1,312 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/structs"
"github.com/gogs/go-gogs-client"
)
var (
_ base.Downloader = &GogsDownloader{}
_ base.DownloaderFactory = &GogsDownloaderFactory{}
)
func init() {
RegisterDownloaderFactory(&GogsDownloaderFactory{})
}
// GogsDownloaderFactory defines a gogs downloader factory
type GogsDownloaderFactory struct {
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GogsDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
baseURL := u.Scheme + "://" + u.Host
repoNameSpace := strings.TrimSuffix(u.Path, ".git")
repoNameSpace = strings.Trim(repoNameSpace, "/")
fields := strings.Split(repoNameSpace, "/")
if len(fields) < 2 {
return nil, fmt.Errorf("invalid path: %s", repoNameSpace)
}
log.Trace("Create gogs downloader. BaseURL: %s RepoOwner: %s RepoName: %s", baseURL, fields[0], fields[1])
return NewGogsDownloader(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, fields[0], fields[1]), nil
}
// GitServiceType returns the type of git service
func (f *GogsDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.GogsService
}
// GogsDownloader implements a Downloader interface to get repository informations
// from gogs via API
type GogsDownloader struct {
base.NullDownloader
ctx context.Context
client *gogs.Client
baseURL string
repoOwner string
repoName string
userName string
password string
openIssuesFinished bool
openIssuesPages int
transport http.RoundTripper
}
// SetContext set context
func (g *GogsDownloader) SetContext(ctx context.Context) {
g.ctx = ctx
}
// NewGogsDownloader creates a gogs Downloader via gogs API
func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader {
var downloader = GogsDownloader{
ctx: ctx,
baseURL: baseURL,
userName: userName,
password: password,
repoOwner: repoOwner,
repoName: repoName,
}
var client *gogs.Client
if len(token) != 0 {
client = gogs.NewClient(baseURL, token)
downloader.userName = token
} else {
downloader.transport = &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return nil, nil
},
}
client = gogs.NewClient(baseURL, "")
client.SetHTTPClient(&http.Client{
Transport: &downloader,
})
}
downloader.client = client
return &downloader
}
// RoundTrip wraps the provided request within this downloader's context and passes it to our internal http.Transport.
// This implements http.RoundTripper and makes the gogs client requests cancellable even though it is not cancellable itself
func (g *GogsDownloader) RoundTrip(req *http.Request) (*http.Response, error) {
return g.transport.RoundTrip(req.WithContext(g.ctx))
}
// GetRepoInfo returns a repository information
func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) {
gr, err := g.client.GetRepo(g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
// convert gogs repo to stand Repo
return &base.Repository{
Owner: g.repoOwner,
Name: g.repoName,
IsPrivate: gr.Private,
Description: gr.Description,
CloneURL: gr.CloneURL,
OriginalURL: gr.HTMLURL,
DefaultBranch: gr.DefaultBranch,
}, nil
}
// GetMilestones returns milestones
func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100
var milestones = make([]*base.Milestone, 0, perPage)
ms, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
t := time.Now()
for _, m := range ms {
milestones = append(milestones, &base.Milestone{
Title: m.Title,
Description: m.Description,
Deadline: m.Deadline,
State: string(m.State),
Created: t,
Updated: &t,
Closed: m.Closed,
})
}
return milestones, nil
}
// GetLabels returns labels
func (g *GogsDownloader) GetLabels() ([]*base.Label, error) {
var perPage = 100
var labels = make([]*base.Label, 0, perPage)
ls, err := g.client.ListRepoLabels(g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
for _, label := range ls {
labels = append(labels, convertGogsLabel(label))
}
return labels, nil
}
// GetIssues returns issues according start and limit, perPage is not supported
func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) {
var state string
if g.openIssuesFinished {
state = string(gogs.STATE_CLOSED)
page -= g.openIssuesPages
} else {
state = string(gogs.STATE_OPEN)
g.openIssuesPages = page
}
issues, isEnd, err := g.getIssues(page, state)
if err != nil {
return nil, false, err
}
if isEnd {
if g.openIssuesFinished {
return issues, true, nil
}
g.openIssuesFinished = true
}
return issues, false, nil
}
func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, error) {
var allIssues = make([]*base.Issue, 0, 10)
issues, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{
Page: page,
State: state,
})
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
for _, issue := range issues {
if issue.PullRequest != nil {
continue
}
allIssues = append(allIssues, convertGogsIssue(issue))
}
return allIssues, len(issues) == 0, nil
}
// GetComments returns comments according issueNumber
func (g *GogsDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
var allComments = make([]*base.Comment, 0, 100)
comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, issueNumber)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
for _, comment := range comments {
if len(comment.Body) == 0 || comment.Poster == nil {
continue
}
allComments = append(allComments, &base.Comment{
IssueIndex: issueNumber,
PosterID: comment.Poster.ID,
PosterName: comment.Poster.Login,
PosterEmail: comment.Poster.Email,
Content: comment.Body,
Created: comment.Created,
Updated: comment.Updated,
})
}
return allComments, nil
}
// GetTopics return repository topics
func (g *GogsDownloader) GetTopics() ([]string, error) {
return []string{}, nil
}
// FormatCloneURL add authentification into remote URLs
func (g *GogsDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
if len(opts.AuthToken) != 0 {
u.User = url.UserPassword(opts.AuthToken, "")
} else {
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
}
return u.String(), nil
}
return remoteAddr, nil
}
func convertGogsIssue(issue *gogs.Issue) *base.Issue {
var milestone string
if issue.Milestone != nil {
milestone = issue.Milestone.Title
}
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, convertGogsLabel(l))
}
var closed *time.Time
if issue.State == gogs.STATE_CLOSED {
// gogs client haven't provide closed, so we use updated instead
closed = &issue.Updated
}
return &base.Issue{
Title: issue.Title,
Number: issue.Index,
PosterName: issue.Poster.Login,
PosterEmail: issue.Poster.Email,
Content: issue.Body,
Milestone: milestone,
State: string(issue.State),
Created: issue.Created,
Labels: labels,
Closed: closed,
}
}
func convertGogsLabel(label *gogs.Label) *base.Label {
return &base.Label{
Name: label.Name,
Color: label.Color,
}
}

122
modules/migrations/gogs_test.go Normal file
View file

@ -0,0 +1,122 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"net/http"
"os"
"testing"
"time"
"code.gitea.io/gitea/modules/migrations/base"
"github.com/stretchr/testify/assert"
)
func TestGogsDownloadRepo(t *testing.T) {
// Skip tests if Gogs token is not found
gogsPersonalAccessToken := os.Getenv("GOGS_READ_TOKEN")
if len(gogsPersonalAccessToken) == 0 {
t.Skip("skipped test because GOGS_READ_TOKEN was not in the environment")
}
resp, err := http.Get("https://try.gogs.io/lunnytest/TESTREPO")
if err != nil || resp.StatusCode/100 != 2 {
// skip and don't run test
t.Skipf("visit test repo failed, ignored")
return
}
downloader := NewGogsDownloader(context.Background(), "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO")
repo, err := downloader.GetRepoInfo()
assert.NoError(t, err)
assert.EqualValues(t, &base.Repository{
Name: "TESTREPO",
Owner: "lunnytest",
Description: "",
CloneURL: "https://try.gogs.io/lunnytest/TESTREPO.git",
}, repo)
milestones, err := downloader.GetMilestones()
assert.NoError(t, err)
assert.True(t, len(milestones) == 1)
for _, milestone := range milestones {
switch milestone.Title {
case "1.0":
assert.EqualValues(t, "open", milestone.State)
}
}
labels, err := downloader.GetLabels()
assert.NoError(t, err)
assert.Len(t, labels, 7)
for _, l := range labels {
switch l.Name {
case "bug":
assertLabelEqual(t, "bug", "ee0701", "", l)
case "duplicated":
assertLabelEqual(t, "duplicated", "cccccc", "", l)
case "enhancement":
assertLabelEqual(t, "enhancement", "84b6eb", "", l)
case "help wanted":
assertLabelEqual(t, "help wanted", "128a0c", "", l)
case "invalid":
assertLabelEqual(t, "invalid", "e6e6e6", "", l)
case "question":
assertLabelEqual(t, "question", "cc317c", "", l)
case "wontfix":
assertLabelEqual(t, "wontfix", "ffffff", "", l)
}
}
_, err = downloader.GetReleases()
assert.Error(t, err)
// downloader.GetIssues()
issues, isEnd, err := downloader.GetIssues(1, 8)
assert.NoError(t, err)
assert.EqualValues(t, 1, len(issues))
assert.False(t, isEnd)
assert.EqualValues(t, []*base.Issue{
{
Number: 1,
Title: "test",
Content: "test",
Milestone: "",
PosterName: "lunny",
PosterEmail: "xiaolunwen@gmail.com",
State: "open",
Created: time.Date(2019, 06, 11, 8, 16, 44, 0, time.UTC),
Labels: []*base.Label{
{
Name: "bug",
Color: "ee0701",
},
},
},
}, issues)
// downloader.GetComments()
comments, err := downloader.GetComments(1)
assert.NoError(t, err)
assert.EqualValues(t, 1, len(comments))
assert.EqualValues(t, []*base.Comment{
{
PosterName: "lunny",
PosterEmail: "xiaolunwen@gmail.com",
Created: time.Date(2019, 06, 11, 8, 19, 50, 0, time.UTC),
Updated: time.Date(2019, 06, 11, 8, 19, 50, 0, time.UTC),
Content: `1111`,
},
}, comments)
// downloader.GetPullRequests()
_, _, err = downloader.GetPullRequests(1, 3)
assert.Error(t, err)
}

83
modules/migrations/migrate.go
View file

@ -133,15 +133,22 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio
func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error {
repo, err := downloader.GetRepoInfo()
if err != nil {
return err
if !base.IsErrNotSupported(err) {
return err
}
log.Info("migrating repo infos is not supported, ignored")
}
repo.IsPrivate = opts.Private
repo.IsMirror = opts.Mirror
if opts.Description != "" {
repo.Description = opts.Description
}
if repo.CloneURL, err = downloader.FormatCloneURL(opts, repo.CloneURL); err != nil {
return err
}
log.Trace("migrating git data")
if err := uploader.CreateRepo(repo, opts); err != nil {
if err = uploader.CreateRepo(repo, opts); err != nil {
return err
}
defer uploader.Close()
@ -149,10 +156,13 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
log.Trace("migrating topics")
topics, err := downloader.GetTopics()
if err != nil {
return err
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating topics is not supported, ignored")
}
if len(topics) > 0 {
if err := uploader.CreateTopics(topics...); err != nil {
if len(topics) != 0 {
if err = uploader.CreateTopics(topics...); err != nil {
return err
}
}
@ -161,7 +171,10 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
log.Trace("migrating milestones")
milestones, err := downloader.GetMilestones()
if err != nil {
return err
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating milestones is not supported, ignored")
}
msBatchSize := uploader.MaxBatchInsertSize("milestone")
@ -181,7 +194,10 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
log.Trace("migrating labels")
labels, err := downloader.GetLabels()
if err != nil {
return err
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating labels is not supported, ignored")
}
lbBatchSize := uploader.MaxBatchInsertSize("label")
@ -201,7 +217,10 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
log.Trace("migrating releases")
releases, err := downloader.GetReleases()
if err != nil {
return err
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating releases is not supported, ignored")
}
relBatchSize := uploader.MaxBatchInsertSize("release")
@ -210,14 +229,14 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
relBatchSize = len(releases)
}
if err := uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
return err
}
releases = releases[relBatchSize:]
}
// Once all releases (if any) are inserted, sync any remaining non-release tags
if err := uploader.SyncTags(); err != nil {
if err = uploader.SyncTags(); err != nil {
return err
}
}
@ -234,7 +253,11 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
for i := 1; ; i++ {
issues, isEnd, err := downloader.GetIssues(i, issueBatchSize)
if err != nil {
return err
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating issues is not supported, ignored")
break
}
if err := uploader.CreateIssues(issues...); err != nil {
@ -247,13 +270,16 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
log.Trace("migrating issue %d's comments", issue.Number)
comments, err := downloader.GetComments(issue.Number)
if err != nil {
return err
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating comments is not supported, ignored")
}
allComments = append(allComments, comments...)
if len(allComments) >= commentBatchSize {
if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
return err
}
@ -262,7 +288,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
}
if len(allComments) > 0 {
if err := uploader.CreateComments(allComments...); err != nil {
if err = uploader.CreateComments(allComments...); err != nil {
return err
}
}
@ -280,7 +306,11 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
for i := 1; ; i++ {
prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)
if err != nil {
return err
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating pull requests is not supported, ignored")
break
}
if err := uploader.CreatePullRequests(prs...); err != nil {
@ -294,20 +324,23 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
log.Trace("migrating pull request %d's comments", pr.Number)
comments, err := downloader.GetComments(pr.Number)
if err != nil {
return err
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating comments is not supported, ignored")
}
allComments = append(allComments, comments...)
if len(allComments) >= commentBatchSize {
if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
return err
}
allComments = allComments[commentBatchSize:]
}
}
if len(allComments) > 0 {
if err := uploader.CreateComments(allComments...); err != nil {
if err = uploader.CreateComments(allComments...); err != nil {
return err
}
}
@ -323,26 +356,30 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
}
reviews, err := downloader.GetReviews(number)
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating reviews is not supported, ignored")
break
}
if pr.OriginalNumber > 0 {
for i := range reviews {
reviews[i].IssueIndex = pr.Number
}
}
if err != nil {
return err
}
allReviews = append(allReviews, reviews...)
if len(allReviews) >= reviewBatchSize {
if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
return err
}
allReviews = allReviews[reviewBatchSize:]
}
}
if len(allReviews) > 0 {
if err := uploader.CreateReviews(allReviews...); err != nil {
if err = uploader.CreateReviews(allReviews...); err != nil {
return err
}
}

1
modules/migrations/restore.go
View file

@ -19,6 +19,7 @@ import (
// RepositoryRestorer implements an Downloader from the local directory
type RepositoryRestorer struct {
base.NullDownloader
ctx context.Context
baseDir string
repoOwner string