Add a storage layer for attachments (#11387)

* Add a storage layer for attachments

* Fix some bug

* fix test

* Fix copyright head and lint

* Fix bug

* Add setting for minio and flags for migrate-storage

* Add documents

* fix lint

* Add test for minio store type on attachments

* fix test

* fix test

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Add warning when storage migrated successfully

* Fix drone

* fix test

* rebase

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* remove log on xorm

* Fi download bug

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* Add URL function to serve attachments directly from S3/Minio

* Add ability to enable/disable redirection in attachment configuration

* Fix typo

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* don't change unrelated files

* Fix lint

* Fix build

* update go.mod and go.sum

* Use github.com/minio/minio-go/v6

* Remove unused function

* Upgrade minio to v7 and some other improvements

* fix lint

* Fix go mod

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
Co-authored-by: Tyler <tystuyfzand@gmail.com>

modules/storage/local.go

@@ -0,0 +1,70 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package storage
+
+import (
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+var (
+	_ ObjectStorage = &LocalStorage{}
+)
+
+// LocalStorage represents a local files storage
+type LocalStorage struct {
+	dir string
+}
+
+// NewLocalStorage returns a local files
+func NewLocalStorage(bucket string) (*LocalStorage, error) {
+	if err := os.MkdirAll(bucket, os.ModePerm); err != nil {
+		return nil, err
+	}
+
+	return &LocalStorage{
+		dir: bucket,
+	}, nil
+}
+
+// Open a file
+func (l *LocalStorage) Open(path string) (io.ReadCloser, error) {
+	return os.Open(filepath.Join(l.dir, path))
+}
+
+// Save a file
+func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) {
+	p := filepath.Join(l.dir, path)
+	if err := os.MkdirAll(filepath.Dir(p), os.ModePerm); err != nil {
+		return 0, err
+	}
+
+	// always override
+	if err := util.Remove(p); err != nil {
+		return 0, err
+	}
+
+	f, err := os.Create(p)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	return io.Copy(f, r)
+}
+
+// Delete delete a file
+func (l *LocalStorage) Delete(path string) error {
+	p := filepath.Join(l.dir, path)
+	return util.Remove(p)
+}
+
+// URL gets the redirect URL to a file
+func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
+	return nil, ErrURLNotSupported
+}

modules/storage/minio.go

@@ -0,0 +1,101 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package storage
+
+import (
+	"context"
+	"io"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+var (
+	_            ObjectStorage = &MinioStorage{}
+	quoteEscaper               = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+)
+
+// MinioStorage returns a minio bucket storage
+type MinioStorage struct {
+	ctx      context.Context
+	client   *minio.Client
+	bucket   string
+	basePath string
+}
+
+// NewMinioStorage returns a minio storage
+func NewMinioStorage(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucket, location, basePath string, useSSL bool) (*MinioStorage, error) {
+	minioClient, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: useSSL,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if err := minioClient.MakeBucket(ctx, bucket, minio.MakeBucketOptions{
+		Region: location,
+	}); err != nil {
+		// Check to see if we already own this bucket (which happens if you run this twice)
+		exists, errBucketExists := minioClient.BucketExists(ctx, bucket)
+		if !exists || errBucketExists != nil {
+			return nil, err
+		}
+	}
+
+	return &MinioStorage{
+		ctx:      ctx,
+		client:   minioClient,
+		bucket:   bucket,
+		basePath: basePath,
+	}, nil
+}
+
+func (m *MinioStorage) buildMinioPath(p string) string {
+	return strings.TrimPrefix(path.Join(m.basePath, p), "/")
+}
+
+// Open open a file
+func (m *MinioStorage) Open(path string) (io.ReadCloser, error) {
+	var opts = minio.GetObjectOptions{}
+	object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
+	if err != nil {
+		return nil, err
+	}
+	return object, nil
+}
+
+// Save save a file to minio
+func (m *MinioStorage) Save(path string, r io.Reader) (int64, error) {
+	uploadInfo, err := m.client.PutObject(
+		m.ctx,
+		m.bucket,
+		m.buildMinioPath(path),
+		r,
+		-1,
+		minio.PutObjectOptions{ContentType: "application/octet-stream"},
+	)
+	if err != nil {
+		return 0, err
+	}
+	return uploadInfo.Size, nil
+}
+
+// Delete delete a file
+func (m *MinioStorage) Delete(path string) error {
+	return m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{})
+}
+
+// URL gets the redirect URL to a file. The presigned link is valid for 5 minutes.
+func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
+	reqParams := make(url.Values)
+	// TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we?
+	reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
+	return m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)
+}

modules/storage/storage.go

@@ -0,0 +1,73 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package storage
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var (
+	// ErrURLNotSupported represents url is not supported
+	ErrURLNotSupported = errors.New("url method not supported")
+)
+
+// ObjectStorage represents an object storage to handle a bucket and files
+type ObjectStorage interface {
+	Save(path string, r io.Reader) (int64, error)
+	Open(path string) (io.ReadCloser, error)
+	Delete(path string) error
+	URL(path, name string) (*url.URL, error)
+}
+
+// Copy copys a file from source ObjectStorage to dest ObjectStorage
+func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, srcPath string) (int64, error) {
+	f, err := srcStorage.Open(srcPath)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+
+	return dstStorage.Save(dstPath, f)
+}
+
+var (
+	// Attachments represents attachments storage
+	Attachments ObjectStorage
+)
+
+// Init init the stoarge
+func Init() error {
+	var err error
+	switch setting.Attachment.StoreType {
+	case "local":
+		Attachments, err = NewLocalStorage(setting.Attachment.Path)
+	case "minio":
+		minio := setting.Attachment.Minio
+		Attachments, err = NewMinioStorage(
+			context.Background(),
+			minio.Endpoint,
+			minio.AccessKeyID,
+			minio.SecretAccessKey,
+			minio.Bucket,
+			minio.Location,
+			minio.BasePath,
+			minio.UseSSL,
+		)
+	default:
+		return fmt.Errorf("Unsupported attachment store type: %s", setting.Attachment.StoreType)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}