Add default storage configurations (#12813)

Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: zeripath <art27@cantab.net>

modules/storage/local.go

@@ -59,7 +59,7 @@ func (l *LocalStorage) Save(path string, r io.Reader) (int64, error) {
 }
 
 // Stat returns the info of the file
-func (l *LocalStorage) Stat(path string) (ObjectInfo, error) {
+func (l *LocalStorage) Stat(path string) (os.FileInfo, error) {
 	return os.Stat(filepath.Join(l.dir, path))
 }
 
@@ -73,3 +73,28 @@ func (l *LocalStorage) Delete(path string) error {
 func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
 	return nil, ErrURLNotSupported
 }
+
+// IterateObjects iterates across the objects in the local storage
+func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error {
+	return filepath.Walk(l.dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if path == l.dir {
+			return nil
+		}
+		if info.IsDir() {
+			return nil
+		}
+		relPath, err := filepath.Rel(l.dir, path)
+		if err != nil {
+			return err
+		}
+		obj, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer obj.Close()
+		return fn(relPath, obj)
+	})
+}

modules/storage/minio.go

@@ -22,6 +22,19 @@ var (
 	quoteEscaper               = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
 )
 
+type minioObject struct {
+	*minio.Object
+}
+
+func (m *minioObject) Stat() (os.FileInfo, error) {
+	oi, err := m.Object.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	return &minioFileInfo{oi}, nil
+}
+
 // MinioStorage returns a minio bucket storage
 type MinioStorage struct {
 	ctx      context.Context
@@ -69,7 +82,7 @@ func (m *MinioStorage) Open(path string) (Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	return object, nil
+	return &minioObject{object}, nil
 }
 
 // Save save a file to minio
@@ -104,8 +117,20 @@ func (m minioFileInfo) ModTime() time.Time {
 	return m.LastModified
 }
 
+func (m minioFileInfo) IsDir() bool {
+	return strings.HasSuffix(m.ObjectInfo.Key, "/")
+}
+
+func (m minioFileInfo) Mode() os.FileMode {
+	return os.ModePerm
+}
+
+func (m minioFileInfo) Sys() interface{} {
+	return nil
+}
+
 // Stat returns the stat information of the object
-func (m *MinioStorage) Stat(path string) (ObjectInfo, error) {
+func (m *MinioStorage) Stat(path string) (os.FileInfo, error) {
 	info, err := m.client.StatObject(
 		m.ctx,
 		m.bucket,
@@ -135,3 +160,26 @@ func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
 	reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
 	return m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)
 }
+
+// IterateObjects iterates across the objects in the miniostorage
+func (m *MinioStorage) IterateObjects(fn func(path string, obj Object) error) error {
+	var opts = minio.GetObjectOptions{}
+	lobjectCtx, cancel := context.WithCancel(m.ctx)
+	defer cancel()
+	for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{
+		Prefix:    m.basePath,
+		Recursive: true,
+	}) {
+		object, err := m.client.GetObject(lobjectCtx, m.bucket, mObjInfo.Key, opts)
+		if err != nil {
+			return err
+		}
+		if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
+			defer object.Close()
+			return fn(strings.TrimPrefix(m.basePath, mObjInfo.Key), &minioObject{object})
+		}(object, fn); err != nil {
+			return err
+		}
+	}
+	return nil
+}

modules/storage/storage.go

@@ -10,7 +10,7 @@ import (
 	"fmt"
 	"io"
 	"net/url"
-	"time"
+	"os"
 
 	"code.gitea.io/gitea/modules/setting"
 )
@@ -18,28 +18,25 @@ import (
 var (
 	// ErrURLNotSupported represents url is not supported
 	ErrURLNotSupported = errors.New("url method not supported")
+	// ErrIterateObjectsNotSupported represents IterateObjects not supported
+	ErrIterateObjectsNotSupported = errors.New("iterateObjects method not supported")
 )
 
 // Object represents the object on the storage
 type Object interface {
 	io.ReadCloser
 	io.Seeker
-}
-
-// ObjectInfo represents the object info on the storage
-type ObjectInfo interface {
-	Name() string
-	Size() int64
-	ModTime() time.Time
+	Stat() (os.FileInfo, error)
 }
 
 // ObjectStorage represents an object storage to handle a bucket and files
 type ObjectStorage interface {
 	Open(path string) (Object, error)
 	Save(path string, r io.Reader) (int64, error)
-	Stat(path string) (ObjectInfo, error)
+	Stat(path string) (os.FileInfo, error)
 	Delete(path string) error
 	URL(path, name string) (*url.URL, error)
+	IterateObjects(func(path string, obj Object) error) error
 }
 
 // Copy copys a file from source ObjectStorage to dest ObjectStorage
@@ -70,14 +67,15 @@ func Init() error {
 	return initLFS()
 }
 
-func initAttachments() error {
+func initStorage(storageCfg setting.Storage) (ObjectStorage, error) {
 	var err error
-	switch setting.Attachment.StoreType {
-	case "local":
-		Attachments, err = NewLocalStorage(setting.Attachment.Path)
-	case "minio":
-		minio := setting.Attachment.Minio
-		Attachments, err = NewMinioStorage(
+	var s ObjectStorage
+	switch storageCfg.Type {
+	case setting.LocalStorageType:
+		s, err = NewLocalStorage(storageCfg.Path)
+	case setting.MinioStorageType:
+		minio := storageCfg.Minio
+		s, err = NewMinioStorage(
 			context.Background(),
 			minio.Endpoint,
 			minio.AccessKeyID,
@@ -88,40 +86,22 @@ func initAttachments() error {
 			minio.UseSSL,
 		)
 	default:
-		return fmt.Errorf("Unsupported attachment store type: %s", setting.Attachment.StoreType)
+		return nil, fmt.Errorf("Unsupported attachment store type: %s", storageCfg.Type)
 	}
 
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	return nil
+	return s, nil
 }
 
-func initLFS() error {
-	var err error
-	switch setting.LFS.StoreType {
-	case "local":
-		LFS, err = NewLocalStorage(setting.LFS.ContentPath)
-	case "minio":
-		minio := setting.LFS.Minio
-		LFS, err = NewMinioStorage(
-			context.Background(),
-			minio.Endpoint,
-			minio.AccessKeyID,
-			minio.SecretAccessKey,
-			minio.Bucket,
-			minio.Location,
-			minio.BasePath,
-			minio.UseSSL,
-		)
-	default:
-		return fmt.Errorf("Unsupported LFS store type: %s", setting.LFS.StoreType)
-	}
-
-	if err != nil {
-		return err
-	}
-
-	return nil
+func initAttachments() (err error) {
+	Attachments, err = initStorage(setting.Attachment.Storage)
+	return
+}
+
+func initLFS() (err error) {
+	LFS, err = initStorage(setting.LFS.Storage)
+	return
 }