mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-05-14 05:52:43 +00:00
LFS support to be stored on minio (#12518)
* LFS support to be stored on minio * Fix test * Fix lint * Fix lint * Fix check * Fix test * Update documents and add migration for LFS * Fix some bugs
This commit is contained in:
Lunny Xiao
GitHub
parent
e4b3f35b8d
commit
7a5465fc56
18 changed files with 423 additions and 203 deletions
|
|
@ -10,11 +10,10 @@ import (
|
|||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
"code.gitea.io/gitea/modules/storage"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -24,17 +23,15 @@ var (
|
|||
|
||||
// ContentStore provides a simple file system based storage.
|
||||
type ContentStore struct {
|
||||
BasePath string
|
||||
storage.ObjectStorage
|
||||
}
|
||||
|
||||
// Get takes a Meta object and retrieves the content from the store, returning
|
||||
// it as an io.Reader. If fromByte > 0, the reader starts from that byte
|
||||
func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
|
||||
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
|
||||
|
||||
f, err := os.Open(path)
|
||||
f, err := s.Open(meta.RelativePath())
|
||||
if err != nil {
|
||||
log.Error("Whilst trying to read LFS OID[%s]: Unable to open %s Error: %v", meta.Oid, path, err)
|
||||
log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
|
||||
return nil, err
|
||||
}
|
||||
if fromByte > 0 {
|
||||
|
|
@ -48,82 +45,55 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
|
|||
|
||||
// Put takes a Meta object and an io.Reader and writes the content to the store.
|
||||
func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
|
||||
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
|
||||
tmpPath := path + ".tmp"
|
||||
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0750); err != nil {
|
||||
log.Error("Whilst putting LFS OID[%s]: Unable to create the LFS directory: %s Error: %v", meta.Oid, dir, err)
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
|
||||
if err != nil {
|
||||
log.Error("Whilst putting LFS OID[%s]: Unable to open temporary file for writing: %s Error: %v", tmpPath, err)
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := util.Remove(tmpPath); err != nil {
|
||||
log.Warn("Unable to remove temporary path: %s: Error: %v", tmpPath, err)
|
||||
}
|
||||
}()
|
||||
|
||||
hash := sha256.New()
|
||||
hw := io.MultiWriter(hash, file)
|
||||
|
||||
written, err := io.Copy(hw, r)
|
||||
rd := io.TeeReader(r, hash)
|
||||
p := meta.RelativePath()
|
||||
written, err := s.Save(p, rd)
|
||||
if err != nil {
|
||||
log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, tmpPath, err)
|
||||
file.Close()
|
||||
log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
|
||||
return err
|
||||
}
|
||||
file.Close()
|
||||
|
||||
if written != meta.Size {
|
||||
if err := s.Delete(p); err != nil {
|
||||
log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
|
||||
}
|
||||
return errSizeMismatch
|
||||
}
|
||||
|
||||
shaStr := hex.EncodeToString(hash.Sum(nil))
|
||||
if shaStr != meta.Oid {
|
||||
if err := s.Delete(p); err != nil {
|
||||
log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
|
||||
}
|
||||
return errHashMismatch
|
||||
}
|
||||
|
||||
if err := os.Rename(tmpPath, path); err != nil {
|
||||
log.Error("Whilst putting LFS OID[%s]: Unable to move tmp file to final destination: %s Error: %v", meta.Oid, path, err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists returns true if the object exists in the content store.
|
||||
func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool {
|
||||
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
return false
|
||||
func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
|
||||
_, err := s.ObjectStorage.Stat(meta.RelativePath())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Verify returns true if the object exists in the content store and size is correct.
|
||||
func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
|
||||
path := filepath.Join(s.BasePath, transformKey(meta.Oid))
|
||||
|
||||
fi, err := os.Stat(path)
|
||||
if os.IsNotExist(err) || err == nil && fi.Size() != meta.Size {
|
||||
p := meta.RelativePath()
|
||||
fi, err := s.ObjectStorage.Stat(p)
|
||||
if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", path, meta.Oid, err)
|
||||
log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err)
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func transformKey(key string) string {
|
||||
if len(key) < 5 {
|
||||
return key
|
||||
}
|
||||
|
||||
return filepath.Join(key[0:2], key[2:4], key[4:])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/base"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/storage"
|
||||
)
|
||||
|
||||
// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
|
||||
|
|
@ -53,9 +54,10 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
|
|||
return nil
|
||||
}
|
||||
|
||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
||||
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||
meta := &models.LFSMetaObject{Oid: oid, Size: size}
|
||||
if !contentStore.Exists(meta) {
|
||||
exist, err := contentStore.Exists(meta)
|
||||
if err != nil || !exist {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -64,6 +66,6 @@ func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
|
|||
|
||||
// ReadMetaObject will read a models.LFSMetaObject and return a reader
|
||||
func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
|
||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
||||
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||
return contentStore.Get(meta, 0)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import (
|
|||
"code.gitea.io/gitea/modules/context"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/storage"
|
||||
|
||||
"gitea.com/macaron/macaron"
|
||||
"github.com/dgrijalva/jwt-go"
|
||||
|
|
@ -187,7 +188,7 @@ func getContentHandler(ctx *context.Context) {
|
|||
}
|
||||
}
|
||||
|
||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
||||
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||
content, err := contentStore.Get(meta, fromByte)
|
||||
if err != nil {
|
||||
// Errors are logged in contentStore.Get
|
||||
|
|
@ -288,8 +289,14 @@ func PostHandler(ctx *context.Context) {
|
|||
ctx.Resp.Header().Set("Content-Type", metaMediaType)
|
||||
|
||||
sentStatus := 202
|
||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
||||
if meta.Existing && contentStore.Exists(meta) {
|
||||
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||
exist, err := contentStore.Exists(meta)
|
||||
if err != nil {
|
||||
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err)
|
||||
writeStatus(ctx, 500)
|
||||
return
|
||||
}
|
||||
if meta.Existing && exist {
|
||||
sentStatus = 200
|
||||
}
|
||||
ctx.Resp.WriteHeader(sentStatus)
|
||||
|
|
@ -343,12 +350,20 @@ func BatchHandler(ctx *context.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
||||
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||
|
||||
meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
|
||||
if err == nil && contentStore.Exists(meta) { // Object is found and exists
|
||||
responseObjects = append(responseObjects, Represent(object, meta, true, false))
|
||||
continue
|
||||
if err == nil { // Object is found and exists
|
||||
exist, err := contentStore.Exists(meta)
|
||||
if err != nil {
|
||||
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
|
||||
writeStatus(ctx, 500)
|
||||
return
|
||||
}
|
||||
if exist {
|
||||
responseObjects = append(responseObjects, Represent(object, meta, true, false))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize {
|
||||
|
|
@ -360,7 +375,13 @@ func BatchHandler(ctx *context.Context) {
|
|||
// Object is not found
|
||||
meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
|
||||
if err == nil {
|
||||
responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !contentStore.Exists(meta)))
|
||||
exist, err := contentStore.Exists(meta)
|
||||
if err != nil {
|
||||
log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
|
||||
writeStatus(ctx, 500)
|
||||
return
|
||||
}
|
||||
responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !exist))
|
||||
} else {
|
||||
log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err)
|
||||
}
|
||||
|
|
@ -387,7 +408,7 @@ func PutHandler(ctx *context.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
||||
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||
bodyReader := ctx.Req.Body().ReadCloser()
|
||||
defer bodyReader.Close()
|
||||
if err := contentStore.Put(meta, bodyReader); err != nil {
|
||||
|
|
@ -429,7 +450,7 @@ func VerifyHandler(ctx *context.Context) {
|
|||
return
|
||||
}
|
||||
|
||||
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
||||
contentStore := &ContentStore{ObjectStorage: storage.LFS}
|
||||
ok, err := contentStore.Verify(meta)
|
||||
if err != nil {
|
||||
// Error will be logged in Verify
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue