mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-05-21 09:21:15 +00:00
Rework repository archive (#14723)
* Use storage to store archive files * Fix backend lint * Add archiver table on database * Finish archive download * Fix test * Add database migrations * Add status for archiver * Fix lint * Add queue * Add doctor to check and delete old archives * Improve archive queue * Fix tests * improve archive storage * Delete repo archives * Add missing fixture * fix fixture * Fix fixture * Fix test * Fix archiver cleaning * Fix bug * Add docs for repository archive storage * remove repo-archive configuration * Fix test * Fix test * Fix lint Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
Lunny Xiao
GitHub
parent
c9c7afda1a
commit
b223d36195
25 changed files with 648 additions and 480 deletions
|
|
@ -6,108 +6,75 @@ package archiver
|
|||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/test"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var queueMutex sync.Mutex
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
models.MainTest(m, filepath.Join("..", ".."))
|
||||
}
|
||||
|
||||
func waitForCount(t *testing.T, num int) {
|
||||
var numQueued int
|
||||
|
||||
// Wait for up to 10 seconds for the queue to be impacted.
|
||||
timeout := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
numQueued = len(archiveInProgress)
|
||||
if numQueued == num || time.Now().After(timeout) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
assert.Len(t, archiveInProgress, num)
|
||||
}
|
||||
|
||||
func releaseOneEntry(t *testing.T, inFlight []*ArchiveRequest) {
|
||||
var nowQueued, numQueued int
|
||||
|
||||
numQueued = len(archiveInProgress)
|
||||
|
||||
// Release one, then wait up to 10 seconds for it to complete.
|
||||
queueMutex.Lock()
|
||||
archiveQueueReleaseCond.Signal()
|
||||
queueMutex.Unlock()
|
||||
timeout := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
nowQueued = len(archiveInProgress)
|
||||
if nowQueued != numQueued || time.Now().After(timeout) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure we didn't just timeout.
|
||||
assert.NotEqual(t, numQueued, nowQueued)
|
||||
|
||||
// Also make sure that we released only one.
|
||||
assert.Equal(t, numQueued-1, nowQueued)
|
||||
}
|
||||
|
||||
func TestArchive_Basic(t *testing.T) {
|
||||
assert.NoError(t, models.PrepareTestDatabase())
|
||||
|
||||
archiveQueueMutex = &queueMutex
|
||||
archiveQueueStartCond = sync.NewCond(&queueMutex)
|
||||
archiveQueueReleaseCond = sync.NewCond(&queueMutex)
|
||||
defer func() {
|
||||
archiveQueueMutex = nil
|
||||
archiveQueueStartCond = nil
|
||||
archiveQueueReleaseCond = nil
|
||||
}()
|
||||
|
||||
ctx := test.MockContext(t, "user27/repo49")
|
||||
firstCommit, secondCommit := "51f84af23134", "aacbdfe9e1c4"
|
||||
|
||||
bogusReq := DeriveRequestFrom(ctx, firstCommit+".zip")
|
||||
assert.Nil(t, bogusReq)
|
||||
|
||||
test.LoadRepo(t, ctx, 49)
|
||||
bogusReq = DeriveRequestFrom(ctx, firstCommit+".zip")
|
||||
assert.Nil(t, bogusReq)
|
||||
|
||||
test.LoadGitRepo(t, ctx)
|
||||
defer ctx.Repo.GitRepo.Close()
|
||||
|
||||
bogusReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, bogusReq)
|
||||
assert.EqualValues(t, firstCommit+".zip", bogusReq.GetArchiveName())
|
||||
|
||||
// Check a series of bogus requests.
|
||||
// Step 1, valid commit with a bad extension.
|
||||
bogusReq = DeriveRequestFrom(ctx, firstCommit+".dilbert")
|
||||
bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".dilbert")
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, bogusReq)
|
||||
|
||||
// Step 2, missing commit.
|
||||
bogusReq = DeriveRequestFrom(ctx, "dbffff.zip")
|
||||
bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "dbffff.zip")
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, bogusReq)
|
||||
|
||||
// Step 3, doesn't look like branch/tag/commit.
|
||||
bogusReq = DeriveRequestFrom(ctx, "db.zip")
|
||||
bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "db.zip")
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, bogusReq)
|
||||
|
||||
bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "master.zip")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, bogusReq)
|
||||
assert.EqualValues(t, "master.zip", bogusReq.GetArchiveName())
|
||||
|
||||
bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "test/archive.zip")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, bogusReq)
|
||||
assert.EqualValues(t, "test-archive.zip", bogusReq.GetArchiveName())
|
||||
|
||||
// Now two valid requests, firstCommit with valid extensions.
|
||||
zipReq := DeriveRequestFrom(ctx, firstCommit+".zip")
|
||||
zipReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, zipReq)
|
||||
|
||||
tgzReq := DeriveRequestFrom(ctx, firstCommit+".tar.gz")
|
||||
tgzReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".tar.gz")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, tgzReq)
|
||||
|
||||
secondReq := DeriveRequestFrom(ctx, secondCommit+".zip")
|
||||
secondReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".zip")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, secondReq)
|
||||
|
||||
inFlight := make([]*ArchiveRequest, 3)
|
||||
|
|
@ -128,41 +95,9 @@ func TestArchive_Basic(t *testing.T) {
|
|||
|
||||
// Sleep two seconds to make sure the queue doesn't change.
|
||||
time.Sleep(2 * time.Second)
|
||||
assert.Len(t, archiveInProgress, 3)
|
||||
|
||||
// Release them all, they'll then stall at the archiveQueueReleaseCond while
|
||||
// we examine the queue state.
|
||||
queueMutex.Lock()
|
||||
archiveQueueStartCond.Broadcast()
|
||||
queueMutex.Unlock()
|
||||
|
||||
// Iterate through all of the in-flight requests and wait for their
|
||||
// completion.
|
||||
for _, req := range inFlight {
|
||||
req.WaitForCompletion(ctx)
|
||||
}
|
||||
|
||||
for _, req := range inFlight {
|
||||
assert.True(t, req.IsComplete())
|
||||
exist, err := util.IsExist(req.GetArchivePath())
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, exist)
|
||||
}
|
||||
|
||||
arbitraryReq := inFlight[0]
|
||||
// Reopen the channel so we don't double-close, mark it incomplete. We're
|
||||
// going to run it back through the archiver, and it should get marked
|
||||
// complete again.
|
||||
arbitraryReq.cchan = make(chan struct{})
|
||||
arbitraryReq.archiveComplete = false
|
||||
doArchive(arbitraryReq)
|
||||
assert.True(t, arbitraryReq.IsComplete())
|
||||
|
||||
// Queues should not have drained yet, because we haven't released them.
|
||||
// Do so now.
|
||||
assert.Len(t, archiveInProgress, 3)
|
||||
|
||||
zipReq2 := DeriveRequestFrom(ctx, firstCommit+".zip")
|
||||
zipReq2, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip")
|
||||
assert.NoError(t, err)
|
||||
// This zipReq should match what's sitting in the queue, as we haven't
|
||||
// let it release yet. From the consumer's point of view, this looks like
|
||||
// a long-running archive task.
|
||||
|
|
@ -173,46 +108,22 @@ func TestArchive_Basic(t *testing.T) {
|
|||
// predecessor has cleared out of the queue.
|
||||
ArchiveRepository(zipReq2)
|
||||
|
||||
// Make sure the queue hasn't grown any.
|
||||
assert.Len(t, archiveInProgress, 3)
|
||||
|
||||
// Make sure the queue drains properly
|
||||
releaseOneEntry(t, inFlight)
|
||||
assert.Len(t, archiveInProgress, 2)
|
||||
releaseOneEntry(t, inFlight)
|
||||
assert.Len(t, archiveInProgress, 1)
|
||||
releaseOneEntry(t, inFlight)
|
||||
assert.Empty(t, archiveInProgress)
|
||||
|
||||
// Now we'll submit a request and TimedWaitForCompletion twice, before and
|
||||
// after we release it. We should trigger both the timeout and non-timeout
|
||||
// cases.
|
||||
var completed, timedout bool
|
||||
timedReq := DeriveRequestFrom(ctx, secondCommit+".tar.gz")
|
||||
timedReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, secondCommit+".tar.gz")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, timedReq)
|
||||
ArchiveRepository(timedReq)
|
||||
|
||||
// Guaranteed to timeout; we haven't signalled the request to start..
|
||||
completed, timedout = timedReq.TimedWaitForCompletion(ctx, 2*time.Second)
|
||||
assert.False(t, completed)
|
||||
assert.True(t, timedout)
|
||||
|
||||
queueMutex.Lock()
|
||||
archiveQueueStartCond.Broadcast()
|
||||
queueMutex.Unlock()
|
||||
|
||||
// Shouldn't timeout, we've now signalled it and it's a small request.
|
||||
completed, timedout = timedReq.TimedWaitForCompletion(ctx, 15*time.Second)
|
||||
assert.True(t, completed)
|
||||
assert.False(t, timedout)
|
||||
|
||||
zipReq2 = DeriveRequestFrom(ctx, firstCommit+".zip")
|
||||
zipReq2, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip")
|
||||
assert.NoError(t, err)
|
||||
// Now, we're guaranteed to have released the original zipReq from the queue.
|
||||
// Ensure that we don't get handed back the released entry somehow, but they
|
||||
// should remain functionally equivalent in all fields. The exception here
|
||||
// is zipReq.cchan, which will be non-nil because it's a completed request.
|
||||
// It's fine to go ahead and set it to nil now.
|
||||
zipReq.cchan = nil
|
||||
|
||||
assert.Equal(t, zipReq, zipReq2)
|
||||
assert.False(t, zipReq == zipReq2)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue