mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-05-15 22:42:43 +00:00
Make LFS http_client parallel within a batch. (#32369)
Signed-off-by: Royce Remer <royceremer@gmail.com> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Royce Remer
Otto Richter
parent
3c9b3ddf5c
commit
f6273e2250
7 changed files with 183 additions and 153 deletions
|
|
@ -12,6 +12,8 @@ import (
|
|||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/test"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
|
@ -184,93 +186,84 @@ func TestHTTPClientDownload(t *testing.T) {
|
|||
|
||||
cases := []struct {
|
||||
endpoint string
|
||||
expectederror string
|
||||
expectedError string
|
||||
}{
|
||||
// case 0
|
||||
{
|
||||
endpoint: "https://status-not-ok.io",
|
||||
expectederror: io.ErrUnexpectedEOF.Error(),
|
||||
expectedError: io.ErrUnexpectedEOF.Error(),
|
||||
},
|
||||
// case 1
|
||||
{
|
||||
endpoint: "https://invalid-json-response.io",
|
||||
expectederror: "invalid json",
|
||||
expectedError: "invalid json",
|
||||
},
|
||||
// case 2
|
||||
{
|
||||
endpoint: "https://valid-batch-request-download.io",
|
||||
expectederror: "",
|
||||
expectedError: "",
|
||||
},
|
||||
// case 3
|
||||
{
|
||||
endpoint: "https://response-no-objects.io",
|
||||
expectederror: "",
|
||||
expectedError: "",
|
||||
},
|
||||
// case 4
|
||||
{
|
||||
endpoint: "https://unknown-transfer-adapter.io",
|
||||
expectederror: "TransferAdapter not found: ",
|
||||
expectedError: "TransferAdapter not found: ",
|
||||
},
|
||||
// case 5
|
||||
{
|
||||
endpoint: "https://error-in-response-objects.io",
|
||||
expectederror: "Object not found",
|
||||
expectedError: "Object not found",
|
||||
},
|
||||
// case 6
|
||||
{
|
||||
endpoint: "https://empty-actions-map.io",
|
||||
expectederror: "missing action 'download'",
|
||||
expectedError: "missing action 'download'",
|
||||
},
|
||||
// case 7
|
||||
{
|
||||
endpoint: "https://download-actions-map.io",
|
||||
expectederror: "",
|
||||
expectedError: "",
|
||||
},
|
||||
// case 8
|
||||
{
|
||||
endpoint: "https://upload-actions-map.io",
|
||||
expectederror: "missing action 'download'",
|
||||
expectedError: "missing action 'download'",
|
||||
},
|
||||
// case 9
|
||||
{
|
||||
endpoint: "https://verify-actions-map.io",
|
||||
expectederror: "missing action 'download'",
|
||||
expectedError: "missing action 'download'",
|
||||
},
|
||||
// case 10
|
||||
{
|
||||
endpoint: "https://unknown-actions-map.io",
|
||||
expectederror: "missing action 'download'",
|
||||
expectedError: "missing action 'download'",
|
||||
},
|
||||
// case 11
|
||||
{
|
||||
endpoint: "https://legacy-batch-request-download.io",
|
||||
expectederror: "",
|
||||
expectedError: "",
|
||||
},
|
||||
}
|
||||
|
||||
for n, c := range cases {
|
||||
client := &HTTPClient{
|
||||
client: hc,
|
||||
endpoint: c.endpoint,
|
||||
transfers: map[string]TransferAdapter{
|
||||
"dummy": dummy,
|
||||
},
|
||||
}
|
||||
|
||||
err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error {
|
||||
if objectError != nil {
|
||||
return objectError
|
||||
defer test.MockVariableValue(&setting.LFSClient.BatchOperationConcurrency, 3)()
|
||||
for _, c := range cases {
|
||||
t.Run(c.endpoint, func(t *testing.T) {
|
||||
client := &HTTPClient{
|
||||
client: hc,
|
||||
endpoint: c.endpoint,
|
||||
transfers: map[string]TransferAdapter{
|
||||
"dummy": dummy,
|
||||
},
|
||||
}
|
||||
|
||||
err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error {
|
||||
if objectError != nil {
|
||||
return objectError
|
||||
}
|
||||
b, err := io.ReadAll(content)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []byte("dummy"), b)
|
||||
return nil
|
||||
})
|
||||
if c.expectedError != "" {
|
||||
assert.ErrorContains(t, err, c.expectedError)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
b, err := io.ReadAll(content)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []byte("dummy"), b)
|
||||
return nil
|
||||
})
|
||||
if len(c.expectederror) > 0 {
|
||||
assert.Contains(t, err.Error(), c.expectederror, "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
|
||||
} else {
|
||||
require.NoError(t, err, "case %d", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -297,81 +290,73 @@ func TestHTTPClientUpload(t *testing.T) {
|
|||
|
||||
cases := []struct {
|
||||
endpoint string
|
||||
expectederror string
|
||||
expectedError string
|
||||
}{
|
||||
// case 0
|
||||
{
|
||||
endpoint: "https://status-not-ok.io",
|
||||
expectederror: io.ErrUnexpectedEOF.Error(),
|
||||
expectedError: io.ErrUnexpectedEOF.Error(),
|
||||
},
|
||||
// case 1
|
||||
{
|
||||
endpoint: "https://invalid-json-response.io",
|
||||
expectederror: "invalid json",
|
||||
expectedError: "invalid json",
|
||||
},
|
||||
// case 2
|
||||
{
|
||||
endpoint: "https://valid-batch-request-upload.io",
|
||||
expectederror: "",
|
||||
expectedError: "",
|
||||
},
|
||||
// case 3
|
||||
{
|
||||
endpoint: "https://response-no-objects.io",
|
||||
expectederror: "",
|
||||
expectedError: "",
|
||||
},
|
||||
// case 4
|
||||
{
|
||||
endpoint: "https://unknown-transfer-adapter.io",
|
||||
expectederror: "TransferAdapter not found: ",
|
||||
expectedError: "TransferAdapter not found: ",
|
||||
},
|
||||
// case 5
|
||||
{
|
||||
endpoint: "https://error-in-response-objects.io",
|
||||
expectederror: "Object not found",
|
||||
expectedError: "Object not found",
|
||||
},
|
||||
// case 6
|
||||
{
|
||||
endpoint: "https://empty-actions-map.io",
|
||||
expectederror: "",
|
||||
expectedError: "",
|
||||
},
|
||||
// case 7
|
||||
{
|
||||
endpoint: "https://download-actions-map.io",
|
||||
expectederror: "missing action 'upload'",
|
||||
expectedError: "missing action 'upload'",
|
||||
},
|
||||
// case 8
|
||||
{
|
||||
endpoint: "https://upload-actions-map.io",
|
||||
expectederror: "",
|
||||
expectedError: "",
|
||||
},
|
||||
// case 9
|
||||
{
|
||||
endpoint: "https://verify-actions-map.io",
|
||||
expectederror: "missing action 'upload'",
|
||||
expectedError: "missing action 'upload'",
|
||||
},
|
||||
// case 10
|
||||
{
|
||||
endpoint: "https://unknown-actions-map.io",
|
||||
expectederror: "missing action 'upload'",
|
||||
expectedError: "missing action 'upload'",
|
||||
},
|
||||
}
|
||||
|
||||
for n, c := range cases {
|
||||
client := &HTTPClient{
|
||||
client: hc,
|
||||
endpoint: c.endpoint,
|
||||
transfers: map[string]TransferAdapter{
|
||||
"dummy": dummy,
|
||||
},
|
||||
}
|
||||
defer test.MockVariableValue(&setting.LFSClient.BatchOperationConcurrency, 3)()
|
||||
for _, c := range cases {
|
||||
t.Run(c.endpoint, func(t *testing.T) {
|
||||
client := &HTTPClient{
|
||||
client: hc,
|
||||
endpoint: c.endpoint,
|
||||
transfers: map[string]TransferAdapter{
|
||||
"dummy": dummy,
|
||||
},
|
||||
}
|
||||
|
||||
err := client.Upload(context.Background(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) {
|
||||
return io.NopCloser(new(bytes.Buffer)), objectError
|
||||
err := client.Upload(context.Background(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) {
|
||||
return io.NopCloser(new(bytes.Buffer)), objectError
|
||||
})
|
||||
if c.expectedError != "" {
|
||||
assert.ErrorContains(t, err, c.expectedError)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
})
|
||||
if len(c.expectederror) > 0 {
|
||||
assert.Contains(t, err.Error(), c.expectederror, "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
|
||||
} else {
|
||||
require.NoError(t, err, "case %d", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue