mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2025-06-25 18:40:51 +00:00
feat: support artifact uploads for OCI container packages (#8070)
# Fix OCI artifact uploads with`oras` ## Problem ORAS (OCI Registry As Storage) artifact uploads were failing with several HTTP-related errors when pushing to Forgejo's container registry. This prevented users from storing OCI artifacts like `artifacthub-repo.yaml` in commands like `oras push [...] artifacthub-repo.yaml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml`. This has been discussed previously in https://github.com/go-gitea/gitea/issues/25846 ## Root Causes and Fixes ### 1. Missing Content-Length for Empty Blobs **Issue**: Empty blobs (size 0) were not getting the required `Content-Length: 0` header, causing ORAS to fail with "unknown response Content-Length". **Fix**: Changed the condition in `setResponseHeaders` from `if h.ContentLength != 0` to `if h.ContentLength >= 0` to ensure the Content-Length header is always set for valid blob sizes. ```go // Before if h.ContentLength != 0 { resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength, 10)) } // After if h.ContentLength >= 0 { resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength, 10)) } ``` ### 2. Content-Length Mismatch in JSON Error Responses **Issue**: The `jsonResponse` function was calling `WriteHeader()` before writing JSON content, causing "wrote more than the declared Content-Length" errors when the HTTP stack calculated a different Content-Length than what was actually written. **Fix**: Modified `jsonResponse` to buffer JSON content first, calculate the exact Content-Length, then write the complete response. ### 3. Incomplete HTTP Responses in Error Handling **Issue**: The `apiError` function was only setting response headers without writing any response body, causing EOF errors when clients expected a complete HTTP response. **Fix**: Updated `apiError` to write proper JSON error responses following the OCI Distribution Specification format with `code` and `message` fields. ### 4. Empty Config Blob Handling for OCI Artifacts **Issue**: OCI artifacts often have empty config blobs (required by spec but contain no data). The JSON decoder was failing with EOF when trying to parse these empty configs. **Fix**: Added EOF handling in `parseOCIImageConfig` to return a valid default metadata object for empty config blobs. ```go if err := json.NewDecoder(r).Decode(&image); err != nil { // Handle empty config blobs (common in OCI artifacts) if err == io.EOF { return &Metadata{ Type: TypeOCI, Platform: DefaultPlatform, }, nil } return nil, err } ``` ## Testing Verified that ORAS artifact uploads now work correctly: ```bash oras push registry/owner/package:artifacthub.io \ --config /dev/null:application/vnd.cncf.artifacthub.config.v1+yaml \ artifacthub-repo.yaml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml ``` ### Tests - I added test coverage for Go changes... - [x] in their respective `*_test.go` for unit tests. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [ ] I do not want this change to show in the release notes. - [x] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title. Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/8070 Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org> Co-authored-by: pat-s <patrick.schratz@gmail.com> Co-committed-by: pat-s <patrick.schratz@gmail.com>
This commit is contained in:
pat-s
Earl Warren
parent
b2b039b6e7
commit
bd6f3243ab
6 changed files with 355 additions and 39 deletions
|
|
@ -56,7 +56,7 @@ func TestPackageContainer(t *testing.T) {
|
|||
return values
|
||||
}
|
||||
|
||||
images := []string{"test", "te/st"}
|
||||
images := []string{"test", "te/st", "oras-artifact"}
|
||||
tags := []string{"latest", "main"}
|
||||
multiTag := "multi"
|
||||
|
||||
|
|
@ -177,6 +177,90 @@ func TestPackageContainer(t *testing.T) {
|
|||
assert.Equal(t, "registry/2.0", resp.Header().Get("Docker-Distribution-Api-Version"))
|
||||
})
|
||||
|
||||
t.Run("ORAS Artifact Upload", func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
image := "oras-artifact"
|
||||
url := fmt.Sprintf("%sv2/%s/%s", setting.AppURL, user.Name, image)
|
||||
|
||||
// Empty config blob (common in ORAS artifacts)
|
||||
emptyConfigDigest := "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
emptyConfigContent := ""
|
||||
|
||||
// Upload empty config blob
|
||||
req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, emptyConfigDigest), bytes.NewReader([]byte(emptyConfigContent))).
|
||||
AddTokenAuth(userToken)
|
||||
resp := MakeRequest(t, req, http.StatusCreated)
|
||||
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, emptyConfigDigest), resp.Header().Get("Location"))
|
||||
assert.Equal(t, emptyConfigDigest, resp.Header().Get("Docker-Content-Digest"))
|
||||
|
||||
// Verify empty blob exists and has correct Content-Length
|
||||
req = NewRequest(t, "HEAD", fmt.Sprintf("%s/blobs/%s", url, emptyConfigDigest)).
|
||||
AddTokenAuth(userToken)
|
||||
resp = MakeRequest(t, req, http.StatusOK)
|
||||
assert.Equal(t, "0", resp.Header().Get("Content-Length")) // This was the main fix
|
||||
assert.Equal(t, emptyConfigDigest, resp.Header().Get("Docker-Content-Digest"))
|
||||
|
||||
// Upload a small data blob (e.g., artifacthub metadata)
|
||||
artifactData := `{"name":"test-artifact","version":"1.0.0"}`
|
||||
artifactDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(artifactData)))
|
||||
|
||||
req = NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, artifactDigest), bytes.NewReader([]byte(artifactData))).
|
||||
AddTokenAuth(userToken)
|
||||
resp = MakeRequest(t, req, http.StatusCreated)
|
||||
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, artifactDigest), resp.Header().Get("Location"))
|
||||
|
||||
// Create OCI artifact manifest
|
||||
artifactManifest := fmt.Sprintf(`{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"artifactType": "application/vnd.cncf.artifacthub.config.v1+yaml",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.cncf.artifacthub.config.v1+yaml",
|
||||
"digest": "%s",
|
||||
"size": %d
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml",
|
||||
"digest": "%s",
|
||||
"size": %d
|
||||
}
|
||||
]
|
||||
}`, emptyConfigDigest, len(emptyConfigContent), artifactDigest, len(artifactData))
|
||||
|
||||
artifactManifestDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(artifactManifest)))
|
||||
|
||||
// Upload artifact manifest
|
||||
req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/artifact-v1", url), bytes.NewReader([]byte(artifactManifest))).
|
||||
AddTokenAuth(userToken).
|
||||
SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json")
|
||||
resp = MakeRequest(t, req, http.StatusCreated)
|
||||
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/manifests/artifact-v1", user.Name, image), resp.Header().Get("Location"))
|
||||
assert.Equal(t, artifactManifestDigest, resp.Header().Get("Docker-Content-Digest"))
|
||||
|
||||
// Verify manifest can be retrieved
|
||||
req = NewRequest(t, "GET", fmt.Sprintf("%s/manifests/artifact-v1", url)).
|
||||
AddTokenAuth(userToken).
|
||||
SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json")
|
||||
resp = MakeRequest(t, req, http.StatusOK)
|
||||
assert.Equal(t, "application/vnd.oci.image.manifest.v1+json", resp.Header().Get("Content-Type"))
|
||||
assert.Equal(t, artifactManifestDigest, resp.Header().Get("Docker-Content-Digest"))
|
||||
|
||||
// Verify package was created with correct metadata
|
||||
pvs, err := packages_model.GetVersionsByPackageType(db.DefaultContext, user.ID, packages_model.TypeContainer)
|
||||
require.NoError(t, err)
|
||||
|
||||
found := false
|
||||
for _, pv := range pvs {
|
||||
if pv.LowerVersion == "artifact-v1" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "ORAS artifact package should be created")
|
||||
})
|
||||
|
||||
for _, image := range images {
|
||||
t.Run(fmt.Sprintf("[Image:%s]", image), func(t *testing.T) {
|
||||
url := fmt.Sprintf("%sv2/%s/%s", setting.AppURL, user.Name, image)
|
||||
|
|
@ -604,36 +688,76 @@ func TestPackageContainer(t *testing.T) {
|
|||
t.Run("GetTagList", func(t *testing.T) {
|
||||
defer tests.PrintCurrentTest(t)()
|
||||
|
||||
cases := []struct {
|
||||
var cases []struct {
|
||||
URL string
|
||||
ExpectedTags []string
|
||||
ExpectedLink string
|
||||
}{
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list", url),
|
||||
ExpectedTags: []string{"latest", "main", "multi"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?n=0", url),
|
||||
ExpectedTags: []string{},
|
||||
ExpectedLink: "",
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?n=2", url),
|
||||
ExpectedTags: []string{"latest", "main"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=main&n=2>; rel="next"`, user.Name, image),
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?last=main", url),
|
||||
ExpectedTags: []string{"multi"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?n=1&last=latest", url),
|
||||
ExpectedTags: []string{"main"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=main&n=1>; rel="next"`, user.Name, image),
|
||||
},
|
||||
}
|
||||
|
||||
if image == "oras-artifact" {
|
||||
cases = []struct {
|
||||
URL string
|
||||
ExpectedTags []string
|
||||
ExpectedLink string
|
||||
}{
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list", url),
|
||||
ExpectedTags: []string{"artifact-v1", "latest", "main", "multi"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?n=0", url),
|
||||
ExpectedTags: []string{},
|
||||
ExpectedLink: "",
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?n=2", url),
|
||||
ExpectedTags: []string{"artifact-v1", "latest"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=latest&n=2>; rel="next"`, user.Name, image),
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?last=main", url),
|
||||
ExpectedTags: []string{"multi"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?n=1&last=latest", url),
|
||||
ExpectedTags: []string{"main"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=main&n=1>; rel="next"`, user.Name, image),
|
||||
},
|
||||
}
|
||||
} else {
|
||||
cases = []struct {
|
||||
URL string
|
||||
ExpectedTags []string
|
||||
ExpectedLink string
|
||||
}{
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list", url),
|
||||
ExpectedTags: []string{"latest", "main", "multi"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?n=0", url),
|
||||
ExpectedTags: []string{},
|
||||
ExpectedLink: "",
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?n=2", url),
|
||||
ExpectedTags: []string{"latest", "main"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=main&n=2>; rel="next"`, user.Name, image),
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?last=main", url),
|
||||
ExpectedTags: []string{"multi"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
|
||||
},
|
||||
{
|
||||
URL: fmt.Sprintf("%s/tags/list?n=1&last=latest", url),
|
||||
ExpectedTags: []string{"main"},
|
||||
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=main&n=1>; rel="next"`, user.Name, image),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
|
|
@ -660,7 +784,11 @@ func TestPackageContainer(t *testing.T) {
|
|||
|
||||
var apiPackages []*api.Package
|
||||
DecodeJSON(t, resp, &apiPackages)
|
||||
assert.Len(t, apiPackages, 4) // "latest", "main", "multi", "sha256:..."
|
||||
if image == "oras-artifact" {
|
||||
assert.Len(t, apiPackages, 5) // "artifact-v1", "latest", "main", "multi", "sha256:..."
|
||||
} else {
|
||||
assert.Len(t, apiPackages, 4) // "latest", "main", "multi", "sha256:..."
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Delete", func(t *testing.T) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue