feat: support artifact uploads for OCI container packages (#8070)

# Fix OCI artifact uploads with`oras`

## Problem

ORAS (OCI Registry As Storage) artifact uploads were failing with several HTTP-related errors when pushing to Forgejo's container registry. This prevented users from storing OCI artifacts like `artifacthub-repo.yaml` in commands like `oras push [...] artifacthub-repo.yaml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml`.

This has been discussed previously in https://github.com/go-gitea/gitea/issues/25846

## Root Causes and Fixes

### 1. Missing Content-Length for Empty Blobs

**Issue**: Empty blobs (size 0) were not getting the required `Content-Length: 0` header, causing ORAS to fail with "unknown response Content-Length".

**Fix**: Changed the condition in `setResponseHeaders` from `if h.ContentLength != 0` to `if h.ContentLength >= 0` to ensure the Content-Length header is always set for valid blob sizes.

```go
// Before
if h.ContentLength != 0 {
    resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength, 10))
}

// After
if h.ContentLength >= 0 {
    resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength, 10))
}
```

### 2. Content-Length Mismatch in JSON Error Responses

**Issue**: The `jsonResponse` function was calling `WriteHeader()` before writing JSON content, causing "wrote more than the declared Content-Length" errors when the HTTP stack calculated a different Content-Length than what was actually written.

**Fix**: Modified `jsonResponse` to buffer JSON content first, calculate the exact Content-Length, then write the complete response.

### 3. Incomplete HTTP Responses in Error Handling

**Issue**: The `apiError` function was only setting response headers without writing any response body, causing EOF errors when clients expected a complete HTTP response.

**Fix**: Updated `apiError` to write proper JSON error responses following the OCI Distribution Specification format with `code` and `message` fields.

### 4. Empty Config Blob Handling for OCI Artifacts

**Issue**: OCI artifacts often have empty config blobs (required by spec but contain no data). The JSON decoder was failing with EOF when trying to parse these empty configs.

**Fix**: Added EOF handling in `parseOCIImageConfig` to return a valid default metadata object for empty config blobs.

```go
if err := json.NewDecoder(r).Decode(&image); err != nil {
    // Handle empty config blobs (common in OCI artifacts)
    if err == io.EOF {
        return &Metadata{
            Type:     TypeOCI,
            Platform: DefaultPlatform,
        }, nil
    }
    return nil, err
}
```

## Testing

Verified that ORAS artifact uploads now work correctly:

```bash
oras push registry/owner/package:artifacthub.io \
  --config /dev/null:application/vnd.cncf.artifacthub.config.v1+yaml \
  artifacthub-repo.yaml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml
```

### Tests

- I added test coverage for Go changes...
  - [x] in their respective `*_test.go` for unit tests.
  - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I added test coverage for JavaScript changes...
  - [ ] in `web_src/js/*.test.js` if it can be unit tested.
  - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)).

### Documentation

- [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
- [x] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [ ] I do not want this change to show in the release notes.
- [x] I want the title to show in the release notes with a link to this pull request.
- [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/8070
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: pat-s <patrick.schratz@gmail.com>
Co-committed-by: pat-s <patrick.schratz@gmail.com>
This commit is contained in:
pat-s 2025-06-09 10:14:53 +02:00 committed by Earl Warren
parent b2b039b6e7
commit bd6f3243ab
6 changed files with 355 additions and 39 deletions

View file

@ -408,7 +408,7 @@ local addIssueLabelsOverrides(labels) =
regex: '',
type: 'query',
multi: true,
allValue: '.+'
allValue: '.+',
},
)
.addTemplate(
@ -423,7 +423,7 @@ local addIssueLabelsOverrides(labels) =
regex: '',
type: 'query',
multi: true,
allValue: '.+'
allValue: '.+',
},
)
.addTemplate(

View file

@ -84,6 +84,13 @@ func ParseImageConfig(mt string, r io.Reader) (*Metadata, error) {
func parseOCIImageConfig(r io.Reader) (*Metadata, error) {
var image oci.Image
if err := json.NewDecoder(r).Decode(&image); err != nil {
// Handle empty config blobs (common in OCI artifacts)
if err == io.EOF {
return &Metadata{
Type: TypeOCI,
Platform: DefaultPlatform,
}, nil
}
return nil, err
}

View file

@ -4,6 +4,7 @@
package container
import (
"io"
"strings"
"testing"
@ -60,3 +61,49 @@ func TestParseImageConfig(t *testing.T) {
assert.Equal(t, projectURL, metadata.ProjectURL)
assert.Equal(t, repositoryURL, metadata.RepositoryURL)
}
func TestParseImageConfigEmptyBlob(t *testing.T) {
t.Run("Empty config blob (EOF)", func(t *testing.T) {
// Test empty reader (simulates empty config blob common in OCI artifacts)
metadata, err := ParseImageConfig(oci.MediaTypeImageManifest, strings.NewReader(""))
require.NoError(t, err)
assert.Equal(t, TypeOCI, metadata.Type)
assert.Equal(t, DefaultPlatform, metadata.Platform)
assert.Empty(t, metadata.Description)
assert.Empty(t, metadata.Authors)
assert.Empty(t, metadata.Labels)
assert.Empty(t, metadata.Manifests)
})
t.Run("Empty JSON object", func(t *testing.T) {
// Test minimal valid JSON config
metadata, err := ParseImageConfig(oci.MediaTypeImageManifest, strings.NewReader("{}"))
require.NoError(t, err)
assert.Equal(t, TypeOCI, metadata.Type)
assert.Equal(t, DefaultPlatform, metadata.Platform)
assert.Empty(t, metadata.Description)
assert.Empty(t, metadata.Authors)
})
t.Run("Invalid JSON still returns error", func(t *testing.T) {
// Test that actual JSON errors (not EOF) are still returned
_, err := ParseImageConfig(oci.MediaTypeImageManifest, strings.NewReader("{invalid json"))
require.Error(t, err)
assert.NotEqual(t, io.EOF, err)
})
t.Run("OCI artifact with empty config", func(t *testing.T) {
// Test OCI artifact scenario with minimal config
configOCI := `{"config": {}}`
metadata, err := ParseImageConfig(oci.MediaTypeImageManifest, strings.NewReader(configOCI))
require.NoError(t, err)
assert.Equal(t, TypeOCI, metadata.Type)
assert.Equal(t, DefaultPlatform, metadata.Platform)
assert.Empty(t, metadata.Description)
assert.Empty(t, metadata.Authors)
assert.Empty(t, metadata.ImageLayers)
})
}

View file

@ -4,6 +4,7 @@
package container
import (
"bytes"
"errors"
"fmt"
"io"
@ -62,9 +63,6 @@ func setResponseHeaders(resp http.ResponseWriter, h *containerHeaders) {
if h.ContentType != "" {
resp.Header().Set("Content-Type", h.ContentType)
}
if h.ContentLength != 0 {
resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength, 10))
}
if h.UploadUUID != "" {
resp.Header().Set("Docker-Upload-Uuid", h.UploadUUID)
}
@ -72,17 +70,29 @@ func setResponseHeaders(resp http.ResponseWriter, h *containerHeaders) {
resp.Header().Set("Docker-Content-Digest", h.ContentDigest)
resp.Header().Set("ETag", fmt.Sprintf(`"%s"`, h.ContentDigest))
}
if h.ContentLength >= 0 {
resp.Header().Set("Content-Length", strconv.FormatInt(h.ContentLength, 10))
}
resp.Header().Set("Docker-Distribution-Api-Version", "registry/2.0")
resp.WriteHeader(h.Status)
}
func jsonResponse(ctx *context.Context, status int, obj any) {
setResponseHeaders(ctx.Resp, &containerHeaders{
Status: status,
ContentType: "application/json",
})
if err := json.NewEncoder(ctx.Resp).Encode(obj); err != nil {
// Buffer the JSON content first to calculate correct Content-Length
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(obj); err != nil {
log.Error("JSON encode: %v", err)
return
}
setResponseHeaders(ctx.Resp, &containerHeaders{
Status: status,
ContentType: "application/json",
ContentLength: int64(buf.Len()),
})
if _, err := buf.WriteTo(ctx.Resp); err != nil {
log.Error("JSON write: %v", err)
}
}

View file

@ -0,0 +1,124 @@
// Copyright 2025 The Forgejo Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSetResponseHeaders(t *testing.T) {
t.Run("Content-Length for empty content", func(t *testing.T) {
recorder := httptest.NewRecorder()
setResponseHeaders(recorder, &containerHeaders{
Status: http.StatusOK,
ContentLength: 0, // Empty blob
ContentDigest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
})
assert.Equal(t, "0", recorder.Header().Get("Content-Length"))
assert.Equal(t, "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", recorder.Header().Get("Docker-Content-Digest"))
assert.Equal(t, "registry/2.0", recorder.Header().Get("Docker-Distribution-Api-Version"))
assert.Equal(t, http.StatusOK, recorder.Code)
})
t.Run("Content-Length for non-empty content", func(t *testing.T) {
recorder := httptest.NewRecorder()
setResponseHeaders(recorder, &containerHeaders{
Status: http.StatusOK,
ContentLength: 1024,
ContentDigest: "sha256:abcd1234",
})
assert.Equal(t, "1024", recorder.Header().Get("Content-Length"))
assert.Equal(t, "sha256:abcd1234", recorder.Header().Get("Docker-Content-Digest"))
})
t.Run("All headers set correctly", func(t *testing.T) {
recorder := httptest.NewRecorder()
setResponseHeaders(recorder, &containerHeaders{
Status: http.StatusAccepted,
ContentLength: 512,
ContentDigest: "sha256:test123",
ContentType: "application/vnd.oci.image.manifest.v1+json",
Location: "/v2/test/repo/blobs/uploads/uuid123",
Range: "0-511",
UploadUUID: "uuid123",
})
assert.Equal(t, "512", recorder.Header().Get("Content-Length"))
assert.Equal(t, "sha256:test123", recorder.Header().Get("Docker-Content-Digest"))
assert.Equal(t, "application/vnd.oci.image.manifest.v1+json", recorder.Header().Get("Content-Type"))
assert.Equal(t, "/v2/test/repo/blobs/uploads/uuid123", recorder.Header().Get("Location"))
assert.Equal(t, "0-511", recorder.Header().Get("Range"))
assert.Equal(t, "uuid123", recorder.Header().Get("Docker-Upload-Uuid"))
assert.Equal(t, "registry/2.0", recorder.Header().Get("Docker-Distribution-Api-Version"))
assert.Equal(t, `"sha256:test123"`, recorder.Header().Get("ETag"))
assert.Equal(t, http.StatusAccepted, recorder.Code)
})
}
// TestResponseHeadersForEmptyBlobs tests the core fix for ORAS empty blob support
func TestResponseHeadersForEmptyBlobs(t *testing.T) {
t.Run("Content-Length set for empty blob", func(t *testing.T) {
recorder := httptest.NewRecorder()
// This tests the main fix: empty blobs should have Content-Length: 0
setResponseHeaders(recorder, &containerHeaders{
Status: http.StatusOK,
ContentLength: 0, // Empty blob (like empty config in ORAS artifacts)
ContentDigest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
})
// The key fix: Content-Length should be set even for 0-byte blobs
assert.Equal(t, "0", recorder.Header().Get("Content-Length"))
assert.Equal(t, "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", recorder.Header().Get("Docker-Content-Digest"))
assert.Equal(t, "registry/2.0", recorder.Header().Get("Docker-Distribution-Api-Version"))
assert.Equal(t, `"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"`, recorder.Header().Get("ETag"))
assert.Equal(t, http.StatusOK, recorder.Code)
})
t.Run("Content-Length set for regular blob", func(t *testing.T) {
recorder := httptest.NewRecorder()
setResponseHeaders(recorder, &containerHeaders{
Status: http.StatusOK,
ContentLength: 1024,
ContentDigest: "sha256:abcd1234",
})
assert.Equal(t, "1024", recorder.Header().Get("Content-Length"))
assert.Equal(t, "sha256:abcd1234", recorder.Header().Get("Docker-Content-Digest"))
})
t.Run("All headers set correctly", func(t *testing.T) {
recorder := httptest.NewRecorder()
setResponseHeaders(recorder, &containerHeaders{
Status: http.StatusAccepted,
ContentLength: 512,
ContentDigest: "sha256:test123",
ContentType: "application/vnd.oci.image.manifest.v1+json",
Location: "/v2/test/repo/blobs/uploads/uuid123",
Range: "0-511",
UploadUUID: "uuid123",
})
assert.Equal(t, "512", recorder.Header().Get("Content-Length"))
assert.Equal(t, "sha256:test123", recorder.Header().Get("Docker-Content-Digest"))
assert.Equal(t, "application/vnd.oci.image.manifest.v1+json", recorder.Header().Get("Content-Type"))
assert.Equal(t, "/v2/test/repo/blobs/uploads/uuid123", recorder.Header().Get("Location"))
assert.Equal(t, "0-511", recorder.Header().Get("Range"))
assert.Equal(t, "uuid123", recorder.Header().Get("Docker-Upload-Uuid"))
assert.Equal(t, "registry/2.0", recorder.Header().Get("Docker-Distribution-Api-Version"))
assert.Equal(t, `"sha256:test123"`, recorder.Header().Get("ETag"))
assert.Equal(t, http.StatusAccepted, recorder.Code)
})
}

View file

@ -56,7 +56,7 @@ func TestPackageContainer(t *testing.T) {
return values
}
images := []string{"test", "te/st"}
images := []string{"test", "te/st", "oras-artifact"}
tags := []string{"latest", "main"}
multiTag := "multi"
@ -177,6 +177,90 @@ func TestPackageContainer(t *testing.T) {
assert.Equal(t, "registry/2.0", resp.Header().Get("Docker-Distribution-Api-Version"))
})
t.Run("ORAS Artifact Upload", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
image := "oras-artifact"
url := fmt.Sprintf("%sv2/%s/%s", setting.AppURL, user.Name, image)
// Empty config blob (common in ORAS artifacts)
emptyConfigDigest := "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
emptyConfigContent := ""
// Upload empty config blob
req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, emptyConfigDigest), bytes.NewReader([]byte(emptyConfigContent))).
AddTokenAuth(userToken)
resp := MakeRequest(t, req, http.StatusCreated)
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, emptyConfigDigest), resp.Header().Get("Location"))
assert.Equal(t, emptyConfigDigest, resp.Header().Get("Docker-Content-Digest"))
// Verify empty blob exists and has correct Content-Length
req = NewRequest(t, "HEAD", fmt.Sprintf("%s/blobs/%s", url, emptyConfigDigest)).
AddTokenAuth(userToken)
resp = MakeRequest(t, req, http.StatusOK)
assert.Equal(t, "0", resp.Header().Get("Content-Length")) // This was the main fix
assert.Equal(t, emptyConfigDigest, resp.Header().Get("Docker-Content-Digest"))
// Upload a small data blob (e.g., artifacthub metadata)
artifactData := `{"name":"test-artifact","version":"1.0.0"}`
artifactDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(artifactData)))
req = NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, artifactDigest), bytes.NewReader([]byte(artifactData))).
AddTokenAuth(userToken)
resp = MakeRequest(t, req, http.StatusCreated)
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, artifactDigest), resp.Header().Get("Location"))
// Create OCI artifact manifest
artifactManifest := fmt.Sprintf(`{
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
"artifactType": "application/vnd.cncf.artifacthub.config.v1+yaml",
"config": {
"mediaType": "application/vnd.cncf.artifacthub.config.v1+yaml",
"digest": "%s",
"size": %d
},
"layers": [
{
"mediaType": "application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml",
"digest": "%s",
"size": %d
}
]
}`, emptyConfigDigest, len(emptyConfigContent), artifactDigest, len(artifactData))
artifactManifestDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(artifactManifest)))
// Upload artifact manifest
req = NewRequestWithBody(t, "PUT", fmt.Sprintf("%s/manifests/artifact-v1", url), bytes.NewReader([]byte(artifactManifest))).
AddTokenAuth(userToken).
SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json")
resp = MakeRequest(t, req, http.StatusCreated)
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/manifests/artifact-v1", user.Name, image), resp.Header().Get("Location"))
assert.Equal(t, artifactManifestDigest, resp.Header().Get("Docker-Content-Digest"))
// Verify manifest can be retrieved
req = NewRequest(t, "GET", fmt.Sprintf("%s/manifests/artifact-v1", url)).
AddTokenAuth(userToken).
SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json")
resp = MakeRequest(t, req, http.StatusOK)
assert.Equal(t, "application/vnd.oci.image.manifest.v1+json", resp.Header().Get("Content-Type"))
assert.Equal(t, artifactManifestDigest, resp.Header().Get("Docker-Content-Digest"))
// Verify package was created with correct metadata
pvs, err := packages_model.GetVersionsByPackageType(db.DefaultContext, user.ID, packages_model.TypeContainer)
require.NoError(t, err)
found := false
for _, pv := range pvs {
if pv.LowerVersion == "artifact-v1" {
found = true
break
}
}
assert.True(t, found, "ORAS artifact package should be created")
})
for _, image := range images {
t.Run(fmt.Sprintf("[Image:%s]", image), func(t *testing.T) {
url := fmt.Sprintf("%sv2/%s/%s", setting.AppURL, user.Name, image)
@ -604,36 +688,76 @@ func TestPackageContainer(t *testing.T) {
t.Run("GetTagList", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
cases := []struct {
var cases []struct {
URL string
ExpectedTags []string
ExpectedLink string
}{
{
URL: fmt.Sprintf("%s/tags/list", url),
ExpectedTags: []string{"latest", "main", "multi"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
},
{
URL: fmt.Sprintf("%s/tags/list?n=0", url),
ExpectedTags: []string{},
ExpectedLink: "",
},
{
URL: fmt.Sprintf("%s/tags/list?n=2", url),
ExpectedTags: []string{"latest", "main"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=main&n=2>; rel="next"`, user.Name, image),
},
{
URL: fmt.Sprintf("%s/tags/list?last=main", url),
ExpectedTags: []string{"multi"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
},
{
URL: fmt.Sprintf("%s/tags/list?n=1&last=latest", url),
ExpectedTags: []string{"main"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=main&n=1>; rel="next"`, user.Name, image),
},
}
if image == "oras-artifact" {
cases = []struct {
URL string
ExpectedTags []string
ExpectedLink string
}{
{
URL: fmt.Sprintf("%s/tags/list", url),
ExpectedTags: []string{"artifact-v1", "latest", "main", "multi"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
},
{
URL: fmt.Sprintf("%s/tags/list?n=0", url),
ExpectedTags: []string{},
ExpectedLink: "",
},
{
URL: fmt.Sprintf("%s/tags/list?n=2", url),
ExpectedTags: []string{"artifact-v1", "latest"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=latest&n=2>; rel="next"`, user.Name, image),
},
{
URL: fmt.Sprintf("%s/tags/list?last=main", url),
ExpectedTags: []string{"multi"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
},
{
URL: fmt.Sprintf("%s/tags/list?n=1&last=latest", url),
ExpectedTags: []string{"main"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=main&n=1>; rel="next"`, user.Name, image),
},
}
} else {
cases = []struct {
URL string
ExpectedTags []string
ExpectedLink string
}{
{
URL: fmt.Sprintf("%s/tags/list", url),
ExpectedTags: []string{"latest", "main", "multi"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
},
{
URL: fmt.Sprintf("%s/tags/list?n=0", url),
ExpectedTags: []string{},
ExpectedLink: "",
},
{
URL: fmt.Sprintf("%s/tags/list?n=2", url),
ExpectedTags: []string{"latest", "main"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=main&n=2>; rel="next"`, user.Name, image),
},
{
URL: fmt.Sprintf("%s/tags/list?last=main", url),
ExpectedTags: []string{"multi"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=multi>; rel="next"`, user.Name, image),
},
{
URL: fmt.Sprintf("%s/tags/list?n=1&last=latest", url),
ExpectedTags: []string{"main"},
ExpectedLink: fmt.Sprintf(`</v2/%s/%s/tags/list?last=main&n=1>; rel="next"`, user.Name, image),
},
}
}
for _, c := range cases {
@ -660,7 +784,11 @@ func TestPackageContainer(t *testing.T) {
var apiPackages []*api.Package
DecodeJSON(t, resp, &apiPackages)
assert.Len(t, apiPackages, 4) // "latest", "main", "multi", "sha256:..."
if image == "oras-artifact" {
assert.Len(t, apiPackages, 5) // "artifact-v1", "latest", "main", "multi", "sha256:..."
} else {
assert.Len(t, apiPackages, 4) // "latest", "main", "multi", "sha256:..."
}
})
t.Run("Delete", func(t *testing.T) {