Attachments: Add extension support, allow all types for releases (#12465)

* Attachments: Add extension support, allow all types for releases

- Add support for file extensions, matching the `accept` attribute of `<input type="file">`
- Add support for type wildcard mime types, e.g. `image/*`
- Create repository.release.ALLOWED_TYPES setting (default unrestricted)
- Change default for attachment.ALLOWED_TYPES to a list of extensions
- Split out POST /attachments into two endpoints for issue/pr and
  releases to prevent circumvention of allowed types check

Fixes: https://github.com/go-gitea/gitea/pull/10172
Fixes: https://github.com/go-gitea/gitea/issues/7266
Fixes: https://github.com/go-gitea/gitea/pull/12460
Ref: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/file#Unique_file_type_specifiers

* rename function

* extract GET routes out of RepoMustNotBeArchived

Co-authored-by: Lauris BH <lauris@nix.lv>

modules/secret/secret.go

@@ -5,8 +5,14 @@
 package secret
 
 import (
+	"crypto/aes"
+	"crypto/cipher"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"io"
 )
 
 // New creats a new secret
@@ -31,3 +37,65 @@ func randomString(len int64) (string, error) {
 	b, err := randomBytes(len)
 	return base64.URLEncoding.EncodeToString(b), err
 }
+
+// AesEncrypt encrypts text and given key with AES.
+func AesEncrypt(key, text []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	b := base64.StdEncoding.EncodeToString(text)
+	ciphertext := make([]byte, aes.BlockSize+len(b))
+	iv := ciphertext[:aes.BlockSize]
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		return nil, err
+	}
+	cfb := cipher.NewCFBEncrypter(block, iv)
+	cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
+	return ciphertext, nil
+}
+
+// AesDecrypt decrypts text and given key with AES.
+func AesDecrypt(key, text []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	if len(text) < aes.BlockSize {
+		return nil, errors.New("ciphertext too short")
+	}
+	iv := text[:aes.BlockSize]
+	text = text[aes.BlockSize:]
+	cfb := cipher.NewCFBDecrypter(block, iv)
+	cfb.XORKeyStream(text, text)
+	data, err := base64.StdEncoding.DecodeString(string(text))
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+// EncryptSecret encrypts a string with given key into a hex string
+func EncryptSecret(key string, str string) (string, error) {
+	keyHash := sha256.Sum256([]byte(key))
+	plaintext := []byte(str)
+	ciphertext, err := AesEncrypt(keyHash[:], plaintext)
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(ciphertext), nil
+}
+
+// DecryptSecret decrypts a previously encrypted hex string
+func DecryptSecret(key string, cipherhex string) (string, error) {
+	keyHash := sha256.Sum256([]byte(key))
+	ciphertext, err := hex.DecodeString(cipherhex)
+	if err != nil {
+		return "", err
+	}
+	plaintext, err := AesDecrypt(keyHash[:], ciphertext)
+	if err != nil {
+		return "", err
+	}
+	return string(plaintext), nil
+}

modules/secret/secret_test.go

@@ -20,3 +20,16 @@ func TestNew(t *testing.T) {
 	// check if secrets
 	assert.NotEqual(t, result, result2)
 }
+
+func TestEncryptDecrypt(t *testing.T) {
+	var hex string
+	var str string
+
+	hex, _ = EncryptSecret("foo", "baz")
+	str, _ = DecryptSecret("foo", hex)
+	assert.Equal(t, str, "baz")
+
+	hex, _ = EncryptSecret("bar", "baz")
+	str, _ = DecryptSecret("foo", hex)
+	assert.NotEqual(t, str, "baz")
+}

modules/setting/attachment.go

@@ -6,7 +6,6 @@ package setting
 
 import (
 	"path/filepath"
-	"strings"
 
 	"code.gitea.io/gitea/modules/log"
 )
@@ -65,7 +64,7 @@ func newAttachmentService() {
 		Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/")
 	}
 
-	Attachment.AllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
+	Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".docx,.gif,.gz,.jpeg,.jpg,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip")
 	Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4)
 	Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5)
 	Attachment.Enabled = sec.Key("ENABLED").MustBool(true)

modules/setting/repository.go

@@ -58,7 +58,7 @@ var (
 		Upload struct {
 			Enabled      bool
 			TempPath     string
-			AllowedTypes []string `delim:"|"`
+			AllowedTypes string
 			FileMaxSize  int64
 			MaxFiles     int
 		} `ini:"-"`
@@ -85,6 +85,10 @@ var (
 			LockReasons []string
 		} `ini:"repository.issue"`
 
+		Release struct {
+			AllowedTypes string
+		} `ini:"repository.release"`
+
 		Signing struct {
 			SigningKey        string
 			SigningName       string
@@ -165,13 +169,13 @@ var (
 		Upload: struct {
 			Enabled      bool
 			TempPath     string
-			AllowedTypes []string `delim:"|"`
+			AllowedTypes string
 			FileMaxSize  int64
 			MaxFiles     int
 		}{
 			Enabled:      true,
 			TempPath:     "data/tmp/uploads",
-			AllowedTypes: []string{},
+			AllowedTypes: "",
 			FileMaxSize:  3,
 			MaxFiles:     5,
 		},
@@ -213,6 +217,12 @@ var (
 			LockReasons: strings.Split("Too heated,Off-topic,Spam,Resolved", ","),
 		},
 
+		Release: struct {
+			AllowedTypes string
+		}{
+			AllowedTypes: "",
+		},
+
 		// Signing settings
 		Signing: struct {
 			SigningKey        string

modules/upload/filetype.go

@@ -1,46 +0,0 @@
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package upload
-
-import (
-	"fmt"
-	"net/http"
-	"strings"
-
-	"code.gitea.io/gitea/modules/log"
-)
-
-// ErrFileTypeForbidden not allowed file type error
-type ErrFileTypeForbidden struct {
-	Type string
-}
-
-// IsErrFileTypeForbidden checks if an error is a ErrFileTypeForbidden.
-func IsErrFileTypeForbidden(err error) bool {
-	_, ok := err.(ErrFileTypeForbidden)
-	return ok
-}
-
-func (err ErrFileTypeForbidden) Error() string {
-	return fmt.Sprintf("File type is not allowed: %s", err.Type)
-}
-
-// VerifyAllowedContentType validates a file is allowed to be uploaded.
-func VerifyAllowedContentType(buf []byte, allowedTypes []string) error {
-	fileType := http.DetectContentType(buf)
-
-	for _, t := range allowedTypes {
-		t := strings.Trim(t, " ")
-
-		if t == "*/*" || t == fileType ||
-			// Allow directives after type, like 'text/plain; charset=utf-8'
-			strings.HasPrefix(fileType, t+";") {
-			return nil
-		}
-	}
-
-	log.Info("Attachment with type %s blocked from upload", fileType)
-	return ErrFileTypeForbidden{Type: fileType}
-}

modules/upload/filetype_test.go

@@ -1,47 +0,0 @@
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package upload
-
-import (
-	"bytes"
-	"compress/gzip"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestUpload(t *testing.T) {
-	testContent := []byte(`This is a plain text file.`)
-	var b bytes.Buffer
-	w := gzip.NewWriter(&b)
-	w.Write(testContent)
-	w.Close()
-
-	kases := []struct {
-		data         []byte
-		allowedTypes []string
-		err          error
-	}{
-		{
-			data:         testContent,
-			allowedTypes: []string{"text/plain"},
-			err:          nil,
-		},
-		{
-			data:         testContent,
-			allowedTypes: []string{"application/x-gzip"},
-			err:          ErrFileTypeForbidden{"text/plain; charset=utf-8"},
-		},
-		{
-			data:         b.Bytes(),
-			allowedTypes: []string{"application/x-gzip"},
-			err:          nil,
-		},
-	}
-
-	for _, kase := range kases {
-		assert.Equal(t, kase.err, VerifyAllowedContentType(kase.data, kase.allowedTypes))
-	}
-}

modules/upload/upload.go

@@ -0,0 +1,94 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package upload
+
+import (
+	"net/http"
+	"path"
+	"regexp"
+	"strings"
+
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// ErrFileTypeForbidden not allowed file type error
+type ErrFileTypeForbidden struct {
+	Type string
+}
+
+// IsErrFileTypeForbidden checks if an error is a ErrFileTypeForbidden.
+func IsErrFileTypeForbidden(err error) bool {
+	_, ok := err.(ErrFileTypeForbidden)
+	return ok
+}
+
+func (err ErrFileTypeForbidden) Error() string {
+	return "This file extension or type is not allowed to be uploaded."
+}
+
+var mimeTypeSuffixRe = regexp.MustCompile(`;.*$`)
+var wildcardTypeRe = regexp.MustCompile(`^[a-z]+/\*$`)
+
+// Verify validates whether a file is allowed to be uploaded.
+func Verify(buf []byte, fileName string, allowedTypesStr string) error {
+	allowedTypesStr = strings.ReplaceAll(allowedTypesStr, "|", ",") // compat for old config format
+
+	allowedTypes := []string{}
+	for _, entry := range strings.Split(allowedTypesStr, ",") {
+		entry = strings.ToLower(strings.TrimSpace(entry))
+		if entry != "" {
+			allowedTypes = append(allowedTypes, entry)
+		}
+	}
+
+	if len(allowedTypes) == 0 {
+		return nil // everything is allowed
+	}
+
+	fullMimeType := http.DetectContentType(buf)
+	mimeType := strings.TrimSpace(mimeTypeSuffixRe.ReplaceAllString(fullMimeType, ""))
+	extension := strings.ToLower(path.Ext(fileName))
+
+	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/file#Unique_file_type_specifiers
+	for _, allowEntry := range allowedTypes {
+		if allowEntry == "*/*" {
+			return nil // everything allowed
+		} else if strings.HasPrefix(allowEntry, ".") && allowEntry == extension {
+			return nil // extension is allowed
+		} else if mimeType == allowEntry {
+			return nil // mime type is allowed
+		} else if wildcardTypeRe.MatchString(allowEntry) && strings.HasPrefix(mimeType, allowEntry[:len(allowEntry)-1]) {
+			return nil // wildcard match, e.g. image/*
+		}
+	}
+
+	log.Info("Attachment with type %s blocked from upload", fullMimeType)
+	return ErrFileTypeForbidden{Type: fullMimeType}
+}
+
+// AddUploadContext renders template values for dropzone
+func AddUploadContext(ctx *context.Context, uploadType string) {
+	if uploadType == "release" {
+		ctx.Data["UploadUrl"] = ctx.Repo.RepoLink + "/releases/attachments"
+		ctx.Data["UploadRemoveUrl"] = ctx.Repo.RepoLink + "/releases/attachments/remove"
+		ctx.Data["UploadAccepts"] = strings.Replace(setting.Repository.Release.AllowedTypes, "|", ",", -1)
+		ctx.Data["UploadMaxFiles"] = setting.Attachment.MaxFiles
+		ctx.Data["UploadMaxSize"] = setting.Attachment.MaxSize
+	} else if uploadType == "comment" {
+		ctx.Data["UploadUrl"] = ctx.Repo.RepoLink + "/issues/attachments"
+		ctx.Data["UploadRemoveUrl"] = ctx.Repo.RepoLink + "/issues/attachments/remove"
+		ctx.Data["UploadAccepts"] = strings.Replace(setting.Attachment.AllowedTypes, "|", ",", -1)
+		ctx.Data["UploadMaxFiles"] = setting.Attachment.MaxFiles
+		ctx.Data["UploadMaxSize"] = setting.Attachment.MaxSize
+	} else if uploadType == "repo" {
+		ctx.Data["UploadUrl"] = ctx.Repo.RepoLink + "/upload-file"
+		ctx.Data["UploadRemoveUrl"] = ctx.Repo.RepoLink + "/upload-remove"
+		ctx.Data["UploadAccepts"] = strings.Replace(setting.Repository.Upload.AllowedTypes, "|", ",", -1)
+		ctx.Data["UploadMaxFiles"] = setting.Repository.Upload.MaxFiles
+		ctx.Data["UploadMaxSize"] = setting.Repository.Upload.FileMaxSize
+	}
+}

modules/upload/upload_test.go

@@ -0,0 +1,195 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package upload
+
+import (
+	"bytes"
+	"compress/gzip"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUpload(t *testing.T) {
+	testContent := []byte(`This is a plain text file.`)
+	var b bytes.Buffer
+	w := gzip.NewWriter(&b)
+	w.Write(testContent)
+	w.Close()
+
+	kases := []struct {
+		data         []byte
+		fileName     string
+		allowedTypes string
+		err          error
+	}{
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "dir/test.txt",
+			allowedTypes: "",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "../../../test.txt",
+			allowedTypes: "",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: ",",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "|",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "*/*",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "*/*,",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "*/*|",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "text/plain",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "dir/test.txt",
+			allowedTypes: "text/plain",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "/dir.txt/test.js",
+			allowedTypes: ".js",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: " text/plain ",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: ".txt",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: " .txt,.js",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: " .txt|.js",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "../../test.txt",
+			allowedTypes: " .txt|.js",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: " .txt ,.js ",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "text/plain, .txt",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "text/*",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "text/*,.js",
+			err:          nil,
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "text/**",
+			err:          ErrFileTypeForbidden{"text/plain; charset=utf-8"},
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: "application/x-gzip",
+			err:          ErrFileTypeForbidden{"text/plain; charset=utf-8"},
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: ".zip",
+			err:          ErrFileTypeForbidden{"text/plain; charset=utf-8"},
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: ".zip,.txtx",
+			err:          ErrFileTypeForbidden{"text/plain; charset=utf-8"},
+		},
+		{
+			data:         testContent,
+			fileName:     "test.txt",
+			allowedTypes: ".zip|.txtx",
+			err:          ErrFileTypeForbidden{"text/plain; charset=utf-8"},
+		},
+		{
+			data:         b.Bytes(),
+			fileName:     "test.txt",
+			allowedTypes: "application/x-gzip",
+			err:          nil,
+		},
+	}
+
+	for _, kase := range kases {
+		assert.Equal(t, kase.err, Verify(kase.data, kase.fileName, kase.allowedTypes))
+	}
+}