chore: replace github.com/go-testfixtures/testfixtures (#7715)

- Replaces `github.com/go-testfixtures/testfixtures` with a homebrew solution that is fully compatible.
- The reason to replace this library is that it pulls in a lot of other libraries which is causing issues: (1) the test binary becomes bigger than necessary which really shows in incremental build times (this patch removes 27.6MiB of the integration test binary) (2) it pulls in libraries (mainly database drivers) that are not used and are not easy to upgrade in case of a security vulnerability, causing CI failures.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/7715
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Gusted <postmaster@gusted.xyz>
Co-committed-by: Gusted <postmaster@gusted.xyz>

models/unittest/fixture_loader.go

@@ -0,0 +1,198 @@
+// Copyright 2025 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package unittest
+
+import (
+	"database/sql"
+	"encoding/hex"
+	"encoding/json" //nolint:depguard
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+type insertSQL struct {
+	statement string
+	values    []any
+}
+
+type fixtureFile struct {
+	name       string
+	insertSQLs []insertSQL
+}
+
+type loader struct {
+	db      *sql.DB
+	dialect string
+
+	fixtureFiles []*fixtureFile
+}
+
+func newFixtureLoader(db *sql.DB, dialect string, fixturePaths []string) (*loader, error) {
+	l := &loader{
+		db:           db,
+		dialect:      dialect,
+		fixtureFiles: []*fixtureFile{},
+	}
+
+	// Load fixtures
+	for _, fixturePath := range fixturePaths {
+		stat, err := os.Stat(fixturePath)
+		if err != nil {
+			return nil, err
+		}
+
+		// If fixture path is a directory, then read read the files of the directory
+		// and use those as fixture files.
+		if stat.IsDir() {
+			files, err := os.ReadDir(fixturePath)
+			if err != nil {
+				return nil, err
+			}
+			for _, file := range files {
+				if !file.IsDir() {
+					fixtureFile, err := l.buildFixtureFile(filepath.Join(fixturePath, file.Name()))
+					if err != nil {
+						return nil, err
+					}
+					l.fixtureFiles = append(l.fixtureFiles, fixtureFile)
+				}
+			}
+		} else {
+			fixtureFile, err := l.buildFixtureFile(fixturePath)
+			if err != nil {
+				return nil, err
+			}
+			l.fixtureFiles = append(l.fixtureFiles, fixtureFile)
+		}
+	}
+
+	return l, nil
+}
+
+// quoteKeyword returns the quoted string of keyword.
+func (l *loader) quoteKeyword(keyword string) string {
+	switch l.dialect {
+	case "sqlite3":
+		return `"` + keyword + `"`
+	case "mysql":
+		return "`" + keyword + "`"
+	case "postgres":
+		parts := strings.Split(keyword, ".")
+		for i, p := range parts {
+			parts[i] = `"` + p + `"`
+		}
+		return strings.Join(parts, ".")
+	default:
+		return "invalid"
+	}
+}
+
+// placeholder returns the placeholder string.
+func (l *loader) placeholder(index int) string {
+	if l.dialect == "postgres" {
+		return fmt.Sprintf("$%d", index)
+	}
+	return "?"
+}
+
+func (l *loader) buildFixtureFile(fixturePath string) (*fixtureFile, error) {
+	f, err := os.Open(fixturePath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var records []map[string]any
+	if err := yaml.NewDecoder(f).Decode(&records); err != nil {
+		return nil, err
+	}
+
+	fixture := &fixtureFile{
+		name:       filepath.Base(strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))),
+		insertSQLs: []insertSQL{},
+	}
+
+	for _, record := range records {
+		columns := []string{}
+		sqlValues := []string{}
+		values := []any{}
+		i := 1
+
+		for key, value := range record {
+			columns = append(columns, l.quoteKeyword(key))
+
+			switch v := value.(type) {
+			case string:
+				// Try to decode hex.
+				if strings.HasPrefix(v, "0x") {
+					value, err = hex.DecodeString(strings.TrimPrefix(v, "0x"))
+					if err != nil {
+						return nil, err
+					}
+				}
+			case []any:
+				// Decode array.
+				var bytes []byte
+				bytes, err = json.Marshal(v)
+				if err != nil {
+					return nil, err
+				}
+				value = string(bytes)
+			}
+
+			values = append(values, value)
+
+			sqlValues = append(sqlValues, l.placeholder(i))
+			i++
+		}
+
+		// Construct the insert SQL.
+		fixture.insertSQLs = append(fixture.insertSQLs, insertSQL{
+			statement: fmt.Sprintf(
+				"INSERT INTO %s (%s) VALUES (%s)",
+				l.quoteKeyword(fixture.name),
+				strings.Join(columns, ", "),
+				strings.Join(sqlValues, ", "),
+			),
+			values: values,
+		})
+	}
+
+	return fixture, nil
+}
+
+func (l *loader) Load() error {
+	// Start transaction.
+	tx, err := l.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		_ = tx.Rollback()
+	}()
+
+	// Clean the table and re-insert the fixtures.
+	tableDeleted := map[string]struct{}{}
+	for _, fixture := range l.fixtureFiles {
+		if _, ok := tableDeleted[fixture.name]; !ok {
+			if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s", l.quoteKeyword(fixture.name))); err != nil {
+				return fmt.Errorf("cannot delete table %s: %w", fixture.name, err)
+			}
+			tableDeleted[fixture.name] = struct{}{}
+		}
+
+		for _, insertSQL := range fixture.insertSQLs {
+			if _, err := tx.Exec(insertSQL.statement, insertSQL.values...); err != nil {
+				return fmt.Errorf("cannot insert %q with values %q: %w", insertSQL.statement, insertSQL.values, err)
+			}
+		}
+	}
+
+	return tx.Commit()
+}

models/unittest/fixtures.go

@@ -6,7 +6,6 @@ package unittest
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"time"
 
@@ -14,12 +13,11 @@ import (
 	"forgejo.org/modules/auth/password/hash"
 	"forgejo.org/modules/setting"
 
-	"github.com/go-testfixtures/testfixtures/v3"
 	"xorm.io/xorm"
 	"xorm.io/xorm/schemas"
 )
 
-var fixturesLoader *testfixtures.Loader
+var fixturesLoader *loader
 
 // GetXORMEngine gets the XORM engine
 func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine, err error) {
@@ -31,6 +29,7 @@ func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine, err error) {
 
 func OverrideFixtures(dir string) func() {
 	old := fixturesLoader
+
 	opts := FixturesOptions{
 		Dir:  filepath.Join(setting.AppWorkPath, "models/fixtures/"),
 		Base: setting.AppWorkPath,
@@ -39,6 +38,7 @@ func OverrideFixtures(dir string) func() {
 	if err := InitFixtures(opts); err != nil {
 		panic(err)
 	}
+
 	return func() {
 		fixturesLoader = old
 	}
@@ -50,19 +50,20 @@ func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
 	if err != nil {
 		return err
 	}
-	var fixtureOptionFiles func(*testfixtures.Loader) error
+
+	fixturePaths := []string{}
 	if opts.Dir != "" {
-		fixtureOptionFiles = testfixtures.Directory(opts.Dir)
+		fixturePaths = append(fixturePaths, opts.Dir)
 	} else {
-		fixtureOptionFiles = testfixtures.Files(opts.Files...)
+		fixturePaths = append(fixturePaths, opts.Files...)
 	}
-	var fixtureOptionDirs []func(*testfixtures.Loader) error
 	if opts.Dirs != nil {
 		for _, dir := range opts.Dirs {
-			fixtureOptionDirs = append(fixtureOptionDirs, testfixtures.Directory(filepath.Join(opts.Base, dir)))
+			fixturePaths = append(fixturePaths, filepath.Join(opts.Base, dir))
 		}
 	}
-	dialect := "unknown"
+
+	var dialect string
 	switch e.Dialect().URI().DBType {
 	case schemas.POSTGRES:
 		dialect = "postgres"
@@ -71,22 +72,10 @@ func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
 	case schemas.SQLITE:
 		dialect = "sqlite3"
 	default:
-		fmt.Println("Unsupported RDBMS for integration tests")
-		os.Exit(1)
-	}
-	loaderOptions := []func(loader *testfixtures.Loader) error{
-		testfixtures.Database(e.DB().DB),
-		testfixtures.Dialect(dialect),
-		testfixtures.DangerousSkipTestDatabaseCheck(),
-		fixtureOptionFiles,
-	}
-	loaderOptions = append(loaderOptions, fixtureOptionDirs...)
-
-	if e.Dialect().URI().DBType == schemas.POSTGRES {
-		loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
+		panic("Unsupported RDBMS for test")
 	}
 
-	fixturesLoader, err = testfixtures.New(loaderOptions...)
+	fixturesLoader, err = newFixtureLoader(e.DB().DB, dialect, fixturePaths)
 	if err != nil {
 		return err
 	}