gitea/tests/test_utils.go
wxiaoguang ecbb03dc6d
Improve testing and try to fix MySQL hanging ()
By some CI fine tunes (`run tests`), SQLite & MSSQL could complete
in about 12~13 minutes (before > 14), MySQL could complete in 18 minutes
(before: about 23 or even > 30)

Major changes:

1. use tmpfs for MySQL storage
1. run `make test-mysql` instead of `make integration-test-coverage`
because the code coverage is not really used at the moment.
1. refactor testlogger to make it more reliable and be able to report
stuck stacktrace
1. do not requeue failed items when a queue is being flushed (failed
items would keep failing and make flush uncompleted)
1. reduce the file sizes for testing
1. use math ChaCha20 random data instead of crypot/rand (for testing
purpose only)
1. no need to `DeleteRepository` in `TestLinguist`
1. other related refactoring to make code easier to maintain
2024-11-15 23:45:07 +08:00

256 lines
8.7 KiB
Go

// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//nolint:forbidigo
package tests
import (
"context"
"database/sql"
"fmt"
"os"
"path/filepath"
"testing"
"code.gitea.io/gitea/models/db"
packages_model "code.gitea.io/gitea/models/packages"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/testlogger"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/routers"
"github.com/stretchr/testify/assert"
)
func InitTest(requireGitea bool) {
testlogger.Init()
giteaRoot := base.SetupGiteaRoot()
if giteaRoot == "" {
testlogger.Fatalf("Environment variable $GITEA_ROOT not set\n")
}
// TODO: Speedup tests that rely on the event source ticker, confirm whether there is any bug or failure.
// setting.UI.Notification.EventSourceUpdateTime = time.Second
setting.AppWorkPath = giteaRoot
setting.CustomPath = filepath.Join(setting.AppWorkPath, "custom")
if requireGitea {
giteaBinary := "gitea"
if setting.IsWindows {
giteaBinary += ".exe"
}
setting.AppPath = filepath.Join(giteaRoot, giteaBinary)
if _, err := os.Stat(setting.AppPath); err != nil {
testlogger.Fatalf("Could not find gitea binary at %s\n", setting.AppPath)
}
}
giteaConf := os.Getenv("GITEA_CONF")
if giteaConf == "" {
// By default, use sqlite.ini for testing, then IDE like GoLand can start the test process with debugger.
// It's easier for developers to debug bugs step by step with a debugger.
// Notice: when doing "ssh push", Gitea executes sub processes, debugger won't work for the sub processes.
giteaConf = "tests/sqlite.ini"
_ = os.Setenv("GITEA_CONF", giteaConf)
fmt.Printf("Environment variable $GITEA_CONF not set, use default: %s\n", giteaConf)
if !setting.EnableSQLite3 {
testlogger.Fatalf(`sqlite3 requires: import _ "github.com/mattn/go-sqlite3" or -tags sqlite,sqlite_unlock_notify` + "\n")
}
}
if !filepath.IsAbs(giteaConf) {
setting.CustomConf = filepath.Join(giteaRoot, giteaConf)
} else {
setting.CustomConf = giteaConf
}
unittest.InitSettings()
setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master"
_ = util.RemoveAll(repo_module.LocalCopyPath())
if err := git.InitFull(context.Background()); err != nil {
log.Fatal("git.InitOnceWithSync: %v", err)
}
setting.LoadDBSetting()
if err := storage.Init(); err != nil {
testlogger.Fatalf("Init storage failed: %v\n", err)
}
switch {
case setting.Database.Type.IsMySQL():
connType := "tcp"
if len(setting.Database.Host) > 0 && setting.Database.Host[0] == '/' { // looks like a unix socket
connType = "unix"
}
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@%s(%s)/",
setting.Database.User, setting.Database.Passwd, connType, setting.Database.Host))
defer db.Close()
if err != nil {
log.Fatal("sql.Open: %v", err)
}
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
log.Fatal("db.Exec: %v", err)
}
case setting.Database.Type.IsPostgreSQL():
var db *sql.DB
var err error
if setting.Database.Host[0] == '/' {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host))
} else {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
}
defer db.Close()
if err != nil {
log.Fatal("sql.Open: %v", err)
}
dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
if err != nil {
log.Fatal("db.Query: %v", err)
}
defer dbrows.Close()
if !dbrows.Next() {
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
log.Fatal("db.Exec: CREATE DATABASE: %v", err)
}
}
// Check if we need to setup a specific schema
if len(setting.Database.Schema) == 0 {
break
}
db.Close()
if setting.Database.Host[0] == '/' {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host))
} else {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
}
// This is a different db object; requires a different Close()
defer db.Close()
if err != nil {
log.Fatal("sql.Open: %v", err)
}
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
if err != nil {
log.Fatal("db.Query: %v", err)
}
defer schrows.Close()
if !schrows.Next() {
// Create and setup a DB schema
if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
log.Fatal("db.Exec: CREATE SCHEMA: %v", err)
}
}
case setting.Database.Type.IsMSSQL():
host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
host, port, "master", setting.Database.User, setting.Database.Passwd))
if err != nil {
log.Fatal("sql.Open: %v", err)
}
if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil {
log.Fatal("db.Exec: %v", err)
}
defer db.Close()
}
routers.InitWebInstalled(graceful.GetManager().HammerContext())
}
func PrepareAttachmentsStorage(t testing.TB) {
// prepare attachments directory and files
assert.NoError(t, storage.Clean(storage.Attachments))
s, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{
Path: filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "attachments"),
})
assert.NoError(t, err)
assert.NoError(t, s.IterateObjects("", func(p string, obj storage.Object) error {
_, err = storage.Copy(storage.Attachments, p, s, p)
return err
}))
}
func PrepareGitRepoDirectory(t testing.TB) {
if !assert.NotEmpty(t, setting.RepoRootPath) {
return
}
assert.NoError(t, unittest.SyncDirs(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
}
func PrepareArtifactsStorage(t testing.TB) {
// prepare actions artifacts directory and files
assert.NoError(t, storage.Clean(storage.ActionsArtifacts))
s, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{
Path: filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "artifacts"),
})
assert.NoError(t, err)
assert.NoError(t, s.IterateObjects("", func(p string, obj storage.Object) error {
_, err = storage.Copy(storage.ActionsArtifacts, p, s, p)
return err
}))
}
func PrepareLFSStorage(t testing.TB) {
// load LFS object fixtures
// (LFS storage can be on any of several backends, including remote servers, so init it with the storage API)
lfsFixtures, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{
Path: filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta"),
})
assert.NoError(t, err)
assert.NoError(t, storage.Clean(storage.LFS))
assert.NoError(t, lfsFixtures.IterateObjects("", func(path string, _ storage.Object) error {
_, err := storage.Copy(storage.LFS, path, lfsFixtures, path)
return err
}))
}
func PrepareCleanPackageData(t testing.TB) {
// clear all package data
assert.NoError(t, db.TruncateBeans(db.DefaultContext,
&packages_model.Package{},
&packages_model.PackageVersion{},
&packages_model.PackageFile{},
&packages_model.PackageBlob{},
&packages_model.PackageProperty{},
&packages_model.PackageBlobUpload{},
&packages_model.PackageCleanupRule{},
))
assert.NoError(t, storage.Clean(storage.Packages))
}
func PrepareTestEnv(t testing.TB, skip ...int) func() {
t.Helper()
deferFn := PrintCurrentTest(t, util.OptionalArg(skip)+1)
// load database fixtures
assert.NoError(t, unittest.LoadFixtures())
// do not add more Prepare* functions here, only call necessary ones in the related test functions
PrepareGitRepoDirectory(t)
PrepareLFSStorage(t)
PrepareCleanPackageData(t)
return deferFn
}
func PrintCurrentTest(t testing.TB, skip ...int) func() {
t.Helper()
return testlogger.PrintCurrentTest(t, util.OptionalArg(skip)+1)
}