Create Proper Migration Tests (#15116)

* Create Proper Migration tests

Unfortunately our testing regime has so far meant that migrations do not
get proper testing.

This PR begins the process of creating migration tests for this.

* Add test for v176

* fix mssql drop db

Signed-off-by: Andrew Thornton <art27@cantab.net>
tokarchuk/v1.17
zeripath 4 years ago committed by GitHub
parent 750ac52db2
commit 39ef6f83d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 37
      Makefile
  2. 29
      models/migrations/fixtures/Test_deleteOrphanedIssueLabels/issue_label.yml
  3. 43
      models/migrations/fixtures/Test_deleteOrphanedIssueLabels/label.yml
  4. 52
      models/migrations/fixtures/Test_removeInvalidLabels/comment.yml
  5. 21
      models/migrations/fixtures/Test_removeInvalidLabels/issue.yml
  6. 35
      models/migrations/fixtures/Test_removeInvalidLabels/issue_label.yml
  7. 26
      models/migrations/fixtures/Test_removeInvalidLabels/label.yml
  8. 17
      models/migrations/fixtures/Test_removeInvalidLabels/repository.yml
  9. 17
      models/migrations/migrations.go
  10. 338
      models/migrations/migrations_test.go
  11. 188
      models/migrations/testlogger_test.go
  12. 3
      models/migrations/v176.go
  13. 128
      models/migrations/v176_test.go
  14. 1
      models/migrations/v177.go
  15. 88
      models/migrations/v177_test.go
  16. 7
      models/models.go
  17. 26
      models/test_fixtures.go

@ -89,7 +89,7 @@ LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(G
LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64 LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64
GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/integrations/migration-test,$(filter-out code.gitea.io/gitea/integrations,$(shell $(GO) list -mod=vendor ./... | grep -v /vendor/))) GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/models/migrations code.gitea.io/gitea/integrations/migration-test code.gitea.io/gitea/integrations,$(shell $(GO) list -mod=vendor ./... | grep -v /vendor/))
FOMANTIC_CONFIGS := semantic.json web_src/fomantic/theme.config.less web_src/fomantic/_site/globals/site.variables FOMANTIC_CONFIGS := semantic.json web_src/fomantic/theme.config.less web_src/fomantic/_site/globals/site.variables
FOMANTIC_DEST := web_src/fomantic/build/semantic.js web_src/fomantic/build/semantic.css FOMANTIC_DEST := web_src/fomantic/build/semantic.js web_src/fomantic/build/semantic.css
@ -399,8 +399,9 @@ test-sqlite\#%: integrations.sqlite.test generate-ini-sqlite
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/sqlite.ini ./integrations.sqlite.test -test.run $(subst .,/,$*) GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/sqlite.ini ./integrations.sqlite.test -test.run $(subst .,/,$*)
.PHONY: test-sqlite-migration .PHONY: test-sqlite-migration
test-sqlite-migration: migrations.sqlite.test generate-ini-sqlite test-sqlite-migration: migrations.sqlite.test migrations.individual.sqlite.test generate-ini-sqlite
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/sqlite.ini ./migrations.sqlite.test GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/sqlite.ini ./migrations.sqlite.test
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/sqlite.ini ./migrations.individual.sqlite.test
generate-ini-mysql: generate-ini-mysql:
sed -e 's|{{TEST_MYSQL_HOST}}|${TEST_MYSQL_HOST}|g' \ sed -e 's|{{TEST_MYSQL_HOST}}|${TEST_MYSQL_HOST}|g' \
@ -419,8 +420,9 @@ test-mysql\#%: integrations.mysql.test generate-ini-mysql
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql.ini ./integrations.mysql.test -test.run $(subst .,/,$*) GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql.ini ./integrations.mysql.test -test.run $(subst .,/,$*)
.PHONY: test-mysql-migration .PHONY: test-mysql-migration
test-mysql-migration: migrations.mysql.test generate-ini-mysql test-mysql-migration: migrations.mysql.test migrations.individual.mysql.test generate-ini-mysql
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql.ini ./migrations.mysql.test GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql.ini ./migrations.mysql.test
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql.ini ./migrations.individual.mysql.test
generate-ini-mysql8: generate-ini-mysql8:
sed -e 's|{{TEST_MYSQL8_HOST}}|${TEST_MYSQL8_HOST}|g' \ sed -e 's|{{TEST_MYSQL8_HOST}}|${TEST_MYSQL8_HOST}|g' \
@ -439,8 +441,9 @@ test-mysql8\#%: integrations.mysql8.test generate-ini-mysql8
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql8.ini ./integrations.mysql8.test -test.run $(subst .,/,$*) GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql8.ini ./integrations.mysql8.test -test.run $(subst .,/,$*)
.PHONY: test-mysql8-migration .PHONY: test-mysql8-migration
test-mysql8-migration: migrations.mysql8.test generate-ini-mysql8 test-mysql8-migration: migrations.mysql8.test migrations.individual.mysql8.test generate-ini-mysql8
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql8.ini ./migrations.mysql8.test GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql8.ini ./migrations.mysql8.test
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mysql8.ini ./migrations.individual.mysql8.test
generate-ini-pgsql: generate-ini-pgsql:
sed -e 's|{{TEST_PGSQL_HOST}}|${TEST_PGSQL_HOST}|g' \ sed -e 's|{{TEST_PGSQL_HOST}}|${TEST_PGSQL_HOST}|g' \
@ -460,8 +463,9 @@ test-pgsql\#%: integrations.pgsql.test generate-ini-pgsql
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/pgsql.ini ./integrations.pgsql.test -test.run $(subst .,/,$*) GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/pgsql.ini ./integrations.pgsql.test -test.run $(subst .,/,$*)
.PHONY: test-pgsql-migration .PHONY: test-pgsql-migration
test-pgsql-migration: migrations.pgsql.test generate-ini-pgsql test-pgsql-migration: migrations.pgsql.test migrations.individual.pgsql.test generate-ini-pgsql
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/pgsql.ini ./migrations.pgsql.test GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/pgsql.ini ./migrations.pgsql.test
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/pgsql.ini ./migrations.individual.pgsql.test
generate-ini-mssql: generate-ini-mssql:
sed -e 's|{{TEST_MSSQL_HOST}}|${TEST_MSSQL_HOST}|g' \ sed -e 's|{{TEST_MSSQL_HOST}}|${TEST_MSSQL_HOST}|g' \
@ -480,8 +484,9 @@ test-mssql\#%: integrations.mssql.test generate-ini-mssql
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mssql.ini ./integrations.mssql.test -test.run $(subst .,/,$*) GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mssql.ini ./integrations.mssql.test -test.run $(subst .,/,$*)
.PHONY: test-mssql-migration .PHONY: test-mssql-migration
test-mssql-migration: migrations.mssql.test generate-ini-mssql test-mssql-migration: migrations.mssql.test migrations.individual.mssql.test generate-ini-mssql
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mssql.ini ./migrations.mssql.test -test.failfast GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mssql.ini ./migrations.mssql.test -test.failfast
GITEA_ROOT="$(CURDIR)" GITEA_CONF=integrations/mssql.ini ./migrations.individual.mssql.test -test.failfast
.PHONY: bench-sqlite .PHONY: bench-sqlite
bench-sqlite: integrations.sqlite.test generate-ini-sqlite bench-sqlite: integrations.sqlite.test generate-ini-sqlite
@ -541,6 +546,26 @@ migrations.mssql.test: $(GO_SOURCES)
migrations.sqlite.test: $(GO_SOURCES) migrations.sqlite.test: $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags '$(TEST_TAGS)' $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags '$(TEST_TAGS)'
.PHONY: migrations.individual.mysql.test
migrations.individual.mysql.test: $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/models/migrations -o migrations.individual.mysql.test
.PHONY: migrations.individual.mysql8.test
migrations.individual.mysql8.test: $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/models/migrations -o migrations.individual.mysql8.test
.PHONY: migrations.individual.pgsql.test
migrations.individual.pgsql.test: $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/models/migrations -o migrations.individual.pgsql.test
.PHONY: migrations.individual.mssql.test
migrations.individual.mssql.test: $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/models/migrations -o migrations.individual.mssql.test
.PHONY: migrations.individual.sqlite.test
migrations.individual.sqlite.test: $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/models/migrations -o migrations.individual.sqlite.test -tags '$(TEST_TAGS)'
.PHONY: check .PHONY: check
check: test check: test

@ -0,0 +1,29 @@
# Issue_Label 1 should not be deleted
-
id: 1
issue_id: 1
label_id: 1
# Issue_label 2 should be deleted
-
id: 2
issue_id: 5
label_id: 99
# Issue_Label 3 should not be deleted
-
id: 3
issue_id: 2
label_id: 1
# Issue_Label 4 should not be deleted
-
id: 4
issue_id: 2
label_id: 4
-
id: 5
issue_id: 2
label_id: 87

@ -0,0 +1,43 @@
-
id: 1
repo_id: 1
org_id: 0
name: label1
color: '#abcdef'
num_issues: 2
num_closed_issues: 0
-
id: 2
repo_id: 1
org_id: 0
name: label2
color: '#000000'
num_issues: 1
num_closed_issues: 1
-
id: 3
repo_id: 0
org_id: 3
name: orglabel3
color: '#abcdef'
num_issues: 0
num_closed_issues: 0
-
id: 4
repo_id: 0
org_id: 3
name: orglabel4
color: '#000000'
num_issues: 1
num_closed_issues: 0
-
id: 5
repo_id: 10
org_id: 0
name: pull-test-label
color: '#000000'
num_issues: 0
num_closed_issues: 0

@ -0,0 +1,52 @@
# type Comment struct {
# ID int64 `xorm:"pk autoincr"`
# Type int `xorm:"INDEX"`
# IssueID int64 `xorm:"INDEX"`
# LabelID int64
# }
#
# we are only interested in type 7
#
-
id: 1 # Should remain
type: 6
issue_id: 1
label_id: 0
should_remain: true
-
id: 2 # Should remain
type: 7
issue_id: 1 # repo_id: 1
label_id: 1 # repo_id: 1
should_remain: true
-
id: 3 # Should remain
type: 7
issue_id: 2 # repo_id: 2 owner_id: 1
label_id: 2 # org_id: 1
should_remain: true
-
id: 4 # Should be DELETED
type: 7
issue_id: 1 # repo_id: 1
label_id: 3 # repo_id: 2
should_remain: false
-
id: 5 # Should remain
type: 7
issue_id: 3 # repo_id: 1
label_id: 1 # repo_id: 1
should_remain: true
-
id: 6 # Should be DELETED
type: 7
issue_id: 3 # repo_id: 1 owner_id: 2
label_id: 2 # org_id: 1
should_remain: false
-
id: 7 # Should be DELETED
type: 7
issue_id: 3 # repo_id: 1 owner_id: 2
label_id: 5 # repo_id: 3
should_remain: false

@ -0,0 +1,21 @@
# type Issue struct {
# ID int64 `xorm:"pk autoincr"`
# RepoID int64 `xorm:"INDEX UNIQUE(repo_index)"`
# Index int64 `xorm:"UNIQUE(repo_index)"` // Index in one repository.
# }
-
id: 1
repo_id: 1
index: 1
-
id: 2
repo_id: 2
index: 1
-
id: 3
repo_id: 1
index: 2
-
id: 4
repo_id: 3
index: 1

@ -0,0 +1,35 @@
# type IssueLabel struct {
# ID int64 `xorm:"pk autoincr"`
# IssueID int64 `xorm:"UNIQUE(s)"`
# LabelID int64 `xorm:"UNIQUE(s)"`
# }
-
id: 1 # Should remain - matches comment 2
issue_id: 1
label_id: 1
should_remain: true
-
id: 2 # Should remain
issue_id: 2
label_id: 2
should_remain: true
-
id: 3 # Should be deleted
issue_id: 1
label_id: 3
should_remain: false
-
id: 4 # Should remain
issue_id: 3
label_id: 1
should_remain: true
-
id: 5 # Should be deleted
issue_id: 3
label_id: 2
should_remain: false
-
id: 6 # Should be deleted
issue_id: 3
label_id: 5
should_remain: false

@ -0,0 +1,26 @@
# type Label struct {
# ID int64 `xorm:"pk autoincr"`
# RepoID int64 `xorm:"INDEX"`
# OrgID int64 `xorm:"INDEX"`
# }
-
id: 1
repo_id: 1
org_id: 0
-
id: 2
repo_id: 0
org_id: 1
-
id: 3
repo_id: 2
org_id: 0
-
id: 4
repo_id: 1
org_id: 0
-
id: 5
repo_id: 3
org_id: 0

@ -0,0 +1,17 @@
# type Repository struct {
# ID int64 `xorm:"pk autoincr"`
# OwnerID int64 `xorm:"UNIQUE(s) index"`
# LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
# }
-
id: 1
owner_id: 2
lower_name: "repo1"
-
id: 2
owner_id: 1
lower_name: "repo2"
-
id: 3
owner_id: 2
lower_name: "repo3"

@ -819,9 +819,24 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
} }
for _, constraint := range constraints { for _, constraint := range constraints {
if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` DROP CONSTRAINT `%s`", tableName, constraint)); err != nil { if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` DROP CONSTRAINT `%s`", tableName, constraint)); err != nil {
return fmt.Errorf("Drop table `%s` constraint `%s`: %v", tableName, constraint, err) return fmt.Errorf("Drop table `%s` default constraint `%s`: %v", tableName, constraint, err)
} }
} }
sql = fmt.Sprintf("SELECT DISTINCT Name FROM SYS.INDEXES INNER JOIN SYS.INDEX_COLUMNS ON INDEXES.INDEX_ID = INDEX_COLUMNS.INDEX_ID AND INDEXES.OBJECT_ID = INDEX_COLUMNS.OBJECT_ID WHERE INDEXES.OBJECT_ID = OBJECT_ID('%[1]s') AND INDEX_COLUMNS.COLUMN_ID IN (SELECT column_id FROM sys.columns WHERE lower(NAME) IN (%[2]s) AND object_id = OBJECT_ID('%[1]s'))",
tableName, strings.ReplaceAll(cols, "`", "'"))
constraints = make([]string, 0)
if err := sess.SQL(sql).Find(&constraints); err != nil {
return fmt.Errorf("Find constraints: %v", err)
}
for _, constraint := range constraints {
if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` DROP CONSTRAINT IF EXISTS `%s`", tableName, constraint)); err != nil {
return fmt.Errorf("Drop table `%s` index constraint `%s`: %v", tableName, constraint, err)
}
if _, err := sess.Exec(fmt.Sprintf("DROP INDEX IF EXISTS `%[2]s` ON `%[1]s`", tableName, constraint)); err != nil {
return fmt.Errorf("Drop index `%[2]s` on `%[1]s`: %v", tableName, constraint, err)
}
}
if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` DROP COLUMN %s", tableName, cols)); err != nil { if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` DROP COLUMN %s", tableName, cols)); err != nil {
return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err) return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
} }

@ -0,0 +1,338 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"database/sql"
"fmt"
"os"
"path"
"path/filepath"
"runtime"
"testing"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
"github.com/stretchr/testify/assert"
"github.com/unknwon/com"
"xorm.io/xorm"
"xorm.io/xorm/names"
)
func TestMain(m *testing.M) {
giteaRoot := base.SetupGiteaRoot()
if giteaRoot == "" {
fmt.Println("Environment variable $GITEA_ROOT not set")
os.Exit(1)
}
giteaBinary := "gitea"
if runtime.GOOS == "windows" {
giteaBinary += ".exe"
}
setting.AppPath = path.Join(giteaRoot, giteaBinary)
if _, err := os.Stat(setting.AppPath); err != nil {
fmt.Printf("Could not find gitea binary at %s\n", setting.AppPath)
os.Exit(1)
}
giteaConf := os.Getenv("GITEA_CONF")
if giteaConf == "" {
giteaConf = path.Join(filepath.Dir(setting.AppPath), "integrations/sqlite.ini")
fmt.Printf("Environment variable $GITEA_CONF not set - defaulting to %s\n", giteaConf)
}
if !path.IsAbs(giteaConf) {
setting.CustomConf = path.Join(giteaRoot, giteaConf)
} else {
setting.CustomConf = giteaConf
}
setting.SetCustomPathAndConf("", "", "")
setting.NewContext()
setting.CheckLFSVersion()
setting.InitDBConfig()
setting.NewLogServices(true)
exitStatus := m.Run()
if err := removeAllWithRetry(setting.RepoRootPath); err != nil {
fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err)
}
if err := removeAllWithRetry(setting.AppDataPath); err != nil {
fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err)
}
os.Exit(exitStatus)
}
func removeAllWithRetry(dir string) error {
var err error
for i := 0; i < 20; i++ {
err = os.RemoveAll(dir)
if err == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
return err
}
// SetEngine sets the xorm.Engine
func SetEngine() (*xorm.Engine, error) {
x, err := models.GetNewEngine()
if err != nil {
return x, fmt.Errorf("Failed to connect to database: %v", err)
}
x.SetMapper(names.GonicMapper{})
// WARNING: for serv command, MUST remove the output to os.stdout,
// so use log file to instead print to stdout.
x.SetLogger(models.NewXORMLogger(setting.Database.LogSQL))
x.ShowSQL(setting.Database.LogSQL)
x.SetMaxOpenConns(setting.Database.MaxOpenConns)
x.SetMaxIdleConns(setting.Database.MaxIdleConns)
x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
return x, nil
}
func deleteDB() error {
switch {
case setting.Database.UseSQLite3:
if err := util.Remove(setting.Database.Path); err != nil {
return err
}
return os.MkdirAll(path.Dir(setting.Database.Path), os.ModePerm)
case setting.Database.UseMySQL:
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/",
setting.Database.User, setting.Database.Passwd, setting.Database.Host))
if err != nil {
return err
}
defer db.Close()
if _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name)); err != nil {
return err
}
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
return err
}
return nil
case setting.Database.UsePostgreSQL:
db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
if err != nil {
return err
}
defer db.Close()
if _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name)); err != nil {
return err
}
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
return err
}
db.Close()
// Check if we need to setup a specific schema
if len(setting.Database.Schema) != 0 {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
if err != nil {
return err
}
defer db.Close()
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
if err != nil {
return err
}
defer schrows.Close()
if !schrows.Next() {
// Create and setup a DB schema
_, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema))
if err != nil {
return err
}
}
// Make the user's default search path the created schema; this will affect new connections
_, err = db.Exec(fmt.Sprintf(`ALTER USER "%s" SET search_path = %s`, setting.Database.User, setting.Database.Schema))
if err != nil {
return err
}
return nil
}
case setting.Database.UseMSSQL:
host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
host, port, "master", setting.Database.User, setting.Database.Passwd))
if err != nil {
return err
}
defer db.Close()
if _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS [%s]", setting.Database.Name)); err != nil {
return err
}
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE [%s]", setting.Database.Name)); err != nil {
return err
}
}
return nil
}
// prepareTestEnv prepares the test environment and reset the database. The skip parameter should usually be 0.
// Provide models to be sync'd with the database - in particular any models you expect fixtures to be loaded from.
//
// fixtures in `models/migrations/fixtures/<TestName>` will be loaded automatically
func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.Engine, func()) {
t.Helper()
ourSkip := 2
ourSkip += skip
deferFn := PrintCurrentTest(t, ourSkip)
assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
assert.NoError(t, com.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"),
setting.RepoRootPath))
if err := deleteDB(); err != nil {
t.Errorf("unable to reset database: %v", err)
return nil, deferFn
}
x, err := SetEngine()
assert.NoError(t, err)
if x != nil {
oldDefer := deferFn
deferFn = func() {
oldDefer()
if err := x.Close(); err != nil {
t.Errorf("error during close: %v", err)
}
}
}
if err != nil {
return x, deferFn
}
if len(syncModels) > 0 {
if err := x.Sync2(syncModels...); err != nil {
t.Errorf("error during sync: %v", err)
return x, deferFn
}
}
fixturesDir := filepath.Join(filepath.Dir(setting.AppPath), "models", "migrations", "fixtures", t.Name())
if _, err := os.Stat(fixturesDir); err == nil {
t.Logf("initializing fixtures from: %s", fixturesDir)
if err := models.InitFixtures(fixturesDir, x); err != nil {
t.Errorf("error whilst initializing fixtures from %s: %v", fixturesDir, err)
return x, deferFn
}
if err := models.LoadFixtures(x); err != nil {
t.Errorf("error whilst loading fixtures from %s: %v", fixturesDir, err)
return x, deferFn
}
} else if !os.IsNotExist(err) {
t.Errorf("unexpected error whilst checking for existence of fixtures: %v", err)
} else {
t.Logf("no fixtures found in: %s", fixturesDir)
}
return x, deferFn
}
func Test_dropTableColumns(t *testing.T) {
x, deferable := prepareTestEnv(t, 0)
if x == nil || t.Failed() {
defer deferable()
return
}
defer deferable()
type DropTest struct {
ID int64 `xorm:"pk autoincr"`
FirstColumn string
ToDropColumn string `xorm:"unique"`
AnotherColumn int64
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}
columns := []string{
"first_column",
"to_drop_column",
"another_column",
"created_unix",
"updated_unix",
}
for i := range columns {
x.SetMapper(names.GonicMapper{})
if err := x.Sync2(new(DropTest)); err != nil {
t.Errorf("unable to create DropTest table: %v", err)
return
}
sess := x.NewSession()
if err := sess.Begin(); err != nil {
sess.Close()
t.Errorf("unable to begin transaction: %v", err)
return
}
if err := dropTableColumns(sess, "drop_test", columns[i:]...); err != nil {
sess.Close()
t.Errorf("Unable to drop columns[%d:]: %s from drop_test: %v", i, columns[i:], err)
return
}
if err := sess.Commit(); err != nil {
sess.Close()
t.Errorf("unable to commit transaction: %v", err)
return
}
sess.Close()
if err := x.DropTables(new(DropTest)); err != nil {
t.Errorf("unable to drop table: %v", err)
return
}
for j := range columns[i+1:] {
x.SetMapper(names.GonicMapper{})
if err := x.Sync2(new(DropTest)); err != nil {
t.Errorf("unable to create DropTest table: %v", err)
return
}
dropcols := append([]string{columns[i]}, columns[j+i+1:]...)
sess := x.NewSession()
if err := sess.Begin(); err != nil {
sess.Close()
t.Errorf("unable to begin transaction: %v", err)
return
}
if err := dropTableColumns(sess, "drop_test", dropcols...); err != nil {
sess.Close()
t.Errorf("Unable to drop columns: %s from drop_test: %v", dropcols, err)
return
}
if err := sess.Commit(); err != nil {
sess.Close()
t.Errorf("unable to commit transaction: %v", err)
return
}
sess.Close()
if err := x.DropTables(new(DropTest)); err != nil {
t.Errorf("unable to drop table: %v", err)
return
}
}
}
}

@ -0,0 +1,188 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"os"
"runtime"
"strings"
"sync"
"testing"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
jsoniter "github.com/json-iterator/go"
)
var (
prefix string
slowTest = 10 * time.Second
slowFlush = 5 * time.Second
)
// TestLogger is a logger which will write to the testing log
type TestLogger struct {
log.WriterLogger
}
var writerCloser = &testLoggerWriterCloser{}
type testLoggerWriterCloser struct {
sync.RWMutex
t []*testing.TB
}
func (w *testLoggerWriterCloser) setT(t *testing.TB) {
w.Lock()
w.t = append(w.t, t)
w.Unlock()
}
func (w *testLoggerWriterCloser) Write(p []byte) (int, error) {
w.RLock()
var t *testing.TB
if len(w.t) > 0 {
t = w.t[len(w.t)-1]
}
w.RUnlock()
if t != nil && *t != nil {
if len(p) > 0 && p[len(p)-1] == '\n' {
p = p[:len(p)-1]
}
defer func() {
err := recover()
if err == nil {
return
}
var errString string
errErr, ok := err.(error)
if ok {
errString = errErr.Error()
} else {
errString, ok = err.(string)
}
if !ok {
panic(err)
}
if !strings.HasPrefix(errString, "Log in goroutine after ") {
panic(err)
}
}()
(*t).Log(string(p))
return len(p), nil
}
return len(p), nil
}
func (w *testLoggerWriterCloser) Close() error {
w.Lock()
if len(w.t) > 0 {
w.t = w.t[:len(w.t)-1]
}
w.Unlock()
return nil
}
// PrintCurrentTest prints the current test to os.Stdout
func PrintCurrentTest(t testing.TB, skip ...int) func() {
start := time.Now()
actualSkip := 1
if len(skip) > 0 {
actualSkip = skip[0]
}
_, filename, line, _ := runtime.Caller(actualSkip)
if log.CanColorStdout {
fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", fmt.Formatter(log.NewColoredValue(t.Name())), strings.TrimPrefix(filename, prefix), line)
} else {
fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line)
}
writerCloser.setT(&t)
return func() {
took := time.Since(start)
if took > slowTest {
if log.CanColorStdout {
fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgYellow)), fmt.Formatter(log.NewColoredValue(took, log.Bold, log.FgYellow)))
} else {
fmt.Fprintf(os.Stdout, "+++ %s is a slow tets (took %v)\n", t.Name(), took)
}
}
timer := time.AfterFunc(slowFlush, func() {
if log.CanColorStdout {
fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), slowFlush)
} else {
fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), slowFlush)
}
})
if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
t.Errorf("Flushing queues failed with error %v", err)
}
timer.Stop()
flushTook := time.Since(start) - took
if flushTook > slowFlush {
if log.CanColorStdout {
fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), fmt.Formatter(log.NewColoredValue(flushTook, log.Bold, log.FgRed)))
} else {
fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook)
}
}
_ = writerCloser.Close()
}
}
// Printf takes a format and args and prints the string to os.Stdout
func Printf(format string, args ...interface{}) {
if log.CanColorStdout {
for i := 0; i < len(args); i++ {
args[i] = log.NewColoredValue(args[i])
}
}
fmt.Fprintf(os.Stdout, "\t"+format, args...)
}
// NewTestLogger creates a TestLogger as a log.LoggerProvider
func NewTestLogger() log.LoggerProvider {
logger := &TestLogger{}
logger.Colorize = log.CanColorStdout
logger.Level = log.TRACE
return logger
}
// Init inits connection writer with json config.
// json config only need key "level".
func (log *TestLogger) Init(config string) error {
json := jsoniter.ConfigCompatibleWithStandardLibrary
err := json.Unmarshal([]byte(config), log)
if err != nil {
return err
}
log.NewWriterLogger(writerCloser)
return nil
}
// Flush when log should be flushed
func (log *TestLogger) Flush() {
}
//ReleaseReopen does nothing
func (log *TestLogger) ReleaseReopen() error {
return nil
}
// GetName returns the default name for this implementation
func (log *TestLogger) GetName() string {
return "test"
}
func init() {
log.Register("test", NewTestLogger)
_, filename, _, _ := runtime.Caller(0)
prefix = strings.TrimSuffix(filename, "integrations/testlogger.go")
}

@ -8,6 +8,9 @@ import (
"xorm.io/xorm" "xorm.io/xorm"
) )
// removeInvalidLabels looks through the database to look for comments and issue_labels
// that refer to labels do not belong to the repository or organization that repository
// that the issue is in
func removeInvalidLabels(x *xorm.Engine) error { func removeInvalidLabels(x *xorm.Engine) error {
type Comment struct { type Comment struct {
ID int64 `xorm:"pk autoincr"` ID int64 `xorm:"pk autoincr"`

@ -0,0 +1,128 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"testing"
"github.com/stretchr/testify/assert"
)
func Test_removeInvalidLabels(t *testing.T) {
// Models used by the migration
type Comment struct {
ID int64 `xorm:"pk autoincr"`
Type int `xorm:"INDEX"`
IssueID int64 `xorm:"INDEX"`
LabelID int64
ShouldRemain bool // <- Flag for testing the migration
}
type Issue struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX UNIQUE(repo_index)"`
Index int64 `xorm:"UNIQUE(repo_index)"` // Index in one repository.
}
type Repository struct {
ID int64 `xorm:"pk autoincr"`
OwnerID int64 `xorm:"UNIQUE(s) index"`
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
}
type Label struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX"`
OrgID int64 `xorm:"INDEX"`
}
type IssueLabel struct {
ID int64 `xorm:"pk autoincr"`
IssueID int64 `xorm:"UNIQUE(s)"`
LabelID int64 `xorm:"UNIQUE(s)"`
ShouldRemain bool // <- Flag for testing the migration
}
// load and prepare the test database
x, deferable := prepareTestEnv(t, 0, new(Comment), new(Issue), new(Repository), new(IssueLabel), new(Label))
if x == nil || t.Failed() {
defer deferable()
return
}
defer deferable()
var issueLabels []*IssueLabel
ilPreMigration := map[int64]*IssueLabel{}
ilPostMigration := map[int64]*IssueLabel{}
var comments []*Comment
comPreMigration := map[int64]*Comment{}
comPostMigration := map[int64]*Comment{}
// Get pre migration values
if err := x.Find(&issueLabels); err != nil {
t.Errorf("Unable to find issueLabels: %v", err)
return
}
for _, issueLabel := range issueLabels {
ilPreMigration[issueLabel.ID] = issueLabel
}
if err := x.Find(&comments); err != nil {
t.Errorf("Unable to find comments: %v", err)
return
}
for _, comment := range comments {
comPreMigration[comment.ID] = comment
}
// Run the migration
if err := removeInvalidLabels(x); err != nil {
t.Errorf("unable to RemoveInvalidLabels: %v", err)
}
// Get the post migration values
issueLabels = issueLabels[:0]
if err := x.Find(&issueLabels); err != nil {
t.Errorf("Unable to find issueLabels: %v", err)
return
}
for _, issueLabel := range issueLabels {
ilPostMigration[issueLabel.ID] = issueLabel
}
comments = comments[:0]
if err := x.Find(&comments); err != nil {
t.Errorf("Unable to find comments: %v", err)
return
}
for _, comment := range comments {
comPostMigration[comment.ID] = comment
}
// Finally test results of the migration
for id, comment := range comPreMigration {
post, ok := comPostMigration[id]
if ok {
if !comment.ShouldRemain {
t.Errorf("Comment[%d] remained but should have been deleted", id)
}
assert.Equal(t, comment, post)
} else if comment.ShouldRemain {
t.Errorf("Comment[%d] was deleted but should have remained", id)
}
}
for id, il := range ilPreMigration {
post, ok := ilPostMigration[id]
if ok {
if !il.ShouldRemain {
t.Errorf("IssueLabel[%d] remained but should have been deleted", id)
}
assert.Equal(t, il, post)
} else if il.ShouldRemain {
t.Errorf("IssueLabel[%d] was deleted but should have remained", id)
}
}
}

@ -10,6 +10,7 @@ import (
"xorm.io/xorm" "xorm.io/xorm"
) )
// deleteOrphanedIssueLabels looks through the database for issue_labels where the label no longer exists and deletes them.
func deleteOrphanedIssueLabels(x *xorm.Engine) error { func deleteOrphanedIssueLabels(x *xorm.Engine) error {
type IssueLabel struct { type IssueLabel struct {
ID int64 `xorm:"pk autoincr"` ID int64 `xorm:"pk autoincr"`

@ -0,0 +1,88 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"testing"
"code.gitea.io/gitea/modules/timeutil"
"github.com/stretchr/testify/assert"
)
func Test_deleteOrphanedIssueLabels(t *testing.T) {
// Create the models used in the migration
type IssueLabel struct {
ID int64 `xorm:"pk autoincr"`
IssueID int64 `xorm:"UNIQUE(s)"`
LabelID int64 `xorm:"UNIQUE(s)"`
}
type Label struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX"`
OrgID int64 `xorm:"INDEX"`
Name string
Description string
Color string `xorm:"VARCHAR(7)"`
NumIssues int
NumClosedIssues int
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}
// Prepare and load the testing database
x, deferable := prepareTestEnv(t, 0, new(IssueLabel), new(Label))
if x == nil || t.Failed() {
defer deferable()
return
}
defer deferable()
var issueLabels []*IssueLabel
preMigration := map[int64]*IssueLabel{}
postMigration := map[int64]*IssueLabel{}
// Load issue labels that exist in the database pre-migration
if err := x.Find(&issueLabels); err != nil {
assert.NoError(t, err)
return
}
for _, issueLabel := range issueLabels {
preMigration[issueLabel.ID] = issueLabel
}
// Run the migration
if err := deleteOrphanedIssueLabels(x); err != nil {
assert.NoError(t, err)
return
}
// Load the remaining issue-labels
issueLabels = issueLabels[:0]
if err := x.Find(&issueLabels); err != nil {
assert.NoError(t, err)
return
}
for _, issueLabel := range issueLabels {
postMigration[issueLabel.ID] = issueLabel
}
// Now test what is left
if _, ok := postMigration[2]; ok {
t.Errorf("Orphaned Label[2] survived the migration")
return
}
if _, ok := postMigration[5]; ok {
t.Errorf("Orphaned Label[5] survived the migration")
return
}
for id, post := range postMigration {
pre := preMigration[id]
assert.Equal(t, pre, post, "migration changed issueLabel %d", id)
}
}

@ -142,7 +142,8 @@ func init() {
} }
} }
func getEngine() (*xorm.Engine, error) { // GetNewEngine returns a new xorm engine from the configuration
func GetNewEngine() (*xorm.Engine, error) {
connStr, err := setting.DBConnStr() connStr, err := setting.DBConnStr()
if err != nil { if err != nil {
return nil, err return nil, err
@ -172,7 +173,7 @@ func getEngine() (*xorm.Engine, error) {
// NewTestEngine sets a new test xorm.Engine // NewTestEngine sets a new test xorm.Engine
func NewTestEngine() (err error) { func NewTestEngine() (err error) {
x, err = getEngine() x, err = GetNewEngine()
if err != nil { if err != nil {
return fmt.Errorf("Connect to database: %v", err) return fmt.Errorf("Connect to database: %v", err)
} }
@ -185,7 +186,7 @@ func NewTestEngine() (err error) {
// SetEngine sets the xorm.Engine // SetEngine sets the xorm.Engine
func SetEngine() (err error) { func SetEngine() (err error) {
x, err = getEngine() x, err = GetNewEngine()
if err != nil { if err != nil {
return fmt.Errorf("Failed to connect to database: %v", err) return fmt.Errorf("Failed to connect to database: %v", err)
} }

@ -10,16 +10,22 @@ import (
"time" "time"
"github.com/go-testfixtures/testfixtures/v3" "github.com/go-testfixtures/testfixtures/v3"
"xorm.io/xorm"
"xorm.io/xorm/schemas" "xorm.io/xorm/schemas"
) )
var fixtures *testfixtures.Loader var fixtures *testfixtures.Loader
// InitFixtures initialize test fixtures for a test database // InitFixtures initialize test fixtures for a test database
func InitFixtures(dir string) (err error) { func InitFixtures(dir string, engine ...*xorm.Engine) (err error) {
e := x
if len(engine) == 1 {
e = engine[0]
}
testfiles := testfixtures.Directory(dir) testfiles := testfixtures.Directory(dir)
dialect := "unknown" dialect := "unknown"
switch x.Dialect().URI().DBType { switch e.Dialect().URI().DBType {
case schemas.POSTGRES: case schemas.POSTGRES:
dialect = "postgres" dialect = "postgres"
case schemas.MYSQL: case schemas.MYSQL:
@ -33,13 +39,13 @@ func InitFixtures(dir string) (err error) {
os.Exit(1) os.Exit(1)
} }
loaderOptions := []func(loader *testfixtures.Loader) error{ loaderOptions := []func(loader *testfixtures.Loader) error{
testfixtures.Database(x.DB().DB), testfixtures.Database(e.DB().DB),
testfixtures.Dialect(dialect), testfixtures.Dialect(dialect),
testfixtures.DangerousSkipTestDatabaseCheck(), testfixtures.DangerousSkipTestDatabaseCheck(),
testfiles, testfiles,
} }
if x.Dialect().URI().DBType == schemas.POSTGRES { if e.Dialect().URI().DBType == schemas.POSTGRES {
loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences()) loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
} }
@ -52,7 +58,11 @@ func InitFixtures(dir string) (err error) {
} }
// LoadFixtures load fixtures for a test database // LoadFixtures load fixtures for a test database
func LoadFixtures() error { func LoadFixtures(engine ...*xorm.Engine) error {
e := x
if len(engine) == 1 {
e = engine[0]
}
var err error var err error
// Database transaction conflicts could occur and result in ROLLBACK // Database transaction conflicts could occur and result in ROLLBACK
// As a simple workaround, we just retry 20 times. // As a simple workaround, we just retry 20 times.
@ -67,8 +77,8 @@ func LoadFixtures() error {
fmt.Printf("LoadFixtures failed after retries: %v\n", err) fmt.Printf("LoadFixtures failed after retries: %v\n", err)
} }
// Now if we're running postgres we need to tell it to update the sequences // Now if we're running postgres we need to tell it to update the sequences
if x.Dialect().URI().DBType == schemas.POSTGRES { if e.Dialect().URI().DBType == schemas.POSTGRES {
results, err := x.QueryString(`SELECT 'SELECT SETVAL(' || results, err := e.QueryString(`SELECT 'SELECT SETVAL(' ||
quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) || quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) ||
', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' || ', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' ||
quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';' quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';'
@ -90,7 +100,7 @@ func LoadFixtures() error {
} }
for _, r := range results { for _, r := range results {
for _, value := range r { for _, value := range r {
_, err = x.Exec(value) _, err = e.Exec(value)
if err != nil { if err != nil {
fmt.Printf("Failed to update sequence: %s Error: %v\n", value, err) fmt.Printf("Failed to update sequence: %s Error: %v\n", value, err)
return err return err

Loading…
Cancel
Save