diff --git a/cmd/cmd.go b/cmd/cmd.go index b89dd5d8b..fd713b9af 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -16,6 +16,7 @@ import ( "syscall" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -57,15 +58,17 @@ func confirm() (bool, error) { } func initDB(ctx context.Context) error { - return initDBDisableConsole(ctx, false) -} - -func initDBDisableConsole(ctx context.Context, disableConsole bool) error { - setting.NewContext() + setting.LoadFromExisting() setting.InitDBConfig() - setting.NewXORMLogService(disableConsole) + setting.NewXORMLogService(false) + + if setting.Database.Type == "" { + log.Fatal(`Database settings are missing from the configuration file: %q. +Ensure you are running in the correct environment or set the correct configuration file with -c. +If this is the intended configuration file complete the [database] section.`, setting.CustomConf) + } if err := db.InitEngine(ctx); err != nil { - return fmt.Errorf("models.SetEngine: %v", err) + return fmt.Errorf("unable to initialise the database using the configuration in %q. Error: %v", setting.CustomConf, err) } return nil } diff --git a/cmd/convert.go b/cmd/convert.go index 0b2e240b3..6d4d99a25 100644 --- a/cmd/convert.go +++ b/cmd/convert.go @@ -35,7 +35,6 @@ func runConvert(ctx *cli.Context) error { log.Info("Custom path: %s", setting.CustomPath) log.Info("Log path: %s", setting.LogRootPath) log.Info("Configuration file: %s", setting.CustomConf) - setting.InitDBConfig() if !setting.Database.UseMySQL { fmt.Println("This command can only be used with a MySQL database") diff --git a/cmd/doctor.go b/cmd/doctor.go index 27f91e41b..54c18b83d 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -87,7 +87,7 @@ func runRecreateTable(ctx *cli.Context) error { golog.SetPrefix("") golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT))) - setting.NewContext() + setting.LoadFromExisting() setting.InitDBConfig() setting.EnableXORMLog = ctx.Bool("debug") diff --git a/cmd/dump.go b/cmd/dump.go index efb939720..c158ee21f 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -159,7 +159,8 @@ func runDump(ctx *cli.Context) error { fatal("Deleting default logger failed. Can not write to stdout: %v", err) } } - setting.NewContext() + setting.LoadFromExisting() + // make sure we are logging to the console no matter what the configuration tells us do to if _, err := setting.Cfg.Section("log").NewKey("MODE", "console"); err != nil { fatal("Setting logging mode to console failed: %v", err) diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go index 31f4574c2..1a9344dbe 100644 --- a/cmd/dump_repo.go +++ b/cmd/dump_repo.go @@ -88,7 +88,6 @@ func runDumpRepository(ctx *cli.Context) error { log.Info("Custom path: %s", setting.CustomPath) log.Info("Log path: %s", setting.LogRootPath) log.Info("Configuration file: %s", setting.CustomConf) - setting.InitDBConfig() var ( serviceType structs.GitServiceType diff --git a/cmd/embedded.go b/cmd/embedded.go index 2aeaba478..c608667bf 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -115,7 +115,7 @@ func initEmbeddedExtractor(c *cli.Context) error { log.DelNamedLogger(log.DEFAULT) // Read configuration file - setting.NewContext() + setting.LoadAllowEmpty() pats, err := getPatterns(c.Args()) if err != nil { diff --git a/cmd/mailer.go b/cmd/mailer.go index a3d6baaa2..35fcb302f 100644 --- a/cmd/mailer.go +++ b/cmd/mailer.go @@ -18,7 +18,7 @@ func runSendMail(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() - setting.NewContext() + setting.LoadFromExisting() if err := argsSet(c, "title"); err != nil { return err diff --git a/cmd/migrate.go b/cmd/migrate.go index 054772d9e..49a13adeb 100644 --- a/cmd/migrate.go +++ b/cmd/migrate.go @@ -36,7 +36,6 @@ func runMigrate(ctx *cli.Context) error { log.Info("Custom path: %s", setting.CustomPath) log.Info("Log path: %s", setting.LogRootPath) log.Info("Configuration file: %s", setting.CustomConf) - setting.InitDBConfig() if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil { log.Fatal("Failed to initialize ORM engine: %v", err) diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index afb0fc02a..f8e2378ca 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -121,7 +121,6 @@ func runMigrateStorage(ctx *cli.Context) error { log.Info("Custom path: %s", setting.CustomPath) log.Info("Log path: %s", setting.LogRootPath) log.Info("Configuration file: %s", setting.CustomConf) - setting.InitDBConfig() if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil { log.Fatal("Failed to initialize ORM engine: %v", err) diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index 1208796c9..357bd92c7 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -50,7 +50,7 @@ func runRestoreRepository(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() - setting.NewContext() + setting.LoadFromExisting() statusCode, errStr := private.RestoreRepo( ctx, diff --git a/cmd/serv.go b/cmd/serv.go index e1a1d4cb7..e34e300cb 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -58,7 +58,7 @@ func setup(logPath string, debug bool) { } else { _ = log.NewLogger(1000, "console", "console", `{"level":"fatal","stacktracelevel":"NONE","stderr":true}`) } - setting.NewContext() + setting.LoadFromExisting() if debug { setting.RunMode = "dev" } diff --git a/cmd/web.go b/cmd/web.go index 4b6dfa71a..fbd62191a 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -124,6 +124,10 @@ func runWeb(ctx *cli.Context) error { } c := install.Routes() err := listen(c, false) + if err != nil { + log.Critical("Unable to open listener for installer. Is Gitea already running?") + graceful.GetManager().DoGracefulShutdown() + } select { case <-graceful.GetManager().IsShutdown(): <-graceful.GetManager().Done() @@ -145,7 +149,15 @@ func runWeb(ctx *cli.Context) error { log.Info("Global init") // Perform global initialization - routers.GlobalInit(graceful.GetManager().HammerContext()) + setting.LoadFromExisting() + routers.GlobalInitInstalled(graceful.GetManager().HammerContext()) + + // We check that AppDataPath exists here (it should have been created during installation) + // We can't check it in `GlobalInitInstalled`, because some integration tests + // use cmd -> GlobalInitInstalled, but the AppDataPath doesn't exist during those tests. + if _, err := os.Stat(setting.AppDataPath); err != nil { + log.Fatal("Can not find APP_DATA_PATH '%s'", setting.AppDataPath) + } // Override the provided port number within the configuration if ctx.IsSet("port") { diff --git a/contrib/environment-to-ini/environment-to-ini.go b/contrib/environment-to-ini/environment-to-ini.go index 03c876aa6..ccda03fa9 100644 --- a/contrib/environment-to-ini/environment-to-ini.go +++ b/contrib/environment-to-ini/environment-to-ini.go @@ -156,6 +156,7 @@ func runEnvironmentToIni(c *cli.Context) error { destination = setting.CustomConf } if destination != setting.CustomConf || changed { + log.Info("Settings saved to: %q", destination) err = cfg.SaveTo(destination) if err != nil { return err diff --git a/contrib/pr/checkout.go b/contrib/pr/checkout.go index 1e2a9714e..18c0585f4 100644 --- a/contrib/pr/checkout.go +++ b/contrib/pr/checkout.go @@ -49,7 +49,7 @@ func runPR() { log.Fatal(err) } setting.SetCustomPathAndConf("", "", "") - setting.NewContext() + setting.LoadAllowEmpty() setting.RepoRootPath, err = os.MkdirTemp(os.TempDir(), "repos") if err != nil { diff --git a/integrations/integration_test.go b/integrations/integration_test.go index 710e2b3b3..6dfc7350d 100644 --- a/integrations/integration_test.go +++ b/integrations/integration_test.go @@ -164,8 +164,8 @@ func initIntegrationTest() { } setting.SetCustomPathAndConf("", "", "") - setting.NewContext() - util.RemoveAll(models.LocalCopyPath()) + setting.LoadForTest() + _ = util.RemoveAll(models.LocalCopyPath()) git.CheckLFSVersion() setting.InitDBConfig() if err := storage.Init(); err != nil { @@ -240,7 +240,8 @@ func initIntegrationTest() { } defer db.Close() } - routers.GlobalInit(graceful.GetManager().HammerContext()) + + routers.GlobalInitInstalled(graceful.GetManager().HammerContext()) } func prepareTestEnv(t testing.TB, skip ...int) func() { @@ -254,6 +255,7 @@ func prepareTestEnv(t testing.TB, skip ...int) func() { assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath)) + return deferFn } diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index 1e74fcc94..57354c39c 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -56,7 +56,7 @@ func initMigrationTest(t *testing.T) func() { setting.CustomConf = giteaConf } - setting.NewContext() + setting.LoadForTest() assert.True(t, len(setting.RepoRootPath) != 0) assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) diff --git a/models/db/engine.go b/models/db/engine.go index 0f744d027..63e1d5547 100755 --- a/models/db/engine.go +++ b/models/db/engine.go @@ -8,7 +8,6 @@ package db import ( "context" "database/sql" - "errors" "fmt" "io" "reflect" @@ -92,8 +91,8 @@ func init() { } } -// NewEngine returns a new xorm engine from the configuration -func NewEngine() (*xorm.Engine, error) { +// newXORMEngine returns a new XORM engine from the configuration +func newXORMEngine() (*xorm.Engine, error) { connStr, err := setting.DBConnStr() if err != nil { return nil, err @@ -126,40 +125,49 @@ func SyncAllTables() error { return x.StoreEngine("InnoDB").Sync2(tables...) } -// InitEngine sets the xorm.Engine -func InitEngine(ctx context.Context) (err error) { - x, err = NewEngine() +// InitEngine initializes the xorm.Engine and sets it as db.DefaultContext +func InitEngine(ctx context.Context) error { + xormEngine, err := newXORMEngine() if err != nil { - return fmt.Errorf("Failed to connect to database: %v", err) + return fmt.Errorf("failed to connect to database: %v", err) } - x.SetMapper(names.GonicMapper{}) + xormEngine.SetMapper(names.GonicMapper{}) // WARNING: for serv command, MUST remove the output to os.stdout, // so use log file to instead print to stdout. - x.SetLogger(NewXORMLogger(setting.Database.LogSQL)) - x.ShowSQL(setting.Database.LogSQL) - x.SetMaxOpenConns(setting.Database.MaxOpenConns) - x.SetMaxIdleConns(setting.Database.MaxIdleConns) - x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) + xormEngine.SetLogger(NewXORMLogger(setting.Database.LogSQL)) + xormEngine.ShowSQL(setting.Database.LogSQL) + xormEngine.SetMaxOpenConns(setting.Database.MaxOpenConns) + xormEngine.SetMaxIdleConns(setting.Database.MaxIdleConns) + xormEngine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) + xormEngine.SetDefaultContext(ctx) + + SetDefaultEngine(ctx, xormEngine) + return nil +} +// SetDefaultEngine sets the default engine for db +func SetDefaultEngine(ctx context.Context, eng *xorm.Engine) { + x = eng DefaultContext = &Context{ Context: ctx, e: x, } - x.SetDefaultContext(ctx) - return nil } -// SetEngine is used by unit test code -func SetEngine(eng *xorm.Engine) { - x = eng - DefaultContext = &Context{ - Context: context.Background(), - e: x, +// UnsetDefaultEngine closes and unsets the default engine +// We hope the SetDefaultEngine and UnsetDefaultEngine can be paired, but it's impossible now, +// there are many calls to InitEngine -> SetDefaultEngine directly to overwrite the `x` and DefaultContext without close +// Global database engine related functions are all racy and there is no graceful close right now. +func UnsetDefaultEngine() { + if x != nil { + _ = x.Close() + x = nil } + DefaultContext = nil } -// InitEngineWithMigration initializes a new xorm.Engine +// InitEngineWithMigration initializes a new xorm.Engine and sets it as the db.DefaultContext // This function must never call .Sync2() if the provided migration function fails. // When called from the "doctor" command, the migration function is a version check // that prevents the doctor from fixing anything in the database if the migration level @@ -226,14 +234,6 @@ func NamesToBean(names ...string) ([]interface{}, error) { return beans, nil } -// Ping tests if database is alive -func Ping() error { - if x != nil { - return x.Ping() - } - return errors.New("database not configured") -} - // DumpDatabase dumps all data from database according the special database SQL syntax to file system. func DumpDatabase(filePath, dbType string) error { var tbs []*schemas.Table @@ -291,11 +291,3 @@ func GetMaxID(beanOrTableName interface{}) (maxID int64, err error) { _, err = x.Select("MAX(id)").Table(beanOrTableName).Get(&maxID) return } - -// FindByMaxID filled results as the condition from database -func FindByMaxID(maxID int64, limit int, results interface{}) error { - return x.Where("id <= ?", maxID). - OrderBy("id DESC"). - Limit(limit). - Find(results) -} diff --git a/models/db/install/db.go b/models/db/install/db.go new file mode 100644 index 000000000..363a8c867 --- /dev/null +++ b/models/db/install/db.go @@ -0,0 +1,65 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package install + +import ( + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/setting" + + "xorm.io/xorm" +) + +func getXORMEngine() *xorm.Engine { + return db.DefaultContext.(*db.Context).Engine().(*xorm.Engine) +} + +// CheckDatabaseConnection checks the database connection +func CheckDatabaseConnection() error { + e := db.GetEngine(db.DefaultContext) + _, err := e.Exec("SELECT 1") + return err +} + +// GetMigrationVersion gets the database migration version +func GetMigrationVersion() (int64, error) { + var installedDbVersion int64 + x := getXORMEngine() + exist, err := x.IsTableExist("version") + if err != nil { + return 0, err + } + if !exist { + return 0, nil + } + _, err = x.Table("version").Cols("version").Get(&installedDbVersion) + if err != nil { + return 0, err + } + return installedDbVersion, nil +} + +// HasPostInstallationUsers checks whether there are users after installation +func HasPostInstallationUsers() (bool, error) { + x := getXORMEngine() + exist, err := x.IsTableExist("user") + if err != nil { + return false, err + } + if !exist { + return false, nil + } + + // if there are 2 or more users in database, we consider there are users created after installation + threshold := 2 + if !setting.IsProd { + // to debug easily, with non-prod RUN_MODE, we only check the count to 1 + threshold = 1 + } + res, err := x.Table("user").Cols("id").Limit(threshold).Query() + if err != nil { + return false, err + } + return len(res) >= threshold, nil +} diff --git a/models/migrations/migrations_test.go b/models/migrations/migrations_test.go index f46070cf8..2effcb99e 100644 --- a/models/migrations/migrations_test.go +++ b/models/migrations/migrations_test.go @@ -5,6 +5,7 @@ package migrations import ( + "context" "database/sql" "fmt" "os" @@ -57,7 +58,7 @@ func TestMain(m *testing.M) { } setting.SetCustomPathAndConf("", "", "") - setting.NewContext() + setting.LoadForTest() git.CheckLFSVersion() setting.InitDBConfig() setting.NewLogServices(true) @@ -85,21 +86,11 @@ func removeAllWithRetry(dir string) error { return err } -// newEngine sets the xorm.Engine -func newEngine() (*xorm.Engine, error) { - x, err := db.NewEngine() - if err != nil { - return x, fmt.Errorf("Failed to connect to database: %v", err) +func newXORMEngine() (*xorm.Engine, error) { + if err := db.InitEngine(context.Background()); err != nil { + return nil, err } - - x.SetMapper(names.GonicMapper{}) - // WARNING: for serv command, MUST remove the output to os.stdout, - // so use log file to instead print to stdout. - x.SetLogger(db.NewXORMLogger(setting.Database.LogSQL)) - x.ShowSQL(setting.Database.LogSQL) - x.SetMaxOpenConns(setting.Database.MaxOpenConns) - x.SetMaxIdleConns(setting.Database.MaxIdleConns) - x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) + x := unittest.GetXORMEngine() return x, nil } @@ -213,7 +204,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En return nil, deferFn } - x, err := newEngine() + x, err := newXORMEngine() assert.NoError(t, err) if x != nil { oldDefer := deferFn diff --git a/models/ssh_key_test.go b/models/ssh_key_test.go index cc78f2604..b52a36bdb 100644 --- a/models/ssh_key_test.go +++ b/models/ssh_key_test.go @@ -16,7 +16,7 @@ import ( func init() { setting.SetCustomPathAndConf("", "", "") - setting.NewContext() + setting.LoadForTest() } func Test_SSHParsePublicKey(t *testing.T) { diff --git a/models/unittest/fixtures.go b/models/unittest/fixtures.go index 6277d1c72..e39acd766 100644 --- a/models/unittest/fixtures.go +++ b/models/unittest/fixtures.go @@ -18,7 +18,8 @@ import ( var fixtures *testfixtures.Loader -func getXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) { +// GetXORMEngine gets the XORM engine +func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) { if len(engine) == 1 { return engine[0] } @@ -27,7 +28,7 @@ func getXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) { // InitFixtures initialize test fixtures for a test database func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) { - e := getXORMEngine(engine...) + e := GetXORMEngine(engine...) var testfiles func(*testfixtures.Loader) error if opts.Dir != "" { testfiles = testfixtures.Directory(opts.Dir) @@ -69,7 +70,7 @@ func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) { // LoadFixtures load fixtures for a test database func LoadFixtures(engine ...*xorm.Engine) error { - e := getXORMEngine(engine...) + e := GetXORMEngine(engine...) var err error // Database transaction conflicts could occur and result in ROLLBACK // As a simple workaround, we just retry 20 times. diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 8771bc1d2..94c93755e 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -5,6 +5,7 @@ package unittest import ( + "context" "fmt" "net/url" "os" @@ -124,7 +125,7 @@ func CreateTestEngine(opts FixturesOptions) error { return err } x.SetMapper(names.GonicMapper{}) - db.SetEngine(x) + db.SetDefaultEngine(context.Background(), x) if err = db.SyncAllTables(); err != nil { return err diff --git a/modules/doctor/doctor.go b/modules/doctor/doctor.go index 6451788f9..20a32f186 100644 --- a/modules/doctor/doctor.go +++ b/modules/doctor/doctor.go @@ -44,7 +44,7 @@ func (w *wrappedLevelLogger) Log(skip int, level log.Level, format string, v ... } func initDBDisableConsole(ctx context.Context, disableConsole bool) error { - setting.NewContext() + setting.LoadFromExisting() setting.InitDBConfig() setting.NewXORMLogService(disableConsole) diff --git a/modules/doctor/paths.go b/modules/doctor/paths.go index 88172d315..b4eab631b 100644 --- a/modules/doctor/paths.go +++ b/modules/doctor/paths.go @@ -67,7 +67,7 @@ func checkConfigurationFiles(logger log.Logger, autofix bool) error { return err } - setting.NewContext() + setting.LoadFromExisting() configurationFiles := []configurationFile{ {"Configuration File Path", setting.CustomConf, false, true, false}, diff --git a/modules/private/internal.go b/modules/private/internal.go index f5b5db0ca..0a39ca7b8 100644 --- a/modules/private/internal.go +++ b/modules/private/internal.go @@ -13,14 +13,18 @@ import ( "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" ) func newRequest(ctx context.Context, url, method string) *httplib.Request { + if setting.InternalToken == "" { + log.Fatal(`The INTERNAL_TOKEN setting is missing from the configuration file: %q. +Ensure you are running in the correct environment or set the correct configuration file with -c.`, setting.CustomConf) + } return httplib.NewRequest(url, method). SetContext(ctx). - Header("Authorization", - fmt.Sprintf("Bearer %s", setting.InternalToken)) + Header("Authorization", fmt.Sprintf("Bearer %s", setting.InternalToken)) } // Response internal request response @@ -44,9 +48,6 @@ func newInternalRequest(ctx context.Context, url, method string) *httplib.Reques }) if setting.Protocol == setting.UnixSocket { req.SetTransport(&http.Transport{ - Dial: func(_, _ string) (net.Conn, error) { - return net.Dial("unix", setting.HTTPAddr) - }, DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { var d net.Dialer return d.DialContext(ctx, "unix", setting.HTTPAddr) diff --git a/modules/setting/directory.go b/modules/setting/directory.go new file mode 100644 index 000000000..5dcdd89c0 --- /dev/null +++ b/modules/setting/directory.go @@ -0,0 +1,40 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "fmt" + "os" +) + +// PrepareAppDataPath creates app data directory if necessary +func PrepareAppDataPath() error { + // FIXME: There are too many calls to MkdirAll in old code. It is incorrect. + // For example, if someDir=/mnt/vol1/gitea-home/data, if the mount point /mnt/vol1 is not mounted when Gitea runs, + // then gitea will make new empty directories in /mnt/vol1, all are stored in the root filesystem. + // The correct behavior should be: creating parent directories is end users' duty. We only create sub-directories in existing parent directories. + // For quickstart, the parent directories should be created automatically for first startup (eg: a flag or a check of INSTALL_LOCK). + // Now we can take the first step to do correctly (using Mkdir) in other packages, and prepare the AppDataPath here, then make a refactor in future. + + st, err := os.Stat(AppDataPath) + + if os.IsNotExist(err) { + err = os.MkdirAll(AppDataPath, os.ModePerm) + if err != nil { + return fmt.Errorf("unable to create the APP_DATA_PATH directory: %q, Error: %v", AppDataPath, err) + } + return nil + } + + if err != nil { + return fmt.Errorf("unable to use APP_DATA_PATH %q. Error: %v", AppDataPath, err) + } + + if !st.IsDir() /* also works for symlink */ { + return fmt.Errorf("the APP_DATA_PATH %q is not a directory (or symlink to a directory) and can't be used", AppDataPath) + } + + return nil +} diff --git a/modules/setting/i18n.go b/modules/setting/i18n.go new file mode 100644 index 000000000..113e1b558 --- /dev/null +++ b/modules/setting/i18n.go @@ -0,0 +1,51 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +// defaultI18nLangNames must be a slice, we need the order +var defaultI18nLangNames = []string{ + "en-US", "English", + "zh-CN", "简体中文", + "zh-HK", "繁體中文(香港)", + "zh-TW", "繁體中文(台灣)", + "de-DE", "Deutsch", + "fr-FR", "français", + "nl-NL", "Nederlands", + "lv-LV", "latviešu", + "ru-RU", "русский", + "uk-UA", "Українська", + "ja-JP", "日本語", + "es-ES", "español", + "pt-BR", "português do Brasil", + "pt-PT", "Português de Portugal", + "pl-PL", "polski", + "bg-BG", "български", + "it-IT", "italiano", + "fi-FI", "suomi", + "tr-TR", "Türkçe", + "cs-CZ", "čeština", + "sr-SP", "српски", + "sv-SE", "svenska", + "ko-KR", "한국어", + "el-GR", "ελληνικά", + "fa-IR", "فارسی", + "hu-HU", "magyar nyelv", + "id-ID", "bahasa Indonesia", + "ml-IN", "മലയാളം", +} + +func defaultI18nLangs() (res []string) { + for i := 0; i < len(defaultI18nLangNames); i += 2 { + res = append(res, defaultI18nLangNames[i]) + } + return +} + +func defaultI18nNames() (res []string) { + for i := 0; i < len(defaultI18nLangNames); i += 2 { + res = append(res, defaultI18nLangNames[i+1]) + } + return +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 15cdc1fe3..d219dbaaf 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -546,9 +546,27 @@ func SetCustomPathAndConf(providedCustom, providedConf, providedWorkPath string) } } -// NewContext initializes configuration context. +// LoadFromExisting initializes setting options from an existing config file (app.ini) +func LoadFromExisting() { + loadFromConf(false) +} + +// LoadAllowEmpty initializes setting options, it's also fine that if the config file (app.ini) doesn't exist +func LoadAllowEmpty() { + loadFromConf(true) +} + +// LoadForTest initializes setting options for tests +func LoadForTest() { + loadFromConf(true) + if err := PrepareAppDataPath(); err != nil { + log.Fatal("Can not prepare APP_DATA_PATH: %v", err) + } +} + +// loadFromConf initializes configuration context. // NOTE: do not print any log except error. -func NewContext() { +func loadFromConf(allowEmpty bool) { Cfg = ini.Empty() if WritePIDFile && len(PIDFile) > 0 { @@ -563,9 +581,10 @@ func NewContext() { if err := Cfg.Append(CustomConf); err != nil { log.Fatal("Failed to load custom conf '%s': %v", CustomConf, err) } - } else { - log.Warn("Custom config '%s' not found, ignore this if you're running first time", CustomConf) - } + } else if !allowEmpty { + log.Fatal("Unable to find configuration file: %q.\nEnsure you are running in the correct environment or set the correct configuration file with -c.", CustomConf) + } // else: no config file, a config file might be created at CustomConf later (might not) + Cfg.NameMapper = ini.SnackCase homeDir, err := com.HomeDir() @@ -698,18 +717,7 @@ func NewContext() { StaticRootPath = sec.Key("STATIC_ROOT_PATH").MustString(StaticRootPath) StaticCacheTime = sec.Key("STATIC_CACHE_TIME").MustDuration(6 * time.Hour) AppDataPath = sec.Key("APP_DATA_PATH").MustString(path.Join(AppWorkPath, "data")) - if _, err = os.Stat(AppDataPath); err != nil { - // FIXME: There are too many calls to MkdirAll in old code. It is incorrect. - // For example, if someDir=/mnt/vol1/gitea-home/data, if the mount point /mnt/vol1 is not mounted when Gitea runs, - // then gitea will make new empty directories in /mnt/vol1, all are stored in the root filesystem. - // The correct behavior should be: creating parent directories is end users' duty. We only create sub-directories in existing parent directories. - // For quickstart, the parent directories should be created automatically for first startup (eg: a flag or a check of INSTALL_LOCK). - // Now we can take the first step to do correctly (using Mkdir) in other packages, and prepare the AppDataPath here, then make a refactor in future. - err = os.MkdirAll(AppDataPath, os.ModePerm) - if err != nil { - log.Fatal("Failed to create the directory for app data path '%s'", AppDataPath) - } - } + EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(path.Join(AppWorkPath, "data/tmp/pprof")) @@ -864,6 +872,10 @@ func NewContext() { SuccessfulTokensCacheSize = sec.Key("SUCCESSFUL_TOKENS_CACHE_SIZE").MustInt(20) InternalToken = loadInternalToken(sec) + if InstallLock && InternalToken == "" { + // if Gitea has been installed but the InternalToken hasn't been generated (upgrade from an old release), we should generate + generateSaveInternalToken() + } cfgdata := sec.Key("PASSWORD_COMPLEXITY").Strings(",") if len(cfgdata) == 0 { @@ -975,19 +987,11 @@ func NewContext() { Langs = Cfg.Section("i18n").Key("LANGS").Strings(",") if len(Langs) == 0 { - Langs = []string{ - "en-US", "zh-CN", "zh-HK", "zh-TW", "de-DE", "fr-FR", "nl-NL", "lv-LV", - "ru-RU", "uk-UA", "ja-JP", "es-ES", "pt-BR", "pt-PT", "pl-PL", "bg-BG", - "it-IT", "fi-FI", "tr-TR", "cs-CZ", "sr-SP", "sv-SE", "ko-KR", "el-GR", - "fa-IR", "hu-HU", "id-ID", "ml-IN"} + Langs = defaultI18nLangs() } Names = Cfg.Section("i18n").Key("NAMES").Strings(",") if len(Names) == 0 { - Names = []string{"English", "简体中文", "繁體中文(香港)", "繁體中文(台灣)", "Deutsch", - "français", "Nederlands", "latviešu", "русский", "Українська", "日本語", - "español", "português do Brasil", "Português de Portugal", "polski", "български", - "italiano", "suomi", "Türkçe", "čeština", "српски", "svenska", "한국어", "ελληνικά", - "فارسی", "magyar nyelv", "bahasa Indonesia", "മലയാളം"} + Names = defaultI18nNames() } ShowFooterBranding = Cfg.Section("other").Key("SHOW_FOOTER_BRANDING").MustBool(false) @@ -1054,8 +1058,8 @@ func parseAuthorizedPrincipalsAllow(values []string) ([]string, bool) { func loadInternalToken(sec *ini.Section) string { uri := sec.Key("INTERNAL_TOKEN_URI").String() - if len(uri) == 0 { - return loadOrGenerateInternalToken(sec) + if uri == "" { + return sec.Key("INTERNAL_TOKEN").String() } tempURI, err := url.Parse(uri) if err != nil { @@ -1092,21 +1096,17 @@ func loadInternalToken(sec *ini.Section) string { return "" } -func loadOrGenerateInternalToken(sec *ini.Section) string { - var err error - token := sec.Key("INTERNAL_TOKEN").String() - if len(token) == 0 { - token, err = generate.NewInternalToken() - if err != nil { - log.Fatal("Error generate internal token: %v", err) - } - - // Save secret - CreateOrAppendToCustomConf(func(cfg *ini.File) { - cfg.Section("security").Key("INTERNAL_TOKEN").SetValue(token) - }) +// generateSaveInternalToken generates and saves the internal token to app.ini +func generateSaveInternalToken() { + token, err := generate.NewInternalToken() + if err != nil { + log.Fatal("Error generate internal token: %v", err) } - return token + + InternalToken = token + CreateOrAppendToCustomConf(func(cfg *ini.File) { + cfg.Section("security").Key("INTERNAL_TOKEN").SetValue(token) + }) } // MakeAbsoluteAssetURL returns the absolute asset url prefix without a trailing slash @@ -1186,6 +1186,8 @@ func CreateOrAppendToCustomConf(callback func(cfg *ini.File)) { callback(cfg) + log.Info("Settings saved to: %q", CustomConf) + if err := os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm); err != nil { log.Fatal("failed to create '%s': %v", CustomConf, err) return diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 523f1c78d..a2f6389e5 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -138,6 +138,11 @@ ssl_mode = SSL charset = Charset path = Path sqlite_helper = File path for the SQLite3 database.
Enter an absolute path if you run Gitea as a service. +reinstall_error = You are trying to install into an existing Gitea database +reinstall_confirm_message = Re-installing with an existing Gitea database can cause multiple problems. In most cases, you should use your existing "app.ini" to run Gitea. If you know what you are doing, confirm the following: +reinstall_confirm_check_1 = The data encrypted by the SECRET_KEY in app.ini may be lost: users may not be able to log in with 2FA/OTP & mirrors may not function correctly. By checking this box you confirm that the current app.ini file contains the correct the SECRET_KEY. +reinstall_confirm_check_2 = The repositories and settings may need to be re-synchronized. By checking this box you confirm that you will resynchronize the hooks for the repositories and authorized_keys file manually. You confirm that you will ensure that repository and mirror settings are correct. +reinstall_confirm_check_3 = You confirm that you are absolutely sure that this Gitea is running with the correct app.ini location and that you are sure that you have have to re-install. You confirm that you acknowledge the above risks. err_empty_db_path = The SQLite3 database path cannot be empty. no_admin_and_disable_registration = You cannot disable user self-registration without creating an administrator account. err_empty_admin_password = The administrator password cannot be empty. @@ -203,8 +208,12 @@ install_btn_confirm = Install Gitea test_git_failed = Could not test 'git' command: %v sqlite3_not_available = This Gitea version does not support SQLite3. Please download the official binary version from %s (not the 'gobuild' version). invalid_db_setting = The database settings are invalid: %v +invalid_db_table = The database table '%s' is invalid: %v invalid_repo_path = The repository root path is invalid: %v +invalid_app_data_path = The app data path is invalid: %v run_user_not_match = The 'run as' username is not the current username: %s -> %s +internal_token_failed = Failed to generate internal token: %v +secret_key_failed = Failed to generate secret key: %v save_config_failed = Failed to save configuration: %v invalid_admin_setting = Administrator account setting is invalid: %v install_success = Welcome! Thank you for choosing Gitea. Have fun and take care! diff --git a/routers/init.go b/routers/init.go index 58c738409..4cce7992d 100644 --- a/routers/init.go +++ b/routers/init.go @@ -95,9 +95,8 @@ func syncAppPathForGit(ctx context.Context) error { return nil } -// GlobalInit is for global configuration reload-able. -func GlobalInit(ctx context.Context) { - setting.NewContext() +// GlobalInitInstalled is for global installed configuration. +func GlobalInitInstalled(ctx context.Context) { if !setting.InstallLock { log.Fatal("Gitea is not installed") } diff --git a/routers/install/install.go b/routers/install/install.go index bd19ab5eb..b2f04b14d 100644 --- a/routers/install/install.go +++ b/routers/install/install.go @@ -15,6 +15,7 @@ import ( "time" "code.gitea.io/gitea/models/db" + db_install "code.gitea.io/gitea/models/db/install" "code.gitea.io/gitea/models/migrations" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" @@ -161,10 +162,81 @@ func Install(ctx *context.Context) { ctx.HTML(http.StatusOK, tplInstall) } +func checkDatabase(ctx *context.Context, form *forms.InstallForm) bool { + var err error + + if (setting.Database.Type == "sqlite3") && + len(setting.Database.Path) == 0 { + ctx.Data["Err_DbPath"] = true + ctx.RenderWithErr(ctx.Tr("install.err_empty_db_path"), tplInstall, form) + return false + } + + // Check if the user is trying to re-install in an installed database + db.UnsetDefaultEngine() + defer db.UnsetDefaultEngine() + + if err = db.InitEngine(ctx); err != nil { + if strings.Contains(err.Error(), `Unknown database type: sqlite3`) { + ctx.Data["Err_DbType"] = true + ctx.RenderWithErr(ctx.Tr("install.sqlite3_not_available", "https://docs.gitea.io/en-us/install-from-binary/"), tplInstall, form) + } else { + ctx.Data["Err_DbSetting"] = true + ctx.RenderWithErr(ctx.Tr("install.invalid_db_setting", err), tplInstall, form) + } + return false + } + + err = db_install.CheckDatabaseConnection() + if err != nil { + ctx.Data["Err_DbSetting"] = true + ctx.RenderWithErr(ctx.Tr("install.invalid_db_setting", err), tplInstall, form) + return false + } + + hasPostInstallationUser, err := db_install.HasPostInstallationUsers() + if err != nil { + ctx.Data["Err_DbSetting"] = true + ctx.RenderWithErr(ctx.Tr("install.invalid_db_table", "user", err), tplInstall, form) + return false + } + dbMigrationVersion, err := db_install.GetMigrationVersion() + if err != nil { + ctx.Data["Err_DbSetting"] = true + ctx.RenderWithErr(ctx.Tr("install.invalid_db_table", "version", err), tplInstall, form) + return false + } + + if hasPostInstallationUser && dbMigrationVersion > 0 { + log.Error("The database is likely to have been used by Gitea before, database migration version=%d", dbMigrationVersion) + confirmed := form.ReinstallConfirmFirst && form.ReinstallConfirmSecond && form.ReinstallConfirmThird + if !confirmed { + ctx.Data["Err_DbInstalledBefore"] = true + ctx.RenderWithErr(ctx.Tr("install.reinstall_error"), tplInstall, form) + return false + } + + log.Info("User confirmed reinstallation of Gitea into a pre-existing database") + } + + if hasPostInstallationUser || dbMigrationVersion > 0 { + log.Info("Gitea will be installed in a database with: hasPostInstallationUser=%v, dbMigrationVersion=%v", hasPostInstallationUser, dbMigrationVersion) + } + + return true +} + // SubmitInstall response for submit install items func SubmitInstall(ctx *context.Context) { - form := *web.GetForm(ctx).(*forms.InstallForm) var err error + + form := *web.GetForm(ctx).(*forms.InstallForm) + + // fix form values + if form.AppURL != "" && form.AppURL[len(form.AppURL)-1] != '/' { + form.AppURL += "/" + } + ctx.Data["CurDbOption"] = form.DbType if ctx.HasError() { @@ -186,9 +258,9 @@ func SubmitInstall(ctx *context.Context) { return } - // Pass basic check, now test configuration. - // Test database setting. + // ---- Basic checks are passed, now test configuration. + // Test database setting. setting.Database.Type = setting.GetDBTypeByName(form.DbType) setting.Database.Host = form.DbHost setting.Database.User = form.DbUser @@ -201,22 +273,13 @@ func SubmitInstall(ctx *context.Context) { setting.Database.LogSQL = !setting.IsProd setting.PasswordHashAlgo = form.PasswordAlgorithm - if (setting.Database.Type == "sqlite3") && - len(setting.Database.Path) == 0 { - ctx.Data["Err_DbPath"] = true - ctx.RenderWithErr(ctx.Tr("install.err_empty_db_path"), tplInstall, &form) + if !checkDatabase(ctx, &form) { return } - // Set test engine. - if err = db.InitEngineWithMigration(ctx, migrations.Migrate); err != nil { - if strings.Contains(err.Error(), `Unknown database type: sqlite3`) { - ctx.Data["Err_DbType"] = true - ctx.RenderWithErr(ctx.Tr("install.sqlite3_not_available", "https://docs.gitea.io/en-us/install-from-binary/"), tplInstall, &form) - } else { - ctx.Data["Err_DbSetting"] = true - ctx.RenderWithErr(ctx.Tr("install.invalid_db_setting", err), tplInstall, &form) - } + // Prepare AppDataPath, it is very important for Gitea + if err = setting.PrepareAppDataPath(); err != nil { + ctx.RenderWithErr(ctx.Tr("install.invalid_app_data_path", err), tplInstall, &form) return } @@ -299,9 +362,14 @@ func SubmitInstall(ctx *context.Context) { } } - if form.AppURL[len(form.AppURL)-1] != '/' { - form.AppURL += "/" + // Init the engine with migration + if err = db.InitEngineWithMigration(ctx, migrations.Migrate); err != nil { + db.UnsetDefaultEngine() + ctx.Data["Err_DbSetting"] = true + ctx.RenderWithErr(ctx.Tr("install.invalid_db_setting", err), tplInstall, &form) + return } + db.UnsetDefaultEngine() // Save settings. cfg := ini.Empty() @@ -344,12 +412,12 @@ func SubmitInstall(ctx *context.Context) { if form.LFSRootPath != "" { cfg.Section("server").Key("LFS_START_SERVER").SetValue("true") cfg.Section("server").Key("LFS_CONTENT_PATH").SetValue(form.LFSRootPath) - var secretKey string - if secretKey, err = generate.NewJwtSecretBase64(); err != nil { + var lfsJwtSecret string + if lfsJwtSecret, err = generate.NewJwtSecretBase64(); err != nil { ctx.RenderWithErr(ctx.Tr("install.lfs_jwt_secret_failed", err), tplInstall, &form) return } - cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(secretKey) + cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(lfsJwtSecret) } else { cfg.Section("server").Key("LFS_START_SERVER").SetValue("false") } @@ -390,16 +458,30 @@ func SubmitInstall(ctx *context.Context) { cfg.Section("log").Key("ROUTER").SetValue("console") cfg.Section("security").Key("INSTALL_LOCK").SetValue("true") - var secretKey string - if secretKey, err = generate.NewSecretKey(); err != nil { - ctx.RenderWithErr(ctx.Tr("install.secret_key_failed", err), tplInstall, &form) + + var internalToken string + if internalToken, err = generate.NewInternalToken(); err != nil { + ctx.RenderWithErr(ctx.Tr("install.internal_token_failed", err), tplInstall, &form) return } - cfg.Section("security").Key("SECRET_KEY").SetValue(secretKey) + cfg.Section("security").Key("INTERNAL_TOKEN").SetValue(internalToken) + + // if there is already a SECRET_KEY, we should not overwrite it, otherwise the encrypted data will not be able to be decrypted + if setting.SecretKey == "" { + var secretKey string + if secretKey, err = generate.NewSecretKey(); err != nil { + ctx.RenderWithErr(ctx.Tr("install.secret_key_failed", err), tplInstall, &form) + return + } + cfg.Section("security").Key("SECRET_KEY").SetValue(secretKey) + } + if len(form.PasswordAlgorithm) > 0 { cfg.Section("security").Key("PASSWORD_HASH_ALGO").SetValue(form.PasswordAlgorithm) } + log.Info("Save settings to custom config file %s", setting.CustomConf) + err = os.MkdirAll(filepath.Dir(setting.CustomConf), os.ModePerm) if err != nil { ctx.RenderWithErr(ctx.Tr("install.save_config_failed", err), tplInstall, &form) @@ -411,8 +493,10 @@ func SubmitInstall(ctx *context.Context) { return } - // Re-read settings - ReloadSettings(ctx) + // ---- All checks are passed + + // Reload settings (and re-initialize database connection) + reloadSettings(ctx) // Create admin account if len(form.AdminName) > 0 { diff --git a/routers/install/setting.go b/routers/install/setting.go index e2af66cfb..cf0a01ce3 100644 --- a/routers/install/setting.go +++ b/routers/install/setting.go @@ -16,17 +16,17 @@ import ( // PreloadSettings preloads the configuration to check if we need to run install func PreloadSettings(ctx context.Context) bool { - setting.NewContext() + setting.LoadAllowEmpty() if !setting.InstallLock { log.Info("AppPath: %s", setting.AppPath) log.Info("AppWorkPath: %s", setting.AppWorkPath) log.Info("Custom path: %s", setting.CustomPath) log.Info("Log path: %s", setting.LogRootPath) log.Info("Configuration file: %s", setting.CustomConf) - log.Info("Preparing to run install page") + log.Info("Prepare to run install page") translation.InitLocales() if setting.EnableSQLite3 { - log.Info("SQLite3 Supported") + log.Info("SQLite3 is supported") } setting.InitDBConfig() setting.NewServicesForInstall() @@ -36,9 +36,9 @@ func PreloadSettings(ctx context.Context) bool { return !setting.InstallLock } -// ReloadSettings rereads the settings and starts up the database -func ReloadSettings(ctx context.Context) { - setting.NewContext() +// reloadSettings reloads the existing settings and starts up the database +func reloadSettings(ctx context.Context) { + setting.LoadFromExisting() setting.InitDBConfig() if setting.InstallLock { if err := common.InitDBEngine(ctx); err == nil { diff --git a/services/forms/user_form.go b/services/forms/user_form.go index 9f86bf616..f114682b9 100644 --- a/services/forms/user_form.go +++ b/services/forms/user_form.go @@ -67,6 +67,11 @@ type InstallForm struct { AdminPasswd string `binding:"OmitEmpty;MaxSize(255)" locale:"install.admin_password"` AdminConfirmPasswd string AdminEmail string `binding:"OmitEmpty;MinSize(3);MaxSize(254);Include(@)" locale:"install.admin_email"` + + // ReinstallConfirmFirst we can not use 1/2/3 or A/B/C here, there is a framework bug, can not parse "reinstall_confirm_1" or "reinstall_confirm_a" + ReinstallConfirmFirst bool + ReinstallConfirmSecond bool + ReinstallConfirmThird bool } // Validate validates the fields diff --git a/templates/base/head.tmpl b/templates/base/head.tmpl index d529e6bfd..dd8e68761 100644 --- a/templates/base/head.tmpl +++ b/templates/base/head.tmpl @@ -51,6 +51,8 @@ copy_error: '{{.i18n.Tr "copy_error"}}', } }; + {{/* in case some pages don't render the pageData, we make sure it is an object to prevent null access */}} + window.config.pageData = window.config.pageData || {}; diff --git a/templates/install.tmpl b/templates/install.tmpl index a004e1939..1fa0929af 100644 --- a/templates/install.tmpl +++ b/templates/install.tmpl @@ -91,6 +91,30 @@ + {{if .Err_DbInstalledBefore}} +
+

{{.i18n.Tr "install.reinstall_confirm_message"}}

+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+ {{end}} +

{{.i18n.Tr "install.general_title"}}

diff --git a/web_src/less/_install.less b/web_src/less/_install.less index 3f59a99aa..c6d93c514 100644 --- a/web_src/less/_install.less +++ b/web_src/less/_install.less @@ -4,7 +4,7 @@ form { @input-padding: 320px !important; - label { + .inline.field label { text-align: right; width: @input-padding; } @@ -20,19 +20,32 @@ margin-left: @input-padding+15px; } - &.optional .title { - margin-left: 38%; + &.optional { + .title { + margin-left: 38%; + } + .checkbox { + margin-left: 40% !important; + label { + width: auto !important; + } + } } } } .ui { - .checkbox { - margin-left: 40% !important; - - label { - width: auto !important; - } + .reinstall-message { + width: 70%; + margin: 20px auto; + color: red; + text-align: left; + font-weight: bold; + } + .reinstall-confirm { + width: 70%; + text-align: left; + margin: 10px auto; } } }