Refactor Cron and merge dashboard tasks (#10745)
* Refactor Cron and merge dashboard tasks * Merge Cron and Dashboard tasks * Make every cron task report a system notice on completion * Refactor the creation of these tasks * Ensure that execution counts of tasks is correct * Allow cron tasks to be started from the cron page * golangci-lint fixes * Enforce that only one task with the same name can be registered Signed-off-by: Andrew Thornton <art27@cantab.net> * fix name check Signed-off-by: Andrew Thornton <art27@cantab.net> * as per @guillep2k * as per @lafriks Signed-off-by: Andrew Thornton <art27@cantab.net> * Add git.CommandContext variants Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: techknowlogick <techknowlogick@gitea.io>tokarchuk/v1.17
zeripath
5 years ago
committed by
GitHub
parent
c18144086f
commit
9a2e47b23a
@ -0,0 +1,72 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cron |
||||
|
||||
import ( |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"github.com/unknwon/i18n" |
||||
) |
||||
|
||||
// Config represents a basic configuration interface that cron task
|
||||
type Config interface { |
||||
IsEnabled() bool |
||||
DoRunAtStart() bool |
||||
GetSchedule() string |
||||
FormatMessage(name, status string, doer *models.User, args ...interface{}) string |
||||
} |
||||
|
||||
// BaseConfig represents the basic config for a Cron task
|
||||
type BaseConfig struct { |
||||
Enabled bool |
||||
RunAtStart bool |
||||
Schedule string |
||||
} |
||||
|
||||
// OlderThanConfig represents a cron task with OlderThan setting
|
||||
type OlderThanConfig struct { |
||||
BaseConfig |
||||
OlderThan time.Duration |
||||
} |
||||
|
||||
// UpdateExistingConfig represents a cron task with UpdateExisting setting
|
||||
type UpdateExistingConfig struct { |
||||
BaseConfig |
||||
UpdateExisting bool |
||||
} |
||||
|
||||
// GetSchedule returns the schedule for the base config
|
||||
func (b *BaseConfig) GetSchedule() string { |
||||
return b.Schedule |
||||
} |
||||
|
||||
// IsEnabled returns the enabled status for the config
|
||||
func (b *BaseConfig) IsEnabled() bool { |
||||
return b.Enabled |
||||
} |
||||
|
||||
// DoRunAtStart returns whether the task should be run at the start
|
||||
func (b *BaseConfig) DoRunAtStart() bool { |
||||
return b.RunAtStart |
||||
} |
||||
|
||||
// FormatMessage returns a message for the task
|
||||
func (b *BaseConfig) FormatMessage(name, status string, doer *models.User, args ...interface{}) string { |
||||
realArgs := make([]interface{}, 0, len(args)+2) |
||||
realArgs = append(realArgs, i18n.Tr("en-US", "admin.dashboard."+name)) |
||||
if doer == nil { |
||||
realArgs = append(realArgs, "(Cron)") |
||||
} else { |
||||
realArgs = append(realArgs, doer.Name) |
||||
} |
||||
if len(args) > 0 { |
||||
realArgs = append(realArgs, args...) |
||||
} |
||||
if doer == nil || (doer.ID == -1 && doer.Name == "(Cron)") { |
||||
return i18n.Tr("en-US", "admin.dashboard.cron."+status, realArgs...) |
||||
} |
||||
return i18n.Tr("en-US", "admin.dashboard.task."+status, realArgs...) |
||||
} |
||||
@ -0,0 +1,166 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cron |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"reflect" |
||||
"sync" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"code.gitea.io/gitea/modules/graceful" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/process" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
) |
||||
|
||||
var lock = sync.Mutex{} |
||||
var started = false |
||||
var tasks = []*Task{} |
||||
var tasksMap = map[string]*Task{} |
||||
|
||||
// Task represents a Cron task
|
||||
type Task struct { |
||||
lock sync.Mutex |
||||
Name string |
||||
config Config |
||||
fun func(context.Context, *models.User, Config) error |
||||
ExecTimes int64 |
||||
} |
||||
|
||||
// DoRunAtStart returns if this task should run at the start
|
||||
func (t *Task) DoRunAtStart() bool { |
||||
return t.config.DoRunAtStart() |
||||
} |
||||
|
||||
// IsEnabled returns if this task is enabled as cron task
|
||||
func (t *Task) IsEnabled() bool { |
||||
return t.config.IsEnabled() |
||||
} |
||||
|
||||
// GetConfig will return a copy of the task's config
|
||||
func (t *Task) GetConfig() Config { |
||||
if reflect.TypeOf(t.config).Kind() == reflect.Ptr { |
||||
// Pointer:
|
||||
return reflect.New(reflect.ValueOf(t.config).Elem().Type()).Interface().(Config) |
||||
} |
||||
// Not pointer:
|
||||
return reflect.New(reflect.TypeOf(t.config)).Elem().Interface().(Config) |
||||
} |
||||
|
||||
// Run will run the task incrementing the cron counter with no user defined
|
||||
func (t *Task) Run() { |
||||
t.RunWithUser(&models.User{ |
||||
ID: -1, |
||||
Name: "(Cron)", |
||||
LowerName: "(cron)", |
||||
}, t.config) |
||||
} |
||||
|
||||
// RunWithUser will run the task incrementing the cron counter at the time with User
|
||||
func (t *Task) RunWithUser(doer *models.User, config Config) { |
||||
if !taskStatusTable.StartIfNotRunning(t.Name) { |
||||
return |
||||
} |
||||
t.lock.Lock() |
||||
if config == nil { |
||||
config = t.config |
||||
} |
||||
t.ExecTimes++ |
||||
t.lock.Unlock() |
||||
defer func() { |
||||
taskStatusTable.Stop(t.Name) |
||||
if err := recover(); err != nil { |
||||
// Recover a panic within the
|
||||
combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2)) |
||||
log.Error("PANIC whilst running task: %s Value: %v", t.Name, combinedErr) |
||||
} |
||||
}() |
||||
graceful.GetManager().RunWithShutdownContext(func(baseCtx context.Context) { |
||||
ctx, cancel := context.WithCancel(baseCtx) |
||||
defer cancel() |
||||
pm := process.GetManager() |
||||
pid := pm.Add(config.FormatMessage(t.Name, "process", doer), cancel) |
||||
defer pm.Remove(pid) |
||||
if err := t.fun(ctx, doer, config); err != nil { |
||||
if models.IsErrCancelled(err) { |
||||
message := err.(models.ErrCancelled).Message |
||||
if err := models.CreateNotice(models.NoticeTask, config.FormatMessage(t.Name, "aborted", doer, message)); err != nil { |
||||
log.Error("CreateNotice: %v", err) |
||||
} |
||||
return |
||||
} |
||||
if err := models.CreateNotice(models.NoticeTask, config.FormatMessage(t.Name, "error", doer, err)); err != nil { |
||||
log.Error("CreateNotice: %v", err) |
||||
} |
||||
return |
||||
} |
||||
if err := models.CreateNotice(models.NoticeTask, config.FormatMessage(t.Name, "finished", doer)); err != nil { |
||||
log.Error("CreateNotice: %v", err) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
// GetTask gets the named task
|
||||
func GetTask(name string) *Task { |
||||
lock.Lock() |
||||
defer lock.Unlock() |
||||
log.Info("Getting %s in %v", name, tasksMap[name]) |
||||
|
||||
return tasksMap[name] |
||||
} |
||||
|
||||
// RegisterTask allows a task to be registered with the cron service
|
||||
func RegisterTask(name string, config Config, fun func(context.Context, *models.User, Config) error) error { |
||||
log.Debug("Registering task: %s", name) |
||||
_, err := setting.GetCronSettings(name, config) |
||||
if err != nil { |
||||
log.Error("Unable to register cron task with name: %s Error: %v", name, err) |
||||
return err |
||||
} |
||||
|
||||
task := &Task{ |
||||
Name: name, |
||||
config: config, |
||||
fun: fun, |
||||
} |
||||
lock.Lock() |
||||
locked := true |
||||
defer func() { |
||||
if locked { |
||||
lock.Unlock() |
||||
} |
||||
}() |
||||
if _, has := tasksMap[task.Name]; has { |
||||
log.Error("A task with this name: %s has already been registered", name) |
||||
return fmt.Errorf("duplicate task with name: %s", task.Name) |
||||
} |
||||
|
||||
if config.IsEnabled() { |
||||
// We cannot use the entry return as there is no way to lock it
|
||||
if _, err = c.AddJob(name, config.GetSchedule(), task); err != nil { |
||||
log.Error("Unable to register cron task with name: %s Error: %v", name, err) |
||||
return err |
||||
} |
||||
} |
||||
|
||||
tasks = append(tasks, task) |
||||
tasksMap[task.Name] = task |
||||
if started && config.IsEnabled() && config.DoRunAtStart() { |
||||
lock.Unlock() |
||||
locked = false |
||||
task.Run() |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// RegisterTaskFatal will register a task but if there is an error log.Fatal
|
||||
func RegisterTaskFatal(name string, config Config, fun func(context.Context, *models.User, Config) error) { |
||||
if err := RegisterTask(name, config, fun); err != nil { |
||||
log.Fatal("Unable to register cron task %s Error: %v", name, err) |
||||
} |
||||
} |
||||
@ -0,0 +1,118 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cron |
||||
|
||||
import ( |
||||
"context" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"code.gitea.io/gitea/modules/migrations" |
||||
repository_service "code.gitea.io/gitea/modules/repository" |
||||
mirror_service "code.gitea.io/gitea/services/mirror" |
||||
) |
||||
|
||||
func registerUpdateMirrorTask() { |
||||
RegisterTaskFatal("update_mirrors", &BaseConfig{ |
||||
Enabled: true, |
||||
RunAtStart: false, |
||||
Schedule: "@every 10m", |
||||
}, func(ctx context.Context, _ *models.User, _ Config) error { |
||||
return mirror_service.Update(ctx) |
||||
}) |
||||
} |
||||
|
||||
func registerRepoHealthCheck() { |
||||
type RepoHealthCheckConfig struct { |
||||
BaseConfig |
||||
Timeout time.Duration |
||||
Args []string `delim:" "` |
||||
} |
||||
RegisterTaskFatal("repo_health_check", &RepoHealthCheckConfig{ |
||||
BaseConfig: BaseConfig{ |
||||
Enabled: true, |
||||
RunAtStart: false, |
||||
Schedule: "@every 24h", |
||||
}, |
||||
Timeout: 60 * time.Second, |
||||
Args: []string{}, |
||||
}, func(ctx context.Context, _ *models.User, config Config) error { |
||||
rhcConfig := config.(*RepoHealthCheckConfig) |
||||
return repository_service.GitFsck(ctx, rhcConfig.Timeout, rhcConfig.Args) |
||||
}) |
||||
} |
||||
|
||||
func registerCheckRepoStats() { |
||||
RegisterTaskFatal("check_repo_stats", &BaseConfig{ |
||||
Enabled: true, |
||||
RunAtStart: true, |
||||
Schedule: "@every 24h", |
||||
}, func(ctx context.Context, _ *models.User, _ Config) error { |
||||
return models.CheckRepoStats(ctx) |
||||
}) |
||||
} |
||||
|
||||
func registerArchiveCleanup() { |
||||
RegisterTaskFatal("archive_cleanup", &OlderThanConfig{ |
||||
BaseConfig: BaseConfig{ |
||||
Enabled: true, |
||||
RunAtStart: true, |
||||
Schedule: "@every 24h", |
||||
}, |
||||
OlderThan: 24 * time.Hour, |
||||
}, func(ctx context.Context, _ *models.User, config Config) error { |
||||
acConfig := config.(*OlderThanConfig) |
||||
return models.DeleteOldRepositoryArchives(ctx, acConfig.OlderThan) |
||||
}) |
||||
} |
||||
|
||||
func registerSyncExternalUsers() { |
||||
RegisterTaskFatal("sync_external_users", &UpdateExistingConfig{ |
||||
BaseConfig: BaseConfig{ |
||||
Enabled: true, |
||||
RunAtStart: false, |
||||
Schedule: "@every 24h", |
||||
}, |
||||
UpdateExisting: true, |
||||
}, func(ctx context.Context, _ *models.User, config Config) error { |
||||
realConfig := config.(*UpdateExistingConfig) |
||||
return models.SyncExternalUsers(ctx, realConfig.UpdateExisting) |
||||
}) |
||||
} |
||||
|
||||
func registerDeletedBranchesCleanup() { |
||||
RegisterTaskFatal("deleted_branches_cleanup", &OlderThanConfig{ |
||||
BaseConfig: BaseConfig{ |
||||
Enabled: true, |
||||
RunAtStart: true, |
||||
Schedule: "@every 24h", |
||||
}, |
||||
OlderThan: 24 * time.Hour, |
||||
}, func(ctx context.Context, _ *models.User, config Config) error { |
||||
realConfig := config.(*OlderThanConfig) |
||||
models.RemoveOldDeletedBranches(ctx, realConfig.OlderThan) |
||||
return nil |
||||
}) |
||||
} |
||||
|
||||
func registerUpdateMigrationPosterID() { |
||||
RegisterTaskFatal("update_migration_poster_id", &BaseConfig{ |
||||
Enabled: true, |
||||
RunAtStart: true, |
||||
Schedule: "@every 24h", |
||||
}, func(ctx context.Context, _ *models.User, _ Config) error { |
||||
return migrations.UpdateMigrationPosterID(ctx) |
||||
}) |
||||
} |
||||
|
||||
func initBasicTasks() { |
||||
registerUpdateMirrorTask() |
||||
registerRepoHealthCheck() |
||||
registerCheckRepoStats() |
||||
registerArchiveCleanup() |
||||
registerSyncExternalUsers() |
||||
registerDeletedBranchesCleanup() |
||||
registerUpdateMigrationPosterID() |
||||
} |
||||
@ -0,0 +1,119 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cron |
||||
|
||||
import ( |
||||
"context" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
repo_module "code.gitea.io/gitea/modules/repository" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
) |
||||
|
||||
func registerDeleteInactiveUsers() { |
||||
RegisterTaskFatal("delete_inactive_accounts", &OlderThanConfig{ |
||||
BaseConfig: BaseConfig{ |
||||
Enabled: false, |
||||
RunAtStart: false, |
||||
Schedule: "@annually", |
||||
}, |
||||
OlderThan: 0 * time.Second, |
||||
}, func(ctx context.Context, _ *models.User, config Config) error { |
||||
olderThanConfig := config.(*OlderThanConfig) |
||||
return models.DeleteInactiveUsers(ctx, olderThanConfig.OlderThan) |
||||
}) |
||||
} |
||||
|
||||
func registerDeleteRepositoryArchives() { |
||||
RegisterTaskFatal("delete_repo_archives", &BaseConfig{ |
||||
Enabled: false, |
||||
RunAtStart: false, |
||||
Schedule: "@annually", |
||||
}, func(ctx context.Context, _ *models.User, _ Config) error { |
||||
return models.DeleteRepositoryArchives(ctx) |
||||
}) |
||||
} |
||||
|
||||
func registerGarbageCollectRepositories() { |
||||
type RepoHealthCheckConfig struct { |
||||
BaseConfig |
||||
Timeout time.Duration |
||||
Args []string `delim:" "` |
||||
} |
||||
RegisterTaskFatal("git_gc_repos", &RepoHealthCheckConfig{ |
||||
BaseConfig: BaseConfig{ |
||||
Enabled: false, |
||||
RunAtStart: false, |
||||
Schedule: "@every 72h", |
||||
}, |
||||
Timeout: time.Duration(setting.Git.Timeout.GC) * time.Second, |
||||
Args: setting.Git.GCArgs, |
||||
}, func(ctx context.Context, _ *models.User, config Config) error { |
||||
rhcConfig := config.(*RepoHealthCheckConfig) |
||||
return repo_module.GitGcRepos(ctx, rhcConfig.Timeout, rhcConfig.Args...) |
||||
}) |
||||
} |
||||
|
||||
func registerRewriteAllPublicKeys() { |
||||
RegisterTaskFatal("resync_all_sshkeys", &BaseConfig{ |
||||
Enabled: false, |
||||
RunAtStart: false, |
||||
Schedule: "@every 72h", |
||||
}, func(_ context.Context, _ *models.User, _ Config) error { |
||||
return models.RewriteAllPublicKeys() |
||||
}) |
||||
} |
||||
|
||||
func registerRepositoryUpdateHook() { |
||||
RegisterTaskFatal("resync_all_hooks", &BaseConfig{ |
||||
Enabled: false, |
||||
RunAtStart: false, |
||||
Schedule: "@every 72h", |
||||
}, func(ctx context.Context, _ *models.User, _ Config) error { |
||||
return repo_module.SyncRepositoryHooks(ctx) |
||||
}) |
||||
} |
||||
|
||||
func registerReinitMissingRepositories() { |
||||
RegisterTaskFatal("reinit_missing_repos", &BaseConfig{ |
||||
Enabled: false, |
||||
RunAtStart: false, |
||||
Schedule: "@every 72h", |
||||
}, func(ctx context.Context, _ *models.User, _ Config) error { |
||||
return repo_module.ReinitMissingRepositories(ctx) |
||||
}) |
||||
} |
||||
|
||||
func registerDeleteMissingRepositories() { |
||||
RegisterTaskFatal("delete_missing_repos", &BaseConfig{ |
||||
Enabled: false, |
||||
RunAtStart: false, |
||||
Schedule: "@every 72h", |
||||
}, func(ctx context.Context, user *models.User, _ Config) error { |
||||
return repo_module.DeleteMissingRepositories(ctx, user) |
||||
}) |
||||
} |
||||
|
||||
func registerRemoveRandomAvatars() { |
||||
RegisterTaskFatal("delete_generated_repository_avatars", &BaseConfig{ |
||||
Enabled: false, |
||||
RunAtStart: false, |
||||
Schedule: "@every 72h", |
||||
}, func(ctx context.Context, _ *models.User, _ Config) error { |
||||
return models.RemoveRandomAvatars(ctx) |
||||
}) |
||||
} |
||||
|
||||
func initExtendedTasks() { |
||||
registerDeleteInactiveUsers() |
||||
registerDeleteRepositoryArchives() |
||||
registerGarbageCollectRepositories() |
||||
registerRewriteAllPublicKeys() |
||||
registerRepositoryUpdateHook() |
||||
registerReinitMissingRepositories() |
||||
registerDeleteMissingRepositories() |
||||
registerRemoveRandomAvatars() |
||||
} |
||||
Loading…
Reference in new issue