Add Status Updates whilst Gitea migrations are occurring (#15076)

* Add migrating message

Signed-off-by: Andrew Thornton <art27@cantab.net>

* simplify messenger

Signed-off-by: Andrew Thornton <art27@cantab.net>

* make messenger an interface

Signed-off-by: Andrew Thornton <art27@cantab.net>

* rename

Signed-off-by: Andrew Thornton <art27@cantab.net>

* prepare for merge

Signed-off-by: Andrew Thornton <art27@cantab.net>

* as per tech

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: 6543 <6543@obermui.de>
tokarchuk/v1.17
zeripath 3 years ago committed by GitHub
parent 047c39e91b
commit 6d69df2804
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      models/migrations/migrations.go
  2. 47
      models/migrations/v184.go
  3. 8
      models/task.go
  4. 11
      modules/migrations/base/messenger.go
  5. 4
      modules/migrations/dump.go
  6. 2
      modules/migrations/gitea_uploader_test.go
  7. 17
      modules/migrations/migrate.go
  8. 14
      modules/task/migrate.go
  9. 8
      options/locale/locale_en-US.ini
  10. 2
      routers/api/v1/repo/migrate.go
  11. 18
      routers/web/user/task.go
  12. 1
      templates/repo/migrate/migrating.tmpl
  13. 7
      web_src/js/index.js

@ -317,6 +317,8 @@ var migrations = []Migration{
NewMigration("Add issue resource index table", addIssueResourceIndexTable), NewMigration("Add issue resource index table", addIssueResourceIndexTable),
// v183 -> v184 // v183 -> v184
NewMigration("Create PushMirror table", createPushMirrorTable), NewMigration("Create PushMirror table", createPushMirrorTable),
// v184 -> v185
NewMigration("Rename Task errors to message", renameTaskErrorsToMessage),
} }
// GetCurrentDBVersion returns the current db version // GetCurrentDBVersion returns the current db version

@ -0,0 +1,47 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"code.gitea.io/gitea/modules/setting"
"xorm.io/xorm"
)
func renameTaskErrorsToMessage(x *xorm.Engine) error {
type Task struct {
Errors string `xorm:"TEXT"` // if task failed, saved the error reason
Type int
Status int `xorm:"index"`
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if err := sess.Sync2(new(Task)); err != nil {
return fmt.Errorf("error on Sync2: %v", err)
}
switch {
case setting.Database.UseMySQL:
if _, err := sess.Exec("ALTER TABLE `task` CHANGE errors message text"); err != nil {
return err
}
case setting.Database.UseMSSQL:
if _, err := sess.Exec("sp_rename 'task.errors', 'message', 'COLUMN'"); err != nil {
return err
}
default:
if _, err := sess.Exec("ALTER TABLE `task` RENAME COLUMN errors TO message"); err != nil {
return err
}
}
return sess.Commit()
}

@ -32,10 +32,16 @@ type Task struct {
StartTime timeutil.TimeStamp StartTime timeutil.TimeStamp
EndTime timeutil.TimeStamp EndTime timeutil.TimeStamp
PayloadContent string `xorm:"TEXT"` PayloadContent string `xorm:"TEXT"`
Errors string `xorm:"TEXT"` // if task failed, saved the error reason Message string `xorm:"TEXT"` // if task failed, saved the error reason
Created timeutil.TimeStamp `xorm:"created"` Created timeutil.TimeStamp `xorm:"created"`
} }
// TranslatableMessage represents JSON struct that can be translated with a Locale
type TranslatableMessage struct {
Format string
Args []interface{} `json:"omitempty"`
}
// LoadRepo loads repository of the task // LoadRepo loads repository of the task
func (task *Task) LoadRepo() error { func (task *Task) LoadRepo() error {
return task.loadRepo(x) return task.loadRepo(x)

@ -0,0 +1,11 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package base
// Messenger is a formatting function similar to i18n.Tr
type Messenger func(key string, args ...interface{})
// NilMessenger represents an empty formatting function
func NilMessenger(string, ...interface{}) {}

@ -555,7 +555,7 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi
return err return err
} }
if err := migrateRepository(downloader, uploader, opts); err != nil { if err := migrateRepository(downloader, uploader, opts, nil); err != nil {
if err1 := uploader.Rollback(); err1 != nil { if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1) log.Error("rollback failed: %v", err1)
} }
@ -620,7 +620,7 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName
} }
updateOptionsUnits(&migrateOpts, units) updateOptionsUnits(&migrateOpts, units)
if err = migrateRepository(downloader, uploader, migrateOpts); err != nil { if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil {
if err1 := uploader.Rollback(); err1 != nil { if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1) log.Error("rollback failed: %v", err1)
} }

@ -47,7 +47,7 @@ func TestGiteaUploadRepo(t *testing.T) {
PullRequests: true, PullRequests: true,
Private: true, Private: true,
Mirror: false, Mirror: false,
}) }, nil)
assert.NoError(t, err) assert.NoError(t, err)
repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository) repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)

@ -99,7 +99,7 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
} }
// MigrateRepository migrate repository according MigrateOptions // MigrateRepository migrate repository according MigrateOptions
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*models.Repository, error) {
err := IsMigrateURLAllowed(opts.CloneAddr, doer) err := IsMigrateURLAllowed(opts.CloneAddr, doer)
if err != nil { if err != nil {
return nil, err return nil, err
@ -118,7 +118,7 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
uploader.gitServiceType = opts.GitServiceType uploader.gitServiceType = opts.GitServiceType
if err := migrateRepository(downloader, uploader, opts); err != nil { if err := migrateRepository(downloader, uploader, opts, messenger); err != nil {
if err1 := uploader.Rollback(); err1 != nil { if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1) log.Error("rollback failed: %v", err1)
} }
@ -167,7 +167,11 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio
// migrateRepository will download information and then upload it to Uploader, this is a simple // migrateRepository will download information and then upload it to Uploader, this is a simple
// process for small repository. For a big repository, save all the data to disk // process for small repository. For a big repository, save all the data to disk
// before upload is better // before upload is better
func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error { func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
if messenger == nil {
messenger = base.NilMessenger
}
repo, err := downloader.GetRepoInfo() repo, err := downloader.GetRepoInfo()
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
@ -185,12 +189,14 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
} }
log.Trace("migrating git data from %s", repo.CloneURL) log.Trace("migrating git data from %s", repo.CloneURL)
messenger("repo.migrate.migrating_git")
if err = uploader.CreateRepo(repo, opts); err != nil { if err = uploader.CreateRepo(repo, opts); err != nil {
return err return err
} }
defer uploader.Close() defer uploader.Close()
log.Trace("migrating topics") log.Trace("migrating topics")
messenger("repo.migrate.migrating_topics")
topics, err := downloader.GetTopics() topics, err := downloader.GetTopics()
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
@ -206,6 +212,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.Milestones { if opts.Milestones {
log.Trace("migrating milestones") log.Trace("migrating milestones")
messenger("repo.migrate.migrating_milestones")
milestones, err := downloader.GetMilestones() milestones, err := downloader.GetMilestones()
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
@ -229,6 +236,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.Labels { if opts.Labels {
log.Trace("migrating labels") log.Trace("migrating labels")
messenger("repo.migrate.migrating_labels")
labels, err := downloader.GetLabels() labels, err := downloader.GetLabels()
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
@ -252,6 +260,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.Releases { if opts.Releases {
log.Trace("migrating releases") log.Trace("migrating releases")
messenger("repo.migrate.migrating_releases")
releases, err := downloader.GetReleases() releases, err := downloader.GetReleases()
if err != nil { if err != nil {
if !base.IsErrNotSupported(err) { if !base.IsErrNotSupported(err) {
@ -285,6 +294,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.Issues { if opts.Issues {
log.Trace("migrating issues and comments") log.Trace("migrating issues and comments")
messenger("repo.migrate.migrating_issues")
var issueBatchSize = uploader.MaxBatchInsertSize("issue") var issueBatchSize = uploader.MaxBatchInsertSize("issue")
for i := 1; ; i++ { for i := 1; ; i++ {
@ -339,6 +349,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.PullRequests { if opts.PullRequests {
log.Trace("migrating pull requests and comments") log.Trace("migrating pull requests and comments")
messenger("repo.migrate.migrating_pulls")
var prBatchSize = uploader.MaxBatchInsertSize("pullrequest") var prBatchSize = uploader.MaxBatchInsertSize("pullrequest")
for i := 1; ; i++ { for i := 1; ; i++ {
prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize) prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)

@ -20,6 +20,7 @@ import (
"code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
jsoniter "github.com/json-iterator/go"
) )
func handleCreateError(owner *models.User, err error) error { func handleCreateError(owner *models.User, err error) error {
@ -56,7 +57,7 @@ func runMigrateTask(t *models.Task) (err error) {
t.EndTime = timeutil.TimeStampNow() t.EndTime = timeutil.TimeStampNow()
t.Status = structs.TaskStatusFailed t.Status = structs.TaskStatusFailed
t.Errors = err.Error() t.Message = err.Error()
t.RepoID = 0 t.RepoID = 0
if err := t.UpdateCols("status", "errors", "repo_id", "end_time"); err != nil { if err := t.UpdateCols("status", "errors", "repo_id", "end_time"); err != nil {
log.Error("Task UpdateCols failed: %v", err) log.Error("Task UpdateCols failed: %v", err)
@ -106,7 +107,16 @@ func runMigrateTask(t *models.Task) (err error) {
return return
} }
repo, err = migrations.MigrateRepository(ctx, t.Doer, t.Owner.Name, *opts) repo, err = migrations.MigrateRepository(ctx, t.Doer, t.Owner.Name, *opts, func(format string, args ...interface{}) {
message := models.TranslatableMessage{
Format: format,
Args: args,
}
json := jsoniter.ConfigCompatibleWithStandardLibrary
bs, _ := json.Marshal(message)
t.Message = string(bs)
_ = t.UpdateCols("message")
})
if err == nil { if err == nil {
log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name)
return return

@ -824,11 +824,19 @@ migrated_from_fake = Migrated From %[1]s
migrate.migrate = Migrate From %s migrate.migrate = Migrate From %s
migrate.migrating = Migrating from <b>%s</b> ... migrate.migrating = Migrating from <b>%s</b> ...
migrate.migrating_failed = Migrating from <b>%s</b> failed. migrate.migrating_failed = Migrating from <b>%s</b> failed.
migrate.migrating_failed.error = Error: %s
migrate.github.description = Migrating data from Github.com or Github Enterprise. migrate.github.description = Migrating data from Github.com or Github Enterprise.
migrate.git.description = Migrating or Mirroring git data from Git services migrate.git.description = Migrating or Mirroring git data from Git services
migrate.gitlab.description = Migrating data from GitLab.com or Self-Hosted gitlab server. migrate.gitlab.description = Migrating data from GitLab.com or Self-Hosted gitlab server.
migrate.gitea.description = Migrating data from Gitea.com or Self-Hosted Gitea server. migrate.gitea.description = Migrating data from Gitea.com or Self-Hosted Gitea server.
migrate.gogs.description = Migrating data from notabug.org or other Self-Hosted Gogs server. migrate.gogs.description = Migrating data from notabug.org or other Self-Hosted Gogs server.
migrate.migrating_git = Migrating Git Data
migrate.migrating_topics = Migrating Topics
migrate.migrating_milestones = Migrating Milestones
migrate.migrating_labels = Migrating Labels
migrate.migrating_releases = Migrating Releases
migrate.migrating_issues = Migrating Issues
migrate.migrating_pulls = Migrating Pull Requests
mirror_from = mirror of mirror_from = mirror of
forked_from = forked from forked_from = forked from

@ -199,7 +199,7 @@ func Migrate(ctx *context.APIContext) {
} }
}() }()
if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, repoOwner.Name, opts); err != nil { if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, repoOwner.Name, opts, nil); err != nil {
handleMigrateError(ctx, repoOwner, remoteAddr, err) handleMigrateError(ctx, repoOwner, remoteAddr, err)
return return
} }

@ -9,6 +9,7 @@ import (
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
jsoniter "github.com/json-iterator/go"
) )
// TaskStatus returns task's status // TaskStatus returns task's status
@ -21,9 +22,24 @@ func TaskStatus(ctx *context.Context) {
return return
} }
message := task.Message
if task.Message != "" && task.Message[0] == '{' {
// assume message is actually a translatable string
json := jsoniter.ConfigCompatibleWithStandardLibrary
var translatableMessage models.TranslatableMessage
if err := json.Unmarshal([]byte(message), &translatableMessage); err != nil {
translatableMessage = models.TranslatableMessage{
Format: "migrate.migrating_failed.error",
Args: []interface{}{task.Message},
}
}
message = ctx.Tr(translatableMessage.Format, translatableMessage.Args...)
}
ctx.JSON(http.StatusOK, map[string]interface{}{ ctx.JSON(http.StatusOK, map[string]interface{}{
"status": task.Status, "status": task.Status,
"err": task.Errors, "message": message,
"repo-id": task.RepoID, "repo-id": task.RepoID,
"repo-name": opts.RepoName, "repo-name": opts.RepoName,
"start": task.StartTime, "start": task.StartTime,

@ -22,6 +22,7 @@
<div class="sixteen wide center aligned centered column"> <div class="sixteen wide center aligned centered column">
<div id="repo_migrating_progress"> <div id="repo_migrating_progress">
<p>{{.i18n.Tr "repo.migrate.migrating" .CloneAddr | Safe}}</p> <p>{{.i18n.Tr "repo.migrate.migrating" .CloneAddr | Safe}}</p>
<p id="repo_migrating_progress_message"></p>
</div> </div>
<div id="repo_migrating_failed" hidden> <div id="repo_migrating_failed" hidden>
<p>{{.i18n.Tr "repo.migrate.migrating_failed" .CloneAddr | Safe}}</p> <p>{{.i18n.Tr "repo.migrate.migrating_failed" .CloneAddr | Safe}}</p>

@ -202,6 +202,7 @@ function initRepoStatusChecker() {
const migrating = $('#repo_migrating'); const migrating = $('#repo_migrating');
$('#repo_migrating_failed').hide(); $('#repo_migrating_failed').hide();
$('#repo_migrating_failed_image').hide(); $('#repo_migrating_failed_image').hide();
$('#repo_migrating_progress_message').hide();
if (migrating) { if (migrating) {
const task = migrating.attr('task'); const task = migrating.attr('task');
if (typeof task === 'undefined') { if (typeof task === 'undefined') {
@ -223,9 +224,13 @@ function initRepoStatusChecker() {
$('#repo_migrating').hide(); $('#repo_migrating').hide();
$('#repo_migrating_failed').show(); $('#repo_migrating_failed').show();
$('#repo_migrating_failed_image').show(); $('#repo_migrating_failed_image').show();
$('#repo_migrating_failed_error').text(xhr.responseJSON.err); $('#repo_migrating_failed_error').text(xhr.responseJSON.message);
return; return;
} }
if (xhr.responseJSON.message) {
$('#repo_migrating_progress_message').show();
$('#repo_migrating_progress_message').text(xhr.responseJSON.message);
}
setTimeout(() => { setTimeout(() => {
initRepoStatusChecker(); initRepoStatusChecker();
}, 2000); }, 2000);

Loading…
Cancel
Save