Improve mirror iterator (#18928)

* Improve mirror iterator

* fix test
tokarchuk/v1.17
Lunny Xiao 3 years ago committed by GitHub
parent 59959ab222
commit b75ad7b87f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      models/repo/mirror.go
  2. 3
      models/repo/pushmirror.go
  3. 2
      models/repo/pushmirror_test.go
  4. 30
      services/mirror/mirror.go

@ -120,11 +120,12 @@ func DeleteMirrorByRepoID(repoID int64) error {
}
// MirrorsIterate iterates all mirror repositories.
func MirrorsIterate(f func(idx int, bean interface{}) error) error {
func MirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext).
Where("next_update_unix<=?", time.Now().Unix()).
And("next_update_unix!=0").
OrderBy("updated_unix ASC").
Limit(limit).
Iterate(new(Mirror), f)
}

@ -101,10 +101,11 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
}
// PushMirrorsIterate iterates all push-mirror repositories.
func PushMirrorsIterate(f func(idx int, bean interface{}) error) error {
func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext).
Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
And("`interval` != 0").
OrderBy("last_update ASC").
Limit(limit).
Iterate(new(PushMirror), f)
}

@ -40,7 +40,7 @@ func TestPushMirrorsIterate(t *testing.T) {
time.Sleep(1 * time.Millisecond)
PushMirrorsIterate(func(idx int, bean interface{}) error {
PushMirrorsIterate(1, func(idx int, bean interface{}) error {
m, ok := bean.(*PushMirror)
assert.True(t, ok)
assert.Equal(t, "test-1", m.RemoteName)

@ -55,9 +55,7 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error {
}
log.Trace("Doing: Update")
requested := 0
handler := func(idx int, bean interface{}, limit int) error {
handler := func(idx int, bean interface{}) error {
var item SyncRequest
var repo *repo_model.Repository
if m, ok := bean.(*repo_model.Mirror); ok {
@ -104,35 +102,35 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error {
}
return err
}
requested++
if limit > 0 && requested > limit {
return errLimit
}
return nil
}
pullMirrorsRequested := 0
if pullLimit != 0 {
requested = 0
if err := repo_model.MirrorsIterate(func(idx int, bean interface{}) error {
return handler(idx, bean, pullLimit)
if err := repo_model.MirrorsIterate(pullLimit, func(idx int, bean interface{}) error {
if err := handler(idx, bean); err != nil {
return err
}
pullMirrorsRequested++
return nil
}); err != nil && err != errLimit {
log.Error("MirrorsIterate: %v", err)
return err
}
pullMirrorsRequested, requested = requested, 0
}
pushMirrorsRequested := 0
if pushLimit != 0 {
requested = 0
if err := repo_model.PushMirrorsIterate(func(idx int, bean interface{}) error {
return handler(idx, bean, pushLimit)
if err := repo_model.PushMirrorsIterate(pushLimit, func(idx int, bean interface{}) error {
if err := handler(idx, bean); err != nil {
return err
}
pushMirrorsRequested++
return nil
}); err != nil && err != errLimit {
log.Error("PushMirrorsIterate: %v", err)
return err
}
pushMirrorsRequested, requested = requested, 0
}
log.Trace("Finished: Update: %d pull mirrors and %d push mirrors queued", pullMirrorsRequested, pushMirrorsRequested)
return nil

Loading…
Cancel
Save