third_party
/
gitea
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Sleep longer if request speed is over github limitation (#9335)

Browse Source 
* Sleep longer if request speed is over github limitation

* improve code

* remove unused code

* fix lint

* Use github's rate limit remain value to determine how long to sleep

* Save reset time when finished github api request

* fix bug

* fix lint

* Add context.Context for sleep

* fix test

* improve code

* fix bug and lint

* fix import order
tokarchuk/v1.17
Lunny Xiao 5 years ago committed by techknowlogick
parent d1a49977b0
commit ffc904b1e0
8 changed files with 86 additions and 16 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 11
      modules/migrations/base/downloader.go
  2. 6
      modules/migrations/git.go
  3. 5
      modules/migrations/gitea.go
  4. 3
      modules/migrations/gitea_test.go
  5. 64
      modules/migrations/github.go
  6. 7
      modules/migrations/migrate.go
  7. 3
      modules/task/migrate.go
  8. 3
      routers/api/v1/repo/repo.go

11
modules/migrations/base/downloader.go
Unescape View File

@ -6,6 +6,7 @@
package base
import (
"context"
"time"
"code.gitea.io/gitea/modules/structs"
@ -13,6 +14,7 @@ import (
// Downloader downloads the site repo informations
type Downloader interface {
SetContext(context.Context)
GetRepoInfo() (*Repository, error)
GetTopics() ([]string, error)
GetMilestones() ([]*Milestone, error)
@ -30,6 +32,10 @@ type DownloaderFactory interface {
GitServiceType() structs.GitServiceType
}
var (
_ Downloader = &RetryDownloader{}
)
// RetryDownloader retry the downloads
type RetryDownloader struct {
Downloader
@ -46,6 +52,11 @@ func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *Retr
}
}
// SetContext set context
func (d *RetryDownloader) SetContext(ctx context.Context) {
d.Downloader.SetContext(ctx)
}
// GetRepoInfo returns a repository information with retry
func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
var (

6
modules/migrations/git.go
Unescape View File

@ -5,6 +5,8 @@
package migrations
import (
"context"
"code.gitea.io/gitea/modules/migrations/base"
)
@ -28,6 +30,10 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl
}
}
// SetContext set context
func (g *PlainGitDownloader) SetContext(ctx context.Context) {
}
// GetRepoInfo returns a repository information
func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
// convert github repo to stand Repo

5
modules/migrations/gitea.go
Unescape View File

@ -6,6 +6,7 @@
package migrations
import (
"context"
"fmt"
"io"
"net/http"
@ -35,6 +36,7 @@ var (
// GiteaLocalUploader implements an Uploader to gitea sites
type GiteaLocalUploader struct {
ctx context.Context
doer *models.User
repoOwner string
repoName string
@ -49,8 +51,9 @@ type GiteaLocalUploader struct {
}
// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
return &GiteaLocalUploader{
ctx: ctx,
doer: doer,
repoOwner: repoOwner,
repoName: repoName,

3
modules/migrations/gitea_test.go
Unescape View File

@ -10,6 +10,7 @@ import (
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
@ -27,7 +28,7 @@ func TestGiteaUploadRepo(t *testing.T) {
var (
downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder")
repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05")
uploader = NewGiteaLocalUploader(user, user.Name, repoName)
uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
)
err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{

64
modules/migrations/github.go
Unescape View File

@ -11,6 +11,7 @@ import (
"net/http"
"net/url"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
@ -73,6 +74,7 @@ type GithubDownloaderV3 struct {
repoName string
userName string
password string
rate *github.Rate
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
@ -107,12 +109,39 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith
return &downloader
}
// SetContext set context
func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
g.ctx = ctx
}
func (g *GithubDownloaderV3) sleep() {
for g.rate != nil && g.rate.Remaining <= 0 {
timer := time.NewTimer(time.Until(g.rate.Reset.Time))
select {
case <-g.ctx.Done():
timer.Stop()
return
case <-timer.C:
}
rates, _, err := g.client.RateLimits(g.ctx)
if err != nil {
log.Error("g.client.RateLimits: %s", err)
}
g.rate = rates.GetCore()
}
}
// GetRepoInfo returns a repository information
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
g.sleep()
gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
g.rate = &resp.Rate
// convert github repo to stand Repo
return &base.Repository{
Owner: g.repoOwner,
@ -126,8 +155,13 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
// GetTopics return github topics
func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
r, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
return r.Topics, err
g.sleep()
r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
g.rate = &resp.Rate
return r.Topics, nil
}
// GetMilestones returns milestones
@ -135,7 +169,8 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
g.sleep()
ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
&github.MilestoneListOptions{
State: "all",
ListOptions: github.ListOptions{
@ -145,6 +180,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
if err != nil {
return nil, err
}
g.rate = &resp.Rate
for _, m := range ms {
var desc string
@ -189,7 +225,8 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
var perPage = 100
var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
g.sleep()
ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
@ -197,6 +234,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
if err != nil {
return nil, err
}
g.rate = &resp.Rate
for _, label := range ls {
labels = append(labels, convertGithubLabel(label))
@ -260,7 +298,8 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
var perPage = 100
var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
g.sleep()
ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
@ -268,6 +307,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
if err != nil {
return nil, err
}
g.rate = &resp.Rate
for _, release := range ls {
releases = append(releases, g.convertGithubRelease(release))
@ -304,11 +344,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
}
var allIssues = make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
g.sleep()
issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
g.rate = &resp.Rate
for _, issue := range issues {
if issue.IsPullRequest() {
continue
@ -365,10 +406,12 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
},
}
for {
g.sleep()
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
g.rate = &resp.Rate
for _, comment := range comments {
var email string
if comment.User.Email != nil {
@ -408,11 +451,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
},
}
var allPRs = make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
g.sleep()
prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
g.rate = &resp.Rate
for _, pr := range prs {
var body string
if pr.Body != nil {

7
modules/migrations/migrate.go
Unescape View File

@ -6,6 +6,7 @@
package migrations
import (
"context"
"fmt"
"code.gitea.io/gitea/models"
@ -28,10 +29,10 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) {
}
// MigrateRepository migrate repository according MigrateOptions
func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
var (
downloader base.Downloader
uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName)
uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
theFactory base.DownloaderFactory
)
@ -69,6 +70,8 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
}
downloader.SetContext(ctx)
if err := migrateRepository(downloader, uploader, opts); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)

3
modules/task/migrate.go
Unescape View File

@ -11,6 +11,7 @@ import (
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/notification"
@ -95,7 +96,7 @@ func runMigrateTask(t *models.Task) (err error) {
}
opts.MigrateToRepoID = t.RepoID
repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts)
repo, err := migrations.MigrateRepository(graceful.GetManager().HammerContext(), t.Doer, t.Owner.Name, *opts)
if err == nil {
log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name)
return nil

3
routers/api/v1/repo/repo.go
Unescape View File

@ -18,6 +18,7 @@ import (
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/convert"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/notification"
@ -481,7 +482,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
}
}()
if _, err = migrations.MigrateRepository(ctx.User, ctxUser.Name, opts); err != nil {
if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, ctxUser.Name, opts); err != nil {
handleMigrateError(ctx, ctxUser, remoteAddr, err)
return
}

Write Preview
Loading…
Cancel
Save