From e3e44a59d01da3af2be3a830f4a90394e7af4ff4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 14 Oct 2019 14:10:42 +0800 Subject: [PATCH] Update migrated repositories' issues/comments/prs poster id if user has a github external user saved (#7751) * update migrated issues/comments when login as github * add get userid when migrating or login with github oauth2 * fix lint * add migrations for repository service type * fix build * remove unnecessary dependencies on migrations * add cron task to update migrations poster ids and fix posterid when migrating * fix lint * fix lint * improve code * fix lint * improve code * replace releases publish id to actual author id * fix import * fix bug * fix lint * fix rawdata definition * fix some bugs * fix error message --- custom/conf/app.ini.sample | 5 + .../doc/advanced/config-cheat-sheet.en-us.md | 4 + .../doc/advanced/config-cheat-sheet.zh-cn.md | 6 +- models/external_login_user.go | 143 +++++++++-- models/issue.go | 16 +- models/issue_comment.go | 21 ++ models/migrations/migrations.go | 2 + models/migrations/v100.go | 83 ++++++ models/release.go | 14 + models/repo.go | 22 +- modules/cron/cron.go | 23 +- modules/migrations/base/downloader.go | 3 + modules/migrations/gitea.go | 239 +++++++++++++----- modules/migrations/github.go | 8 +- modules/migrations/migrate.go | 7 + modules/migrations/update.go | 59 +++++ modules/setting/cron.go | 8 + modules/structs/repo.go | 39 +++ routers/api/v1/repo/repo.go | 37 ++- routers/user/auth.go | 88 ++++--- services/externalaccount/user.go | 66 +++++ 21 files changed, 737 insertions(+), 156 deletions(-) create mode 100644 models/migrations/v100.go create mode 100644 modules/migrations/update.go create mode 100644 services/externalaccount/user.go diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index dd14089d2..fd8d928ed 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -690,6 +690,11 @@ SCHEDULE = @every 24h ; or only create new users if UPDATE_EXISTING is set to false UPDATE_EXISTING = true +; Update migrated repositories' issues and comments' posterid, it will always attempt synchronization when the instance starts. +[cron.update_migration_post_id] +; Interval as a duration between each synchronization. (default every 24h) +SCHEDULE = @every 24h + [git] ; The path of git executable. If empty, Gitea searches through the PATH environment. PATH = diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index ed34be032..b927793a5 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -419,6 +419,10 @@ NB: You must `REDIRECT_MACARON_LOG` and have `DISABLE_ROUTER_LOG` set to `false` - `RUN_AT_START`: **true**: Run repository statistics check at start time. - `SCHEDULE`: **@every 24h**: Cron syntax for scheduling repository statistics check. +### Cron - Update Migration Poster ID (`cron.update_migration_post_id`) + +- `SCHEDULE`: **@every 24h** : Interval as a duration between each synchronization, it will always attempt synchronization when the instance starts. + ## Git (`git`) - `PATH`: **""**: The path of git executable. If empty, Gitea searches through the PATH environment. diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 01ba821a4..ab73e2059 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -196,7 +196,11 @@ menu: ### Cron - Repository Statistics Check (`cron.check_repo_stats`) - `RUN_AT_START`: 是否启动时自动运行仓库统计。 -- `SCHEDULE`: 藏亏统计时的Cron 语法，比如：`@every 24h`. +- `SCHEDULE`: 仓库统计时的Cron 语法，比如：`@every 24h`. + +### Cron - Update Migration Poster ID (`cron.update_migration_post_id`) + +- `SCHEDULE`: **@every 24h** : 每次同步的间隔时间。此任务总是在启动时自动进行。 ## Git (`git`) diff --git a/models/external_login_user.go b/models/external_login_user.go index 21a3cbbd3..5058fd1b4 100644 --- a/models/external_login_user.go +++ b/models/external_login_user.go @@ -4,13 +4,34 @@ package models -import "github.com/markbates/goth" +import ( + "time" + + "code.gitea.io/gitea/modules/structs" + + "github.com/markbates/goth" + "xorm.io/builder" +) // ExternalLoginUser makes the connecting between some existing user and additional external login sources type ExternalLoginUser struct { - ExternalID string `xorm:"pk NOT NULL"` - UserID int64 `xorm:"INDEX NOT NULL"` - LoginSourceID int64 `xorm:"pk NOT NULL"` + ExternalID string `xorm:"pk NOT NULL"` + UserID int64 `xorm:"INDEX NOT NULL"` + LoginSourceID int64 `xorm:"pk NOT NULL"` + RawData map[string]interface{} `xorm:"TEXT JSON"` + Provider string `xorm:"index VARCHAR(25)"` + Email string + Name string + FirstName string + LastName string + NickName string + Description string + AvatarURL string + Location string + AccessToken string + AccessTokenSecret string + RefreshToken string + ExpiresAt time.Time } // GetExternalLogin checks if a externalID in loginSourceID scope already exists @@ -32,23 +53,15 @@ func ListAccountLinks(user *User) ([]*ExternalLoginUser, error) { return externalAccounts, nil } -// LinkAccountToUser link the gothUser to the user -func LinkAccountToUser(user *User, gothUser goth.User) error { - loginSource, err := GetActiveOAuth2LoginSourceByName(gothUser.Provider) - if err != nil { - return err - } - - externalLoginUser := &ExternalLoginUser{ - ExternalID: gothUser.UserID, - UserID: user.ID, - LoginSourceID: loginSource.ID, - } - has, err := x.Get(externalLoginUser) +// LinkExternalToUser link the external user to the user +func LinkExternalToUser(user *User, externalLoginUser *ExternalLoginUser) error { + has, err := x.Where("external_id=? AND login_source_id=?", externalLoginUser.ExternalID, externalLoginUser.LoginSourceID). + NoAutoCondition(). + Exist(externalLoginUser) if err != nil { return err } else if has { - return ErrExternalLoginUserAlreadyExist{gothUser.UserID, user.ID, loginSource.ID} + return ErrExternalLoginUserAlreadyExist{externalLoginUser.ExternalID, user.ID, externalLoginUser.LoginSourceID} } _, err = x.Insert(externalLoginUser) @@ -72,3 +85,97 @@ func removeAllAccountLinks(e Engine, user *User) error { _, err := e.Delete(&ExternalLoginUser{UserID: user.ID}) return err } + +// GetUserIDByExternalUserID get user id according to provider and userID +func GetUserIDByExternalUserID(provider string, userID string) (int64, error) { + var id int64 + _, err := x.Table("external_login_user"). + Select("user_id"). + Where("provider=?", provider). + And("external_id=?", userID). + Get(&id) + if err != nil { + return 0, err + } + return id, nil +} + +// UpdateExternalUser updates external user's information +func UpdateExternalUser(user *User, gothUser goth.User) error { + loginSource, err := GetActiveOAuth2LoginSourceByName(gothUser.Provider) + if err != nil { + return err + } + externalLoginUser := &ExternalLoginUser{ + ExternalID: gothUser.UserID, + UserID: user.ID, + LoginSourceID: loginSource.ID, + RawData: gothUser.RawData, + Provider: gothUser.Provider, + Email: gothUser.Email, + Name: gothUser.Name, + FirstName: gothUser.FirstName, + LastName: gothUser.LastName, + NickName: gothUser.NickName, + Description: gothUser.Description, + AvatarURL: gothUser.AvatarURL, + Location: gothUser.Location, + AccessToken: gothUser.AccessToken, + AccessTokenSecret: gothUser.AccessTokenSecret, + RefreshToken: gothUser.RefreshToken, + ExpiresAt: gothUser.ExpiresAt, + } + + has, err := x.Where("external_id=? AND login_source_id=?", gothUser.UserID, loginSource.ID). + NoAutoCondition(). + Exist(externalLoginUser) + if err != nil { + return err + } else if !has { + return ErrExternalLoginUserNotExist{user.ID, loginSource.ID} + } + + _, err = x.Where("external_id=? AND login_source_id=?", gothUser.UserID, loginSource.ID).AllCols().Update(externalLoginUser) + return err +} + +// FindExternalUserOptions represents an options to find external users +type FindExternalUserOptions struct { + Provider string + Limit int + Start int +} + +func (opts FindExternalUserOptions) toConds() builder.Cond { + var cond = builder.NewCond() + if len(opts.Provider) > 0 { + cond = cond.And(builder.Eq{"provider": opts.Provider}) + } + return cond +} + +// FindExternalUsersByProvider represents external users via provider +func FindExternalUsersByProvider(opts FindExternalUserOptions) ([]ExternalLoginUser, error) { + var users []ExternalLoginUser + err := x.Where(opts.toConds()). + Limit(opts.Limit, opts.Start). + Asc("id"). + Find(&users) + if err != nil { + return nil, err + } + return users, nil +} + +// UpdateMigrationsByType updates all migrated repositories' posterid from gitServiceType to replace originalAuthorID to posterID +func UpdateMigrationsByType(tp structs.GitServiceType, externalUserID, userID int64) error { + if err := UpdateIssuesMigrationsByType(tp, externalUserID, userID); err != nil { + return err + } + + if err := UpdateCommentsMigrationsByType(tp, externalUserID, userID); err != nil { + return err + } + + return UpdateReleasesMigrationsByType(tp, externalUserID, userID) +} diff --git a/models/issue.go b/models/issue.go index 8ce7d496a..fc675a3ff 100644 --- a/models/issue.go +++ b/models/issue.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" @@ -32,7 +33,7 @@ type Issue struct { PosterID int64 `xorm:"INDEX"` Poster *User `xorm:"-"` OriginalAuthor string - OriginalAuthorID int64 + OriginalAuthorID int64 `xorm:"index"` Title string `xorm:"name"` Content string `xorm:"TEXT"` RenderedContent string `xorm:"-"` @@ -1947,3 +1948,16 @@ func (issue *Issue) ResolveMentionsByVisibility(ctx DBContext, doer *User, menti return } + +// UpdateIssuesMigrationsByType updates all migrated repositories' issues from gitServiceType to replace originalAuthorID to posterID +func UpdateIssuesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID, posterID int64) error { + _, err := x.Table("issue"). + Where("repo_id IN (SELECT id FROM repository WHERE original_service_type = ?)", gitServiceType). + And("original_author_id = ?", originalAuthorID). + Update(map[string]interface{}{ + "poster_id": posterID, + "original_author": "", + "original_author_id": 0, + }) + return err +} diff --git a/models/issue_comment.go b/models/issue_comment.go index 7d38302b9..3a090c3b1 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/references" + "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" @@ -1022,3 +1023,23 @@ func fetchCodeCommentsByReview(e Engine, issue *Issue, currentUser *User, review func FetchCodeComments(issue *Issue, currentUser *User) (CodeComments, error) { return fetchCodeComments(x, issue, currentUser) } + +// UpdateCommentsMigrationsByType updates comments' migrations information via given git service type and original id and poster id +func UpdateCommentsMigrationsByType(tp structs.GitServiceType, originalAuthorID, posterID int64) error { + _, err := x.Table("comment"). + Where(builder.In("issue_id", + builder.Select("issue.id"). + From("issue"). + InnerJoin("repository", "issue.repo_id = repository.id"). + Where(builder.Eq{ + "repository.original_service_type": tp, + }), + )). + And("comment.original_author_id = ?", originalAuthorID). + Update(map[string]interface{}{ + "poster_id": posterID, + "original_author": "", + "original_author_id": 0, + }) + return err +} diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index ef5cd377a..60a416c6e 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -254,6 +254,8 @@ var migrations = []Migration{ NewMigration("add original author name and id on migrated release", addOriginalAuthorOnMigratedReleases), // v99 -> v100 NewMigration("add task table and status column for repository table", addTaskTable), + // v100 -> v101 + NewMigration("update migration repositories' service type", updateMigrationServiceTypes), } // Migrate database to current version diff --git a/models/migrations/v100.go b/models/migrations/v100.go new file mode 100644 index 000000000..ac3b73e2a --- /dev/null +++ b/models/migrations/v100.go @@ -0,0 +1,83 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "net/url" + "strings" + "time" + + "github.com/go-xorm/xorm" +) + +func updateMigrationServiceTypes(x *xorm.Engine) error { + type Repository struct { + ID int64 + OriginalServiceType int `xorm:"index default(0)"` + OriginalURL string `xorm:"VARCHAR(2048)"` + } + + if err := x.Sync2(new(Repository)); err != nil { + return err + } + + var last int + const batchSize = 50 + for { + var results = make([]Repository, 0, batchSize) + err := x.Where("original_url <> '' AND original_url IS NOT NULL"). + And("original_service_type = 0 OR original_service_type IS NULL"). + OrderBy("id"). + Limit(batchSize, last). + Find(&results) + if err != nil { + return err + } + if len(results) == 0 { + break + } + last += len(results) + + const PlainGitService = 1 // 1 plain git service + const GithubService = 2 // 2 github.com + + for _, res := range results { + u, err := url.Parse(res.OriginalURL) + if err != nil { + return err + } + var serviceType = PlainGitService + if strings.EqualFold(u.Host, "github.com") { + serviceType = GithubService + } + _, err = x.Exec("UPDATE repository SET original_service_type = ? WHERE id = ?", serviceType, res.ID) + if err != nil { + return err + } + } + } + + type ExternalLoginUser struct { + ExternalID string `xorm:"pk NOT NULL"` + UserID int64 `xorm:"INDEX NOT NULL"` + LoginSourceID int64 `xorm:"pk NOT NULL"` + RawData map[string]interface{} `xorm:"TEXT JSON"` + Provider string `xorm:"index VARCHAR(25)"` + Email string + Name string + FirstName string + LastName string + NickName string + Description string + AvatarURL string + Location string + AccessToken string + AccessTokenSecret string + RefreshToken string + ExpiresAt time.Time + } + + return x.Sync2(new(ExternalLoginUser)) +} diff --git a/models/release.go b/models/release.go index 243cc2fa3..03685e0a4 100644 --- a/models/release.go +++ b/models/release.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" @@ -366,3 +367,16 @@ func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error { } return nil } + +// UpdateReleasesMigrationsByType updates all migrated repositories' releases from gitServiceType to replace originalAuthorID to posterID +func UpdateReleasesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID, posterID int64) error { + _, err := x.Table("release"). + Where("repo_id IN (SELECT id FROM repository WHERE original_service_type = ?)", gitServiceType). + And("original_author_id = ?", originalAuthorID). + Update(map[string]interface{}{ + "publisher_id": posterID, + "original_author": "", + "original_author_id": 0, + }) + return err +} diff --git a/models/repo.go b/models/repo.go index 23b1c2ef5..aa2cf06f3 100644 --- a/models/repo.go +++ b/models/repo.go @@ -32,6 +32,7 @@ import ( "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/timeutil" @@ -137,16 +138,17 @@ const ( // Repository represents a git repository. type Repository struct { - ID int64 `xorm:"pk autoincr"` - OwnerID int64 `xorm:"UNIQUE(s) index"` - OwnerName string `xorm:"-"` - Owner *User `xorm:"-"` - LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Name string `xorm:"INDEX NOT NULL"` - Description string `xorm:"TEXT"` - Website string `xorm:"VARCHAR(2048)"` - OriginalURL string `xorm:"VARCHAR(2048)"` - DefaultBranch string + ID int64 `xorm:"pk autoincr"` + OwnerID int64 `xorm:"UNIQUE(s) index"` + OwnerName string `xorm:"-"` + Owner *User `xorm:"-"` + LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"` + Name string `xorm:"INDEX NOT NULL"` + Description string `xorm:"TEXT"` + Website string `xorm:"VARCHAR(2048)"` + OriginalServiceType structs.GitServiceType `xorm:"index"` + OriginalURL string `xorm:"VARCHAR(2048)"` + DefaultBranch string NumWatches int NumStars int diff --git a/modules/cron/cron.go b/modules/cron/cron.go index 089f0fa76..795fafb51 100644 --- a/modules/cron/cron.go +++ b/modules/cron/cron.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/sync" mirror_service "code.gitea.io/gitea/services/mirror" @@ -18,12 +19,13 @@ import ( ) const ( - mirrorUpdate = "mirror_update" - gitFsck = "git_fsck" - checkRepos = "check_repos" - archiveCleanup = "archive_cleanup" - syncExternalUsers = "sync_external_users" - deletedBranchesCleanup = "deleted_branches_cleanup" + mirrorUpdate = "mirror_update" + gitFsck = "git_fsck" + checkRepos = "check_repos" + archiveCleanup = "archive_cleanup" + syncExternalUsers = "sync_external_users" + deletedBranchesCleanup = "deleted_branches_cleanup" + updateMigrationPosterID = "update_migration_post_id" ) var c = cron.New() @@ -117,6 +119,15 @@ func NewContext() { go WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches)() } } + + entry, err = c.AddFunc("Update migrated repositories' issues and comments' posterid", setting.Cron.UpdateMigrationPosterID.Schedule, WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID)) + if err != nil { + log.Fatal("Cron[Update migrated repositories]: %v", err) + } + entry.Prev = time.Now() + entry.ExecTimes++ + go WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID)() + c.Start() } diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index ab5ca6dec..69c2adb9e 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -5,6 +5,8 @@ package base +import "code.gitea.io/gitea/modules/structs" + // Downloader downloads the site repo informations type Downloader interface { GetRepoInfo() (*Repository, error) @@ -21,4 +23,5 @@ type Downloader interface { type DownloaderFactory interface { Match(opts MigrateOptions) (bool, error) New(opts MigrateOptions) (Downloader, error) + GitServiceType() structs.GitServiceType } diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index ab3b0b9f6..2452a7a88 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -34,15 +34,17 @@ var ( // GiteaLocalUploader implements an Uploader to gitea sites type GiteaLocalUploader struct { - doer *models.User - repoOwner string - repoName string - repo *models.Repository - labels sync.Map - milestones sync.Map - issues sync.Map - gitRepo *git.Repository - prHeadCache map[string]struct{} + doer *models.User + repoOwner string + repoName string + repo *models.Repository + labels sync.Map + milestones sync.Map + issues sync.Map + gitRepo *git.Repository + prHeadCache map[string]struct{} + userMap map[int64]int64 // external user id mapping to user id + gitServiceType structs.GitServiceType } // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1 @@ -52,6 +54,7 @@ func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *Gitea repoOwner: repoOwner, repoName: repoName, prHeadCache: make(map[string]struct{}), + userMap: make(map[int64]int64), } } @@ -109,13 +112,15 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate } r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{ - RepoName: g.repoName, - Description: repo.Description, - Mirror: repo.IsMirror, - CloneAddr: remoteAddr, - Private: repo.IsPrivate, - Wiki: opts.Wiki, - Releases: opts.Releases, // if didn't get releases, then sync them from tags + RepoName: g.repoName, + Description: repo.Description, + OriginalURL: repo.OriginalURL, + GitServiceType: opts.GitServiceType, + Mirror: repo.IsMirror, + CloneAddr: remoteAddr, + Private: repo.IsPrivate, + Wiki: opts.Wiki, + Releases: opts.Releases, // if didn't get releases, then sync them from tags }) g.repo = r @@ -193,20 +198,38 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { var rels = make([]*models.Release, 0, len(releases)) for _, release := range releases { var rel = models.Release{ - RepoID: g.repo.ID, - PublisherID: g.doer.ID, - TagName: release.TagName, - LowerTagName: strings.ToLower(release.TagName), - Target: release.TargetCommitish, - Title: release.Name, - Sha1: release.TargetCommitish, - Note: release.Body, - IsDraft: release.Draft, - IsPrerelease: release.Prerelease, - IsTag: false, - CreatedUnix: timeutil.TimeStamp(release.Created.Unix()), - OriginalAuthor: release.PublisherName, - OriginalAuthorID: release.PublisherID, + RepoID: g.repo.ID, + TagName: release.TagName, + LowerTagName: strings.ToLower(release.TagName), + Target: release.TargetCommitish, + Title: release.Name, + Sha1: release.TargetCommitish, + Note: release.Body, + IsDraft: release.Draft, + IsPrerelease: release.Prerelease, + IsTag: false, + CreatedUnix: timeutil.TimeStamp(release.Created.Unix()), + } + + userid, ok := g.userMap[release.PublisherID] + tp := g.gitServiceType.Name() + if !ok && tp != "" { + var err error + userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID)) + if err != nil { + log.Error("GetUserIDByExternalUserID: %v", err) + } + if userid > 0 { + g.userMap[release.PublisherID] = userid + } + } + + if userid > 0 { + rel.PublisherID = userid + } else { + rel.PublisherID = g.doer.ID + rel.OriginalAuthor = release.PublisherName + rel.OriginalAuthorID = release.PublisherID } // calc NumCommits @@ -284,20 +307,39 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } var is = models.Issue{ - RepoID: g.repo.ID, - Repo: g.repo, - Index: issue.Number, - PosterID: g.doer.ID, - OriginalAuthor: issue.PosterName, - OriginalAuthorID: issue.PosterID, - Title: issue.Title, - Content: issue.Content, - IsClosed: issue.State == "closed", - IsLocked: issue.IsLocked, - MilestoneID: milestoneID, - Labels: labels, - CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()), + RepoID: g.repo.ID, + Repo: g.repo, + Index: issue.Number, + Title: issue.Title, + Content: issue.Content, + IsClosed: issue.State == "closed", + IsLocked: issue.IsLocked, + MilestoneID: milestoneID, + Labels: labels, + CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()), + } + + userid, ok := g.userMap[issue.PosterID] + tp := g.gitServiceType.Name() + if !ok && tp != "" { + var err error + userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID)) + if err != nil { + log.Error("GetUserIDByExternalUserID: %v", err) + } + if userid > 0 { + g.userMap[issue.PosterID] = userid + } + } + + if userid > 0 { + is.PosterID = userid + } else { + is.PosterID = g.doer.ID + is.OriginalAuthor = issue.PosterName + is.OriginalAuthorID = issue.PosterID } + if issue.Closed != nil { is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix()) } @@ -331,15 +373,35 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { issueID = issueIDStr.(int64) } - cms = append(cms, &models.Comment{ - IssueID: issueID, - Type: models.CommentTypeComment, - PosterID: g.doer.ID, - OriginalAuthor: comment.PosterName, - OriginalAuthorID: comment.PosterID, - Content: comment.Content, - CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()), - }) + userid, ok := g.userMap[comment.PosterID] + tp := g.gitServiceType.Name() + if !ok && tp != "" { + var err error + userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID)) + if err != nil { + log.Error("GetUserIDByExternalUserID: %v", err) + } + if userid > 0 { + g.userMap[comment.PosterID] = userid + } + } + + cm := models.Comment{ + IssueID: issueID, + Type: models.CommentTypeComment, + Content: comment.Content, + CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()), + } + + if userid > 0 { + cm.PosterID = userid + } else { + cm.PosterID = g.doer.ID + cm.OriginalAuthor = comment.PosterName + cm.OriginalAuthorID = comment.PosterID + } + + cms = append(cms, &cm) // TODO: Reactions } @@ -355,6 +417,28 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error if err != nil { return err } + + userid, ok := g.userMap[pr.PosterID] + tp := g.gitServiceType.Name() + if !ok && tp != "" { + var err error + userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID)) + if err != nil { + log.Error("GetUserIDByExternalUserID: %v", err) + } + if userid > 0 { + g.userMap[pr.PosterID] = userid + } + } + + if userid > 0 { + gpr.Issue.PosterID = userid + } else { + gpr.Issue.PosterID = g.doer.ID + gpr.Issue.OriginalAuthor = pr.PosterName + gpr.Issue.OriginalAuthorID = pr.PosterID + } + gprs = append(gprs, gpr) } if err := models.InsertPullRequests(gprs...); err != nil { @@ -460,6 +544,40 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR head = pr.Head.Ref } + var issue = models.Issue{ + RepoID: g.repo.ID, + Repo: g.repo, + Title: pr.Title, + Index: pr.Number, + Content: pr.Content, + MilestoneID: milestoneID, + IsPull: true, + IsClosed: pr.State == "closed", + IsLocked: pr.IsLocked, + Labels: labels, + CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()), + } + + userid, ok := g.userMap[pr.PosterID] + if !ok { + var err error + userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", pr.PosterID)) + if err != nil { + log.Error("GetUserIDByExternalUserID: %v", err) + } + if userid > 0 { + g.userMap[pr.PosterID] = userid + } + } + + if userid > 0 { + issue.PosterID = userid + } else { + issue.PosterID = g.doer.ID + issue.OriginalAuthor = pr.PosterName + issue.OriginalAuthorID = pr.PosterID + } + var pullRequest = models.PullRequest{ HeadRepoID: g.repo.ID, HeadBranch: head, @@ -470,22 +588,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR Index: pr.Number, HasMerged: pr.Merged, - Issue: &models.Issue{ - RepoID: g.repo.ID, - Repo: g.repo, - Title: pr.Title, - Index: pr.Number, - PosterID: g.doer.ID, - OriginalAuthor: pr.PosterName, - OriginalAuthorID: pr.PosterID, - Content: pr.Content, - MilestoneID: milestoneID, - IsPull: true, - IsClosed: pr.State == "closed", - IsLocked: pr.IsLocked, - Labels: labels, - CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()), - }, + Issue: &issue, } if pullRequest.Issue.IsClosed && pr.Closed != nil { diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 1c5d96c03..00d137a3d 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/structs" "github.com/google/go-github/v24/github" "golang.org/x/oauth2" @@ -39,7 +40,7 @@ func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error return false, err } - return u.Host == "github.com" && opts.AuthUsername != "", nil + return strings.EqualFold(u.Host, "github.com") && opts.AuthUsername != "", nil } // New returns a Downloader related to this factory according MigrateOptions @@ -58,6 +59,11 @@ func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Download return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil } +// GitServiceType returns the type of git service +func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType { + return structs.GithubService +} + // GithubDownloaderV3 implements a Downloader interface to get repository informations // from github via APIv3 type GithubDownloaderV3 struct { diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 3f5c0d111..bbc1dc2d5 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/structs" ) // MigrateOptions is equal to base.MigrateOptions @@ -30,6 +31,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt var ( downloader base.Downloader uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName) + theFactory base.DownloaderFactory ) for _, factory := range factories { @@ -40,6 +42,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt if err != nil { return nil, err } + theFactory = factory break } } @@ -52,10 +55,14 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt opts.Comments = false opts.Issues = false opts.PullRequests = false + opts.GitServiceType = structs.PlainGitService downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr) log.Trace("Will migrate from git: %s", opts.CloneAddr) + } else if opts.GitServiceType == structs.NotMigrated { + opts.GitServiceType = theFactory.GitServiceType() } + uploader.gitServiceType = opts.GitServiceType if err := migrateRepository(downloader, uploader, opts); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) diff --git a/modules/migrations/update.go b/modules/migrations/update.go new file mode 100644 index 000000000..df626ddd9 --- /dev/null +++ b/modules/migrations/update.go @@ -0,0 +1,59 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "strconv" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/structs" +) + +// UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID +func UpdateMigrationPosterID() { + for _, gitService := range structs.SupportedFullGitService { + if err := updateMigrationPosterIDByGitService(gitService); err != nil { + log.Error("updateMigrationPosterIDByGitService failed: %v", err) + } + } +} + +func updateMigrationPosterIDByGitService(tp structs.GitServiceType) error { + provider := tp.Name() + if len(provider) == 0 { + return nil + } + + const batchSize = 100 + var start int + for { + users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{ + Provider: provider, + Start: start, + Limit: batchSize, + }) + if err != nil { + return err + } + + for _, user := range users { + externalUserID, err := strconv.ParseInt(user.ExternalID, 10, 64) + if err != nil { + log.Warn("Parse externalUser %#v 's userID failed: %v", user, err) + continue + } + if err := models.UpdateMigrationsByType(tp, externalUserID, user.UserID); err != nil { + log.Error("UpdateMigrationsByType type %s external user id %v to local user id %v failed: %v", tp.Name(), user.ExternalID, user.UserID, err) + } + } + + if len(users) < batchSize { + break + } + start += len(users) + } + return nil +} diff --git a/modules/setting/cron.go b/modules/setting/cron.go index c544c6c22..77f55168a 100644 --- a/modules/setting/cron.go +++ b/modules/setting/cron.go @@ -49,6 +49,9 @@ var ( Schedule string OlderThan time.Duration } `ini:"cron.deleted_branches_cleanup"` + UpdateMigrationPosterID struct { + Schedule string + } `ini:"cron.update_migration_poster_id"` }{ UpdateMirror: struct { Enabled bool @@ -114,6 +117,11 @@ var ( Schedule: "@every 24h", OlderThan: 24 * time.Hour, }, + UpdateMigrationPosterID: struct { + Schedule string + }{ + Schedule: "@every 24h", + }, } ) diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 57f1768a0..be6a3d4b4 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -153,6 +153,43 @@ type EditRepoOption struct { Archived *bool `json:"archived,omitempty"` } +// GitServiceType represents a git service +type GitServiceType int + +// enumerate all GitServiceType +const ( + NotMigrated GitServiceType = iota // 0 not migrated from external sites + PlainGitService // 1 plain git service + GithubService // 2 github.com + GiteaService // 3 gitea service + GitlabService // 4 gitlab service + GogsService // 5 gogs service +) + +// Name represents the service type's name +// WARNNING: the name have to be equal to that on goth's library +func (gt GitServiceType) Name() string { + switch gt { + case GithubService: + return "github" + case GiteaService: + return "gitea" + case GitlabService: + return "gitlab" + case GogsService: + return "gogs" + } + return "" +} + +var ( + // SupportedFullGitService represents all git services supported to migrate issues/labels/prs and etc. + // TODO: add to this list after new git service added + SupportedFullGitService = []GitServiceType{ + GithubService, + } +) + // MigrateRepoOption options for migrating a repository from an external service type MigrateRepoOption struct { // required: true @@ -166,6 +203,8 @@ type MigrateRepoOption struct { Mirror bool `json:"mirror"` Private bool `json:"private"` Description string `json:"description"` + OriginalURL string + GitServiceType GitServiceType Wiki bool Issues bool Milestones bool diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 08c0635bc..a4417107e 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -8,6 +8,7 @@ package repo import ( "fmt" "net/http" + "net/url" "strings" "code.gitea.io/gitea/models" @@ -17,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/validation" @@ -397,21 +399,28 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { return } + var gitServiceType = structs.PlainGitService + u, err := url.Parse(remoteAddr) + if err == nil && strings.EqualFold(u.Host, "github.com") { + gitServiceType = structs.GithubService + } + var opts = migrations.MigrateOptions{ - CloneAddr: remoteAddr, - RepoName: form.RepoName, - Description: form.Description, - Private: form.Private || setting.Repository.ForcePrivate, - Mirror: form.Mirror, - AuthUsername: form.AuthUsername, - AuthPassword: form.AuthPassword, - Wiki: form.Wiki, - Issues: form.Issues, - Milestones: form.Milestones, - Labels: form.Labels, - Comments: true, - PullRequests: form.PullRequests, - Releases: form.Releases, + CloneAddr: remoteAddr, + RepoName: form.RepoName, + Description: form.Description, + Private: form.Private || setting.Repository.ForcePrivate, + Mirror: form.Mirror, + AuthUsername: form.AuthUsername, + AuthPassword: form.AuthPassword, + Wiki: form.Wiki, + Issues: form.Issues, + Milestones: form.Milestones, + Labels: form.Labels, + Comments: true, + PullRequests: form.PullRequests, + Releases: form.Releases, + GitServiceType: gitServiceType, } if opts.Mirror { opts.Issues = false diff --git a/routers/user/auth.go b/routers/user/auth.go index 3def867f6..212d535a0 100644 --- a/routers/user/auth.go +++ b/routers/user/auth.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/externalaccount" "code.gitea.io/gitea/services/mailer" "gitea.com/macaron/captcha" @@ -277,7 +278,7 @@ func TwoFactorPost(ctx *context.Context, form auth.TwoFactorAuthForm) { return } - err = models.LinkAccountToUser(u, gothUser.(goth.User)) + err = externalaccount.LinkAccountToUser(u, gothUser.(goth.User)) if err != nil { ctx.ServerError("UserSignIn", err) return @@ -452,7 +453,7 @@ func U2FSign(ctx *context.Context, signResp u2f.SignResponse) { return } - err = models.LinkAccountToUser(user, gothUser.(goth.User)) + err = externalaccount.LinkAccountToUser(user, gothUser.(goth.User)) if err != nil { ctx.ServerError("UserSignIn", err) return @@ -601,36 +602,42 @@ func handleOAuth2SignIn(u *models.User, gothUser goth.User, ctx *context.Context // Instead, redirect them to the 2FA authentication page. _, err = models.GetTwoFactorByUID(u.ID) if err != nil { - if models.IsErrTwoFactorNotEnrolled(err) { - err = ctx.Session.Set("uid", u.ID) - if err != nil { - log.Error(fmt.Sprintf("Error setting session: %v", err)) - } - err = ctx.Session.Set("uname", u.Name) - if err != nil { - log.Error(fmt.Sprintf("Error setting session: %v", err)) - } + if !models.IsErrTwoFactorNotEnrolled(err) { + ctx.ServerError("UserSignIn", err) + return + } - // Clear whatever CSRF has right now, force to generate a new one - ctx.SetCookie(setting.CSRFCookieName, "", -1, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) + err = ctx.Session.Set("uid", u.ID) + if err != nil { + log.Error(fmt.Sprintf("Error setting session: %v", err)) + } + err = ctx.Session.Set("uname", u.Name) + if err != nil { + log.Error(fmt.Sprintf("Error setting session: %v", err)) + } - // Register last login - u.SetLastLogin() - if err := models.UpdateUserCols(u, "last_login_unix"); err != nil { - ctx.ServerError("UpdateUserCols", err) - return - } + // Clear whatever CSRF has right now, force to generate a new one + ctx.SetCookie(setting.CSRFCookieName, "", -1, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) - if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 { - ctx.SetCookie("redirect_to", "", -1, setting.AppSubURL, "", setting.SessionConfig.Secure, true) - ctx.RedirectToFirst(redirectTo) - return - } + // Register last login + u.SetLastLogin() + if err := models.UpdateUserCols(u, "last_login_unix"); err != nil { + ctx.ServerError("UpdateUserCols", err) + return + } - ctx.Redirect(setting.AppSubURL + "/") - } else { - ctx.ServerError("UserSignIn", err) + // update external user information + if err := models.UpdateExternalUser(u, gothUser); err != nil { + log.Error("UpdateExternalUser failed: %v", err) + } + + if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 { + ctx.SetCookie("redirect_to", "", -1, setting.AppSubURL, "", setting.SessionConfig.Secure, true) + ctx.RedirectToFirst(redirectTo) + return } + + ctx.Redirect(setting.AppSubURL + "/") return } @@ -675,7 +682,7 @@ func oAuth2UserLoginCallback(loginSource *models.LoginSource, request *http.Requ } if hasUser { - return user, goth.User{}, nil + return user, gothUser, nil } // search in external linked users @@ -689,7 +696,7 @@ func oAuth2UserLoginCallback(loginSource *models.LoginSource, request *http.Requ } if hasUser { user, err = models.GetUserByID(externalLoginUser.UserID) - return user, goth.User{}, err + return user, gothUser, err } // no user found to login @@ -789,16 +796,18 @@ func LinkAccountPostSignIn(ctx *context.Context, signInForm auth.SignInForm) { // Instead, redirect them to the 2FA authentication page. _, err = models.GetTwoFactorByUID(u.ID) if err != nil { - if models.IsErrTwoFactorNotEnrolled(err) { - err = models.LinkAccountToUser(u, gothUser.(goth.User)) - if err != nil { - ctx.ServerError("UserLinkAccount", err) - } else { - handleSignIn(ctx, u, signInForm.Remember) - } - } else { + if !models.IsErrTwoFactorNotEnrolled(err) { + ctx.ServerError("UserLinkAccount", err) + return + } + + err = externalaccount.LinkAccountToUser(u, gothUser.(goth.User)) + if err != nil { ctx.ServerError("UserLinkAccount", err) + return } + + handleSignIn(ctx, u, signInForm.Remember) return } @@ -947,6 +956,11 @@ func LinkAccountPostRegister(ctx *context.Context, cpt *captcha.Captcha, form au } } + // update external user information + if err := models.UpdateExternalUser(u, gothUser.(goth.User)); err != nil { + log.Error("UpdateExternalUser failed: %v", err) + } + // Send confirmation email if setting.Service.RegisterEmailConfirm && u.ID > 1 { mailer.SendActivateAccountMail(ctx.Locale, u) diff --git a/services/externalaccount/user.go b/services/externalaccount/user.go new file mode 100644 index 000000000..800546f12 --- /dev/null +++ b/services/externalaccount/user.go @@ -0,0 +1,66 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package externalaccount + +import ( + "strconv" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/structs" + + "github.com/markbates/goth" +) + +// LinkAccountToUser link the gothUser to the user +func LinkAccountToUser(user *models.User, gothUser goth.User) error { + loginSource, err := models.GetActiveOAuth2LoginSourceByName(gothUser.Provider) + if err != nil { + return err + } + + externalLoginUser := &models.ExternalLoginUser{ + ExternalID: gothUser.UserID, + UserID: user.ID, + LoginSourceID: loginSource.ID, + RawData: gothUser.RawData, + Provider: gothUser.Provider, + Email: gothUser.Email, + Name: gothUser.Name, + FirstName: gothUser.FirstName, + LastName: gothUser.LastName, + NickName: gothUser.NickName, + Description: gothUser.Description, + AvatarURL: gothUser.AvatarURL, + Location: gothUser.Location, + AccessToken: gothUser.AccessToken, + AccessTokenSecret: gothUser.AccessTokenSecret, + RefreshToken: gothUser.RefreshToken, + ExpiresAt: gothUser.ExpiresAt, + } + + if err := models.LinkExternalToUser(user, externalLoginUser); err != nil { + return err + } + + externalID, err := strconv.ParseInt(externalLoginUser.ExternalID, 10, 64) + if err != nil { + return err + } + + var tp structs.GitServiceType + for _, s := range structs.SupportedFullGitService { + if strings.EqualFold(s.Name(), gothUser.Provider) { + tp = s + break + } + } + + if tp.Name() != "" { + return models.UpdateMigrationsByType(tp, externalID, user.ID) + } + + return nil +}