Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)
* Dump github/gitlab repository data to a local directory * Fix lint * Adjust directory structure * Allow migration special units * Allow migration ignore release assets * Fix lint * Add restore repository * stage the changes * Merge * Fix lint * Update the interface * Add some restore methods * Finish restore * Add comments * Fix restore * Add a token flag * Fix bug * Fix test * Fix test * Fix bug * Fix bug * Fix lint * Fix restore * refactor downloader * fmt * Fix bug isEnd detection on getIssues * Refactor maxPerPage * Remove unused codes * Remove unused codes * Fix bug * Fix restore * Fix dump * Uploader should not depend downloader * use release attachment name but not id * Fix restore bug * Fix lint * Fix restore bug * Add a method of DownloadFunc for base.Release to make uploader not depend on downloader * fix Release yml marshal * Fix trace information * Fix bug when dump & restore * Save relative path on yml file * Fix bug * Use relative path * Update docs * Use git service string but not int * Recognize clone addr to service typetokarchuk/v1.17
Lunny Xiao
4 years ago
committed by
GitHub
parent
212fa340cf
commit
dd08853b10
@ -0,0 +1,162 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"strings" |
||||
|
||||
"code.gitea.io/gitea/modules/convert" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/migrations" |
||||
"code.gitea.io/gitea/modules/migrations/base" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/structs" |
||||
|
||||
"github.com/urfave/cli" |
||||
) |
||||
|
||||
// CmdDumpRepository represents the available dump repository sub-command.
|
||||
var CmdDumpRepository = cli.Command{ |
||||
Name: "dump-repo", |
||||
Usage: "Dump the repository from git/github/gitea/gitlab", |
||||
Description: "This is a command for dumping the repository data.", |
||||
Action: runDumpRepository, |
||||
Flags: []cli.Flag{ |
||||
cli.StringFlag{ |
||||
Name: "git_service", |
||||
Value: "", |
||||
Usage: "Git service, git, github, gitea, gitlab. If clone_addr could be recognized, this could be ignored.", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "repo_dir, r", |
||||
Value: "./data", |
||||
Usage: "Repository dir path to store the data", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "clone_addr", |
||||
Value: "", |
||||
Usage: "The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "auth_username", |
||||
Value: "", |
||||
Usage: "The username to visit the clone_addr", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "auth_password", |
||||
Value: "", |
||||
Usage: "The password to visit the clone_addr", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "auth_token", |
||||
Value: "", |
||||
Usage: "The personal token to visit the clone_addr", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "owner_name", |
||||
Value: "", |
||||
Usage: "The data will be stored on a directory with owner name if not empty", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "repo_name", |
||||
Value: "", |
||||
Usage: "The data will be stored on a directory with repository name if not empty", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "units", |
||||
Value: "", |
||||
Usage: `Which items will be migrated, one or more units should be separated as comma.
|
||||
wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`, |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
func runDumpRepository(ctx *cli.Context) error { |
||||
if err := initDB(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
log.Trace("AppPath: %s", setting.AppPath) |
||||
log.Trace("AppWorkPath: %s", setting.AppWorkPath) |
||||
log.Trace("Custom path: %s", setting.CustomPath) |
||||
log.Trace("Log path: %s", setting.LogRootPath) |
||||
setting.InitDBConfig() |
||||
|
||||
var ( |
||||
serviceType structs.GitServiceType |
||||
cloneAddr = ctx.String("clone_addr") |
||||
serviceStr = ctx.String("git_service") |
||||
) |
||||
|
||||
if strings.HasPrefix(strings.ToLower(cloneAddr), "https://github.com/") { |
||||
serviceStr = "github" |
||||
} else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitlab.com/") { |
||||
serviceStr = "gitlab" |
||||
} else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitea.com/") { |
||||
serviceStr = "gitea" |
||||
} |
||||
if serviceStr == "" { |
||||
return errors.New("git_service missed or clone_addr cannot be recognized") |
||||
} |
||||
serviceType = convert.ToGitServiceType(serviceStr) |
||||
|
||||
var opts = base.MigrateOptions{ |
||||
GitServiceType: serviceType, |
||||
CloneAddr: cloneAddr, |
||||
AuthUsername: ctx.String("auth_username"), |
||||
AuthPassword: ctx.String("auth_password"), |
||||
AuthToken: ctx.String("auth_token"), |
||||
RepoName: ctx.String("repo_name"), |
||||
} |
||||
|
||||
if len(ctx.String("units")) == 0 { |
||||
opts.Wiki = true |
||||
opts.Issues = true |
||||
opts.Milestones = true |
||||
opts.Labels = true |
||||
opts.Releases = true |
||||
opts.Comments = true |
||||
opts.PullRequests = true |
||||
opts.ReleaseAssets = true |
||||
} else { |
||||
units := strings.Split(ctx.String("units"), ",") |
||||
for _, unit := range units { |
||||
switch strings.ToLower(unit) { |
||||
case "wiki": |
||||
opts.Wiki = true |
||||
case "issues": |
||||
opts.Issues = true |
||||
case "milestones": |
||||
opts.Milestones = true |
||||
case "labels": |
||||
opts.Labels = true |
||||
case "releases": |
||||
opts.Releases = true |
||||
case "release_assets": |
||||
opts.ReleaseAssets = true |
||||
case "comments": |
||||
opts.Comments = true |
||||
case "pull_requests": |
||||
opts.PullRequests = true |
||||
} |
||||
} |
||||
} |
||||
|
||||
if err := migrations.DumpRepository( |
||||
context.Background(), |
||||
ctx.String("repo_dir"), |
||||
ctx.String("owner_name"), |
||||
opts, |
||||
); err != nil { |
||||
log.Fatal("Failed to dump repository: %v", err) |
||||
return err |
||||
} |
||||
|
||||
log.Trace("Dump finished!!!") |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,119 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"context" |
||||
"strings" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/migrations" |
||||
"code.gitea.io/gitea/modules/migrations/base" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/storage" |
||||
pull_service "code.gitea.io/gitea/services/pull" |
||||
|
||||
"github.com/urfave/cli" |
||||
) |
||||
|
||||
// CmdRestoreRepository represents the available restore a repository sub-command.
|
||||
var CmdRestoreRepository = cli.Command{ |
||||
Name: "restore-repo", |
||||
Usage: "Restore the repository from disk", |
||||
Description: "This is a command for restoring the repository data.", |
||||
Action: runRestoreRepository, |
||||
Flags: []cli.Flag{ |
||||
cli.StringFlag{ |
||||
Name: "repo_dir, r", |
||||
Value: "./data", |
||||
Usage: "Repository dir path to restore from", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "owner_name", |
||||
Value: "", |
||||
Usage: "Restore destination owner name", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "repo_name", |
||||
Value: "", |
||||
Usage: "Restore destination repository name", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "units", |
||||
Value: "", |
||||
Usage: `Which items will be restored, one or more units should be separated as comma.
|
||||
wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`, |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
func runRestoreRepository(ctx *cli.Context) error { |
||||
if err := initDB(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
log.Trace("AppPath: %s", setting.AppPath) |
||||
log.Trace("AppWorkPath: %s", setting.AppWorkPath) |
||||
log.Trace("Custom path: %s", setting.CustomPath) |
||||
log.Trace("Log path: %s", setting.LogRootPath) |
||||
setting.InitDBConfig() |
||||
|
||||
if err := storage.Init(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err := pull_service.Init(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
var opts = base.MigrateOptions{ |
||||
RepoName: ctx.String("repo_name"), |
||||
} |
||||
|
||||
if len(ctx.String("units")) == 0 { |
||||
opts.Wiki = true |
||||
opts.Issues = true |
||||
opts.Milestones = true |
||||
opts.Labels = true |
||||
opts.Releases = true |
||||
opts.Comments = true |
||||
opts.PullRequests = true |
||||
opts.ReleaseAssets = true |
||||
} else { |
||||
units := strings.Split(ctx.String("units"), ",") |
||||
for _, unit := range units { |
||||
switch strings.ToLower(unit) { |
||||
case "wiki": |
||||
opts.Wiki = true |
||||
case "issues": |
||||
opts.Issues = true |
||||
case "milestones": |
||||
opts.Milestones = true |
||||
case "labels": |
||||
opts.Labels = true |
||||
case "releases": |
||||
opts.Releases = true |
||||
case "release_assets": |
||||
opts.ReleaseAssets = true |
||||
case "comments": |
||||
opts.Comments = true |
||||
case "pull_requests": |
||||
opts.PullRequests = true |
||||
} |
||||
} |
||||
} |
||||
|
||||
if err := migrations.RestoreRepository( |
||||
context.Background(), |
||||
ctx.String("repo_dir"), |
||||
ctx.String("owner_name"), |
||||
ctx.String("repo_name"), |
||||
); err != nil { |
||||
log.Fatal("Failed to restore repository: %v", err) |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,591 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"io" |
||||
"net/http" |
||||
"net/url" |
||||
"os" |
||||
"path/filepath" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/migrations/base" |
||||
"code.gitea.io/gitea/modules/repository" |
||||
|
||||
"gopkg.in/yaml.v2" |
||||
) |
||||
|
||||
var ( |
||||
_ base.Uploader = &RepositoryDumper{} |
||||
) |
||||
|
||||
// RepositoryDumper implements an Uploader to the local directory
|
||||
type RepositoryDumper struct { |
||||
ctx context.Context |
||||
baseDir string |
||||
repoOwner string |
||||
repoName string |
||||
opts base.MigrateOptions |
||||
milestoneFile *os.File |
||||
labelFile *os.File |
||||
releaseFile *os.File |
||||
issueFile *os.File |
||||
commentFiles map[int64]*os.File |
||||
pullrequestFile *os.File |
||||
reviewFiles map[int64]*os.File |
||||
|
||||
gitRepo *git.Repository |
||||
prHeadCache map[string]struct{} |
||||
} |
||||
|
||||
// NewRepositoryDumper creates an gitea Uploader
|
||||
func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) (*RepositoryDumper, error) { |
||||
baseDir = filepath.Join(baseDir, repoOwner, repoName) |
||||
if err := os.MkdirAll(baseDir, os.ModePerm); err != nil { |
||||
return nil, err |
||||
} |
||||
return &RepositoryDumper{ |
||||
ctx: ctx, |
||||
opts: opts, |
||||
baseDir: baseDir, |
||||
repoOwner: repoOwner, |
||||
repoName: repoName, |
||||
prHeadCache: make(map[string]struct{}), |
||||
commentFiles: make(map[int64]*os.File), |
||||
reviewFiles: make(map[int64]*os.File), |
||||
}, nil |
||||
} |
||||
|
||||
// MaxBatchInsertSize returns the table's max batch insert size
|
||||
func (g *RepositoryDumper) MaxBatchInsertSize(tp string) int { |
||||
return 1000 |
||||
} |
||||
|
||||
func (g *RepositoryDumper) gitPath() string { |
||||
return filepath.Join(g.baseDir, "git") |
||||
} |
||||
|
||||
func (g *RepositoryDumper) wikiPath() string { |
||||
return filepath.Join(g.baseDir, "wiki") |
||||
} |
||||
|
||||
func (g *RepositoryDumper) commentDir() string { |
||||
return filepath.Join(g.baseDir, "comments") |
||||
} |
||||
|
||||
func (g *RepositoryDumper) reviewDir() string { |
||||
return filepath.Join(g.baseDir, "reviews") |
||||
} |
||||
|
||||
func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) { |
||||
if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 { |
||||
u, err := url.Parse(remoteAddr) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword) |
||||
if len(g.opts.AuthToken) > 0 { |
||||
u.User = url.UserPassword("oauth2", g.opts.AuthToken) |
||||
} |
||||
remoteAddr = u.String() |
||||
} |
||||
|
||||
return remoteAddr, nil |
||||
} |
||||
|
||||
// CreateRepo creates a repository
|
||||
func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { |
||||
f, err := os.Create(filepath.Join(g.baseDir, "repo.yml")) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer f.Close() |
||||
|
||||
bs, err := yaml.Marshal(map[string]interface{}{ |
||||
"name": repo.Name, |
||||
"owner": repo.Owner, |
||||
"description": repo.Description, |
||||
"clone_addr": opts.CloneAddr, |
||||
"original_url": repo.OriginalURL, |
||||
"is_private": opts.Private, |
||||
"service_type": opts.GitServiceType, |
||||
"wiki": opts.Wiki, |
||||
"issues": opts.Issues, |
||||
"milestones": opts.Milestones, |
||||
"labels": opts.Labels, |
||||
"releases": opts.Releases, |
||||
"comments": opts.Comments, |
||||
"pulls": opts.PullRequests, |
||||
"assets": opts.ReleaseAssets, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err := f.Write(bs); err != nil { |
||||
return err |
||||
} |
||||
|
||||
repoPath := g.gitPath() |
||||
if err := os.MkdirAll(repoPath, os.ModePerm); err != nil { |
||||
return err |
||||
} |
||||
|
||||
migrateTimeout := 2 * time.Hour |
||||
|
||||
remoteAddr, err := g.setURLToken(repo.CloneURL) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{ |
||||
Mirror: true, |
||||
Quiet: true, |
||||
Timeout: migrateTimeout, |
||||
}) |
||||
if err != nil { |
||||
return fmt.Errorf("Clone: %v", err) |
||||
} |
||||
|
||||
if opts.Wiki { |
||||
wikiPath := g.wikiPath() |
||||
wikiRemotePath := repository.WikiRemoteURL(remoteAddr) |
||||
if len(wikiRemotePath) > 0 { |
||||
if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil { |
||||
return fmt.Errorf("Failed to remove %s: %v", wikiPath, err) |
||||
} |
||||
|
||||
if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ |
||||
Mirror: true, |
||||
Quiet: true, |
||||
Timeout: migrateTimeout, |
||||
Branch: "master", |
||||
}); err != nil { |
||||
log.Warn("Clone wiki: %v", err) |
||||
if err := os.RemoveAll(wikiPath); err != nil { |
||||
return fmt.Errorf("Failed to remove %s: %v", wikiPath, err) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
g.gitRepo, err = git.OpenRepository(g.gitPath()) |
||||
return err |
||||
} |
||||
|
||||
// Close closes this uploader
|
||||
func (g *RepositoryDumper) Close() { |
||||
if g.gitRepo != nil { |
||||
g.gitRepo.Close() |
||||
} |
||||
if g.milestoneFile != nil { |
||||
g.milestoneFile.Close() |
||||
} |
||||
if g.labelFile != nil { |
||||
g.labelFile.Close() |
||||
} |
||||
if g.releaseFile != nil { |
||||
g.releaseFile.Close() |
||||
} |
||||
if g.issueFile != nil { |
||||
g.issueFile.Close() |
||||
} |
||||
for _, f := range g.commentFiles { |
||||
f.Close() |
||||
} |
||||
if g.pullrequestFile != nil { |
||||
g.pullrequestFile.Close() |
||||
} |
||||
for _, f := range g.reviewFiles { |
||||
f.Close() |
||||
} |
||||
} |
||||
|
||||
// CreateTopics creates topics
|
||||
func (g *RepositoryDumper) CreateTopics(topics ...string) error { |
||||
f, err := os.Create(filepath.Join(g.baseDir, "topic.yml")) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer f.Close() |
||||
|
||||
bs, err := yaml.Marshal(map[string]interface{}{ |
||||
"topics": topics, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err := f.Write(bs); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// CreateMilestones creates milestones
|
||||
func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error { |
||||
var err error |
||||
if g.milestoneFile == nil { |
||||
g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml")) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
bs, err := yaml.Marshal(milestones) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err := g.milestoneFile.Write(bs); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// CreateLabels creates labels
|
||||
func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { |
||||
var err error |
||||
if g.labelFile == nil { |
||||
g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml")) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
bs, err := yaml.Marshal(labels) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err := g.labelFile.Write(bs); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// CreateReleases creates releases
|
||||
func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { |
||||
if g.opts.ReleaseAssets { |
||||
for _, release := range releases { |
||||
attachDir := filepath.Join("release_assets", release.TagName) |
||||
if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil { |
||||
return err |
||||
} |
||||
for _, asset := range release.Assets { |
||||
attachLocalPath := filepath.Join(attachDir, asset.Name) |
||||
// download attachment
|
||||
|
||||
err := func(attachPath string) error { |
||||
var rc io.ReadCloser |
||||
var err error |
||||
if asset.DownloadURL == nil { |
||||
rc, err = asset.DownloadFunc() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} else { |
||||
resp, err := http.Get(*asset.DownloadURL) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
rc = resp.Body |
||||
} |
||||
defer rc.Close() |
||||
|
||||
fw, err := os.Create(attachPath) |
||||
if err != nil { |
||||
return fmt.Errorf("Create: %v", err) |
||||
} |
||||
defer fw.Close() |
||||
|
||||
_, err = io.Copy(fw, rc) |
||||
return err |
||||
}(filepath.Join(g.baseDir, attachLocalPath)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source
|
||||
} |
||||
} |
||||
} |
||||
|
||||
var err error |
||||
if g.releaseFile == nil { |
||||
g.releaseFile, err = os.Create(filepath.Join(g.baseDir, "release.yml")) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
bs, err := yaml.Marshal(releases) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err := g.releaseFile.Write(bs); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// SyncTags syncs releases with tags in the database
|
||||
func (g *RepositoryDumper) SyncTags() error { |
||||
return nil |
||||
} |
||||
|
||||
// CreateIssues creates issues
|
||||
func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error { |
||||
var err error |
||||
if g.issueFile == nil { |
||||
g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml")) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
bs, err := yaml.Marshal(issues) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err := g.issueFile.Write(bs); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File, itemsMap map[int64][]interface{}) error { |
||||
if err := os.MkdirAll(dir, os.ModePerm); err != nil { |
||||
return err |
||||
} |
||||
|
||||
for number, items := range itemsMap { |
||||
var err error |
||||
itemFile := itemFiles[number] |
||||
if itemFile == nil { |
||||
itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number))) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
itemFiles[number] = itemFile |
||||
} |
||||
|
||||
bs, err := yaml.Marshal(items) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err := itemFile.Write(bs); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// CreateComments creates comments of issues
|
||||
func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { |
||||
var commentsMap = make(map[int64][]interface{}, len(comments)) |
||||
for _, comment := range comments { |
||||
commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment) |
||||
} |
||||
|
||||
return g.createItems(g.commentDir(), g.commentFiles, commentsMap) |
||||
} |
||||
|
||||
// CreatePullRequests creates pull requests
|
||||
func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { |
||||
for _, pr := range prs { |
||||
// download patch file
|
||||
err := func() error { |
||||
u, err := g.setURLToken(pr.PatchURL) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
resp, err := http.Get(u) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer resp.Body.Close() |
||||
pullDir := filepath.Join(g.gitPath(), "pulls") |
||||
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { |
||||
return err |
||||
} |
||||
fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)) |
||||
f, err := os.Create(fPath) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer f.Close() |
||||
if _, err = io.Copy(f, resp.Body); err != nil { |
||||
return err |
||||
} |
||||
pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number) |
||||
|
||||
return nil |
||||
}() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
// set head information
|
||||
pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) |
||||
if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { |
||||
return err |
||||
} |
||||
p, err := os.Create(filepath.Join(pullHead, "head")) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
_, err = p.WriteString(pr.Head.SHA) |
||||
p.Close() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if pr.IsForkPullRequest() && pr.State != "closed" { |
||||
if pr.Head.OwnerName != "" { |
||||
remote := pr.Head.OwnerName |
||||
_, ok := g.prHeadCache[remote] |
||||
if !ok { |
||||
// git remote add
|
||||
// TODO: how to handle private CloneURL?
|
||||
err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true) |
||||
if err != nil { |
||||
log.Error("AddRemote failed: %s", err) |
||||
} else { |
||||
g.prHeadCache[remote] = struct{}{} |
||||
ok = true |
||||
} |
||||
} |
||||
|
||||
if ok { |
||||
_, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.gitPath()) |
||||
if err != nil { |
||||
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) |
||||
} else { |
||||
headBranch := filepath.Join(g.gitPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) |
||||
if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { |
||||
return err |
||||
} |
||||
b, err := os.Create(headBranch) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
_, err = b.WriteString(pr.Head.SHA) |
||||
b.Close() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
var err error |
||||
if g.pullrequestFile == nil { |
||||
if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { |
||||
return err |
||||
} |
||||
g.pullrequestFile, err = os.Create(filepath.Join(g.baseDir, "pull_request.yml")) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
bs, err := yaml.Marshal(prs) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if _, err := g.pullrequestFile.Write(bs); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// CreateReviews create pull request reviews
|
||||
func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error { |
||||
var reviewsMap = make(map[int64][]interface{}, len(reviews)) |
||||
for _, review := range reviews { |
||||
reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review) |
||||
} |
||||
|
||||
return g.createItems(g.reviewDir(), g.reviewFiles, reviewsMap) |
||||
} |
||||
|
||||
// Rollback when migrating failed, this will rollback all the changes.
|
||||
func (g *RepositoryDumper) Rollback() error { |
||||
g.Close() |
||||
return os.RemoveAll(g.baseDir) |
||||
} |
||||
|
||||
// Finish when migrating succeed, this will update something.
|
||||
func (g *RepositoryDumper) Finish() error { |
||||
return nil |
||||
} |
||||
|
||||
// DumpRepository dump repository according MigrateOptions to a local directory
|
||||
func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error { |
||||
downloader, err := newDownloader(ctx, ownerName, opts) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if err := migrateRepository(downloader, uploader, opts); err != nil { |
||||
if err1 := uploader.Rollback(); err1 != nil { |
||||
log.Error("rollback failed: %v", err1) |
||||
} |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// RestoreRepository restore a repository from the disk directory
|
||||
func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string) error { |
||||
doer, err := models.GetAdminUser() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName) |
||||
downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if err = migrateRepository(downloader, uploader, base.MigrateOptions{ |
||||
Wiki: true, |
||||
Issues: true, |
||||
Milestones: true, |
||||
Labels: true, |
||||
Releases: true, |
||||
Comments: true, |
||||
PullRequests: true, |
||||
ReleaseAssets: true, |
||||
}); err != nil { |
||||
if err1 := uploader.Rollback(); err1 != nil { |
||||
log.Error("rollback failed: %v", err1) |
||||
} |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,276 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"os" |
||||
"path/filepath" |
||||
"strconv" |
||||
|
||||
"code.gitea.io/gitea/modules/migrations/base" |
||||
|
||||
"gopkg.in/yaml.v2" |
||||
) |
||||
|
||||
// RepositoryRestorer implements an Downloader from the local directory
|
||||
type RepositoryRestorer struct { |
||||
ctx context.Context |
||||
baseDir string |
||||
repoOwner string |
||||
repoName string |
||||
} |
||||
|
||||
// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder
|
||||
func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) (*RepositoryRestorer, error) { |
||||
baseDir, err := filepath.Abs(baseDir) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return &RepositoryRestorer{ |
||||
ctx: ctx, |
||||
baseDir: baseDir, |
||||
repoOwner: owner, |
||||
repoName: repoName, |
||||
}, nil |
||||
} |
||||
|
||||
func (r *RepositoryRestorer) commentDir() string { |
||||
return filepath.Join(r.baseDir, "comments") |
||||
} |
||||
|
||||
func (r *RepositoryRestorer) reviewDir() string { |
||||
return filepath.Join(r.baseDir, "reviews") |
||||
} |
||||
|
||||
// SetContext set context
|
||||
func (r *RepositoryRestorer) SetContext(ctx context.Context) { |
||||
r.ctx = ctx |
||||
} |
||||
|
||||
// GetRepoInfo returns a repository information
|
||||
func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { |
||||
p := filepath.Join(r.baseDir, "repo.yml") |
||||
bs, err := ioutil.ReadFile(p) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
var opts = make(map[string]string) |
||||
err = yaml.Unmarshal(bs, &opts) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
isPrivate, _ := strconv.ParseBool(opts["is_private"]) |
||||
|
||||
return &base.Repository{ |
||||
Owner: r.repoOwner, |
||||
Name: r.repoName, |
||||
IsPrivate: isPrivate, |
||||
Description: opts["description"], |
||||
OriginalURL: opts["original_url"], |
||||
CloneURL: opts["clone_addr"], |
||||
DefaultBranch: opts["default_branch"], |
||||
}, nil |
||||
} |
||||
|
||||
// GetTopics return github topics
|
||||
func (r *RepositoryRestorer) GetTopics() ([]string, error) { |
||||
p := filepath.Join(r.baseDir, "topic.yml") |
||||
|
||||
var topics = struct { |
||||
Topics []string `yaml:"topics"` |
||||
}{} |
||||
|
||||
bs, err := ioutil.ReadFile(p) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
err = yaml.Unmarshal(bs, &topics) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return topics.Topics, nil |
||||
} |
||||
|
||||
// GetMilestones returns milestones
|
||||
func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { |
||||
var milestones = make([]*base.Milestone, 0, 10) |
||||
p := filepath.Join(r.baseDir, "milestone.yml") |
||||
_, err := os.Stat(p) |
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
return nil, nil |
||||
} |
||||
return nil, err |
||||
} |
||||
|
||||
bs, err := ioutil.ReadFile(p) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
err = yaml.Unmarshal(bs, &milestones) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return milestones, nil |
||||
} |
||||
|
||||
// GetReleases returns releases
|
||||
func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { |
||||
var releases = make([]*base.Release, 0, 10) |
||||
p := filepath.Join(r.baseDir, "release.yml") |
||||
_, err := os.Stat(p) |
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
return nil, nil |
||||
} |
||||
return nil, err |
||||
} |
||||
|
||||
bs, err := ioutil.ReadFile(p) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
err = yaml.Unmarshal(bs, &releases) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
for _, rel := range releases { |
||||
for _, asset := range rel.Assets { |
||||
*asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL) |
||||
} |
||||
} |
||||
return releases, nil |
||||
} |
||||
|
||||
// GetLabels returns labels
|
||||
func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { |
||||
var labels = make([]*base.Label, 0, 10) |
||||
p := filepath.Join(r.baseDir, "label.yml") |
||||
_, err := os.Stat(p) |
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
return nil, nil |
||||
} |
||||
return nil, err |
||||
} |
||||
|
||||
bs, err := ioutil.ReadFile(p) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
err = yaml.Unmarshal(bs, &labels) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return labels, nil |
||||
} |
||||
|
||||
// GetIssues returns issues according start and limit
|
||||
func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { |
||||
var issues = make([]*base.Issue, 0, 10) |
||||
p := filepath.Join(r.baseDir, "issue.yml") |
||||
_, err := os.Stat(p) |
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
return nil, true, nil |
||||
} |
||||
return nil, false, err |
||||
} |
||||
|
||||
bs, err := ioutil.ReadFile(p) |
||||
if err != nil { |
||||
return nil, false, err |
||||
} |
||||
|
||||
err = yaml.Unmarshal(bs, &issues) |
||||
if err != nil { |
||||
return nil, false, err |
||||
} |
||||
return issues, true, nil |
||||
} |
||||
|
||||
// GetComments returns comments according issueNumber
|
||||
func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) { |
||||
var comments = make([]*base.Comment, 0, 10) |
||||
p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber)) |
||||
_, err := os.Stat(p) |
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
return nil, nil |
||||
} |
||||
return nil, err |
||||
} |
||||
|
||||
bs, err := ioutil.ReadFile(p) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
err = yaml.Unmarshal(bs, &comments) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return comments, nil |
||||
} |
||||
|
||||
// GetPullRequests returns pull requests according page and perPage
|
||||
func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { |
||||
var pulls = make([]*base.PullRequest, 0, 10) |
||||
p := filepath.Join(r.baseDir, "pull_request.yml") |
||||
_, err := os.Stat(p) |
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
return nil, true, nil |
||||
} |
||||
return nil, false, err |
||||
} |
||||
|
||||
bs, err := ioutil.ReadFile(p) |
||||
if err != nil { |
||||
return nil, false, err |
||||
} |
||||
|
||||
err = yaml.Unmarshal(bs, &pulls) |
||||
if err != nil { |
||||
return nil, false, err |
||||
} |
||||
for _, pr := range pulls { |
||||
pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL) |
||||
} |
||||
return pulls, true, nil |
||||
} |
||||
|
||||
// GetReviews returns pull requests review
|
||||
func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { |
||||
var reviews = make([]*base.Review, 0, 10) |
||||
p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber)) |
||||
_, err := os.Stat(p) |
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
return nil, nil |
||||
} |
||||
return nil, err |
||||
} |
||||
|
||||
bs, err := ioutil.ReadFile(p) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
err = yaml.Unmarshal(bs, &reviews) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return reviews, nil |
||||
} |
||||
@ -0,0 +1,40 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package uri |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io" |
||||
"net/http" |
||||
"net/url" |
||||
"os" |
||||
"strings" |
||||
) |
||||
|
||||
// ErrURISchemeNotSupported represents a scheme error
|
||||
type ErrURISchemeNotSupported struct { |
||||
Scheme string |
||||
} |
||||
|
||||
func (e ErrURISchemeNotSupported) Error() string { |
||||
return fmt.Sprintf("Unsupported scheme: %v", e.Scheme) |
||||
} |
||||
|
||||
// Open open a local file or a remote file
|
||||
func Open(uriStr string) (io.ReadCloser, error) { |
||||
u, err := url.Parse(uriStr) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
switch strings.ToLower(u.Scheme) { |
||||
case "http", "https": |
||||
f, err := http.Get(uriStr) |
||||
return f.Body, err |
||||
case "file": |
||||
return os.Open(u.Path) |
||||
default: |
||||
return nil, ErrURISchemeNotSupported{Scheme: u.Scheme} |
||||
} |
||||
} |
||||
@ -0,0 +1,20 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package uri |
||||
|
||||
import ( |
||||
"path/filepath" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestReadURI(t *testing.T) { |
||||
p, err := filepath.Abs("./uri.go") |
||||
assert.NoError(t, err) |
||||
f, err := Open("file://" + p) |
||||
assert.NoError(t, err) |
||||
defer f.Close() |
||||
} |
||||
Loading…
Reference in new issue