You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
gitea/modules/repository/generate.go

358 lines
11 KiB

// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package repository
import (
"bufio"
"bytes"
"context"
"fmt"
"os"
"path"
"path/filepath"
"strings"
"time"
"code.gitea.io/gitea/models"
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
"github.com/gobwas/glob"
"github.com/huandu/xstrings"
)
type transformer struct {
Name string
Transform func(string) string
}
type expansion struct {
Name string
Value string
Transformers []transformer
}
var defaultTransformers = []transformer{
{Name: "SNAKE", Transform: xstrings.ToSnakeCase},
{Name: "KEBAB", Transform: xstrings.ToKebabCase},
{Name: "CAMEL", Transform: func(str string) string {
return xstrings.FirstRuneToLower(xstrings.ToCamelCase(str))
}},
{Name: "PASCAL", Transform: xstrings.ToCamelCase},
{Name: "LOWER", Transform: strings.ToLower},
{Name: "UPPER", Transform: strings.ToUpper},
{Name: "TITLE", Transform: util.ToTitleCase},
}
func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository) string {
expansions := []expansion{
{Name: "REPO_NAME", Value: generateRepo.Name, Transformers: defaultTransformers},
{Name: "TEMPLATE_NAME", Value: templateRepo.Name, Transformers: defaultTransformers},
{Name: "REPO_DESCRIPTION", Value: generateRepo.Description, Transformers: nil},
{Name: "TEMPLATE_DESCRIPTION", Value: templateRepo.Description, Transformers: nil},
{Name: "REPO_OWNER", Value: generateRepo.OwnerName, Transformers: defaultTransformers},
{Name: "TEMPLATE_OWNER", Value: templateRepo.OwnerName, Transformers: defaultTransformers},
{Name: "REPO_LINK", Value: generateRepo.Link(), Transformers: nil},
{Name: "TEMPLATE_LINK", Value: templateRepo.Link(), Transformers: nil},
{Name: "REPO_HTTPS_URL", Value: generateRepo.CloneLink().HTTPS, Transformers: nil},
{Name: "TEMPLATE_HTTPS_URL", Value: templateRepo.CloneLink().HTTPS, Transformers: nil},
{Name: "REPO_SSH_URL", Value: generateRepo.CloneLink().SSH, Transformers: nil},
{Name: "TEMPLATE_SSH_URL", Value: templateRepo.CloneLink().SSH, Transformers: nil},
}
expansionMap := make(map[string]string)
for _, e := range expansions {
expansionMap[e.Name] = e.Value
for _, tr := range e.Transformers {
expansionMap[fmt.Sprintf("%s_%s", e.Name, tr.Name)] = tr.Transform(e.Value)
}
}
return os.Expand(src, func(key string) string {
if expansion, ok := expansionMap[key]; ok {
return expansion
}
return key
})
}
// GiteaTemplate holds information about a .gitea/template file
type GiteaTemplate struct {
Path string
Content []byte
globs []glob.Glob
}
// Globs parses the .gitea/template globs or returns them if they were already parsed
func (gt GiteaTemplate) Globs() []glob.Glob {
if gt.globs != nil {
return gt.globs
}
gt.globs = make([]glob.Glob, 0)
scanner := bufio.NewScanner(bytes.NewReader(gt.Content))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
g, err := glob.Compile(line, '/')
if err != nil {
log.Info("Invalid glob expression '%s' (skipped): %v", line, err)
continue
}
gt.globs = append(gt.globs, g)
}
return gt.globs
}
func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) {
gtPath := filepath.Join(tmpDir, ".gitea", "template")
if _, err := os.Stat(gtPath); os.IsNotExist(err) {
return nil, nil
} else if err != nil {
return nil, err
}
content, err := os.ReadFile(gtPath)
if err != nil {
return nil, err
}
gt := &GiteaTemplate{
Path: gtPath,
Content: content,
}
return gt, nil
}
func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository, tmpDir string) error {
commitTimeStr := time.Now().Format(time.RFC3339)
authorSig := repo.Owner.NewGitSig()
// Because this may call hooks we should pass in the environment
env := append(os.Environ(),
"GIT_AUTHOR_NAME="+authorSig.Name,
"GIT_AUTHOR_EMAIL="+authorSig.Email,
"GIT_AUTHOR_DATE="+commitTimeStr,
"GIT_COMMITTER_NAME="+authorSig.Name,
"GIT_COMMITTER_EMAIL="+authorSig.Email,
"GIT_COMMITTER_DATE="+commitTimeStr,
)
// Clone to temporary path and do the init commit.
templateRepoPath := templateRepo.RepoPath()
if err := git.Clone(ctx, templateRepoPath, tmpDir, git.CloneRepoOptions{
Depth: 1,
Branch: templateRepo.DefaultBranch,
}); err != nil {
return fmt.Errorf("git clone: %v", err)
}
if err := util.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
return fmt.Errorf("remove git dir: %v", err)
}
// Variable expansion
gt, err := checkGiteaTemplate(tmpDir)
if err != nil {
return fmt.Errorf("checkGiteaTemplate: %v", err)
}
if gt != nil {
if err := util.Remove(gt.Path); err != nil {
return fmt.Errorf("remove .giteatemplate: %v", err)
}
// Avoid walking tree if there are no globs
if len(gt.Globs()) > 0 {
tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/"
if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error {
if walkErr != nil {
return walkErr
}
if info.IsDir() {
return nil
}
base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash)
for _, g := range gt.Globs() {
if g.Match(base) {
content, err := os.ReadFile(path)
if err != nil {
return err
}
if err := os.WriteFile(path,
[]byte(generateExpansion(string(content), templateRepo, generateRepo)),
0o644); err != nil {
return err
}
break
}
}
return nil
}); err != nil {
return err
}
}
}
if err := git.InitRepository(ctx, tmpDir, false); err != nil {
return err
}
repoPath := repo.RepoPath()
if stdout, _, err := git.NewCommand(ctx, "remote", "add", "origin", repoPath).
SetDescription(fmt.Sprintf("generateRepoCommit (git remote add): %s to %s", templateRepoPath, tmpDir)).
RunStdString(&git.RunOpts{Dir: tmpDir, Env: env}); err != nil {
log.Error("Unable to add %v as remote origin to temporary repo to %s: stdout %s\nError: %v", repo, tmpDir, stdout, err)
return fmt.Errorf("git remote add: %v", err)
}
// set default branch based on whether it's specified in the newly generated repo or not
defaultBranch := repo.DefaultBranch
if strings.TrimSpace(defaultBranch) == "" {
defaultBranch = templateRepo.DefaultBranch
}
return initRepoCommit(ctx, tmpDir, repo, repo.Owner, defaultBranch)
}
func generateGitContent(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository) (err error) {
tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name)
if err != nil {
return fmt.Errorf("Failed to create temp dir for repository %s: %v", repo.RepoPath(), err)
}
defer func() {
if err := util.RemoveAll(tmpDir); err != nil {
log.Error("RemoveAll: %v", err)
}
}()
if err = generateRepoCommit(ctx, repo, templateRepo, generateRepo, tmpDir); err != nil {
return fmt.Errorf("generateRepoCommit: %v", err)
}
// re-fetch repo
if repo, err = repo_model.GetRepositoryByIDCtx(ctx, repo.ID); err != nil {
return fmt.Errorf("getRepositoryByID: %v", err)
}
// if there was no default branch supplied when generating the repo, use the default one from the template
if strings.TrimSpace(repo.DefaultBranch) == "" {
repo.DefaultBranch = templateRepo.DefaultBranch
}
gitRepo, err := git.OpenRepository(ctx, repo.RepoPath())
if err != nil {
return fmt.Errorf("openRepository: %v", err)
}
defer gitRepo.Close()
if err = gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil {
return fmt.Errorf("setDefaultBranch: %v", err)
}
if err = UpdateRepository(ctx, repo, false); err != nil {
return fmt.Errorf("updateRepository: %v", err)
}
return nil
}
// GenerateGitContent generates git content from a template repository
func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error {
if err := generateGitContent(ctx, generateRepo, templateRepo, generateRepo); err != nil {
return err
}
if err := UpdateRepoSize(ctx, generateRepo); err != nil {
return fmt.Errorf("failed to update size for repository: %v", err)
}
if err := git_model.CopyLFS(ctx, generateRepo, templateRepo); err != nil {
return fmt.Errorf("failed to copy LFS: %v", err)
}
return nil
}
// GenerateRepoOptions contains the template units to generate
type GenerateRepoOptions struct {
Name string
DefaultBranch string
Description string
Private bool
GitContent bool
Topics bool
GitHooks bool
Webhooks bool
Avatar bool
IssueLabels bool
}
// IsValid checks whether at least one option is chosen for generation
func (gro GenerateRepoOptions) IsValid() bool {
return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels // or other items as they are added
}
// GenerateRepository generates a repository from a template
func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) {
generateRepo := &repo_model.Repository{
OwnerID: owner.ID,
Owner: owner,
OwnerName: owner.Name,
Name: opts.Name,
LowerName: strings.ToLower(opts.Name),
Description: opts.Description,
DefaultBranch: opts.DefaultBranch,
IsPrivate: opts.Private,
IsEmpty: !opts.GitContent || templateRepo.IsEmpty,
IsFsckEnabled: templateRepo.IsFsckEnabled,
TemplateID: templateRepo.ID,
Add configurable Trust Models (#11712) * Add configurable Trust Models Gitea's default signature verification model differs from GitHub. GitHub uses signatures to verify that the committer is who they say they are - meaning that when GitHub makes a signed commit it must be the committer. The GitHub model prevents re-publishing of commits after revocation of a key and prevents re-signing of other people's commits to create a completely trusted repository signed by one key or a set of trusted keys. The default behaviour of Gitea in contrast is to always display the avatar and information related to a signature. This allows signatures to be decoupled from the committer. That being said, allowing arbitary users to present other peoples commits as theirs is not necessarily desired therefore we have a trust model whereby signatures from collaborators are marked trusted, signatures matching the commit line are marked untrusted and signatures that match a user in the db but not the committer line are marked unmatched. The problem with this model is that this conflicts with Github therefore we need to provide an option to allow users to choose the Github model should they wish to. Signed-off-by: Andrew Thornton <art27@cantab.net> * Adjust locale strings Signed-off-by: Andrew Thornton <art27@cantab.net> * as per @6543 Co-authored-by: 6543 <6543@obermui.de> * Update models/gpg_key.go * Add migration for repository Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
4 years ago
TrustModel: templateRepo.TrustModel,
}
Adopt repositories (#12920) * Don't automatically delete repository files if they are present Prior to this PR Gitea would delete any repository files if they are present during creation or migration. This can in certain circumstances lead to data-loss and is slightly unpleasant. This PR provides a mechanism for Gitea to adopt repositories on creation and otherwise requires an explicit flag for deletion. PushCreate is slightly different - the create will cause adoption if that is allowed otherwise it will delete the data if that is allowed. Signed-off-by: Andrew Thornton <art27@cantab.net> * Update swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * Fix tests and migrate overwrite Signed-off-by: Andrew Thornton <art27@cantab.net> * as per @lunny Only offer to adopt or overwrite if the user can do that. Allow the site administrator to adopt or overwrite in all circumstances Signed-off-by: Andrew Thornton <art27@cantab.net> * Use setting.Repository.DefaultBranch for the default branch Signed-off-by: Andrew Thornton <art27@cantab.net> * Always set setting.Repository.DefaultBranch Signed-off-by: Andrew Thornton <art27@cantab.net> * update swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * update templates Signed-off-by: Andrew Thornton <art27@cantab.net> * ensure repo closed Signed-off-by: Andrew Thornton <art27@cantab.net> * Rewrite of adoption as per @6543 and @lunny Signed-off-by: Andrew Thornton <art27@cantab.net> * Apply suggestions from code review * update swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * missing not Signed-off-by: Andrew Thornton <art27@cantab.net> * add modals and flash reporting Signed-off-by: Andrew Thornton <art27@cantab.net> * Make the unadopted page searchable Signed-off-by: Andrew Thornton <art27@cantab.net> * Add API Signed-off-by: Andrew Thornton <art27@cantab.net> * Fix swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * fix swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * Handle empty and non-master branched repositories Signed-off-by: Andrew Thornton <art27@cantab.net> * placate lint Signed-off-by: Andrew Thornton <art27@cantab.net> * remove commented out code Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
4 years ago
if err = models.CreateRepository(ctx, doer, owner, generateRepo, false); err != nil {
return nil, err
}
Adopt repositories (#12920) * Don't automatically delete repository files if they are present Prior to this PR Gitea would delete any repository files if they are present during creation or migration. This can in certain circumstances lead to data-loss and is slightly unpleasant. This PR provides a mechanism for Gitea to adopt repositories on creation and otherwise requires an explicit flag for deletion. PushCreate is slightly different - the create will cause adoption if that is allowed otherwise it will delete the data if that is allowed. Signed-off-by: Andrew Thornton <art27@cantab.net> * Update swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * Fix tests and migrate overwrite Signed-off-by: Andrew Thornton <art27@cantab.net> * as per @lunny Only offer to adopt or overwrite if the user can do that. Allow the site administrator to adopt or overwrite in all circumstances Signed-off-by: Andrew Thornton <art27@cantab.net> * Use setting.Repository.DefaultBranch for the default branch Signed-off-by: Andrew Thornton <art27@cantab.net> * Always set setting.Repository.DefaultBranch Signed-off-by: Andrew Thornton <art27@cantab.net> * update swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * update templates Signed-off-by: Andrew Thornton <art27@cantab.net> * ensure repo closed Signed-off-by: Andrew Thornton <art27@cantab.net> * Rewrite of adoption as per @6543 and @lunny Signed-off-by: Andrew Thornton <art27@cantab.net> * Apply suggestions from code review * update swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * missing not Signed-off-by: Andrew Thornton <art27@cantab.net> * add modals and flash reporting Signed-off-by: Andrew Thornton <art27@cantab.net> * Make the unadopted page searchable Signed-off-by: Andrew Thornton <art27@cantab.net> * Add API Signed-off-by: Andrew Thornton <art27@cantab.net> * Fix swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * fix swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * Handle empty and non-master branched repositories Signed-off-by: Andrew Thornton <art27@cantab.net> * placate lint Signed-off-by: Andrew Thornton <art27@cantab.net> * remove commented out code Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
4 years ago
repoPath := generateRepo.RepoPath()
isExist, err := util.IsExist(repoPath)
if err != nil {
log.Error("Unable to check if %s exists. Error: %v", repoPath, err)
return nil, err
}
if isExist {
return nil, repo_model.ErrRepoFilesAlreadyExist{
Adopt repositories (#12920) * Don't automatically delete repository files if they are present Prior to this PR Gitea would delete any repository files if they are present during creation or migration. This can in certain circumstances lead to data-loss and is slightly unpleasant. This PR provides a mechanism for Gitea to adopt repositories on creation and otherwise requires an explicit flag for deletion. PushCreate is slightly different - the create will cause adoption if that is allowed otherwise it will delete the data if that is allowed. Signed-off-by: Andrew Thornton <art27@cantab.net> * Update swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * Fix tests and migrate overwrite Signed-off-by: Andrew Thornton <art27@cantab.net> * as per @lunny Only offer to adopt or overwrite if the user can do that. Allow the site administrator to adopt or overwrite in all circumstances Signed-off-by: Andrew Thornton <art27@cantab.net> * Use setting.Repository.DefaultBranch for the default branch Signed-off-by: Andrew Thornton <art27@cantab.net> * Always set setting.Repository.DefaultBranch Signed-off-by: Andrew Thornton <art27@cantab.net> * update swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * update templates Signed-off-by: Andrew Thornton <art27@cantab.net> * ensure repo closed Signed-off-by: Andrew Thornton <art27@cantab.net> * Rewrite of adoption as per @6543 and @lunny Signed-off-by: Andrew Thornton <art27@cantab.net> * Apply suggestions from code review * update swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * missing not Signed-off-by: Andrew Thornton <art27@cantab.net> * add modals and flash reporting Signed-off-by: Andrew Thornton <art27@cantab.net> * Make the unadopted page searchable Signed-off-by: Andrew Thornton <art27@cantab.net> * Add API Signed-off-by: Andrew Thornton <art27@cantab.net> * Fix swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * fix swagger Signed-off-by: Andrew Thornton <art27@cantab.net> * Handle empty and non-master branched repositories Signed-off-by: Andrew Thornton <art27@cantab.net> * placate lint Signed-off-by: Andrew Thornton <art27@cantab.net> * remove commented out code Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
4 years ago
Uname: generateRepo.OwnerName,
Name: generateRepo.Name,
}
}
if err = checkInitRepository(ctx, owner.Name, generateRepo.Name); err != nil {
return generateRepo, err
}
if err = CheckDaemonExportOK(ctx, generateRepo); err != nil {
return generateRepo, fmt.Errorf("checkDaemonExportOK: %v", err)
}
if stdout, _, err := git.NewCommand(ctx, "update-server-info").
SetDescription(fmt.Sprintf("GenerateRepository(git update-server-info): %s", repoPath)).
RunStdString(&git.RunOpts{Dir: repoPath}); err != nil {
log.Error("GenerateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", generateRepo, stdout, err)
return generateRepo, fmt.Errorf("error in GenerateRepository(git update-server-info): %v", err)
}
return generateRepo, nil
}