third_party
/
gitea
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Migrate temp_repo.go to use git.NewCommand (#8918)

Browse Source 
This PR migrates temp_repo.go to use git.NewCommand instead creating processes by itself - this fixes the problem underlying PR #8905.

There are other places that run git outside of the controlled locale defined in #8548 but temp_repo.go is the only cause of failure of local testing in cases where English is not the default - implying that error messages from those other commands are not interpreted.

Replaces #8905
tokarchuk/v1.17
zeripath 5 years ago committed by GitHub
parent 8d9e625f83
commit 74bb292fe3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 93 additions and 163 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 11
      modules/git/command.go
  2. 245
      modules/repofiles/temp_repo.go

11
modules/git/command.go
Unescape View File

@ -67,6 +67,13 @@ func (c *Command) RunInDirTimeoutEnvPipeline(env []string, timeout time.Duration
// RunInDirTimeoutEnvFullPipeline executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin.
func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader) error {
return c.RunInDirTimeoutEnvFullPipelineFunc(env, timeout, dir, stdout, stderr, stdin, nil)
}
// RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc)) error {
if timeout == -1 {
timeout = DefaultCommandExecutionTimeout
}
@ -98,6 +105,10 @@ func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Dura
pid := process.GetManager().Add(fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir), cmd)
defer process.GetManager().Remove(pid)
if fn != nil {
fn(ctx, cancel)
}
if err := cmd.Wait(); err != nil {
return err
}

245
modules/repofiles/temp_repo.go
Unescape View File

@ -10,7 +10,6 @@ import (
"fmt"
"io"
"os"
"os/exec"
"regexp"
"strings"
"time"
@ -18,7 +17,6 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/gitdiff"
@ -51,9 +49,8 @@ func (t *TemporaryUploadRepository) Close() {
// Clone the base repository to our path and set branch as the HEAD
func (t *TemporaryUploadRepository) Clone(branch string) error {
if _, stderr, err := process.GetManager().ExecTimeout(5*time.Minute,
fmt.Sprintf("Clone (git clone -s --bare): %s", t.basePath),
git.GitExecutable, "clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath); err != nil {
if _, err := git.NewCommand("clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath).Run(); err != nil {
stderr := err.Error()
if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched {
return git.ErrBranchNotExist{
Name: branch,
@ -79,11 +76,8 @@ func (t *TemporaryUploadRepository) Clone(branch string) error {
// SetDefaultIndex sets the git index to our HEAD
func (t *TemporaryUploadRepository) SetDefaultIndex() error {
if _, stderr, err := process.GetManager().ExecDir(5*time.Minute,
t.basePath,
fmt.Sprintf("SetDefaultIndex (git read-tree HEAD): %s", t.basePath),
git.GitExecutable, "read-tree", "HEAD"); err != nil {
return fmt.Errorf("SetDefaultIndex: %v %s", err, stderr)
if _, err := git.NewCommand("read-tree", "HEAD").RunInDir(t.basePath); err != nil {
return fmt.Errorf("SetDefaultIndex: %v", err)
}
return nil
}
@ -93,10 +87,6 @@ func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, erro
stdOut := new(bytes.Buffer)
stdErr := new(bytes.Buffer)
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmdArgs := []string{"ls-files", "-z", "--"}
for _, arg := range filenames {
if arg != "" {
@ -104,22 +94,9 @@ func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, erro
}
}
cmd := exec.CommandContext(ctx, git.GitExecutable, cmdArgs...)
desc := fmt.Sprintf("lsFiles: (git ls-files) %v", cmdArgs)
cmd.Dir = t.basePath
cmd.Stdout = stdOut
cmd.Stderr = stdErr
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
}
pid := process.GetManager().Add(desc, cmd)
err := cmd.Wait()
process.GetManager().Remove(pid)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
if err := git.NewCommand(cmdArgs...).RunInDirPipeline(t.basePath, stdOut, stdErr); err != nil {
log.Error("Unable to run git ls-files for temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String())
err = fmt.Errorf("Unable to run git ls-files for temporary repo of: %s Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String())
return nil, err
}
@ -128,7 +105,7 @@ func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, erro
filelist = append(filelist, string(line))
}
return filelist, err
return filelist, nil
}
// RemoveFilesFromIndex removes the given files from the index
@ -144,90 +121,50 @@ func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) er
}
}
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmdArgs := []string{"update-index", "--remove", "-z", "--index-info"}
cmd := exec.CommandContext(ctx, git.GitExecutable, cmdArgs...)
desc := fmt.Sprintf("removeFilesFromIndex: (git update-index) %v", filenames)
cmd.Dir = t.basePath
cmd.Stdout = stdOut
cmd.Stderr = stdErr
cmd.Stdin = bytes.NewReader(stdIn.Bytes())
if err := cmd.Start(); err != nil {
return fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
}
pid := process.GetManager().Add(desc, cmd)
err := cmd.Wait()
process.GetManager().Remove(pid)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
if err := git.NewCommand("update-index", "--remove", "-z", "--index-info").RunInDirFullPipeline(t.basePath, stdOut, stdErr, stdIn); err != nil {
log.Error("Unable to update-index for temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String())
return fmt.Errorf("Unable to update-index for temporary repo: %s Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String())
}
return err
return nil
}
// HashObject writes the provided content to the object db and returns its hash
func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) {
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
hashCmd := exec.CommandContext(ctx, git.GitExecutable, "hash-object", "-w", "--stdin")
hashCmd.Dir = t.basePath
hashCmd.Stdin = content
stdOutBuffer := new(bytes.Buffer)
stdErrBuffer := new(bytes.Buffer)
hashCmd.Stdout = stdOutBuffer
hashCmd.Stderr = stdErrBuffer
desc := fmt.Sprintf("hashObject: (git hash-object)")
if err := hashCmd.Start(); err != nil {
return "", fmt.Errorf("git hash-object: %s", err)
}
pid := process.GetManager().Add(desc, hashCmd)
err := hashCmd.Wait()
process.GetManager().Remove(pid)
stdOut := new(bytes.Buffer)
stdErr := new(bytes.Buffer)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOutBuffer, stdErrBuffer)
return "", err
if err := git.NewCommand("hash-object", "-w", "--stdin").RunInDirFullPipeline(t.basePath, stdOut, stdErr, content); err != nil {
log.Error("Unable to hash-object to temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String())
return "", fmt.Errorf("Unable to hash-object to temporary repo: %s Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String())
}
return strings.TrimSpace(stdOutBuffer.String()), nil
return strings.TrimSpace(stdOut.String()), nil
}
// AddObjectToIndex adds the provided object hash to the index with the provided mode and path
func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error {
if _, stderr, err := process.GetManager().ExecDir(5*time.Minute,
t.basePath,
fmt.Sprintf("addObjectToIndex (git update-index): %s", t.basePath),
git.GitExecutable, "update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath); err != nil {
if _, err := git.NewCommand("update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath).RunInDir(t.basePath); err != nil {
stderr := err.Error()
if matched, _ := regexp.MatchString(".*Invalid path '.*", stderr); matched {
return models.ErrFilePathInvalid{
Message: objectPath,
Path: objectPath,
}
}
return fmt.Errorf("git update-index: %s", stderr)
log.Error("Unable to add object to index: %s %s %s in temporary repo %s(%s) Error: %v", mode, objectHash, objectPath, t.repo.FullName(), t.basePath, err)
return fmt.Errorf("Unable to add object to index at %s in temporary repo %s Error: %v", objectPath, t.repo.FullName(), err)
}
return nil
}
// WriteTree writes the current index as a tree to the object db and returns its hash
func (t *TemporaryUploadRepository) WriteTree() (string, error) {
treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute,
t.basePath,
fmt.Sprintf("WriteTree (git write-tree): %s", t.basePath),
git.GitExecutable, "write-tree")
stdout, err := git.NewCommand("write-tree").RunInDir(t.basePath)
if err != nil {
return "", fmt.Errorf("git write-tree: %s", stderr)
log.Error("Unable to write tree in temporary repo: %s(%s): Error: %v", t.repo.FullName(), t.basePath, err)
return "", fmt.Errorf("Unable to write-tree in temporary repo for: %s Error: %v", t.repo.FullName(), err)
}
return strings.TrimSpace(treeHash), nil
return strings.TrimSpace(stdout), nil
}
// GetLastCommit gets the last commit ID SHA of the repo
@ -240,14 +177,12 @@ func (t *TemporaryUploadRepository) GetLastCommitByRef(ref string) (string, erro
if ref == "" {
ref = "HEAD"
}
treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute,
t.basePath,
fmt.Sprintf("GetLastCommit (git rev-parse %s): %s", ref, t.basePath),
git.GitExecutable, "rev-parse", ref)
stdout, err := git.NewCommand("rev-parse", ref).RunInDir(t.basePath)
if err != nil {
return "", fmt.Errorf("git rev-parse %s: %s", ref, stderr)
log.Error("Unable to get last ref for %s in temporary repo: %s(%s): Error: %v", ref, t.repo.FullName(), t.basePath, err)
return "", fmt.Errorf("Unable to rev-parse %s in temporary repo for: %s Error: %v", ref, t.repo.FullName(), err)
}
return strings.TrimSpace(treeHash), nil
return strings.TrimSpace(stdout), nil
}
// CommitTree creates a commit from a given tree for the user with provided message
@ -287,16 +222,15 @@ func (t *TemporaryUploadRepository) CommitTree(author, committer *models.User, t
}
}
commitHash, stderr, err := process.GetManager().ExecDirEnvStdIn(5*time.Minute,
t.basePath,
fmt.Sprintf("commitTree (git commit-tree): %s", t.basePath),
env,
messageBytes,
git.GitExecutable, args...)
if err != nil {
return "", fmt.Errorf("git commit-tree: %s", stderr)
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
if err := git.NewCommand(args...).RunInDirTimeoutEnvFullPipeline(env, -1, t.basePath, stdout, stderr, messageBytes); err != nil {
log.Error("Unable to commit-tree in temporary repo: %s (%s) Error: %v\nStdout: %s\nStderr: %s",
t.repo.FullName(), t.basePath, err, stdout, stderr)
return "", fmt.Errorf("Unable to commit-tree in temporary repo: %s Error: %v\nStdout: %s\nStderr: %s",
t.repo.FullName(), err, stdout, stderr)
}
return strings.TrimSpace(commitHash), nil
return strings.TrimSpace(stdout.String()), nil
}
// Push the provided commitHash to the repository branch by the provided user
@ -304,47 +238,48 @@ func (t *TemporaryUploadRepository) Push(doer *models.User, commitHash string, b
// Because calls hooks we need to pass in the environment
env := models.PushingEnvironment(doer, t.repo)
if _, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute,
t.basePath,
fmt.Sprintf("actuallyPush (git push): %s", t.basePath),
env,
git.GitExecutable, "push", t.repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)); err != nil {
return fmt.Errorf("git push: %s", stderr)
if _, err := git.NewCommand("push", t.repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)).RunInDirWithEnv(t.basePath, env); err != nil {
log.Error("Unable to push back to repo from temporary repo: %s (%s) Error: %v",
t.repo.FullName(), t.basePath, err)
return fmt.Errorf("Unable to push back to repo from temporary repo: %s (%s) Error: %v",
t.repo.FullName(), t.basePath, err)
}
return nil
}
// DiffIndex returns a Diff of the current index to the head
func (t *TemporaryUploadRepository) DiffIndex() (diff *gitdiff.Diff, err error) {
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
stdErr := new(bytes.Buffer)
cmd := exec.CommandContext(ctx, git.GitExecutable, "diff-index", "--cached", "-p", "HEAD")
cmd.Dir = t.basePath
cmd.Stderr = stdErr
stdout, err := cmd.StdoutPipe()
func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) {
stdoutReader, stdoutWriter, err := os.Pipe()
if err != nil {
return nil, fmt.Errorf("StdoutPipe: %v stderr %s", err, stdErr.String())
}
if err = cmd.Start(); err != nil {
return nil, fmt.Errorf("Start: %v stderr %s", err, stdErr.String())
}
pid := process.GetManager().Add(fmt.Sprintf("diffIndex [repo_path: %s]", t.repo.RepoPath()), cmd)
defer process.GetManager().Remove(pid)
diff, err = gitdiff.ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout)
if err != nil {
return nil, fmt.Errorf("ParsePatch: %v", err)
}
if err = cmd.Wait(); err != nil {
return nil, fmt.Errorf("Wait: %v", err)
log.Error("Unable to open stdout pipe: %v", err)
return nil, fmt.Errorf("Unable to open stdout pipe: %v", err)
}
defer func() {
_ = stdoutReader.Close()
_ = stdoutWriter.Close()
}()
stderr := new(bytes.Buffer)
var diff *gitdiff.Diff
var finalErr error
if err := git.NewCommand("diff-index", "--cached", "-p", "HEAD").
RunInDirTimeoutEnvFullPipelineFunc(nil, 30*time.Second, t.basePath, stdoutWriter, stderr, nil, func(ctx context.Context, cancel context.CancelFunc) {
_ = stdoutWriter.Close()
diff, finalErr = gitdiff.ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdoutReader)
if finalErr != nil {
log.Error("ParsePatch: %v", finalErr)
cancel()
}
_ = stdoutReader.Close()
}); err != nil {
if finalErr != nil {
log.Error("Unable to ParsePatch in temporary repo %s (%s). Error: %v", t.repo.FullName(), t.basePath, finalErr)
return nil, finalErr
}
log.Error("Unable to run diff-index pipeline in temporary repo %s (%s). Error: %v\nStderr: %s",
t.repo.FullName(), t.basePath, err, stderr)
return nil, fmt.Errorf("Unable to run diff-index pipeline in temporary repo %s. Error: %v\nStderr: %s",
t.repo.FullName(), err, stderr)
}
return diff, nil
@ -358,12 +293,8 @@ func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...str
return nil, err
}
stdOut := new(bytes.Buffer)
stdErr := new(bytes.Buffer)
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
cmdArgs := []string{"check-attr", "-z", attribute}
@ -379,26 +310,14 @@ func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...str
}
}
cmd := exec.CommandContext(ctx, git.GitExecutable, cmdArgs...)
desc := fmt.Sprintf("checkAttr: (git check-attr) %s %v", attribute, cmdArgs)
cmd.Dir = t.basePath
cmd.Stdout = stdOut
cmd.Stderr = stdErr
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
}
pid := process.GetManager().Add(desc, cmd)
err = cmd.Wait()
process.GetManager().Remove(pid)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
return nil, err
if err := git.NewCommand(cmdArgs...).RunInDirPipeline(t.basePath, stdout, stderr); err != nil {
log.Error("Unable to check-attr in temporary repo: %s (%s) Error: %v\nStdout: %s\nStderr: %s",
t.repo.FullName(), t.basePath, err, stdout, stderr)
return nil, fmt.Errorf("Unable to check-attr in temporary repo: %s Error: %v\nStdout: %s\nStderr: %s",
t.repo.FullName(), err, stdout, stderr)
}
fields := bytes.Split(stdOut.Bytes(), []byte{'\000'})
fields := bytes.Split(stdout.Bytes(), []byte{'\000'})
if len(fields)%3 != 1 {
return nil, fmt.Errorf("Wrong number of fields in return from check-attr")

Write Preview
Loading…
Cancel
Save