third_party
/
gitea
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Migrate temp_repo.go to use git.NewCommand (#8918)

Browse Source 
This PR migrates temp_repo.go to use git.NewCommand instead creating processes by itself - this fixes the problem underlying PR #8905.

There are other places that run git outside of the controlled locale defined in #8548 but temp_repo.go is the only cause of failure of local testing in cases where English is not the default - implying that error messages from those other commands are not interpreted.

Replaces #8905
tokarchuk/v1.17
zeripath 5 years ago committed by GitHub
parent 8d9e625f83
commit 74bb292fe3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 93 additions and 163 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 11
      modules/git/command.go
  2. 245
      modules/repofiles/temp_repo.go

11
modules/git/command.go
Unescape View File

@ -67,6 +67,13 @@ func (c *Command) RunInDirTimeoutEnvPipeline(env []string, timeout time.Duration
// RunInDirTimeoutEnvFullPipeline executes the command in given directory with given timeout, // RunInDirTimeoutEnvFullPipeline executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. // it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin.
func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader) error { func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader) error {
return c.RunInDirTimeoutEnvFullPipelineFunc(env, timeout, dir, stdout, stderr, stdin, nil)
}
// RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc)) error {
if timeout == -1 { if timeout == -1 {
timeout = DefaultCommandExecutionTimeout timeout = DefaultCommandExecutionTimeout
} }
@ -98,6 +105,10 @@ func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Dura
pid := process.GetManager().Add(fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir), cmd) pid := process.GetManager().Add(fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir), cmd)
defer process.GetManager().Remove(pid) defer process.GetManager().Remove(pid)
if fn != nil {
fn(ctx, cancel)
}
if err := cmd.Wait(); err != nil { if err := cmd.Wait(); err != nil {
return err return err
} }

245
modules/repofiles/temp_repo.go
Unescape View File

@ -10,7 +10,6 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"os/exec"
"regexp" "regexp"
"strings" "strings"
"time" "time"
@ -18,7 +17,6 @@ import (
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/gitdiff" "code.gitea.io/gitea/services/gitdiff"
@ -51,9 +49,8 @@ func (t *TemporaryUploadRepository) Close() {
// Clone the base repository to our path and set branch as the HEAD // Clone the base repository to our path and set branch as the HEAD
func (t *TemporaryUploadRepository) Clone(branch string) error { func (t *TemporaryUploadRepository) Clone(branch string) error {
if _, stderr, err := process.GetManager().ExecTimeout(5*time.Minute, if _, err := git.NewCommand("clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath).Run(); err != nil {
fmt.Sprintf("Clone (git clone -s --bare): %s", t.basePath), stderr := err.Error()
git.GitExecutable, "clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath); err != nil {
if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched { if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched {
return git.ErrBranchNotExist{ return git.ErrBranchNotExist{
Name: branch, Name: branch,
@ -79,11 +76,8 @@ func (t *TemporaryUploadRepository) Clone(branch string) error {
// SetDefaultIndex sets the git index to our HEAD // SetDefaultIndex sets the git index to our HEAD
func (t *TemporaryUploadRepository) SetDefaultIndex() error { func (t *TemporaryUploadRepository) SetDefaultIndex() error {
if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, if _, err := git.NewCommand("read-tree", "HEAD").RunInDir(t.basePath); err != nil {
t.basePath, return fmt.Errorf("SetDefaultIndex: %v", err)
fmt.Sprintf("SetDefaultIndex (git read-tree HEAD): %s", t.basePath),
git.GitExecutable, "read-tree", "HEAD"); err != nil {
return fmt.Errorf("SetDefaultIndex: %v %s", err, stderr)
} }
return nil return nil
} }
@ -93,10 +87,6 @@ func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, erro
stdOut := new(bytes.Buffer) stdOut := new(bytes.Buffer)
stdErr := new(bytes.Buffer) stdErr := new(bytes.Buffer)
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmdArgs := []string{"ls-files", "-z", "--"} cmdArgs := []string{"ls-files", "-z", "--"}
for _, arg := range filenames { for _, arg := range filenames {
if arg != "" { if arg != "" {
@ -104,22 +94,9 @@ func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, erro
} }
} }
cmd := exec.CommandContext(ctx, git.GitExecutable, cmdArgs...) if err := git.NewCommand(cmdArgs...).RunInDirPipeline(t.basePath, stdOut, stdErr); err != nil {
desc := fmt.Sprintf("lsFiles: (git ls-files) %v", cmdArgs) log.Error("Unable to run git ls-files for temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String())
cmd.Dir = t.basePath err = fmt.Errorf("Unable to run git ls-files for temporary repo of: %s Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String())
cmd.Stdout = stdOut
cmd.Stderr = stdErr
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
}
pid := process.GetManager().Add(desc, cmd)
err := cmd.Wait()
process.GetManager().Remove(pid)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
return nil, err return nil, err
} }
@ -128,7 +105,7 @@ func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, erro
filelist = append(filelist, string(line)) filelist = append(filelist, string(line))
} }
return filelist, err return filelist, nil
} }
// RemoveFilesFromIndex removes the given files from the index // RemoveFilesFromIndex removes the given files from the index
@ -144,90 +121,50 @@ func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) er
} }
} }
timeout := 5 * time.Minute if err := git.NewCommand("update-index", "--remove", "-z", "--index-info").RunInDirFullPipeline(t.basePath, stdOut, stdErr, stdIn); err != nil {
ctx, cancel := context.WithTimeout(context.Background(), timeout) log.Error("Unable to update-index for temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String())
defer cancel() return fmt.Errorf("Unable to update-index for temporary repo: %s Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String())
cmdArgs := []string{"update-index", "--remove", "-z", "--index-info"}
cmd := exec.CommandContext(ctx, git.GitExecutable, cmdArgs...)
desc := fmt.Sprintf("removeFilesFromIndex: (git update-index) %v", filenames)
cmd.Dir = t.basePath
cmd.Stdout = stdOut
cmd.Stderr = stdErr
cmd.Stdin = bytes.NewReader(stdIn.Bytes())
if err := cmd.Start(); err != nil {
return fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
}
pid := process.GetManager().Add(desc, cmd)
err := cmd.Wait()
process.GetManager().Remove(pid)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
} }
return nil
return err
} }
// HashObject writes the provided content to the object db and returns its hash // HashObject writes the provided content to the object db and returns its hash
func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) { func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) {
timeout := 5 * time.Minute stdOut := new(bytes.Buffer)
ctx, cancel := context.WithTimeout(context.Background(), timeout) stdErr := new(bytes.Buffer)
defer cancel()
hashCmd := exec.CommandContext(ctx, git.GitExecutable, "hash-object", "-w", "--stdin")
hashCmd.Dir = t.basePath
hashCmd.Stdin = content
stdOutBuffer := new(bytes.Buffer)
stdErrBuffer := new(bytes.Buffer)
hashCmd.Stdout = stdOutBuffer
hashCmd.Stderr = stdErrBuffer
desc := fmt.Sprintf("hashObject: (git hash-object)")
if err := hashCmd.Start(); err != nil {
return "", fmt.Errorf("git hash-object: %s", err)
}
pid := process.GetManager().Add(desc, hashCmd)
err := hashCmd.Wait()
process.GetManager().Remove(pid)
if err != nil { if err := git.NewCommand("hash-object", "-w", "--stdin").RunInDirFullPipeline(t.basePath, stdOut, stdErr, content); err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOutBuffer, stdErrBuffer) log.Error("Unable to hash-object to temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String())
return "", err return "", fmt.Errorf("Unable to hash-object to temporary repo: %s Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String())
} }
return strings.TrimSpace(stdOutBuffer.String()), nil return strings.TrimSpace(stdOut.String()), nil
} }
// AddObjectToIndex adds the provided object hash to the index with the provided mode and path // AddObjectToIndex adds the provided object hash to the index with the provided mode and path
func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error { func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error {
if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, if _, err := git.NewCommand("update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath).RunInDir(t.basePath); err != nil {
t.basePath, stderr := err.Error()
fmt.Sprintf("addObjectToIndex (git update-index): %s", t.basePath),
git.GitExecutable, "update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath); err != nil {
if matched, _ := regexp.MatchString(".*Invalid path '.*", stderr); matched { if matched, _ := regexp.MatchString(".*Invalid path '.*", stderr); matched {
return models.ErrFilePathInvalid{ return models.ErrFilePathInvalid{
Message: objectPath, Message: objectPath,
Path: objectPath, Path: objectPath,
} }
} }
return fmt.Errorf("git update-index: %s", stderr) log.Error("Unable to add object to index: %s %s %s in temporary repo %s(%s) Error: %v", mode, objectHash, objectPath, t.repo.FullName(), t.basePath, err)
return fmt.Errorf("Unable to add object to index at %s in temporary repo %s Error: %v", objectPath, t.repo.FullName(), err)
} }
return nil return nil
} }
// WriteTree writes the current index as a tree to the object db and returns its hash // WriteTree writes the current index as a tree to the object db and returns its hash
func (t *TemporaryUploadRepository) WriteTree() (string, error) { func (t *TemporaryUploadRepository) WriteTree() (string, error) {
treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute, stdout, err := git.NewCommand("write-tree").RunInDir(t.basePath)
t.basePath,
fmt.Sprintf("WriteTree (git write-tree): %s", t.basePath),
git.GitExecutable, "write-tree")
if err != nil { if err != nil {
return "", fmt.Errorf("git write-tree: %s", stderr) log.Error("Unable to write tree in temporary repo: %s(%s): Error: %v", t.repo.FullName(), t.basePath, err)
return "", fmt.Errorf("Unable to write-tree in temporary repo for: %s Error: %v", t.repo.FullName(), err)
} }
return strings.TrimSpace(treeHash), nil return strings.TrimSpace(stdout), nil
} }
// GetLastCommit gets the last commit ID SHA of the repo // GetLastCommit gets the last commit ID SHA of the repo
@ -240,14 +177,12 @@ func (t *TemporaryUploadRepository) GetLastCommitByRef(ref string) (string, erro
if ref == "" { if ref == "" {
ref = "HEAD" ref = "HEAD"
} }
treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute, stdout, err := git.NewCommand("rev-parse", ref).RunInDir(t.basePath)
t.basePath,
fmt.Sprintf("GetLastCommit (git rev-parse %s): %s", ref, t.basePath),
git.GitExecutable, "rev-parse", ref)
if err != nil { if err != nil {
return "", fmt.Errorf("git rev-parse %s: %s", ref, stderr) log.Error("Unable to get last ref for %s in temporary repo: %s(%s): Error: %v", ref, t.repo.FullName(), t.basePath, err)
return "", fmt.Errorf("Unable to rev-parse %s in temporary repo for: %s Error: %v", ref, t.repo.FullName(), err)
} }
return strings.TrimSpace(treeHash), nil return strings.TrimSpace(stdout), nil
} }
// CommitTree creates a commit from a given tree for the user with provided message // CommitTree creates a commit from a given tree for the user with provided message
@ -287,16 +222,15 @@ func (t *TemporaryUploadRepository) CommitTree(author, committer *models.User, t
} }
} }
commitHash, stderr, err := process.GetManager().ExecDirEnvStdIn(5*time.Minute, stdout := new(bytes.Buffer)
t.basePath, stderr := new(bytes.Buffer)
fmt.Sprintf("commitTree (git commit-tree): %s", t.basePath), if err := git.NewCommand(args...).RunInDirTimeoutEnvFullPipeline(env, -1, t.basePath, stdout, stderr, messageBytes); err != nil {
env, log.Error("Unable to commit-tree in temporary repo: %s (%s) Error: %v\nStdout: %s\nStderr: %s",
messageBytes, t.repo.FullName(), t.basePath, err, stdout, stderr)
git.GitExecutable, args...) return "", fmt.Errorf("Unable to commit-tree in temporary repo: %s Error: %v\nStdout: %s\nStderr: %s",
if err != nil { t.repo.FullName(), err, stdout, stderr)
return "", fmt.Errorf("git commit-tree: %s", stderr)
} }
return strings.TrimSpace(commitHash), nil return strings.TrimSpace(stdout.String()), nil
} }
// Push the provided commitHash to the repository branch by the provided user // Push the provided commitHash to the repository branch by the provided user
@ -304,47 +238,48 @@ func (t *TemporaryUploadRepository) Push(doer *models.User, commitHash string, b
// Because calls hooks we need to pass in the environment // Because calls hooks we need to pass in the environment
env := models.PushingEnvironment(doer, t.repo) env := models.PushingEnvironment(doer, t.repo)
if _, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, if _, err := git.NewCommand("push", t.repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)).RunInDirWithEnv(t.basePath, env); err != nil {
t.basePath, log.Error("Unable to push back to repo from temporary repo: %s (%s) Error: %v",
fmt.Sprintf("actuallyPush (git push): %s", t.basePath), t.repo.FullName(), t.basePath, err)
env, return fmt.Errorf("Unable to push back to repo from temporary repo: %s (%s) Error: %v",
git.GitExecutable, "push", t.repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)); err != nil { t.repo.FullName(), t.basePath, err)
return fmt.Errorf("git push: %s", stderr)
} }
return nil return nil
} }
// DiffIndex returns a Diff of the current index to the head // DiffIndex returns a Diff of the current index to the head
func (t *TemporaryUploadRepository) DiffIndex() (diff *gitdiff.Diff, err error) { func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) {
timeout := 5 * time.Minute stdoutReader, stdoutWriter, err := os.Pipe()
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
stdErr := new(bytes.Buffer)
cmd := exec.CommandContext(ctx, git.GitExecutable, "diff-index", "--cached", "-p", "HEAD")
cmd.Dir = t.basePath
cmd.Stderr = stdErr
stdout, err := cmd.StdoutPipe()
if err != nil { if err != nil {
return nil, fmt.Errorf("StdoutPipe: %v stderr %s", err, stdErr.String()) log.Error("Unable to open stdout pipe: %v", err)
} return nil, fmt.Errorf("Unable to open stdout pipe: %v", err)
}
if err = cmd.Start(); err != nil { defer func() {
return nil, fmt.Errorf("Start: %v stderr %s", err, stdErr.String()) _ = stdoutReader.Close()
} _ = stdoutWriter.Close()
}()
pid := process.GetManager().Add(fmt.Sprintf("diffIndex [repo_path: %s]", t.repo.RepoPath()), cmd) stderr := new(bytes.Buffer)
defer process.GetManager().Remove(pid) var diff *gitdiff.Diff
var finalErr error
diff, err = gitdiff.ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout)
if err != nil { if err := git.NewCommand("diff-index", "--cached", "-p", "HEAD").
return nil, fmt.Errorf("ParsePatch: %v", err) RunInDirTimeoutEnvFullPipelineFunc(nil, 30*time.Second, t.basePath, stdoutWriter, stderr, nil, func(ctx context.Context, cancel context.CancelFunc) {
} _ = stdoutWriter.Close()
diff, finalErr = gitdiff.ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdoutReader)
if err = cmd.Wait(); err != nil { if finalErr != nil {
return nil, fmt.Errorf("Wait: %v", err) log.Error("ParsePatch: %v", finalErr)
cancel()
}
_ = stdoutReader.Close()
}); err != nil {
if finalErr != nil {
log.Error("Unable to ParsePatch in temporary repo %s (%s). Error: %v", t.repo.FullName(), t.basePath, finalErr)
return nil, finalErr
}
log.Error("Unable to run diff-index pipeline in temporary repo %s (%s). Error: %v\nStderr: %s",
t.repo.FullName(), t.basePath, err, stderr)
return nil, fmt.Errorf("Unable to run diff-index pipeline in temporary repo %s. Error: %v\nStderr: %s",
t.repo.FullName(), err, stderr)
} }
return diff, nil return diff, nil
@ -358,12 +293,8 @@ func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...str
return nil, err return nil, err
} }
stdOut := new(bytes.Buffer) stdout := new(bytes.Buffer)
stdErr := new(bytes.Buffer) stderr := new(bytes.Buffer)
timeout := 5 * time.Minute
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmdArgs := []string{"check-attr", "-z", attribute} cmdArgs := []string{"check-attr", "-z", attribute}
@ -379,26 +310,14 @@ func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...str
} }
} }
cmd := exec.CommandContext(ctx, git.GitExecutable, cmdArgs...) if err := git.NewCommand(cmdArgs...).RunInDirPipeline(t.basePath, stdout, stderr); err != nil {
desc := fmt.Sprintf("checkAttr: (git check-attr) %s %v", attribute, cmdArgs) log.Error("Unable to check-attr in temporary repo: %s (%s) Error: %v\nStdout: %s\nStderr: %s",
cmd.Dir = t.basePath t.repo.FullName(), t.basePath, err, stdout, stderr)
cmd.Stdout = stdOut return nil, fmt.Errorf("Unable to check-attr in temporary repo: %s Error: %v\nStdout: %s\nStderr: %s",
cmd.Stderr = stdErr t.repo.FullName(), err, stdout, stderr)
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
}
pid := process.GetManager().Add(desc, cmd)
err = cmd.Wait()
process.GetManager().Remove(pid)
if err != nil {
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
return nil, err
} }
fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) fields := bytes.Split(stdout.Bytes(), []byte{'\000'})
if len(fields)%3 != 1 { if len(fields)%3 != 1 {
return nil, fmt.Errorf("Wrong number of fields in return from check-attr") return nil, fmt.Errorf("Wrong number of fields in return from check-attr")

Write Preview
Loading…
Cancel
Save