@ -5,325 +5,237 @@
package git package git
import ( import (
"bufio" "github.com/emirpasic/gods/trees/binaryheap"
"context" "gopkg.in/src-d/go-git.v4/plumbing"
"fmt" "gopkg.in/src-d/go-git.v4/plumbing/object"
"os/exec"
"path"
"runtime"
"strconv"
"strings"
"sync"
"time"
) )
const ( // GetCommitsInfo gets information of all commits that are corresponding to these entries
// parameters for searching for commit infos. If the untargeted search has
func ( tes Entries ) GetCommitsInfo ( commit * Commit , treePath string , cache LastCommitCache ) ( [ ] [ ] interface { } , * Commit , error ) {
// not found any entries in the past 5 commits, and 12 or fewer entries
entryPaths := make ( [ ] string , len ( tes ) + 1 )
// remain, then we'll just let the targeted-searching threads finish off,
// Get the commit for the treePath itself
// and stop the untargeted search to not interfere.
entryPaths [ 0 ] = ""
deferToTargetedSearchColdStreak = 5 for i , entry := range tes {
deferToTargetedSearchNumRemainingEntries = 12 entryPaths [ i + 1 ] = entry . Name ( )
)
// getCommitsInfoState shared state while getting commit info for entries
type getCommitsInfoState struct {
lock sync . Mutex
/* read-only fields, can be read without the mutex */
// entries and entryPaths are read-only after initialization, so they can
// safely be read without the mutex
entries [ ] * TreeEntry
// set of filepaths to get info for
entryPaths map [ string ] struct { }
treePath string
headCommit * Commit
/* mutable fields, must hold mutex to read or write */
// map from filepath to commit
commits map [ string ] * Commit
// set of filepaths that have been or are being searched for in a target search
targetedPaths map [ string ] struct { }
}
func ( state * getCommitsInfoState ) numRemainingEntries ( ) int {
state . lock . Lock ( )
defer state . lock . Unlock ( )
return len ( state . entries ) - len ( state . commits )
}
// getTargetEntryPath Returns the next path for a targeted-searching thread to
// search for, or returns the empty string if nothing left to search for
func ( state * getCommitsInfoState ) getTargetedEntryPath ( ) string {
var targetedEntryPath string
state . lock . Lock ( )
defer state . lock . Unlock ( )
for _ , entry := range state . entries {
entryPath := path . Join ( state . treePath , entry . Name ( ) )
if _ , ok := state . commits [ entryPath ] ; ok {
continue
} else if _ , ok = state . targetedPaths [ entryPath ] ; ok {
continue
}
targetedEntryPath = entryPath
state . targetedPaths [ entryPath ] = struct { } { }
break
} }
return targetedEntryPath
}
// repeatedly perform targeted searches for unpopulated entries
c , err := commit . repo . gogitRepo . CommitObject ( plumbing . Hash ( commit . ID ) )
func targetedSearch ( state * getCommitsInfoState , done chan error , cache LastCommitCache ) {
for {
entryPath := state . getTargetedEntryPath ( )
if len ( entryPath ) == 0 {
done <- nil
return
}
if cache != nil {
commit , err := cache . Get ( state . headCommit . repo . Path , state . headCommit . ID . String ( ) , entryPath )
if err == nil && commit != nil {
state . update ( entryPath , commit )
continue
}
}
command := NewCommand ( "rev-list" , "-1" , state . headCommit . ID . String ( ) , "--" , entryPath )
output , err := command . RunInDir ( state . headCommit . repo . Path )
if err != nil {
done <- err
return
}
id , err := NewIDFromString ( strings . TrimSpace ( output ) )
if err != nil {
done <- err
return
}
commit , err := state . headCommit . repo . getCommit ( id )
if err != nil { if err != nil {
done <- err return nil , nil , err
return
} }
state . update ( entryPath , commit )
if cache != nil {
cache . Put ( state . headCommit . repo . Path , state . headCommit . ID . String ( ) , entryPath , commit )
}
}
}
func initGetCommitInfoState ( entries Entries , headCommit * Commit , treePath string ) * getCommitsInfoState { revs , err := getLastCommitForPaths ( c , treePath , entryPaths )
entryPaths := make ( map [ string ] struct { } , len ( entries ) ) if err != nil {
for _ , entry := range entries { return nil , nil , err
entryPaths [ path . Join ( treePath , entry . Name ( ) ) ] = struct { } { }
}
if treePath = path . Clean ( treePath ) ; treePath == "." {
treePath = ""
}
return & getCommitsInfoState {
entries : entries ,
entryPaths : entryPaths ,
commits : make ( map [ string ] * Commit , len ( entries ) ) ,
targetedPaths : make ( map [ string ] struct { } , len ( entries ) ) ,
treePath : treePath ,
headCommit : headCommit ,
} }
}
// GetCommitsInfo gets information of all commits that are corresponding to these entries
commit . repo . gogitStorage . Close ( )
func ( tes Entries ) GetCommitsInfo ( commit * Commit , treePath string , cache LastCommitCache ) ( [ ] [ ] interface { } , error ) {
state := initGetCommitInfoState ( tes , commit , treePath )
if err := getCommitsInfo ( state , cache ) ; err != nil {
return nil , err
}
if len ( state . commits ) < len ( state . entryPaths ) {
return nil , fmt . Errorf ( "could not find commits for all entries" )
}
commitsInfo := make ( [ ] [ ] interface { } , len ( tes ) ) commitsInfo := make ( [ ] [ ] interface { } , len ( tes ) )
for i , entry := range tes { for i , entry := range tes {
commit , ok := state . commits [ path . Join ( treePath , entry . Name ( ) ) ] if rev , ok := revs [ entry . Name ( ) ] ; ok {
if ! ok { entryCommit := convertCommit ( rev )
return nil , fmt . Errorf ( "could not find commit for %s" , entry . Name ( ) ) if entry . IsSubModule ( ) {
}
switch entry . Type {
case ObjectCommit :
subModuleURL := "" subModuleURL := ""
if subModule , err := state . headC ommit. GetSubModule ( entry . Name ( ) ) ; err != nil { if subModule , err := commit . GetSubModule ( entry . Name ( ) ) ; err != nil {
return nil , err return nil , nil , err
} else if subModule != nil { } else if subModule != nil {
subModuleURL = subModule . URL subModuleURL = subModule . URL
} }
subModuleFile := NewSubModuleFile ( c ommit, subModuleURL , entry . ID . String ( ) ) subModuleFile := NewSubModuleFile ( entryCommit , subModuleURL , entry . ID . String ( ) )
commitsInfo [ i ] = [ ] interface { } { entry , subModuleFile } commitsInfo [ i ] = [ ] interface { } { entry , subModuleFile }
default : } else {
commitsInfo [ i ] = [ ] interface { } { entry , commit } commitsInfo [ i ] = [ ] interface { } { entry , entryCommit }
}
} else {
commitsInfo [ i ] = [ ] interface { } { entry , nil }
} }
} }
return commitsInfo , nil
// Retrieve the commit for the treePath itself (see above). We basically
// get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path.
var treeCommit * Commit
if rev , ok := revs [ "" ] ; ok {
treeCommit = convertCommit ( rev )
}
return commitsInfo , treeCommit , nil
} }
func ( state * getCommitsInfoState ) cleanEntryPath ( rawEntryPath string ) ( string , error ) { type commitAndPaths struct {
if rawEntryPath [ 0 ] == '"' { commit * object . Commit
var err error // Paths that are still on the branch represented by commit
rawEntryPath , err = strconv . Unquote ( rawEntryPath ) paths [ ] string
// Set of hashes for the paths
hashes map [ string ] plumbing . Hash
}
func getCommitTree ( c * object . Commit , treePath string ) ( * object . Tree , error ) {
tree , err := c . Tree ( )
if err != nil { if err != nil {
return rawEntryPath , err return nil , err
} }
// Optimize deep traversals by focusing only on the specific tree
if treePath != "" {
tree , err = tree . Tree ( treePath )
if err != nil {
return nil , err
} }
var entryNameStartIndex int
if len ( state . treePath ) > 0 {
entryNameStartIndex = len ( state . treePath ) + 1
} }
if index := strings . IndexByte ( rawEntryPath [ entryNameStartIndex : ] , '/' ) ; index >= 0 { return tree , nil
return rawEntryPath [ : entryNameStartIndex + index ] , nil
}
return rawEntryPath , nil
} }
// update report that the given path was last modified by the given commit.
func getFullPath ( treePath , path string ) string {
// Returns whether state.commits was updated
if treePath != "" {
func ( state * getCommitsInfoState ) update ( entryPath string , commit * Commit ) bool { if path != "" {
if _ , ok := state . entryPaths [ entryPath ] ; ! ok { return treePath + "/" + path
return false
} }
return treePath
var updated bool }
state . lock . Lock ( ) return path
defer state . lock . Unlock ( )
if _ , ok := state . commits [ entryPath ] ; ! ok {
state . commits [ entryPath ] = commit
updated = true
}
return updated
} }
const getCommitsInfoPretty = "--pretty=format:%H %ct %s" func getFileHashes ( c * object . Commit , treePath string , paths [ ] string ) ( map [ string ] plumbing . Hash , error ) {
tree , err := getCommitTree ( c , treePath )
if err == object . ErrDirectoryNotFound {
// The whole tree didn't exist, so return empty map
return make ( map [ string ] plumbing . Hash ) , nil
}
if err != nil {
return nil , err
}
func getCommitsInfo ( state * getCommitsInfoState , cache LastCommitCache ) error { hashes := make ( map [ string ] plumbing . Hash )
ctx , cancel := context . WithTimeout ( context . Background ( ) , 1 * time . Minute ) for _ , path := range paths {
defer cancel ( ) if path != "" {
entry , err := tree . FindEntry ( path )
if err == nil {
hashes [ path ] = entry . Hash
}
} else {
hashes [ path ] = tree . Hash
}
}
args := [ ] string { "log" , state . headCommit . ID . String ( ) , getCommitsInfoPretty , "--name-status" , "-c" } return hashes , nil
if len ( state . treePath ) > 0 { }
args = append ( args , "--" , state . treePath )
func getLastCommitForPaths ( c * object . Commit , treePath string , paths [ ] string ) ( map [ string ] * object . Commit , error ) {
// We do a tree traversal with nodes sorted by commit time
seen := make ( map [ plumbing . Hash ] bool )
heap := binaryheap . NewWith ( func ( a , b interface { } ) int {
if a . ( * commitAndPaths ) . commit . Committer . When . Before ( b . ( * commitAndPaths ) . commit . Committer . When ) {
return 1
} }
cmd := exec . CommandContext ( ctx , "git" , args ... ) return - 1
cmd . Dir = state . headCommit . repo . Path } )
readCloser , err := cmd . StdoutPipe ( ) result := make ( map [ string ] * object . Commit )
initialHashes , err := getFileHashes ( c , treePath , paths )
if err != nil { if err != nil {
return err return nil , err
} }
if err := cmd . Start ( ) ; err != nil { // Start search from the root commit and with full set of paths
return err heap . Push ( & commitAndPaths { c , paths , initialHashes } )
}
// it's okay to ignore the error returned by cmd.Wait(); we expect the
// subprocess to sometimes have a non-zero exit status, since we may
// prematurely close stdout, resulting in a broken pipe.
defer cmd . Wait ( )
numThreads := runtime . NumCPU ( ) for {
done := make ( chan error , numThreads ) cIn , ok := heap . Pop ( )
for i := 0 ; i < numThreads ; i ++ { if ! ok {
go targetedSearch ( state , done , cache ) break
} }
current := cIn . ( * commitAndPaths )
currentID := current . commit . ID ( )
scanner := bufio . NewScanner ( readCloser ) if seen [ currentID ] {
err = state . processGitLogOutput ( scanner ) continue
// it is important that we close stdout here; if we do not close
// stdout, the subprocess will keep running, and the deffered call
// cmd.Wait() may block for a long time.
if closeErr := readCloser . Close ( ) ; closeErr != nil && err == nil {
err = closeErr
} }
seen [ currentID ] = true
for i := 0 ; i < numThreads ; i ++ { // Load the parent commits for the one we are currently examining
doneErr := <- done numParents := current . commit . NumParents ( )
if doneErr != nil && err == nil { var parents [ ] * object . Commit
err = doneErr for i := 0 ; i < numParents ; i ++ {
parent , err := current . commit . Parent ( i )
if err != nil {
break
} }
parents = append ( parents , parent )
} }
return err
}
func ( state * getCommitsInfoState ) processGitLogOutput ( scanner * bufio . Scanner ) error { // Examine the current commit and set of interesting paths
// keep a local cache of seen paths to avoid acquiring a lock for paths
numOfParentsWithPath := make ( [ ] int , len ( current . paths ) )
// we've already seen
pathChanged := make ( [ ] bool , len ( current . paths ) )
seenPaths := make ( map [ string ] struct { } , len ( state . entryPaths ) ) parentHashes := make ( [ ] map [ string ] plumbing . Hash , len ( parents ) )
// number of consecutive commits without any finds
for j , parent := range parents {
coldStreak := 0 parentHashes [ j ] , err = getFileHashes ( parent , treePath , current . paths )
var commit * Commit if err != nil {
var err error
for scanner . Scan ( ) {
line := scanner . Text ( )
if len ( line ) == 0 { // in-between commits
numRemainingEntries := state . numRemainingEntries ( )
if numRemainingEntries == 0 {
break break
} }
if coldStreak >= deferToTargetedSearchColdStreak &&
numRemainingEntries <= deferToTargetedSearchNumRemainingEntries { for i , path := range current . paths {
// stop this untargeted search, and let the targeted-search threads
if parentHashes [ j ] [ path ] != plumbing . ZeroHash {
// finish the work
numOfParentsWithPath [ i ] ++
break if parentHashes [ j ] [ path ] != current . hashes [ path ] {
pathChanged [ i ] = true
} }
continue
} }
if line [ 0 ] >= 'A' && line [ 0 ] <= 'X' { // a file was changed by the current commit
// look for the last tab, since for copies (C) and renames (R) two
// filenames are printed: src, then dest
tabIndex := strings . LastIndexByte ( line , '\t' )
if tabIndex < 1 {
return fmt . Errorf ( "misformatted line: %s" , line )
} }
entryPath , err := state . cleanEntryPath ( line [ tabIndex + 1 : ] )
if err != nil {
return err
} }
if _ , ok := seenPaths [ entryPath ] ; ! ok {
if state . update ( entryPath , commit ) { var remainingPaths [ ] string
coldStreak = 0 for i , path := range current . paths {
switch numOfParentsWithPath [ i ] {
case 0 :
// The path didn't exist in any parent, so it must have been created by
// this commit. The results could already contain some newer change from
// different path, so don't override that.
if result [ path ] == nil {
result [ path ] = current . commit
}
case 1 :
// The file is present on exactly one parent, so check if it was changed
// and save the revision if it did.
if pathChanged [ i ] {
if result [ path ] == nil {
result [ path ] = current . commit
} }
seenPaths [ entryPath ] = struct { } { } } else {
remainingPaths = append ( remainingPaths , path )
}
default :
// The file is present on more than one of the parent paths, so this is
// a merge. We have to examine all the parent trees to find out where
// the change occurred. pathChanged[i] would tell us that the file was
// changed during the merge, but it wouldn't tell us the relevant commit
// that introduced it.
remainingPaths = append ( remainingPaths , path )
} }
}
if len ( remainingPaths ) > 0 {
// Add the parent nodes along with remaining paths to the heap for further
// processing.
for j , parent := range parents {
if seen [ parent . ID ( ) ] {
continue continue
} }
// a new commit
// Combine remainingPath with paths available on the parent branch
commit , err = parseCommitInfo ( line ) // and make union of them
if err != nil { var remainingPathsForParent [ ] string
return err for _ , path := range remainingPaths {
if parentHashes [ j ] [ path ] != plumbing . ZeroHash {
remainingPathsForParent = append ( remainingPathsForParent , path )
} }
coldStreak ++
} }
return scanner . Err ( )
}
// parseCommitInfo parse a commit from a line of `git log` output. Expects the
heap . Push ( & commitAndPaths { parent , remainingPathsForParent , parentHashes [ j ] } )
// line to be formatted according to getCommitsInfoPretty.
func parseCommitInfo ( line string ) ( * Commit , error ) {
if len ( line ) < 43 {
return nil , fmt . Errorf ( "invalid git output: %s" , line )
}
ref , err := NewIDFromString ( line [ : 40 ] )
if err != nil {
return nil , err
} }
spaceIndex := strings . IndexByte ( line [ 41 : ] , ' ' )
if spaceIndex < 0 {
return nil , fmt . Errorf ( "invalid git output: %s" , line )
} }
unixSeconds , err := strconv . Atoi ( line [ 41 : 41 + spaceIndex ] )
if err != nil {
return nil , err
} }
message := line [ spaceIndex + 42 : ]
return & Commit { return result , nil
ID : ref ,
CommitMessage : message ,
Committer : & Signature {
When : time . Unix ( int64 ( unixSeconds ) , 0 ) ,
} ,
} , nil
} }
Filip Navara
6 years ago
committed by
Lunny Xiao