@ -5,325 +5,237 @@
package git package git
import ( import (
"bufio" "github.com/emirpasic/gods/trees/binaryheap"
"context" "gopkg.in/src-d/go-git.v4/plumbing"
"fmt" "gopkg.in/src-d/go-git.v4/plumbing/object"
"os/exec"
"path"
"runtime"
"strconv"
"strings"
"sync"
"time"
) )
const ( // GetCommitsInfo gets information of all commits that are corresponding to these entries
// parameters for searching for commit infos. If the untargeted search has
func ( tes Entries ) GetCommitsInfo ( commit * Commit , treePath string , cache LastCommitCache ) ( [ ] [ ] interface { } , * Commit , error ) {
// not found any entries in the past 5 commits, and 12 or fewer entries
entryPaths := make ( [ ] string , len ( tes ) + 1 )
// remain, then we'll just let the targeted-searching threads finish off,
// Get the commit for the treePath itself
// and stop the untargeted search to not interfere.
entryPaths [ 0 ] = ""
deferToTargetedSearchColdStreak = 5 for i , entry := range tes {
deferToTargetedSearchNumRemainingEntries = 12 entryPaths [ i + 1 ] = entry . Name ( )
) }
// getCommitsInfoState shared state while getting commit info for entries
type getCommitsInfoState struct {
lock sync . Mutex
/* read-only fields, can be read without the mutex */
// entries and entryPaths are read-only after initialization, so they can
// safely be read without the mutex
entries [ ] * TreeEntry
// set of filepaths to get info for
entryPaths map [ string ] struct { }
treePath string
headCommit * Commit
/* mutable fields, must hold mutex to read or write */ c , err := commit . repo . gogitRepo . CommitObject ( plumbing . Hash ( commit . ID ) )
// map from filepath to commit
if err != nil {
commits map [ string ] * Commit return nil , nil , err
// set of filepaths that have been or are being searched for in a target search
targetedPaths map [ string ] struct { }
} }
func ( state * getCommitsInfoState ) numRemainingEntries ( ) int { revs , err := getLastCommitForPaths ( c , treePath , entryPaths )
state . lock . Lock ( ) if err != nil {
defer state . lock . Unlock ( ) return nil , nil , err
return len ( state . entries ) - len ( state . commits )
} }
// getTargetEntryPath Returns the next path for a targeted-searching thread to
commit . repo . gogitStorage . Close ( )
// search for, or returns the empty string if nothing left to search for
func ( state * getCommitsInfoState ) getTargetedEntryPath ( ) string { commitsInfo := make ( [ ] [ ] interface { } , len ( tes ) )
var targetedEntryPath string for i , entry := range tes {
state . lock . Lock ( ) if rev , ok := revs [ entry . Name ( ) ] ; ok {
defer state . lock . Unlock ( ) entryCommit := convertCommit ( rev )
for _ , entry := range state . entries { if entry . IsSubModule ( ) {
entryPath := path . Join ( state . treePath , entry . Name ( ) ) subModuleURL := ""
if _ , ok := state . commits [ entryPath ] ; ok { if subModule , err := commit . GetSubModule ( entry . Name ( ) ) ; err != nil {
continue return nil , nil , err
} else if _ , ok = state . targetedPaths [ entryPath ] ; ok { } else if subModule != nil {
continue subModuleURL = subModule . URL
} }
targetedEntryPath = entryPath subModuleFile := NewSubModuleFile ( entryCommit , subModuleURL , entry . ID . String ( ) )
state . targetedPaths [ entryPath ] = struct { } { } commitsInfo [ i ] = [ ] interface { } { entry , subModuleFile }
break } else {
commitsInfo [ i ] = [ ] interface { } { entry , entryCommit }
}
} else {
commitsInfo [ i ] = [ ] interface { } { entry , nil }
} }
return targetedEntryPath
} }
// repeatedly perform targeted searches for unpopulated entries
// Retrieve the commit for the treePath itself (see above). We basically
func targetedSearch ( state * getCommitsInfoState , done chan error , cache LastCommitCache ) { // get it for free during the tree traversal and it's used for listing
for { // pages to display information about newest commit for a given path.
entryPath := state . getTargetedEntryPath ( ) var treeCommit * Commit
if len ( entryPath ) == 0 { if rev , ok := revs [ "" ] ; ok {
done <- nil treeCommit = convertCommit ( rev )
return
}
if cache != nil {
commit , err := cache . Get ( state . headCommit . repo . Path , state . headCommit . ID . String ( ) , entryPath )
if err == nil && commit != nil {
state . update ( entryPath , commit )
continue
} }
return commitsInfo , treeCommit , nil
} }
command := NewCommand ( "rev-list" , "-1" , state . headCommit . ID . String ( ) , "--" , entryPath )
output , err := command . RunInDir ( state . headCommit . repo . Path ) type commitAndPaths struct {
if err != nil { commit * object . Commit
done <- err // Paths that are still on the branch represented by commit
return paths [ ] string
// Set of hashes for the paths
hashes map [ string ] plumbing . Hash
} }
id , err := NewIDFromString ( strings . TrimSpace ( output ) )
func getCommitTree ( c * object . Commit , treePath string ) ( * object . Tree , error ) {
tree , err := c . Tree ( )
if err != nil { if err != nil {
done <- err return nil , err
return
} }
commit , err := state . headCommit . repo . getCommit ( id )
// Optimize deep traversals by focusing only on the specific tree
if treePath != "" {
tree , err = tree . Tree ( treePath )
if err != nil { if err != nil {
done <- err return nil , err
return
}
state . update ( entryPath , commit )
if cache != nil {
cache . Put ( state . headCommit . repo . Path , state . headCommit . ID . String ( ) , entryPath , commit )
}
} }
} }
func initGetCommitInfoState ( entries Entries , headCommit * Commit , treePath string ) * getCommitsInfoState { return tree , nil
entryPaths := make ( map [ string ] struct { } , len ( entries ) )
for _ , entry := range entries {
entryPaths [ path . Join ( treePath , entry . Name ( ) ) ] = struct { } { }
}
if treePath = path . Clean ( treePath ) ; treePath == "." {
treePath = ""
}
return & getCommitsInfoState {
entries : entries ,
entryPaths : entryPaths ,
commits : make ( map [ string ] * Commit , len ( entries ) ) ,
targetedPaths : make ( map [ string ] struct { } , len ( entries ) ) ,
treePath : treePath ,
headCommit : headCommit ,
}
} }
// GetCommitsInfo gets information of all commits that are corresponding to these entries
func getFullPath ( treePath , path string ) string {
func ( tes Entries ) GetCommitsInfo ( commit * Commit , treePath string , cache LastCommitCache ) ( [ ] [ ] interface { } , error ) { if treePath != "" {
state := initGetCommitInfoState ( tes , commit , treePath ) if path != "" {
if err := getCommitsInfo ( state , cache ) ; err != nil { return treePath + "/" + path
return nil , err
} }
if len ( state . commits ) < len ( state . entryPaths ) { return treePath
return nil , fmt . Errorf ( "could not find commits for all entries" ) }
return path
} }
commitsInfo := make ( [ ] [ ] interface { } , len ( tes ) ) func getFileHashes ( c * object . Commit , treePath string , paths [ ] string ) ( map [ string ] plumbing . Hash , error ) {
for i , entry := range tes { tree , err := getCommitTree ( c , treePath )
commit , ok := state . commits [ path . Join ( treePath , entry . Name ( ) ) ] if err == object . ErrDirectoryNotFound {
if ! ok { // The whole tree didn't exist, so return empty map
return nil , fmt . Errorf ( "could not find commit for %s" , entry . Name ( ) ) return make ( map [ string ] plumbing . Hash ) , nil
} }
switch entry . Type { if err != nil {
case ObjectCommit :
subModuleURL := ""
if subModule , err := state . headCommit . GetSubModule ( entry . Name ( ) ) ; err != nil {
return nil , err return nil , err
} else if subModule != nil {
subModuleURL = subModule . URL
}
subModuleFile := NewSubModuleFile ( commit , subModuleURL , entry . ID . String ( ) )
commitsInfo [ i ] = [ ] interface { } { entry , subModuleFile }
default :
commitsInfo [ i ] = [ ] interface { } { entry , commit }
}
}
return commitsInfo , nil
} }
func ( state * getCommitsInfoState ) cleanEntryPath ( rawEntryPath string ) ( string , error ) { hashes := make ( map [ string ] plumbing . Hash )
if rawEntryPath [ 0 ] == '"' { for _ , path := range paths {
var err error if path != "" {
rawEntryPath , err = strconv . Unquote ( rawEntryP ath) entry , err := tree . FindEntry ( path )
if err ! =nil { if err == nil {
return rawEntryPath , err hashes [ path ] = entry . Hash
} }
} else {
hashes [ path ] = tree . Hash
} }
var entryNameStartIndex int
if len ( state . treePath ) > 0 {
entryNameStartIndex = len ( state . treePath ) + 1
} }
if index := strings . IndexByte ( rawEntryPath [ entryNameStartIndex : ] , '/' ) ; index >= 0 { return hashes , nil
return rawEntryPath [ : entryNameStartIndex + index ] , nil
}
return rawEntryPath , nil
} }
// update report that the given path was last modified by the given commit.
func getLastCommitForPaths ( c * object . Commit , treePath string , paths [ ] string ) ( map [ string ] * object . Commit , error ) {
// Returns whether state.commits was updated
// We do a tree traversal with nodes sorted by commit time
func ( state * getCommitsInfoState ) update ( entryPath string , commit * Commit ) bool { seen := make ( map [ plumbing . Hash ] bool )
if _ , ok := state . entryPaths [ entryPath ] ; ! ok { heap := binaryheap . NewWith ( func ( a , b interface { } ) int {
return false if a . ( * commitAndPaths ) . commit . Committer . When . Before ( b . ( * commitAndPaths ) . commit . Committer . When ) {
return 1
} }
return - 1
} )
var updated bool result := make ( map [ string ] * object . Commit )
state . lock . Lock ( ) initialHashes , err := getFileHashes ( c , treePath , paths )
defer state . lock . Unlock ( ) if err != nil {
if _ , ok := state . commits [ entryPath ] ; ! ok { return nil , err
state . commits [ entryPath ] = commit
updated = true
}
return updated
} }
const getCommitsInfoPretty = "--pretty=format:%H %ct %s" // Start search from the root commit and with full set of paths
heap . Push ( & commitAndPaths { c , paths , initialHashes } )
func getCommitsInfo ( state * getCommitsInfoState , cache LastCommitCache ) error { for {
ctx , cancel := context . WithTimeout ( context . Background ( ) , 1 * time . Minute ) cIn , ok := heap . Pop ( )
defer cancel ( ) if ! ok {
break
}
current := cIn . ( * commitAndPaths )
currentID := current . commit . ID ( )
args := [ ] string { "log" , state . headCommit . ID . String ( ) , getCommitsInfoPretty , "--name-status" , "-c" } if seen [ currentID ] {
if len ( state . treePath ) > 0 { continue
args = append ( args , "--" , state . treePath )
} }
cmd := exec . CommandContext ( ctx , "git" , args ... ) seen [ currentID ] = true
cmd . Dir = state . headCommit . repo . Path
readCloser , err := cmd . StdoutPipe ( ) // Load the parent commits for the one we are currently examining
numParents := current . commit . NumParents ( )
var parents [ ] * object . Commit
for i := 0 ; i < numParents ; i ++ {
parent , err := current . commit . Parent ( i )
if err != nil { if err != nil {
return err break
} }
parents = append ( parents , parent )
if err := cmd . Start ( ) ; err != nil {
return err
} }
// it's okay to ignore the error returned by cmd.Wait(); we expect the
// subprocess to sometimes have a non-zero exit status, since we may
// prematurely close stdout, resulting in a broken pipe.
defer cmd . Wait ( )
numThreads := runtime . NumCPU ( ) // Examine the current commit and set of interesting paths
done := make ( chan error , numThreads ) numOfParentsWithPath := make ( [ ] int , len ( current . paths ) )
for i := 0 ; i < numThreads ; i ++ { pathChanged := make ( [ ] bool , len ( current . paths ) )
go targetedSearch ( state , done , cache ) parentHashes := make ( [ ] map [ string ] plumbing . Hash , len ( parents ) )
for j , parent := range parents {
parentHashes [ j ] , err = getFileHashes ( parent , treePath , current . paths )
if err != nil {
break
} }
scanner := bufio . NewScanner ( readCloser ) for i , path := range current . paths {
err = state . processGitLogOutput ( scanner ) if parentHashes [ j ] [ path ] != plumbing . ZeroHash {
numOfParentsWithPath [ i ] ++
// it is important that we close stdout here; if we do not close
if parentHashes [ j ] [ path ] != current . hashes [ path ] {
// stdout, the subprocess will keep running, and the deffered call
pathChanged [ i ] = true
// cmd.Wait() may block for a long time.
if closeErr := readCloser . Close ( ) ; closeErr != nil && err == nil {
err = closeErr
} }
for i := 0 ; i < numThreads ; i ++ {
doneErr := <- done
if doneErr != nil && err == nil {
err = doneErr
} }
} }
return err
} }
func ( state * getCommitsInfoState ) processGitLogOutput ( scanner * bufio . Scanner ) error { var remainingPaths [ ] string
// keep a local cache of seen paths to avoid acquiring a lock for paths
for i , path := range current . paths {
// we've already seen
switch numOfParentsWithPath [ i ] {
seenPaths := make ( map [ string ] struct { } , len ( state . entryPaths ) ) case 0 :
// number of consecutive commits without any finds
// The path didn't exist in any parent, so it must have been created by
coldStreak := 0 // this commit. The results could already contain some newer change from
var commit * Commit // different path, so don't override that.
var err error if result [ path ] == nil {
for scanner . Scan ( ) { result [ path ] = current . commit
line := scanner . Text ( )
if len ( line ) == 0 { // in-between commits
numRemainingEntries := state . numRemainingEntries ( )
if numRemainingEntries == 0 {
break
}
if coldStreak >= deferToTargetedSearchColdStreak &&
numRemainingEntries <= deferToTargetedSearchNumRemainingEntries {
// stop this untargeted search, and let the targeted-search threads
// finish the work
break
} }
continue case 1 :
// The file is present on exactly one parent, so check if it was changed
// and save the revision if it did.
if pathChanged [ i ] {
if result [ path ] == nil {
result [ path ] = current . commit
} }
if line [ 0 ] >= 'A' && line [ 0 ] <= 'X' { // a file was changed by the current commit
} else {
// look for the last tab, since for copies (C) and renames (R) two
remainingPaths = append ( remainingPaths , path )
// filenames are printed: src, then dest
tabIndex := strings . LastIndexByte ( line , '\t' )
if tabIndex < 1 {
return fmt . Errorf ( "misformatted line: %s" , line )
} }
entryPath , err := state . cleanEntryPath ( line [ tabIndex + 1 : ] ) default :
if err != nil { // The file is present on more than one of the parent paths, so this is
return err // a merge. We have to examine all the parent trees to find out where
} // the change occurred. pathChanged[i] would tell us that the file was
if _ , ok := seenPaths [ entryPath ] ; ! ok { // changed during the merge, but it wouldn't tell us the relevant commit
if state . update ( entryPath , commit ) { // that introduced it.
coldStreak = 0 remainingPaths = append ( remainingPaths , path )
} }
seenPaths [ entryPath ] = struct { } { }
} }
if len ( remainingPaths ) > 0 {
// Add the parent nodes along with remaining paths to the heap for further
// processing.
for j , parent := range parents {
if seen [ parent . ID ( ) ] {
continue continue
} }
// a new commit
// Combine remainingPath with paths available on the parent branch
commit , err = parseCommitInfo ( line ) // and make union of them
if err != nil { var remainingPathsForParent [ ] string
return err for _ , path := range remainingPaths {
} if parentHashes [ j ] [ path ] != plumbing . ZeroHash {
coldStreak ++ remainingPathsForParent = append ( remainingPathsForParent , path )
} }
return scanner . Err ( )
} }
// parseCommitInfo parse a commit from a line of `git log` output. Expects the
heap . Push ( & commitAndPaths { parent , remainingPathsForParent , parentHashes [ j ] } )
// line to be formatted according to getCommitsInfoPretty.
func parseCommitInfo ( line string ) ( * Commit , error ) {
if len ( line ) < 43 {
return nil , fmt . Errorf ( "invalid git output: %s" , line )
} }
ref , err := NewIDFromString ( line [ : 40 ] )
if err != nil {
return nil , err
} }
spaceIndex := strings . IndexByte ( line [ 41 : ] , ' ' )
if spaceIndex < 0 {
return nil , fmt . Errorf ( "invalid git output: %s" , line )
} }
unixSeconds , err := strconv . Atoi ( line [ 41 : 41 + spaceIndex ] )
if err != nil { return result , nil
return nil , err
}
message := line [ spaceIndex + 42 : ]
return & Commit {
ID : ref ,
CommitMessage : message ,
Committer : & Signature {
When : time . Unix ( int64 ( unixSeconds ) , 0 ) ,
} ,
} , nil
} }
Filip Navara
6 years ago
committed by
Lunny Xiao