Slightly simplify LastCommitCache (#20444)

The LastCommitCache code is a little complex and there is unnecessary
duplication between the gogit and nogogit variants.

This PR adds the LastCommitCache as a field to the git.Repository and
pre-creates it in the ReferencesGit helpers etc. There has been some
simplification and unification of the variant code.

Signed-off-by: Andrew Thornton <art27@cantab.net>
tokarchuk/v1.18
zeripath 2 years ago committed by GitHub
parent 690272d2e2
commit a2cfcdb91a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      modules/context/repo.go
  2. 3
      modules/git/commit.go
  3. 31
      modules/git/commit_info_gogit.go
  4. 21
      modules/git/commit_info_nogogit.go
  5. 4
      modules/git/commit_info_test.go
  6. 87
      modules/git/last_commit_cache.go
  7. 62
      modules/git/last_commit_cache_gogit.go
  8. 63
      modules/git/last_commit_cache_nogogit.go
  9. 6
      modules/git/log_name_status.go
  10. 2
      modules/git/notes_gogit.go
  11. 2
      modules/git/notes_nogogit.go
  12. 5
      modules/git/repo_base_gogit.go
  13. 5
      modules/git/repo_base_nogogit.go
  14. 8
      routers/api/v1/repo/file.go
  15. 24
      routers/api/v1/utils/git.go
  16. 8
      routers/web/repo/download.go
  17. 8
      routers/web/repo/view.go
  18. 2
      routers/web/web.go
  19. 16
      services/repository/cache.go

@ -1001,6 +1001,8 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
return return
} }
ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount
ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(ctx.Repo.CommitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache())
return cancel return cancel
} }
} }

@ -80,6 +80,9 @@ func (c *Commit) ParentCount() int {
// GetCommitByPath return the commit of relative path object. // GetCommitByPath return the commit of relative path object.
func (c *Commit) GetCommitByPath(relpath string) (*Commit, error) { func (c *Commit) GetCommitByPath(relpath string) (*Commit, error) {
if c.repo.LastCommitCache != nil {
return c.repo.LastCommitCache.GetCommitByPath(c.ID.String(), relpath)
}
return c.repo.getCommitByPathWithID(c.ID, relpath) return c.repo.getCommitByPathWithID(c.ID, relpath)
} }

@ -17,7 +17,7 @@ import (
) )
// GetCommitsInfo gets information of all commits that are corresponding to these entries // GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) {
entryPaths := make([]string, len(tes)+1) entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself // Get the commit for the treePath itself
entryPaths[0] = "" entryPaths[0] = ""
@ -35,15 +35,15 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
return nil, nil, err return nil, nil, err
} }
var revs map[string]*object.Commit var revs map[string]*Commit
if cache != nil { if commit.repo.LastCommitCache != nil {
var unHitPaths []string var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
if len(unHitPaths) > 0 { if len(unHitPaths) > 0 {
revs2, err := GetLastCommitForPaths(ctx, cache, c, treePath, unHitPaths) revs2, err := GetLastCommitForPaths(ctx, commit.repo.LastCommitCache, c, treePath, unHitPaths)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -68,8 +68,7 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
} }
// Check if we have found a commit for this entry in time // Check if we have found a commit for this entry in time
if rev, ok := revs[entry.Name()]; ok { if entryCommit, ok := revs[entry.Name()]; ok {
entryCommit := convertCommit(rev)
commitsInfo[i].Commit = entryCommit commitsInfo[i].Commit = entryCommit
} }
@ -96,10 +95,10 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
// get it for free during the tree traversal and it's used for listing // get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path. // pages to display information about newest commit for a given path.
var treeCommit *Commit var treeCommit *Commit
var ok bool
if treePath == "" { if treePath == "" {
treeCommit = commit treeCommit = commit
} else if rev, ok := revs[""]; ok { } else if treeCommit, ok = revs[""]; ok {
treeCommit = convertCommit(rev)
treeCommit.repo = commit.repo treeCommit.repo = commit.repo
} }
return commitsInfo, treeCommit, nil return commitsInfo, treeCommit, nil
@ -155,16 +154,16 @@ func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[
return hashes, nil return hashes, nil
} }
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) { func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) {
var unHitEntryPaths []string var unHitEntryPaths []string
results := make(map[string]*object.Commit) results := make(map[string]*Commit)
for _, p := range paths { for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
if lastCommit != nil { if lastCommit != nil {
results[p] = lastCommit.(*object.Commit) results[p] = lastCommit
continue continue
} }
@ -175,7 +174,7 @@ func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cac
} }
// GetLastCommitForPaths returns last commit information // GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) { func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*Commit, error) {
refSha := c.ID().String() refSha := c.ID().String()
// We do a tree traversal with nodes sorted by commit time // We do a tree traversal with nodes sorted by commit time
@ -293,13 +292,13 @@ heaploop:
} }
// Post-processing // Post-processing
result := make(map[string]*object.Commit) result := make(map[string]*Commit)
for path, commitNode := range resultNodes { for path, commitNode := range resultNodes {
var err error commit, err := commitNode.Commit()
result[path], err = commitNode.Commit()
if err != nil { if err != nil {
return nil, err return nil, err
} }
result[path] = convertCommit(commit)
} }
return result, nil return result, nil

@ -17,7 +17,7 @@ import (
) )
// GetCommitsInfo gets information of all commits that are corresponding to these entries // GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) {
entryPaths := make([]string, len(tes)+1) entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself // Get the commit for the treePath itself
entryPaths[0] = "" entryPaths[0] = ""
@ -28,15 +28,15 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
var err error var err error
var revs map[string]*Commit var revs map[string]*Commit
if cache != nil { if commit.repo.LastCommitCache != nil {
var unHitPaths []string var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, cache) revs, unHitPaths, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
if len(unHitPaths) > 0 { if len(unHitPaths) > 0 {
sort.Strings(unHitPaths) sort.Strings(unHitPaths)
commits, err := GetLastCommitForPaths(ctx, cache, commit, treePath, unHitPaths) commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@ -47,7 +47,7 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
} }
} else { } else {
sort.Strings(entryPaths) sort.Strings(entryPaths)
revs, err = GetLastCommitForPaths(ctx, nil, commit, treePath, entryPaths) revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths)
} }
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
@ -99,18 +99,15 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
} }
func getLastCommitForPathsByCache(ctx context.Context, commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) { func getLastCommitForPathsByCache(ctx context.Context, commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) {
wr, rd, cancel := cache.repo.CatFileBatch(ctx)
defer cancel()
var unHitEntryPaths []string var unHitEntryPaths []string
results := make(map[string]*Commit) results := make(map[string]*Commit)
for _, p := range paths { for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p), wr, rd) lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
if lastCommit != nil { if lastCommit != nil {
results[p] = lastCommit.(*Commit) results[p] = lastCommit
continue continue
} }
@ -121,9 +118,9 @@ func getLastCommitForPathsByCache(ctx context.Context, commitID, treePath string
} }
// GetLastCommitForPaths returns last commit information // GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) { func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) {
// We read backwards from the commit to obtain all of the commits // We read backwards from the commit to obtain all of the commits
revs, err := WalkGitLog(ctx, cache, commit.repo, commit, treePath, paths...) revs, err := WalkGitLog(ctx, commit.repo, commit, treePath, paths...)
if err != nil { if err != nil {
return nil, err return nil, err
} }

@ -91,7 +91,7 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) {
} }
// FIXME: Context.TODO() - if graceful has started we should use its Shutdown context otherwise use install signals in TestMain. // FIXME: Context.TODO() - if graceful has started we should use its Shutdown context otherwise use install signals in TestMain.
commitsInfo, treeCommit, err := entries.GetCommitsInfo(context.TODO(), commit, testCase.Path, nil) commitsInfo, treeCommit, err := entries.GetCommitsInfo(context.TODO(), commit, testCase.Path)
assert.NoError(t, err, "Unable to get commit information for entries of subtree: %s in commit: %s from testcase due to error: %v", testCase.Path, testCase.CommitID, err) assert.NoError(t, err, "Unable to get commit information for entries of subtree: %s in commit: %s from testcase due to error: %v", testCase.Path, testCase.CommitID, err)
if err != nil { if err != nil {
t.FailNow() t.FailNow()
@ -170,7 +170,7 @@ func BenchmarkEntries_GetCommitsInfo(b *testing.B) {
b.ResetTimer() b.ResetTimer()
b.Run(benchmark.name, func(b *testing.B) { b.Run(benchmark.name, func(b *testing.B) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, _, err := entries.GetCommitsInfo(context.Background(), commit, "", nil) _, _, err := entries.GetCommitsInfo(context.Background(), commit, "")
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }

@ -9,6 +9,7 @@ import (
"fmt" "fmt"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
) )
// Cache represents a caching interface // Cache represents a caching interface
@ -19,16 +20,96 @@ type Cache interface {
Get(key string) interface{} Get(key string) interface{}
} }
func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string { func getCacheKey(repoPath, commitID, entryPath string) string {
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath))) hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, commitID, entryPath)))
return fmt.Sprintf("last_commit:%x", hashBytes) return fmt.Sprintf("last_commit:%x", hashBytes)
} }
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl func() int64
repo *Repository
commitCache map[string]*Commit
cache Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(count int64, repoPath string, gitRepo *Repository, cache Cache) *LastCommitCache {
if cache == nil {
return nil
}
if !setting.CacheService.LastCommit.Enabled || count < setting.CacheService.LastCommit.CommitsCount {
return nil
}
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
ttl: setting.LastCommitCacheTTLSeconds,
cache: cache,
}
}
// Put put the last commit id with commit and entry path // Put put the last commit id with commit and entry path
func (c *LastCommitCache) Put(ref, entryPath, commitID string) error { func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
if c == nil || c.cache == nil { if c == nil || c.cache == nil {
return nil return nil
} }
log.Debug("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID) log.Debug("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl()) return c.cache.Put(getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl())
}
// Get gets the last commit information by commit id and entry path
func (c *LastCommitCache) Get(ref, entryPath string) (*Commit, error) {
if c == nil || c.cache == nil {
return nil, nil
}
commitID, ok := c.cache.Get(getCacheKey(c.repoPath, ref, entryPath)).(string)
if !ok || commitID == "" {
return nil, nil
}
log.Debug("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, commitID)
if c.commitCache != nil {
if commit, ok := c.commitCache[commitID]; ok {
log.Debug("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, commitID)
return commit, nil
}
}
commit, err := c.repo.GetCommit(commitID)
if err != nil {
return nil, err
}
if c.commitCache == nil {
c.commitCache = make(map[string]*Commit)
}
c.commitCache[commitID] = commit
return commit, nil
}
// GetCommitByPath gets the last commit for the entry in the provided commit
func (c *LastCommitCache) GetCommitByPath(commitID, entryPath string) (*Commit, error) {
sha1, err := NewIDFromString(commitID)
if err != nil {
return nil, err
}
lastCommit, err := c.Get(sha1.String(), entryPath)
if err != nil || lastCommit != nil {
return lastCommit, err
}
lastCommit, err = c.repo.getCommitByPathWithID(sha1, entryPath)
if err != nil {
return nil, err
}
if err := c.Put(commitID, entryPath, lastCommit.ID.String()); err != nil {
log.Error("Unable to cache %s as the last commit for %q in %s %s. Error %v", lastCommit.ID.String(), entryPath, commitID, c.repoPath, err)
}
return lastCommit, nil
} }

@ -9,71 +9,25 @@ package git
import ( import (
"context" "context"
"code.gitea.io/gitea/modules/log"
"github.com/go-git/go-git/v5/plumbing/object"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
) )
// LastCommitCache represents a cache to store last commit // CacheCommit will cache the commit from the gitRepository
type LastCommitCache struct { func (c *Commit) CacheCommit(ctx context.Context) error {
repoPath string if c.repo.LastCommitCache == nil {
ttl func() int64
repo *Repository
commitCache map[string]*object.Commit
cache Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64, cache Cache) *LastCommitCache {
if cache == nil {
return nil return nil
} }
return &LastCommitCache{ commitNodeIndex, _ := c.repo.CommitNodeIndex()
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*object.Commit),
ttl: ttl,
cache: cache,
}
}
// Get get the last commit information by commit id and entry path
func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) {
v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log.Debug("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log.Debug("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
if err != nil {
return nil, err
}
commit, err := c.repo.GoGitRepo().CommitObject(id)
if err != nil {
return nil, err
}
c.commitCache[vs] = commit
return commit, nil
}
return nil, nil
}
// CacheCommit will cache the commit from the gitRepository
func (c *LastCommitCache) CacheCommit(ctx context.Context, commit *Commit) error {
commitNodeIndex, _ := commit.repo.CommitNodeIndex()
index, err := commitNodeIndex.Get(commit.ID) index, err := commitNodeIndex.Get(c.ID)
if err != nil { if err != nil {
return err return err
} }
return c.recursiveCache(ctx, index, &commit.Tree, "", 1) return c.recursiveCache(ctx, index, &c.Tree, "", 1)
} }
func (c *LastCommitCache) recursiveCache(ctx context.Context, index cgobject.CommitNode, tree *Tree, treePath string, level int) error { func (c *Commit) recursiveCache(ctx context.Context, index cgobject.CommitNode, tree *Tree, treePath string, level int) error {
if level == 0 { if level == 0 {
return nil return nil
} }
@ -90,7 +44,7 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, index cgobject.Com
entryMap[entry.Name()] = entry entryMap[entry.Name()] = entry
} }
commits, err := GetLastCommitForPaths(ctx, c, index, treePath, entryPaths) commits, err := GetLastCommitForPaths(ctx, c.repo.LastCommitCache, index, treePath, entryPaths)
if err != nil { if err != nil {
return err return err
} }

@ -7,67 +7,18 @@
package git package git
import ( import (
"bufio"
"context" "context"
"code.gitea.io/gitea/modules/log"
) )
// LastCommitCache represents a cache to store last commit // CacheCommit will cache the commit from the gitRepository
type LastCommitCache struct { func (c *Commit) CacheCommit(ctx context.Context) error {
repoPath string if c.repo.LastCommitCache == nil {
ttl func() int64
repo *Repository
commitCache map[string]*Commit
cache Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64, cache Cache) *LastCommitCache {
if cache == nil {
return nil return nil
} }
return &LastCommitCache{ return c.recursiveCache(ctx, &c.Tree, "", 1)
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*Commit),
ttl: ttl,
cache: cache,
}
}
// Get get the last commit information by commit id and entry path
func (c *LastCommitCache) Get(ref, entryPath string, wr WriteCloserError, rd *bufio.Reader) (interface{}, error) {
v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log.Debug("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log.Debug("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
if err != nil {
return nil, err
}
if _, err := wr.Write([]byte(vs + "\n")); err != nil {
return nil, err
}
commit, err := c.repo.getCommitFromBatchReader(rd, id)
if err != nil {
return nil, err
}
c.commitCache[vs] = commit
return commit, nil
}
return nil, nil
}
// CacheCommit will cache the commit from the gitRepository
func (c *LastCommitCache) CacheCommit(ctx context.Context, commit *Commit) error {
return c.recursiveCache(ctx, commit, &commit.Tree, "", 1)
} }
func (c *LastCommitCache) recursiveCache(ctx context.Context, commit *Commit, tree *Tree, treePath string, level int) error { func (c *Commit) recursiveCache(ctx context.Context, tree *Tree, treePath string, level int) error {
if level == 0 { if level == 0 {
return nil return nil
} }
@ -82,7 +33,7 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, commit *Commit, tr
entryPaths[i] = entry.Name() entryPaths[i] = entry.Name()
} }
_, err = WalkGitLog(ctx, c, commit.repo, commit, treePath, entryPaths...) _, err = WalkGitLog(ctx, c.repo, c, treePath, entryPaths...)
if err != nil { if err != nil {
return err return err
} }
@ -94,7 +45,7 @@ func (c *LastCommitCache) recursiveCache(ctx context.Context, commit *Commit, tr
if err != nil { if err != nil {
return err return err
} }
if err := c.recursiveCache(ctx, commit, subTree, treeEntry.Name(), level-1); err != nil { if err := c.recursiveCache(ctx, subTree, treeEntry.Name(), level-1); err != nil {
return err return err
} }
} }

@ -281,7 +281,7 @@ func (g *LogNameStatusRepoParser) Close() {
} }
// WalkGitLog walks the git log --name-status for the head commit in the provided treepath and files // WalkGitLog walks the git log --name-status for the head commit in the provided treepath and files
func WalkGitLog(ctx context.Context, cache *LastCommitCache, repo *Repository, head *Commit, treepath string, paths ...string) (map[string]string, error) { func WalkGitLog(ctx context.Context, repo *Repository, head *Commit, treepath string, paths ...string) (map[string]string, error) {
headRef := head.ID.String() headRef := head.ID.String()
tree, err := head.SubTree(treepath) tree, err := head.SubTree(treepath)
@ -374,14 +374,14 @@ heaploop:
changed[i] = false changed[i] = false
if results[i] == "" { if results[i] == "" {
results[i] = current.CommitID results[i] = current.CommitID
if err := cache.Put(headRef, path.Join(treepath, paths[i]), current.CommitID); err != nil { if err := repo.LastCommitCache.Put(headRef, path.Join(treepath, paths[i]), current.CommitID); err != nil {
return nil, err return nil, err
} }
delete(path2idx, paths[i]) delete(path2idx, paths[i])
remaining-- remaining--
if results[0] == "" { if results[0] == "" {
results[0] = current.CommitID results[0] = current.CommitID
if err := cache.Put(headRef, treepath, current.CommitID); err != nil { if err := repo.LastCommitCache.Put(headRef, treepath, current.CommitID); err != nil {
return nil, err return nil, err
} }
delete(path2idx, "") delete(path2idx, "")

@ -83,7 +83,7 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note)
log.Error("Unable to get the commit for the path %q. Error: %v", path, err) log.Error("Unable to get the commit for the path %q. Error: %v", path, err)
return err return err
} }
note.Commit = convertCommit(lastCommits[path]) note.Commit = lastCommits[path]
return nil return nil
} }

@ -81,7 +81,7 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note)
path = path[idx+1:] path = path[idx+1:]
} }
lastCommits, err := GetLastCommitForPaths(ctx, nil, notes, treePath, []string{path}) lastCommits, err := GetLastCommitForPaths(ctx, notes, treePath, []string{path})
if err != nil { if err != nil {
log.Error("Unable to get the commit for the path %q. Error: %v", treePath, err) log.Error("Unable to get the commit for the path %q. Error: %v", treePath, err)
return err return err

@ -31,7 +31,8 @@ type Repository struct {
gogitStorage *filesystem.Storage gogitStorage *filesystem.Storage
gpgSettings *GPGSettings gpgSettings *GPGSettings
Ctx context.Context Ctx context.Context
LastCommitCache *LastCommitCache
} }
// openRepositoryWithDefaultContext opens the repository at the given path with DefaultContext. // openRepositoryWithDefaultContext opens the repository at the given path with DefaultContext.
@ -79,6 +80,8 @@ func (repo *Repository) Close() (err error) {
if err := repo.gogitStorage.Close(); err != nil { if err := repo.gogitStorage.Close(); err != nil {
gitealog.Error("Error closing storage: %v", err) gitealog.Error("Error closing storage: %v", err)
} }
repo.LastCommitCache = nil
repo.tagCache = nil
return return
} }

@ -32,7 +32,8 @@ type Repository struct {
checkReader *bufio.Reader checkReader *bufio.Reader
checkWriter WriteCloserError checkWriter WriteCloserError
Ctx context.Context Ctx context.Context
LastCommitCache *LastCommitCache
} }
// openRepositoryWithDefaultContext opens the repository at the given path with DefaultContext. // openRepositoryWithDefaultContext opens the repository at the given path with DefaultContext.
@ -101,5 +102,7 @@ func (repo *Repository) Close() (err error) {
repo.checkReader = nil repo.checkReader = nil
repo.checkWriter = nil repo.checkWriter = nil
} }
repo.LastCommitCache = nil
repo.tagCache = nil
return err return err
} }

@ -18,7 +18,6 @@ import (
git_model "code.gitea.io/gitea/models/git" git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo" repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/models/unit"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/httpcache"
@ -240,12 +239,7 @@ func getBlobForEntry(ctx *context.APIContext) (blob *git.Blob, entry *git.TreeEn
return return
} }
var c *git.LastCommitCache info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:])
if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount {
c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache())
}
info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:], c)
if err != nil { if err != nil {
ctx.Error(http.StatusInternalServerError, "GetCommitsInfo", err) ctx.Error(http.StatusInternalServerError, "GetCommitsInfo", err)
return return

@ -8,8 +8,10 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
) )
// ResolveRefOrSha resolve ref to sha if exist // ResolveRefOrSha resolve ref to sha if exist
@ -19,6 +21,7 @@ func ResolveRefOrSha(ctx *context.APIContext, ref string) string {
return "" return ""
} }
sha := ref
// Search branches and tags // Search branches and tags
for _, refType := range []string{"heads", "tags"} { for _, refType := range []string{"heads", "tags"} {
refSHA, lastMethodName, err := searchRefCommitByType(ctx, refType, ref) refSHA, lastMethodName, err := searchRefCommitByType(ctx, refType, ref)
@ -27,10 +30,27 @@ func ResolveRefOrSha(ctx *context.APIContext, ref string) string {
return "" return ""
} }
if refSHA != "" { if refSHA != "" {
return refSHA sha = refSHA
break
} }
} }
return ref
if ctx.Repo.GitRepo != nil && ctx.Repo.GitRepo.LastCommitCache == nil {
commitsCount, err := cache.GetInt64(ctx.Repo.Repository.GetCommitsCountCacheKey(ref, true), func() (int64, error) {
commit, err := ctx.Repo.GitRepo.GetCommit(sha)
if err != nil {
return 0, err
}
return commit.CommitsCount()
})
if err != nil {
log.Error("Unable to get commits count for %s in %s. Error: %v", sha, ctx.Repo.Repository.FullName(), err)
return sha
}
ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(commitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache())
}
return sha
} }
// GetGitRefs return git references based on filter // GetGitRefs return git references based on filter

@ -10,7 +10,6 @@ import (
"time" "time"
git_model "code.gitea.io/gitea/models/git" git_model "code.gitea.io/gitea/models/git"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/httpcache"
@ -99,12 +98,7 @@ func getBlobForEntry(ctx *context.Context) (blob *git.Blob, lastModified time.Ti
return return
} }
var c *git.LastCommitCache info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:])
if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount {
c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache())
}
info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:], c)
if err != nil { if err != nil {
ctx.ServerError("GetCommitsInfo", err) ctx.ServerError("GetCommitsInfo", err)
return return

@ -27,7 +27,6 @@ import (
unit_model "code.gitea.io/gitea/models/unit" unit_model "code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
@ -812,11 +811,6 @@ func renderDirectoryFiles(ctx *context.Context, timeout time.Duration) git.Entri
defer cancel() defer cancel()
} }
var c *git.LastCommitCache
if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount {
c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache())
}
selected := map[string]bool{} selected := map[string]bool{}
for _, pth := range ctx.FormStrings("f[]") { for _, pth := range ctx.FormStrings("f[]") {
selected[pth] = true selected[pth] = true
@ -833,7 +827,7 @@ func renderDirectoryFiles(ctx *context.Context, timeout time.Duration) git.Entri
} }
var latestCommit *git.Commit var latestCommit *git.Commit
ctx.Data["Files"], latestCommit, err = entries.GetCommitsInfo(commitInfoCtx, ctx.Repo.Commit, ctx.Repo.TreePath, c) ctx.Data["Files"], latestCommit, err = entries.GetCommitsInfo(commitInfoCtx, ctx.Repo.Commit, ctx.Repo.TreePath)
if err != nil { if err != nil {
ctx.ServerError("GetCommitsInfo", err) ctx.ServerError("GetCommitsInfo", err)
return nil return nil

@ -12,6 +12,7 @@ import (
"code.gitea.io/gitea/models/perm" "code.gitea.io/gitea/models/perm"
"code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/models/unit"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/httpcache"
@ -1011,6 +1012,7 @@ func RegisterRoutes(m *web.Route) {
return return
} }
ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount
ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(ctx.Repo.CommitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache())
}) })
}, ignSignIn, context.RepoAssignment, context.UnitTypes(), reqRepoReleaseReader) }, ignSignIn, context.RepoAssignment, context.UnitTypes(), reqRepoReleaseReader)

@ -34,15 +34,13 @@ func CacheRef(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Rep
return err return err
} }
commitsCount, err := cache.GetInt64(repo.GetCommitsCountCacheKey(getRefName(fullRefName), true), commit.CommitsCount) if gitRepo.LastCommitCache == nil {
if err != nil { commitsCount, err := cache.GetInt64(repo.GetCommitsCountCacheKey(getRefName(fullRefName), true), commit.CommitsCount)
return err if err != nil {
return err
}
gitRepo.LastCommitCache = git.NewLastCommitCache(commitsCount, repo.FullName(), gitRepo, cache.GetCache())
} }
if commitsCount < setting.CacheService.LastCommit.CommitsCount {
return nil
}
commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache())
return commitCache.CacheCommit(ctx, commit) return commit.CacheCommit(ctx)
} }

Loading…
Cancel
Save