Add basic repository lfs management (#7199)
This PR adds basic repository LFS management UI including the ability to find all possible pointers within the repository. Locks are not managed at present but would be addable through some simple additions. * Add basic repository lfs management * add auto-associate function * Add functionality to find commits with this lfs file * Add link to find commits on the lfs file view * Adjust commit view to state the likely branch causing the commit * Only read Oid from databasetokarchuk/v1.17
zeripath
5 years ago
committed by
GitHub
parent
af8957bc4c
commit
5e6a008fba
@ -0,0 +1,94 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package pipeline |
||||
|
||||
import ( |
||||
"bufio" |
||||
"bytes" |
||||
"fmt" |
||||
"io" |
||||
"strconv" |
||||
"strings" |
||||
"sync" |
||||
|
||||
"code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/log" |
||||
) |
||||
|
||||
// CatFileBatchCheck runs cat-file with --batch-check
|
||||
func CatFileBatchCheck(shasToCheckReader *io.PipeReader, catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { |
||||
defer wg.Done() |
||||
defer shasToCheckReader.Close() |
||||
defer catFileCheckWriter.Close() |
||||
|
||||
stderr := new(bytes.Buffer) |
||||
var errbuf strings.Builder |
||||
cmd := git.NewCommand("cat-file", "--batch-check") |
||||
if err := cmd.RunInDirFullPipeline(tmpBasePath, catFileCheckWriter, stderr, shasToCheckReader); err != nil { |
||||
_ = catFileCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check [%s]: %v - %s", tmpBasePath, err, errbuf.String())) |
||||
} |
||||
} |
||||
|
||||
// CatFileBatchCheckAllObjects runs cat-file with --batch-check --batch-all
|
||||
func CatFileBatchCheckAllObjects(catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string, errChan chan<- error) { |
||||
defer wg.Done() |
||||
defer catFileCheckWriter.Close() |
||||
|
||||
stderr := new(bytes.Buffer) |
||||
var errbuf strings.Builder |
||||
cmd := git.NewCommand("cat-file", "--batch-check", "--batch-all-objects") |
||||
if err := cmd.RunInDirPipeline(tmpBasePath, catFileCheckWriter, stderr); err != nil { |
||||
log.Error("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String()) |
||||
err = fmt.Errorf("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String()) |
||||
_ = catFileCheckWriter.CloseWithError(err) |
||||
errChan <- err |
||||
} |
||||
} |
||||
|
||||
// CatFileBatch runs cat-file --batch
|
||||
func CatFileBatch(shasToBatchReader *io.PipeReader, catFileBatchWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { |
||||
defer wg.Done() |
||||
defer shasToBatchReader.Close() |
||||
defer catFileBatchWriter.Close() |
||||
|
||||
stderr := new(bytes.Buffer) |
||||
var errbuf strings.Builder |
||||
if err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(tmpBasePath, catFileBatchWriter, stderr, shasToBatchReader); err != nil { |
||||
_ = shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())) |
||||
} |
||||
} |
||||
|
||||
// BlobsLessThan1024FromCatFileBatchCheck reads a pipeline from cat-file --batch-check and returns the blobs <1024 in size
|
||||
func BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) { |
||||
defer wg.Done() |
||||
defer catFileCheckReader.Close() |
||||
scanner := bufio.NewScanner(catFileCheckReader) |
||||
defer func() { |
||||
_ = shasToBatchWriter.CloseWithError(scanner.Err()) |
||||
}() |
||||
for scanner.Scan() { |
||||
line := scanner.Text() |
||||
if len(line) == 0 { |
||||
continue |
||||
} |
||||
fields := strings.Split(line, " ") |
||||
if len(fields) < 3 || fields[1] != "blob" { |
||||
continue |
||||
} |
||||
size, _ := strconv.Atoi(fields[2]) |
||||
if size > 1024 { |
||||
continue |
||||
} |
||||
toWrite := []byte(fields[0] + "\n") |
||||
for len(toWrite) > 0 { |
||||
n, err := shasToBatchWriter.Write(toWrite) |
||||
if err != nil { |
||||
_ = catFileCheckReader.CloseWithError(err) |
||||
break |
||||
} |
||||
toWrite = toWrite[n:] |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,28 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package pipeline |
||||
|
||||
import ( |
||||
"bytes" |
||||
"fmt" |
||||
"io" |
||||
"strings" |
||||
"sync" |
||||
|
||||
"code.gitea.io/gitea/modules/git" |
||||
) |
||||
|
||||
// NameRevStdin runs name-rev --stdin
|
||||
func NameRevStdin(shasToNameReader *io.PipeReader, nameRevStdinWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) { |
||||
defer wg.Done() |
||||
defer shasToNameReader.Close() |
||||
defer nameRevStdinWriter.Close() |
||||
|
||||
stderr := new(bytes.Buffer) |
||||
var errbuf strings.Builder |
||||
if err := git.NewCommand("name-rev", "--stdin", "--name-only", "--always").RunInDirFullPipeline(tmpBasePath, nameRevStdinWriter, stderr, shasToNameReader); err != nil { |
||||
_ = shasToNameReader.CloseWithError(fmt.Errorf("git name-rev [%s]: %v - %s", tmpBasePath, err, errbuf.String())) |
||||
} |
||||
} |
||||
@ -0,0 +1,75 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package pipeline |
||||
|
||||
import ( |
||||
"bufio" |
||||
"bytes" |
||||
"fmt" |
||||
"io" |
||||
"strings" |
||||
"sync" |
||||
|
||||
"code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/log" |
||||
) |
||||
|
||||
// RevListAllObjects runs rev-list --objects --all and writes to a pipewriter
|
||||
func RevListAllObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, basePath string, errChan chan<- error) { |
||||
defer wg.Done() |
||||
defer revListWriter.Close() |
||||
|
||||
stderr := new(bytes.Buffer) |
||||
var errbuf strings.Builder |
||||
cmd := git.NewCommand("rev-list", "--objects", "--all") |
||||
if err := cmd.RunInDirPipeline(basePath, revListWriter, stderr); err != nil { |
||||
log.Error("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String()) |
||||
err = fmt.Errorf("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String()) |
||||
_ = revListWriter.CloseWithError(err) |
||||
errChan <- err |
||||
} |
||||
} |
||||
|
||||
// RevListObjects run rev-list --objects from headSHA to baseSHA
|
||||
func RevListObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath, headSHA, baseSHA string, errChan chan<- error) { |
||||
defer wg.Done() |
||||
defer revListWriter.Close() |
||||
stderr := new(bytes.Buffer) |
||||
var errbuf strings.Builder |
||||
cmd := git.NewCommand("rev-list", "--objects", headSHA, "--not", baseSHA) |
||||
if err := cmd.RunInDirPipeline(tmpBasePath, revListWriter, stderr); err != nil { |
||||
log.Error("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String()) |
||||
errChan <- fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String()) |
||||
} |
||||
} |
||||
|
||||
// BlobsFromRevListObjects reads a RevListAllObjects and only selects blobs
|
||||
func BlobsFromRevListObjects(revListReader *io.PipeReader, shasToCheckWriter *io.PipeWriter, wg *sync.WaitGroup) { |
||||
defer wg.Done() |
||||
defer revListReader.Close() |
||||
scanner := bufio.NewScanner(revListReader) |
||||
defer func() { |
||||
_ = shasToCheckWriter.CloseWithError(scanner.Err()) |
||||
}() |
||||
for scanner.Scan() { |
||||
line := scanner.Text() |
||||
if len(line) == 0 { |
||||
continue |
||||
} |
||||
fields := strings.Split(line, " ") |
||||
if len(fields) < 2 || len(fields[1]) == 0 { |
||||
continue |
||||
} |
||||
toWrite := []byte(fields[0] + "\n") |
||||
for len(toWrite) > 0 { |
||||
n, err := shasToCheckWriter.Write(toWrite) |
||||
if err != nil { |
||||
_ = revListReader.CloseWithError(err) |
||||
break |
||||
} |
||||
toWrite = toWrite[n:] |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,551 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package repo |
||||
|
||||
import ( |
||||
"bufio" |
||||
"bytes" |
||||
"fmt" |
||||
gotemplate "html/template" |
||||
"io" |
||||
"io/ioutil" |
||||
"os" |
||||
"path/filepath" |
||||
"sort" |
||||
"strconv" |
||||
"strings" |
||||
"sync" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"code.gitea.io/gitea/modules/base" |
||||
"code.gitea.io/gitea/modules/charset" |
||||
"code.gitea.io/gitea/modules/context" |
||||
"code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/git/pipeline" |
||||
"code.gitea.io/gitea/modules/lfs" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
|
||||
"github.com/mcuadros/go-version" |
||||
"github.com/unknwon/com" |
||||
gogit "gopkg.in/src-d/go-git.v4" |
||||
"gopkg.in/src-d/go-git.v4/plumbing" |
||||
"gopkg.in/src-d/go-git.v4/plumbing/object" |
||||
) |
||||
|
||||
const ( |
||||
tplSettingsLFS base.TplName = "repo/settings/lfs" |
||||
tplSettingsLFSFile base.TplName = "repo/settings/lfs_file" |
||||
tplSettingsLFSFileFind base.TplName = "repo/settings/lfs_file_find" |
||||
tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers" |
||||
) |
||||
|
||||
// LFSFiles shows a repository's LFS files
|
||||
func LFSFiles(ctx *context.Context) { |
||||
if !setting.LFS.StartServer { |
||||
ctx.NotFound("LFSFiles", nil) |
||||
return |
||||
} |
||||
page := ctx.QueryInt("page") |
||||
if page <= 1 { |
||||
page = 1 |
||||
} |
||||
total, err := ctx.Repo.Repository.CountLFSMetaObjects() |
||||
if err != nil { |
||||
ctx.ServerError("LFSFiles", err) |
||||
return |
||||
} |
||||
|
||||
pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5) |
||||
ctx.Data["Title"] = ctx.Tr("repo.settings.lfs") |
||||
ctx.Data["PageIsSettingsLFS"] = true |
||||
lfsMetaObjects, err := ctx.Repo.Repository.GetLFSMetaObjects(pager.Paginater.Current(), setting.UI.ExplorePagingNum) |
||||
if err != nil { |
||||
ctx.ServerError("LFSFiles", err) |
||||
return |
||||
} |
||||
ctx.Data["LFSFiles"] = lfsMetaObjects |
||||
ctx.Data["Page"] = pager |
||||
ctx.HTML(200, tplSettingsLFS) |
||||
} |
||||
|
||||
// LFSFileGet serves a single LFS file
|
||||
func LFSFileGet(ctx *context.Context) { |
||||
if !setting.LFS.StartServer { |
||||
ctx.NotFound("LFSFileGet", nil) |
||||
return |
||||
} |
||||
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" |
||||
oid := ctx.Params("oid") |
||||
ctx.Data["Title"] = oid |
||||
ctx.Data["PageIsSettingsLFS"] = true |
||||
meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(oid) |
||||
if err != nil { |
||||
if err == models.ErrLFSObjectNotExist { |
||||
ctx.NotFound("LFSFileGet", nil) |
||||
return |
||||
} |
||||
ctx.ServerError("LFSFileGet", err) |
||||
return |
||||
} |
||||
ctx.Data["LFSFile"] = meta |
||||
dataRc, err := lfs.ReadMetaObject(meta) |
||||
if err != nil { |
||||
ctx.ServerError("LFSFileGet", err) |
||||
return |
||||
} |
||||
defer dataRc.Close() |
||||
buf := make([]byte, 1024) |
||||
n, err := dataRc.Read(buf) |
||||
if err != nil { |
||||
ctx.ServerError("Data", err) |
||||
return |
||||
} |
||||
buf = buf[:n] |
||||
|
||||
isTextFile := base.IsTextFile(buf) |
||||
ctx.Data["IsTextFile"] = isTextFile |
||||
|
||||
fileSize := meta.Size |
||||
ctx.Data["FileSize"] = meta.Size |
||||
ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, "direct") |
||||
switch { |
||||
case isTextFile: |
||||
if fileSize >= setting.UI.MaxDisplayFileSize { |
||||
ctx.Data["IsFileTooLarge"] = true |
||||
break |
||||
} |
||||
|
||||
d, _ := ioutil.ReadAll(dataRc) |
||||
buf = charset.ToUTF8WithFallback(append(buf, d...)) |
||||
|
||||
// Building code view blocks with line number on server side.
|
||||
var fileContent string |
||||
if content, err := charset.ToUTF8WithErr(buf); err != nil { |
||||
log.Error("ToUTF8WithErr: %v", err) |
||||
fileContent = string(buf) |
||||
} else { |
||||
fileContent = content |
||||
} |
||||
|
||||
var output bytes.Buffer |
||||
lines := strings.Split(fileContent, "\n") |
||||
//Remove blank line at the end of file
|
||||
if len(lines) > 0 && lines[len(lines)-1] == "" { |
||||
lines = lines[:len(lines)-1] |
||||
} |
||||
for index, line := range lines { |
||||
line = gotemplate.HTMLEscapeString(line) |
||||
if index != len(lines)-1 { |
||||
line += "\n" |
||||
} |
||||
output.WriteString(fmt.Sprintf(`<li class="L%d" rel="L%d">%s</li>`, index+1, index+1, line)) |
||||
} |
||||
ctx.Data["FileContent"] = gotemplate.HTML(output.String()) |
||||
|
||||
output.Reset() |
||||
for i := 0; i < len(lines); i++ { |
||||
output.WriteString(fmt.Sprintf(`<span id="L%d">%d</span>`, i+1, i+1)) |
||||
} |
||||
ctx.Data["LineNums"] = gotemplate.HTML(output.String()) |
||||
|
||||
case base.IsPDFFile(buf): |
||||
ctx.Data["IsPDFFile"] = true |
||||
case base.IsVideoFile(buf): |
||||
ctx.Data["IsVideoFile"] = true |
||||
case base.IsAudioFile(buf): |
||||
ctx.Data["IsAudioFile"] = true |
||||
case base.IsImageFile(buf): |
||||
ctx.Data["IsImageFile"] = true |
||||
} |
||||
ctx.HTML(200, tplSettingsLFSFile) |
||||
} |
||||
|
||||
// LFSDelete disassociates the provided oid from the repository and if the lfs file is no longer associated with any repositories - deletes it
|
||||
func LFSDelete(ctx *context.Context) { |
||||
if !setting.LFS.StartServer { |
||||
ctx.NotFound("LFSDelete", nil) |
||||
return |
||||
} |
||||
oid := ctx.Params("oid") |
||||
count, err := ctx.Repo.Repository.RemoveLFSMetaObjectByOid(oid) |
||||
if err != nil { |
||||
ctx.ServerError("LFSDelete", err) |
||||
return |
||||
} |
||||
// FIXME: Warning: the LFS store is not locked - and can't be locked - there could be a race condition here
|
||||
// Please note a similar condition happens in models/repo.go DeleteRepository
|
||||
if count == 0 { |
||||
oidPath := filepath.Join(oid[0:2], oid[2:4], oid[4:]) |
||||
err = os.Remove(filepath.Join(setting.LFS.ContentPath, oidPath)) |
||||
if err != nil { |
||||
ctx.ServerError("LFSDelete", err) |
||||
return |
||||
} |
||||
} |
||||
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") |
||||
} |
||||
|
||||
type lfsResult struct { |
||||
Name string |
||||
SHA string |
||||
Summary string |
||||
When time.Time |
||||
ParentHashes []plumbing.Hash |
||||
BranchName string |
||||
FullCommitName string |
||||
} |
||||
|
||||
type lfsResultSlice []*lfsResult |
||||
|
||||
func (a lfsResultSlice) Len() int { return len(a) } |
||||
func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } |
||||
func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) } |
||||
|
||||
// LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha
|
||||
func LFSFileFind(ctx *context.Context) { |
||||
if !setting.LFS.StartServer { |
||||
ctx.NotFound("LFSFind", nil) |
||||
return |
||||
} |
||||
oid := ctx.Query("oid") |
||||
size := ctx.QueryInt64("size") |
||||
if len(oid) == 0 || size == 0 { |
||||
ctx.NotFound("LFSFind", nil) |
||||
return |
||||
} |
||||
sha := ctx.Query("sha") |
||||
ctx.Data["Title"] = oid |
||||
ctx.Data["PageIsSettingsLFS"] = true |
||||
var hash plumbing.Hash |
||||
if len(sha) == 0 { |
||||
meta := models.LFSMetaObject{Oid: oid, Size: size} |
||||
pointer := meta.Pointer() |
||||
hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer)) |
||||
sha = hash.String() |
||||
} else { |
||||
hash = plumbing.NewHash(sha) |
||||
} |
||||
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" |
||||
ctx.Data["Oid"] = oid |
||||
ctx.Data["Size"] = size |
||||
ctx.Data["SHA"] = sha |
||||
|
||||
resultsMap := map[string]*lfsResult{} |
||||
results := make([]*lfsResult, 0) |
||||
|
||||
basePath := ctx.Repo.Repository.RepoPath() |
||||
gogitRepo := ctx.Repo.GitRepo.GoGitRepo() |
||||
|
||||
commitsIter, err := gogitRepo.Log(&gogit.LogOptions{ |
||||
Order: gogit.LogOrderCommitterTime, |
||||
All: true, |
||||
}) |
||||
if err != nil { |
||||
log.Error("Failed to get GoGit CommitsIter: %v", err) |
||||
ctx.ServerError("LFSFind: Iterate Commits", err) |
||||
return |
||||
} |
||||
|
||||
err = commitsIter.ForEach(func(gitCommit *object.Commit) error { |
||||
tree, err := gitCommit.Tree() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
treeWalker := object.NewTreeWalker(tree, true, nil) |
||||
defer treeWalker.Close() |
||||
for { |
||||
name, entry, err := treeWalker.Next() |
||||
if err == io.EOF { |
||||
break |
||||
} |
||||
if entry.Hash == hash { |
||||
result := lfsResult{ |
||||
Name: name, |
||||
SHA: gitCommit.Hash.String(), |
||||
Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0], |
||||
When: gitCommit.Author.When, |
||||
ParentHashes: gitCommit.ParentHashes, |
||||
} |
||||
resultsMap[gitCommit.Hash.String()+":"+name] = &result |
||||
} |
||||
} |
||||
return nil |
||||
}) |
||||
if err != nil && err != io.EOF { |
||||
log.Error("Failure in CommitIter.ForEach: %v", err) |
||||
ctx.ServerError("LFSFind: IterateCommits ForEach", err) |
||||
return |
||||
} |
||||
|
||||
for _, result := range resultsMap { |
||||
hasParent := false |
||||
for _, parentHash := range result.ParentHashes { |
||||
if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent { |
||||
break |
||||
} |
||||
} |
||||
if !hasParent { |
||||
results = append(results, result) |
||||
} |
||||
} |
||||
|
||||
sort.Sort(lfsResultSlice(results)) |
||||
|
||||
// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
|
||||
shasToNameReader, shasToNameWriter := io.Pipe() |
||||
nameRevStdinReader, nameRevStdinWriter := io.Pipe() |
||||
errChan := make(chan error, 1) |
||||
wg := sync.WaitGroup{} |
||||
wg.Add(3) |
||||
|
||||
go func() { |
||||
defer wg.Done() |
||||
scanner := bufio.NewScanner(nameRevStdinReader) |
||||
i := 0 |
||||
for scanner.Scan() { |
||||
line := scanner.Text() |
||||
if len(line) == 0 { |
||||
continue |
||||
} |
||||
result := results[i] |
||||
result.FullCommitName = line |
||||
result.BranchName = strings.Split(line, "~")[0] |
||||
i++ |
||||
} |
||||
}() |
||||
go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath) |
||||
go func() { |
||||
defer wg.Done() |
||||
defer shasToNameWriter.Close() |
||||
for _, result := range results { |
||||
i := 0 |
||||
if i < len(result.SHA) { |
||||
n, err := shasToNameWriter.Write([]byte(result.SHA)[i:]) |
||||
if err != nil { |
||||
errChan <- err |
||||
break |
||||
} |
||||
i += n |
||||
} |
||||
n := 0 |
||||
for n < 1 { |
||||
n, err = shasToNameWriter.Write([]byte{'\n'}) |
||||
if err != nil { |
||||
errChan <- err |
||||
break |
||||
} |
||||
|
||||
} |
||||
|
||||
} |
||||
}() |
||||
|
||||
wg.Wait() |
||||
|
||||
select { |
||||
case err, has := <-errChan: |
||||
if has { |
||||
ctx.ServerError("LFSPointerFiles", err) |
||||
} |
||||
default: |
||||
} |
||||
|
||||
ctx.Data["Results"] = results |
||||
ctx.HTML(200, tplSettingsLFSFileFind) |
||||
} |
||||
|
||||
// LFSPointerFiles will search the repository for pointer files and report which are missing LFS files in the content store
|
||||
func LFSPointerFiles(ctx *context.Context) { |
||||
if !setting.LFS.StartServer { |
||||
ctx.NotFound("LFSFileGet", nil) |
||||
return |
||||
} |
||||
ctx.Data["PageIsSettingsLFS"] = true |
||||
binVersion, err := git.BinVersion() |
||||
if err != nil { |
||||
log.Fatal("Error retrieving git version: %v", err) |
||||
} |
||||
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" |
||||
|
||||
basePath := ctx.Repo.Repository.RepoPath() |
||||
|
||||
pointerChan := make(chan pointerResult) |
||||
|
||||
catFileCheckReader, catFileCheckWriter := io.Pipe() |
||||
shasToBatchReader, shasToBatchWriter := io.Pipe() |
||||
catFileBatchReader, catFileBatchWriter := io.Pipe() |
||||
errChan := make(chan error, 1) |
||||
wg := sync.WaitGroup{} |
||||
wg.Add(5) |
||||
|
||||
var numPointers, numAssociated, numNoExist, numAssociatable int |
||||
|
||||
go func() { |
||||
defer wg.Done() |
||||
pointers := make([]pointerResult, 0, 50) |
||||
for pointer := range pointerChan { |
||||
pointers = append(pointers, pointer) |
||||
if pointer.InRepo { |
||||
numAssociated++ |
||||
} |
||||
if !pointer.Exists { |
||||
numNoExist++ |
||||
} |
||||
if !pointer.InRepo && pointer.Accessible { |
||||
numAssociatable++ |
||||
} |
||||
} |
||||
numPointers = len(pointers) |
||||
ctx.Data["Pointers"] = pointers |
||||
ctx.Data["NumPointers"] = numPointers |
||||
ctx.Data["NumAssociated"] = numAssociated |
||||
ctx.Data["NumAssociatable"] = numAssociatable |
||||
ctx.Data["NumNoExist"] = numNoExist |
||||
ctx.Data["NumNotAssociated"] = numPointers - numAssociated |
||||
}() |
||||
go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User) |
||||
go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) |
||||
go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) |
||||
if !version.Compare(binVersion, "2.6.0", ">=") { |
||||
revListReader, revListWriter := io.Pipe() |
||||
shasToCheckReader, shasToCheckWriter := io.Pipe() |
||||
wg.Add(2) |
||||
go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath) |
||||
go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg) |
||||
go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan) |
||||
} else { |
||||
go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan) |
||||
} |
||||
wg.Wait() |
||||
|
||||
select { |
||||
case err, has := <-errChan: |
||||
if has { |
||||
ctx.ServerError("LFSPointerFiles", err) |
||||
} |
||||
default: |
||||
} |
||||
ctx.HTML(200, tplSettingsLFSPointers) |
||||
} |
||||
|
||||
type pointerResult struct { |
||||
SHA string |
||||
Oid string |
||||
Size int64 |
||||
InRepo bool |
||||
Exists bool |
||||
Accessible bool |
||||
} |
||||
|
||||
func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { |
||||
defer wg.Done() |
||||
defer catFileBatchReader.Close() |
||||
contentStore := lfs.ContentStore{BasePath: setting.LFS.ContentPath} |
||||
|
||||
bufferedReader := bufio.NewReader(catFileBatchReader) |
||||
buf := make([]byte, 1025) |
||||
for { |
||||
// File descriptor line: sha
|
||||
sha, err := bufferedReader.ReadString(' ') |
||||
if err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
// Throw away the blob
|
||||
if _, err := bufferedReader.ReadString(' '); err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
sizeStr, err := bufferedReader.ReadString('\n') |
||||
if err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1]) |
||||
if err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
pointerBuf := buf[:size+1] |
||||
if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
pointerBuf = pointerBuf[:size] |
||||
// Now we need to check if the pointerBuf is an LFS pointer
|
||||
pointer := lfs.IsPointerFile(&pointerBuf) |
||||
if pointer == nil { |
||||
continue |
||||
} |
||||
|
||||
result := pointerResult{ |
||||
SHA: strings.TrimSpace(sha), |
||||
Oid: pointer.Oid, |
||||
Size: pointer.Size, |
||||
} |
||||
|
||||
// Then we need to check that this pointer is in the db
|
||||
if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil { |
||||
if err != models.ErrLFSObjectNotExist { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
} else { |
||||
result.InRepo = true |
||||
} |
||||
|
||||
result.Exists = contentStore.Exists(pointer) |
||||
|
||||
if result.Exists { |
||||
if !result.InRepo { |
||||
// Can we fix?
|
||||
// OK well that's "simple"
|
||||
// - we need to check whether current user has access to a repo that has access to the file
|
||||
result.Accessible, err = models.LFSObjectAccessible(user, result.Oid) |
||||
if err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
} else { |
||||
result.Accessible = true |
||||
} |
||||
} |
||||
pointerChan <- result |
||||
} |
||||
close(pointerChan) |
||||
} |
||||
|
||||
// LFSAutoAssociate auto associates accessible lfs files
|
||||
func LFSAutoAssociate(ctx *context.Context) { |
||||
if !setting.LFS.StartServer { |
||||
ctx.NotFound("LFSAutoAssociate", nil) |
||||
return |
||||
} |
||||
oids := ctx.QueryStrings("oid") |
||||
metas := make([]*models.LFSMetaObject, len(oids)) |
||||
for i, oid := range oids { |
||||
idx := strings.IndexRune(oid, ' ') |
||||
if idx < 0 || idx+1 > len(oid) { |
||||
ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s", oid)) |
||||
return |
||||
} |
||||
var err error |
||||
metas[i] = &models.LFSMetaObject{} |
||||
metas[i].Size, err = com.StrTo(oid[idx+1:]).Int64() |
||||
if err != nil { |
||||
ctx.ServerError("LFSAutoAssociate", fmt.Errorf("Illegal oid input: %s %v", oid, err)) |
||||
return |
||||
} |
||||
metas[i].Oid = oid[:idx] |
||||
//metas[i].RepositoryID = ctx.Repo.Repository.ID
|
||||
} |
||||
if err := models.LFSAutoAssociate(metas, ctx.User, ctx.Repo.Repository.ID); err != nil { |
||||
ctx.ServerError("LFSAutoAssociate", err) |
||||
return |
||||
} |
||||
ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") |
||||
} |
||||
@ -0,0 +1,62 @@ |
||||
{{template "base/head" .}} |
||||
<div class="repository settings lfs"> |
||||
{{template "repo/header" .}} |
||||
{{template "repo/settings/navbar" .}} |
||||
<div class="ui container"> |
||||
{{template "base/alert" .}} |
||||
<h4 class="ui top attached header"> |
||||
{{.i18n.Tr "repo.settings.lfs_filelist"}} |
||||
<div class="ui right"> |
||||
<a class="ui blue tiny show-panel button" href="{{.Link}}/pointers">{{.i18n.Tr "repo.settings.lfs_findpointerfiles"}}</a> |
||||
</div> |
||||
</h4> |
||||
<table id="lfs-files-table" class="ui attached segment single line table"> |
||||
<tbody> |
||||
{{range .LFSFiles}} |
||||
<tr> |
||||
<td> |
||||
<span class="text sha label"> |
||||
<a href="{{$.Link}}/show/{{.Oid}}" title="{{.Oid}}" class="ui detail icon button brown truncate"> |
||||
{{ShortSha .Oid}} |
||||
</a> |
||||
</span> |
||||
</td> |
||||
<td>{{FileSize .Size}}</td> |
||||
<td>{{TimeSince .CreatedUnix.AsTime $.Lang}}</td> |
||||
<td class="right aligned"> |
||||
<a class="ui blue show-panel button" href="{{$.Link}}/find?oid={{.Oid}}&size={{.Size}}">{{$.i18n.Tr "repo.settings.lfs_findcommits"}}</a> |
||||
<button class="ui basic show-modal icon button" data-modal="#delete-{{.Oid}}"> |
||||
<i class="octicon octicon-trashcan btn-octicon btn-octicon-danger poping up" data-content="{{$.i18n.Tr "repo.editor.delete_this_file"}}" data-position="bottom center" data-variation="tiny inverted"></i> |
||||
</button> |
||||
</td> |
||||
</tr> |
||||
{{else}} |
||||
<tr> |
||||
<td colspan="4">{{.i18n.Tr "repo.settings.lfs_no_lfs_files"}}</td> |
||||
</tr> |
||||
{{end}} |
||||
</tbody> |
||||
</table> |
||||
{{template "base/paginate" .}} |
||||
{{range .LFSFiles}} |
||||
<div class="ui basic modal" id="delete-{{.Oid}}"> |
||||
<div class="ui icon header"> |
||||
{{$.i18n.Tr "repo.settings.lfs_delete" .Oid}} |
||||
</div> |
||||
<div class="content center"> |
||||
<p> |
||||
{{$.i18n.Tr "repo.settings.lfs_delete_warning"}} |
||||
</p> |
||||
<form class="ui form" action="{{$.Link}}/delete/{{.Oid}}" method="post"> |
||||
{{$.CsrfTokenHtml}} |
||||
<div class="center actions"> |
||||
<div class="ui basic cancel inverted button">{{$.i18n.Tr "settings.cancel"}}</div> |
||||
<button class="ui basic inverted yellow button">{{$.i18n.Tr "modal.yes"}}</button> |
||||
</div> |
||||
</form> |
||||
</div> |
||||
</div> |
||||
{{end}} |
||||
</div> |
||||
</div> |
||||
{{template "base/footer" .}} |
||||
@ -0,0 +1,57 @@ |
||||
{{template "base/head" .}} |
||||
<div class="repository settings lfs"> |
||||
{{template "repo/header" .}} |
||||
{{template "repo/settings/navbar" .}} |
||||
<div class="ui container repository file list"> |
||||
{{template "base/alert" .}} |
||||
<div class="tab-size-8 non-diff-file-content"> |
||||
<h4 class="ui top attached header"> |
||||
<a href="{{.LFSFilesLink}}">{{.i18n.Tr "repo.settings.lfs"}}</a> / <span class="truncate sha">{{.LFSFile.Oid}}</span> |
||||
<div class="ui right"> |
||||
<a class="ui blue show-panel button" href="{{.LFSFilesLink}}/find?oid={{.LFSFile.Oid}}&size={{.LFSFile.Size}}">{{$.i18n.Tr "repo.settings.lfs_findcommits"}}</a> |
||||
</div> |
||||
</h4> |
||||
<div class="ui attached table unstackable segment"> |
||||
<div class="file-view {{if .IsMarkup}}markdown{{else if .IsRenderedHTML}}plain-text{{else if .IsTextFile}}code-view{{end}} has-emoji"> |
||||
{{if .IsMarkup}} |
||||
{{if .FileContent}}{{.FileContent | Safe}}{{end}} |
||||
{{else if .IsRenderedHTML}} |
||||
<pre>{{if .FileContent}}{{.FileContent | Str2html}}{{end}}</pre> |
||||
{{else if not .IsTextFile}} |
||||
<div class="view-raw ui center"> |
||||
{{if .IsImageFile}} |
||||
<img src="{{EscapePound $.RawFileLink}}"> |
||||
{{else if .IsVideoFile}} |
||||
<video controls src="{{EscapePound $.RawFileLink}}"> |
||||
<strong>{{.i18n.Tr "repo.video_not_supported_in_browser"}}</strong> |
||||
</video> |
||||
{{else if .IsAudioFile}} |
||||
<audio controls src="{{EscapePound $.RawFileLink}}"> |
||||
<strong>{{.i18n.Tr "repo.audio_not_supported_in_browser"}}</strong> |
||||
</audio> |
||||
{{else if .IsPDFFile}} |
||||
<iframe width="100%" height="600px" src="{{AppSubUrl}}/vendor/plugins/pdfjs/web/viewer.html?file={{EscapePound $.RawFileLink}}"></iframe> |
||||
{{else}} |
||||
<a href="{{EscapePound $.RawFileLink}}" rel="nofollow" class="btn btn-gray btn-radius">{{.i18n.Tr "repo.file_view_raw"}}</a> |
||||
{{end}} |
||||
</div> |
||||
{{else if .FileSize}} |
||||
<table> |
||||
<tbody> |
||||
<tr> |
||||
{{if .IsFileTooLarge}} |
||||
<td><strong>{{.i18n.Tr "repo.file_too_large"}}</strong></td> |
||||
{{else}} |
||||
<td class="lines-num">{{.LineNums}}</td> |
||||
<td class="lines-code"><pre><code class="{{.HighlightClass}}"><ol class="linenums">{{.FileContent}}</ol></code></pre></td> |
||||
{{end}} |
||||
</tr> |
||||
</tbody> |
||||
</table> |
||||
{{end}} |
||||
</div> |
||||
</div> |
||||
</div> |
||||
</div> |
||||
</div> |
||||
{{template "base/footer" .}} |
||||
@ -0,0 +1,52 @@ |
||||
{{template "base/head" .}} |
||||
<div class="repository settings lfs"> |
||||
{{template "repo/header" .}} |
||||
{{template "repo/settings/navbar" .}} |
||||
<div class="ui container repository file list"> |
||||
{{template "base/alert" .}} |
||||
<div class="tab-size-8 non-diff-file-content"> |
||||
<h4 class="ui top attached header"> |
||||
<a href="{{.LFSFilesLink}}">{{.i18n.Tr "repo.settings.lfs"}}</a> / <span class="truncate sha">{{.Oid}}</span> |
||||
</h4> |
||||
<table id="lfs-files-find-table" class="ui attached segment single line table"> |
||||
<tbody> |
||||
{{range .Results}} |
||||
<tr> |
||||
<td> |
||||
<span class="octicon octicon-file-text"></span> |
||||
<a href="{{EscapePound $.RepoLink}}/src/commit/{{.SHA}}/{{EscapePound .Name}}" title="{{.Name}}">{{.Name}}</a> |
||||
</td> |
||||
<td class="message has-emoji"> |
||||
<span class="truncate"> |
||||
<a href="{{$.RepoLink}}/commit/{{.SHA}}" title="{{.Summary}}"> |
||||
{{.Summary}} |
||||
</a> |
||||
</span> |
||||
</td> |
||||
<td> |
||||
<span class="text grey"><i class="octicon octicon-git-branch"></i>{{.BranchName}}</span> |
||||
</td> |
||||
<td> |
||||
{{if .ParentHashes}} |
||||
{{$.i18n.Tr "repo.diff.parent"}} |
||||
{{range .ParentHashes}} |
||||
<a class="ui blue sha label" href="{{$.RepoLink}}/commit/{{.String}}">{{ShortSha .String}}</a> |
||||
{{end}} |
||||
{{end}} |
||||
<div class="mobile-only"></div> |
||||
{{$.i18n.Tr "repo.diff.commit"}} |
||||
<a class="ui blue sha label" href="{{$.RepoLink}}/commit/{{.SHA}}">{{ShortSha .SHA}}</a> |
||||
</td> |
||||
<td>{{TimeSince .When $.Lang}}</td> |
||||
</tr> |
||||
{{else}} |
||||
<tr> |
||||
<td colspan="5">{{.i18n.Tr "repo.settings.lfs_lfs_file_no_commits"}}</td> |
||||
</tr> |
||||
{{end}} |
||||
</tbody> |
||||
</table> |
||||
</div> |
||||
</div> |
||||
</div> |
||||
{{template "base/footer" .}} |
||||
@ -0,0 +1,71 @@ |
||||
{{template "base/head" .}} |
||||
<div class="repository settings lfs"> |
||||
{{template "repo/header" .}} |
||||
{{template "repo/settings/navbar" .}} |
||||
<div class="ui container"> |
||||
{{template "base/alert" .}} |
||||
<h4 class="ui top attached header"> |
||||
{{.i18n.Tr "repo.settings.lfs_pointers.found" .NumPointers .NumAssociated .NumNotAssociated .NumNoExist }} |
||||
{{if gt .NumAssociatable 0}} |
||||
<div class="ui right"> |
||||
<form class="ui form" method="post" action="{{$.Link}}/associate"> |
||||
{{.CsrfTokenHtml}} |
||||
{{range .Pointers}} |
||||
{{if and (not .InRepo) .Exists .Accessible}} |
||||
<input type="hidden" name="oid" value="{{.Oid}} {{.Size}}"/> |
||||
{{end}} |
||||
{{end}} |
||||
<button class="ui green button">{{$.i18n.Tr "repo.settings.lfs_pointers.associateAccessible" $.NumAssociatable}}</button> |
||||
</form> |
||||
</div> |
||||
{{end}} |
||||
</h4> |
||||
<div class="ui attached segment"> |
||||
<table id="lfs-files-table" class="ui fixed single line table"> |
||||
<thead> |
||||
<tr> |
||||
<th class="three wide">{{.i18n.Tr "repo.settings.lfs_pointers.sha"}}</th> |
||||
<th class="four wide">{{.i18n.Tr "repo.settings.lfs_pointers.oid"}}</th> |
||||
<th class="three wide"></th> |
||||
<th class="two wide">{{.i18n.Tr "repo.settings.lfs_pointers.inRepo"}}</th> |
||||
<th class="two wide">{{.i18n.Tr "repo.settings.lfs_pointers.exists"}}</th> |
||||
<th class="two wide">{{.i18n.Tr "repo.settings.lfs_pointers.accessible"}}</th> |
||||
</tr> |
||||
</thead> |
||||
<tbody> |
||||
{{range .Pointers}} |
||||
<tr> |
||||
<td> |
||||
<span class="text sha label" title="{{.SHA}}"> |
||||
<a href="{{$.RepoLink}}/raw/blob/{{.SHA}}" rel="nofollow" target="_blank" class="ui detail icon button truncate"> |
||||
{{ShortSha .SHA}} |
||||
</a> |
||||
</span> |
||||
</td> |
||||
<td> |
||||
<span class="text sha label" title="{{.Oid}}"> |
||||
{{if and .Exists .InRepo}} |
||||
<a href="{{$.LFSFilesLink}}/show/{{.Oid}}" rel="nofollow" target="_blank" class="ui text detail icon button brown truncate"> |
||||
{{ShortSha .Oid}} |
||||
</a> |
||||
{{else}} |
||||
<span class="ui detail icon button brown disabled truncate"> |
||||
{{ShortSha .Oid}} |
||||
</span> |
||||
{{end}} |
||||
</span> |
||||
</td> |
||||
<td> |
||||
<a class="ui blue show-panel button" href="{{$.LFSFilesLink}}/find?oid={{.Oid}}&size={{.Size}}&sha={{.SHA}}">{{$.i18n.Tr "repo.settings.lfs_findcommits"}}</a> |
||||
</td> |
||||
<td><i class="fa fa{{if .InRepo}}-check{{end}}-square-o"></i></td> |
||||
<td><i class="fa fa{{if .Exists}}-check{{end}}-square-o"></i></td> |
||||
<td><i class="fa fa{{if .Accessible}}-check{{end}}-square-o"></i></td> |
||||
</tr> |
||||
{{end}} |
||||
</tbody> |
||||
</table> |
||||
</div> |
||||
</div> |
||||
</div> |
||||
{{template "base/footer" .}} |
||||
Loading…
Reference in new issue