Add LFS Migration and Mirror (#14726)
* Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: zeripath <art27@cantab.net>tokarchuk/v1.17
KN4CK3R
4 years ago
committed by
GitHub
parent
f544414a23
commit
c03e488e14
@ -0,0 +1,49 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package integrations |
||||
|
||||
import ( |
||||
"net/http" |
||||
"path" |
||||
"testing" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"code.gitea.io/gitea/modules/lfs" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
api "code.gitea.io/gitea/modules/structs" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestAPIRepoLFSMigrateLocal(t *testing.T) { |
||||
defer prepareTestEnv(t)() |
||||
|
||||
oldImportLocalPaths := setting.ImportLocalPaths |
||||
oldAllowLocalNetworks := setting.Migrations.AllowLocalNetworks |
||||
setting.ImportLocalPaths = true |
||||
setting.Migrations.AllowLocalNetworks = true |
||||
|
||||
user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User) |
||||
session := loginUser(t, user.Name) |
||||
token := getTokenForLoggedInUser(t, session) |
||||
|
||||
req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate?token="+token, &api.MigrateRepoOptions{ |
||||
CloneAddr: path.Join(setting.RepoRootPath, "migration/lfs-test.git"), |
||||
RepoOwnerID: user.ID, |
||||
RepoName: "lfs-test-local", |
||||
LFS: true, |
||||
}) |
||||
resp := MakeRequest(t, req, NoExpectedStatus) |
||||
assert.EqualValues(t, http.StatusCreated, resp.Code) |
||||
|
||||
store := lfs.NewContentStore() |
||||
ok, _ := store.Verify(lfs.Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6}) |
||||
assert.True(t, ok) |
||||
ok, _ = store.Verify(lfs.Pointer{Oid: "d6f175817f886ec6fbbc1515326465fa96c3bfd54a4ea06cfd6dbbd8340e0152", Size: 6}) |
||||
assert.True(t, ok) |
||||
|
||||
setting.ImportLocalPaths = oldImportLocalPaths |
||||
setting.Migrations.AllowLocalNetworks = oldAllowLocalNetworks |
||||
} |
||||
@ -0,0 +1 @@ |
||||
ref: refs/heads/master |
||||
@ -0,0 +1,7 @@ |
||||
[core] |
||||
bare = false |
||||
repositoryformatversion = 0 |
||||
filemode = false |
||||
symlinks = false |
||||
ignorecase = true |
||||
logallrefupdates = true |
||||
@ -0,0 +1 @@ |
||||
Unnamed repository; edit this file 'description' to name the repository. |
||||
@ -0,0 +1,3 @@ |
||||
#!/bin/sh |
||||
command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout.\n"; exit 2; } |
||||
git lfs post-checkout "$@" |
||||
@ -0,0 +1,3 @@ |
||||
#!/bin/sh |
||||
command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-commit.\n"; exit 2; } |
||||
git lfs post-commit "$@" |
||||
@ -0,0 +1,3 @@ |
||||
#!/bin/sh |
||||
command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-merge.\n"; exit 2; } |
||||
git lfs post-merge "$@" |
||||
@ -0,0 +1,3 @@ |
||||
#!/bin/sh |
||||
command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/pre-push.\n"; exit 2; } |
||||
git lfs pre-push "$@" |
||||
Binary file not shown.
@ -0,0 +1 @@ |
||||
dummy2 |
||||
@ -0,0 +1 @@ |
||||
dummy1 |
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1 @@ |
||||
546244003622c64b2fc3c2cd544d7a29882c8383 |
||||
@ -0,0 +1,117 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package integrations |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io/ioutil" |
||||
"net/url" |
||||
"os" |
||||
"path/filepath" |
||||
"testing" |
||||
|
||||
"code.gitea.io/gitea/modules/lfs" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func str2url(raw string) *url.URL { |
||||
u, _ := url.Parse(raw) |
||||
return u |
||||
} |
||||
|
||||
func TestDetermineLocalEndpoint(t *testing.T) { |
||||
defer prepareTestEnv(t)() |
||||
|
||||
root, _ := ioutil.TempDir("", "lfs_test") |
||||
defer os.RemoveAll(root) |
||||
|
||||
rootdotgit, _ := ioutil.TempDir("", "lfs_test") |
||||
defer os.RemoveAll(rootdotgit) |
||||
os.Mkdir(filepath.Join(rootdotgit, ".git"), 0700) |
||||
|
||||
lfsroot, _ := ioutil.TempDir("", "lfs_test") |
||||
defer os.RemoveAll(lfsroot) |
||||
|
||||
// Test cases
|
||||
var cases = []struct { |
||||
cloneurl string |
||||
lfsurl string |
||||
expected *url.URL |
||||
}{ |
||||
// case 0
|
||||
{ |
||||
cloneurl: root, |
||||
lfsurl: "", |
||||
expected: str2url(fmt.Sprintf("file://%s", root)), |
||||
}, |
||||
// case 1
|
||||
{ |
||||
cloneurl: root, |
||||
lfsurl: lfsroot, |
||||
expected: str2url(fmt.Sprintf("file://%s", lfsroot)), |
||||
}, |
||||
// case 2
|
||||
{ |
||||
cloneurl: "https://git.com/repo.git", |
||||
lfsurl: lfsroot, |
||||
expected: str2url(fmt.Sprintf("file://%s", lfsroot)), |
||||
}, |
||||
// case 3
|
||||
{ |
||||
cloneurl: rootdotgit, |
||||
lfsurl: "", |
||||
expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))), |
||||
}, |
||||
// case 4
|
||||
{ |
||||
cloneurl: "", |
||||
lfsurl: rootdotgit, |
||||
expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))), |
||||
}, |
||||
// case 5
|
||||
{ |
||||
cloneurl: rootdotgit, |
||||
lfsurl: rootdotgit, |
||||
expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))), |
||||
}, |
||||
// case 6
|
||||
{ |
||||
cloneurl: fmt.Sprintf("file://%s", root), |
||||
lfsurl: "", |
||||
expected: str2url(fmt.Sprintf("file://%s", root)), |
||||
}, |
||||
// case 7
|
||||
{ |
||||
cloneurl: fmt.Sprintf("file://%s", root), |
||||
lfsurl: fmt.Sprintf("file://%s", lfsroot), |
||||
expected: str2url(fmt.Sprintf("file://%s", lfsroot)), |
||||
}, |
||||
// case 8
|
||||
{ |
||||
cloneurl: root, |
||||
lfsurl: fmt.Sprintf("file://%s", lfsroot), |
||||
expected: str2url(fmt.Sprintf("file://%s", lfsroot)), |
||||
}, |
||||
// case 9
|
||||
{ |
||||
cloneurl: "", |
||||
lfsurl: "/does/not/exist", |
||||
expected: nil, |
||||
}, |
||||
// case 10
|
||||
{ |
||||
cloneurl: "", |
||||
lfsurl: "file:///does/not/exist", |
||||
expected: str2url("file:///does/not/exist"), |
||||
}, |
||||
} |
||||
|
||||
for n, c := range cases { |
||||
ep := lfs.DetermineEndpoint(c.cloneurl, c.lfsurl) |
||||
|
||||
assert.Equal(t, c.expected, ep, "case %d: error should match", n) |
||||
} |
||||
} |
||||
@ -0,0 +1,18 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations |
||||
|
||||
import ( |
||||
"xorm.io/xorm" |
||||
) |
||||
|
||||
func addLFSMirrorColumns(x *xorm.Engine) error { |
||||
type Mirror struct { |
||||
LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"` |
||||
LFSEndpoint string `xorm:"lfs_endpoint TEXT"` |
||||
} |
||||
|
||||
return x.Sync2(new(Mirror)) |
||||
} |
||||
@ -0,0 +1,24 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"context" |
||||
"io" |
||||
"net/url" |
||||
) |
||||
|
||||
// Client is used to communicate with a LFS source
|
||||
type Client interface { |
||||
Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) |
||||
} |
||||
|
||||
// NewClient creates a LFS client
|
||||
func NewClient(endpoint *url.URL) Client { |
||||
if endpoint.Scheme == "file" { |
||||
return newFilesystemClient(endpoint) |
||||
} |
||||
return newHTTPClient(endpoint) |
||||
} |
||||
@ -0,0 +1,23 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"net/url" |
||||
|
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestNewClient(t *testing.T) { |
||||
u, _ := url.Parse("file:///test") |
||||
c := NewClient(u) |
||||
assert.IsType(t, &FilesystemClient{}, c) |
||||
|
||||
u, _ = url.Parse("https://test.com/lfs") |
||||
c = NewClient(u) |
||||
assert.IsType(t, &HTTPClient{}, c) |
||||
} |
||||
@ -0,0 +1,106 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"fmt" |
||||
"net/url" |
||||
"os" |
||||
"path" |
||||
"path/filepath" |
||||
"strings" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
) |
||||
|
||||
// DetermineEndpoint determines an endpoint from the clone url or uses the specified LFS url.
|
||||
func DetermineEndpoint(cloneurl, lfsurl string) *url.URL { |
||||
if len(lfsurl) > 0 { |
||||
return endpointFromURL(lfsurl) |
||||
} |
||||
return endpointFromCloneURL(cloneurl) |
||||
} |
||||
|
||||
func endpointFromCloneURL(rawurl string) *url.URL { |
||||
ep := endpointFromURL(rawurl) |
||||
if ep == nil { |
||||
return ep |
||||
} |
||||
|
||||
if strings.HasSuffix(ep.Path, "/") { |
||||
ep.Path = ep.Path[:len(ep.Path)-1] |
||||
} |
||||
|
||||
if ep.Scheme == "file" { |
||||
return ep |
||||
} |
||||
|
||||
if path.Ext(ep.Path) == ".git" { |
||||
ep.Path += "/info/lfs" |
||||
} else { |
||||
ep.Path += ".git/info/lfs" |
||||
} |
||||
|
||||
return ep |
||||
} |
||||
|
||||
func endpointFromURL(rawurl string) *url.URL { |
||||
if strings.HasPrefix(rawurl, "/") { |
||||
return endpointFromLocalPath(rawurl) |
||||
} |
||||
|
||||
u, err := url.Parse(rawurl) |
||||
if err != nil { |
||||
log.Error("lfs.endpointFromUrl: %v", err) |
||||
return nil |
||||
} |
||||
|
||||
switch u.Scheme { |
||||
case "http", "https": |
||||
return u |
||||
case "git": |
||||
u.Scheme = "https" |
||||
return u |
||||
case "file": |
||||
return u |
||||
default: |
||||
if _, err := os.Stat(rawurl); err == nil { |
||||
return endpointFromLocalPath(rawurl) |
||||
} |
||||
|
||||
log.Error("lfs.endpointFromUrl: unknown url") |
||||
return nil |
||||
} |
||||
} |
||||
|
||||
func endpointFromLocalPath(path string) *url.URL { |
||||
var slash string |
||||
if abs, err := filepath.Abs(path); err == nil { |
||||
if !strings.HasPrefix(abs, "/") { |
||||
slash = "/" |
||||
} |
||||
path = abs |
||||
} |
||||
|
||||
var gitpath string |
||||
if filepath.Base(path) == ".git" { |
||||
gitpath = path |
||||
path = filepath.Dir(path) |
||||
} else { |
||||
gitpath = filepath.Join(path, ".git") |
||||
} |
||||
|
||||
if _, err := os.Stat(gitpath); err == nil { |
||||
path = gitpath |
||||
} else if _, err := os.Stat(path); err != nil { |
||||
return nil |
||||
} |
||||
|
||||
path = fmt.Sprintf("file://%s%s", slash, filepath.ToSlash(path)) |
||||
|
||||
u, _ := url.Parse(path) |
||||
|
||||
return u |
||||
} |
||||
@ -0,0 +1,75 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"net/url" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func str2url(raw string) *url.URL { |
||||
u, _ := url.Parse(raw) |
||||
return u |
||||
} |
||||
|
||||
func TestDetermineEndpoint(t *testing.T) { |
||||
// Test cases
|
||||
var cases = []struct { |
||||
cloneurl string |
||||
lfsurl string |
||||
expected *url.URL |
||||
}{ |
||||
// case 0
|
||||
{ |
||||
cloneurl: "", |
||||
lfsurl: "", |
||||
expected: nil, |
||||
}, |
||||
// case 1
|
||||
{ |
||||
cloneurl: "https://git.com/repo", |
||||
lfsurl: "", |
||||
expected: str2url("https://git.com/repo.git/info/lfs"), |
||||
}, |
||||
// case 2
|
||||
{ |
||||
cloneurl: "https://git.com/repo.git", |
||||
lfsurl: "", |
||||
expected: str2url("https://git.com/repo.git/info/lfs"), |
||||
}, |
||||
// case 3
|
||||
{ |
||||
cloneurl: "", |
||||
lfsurl: "https://gitlfs.com/repo", |
||||
expected: str2url("https://gitlfs.com/repo"), |
||||
}, |
||||
// case 4
|
||||
{ |
||||
cloneurl: "https://git.com/repo.git", |
||||
lfsurl: "https://gitlfs.com/repo", |
||||
expected: str2url("https://gitlfs.com/repo"), |
||||
}, |
||||
// case 5
|
||||
{ |
||||
cloneurl: "git://git.com/repo.git", |
||||
lfsurl: "", |
||||
expected: str2url("https://git.com/repo.git/info/lfs"), |
||||
}, |
||||
// case 6
|
||||
{ |
||||
cloneurl: "", |
||||
lfsurl: "git://gitlfs.com/repo", |
||||
expected: str2url("https://gitlfs.com/repo"), |
||||
}, |
||||
} |
||||
|
||||
for n, c := range cases { |
||||
ep := DetermineEndpoint(c.cloneurl, c.lfsurl) |
||||
|
||||
assert.Equal(t, c.expected, ep, "case %d: error should match", n) |
||||
} |
||||
} |
||||
@ -0,0 +1,50 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"context" |
||||
"io" |
||||
"net/url" |
||||
"os" |
||||
"path/filepath" |
||||
|
||||
"code.gitea.io/gitea/modules/util" |
||||
) |
||||
|
||||
// FilesystemClient is used to read LFS data from a filesystem path
|
||||
type FilesystemClient struct { |
||||
lfsdir string |
||||
} |
||||
|
||||
func newFilesystemClient(endpoint *url.URL) *FilesystemClient { |
||||
path, _ := util.FileURLToPath(endpoint) |
||||
|
||||
lfsdir := filepath.Join(path, "lfs", "objects") |
||||
|
||||
client := &FilesystemClient{lfsdir} |
||||
|
||||
return client |
||||
} |
||||
|
||||
func (c *FilesystemClient) objectPath(oid string) string { |
||||
return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid) |
||||
} |
||||
|
||||
// Download reads the specific LFS object from the target repository
|
||||
func (c *FilesystemClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { |
||||
objectPath := c.objectPath(oid) |
||||
|
||||
if _, err := os.Stat(objectPath); os.IsNotExist(err) { |
||||
return nil, err |
||||
} |
||||
|
||||
file, err := os.Open(objectPath) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return file, nil |
||||
} |
||||
@ -0,0 +1,129 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"encoding/json" |
||||
"errors" |
||||
"fmt" |
||||
"io" |
||||
"net/http" |
||||
"net/url" |
||||
"strings" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
) |
||||
|
||||
// HTTPClient is used to communicate with the LFS server
|
||||
// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
|
||||
type HTTPClient struct { |
||||
client *http.Client |
||||
endpoint string |
||||
transfers map[string]TransferAdapter |
||||
} |
||||
|
||||
func newHTTPClient(endpoint *url.URL) *HTTPClient { |
||||
hc := &http.Client{} |
||||
|
||||
client := &HTTPClient{ |
||||
client: hc, |
||||
endpoint: strings.TrimSuffix(endpoint.String(), "/"), |
||||
transfers: make(map[string]TransferAdapter), |
||||
} |
||||
|
||||
basic := &BasicTransferAdapter{hc} |
||||
|
||||
client.transfers[basic.Name()] = basic |
||||
|
||||
return client |
||||
} |
||||
|
||||
func (c *HTTPClient) transferNames() []string { |
||||
keys := make([]string, len(c.transfers)) |
||||
|
||||
i := 0 |
||||
for k := range c.transfers { |
||||
keys[i] = k |
||||
i++ |
||||
} |
||||
|
||||
return keys |
||||
} |
||||
|
||||
func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { |
||||
url := fmt.Sprintf("%s/objects/batch", c.endpoint) |
||||
|
||||
request := &BatchRequest{operation, c.transferNames(), nil, objects} |
||||
|
||||
payload := new(bytes.Buffer) |
||||
err := json.NewEncoder(payload).Encode(request) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("lfs.HTTPClient.batch json.Encode: %w", err) |
||||
} |
||||
|
||||
log.Trace("lfs.HTTPClient.batch NewRequestWithContext: %s", url) |
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, payload) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("lfs.HTTPClient.batch http.NewRequestWithContext: %w", err) |
||||
} |
||||
req.Header.Set("Content-type", MediaType) |
||||
req.Header.Set("Accept", MediaType) |
||||
|
||||
res, err := c.client.Do(req) |
||||
if err != nil { |
||||
select { |
||||
case <-ctx.Done(): |
||||
return nil, ctx.Err() |
||||
default: |
||||
} |
||||
return nil, fmt.Errorf("lfs.HTTPClient.batch http.Do: %w", err) |
||||
} |
||||
defer res.Body.Close() |
||||
|
||||
if res.StatusCode != http.StatusOK { |
||||
return nil, fmt.Errorf("lfs.HTTPClient.batch: Unexpected servers response: %s", res.Status) |
||||
} |
||||
|
||||
var response BatchResponse |
||||
err = json.NewDecoder(res.Body).Decode(&response) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("lfs.HTTPClient.batch json.Decode: %w", err) |
||||
} |
||||
|
||||
if len(response.Transfer) == 0 { |
||||
response.Transfer = "basic" |
||||
} |
||||
|
||||
return &response, nil |
||||
} |
||||
|
||||
// Download reads the specific LFS object from the LFS server
|
||||
func (c *HTTPClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { |
||||
var objects []Pointer |
||||
objects = append(objects, Pointer{oid, size}) |
||||
|
||||
result, err := c.batch(ctx, "download", objects) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
transferAdapter, ok := c.transfers[result.Transfer] |
||||
if !ok { |
||||
return nil, fmt.Errorf("lfs.HTTPClient.Download Transferadapter not found: %s", result.Transfer) |
||||
} |
||||
|
||||
if len(result.Objects) == 0 { |
||||
return nil, errors.New("lfs.HTTPClient.Download: No objects in result") |
||||
} |
||||
|
||||
content, err := transferAdapter.Download(ctx, result.Objects[0]) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return content, nil |
||||
} |
||||
@ -0,0 +1,144 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"encoding/json" |
||||
"io" |
||||
"io/ioutil" |
||||
"net/http" |
||||
"strings" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
type RoundTripFunc func(req *http.Request) *http.Response |
||||
|
||||
func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { |
||||
return f(req), nil |
||||
} |
||||
|
||||
type DummyTransferAdapter struct { |
||||
} |
||||
|
||||
func (a *DummyTransferAdapter) Name() string { |
||||
return "dummy" |
||||
} |
||||
|
||||
func (a *DummyTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { |
||||
return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil |
||||
} |
||||
|
||||
func TestHTTPClientDownload(t *testing.T) { |
||||
oid := "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041" |
||||
size := int64(6) |
||||
|
||||
roundTripHandler := func(req *http.Request) *http.Response { |
||||
url := req.URL.String() |
||||
if strings.Contains(url, "status-not-ok") { |
||||
return &http.Response{StatusCode: http.StatusBadRequest} |
||||
} |
||||
if strings.Contains(url, "invalid-json-response") { |
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))} |
||||
} |
||||
if strings.Contains(url, "valid-batch-request-download") { |
||||
assert.Equal(t, "POST", req.Method) |
||||
assert.Equal(t, MediaType, req.Header.Get("Content-type"), "case %s: error should match", url) |
||||
assert.Equal(t, MediaType, req.Header.Get("Accept"), "case %s: error should match", url) |
||||
|
||||
var batchRequest BatchRequest |
||||
err := json.NewDecoder(req.Body).Decode(&batchRequest) |
||||
assert.NoError(t, err) |
||||
|
||||
assert.Equal(t, "download", batchRequest.Operation) |
||||
assert.Equal(t, 1, len(batchRequest.Objects)) |
||||
assert.Equal(t, oid, batchRequest.Objects[0].Oid) |
||||
assert.Equal(t, size, batchRequest.Objects[0].Size) |
||||
|
||||
batchResponse := &BatchResponse{ |
||||
Transfer: "dummy", |
||||
Objects: make([]*ObjectResponse, 1), |
||||
} |
||||
|
||||
payload := new(bytes.Buffer) |
||||
json.NewEncoder(payload).Encode(batchResponse) |
||||
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} |
||||
} |
||||
if strings.Contains(url, "invalid-response-no-objects") { |
||||
batchResponse := &BatchResponse{Transfer: "dummy"} |
||||
|
||||
payload := new(bytes.Buffer) |
||||
json.NewEncoder(payload).Encode(batchResponse) |
||||
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} |
||||
} |
||||
if strings.Contains(url, "unknown-transfer-adapter") { |
||||
batchResponse := &BatchResponse{Transfer: "unknown_adapter"} |
||||
|
||||
payload := new(bytes.Buffer) |
||||
json.NewEncoder(payload).Encode(batchResponse) |
||||
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} |
||||
} |
||||
|
||||
t.Errorf("Unknown test case: %s", url) |
||||
|
||||
return nil |
||||
} |
||||
|
||||
hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)} |
||||
dummy := &DummyTransferAdapter{} |
||||
|
||||
var cases = []struct { |
||||
endpoint string |
||||
expectederror string |
||||
}{ |
||||
// case 0
|
||||
{ |
||||
endpoint: "https://status-not-ok.io", |
||||
expectederror: "Unexpected servers response: ", |
||||
}, |
||||
// case 1
|
||||
{ |
||||
endpoint: "https://invalid-json-response.io", |
||||
expectederror: "json.Decode: ", |
||||
}, |
||||
// case 2
|
||||
{ |
||||
endpoint: "https://valid-batch-request-download.io", |
||||
expectederror: "", |
||||
}, |
||||
// case 3
|
||||
{ |
||||
endpoint: "https://invalid-response-no-objects.io", |
||||
expectederror: "No objects in result", |
||||
}, |
||||
// case 4
|
||||
{ |
||||
endpoint: "https://unknown-transfer-adapter.io", |
||||
expectederror: "Transferadapter not found: ", |
||||
}, |
||||
} |
||||
|
||||
for n, c := range cases { |
||||
client := &HTTPClient{ |
||||
client: hc, |
||||
endpoint: c.endpoint, |
||||
transfers: make(map[string]TransferAdapter), |
||||
} |
||||
client.transfers["dummy"] = dummy |
||||
|
||||
_, err := client.Download(context.Background(), oid, size) |
||||
if len(c.expectederror) > 0 { |
||||
assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) |
||||
} else { |
||||
assert.NoError(t, err, "case %d", n) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,123 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"crypto/sha256" |
||||
"encoding/hex" |
||||
"errors" |
||||
"fmt" |
||||
"io" |
||||
"path" |
||||
"regexp" |
||||
"strconv" |
||||
"strings" |
||||
) |
||||
|
||||
const ( |
||||
blobSizeCutoff = 1024 |
||||
|
||||
// MetaFileIdentifier is the string appearing at the first line of LFS pointer files.
|
||||
// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
|
||||
MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" |
||||
|
||||
// MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
|
||||
MetaFileOidPrefix = "oid sha256:" |
||||
) |
||||
|
||||
var ( |
||||
// ErrMissingPrefix occurs if the content lacks the LFS prefix
|
||||
ErrMissingPrefix = errors.New("Content lacks the LFS prefix") |
||||
|
||||
// ErrInvalidStructure occurs if the content has an invalid structure
|
||||
ErrInvalidStructure = errors.New("Content has an invalid structure") |
||||
|
||||
// ErrInvalidOIDFormat occurs if the oid has an invalid format
|
||||
ErrInvalidOIDFormat = errors.New("OID has an invalid format") |
||||
) |
||||
|
||||
// ReadPointer tries to read LFS pointer data from the reader
|
||||
func ReadPointer(reader io.Reader) (Pointer, error) { |
||||
buf := make([]byte, blobSizeCutoff) |
||||
n, err := io.ReadFull(reader, buf) |
||||
if err != nil && err != io.ErrUnexpectedEOF { |
||||
return Pointer{}, err |
||||
} |
||||
buf = buf[:n] |
||||
|
||||
return ReadPointerFromBuffer(buf) |
||||
} |
||||
|
||||
var oidPattern = regexp.MustCompile(`^[a-f\d]{64}$`) |
||||
|
||||
// ReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or an error otherwise.
|
||||
func ReadPointerFromBuffer(buf []byte) (Pointer, error) { |
||||
var p Pointer |
||||
|
||||
headString := string(buf) |
||||
if !strings.HasPrefix(headString, MetaFileIdentifier) { |
||||
return p, ErrMissingPrefix |
||||
} |
||||
|
||||
splitLines := strings.Split(headString, "\n") |
||||
if len(splitLines) < 3 { |
||||
return p, ErrInvalidStructure |
||||
} |
||||
|
||||
oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix) |
||||
if len(oid) != 64 || !oidPattern.MatchString(oid) { |
||||
return p, ErrInvalidOIDFormat |
||||
} |
||||
size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) |
||||
if err != nil { |
||||
return p, err |
||||
} |
||||
|
||||
p.Oid = oid |
||||
p.Size = size |
||||
|
||||
return p, nil |
||||
} |
||||
|
||||
// IsValid checks if the pointer has a valid structure.
|
||||
// It doesn't check if the pointed-to-content exists.
|
||||
func (p Pointer) IsValid() bool { |
||||
if len(p.Oid) != 64 { |
||||
return false |
||||
} |
||||
if !oidPattern.MatchString(p.Oid) { |
||||
return false |
||||
} |
||||
if p.Size < 0 { |
||||
return false |
||||
} |
||||
return true |
||||
} |
||||
|
||||
// StringContent returns the string representation of the pointer
|
||||
// https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#the-pointer
|
||||
func (p Pointer) StringContent() string { |
||||
return fmt.Sprintf("%s\n%s%s\nsize %d\n", MetaFileIdentifier, MetaFileOidPrefix, p.Oid, p.Size) |
||||
} |
||||
|
||||
// RelativePath returns the relative storage path of the pointer
|
||||
func (p Pointer) RelativePath() string { |
||||
if len(p.Oid) < 5 { |
||||
return p.Oid |
||||
} |
||||
|
||||
return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:]) |
||||
} |
||||
|
||||
// GeneratePointer generates a pointer for arbitrary content
|
||||
func GeneratePointer(content io.Reader) (Pointer, error) { |
||||
h := sha256.New() |
||||
c, err := io.Copy(h, content) |
||||
if err != nil { |
||||
return Pointer{}, err |
||||
} |
||||
sum := h.Sum(nil) |
||||
return Pointer{Oid: hex.EncodeToString(sum), Size: c}, nil |
||||
} |
||||
@ -0,0 +1,64 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build gogit
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
|
||||
"code.gitea.io/gitea/modules/git" |
||||
|
||||
"github.com/go-git/go-git/v5/plumbing/object" |
||||
) |
||||
|
||||
// SearchPointerBlobs scans the whole repository for LFS pointer files
|
||||
func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) { |
||||
gitRepo := repo.GoGitRepo() |
||||
|
||||
err := func() error { |
||||
blobs, err := gitRepo.BlobObjects() |
||||
if err != nil { |
||||
return fmt.Errorf("lfs.SearchPointerBlobs BlobObjects: %w", err) |
||||
} |
||||
|
||||
return blobs.ForEach(func(blob *object.Blob) error { |
||||
select { |
||||
case <-ctx.Done(): |
||||
return ctx.Err() |
||||
default: |
||||
} |
||||
|
||||
if blob.Size > blobSizeCutoff { |
||||
return nil |
||||
} |
||||
|
||||
reader, err := blob.Reader() |
||||
if err != nil { |
||||
return fmt.Errorf("lfs.SearchPointerBlobs blob.Reader: %w", err) |
||||
} |
||||
defer reader.Close() |
||||
|
||||
pointer, _ := ReadPointer(reader) |
||||
if pointer.IsValid() { |
||||
pointerChan <- PointerBlob{Hash: blob.Hash.String(), Pointer: pointer} |
||||
} |
||||
|
||||
return nil |
||||
}) |
||||
}() |
||||
|
||||
if err != nil { |
||||
select { |
||||
case <-ctx.Done(): |
||||
default: |
||||
errChan <- err |
||||
} |
||||
} |
||||
|
||||
close(pointerChan) |
||||
close(errChan) |
||||
} |
||||
@ -0,0 +1,110 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !gogit
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"bufio" |
||||
"context" |
||||
"io" |
||||
"strconv" |
||||
"sync" |
||||
|
||||
"code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/git/pipeline" |
||||
) |
||||
|
||||
// SearchPointerBlobs scans the whole repository for LFS pointer files
|
||||
func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) { |
||||
basePath := repo.Path |
||||
|
||||
catFileCheckReader, catFileCheckWriter := io.Pipe() |
||||
shasToBatchReader, shasToBatchWriter := io.Pipe() |
||||
catFileBatchReader, catFileBatchWriter := io.Pipe() |
||||
|
||||
wg := sync.WaitGroup{} |
||||
wg.Add(4) |
||||
|
||||
// Create the go-routines in reverse order.
|
||||
|
||||
// 4. Take the output of cat-file --batch and check if each file in turn
|
||||
// to see if they're pointers to files in the LFS store
|
||||
go createPointerResultsFromCatFileBatch(ctx, catFileBatchReader, &wg, pointerChan) |
||||
|
||||
// 3. Take the shas of the blobs and batch read them
|
||||
go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) |
||||
|
||||
// 2. From the provided objects restrict to blobs <=1k
|
||||
go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) |
||||
|
||||
// 1. Run batch-check on all objects in the repository
|
||||
if git.CheckGitVersionAtLeast("2.6.0") != nil { |
||||
revListReader, revListWriter := io.Pipe() |
||||
shasToCheckReader, shasToCheckWriter := io.Pipe() |
||||
wg.Add(2) |
||||
go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath) |
||||
go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg) |
||||
go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan) |
||||
} else { |
||||
go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan) |
||||
} |
||||
wg.Wait() |
||||
|
||||
close(pointerChan) |
||||
close(errChan) |
||||
} |
||||
|
||||
func createPointerResultsFromCatFileBatch(ctx context.Context, catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- PointerBlob) { |
||||
defer wg.Done() |
||||
defer catFileBatchReader.Close() |
||||
|
||||
bufferedReader := bufio.NewReader(catFileBatchReader) |
||||
buf := make([]byte, 1025) |
||||
|
||||
loop: |
||||
for { |
||||
select { |
||||
case <-ctx.Done(): |
||||
break loop |
||||
default: |
||||
} |
||||
|
||||
// File descriptor line: sha
|
||||
sha, err := bufferedReader.ReadString(' ') |
||||
if err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
// Throw away the blob
|
||||
if _, err := bufferedReader.ReadString(' '); err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
sizeStr, err := bufferedReader.ReadString('\n') |
||||
if err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1]) |
||||
if err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
pointerBuf := buf[:size+1] |
||||
if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil { |
||||
_ = catFileBatchReader.CloseWithError(err) |
||||
break |
||||
} |
||||
pointerBuf = pointerBuf[:size] |
||||
// Now we need to check if the pointerBuf is an LFS pointer
|
||||
pointer, _ := ReadPointerFromBuffer(pointerBuf) |
||||
if !pointer.IsValid() { |
||||
continue |
||||
} |
||||
|
||||
pointerChan <- PointerBlob{Hash: sha, Pointer: pointer} |
||||
} |
||||
} |
||||
@ -0,0 +1,103 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"path" |
||||
"strings" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestStringContent(t *testing.T) { |
||||
p := Pointer{Oid: "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393", Size: 1234} |
||||
expected := "version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n" |
||||
assert.Equal(t, p.StringContent(), expected) |
||||
} |
||||
|
||||
func TestRelativePath(t *testing.T) { |
||||
p := Pointer{Oid: "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393"} |
||||
expected := path.Join("4d", "7a", "214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") |
||||
assert.Equal(t, p.RelativePath(), expected) |
||||
|
||||
p2 := Pointer{Oid: "4d7a"} |
||||
assert.Equal(t, p2.RelativePath(), "4d7a") |
||||
} |
||||
|
||||
func TestIsValid(t *testing.T) { |
||||
p := Pointer{} |
||||
assert.False(t, p.IsValid()) |
||||
|
||||
p = Pointer{Oid: "123"} |
||||
assert.False(t, p.IsValid()) |
||||
|
||||
p = Pointer{Oid: "z4cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc"} |
||||
assert.False(t, p.IsValid()) |
||||
|
||||
p = Pointer{Oid: "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc"} |
||||
assert.True(t, p.IsValid()) |
||||
|
||||
p = Pointer{Oid: "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc", Size: -1} |
||||
assert.False(t, p.IsValid()) |
||||
} |
||||
|
||||
func TestGeneratePointer(t *testing.T) { |
||||
p, err := GeneratePointer(strings.NewReader("Gitea")) |
||||
assert.NoError(t, err) |
||||
assert.True(t, p.IsValid()) |
||||
assert.Equal(t, p.Oid, "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc") |
||||
assert.Equal(t, p.Size, int64(5)) |
||||
} |
||||
|
||||
func TestReadPointerFromBuffer(t *testing.T) { |
||||
p, err := ReadPointerFromBuffer([]byte{}) |
||||
assert.ErrorIs(t, err, ErrMissingPrefix) |
||||
assert.False(t, p.IsValid()) |
||||
|
||||
p, err = ReadPointerFromBuffer([]byte("test")) |
||||
assert.ErrorIs(t, err, ErrMissingPrefix) |
||||
assert.False(t, p.IsValid()) |
||||
|
||||
p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\n")) |
||||
assert.ErrorIs(t, err, ErrInvalidStructure) |
||||
assert.False(t, p.IsValid()) |
||||
|
||||
p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a\nsize 1234\n")) |
||||
assert.ErrorIs(t, err, ErrInvalidOIDFormat) |
||||
assert.False(t, p.IsValid()) |
||||
|
||||
p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a2146z4ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n")) |
||||
assert.ErrorIs(t, err, ErrInvalidOIDFormat) |
||||
assert.False(t, p.IsValid()) |
||||
|
||||
p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\ntest 1234\n")) |
||||
assert.Error(t, err) |
||||
assert.False(t, p.IsValid()) |
||||
|
||||
p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize test\n")) |
||||
assert.Error(t, err) |
||||
assert.False(t, p.IsValid()) |
||||
|
||||
p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n")) |
||||
assert.NoError(t, err) |
||||
assert.True(t, p.IsValid()) |
||||
assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") |
||||
assert.Equal(t, p.Size, int64(1234)) |
||||
|
||||
p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\ntest")) |
||||
assert.NoError(t, err) |
||||
assert.True(t, p.IsValid()) |
||||
assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") |
||||
assert.Equal(t, p.Size, int64(1234)) |
||||
} |
||||
|
||||
func TestReadPointer(t *testing.T) { |
||||
p, err := ReadPointer(strings.NewReader("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n")) |
||||
assert.NoError(t, err) |
||||
assert.True(t, p.IsValid()) |
||||
assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") |
||||
assert.Equal(t, p.Size, int64(1234)) |
||||
} |
||||
@ -1,71 +0,0 @@ |
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"io" |
||||
"strconv" |
||||
"strings" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"code.gitea.io/gitea/modules/base" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/storage" |
||||
) |
||||
|
||||
// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
|
||||
func ReadPointerFile(reader io.Reader) (*models.LFSMetaObject, *[]byte) { |
||||
if !setting.LFS.StartServer { |
||||
return nil, nil |
||||
} |
||||
|
||||
buf := make([]byte, 1024) |
||||
n, _ := reader.Read(buf) |
||||
buf = buf[:n] |
||||
|
||||
if isTextFile := base.IsTextFile(buf); !isTextFile { |
||||
return nil, nil |
||||
} |
||||
|
||||
return IsPointerFile(&buf), &buf |
||||
} |
||||
|
||||
// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file
|
||||
func IsPointerFile(buf *[]byte) *models.LFSMetaObject { |
||||
if !setting.LFS.StartServer { |
||||
return nil |
||||
} |
||||
|
||||
headString := string(*buf) |
||||
if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) { |
||||
return nil |
||||
} |
||||
|
||||
splitLines := strings.Split(headString, "\n") |
||||
if len(splitLines) < 3 { |
||||
return nil |
||||
} |
||||
|
||||
oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix) |
||||
size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) |
||||
if len(oid) != 64 || err != nil { |
||||
return nil |
||||
} |
||||
|
||||
contentStore := &ContentStore{ObjectStorage: storage.LFS} |
||||
meta := &models.LFSMetaObject{Oid: oid, Size: size} |
||||
exist, err := contentStore.Exists(meta) |
||||
if err != nil || !exist { |
||||
return nil |
||||
} |
||||
|
||||
return meta |
||||
} |
||||
|
||||
// ReadMetaObject will read a models.LFSMetaObject and return a reader
|
||||
func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) { |
||||
contentStore := &ContentStore{ObjectStorage: storage.LFS} |
||||
return contentStore.Get(meta) |
||||
} |
||||
@ -0,0 +1,69 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"time" |
||||
) |
||||
|
||||
const ( |
||||
// MediaType contains the media type for LFS server requests
|
||||
MediaType = "application/vnd.git-lfs+json" |
||||
) |
||||
|
||||
// BatchRequest contains multiple requests processed in one batch operation.
|
||||
// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#requests
|
||||
type BatchRequest struct { |
||||
Operation string `json:"operation"` |
||||
Transfers []string `json:"transfers,omitempty"` |
||||
Ref *Reference `json:"ref,omitempty"` |
||||
Objects []Pointer `json:"objects"` |
||||
} |
||||
|
||||
// Reference contains a git reference.
|
||||
// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#ref-property
|
||||
type Reference struct { |
||||
Name string `json:"name"` |
||||
} |
||||
|
||||
// Pointer contains LFS pointer data
|
||||
type Pointer struct { |
||||
Oid string `json:"oid" xorm:"UNIQUE(s) INDEX NOT NULL"` |
||||
Size int64 `json:"size" xorm:"NOT NULL"` |
||||
} |
||||
|
||||
// BatchResponse contains multiple object metadata Representation structures
|
||||
// for use with the batch API.
|
||||
// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#successful-responses
|
||||
type BatchResponse struct { |
||||
Transfer string `json:"transfer,omitempty"` |
||||
Objects []*ObjectResponse `json:"objects"` |
||||
} |
||||
|
||||
// ObjectResponse is object metadata as seen by clients of the LFS server.
|
||||
type ObjectResponse struct { |
||||
Pointer |
||||
Actions map[string]*Link `json:"actions"` |
||||
Error *ObjectError `json:"error,omitempty"` |
||||
} |
||||
|
||||
// Link provides a structure used to build a hypermedia representation of an HTTP link.
|
||||
type Link struct { |
||||
Href string `json:"href"` |
||||
Header map[string]string `json:"header,omitempty"` |
||||
ExpiresAt time.Time `json:"expires_at,omitempty"` |
||||
} |
||||
|
||||
// ObjectError defines the JSON structure returned to the client in case of an error
|
||||
type ObjectError struct { |
||||
Code int `json:"code"` |
||||
Message string `json:"message"` |
||||
} |
||||
|
||||
// PointerBlob associates a Git blob with a Pointer.
|
||||
type PointerBlob struct { |
||||
Hash string |
||||
Pointer |
||||
} |
||||
@ -0,0 +1,58 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"fmt" |
||||
"io" |
||||
"net/http" |
||||
) |
||||
|
||||
// TransferAdapter represents an adapter for downloading/uploading LFS objects
|
||||
type TransferAdapter interface { |
||||
Name() string |
||||
Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) |
||||
//Upload(ctx context.Context, reader io.Reader) error
|
||||
} |
||||
|
||||
// BasicTransferAdapter implements the "basic" adapter
|
||||
type BasicTransferAdapter struct { |
||||
client *http.Client |
||||
} |
||||
|
||||
// Name returns the name of the adapter
|
||||
func (a *BasicTransferAdapter) Name() string { |
||||
return "basic" |
||||
} |
||||
|
||||
// Download reads the download location and downloads the data
|
||||
func (a *BasicTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { |
||||
download, ok := r.Actions["download"] |
||||
if !ok { |
||||
return nil, errors.New("lfs.BasicTransferAdapter.Download: Action 'download' not found") |
||||
} |
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", download.Href, nil) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.NewRequestWithContext: %w", err) |
||||
} |
||||
for key, value := range download.Header { |
||||
req.Header.Set(key, value) |
||||
} |
||||
|
||||
res, err := a.client.Do(req) |
||||
if err != nil { |
||||
select { |
||||
case <-ctx.Done(): |
||||
return nil, ctx.Err() |
||||
default: |
||||
} |
||||
return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.Do: %w", err) |
||||
} |
||||
|
||||
return res.Body, nil |
||||
} |
||||
@ -0,0 +1,78 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package lfs |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"io/ioutil" |
||||
"net/http" |
||||
"strings" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestBasicTransferAdapterName(t *testing.T) { |
||||
a := &BasicTransferAdapter{} |
||||
|
||||
assert.Equal(t, "basic", a.Name()) |
||||
} |
||||
|
||||
func TestBasicTransferAdapterDownload(t *testing.T) { |
||||
roundTripHandler := func(req *http.Request) *http.Response { |
||||
url := req.URL.String() |
||||
if strings.Contains(url, "valid-download-request") { |
||||
assert.Equal(t, "GET", req.Method) |
||||
assert.Equal(t, "test-value", req.Header.Get("test-header")) |
||||
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("dummy"))} |
||||
} |
||||
|
||||
t.Errorf("Unknown test case: %s", url) |
||||
|
||||
return nil |
||||
} |
||||
|
||||
hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)} |
||||
a := &BasicTransferAdapter{hc} |
||||
|
||||
var cases = []struct { |
||||
response *ObjectResponse |
||||
expectederror string |
||||
}{ |
||||
// case 0
|
||||
{ |
||||
response: &ObjectResponse{}, |
||||
expectederror: "Action 'download' not found", |
||||
}, |
||||
// case 1
|
||||
{ |
||||
response: &ObjectResponse{ |
||||
Actions: map[string]*Link{"upload": nil}, |
||||
}, |
||||
expectederror: "Action 'download' not found", |
||||
}, |
||||
// case 2
|
||||
{ |
||||
response: &ObjectResponse{ |
||||
Actions: map[string]*Link{"download": { |
||||
Href: "https://valid-download-request.io", |
||||
Header: map[string]string{"test-header": "test-value"}, |
||||
}}, |
||||
}, |
||||
expectederror: "", |
||||
}, |
||||
} |
||||
|
||||
for n, c := range cases { |
||||
_, err := a.Download(context.Background(), c.response) |
||||
if len(c.expectederror) > 0 { |
||||
assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) |
||||
} else { |
||||
assert.NoError(t, err, "case %d", n) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,58 @@ |
||||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package util |
||||
|
||||
import ( |
||||
"net/url" |
||||
"runtime" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestFileURLToPath(t *testing.T) { |
||||
var cases = []struct { |
||||
url string |
||||
expected string |
||||
haserror bool |
||||
windows bool |
||||
}{ |
||||
// case 0
|
||||
{ |
||||
url: "", |
||||
haserror: true, |
||||
}, |
||||
// case 1
|
||||
{ |
||||
url: "http://test.io", |
||||
haserror: true, |
||||
}, |
||||
// case 2
|
||||
{ |
||||
url: "file:///path", |
||||
expected: "/path", |
||||
}, |
||||
// case 3
|
||||
{ |
||||
url: "file:///C:/path", |
||||
expected: "C:/path", |
||||
windows: true, |
||||
}, |
||||
} |
||||
|
||||
for n, c := range cases { |
||||
if c.windows && runtime.GOOS != "windows" { |
||||
continue |
||||
} |
||||
u, _ := url.Parse(c.url) |
||||
p, err := FileURLToPath(u) |
||||
if c.haserror { |
||||
assert.Error(t, err, "case %d: should return error", n) |
||||
} else { |
||||
assert.NoError(t, err, "case %d: should not return error", n) |
||||
assert.Equal(t, c.expected, p, "case %d: should be equal", n) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,29 @@ |
||||
<div class="inline field"> |
||||
<label>{{.i18n.Tr "repo.migrate_options"}}</label> |
||||
<div class="ui checkbox"> |
||||
{{if .DisableMirrors}} |
||||
<input id="mirror" name="mirror" type="checkbox" readonly> |
||||
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label> |
||||
{{else}} |
||||
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}> |
||||
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label> |
||||
{{end}} |
||||
</div> |
||||
</div> |
||||
{{if .LFSActive}} |
||||
<div class="inline field"> |
||||
<label></label> |
||||
<div class="ui checkbox"> |
||||
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}> |
||||
<label>{{.i18n.Tr "repo.migrate_options_lfs"}}</label> |
||||
</div> |
||||
<span id="lfs_settings" style="display:none">(<a id="lfs_settings_show" href="#">{{.i18n.Tr "repo.settings.advanced_settings"}}</a>)</span> |
||||
</div> |
||||
<div id="lfs_endpoint" style="display:none"> |
||||
<span class="help">{{.i18n.Tr "repo.migrate_options_lfs_endpoint.description" "https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md#server-discovery" | Str2html}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate_options_lfs_endpoint.description.local"}}{{end}}</span> |
||||
<div class="inline field {{if .Err_LFSEndpoint}}error{{end}}"> |
||||
<label>{{.i18n.Tr "repo.migrate_options_lfs_endpoint.label"}}</label> |
||||
<input name="lfs_endpoint" value="{{.lfs_endpoint}}" placeholder="{{.i18n.Tr "repo.migrate_options_lfs_endpoint.placeholder"}}"> |
||||
</div> |
||||
</div> |
||||
{{end}} |
||||
Loading…
Reference in new issue