Allow admin to associate missing LFS objects for repositories (#18143)

This PR reworked the Find pointer files feature in Settings -> LFS page.

When a LFS object is missing from database but exists in LFS content store, admin can associate it to the repository by clicking the Associate button.

This PR is not perfect (because the LFS module itself should be improved too), it's just a nice-to-have feature to help users recover their LFS repositories (eg: database was lost / table was truncated)
tokarchuk/v1.17
wxiaoguang 3 years ago committed by GitHub
parent 25a290e320
commit 385dc6a992
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 53
      models/lfs.go
  2. 28
      routers/web/repo/lfs.go
  3. 2
      templates/repo/settings/lfs_pointers.tmpl

@ -7,11 +7,13 @@ package models
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo" repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user" user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/timeutil"
"xorm.io/builder" "xorm.io/builder"
@ -145,6 +147,11 @@ func LFSObjectAccessible(user *user_model.User, oid string) (bool, error) {
return count > 0, err return count > 0, err
} }
// LFSObjectIsAssociated checks if a provided Oid is associated
func LFSObjectIsAssociated(oid string) (bool, error) {
return db.GetEngine(db.DefaultContext).Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
}
// LFSAutoAssociate auto associates accessible LFSMetaObjects // LFSAutoAssociate auto associates accessible LFSMetaObjects
func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int64) error { func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int64) error {
ctx, committer, err := db.TxContext() ctx, committer, err := db.TxContext()
@ -162,23 +169,39 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6
oidMap[meta.Oid] = meta oidMap[meta.Oid] = meta
} }
cond := builder.NewCond()
if !user.IsAdmin { if !user.IsAdmin {
cond = builder.In("`lfs_meta_object`.repository_id", newMetas := make([]*LFSMetaObject, 0, len(metas))
builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user))) cond := builder.In(
} "`lfs_meta_object`.repository_id",
newMetas := make([]*LFSMetaObject, 0, len(metas)) builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user)),
if err := sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil { )
return err err = sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas)
} if err != nil {
for i := range newMetas { return err
newMetas[i].Size = oidMap[newMetas[i].Oid].Size }
newMetas[i].RepositoryID = repoID if len(newMetas) != len(oidMap) {
} return fmt.Errorf("unable collect all LFS objects from database, expected %d, actually %d", len(oidMap), len(newMetas))
if err := db.Insert(ctx, newMetas); err != nil { }
return err for i := range newMetas {
newMetas[i].Size = oidMap[newMetas[i].Oid].Size
newMetas[i].RepositoryID = repoID
}
if err = db.Insert(ctx, newMetas); err != nil {
return err
}
} else {
// admin can associate any LFS object to any repository, and we do not care about errors (eg: duplicated unique key),
// even if error occurs, it won't hurt users and won't make things worse
for i := range metas {
_, err = sess.Insert(&LFSMetaObject{
Pointer: lfs.Pointer{Oid: metas[i].Oid, Size: metas[i].Size},
RepositoryID: repoID,
})
if err != nil {
log.Warn("failed to insert LFS meta object into database, err=%v", err)
}
}
} }
return committer.Commit() return committer.Commit()
} }

@ -421,12 +421,13 @@ func LFSPointerFiles(ctx *context.Context) {
var numAssociated, numNoExist, numAssociatable int var numAssociated, numNoExist, numAssociatable int
type pointerResult struct { type pointerResult struct {
SHA string SHA string
Oid string Oid string
Size int64 Size int64
InRepo bool InRepo bool
Exists bool Exists bool
Accessible bool Accessible bool
Associatable bool
} }
results := []pointerResult{} results := []pointerResult{}
@ -461,22 +462,29 @@ func LFSPointerFiles(ctx *context.Context) {
// Can we fix? // Can we fix?
// OK well that's "simple" // OK well that's "simple"
// - we need to check whether current user has access to a repo that has access to the file // - we need to check whether current user has access to a repo that has access to the file
result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid) result.Associatable, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid)
if err != nil { if err != nil {
return err return err
} }
} else { if !result.Associatable {
result.Accessible = true associated, err := models.LFSObjectIsAssociated(pointerBlob.Oid)
if err != nil {
return err
}
result.Associatable = !associated
}
} }
} }
result.Accessible = result.InRepo || result.Associatable
if result.InRepo { if result.InRepo {
numAssociated++ numAssociated++
} }
if !result.Exists { if !result.Exists {
numNoExist++ numNoExist++
} }
if !result.InRepo && result.Accessible { if result.Associatable {
numAssociatable++ numAssociatable++
} }

@ -11,7 +11,7 @@
<form class="ui form" method="post" action="{{$.Link}}/associate"> <form class="ui form" method="post" action="{{$.Link}}/associate">
{{.CsrfTokenHtml}} {{.CsrfTokenHtml}}
{{range .Pointers}} {{range .Pointers}}
{{if and (not .InRepo) .Exists .Accessible}} {{if .Associatable}}
<input type="hidden" name="oid" value="{{.Oid}} {{.Size}}"/> <input type="hidden" name="oid" value="{{.Oid}} {{.Size}}"/>
{{end}} {{end}}
{{end}} {{end}}

Loading…
Cancel
Save