@ -6,22 +6,20 @@
package archiver package archiver
import ( import (
"errors"
"fmt"
"io" "io"
"io/ioutil"
"os" "os"
"path"
"regexp" "regexp"
"strings" "strings"
"sync"
"time"
"code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util " "code.gitea.io/gitea/modules/storage "
) )
// ArchiveRequest defines the parameters of an archive request, which notably
// ArchiveRequest defines the parameters of an archive request, which notably
@ -30,223 +28,174 @@ import (
// This is entirely opaque to external entities, though, and mostly used as a
// This is entirely opaque to external entities, though, and mostly used as a
// handle elsewhere.
// handle elsewhere.
type ArchiveRequest struct { type ArchiveRequest struct {
uri string RepoID int64
repo * git . Repository
refName string refName string
ext string Type git . ArchiveType
archivePath string CommitID string
archiveType git . ArchiveType
archiveComplete bool
commit * git . Commit
cchan chan struct { }
} }
var archiveInProgress [ ] * ArchiveRequest
var archiveMutex sync . Mutex
// SHA1 hashes will only go up to 40 characters, but SHA256 hashes will go all
// SHA1 hashes will only go up to 40 characters, but SHA256 hashes will go all
// the way to 64.
// the way to 64.
var shaRegex = regexp . MustCompile ( ` ^[0-9a-f] { 4,64}$ ` ) var shaRegex = regexp . MustCompile ( ` ^[0-9a-f] { 4,64}$ ` )
// These facilitate testing, by allowing the unit tests to control (to some extent)
// NewRequest creates an archival request, based on the URI. The
// the goroutine used for processing the queue.
var archiveQueueMutex * sync . Mutex
var archiveQueueStartCond * sync . Cond
var archiveQueueReleaseCond * sync . Cond
// GetArchivePath returns the path from which we can serve this archive.
func ( aReq * ArchiveRequest ) GetArchivePath ( ) string {
return aReq . archivePath
}
// GetArchiveName returns the name of the caller, based on the ref used by the
// caller to create this request.
func ( aReq * ArchiveRequest ) GetArchiveName ( ) string {
return aReq . refName + aReq . ext
}
// IsComplete returns the completion status of this request.
func ( aReq * ArchiveRequest ) IsComplete ( ) bool {
return aReq . archiveComplete
}
// WaitForCompletion will wait for this request to complete, with no timeout.
// It returns whether the archive was actually completed, as the channel could
// have also been closed due to an error.
func ( aReq * ArchiveRequest ) WaitForCompletion ( ctx * context . Context ) bool {
select {
case <- aReq . cchan :
case <- ctx . Done ( ) :
}
return aReq . IsComplete ( )
}
// TimedWaitForCompletion will wait for this request to complete, with timeout
// happening after the specified Duration. It returns whether the archive is
// now complete and whether we hit the timeout or not. The latter may not be
// useful if the request is complete or we started to shutdown.
func ( aReq * ArchiveRequest ) TimedWaitForCompletion ( ctx * context . Context , dur time . Duration ) ( bool , bool ) {
timeout := false
select {
case <- time . After ( dur ) :
timeout = true
case <- aReq . cchan :
case <- ctx . Done ( ) :
}
return aReq . IsComplete ( ) , timeout
}
// The caller must hold the archiveMutex across calls to getArchiveRequest.
func getArchiveRequest ( repo * git . Repository , commit * git . Commit , archiveType git . ArchiveType ) * ArchiveRequest {
for _ , r := range archiveInProgress {
// Need to be referring to the same repository.
if r . repo . Path == repo . Path && r . commit . ID == commit . ID && r . archiveType == archiveType {
return r
}
}
return nil
}
// DeriveRequestFrom creates an archival request, based on the URI. The
// resulting ArchiveRequest is suitable for being passed to ArchiveRepository()
// resulting ArchiveRequest is suitable for being passed to ArchiveRepository()
// if it's determined that the request still needs to be satisfied.
// if it's determined that the request still needs to be satisfied.
func DeriveRequestFrom ( ctx * context . Context , uri string ) * ArchiveRequest { func NewRequest ( repoID int64 , repo * git . Repository , uri string ) ( * ArchiveRequest , error ) {
if ctx . Repo == nil || ctx . Repo . GitRepo == nil {
log . Trace ( "Repo not initialized" )
return nil
}
r := & ArchiveRequest { r := & ArchiveRequest {
uri : uri , RepoID : repoID ,
repo : ctx . Repo . GitRepo ,
} }
var ext string
switch { switch {
case strings . HasSuffix ( uri , ".zip" ) : case strings . HasSuffix ( uri , ".zip" ) :
r . ext = ".zip" ext = ".zip"
r . archivePath = path . Join ( r . repo . Path , "archives/zip" ) r . Type = git . ZIP
r . archiveType = git . ZIP
case strings . HasSuffix ( uri , ".tar.gz" ) : case strings . HasSuffix ( uri , ".tar.gz" ) :
r . ext = ".tar.gz" ext = ".tar.gz"
r . archivePath = path . Join ( r . repo . Path , "archives/targz" ) r . Type = git . TARGZ
r . archiveType = git . TARGZ
default : default :
log . Trace ( "Unknown format: %s" , uri ) return nil , fmt . Errorf ( "Unknown format: %s" , uri )
return nil
} }
r . refName = strings . TrimSuffix ( r . uri , r . ext ) r . refName = strings . TrimSuffix ( uri , ext )
isDir , err := util . IsDir ( r . archivePath )
var err error
// Get corresponding commit.
if repo . IsBranchExist ( r . refName ) {
r . CommitID , err = repo . GetBranchCommitID ( r . refName )
if err != nil { if err != nil {
ctx . ServerError ( "Download -> util.IsDir(archivePath)" , err ) return nil , err
return nil
} }
if ! isDir { } else if repo . IsTagExist ( r . refName ) {
if err := os . MkdirAll ( r . archivePath , os . ModePerm ) ; err != nil { r . CommitID , err = repo . GetTagCommitID ( r . refName )
ctx . ServerError ( "Download -> os.MkdirAll(archivePath)" , err ) if err != nil {
return nil return nil , err
}
} else if shaRegex . MatchString ( r . refName ) {
if repo . IsCommitExist ( r . refName ) {
r . CommitID = r . refName
} else {
return nil , git . ErrNotExist {
ID : r . refName ,
} }
} }
} else {
return nil , fmt . Errorf ( "Unknow ref %s type" , r . refName )
}
// Get corresponding commit.
return r , nil
if r . repo . IsBranchExist ( r . refName ) { }
r . commit , err = r . repo . GetBranchCommit ( r . refName )
// GetArchiveName returns the name of the caller, based on the ref used by the
// caller to create this request.
func ( aReq * ArchiveRequest ) GetArchiveName ( ) string {
return strings . ReplaceAll ( aReq . refName , "/" , "-" ) + "." + aReq . Type . String ( )
}
func doArchive ( r * ArchiveRequest ) ( * models . RepoArchiver , error ) {
ctx , commiter , err := models . TxDBContext ( )
if err != nil { if err != nil {
ctx . ServerError ( "GetBranchCommit" , err ) return nil , err
return nil
} }
} else if r . repo . IsTagExist ( r . refName ) { defer commiter . Close ( )
r . commit , err = r . repo . GetTagCommit ( r . refName )
archiver , err := models . GetRepoArchiver ( ctx , r . RepoID , r . Type , r . CommitID )
if err != nil { if err != nil {
ctx . ServerError ( "GetTagCommit" , err ) return nil , err
return nil
} }
} else if shaRegex . MatchString ( r . refName ) {
r . commit , err = r . repo . GetCommit ( r . refName ) if archiver != nil {
if err != nil { // FIXME: If another process are generating it, we think it's not ready and just return
ctx . NotFound ( "GetCommit" , nil ) // Or we should wait until the archive generated.
return nil if archiver . Status == models . RepoArchiverGenerating {
return nil , nil
} }
} else { } else {
ctx . NotFound ( "DeriveRequestFrom" , nil ) archiver = & models . RepoArchiver {
return nil RepoID : r . RepoID ,
Type : r . Type ,
CommitID : r . CommitID ,
Status : models . RepoArchiverGenerating ,
}
if err := models . AddRepoArchiver ( ctx , archiver ) ; err != nil {
return nil , err
} }
archiveMutex . Lock ( )
defer archiveMutex . Unlock ( )
if rExisting := getArchiveRequest ( r . repo , r . commit , r . archiveType ) ; rExisting != nil {
return rExisting
} }
r . archivePath = path . Join ( r . archivePath , base . ShortSha ( r . commit . ID . String ( ) ) + r . ext ) rPath , err := archiver . RelativePath ( )
r . archiveComplete , err = util . IsFile ( r . archivePath )
if err != nil { if err != nil {
ctx . ServerError ( "util.IsFile" , err ) return nil , err
return nil
} }
return r
}
func doArchive ( r * ArchiveRequest ) { _ , err = storage . RepoArchives . Stat ( rPath )
var ( if err == nil {
err error if archiver . Status == models . RepoArchiverGenerating {
tmpArchive * os . File archiver . Status = models . RepoArchiverReady
destArchive * os . File return archiver , models . UpdateRepoArchiverStatus ( ctx , archiver )
) }
return archiver , nil
}
// Close the channel to indicate to potential waiters that this request
if ! errors . Is ( err , os . ErrNotExist ) {
// has finished.
return nil , fmt . Errorf ( "unable to stat archive: %v" , err )
defer close ( r . cchan ) }
// It could have happened that we enqueued two archival requests, due to
rd , w := io . Pipe ( )
// race conditions and difficulties in locking. Do one last check that
defer func ( ) {
// the archive we're referring to doesn't already exist. If it does exist,
w . Close ( )
// then just mark the request as complete and move on.
rd . Close ( )
isFile , err := util . IsFile ( r . archivePath ) } ( )
var done = make ( chan error )
repo , err := archiver . LoadRepo ( )
if err != nil { if err != nil {
log . Error ( "Unable to check if %s util.IsFile: %v. Will ignore and recreate." , r . archivePath , err ) return nil , fmt . Errorf ( "archiver.LoadRepo failed: %v" , err )
}
if isFile {
r . archiveComplete = true
return
} }
// Create a temporary file to use while the archive is being built. We
gitRepo , err := git . OpenRepository ( repo . RepoPath ( ) )
// will then copy it into place (r.archivePath) once it's fully
// constructed.
tmpArchive , err = ioutil . TempFile ( "" , "archive" )
if err != nil { if err != nil {
log . Error ( "Unable to create a temporary archive file! Error: %v" , err ) return nil , err
return
} }
defer gitRepo . Close ( )
go func ( done chan error , w * io . PipeWriter , archiver * models . RepoArchiver , gitRepo * git . Repository ) {
defer func ( ) { defer func ( ) {
tmpArchive . Close ( ) if r := recover ( ) ; r != nil {
os . Remove ( tmpArchive . Name ( ) ) done <- fmt . Errorf ( "%v" , r )
}
} ( ) } ( )
if err = r . commit . CreateArchive ( graceful . GetManager ( ) . ShutdownContext ( ) , tmpArchive . Name ( ) , git . CreateArchiveOpts { err = gitRepo . CreateArchive (
Format : r . archiveType , graceful . GetManager ( ) . ShutdownContext ( ) ,
Prefix : setting . Repository . PrefixArchiveFiles , archiver . Type ,
} ) ; err != nil { w ,
log . Error ( "Download -> CreateArchive " + tmpArchive . Name ( ) , err ) setting . Repository . PrefixArchiveFiles ,
return archiver . CommitID ,
} )
_ = w . CloseWithError ( err )
done <- err
} ( done , w , archiver , gitRepo )
// Now we copy it into place
// TODO: add lfs data to zip
if destArchive , err = os . Create ( r . archivePath ) ; err != nil { // TODO: add submodule data to zip
log . Error ( "Unable to open archive " + r . archivePath )
return if _ , err := storage . RepoArchives . Save ( rPath , rd , - 1 ) ; err != nil {
return nil , fmt . Errorf ( "unable to write archive: %v" , err )
} }
_ , err = io . Copy ( destArchive , tmpArchive )
destArchive . Close ( ) err = <- done
if err != nil { if err != nil {
log . Error ( "Unable to write archive " + r . archivePath ) return nil , err
return
} }
// Block any attempt to finalize creating a new request if we're marking
if archiver . Status == models . RepoArchiverGenerating {
r . archiveComplete = true archiver . Status = models . RepoArchiverReady
if err = models . UpdateRepoArchiverStatus ( ctx , archiver ) ; err != nil {
return nil , err
}
}
return archiver , commiter . Commit ( )
} }
// ArchiveRepository satisfies the ArchiveRequest being passed in. Processing
// ArchiveRepository satisfies the ArchiveRequest being passed in. Processing
@ -255,65 +204,46 @@ func doArchive(r *ArchiveRequest) {
// anything. In all cases, the caller should be examining the *ArchiveRequest
// anything. In all cases, the caller should be examining the *ArchiveRequest
// being returned for completion, as it may be different than the one they passed
// being returned for completion, as it may be different than the one they passed
// in.
// in.
func ArchiveRepository ( request * ArchiveRequest ) * ArchiveRequest { func ArchiveRepository ( request * ArchiveRequest ) ( * models . RepoArchiver , error ) {
// We'll return the request that's already been enqueued if it has been
return doArchive ( request )
// enqueued, or we'll immediately enqueue it if it has not been enqueued
}
// and it is not marked complete.
archiveMutex . Lock ( ) var archiverQueue queue . UniqueQueue
defer archiveMutex . Unlock ( )
if rExisting := getArchiveRequest ( request . repo , request . commit , request . archiveType ) ; rExisting != nil { // Init initlize archive
return rExisting func Init ( ) error {
handler := func ( data ... queue . Data ) {
for _ , datum := range data {
archiveReq , ok := datum . ( * ArchiveRequest )
if ! ok {
log . Error ( "Unable to process provided datum: %v - not possible to cast to IndexerData" , datum )
continue
}
log . Trace ( "ArchiverData Process: %#v" , archiveReq )
if _ , err := doArchive ( archiveReq ) ; err != nil {
log . Error ( "Archive %v faild: %v" , datum , err )
}
} }
if request . archiveComplete {
return request
} }
request . cchan = make ( chan struct { } ) archiverQueue = queue . CreateUniqueQueue ( "repo-archive" , handler , new ( ArchiveRequest ) )
archiveInProgress = append ( archiveInProgress , request ) if archiverQueue == nil {
go func ( ) { return errors . New ( "unable to create codes indexer queue" )
// Wait to start, if we have the Cond for it. This is currently only
// useful for testing, so that the start and release of queued entries
// can be controlled to examine the queue.
if archiveQueueStartCond != nil {
archiveQueueMutex . Lock ( )
archiveQueueStartCond . Wait ( )
archiveQueueMutex . Unlock ( )
} }
// Drop the mutex while we process the request. This may take a long
go graceful . GetManager ( ) . RunWithShutdownFns ( archiverQueue . Run )
// time, and it's not necessary now that we've added the reequest to
// archiveInProgress.
doArchive ( request )
if archiveQueueReleaseCond != nil { return nil
archiveQueueMutex . Lock ( ) }
archiveQueueReleaseCond . Wait ( )
archiveQueueMutex . Unlock ( )
}
// Purge this request from the list. To do so, we'll just take the
// StartArchive push the archive request to the queue
// index at which we ended up at and swap the final element into that
func StartArchive ( request * ArchiveRequest ) error {
// position, then chop off the now-redundant final element. The slice
has , err := archiverQueue . Has ( request )
// may have change in between these two segments and we may have moved,
if err != nil {
// so we search for it here. We could perhaps avoid this search
return err
// entirely if len(archiveInProgress) == 1, but we should verify
// correctness.
archiveMutex . Lock ( )
defer archiveMutex . Unlock ( )
idx := - 1
for _idx , req := range archiveInProgress {
if req == request {
idx = _idx
break
}
} }
if idx == - 1 { if has {
log . Error ( "ArchiveRepository: Failed to find request for removal." ) return nil
return
} }
archiveInProgress = append ( archiveInProgress [ : idx ] , archiveInProgress [ idx + 1 : ] ... ) return archiverQueue . Push ( request )
} ( )
return request
} }
Lunny Xiao
3 years ago
committed by
GitHub