Storage configuration support `[storage]` (#13314)

* Fix minio bug

* Add tests for storage configuration

* Change the Seek flag to keep compitable minio?

* Fix test when first-byte-pos of all ranges is greater than the resource length

Co-authored-by: techknowlogick <techknowlogick@gitea.io>
tokarchuk/v1.17
Lunny Xiao 4 years ago committed by GitHub
parent e7750e0f6a
commit e4e85a3e51
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 29
      integrations/integration_test.go
  2. 3
      integrations/lfs_getobject_test.go
  3. 26
      integrations/mysql.ini.tmpl
  4. 23
      modules/lfs/content_store.go
  5. 4
      modules/lfs/server.go
  6. 6
      modules/setting/storage.go

@ -11,7 +11,6 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"log"
"net/http" "net/http"
"net/http/cookiejar" "net/http/cookiejar"
"net/http/httptest" "net/http/httptest"
@ -27,8 +26,10 @@ import (
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/routers" "code.gitea.io/gitea/routers"
"code.gitea.io/gitea/routers/routes" "code.gitea.io/gitea/routers/routes"
@ -59,6 +60,8 @@ func NewNilResponseRecorder() *NilResponseRecorder {
} }
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
defer log.Close()
managerCtx, cancel := context.WithCancel(context.Background()) managerCtx, cancel := context.WithCancel(context.Background())
graceful.InitManager(managerCtx) graceful.InitManager(managerCtx)
defer cancel() defer cancel()
@ -142,6 +145,10 @@ func initIntegrationTest() {
util.RemoveAll(models.LocalCopyPath()) util.RemoveAll(models.LocalCopyPath())
setting.CheckLFSVersion() setting.CheckLFSVersion()
setting.InitDBConfig() setting.InitDBConfig()
if err := storage.Init(); err != nil {
fmt.Printf("Init storage failed: %v", err)
os.Exit(1)
}
switch { switch {
case setting.Database.UseMySQL: case setting.Database.UseMySQL:
@ -149,27 +156,27 @@ func initIntegrationTest() {
setting.Database.User, setting.Database.Passwd, setting.Database.Host)) setting.Database.User, setting.Database.Passwd, setting.Database.Host))
defer db.Close() defer db.Close()
if err != nil { if err != nil {
log.Fatalf("sql.Open: %v", err) log.Fatal("sql.Open: %v", err)
} }
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil { if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
log.Fatalf("db.Exec: %v", err) log.Fatal("db.Exec: %v", err)
} }
case setting.Database.UsePostgreSQL: case setting.Database.UsePostgreSQL:
db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s", db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode)) setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
defer db.Close() defer db.Close()
if err != nil { if err != nil {
log.Fatalf("sql.Open: %v", err) log.Fatal("sql.Open: %v", err)
} }
dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name)) dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
if err != nil { if err != nil {
log.Fatalf("db.Query: %v", err) log.Fatal("db.Query: %v", err)
} }
defer dbrows.Close() defer dbrows.Close()
if !dbrows.Next() { if !dbrows.Next() {
if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil { if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
log.Fatalf("db.Exec: CREATE DATABASE: %v", err) log.Fatal("db.Exec: CREATE DATABASE: %v", err)
} }
} }
// Check if we need to setup a specific schema // Check if we need to setup a specific schema
@ -183,18 +190,18 @@ func initIntegrationTest() {
// This is a different db object; requires a different Close() // This is a different db object; requires a different Close()
defer db.Close() defer db.Close()
if err != nil { if err != nil {
log.Fatalf("sql.Open: %v", err) log.Fatal("sql.Open: %v", err)
} }
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema)) schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
if err != nil { if err != nil {
log.Fatalf("db.Query: %v", err) log.Fatal("db.Query: %v", err)
} }
defer schrows.Close() defer schrows.Close()
if !schrows.Next() { if !schrows.Next() {
// Create and setup a DB schema // Create and setup a DB schema
if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil { if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
log.Fatalf("db.Exec: CREATE SCHEMA: %v", err) log.Fatal("db.Exec: CREATE SCHEMA: %v", err)
} }
} }
@ -203,10 +210,10 @@ func initIntegrationTest() {
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
host, port, "master", setting.Database.User, setting.Database.Passwd)) host, port, "master", setting.Database.User, setting.Database.Passwd))
if err != nil { if err != nil {
log.Fatalf("sql.Open: %v", err) log.Fatal("sql.Open: %v", err)
} }
if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil { if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil {
log.Fatalf("db.Exec: %v", err) log.Fatal("db.Exec: %v", err)
} }
defer db.Close() defer db.Close()
} }

@ -78,6 +78,7 @@ func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, exp
} }
} }
} }
resp := session.MakeRequest(t, req, expectedStatus) resp := session.MakeRequest(t, req, expectedStatus)
return resp return resp
@ -210,7 +211,7 @@ func TestGetLFSRange(t *testing.T) {
{"bytes=0-10", "123456789\n", http.StatusPartialContent}, {"bytes=0-10", "123456789\n", http.StatusPartialContent},
// end-range bigger than length-1 is ignored // end-range bigger than length-1 is ignored
{"bytes=0-11", "123456789\n", http.StatusPartialContent}, {"bytes=0-11", "123456789\n", http.StatusPartialContent},
{"bytes=11-", "", http.StatusPartialContent}, {"bytes=11-", "Requested Range Not Satisfiable", http.StatusRequestedRangeNotSatisfiable},
// incorrect header value cause whole header to be ignored // incorrect header value cause whole header to be ignored
{"bytes=-", "123456789\n", http.StatusOK}, {"bytes=-", "123456789\n", http.StatusOK},
{"foobar", "123456789\n", http.StatusOK}, {"foobar", "123456789\n", http.StatusOK},

@ -45,19 +45,21 @@ START_SSH_SERVER = true
OFFLINE_MODE = false OFFLINE_MODE = false
LFS_START_SERVER = true LFS_START_SERVER = true
LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
LFS_STORE_TYPE = minio
LFS_SERVE_DIRECT = false [lfs]
LFS_MINIO_ENDPOINT = minio:9000 MINIO_BASE_PATH = lfs/
LFS_MINIO_ACCESS_KEY_ID = 123456
LFS_MINIO_SECRET_ACCESS_KEY = 12345678
LFS_MINIO_BUCKET = gitea
LFS_MINIO_LOCATION = us-east-1
LFS_MINIO_BASE_PATH = lfs/
LFS_MINIO_USE_SSL = false
[attachment] [attachment]
MINIO_BASE_PATH = attachments/
[avatars]
MINIO_BASE_PATH = avatars/
[repo-avatars]
MINIO_BASE_PATH = repo-avatars/
[storage]
STORAGE_TYPE = minio STORAGE_TYPE = minio
SERVE_DIRECT = false SERVE_DIRECT = false
MINIO_ENDPOINT = minio:9000 MINIO_ENDPOINT = minio:9000
@ -65,7 +67,6 @@ MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678 MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = gitea MINIO_BUCKET = gitea
MINIO_LOCATION = us-east-1 MINIO_LOCATION = us-east-1
MINIO_BASE_PATH = attachments/
MINIO_USE_SSL = false MINIO_USE_SSL = false
[mailer] [mailer]
@ -88,9 +89,6 @@ ENABLE_NOTIFY_MAIL = true
DISABLE_GRAVATAR = false DISABLE_GRAVATAR = false
ENABLE_FEDERATED_AVATAR = false ENABLE_FEDERATED_AVATAR = false
AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/avatars
REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/repo-avatars
[session] [session]
PROVIDER = file PROVIDER = file
PROVIDER_CONFIG = integrations/gitea-integration-mysql/data/sessions PROVIDER_CONFIG = integrations/gitea-integration-mysql/data/sessions

@ -8,6 +8,7 @@ import (
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"errors" "errors"
"fmt"
"io" "io"
"os" "os"
@ -21,6 +22,21 @@ var (
errSizeMismatch = errors.New("Content size does not match") errSizeMismatch = errors.New("Content size does not match")
) )
// ErrRangeNotSatisfiable represents an error which request range is not satisfiable.
type ErrRangeNotSatisfiable struct {
FromByte int64
}
func (err ErrRangeNotSatisfiable) Error() string {
return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
}
// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable
func IsErrRangeNotSatisfiable(err error) bool {
_, ok := err.(ErrRangeNotSatisfiable)
return ok
}
// ContentStore provides a simple file system based storage. // ContentStore provides a simple file system based storage.
type ContentStore struct { type ContentStore struct {
storage.ObjectStorage storage.ObjectStorage
@ -35,7 +51,12 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
return nil, err return nil, err
} }
if fromByte > 0 { if fromByte > 0 {
_, err = f.Seek(fromByte, os.SEEK_CUR) if fromByte >= meta.Size {
return nil, ErrRangeNotSatisfiable{
FromByte: fromByte,
}
}
_, err = f.Seek(fromByte, io.SeekStart)
if err != nil { if err != nil {
log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err) log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
} }

@ -191,8 +191,12 @@ func getContentHandler(ctx *context.Context) {
contentStore := &ContentStore{ObjectStorage: storage.LFS} contentStore := &ContentStore{ObjectStorage: storage.LFS}
content, err := contentStore.Get(meta, fromByte) content, err := contentStore.Get(meta, fromByte)
if err != nil { if err != nil {
if IsErrRangeNotSatisfiable(err) {
writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable)
} else {
// Errors are logged in contentStore.Get // Errors are logged in contentStore.Get
writeStatus(ctx, 404) writeStatus(ctx, 404)
}
return return
} }
defer content.Close() defer content.Close()

@ -32,14 +32,12 @@ func (s *Storage) MapTo(v interface{}) error {
} }
func getStorage(name, typ string, overrides ...*ini.Section) Storage { func getStorage(name, typ string, overrides ...*ini.Section) Storage {
sectionName := "storage" const sectionName = "storage"
if len(name) > 0 {
sectionName = sectionName + "." + typ
}
sec := Cfg.Section(sectionName) sec := Cfg.Section(sectionName)
if len(overrides) == 0 { if len(overrides) == 0 {
overrides = []*ini.Section{ overrides = []*ini.Section{
Cfg.Section(sectionName + "." + typ),
Cfg.Section(sectionName + "." + name), Cfg.Section(sectionName + "." + name),
} }
} }

Loading…
Cancel
Save