Share HTML template renderers and create a watcher framework (#20218)
The recovery, API, Web and package frameworks all create their own HTML Renderers. This increases the memory requirements of Gitea unnecessarily with duplicate templates being kept in memory. Further the reloading framework in dev mode for these involves locking and recompiling all of the templates on each load. This will potentially hide concurrency issues and it is inefficient. This PR stores the templates renderer in the context and stores this context in the NormalRoutes, it then creates a fsnotify.Watcher framework to watch files. The watching framework is then extended to the mailer templates which were previously not being reloaded in dev. Then the locales are simplified to a similar structure. Fix #20210 Fix #20211 Fix #20217 Signed-off-by: Andrew Thornton <art27@cantab.net>tokarchuk/v1.18
zeripath
2 years ago
committed by
GitHub
parent
c21d6511a8
commit
bb0ff77e46
@ -0,0 +1,40 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package options |
||||
|
||||
import ( |
||||
"fmt" |
||||
"io/fs" |
||||
"os" |
||||
"path/filepath" |
||||
|
||||
"code.gitea.io/gitea/modules/util" |
||||
) |
||||
|
||||
func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error { |
||||
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { |
||||
// name is the path relative to the root
|
||||
name := path[len(root):] |
||||
if len(name) > 0 && name[0] == '/' { |
||||
name = name[1:] |
||||
} |
||||
if err != nil { |
||||
if os.IsNotExist(err) { |
||||
return callback(path, name, d, err) |
||||
} |
||||
return err |
||||
} |
||||
if util.CommonSkip(d.Name()) { |
||||
if d.IsDir() { |
||||
return fs.SkipDir |
||||
} |
||||
return nil |
||||
} |
||||
return callback(path, name, d, err) |
||||
}); err != nil && !os.IsNotExist(err) { |
||||
return fmt.Errorf("unable to get files for assets in %s: %w", root, err) |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,52 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package templates |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/watcher" |
||||
|
||||
"github.com/unrolled/render" |
||||
) |
||||
|
||||
var rendererKey interface{} = "templatesHtmlRendereer" |
||||
|
||||
// HTMLRenderer returns the current html renderer for the context or creates and stores one within the context for future use
|
||||
func HTMLRenderer(ctx context.Context) (context.Context, *render.Render) { |
||||
rendererInterface := ctx.Value(rendererKey) |
||||
if rendererInterface != nil { |
||||
renderer, ok := rendererInterface.(*render.Render) |
||||
if ok { |
||||
return ctx, renderer |
||||
} |
||||
} |
||||
|
||||
rendererType := "static" |
||||
if !setting.IsProd { |
||||
rendererType = "auto-reloading" |
||||
} |
||||
log.Log(1, log.DEBUG, "Creating "+rendererType+" HTML Renderer") |
||||
|
||||
renderer := render.New(render.Options{ |
||||
Extensions: []string{".tmpl"}, |
||||
Directory: "templates", |
||||
Funcs: NewFuncMap(), |
||||
Asset: GetAsset, |
||||
AssetNames: GetTemplateAssetNames, |
||||
UseMutexLock: !setting.IsProd, |
||||
IsDevelopment: false, |
||||
DisableHTTPErrorRendering: true, |
||||
}) |
||||
if !setting.IsProd { |
||||
watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{ |
||||
PathsCallback: walkTemplateFiles, |
||||
BetweenCallback: renderer.CompileTemplates, |
||||
}) |
||||
} |
||||
return context.WithValue(ctx, rendererKey, renderer), renderer |
||||
} |
||||
@ -0,0 +1,92 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package templates |
||||
|
||||
import ( |
||||
"context" |
||||
"html/template" |
||||
"io/fs" |
||||
"os" |
||||
"strings" |
||||
texttmpl "text/template" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/watcher" |
||||
) |
||||
|
||||
// Mailer provides the templates required for sending notification mails.
|
||||
func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) { |
||||
for _, funcs := range NewTextFuncMap() { |
||||
subjectTemplates.Funcs(funcs) |
||||
} |
||||
for _, funcs := range NewFuncMap() { |
||||
bodyTemplates.Funcs(funcs) |
||||
} |
||||
|
||||
refreshTemplates := func() { |
||||
for _, assetPath := range BuiltinAssetNames() { |
||||
if !strings.HasPrefix(assetPath, "mail/") { |
||||
continue |
||||
} |
||||
|
||||
if !strings.HasSuffix(assetPath, ".tmpl") { |
||||
continue |
||||
} |
||||
|
||||
content, err := BuiltinAsset(assetPath) |
||||
if err != nil { |
||||
log.Warn("Failed to read embedded %s template. %v", assetPath, err) |
||||
continue |
||||
} |
||||
|
||||
assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/") |
||||
|
||||
log.Trace("Adding built-in mailer template for %s", assetName) |
||||
buildSubjectBodyTemplate(subjectTemplates, |
||||
bodyTemplates, |
||||
assetName, |
||||
content) |
||||
} |
||||
|
||||
if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error { |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if d.IsDir() { |
||||
return nil |
||||
} |
||||
|
||||
content, err := os.ReadFile(path) |
||||
if err != nil { |
||||
log.Warn("Failed to read custom %s template. %v", path, err) |
||||
return nil |
||||
} |
||||
|
||||
assetName := strings.TrimSuffix(name, ".tmpl") |
||||
log.Trace("Adding mailer template for %s from %q", assetName, path) |
||||
buildSubjectBodyTemplate(subjectTemplates, |
||||
bodyTemplates, |
||||
assetName, |
||||
content) |
||||
return nil |
||||
}); err != nil && !os.IsNotExist(err) { |
||||
log.Warn("Error whilst walking mailer templates directories. %v", err) |
||||
} |
||||
} |
||||
|
||||
refreshTemplates() |
||||
|
||||
if !setting.IsProd { |
||||
// Now subjectTemplates and bodyTemplates are both synchronized
|
||||
// thus it is safe to call refresh from a different goroutine
|
||||
watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{ |
||||
PathsCallback: walkMailerTemplates, |
||||
BetweenCallback: refreshTemplates, |
||||
}) |
||||
} |
||||
|
||||
return subjectTemplates, bodyTemplates |
||||
} |
||||
@ -0,0 +1,12 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package i18n |
||||
|
||||
import "errors" |
||||
|
||||
var ( |
||||
ErrLocaleAlreadyExist = errors.New("lang already exists") |
||||
ErrUncertainArguments = errors.New("arguments to i18n should not contain uncertain slices") |
||||
) |
||||
@ -0,0 +1,42 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package i18n |
||||
|
||||
import ( |
||||
"fmt" |
||||
"reflect" |
||||
) |
||||
|
||||
// Format formats provided arguments for a given translated message
|
||||
func Format(format string, args ...interface{}) (msg string, err error) { |
||||
if len(args) == 0 { |
||||
return format, nil |
||||
} |
||||
|
||||
fmtArgs := make([]interface{}, 0, len(args)) |
||||
for _, arg := range args { |
||||
val := reflect.ValueOf(arg) |
||||
if val.Kind() == reflect.Slice { |
||||
// Previously, we would accept Tr(lang, key, a, [b, c], d, [e, f]) as Sprintf(msg, a, b, c, d, e, f)
|
||||
// but this is an unstable behavior.
|
||||
//
|
||||
// So we restrict the accepted arguments to either:
|
||||
//
|
||||
// 1. Tr(lang, key, [slice-items]) as Sprintf(msg, items...)
|
||||
// 2. Tr(lang, key, args...) as Sprintf(msg, args...)
|
||||
if len(args) == 1 { |
||||
for i := 0; i < val.Len(); i++ { |
||||
fmtArgs = append(fmtArgs, val.Index(i).Interface()) |
||||
} |
||||
} else { |
||||
err = ErrUncertainArguments |
||||
break |
||||
} |
||||
} else { |
||||
fmtArgs = append(fmtArgs, arg) |
||||
} |
||||
} |
||||
return fmt.Sprintf(format, fmtArgs...), err |
||||
} |
||||
@ -0,0 +1,161 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package i18n |
||||
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
|
||||
"gopkg.in/ini.v1" |
||||
) |
||||
|
||||
// This file implements the static LocaleStore that will not watch for changes
|
||||
|
||||
type locale struct { |
||||
store *localeStore |
||||
langName string |
||||
idxToMsgMap map[int]string // the map idx is generated by store's trKeyToIdxMap
|
||||
} |
||||
|
||||
type localeStore struct { |
||||
// After initializing has finished, these fields are read-only.
|
||||
langNames []string |
||||
langDescs []string |
||||
|
||||
localeMap map[string]*locale |
||||
trKeyToIdxMap map[string]int |
||||
|
||||
defaultLang string |
||||
} |
||||
|
||||
// NewLocaleStore creates a static locale store
|
||||
func NewLocaleStore() LocaleStore { |
||||
return &localeStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)} |
||||
} |
||||
|
||||
// AddLocaleByIni adds locale by ini into the store
|
||||
// if source is a string, then the file is loaded
|
||||
// if source is a []byte, then the content is used
|
||||
func (store *localeStore) AddLocaleByIni(langName, langDesc string, source interface{}) error { |
||||
if _, ok := store.localeMap[langName]; ok { |
||||
return ErrLocaleAlreadyExist |
||||
} |
||||
|
||||
store.langNames = append(store.langNames, langName) |
||||
store.langDescs = append(store.langDescs, langDesc) |
||||
|
||||
l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string)} |
||||
store.localeMap[l.langName] = l |
||||
|
||||
iniFile, err := ini.LoadSources(ini.LoadOptions{ |
||||
IgnoreInlineComment: true, |
||||
UnescapeValueCommentSymbols: true, |
||||
}, source) |
||||
if err != nil { |
||||
return fmt.Errorf("unable to load ini: %w", err) |
||||
} |
||||
iniFile.BlockMode = false |
||||
|
||||
for _, section := range iniFile.Sections() { |
||||
for _, key := range section.Keys() { |
||||
var trKey string |
||||
if section.Name() == "" || section.Name() == "DEFAULT" { |
||||
trKey = key.Name() |
||||
} else { |
||||
trKey = section.Name() + "." + key.Name() |
||||
} |
||||
idx, ok := store.trKeyToIdxMap[trKey] |
||||
if !ok { |
||||
idx = len(store.trKeyToIdxMap) |
||||
store.trKeyToIdxMap[trKey] = idx |
||||
} |
||||
l.idxToMsgMap[idx] = key.Value() |
||||
} |
||||
} |
||||
iniFile = nil |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (store *localeStore) HasLang(langName string) bool { |
||||
_, ok := store.localeMap[langName] |
||||
return ok |
||||
} |
||||
|
||||
func (store *localeStore) ListLangNameDesc() (names, desc []string) { |
||||
return store.langNames, store.langDescs |
||||
} |
||||
|
||||
// SetDefaultLang sets default language as a fallback
|
||||
func (store *localeStore) SetDefaultLang(lang string) { |
||||
store.defaultLang = lang |
||||
} |
||||
|
||||
// Tr translates content to target language. fall back to default language.
|
||||
func (store *localeStore) Tr(lang, trKey string, trArgs ...interface{}) string { |
||||
l, _ := store.Locale(lang) |
||||
|
||||
return l.Tr(trKey, trArgs...) |
||||
} |
||||
|
||||
// Has returns whether the given language has a translation for the provided key
|
||||
func (store *localeStore) Has(lang, trKey string) bool { |
||||
l, _ := store.Locale(lang) |
||||
|
||||
return l.Has(trKey) |
||||
} |
||||
|
||||
// Locale returns the locale for the lang or the default language
|
||||
func (store *localeStore) Locale(lang string) (Locale, bool) { |
||||
l, found := store.localeMap[lang] |
||||
if !found { |
||||
var ok bool |
||||
l, ok = store.localeMap[store.defaultLang] |
||||
if !ok { |
||||
// no default - return an empty locale
|
||||
l = &locale{store: store, idxToMsgMap: make(map[int]string)} |
||||
} |
||||
} |
||||
return l, found |
||||
} |
||||
|
||||
// Close implements io.Closer
|
||||
func (store *localeStore) Close() error { |
||||
return nil |
||||
} |
||||
|
||||
// Tr translates content to locale language. fall back to default language.
|
||||
func (l *locale) Tr(trKey string, trArgs ...interface{}) string { |
||||
format := trKey |
||||
|
||||
idx, ok := l.store.trKeyToIdxMap[trKey] |
||||
if ok { |
||||
if msg, ok := l.idxToMsgMap[idx]; ok { |
||||
format = msg // use the found translation
|
||||
} else if def, ok := l.store.localeMap[l.store.defaultLang]; ok { |
||||
// try to use default locale's translation
|
||||
if msg, ok := def.idxToMsgMap[idx]; ok { |
||||
format = msg |
||||
} |
||||
} |
||||
} |
||||
|
||||
msg, err := Format(format, trArgs...) |
||||
if err != nil { |
||||
log.Error("Error whilst formatting %q in %s: %v", trKey, l.langName, err) |
||||
} |
||||
return msg |
||||
} |
||||
|
||||
// Has returns whether a key is present in this locale or not
|
||||
func (l *locale) Has(trKey string) bool { |
||||
idx, ok := l.store.trKeyToIdxMap[trKey] |
||||
if !ok { |
||||
return false |
||||
} |
||||
_, ok = l.idxToMsgMap[idx] |
||||
return ok |
||||
} |
||||
@ -0,0 +1,115 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package watcher |
||||
|
||||
import ( |
||||
"context" |
||||
"io/fs" |
||||
"os" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/process" |
||||
|
||||
"github.com/fsnotify/fsnotify" |
||||
) |
||||
|
||||
// CreateWatcherOpts are options to configure the watcher
|
||||
type CreateWatcherOpts struct { |
||||
// PathsCallback is used to set the required paths to watch
|
||||
PathsCallback func(func(path, name string, d fs.DirEntry, err error) error) error |
||||
|
||||
// BeforeCallback is called before any files are watched
|
||||
BeforeCallback func() |
||||
|
||||
// Between Callback is called between after a watched event has occurred
|
||||
BetweenCallback func() |
||||
|
||||
// AfterCallback is called as this watcher ends
|
||||
AfterCallback func() |
||||
} |
||||
|
||||
// CreateWatcher creates a watcher labelled with the provided description and running with the provided options.
|
||||
// The created watcher will create a subcontext from the provided ctx and register it with the process manager.
|
||||
func CreateWatcher(ctx context.Context, desc string, opts *CreateWatcherOpts) { |
||||
go run(ctx, desc, opts) |
||||
} |
||||
|
||||
func run(ctx context.Context, desc string, opts *CreateWatcherOpts) { |
||||
if opts.BeforeCallback != nil { |
||||
opts.BeforeCallback() |
||||
} |
||||
if opts.AfterCallback != nil { |
||||
defer opts.AfterCallback() |
||||
} |
||||
ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Watcher: "+desc, process.SystemProcessType, true) |
||||
defer finished() |
||||
|
||||
log.Trace("Watcher loop starting for %s", desc) |
||||
defer log.Trace("Watcher loop ended for %s", desc) |
||||
|
||||
watcher, err := fsnotify.NewWatcher() |
||||
if err != nil { |
||||
log.Error("Unable to create watcher for %s: %v", desc, err) |
||||
return |
||||
} |
||||
if err := opts.PathsCallback(func(path, _ string, d fs.DirEntry, err error) error { |
||||
if err != nil && !os.IsNotExist(err) { |
||||
return err |
||||
} |
||||
log.Trace("Watcher: %s watching %q", desc, path) |
||||
_ = watcher.Add(path) |
||||
return nil |
||||
}); err != nil { |
||||
log.Error("Unable to create watcher for %s: %v", desc, err) |
||||
_ = watcher.Close() |
||||
return |
||||
} |
||||
|
||||
// Note we don't call the BetweenCallback here
|
||||
|
||||
for { |
||||
select { |
||||
case event, ok := <-watcher.Events: |
||||
if !ok { |
||||
_ = watcher.Close() |
||||
return |
||||
} |
||||
log.Debug("Watched file for %s had event: %v", desc, event) |
||||
case err, ok := <-watcher.Errors: |
||||
if !ok { |
||||
_ = watcher.Close() |
||||
return |
||||
} |
||||
log.Error("Error whilst watching files for %s: %v", desc, err) |
||||
case <-ctx.Done(): |
||||
_ = watcher.Close() |
||||
return |
||||
} |
||||
|
||||
// Recreate the watcher - only call the BetweenCallback after the new watcher is set-up
|
||||
_ = watcher.Close() |
||||
watcher, err = fsnotify.NewWatcher() |
||||
if err != nil { |
||||
log.Error("Unable to create watcher for %s: %v", desc, err) |
||||
return |
||||
} |
||||
if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error { |
||||
if err != nil { |
||||
return err |
||||
} |
||||
_ = watcher.Add(path) |
||||
return nil |
||||
}); err != nil { |
||||
log.Error("Unable to create watcher for %s: %v", desc, err) |
||||
_ = watcher.Close() |
||||
return |
||||
} |
||||
|
||||
// Inform our BetweenCallback that there has been an event
|
||||
if opts.BetweenCallback != nil { |
||||
opts.BetweenCallback() |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue