Implement "embedded" command to extract static resources (#9982)
* draft * Implement extract command * Fix nits and force args on extract * Add !bindata stub, support Windows, fmt * fix vendored flag * Remove leading slash for matching * Add docs * Fix typos * Add embedded view command Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>tokarchuk/v1.17
guillep2k
5 years ago
committed by
GitHub
parent
9b9dd19d7d
commit
bcb52aef09
@ -0,0 +1,332 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build bindata
|
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"errors" |
||||
"fmt" |
||||
"os" |
||||
"path/filepath" |
||||
"sort" |
||||
"strings" |
||||
|
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/options" |
||||
"code.gitea.io/gitea/modules/public" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/templates" |
||||
|
||||
"github.com/gobwas/glob" |
||||
"github.com/urfave/cli" |
||||
) |
||||
|
||||
// Cmdembedded represents the available extract sub-command.
|
||||
var ( |
||||
Cmdembedded = cli.Command{ |
||||
Name: "embedded", |
||||
Usage: "Extract embedded resources", |
||||
Description: "A command for extracting embedded resources, like templates and images", |
||||
Subcommands: []cli.Command{ |
||||
subcmdList, |
||||
subcmdView, |
||||
subcmdExtract, |
||||
}, |
||||
} |
||||
|
||||
subcmdList = cli.Command{ |
||||
Name: "list", |
||||
Usage: "List files matching the given pattern", |
||||
Action: runList, |
||||
Flags: []cli.Flag{ |
||||
cli.BoolFlag{ |
||||
Name: "include-vendored,vendor", |
||||
Usage: "Include files under public/vendor as well", |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
subcmdView = cli.Command{ |
||||
Name: "view", |
||||
Usage: "View a file matching the given pattern", |
||||
Action: runView, |
||||
Flags: []cli.Flag{ |
||||
cli.BoolFlag{ |
||||
Name: "include-vendored,vendor", |
||||
Usage: "Include files under public/vendor as well", |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
subcmdExtract = cli.Command{ |
||||
Name: "extract", |
||||
Usage: "Extract resources", |
||||
Action: runExtract, |
||||
Flags: []cli.Flag{ |
||||
cli.BoolFlag{ |
||||
Name: "include-vendored,vendor", |
||||
Usage: "Include files under public/vendor as well", |
||||
}, |
||||
cli.BoolFlag{ |
||||
Name: "overwrite", |
||||
Usage: "Overwrite files if they already exist", |
||||
}, |
||||
cli.BoolFlag{ |
||||
Name: "rename", |
||||
Usage: "Rename files as {name}.bak if they already exist (overwrites previous .bak)", |
||||
}, |
||||
cli.BoolFlag{ |
||||
Name: "custom", |
||||
Usage: "Extract to the 'custom' directory as per app.ini", |
||||
}, |
||||
cli.StringFlag{ |
||||
Name: "destination,dest-dir", |
||||
Usage: "Extract to the specified directory", |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
sections map[string]*section |
||||
assets []asset |
||||
) |
||||
|
||||
type section struct { |
||||
Path string |
||||
Names func() []string |
||||
IsDir func(string) (bool, error) |
||||
Asset func(string) ([]byte, error) |
||||
} |
||||
|
||||
type asset struct { |
||||
Section *section |
||||
Name string |
||||
Path string |
||||
} |
||||
|
||||
func initEmbeddedExtractor(c *cli.Context) error { |
||||
|
||||
// Silence the console logger
|
||||
log.DelNamedLogger("console") |
||||
log.DelNamedLogger(log.DEFAULT) |
||||
|
||||
// Read configuration file
|
||||
setting.NewContext() |
||||
|
||||
pats, err := getPatterns(c.Args()) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
sections := make(map[string]*section, 3) |
||||
|
||||
sections["public"] = §ion{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset} |
||||
sections["options"] = §ion{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset} |
||||
sections["templates"] = §ion{Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir, Asset: templates.Asset} |
||||
|
||||
for _, sec := range sections { |
||||
assets = append(assets, buildAssetList(sec, pats, c)...) |
||||
} |
||||
|
||||
// Sort assets
|
||||
sort.SliceStable(assets, func(i, j int) bool { |
||||
return assets[i].Path < assets[j].Path |
||||
}) |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func runList(c *cli.Context) error { |
||||
if err := runListDo(c); err != nil { |
||||
fmt.Fprintf(os.Stderr, "%v\n", err) |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func runView(c *cli.Context) error { |
||||
if err := runViewDo(c); err != nil { |
||||
fmt.Fprintf(os.Stderr, "%v\n", err) |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func runExtract(c *cli.Context) error { |
||||
if err := runExtractDo(c); err != nil { |
||||
fmt.Fprintf(os.Stderr, "%v\n", err) |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func runListDo(c *cli.Context) error { |
||||
if err := initEmbeddedExtractor(c); err != nil { |
||||
return err |
||||
} |
||||
|
||||
for _, a := range assets { |
||||
fmt.Println(a.Path) |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func runViewDo(c *cli.Context) error { |
||||
if err := initEmbeddedExtractor(c); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if len(assets) == 0 { |
||||
return fmt.Errorf("No files matched the given pattern") |
||||
} else if len(assets) > 1 { |
||||
return fmt.Errorf("Too many files matched the given pattern; try to be more specific") |
||||
} |
||||
|
||||
data, err := assets[0].Section.Asset(assets[0].Name) |
||||
if err != nil { |
||||
return fmt.Errorf("%s: %v", assets[0].Path, err) |
||||
} |
||||
|
||||
if _, err = os.Stdout.Write(data); err != nil { |
||||
return fmt.Errorf("%s: %v", assets[0].Path, err) |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func runExtractDo(c *cli.Context) error { |
||||
if err := initEmbeddedExtractor(c); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if len(c.Args()) == 0 { |
||||
return fmt.Errorf("A list of pattern of files to extract is mandatory (e.g. '**' for all)") |
||||
} |
||||
|
||||
destdir := "." |
||||
|
||||
if c.IsSet("destination") { |
||||
destdir = c.String("destination") |
||||
} else if c.Bool("custom") { |
||||
destdir = setting.CustomPath |
||||
fmt.Println("Using app.ini at", setting.CustomConf) |
||||
} |
||||
|
||||
fi, err := os.Stat(destdir) |
||||
if errors.Is(err, os.ErrNotExist) { |
||||
// In case Windows users attempt to provide a forward-slash path
|
||||
wdestdir := filepath.FromSlash(destdir) |
||||
if wfi, werr := os.Stat(wdestdir); werr == nil { |
||||
destdir = wdestdir |
||||
fi = wfi |
||||
err = nil |
||||
} |
||||
} |
||||
if err != nil { |
||||
return fmt.Errorf("%s: %s", destdir, err) |
||||
} else if !fi.IsDir() { |
||||
return fmt.Errorf("%s is not a directory.", destdir) |
||||
} |
||||
|
||||
fmt.Printf("Extracting to %s:\n", destdir) |
||||
|
||||
overwrite := c.Bool("overwrite") |
||||
rename := c.Bool("rename") |
||||
|
||||
for _, a := range assets { |
||||
if err := extractAsset(destdir, a, overwrite, rename); err != nil { |
||||
// Non-fatal error
|
||||
fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err) |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func extractAsset(d string, a asset, overwrite, rename bool) error { |
||||
dest := filepath.Join(d, filepath.FromSlash(a.Path)) |
||||
dir := filepath.Dir(dest) |
||||
|
||||
data, err := a.Section.Asset(a.Name) |
||||
if err != nil { |
||||
return fmt.Errorf("%s: %v", a.Path, err) |
||||
} |
||||
|
||||
if err := os.MkdirAll(dir, os.ModePerm); err != nil { |
||||
return fmt.Errorf("%s: %v", dir, err) |
||||
} |
||||
|
||||
perms := os.ModePerm & 0666 |
||||
|
||||
fi, err := os.Lstat(dest) |
||||
if err != nil { |
||||
if !errors.Is(err, os.ErrNotExist) { |
||||
return fmt.Errorf("%s: %v", dest, err) |
||||
} |
||||
} else if !overwrite && !rename { |
||||
fmt.Printf("%s already exists; skipped.\n", dest) |
||||
return nil |
||||
} else if !fi.Mode().IsRegular() { |
||||
return fmt.Errorf("%s already exists, but it's not a regular file", dest) |
||||
} else if rename { |
||||
if err := os.Rename(dest, dest+".bak"); err != nil { |
||||
return fmt.Errorf("Error creating backup for %s: %v", dest, err) |
||||
} |
||||
// Attempt to respect file permissions mask (even if user:group will be set anew)
|
||||
perms = fi.Mode() |
||||
} |
||||
|
||||
file, err := os.OpenFile(dest, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, perms) |
||||
if err != nil { |
||||
return fmt.Errorf("%s: %v", dest, err) |
||||
} |
||||
defer file.Close() |
||||
|
||||
if _, err = file.Write(data); err != nil { |
||||
return fmt.Errorf("%s: %v", dest, err) |
||||
} |
||||
|
||||
fmt.Println(dest) |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { |
||||
var results = make([]asset, 0, 64) |
||||
for _, name := range sec.Names() { |
||||
if isdir, err := sec.IsDir(name); !isdir && err == nil { |
||||
if sec.Path == "public" && |
||||
strings.HasPrefix(name, "vendor/") && |
||||
!c.Bool("include-vendored") { |
||||
continue |
||||
} |
||||
matchName := sec.Path + "/" + name |
||||
for _, g := range globs { |
||||
if g.Match(matchName) { |
||||
results = append(results, asset{Section: sec, |
||||
Name: name, |
||||
Path: sec.Path + "/" + name}) |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
return results |
||||
} |
||||
|
||||
func getPatterns(args []string) ([]glob.Glob, error) { |
||||
if len(args) == 0 { |
||||
args = []string{"**"} |
||||
} |
||||
pat := make([]glob.Glob, len(args)) |
||||
for i := range args { |
||||
if g, err := glob.Compile(args[i], '/'); err != nil { |
||||
return nil, fmt.Errorf("'%s': Invalid glob pattern: %v", args[i], err) |
||||
} else { |
||||
pat[i] = g |
||||
} |
||||
} |
||||
return pat, nil |
||||
} |
||||
@ -0,0 +1,30 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// +build !bindata
|
||||
|
||||
package cmd |
||||
|
||||
import ( |
||||
"fmt" |
||||
"os" |
||||
|
||||
"github.com/urfave/cli" |
||||
) |
||||
|
||||
// Cmdembedded represents the available extract sub-command.
|
||||
var ( |
||||
Cmdembedded = cli.Command{ |
||||
Name: "embedded", |
||||
Usage: "Extract embedded resources", |
||||
Description: "A command for extracting embedded resources, like templates and images", |
||||
Action: extractorNotImplemented, |
||||
} |
||||
) |
||||
|
||||
func extractorNotImplemented(c *cli.Context) error { |
||||
err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in builds without bindata") |
||||
fmt.Fprintf(os.Stderr, "%s\n", err) |
||||
return err |
||||
} |
||||
@ -0,0 +1,115 @@ |
||||
--- |
||||
date: "2020-01-25T21:00:00-03:00" |
||||
title: "Embedded data extraction tool" |
||||
slug: "cmd-embedded" |
||||
weight: 40 |
||||
toc: true |
||||
draft: false |
||||
menu: |
||||
sidebar: |
||||
parent: "advanced" |
||||
name: "Embedded data extraction tool" |
||||
weight: 40 |
||||
identifier: "cmd-embedded" |
||||
--- |
||||
|
||||
# Embedded data extraction tool |
||||
|
||||
Gitea's executable contains all the resources required to run: templates, images, style-sheets |
||||
and translations. Any of them can be overridden by placing a replacement in a matching path |
||||
inside the `custom` directory (see [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}})). |
||||
|
||||
To obtain a copy of the embedded resources ready for editing, the `embedded` command from the CLI |
||||
can be used from the OS shell interface. |
||||
|
||||
## Listing resources |
||||
|
||||
To list resources embedded in Gitea's executable, use the following syntax: |
||||
|
||||
``` |
||||
gitea embedded list [--include-vendored] [patterns...] |
||||
``` |
||||
|
||||
The `--include-vendored` flag makes the command include vendored files, which are |
||||
normally excluded; that is, files from external libraries that are required for Gitea |
||||
(e.g. [font-awesome](https://fontawesome.com/), [octicons](https://octicons.github.com/), etc). |
||||
|
||||
A list of file search patterns can be provided. Gitea uses [gobwas/glob](https://github.com/gobwas/glob) |
||||
for its glob syntax. Here are some examples: |
||||
|
||||
- List all template files, in any virtual directory: `**.tmpl` |
||||
- List all mail template files: `templates/mail/**.tmpl` |
||||
- List all files inside `public/img`: `public/img/**` |
||||
|
||||
Don't forget to use quotes for the patterns, as spaces, `*` and other characters might have |
||||
a special meaning for your command shell. |
||||
|
||||
If no pattern is provided, all files are listed. |
||||
|
||||
#### Example |
||||
|
||||
Listing all embedded files with `openid` in their path: |
||||
|
||||
``` |
||||
$ gitea embedded list '**openid**' |
||||
public/img/auth/openid_connect.png |
||||
public/img/openid-16x16.png |
||||
templates/user/auth/finalize_openid.tmpl |
||||
templates/user/auth/signin_openid.tmpl |
||||
templates/user/auth/signup_openid_connect.tmpl |
||||
templates/user/auth/signup_openid_navbar.tmpl |
||||
templates/user/auth/signup_openid_register.tmpl |
||||
templates/user/settings/security_openid.tmpl |
||||
``` |
||||
|
||||
## Extracting resources |
||||
|
||||
To extract resources embedded in Gitea's executable, use the following syntax: |
||||
|
||||
``` |
||||
gitea [--config {file}] embedded extract [--destination {dir}|--custom] [--overwrite|--rename] [--include-vendored] {patterns...} |
||||
``` |
||||
|
||||
The `--config` option tells gitea the location of the `app.ini` configuration file if |
||||
it's not in its default location. This option is only used with the `--custom` flag. |
||||
|
||||
The `--destination` option tells gitea the directory where the files must be extracted to. |
||||
The default is the current directory. |
||||
|
||||
The `--custom` flag tells gitea to extract the files directly into the `custom` directory. |
||||
For this to work, the command needs to know the location of the `app.ini` configuration |
||||
file (`--config`) and, depending of the configuration, be ran from the directory where |
||||
gitea normally starts. See [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}}) for details. |
||||
|
||||
The `--overwrite` flag allows any existing files in the destination directory to be overwritten. |
||||
|
||||
The `--rename` flag tells gitea to rename any existing files in the destination directory |
||||
as `filename.bak`. Previous `.bak` files are overwritten. |
||||
|
||||
At least one file search pattern must be provided; see `list` subcomand above for pattern |
||||
syntax and examples. |
||||
|
||||
#### Important notice |
||||
|
||||
Make sure to **only extract those files that require customization**. Files that |
||||
are present in the `custom` directory are not upgraded by Gitea's upgrade process. |
||||
When Gitea is upgraded to a new version (by replacing the executable), many of the |
||||
embedded files will suffer changes. Gitea will honor and use any files found |
||||
in the `custom` directory, even if they are old and incompatible. |
||||
|
||||
#### Example |
||||
|
||||
Extracting mail templates to a temporary directory: |
||||
|
||||
``` |
||||
$ mkdir tempdir |
||||
$ gitea embedded extract --destination tempdir 'templates/mail/**.tmpl' |
||||
Extracting to tempdir: |
||||
tempdir/templates/mail/auth/activate.tmpl |
||||
tempdir/templates/mail/auth/activate_email.tmpl |
||||
tempdir/templates/mail/auth/register_notify.tmpl |
||||
tempdir/templates/mail/auth/reset_passwd.tmpl |
||||
tempdir/templates/mail/issue/assigned.tmpl |
||||
tempdir/templates/mail/issue/default.tmpl |
||||
tempdir/templates/mail/notify/collaborator.tmpl |
||||
``` |
||||
Loading…
Reference in new issue