Validate migration files (#18203)
JSON Schema validation for data used by Gitea during migrations Discussion at https://forum.forgefriends.org/t/common-json-schema-for-repository-information/563 Co-authored-by: Loïc Dachary <loic@dachary.org>tokarchuk/v1.17
parent
49dd906753
commit
3bb028cc46
@ -0,0 +1,112 @@ |
|||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package migration |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"os" |
||||||
|
"strings" |
||||||
|
|
||||||
|
"code.gitea.io/gitea/modules/json" |
||||||
|
"code.gitea.io/gitea/modules/log" |
||||||
|
|
||||||
|
"github.com/santhosh-tekuri/jsonschema/v5" |
||||||
|
"gopkg.in/yaml.v2" |
||||||
|
) |
||||||
|
|
||||||
|
// Load project data from file, with optional validation
|
||||||
|
func Load(filename string, data interface{}, validation bool) error { |
||||||
|
isJSON := strings.HasSuffix(filename, ".json") |
||||||
|
|
||||||
|
bs, err := os.ReadFile(filename) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
if validation { |
||||||
|
err := validate(bs, data, isJSON) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
} |
||||||
|
return unmarshal(bs, data, isJSON) |
||||||
|
} |
||||||
|
|
||||||
|
func unmarshal(bs []byte, data interface{}, isJSON bool) error { |
||||||
|
if isJSON { |
||||||
|
return json.Unmarshal(bs, data) |
||||||
|
} |
||||||
|
return yaml.Unmarshal(bs, data) |
||||||
|
} |
||||||
|
|
||||||
|
func getSchema(filename string) (*jsonschema.Schema, error) { |
||||||
|
c := jsonschema.NewCompiler() |
||||||
|
c.LoadURL = openSchema |
||||||
|
return c.Compile(filename) |
||||||
|
} |
||||||
|
|
||||||
|
func validate(bs []byte, datatype interface{}, isJSON bool) error { |
||||||
|
var v interface{} |
||||||
|
err := unmarshal(bs, &v, isJSON) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
if !isJSON { |
||||||
|
v, err = toStringKeys(v) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
var schemaFilename string |
||||||
|
switch datatype := datatype.(type) { |
||||||
|
case *[]*Issue: |
||||||
|
schemaFilename = "issue.json" |
||||||
|
case *[]*Milestone: |
||||||
|
schemaFilename = "milestone.json" |
||||||
|
default: |
||||||
|
return fmt.Errorf("file_format:validate: %T has not a validation implemented", datatype) |
||||||
|
} |
||||||
|
|
||||||
|
sch, err := getSchema(schemaFilename) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
err = sch.Validate(v) |
||||||
|
if err != nil { |
||||||
|
log.Error("migration validation with %s failed for\n%s", schemaFilename, string(bs)) |
||||||
|
} |
||||||
|
return err |
||||||
|
} |
||||||
|
|
||||||
|
func toStringKeys(val interface{}) (interface{}, error) { |
||||||
|
var err error |
||||||
|
switch val := val.(type) { |
||||||
|
case map[interface{}]interface{}: |
||||||
|
m := make(map[string]interface{}) |
||||||
|
for k, v := range val { |
||||||
|
k, ok := k.(string) |
||||||
|
if !ok { |
||||||
|
return nil, fmt.Errorf("found non-string key %T %s", k, k) |
||||||
|
} |
||||||
|
m[k], err = toStringKeys(v) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
} |
||||||
|
return m, nil |
||||||
|
case []interface{}: |
||||||
|
l := make([]interface{}, len(val)) |
||||||
|
for i, v := range val { |
||||||
|
l[i], err = toStringKeys(v) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
} |
||||||
|
return l, nil |
||||||
|
default: |
||||||
|
return val, nil |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,39 @@ |
|||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package migration |
||||||
|
|
||||||
|
import ( |
||||||
|
"strings" |
||||||
|
"testing" |
||||||
|
|
||||||
|
"github.com/santhosh-tekuri/jsonschema/v5" |
||||||
|
"github.com/stretchr/testify/assert" |
||||||
|
) |
||||||
|
|
||||||
|
func TestMigrationJSON_IssueOK(t *testing.T) { |
||||||
|
issues := make([]*Issue, 0, 10) |
||||||
|
err := Load("file_format_testdata/issue_a.json", &issues, true) |
||||||
|
assert.NoError(t, err) |
||||||
|
err = Load("file_format_testdata/issue_a.yml", &issues, true) |
||||||
|
assert.NoError(t, err) |
||||||
|
} |
||||||
|
|
||||||
|
func TestMigrationJSON_IssueFail(t *testing.T) { |
||||||
|
issues := make([]*Issue, 0, 10) |
||||||
|
err := Load("file_format_testdata/issue_b.json", &issues, true) |
||||||
|
if _, ok := err.(*jsonschema.ValidationError); ok { |
||||||
|
errors := strings.Split(err.(*jsonschema.ValidationError).GoString(), "\n") |
||||||
|
assert.Contains(t, errors[1], "missing properties") |
||||||
|
assert.Contains(t, errors[1], "poster_id") |
||||||
|
} else { |
||||||
|
t.Fatalf("got: type %T with value %s, want: *jsonschema.ValidationError", err, err) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func TestMigrationJSON_MilestoneOK(t *testing.T) { |
||||||
|
milestones := make([]*Milestone, 0, 10) |
||||||
|
err := Load("file_format_testdata/milestones.json", &milestones, true) |
||||||
|
assert.NoError(t, err) |
||||||
|
} |
@ -0,0 +1,14 @@ |
|||||||
|
[ |
||||||
|
{ |
||||||
|
"number": 1, |
||||||
|
"poster_id": 1, |
||||||
|
"poster_name": "name_a", |
||||||
|
"title": "title_a", |
||||||
|
"content": "content_a", |
||||||
|
"state": "closed", |
||||||
|
"is_locked": false, |
||||||
|
"created": "1985-04-12T23:20:50.52Z", |
||||||
|
"updated": "1986-04-12T23:20:50.52Z", |
||||||
|
"closed": "1987-04-12T23:20:50.52Z" |
||||||
|
} |
||||||
|
] |
@ -0,0 +1,10 @@ |
|||||||
|
- number: 1 |
||||||
|
poster_id: 1 |
||||||
|
poster_name: name_a |
||||||
|
title: title_a |
||||||
|
content: content_a |
||||||
|
state: closed |
||||||
|
is_locked: false |
||||||
|
created: 2021-05-27T15:24:13+02:00 |
||||||
|
updated: 2021-11-11T10:52:45+01:00 |
||||||
|
closed: 2021-11-11T10:52:45+01:00 |
@ -0,0 +1,5 @@ |
|||||||
|
[ |
||||||
|
{ |
||||||
|
"number": 1 |
||||||
|
} |
||||||
|
] |
@ -0,0 +1,20 @@ |
|||||||
|
[ |
||||||
|
{ |
||||||
|
"title": "title_a", |
||||||
|
"description": "description_a", |
||||||
|
"deadline": "1988-04-12T23:20:50.52Z", |
||||||
|
"created": "1985-04-12T23:20:50.52Z", |
||||||
|
"updated": "1986-04-12T23:20:50.52Z", |
||||||
|
"closed": "1987-04-12T23:20:50.52Z", |
||||||
|
"state": "closed" |
||||||
|
}, |
||||||
|
{ |
||||||
|
"title": "title_b", |
||||||
|
"description": "description_b", |
||||||
|
"deadline": "1998-04-12T23:20:50.52Z", |
||||||
|
"created": "1995-04-12T23:20:50.52Z", |
||||||
|
"updated": "1996-04-12T23:20:50.52Z", |
||||||
|
"closed": null, |
||||||
|
"state": "open" |
||||||
|
} |
||||||
|
] |
@ -0,0 +1,114 @@ |
|||||||
|
{ |
||||||
|
"title": "Issue", |
||||||
|
"description": "Issues associated to a repository within a forge (Gitea, GitLab, etc.).", |
||||||
|
|
||||||
|
"type": "array", |
||||||
|
"items": { |
||||||
|
"type": "object", |
||||||
|
"additionalProperties": false, |
||||||
|
"properties": { |
||||||
|
"number": { |
||||||
|
"description": "Unique identifier, relative to the repository.", |
||||||
|
"type": "number" |
||||||
|
}, |
||||||
|
"poster_id": { |
||||||
|
"description": "Unique identifier of the user who authored the issue.", |
||||||
|
"type": "number" |
||||||
|
}, |
||||||
|
"poster_name": { |
||||||
|
"description": "Name of the user who authored the issue.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"poster_email": { |
||||||
|
"description": "Email of the user who authored the issue.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"title": { |
||||||
|
"description": "Short description displayed as the title.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"content": { |
||||||
|
"description": "Long, multiline, description.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"ref": { |
||||||
|
"description": "Target branch in the repository.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"milestone": { |
||||||
|
"description": "Name of the milestone.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"state": { |
||||||
|
"description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.", |
||||||
|
"enum": [ |
||||||
|
"closed", |
||||||
|
"open" |
||||||
|
] |
||||||
|
}, |
||||||
|
"is_locked": { |
||||||
|
"description": "A locked issue can only be modified by privileged users.", |
||||||
|
"type": "boolean" |
||||||
|
}, |
||||||
|
"created": { |
||||||
|
"description": "Creation time.", |
||||||
|
"type": "string", |
||||||
|
"format": "date-time" |
||||||
|
}, |
||||||
|
"updated": { |
||||||
|
"description": "Last update time.", |
||||||
|
"type": "string", |
||||||
|
"format": "date-time" |
||||||
|
}, |
||||||
|
"closed": { |
||||||
|
"description": "The last time 'state' changed to 'closed'.", |
||||||
|
"anyOf": [ |
||||||
|
{ |
||||||
|
"type": "string", |
||||||
|
"format": "date-time" |
||||||
|
}, |
||||||
|
{ |
||||||
|
"type": "null" |
||||||
|
} |
||||||
|
] |
||||||
|
}, |
||||||
|
"labels": { |
||||||
|
"description": "List of labels.", |
||||||
|
"type": "array", |
||||||
|
"items": { |
||||||
|
"$ref": "label.json" |
||||||
|
} |
||||||
|
}, |
||||||
|
"reactions": { |
||||||
|
"description": "List of reactions.", |
||||||
|
"type": "array", |
||||||
|
"items": { |
||||||
|
"$ref": "reaction.json" |
||||||
|
} |
||||||
|
}, |
||||||
|
"assignees": { |
||||||
|
"description": "List of assignees.", |
||||||
|
"type": "array", |
||||||
|
"items": { |
||||||
|
"description": "Name of a user assigned to the issue.", |
||||||
|
"type": "string" |
||||||
|
} |
||||||
|
} |
||||||
|
}, |
||||||
|
"required": [ |
||||||
|
"number", |
||||||
|
"poster_id", |
||||||
|
"poster_name", |
||||||
|
"title", |
||||||
|
"content", |
||||||
|
"state", |
||||||
|
"is_locked", |
||||||
|
"created", |
||||||
|
"updated" |
||||||
|
] |
||||||
|
}, |
||||||
|
|
||||||
|
"$schema": "http://json-schema.org/draft-04/schema#", |
||||||
|
"$id": "http://example.com/issue.json", |
||||||
|
"$$target": "issue.json" |
||||||
|
} |
@ -0,0 +1,28 @@ |
|||||||
|
{ |
||||||
|
"title": "Label", |
||||||
|
"description": "Label associated to an issue.", |
||||||
|
|
||||||
|
"type": "object", |
||||||
|
"additionalProperties": false, |
||||||
|
"properties": { |
||||||
|
"name": { |
||||||
|
"description": "Name of the label, unique within the repository.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"color": { |
||||||
|
"description": "Color code of the label.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"description": { |
||||||
|
"description": "Long, multiline, description.", |
||||||
|
"type": "string" |
||||||
|
} |
||||||
|
}, |
||||||
|
"required": [ |
||||||
|
"name" |
||||||
|
], |
||||||
|
|
||||||
|
"$schema": "http://json-schema.org/draft-04/schema#", |
||||||
|
"$id": "label.json", |
||||||
|
"$$target": "label.json" |
||||||
|
} |
@ -0,0 +1,67 @@ |
|||||||
|
{ |
||||||
|
"title": "Milestone", |
||||||
|
"description": "Milestone associated to a repository within a forge.", |
||||||
|
|
||||||
|
"type": "array", |
||||||
|
"items": { |
||||||
|
"type": "object", |
||||||
|
"additionalProperties": false, |
||||||
|
"properties": { |
||||||
|
"title": { |
||||||
|
"description": "Short description.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"description": { |
||||||
|
"description": "Long, multiline, description.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"deadline": { |
||||||
|
"description": "Deadline after which the milestone is overdue.", |
||||||
|
"type": "string", |
||||||
|
"format": "date-time" |
||||||
|
}, |
||||||
|
"created": { |
||||||
|
"description": "Creation time.", |
||||||
|
"type": "string", |
||||||
|
"format": "date-time" |
||||||
|
}, |
||||||
|
"updated": { |
||||||
|
"description": "Last update time.", |
||||||
|
"type": "string", |
||||||
|
"format": "date-time" |
||||||
|
}, |
||||||
|
"closed": { |
||||||
|
"description": "The last time 'state' changed to 'closed'.", |
||||||
|
"anyOf": [ |
||||||
|
{ |
||||||
|
"type": "string", |
||||||
|
"format": "date-time" |
||||||
|
}, |
||||||
|
{ |
||||||
|
"type": "null" |
||||||
|
} |
||||||
|
] |
||||||
|
}, |
||||||
|
"state": { |
||||||
|
"description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.", |
||||||
|
"enum": [ |
||||||
|
"closed", |
||||||
|
"open" |
||||||
|
] |
||||||
|
} |
||||||
|
}, |
||||||
|
"required": [ |
||||||
|
"title", |
||||||
|
"description", |
||||||
|
"deadline", |
||||||
|
"created", |
||||||
|
"updated", |
||||||
|
"closed", |
||||||
|
"state" |
||||||
|
] |
||||||
|
}, |
||||||
|
|
||||||
|
"$schema": "http://json-schema.org/draft-04/schema#", |
||||||
|
"$id": "http://example.com/milestone.json", |
||||||
|
"$$target": "milestone.json" |
||||||
|
} |
@ -0,0 +1,29 @@ |
|||||||
|
{ |
||||||
|
"title": "Reaction", |
||||||
|
"description": "Reaction associated to an issue or a comment.", |
||||||
|
|
||||||
|
"type": "object", |
||||||
|
"additionalProperties": false, |
||||||
|
"properties": { |
||||||
|
"user_id": { |
||||||
|
"description": "Unique identifier of the user who authored the reaction.", |
||||||
|
"type": "number" |
||||||
|
}, |
||||||
|
"user_name": { |
||||||
|
"description": "Name of the user who authored the reaction.", |
||||||
|
"type": "string" |
||||||
|
}, |
||||||
|
"content": { |
||||||
|
"description": "Representation of the reaction", |
||||||
|
"type": "string" |
||||||
|
} |
||||||
|
}, |
||||||
|
"required": [ |
||||||
|
"user_id", |
||||||
|
"content" |
||||||
|
], |
||||||
|
|
||||||
|
"$schema": "http://json-schema.org/draft-04/schema#", |
||||||
|
"$id": "http://example.com/reaction.json", |
||||||
|
"$$target": "reaction.json" |
||||||
|
} |
@ -0,0 +1,10 @@ |
|||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
//go:build bindata
|
||||||
|
// +build bindata
|
||||||
|
|
||||||
|
package migration |
||||||
|
|
||||||
|
//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas migration bindata.go
|
@ -0,0 +1,40 @@ |
|||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
//go:build !bindata
|
||||||
|
// +build !bindata
|
||||||
|
|
||||||
|
package migration |
||||||
|
|
||||||
|
import ( |
||||||
|
"io" |
||||||
|
"net/url" |
||||||
|
"os" |
||||||
|
"path" |
||||||
|
"path/filepath" |
||||||
|
) |
||||||
|
|
||||||
|
func openSchema(s string) (io.ReadCloser, error) { |
||||||
|
u, err := url.Parse(s) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
basename := path.Base(u.Path) |
||||||
|
filename := basename |
||||||
|
//
|
||||||
|
// Schema reference each other within the schemas directory but
|
||||||
|
// the tests run in the parent directory.
|
||||||
|
//
|
||||||
|
if _, err := os.Stat(filename); os.IsNotExist(err) { |
||||||
|
filename = filepath.Join("schemas", basename) |
||||||
|
//
|
||||||
|
// Integration tests run from the git root directory, not the
|
||||||
|
// directory in which the test source is located.
|
||||||
|
//
|
||||||
|
if _, err := os.Stat(filename); os.IsNotExist(err) { |
||||||
|
filename = filepath.Join("modules/migration/schemas", basename) |
||||||
|
} |
||||||
|
} |
||||||
|
return os.Open(filename) |
||||||
|
} |
@ -0,0 +1,17 @@ |
|||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
//go:build bindata
|
||||||
|
// +build bindata
|
||||||
|
|
||||||
|
package migration |
||||||
|
|
||||||
|
import ( |
||||||
|
"io" |
||||||
|
"path" |
||||||
|
) |
||||||
|
|
||||||
|
func openSchema(filename string) (io.ReadCloser, error) { |
||||||
|
return Assets.Open(path.Base(filename)) |
||||||
|
} |
Loading…
Reference in new issue