Validate migration files (#18203)
JSON Schema validation for data used by Gitea during migrations Discussion at https://forum.forgefriends.org/t/common-json-schema-for-repository-information/563 Co-authored-by: Loïc Dachary <loic@dachary.org>tokarchuk/v1.17
Aravinth Manivannan
3 years ago
committed by
GitHub
parent
49dd906753
commit
3bb028cc46
@ -0,0 +1,112 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migration |
||||
|
||||
import ( |
||||
"fmt" |
||||
"os" |
||||
"strings" |
||||
|
||||
"code.gitea.io/gitea/modules/json" |
||||
"code.gitea.io/gitea/modules/log" |
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v5" |
||||
"gopkg.in/yaml.v2" |
||||
) |
||||
|
||||
// Load project data from file, with optional validation
|
||||
func Load(filename string, data interface{}, validation bool) error { |
||||
isJSON := strings.HasSuffix(filename, ".json") |
||||
|
||||
bs, err := os.ReadFile(filename) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if validation { |
||||
err := validate(bs, data, isJSON) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return unmarshal(bs, data, isJSON) |
||||
} |
||||
|
||||
func unmarshal(bs []byte, data interface{}, isJSON bool) error { |
||||
if isJSON { |
||||
return json.Unmarshal(bs, data) |
||||
} |
||||
return yaml.Unmarshal(bs, data) |
||||
} |
||||
|
||||
func getSchema(filename string) (*jsonschema.Schema, error) { |
||||
c := jsonschema.NewCompiler() |
||||
c.LoadURL = openSchema |
||||
return c.Compile(filename) |
||||
} |
||||
|
||||
func validate(bs []byte, datatype interface{}, isJSON bool) error { |
||||
var v interface{} |
||||
err := unmarshal(bs, &v, isJSON) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if !isJSON { |
||||
v, err = toStringKeys(v) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
var schemaFilename string |
||||
switch datatype := datatype.(type) { |
||||
case *[]*Issue: |
||||
schemaFilename = "issue.json" |
||||
case *[]*Milestone: |
||||
schemaFilename = "milestone.json" |
||||
default: |
||||
return fmt.Errorf("file_format:validate: %T has not a validation implemented", datatype) |
||||
} |
||||
|
||||
sch, err := getSchema(schemaFilename) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
err = sch.Validate(v) |
||||
if err != nil { |
||||
log.Error("migration validation with %s failed for\n%s", schemaFilename, string(bs)) |
||||
} |
||||
return err |
||||
} |
||||
|
||||
func toStringKeys(val interface{}) (interface{}, error) { |
||||
var err error |
||||
switch val := val.(type) { |
||||
case map[interface{}]interface{}: |
||||
m := make(map[string]interface{}) |
||||
for k, v := range val { |
||||
k, ok := k.(string) |
||||
if !ok { |
||||
return nil, fmt.Errorf("found non-string key %T %s", k, k) |
||||
} |
||||
m[k], err = toStringKeys(v) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
return m, nil |
||||
case []interface{}: |
||||
l := make([]interface{}, len(val)) |
||||
for i, v := range val { |
||||
l[i], err = toStringKeys(v) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
return l, nil |
||||
default: |
||||
return val, nil |
||||
} |
||||
} |
||||
@ -0,0 +1,39 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migration |
||||
|
||||
import ( |
||||
"strings" |
||||
"testing" |
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v5" |
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestMigrationJSON_IssueOK(t *testing.T) { |
||||
issues := make([]*Issue, 0, 10) |
||||
err := Load("file_format_testdata/issue_a.json", &issues, true) |
||||
assert.NoError(t, err) |
||||
err = Load("file_format_testdata/issue_a.yml", &issues, true) |
||||
assert.NoError(t, err) |
||||
} |
||||
|
||||
func TestMigrationJSON_IssueFail(t *testing.T) { |
||||
issues := make([]*Issue, 0, 10) |
||||
err := Load("file_format_testdata/issue_b.json", &issues, true) |
||||
if _, ok := err.(*jsonschema.ValidationError); ok { |
||||
errors := strings.Split(err.(*jsonschema.ValidationError).GoString(), "\n") |
||||
assert.Contains(t, errors[1], "missing properties") |
||||
assert.Contains(t, errors[1], "poster_id") |
||||
} else { |
||||
t.Fatalf("got: type %T with value %s, want: *jsonschema.ValidationError", err, err) |
||||
} |
||||
} |
||||
|
||||
func TestMigrationJSON_MilestoneOK(t *testing.T) { |
||||
milestones := make([]*Milestone, 0, 10) |
||||
err := Load("file_format_testdata/milestones.json", &milestones, true) |
||||
assert.NoError(t, err) |
||||
} |
||||
@ -0,0 +1,14 @@ |
||||
[ |
||||
{ |
||||
"number": 1, |
||||
"poster_id": 1, |
||||
"poster_name": "name_a", |
||||
"title": "title_a", |
||||
"content": "content_a", |
||||
"state": "closed", |
||||
"is_locked": false, |
||||
"created": "1985-04-12T23:20:50.52Z", |
||||
"updated": "1986-04-12T23:20:50.52Z", |
||||
"closed": "1987-04-12T23:20:50.52Z" |
||||
} |
||||
] |
||||
@ -0,0 +1,10 @@ |
||||
- number: 1 |
||||
poster_id: 1 |
||||
poster_name: name_a |
||||
title: title_a |
||||
content: content_a |
||||
state: closed |
||||
is_locked: false |
||||
created: 2021-05-27T15:24:13+02:00 |
||||
updated: 2021-11-11T10:52:45+01:00 |
||||
closed: 2021-11-11T10:52:45+01:00 |
||||
@ -0,0 +1,5 @@ |
||||
[ |
||||
{ |
||||
"number": 1 |
||||
} |
||||
] |
||||
@ -0,0 +1,20 @@ |
||||
[ |
||||
{ |
||||
"title": "title_a", |
||||
"description": "description_a", |
||||
"deadline": "1988-04-12T23:20:50.52Z", |
||||
"created": "1985-04-12T23:20:50.52Z", |
||||
"updated": "1986-04-12T23:20:50.52Z", |
||||
"closed": "1987-04-12T23:20:50.52Z", |
||||
"state": "closed" |
||||
}, |
||||
{ |
||||
"title": "title_b", |
||||
"description": "description_b", |
||||
"deadline": "1998-04-12T23:20:50.52Z", |
||||
"created": "1995-04-12T23:20:50.52Z", |
||||
"updated": "1996-04-12T23:20:50.52Z", |
||||
"closed": null, |
||||
"state": "open" |
||||
} |
||||
] |
||||
@ -0,0 +1,114 @@ |
||||
{ |
||||
"title": "Issue", |
||||
"description": "Issues associated to a repository within a forge (Gitea, GitLab, etc.).", |
||||
|
||||
"type": "array", |
||||
"items": { |
||||
"type": "object", |
||||
"additionalProperties": false, |
||||
"properties": { |
||||
"number": { |
||||
"description": "Unique identifier, relative to the repository.", |
||||
"type": "number" |
||||
}, |
||||
"poster_id": { |
||||
"description": "Unique identifier of the user who authored the issue.", |
||||
"type": "number" |
||||
}, |
||||
"poster_name": { |
||||
"description": "Name of the user who authored the issue.", |
||||
"type": "string" |
||||
}, |
||||
"poster_email": { |
||||
"description": "Email of the user who authored the issue.", |
||||
"type": "string" |
||||
}, |
||||
"title": { |
||||
"description": "Short description displayed as the title.", |
||||
"type": "string" |
||||
}, |
||||
"content": { |
||||
"description": "Long, multiline, description.", |
||||
"type": "string" |
||||
}, |
||||
"ref": { |
||||
"description": "Target branch in the repository.", |
||||
"type": "string" |
||||
}, |
||||
"milestone": { |
||||
"description": "Name of the milestone.", |
||||
"type": "string" |
||||
}, |
||||
"state": { |
||||
"description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.", |
||||
"enum": [ |
||||
"closed", |
||||
"open" |
||||
] |
||||
}, |
||||
"is_locked": { |
||||
"description": "A locked issue can only be modified by privileged users.", |
||||
"type": "boolean" |
||||
}, |
||||
"created": { |
||||
"description": "Creation time.", |
||||
"type": "string", |
||||
"format": "date-time" |
||||
}, |
||||
"updated": { |
||||
"description": "Last update time.", |
||||
"type": "string", |
||||
"format": "date-time" |
||||
}, |
||||
"closed": { |
||||
"description": "The last time 'state' changed to 'closed'.", |
||||
"anyOf": [ |
||||
{ |
||||
"type": "string", |
||||
"format": "date-time" |
||||
}, |
||||
{ |
||||
"type": "null" |
||||
} |
||||
] |
||||
}, |
||||
"labels": { |
||||
"description": "List of labels.", |
||||
"type": "array", |
||||
"items": { |
||||
"$ref": "label.json" |
||||
} |
||||
}, |
||||
"reactions": { |
||||
"description": "List of reactions.", |
||||
"type": "array", |
||||
"items": { |
||||
"$ref": "reaction.json" |
||||
} |
||||
}, |
||||
"assignees": { |
||||
"description": "List of assignees.", |
||||
"type": "array", |
||||
"items": { |
||||
"description": "Name of a user assigned to the issue.", |
||||
"type": "string" |
||||
} |
||||
} |
||||
}, |
||||
"required": [ |
||||
"number", |
||||
"poster_id", |
||||
"poster_name", |
||||
"title", |
||||
"content", |
||||
"state", |
||||
"is_locked", |
||||
"created", |
||||
"updated" |
||||
] |
||||
}, |
||||
|
||||
"$schema": "http://json-schema.org/draft-04/schema#", |
||||
"$id": "http://example.com/issue.json", |
||||
"$$target": "issue.json" |
||||
} |
||||
@ -0,0 +1,28 @@ |
||||
{ |
||||
"title": "Label", |
||||
"description": "Label associated to an issue.", |
||||
|
||||
"type": "object", |
||||
"additionalProperties": false, |
||||
"properties": { |
||||
"name": { |
||||
"description": "Name of the label, unique within the repository.", |
||||
"type": "string" |
||||
}, |
||||
"color": { |
||||
"description": "Color code of the label.", |
||||
"type": "string" |
||||
}, |
||||
"description": { |
||||
"description": "Long, multiline, description.", |
||||
"type": "string" |
||||
} |
||||
}, |
||||
"required": [ |
||||
"name" |
||||
], |
||||
|
||||
"$schema": "http://json-schema.org/draft-04/schema#", |
||||
"$id": "label.json", |
||||
"$$target": "label.json" |
||||
} |
||||
@ -0,0 +1,67 @@ |
||||
{ |
||||
"title": "Milestone", |
||||
"description": "Milestone associated to a repository within a forge.", |
||||
|
||||
"type": "array", |
||||
"items": { |
||||
"type": "object", |
||||
"additionalProperties": false, |
||||
"properties": { |
||||
"title": { |
||||
"description": "Short description.", |
||||
"type": "string" |
||||
}, |
||||
"description": { |
||||
"description": "Long, multiline, description.", |
||||
"type": "string" |
||||
}, |
||||
"deadline": { |
||||
"description": "Deadline after which the milestone is overdue.", |
||||
"type": "string", |
||||
"format": "date-time" |
||||
}, |
||||
"created": { |
||||
"description": "Creation time.", |
||||
"type": "string", |
||||
"format": "date-time" |
||||
}, |
||||
"updated": { |
||||
"description": "Last update time.", |
||||
"type": "string", |
||||
"format": "date-time" |
||||
}, |
||||
"closed": { |
||||
"description": "The last time 'state' changed to 'closed'.", |
||||
"anyOf": [ |
||||
{ |
||||
"type": "string", |
||||
"format": "date-time" |
||||
}, |
||||
{ |
||||
"type": "null" |
||||
} |
||||
] |
||||
}, |
||||
"state": { |
||||
"description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.", |
||||
"enum": [ |
||||
"closed", |
||||
"open" |
||||
] |
||||
} |
||||
}, |
||||
"required": [ |
||||
"title", |
||||
"description", |
||||
"deadline", |
||||
"created", |
||||
"updated", |
||||
"closed", |
||||
"state" |
||||
] |
||||
}, |
||||
|
||||
"$schema": "http://json-schema.org/draft-04/schema#", |
||||
"$id": "http://example.com/milestone.json", |
||||
"$$target": "milestone.json" |
||||
} |
||||
@ -0,0 +1,29 @@ |
||||
{ |
||||
"title": "Reaction", |
||||
"description": "Reaction associated to an issue or a comment.", |
||||
|
||||
"type": "object", |
||||
"additionalProperties": false, |
||||
"properties": { |
||||
"user_id": { |
||||
"description": "Unique identifier of the user who authored the reaction.", |
||||
"type": "number" |
||||
}, |
||||
"user_name": { |
||||
"description": "Name of the user who authored the reaction.", |
||||
"type": "string" |
||||
}, |
||||
"content": { |
||||
"description": "Representation of the reaction", |
||||
"type": "string" |
||||
} |
||||
}, |
||||
"required": [ |
||||
"user_id", |
||||
"content" |
||||
], |
||||
|
||||
"$schema": "http://json-schema.org/draft-04/schema#", |
||||
"$id": "http://example.com/reaction.json", |
||||
"$$target": "reaction.json" |
||||
} |
||||
@ -0,0 +1,10 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build bindata
|
||||
// +build bindata
|
||||
|
||||
package migration |
||||
|
||||
//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas migration bindata.go
|
||||
@ -0,0 +1,40 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build !bindata
|
||||
// +build !bindata
|
||||
|
||||
package migration |
||||
|
||||
import ( |
||||
"io" |
||||
"net/url" |
||||
"os" |
||||
"path" |
||||
"path/filepath" |
||||
) |
||||
|
||||
func openSchema(s string) (io.ReadCloser, error) { |
||||
u, err := url.Parse(s) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
basename := path.Base(u.Path) |
||||
filename := basename |
||||
//
|
||||
// Schema reference each other within the schemas directory but
|
||||
// the tests run in the parent directory.
|
||||
//
|
||||
if _, err := os.Stat(filename); os.IsNotExist(err) { |
||||
filename = filepath.Join("schemas", basename) |
||||
//
|
||||
// Integration tests run from the git root directory, not the
|
||||
// directory in which the test source is located.
|
||||
//
|
||||
if _, err := os.Stat(filename); os.IsNotExist(err) { |
||||
filename = filepath.Join("modules/migration/schemas", basename) |
||||
} |
||||
} |
||||
return os.Open(filename) |
||||
} |
||||
@ -0,0 +1,17 @@ |
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
//go:build bindata
|
||||
// +build bindata
|
||||
|
||||
package migration |
||||
|
||||
import ( |
||||
"io" |
||||
"path" |
||||
) |
||||
|
||||
func openSchema(filename string) (io.ReadCloser, error) { |
||||
return Assets.Open(path.Base(filename)) |
||||
} |
||||
Loading…
Reference in new issue