third_party
/
gitea
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix postgres ID sequences broken by recreate-table (#15015)

Browse Source 
* Fix postgres ID sequences broken by recreate-table

Unfortunately there is a subtle problem with recreatetable on postgres which
leads to the sequences not being renamed and not being left at 0.

Fix #14725

Signed-off-by: Andrew Thornton <art27@cantab.net>

* let us try information_schema instead

Signed-off-by: Andrew Thornton <art27@cantab.net>

* try again

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Co-authored-by: 6543 <6543@obermui.de>
tokarchuk/v1.17
zeripath 4 years ago committed by GitHub
parent 71aca93dec
commit 070c57867d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 198 additions and 1 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 61
      models/consistency.go
  2. 65
      models/migrations/migrations.go
  3. 53
      models/migrations/v175.go
  4. 20
      modules/doctor/dbconsistency.go

61
models/consistency.go
Unescape View File

@ -5,10 +5,13 @@
package models
import (
"fmt"
"reflect"
"regexp"
"strings"
"testing"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
"xorm.io/builder"
)
@ -315,3 +318,61 @@ func CountCommentTypeLabelWithEmptyLabel() (int64, error) {
func FixCommentTypeLabelWithEmptyLabel() (int64, error) {
return x.Where(builder.Eq{"type": CommentTypeLabel, "label_id": 0}).Delete(new(Comment))
}
// CountBadSequences looks for broken sequences from recreate-table mistakes
func CountBadSequences() (int64, error) {
if !setting.Database.UsePostgreSQL {
return 0, nil
}
sess := x.NewSession()
defer sess.Close()
var sequences []string
schema := sess.Engine().Dialect().URI().Schema
sess.Engine().SetSchema("")
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
return 0, err
}
sess.Engine().SetSchema(schema)
return int64(len(sequences)), nil
}
// FixBadSequences fixes for broken sequences from recreate-table mistakes
func FixBadSequences() error {
if !setting.Database.UsePostgreSQL {
return nil
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
var sequences []string
schema := sess.Engine().Dialect().URI().Schema
sess.Engine().SetSchema("")
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
return err
}
sess.Engine().SetSchema(schema)
sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
for _, sequence := range sequences {
tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
newSequenceName := tableName + "_id_seq"
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
return err
}
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
return err
}
}
return sess.Commit()
}

65
models/migrations/migrations.go
Unescape View File

@ -296,6 +296,8 @@ var migrations = []Migration{
NewMigration("Add time_id column to Comment", addTimeIDCommentColumn),
// v174 -> v175
NewMigration("create repo transfer table", addRepoTransfer),
// v175 -> v176
NewMigration("Fix Postgres ID Sequences broken by recreate-table", fixPostgresIDSequences),
}
// GetCurrentDBVersion returns the current db version
@ -565,6 +567,31 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
return err
}
case setting.Database.UsePostgreSQL:
var originalSequences []string
type sequenceData struct {
LastValue int `xorm:"'last_value'"`
IsCalled bool `xorm:"'is_called'"`
}
sequenceMap := map[string]sequenceData{}
schema := sess.Engine().Dialect().URI().Schema
sess.Engine().SetSchema("")
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&originalSequences); err != nil {
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
return err
}
sess.Engine().SetSchema(schema)
for _, sequence := range originalSequences {
sequenceData := sequenceData{}
if _, err := sess.Table(sequence).Cols("last_value", "is_called").Get(&sequenceData); err != nil {
log.Error("Unable to get last_value and is_called from %s. Error: %v", sequence, err)
return err
}
sequenceMap[sequence] = sequenceData
}
// CASCADE causes postgres to drop all the constraints on the old table
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
log.Error("Unable to drop old table %s. Error: %v", tableName, err)
@ -578,7 +605,6 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
}
var indices []string
schema := sess.Engine().Dialect().URI().Schema
sess.Engine().SetSchema("")
if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
@ -594,6 +620,43 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
}
}
var sequences []string
sess.Engine().SetSchema("")
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&sequences); err != nil {
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
return err
}
sess.Engine().SetSchema(schema)
for _, sequence := range sequences {
newSequenceName := strings.Replace(sequence, "tmp_recreate__", "", 1)
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
log.Error("Unable to rename %s sequence to %s. Error: %v", sequence, newSequenceName, err)
return err
}
val, ok := sequenceMap[newSequenceName]
if newSequenceName == tableName+"_id_seq" {
if ok && val.LastValue != 0 {
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
return err
}
} else {
// We're going to try to guess this
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
log.Error("Unable to reset %s. Error: %v", newSequenceName, err)
return err
}
}
} else if ok {
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
return err
}
}
}
case setting.Database.UseMSSQL:
// MSSQL will drop all the constraints on the old table
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {

53
models/migrations/v175.go
Unescape View File

@ -0,0 +1,53 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"fmt"
"regexp"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"xorm.io/xorm"
)
func fixPostgresIDSequences(x *xorm.Engine) error {
if !setting.Database.UsePostgreSQL {
return nil
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
var sequences []string
schema := sess.Engine().Dialect().URI().Schema
sess.Engine().SetSchema("")
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
log.Error("Unable to find sequences: %v", err)
return err
}
sess.Engine().SetSchema(schema)
sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
for _, sequence := range sequences {
tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
newSequenceName := tableName + "_id_seq"
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
log.Error("Unable to rename %s to %s. Error: %v", sequence, newSequenceName, err)
return err
}
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
log.Error("Unable to reset sequence %s for %s. Error: %v", newSequenceName, tableName, err)
return err
}
}
return sess.Commit()
}

20
modules/doctor/dbconsistency.go
Unescape View File

@ -10,6 +10,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/migrations"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
func checkDBConsistency(logger log.Logger, autofix bool) error {
@ -131,6 +132,25 @@ func checkDBConsistency(logger log.Logger, autofix bool) error {
}
// TODO: function to recalc all counters
if setting.Database.UsePostgreSQL {
count, err = models.CountBadSequences()
if err != nil {
logger.Critical("Error: %v whilst checking sequence values")
}
if count > 0 {
if autofix {
err := models.FixBadSequences()
if err != nil {
logger.Critical("Error: %v whilst attempting to fix sequences")
return err
}
logger.Info("%d sequences updated", count)
} else {
logger.Warn("%d sequences with incorrect values", count)
}
}
}
return nil
}

Write Preview
Loading…
Cancel
Save