From 91ff87c0e5caf9790cff7d2683b95a10250edcc2 Mon Sep 17 00:00:00 2001 From: Alexey Terentyev Date: Wed, 27 Jun 2018 08:23:10 +0300 Subject: [PATCH] Fixed violation of the unique constraint for v68 migration (#4297) --- models/migrations/v68.go | 147 ++++++++++++++++++++++++++------------- models/topic.go | 12 ++-- 2 files changed, 106 insertions(+), 53 deletions(-) diff --git a/models/migrations/v68.go b/models/migrations/v68.go index d6a0d04c5..e27b896c8 100644 --- a/models/migrations/v68.go +++ b/models/migrations/v68.go @@ -5,19 +5,47 @@ package migrations import ( + "fmt" + "regexp" "strings" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "github.com/go-xorm/xorm" ) +var topicPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +func validateTopic(topic string) bool { + return len(topic) <= 35 && topicPattern.MatchString(topic) +} + func reformatAndRemoveIncorrectTopics(x *xorm.Engine) (err error) { log.Info("This migration could take up to minutes, please be patient.") + type Topic struct { - ID int64 - Name string `xorm:"unique"` + ID int64 + Name string `xorm:"UNIQUE"` + RepoCount int + CreatedUnix int64 `xorm:"INDEX created"` + UpdatedUnix int64 `xorm:"INDEX updated"` + } + + type RepoTopic struct { + RepoID int64 `xorm:"UNIQUE(s)"` + TopicID int64 `xorm:"UNIQUE(s)"` + } + + type Repository struct { + ID int64 `xorm:"pk autoincr"` + Topics []string `xorm:"TEXT JSON"` + } + + if err := x.Sync2(new(Topic)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + if err := x.Sync2(new(RepoTopic)); err != nil { + return fmt.Errorf("Sync2: %v", err) } sess := x.NewSession() @@ -25,79 +53,99 @@ func reformatAndRemoveIncorrectTopics(x *xorm.Engine) (err error) { const batchSize = 100 touchedRepo := make(map[int64]struct{}) - topics := make([]*Topic, 0, batchSize) delTopicIDs := make([]int64, 0, batchSize) - ids := make([]int64, 0, 30) + log.Info("Validating existed topics...") if err := sess.Begin(); err != nil { return err } - log.Info("Validating existed topics...") for start := 0; ; start += batchSize { - topics = topics[:0] - if err := sess.Asc("id").Limit(batchSize, start).Find(&topics); err != nil { + topics := make([]*Topic, 0, batchSize) + if err := x.Cols("id", "name").Asc("id").Limit(batchSize, start).Find(&topics); err != nil { return err } if len(topics) == 0 { break } for _, topic := range topics { - if models.ValidateTopic(topic.Name) { + if validateTopic(topic.Name) { continue } + log.Info("Incorrect topic: id = %v, name = %q", topic.ID, topic.Name) + topic.Name = strings.Replace(strings.TrimSpace(strings.ToLower(topic.Name)), " ", "-", -1) + ids := make([]int64, 0, 30) if err := sess.Table("repo_topic").Cols("repo_id"). Where("topic_id = ?", topic.ID).Find(&ids); err != nil { return err } + log.Info("Touched repo ids: %v", ids) for _, id := range ids { touchedRepo[id] = struct{}{} } - if models.ValidateTopic(topic.Name) { - log.Info("Updating topic: id = %v, name = %v", topic.ID, topic.Name) - if _, err := sess.Table("topic").ID(topic.ID). - Update(&Topic{Name: topic.Name}); err != nil { + if validateTopic(topic.Name) { + unifiedTopic := Topic{Name: topic.Name} + exists, err := sess.Cols("id", "name").Get(&unifiedTopic) + log.Info("Exists topic with the name %q? %v, id = %v", topic.Name, exists, unifiedTopic.ID) + if err != nil { return err } - } else { - delTopicIDs = append(delTopicIDs, topic.ID) + if exists { + log.Info("Updating repo_topic rows with topic_id = %v to topic_id = %v", topic.ID, unifiedTopic.ID) + if _, err := sess.Where("topic_id = ? AND repo_id NOT IN "+ + "(SELECT rt1.repo_id FROM repo_topic rt1 INNER JOIN repo_topic rt2 "+ + "ON rt1.repo_id = rt2.repo_id WHERE rt1.topic_id = ? AND rt2.topic_id = ?)", + topic.ID, topic.ID, unifiedTopic.ID).Update(&RepoTopic{TopicID: unifiedTopic.ID}); err != nil { + return err + } + log.Info("Updating topic `repo_count` field") + if _, err := sess.Exec( + "UPDATE topic SET repo_count = (SELECT COUNT(*) FROM repo_topic WHERE topic_id = ? GROUP BY topic_id) WHERE id = ?", + unifiedTopic.ID, unifiedTopic.ID); err != nil { + return err + } + } else { + log.Info("Updating topic: id = %v, name = %q", topic.ID, topic.Name) + if _, err := sess.Table("topic").ID(topic.ID). + Update(&Topic{Name: topic.Name}); err != nil { + return err + } + continue + } } + delTopicIDs = append(delTopicIDs, topic.ID) } } + if err := sess.Commit(); err != nil { + return err + } + + sess.Init() log.Info("Deleting incorrect topics...") - for start := 0; ; start += batchSize { - if (start + batchSize) < len(delTopicIDs) { - ids = delTopicIDs[start:(start + batchSize)] - } else { - ids = delTopicIDs[start:] - } - - log.Info("Deleting 'repo_topic' rows for topics with ids = %v", ids) - if _, err := sess.In("topic_id", ids).Delete(&models.RepoTopic{}); err != nil { - return err - } - - log.Info("Deleting topics with id = %v", ids) - if _, err := sess.In("id", ids).Delete(&Topic{}); err != nil { - return err - } - - if len(ids) < batchSize { - break - } + if err := sess.Begin(); err != nil { + return err + } + log.Info("Deleting 'repo_topic' rows for topics with ids = %v", delTopicIDs) + if _, err := sess.In("topic_id", delTopicIDs).Delete(&RepoTopic{}); err != nil { + return err + } + log.Info("Deleting topics with id = %v", delTopicIDs) + if _, err := sess.In("id", delTopicIDs).Delete(&Topic{}); err != nil { + return err + } + if err := sess.Commit(); err != nil { + return err } - repoTopics := make([]*models.RepoTopic, 0, batchSize) - delRepoTopics := make([]*models.RepoTopic, 0, batchSize) - tmpRepoTopics := make([]*models.RepoTopic, 0, 30) + delRepoTopics := make([]*RepoTopic, 0, batchSize) log.Info("Checking the number of topics in the repositories...") for start := 0; ; start += batchSize { - repoTopics = repoTopics[:0] - if err := sess.Cols("repo_id").Asc("repo_id").Limit(batchSize, start). + repoTopics := make([]*RepoTopic, 0, batchSize) + if err := x.Cols("repo_id").Asc("repo_id").Limit(batchSize, start). GroupBy("repo_id").Having("COUNT(*) > 25").Find(&repoTopics); err != nil { return err } @@ -109,8 +157,8 @@ func reformatAndRemoveIncorrectTopics(x *xorm.Engine) (err error) { for _, repoTopic := range repoTopics { touchedRepo[repoTopic.RepoID] = struct{}{} - tmpRepoTopics = tmpRepoTopics[:0] - if err := sess.Where("repo_id = ?", repoTopic.RepoID).Find(&tmpRepoTopics); err != nil { + tmpRepoTopics := make([]*RepoTopic, 0, 30) + if err := x.Where("repo_id = ?", repoTopic.RepoID).Find(&tmpRepoTopics); err != nil { return err } @@ -122,13 +170,18 @@ func reformatAndRemoveIncorrectTopics(x *xorm.Engine) (err error) { } } + sess.Init() + log.Info("Deleting superfluous topics for repositories (more than 25 topics)...") + if err := sess.Begin(); err != nil { + return err + } for _, repoTopic := range delRepoTopics { log.Info("Deleting 'repo_topic' rows for 'repository' with id = %v. Topic id = %v", repoTopic.RepoID, repoTopic.TopicID) if _, err := sess.Where("repo_id = ? AND topic_id = ?", repoTopic.RepoID, - repoTopic.TopicID).Delete(&models.RepoTopic{}); err != nil { + repoTopic.TopicID).Delete(&RepoTopic{}); err != nil { return err } if _, err := sess.Exec( @@ -138,17 +191,17 @@ func reformatAndRemoveIncorrectTopics(x *xorm.Engine) (err error) { } } - topicNames := make([]string, 0, 30) log.Info("Updating repositories 'topics' fields...") for repoID := range touchedRepo { + topicNames := make([]string, 0, 30) if err := sess.Table("topic").Cols("name"). - Join("INNER", "repo_topic", "topic.id = repo_topic.topic_id"). - Where("repo_topic.repo_id = ?", repoID).Find(&topicNames); err != nil { + Join("INNER", "repo_topic", "repo_topic.topic_id = topic.id"). + Where("repo_topic.repo_id = ?", repoID).Desc("topic.repo_count").Find(&topicNames); err != nil { return err } log.Info("Updating 'topics' field for repository with id = %v", repoID) if _, err := sess.ID(repoID).Cols("topics"). - Update(&models.Repository{Topics: topicNames}); err != nil { + Update(&Repository{Topics: topicNames}); err != nil { return err } } diff --git a/models/topic.go b/models/topic.go index 247aac5ff..678795a3d 100644 --- a/models/topic.go +++ b/models/topic.go @@ -26,7 +26,7 @@ var topicPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) // Topic represents a topic of repositories type Topic struct { ID int64 - Name string `xorm:"unique"` + Name string `xorm:"UNIQUE"` RepoCount int CreatedUnix util.TimeStamp `xorm:"INDEX created"` UpdatedUnix util.TimeStamp `xorm:"INDEX updated"` @@ -34,8 +34,8 @@ type Topic struct { // RepoTopic represents associated repositories and topics type RepoTopic struct { - RepoID int64 `xorm:"unique(s)"` - TopicID int64 `xorm:"unique(s)"` + RepoID int64 `xorm:"UNIQUE(s)"` + TopicID int64 `xorm:"UNIQUE(s)"` } // ErrTopicNotExist represents an error that a topic is not exist @@ -190,10 +190,10 @@ func SaveTopics(repoID int64, topicNames ...string) error { } } - topicNames = topicNames[:0] + topicNames = make([]string, 0, 25) if err := sess.Table("topic").Cols("name"). - Join("INNER", "repo_topic", "topic.id = repo_topic.topic_id"). - Where("repo_topic.repo_id = ?", repoID).Find(&topicNames); err != nil { + Join("INNER", "repo_topic", "repo_topic.topic_id = topic.id"). + Where("repo_topic.repo_id = ?", repoID).Desc("topic.repo_count").Find(&topicNames); err != nil { return err }