Asynchronously populate the repo indexer (#3366)

* Populate repo indexer in background

* Check if no repos exist

* race cond
This commit is contained in:
Ethan Koenig 2018-01-14 09:34:41 -08:00 committed by Lauris BH
parent a8325dd1cb
commit ccdb94992d
1 changed files with 41 additions and 17 deletions

View File

@ -70,34 +70,58 @@ func InitRepoIndexer() {
if !setting.Indexer.RepoIndexerEnabled { if !setting.Indexer.RepoIndexerEnabled {
return return
} }
indexer.InitRepoIndexer(populateRepoIndexer)
repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength) repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
indexer.InitRepoIndexer(populateRepoIndexerAsynchronously)
go processRepoIndexerOperationQueue() go processRepoIndexerOperationQueue()
} }
// populateRepoIndexer populate the repo indexer with data // populateRepoIndexerAsynchronously asynchronously populates the repo indexer
func populateRepoIndexer() error { // with pre-existing data. This should only be run when the indexer is created
log.Info("Populating repository indexer (this may take a while)") // for the first time.
for page := 1; ; page++ { func populateRepoIndexerAsynchronously() error {
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{ exist, err := x.Table("repository").Exist()
Page: page, if err != nil {
PageSize: RepositoryListDefaultPageSize, return err
OrderBy: SearchOrderByID, } else if !exist {
Private: true, return nil
}) }
var maxRepoID int64
if _, err = x.Select("MAX(id)").Table("repository").Get(&maxRepoID); err != nil {
return err
}
go populateRepoIndexer(maxRepoID)
return nil
}
// populateRepoIndexer populate the repo indexer with pre-existing data. This
// should only be run when the indexer is created for the first time.
func populateRepoIndexer(maxRepoID int64) {
log.Info("Populating the repo indexer with existing repositories")
// start with the maximum existing repo ID and work backwards, so that we
// don't include repos that are created after gitea starts; such repos will
// already be added to the indexer, and we don't need to add them again.
for maxRepoID > 0 {
repos := make([]*Repository, 0, RepositoryListDefaultPageSize)
err := x.Where("id <= ?", maxRepoID).
OrderBy("id DESC").
Limit(RepositoryListDefaultPageSize).
Find(&repos)
if err != nil { if err != nil {
return err log.Error(4, "populateRepoIndexer: %v", err)
return
} else if len(repos) == 0 { } else if len(repos) == 0 {
return nil break
} }
for _, repo := range repos { for _, repo := range repos {
if err = updateRepoIndexer(repo); err != nil { repoIndexerOperationQueue <- repoIndexerOperation{
// only log error, since this should not prevent repo: repo,
// gitea from starting up deleted: false,
log.Error(4, "updateRepoIndexer: repoID=%d, %v", repo.ID, err)
} }
maxRepoID = repo.ID - 1
} }
} }
log.Info("Done populating the repo indexer with existing repositories")
} }
func updateRepoIndexer(repo *Repository) error { func updateRepoIndexer(repo *Repository) error {