From a1d796f521919e653f5ceb64d7d361a64cc0139f Mon Sep 17 00:00:00 2001
From: Lauris BH <lauris@nix.lv>
Date: Fri, 14 Feb 2020 14:42:30 +0200
Subject: [PATCH] Index code and stats only for non-empty repositories (#10251)

Fix test and switch to unique queue

Fix MySQL support when deleting old statistics
---
 models/fixtures/repository.yml |  1 +
 models/repo_indexer.go         |  8 +++++---
 models/repo_language_stats.go  | 13 +++++++++++--
 modules/indexer/stats/queue.go |  4 ++--
 4 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/models/fixtures/repository.yml b/models/fixtures/repository.yml
index 05989d903..3b86dd0f8 100644
--- a/models/fixtures/repository.yml
+++ b/models/fixtures/repository.yml
@@ -4,6 +4,7 @@
   owner_name: user2
   lower_name: repo1
   name: repo1
+  is_empty: false
   is_private: false
   num_issues: 2
   num_closed_issues: 1
diff --git a/models/repo_indexer.go b/models/repo_indexer.go
index 1f5ab928a..de3358f55 100644
--- a/models/repo_indexer.go
+++ b/models/repo_indexer.go
@@ -34,6 +34,8 @@ func GetUnindexedRepos(indexerType RepoIndexerType, maxRepoID int64, page, pageS
 	ids := make([]int64, 0, 50)
 	cond := builder.Cond(builder.IsNull{
 		"repo_indexer_status.id",
+	}).And(builder.Eq{
+		"repository.is_empty": false,
 	})
 	sess := x.Table("repository").Join("LEFT OUTER", "repo_indexer_status", "repository.id = repo_indexer_status.repo_id AND repo_indexer_status.indexer_type = ?", indexerType)
 	if maxRepoID > 0 {
@@ -66,11 +68,11 @@ func (repo *Repository) getIndexerStatus(e Engine, indexerType RepoIndexerType)
 			return repo.StatsIndexerStatus, nil
 		}
 	}
-	status := &RepoIndexerStatus{RepoID: repo.ID, IndexerType: indexerType}
-	has, err := e.Get(status)
-	if err != nil {
+	status := &RepoIndexerStatus{RepoID: repo.ID}
+	if has, err := e.Where("`indexer_type` = ?", indexerType).Get(status); err != nil {
 		return nil, err
 	} else if !has {
+		status.IndexerType = indexerType
 		status.CommitSha = ""
 	}
 	switch indexerType {
diff --git a/models/repo_language_stats.go b/models/repo_language_stats.go
index 4c3171e29..bf8f5f59d 100644
--- a/models/repo_language_stats.go
+++ b/models/repo_language_stats.go
@@ -125,10 +125,19 @@ func (repo *Repository) UpdateLanguageStats(commitID string, stats map[string]fl
 		}
 	}
 	// Delete old languages
-	if _, err := sess.Where("`id` IN (SELECT `id` FROM `language_stat` WHERE `repo_id` = ? AND `commit_id` != ?)", repo.ID, commitID).Delete(&LanguageStat{}); err != nil {
-		return err
+	statsToDelete := make([]int64, 0, len(oldstats))
+	for _, s := range oldstats {
+		if s.CommitID != commitID {
+			statsToDelete = append(statsToDelete, s.ID)
+		}
+	}
+	if len(statsToDelete) > 0 {
+		if _, err := sess.In("`id`", statsToDelete).Delete(&LanguageStat{}); err != nil {
+			return err
+		}
 	}
 
+	// Update indexer status
 	if err = repo.updateIndexerStatus(sess, RepoIndexerTypeStats, commitID); err != nil {
 		return err
 	}
diff --git a/modules/indexer/stats/queue.go b/modules/indexer/stats/queue.go
index 43a4de5ac..9c83f9b41 100644
--- a/modules/indexer/stats/queue.go
+++ b/modules/indexer/stats/queue.go
@@ -14,7 +14,7 @@ import (
 )
 
 // statsQueue represents a queue to handle repository stats updates
-var statsQueue queue.Queue
+var statsQueue queue.UniqueQueue
 
 // handle passed PR IDs and test the PRs
 func handle(data ...queue.Data) {
@@ -27,7 +27,7 @@ func handle(data ...queue.Data) {
 }
 
 func initStatsQueue() error {
-	statsQueue = queue.CreateQueue("repo_stats_update", handle, int64(0)).(queue.Queue)
+	statsQueue = queue.CreateUniqueQueue("repo_stats_update", handle, int64(0)).(queue.UniqueQueue)
 	if statsQueue == nil {
 		return fmt.Errorf("Unable to create repo_stats_update Queue")
 	}