Support elastic search for code search (#10273)
* Support elastic search for code search * Finished elastic search implementation and add some tests * Enable test on drone and added docs * Add new fields to elastic search * Fix bug * remove unused changes * Use indexer alias to keep the gitea indexer version * Improve codes * Some code improvements * The real indexer name changed to xxx.v1 Co-authored-by: zeripath <art27@cantab.net>tokarchuk/v1.17
Lunny Xiao
4 years ago
committed by
GitHub
parent
d257485bc0
commit
9bc69ff26e
@ -0,0 +1,385 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package code |
||||
|
||||
import ( |
||||
"context" |
||||
"encoding/json" |
||||
"fmt" |
||||
"strconv" |
||||
"strings" |
||||
"time" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
"code.gitea.io/gitea/modules/analyze" |
||||
"code.gitea.io/gitea/modules/base" |
||||
"code.gitea.io/gitea/modules/charset" |
||||
"code.gitea.io/gitea/modules/git" |
||||
"code.gitea.io/gitea/modules/log" |
||||
"code.gitea.io/gitea/modules/setting" |
||||
"code.gitea.io/gitea/modules/timeutil" |
||||
|
||||
"github.com/go-enry/go-enry/v2" |
||||
"github.com/olivere/elastic/v7" |
||||
) |
||||
|
||||
const ( |
||||
esRepoIndexerLatestVersion = 1 |
||||
) |
||||
|
||||
var ( |
||||
_ Indexer = &ElasticSearchIndexer{} |
||||
) |
||||
|
||||
// ElasticSearchIndexer implements Indexer interface
|
||||
type ElasticSearchIndexer struct { |
||||
client *elastic.Client |
||||
indexerAliasName string |
||||
} |
||||
|
||||
type elasticLogger struct { |
||||
*log.Logger |
||||
} |
||||
|
||||
func (l elasticLogger) Printf(format string, args ...interface{}) { |
||||
_ = l.Logger.Log(2, l.Logger.GetLevel(), format, args...) |
||||
} |
||||
|
||||
// NewElasticSearchIndexer creates a new elasticsearch indexer
|
||||
func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, bool, error) { |
||||
opts := []elastic.ClientOptionFunc{ |
||||
elastic.SetURL(url), |
||||
elastic.SetSniff(false), |
||||
elastic.SetHealthcheckInterval(10 * time.Second), |
||||
elastic.SetGzip(false), |
||||
} |
||||
|
||||
logger := elasticLogger{log.GetLogger(log.DEFAULT)} |
||||
|
||||
if logger.GetLevel() == log.TRACE || logger.GetLevel() == log.DEBUG { |
||||
opts = append(opts, elastic.SetTraceLog(logger)) |
||||
} else if logger.GetLevel() == log.ERROR || logger.GetLevel() == log.CRITICAL || logger.GetLevel() == log.FATAL { |
||||
opts = append(opts, elastic.SetErrorLog(logger)) |
||||
} else if logger.GetLevel() == log.INFO || logger.GetLevel() == log.WARN { |
||||
opts = append(opts, elastic.SetInfoLog(logger)) |
||||
} |
||||
|
||||
client, err := elastic.NewClient(opts...) |
||||
if err != nil { |
||||
return nil, false, err |
||||
} |
||||
|
||||
indexer := &ElasticSearchIndexer{ |
||||
client: client, |
||||
indexerAliasName: indexerName, |
||||
} |
||||
exists, err := indexer.init() |
||||
|
||||
return indexer, !exists, err |
||||
} |
||||
|
||||
const ( |
||||
defaultMapping = `{ |
||||
"mappings": { |
||||
"properties": { |
||||
"repo_id": { |
||||
"type": "long", |
||||
"index": true |
||||
}, |
||||
"content": { |
||||
"type": "text", |
||||
"index": true |
||||
}, |
||||
"commit_id": { |
||||
"type": "keyword", |
||||
"index": true |
||||
}, |
||||
"language": { |
||||
"type": "keyword", |
||||
"index": true |
||||
}, |
||||
"updated_at": { |
||||
"type": "long", |
||||
"index": true |
||||
} |
||||
} |
||||
} |
||||
}` |
||||
) |
||||
|
||||
func (b *ElasticSearchIndexer) realIndexerName() string { |
||||
return fmt.Sprintf("%s.v%d", b.indexerAliasName, esRepoIndexerLatestVersion) |
||||
} |
||||
|
||||
// Init will initialize the indexer
|
||||
func (b *ElasticSearchIndexer) init() (bool, error) { |
||||
ctx := context.Background() |
||||
exists, err := b.client.IndexExists(b.realIndexerName()).Do(ctx) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
if !exists { |
||||
var mapping = defaultMapping |
||||
|
||||
createIndex, err := b.client.CreateIndex(b.realIndexerName()).BodyString(mapping).Do(ctx) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
if !createIndex.Acknowledged { |
||||
return false, fmt.Errorf("create index %s with %s failed", b.realIndexerName(), mapping) |
||||
} |
||||
} |
||||
|
||||
// check version
|
||||
r, err := b.client.Aliases().Do(ctx) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
|
||||
realIndexerNames := r.IndicesByAlias(b.indexerAliasName) |
||||
if len(realIndexerNames) < 1 { |
||||
res, err := b.client.Alias(). |
||||
Add(b.realIndexerName(), b.indexerAliasName). |
||||
Do(ctx) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
if !res.Acknowledged { |
||||
return false, fmt.Errorf("") |
||||
} |
||||
} else if len(realIndexerNames) >= 1 && realIndexerNames[0] < b.realIndexerName() { |
||||
log.Warn("Found older gitea indexer named %s, but we will create a new one %s and keep the old NOT DELETED. You can delete the old version after the upgrade succeed.", |
||||
realIndexerNames[0], b.realIndexerName()) |
||||
res, err := b.client.Alias(). |
||||
Remove(realIndexerNames[0], b.indexerAliasName). |
||||
Add(b.realIndexerName(), b.indexerAliasName). |
||||
Do(ctx) |
||||
if err != nil { |
||||
return false, err |
||||
} |
||||
if !res.Acknowledged { |
||||
return false, fmt.Errorf("") |
||||
} |
||||
} |
||||
|
||||
return exists, nil |
||||
} |
||||
|
||||
func (b *ElasticSearchIndexer) addUpdate(sha string, update fileUpdate, repo *models.Repository) ([]elastic.BulkableRequest, error) { |
||||
stdout, err := git.NewCommand("cat-file", "-s", update.BlobSha). |
||||
RunInDir(repo.RepoPath()) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if size, err := strconv.Atoi(strings.TrimSpace(stdout)); err != nil { |
||||
return nil, fmt.Errorf("Misformatted git cat-file output: %v", err) |
||||
} else if int64(size) > setting.Indexer.MaxIndexerFileSize { |
||||
return []elastic.BulkableRequest{b.addDelete(update.Filename, repo)}, nil |
||||
} |
||||
|
||||
fileContents, err := git.NewCommand("cat-file", "blob", update.BlobSha). |
||||
RunInDirBytes(repo.RepoPath()) |
||||
if err != nil { |
||||
return nil, err |
||||
} else if !base.IsTextFile(fileContents) { |
||||
// FIXME: UTF-16 files will probably fail here
|
||||
return nil, nil |
||||
} |
||||
|
||||
id := filenameIndexerID(repo.ID, update.Filename) |
||||
|
||||
return []elastic.BulkableRequest{ |
||||
elastic.NewBulkIndexRequest(). |
||||
Index(b.indexerAliasName). |
||||
Id(id). |
||||
Doc(map[string]interface{}{ |
||||
"repo_id": repo.ID, |
||||
"content": string(charset.ToUTF8DropErrors(fileContents)), |
||||
"commit_id": sha, |
||||
"language": analyze.GetCodeLanguage(update.Filename, fileContents), |
||||
"updated_at": timeutil.TimeStampNow(), |
||||
}), |
||||
}, nil |
||||
} |
||||
|
||||
func (b *ElasticSearchIndexer) addDelete(filename string, repo *models.Repository) elastic.BulkableRequest { |
||||
id := filenameIndexerID(repo.ID, filename) |
||||
return elastic.NewBulkDeleteRequest(). |
||||
Index(b.indexerAliasName). |
||||
Id(id) |
||||
} |
||||
|
||||
// Index will save the index data
|
||||
func (b *ElasticSearchIndexer) Index(repo *models.Repository, sha string, changes *repoChanges) error { |
||||
reqs := make([]elastic.BulkableRequest, 0) |
||||
for _, update := range changes.Updates { |
||||
updateReqs, err := b.addUpdate(sha, update, repo) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if len(updateReqs) > 0 { |
||||
reqs = append(reqs, updateReqs...) |
||||
} |
||||
} |
||||
|
||||
for _, filename := range changes.RemovedFilenames { |
||||
reqs = append(reqs, b.addDelete(filename, repo)) |
||||
} |
||||
|
||||
if len(reqs) > 0 { |
||||
_, err := b.client.Bulk(). |
||||
Index(b.indexerAliasName). |
||||
Add(reqs...). |
||||
Do(context.Background()) |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// Delete deletes indexes by ids
|
||||
func (b *ElasticSearchIndexer) Delete(repoID int64) error { |
||||
_, err := b.client.DeleteByQuery(b.indexerAliasName). |
||||
Query(elastic.NewTermsQuery("repo_id", repoID)). |
||||
Do(context.Background()) |
||||
return err |
||||
} |
||||
|
||||
func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) { |
||||
hits := make([]*SearchResult, 0, pageSize) |
||||
for _, hit := range searchResult.Hits.Hits { |
||||
// FIXME: There is no way to get the position the keyword on the content currently on the same request.
|
||||
// So we get it from content, this may made the query slower. See
|
||||
// https://discuss.elastic.co/t/fetching-position-of-keyword-in-matched-document/94291
|
||||
var startIndex, endIndex int = -1, -1 |
||||
c, ok := hit.Highlight["content"] |
||||
if ok && len(c) > 0 { |
||||
var subStr = make([]rune, 0, len(kw)) |
||||
startIndex = strings.IndexFunc(c[0], func(r rune) bool { |
||||
if len(subStr) >= len(kw) { |
||||
subStr = subStr[1:] |
||||
} |
||||
subStr = append(subStr, r) |
||||
return strings.EqualFold(kw, string(subStr)) |
||||
}) |
||||
if startIndex > -1 { |
||||
endIndex = startIndex + len(kw) |
||||
} else { |
||||
panic(fmt.Sprintf("1===%#v", hit.Highlight)) |
||||
} |
||||
} else { |
||||
panic(fmt.Sprintf("2===%#v", hit.Highlight)) |
||||
} |
||||
|
||||
repoID, fileName := parseIndexerID(hit.Id) |
||||
var res = make(map[string]interface{}) |
||||
if err := json.Unmarshal(hit.Source, &res); err != nil { |
||||
return 0, nil, nil, err |
||||
} |
||||
|
||||
language := res["language"].(string) |
||||
|
||||
hits = append(hits, &SearchResult{ |
||||
RepoID: repoID, |
||||
Filename: fileName, |
||||
CommitID: res["commit_id"].(string), |
||||
Content: res["content"].(string), |
||||
UpdatedUnix: timeutil.TimeStamp(res["updated_at"].(float64)), |
||||
Language: language, |
||||
StartIndex: startIndex, |
||||
EndIndex: endIndex, |
||||
Color: enry.GetColor(language), |
||||
}) |
||||
} |
||||
|
||||
return searchResult.TotalHits(), hits, extractAggs(searchResult), nil |
||||
} |
||||
|
||||
func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages { |
||||
var searchResultLanguages []*SearchResultLanguages |
||||
agg, found := searchResult.Aggregations.Terms("language") |
||||
if found { |
||||
searchResultLanguages = make([]*SearchResultLanguages, 0, 10) |
||||
|
||||
for _, bucket := range agg.Buckets { |
||||
searchResultLanguages = append(searchResultLanguages, &SearchResultLanguages{ |
||||
Language: bucket.Key.(string), |
||||
Color: enry.GetColor(bucket.Key.(string)), |
||||
Count: int(bucket.DocCount), |
||||
}) |
||||
} |
||||
} |
||||
return searchResultLanguages |
||||
} |
||||
|
||||
// Search searches for codes and language stats by given conditions.
|
||||
func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) { |
||||
kwQuery := elastic.NewMultiMatchQuery(keyword, "content") |
||||
query := elastic.NewBoolQuery() |
||||
query = query.Must(kwQuery) |
||||
if len(repoIDs) > 0 { |
||||
var repoStrs = make([]interface{}, 0, len(repoIDs)) |
||||
for _, repoID := range repoIDs { |
||||
repoStrs = append(repoStrs, repoID) |
||||
} |
||||
repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...) |
||||
query = query.Must(repoQuery) |
||||
} |
||||
|
||||
var ( |
||||
start int |
||||
kw = "<em>" + keyword + "</em>" |
||||
aggregation = elastic.NewTermsAggregation().Field("language").Size(10).OrderByCountDesc() |
||||
) |
||||
|
||||
if page > 0 { |
||||
start = (page - 1) * pageSize |
||||
} |
||||
|
||||
if len(language) == 0 { |
||||
searchResult, err := b.client.Search(). |
||||
Index(b.indexerAliasName). |
||||
Aggregation("language", aggregation). |
||||
Query(query). |
||||
Highlight(elastic.NewHighlight().Field("content")). |
||||
Sort("repo_id", true). |
||||
From(start).Size(pageSize). |
||||
Do(context.Background()) |
||||
if err != nil { |
||||
return 0, nil, nil, err |
||||
} |
||||
|
||||
return convertResult(searchResult, kw, pageSize) |
||||
} |
||||
|
||||
langQuery := elastic.NewMatchQuery("language", language) |
||||
countResult, err := b.client.Search(). |
||||
Index(b.indexerAliasName). |
||||
Aggregation("language", aggregation). |
||||
Query(query). |
||||
Size(0). // We only needs stats information
|
||||
Do(context.Background()) |
||||
if err != nil { |
||||
return 0, nil, nil, err |
||||
} |
||||
|
||||
query = query.Must(langQuery) |
||||
searchResult, err := b.client.Search(). |
||||
Index(b.indexerAliasName). |
||||
Query(query). |
||||
Highlight(elastic.NewHighlight().Field("content")). |
||||
Sort("repo_id", true). |
||||
From(start).Size(pageSize). |
||||
Do(context.Background()) |
||||
if err != nil { |
||||
return 0, nil, nil, err |
||||
} |
||||
|
||||
total, hits, _, err := convertResult(searchResult, kw, pageSize) |
||||
|
||||
return total, hits, extractAggs(countResult), err |
||||
} |
||||
|
||||
// Close implements indexer
|
||||
func (b *ElasticSearchIndexer) Close() {} |
||||
@ -0,0 +1,36 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package code |
||||
|
||||
import ( |
||||
"os" |
||||
"testing" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestESIndexAndSearch(t *testing.T) { |
||||
models.PrepareTestEnv(t) |
||||
|
||||
u := os.Getenv("TEST_INDEXER_CODE_ES_URL") |
||||
if u == "" { |
||||
t.SkipNow() |
||||
return |
||||
} |
||||
|
||||
indexer, _, err := NewElasticSearchIndexer(u, "gitea_codes") |
||||
if err != nil { |
||||
assert.Fail(t, "Unable to create ES indexer Error: %v", err) |
||||
if indexer != nil { |
||||
indexer.Close() |
||||
} |
||||
return |
||||
} |
||||
defer indexer.Close() |
||||
|
||||
testIndexer("elastic_search", t, indexer) |
||||
} |
||||
@ -0,0 +1,83 @@ |
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package code |
||||
|
||||
import ( |
||||
"path/filepath" |
||||
"testing" |
||||
|
||||
"code.gitea.io/gitea/models" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestMain(m *testing.M) { |
||||
models.MainTest(m, filepath.Join("..", "..", "..")) |
||||
} |
||||
|
||||
func testIndexer(name string, t *testing.T, indexer Indexer) { |
||||
t.Run(name, func(t *testing.T) { |
||||
var repoID int64 = 1 |
||||
err := index(indexer, repoID) |
||||
assert.NoError(t, err) |
||||
var ( |
||||
keywords = []struct { |
||||
RepoIDs []int64 |
||||
Keyword string |
||||
IDs []int64 |
||||
Langs int |
||||
}{ |
||||
{ |
||||
RepoIDs: nil, |
||||
Keyword: "Description", |
||||
IDs: []int64{repoID}, |
||||
Langs: 1, |
||||
}, |
||||
{ |
||||
RepoIDs: []int64{2}, |
||||
Keyword: "Description", |
||||
IDs: []int64{}, |
||||
Langs: 0, |
||||
}, |
||||
{ |
||||
RepoIDs: nil, |
||||
Keyword: "repo1", |
||||
IDs: []int64{repoID}, |
||||
Langs: 1, |
||||
}, |
||||
{ |
||||
RepoIDs: []int64{2}, |
||||
Keyword: "repo1", |
||||
IDs: []int64{}, |
||||
Langs: 0, |
||||
}, |
||||
{ |
||||
RepoIDs: nil, |
||||
Keyword: "non-exist", |
||||
IDs: []int64{}, |
||||
Langs: 0, |
||||
}, |
||||
} |
||||
) |
||||
|
||||
for _, kw := range keywords { |
||||
t.Run(kw.Keyword, func(t *testing.T) { |
||||
total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10) |
||||
assert.NoError(t, err) |
||||
assert.EqualValues(t, len(kw.IDs), total) |
||||
assert.EqualValues(t, kw.Langs, len(langs)) |
||||
|
||||
var ids = make([]int64, 0, len(res)) |
||||
for _, hit := range res { |
||||
ids = append(ids, hit.RepoID) |
||||
assert.EqualValues(t, "# repo1\n\nDescription for repo1", hit.Content) |
||||
} |
||||
assert.EqualValues(t, kw.IDs, ids) |
||||
}) |
||||
} |
||||
|
||||
assert.NoError(t, indexer.Delete(repoID)) |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue