Remove unique filter from repo indexer analyzer. (#7878)
* Remove unique filter from repo indexer analyzer. * Bump repoIndexerLatestVersion to 4 * Corrrect fmt * make vendor to remove unique dependencytokarchuk/v1.17
parent
85202d4784
commit
5661773018
@ -1,53 +0,0 @@ |
|||||||
// Copyright (c) 2018 Couchbase, Inc.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package unique |
|
||||||
|
|
||||||
import ( |
|
||||||
"github.com/blevesearch/bleve/analysis" |
|
||||||
"github.com/blevesearch/bleve/registry" |
|
||||||
) |
|
||||||
|
|
||||||
const Name = "unique" |
|
||||||
|
|
||||||
// UniqueTermFilter retains only the tokens which mark the first occurrence of
|
|
||||||
// a term. Tokens whose term appears in a preceding token are dropped.
|
|
||||||
type UniqueTermFilter struct{} |
|
||||||
|
|
||||||
func NewUniqueTermFilter() *UniqueTermFilter { |
|
||||||
return &UniqueTermFilter{} |
|
||||||
} |
|
||||||
|
|
||||||
func (f *UniqueTermFilter) Filter(input analysis.TokenStream) analysis.TokenStream { |
|
||||||
encounteredTerms := make(map[string]struct{}, len(input)/4) |
|
||||||
j := 0 |
|
||||||
for _, token := range input { |
|
||||||
term := string(token.Term) |
|
||||||
if _, ok := encounteredTerms[term]; ok { |
|
||||||
continue |
|
||||||
} |
|
||||||
encounteredTerms[term] = struct{}{} |
|
||||||
input[j] = token |
|
||||||
j++ |
|
||||||
} |
|
||||||
return input[:j] |
|
||||||
} |
|
||||||
|
|
||||||
func UniqueTermFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { |
|
||||||
return NewUniqueTermFilter(), nil |
|
||||||
} |
|
||||||
|
|
||||||
func init() { |
|
||||||
registry.RegisterTokenFilter(Name, UniqueTermFilterConstructor) |
|
||||||
} |
|
Loading…
Reference in new issue