Upgrade blevesearch dependency to v2.0.1 (#14346)
* Upgrade blevesearch dependency to v2.0.1 * Update rupture to v1.0.0 * Fix testtokarchuk/v1.17
parent
3aa53dc6bc
commit
f5abe2f563
@ -1,110 +0,0 @@ |
|||||||
// Copyright (c) 2015 Couchbase, Inc.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package index |
|
||||||
|
|
||||||
import ( |
|
||||||
"reflect" |
|
||||||
|
|
||||||
"github.com/blevesearch/bleve/analysis" |
|
||||||
"github.com/blevesearch/bleve/document" |
|
||||||
"github.com/blevesearch/bleve/size" |
|
||||||
) |
|
||||||
|
|
||||||
var reflectStaticSizeAnalysisResult int |
|
||||||
|
|
||||||
func init() { |
|
||||||
var ar AnalysisResult |
|
||||||
reflectStaticSizeAnalysisResult = int(reflect.TypeOf(ar).Size()) |
|
||||||
} |
|
||||||
|
|
||||||
type IndexRow interface { |
|
||||||
KeySize() int |
|
||||||
KeyTo([]byte) (int, error) |
|
||||||
Key() []byte |
|
||||||
|
|
||||||
ValueSize() int |
|
||||||
ValueTo([]byte) (int, error) |
|
||||||
Value() []byte |
|
||||||
} |
|
||||||
|
|
||||||
type AnalysisResult struct { |
|
||||||
DocID string |
|
||||||
Rows []IndexRow |
|
||||||
|
|
||||||
// scorch
|
|
||||||
Document *document.Document |
|
||||||
Analyzed []analysis.TokenFrequencies |
|
||||||
Length []int |
|
||||||
} |
|
||||||
|
|
||||||
func (a *AnalysisResult) Size() int { |
|
||||||
rv := reflectStaticSizeAnalysisResult |
|
||||||
for _, analyzedI := range a.Analyzed { |
|
||||||
rv += analyzedI.Size() |
|
||||||
} |
|
||||||
rv += len(a.Length) * size.SizeOfInt |
|
||||||
return rv |
|
||||||
} |
|
||||||
|
|
||||||
type AnalysisWork struct { |
|
||||||
i Index |
|
||||||
d *document.Document |
|
||||||
rc chan *AnalysisResult |
|
||||||
} |
|
||||||
|
|
||||||
func NewAnalysisWork(i Index, d *document.Document, rc chan *AnalysisResult) *AnalysisWork { |
|
||||||
return &AnalysisWork{ |
|
||||||
i: i, |
|
||||||
d: d, |
|
||||||
rc: rc, |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
type AnalysisQueue struct { |
|
||||||
queue chan *AnalysisWork |
|
||||||
done chan struct{} |
|
||||||
} |
|
||||||
|
|
||||||
func (q *AnalysisQueue) Queue(work *AnalysisWork) { |
|
||||||
q.queue <- work |
|
||||||
} |
|
||||||
|
|
||||||
func (q *AnalysisQueue) Close() { |
|
||||||
close(q.done) |
|
||||||
} |
|
||||||
|
|
||||||
func NewAnalysisQueue(numWorkers int) *AnalysisQueue { |
|
||||||
rv := AnalysisQueue{ |
|
||||||
queue: make(chan *AnalysisWork), |
|
||||||
done: make(chan struct{}), |
|
||||||
} |
|
||||||
for i := 0; i < numWorkers; i++ { |
|
||||||
go AnalysisWorker(rv) |
|
||||||
} |
|
||||||
return &rv |
|
||||||
} |
|
||||||
|
|
||||||
func AnalysisWorker(q AnalysisQueue) { |
|
||||||
// read work off the queue
|
|
||||||
for { |
|
||||||
select { |
|
||||||
case <-q.done: |
|
||||||
return |
|
||||||
case w := <-q.queue: |
|
||||||
r := w.i.Analyze(w.d) |
|
||||||
w.rc <- r |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
@ -1,137 +0,0 @@ |
|||||||
// Copyright (c) 2017 Couchbase, Inc.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package segment |
|
||||||
|
|
||||||
import ( |
|
||||||
"github.com/RoaringBitmap/roaring" |
|
||||||
"github.com/blevesearch/bleve/index" |
|
||||||
"github.com/couchbase/vellum" |
|
||||||
) |
|
||||||
|
|
||||||
type EmptySegment struct{} |
|
||||||
|
|
||||||
func (e *EmptySegment) Dictionary(field string) (TermDictionary, error) { |
|
||||||
return &EmptyDictionary{}, nil |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptySegment) VisitDocument(num uint64, visitor DocumentFieldValueVisitor) error { |
|
||||||
return nil |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptySegment) DocID(num uint64) ([]byte, error) { |
|
||||||
return nil, nil |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptySegment) Count() uint64 { |
|
||||||
return 0 |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptySegment) DocNumbers([]string) (*roaring.Bitmap, error) { |
|
||||||
r := roaring.NewBitmap() |
|
||||||
return r, nil |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptySegment) Fields() []string { |
|
||||||
return []string{} |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptySegment) Close() error { |
|
||||||
return nil |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptySegment) Size() uint64 { |
|
||||||
return 0 |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptySegment) AddRef() { |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptySegment) DecRef() error { |
|
||||||
return nil |
|
||||||
} |
|
||||||
|
|
||||||
type EmptyDictionary struct{} |
|
||||||
|
|
||||||
func (e *EmptyDictionary) PostingsList(term []byte, |
|
||||||
except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error) { |
|
||||||
return &EmptyPostingsList{}, nil |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyDictionary) Iterator() DictionaryIterator { |
|
||||||
return &EmptyDictionaryIterator{} |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyDictionary) PrefixIterator(prefix string) DictionaryIterator { |
|
||||||
return &EmptyDictionaryIterator{} |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyDictionary) RangeIterator(start, end string) DictionaryIterator { |
|
||||||
return &EmptyDictionaryIterator{} |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyDictionary) AutomatonIterator(a vellum.Automaton, |
|
||||||
startKeyInclusive, endKeyExclusive []byte) DictionaryIterator { |
|
||||||
return &EmptyDictionaryIterator{} |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyDictionary) OnlyIterator(onlyTerms [][]byte, |
|
||||||
includeCount bool) DictionaryIterator { |
|
||||||
return &EmptyDictionaryIterator{} |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyDictionary) Contains(key []byte) (bool, error) { |
|
||||||
return false, nil |
|
||||||
} |
|
||||||
|
|
||||||
type EmptyDictionaryIterator struct{} |
|
||||||
|
|
||||||
func (e *EmptyDictionaryIterator) Next() (*index.DictEntry, error) { |
|
||||||
return nil, nil |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyDictionaryIterator) Contains(key []byte) (bool, error) { |
|
||||||
return false, nil |
|
||||||
} |
|
||||||
|
|
||||||
type EmptyPostingsList struct{} |
|
||||||
|
|
||||||
func (e *EmptyPostingsList) Iterator(includeFreq, includeNorm, includeLocations bool, |
|
||||||
prealloc PostingsIterator) PostingsIterator { |
|
||||||
return &EmptyPostingsIterator{} |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyPostingsList) Size() int { |
|
||||||
return 0 |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyPostingsList) Count() uint64 { |
|
||||||
return 0 |
|
||||||
} |
|
||||||
|
|
||||||
type EmptyPostingsIterator struct{} |
|
||||||
|
|
||||||
func (e *EmptyPostingsIterator) Next() (Posting, error) { |
|
||||||
return nil, nil |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyPostingsIterator) Advance(uint64) (Posting, error) { |
|
||||||
return nil, nil |
|
||||||
} |
|
||||||
|
|
||||||
func (e *EmptyPostingsIterator) Size() int { |
|
||||||
return 0 |
|
||||||
} |
|
||||||
|
|
||||||
var AnEmptyPostingsIterator = &EmptyPostingsIterator{} |
|
@ -1,176 +0,0 @@ |
|||||||
// Copyright 2014 The Cockroach Authors.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
// implied. See the License for the specific language governing
|
|
||||||
// permissions and limitations under the License.
|
|
||||||
|
|
||||||
// This code originated from:
|
|
||||||
// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go
|
|
||||||
|
|
||||||
// Modified to not use pkg/errors
|
|
||||||
|
|
||||||
package segment |
|
||||||
|
|
||||||
import ( |
|
||||||
"errors" |
|
||||||
"fmt" |
|
||||||
) |
|
||||||
|
|
||||||
const ( |
|
||||||
MaxVarintSize = 9 |
|
||||||
|
|
||||||
// IntMin is chosen such that the range of int tags does not overlap the
|
|
||||||
// ascii character set that is frequently used in testing.
|
|
||||||
IntMin = 0x80 // 128
|
|
||||||
intMaxWidth = 8 |
|
||||||
intZero = IntMin + intMaxWidth // 136
|
|
||||||
intSmall = IntMax - intZero - intMaxWidth // 109
|
|
||||||
// IntMax is the maximum int tag value.
|
|
||||||
IntMax = 0xfd // 253
|
|
||||||
) |
|
||||||
|
|
||||||
// EncodeUvarintAscending encodes the uint64 value using a variable length
|
|
||||||
// (length-prefixed) representation. The length is encoded as a single
|
|
||||||
// byte indicating the number of encoded bytes (-8) to follow. See
|
|
||||||
// EncodeVarintAscending for rationale. The encoded bytes are appended to the
|
|
||||||
// supplied buffer and the final buffer is returned.
|
|
||||||
func EncodeUvarintAscending(b []byte, v uint64) []byte { |
|
||||||
switch { |
|
||||||
case v <= intSmall: |
|
||||||
return append(b, intZero+byte(v)) |
|
||||||
case v <= 0xff: |
|
||||||
return append(b, IntMax-7, byte(v)) |
|
||||||
case v <= 0xffff: |
|
||||||
return append(b, IntMax-6, byte(v>>8), byte(v)) |
|
||||||
case v <= 0xffffff: |
|
||||||
return append(b, IntMax-5, byte(v>>16), byte(v>>8), byte(v)) |
|
||||||
case v <= 0xffffffff: |
|
||||||
return append(b, IntMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) |
|
||||||
case v <= 0xffffffffff: |
|
||||||
return append(b, IntMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), |
|
||||||
byte(v)) |
|
||||||
case v <= 0xffffffffffff: |
|
||||||
return append(b, IntMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), |
|
||||||
byte(v>>8), byte(v)) |
|
||||||
case v <= 0xffffffffffffff: |
|
||||||
return append(b, IntMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), |
|
||||||
byte(v>>16), byte(v>>8), byte(v)) |
|
||||||
default: |
|
||||||
return append(b, IntMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), |
|
||||||
byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
// DecodeUvarintAscending decodes a varint encoded uint64 from the input
|
|
||||||
// buffer. The remainder of the input buffer and the decoded uint64
|
|
||||||
// are returned.
|
|
||||||
func DecodeUvarintAscending(b []byte) ([]byte, uint64, error) { |
|
||||||
if len(b) == 0 { |
|
||||||
return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value") |
|
||||||
} |
|
||||||
length := int(b[0]) - intZero |
|
||||||
b = b[1:] // skip length byte
|
|
||||||
if length <= intSmall { |
|
||||||
return b, uint64(length), nil |
|
||||||
} |
|
||||||
length -= intSmall |
|
||||||
if length < 0 || length > 8 { |
|
||||||
return nil, 0, fmt.Errorf("invalid uvarint length of %d", length) |
|
||||||
} else if len(b) < length { |
|
||||||
return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b) |
|
||||||
} |
|
||||||
var v uint64 |
|
||||||
// It is faster to range over the elements in a slice than to index
|
|
||||||
// into the slice on each loop iteration.
|
|
||||||
for _, t := range b[:length] { |
|
||||||
v = (v << 8) | uint64(t) |
|
||||||
} |
|
||||||
return b[length:], v, nil |
|
||||||
} |
|
||||||
|
|
||||||
// ------------------------------------------------------------
|
|
||||||
|
|
||||||
type MemUvarintReader struct { |
|
||||||
C int // index of next byte to read from S
|
|
||||||
S []byte |
|
||||||
} |
|
||||||
|
|
||||||
func NewMemUvarintReader(s []byte) *MemUvarintReader { |
|
||||||
return &MemUvarintReader{S: s} |
|
||||||
} |
|
||||||
|
|
||||||
// Len returns the number of unread bytes.
|
|
||||||
func (r *MemUvarintReader) Len() int { |
|
||||||
n := len(r.S) - r.C |
|
||||||
if n < 0 { |
|
||||||
return 0 |
|
||||||
} |
|
||||||
return n |
|
||||||
} |
|
||||||
|
|
||||||
var ErrMemUvarintReaderOverflow = errors.New("MemUvarintReader overflow") |
|
||||||
|
|
||||||
// ReadUvarint reads an encoded uint64. The original code this was
|
|
||||||
// based on is at encoding/binary/ReadUvarint().
|
|
||||||
func (r *MemUvarintReader) ReadUvarint() (uint64, error) { |
|
||||||
var x uint64 |
|
||||||
var s uint |
|
||||||
var C = r.C |
|
||||||
var S = r.S |
|
||||||
|
|
||||||
for { |
|
||||||
b := S[C] |
|
||||||
C++ |
|
||||||
|
|
||||||
if b < 0x80 { |
|
||||||
r.C = C |
|
||||||
|
|
||||||
// why 63? The original code had an 'i += 1' loop var and
|
|
||||||
// checked for i > 9 || i == 9 ...; but, we no longer
|
|
||||||
// check for the i var, but instead check here for s,
|
|
||||||
// which is incremented by 7. So, 7*9 == 63.
|
|
||||||
//
|
|
||||||
// why the "extra" >= check? The normal case is that s <
|
|
||||||
// 63, so we check this single >= guard first so that we
|
|
||||||
// hit the normal, nil-error return pathway sooner.
|
|
||||||
if s >= 63 && (s > 63 || s == 63 && b > 1) { |
|
||||||
return 0, ErrMemUvarintReaderOverflow |
|
||||||
} |
|
||||||
|
|
||||||
return x | uint64(b)<<s, nil |
|
||||||
} |
|
||||||
|
|
||||||
x |= uint64(b&0x7f) << s |
|
||||||
s += 7 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
// SkipUvarint skips ahead one encoded uint64.
|
|
||||||
func (r *MemUvarintReader) SkipUvarint() { |
|
||||||
for { |
|
||||||
b := r.S[r.C] |
|
||||||
r.C++ |
|
||||||
|
|
||||||
if b < 0x80 { |
|
||||||
return |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
// SkipBytes skips a count number of bytes.
|
|
||||||
func (r *MemUvarintReader) SkipBytes(count int) { |
|
||||||
r.C = r.C + count |
|
||||||
} |
|
||||||
|
|
||||||
func (r *MemUvarintReader) Reset(s []byte) { |
|
||||||
r.C = 0 |
|
||||||
r.S = s |
|
||||||
} |
|
@ -1,58 +0,0 @@ |
|||||||
// Copyright (c) 2020 Couchbase, Inc.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package segment |
|
||||||
|
|
||||||
import ( |
|
||||||
"github.com/RoaringBitmap/roaring" |
|
||||||
"github.com/blevesearch/bleve/index" |
|
||||||
) |
|
||||||
|
|
||||||
// Plugin represents the essential functions required by a package to plug in
|
|
||||||
// it's segment implementation
|
|
||||||
type Plugin interface { |
|
||||||
|
|
||||||
// Type is the name for this segment plugin
|
|
||||||
Type() string |
|
||||||
|
|
||||||
// Version is a numeric value identifying a specific version of this type.
|
|
||||||
// When incompatible changes are made to a particular type of plugin, the
|
|
||||||
// version must be incremented.
|
|
||||||
Version() uint32 |
|
||||||
|
|
||||||
// New takes a set of AnalysisResults and turns them into a new Segment
|
|
||||||
New(results []*index.AnalysisResult) (Segment, uint64, error) |
|
||||||
|
|
||||||
// Open attempts to open the file at the specified path and
|
|
||||||
// return the corresponding Segment
|
|
||||||
Open(path string) (Segment, error) |
|
||||||
|
|
||||||
// Merge takes a set of Segments, and creates a new segment on disk at
|
|
||||||
// the specified path.
|
|
||||||
// Drops is a set of bitmaps (one for each segment) indicating which
|
|
||||||
// documents can be dropped from the segments during the merge.
|
|
||||||
// If the closeCh channel is closed, Merge will cease doing work at
|
|
||||||
// the next opportunity, and return an error (closed).
|
|
||||||
// StatsReporter can optionally be provided, in which case progress
|
|
||||||
// made during the merge is reported while operation continues.
|
|
||||||
// Returns:
|
|
||||||
// A slice of new document numbers (one for each input segment),
|
|
||||||
// this allows the caller to know a particular document's new
|
|
||||||
// document number in the newly merged segment.
|
|
||||||
// The number of bytes written to the new segment file.
|
|
||||||
// An error, if any occurred.
|
|
||||||
Merge(segments []Segment, drops []*roaring.Bitmap, path string, |
|
||||||
closeCh chan struct{}, s StatsReporter) ( |
|
||||||
[][]uint64, uint64, error) |
|
||||||
} |
|
@ -1,93 +0,0 @@ |
|||||||
// Copyright (c) 2019 Couchbase, Inc.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package scorch |
|
||||||
|
|
||||||
import ( |
|
||||||
"fmt" |
|
||||||
|
|
||||||
"github.com/blevesearch/bleve/index/scorch/segment" |
|
||||||
|
|
||||||
zapv11 "github.com/blevesearch/zap/v11" |
|
||||||
zapv12 "github.com/blevesearch/zap/v12" |
|
||||||
zapv13 "github.com/blevesearch/zap/v13" |
|
||||||
zapv14 "github.com/blevesearch/zap/v14" |
|
||||||
zapv15 "github.com/blevesearch/zap/v15" |
|
||||||
) |
|
||||||
|
|
||||||
var supportedSegmentPlugins map[string]map[uint32]segment.Plugin |
|
||||||
var defaultSegmentPlugin segment.Plugin |
|
||||||
|
|
||||||
func init() { |
|
||||||
ResetPlugins() |
|
||||||
RegisterPlugin(zapv15.Plugin(), false) |
|
||||||
RegisterPlugin(zapv14.Plugin(), false) |
|
||||||
RegisterPlugin(zapv13.Plugin(), false) |
|
||||||
RegisterPlugin(zapv12.Plugin(), false) |
|
||||||
RegisterPlugin(zapv11.Plugin(), true) |
|
||||||
} |
|
||||||
|
|
||||||
func ResetPlugins() { |
|
||||||
supportedSegmentPlugins = map[string]map[uint32]segment.Plugin{} |
|
||||||
} |
|
||||||
|
|
||||||
func RegisterPlugin(plugin segment.Plugin, makeDefault bool) { |
|
||||||
if _, ok := supportedSegmentPlugins[plugin.Type()]; !ok { |
|
||||||
supportedSegmentPlugins[plugin.Type()] = map[uint32]segment.Plugin{} |
|
||||||
} |
|
||||||
supportedSegmentPlugins[plugin.Type()][plugin.Version()] = plugin |
|
||||||
if makeDefault { |
|
||||||
defaultSegmentPlugin = plugin |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
func SupportedSegmentTypes() (rv []string) { |
|
||||||
for k := range supportedSegmentPlugins { |
|
||||||
rv = append(rv, k) |
|
||||||
} |
|
||||||
return |
|
||||||
} |
|
||||||
|
|
||||||
func SupportedSegmentTypeVersions(typ string) (rv []uint32) { |
|
||||||
for k := range supportedSegmentPlugins[typ] { |
|
||||||
rv = append(rv, k) |
|
||||||
} |
|
||||||
return rv |
|
||||||
} |
|
||||||
|
|
||||||
func chooseSegmentPlugin(forcedSegmentType string, |
|
||||||
forcedSegmentVersion uint32) (segment.Plugin, error) { |
|
||||||
if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok { |
|
||||||
if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok { |
|
||||||
return segPlugin, nil |
|
||||||
} |
|
||||||
return nil, fmt.Errorf( |
|
||||||
"unsupported version %d for segment type: %s, supported: %v", |
|
||||||
forcedSegmentVersion, forcedSegmentType, |
|
||||||
SupportedSegmentTypeVersions(forcedSegmentType)) |
|
||||||
} |
|
||||||
return nil, fmt.Errorf("unsupported segment type: %s, supported: %v", |
|
||||||
forcedSegmentType, SupportedSegmentTypes()) |
|
||||||
} |
|
||||||
|
|
||||||
func (s *Scorch) loadSegmentPlugin(forcedSegmentType string, |
|
||||||
forcedSegmentVersion uint32) error { |
|
||||||
segPlugin, err := chooseSegmentPlugin(forcedSegmentType, |
|
||||||
forcedSegmentVersion) |
|
||||||
if err != nil { |
|
||||||
return err |
|
||||||
} |
|
||||||
s.segPlugin = segPlugin |
|
||||||
return nil |
|
||||||
} |
|
@ -0,0 +1,70 @@ |
|||||||
|
// Copyright (c) 2014 Couchbase, Inc.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package analysis |
||||||
|
|
||||||
|
import ( |
||||||
|
index "github.com/blevesearch/bleve_index_api" |
||||||
|
) |
||||||
|
|
||||||
|
func TokenFrequency(tokens TokenStream, arrayPositions []uint64, options index.FieldIndexingOptions) index.TokenFrequencies { |
||||||
|
rv := make(map[string]*index.TokenFreq, len(tokens)) |
||||||
|
|
||||||
|
if options.IncludeTermVectors() { |
||||||
|
tls := make([]index.TokenLocation, len(tokens)) |
||||||
|
tlNext := 0 |
||||||
|
|
||||||
|
for _, token := range tokens { |
||||||
|
tls[tlNext] = index.TokenLocation{ |
||||||
|
ArrayPositions: arrayPositions, |
||||||
|
Start: token.Start, |
||||||
|
End: token.End, |
||||||
|
Position: token.Position, |
||||||
|
} |
||||||
|
|
||||||
|
curr, ok := rv[string(token.Term)] |
||||||
|
if ok { |
||||||
|
curr.Locations = append(curr.Locations, &tls[tlNext]) |
||||||
|
} else { |
||||||
|
curr = &index.TokenFreq{ |
||||||
|
Term: token.Term, |
||||||
|
Locations: []*index.TokenLocation{&tls[tlNext]}, |
||||||
|
} |
||||||
|
rv[string(token.Term)] = curr |
||||||
|
} |
||||||
|
|
||||||
|
if !options.SkipFreqNorm() { |
||||||
|
curr.SetFrequency(curr.Frequency() + 1) |
||||||
|
} |
||||||
|
|
||||||
|
tlNext++ |
||||||
|
} |
||||||
|
} else { |
||||||
|
for _, token := range tokens { |
||||||
|
curr, exists := rv[string(token.Term)] |
||||||
|
if !exists { |
||||||
|
curr = &index.TokenFreq{ |
||||||
|
Term: token.Term, |
||||||
|
} |
||||||
|
rv[string(token.Term)] = curr |
||||||
|
} |
||||||
|
|
||||||
|
if !options.SkipFreqNorm() { |
||||||
|
curr.SetFrequency(curr.Frequency() + 1) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return rv |
||||||
|
} |
@ -1,8 +1,8 @@ |
|||||||
package en |
package en |
||||||
|
|
||||||
import ( |
import ( |
||||||
"github.com/blevesearch/bleve/analysis" |
"github.com/blevesearch/bleve/v2/analysis" |
||||||
"github.com/blevesearch/bleve/registry" |
"github.com/blevesearch/bleve/v2/registry" |
||||||
) |
) |
||||||
|
|
||||||
const StopName = "stop_en" |
const StopName = "stop_en" |
@ -1,18 +1,20 @@ |
|||||||
module github.com/blevesearch/bleve |
module github.com/blevesearch/bleve/v2 |
||||||
|
|
||||||
go 1.13 |
go 1.13 |
||||||
|
|
||||||
require ( |
require ( |
||||||
github.com/RoaringBitmap/roaring v0.4.23 |
github.com/RoaringBitmap/roaring v0.4.23 |
||||||
github.com/blevesearch/blevex v1.0.0 |
github.com/blevesearch/bleve_index_api v1.0.0 |
||||||
github.com/blevesearch/go-porterstemmer v1.0.3 |
github.com/blevesearch/go-porterstemmer v1.0.3 |
||||||
|
github.com/blevesearch/scorch_segment_api v1.0.0 |
||||||
github.com/blevesearch/segment v0.9.0 |
github.com/blevesearch/segment v0.9.0 |
||||||
github.com/blevesearch/snowballstem v0.9.0 |
github.com/blevesearch/snowballstem v0.9.0 |
||||||
github.com/blevesearch/zap/v11 v11.0.14 |
github.com/blevesearch/upsidedown_store_api v1.0.1 |
||||||
github.com/blevesearch/zap/v12 v12.0.14 |
github.com/blevesearch/zapx/v11 v11.1.10 |
||||||
github.com/blevesearch/zap/v13 v13.0.6 |
github.com/blevesearch/zapx/v12 v12.1.10 |
||||||
github.com/blevesearch/zap/v14 v14.0.5 |
github.com/blevesearch/zapx/v13 v13.1.10 |
||||||
github.com/blevesearch/zap/v15 v15.0.3 |
github.com/blevesearch/zapx/v14 v14.1.10 |
||||||
|
github.com/blevesearch/zapx/v15 v15.1.10 |
||||||
github.com/couchbase/moss v0.1.0 |
github.com/couchbase/moss v0.1.0 |
||||||
github.com/couchbase/vellum v1.0.2 |
github.com/couchbase/vellum v1.0.2 |
||||||
github.com/golang/protobuf v1.3.2 |
github.com/golang/protobuf v1.3.2 |
@ -0,0 +1,128 @@ |
|||||||
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= |
||||||
|
github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo= |
||||||
|
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= |
||||||
|
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= |
||||||
|
github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= |
||||||
|
github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= |
||||||
|
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= |
||||||
|
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= |
||||||
|
github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= |
||||||
|
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= |
||||||
|
github.com/blevesearch/scorch_segment_api v1.0.0 h1:BUkCPWDg2gimTEyVDXf85I2buqqt4lh28uaVMiJsIYk= |
||||||
|
github.com/blevesearch/scorch_segment_api v1.0.0/go.mod h1:KgRYmlfYC27NeM6cXOHx8LBgq7jn0atpV8mVWoBKBng= |
||||||
|
github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= |
||||||
|
github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= |
||||||
|
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= |
||||||
|
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= |
||||||
|
github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU= |
||||||
|
github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q= |
||||||
|
github.com/blevesearch/zapx/v11 v11.1.10 h1:8Eo3rXiHsVSP9Sk+4StrrwLrj9vyulhMVPmxTf8ZuDg= |
||||||
|
github.com/blevesearch/zapx/v11 v11.1.10/go.mod h1:DTjbcBqrr/Uo82UBilDC8lEew42gN/OcIyiTNFtSijc= |
||||||
|
github.com/blevesearch/zapx/v12 v12.1.10 h1:sqR+/0Z4dSTovApRqLA1HnilMtQer7a4UvPrNmPzlTM= |
||||||
|
github.com/blevesearch/zapx/v12 v12.1.10/go.mod h1:14NmKnPrnKAIyiEJM566k/Jk+FQpuiflT5d3uaaK3MI= |
||||||
|
github.com/blevesearch/zapx/v13 v13.1.10 h1:zCneEVRJDXwtDfSwh+33Dxguliv192vCK283zdGH4Sw= |
||||||
|
github.com/blevesearch/zapx/v13 v13.1.10/go.mod h1:YsVY6YGpTEAlJOMjdL7EsdBLvjWd8kPa2gwJDNpqLJo= |
||||||
|
github.com/blevesearch/zapx/v14 v14.1.10 h1:nD0vw2jxKogJFfA5WyoS4wNwZlVby3Aq8aW7CZi6YIw= |
||||||
|
github.com/blevesearch/zapx/v14 v14.1.10/go.mod h1:hsULl5eJSxs5NEfBsmeT9qrqdCP+/ecpVZKt60M4V64= |
||||||
|
github.com/blevesearch/zapx/v15 v15.1.10 h1:kZR3b9jO9l6s2B5UHI+1N1llLzJ4nYikkXQTMrDl1vQ= |
||||||
|
github.com/blevesearch/zapx/v15 v15.1.10/go.mod h1:4ypq25bwtSQKzwEF1UERyIhmGTbMT3brY/n4NC5gRnM= |
||||||
|
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= |
||||||
|
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= |
||||||
|
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= |
||||||
|
github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= |
||||||
|
github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= |
||||||
|
github.com/couchbase/moss v0.1.0 h1:HCL+xxHUwmOaL44kMM/gU08OW6QGCui1WVFO58bjhNI= |
||||||
|
github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= |
||||||
|
github.com/couchbase/vellum v1.0.2 h1:BrbP0NKiyDdndMPec8Jjhy0U47CZ0Lgx3xUC2r9rZqw= |
||||||
|
github.com/couchbase/vellum v1.0.2/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4= |
||||||
|
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= |
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= |
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
||||||
|
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= |
||||||
|
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= |
||||||
|
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= |
||||||
|
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= |
||||||
|
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= |
||||||
|
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= |
||||||
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
||||||
|
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= |
||||||
|
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
||||||
|
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= |
||||||
|
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= |
||||||
|
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= |
||||||
|
github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw= |
||||||
|
github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= |
||||||
|
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= |
||||||
|
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= |
||||||
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= |
||||||
|
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= |
||||||
|
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= |
||||||
|
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= |
||||||
|
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= |
||||||
|
github.com/kljensen/snowball v0.6.0 h1:6DZLCcZeL0cLfodx+Md4/OLC6b/bfurWUOUGs1ydfOU= |
||||||
|
github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= |
||||||
|
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= |
||||||
|
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= |
||||||
|
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= |
||||||
|
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= |
||||||
|
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= |
||||||
|
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= |
||||||
|
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= |
||||||
|
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= |
||||||
|
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= |
||||||
|
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= |
||||||
|
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= |
||||||
|
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= |
||||||
|
github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= |
||||||
|
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= |
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= |
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= |
||||||
|
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= |
||||||
|
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= |
||||||
|
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= |
||||||
|
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= |
||||||
|
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= |
||||||
|
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= |
||||||
|
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= |
||||||
|
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= |
||||||
|
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= |
||||||
|
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= |
||||||
|
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= |
||||||
|
github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM= |
||||||
|
github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7Z4dM9/Y= |
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= |
||||||
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= |
||||||
|
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= |
||||||
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= |
||||||
|
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= |
||||||
|
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= |
||||||
|
github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= |
||||||
|
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= |
||||||
|
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= |
||||||
|
github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= |
||||||
|
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= |
||||||
|
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= |
||||||
|
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= |
||||||
|
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= |
||||||
|
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= |
||||||
|
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= |
||||||
|
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
||||||
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
||||||
|
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
||||||
|
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
||||||
|
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
||||||
|
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
||||||
|
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= |
||||||
|
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
||||||
|
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= |
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= |
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= |
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
||||||
|
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= |
||||||
|
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= |
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= |
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= |
||||||
|
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
||||||
|
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= |
||||||
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
@ -0,0 +1,33 @@ |
|||||||
|
// Copyright (c) 2020 Couchbase, Inc.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package scorch |
||||||
|
|
||||||
|
import segment "github.com/blevesearch/scorch_segment_api" |
||||||
|
|
||||||
|
type emptyPostingsIterator struct{} |
||||||
|
|
||||||
|
func (e *emptyPostingsIterator) Next() (segment.Posting, error) { |
||||||
|
return nil, nil |
||||||
|
} |
||||||
|
|
||||||
|
func (e *emptyPostingsIterator) Advance(uint64) (segment.Posting, error) { |
||||||
|
return nil, nil |
||||||
|
} |
||||||
|
|
||||||
|
func (e *emptyPostingsIterator) Size() int { |
||||||
|
return 0 |
||||||
|
} |
||||||
|
|
||||||
|
var anEmptyPostingsIterator = &emptyPostingsIterator{} |
@ -0,0 +1,92 @@ |
|||||||
|
// Copyright 2014 The Cockroach Authors.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
// implied. See the License for the specific language governing
|
||||||
|
// permissions and limitations under the License.
|
||||||
|
|
||||||
|
// This code originated from:
|
||||||
|
// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go
|
||||||
|
|
||||||
|
// Modified to not use pkg/errors
|
||||||
|
|
||||||
|
package scorch |
||||||
|
|
||||||
|
import "fmt" |
||||||
|
|
||||||
|
const ( |
||||||
|
// intMin is chosen such that the range of int tags does not overlap the
|
||||||
|
// ascii character set that is frequently used in testing.
|
||||||
|
intMin = 0x80 // 128
|
||||||
|
intMaxWidth = 8 |
||||||
|
intZero = intMin + intMaxWidth // 136
|
||||||
|
intSmall = intMax - intZero - intMaxWidth // 109
|
||||||
|
// intMax is the maximum int tag value.
|
||||||
|
intMax = 0xfd // 253
|
||||||
|
) |
||||||
|
|
||||||
|
// encodeUvarintAscending encodes the uint64 value using a variable length
|
||||||
|
// (length-prefixed) representation. The length is encoded as a single
|
||||||
|
// byte indicating the number of encoded bytes (-8) to follow. See
|
||||||
|
// EncodeVarintAscending for rationale. The encoded bytes are appended to the
|
||||||
|
// supplied buffer and the final buffer is returned.
|
||||||
|
func encodeUvarintAscending(b []byte, v uint64) []byte { |
||||||
|
switch { |
||||||
|
case v <= intSmall: |
||||||
|
return append(b, intZero+byte(v)) |
||||||
|
case v <= 0xff: |
||||||
|
return append(b, intMax-7, byte(v)) |
||||||
|
case v <= 0xffff: |
||||||
|
return append(b, intMax-6, byte(v>>8), byte(v)) |
||||||
|
case v <= 0xffffff: |
||||||
|
return append(b, intMax-5, byte(v>>16), byte(v>>8), byte(v)) |
||||||
|
case v <= 0xffffffff: |
||||||
|
return append(b, intMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) |
||||||
|
case v <= 0xffffffffff: |
||||||
|
return append(b, intMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), |
||||||
|
byte(v)) |
||||||
|
case v <= 0xffffffffffff: |
||||||
|
return append(b, intMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), |
||||||
|
byte(v>>8), byte(v)) |
||||||
|
case v <= 0xffffffffffffff: |
||||||
|
return append(b, intMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), |
||||||
|
byte(v>>16), byte(v>>8), byte(v)) |
||||||
|
default: |
||||||
|
return append(b, intMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), |
||||||
|
byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// decodeUvarintAscending decodes a varint encoded uint64 from the input
|
||||||
|
// buffer. The remainder of the input buffer and the decoded uint64
|
||||||
|
// are returned.
|
||||||
|
func decodeUvarintAscending(b []byte) ([]byte, uint64, error) { |
||||||
|
if len(b) == 0 { |
||||||
|
return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value") |
||||||
|
} |
||||||
|
length := int(b[0]) - intZero |
||||||
|
b = b[1:] // skip length byte
|
||||||
|
if length <= intSmall { |
||||||
|
return b, uint64(length), nil |
||||||
|
} |
||||||
|
length -= intSmall |
||||||
|
if length < 0 || length > 8 { |
||||||
|
return nil, 0, fmt.Errorf("invalid uvarint length of %d", length) |
||||||
|
} else if len(b) < length { |
||||||
|
return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b) |
||||||
|
} |
||||||
|
var v uint64 |
||||||
|
// It is faster to range over the elements in a slice than to index
|
||||||
|
// into the slice on each loop iteration.
|
||||||
|
for _, t := range b[:length] { |
||||||
|
v = (v << 8) | uint64(t) |
||||||
|
} |
||||||
|
return b[length:], v, nil |
||||||
|
} |
@ -0,0 +1,133 @@ |
|||||||
|
// Copyright (c) 2019 Couchbase, Inc.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package scorch |
||||||
|
|
||||||
|
import ( |
||||||
|
"fmt" |
||||||
|
"github.com/RoaringBitmap/roaring" |
||||||
|
index "github.com/blevesearch/bleve_index_api" |
||||||
|
|
||||||
|
segment "github.com/blevesearch/scorch_segment_api" |
||||||
|
|
||||||
|
zapv11 "github.com/blevesearch/zapx/v11" |
||||||
|
zapv12 "github.com/blevesearch/zapx/v12" |
||||||
|
zapv13 "github.com/blevesearch/zapx/v13" |
||||||
|
zapv14 "github.com/blevesearch/zapx/v14" |
||||||
|
zapv15 "github.com/blevesearch/zapx/v15" |
||||||
|
) |
||||||
|
|
||||||
|
// SegmentPlugin represents the essential functions required by a package to plug in
|
||||||
|
// it's segment implementation
|
||||||
|
type SegmentPlugin interface { |
||||||
|
|
||||||
|
// Type is the name for this segment plugin
|
||||||
|
Type() string |
||||||
|
|
||||||
|
// Version is a numeric value identifying a specific version of this type.
|
||||||
|
// When incompatible changes are made to a particular type of plugin, the
|
||||||
|
// version must be incremented.
|
||||||
|
Version() uint32 |
||||||
|
|
||||||
|
// New takes a set of Documents and turns them into a new Segment
|
||||||
|
New(results []index.Document) (segment.Segment, uint64, error) |
||||||
|
|
||||||
|
// Open attempts to open the file at the specified path and
|
||||||
|
// return the corresponding Segment
|
||||||
|
Open(path string) (segment.Segment, error) |
||||||
|
|
||||||
|
// Merge takes a set of Segments, and creates a new segment on disk at
|
||||||
|
// the specified path.
|
||||||
|
// Drops is a set of bitmaps (one for each segment) indicating which
|
||||||
|
// documents can be dropped from the segments during the merge.
|
||||||
|
// If the closeCh channel is closed, Merge will cease doing work at
|
||||||
|
// the next opportunity, and return an error (closed).
|
||||||
|
// StatsReporter can optionally be provided, in which case progress
|
||||||
|
// made during the merge is reported while operation continues.
|
||||||
|
// Returns:
|
||||||
|
// A slice of new document numbers (one for each input segment),
|
||||||
|
// this allows the caller to know a particular document's new
|
||||||
|
// document number in the newly merged segment.
|
||||||
|
// The number of bytes written to the new segment file.
|
||||||
|
// An error, if any occurred.
|
||||||
|
Merge(segments []segment.Segment, drops []*roaring.Bitmap, path string, |
||||||
|
closeCh chan struct{}, s segment.StatsReporter) ( |
||||||
|
[][]uint64, uint64, error) |
||||||
|
} |
||||||
|
|
||||||
|
var supportedSegmentPlugins map[string]map[uint32]SegmentPlugin |
||||||
|
var defaultSegmentPlugin SegmentPlugin |
||||||
|
|
||||||
|
func init() { |
||||||
|
ResetSegmentPlugins() |
||||||
|
RegisterSegmentPlugin(&zapv15.ZapPlugin{}, true) |
||||||
|
RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false) |
||||||
|
RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false) |
||||||
|
RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false) |
||||||
|
RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false) |
||||||
|
} |
||||||
|
|
||||||
|
func ResetSegmentPlugins() { |
||||||
|
supportedSegmentPlugins = map[string]map[uint32]SegmentPlugin{} |
||||||
|
} |
||||||
|
|
||||||
|
func RegisterSegmentPlugin(plugin SegmentPlugin, makeDefault bool) { |
||||||
|
if _, ok := supportedSegmentPlugins[plugin.Type()]; !ok { |
||||||
|
supportedSegmentPlugins[plugin.Type()] = map[uint32]SegmentPlugin{} |
||||||
|
} |
||||||
|
supportedSegmentPlugins[plugin.Type()][plugin.Version()] = plugin |
||||||
|
if makeDefault { |
||||||
|
defaultSegmentPlugin = plugin |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func SupportedSegmentTypes() (rv []string) { |
||||||
|
for k := range supportedSegmentPlugins { |
||||||
|
rv = append(rv, k) |
||||||
|
} |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
func SupportedSegmentTypeVersions(typ string) (rv []uint32) { |
||||||
|
for k := range supportedSegmentPlugins[typ] { |
||||||
|
rv = append(rv, k) |
||||||
|
} |
||||||
|
return rv |
||||||
|
} |
||||||
|
|
||||||
|
func chooseSegmentPlugin(forcedSegmentType string, |
||||||
|
forcedSegmentVersion uint32) (SegmentPlugin, error) { |
||||||
|
if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok { |
||||||
|
if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok { |
||||||
|
return segPlugin, nil |
||||||
|
} |
||||||
|
return nil, fmt.Errorf( |
||||||
|
"unsupported version %d for segment type: %s, supported: %v", |
||||||
|
forcedSegmentVersion, forcedSegmentType, |
||||||
|
SupportedSegmentTypeVersions(forcedSegmentType)) |
||||||
|
} |
||||||
|
return nil, fmt.Errorf("unsupported segment type: %s, supported: %v", |
||||||
|
forcedSegmentType, SupportedSegmentTypes()) |
||||||
|
} |
||||||
|
|
||||||
|
func (s *Scorch) loadSegmentPlugin(forcedSegmentType string, |
||||||
|
forcedSegmentVersion uint32) error { |
||||||
|
segPlugin, err := chooseSegmentPlugin(forcedSegmentType, |
||||||
|
forcedSegmentVersion) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
s.segPlugin = segPlugin |
||||||
|
return nil |
||||||
|
} |
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue