Update bleve dependency to latest master revision (#6100)
* update bleve to master b17287a86f6cac923a5d886e10618df994eeb54b6724eac2e3b8dde89cfbe3a2 * remove unused pkg from dep file * change bleve from master to recent revisiontokarchuk/v1.17
Lunny Xiao
6 years ago
committed by
techknowlogick
parent
11e316654e
commit
a380cfd8e0
@ -1,22 +0,0 @@ |
||||
The MIT License (MIT) |
||||
|
||||
Copyright (c) 2015 Stephen Merity |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in all |
||||
copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||
SOFTWARE. |
||||
|
||||
@ -1,229 +0,0 @@ |
||||
package govarint |
||||
|
||||
import "encoding/binary" |
||||
import "io" |
||||
|
||||
type U32VarintEncoder interface { |
||||
PutU32(x uint32) int |
||||
Close() |
||||
} |
||||
|
||||
type U32VarintDecoder interface { |
||||
GetU32() (uint32, error) |
||||
} |
||||
|
||||
///
|
||||
|
||||
type U64VarintEncoder interface { |
||||
PutU64(x uint64) int |
||||
Close() |
||||
} |
||||
|
||||
type U64VarintDecoder interface { |
||||
GetU64() (uint64, error) |
||||
} |
||||
|
||||
///
|
||||
|
||||
type U32GroupVarintEncoder struct { |
||||
w io.Writer |
||||
index int |
||||
store [4]uint32 |
||||
temp [17]byte |
||||
} |
||||
|
||||
func NewU32GroupVarintEncoder(w io.Writer) *U32GroupVarintEncoder { return &U32GroupVarintEncoder{w: w} } |
||||
|
||||
func (b *U32GroupVarintEncoder) Flush() (int, error) { |
||||
// TODO: Is it more efficient to have a tailored version that's called only in Close()?
|
||||
// If index is zero, there are no integers to flush
|
||||
if b.index == 0 { |
||||
return 0, nil |
||||
} |
||||
// In the case we're flushing (the group isn't of size four), the non-values should be zero
|
||||
// This ensures the unused entries are all zero in the sizeByte
|
||||
for i := b.index; i < 4; i++ { |
||||
b.store[i] = 0 |
||||
} |
||||
length := 1 |
||||
// We need to reset the size byte to zero as we only bitwise OR into it, we don't overwrite it
|
||||
b.temp[0] = 0 |
||||
for i, x := range b.store { |
||||
size := byte(0) |
||||
shifts := []byte{24, 16, 8, 0} |
||||
for _, shift := range shifts { |
||||
// Always writes at least one byte -- the first one (shift = 0)
|
||||
// Will write more bytes until the rest of the integer is all zeroes
|
||||
if (x>>shift) != 0 || shift == 0 { |
||||
size += 1 |
||||
b.temp[length] = byte(x >> shift) |
||||
length += 1 |
||||
} |
||||
} |
||||
// We store the size in two of the eight bits in the first byte (sizeByte)
|
||||
// 0 means there is one byte in total, hence why we subtract one from size
|
||||
b.temp[0] |= (size - 1) << (uint8(3-i) * 2) |
||||
} |
||||
// If we're flushing without a full group of four, remove the unused bytes we computed
|
||||
// This enables us to realize it's a partial group on decoding thanks to EOF
|
||||
if b.index != 4 { |
||||
length -= 4 - b.index |
||||
} |
||||
_, err := b.w.Write(b.temp[:length]) |
||||
return length, err |
||||
} |
||||
|
||||
func (b *U32GroupVarintEncoder) PutU32(x uint32) (int, error) { |
||||
bytesWritten := 0 |
||||
b.store[b.index] = x |
||||
b.index += 1 |
||||
if b.index == 4 { |
||||
n, err := b.Flush() |
||||
if err != nil { |
||||
return n, err |
||||
} |
||||
bytesWritten += n |
||||
b.index = 0 |
||||
} |
||||
return bytesWritten, nil |
||||
} |
||||
|
||||
func (b *U32GroupVarintEncoder) Close() { |
||||
// On Close, we flush any remaining values that might not have been in a full group
|
||||
b.Flush() |
||||
} |
||||
|
||||
///
|
||||
|
||||
type U32GroupVarintDecoder struct { |
||||
r io.ByteReader |
||||
group [4]uint32 |
||||
pos int |
||||
finished bool |
||||
capacity int |
||||
} |
||||
|
||||
func NewU32GroupVarintDecoder(r io.ByteReader) *U32GroupVarintDecoder { |
||||
return &U32GroupVarintDecoder{r: r, pos: 4, capacity: 4} |
||||
} |
||||
|
||||
func (b *U32GroupVarintDecoder) getGroup() error { |
||||
// We should always receive a sizeByte if there are more values to read
|
||||
sizeByte, err := b.r.ReadByte() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
// Calculate the size of the four incoming 32 bit integers
|
||||
// 0b00 means 1 byte to read, 0b01 = 2, etc
|
||||
b.group[0] = uint32((sizeByte >> 6) & 3) |
||||
b.group[1] = uint32((sizeByte >> 4) & 3) |
||||
b.group[2] = uint32((sizeByte >> 2) & 3) |
||||
b.group[3] = uint32(sizeByte & 3) |
||||
//
|
||||
for index, size := range b.group { |
||||
b.group[index] = 0 |
||||
// Any error that occurs in earlier byte reads should be repeated at the end one
|
||||
// Hence we only catch and report the final ReadByte's error
|
||||
var err error |
||||
switch size { |
||||
case 0: |
||||
var x byte |
||||
x, err = b.r.ReadByte() |
||||
b.group[index] = uint32(x) |
||||
case 1: |
||||
var x, y byte |
||||
x, _ = b.r.ReadByte() |
||||
y, err = b.r.ReadByte() |
||||
b.group[index] = uint32(x)<<8 | uint32(y) |
||||
case 2: |
||||
var x, y, z byte |
||||
x, _ = b.r.ReadByte() |
||||
y, _ = b.r.ReadByte() |
||||
z, err = b.r.ReadByte() |
||||
b.group[index] = uint32(x)<<16 | uint32(y)<<8 | uint32(z) |
||||
case 3: |
||||
var x, y, z, zz byte |
||||
x, _ = b.r.ReadByte() |
||||
y, _ = b.r.ReadByte() |
||||
z, _ = b.r.ReadByte() |
||||
zz, err = b.r.ReadByte() |
||||
b.group[index] = uint32(x)<<24 | uint32(y)<<16 | uint32(z)<<8 | uint32(zz) |
||||
} |
||||
if err != nil { |
||||
if err == io.EOF { |
||||
// If we hit EOF here, we have found a partial group
|
||||
// We've return any valid entries we have read and return EOF once we run out
|
||||
b.capacity = index |
||||
b.finished = true |
||||
break |
||||
} else { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
// Reset the pos pointer to the beginning of the read values
|
||||
b.pos = 0 |
||||
return nil |
||||
} |
||||
|
||||
func (b *U32GroupVarintDecoder) GetU32() (uint32, error) { |
||||
// Check if we have any more values to give out - if not, let's get them
|
||||
if b.pos == b.capacity { |
||||
// If finished is set, there is nothing else to do
|
||||
if b.finished { |
||||
return 0, io.EOF |
||||
} |
||||
err := b.getGroup() |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
// Increment pointer and return the value stored at that point
|
||||
b.pos += 1 |
||||
return b.group[b.pos-1], nil |
||||
} |
||||
|
||||
///
|
||||
|
||||
type Base128Encoder struct { |
||||
w io.Writer |
||||
tmpBytes []byte |
||||
} |
||||
|
||||
func NewU32Base128Encoder(w io.Writer) *Base128Encoder { |
||||
return &Base128Encoder{w: w, tmpBytes: make([]byte, binary.MaxVarintLen32)} |
||||
} |
||||
func NewU64Base128Encoder(w io.Writer) *Base128Encoder { |
||||
return &Base128Encoder{w: w, tmpBytes: make([]byte, binary.MaxVarintLen64)} |
||||
} |
||||
|
||||
func (b *Base128Encoder) PutU32(x uint32) (int, error) { |
||||
writtenBytes := binary.PutUvarint(b.tmpBytes, uint64(x)) |
||||
return b.w.Write(b.tmpBytes[:writtenBytes]) |
||||
} |
||||
|
||||
func (b *Base128Encoder) PutU64(x uint64) (int, error) { |
||||
writtenBytes := binary.PutUvarint(b.tmpBytes, x) |
||||
return b.w.Write(b.tmpBytes[:writtenBytes]) |
||||
} |
||||
|
||||
func (b *Base128Encoder) Close() { |
||||
} |
||||
|
||||
///
|
||||
|
||||
type Base128Decoder struct { |
||||
r io.ByteReader |
||||
} |
||||
|
||||
func NewU32Base128Decoder(r io.ByteReader) *Base128Decoder { return &Base128Decoder{r: r} } |
||||
func NewU64Base128Decoder(r io.ByteReader) *Base128Decoder { return &Base128Decoder{r: r} } |
||||
|
||||
func (b *Base128Decoder) GetU32() (uint32, error) { |
||||
v, err := binary.ReadUvarint(b.r) |
||||
return uint32(v), err |
||||
} |
||||
|
||||
func (b *Base128Decoder) GetU64() (uint64, error) { |
||||
return binary.ReadUvarint(b.r) |
||||
} |
||||
@ -0,0 +1,174 @@ |
||||
// The code here was obtained from:
|
||||
// https://github.com/mmcloughlin/geohash
|
||||
|
||||
// The MIT License (MIT)
|
||||
// Copyright (c) 2015 Michael McLoughlin
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
package geo |
||||
|
||||
import ( |
||||
"math" |
||||
) |
||||
|
||||
// encoding encapsulates an encoding defined by a given base32 alphabet.
|
||||
type encoding struct { |
||||
enc string |
||||
dec [256]byte |
||||
} |
||||
|
||||
// newEncoding constructs a new encoding defined by the given alphabet,
|
||||
// which must be a 32-byte string.
|
||||
func newEncoding(encoder string) *encoding { |
||||
e := new(encoding) |
||||
e.enc = encoder |
||||
for i := 0; i < len(e.dec); i++ { |
||||
e.dec[i] = 0xff |
||||
} |
||||
for i := 0; i < len(encoder); i++ { |
||||
e.dec[encoder[i]] = byte(i) |
||||
} |
||||
return e |
||||
} |
||||
|
||||
// Decode string into bits of a 64-bit word. The string s may be at most 12
|
||||
// characters.
|
||||
func (e *encoding) decode(s string) uint64 { |
||||
x := uint64(0) |
||||
for i := 0; i < len(s); i++ { |
||||
x = (x << 5) | uint64(e.dec[s[i]]) |
||||
} |
||||
return x |
||||
} |
||||
|
||||
// Encode bits of 64-bit word into a string.
|
||||
func (e *encoding) encode(x uint64) string { |
||||
b := [12]byte{} |
||||
for i := 0; i < 12; i++ { |
||||
b[11-i] = e.enc[x&0x1f] |
||||
x >>= 5 |
||||
} |
||||
return string(b[:]) |
||||
} |
||||
|
||||
// Base32Encoding with the Geohash alphabet.
|
||||
var base32encoding = newEncoding("0123456789bcdefghjkmnpqrstuvwxyz") |
||||
|
||||
// BoundingBox returns the region encoded by the given string geohash.
|
||||
func geoBoundingBox(hash string) geoBox { |
||||
bits := uint(5 * len(hash)) |
||||
inthash := base32encoding.decode(hash) |
||||
return geoBoundingBoxIntWithPrecision(inthash, bits) |
||||
} |
||||
|
||||
// Box represents a rectangle in latitude/longitude space.
|
||||
type geoBox struct { |
||||
minLat float64 |
||||
maxLat float64 |
||||
minLng float64 |
||||
maxLng float64 |
||||
} |
||||
|
||||
// Round returns a point inside the box, making an effort to round to minimal
|
||||
// precision.
|
||||
func (b geoBox) round() (lat, lng float64) { |
||||
x := maxDecimalPower(b.maxLat - b.minLat) |
||||
lat = math.Ceil(b.minLat/x) * x |
||||
x = maxDecimalPower(b.maxLng - b.minLng) |
||||
lng = math.Ceil(b.minLng/x) * x |
||||
return |
||||
} |
||||
|
||||
// precalculated for performance
|
||||
var exp232 = math.Exp2(32) |
||||
|
||||
// errorWithPrecision returns the error range in latitude and longitude for in
|
||||
// integer geohash with bits of precision.
|
||||
func errorWithPrecision(bits uint) (latErr, lngErr float64) { |
||||
b := int(bits) |
||||
latBits := b / 2 |
||||
lngBits := b - latBits |
||||
latErr = math.Ldexp(180.0, -latBits) |
||||
lngErr = math.Ldexp(360.0, -lngBits) |
||||
return |
||||
} |
||||
|
||||
// minDecimalPlaces returns the minimum number of decimal places such that
|
||||
// there must exist an number with that many places within any range of width
|
||||
// r. This is intended for returning minimal precision coordinates inside a
|
||||
// box.
|
||||
func maxDecimalPower(r float64) float64 { |
||||
m := int(math.Floor(math.Log10(r))) |
||||
return math.Pow10(m) |
||||
} |
||||
|
||||
// Encode the position of x within the range -r to +r as a 32-bit integer.
|
||||
func encodeRange(x, r float64) uint32 { |
||||
p := (x + r) / (2 * r) |
||||
return uint32(p * exp232) |
||||
} |
||||
|
||||
// Decode the 32-bit range encoding X back to a value in the range -r to +r.
|
||||
func decodeRange(X uint32, r float64) float64 { |
||||
p := float64(X) / exp232 |
||||
x := 2*r*p - r |
||||
return x |
||||
} |
||||
|
||||
// Squash the even bitlevels of X into a 32-bit word. Odd bitlevels of X are
|
||||
// ignored, and may take any value.
|
||||
func squash(X uint64) uint32 { |
||||
X &= 0x5555555555555555 |
||||
X = (X | (X >> 1)) & 0x3333333333333333 |
||||
X = (X | (X >> 2)) & 0x0f0f0f0f0f0f0f0f |
||||
X = (X | (X >> 4)) & 0x00ff00ff00ff00ff |
||||
X = (X | (X >> 8)) & 0x0000ffff0000ffff |
||||
X = (X | (X >> 16)) & 0x00000000ffffffff |
||||
return uint32(X) |
||||
} |
||||
|
||||
// Deinterleave the bits of X into 32-bit words containing the even and odd
|
||||
// bitlevels of X, respectively.
|
||||
func deinterleave(X uint64) (uint32, uint32) { |
||||
return squash(X), squash(X >> 1) |
||||
} |
||||
|
||||
// BoundingBoxIntWithPrecision returns the region encoded by the integer
|
||||
// geohash with the specified precision.
|
||||
func geoBoundingBoxIntWithPrecision(hash uint64, bits uint) geoBox { |
||||
fullHash := hash << (64 - bits) |
||||
latInt, lngInt := deinterleave(fullHash) |
||||
lat := decodeRange(latInt, 90) |
||||
lng := decodeRange(lngInt, 180) |
||||
latErr, lngErr := errorWithPrecision(bits) |
||||
return geoBox{ |
||||
minLat: lat, |
||||
maxLat: lat + latErr, |
||||
minLng: lng, |
||||
maxLng: lng + lngErr, |
||||
} |
||||
} |
||||
|
||||
// ----------------------------------------------------------------------
|
||||
|
||||
// Decode the string geohash to a (lat, lng) point.
|
||||
func GeoHashDecode(hash string) (lat, lng float64) { |
||||
box := geoBoundingBox(hash) |
||||
return box.round() |
||||
} |
||||
@ -0,0 +1,420 @@ |
||||
// Copyright (c) 2018 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scorch |
||||
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"github.com/RoaringBitmap/roaring" |
||||
|
||||
"github.com/blevesearch/bleve/index" |
||||
"github.com/blevesearch/bleve/index/scorch/segment" |
||||
"github.com/blevesearch/bleve/index/scorch/segment/zap" |
||||
) |
||||
|
||||
var OptimizeConjunction = true |
||||
var OptimizeConjunctionUnadorned = true |
||||
var OptimizeDisjunctionUnadorned = true |
||||
|
||||
func (s *IndexSnapshotTermFieldReader) Optimize(kind string, |
||||
octx index.OptimizableContext) (index.OptimizableContext, error) { |
||||
if OptimizeConjunction && kind == "conjunction" { |
||||
return s.optimizeConjunction(octx) |
||||
} |
||||
|
||||
if OptimizeConjunctionUnadorned && kind == "conjunction:unadorned" { |
||||
return s.optimizeConjunctionUnadorned(octx) |
||||
} |
||||
|
||||
if OptimizeDisjunctionUnadorned && kind == "disjunction:unadorned" { |
||||
return s.optimizeDisjunctionUnadorned(octx) |
||||
} |
||||
|
||||
return octx, nil |
||||
} |
||||
|
||||
var OptimizeDisjunctionUnadornedMinChildCardinality = uint64(256) |
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
||||
func (s *IndexSnapshotTermFieldReader) optimizeConjunction( |
||||
octx index.OptimizableContext) (index.OptimizableContext, error) { |
||||
if octx == nil { |
||||
octx = &OptimizeTFRConjunction{snapshot: s.snapshot} |
||||
} |
||||
|
||||
o, ok := octx.(*OptimizeTFRConjunction) |
||||
if !ok { |
||||
return octx, nil |
||||
} |
||||
|
||||
if o.snapshot != s.snapshot { |
||||
return nil, fmt.Errorf("tried to optimize conjunction across different snapshots") |
||||
} |
||||
|
||||
o.tfrs = append(o.tfrs, s) |
||||
|
||||
return o, nil |
||||
} |
||||
|
||||
type OptimizeTFRConjunction struct { |
||||
snapshot *IndexSnapshot |
||||
|
||||
tfrs []*IndexSnapshotTermFieldReader |
||||
} |
||||
|
||||
func (o *OptimizeTFRConjunction) Finish() (index.Optimized, error) { |
||||
if len(o.tfrs) <= 1 { |
||||
return nil, nil |
||||
} |
||||
|
||||
for i := range o.snapshot.segment { |
||||
itr0, ok := o.tfrs[0].iterators[i].(*zap.PostingsIterator) |
||||
if !ok || itr0.ActualBM == nil { |
||||
continue |
||||
} |
||||
|
||||
itr1, ok := o.tfrs[1].iterators[i].(*zap.PostingsIterator) |
||||
if !ok || itr1.ActualBM == nil { |
||||
continue |
||||
} |
||||
|
||||
bm := roaring.And(itr0.ActualBM, itr1.ActualBM) |
||||
|
||||
for _, tfr := range o.tfrs[2:] { |
||||
itr, ok := tfr.iterators[i].(*zap.PostingsIterator) |
||||
if !ok || itr.ActualBM == nil { |
||||
continue |
||||
} |
||||
|
||||
bm.And(itr.ActualBM) |
||||
} |
||||
|
||||
// in this conjunction optimization, the postings iterators
|
||||
// will all share the same AND'ed together actual bitmap. The
|
||||
// regular conjunction searcher machinery will still be used,
|
||||
// but the underlying bitmap will be smaller.
|
||||
for _, tfr := range o.tfrs { |
||||
itr, ok := tfr.iterators[i].(*zap.PostingsIterator) |
||||
if ok && itr.ActualBM != nil { |
||||
itr.ActualBM = bm |
||||
itr.Actual = bm.Iterator() |
||||
} |
||||
} |
||||
} |
||||
|
||||
return nil, nil |
||||
} |
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
||||
// An "unadorned" conjunction optimization is appropriate when
|
||||
// additional or subsidiary information like freq-norm's and
|
||||
// term-vectors are not required, and instead only the internal-id's
|
||||
// are needed.
|
||||
func (s *IndexSnapshotTermFieldReader) optimizeConjunctionUnadorned( |
||||
octx index.OptimizableContext) (index.OptimizableContext, error) { |
||||
if octx == nil { |
||||
octx = &OptimizeTFRConjunctionUnadorned{snapshot: s.snapshot} |
||||
} |
||||
|
||||
o, ok := octx.(*OptimizeTFRConjunctionUnadorned) |
||||
if !ok { |
||||
return nil, nil |
||||
} |
||||
|
||||
if o.snapshot != s.snapshot { |
||||
return nil, fmt.Errorf("tried to optimize unadorned conjunction across different snapshots") |
||||
} |
||||
|
||||
o.tfrs = append(o.tfrs, s) |
||||
|
||||
return o, nil |
||||
} |
||||
|
||||
type OptimizeTFRConjunctionUnadorned struct { |
||||
snapshot *IndexSnapshot |
||||
|
||||
tfrs []*IndexSnapshotTermFieldReader |
||||
} |
||||
|
||||
var OptimizeTFRConjunctionUnadornedTerm = []byte("<conjunction:unadorned>") |
||||
var OptimizeTFRConjunctionUnadornedField = "*" |
||||
|
||||
// Finish of an unadorned conjunction optimization will compute a
|
||||
// termFieldReader with an "actual" bitmap that represents the
|
||||
// constituent bitmaps AND'ed together. This termFieldReader cannot
|
||||
// provide any freq-norm or termVector associated information.
|
||||
func (o *OptimizeTFRConjunctionUnadorned) Finish() (rv index.Optimized, err error) { |
||||
if len(o.tfrs) <= 1 { |
||||
return nil, nil |
||||
} |
||||
|
||||
// We use an artificial term and field because the optimized
|
||||
// termFieldReader can represent multiple terms and fields.
|
||||
oTFR := &IndexSnapshotTermFieldReader{ |
||||
term: OptimizeTFRConjunctionUnadornedTerm, |
||||
field: OptimizeTFRConjunctionUnadornedField, |
||||
snapshot: o.snapshot, |
||||
iterators: make([]segment.PostingsIterator, len(o.snapshot.segment)), |
||||
segmentOffset: 0, |
||||
includeFreq: false, |
||||
includeNorm: false, |
||||
includeTermVectors: false, |
||||
} |
||||
|
||||
var actualBMs []*roaring.Bitmap // Collected from regular posting lists.
|
||||
|
||||
OUTER: |
||||
for i := range o.snapshot.segment { |
||||
actualBMs = actualBMs[:0] |
||||
|
||||
var docNum1HitLast uint64 |
||||
var docNum1HitLastOk bool |
||||
|
||||
for _, tfr := range o.tfrs { |
||||
if _, ok := tfr.iterators[i].(*segment.EmptyPostingsIterator); ok { |
||||
// An empty postings iterator means the entire AND is empty.
|
||||
oTFR.iterators[i] = segment.AnEmptyPostingsIterator |
||||
continue OUTER |
||||
} |
||||
|
||||
itr, ok := tfr.iterators[i].(*zap.PostingsIterator) |
||||
if !ok { |
||||
// We optimize zap postings iterators only.
|
||||
return nil, nil |
||||
} |
||||
|
||||
// If the postings iterator is "1-hit" optimized, then we
|
||||
// can perform several optimizations up-front here.
|
||||
docNum1Hit, ok := itr.DocNum1Hit() |
||||
if ok { |
||||
if docNum1Hit == zap.DocNum1HitFinished { |
||||
// An empty docNum here means the entire AND is empty.
|
||||
oTFR.iterators[i] = segment.AnEmptyPostingsIterator |
||||
continue OUTER |
||||
} |
||||
|
||||
if docNum1HitLastOk && docNum1HitLast != docNum1Hit { |
||||
// The docNum1Hit doesn't match the previous
|
||||
// docNum1HitLast, so the entire AND is empty.
|
||||
oTFR.iterators[i] = segment.AnEmptyPostingsIterator |
||||
continue OUTER |
||||
} |
||||
|
||||
docNum1HitLast = docNum1Hit |
||||
docNum1HitLastOk = true |
||||
|
||||
continue |
||||
} |
||||
|
||||
if itr.ActualBM == nil { |
||||
// An empty actual bitmap means the entire AND is empty.
|
||||
oTFR.iterators[i] = segment.AnEmptyPostingsIterator |
||||
continue OUTER |
||||
} |
||||
|
||||
// Collect the actual bitmap for more processing later.
|
||||
actualBMs = append(actualBMs, itr.ActualBM) |
||||
} |
||||
|
||||
if docNum1HitLastOk { |
||||
// We reach here if all the 1-hit optimized posting
|
||||
// iterators had the same 1-hit docNum, so we can check if
|
||||
// our collected actual bitmaps also have that docNum.
|
||||
for _, bm := range actualBMs { |
||||
if !bm.Contains(uint32(docNum1HitLast)) { |
||||
// The docNum1Hit isn't in one of our actual
|
||||
// bitmaps, so the entire AND is empty.
|
||||
oTFR.iterators[i] = segment.AnEmptyPostingsIterator |
||||
continue OUTER |
||||
} |
||||
} |
||||
|
||||
// The actual bitmaps and docNum1Hits all contain or have
|
||||
// the same 1-hit docNum, so that's our AND'ed result.
|
||||
oTFR.iterators[i], err = zap.PostingsIteratorFrom1Hit( |
||||
docNum1HitLast, zap.NormBits1Hit, false, false) |
||||
if err != nil { |
||||
return nil, nil |
||||
} |
||||
|
||||
continue OUTER |
||||
} |
||||
|
||||
if len(actualBMs) == 0 { |
||||
// If we've collected no actual bitmaps at this point,
|
||||
// then the entire AND is empty.
|
||||
oTFR.iterators[i] = segment.AnEmptyPostingsIterator |
||||
continue OUTER |
||||
} |
||||
|
||||
if len(actualBMs) == 1 { |
||||
// If we've only 1 actual bitmap, then that's our result.
|
||||
oTFR.iterators[i], err = zap.PostingsIteratorFromBitmap( |
||||
actualBMs[0], false, false) |
||||
if err != nil { |
||||
return nil, nil |
||||
} |
||||
|
||||
continue OUTER |
||||
} |
||||
|
||||
// Else, AND together our collected bitmaps as our result.
|
||||
bm := roaring.And(actualBMs[0], actualBMs[1]) |
||||
|
||||
for _, actualBM := range actualBMs[2:] { |
||||
bm.And(actualBM) |
||||
} |
||||
|
||||
oTFR.iterators[i], err = zap.PostingsIteratorFromBitmap( |
||||
bm, false, false) |
||||
if err != nil { |
||||
return nil, nil |
||||
} |
||||
} |
||||
|
||||
return oTFR, nil |
||||
} |
||||
|
||||
// ----------------------------------------------------------------
|
||||
|
||||
// An "unadorned" disjunction optimization is appropriate when
|
||||
// additional or subsidiary information like freq-norm's and
|
||||
// term-vectors are not required, and instead only the internal-id's
|
||||
// are needed.
|
||||
func (s *IndexSnapshotTermFieldReader) optimizeDisjunctionUnadorned( |
||||
octx index.OptimizableContext) (index.OptimizableContext, error) { |
||||
if octx == nil { |
||||
octx = &OptimizeTFRDisjunctionUnadorned{snapshot: s.snapshot} |
||||
} |
||||
|
||||
o, ok := octx.(*OptimizeTFRDisjunctionUnadorned) |
||||
if !ok { |
||||
return nil, nil |
||||
} |
||||
|
||||
if o.snapshot != s.snapshot { |
||||
return nil, fmt.Errorf("tried to optimize unadorned disjunction across different snapshots") |
||||
} |
||||
|
||||
o.tfrs = append(o.tfrs, s) |
||||
|
||||
return o, nil |
||||
} |
||||
|
||||
type OptimizeTFRDisjunctionUnadorned struct { |
||||
snapshot *IndexSnapshot |
||||
|
||||
tfrs []*IndexSnapshotTermFieldReader |
||||
} |
||||
|
||||
var OptimizeTFRDisjunctionUnadornedTerm = []byte("<disjunction:unadorned>") |
||||
var OptimizeTFRDisjunctionUnadornedField = "*" |
||||
|
||||
// Finish of an unadorned disjunction optimization will compute a
|
||||
// termFieldReader with an "actual" bitmap that represents the
|
||||
// constituent bitmaps OR'ed together. This termFieldReader cannot
|
||||
// provide any freq-norm or termVector associated information.
|
||||
func (o *OptimizeTFRDisjunctionUnadorned) Finish() (rv index.Optimized, err error) { |
||||
if len(o.tfrs) <= 1 { |
||||
return nil, nil |
||||
} |
||||
|
||||
for i := range o.snapshot.segment { |
||||
var cMax uint64 |
||||
|
||||
for _, tfr := range o.tfrs { |
||||
itr, ok := tfr.iterators[i].(*zap.PostingsIterator) |
||||
if !ok { |
||||
return nil, nil |
||||
} |
||||
|
||||
if itr.ActualBM != nil { |
||||
c := itr.ActualBM.GetCardinality() |
||||
if cMax < c { |
||||
cMax = c |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Heuristic to skip the optimization if all the constituent
|
||||
// bitmaps are too small, where the processing & resource
|
||||
// overhead to create the OR'ed bitmap outweighs the benefit.
|
||||
if cMax < OptimizeDisjunctionUnadornedMinChildCardinality { |
||||
return nil, nil |
||||
} |
||||
} |
||||
|
||||
// We use an artificial term and field because the optimized
|
||||
// termFieldReader can represent multiple terms and fields.
|
||||
oTFR := &IndexSnapshotTermFieldReader{ |
||||
term: OptimizeTFRDisjunctionUnadornedTerm, |
||||
field: OptimizeTFRDisjunctionUnadornedField, |
||||
snapshot: o.snapshot, |
||||
iterators: make([]segment.PostingsIterator, len(o.snapshot.segment)), |
||||
segmentOffset: 0, |
||||
includeFreq: false, |
||||
includeNorm: false, |
||||
includeTermVectors: false, |
||||
} |
||||
|
||||
var docNums []uint32 // Collected docNum's from 1-hit posting lists.
|
||||
var actualBMs []*roaring.Bitmap // Collected from regular posting lists.
|
||||
|
||||
for i := range o.snapshot.segment { |
||||
docNums = docNums[:0] |
||||
actualBMs = actualBMs[:0] |
||||
|
||||
for _, tfr := range o.tfrs { |
||||
itr, ok := tfr.iterators[i].(*zap.PostingsIterator) |
||||
if !ok { |
||||
return nil, nil |
||||
} |
||||
|
||||
docNum, ok := itr.DocNum1Hit() |
||||
if ok { |
||||
docNums = append(docNums, uint32(docNum)) |
||||
continue |
||||
} |
||||
|
||||
if itr.ActualBM != nil { |
||||
actualBMs = append(actualBMs, itr.ActualBM) |
||||
} |
||||
} |
||||
|
||||
var bm *roaring.Bitmap |
||||
if len(actualBMs) > 2 { |
||||
bm = roaring.HeapOr(actualBMs...) |
||||
} else if len(actualBMs) == 2 { |
||||
bm = roaring.Or(actualBMs[0], actualBMs[1]) |
||||
} else if len(actualBMs) == 1 { |
||||
bm = actualBMs[0].Clone() |
||||
} |
||||
|
||||
if bm == nil { |
||||
bm = roaring.New() |
||||
} |
||||
|
||||
bm.AddMany(docNums) |
||||
|
||||
oTFR.iterators[i], err = zap.PostingsIteratorFromBitmap(bm, false, false) |
||||
if err != nil { |
||||
return nil, nil |
||||
} |
||||
} |
||||
|
||||
return oTFR, nil |
||||
} |
||||
@ -1,110 +0,0 @@ |
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scorch |
||||
|
||||
import ( |
||||
"github.com/blevesearch/bleve/document" |
||||
"github.com/blevesearch/bleve/index" |
||||
) |
||||
|
||||
type Reader struct { |
||||
root *IndexSnapshot // Owns 1 ref-count on the index snapshot.
|
||||
} |
||||
|
||||
func (r *Reader) TermFieldReader(term []byte, field string, includeFreq, |
||||
includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { |
||||
return r.root.TermFieldReader(term, field, includeFreq, includeNorm, includeTermVectors) |
||||
} |
||||
|
||||
// DocIDReader returns an iterator over all doc ids
|
||||
// The caller must close returned instance to release associated resources.
|
||||
func (r *Reader) DocIDReaderAll() (index.DocIDReader, error) { |
||||
return r.root.DocIDReaderAll() |
||||
} |
||||
|
||||
func (r *Reader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { |
||||
return r.root.DocIDReaderOnly(ids) |
||||
} |
||||
|
||||
func (r *Reader) FieldDict(field string) (index.FieldDict, error) { |
||||
return r.root.FieldDict(field) |
||||
} |
||||
|
||||
// FieldDictRange is currently defined to include the start and end terms
|
||||
func (r *Reader) FieldDictRange(field string, startTerm []byte, |
||||
endTerm []byte) (index.FieldDict, error) { |
||||
return r.root.FieldDictRange(field, startTerm, endTerm) |
||||
} |
||||
|
||||
func (r *Reader) FieldDictPrefix(field string, |
||||
termPrefix []byte) (index.FieldDict, error) { |
||||
return r.root.FieldDictPrefix(field, termPrefix) |
||||
} |
||||
|
||||
func (r *Reader) Document(id string) (*document.Document, error) { |
||||
return r.root.Document(id) |
||||
} |
||||
func (r *Reader) DocumentVisitFieldTerms(id index.IndexInternalID, fields []string, |
||||
visitor index.DocumentFieldTermVisitor) error { |
||||
return r.root.DocumentVisitFieldTerms(id, fields, visitor) |
||||
} |
||||
|
||||
func (r *Reader) Fields() ([]string, error) { |
||||
return r.root.Fields() |
||||
} |
||||
|
||||
func (r *Reader) GetInternal(key []byte) ([]byte, error) { |
||||
return r.root.GetInternal(key) |
||||
} |
||||
|
||||
func (r *Reader) DocCount() (uint64, error) { |
||||
return r.root.DocCount() |
||||
} |
||||
|
||||
func (r *Reader) ExternalID(id index.IndexInternalID) (string, error) { |
||||
return r.root.ExternalID(id) |
||||
} |
||||
|
||||
func (r *Reader) InternalID(id string) (index.IndexInternalID, error) { |
||||
return r.root.InternalID(id) |
||||
} |
||||
|
||||
func (r *Reader) DumpAll() chan interface{} { |
||||
rv := make(chan interface{}) |
||||
go func() { |
||||
close(rv) |
||||
}() |
||||
return rv |
||||
} |
||||
|
||||
func (r *Reader) DumpDoc(id string) chan interface{} { |
||||
rv := make(chan interface{}) |
||||
go func() { |
||||
close(rv) |
||||
}() |
||||
return rv |
||||
} |
||||
|
||||
func (r *Reader) DumpFields() chan interface{} { |
||||
rv := make(chan interface{}) |
||||
go func() { |
||||
close(rv) |
||||
}() |
||||
return rv |
||||
} |
||||
|
||||
func (r *Reader) Close() error { |
||||
return r.root.DecRef() |
||||
} |
||||
@ -1,321 +0,0 @@ |
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mem |
||||
|
||||
import ( |
||||
"math" |
||||
"sort" |
||||
|
||||
"github.com/RoaringBitmap/roaring" |
||||
"github.com/blevesearch/bleve/analysis" |
||||
"github.com/blevesearch/bleve/document" |
||||
"github.com/blevesearch/bleve/index" |
||||
) |
||||
|
||||
// NewFromAnalyzedDocs places the analyzed document mutations into a new segment
|
||||
func NewFromAnalyzedDocs(results []*index.AnalysisResult) *Segment { |
||||
s := New() |
||||
|
||||
// ensure that _id field get fieldID 0
|
||||
s.getOrDefineField("_id") |
||||
|
||||
// fill Dicts/DictKeys and preallocate memory
|
||||
s.initializeDict(results) |
||||
|
||||
// walk each doc
|
||||
for _, result := range results { |
||||
s.processDocument(result) |
||||
} |
||||
|
||||
// go back and sort the dictKeys
|
||||
for _, dict := range s.DictKeys { |
||||
sort.Strings(dict) |
||||
} |
||||
|
||||
// compute memory usage of segment
|
||||
s.updateSizeInBytes() |
||||
|
||||
// professional debugging
|
||||
//
|
||||
// log.Printf("fields: %v\n", s.FieldsMap)
|
||||
// log.Printf("fieldsInv: %v\n", s.FieldsInv)
|
||||
// log.Printf("fieldsLoc: %v\n", s.FieldsLoc)
|
||||
// log.Printf("dicts: %v\n", s.Dicts)
|
||||
// log.Printf("dict keys: %v\n", s.DictKeys)
|
||||
// for i, posting := range s.Postings {
|
||||
// log.Printf("posting %d: %v\n", i, posting)
|
||||
// }
|
||||
// for i, freq := range s.Freqs {
|
||||
// log.Printf("freq %d: %v\n", i, freq)
|
||||
// }
|
||||
// for i, norm := range s.Norms {
|
||||
// log.Printf("norm %d: %v\n", i, norm)
|
||||
// }
|
||||
// for i, field := range s.Locfields {
|
||||
// log.Printf("field %d: %v\n", i, field)
|
||||
// }
|
||||
// for i, start := range s.Locstarts {
|
||||
// log.Printf("start %d: %v\n", i, start)
|
||||
// }
|
||||
// for i, end := range s.Locends {
|
||||
// log.Printf("end %d: %v\n", i, end)
|
||||
// }
|
||||
// for i, pos := range s.Locpos {
|
||||
// log.Printf("pos %d: %v\n", i, pos)
|
||||
// }
|
||||
// for i, apos := range s.Locarraypos {
|
||||
// log.Printf("apos %d: %v\n", i, apos)
|
||||
// }
|
||||
// log.Printf("stored: %v\n", s.Stored)
|
||||
// log.Printf("stored types: %v\n", s.StoredTypes)
|
||||
// log.Printf("stored pos: %v\n", s.StoredPos)
|
||||
|
||||
return s |
||||
} |
||||
|
||||
// fill Dicts/DictKeys and preallocate memory for postings
|
||||
func (s *Segment) initializeDict(results []*index.AnalysisResult) { |
||||
var numPostingsLists int |
||||
|
||||
numTermsPerPostingsList := make([]int, 0, 64) // Keyed by postings list id.
|
||||
numLocsPerPostingsList := make([]int, 0, 64) // Keyed by postings list id.
|
||||
|
||||
var numTokenFrequencies int |
||||
var totLocs int |
||||
|
||||
// initial scan for all fieldID's to sort them
|
||||
for _, result := range results { |
||||
for _, field := range result.Document.CompositeFields { |
||||
s.getOrDefineField(field.Name()) |
||||
} |
||||
for _, field := range result.Document.Fields { |
||||
s.getOrDefineField(field.Name()) |
||||
} |
||||
} |
||||
sort.Strings(s.FieldsInv[1:]) // keep _id as first field
|
||||
s.FieldsMap = make(map[string]uint16, len(s.FieldsInv)) |
||||
for fieldID, fieldName := range s.FieldsInv { |
||||
s.FieldsMap[fieldName] = uint16(fieldID + 1) |
||||
} |
||||
|
||||
processField := func(fieldID uint16, tfs analysis.TokenFrequencies) { |
||||
for term, tf := range tfs { |
||||
pidPlus1, exists := s.Dicts[fieldID][term] |
||||
if !exists { |
||||
numPostingsLists++ |
||||
pidPlus1 = uint64(numPostingsLists) |
||||
s.Dicts[fieldID][term] = pidPlus1 |
||||
s.DictKeys[fieldID] = append(s.DictKeys[fieldID], term) |
||||
numTermsPerPostingsList = append(numTermsPerPostingsList, 0) |
||||
numLocsPerPostingsList = append(numLocsPerPostingsList, 0) |
||||
} |
||||
pid := pidPlus1 - 1 |
||||
numTermsPerPostingsList[pid] += 1 |
||||
numLocsPerPostingsList[pid] += len(tf.Locations) |
||||
totLocs += len(tf.Locations) |
||||
} |
||||
numTokenFrequencies += len(tfs) |
||||
} |
||||
|
||||
for _, result := range results { |
||||
// walk each composite field
|
||||
for _, field := range result.Document.CompositeFields { |
||||
fieldID := uint16(s.getOrDefineField(field.Name())) |
||||
_, tf := field.Analyze() |
||||
processField(fieldID, tf) |
||||
} |
||||
|
||||
// walk each field
|
||||
for i, field := range result.Document.Fields { |
||||
fieldID := uint16(s.getOrDefineField(field.Name())) |
||||
tf := result.Analyzed[i] |
||||
processField(fieldID, tf) |
||||
} |
||||
} |
||||
|
||||
s.Postings = make([]*roaring.Bitmap, numPostingsLists) |
||||
for i := 0; i < numPostingsLists; i++ { |
||||
s.Postings[i] = roaring.New() |
||||
} |
||||
s.PostingsLocs = make([]*roaring.Bitmap, numPostingsLists) |
||||
for i := 0; i < numPostingsLists; i++ { |
||||
s.PostingsLocs[i] = roaring.New() |
||||
} |
||||
|
||||
// Preallocate big, contiguous backing arrays.
|
||||
auint64Backing := make([][]uint64, numPostingsLists*4+totLocs) // For Freqs, Locstarts, Locends, Locpos, sub-Locarraypos.
|
||||
uint64Backing := make([]uint64, numTokenFrequencies+totLocs*3) // For sub-Freqs, sub-Locstarts, sub-Locends, sub-Locpos.
|
||||
float32Backing := make([]float32, numTokenFrequencies) // For sub-Norms.
|
||||
uint16Backing := make([]uint16, totLocs) // For sub-Locfields.
|
||||
|
||||
// Point top-level slices to the backing arrays.
|
||||
s.Freqs = auint64Backing[0:numPostingsLists] |
||||
auint64Backing = auint64Backing[numPostingsLists:] |
||||
|
||||
s.Norms = make([][]float32, numPostingsLists) |
||||
|
||||
s.Locfields = make([][]uint16, numPostingsLists) |
||||
|
||||
s.Locstarts = auint64Backing[0:numPostingsLists] |
||||
auint64Backing = auint64Backing[numPostingsLists:] |
||||
|
||||
s.Locends = auint64Backing[0:numPostingsLists] |
||||
auint64Backing = auint64Backing[numPostingsLists:] |
||||
|
||||
s.Locpos = auint64Backing[0:numPostingsLists] |
||||
auint64Backing = auint64Backing[numPostingsLists:] |
||||
|
||||
s.Locarraypos = make([][][]uint64, numPostingsLists) |
||||
|
||||
// Point sub-slices to the backing arrays.
|
||||
for pid, numTerms := range numTermsPerPostingsList { |
||||
s.Freqs[pid] = uint64Backing[0:0] |
||||
uint64Backing = uint64Backing[numTerms:] |
||||
|
||||
s.Norms[pid] = float32Backing[0:0] |
||||
float32Backing = float32Backing[numTerms:] |
||||
} |
||||
|
||||
for pid, numLocs := range numLocsPerPostingsList { |
||||
s.Locfields[pid] = uint16Backing[0:0] |
||||
uint16Backing = uint16Backing[numLocs:] |
||||
|
||||
s.Locstarts[pid] = uint64Backing[0:0] |
||||
uint64Backing = uint64Backing[numLocs:] |
||||
|
||||
s.Locends[pid] = uint64Backing[0:0] |
||||
uint64Backing = uint64Backing[numLocs:] |
||||
|
||||
s.Locpos[pid] = uint64Backing[0:0] |
||||
uint64Backing = uint64Backing[numLocs:] |
||||
|
||||
s.Locarraypos[pid] = auint64Backing[0:0] |
||||
auint64Backing = auint64Backing[numLocs:] |
||||
} |
||||
} |
||||
|
||||
func (s *Segment) processDocument(result *index.AnalysisResult) { |
||||
// used to collate information across fields
|
||||
docMap := make(map[uint16]analysis.TokenFrequencies, len(s.FieldsMap)) |
||||
fieldLens := make(map[uint16]int, len(s.FieldsMap)) |
||||
|
||||
docNum := uint64(s.addDocument()) |
||||
|
||||
processField := func(field uint16, name string, l int, tf analysis.TokenFrequencies) { |
||||
fieldLens[field] += l |
||||
if existingFreqs, ok := docMap[field]; ok { |
||||
existingFreqs.MergeAll(name, tf) |
||||
} else { |
||||
docMap[field] = tf |
||||
} |
||||
} |
||||
|
||||
storeField := func(docNum uint64, field uint16, typ byte, val []byte, pos []uint64) { |
||||
s.Stored[docNum][field] = append(s.Stored[docNum][field], val) |
||||
s.StoredTypes[docNum][field] = append(s.StoredTypes[docNum][field], typ) |
||||
s.StoredPos[docNum][field] = append(s.StoredPos[docNum][field], pos) |
||||
} |
||||
|
||||
// walk each composite field
|
||||
for _, field := range result.Document.CompositeFields { |
||||
fieldID := uint16(s.getOrDefineField(field.Name())) |
||||
l, tf := field.Analyze() |
||||
processField(fieldID, field.Name(), l, tf) |
||||
} |
||||
|
||||
// walk each field
|
||||
for i, field := range result.Document.Fields { |
||||
fieldID := uint16(s.getOrDefineField(field.Name())) |
||||
l := result.Length[i] |
||||
tf := result.Analyzed[i] |
||||
processField(fieldID, field.Name(), l, tf) |
||||
if field.Options().IsStored() { |
||||
storeField(docNum, fieldID, encodeFieldType(field), field.Value(), field.ArrayPositions()) |
||||
} |
||||
|
||||
if field.Options().IncludeDocValues() { |
||||
s.DocValueFields[fieldID] = true |
||||
} |
||||
} |
||||
|
||||
// now that its been rolled up into docMap, walk that
|
||||
for fieldID, tokenFrequencies := range docMap { |
||||
for term, tokenFreq := range tokenFrequencies { |
||||
pid := s.Dicts[fieldID][term] - 1 |
||||
bs := s.Postings[pid] |
||||
bs.AddInt(int(docNum)) |
||||
s.Freqs[pid] = append(s.Freqs[pid], uint64(tokenFreq.Frequency())) |
||||
s.Norms[pid] = append(s.Norms[pid], float32(1.0/math.Sqrt(float64(fieldLens[fieldID])))) |
||||
locationBS := s.PostingsLocs[pid] |
||||
if len(tokenFreq.Locations) > 0 { |
||||
locationBS.AddInt(int(docNum)) |
||||
for _, loc := range tokenFreq.Locations { |
||||
var locf = fieldID |
||||
if loc.Field != "" { |
||||
locf = uint16(s.getOrDefineField(loc.Field)) |
||||
} |
||||
s.Locfields[pid] = append(s.Locfields[pid], locf) |
||||
s.Locstarts[pid] = append(s.Locstarts[pid], uint64(loc.Start)) |
||||
s.Locends[pid] = append(s.Locends[pid], uint64(loc.End)) |
||||
s.Locpos[pid] = append(s.Locpos[pid], uint64(loc.Position)) |
||||
if len(loc.ArrayPositions) > 0 { |
||||
s.Locarraypos[pid] = append(s.Locarraypos[pid], loc.ArrayPositions) |
||||
} else { |
||||
s.Locarraypos[pid] = append(s.Locarraypos[pid], nil) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (s *Segment) getOrDefineField(name string) int { |
||||
fieldIDPlus1, ok := s.FieldsMap[name] |
||||
if !ok { |
||||
fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) |
||||
s.FieldsMap[name] = fieldIDPlus1 |
||||
s.FieldsInv = append(s.FieldsInv, name) |
||||
s.Dicts = append(s.Dicts, make(map[string]uint64)) |
||||
s.DictKeys = append(s.DictKeys, make([]string, 0)) |
||||
} |
||||
return int(fieldIDPlus1 - 1) |
||||
} |
||||
|
||||
func (s *Segment) addDocument() int { |
||||
docNum := len(s.Stored) |
||||
s.Stored = append(s.Stored, map[uint16][][]byte{}) |
||||
s.StoredTypes = append(s.StoredTypes, map[uint16][]byte{}) |
||||
s.StoredPos = append(s.StoredPos, map[uint16][][]uint64{}) |
||||
return docNum |
||||
} |
||||
|
||||
func encodeFieldType(f document.Field) byte { |
||||
fieldType := byte('x') |
||||
switch f.(type) { |
||||
case *document.TextField: |
||||
fieldType = 't' |
||||
case *document.NumericField: |
||||
fieldType = 'n' |
||||
case *document.DateTimeField: |
||||
fieldType = 'd' |
||||
case *document.BooleanField: |
||||
fieldType = 'b' |
||||
case *document.GeoPointField: |
||||
fieldType = 'g' |
||||
case *document.CompositeField: |
||||
fieldType = 'c' |
||||
} |
||||
return fieldType |
||||
} |
||||
@ -1,103 +0,0 @@ |
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mem |
||||
|
||||
import ( |
||||
"sort" |
||||
"strings" |
||||
|
||||
"github.com/RoaringBitmap/roaring" |
||||
"github.com/blevesearch/bleve/index" |
||||
"github.com/blevesearch/bleve/index/scorch/segment" |
||||
) |
||||
|
||||
// Dictionary is the in-memory representation of the term dictionary
|
||||
type Dictionary struct { |
||||
segment *Segment |
||||
field string |
||||
fieldID uint16 |
||||
} |
||||
|
||||
// PostingsList returns the postings list for the specified term
|
||||
func (d *Dictionary) PostingsList(term string, |
||||
except *roaring.Bitmap) (segment.PostingsList, error) { |
||||
return &PostingsList{ |
||||
dictionary: d, |
||||
term: term, |
||||
postingsID: d.segment.Dicts[d.fieldID][term], |
||||
except: except, |
||||
}, nil |
||||
} |
||||
|
||||
// Iterator returns an iterator for this dictionary
|
||||
func (d *Dictionary) Iterator() segment.DictionaryIterator { |
||||
return &DictionaryIterator{ |
||||
d: d, |
||||
} |
||||
} |
||||
|
||||
// PrefixIterator returns an iterator which only visits terms having the
|
||||
// the specified prefix
|
||||
func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { |
||||
offset := sort.SearchStrings(d.segment.DictKeys[d.fieldID], prefix) |
||||
return &DictionaryIterator{ |
||||
d: d, |
||||
prefix: prefix, |
||||
offset: offset, |
||||
} |
||||
} |
||||
|
||||
// RangeIterator returns an iterator which only visits terms between the
|
||||
// start and end terms. NOTE: bleve.index API specifies the end is inclusive.
|
||||
func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { |
||||
offset := sort.SearchStrings(d.segment.DictKeys[d.fieldID], start) |
||||
return &DictionaryIterator{ |
||||
d: d, |
||||
offset: offset, |
||||
end: end, |
||||
} |
||||
} |
||||
|
||||
// DictionaryIterator is an iterator for term dictionary
|
||||
type DictionaryIterator struct { |
||||
d *Dictionary |
||||
prefix string |
||||
end string |
||||
offset int |
||||
|
||||
dictEntry index.DictEntry // reused across Next()'s
|
||||
} |
||||
|
||||
// Next returns the next entry in the dictionary
|
||||
func (d *DictionaryIterator) Next() (*index.DictEntry, error) { |
||||
if d.offset > len(d.d.segment.DictKeys[d.d.fieldID])-1 { |
||||
return nil, nil |
||||
} |
||||
next := d.d.segment.DictKeys[d.d.fieldID][d.offset] |
||||
// check prefix
|
||||
if d.prefix != "" && !strings.HasPrefix(next, d.prefix) { |
||||
return nil, nil |
||||
} |
||||
// check end (bleve.index API demands inclusive end)
|
||||
if d.end != "" && next > d.end { |
||||
return nil, nil |
||||
} |
||||
|
||||
d.offset++ |
||||
postingID := d.d.segment.Dicts[d.d.fieldID][next] |
||||
d.dictEntry.Term = next |
||||
d.dictEntry.Count = d.d.segment.Postings[postingID-1].GetCardinality() |
||||
return &d.dictEntry, nil |
||||
} |
||||
@ -1,178 +0,0 @@ |
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mem |
||||
|
||||
import ( |
||||
"github.com/RoaringBitmap/roaring" |
||||
"github.com/blevesearch/bleve/index/scorch/segment" |
||||
) |
||||
|
||||
// PostingsList is an in-memory represenation of a postings list
|
||||
type PostingsList struct { |
||||
dictionary *Dictionary |
||||
term string |
||||
postingsID uint64 |
||||
except *roaring.Bitmap |
||||
} |
||||
|
||||
// Count returns the number of items on this postings list
|
||||
func (p *PostingsList) Count() uint64 { |
||||
var rv uint64 |
||||
if p.postingsID > 0 { |
||||
rv = p.dictionary.segment.Postings[p.postingsID-1].GetCardinality() |
||||
if p.except != nil { |
||||
except := p.except.GetCardinality() |
||||
if except > rv { |
||||
// avoid underflow
|
||||
except = rv |
||||
} |
||||
rv -= except |
||||
} |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
// Iterator returns an iterator for this postings list
|
||||
func (p *PostingsList) Iterator() segment.PostingsIterator { |
||||
rv := &PostingsIterator{ |
||||
postings: p, |
||||
} |
||||
if p.postingsID > 0 { |
||||
allbits := p.dictionary.segment.Postings[p.postingsID-1] |
||||
rv.locations = p.dictionary.segment.PostingsLocs[p.postingsID-1] |
||||
rv.all = allbits.Iterator() |
||||
if p.except != nil { |
||||
allExcept := allbits.Clone() |
||||
allExcept.AndNot(p.except) |
||||
rv.actual = allExcept.Iterator() |
||||
} else { |
||||
rv.actual = allbits.Iterator() |
||||
} |
||||
} |
||||
|
||||
return rv |
||||
} |
||||
|
||||
// PostingsIterator provides a way to iterate through the postings list
|
||||
type PostingsIterator struct { |
||||
postings *PostingsList |
||||
all roaring.IntIterable |
||||
locations *roaring.Bitmap |
||||
offset int |
||||
locoffset int |
||||
actual roaring.IntIterable |
||||
} |
||||
|
||||
// Next returns the next posting on the postings list, or nil at the end
|
||||
func (i *PostingsIterator) Next() (segment.Posting, error) { |
||||
if i.actual == nil || !i.actual.HasNext() { |
||||
return nil, nil |
||||
} |
||||
n := i.actual.Next() |
||||
allN := i.all.Next() |
||||
|
||||
// n is the next actual hit (excluding some postings)
|
||||
// allN is the next hit in the full postings
|
||||
// if they don't match, adjust offsets to factor in item we're skipping over
|
||||
// incr the all iterator, and check again
|
||||
for allN != n { |
||||
i.locoffset += int(i.postings.dictionary.segment.Freqs[i.postings.postingsID-1][i.offset]) |
||||
i.offset++ |
||||
allN = i.all.Next() |
||||
} |
||||
rv := &Posting{ |
||||
iterator: i, |
||||
docNum: uint64(n), |
||||
offset: i.offset, |
||||
locoffset: i.locoffset, |
||||
hasLoc: i.locations.Contains(n), |
||||
} |
||||
|
||||
i.locoffset += int(i.postings.dictionary.segment.Freqs[i.postings.postingsID-1][i.offset]) |
||||
i.offset++ |
||||
return rv, nil |
||||
} |
||||
|
||||
// Posting is a single entry in a postings list
|
||||
type Posting struct { |
||||
iterator *PostingsIterator |
||||
docNum uint64 |
||||
offset int |
||||
locoffset int |
||||
hasLoc bool |
||||
} |
||||
|
||||
// Number returns the document number of this posting in this segment
|
||||
func (p *Posting) Number() uint64 { |
||||
return p.docNum |
||||
} |
||||
|
||||
// Frequency returns the frequence of occurance of this term in this doc/field
|
||||
func (p *Posting) Frequency() uint64 { |
||||
return p.iterator.postings.dictionary.segment.Freqs[p.iterator.postings.postingsID-1][p.offset] |
||||
} |
||||
|
||||
// Norm returns the normalization factor for this posting
|
||||
func (p *Posting) Norm() float64 { |
||||
return float64(p.iterator.postings.dictionary.segment.Norms[p.iterator.postings.postingsID-1][p.offset]) |
||||
} |
||||
|
||||
// Locations returns the location information for each occurance
|
||||
func (p *Posting) Locations() []segment.Location { |
||||
if !p.hasLoc { |
||||
return nil |
||||
} |
||||
freq := int(p.Frequency()) |
||||
rv := make([]segment.Location, freq) |
||||
for i := 0; i < freq; i++ { |
||||
rv[i] = &Location{ |
||||
p: p, |
||||
offset: p.locoffset + i, |
||||
} |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
// Location represents the location of a single occurance
|
||||
type Location struct { |
||||
p *Posting |
||||
offset int |
||||
} |
||||
|
||||
// Field returns the name of the field (useful in composite fields to know
|
||||
// which original field the value came from)
|
||||
func (l *Location) Field() string { |
||||
return l.p.iterator.postings.dictionary.segment.FieldsInv[l.p.iterator.postings.dictionary.segment.Locfields[l.p.iterator.postings.postingsID-1][l.offset]] |
||||
} |
||||
|
||||
// Start returns the start byte offset of this occurance
|
||||
func (l *Location) Start() uint64 { |
||||
return l.p.iterator.postings.dictionary.segment.Locstarts[l.p.iterator.postings.postingsID-1][l.offset] |
||||
} |
||||
|
||||
// End returns the end byte offset of this occurance
|
||||
func (l *Location) End() uint64 { |
||||
return l.p.iterator.postings.dictionary.segment.Locends[l.p.iterator.postings.postingsID-1][l.offset] |
||||
} |
||||
|
||||
// Pos returns the 1-based phrase position of this occurance
|
||||
func (l *Location) Pos() uint64 { |
||||
return l.p.iterator.postings.dictionary.segment.Locpos[l.p.iterator.postings.postingsID-1][l.offset] |
||||
} |
||||
|
||||
// ArrayPositions returns the array position vector associated with this occurance
|
||||
func (l *Location) ArrayPositions() []uint64 { |
||||
return l.p.iterator.postings.dictionary.segment.Locarraypos[l.p.iterator.postings.postingsID-1][l.offset] |
||||
} |
||||
@ -1,289 +0,0 @@ |
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package mem |
||||
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"github.com/RoaringBitmap/roaring" |
||||
"github.com/blevesearch/bleve/index/scorch/segment" |
||||
) |
||||
|
||||
// _id field is always guaranteed to have fieldID of 0
|
||||
const idFieldID uint16 = 0 |
||||
|
||||
// KNOWN ISSUES
|
||||
// - LIMITATION - we decided whether or not to store term vectors for a field
|
||||
// at the segment level, based on the first definition of a
|
||||
// field we see. in normal bleve usage this is fine, all
|
||||
// instances of a field definition will be the same. however,
|
||||
// advanced users may violate this and provide unique field
|
||||
// definitions with each document. this segment does not
|
||||
// support this usage.
|
||||
|
||||
// TODO
|
||||
// - need better testing of multiple docs, iterating freqs, locations and
|
||||
// and verifying the correct results are returned
|
||||
|
||||
// Segment is an in memory implementation of scorch.Segment
|
||||
type Segment struct { |
||||
|
||||
// FieldsMap adds 1 to field id to avoid zero value issues
|
||||
// name -> field id + 1
|
||||
FieldsMap map[string]uint16 |
||||
|
||||
// FieldsInv is the inverse of FieldsMap
|
||||
// field id -> name
|
||||
FieldsInv []string |
||||
|
||||
// Term dictionaries for each field
|
||||
// field id -> term -> postings list id + 1
|
||||
Dicts []map[string]uint64 |
||||
|
||||
// Terms for each field, where terms are sorted ascending
|
||||
// field id -> []term
|
||||
DictKeys [][]string |
||||
|
||||
// Postings list
|
||||
// postings list id -> bitmap by docNum
|
||||
Postings []*roaring.Bitmap |
||||
|
||||
// Postings list has locations
|
||||
PostingsLocs []*roaring.Bitmap |
||||
|
||||
// Term frequencies
|
||||
// postings list id -> Freqs (one for each hit in bitmap)
|
||||
Freqs [][]uint64 |
||||
|
||||
// Field norms
|
||||
// postings list id -> Norms (one for each hit in bitmap)
|
||||
Norms [][]float32 |
||||
|
||||
// Field/start/end/pos/locarraypos
|
||||
// postings list id -> start/end/pos/locarraypos (one for each freq)
|
||||
Locfields [][]uint16 |
||||
Locstarts [][]uint64 |
||||
Locends [][]uint64 |
||||
Locpos [][]uint64 |
||||
Locarraypos [][][]uint64 |
||||
|
||||
// Stored field values
|
||||
// docNum -> field id -> slice of values (each value []byte)
|
||||
Stored []map[uint16][][]byte |
||||
|
||||
// Stored field types
|
||||
// docNum -> field id -> slice of types (each type byte)
|
||||
StoredTypes []map[uint16][]byte |
||||
|
||||
// Stored field array positions
|
||||
// docNum -> field id -> slice of array positions (each is []uint64)
|
||||
StoredPos []map[uint16][][]uint64 |
||||
|
||||
// For storing the docValue persisted fields
|
||||
DocValueFields map[uint16]bool |
||||
|
||||
// Footprint of the segment, updated when analyzed document mutations
|
||||
// are added into the segment
|
||||
sizeInBytes uint64 |
||||
} |
||||
|
||||
// New builds a new empty Segment
|
||||
func New() *Segment { |
||||
return &Segment{ |
||||
FieldsMap: map[string]uint16{}, |
||||
DocValueFields: map[uint16]bool{}, |
||||
} |
||||
} |
||||
|
||||
func (s *Segment) updateSizeInBytes() { |
||||
var sizeInBytes uint64 |
||||
|
||||
// FieldsMap, FieldsInv
|
||||
for k, _ := range s.FieldsMap { |
||||
sizeInBytes += uint64((len(k)+int(segment.SizeOfString))*2 + |
||||
2 /* size of uint16 */) |
||||
} |
||||
// overhead from the data structures
|
||||
sizeInBytes += (segment.SizeOfMap + segment.SizeOfSlice) |
||||
|
||||
// Dicts, DictKeys
|
||||
for _, entry := range s.Dicts { |
||||
for k, _ := range entry { |
||||
sizeInBytes += uint64((len(k)+int(segment.SizeOfString))*2 + |
||||
8 /* size of uint64 */) |
||||
} |
||||
// overhead from the data structures
|
||||
sizeInBytes += (segment.SizeOfMap + segment.SizeOfSlice) |
||||
} |
||||
sizeInBytes += (segment.SizeOfSlice * 2) |
||||
|
||||
// Postings, PostingsLocs
|
||||
for i := 0; i < len(s.Postings); i++ { |
||||
sizeInBytes += (s.Postings[i].GetSizeInBytes() + segment.SizeOfPointer) + |
||||
(s.PostingsLocs[i].GetSizeInBytes() + segment.SizeOfPointer) |
||||
} |
||||
sizeInBytes += (segment.SizeOfSlice * 2) |
||||
|
||||
// Freqs, Norms
|
||||
for i := 0; i < len(s.Freqs); i++ { |
||||
sizeInBytes += uint64(len(s.Freqs[i])*8 /* size of uint64 */ + |
||||
len(s.Norms[i])*4 /* size of float32 */) + |
||||
(segment.SizeOfSlice * 2) |
||||
} |
||||
sizeInBytes += (segment.SizeOfSlice * 2) |
||||
|
||||
// Location data
|
||||
for i := 0; i < len(s.Locfields); i++ { |
||||
sizeInBytes += uint64(len(s.Locfields[i])*2 /* size of uint16 */ + |
||||
len(s.Locstarts[i])*8 /* size of uint64 */ + |
||||
len(s.Locends[i])*8 /* size of uint64 */ + |
||||
len(s.Locpos[i])*8 /* size of uint64 */) |
||||
|
||||
for j := 0; j < len(s.Locarraypos[i]); j++ { |
||||
sizeInBytes += uint64(len(s.Locarraypos[i][j])*8 /* size of uint64 */) + |
||||
segment.SizeOfSlice |
||||
} |
||||
|
||||
sizeInBytes += (segment.SizeOfSlice * 5) |
||||
} |
||||
sizeInBytes += (segment.SizeOfSlice * 5) |
||||
|
||||
// Stored data
|
||||
for i := 0; i < len(s.Stored); i++ { |
||||
for _, v := range s.Stored[i] { |
||||
sizeInBytes += uint64(2 /* size of uint16 */) |
||||
for _, arr := range v { |
||||
sizeInBytes += uint64(len(arr)) + segment.SizeOfSlice |
||||
} |
||||
sizeInBytes += segment.SizeOfSlice |
||||
} |
||||
|
||||
for _, v := range s.StoredTypes[i] { |
||||
sizeInBytes += uint64(2 /* size of uint16 */ +len(v)) + segment.SizeOfSlice |
||||
} |
||||
|
||||
for _, v := range s.StoredPos[i] { |
||||
sizeInBytes += uint64(2 /* size of uint16 */) |
||||
for _, arr := range v { |
||||
sizeInBytes += uint64(len(arr)*8 /* size of uint64 */) + |
||||
segment.SizeOfSlice |
||||
} |
||||
sizeInBytes += segment.SizeOfSlice |
||||
} |
||||
|
||||
// overhead from map(s) within Stored, StoredTypes, StoredPos
|
||||
sizeInBytes += (segment.SizeOfMap * 3) |
||||
} |
||||
// overhead from data structures: Stored, StoredTypes, StoredPos
|
||||
sizeInBytes += (segment.SizeOfSlice * 3) |
||||
|
||||
// DocValueFields
|
||||
sizeInBytes += uint64(len(s.DocValueFields)*3 /* size of uint16 + bool */) + |
||||
segment.SizeOfMap |
||||
|
||||
// SizeInBytes
|
||||
sizeInBytes += uint64(8) |
||||
|
||||
s.sizeInBytes = sizeInBytes |
||||
} |
||||
|
||||
func (s *Segment) SizeInBytes() uint64 { |
||||
return s.sizeInBytes |
||||
} |
||||
|
||||
func (s *Segment) AddRef() { |
||||
} |
||||
|
||||
func (s *Segment) DecRef() error { |
||||
return nil |
||||
} |
||||
|
||||
// Fields returns the field names used in this segment
|
||||
func (s *Segment) Fields() []string { |
||||
return s.FieldsInv |
||||
} |
||||
|
||||
// VisitDocument invokes the DocFieldValueVistor for each stored field
|
||||
// for the specified doc number
|
||||
func (s *Segment) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { |
||||
// ensure document number exists
|
||||
if int(num) > len(s.Stored)-1 { |
||||
return nil |
||||
} |
||||
docFields := s.Stored[int(num)] |
||||
st := s.StoredTypes[int(num)] |
||||
sp := s.StoredPos[int(num)] |
||||
for field, values := range docFields { |
||||
for i, value := range values { |
||||
keepGoing := visitor(s.FieldsInv[field], st[field][i], value, sp[field][i]) |
||||
if !keepGoing { |
||||
return nil |
||||
} |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (s *Segment) getField(name string) (int, error) { |
||||
fieldID, ok := s.FieldsMap[name] |
||||
if !ok { |
||||
return 0, fmt.Errorf("no field named %s", name) |
||||
} |
||||
return int(fieldID - 1), nil |
||||
} |
||||
|
||||
// Dictionary returns the term dictionary for the specified field
|
||||
func (s *Segment) Dictionary(field string) (segment.TermDictionary, error) { |
||||
fieldID, err := s.getField(field) |
||||
if err != nil { |
||||
// no such field, return empty dictionary
|
||||
return &segment.EmptyDictionary{}, nil |
||||
} |
||||
return &Dictionary{ |
||||
segment: s, |
||||
field: field, |
||||
fieldID: uint16(fieldID), |
||||
}, nil |
||||
} |
||||
|
||||
// Count returns the number of documents in this segment
|
||||
// (this has no notion of deleted docs)
|
||||
func (s *Segment) Count() uint64 { |
||||
return uint64(len(s.Stored)) |
||||
} |
||||
|
||||
// DocNumbers returns a bitset corresponding to the doc numbers of all the
|
||||
// provided _id strings
|
||||
func (s *Segment) DocNumbers(ids []string) (*roaring.Bitmap, error) { |
||||
rv := roaring.New() |
||||
|
||||
// guard against empty segment
|
||||
if len(s.FieldsMap) > 0 { |
||||
idDictionary := s.Dicts[idFieldID] |
||||
|
||||
for _, id := range ids { |
||||
postingID := idDictionary[id] |
||||
if postingID > 0 { |
||||
rv.Or(s.Postings[postingID-1]) |
||||
} |
||||
} |
||||
} |
||||
return rv, nil |
||||
} |
||||
|
||||
// Close releases all resources associated with this segment
|
||||
func (s *Segment) Close() error { |
||||
return nil |
||||
} |
||||
@ -0,0 +1,75 @@ |
||||
// Copyright (c) 2018 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package segment |
||||
|
||||
import ( |
||||
"regexp/syntax" |
||||
|
||||
"github.com/couchbase/vellum/regexp" |
||||
) |
||||
|
||||
func ParseRegexp(pattern string) (a *regexp.Regexp, prefixBeg, prefixEnd []byte, err error) { |
||||
// TODO: potential optimization where syntax.Regexp supports a Simplify() API?
|
||||
|
||||
parsed, err := syntax.Parse(pattern, syntax.Perl) |
||||
if err != nil { |
||||
return nil, nil, nil, err |
||||
} |
||||
|
||||
re, err := regexp.NewParsedWithLimit(pattern, parsed, regexp.DefaultLimit) |
||||
if err != nil { |
||||
return nil, nil, nil, err |
||||
} |
||||
|
||||
prefix := LiteralPrefix(parsed) |
||||
if prefix != "" { |
||||
prefixBeg := []byte(prefix) |
||||
prefixEnd := IncrementBytes(prefixBeg) |
||||
return re, prefixBeg, prefixEnd, nil |
||||
} |
||||
|
||||
return re, nil, nil, nil |
||||
} |
||||
|
||||
// Returns the literal prefix given the parse tree for a regexp
|
||||
func LiteralPrefix(s *syntax.Regexp) string { |
||||
// traverse the left-most branch in the parse tree as long as the
|
||||
// node represents a concatenation
|
||||
for s != nil && s.Op == syntax.OpConcat { |
||||
if len(s.Sub) < 1 { |
||||
return "" |
||||
} |
||||
|
||||
s = s.Sub[0] |
||||
} |
||||
|
||||
if s.Op == syntax.OpLiteral { |
||||
return string(s.Rune) |
||||
} |
||||
|
||||
return "" // no literal prefix
|
||||
} |
||||
|
||||
func IncrementBytes(in []byte) []byte { |
||||
rv := make([]byte, len(in)) |
||||
copy(rv, in) |
||||
for i := len(rv) - 1; i >= 0; i-- { |
||||
rv[i] = rv[i] + 1 |
||||
if rv[i] != 0 { |
||||
return rv // didn't overflow, so stop
|
||||
} |
||||
} |
||||
return nil // overflowed
|
||||
} |
||||
@ -0,0 +1,826 @@ |
||||
// Copyright (c) 2018 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package zap |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/binary" |
||||
"math" |
||||
"sort" |
||||
"sync" |
||||
|
||||
"github.com/RoaringBitmap/roaring" |
||||
"github.com/blevesearch/bleve/analysis" |
||||
"github.com/blevesearch/bleve/document" |
||||
"github.com/blevesearch/bleve/index" |
||||
"github.com/couchbase/vellum" |
||||
"github.com/golang/snappy" |
||||
) |
||||
|
||||
var NewSegmentBufferNumResultsBump int = 100 |
||||
var NewSegmentBufferNumResultsFactor float64 = 1.0 |
||||
var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 |
||||
|
||||
// AnalysisResultsToSegmentBase produces an in-memory zap-encoded
|
||||
// SegmentBase from analysis results
|
||||
func AnalysisResultsToSegmentBase(results []*index.AnalysisResult, |
||||
chunkFactor uint32) (*SegmentBase, uint64, error) { |
||||
s := interimPool.Get().(*interim) |
||||
|
||||
var br bytes.Buffer |
||||
if s.lastNumDocs > 0 { |
||||
// use previous results to initialize the buf with an estimate
|
||||
// size, but note that the interim instance comes from a
|
||||
// global interimPool, so multiple scorch instances indexing
|
||||
// different docs can lead to low quality estimates
|
||||
estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * |
||||
NewSegmentBufferNumResultsFactor) |
||||
estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * |
||||
NewSegmentBufferAvgBytesPerDocFactor) |
||||
br.Grow(estimateAvgBytesPerDoc * estimateNumResults) |
||||
} |
||||
|
||||
s.results = results |
||||
s.chunkFactor = chunkFactor |
||||
s.w = NewCountHashWriter(&br) |
||||
|
||||
storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, |
||||
err := s.convert() |
||||
if err != nil { |
||||
return nil, uint64(0), err |
||||
} |
||||
|
||||
sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkFactor, |
||||
s.FieldsMap, s.FieldsInv, uint64(len(results)), |
||||
storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) |
||||
|
||||
if err == nil && s.reset() == nil { |
||||
s.lastNumDocs = len(results) |
||||
s.lastOutSize = len(br.Bytes()) |
||||
interimPool.Put(s) |
||||
} |
||||
|
||||
return sb, uint64(len(br.Bytes())), err |
||||
} |
||||
|
||||
var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} |
||||
|
||||
// interim holds temporary working data used while converting from
|
||||
// analysis results to a zap-encoded segment
|
||||
type interim struct { |
||||
results []*index.AnalysisResult |
||||
|
||||
chunkFactor uint32 |
||||
|
||||
w *CountHashWriter |
||||
|
||||
// FieldsMap adds 1 to field id to avoid zero value issues
|
||||
// name -> field id + 1
|
||||
FieldsMap map[string]uint16 |
||||
|
||||
// FieldsInv is the inverse of FieldsMap
|
||||
// field id -> name
|
||||
FieldsInv []string |
||||
|
||||
// Term dictionaries for each field
|
||||
// field id -> term -> postings list id + 1
|
||||
Dicts []map[string]uint64 |
||||
|
||||
// Terms for each field, where terms are sorted ascending
|
||||
// field id -> []term
|
||||
DictKeys [][]string |
||||
|
||||
// Fields whose IncludeDocValues is true
|
||||
// field id -> bool
|
||||
IncludeDocValues []bool |
||||
|
||||
// postings id -> bitmap of docNums
|
||||
Postings []*roaring.Bitmap |
||||
|
||||
// postings id -> freq/norm's, one for each docNum in postings
|
||||
FreqNorms [][]interimFreqNorm |
||||
freqNormsBacking []interimFreqNorm |
||||
|
||||
// postings id -> locs, one for each freq
|
||||
Locs [][]interimLoc |
||||
locsBacking []interimLoc |
||||
|
||||
numTermsPerPostingsList []int // key is postings list id
|
||||
numLocsPerPostingsList []int // key is postings list id
|
||||
|
||||
builder *vellum.Builder |
||||
builderBuf bytes.Buffer |
||||
|
||||
metaBuf bytes.Buffer |
||||
|
||||
tmp0 []byte |
||||
tmp1 []byte |
||||
|
||||
lastNumDocs int |
||||
lastOutSize int |
||||
} |
||||
|
||||
func (s *interim) reset() (err error) { |
||||
s.results = nil |
||||
s.chunkFactor = 0 |
||||
s.w = nil |
||||
s.FieldsMap = nil |
||||
s.FieldsInv = nil |
||||
for i := range s.Dicts { |
||||
s.Dicts[i] = nil |
||||
} |
||||
s.Dicts = s.Dicts[:0] |
||||
for i := range s.DictKeys { |
||||
s.DictKeys[i] = s.DictKeys[i][:0] |
||||
} |
||||
s.DictKeys = s.DictKeys[:0] |
||||
for i := range s.IncludeDocValues { |
||||
s.IncludeDocValues[i] = false |
||||
} |
||||
s.IncludeDocValues = s.IncludeDocValues[:0] |
||||
for _, idn := range s.Postings { |
||||
idn.Clear() |
||||
} |
||||
s.Postings = s.Postings[:0] |
||||
s.FreqNorms = s.FreqNorms[:0] |
||||
for i := range s.freqNormsBacking { |
||||
s.freqNormsBacking[i] = interimFreqNorm{} |
||||
} |
||||
s.freqNormsBacking = s.freqNormsBacking[:0] |
||||
s.Locs = s.Locs[:0] |
||||
for i := range s.locsBacking { |
||||
s.locsBacking[i] = interimLoc{} |
||||
} |
||||
s.locsBacking = s.locsBacking[:0] |
||||
s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] |
||||
s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] |
||||
s.builderBuf.Reset() |
||||
if s.builder != nil { |
||||
err = s.builder.Reset(&s.builderBuf) |
||||
} |
||||
s.metaBuf.Reset() |
||||
s.tmp0 = s.tmp0[:0] |
||||
s.tmp1 = s.tmp1[:0] |
||||
s.lastNumDocs = 0 |
||||
s.lastOutSize = 0 |
||||
|
||||
return err |
||||
} |
||||
|
||||
func (s *interim) grabBuf(size int) []byte { |
||||
buf := s.tmp0 |
||||
if cap(buf) < size { |
||||
buf = make([]byte, size) |
||||
s.tmp0 = buf |
||||
} |
||||
return buf[0:size] |
||||
} |
||||
|
||||
type interimStoredField struct { |
||||
vals [][]byte |
||||
typs []byte |
||||
arrayposs [][]uint64 // array positions
|
||||
} |
||||
|
||||
type interimFreqNorm struct { |
||||
freq uint64 |
||||
norm float32 |
||||
numLocs int |
||||
} |
||||
|
||||
type interimLoc struct { |
||||
fieldID uint16 |
||||
pos uint64 |
||||
start uint64 |
||||
end uint64 |
||||
arrayposs []uint64 |
||||
} |
||||
|
||||
func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { |
||||
s.FieldsMap = map[string]uint16{} |
||||
|
||||
s.getOrDefineField("_id") // _id field is fieldID 0
|
||||
|
||||
for _, result := range s.results { |
||||
for _, field := range result.Document.CompositeFields { |
||||
s.getOrDefineField(field.Name()) |
||||
} |
||||
for _, field := range result.Document.Fields { |
||||
s.getOrDefineField(field.Name()) |
||||
} |
||||
} |
||||
|
||||
sort.Strings(s.FieldsInv[1:]) // keep _id as first field
|
||||
|
||||
for fieldID, fieldName := range s.FieldsInv { |
||||
s.FieldsMap[fieldName] = uint16(fieldID + 1) |
||||
} |
||||
|
||||
if cap(s.IncludeDocValues) >= len(s.FieldsInv) { |
||||
s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] |
||||
} else { |
||||
s.IncludeDocValues = make([]bool, len(s.FieldsInv)) |
||||
} |
||||
|
||||
s.prepareDicts() |
||||
|
||||
for _, dict := range s.DictKeys { |
||||
sort.Strings(dict) |
||||
} |
||||
|
||||
s.processDocuments() |
||||
|
||||
storedIndexOffset, err := s.writeStoredFields() |
||||
if err != nil { |
||||
return 0, 0, 0, nil, err |
||||
} |
||||
|
||||
var fdvIndexOffset uint64 |
||||
var dictOffsets []uint64 |
||||
|
||||
if len(s.results) > 0 { |
||||
fdvIndexOffset, dictOffsets, err = s.writeDicts() |
||||
if err != nil { |
||||
return 0, 0, 0, nil, err |
||||
} |
||||
} else { |
||||
dictOffsets = make([]uint64, len(s.FieldsInv)) |
||||
} |
||||
|
||||
fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) |
||||
if err != nil { |
||||
return 0, 0, 0, nil, err |
||||
} |
||||
|
||||
return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil |
||||
} |
||||
|
||||
func (s *interim) getOrDefineField(fieldName string) int { |
||||
fieldIDPlus1, exists := s.FieldsMap[fieldName] |
||||
if !exists { |
||||
fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) |
||||
s.FieldsMap[fieldName] = fieldIDPlus1 |
||||
s.FieldsInv = append(s.FieldsInv, fieldName) |
||||
|
||||
s.Dicts = append(s.Dicts, make(map[string]uint64)) |
||||
|
||||
n := len(s.DictKeys) |
||||
if n < cap(s.DictKeys) { |
||||
s.DictKeys = s.DictKeys[:n+1] |
||||
s.DictKeys[n] = s.DictKeys[n][:0] |
||||
} else { |
||||
s.DictKeys = append(s.DictKeys, []string(nil)) |
||||
} |
||||
} |
||||
|
||||
return int(fieldIDPlus1 - 1) |
||||
} |
||||
|
||||
// fill Dicts and DictKeys from analysis results
|
||||
func (s *interim) prepareDicts() { |
||||
var pidNext int |
||||
|
||||
var totTFs int |
||||
var totLocs int |
||||
|
||||
visitField := func(fieldID uint16, tfs analysis.TokenFrequencies) { |
||||
dict := s.Dicts[fieldID] |
||||
dictKeys := s.DictKeys[fieldID] |
||||
|
||||
for term, tf := range tfs { |
||||
pidPlus1, exists := dict[term] |
||||
if !exists { |
||||
pidNext++ |
||||
pidPlus1 = uint64(pidNext) |
||||
|
||||
dict[term] = pidPlus1 |
||||
dictKeys = append(dictKeys, term) |
||||
|
||||
s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) |
||||
s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) |
||||
} |
||||
|
||||
pid := pidPlus1 - 1 |
||||
|
||||
s.numTermsPerPostingsList[pid] += 1 |
||||
s.numLocsPerPostingsList[pid] += len(tf.Locations) |
||||
|
||||
totLocs += len(tf.Locations) |
||||
} |
||||
|
||||
totTFs += len(tfs) |
||||
|
||||
s.DictKeys[fieldID] = dictKeys |
||||
} |
||||
|
||||
for _, result := range s.results { |
||||
// walk each composite field
|
||||
for _, field := range result.Document.CompositeFields { |
||||
fieldID := uint16(s.getOrDefineField(field.Name())) |
||||
_, tf := field.Analyze() |
||||
visitField(fieldID, tf) |
||||
} |
||||
|
||||
// walk each field
|
||||
for i, field := range result.Document.Fields { |
||||
fieldID := uint16(s.getOrDefineField(field.Name())) |
||||
tf := result.Analyzed[i] |
||||
visitField(fieldID, tf) |
||||
} |
||||
} |
||||
|
||||
numPostingsLists := pidNext |
||||
|
||||
if cap(s.Postings) >= numPostingsLists { |
||||
s.Postings = s.Postings[:numPostingsLists] |
||||
} else { |
||||
postings := make([]*roaring.Bitmap, numPostingsLists) |
||||
copy(postings, s.Postings[:cap(s.Postings)]) |
||||
for i := 0; i < numPostingsLists; i++ { |
||||
if postings[i] == nil { |
||||
postings[i] = roaring.New() |
||||
} |
||||
} |
||||
s.Postings = postings |
||||
} |
||||
|
||||
if cap(s.FreqNorms) >= numPostingsLists { |
||||
s.FreqNorms = s.FreqNorms[:numPostingsLists] |
||||
} else { |
||||
s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) |
||||
} |
||||
|
||||
if cap(s.freqNormsBacking) >= totTFs { |
||||
s.freqNormsBacking = s.freqNormsBacking[:totTFs] |
||||
} else { |
||||
s.freqNormsBacking = make([]interimFreqNorm, totTFs) |
||||
} |
||||
|
||||
freqNormsBacking := s.freqNormsBacking |
||||
for pid, numTerms := range s.numTermsPerPostingsList { |
||||
s.FreqNorms[pid] = freqNormsBacking[0:0] |
||||
freqNormsBacking = freqNormsBacking[numTerms:] |
||||
} |
||||
|
||||
if cap(s.Locs) >= numPostingsLists { |
||||
s.Locs = s.Locs[:numPostingsLists] |
||||
} else { |
||||
s.Locs = make([][]interimLoc, numPostingsLists) |
||||
} |
||||
|
||||
if cap(s.locsBacking) >= totLocs { |
||||
s.locsBacking = s.locsBacking[:totLocs] |
||||
} else { |
||||
s.locsBacking = make([]interimLoc, totLocs) |
||||
} |
||||
|
||||
locsBacking := s.locsBacking |
||||
for pid, numLocs := range s.numLocsPerPostingsList { |
||||
s.Locs[pid] = locsBacking[0:0] |
||||
locsBacking = locsBacking[numLocs:] |
||||
} |
||||
} |
||||
|
||||
func (s *interim) processDocuments() { |
||||
numFields := len(s.FieldsInv) |
||||
reuseFieldLens := make([]int, numFields) |
||||
reuseFieldTFs := make([]analysis.TokenFrequencies, numFields) |
||||
|
||||
for docNum, result := range s.results { |
||||
for i := 0; i < numFields; i++ { // clear these for reuse
|
||||
reuseFieldLens[i] = 0 |
||||
reuseFieldTFs[i] = nil |
||||
} |
||||
|
||||
s.processDocument(uint64(docNum), result, |
||||
reuseFieldLens, reuseFieldTFs) |
||||
} |
||||
} |
||||
|
||||
func (s *interim) processDocument(docNum uint64, |
||||
result *index.AnalysisResult, |
||||
fieldLens []int, fieldTFs []analysis.TokenFrequencies) { |
||||
visitField := func(fieldID uint16, fieldName string, |
||||
ln int, tf analysis.TokenFrequencies) { |
||||
fieldLens[fieldID] += ln |
||||
|
||||
existingFreqs := fieldTFs[fieldID] |
||||
if existingFreqs != nil { |
||||
existingFreqs.MergeAll(fieldName, tf) |
||||
} else { |
||||
fieldTFs[fieldID] = tf |
||||
} |
||||
} |
||||
|
||||
// walk each composite field
|
||||
for _, field := range result.Document.CompositeFields { |
||||
fieldID := uint16(s.getOrDefineField(field.Name())) |
||||
ln, tf := field.Analyze() |
||||
visitField(fieldID, field.Name(), ln, tf) |
||||
} |
||||
|
||||
// walk each field
|
||||
for i, field := range result.Document.Fields { |
||||
fieldID := uint16(s.getOrDefineField(field.Name())) |
||||
ln := result.Length[i] |
||||
tf := result.Analyzed[i] |
||||
visitField(fieldID, field.Name(), ln, tf) |
||||
} |
||||
|
||||
// now that it's been rolled up into fieldTFs, walk that
|
||||
for fieldID, tfs := range fieldTFs { |
||||
dict := s.Dicts[fieldID] |
||||
norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID]))) |
||||
|
||||
for term, tf := range tfs { |
||||
pid := dict[term] - 1 |
||||
bs := s.Postings[pid] |
||||
bs.Add(uint32(docNum)) |
||||
|
||||
s.FreqNorms[pid] = append(s.FreqNorms[pid], |
||||
interimFreqNorm{ |
||||
freq: uint64(tf.Frequency()), |
||||
norm: norm, |
||||
numLocs: len(tf.Locations), |
||||
}) |
||||
|
||||
if len(tf.Locations) > 0 { |
||||
locs := s.Locs[pid] |
||||
|
||||
for _, loc := range tf.Locations { |
||||
var locf = uint16(fieldID) |
||||
if loc.Field != "" { |
||||
locf = uint16(s.getOrDefineField(loc.Field)) |
||||
} |
||||
var arrayposs []uint64 |
||||
if len(loc.ArrayPositions) > 0 { |
||||
arrayposs = loc.ArrayPositions |
||||
} |
||||
locs = append(locs, interimLoc{ |
||||
fieldID: locf, |
||||
pos: uint64(loc.Position), |
||||
start: uint64(loc.Start), |
||||
end: uint64(loc.End), |
||||
arrayposs: arrayposs, |
||||
}) |
||||
} |
||||
|
||||
s.Locs[pid] = locs |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (s *interim) writeStoredFields() ( |
||||
storedIndexOffset uint64, err error) { |
||||
varBuf := make([]byte, binary.MaxVarintLen64) |
||||
metaEncode := func(val uint64) (int, error) { |
||||
wb := binary.PutUvarint(varBuf, val) |
||||
return s.metaBuf.Write(varBuf[:wb]) |
||||
} |
||||
|
||||
data, compressed := s.tmp0[:0], s.tmp1[:0] |
||||
defer func() { s.tmp0, s.tmp1 = data, compressed }() |
||||
|
||||
// keyed by docNum
|
||||
docStoredOffsets := make([]uint64, len(s.results)) |
||||
|
||||
// keyed by fieldID, for the current doc in the loop
|
||||
docStoredFields := map[uint16]interimStoredField{} |
||||
|
||||
for docNum, result := range s.results { |
||||
for fieldID := range docStoredFields { // reset for next doc
|
||||
delete(docStoredFields, fieldID) |
||||
} |
||||
|
||||
for _, field := range result.Document.Fields { |
||||
fieldID := uint16(s.getOrDefineField(field.Name())) |
||||
|
||||
opts := field.Options() |
||||
|
||||
if opts.IsStored() { |
||||
isf := docStoredFields[fieldID] |
||||
isf.vals = append(isf.vals, field.Value()) |
||||
isf.typs = append(isf.typs, encodeFieldType(field)) |
||||
isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) |
||||
docStoredFields[fieldID] = isf |
||||
} |
||||
|
||||
if opts.IncludeDocValues() { |
||||
s.IncludeDocValues[fieldID] = true |
||||
} |
||||
} |
||||
|
||||
var curr int |
||||
|
||||
s.metaBuf.Reset() |
||||
data = data[:0] |
||||
|
||||
// _id field special case optimizes ExternalID() lookups
|
||||
idFieldVal := docStoredFields[uint16(0)].vals[0] |
||||
_, err = metaEncode(uint64(len(idFieldVal))) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
// handle non-"_id" fields
|
||||
for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { |
||||
isf, exists := docStoredFields[uint16(fieldID)] |
||||
if exists { |
||||
curr, data, err = persistStoredFieldValues( |
||||
fieldID, isf.vals, isf.typs, isf.arrayposs, |
||||
curr, metaEncode, data) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
} |
||||
|
||||
metaBytes := s.metaBuf.Bytes() |
||||
|
||||
compressed = snappy.Encode(compressed[:cap(compressed)], data) |
||||
|
||||
docStoredOffsets[docNum] = uint64(s.w.Count()) |
||||
|
||||
_, err := writeUvarints(s.w, |
||||
uint64(len(metaBytes)), |
||||
uint64(len(idFieldVal)+len(compressed))) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
_, err = s.w.Write(metaBytes) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
_, err = s.w.Write(idFieldVal) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
_, err = s.w.Write(compressed) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
|
||||
storedIndexOffset = uint64(s.w.Count()) |
||||
|
||||
for _, docStoredOffset := range docStoredOffsets { |
||||
err = binary.Write(s.w, binary.BigEndian, docStoredOffset) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
|
||||
return storedIndexOffset, nil |
||||
} |
||||
|
||||
func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { |
||||
dictOffsets = make([]uint64, len(s.FieldsInv)) |
||||
|
||||
fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) |
||||
fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) |
||||
|
||||
buf := s.grabBuf(binary.MaxVarintLen64) |
||||
|
||||
tfEncoder := newChunkedIntCoder(uint64(s.chunkFactor), uint64(len(s.results)-1)) |
||||
locEncoder := newChunkedIntCoder(uint64(s.chunkFactor), uint64(len(s.results)-1)) |
||||
fdvEncoder := newChunkedContentCoder(uint64(s.chunkFactor), uint64(len(s.results)-1), s.w, false) |
||||
|
||||
var docTermMap [][]byte |
||||
|
||||
if s.builder == nil { |
||||
s.builder, err = vellum.New(&s.builderBuf, nil) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
} |
||||
|
||||
for fieldID, terms := range s.DictKeys { |
||||
if cap(docTermMap) < len(s.results) { |
||||
docTermMap = make([][]byte, len(s.results)) |
||||
} else { |
||||
docTermMap = docTermMap[0:len(s.results)] |
||||
for docNum := range docTermMap { // reset the docTermMap
|
||||
docTermMap[docNum] = docTermMap[docNum][:0] |
||||
} |
||||
} |
||||
|
||||
dict := s.Dicts[fieldID] |
||||
|
||||
for _, term := range terms { // terms are already sorted
|
||||
pid := dict[term] - 1 |
||||
|
||||
postingsBS := s.Postings[pid] |
||||
|
||||
freqNorms := s.FreqNorms[pid] |
||||
freqNormOffset := 0 |
||||
|
||||
locs := s.Locs[pid] |
||||
locOffset := 0 |
||||
|
||||
postingsItr := postingsBS.Iterator() |
||||
for postingsItr.HasNext() { |
||||
docNum := uint64(postingsItr.Next()) |
||||
|
||||
freqNorm := freqNorms[freqNormOffset] |
||||
|
||||
err = tfEncoder.Add(docNum, |
||||
encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), |
||||
uint64(math.Float32bits(freqNorm.norm))) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
|
||||
if freqNorm.numLocs > 0 { |
||||
numBytesLocs := 0 |
||||
for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { |
||||
numBytesLocs += totalUvarintBytes( |
||||
uint64(loc.fieldID), loc.pos, loc.start, loc.end, |
||||
uint64(len(loc.arrayposs)), loc.arrayposs) |
||||
} |
||||
|
||||
err = locEncoder.Add(docNum, uint64(numBytesLocs)) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
|
||||
for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { |
||||
err = locEncoder.Add(docNum, |
||||
uint64(loc.fieldID), loc.pos, loc.start, loc.end, |
||||
uint64(len(loc.arrayposs))) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
|
||||
err = locEncoder.Add(docNum, loc.arrayposs...) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
} |
||||
|
||||
locOffset += freqNorm.numLocs |
||||
} |
||||
|
||||
freqNormOffset++ |
||||
|
||||
docTermMap[docNum] = append( |
||||
append(docTermMap[docNum], term...), |
||||
termSeparator) |
||||
} |
||||
|
||||
tfEncoder.Close() |
||||
locEncoder.Close() |
||||
|
||||
postingsOffset, err := |
||||
writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
|
||||
if postingsOffset > uint64(0) { |
||||
err = s.builder.Insert([]byte(term), postingsOffset) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
} |
||||
|
||||
tfEncoder.Reset() |
||||
locEncoder.Reset() |
||||
} |
||||
|
||||
err = s.builder.Close() |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
|
||||
// record where this dictionary starts
|
||||
dictOffsets[fieldID] = uint64(s.w.Count()) |
||||
|
||||
vellumData := s.builderBuf.Bytes() |
||||
|
||||
// write out the length of the vellum data
|
||||
n := binary.PutUvarint(buf, uint64(len(vellumData))) |
||||
_, err = s.w.Write(buf[:n]) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
|
||||
// write this vellum to disk
|
||||
_, err = s.w.Write(vellumData) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
|
||||
// reset vellum for reuse
|
||||
s.builderBuf.Reset() |
||||
|
||||
err = s.builder.Reset(&s.builderBuf) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
|
||||
// write the field doc values
|
||||
if s.IncludeDocValues[fieldID] { |
||||
for docNum, docTerms := range docTermMap { |
||||
if len(docTerms) > 0 { |
||||
err = fdvEncoder.Add(uint64(docNum), docTerms) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
} |
||||
} |
||||
err = fdvEncoder.Close() |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
|
||||
fdvOffsetsStart[fieldID] = uint64(s.w.Count()) |
||||
|
||||
_, err = fdvEncoder.Write() |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
|
||||
fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) |
||||
|
||||
fdvEncoder.Reset() |
||||
} else { |
||||
fdvOffsetsStart[fieldID] = fieldNotUninverted |
||||
fdvOffsetsEnd[fieldID] = fieldNotUninverted |
||||
} |
||||
} |
||||
|
||||
fdvIndexOffset = uint64(s.w.Count()) |
||||
|
||||
for i := 0; i < len(fdvOffsetsStart); i++ { |
||||
n := binary.PutUvarint(buf, fdvOffsetsStart[i]) |
||||
_, err := s.w.Write(buf[:n]) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) |
||||
_, err = s.w.Write(buf[:n]) |
||||
if err != nil { |
||||
return 0, nil, err |
||||
} |
||||
} |
||||
|
||||
return fdvIndexOffset, dictOffsets, nil |
||||
} |
||||
|
||||
func encodeFieldType(f document.Field) byte { |
||||
fieldType := byte('x') |
||||
switch f.(type) { |
||||
case *document.TextField: |
||||
fieldType = 't' |
||||
case *document.NumericField: |
||||
fieldType = 'n' |
||||
case *document.DateTimeField: |
||||
fieldType = 'd' |
||||
case *document.BooleanField: |
||||
fieldType = 'b' |
||||
case *document.GeoPointField: |
||||
fieldType = 'g' |
||||
case *document.CompositeField: |
||||
fieldType = 'c' |
||||
} |
||||
return fieldType |
||||
} |
||||
|
||||
// returns the total # of bytes needed to encode the given uint64's
|
||||
// into binary.PutUVarint() encoding
|
||||
func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { |
||||
n = numUvarintBytes(a) |
||||
n += numUvarintBytes(b) |
||||
n += numUvarintBytes(c) |
||||
n += numUvarintBytes(d) |
||||
n += numUvarintBytes(e) |
||||
for _, v := range more { |
||||
n += numUvarintBytes(v) |
||||
} |
||||
return n |
||||
} |
||||
|
||||
// returns # of bytes needed to encode x in binary.PutUvarint() encoding
|
||||
func numUvarintBytes(x uint64) (n int) { |
||||
for x >= 0x80 { |
||||
x >>= 7 |
||||
n++ |
||||
} |
||||
return n + 1 |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
343
vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go
generated
vendored
343
vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go
generated
vendored
@ -0,0 +1,343 @@ |
||||
// Copyright (c) 2018 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher |
||||
|
||||
import ( |
||||
"bytes" |
||||
"container/heap" |
||||
"math" |
||||
"reflect" |
||||
|
||||
"github.com/blevesearch/bleve/index" |
||||
"github.com/blevesearch/bleve/search" |
||||
"github.com/blevesearch/bleve/search/scorer" |
||||
"github.com/blevesearch/bleve/size" |
||||
) |
||||
|
||||
var reflectStaticSizeDisjunctionHeapSearcher int |
||||
var reflectStaticSizeSearcherCurr int |
||||
|
||||
func init() { |
||||
var dhs DisjunctionHeapSearcher |
||||
reflectStaticSizeDisjunctionHeapSearcher = int(reflect.TypeOf(dhs).Size()) |
||||
|
||||
var sc SearcherCurr |
||||
reflectStaticSizeSearcherCurr = int(reflect.TypeOf(sc).Size()) |
||||
} |
||||
|
||||
type SearcherCurr struct { |
||||
searcher search.Searcher |
||||
curr *search.DocumentMatch |
||||
} |
||||
|
||||
type DisjunctionHeapSearcher struct { |
||||
indexReader index.IndexReader |
||||
|
||||
numSearchers int |
||||
scorer *scorer.DisjunctionQueryScorer |
||||
min int |
||||
queryNorm float64 |
||||
initialized bool |
||||
searchers []search.Searcher |
||||
heap []*SearcherCurr |
||||
|
||||
matching []*search.DocumentMatch |
||||
matchingCurrs []*SearcherCurr |
||||
} |
||||
|
||||
func newDisjunctionHeapSearcher(indexReader index.IndexReader, |
||||
searchers []search.Searcher, min float64, options search.SearcherOptions, |
||||
limit bool) ( |
||||
*DisjunctionHeapSearcher, error) { |
||||
if limit && tooManyClauses(len(searchers)) { |
||||
return nil, tooManyClausesErr(len(searchers)) |
||||
} |
||||
|
||||
// build our searcher
|
||||
rv := DisjunctionHeapSearcher{ |
||||
indexReader: indexReader, |
||||
searchers: searchers, |
||||
numSearchers: len(searchers), |
||||
scorer: scorer.NewDisjunctionQueryScorer(options), |
||||
min: int(min), |
||||
matching: make([]*search.DocumentMatch, len(searchers)), |
||||
matchingCurrs: make([]*SearcherCurr, len(searchers)), |
||||
heap: make([]*SearcherCurr, 0, len(searchers)), |
||||
} |
||||
rv.computeQueryNorm() |
||||
return &rv, nil |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) Size() int { |
||||
sizeInBytes := reflectStaticSizeDisjunctionHeapSearcher + size.SizeOfPtr + |
||||
s.scorer.Size() |
||||
|
||||
for _, entry := range s.searchers { |
||||
sizeInBytes += entry.Size() |
||||
} |
||||
|
||||
for _, entry := range s.matching { |
||||
if entry != nil { |
||||
sizeInBytes += entry.Size() |
||||
} |
||||
} |
||||
|
||||
// for matchingCurrs and heap, just use static size * len
|
||||
// since searchers and document matches already counted above
|
||||
sizeInBytes += len(s.matchingCurrs) * reflectStaticSizeSearcherCurr |
||||
sizeInBytes += len(s.heap) * reflectStaticSizeSearcherCurr |
||||
|
||||
return sizeInBytes |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) computeQueryNorm() { |
||||
// first calculate sum of squared weights
|
||||
sumOfSquaredWeights := 0.0 |
||||
for _, searcher := range s.searchers { |
||||
sumOfSquaredWeights += searcher.Weight() |
||||
} |
||||
// now compute query norm from this
|
||||
s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) |
||||
// finally tell all the downstream searchers the norm
|
||||
for _, searcher := range s.searchers { |
||||
searcher.SetQueryNorm(s.queryNorm) |
||||
} |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) initSearchers(ctx *search.SearchContext) error { |
||||
// alloc a single block of SearcherCurrs
|
||||
block := make([]SearcherCurr, len(s.searchers)) |
||||
|
||||
// get all searchers pointing at their first match
|
||||
for i, searcher := range s.searchers { |
||||
curr, err := searcher.Next(ctx) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if curr != nil { |
||||
block[i].searcher = searcher |
||||
block[i].curr = curr |
||||
heap.Push(s, &block[i]) |
||||
} |
||||
} |
||||
|
||||
err := s.updateMatches() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
s.initialized = true |
||||
return nil |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) updateMatches() error { |
||||
matching := s.matching[:0] |
||||
matchingCurrs := s.matchingCurrs[:0] |
||||
|
||||
if len(s.heap) > 0 { |
||||
|
||||
// top of the heap is our next hit
|
||||
next := heap.Pop(s).(*SearcherCurr) |
||||
matching = append(matching, next.curr) |
||||
matchingCurrs = append(matchingCurrs, next) |
||||
|
||||
// now as long as top of heap matches, keep popping
|
||||
for len(s.heap) > 0 && bytes.Compare(next.curr.IndexInternalID, s.heap[0].curr.IndexInternalID) == 0 { |
||||
next = heap.Pop(s).(*SearcherCurr) |
||||
matching = append(matching, next.curr) |
||||
matchingCurrs = append(matchingCurrs, next) |
||||
} |
||||
} |
||||
|
||||
s.matching = matching |
||||
s.matchingCurrs = matchingCurrs |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) Weight() float64 { |
||||
var rv float64 |
||||
for _, searcher := range s.searchers { |
||||
rv += searcher.Weight() |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) SetQueryNorm(qnorm float64) { |
||||
for _, searcher := range s.searchers { |
||||
searcher.SetQueryNorm(qnorm) |
||||
} |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) Next(ctx *search.SearchContext) ( |
||||
*search.DocumentMatch, error) { |
||||
if !s.initialized { |
||||
err := s.initSearchers(ctx) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
var rv *search.DocumentMatch |
||||
found := false |
||||
for !found && len(s.matching) > 0 { |
||||
if len(s.matching) >= s.min { |
||||
found = true |
||||
// score this match
|
||||
rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) |
||||
} |
||||
|
||||
// invoke next on all the matching searchers
|
||||
for _, matchingCurr := range s.matchingCurrs { |
||||
if matchingCurr.curr != rv { |
||||
ctx.DocumentMatchPool.Put(matchingCurr.curr) |
||||
} |
||||
curr, err := matchingCurr.searcher.Next(ctx) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if curr != nil { |
||||
matchingCurr.curr = curr |
||||
heap.Push(s, matchingCurr) |
||||
} |
||||
} |
||||
|
||||
err := s.updateMatches() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
return rv, nil |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) Advance(ctx *search.SearchContext, |
||||
ID index.IndexInternalID) (*search.DocumentMatch, error) { |
||||
if !s.initialized { |
||||
err := s.initSearchers(ctx) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
// if there is anything in matching, toss it back onto the heap
|
||||
for _, matchingCurr := range s.matchingCurrs { |
||||
heap.Push(s, matchingCurr) |
||||
} |
||||
s.matching = s.matching[:0] |
||||
s.matchingCurrs = s.matchingCurrs[:0] |
||||
|
||||
// find all searchers that actually need to be advanced
|
||||
// advance them, using s.matchingCurrs as temp storage
|
||||
for len(s.heap) > 0 && bytes.Compare(s.heap[0].curr.IndexInternalID, ID) < 0 { |
||||
searcherCurr := heap.Pop(s).(*SearcherCurr) |
||||
ctx.DocumentMatchPool.Put(searcherCurr.curr) |
||||
curr, err := searcherCurr.searcher.Advance(ctx, ID) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if curr != nil { |
||||
searcherCurr.curr = curr |
||||
s.matchingCurrs = append(s.matchingCurrs, searcherCurr) |
||||
} |
||||
} |
||||
// now all of the searchers that we advanced have to be pushed back
|
||||
for _, matchingCurr := range s.matchingCurrs { |
||||
heap.Push(s, matchingCurr) |
||||
} |
||||
// reset our temp space
|
||||
s.matchingCurrs = s.matchingCurrs[:0] |
||||
|
||||
err := s.updateMatches() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return s.Next(ctx) |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) Count() uint64 { |
||||
// for now return a worst case
|
||||
var sum uint64 |
||||
for _, searcher := range s.searchers { |
||||
sum += searcher.Count() |
||||
} |
||||
return sum |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) Close() (rv error) { |
||||
for _, searcher := range s.searchers { |
||||
err := searcher.Close() |
||||
if err != nil && rv == nil { |
||||
rv = err |
||||
} |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) Min() int { |
||||
return s.min |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) DocumentMatchPoolSize() int { |
||||
rv := len(s.searchers) |
||||
for _, s := range s.searchers { |
||||
rv += s.DocumentMatchPoolSize() |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
// a disjunction searcher implements the index.Optimizable interface
|
||||
// but only activates on an edge case where the disjunction is a
|
||||
// wrapper around a single Optimizable child searcher
|
||||
func (s *DisjunctionHeapSearcher) Optimize(kind string, octx index.OptimizableContext) ( |
||||
index.OptimizableContext, error) { |
||||
if len(s.searchers) == 1 { |
||||
o, ok := s.searchers[0].(index.Optimizable) |
||||
if ok { |
||||
return o.Optimize(kind, octx) |
||||
} |
||||
} |
||||
|
||||
return octx, nil |
||||
} |
||||
|
||||
// heap impl
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Len() int { return len(s.heap) } |
||||
|
||||
func (s *DisjunctionHeapSearcher) Less(i, j int) bool { |
||||
if s.heap[i].curr == nil { |
||||
return true |
||||
} else if s.heap[j].curr == nil { |
||||
return false |
||||
} |
||||
return bytes.Compare(s.heap[i].curr.IndexInternalID, s.heap[j].curr.IndexInternalID) < 0 |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) Swap(i, j int) { |
||||
s.heap[i], s.heap[j] = s.heap[j], s.heap[i] |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) Push(x interface{}) { |
||||
s.heap = append(s.heap, x.(*SearcherCurr)) |
||||
} |
||||
|
||||
func (s *DisjunctionHeapSearcher) Pop() interface{} { |
||||
old := s.heap |
||||
n := len(old) |
||||
x := old[n-1] |
||||
s.heap = old[0 : n-1] |
||||
return x |
||||
} |
||||
298
vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go
generated
vendored
298
vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go
generated
vendored
@ -0,0 +1,298 @@ |
||||
// Copyright (c) 2018 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher |
||||
|
||||
import ( |
||||
"math" |
||||
"reflect" |
||||
"sort" |
||||
|
||||
"github.com/blevesearch/bleve/index" |
||||
"github.com/blevesearch/bleve/search" |
||||
"github.com/blevesearch/bleve/search/scorer" |
||||
"github.com/blevesearch/bleve/size" |
||||
) |
||||
|
||||
var reflectStaticSizeDisjunctionSliceSearcher int |
||||
|
||||
func init() { |
||||
var ds DisjunctionSliceSearcher |
||||
reflectStaticSizeDisjunctionSliceSearcher = int(reflect.TypeOf(ds).Size()) |
||||
} |
||||
|
||||
type DisjunctionSliceSearcher struct { |
||||
indexReader index.IndexReader |
||||
searchers OrderedSearcherList |
||||
numSearchers int |
||||
queryNorm float64 |
||||
currs []*search.DocumentMatch |
||||
scorer *scorer.DisjunctionQueryScorer |
||||
min int |
||||
matching []*search.DocumentMatch |
||||
matchingIdxs []int |
||||
initialized bool |
||||
} |
||||
|
||||
func newDisjunctionSliceSearcher(indexReader index.IndexReader, |
||||
qsearchers []search.Searcher, min float64, options search.SearcherOptions, |
||||
limit bool) ( |
||||
*DisjunctionSliceSearcher, error) { |
||||
if limit && tooManyClauses(len(qsearchers)) { |
||||
return nil, tooManyClausesErr(len(qsearchers)) |
||||
} |
||||
// build the downstream searchers
|
||||
searchers := make(OrderedSearcherList, len(qsearchers)) |
||||
for i, searcher := range qsearchers { |
||||
searchers[i] = searcher |
||||
} |
||||
// sort the searchers
|
||||
sort.Sort(sort.Reverse(searchers)) |
||||
// build our searcher
|
||||
rv := DisjunctionSliceSearcher{ |
||||
indexReader: indexReader, |
||||
searchers: searchers, |
||||
numSearchers: len(searchers), |
||||
currs: make([]*search.DocumentMatch, len(searchers)), |
||||
scorer: scorer.NewDisjunctionQueryScorer(options), |
||||
min: int(min), |
||||
matching: make([]*search.DocumentMatch, len(searchers)), |
||||
matchingIdxs: make([]int, len(searchers)), |
||||
} |
||||
rv.computeQueryNorm() |
||||
return &rv, nil |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) Size() int { |
||||
sizeInBytes := reflectStaticSizeDisjunctionSliceSearcher + size.SizeOfPtr + |
||||
s.scorer.Size() |
||||
|
||||
for _, entry := range s.searchers { |
||||
sizeInBytes += entry.Size() |
||||
} |
||||
|
||||
for _, entry := range s.currs { |
||||
if entry != nil { |
||||
sizeInBytes += entry.Size() |
||||
} |
||||
} |
||||
|
||||
for _, entry := range s.matching { |
||||
if entry != nil { |
||||
sizeInBytes += entry.Size() |
||||
} |
||||
} |
||||
|
||||
sizeInBytes += len(s.matchingIdxs) * size.SizeOfInt |
||||
|
||||
return sizeInBytes |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) computeQueryNorm() { |
||||
// first calculate sum of squared weights
|
||||
sumOfSquaredWeights := 0.0 |
||||
for _, searcher := range s.searchers { |
||||
sumOfSquaredWeights += searcher.Weight() |
||||
} |
||||
// now compute query norm from this
|
||||
s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) |
||||
// finally tell all the downstream searchers the norm
|
||||
for _, searcher := range s.searchers { |
||||
searcher.SetQueryNorm(s.queryNorm) |
||||
} |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) initSearchers(ctx *search.SearchContext) error { |
||||
var err error |
||||
// get all searchers pointing at their first match
|
||||
for i, searcher := range s.searchers { |
||||
if s.currs[i] != nil { |
||||
ctx.DocumentMatchPool.Put(s.currs[i]) |
||||
} |
||||
s.currs[i], err = searcher.Next(ctx) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
err = s.updateMatches() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
s.initialized = true |
||||
return nil |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) updateMatches() error { |
||||
matching := s.matching[:0] |
||||
matchingIdxs := s.matchingIdxs[:0] |
||||
|
||||
for i := 0; i < len(s.currs); i++ { |
||||
curr := s.currs[i] |
||||
if curr == nil { |
||||
continue |
||||
} |
||||
|
||||
if len(matching) > 0 { |
||||
cmp := curr.IndexInternalID.Compare(matching[0].IndexInternalID) |
||||
if cmp > 0 { |
||||
continue |
||||
} |
||||
|
||||
if cmp < 0 { |
||||
matching = matching[:0] |
||||
matchingIdxs = matchingIdxs[:0] |
||||
} |
||||
} |
||||
|
||||
matching = append(matching, curr) |
||||
matchingIdxs = append(matchingIdxs, i) |
||||
} |
||||
|
||||
s.matching = matching |
||||
s.matchingIdxs = matchingIdxs |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) Weight() float64 { |
||||
var rv float64 |
||||
for _, searcher := range s.searchers { |
||||
rv += searcher.Weight() |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) SetQueryNorm(qnorm float64) { |
||||
for _, searcher := range s.searchers { |
||||
searcher.SetQueryNorm(qnorm) |
||||
} |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) Next(ctx *search.SearchContext) ( |
||||
*search.DocumentMatch, error) { |
||||
if !s.initialized { |
||||
err := s.initSearchers(ctx) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
var err error |
||||
var rv *search.DocumentMatch |
||||
|
||||
found := false |
||||
for !found && len(s.matching) > 0 { |
||||
if len(s.matching) >= s.min { |
||||
found = true |
||||
// score this match
|
||||
rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) |
||||
} |
||||
|
||||
// invoke next on all the matching searchers
|
||||
for _, i := range s.matchingIdxs { |
||||
searcher := s.searchers[i] |
||||
if s.currs[i] != rv { |
||||
ctx.DocumentMatchPool.Put(s.currs[i]) |
||||
} |
||||
s.currs[i], err = searcher.Next(ctx) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
err = s.updateMatches() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
return rv, nil |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) Advance(ctx *search.SearchContext, |
||||
ID index.IndexInternalID) (*search.DocumentMatch, error) { |
||||
if !s.initialized { |
||||
err := s.initSearchers(ctx) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
// get all searchers pointing at their first match
|
||||
var err error |
||||
for i, searcher := range s.searchers { |
||||
if s.currs[i] != nil { |
||||
if s.currs[i].IndexInternalID.Compare(ID) >= 0 { |
||||
continue |
||||
} |
||||
ctx.DocumentMatchPool.Put(s.currs[i]) |
||||
} |
||||
s.currs[i], err = searcher.Advance(ctx, ID) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
err = s.updateMatches() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return s.Next(ctx) |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) Count() uint64 { |
||||
// for now return a worst case
|
||||
var sum uint64 |
||||
for _, searcher := range s.searchers { |
||||
sum += searcher.Count() |
||||
} |
||||
return sum |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) Close() (rv error) { |
||||
for _, searcher := range s.searchers { |
||||
err := searcher.Close() |
||||
if err != nil && rv == nil { |
||||
rv = err |
||||
} |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) Min() int { |
||||
return s.min |
||||
} |
||||
|
||||
func (s *DisjunctionSliceSearcher) DocumentMatchPoolSize() int { |
||||
rv := len(s.currs) |
||||
for _, s := range s.searchers { |
||||
rv += s.DocumentMatchPoolSize() |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
// a disjunction searcher implements the index.Optimizable interface
|
||||
// but only activates on an edge case where the disjunction is a
|
||||
// wrapper around a single Optimizable child searcher
|
||||
func (s *DisjunctionSliceSearcher) Optimize(kind string, octx index.OptimizableContext) ( |
||||
index.OptimizableContext, error) { |
||||
if len(s.searchers) == 1 { |
||||
o, ok := s.searchers[0].(index.Optimizable) |
||||
if ok { |
||||
return o.Optimize(kind, octx) |
||||
} |
||||
} |
||||
|
||||
return octx, nil |
||||
} |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue