diff --git a/go.mod b/go.mod index 406e22af5a..a808a41428 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/MicahParks/keyfunc v1.9.0 github.com/Nerzal/gocloak/v13 v13.8.0 github.com/bbalet/stopwords v1.0.0 - github.com/blevesearch/bleve/v2 v2.3.9 + github.com/blevesearch/bleve/v2 v2.3.10 github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.6.0 github.com/cs3org/go-cs3apis v0.0.0-20230516150832-730ac860c71d @@ -131,21 +131,21 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/bits-and-blooms/bitset v1.2.1 // indirect - github.com/blevesearch/bleve_index_api v1.0.5 // indirect - github.com/blevesearch/geo v0.1.17 // indirect + github.com/blevesearch/bleve_index_api v1.0.6 // indirect + github.com/blevesearch/geo v0.1.18 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.1.5 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.1.6 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect github.com/blevesearch/vellum v1.0.10 // indirect - github.com/blevesearch/zapx/v11 v11.3.9 // indirect - github.com/blevesearch/zapx/v12 v12.3.9 // indirect - github.com/blevesearch/zapx/v13 v13.3.9 // indirect - github.com/blevesearch/zapx/v14 v14.3.9 // indirect - github.com/blevesearch/zapx/v15 v15.3.12 // indirect + github.com/blevesearch/zapx/v11 v11.3.10 // indirect + github.com/blevesearch/zapx/v12 v12.3.10 // indirect + github.com/blevesearch/zapx/v13 v13.3.10 // indirect + github.com/blevesearch/zapx/v14 v14.3.10 // indirect + github.com/blevesearch/zapx/v15 v15.3.13 // indirect github.com/bluele/gcache v0.0.2 // indirect github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f // indirect github.com/bombsimon/logrusr/v3 v3.1.0 // indirect diff --git a/go.sum b/go.sum index 33532871da..e31d90e1e1 100644 --- a/go.sum +++ b/go.sum @@ -894,20 +894,20 @@ github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY github.com/bits-and-blooms/bitset v1.2.1 h1:M+/hrU9xlMp7t4TyTDQW97d3tRPVuKFC6zBEK16QnXY= github.com/bits-and-blooms/bitset v1.2.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blevesearch/bleve/v2 v2.3.9 h1:pUMvK0mxAexqasZcVj8lazmWnEW5XiV0tASIqANiNTQ= -github.com/blevesearch/bleve/v2 v2.3.9/go.mod h1:1PibElcjlQMQHF9uS9mRv58ODQgj4pCWHA1Wfd+qagU= -github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw= -github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= -github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA= -github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= +github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg= +github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA= +github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8= +github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= +github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= +github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.1.5 h1:1g713kpCQZ8u4a3stRGBfrwVOuGRnmxOVU5MQkUPrHU= -github.com/blevesearch/scorch_segment_api/v2 v2.1.5/go.mod h1:f2nOkKS1HcjgIWZgDAErgBdxmr2eyt0Kn7IY+FU1Xe4= +github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA= +github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= @@ -916,16 +916,16 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= -github.com/blevesearch/zapx/v11 v11.3.9 h1:y3ijS4h4MJdmQ07MHASxat4owAixreK2xdo76w9ncrw= -github.com/blevesearch/zapx/v11 v11.3.9/go.mod h1:jcAYnQwlr+LqD2vLjDWjWiZDXDXGFqPbpPDRTd3XmS4= -github.com/blevesearch/zapx/v12 v12.3.9 h1:MXGLlZ03oxXH3DMJTZaBaRj2xb6t4wQVZeZK/wu1M6w= -github.com/blevesearch/zapx/v12 v12.3.9/go.mod h1:QXCMwmOkdLnMDgTN1P4CcuX5F851iUOtOwXbw0HMBYs= -github.com/blevesearch/zapx/v13 v13.3.9 h1:+VAz9V0VmllHXlZV4DCvfYj0nqaZHgF3MeEHwOyRBwQ= -github.com/blevesearch/zapx/v13 v13.3.9/go.mod h1:s+WjNp4WSDtrBVBpa37DUOd7S/Gr/jTZ7ST/MbCVj/0= -github.com/blevesearch/zapx/v14 v14.3.9 h1:wuqxATgsTCNHM9xsOFOeFp8H2heZ/gMX/tsl9lRK8U4= -github.com/blevesearch/zapx/v14 v14.3.9/go.mod h1:MWZ4v8AzFBRurhDzkLvokFW8ljcq9Evm27mkWe8OGbM= -github.com/blevesearch/zapx/v15 v15.3.12 h1:w/kU9aHyfMDEdwHGZzCiakC3HZ9z5gYlXaALDC4Dct8= -github.com/blevesearch/zapx/v15 v15.3.12/go.mod h1:tx53gDJS/7Oa3Je820cmVurqCuJ4dqdAy1kiDMV/IUo= +github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= +github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= +github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= +github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= +github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= +github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= +github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= +github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= +github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= +github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/flexible/flexible.go b/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/flexible/flexible.go index 0eba074cd7..cb5f234d58 100644 --- a/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/flexible/flexible.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/flexible/flexible.go @@ -34,14 +34,14 @@ func New(layouts []string) *DateTimeParser { } } -func (p *DateTimeParser) ParseDateTime(input string) (time.Time, error) { +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) { for _, layout := range p.layouts { rv, err := time.Parse(layout, input) if err == nil { - return rv, nil + return rv, layout, nil } } - return time.Time{}, analysis.ErrInvalidDateTime + return time.Time{}, "", analysis.ErrInvalidDateTime } func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/optional/optional.go b/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/optional/optional.go index 196aa25cba..6dc7bfbcc6 100644 --- a/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/optional/optional.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/datetime/optional/optional.go @@ -26,6 +26,7 @@ const Name = "dateTimeOptional" const rfc3339NoTimezone = "2006-01-02T15:04:05" const rfc3339NoTimezoneNoT = "2006-01-02 15:04:05" +const rfc3339Offset = "2006-01-02 15:04:05 -0700" const rfc3339NoTime = "2006-01-02" var layouts = []string{ @@ -33,6 +34,7 @@ var layouts = []string{ time.RFC3339, rfc3339NoTimezone, rfc3339NoTimezoneNoT, + rfc3339Offset, rfc3339NoTime, } diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/type.go b/vendor/github.com/blevesearch/bleve/v2/analysis/type.go index 9e7bfa16d2..e3a7f201b1 100644 --- a/vendor/github.com/blevesearch/bleve/v2/analysis/type.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/type.go @@ -99,8 +99,11 @@ func (a *DefaultAnalyzer) Analyze(input []byte) TokenStream { var ErrInvalidDateTime = fmt.Errorf("unable to parse datetime with any of the layouts") +var ErrInvalidTimestampString = fmt.Errorf("unable to parse timestamp string") +var ErrInvalidTimestampRange = fmt.Errorf("timestamp out of range") + type DateTimeParser interface { - ParseDateTime(string) (time.Time, error) + ParseDateTime(string) (time.Time, string, error) } type ByteArrayConverter interface { diff --git a/vendor/github.com/blevesearch/bleve/v2/builder.go b/vendor/github.com/blevesearch/bleve/v2/builder.go index dbb7e3ed44..30285a2e44 100644 --- a/vendor/github.com/blevesearch/bleve/v2/builder.go +++ b/vendor/github.com/blevesearch/bleve/v2/builder.go @@ -15,12 +15,12 @@ package bleve import ( - "encoding/json" "fmt" "github.com/blevesearch/bleve/v2/document" "github.com/blevesearch/bleve/v2/index/scorch" "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -63,7 +63,7 @@ func newBuilder(path string, mapping mapping.IndexMapping, config map[string]int // the builder does not have an API to interact with internal storage // however we can pass k/v pairs through the config - mappingBytes, err := json.Marshal(mapping) + mappingBytes, err := util.MarshalJSON(mapping) if err != nil { return nil, err } diff --git a/vendor/github.com/blevesearch/bleve/v2/config.go b/vendor/github.com/blevesearch/bleve/v2/config.go index 0622b359db..d1490bdccb 100644 --- a/vendor/github.com/blevesearch/bleve/v2/config.go +++ b/vendor/github.com/blevesearch/bleve/v2/config.go @@ -16,7 +16,7 @@ package bleve import ( "expvar" - "io/ioutil" + "io" "log" "time" @@ -86,10 +86,10 @@ func init() { initDisk() } -var logger = log.New(ioutil.Discard, "bleve", log.LstdFlags) +var logger = log.New(io.Discard, "bleve", log.LstdFlags) // SetLog sets the logger used for logging -// by default log messages are sent to ioutil.Discard +// by default log messages are sent to io.Discard func SetLog(l *log.Logger) { logger = l } diff --git a/vendor/github.com/blevesearch/bleve/v2/document/field_datetime.go b/vendor/github.com/blevesearch/bleve/v2/document/field_datetime.go index 6506405501..efdd26b604 100644 --- a/vendor/github.com/blevesearch/bleve/v2/document/field_datetime.go +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_datetime.go @@ -15,6 +15,7 @@ package document import ( + "bytes" "fmt" "math" "reflect" @@ -26,6 +27,8 @@ import ( index "github.com/blevesearch/bleve_index_api" ) +var dateTimeValueSeperator = []byte{'\xff'} + var reflectStaticSizeDateTimeField int func init() { @@ -79,17 +82,28 @@ func (n *DateTimeField) AnalyzedTokenFrequencies() index.TokenFrequencies { return n.frequencies } +// split the value into the prefix coded date and the layout +// using the dateTimeValueSeperator as the split point +func (n *DateTimeField) splitValue() (numeric.PrefixCoded, string) { + parts := bytes.SplitN(n.value, dateTimeValueSeperator, 2) + if len(parts) == 1 { + return numeric.PrefixCoded(parts[0]), "" + } + return numeric.PrefixCoded(parts[0]), string(parts[1]) +} + func (n *DateTimeField) Analyze() { + valueWithoutLayout, _ := n.splitValue() tokens := make(analysis.TokenStream, 0) tokens = append(tokens, &analysis.Token{ Start: 0, - End: len(n.value), - Term: n.value, + End: len(valueWithoutLayout), + Term: valueWithoutLayout, Position: 1, Type: analysis.DateTime, }) - original, err := n.value.Int64() + original, err := valueWithoutLayout.Int64() if err == nil { shift := DefaultDateTimePrecisionStep @@ -118,12 +132,13 @@ func (n *DateTimeField) Value() []byte { return n.value } -func (n *DateTimeField) DateTime() (time.Time, error) { - i64, err := n.value.Int64() +func (n *DateTimeField) DateTime() (time.Time, string, error) { + date, layout := n.splitValue() + i64, err := date.Int64() if err != nil { - return time.Time{}, err + return time.Time{}, "", err } - return time.Unix(0, i64).UTC(), nil + return time.Unix(0, i64).UTC(), layout, nil } func (n *DateTimeField) GoString() string { @@ -144,18 +159,26 @@ func NewDateTimeFieldFromBytes(name string, arrayPositions []uint64, value []byt } } -func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time) (*DateTimeField, error) { - return NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, DefaultDateTimeIndexingOptions) +func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time, layout string) (*DateTimeField, error) { + return NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, layout, DefaultDateTimeIndexingOptions) } -func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, options index.FieldIndexingOptions) (*DateTimeField, error) { +func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, layout string, options index.FieldIndexingOptions) (*DateTimeField, error) { if canRepresent(dt) { dtInt64 := dt.UnixNano() prefixCoded := numeric.MustNewPrefixCodedInt64(dtInt64, 0) + // The prefixCoded value is combined with the layout. + // This is necessary because the storage layer stores a fields value as a byte slice + // without storing extra information like layout. So by making value = prefixCoded + layout, + // both pieces of information are stored in the byte slice. + // During a query, the layout is extracted from the byte slice stored to correctly + // format the prefixCoded value. + valueWithLayout := append(prefixCoded, dateTimeValueSeperator...) + valueWithLayout = append(valueWithLayout, []byte(layout)...) return &DateTimeField{ name: name, arrayPositions: arrayPositions, - value: prefixCoded, + value: valueWithLayout, options: options, // not correct, just a place holder until we revisit how fields are // represented and can fix this better diff --git a/vendor/github.com/blevesearch/bleve/v2/geo/geo_s2plugin_impl.go b/vendor/github.com/blevesearch/bleve/v2/geo/geo_s2plugin_impl.go index f743d87fc7..8a034bd9b0 100644 --- a/vendor/github.com/blevesearch/bleve/v2/geo/geo_s2plugin_impl.go +++ b/vendor/github.com/blevesearch/bleve/v2/geo/geo_s2plugin_impl.go @@ -18,8 +18,8 @@ import ( "encoding/json" "sync" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" - "github.com/blevesearch/geo/geojson" "github.com/blevesearch/geo/s2" ) @@ -224,7 +224,7 @@ func (p *Point) Type() string { } func (p *Point) Value() ([]byte, error) { - return json.Marshal(p) + return util.MarshalJSON(p) } func (p *Point) Intersects(s index.GeoJSON) (bool, error) { @@ -267,7 +267,7 @@ func (br *boundedRectangle) Type() string { } func (br *boundedRectangle) Value() ([]byte, error) { - return json.Marshal(br) + return util.MarshalJSON(br) } func (p *boundedRectangle) Intersects(s index.GeoJSON) (bool, error) { @@ -309,7 +309,7 @@ func (bp *boundedPolygon) Type() string { } func (bp *boundedPolygon) Value() ([]byte, error) { - return json.Marshal(bp) + return util.MarshalJSON(bp) } func (p *boundedPolygon) Intersects(s index.GeoJSON) (bool, error) { @@ -355,7 +355,7 @@ func (p *pointDistance) Type() string { } func (p *pointDistance) Value() ([]byte, error) { - return json.Marshal(p) + return util.MarshalJSON(p) } func NewPointDistance(centerLat, centerLon, diff --git a/vendor/github.com/blevesearch/bleve/v2/geo/parse.go b/vendor/github.com/blevesearch/bleve/v2/geo/parse.go index cc7beddc88..01ec1dd81f 100644 --- a/vendor/github.com/blevesearch/bleve/v2/geo/parse.go +++ b/vendor/github.com/blevesearch/bleve/v2/geo/parse.go @@ -24,15 +24,24 @@ import ( // interpret it is as geo point. Supported formats: // Container: // slice length 2 (GeoJSON) -// first element lon, second element lat +// +// first element lon, second element lat +// // string (coordinates separated by comma, or a geohash) -// first element lat, second element lon +// +// first element lat, second element lon +// // map[string]interface{} -// exact keys lat and lon or lng +// +// exact keys lat and lon or lng +// // struct -// w/exported fields case-insensitive match on lat and lon or lng +// +// w/exported fields case-insensitive match on lat and lon or lng +// // struct -// satisfying Later and Loner or Lnger interfaces +// +// satisfying Later and Loner or Lnger interfaces // // in all cases values must be some sort of numeric-like thing: int/uint/float func ExtractGeoPoint(thing interface{}) (lon, lat float64, success bool) { diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go index c5f0e79e24..92adc3fd4b 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go @@ -16,7 +16,6 @@ package scorch import ( "context" - "encoding/json" "fmt" "os" "strings" @@ -25,6 +24,7 @@ import ( "github.com/RoaringBitmap/roaring" "github.com/blevesearch/bleve/v2/index/scorch/mergeplan" + "github.com/blevesearch/bleve/v2/util" segment "github.com/blevesearch/scorch_segment_api/v2" ) @@ -198,12 +198,12 @@ func (s *Scorch) parseMergePlannerOptions() (*mergeplan.MergePlanOptions, error) { mergePlannerOptions := mergeplan.DefaultMergePlanOptions if v, ok := s.config["scorchMergePlanOptions"]; ok { - b, err := json.Marshal(v) + b, err := util.MarshalJSON(v) if err != nil { return &mergePlannerOptions, err } - err = json.Unmarshal(b, &mergePlannerOptions) + err = util.UnmarshalJSON(b, &mergePlannerOptions) if err != nil { return &mergePlannerOptions, err } diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go index b90899f257..217582fe1b 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go @@ -17,7 +17,6 @@ package scorch import ( "bytes" "encoding/binary" - "encoding/json" "fmt" "io" "log" @@ -31,6 +30,7 @@ import ( "time" "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" segment "github.com/blevesearch/scorch_segment_api/v2" bolt "go.etcd.io/bbolt" @@ -324,12 +324,12 @@ func (s *Scorch) parsePersisterOptions() (*persisterOptions, error) { MemoryPressurePauseThreshold: DefaultMemoryPressurePauseThreshold, } if v, ok := s.config["scorchPersisterOptions"]; ok { - b, err := json.Marshal(v) + b, err := util.MarshalJSON(v) if err != nil { return &po, err } - err = json.Unmarshal(b, &po) + err = util.UnmarshalJSON(b, &po) if err != nil { return &po, err } diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/stats.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/stats.go index d265e9d040..dc74d9f29e 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/scorch/stats.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/stats.go @@ -15,9 +15,10 @@ package scorch import ( - "encoding/json" "reflect" "sync/atomic" + + "github.com/blevesearch/bleve/v2/util" ) // Stats tracks statistics about the index, fields that are @@ -151,5 +152,5 @@ func (s *Stats) ToMap() map[string]interface{} { // MarshalJSON implements json.Marshaler, and in contrast to standard // json marshaling provides atomic safety func (s *Stats) MarshalJSON() ([]byte, error) { - return json.Marshal(s.ToMap()) + return util.MarshalJSON(s.ToMap()) } diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go index 901a4bcb06..fff6d06737 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/row.go @@ -880,6 +880,9 @@ func NewStoredRowK(key []byte) (*StoredRow, error) { } rv.doc, err = buf.ReadBytes(ByteSeparator) + if err != nil { + return nil, err + } if len(rv.doc) < 2 { // 1 for min doc id length, 1 for separator err = fmt.Errorf("invalid doc length 0") return nil, err diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/stats.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/stats.go index c1b4ddc018..5fea7c1724 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/stats.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/stats.go @@ -15,9 +15,9 @@ package upsidedown import ( - "encoding/json" "sync/atomic" + "github.com/blevesearch/bleve/v2/util" "github.com/blevesearch/upsidedown_store_api" ) @@ -51,5 +51,5 @@ func (i *indexStat) statsMap() map[string]interface{} { func (i *indexStat) MarshalJSON() ([]byte, error) { m := i.statsMap() - return json.Marshal(m) + return util.MarshalJSON(m) } diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/stats.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/stats.go index e50e552745..d8961832e5 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/stats.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb/stats.go @@ -14,7 +14,9 @@ package boltdb -import "encoding/json" +import ( + "github.com/blevesearch/bleve/v2/util" +) type stats struct { s *Store @@ -22,5 +24,5 @@ type stats struct { func (s *stats) MarshalJSON() ([]byte, error) { bs := s.s.db.Stats() - return json.Marshal(bs) + return util.MarshalJSON(bs) } diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go index 6e14e65767..3756422da8 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go @@ -135,22 +135,22 @@ func (udc *UpsideDownCouch) loadSchema(kvreader store.KVReader) (err error) { } type rowBuffer struct { - buf []byte + buf []byte } var rowBufferPool sync.Pool func GetRowBuffer() *rowBuffer { - if rb, ok := rowBufferPool.Get().(*rowBuffer); ok { - return rb - } else { - buf := make([]byte, RowBufferSize) - return &rowBuffer{buf: buf} - } + if rb, ok := rowBufferPool.Get().(*rowBuffer); ok { + return rb + } else { + buf := make([]byte, RowBufferSize) + return &rowBuffer{buf: buf} + } } func PutRowBuffer(rb *rowBuffer) { - rowBufferPool.Put(rb) + rowBufferPool.Put(rb) } func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRowsAll [][]UpsideDownCouchRow, updateRowsAll [][]UpsideDownCouchRow, deleteRowsAll [][]UpsideDownCouchRow) (err error) { diff --git a/vendor/github.com/blevesearch/bleve/v2/index_impl.go b/vendor/github.com/blevesearch/bleve/v2/index_impl.go index b5f115411e..d5f34a2a3d 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index_impl.go +++ b/vendor/github.com/blevesearch/bleve/v2/index_impl.go @@ -16,11 +16,11 @@ package bleve import ( "context" - "encoding/json" "fmt" "io" "os" "path/filepath" + "strconv" "sync" "sync/atomic" "time" @@ -34,7 +34,9 @@ import ( "github.com/blevesearch/bleve/v2/search/collector" "github.com/blevesearch/bleve/v2/search/facet" "github.com/blevesearch/bleve/v2/search/highlight" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" + "github.com/blevesearch/geo/s2" ) type indexImpl struct { @@ -118,7 +120,7 @@ func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, }(&rv) // now persist the mapping - mappingBytes, err := json.Marshal(mapping) + mappingBytes, err := util.MarshalJSON(mapping) if err != nil { return nil, err } @@ -201,7 +203,7 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde } var im *mapping.IndexMappingImpl - err = json.Unmarshal(mappingBytes, &im) + err = util.UnmarshalJSON(mappingBytes, &im) if err != nil { return nil, fmt.Errorf("error parsing mapping JSON: %v\nmapping contents:\n%s", err, string(mappingBytes)) } @@ -482,6 +484,18 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr ctx = context.WithValue(ctx, search.SearchIOStatsCallbackKey, search.SearchIOStatsCallbackFunc(sendBytesRead)) + var bufPool *s2.GeoBufferPool + getBufferPool := func() *s2.GeoBufferPool { + if bufPool == nil { + bufPool = s2.NewGeoBufferPool(search.MaxGeoBufPoolSize, search.MinGeoBufPoolSize) + } + + return bufPool + } + + ctx = context.WithValue(ctx, search.GeoBufferPoolCallbackKey, + search.GeoBufferPoolCallbackFunc(getBufferPool)) + searcher, err := req.Query.Searcher(ctx, indexReader, i.m, search.SearcherOptions{ Explain: req.Explain, IncludeTermVectors: req.IncludeLocations || req.Highlight != nil, @@ -517,10 +531,23 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr } else if facetRequest.DateTimeRanges != nil { // build date range facet facetBuilder := facet.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size) - dateTimeParser := i.m.DateTimeParserNamed("") for _, dr := range facetRequest.DateTimeRanges { - start, end := dr.ParseDates(dateTimeParser) - facetBuilder.AddRange(dr.Name, start, end) + dateTimeParserName := defaultDateTimeParser + if dr.DateTimeParser != "" { + dateTimeParserName = dr.DateTimeParser + } + dateTimeParser := i.m.DateTimeParserNamed(dateTimeParserName) + if dateTimeParser == nil { + return nil, fmt.Errorf("no date time parser named `%s` registered", dateTimeParserName) + } + start, end, startLayout, endLayout, err := dr.ParseDates(dateTimeParser) + if err != nil { + return nil, fmt.Errorf("ParseDates err: %v, using date time parser named %s", err, dateTimeParserName) + } + if start.IsZero() && end.IsZero() { + return nil, fmt.Errorf("date range query must specify either start, end or both for date range name '%s'", dr.Name) + } + facetBuilder.AddRange(dr.Name, start, end, startLayout, endLayout) } facetsBuilder.Add(facetName, facetBuilder) } else { @@ -648,9 +675,14 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, value = num } case index.DateTimeField: - datetime, err := docF.DateTime() + datetime, layout, err := docF.DateTime() if err == nil { - value = datetime.Format(time.RFC3339) + if layout == "" { + // layout not set probably means it was indexed as a timestamp + value = strconv.FormatInt(datetime.UnixNano(), 10) + } else { + value = datetime.Format(layout) + } } case index.BooleanField: boolean, err := docF.Boolean() diff --git a/vendor/github.com/blevesearch/bleve/v2/index_meta.go b/vendor/github.com/blevesearch/bleve/v2/index_meta.go index 90c7fffac8..14b88dcbc0 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index_meta.go +++ b/vendor/github.com/blevesearch/bleve/v2/index_meta.go @@ -15,12 +15,12 @@ package bleve import ( - "encoding/json" "fmt" "os" "path/filepath" "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -50,7 +50,7 @@ func openIndexMeta(path string) (*indexMeta, error) { return nil, ErrorIndexMetaMissing } var im indexMeta - err = json.Unmarshal(metaBytes, &im) + err = util.UnmarshalJSON(metaBytes, &im) if err != nil { return nil, ErrorIndexMetaCorrupt } @@ -70,7 +70,7 @@ func (i *indexMeta) Save(path string) (err error) { } return err } - metaBytes, err := json.Marshal(i) + metaBytes, err := util.MarshalJSON(i) if err != nil { return err } @@ -94,7 +94,7 @@ func (i *indexMeta) Save(path string) (err error) { } func (i *indexMeta) CopyTo(d index.Directory) (err error) { - metaBytes, err := json.Marshal(i) + metaBytes, err := util.MarshalJSON(i) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping/document.go b/vendor/github.com/blevesearch/bleve/v2/mapping/document.go index 7215e5b927..aacaa0a551 100644 --- a/vendor/github.com/blevesearch/bleve/v2/mapping/document.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/document.go @@ -23,6 +23,7 @@ import ( "time" "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" ) // A DocumentMapping describes how a type of document @@ -149,12 +150,11 @@ func (dm *DocumentMapping) documentMappingForPath(path string) ( current := dm OUTER: for i, pathElement := range pathElements { - for name, subDocMapping := range current.Properties { - if name == pathElement { - current = subDocMapping - continue OUTER - } + if subDocMapping, exists := current.Properties[pathElement]; exists { + current = subDocMapping + continue OUTER } + // no subDocMapping matches this pathElement // only if this is the last element check for field name if i == len(pathElements)-1 { @@ -239,7 +239,7 @@ func (dm *DocumentMapping) AddFieldMapping(fm *FieldMapping) { // UnmarshalJSON offers custom unmarshaling with optional strict validation func (dm *DocumentMapping) UnmarshalJSON(data []byte) error { var tmp map[string]json.RawMessage - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } @@ -252,32 +252,32 @@ func (dm *DocumentMapping) UnmarshalJSON(data []byte) error { for k, v := range tmp { switch k { case "enabled": - err := json.Unmarshal(v, &dm.Enabled) + err := util.UnmarshalJSON(v, &dm.Enabled) if err != nil { return err } case "dynamic": - err := json.Unmarshal(v, &dm.Dynamic) + err := util.UnmarshalJSON(v, &dm.Dynamic) if err != nil { return err } case "default_analyzer": - err := json.Unmarshal(v, &dm.DefaultAnalyzer) + err := util.UnmarshalJSON(v, &dm.DefaultAnalyzer) if err != nil { return err } case "properties": - err := json.Unmarshal(v, &dm.Properties) + err := util.UnmarshalJSON(v, &dm.Properties) if err != nil { return err } case "fields": - err := json.Unmarshal(v, &dm.Fields) + err := util.UnmarshalJSON(v, &dm.Fields) if err != nil { return err } case "struct_tag_key": - err := json.Unmarshal(v, &dm.StructTagKey) + err := util.UnmarshalJSON(v, &dm.StructTagKey) if err != nil { return err } @@ -423,7 +423,7 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, // first see if it can be parsed by the default date parser dateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser) if dateTimeParser != nil { - parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString) + parsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString) if err != nil { // index as text fieldMapping := newTextFieldMappingDynamic(context.im) @@ -431,7 +431,7 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, } else { // index as datetime fieldMapping := newDateTimeFieldMappingDynamic(context.im) - fieldMapping.processTime(parsedDateTime, pathString, path, indexes, context) + fieldMapping.processTime(parsedDateTime, layout, pathString, path, indexes, context) } } } @@ -472,11 +472,11 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, if subDocMapping != nil { // index by explicit mapping for _, fieldMapping := range subDocMapping.Fields { - fieldMapping.processTime(property, pathString, path, indexes, context) + fieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context) } } else if closestDocMapping.Dynamic { fieldMapping := newDateTimeFieldMappingDynamic(context.im) - fieldMapping.processTime(property, pathString, path, indexes, context) + fieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context) } case encoding.TextMarshaler: txt, err := property.MarshalText() diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping/field.go b/vendor/github.com/blevesearch/bleve/v2/mapping/field.go index 511782acc4..82d51f317f 100644 --- a/vendor/github.com/blevesearch/bleve/v2/mapping/field.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/field.go @@ -20,12 +20,12 @@ import ( "net" "time" - "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" - index "github.com/blevesearch/bleve_index_api" - "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/document" "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) // control the default behavior for dynamic fields (those not explicitly mapped) @@ -233,9 +233,9 @@ func (fm *FieldMapping) processString(propertyValueString string, pathString str } dateTimeParser := context.im.DateTimeParserNamed(dateTimeFormat) if dateTimeParser != nil { - parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString) + parsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString) if err == nil { - fm.processTime(parsedDateTime, pathString, path, indexes, context) + fm.processTime(parsedDateTime, layout, pathString, path, indexes, context) } } } else if fm.Type == "IP" { @@ -259,11 +259,11 @@ func (fm *FieldMapping) processFloat64(propertyValFloat float64, pathString stri } } -func (fm *FieldMapping) processTime(propertyValueTime time.Time, pathString string, path []string, indexes []uint64, context *walkContext) { +func (fm *FieldMapping) processTime(propertyValueTime time.Time, layout string, pathString string, path []string, indexes []uint64, context *walkContext) { fieldName := getFieldName(pathString, path, fm) if fm.Type == "datetime" { options := fm.Options() - field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, options) + field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, layout, options) if err == nil { context.doc.AddField(field) } else { @@ -390,7 +390,7 @@ func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) func (fm *FieldMapping) UnmarshalJSON(data []byte) error { var tmp map[string]json.RawMessage - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } @@ -399,52 +399,52 @@ func (fm *FieldMapping) UnmarshalJSON(data []byte) error { for k, v := range tmp { switch k { case "name": - err := json.Unmarshal(v, &fm.Name) + err := util.UnmarshalJSON(v, &fm.Name) if err != nil { return err } case "type": - err := json.Unmarshal(v, &fm.Type) + err := util.UnmarshalJSON(v, &fm.Type) if err != nil { return err } case "analyzer": - err := json.Unmarshal(v, &fm.Analyzer) + err := util.UnmarshalJSON(v, &fm.Analyzer) if err != nil { return err } case "store": - err := json.Unmarshal(v, &fm.Store) + err := util.UnmarshalJSON(v, &fm.Store) if err != nil { return err } case "index": - err := json.Unmarshal(v, &fm.Index) + err := util.UnmarshalJSON(v, &fm.Index) if err != nil { return err } case "include_term_vectors": - err := json.Unmarshal(v, &fm.IncludeTermVectors) + err := util.UnmarshalJSON(v, &fm.IncludeTermVectors) if err != nil { return err } case "include_in_all": - err := json.Unmarshal(v, &fm.IncludeInAll) + err := util.UnmarshalJSON(v, &fm.IncludeInAll) if err != nil { return err } case "date_format": - err := json.Unmarshal(v, &fm.DateFormat) + err := util.UnmarshalJSON(v, &fm.DateFormat) if err != nil { return err } case "docvalues": - err := json.Unmarshal(v, &fm.DocValues) + err := util.UnmarshalJSON(v, &fm.DocValues) if err != nil { return err } case "skip_freq_norm": - err := json.Unmarshal(v, &fm.SkipFreqNorm) + err := util.UnmarshalJSON(v, &fm.SkipFreqNorm) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping/index.go b/vendor/github.com/blevesearch/bleve/v2/mapping/index.go index e2ac99f399..0de4147a47 100644 --- a/vendor/github.com/blevesearch/bleve/v2/mapping/index.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/index.go @@ -17,13 +17,14 @@ package mapping import ( "encoding/json" "fmt" - index "github.com/blevesearch/bleve_index_api" "github.com/blevesearch/bleve/v2/analysis" "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" "github.com/blevesearch/bleve/v2/analysis/datetime/optional" "github.com/blevesearch/bleve/v2/document" "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) var MappingJSONStrict = false @@ -203,7 +204,7 @@ func (im *IndexMappingImpl) mappingForType(docType string) *DocumentMapping { func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error { var tmp map[string]json.RawMessage - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } @@ -226,57 +227,57 @@ func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error { for k, v := range tmp { switch k { case "analysis": - err := json.Unmarshal(v, &im.CustomAnalysis) + err := util.UnmarshalJSON(v, &im.CustomAnalysis) if err != nil { return err } case "type_field": - err := json.Unmarshal(v, &im.TypeField) + err := util.UnmarshalJSON(v, &im.TypeField) if err != nil { return err } case "default_type": - err := json.Unmarshal(v, &im.DefaultType) + err := util.UnmarshalJSON(v, &im.DefaultType) if err != nil { return err } case "default_analyzer": - err := json.Unmarshal(v, &im.DefaultAnalyzer) + err := util.UnmarshalJSON(v, &im.DefaultAnalyzer) if err != nil { return err } case "default_datetime_parser": - err := json.Unmarshal(v, &im.DefaultDateTimeParser) + err := util.UnmarshalJSON(v, &im.DefaultDateTimeParser) if err != nil { return err } case "default_field": - err := json.Unmarshal(v, &im.DefaultField) + err := util.UnmarshalJSON(v, &im.DefaultField) if err != nil { return err } case "default_mapping": - err := json.Unmarshal(v, &im.DefaultMapping) + err := util.UnmarshalJSON(v, &im.DefaultMapping) if err != nil { return err } case "types": - err := json.Unmarshal(v, &im.TypeMapping) + err := util.UnmarshalJSON(v, &im.TypeMapping) if err != nil { return err } case "store_dynamic": - err := json.Unmarshal(v, &im.StoreDynamic) + err := util.UnmarshalJSON(v, &im.StoreDynamic) if err != nil { return err } case "index_dynamic": - err := json.Unmarshal(v, &im.IndexDynamic) + err := util.UnmarshalJSON(v, &im.IndexDynamic) if err != nil { return err } case "docvalues_dynamic": - err := json.Unmarshal(v, &im.DocValuesDynamic) + err := util.UnmarshalJSON(v, &im.DocValuesDynamic) if err != nil { return err } @@ -417,23 +418,6 @@ func (im *IndexMappingImpl) DateTimeParserNamed(name string) analysis.DateTimePa return dateTimeParser } -func (im *IndexMappingImpl) datetimeParserNameForPath(path string) string { - - // first we look for explicit mapping on the field - for _, docMapping := range im.TypeMapping { - pathMapping, _ := docMapping.documentMappingForPath(path) - if pathMapping != nil { - if len(pathMapping.Fields) > 0 { - if pathMapping.Fields[0].Analyzer != "" { - return pathMapping.Fields[0].Analyzer - } - } - } - } - - return im.DefaultDateTimeParser -} - func (im *IndexMappingImpl) AnalyzeText(analyzerName string, text []byte) (analysis.TokenStream, error) { analyzer, err := im.cache.AnalyzerNamed(analyzerName) if err != nil { diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go b/vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go index 2506352e0f..a3e5a54e05 100644 --- a/vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/mapping.go @@ -15,7 +15,7 @@ package mapping import ( - "io/ioutil" + "io" "log" "github.com/blevesearch/bleve/v2/analysis" @@ -37,10 +37,10 @@ type bleveClassifier interface { BleveType() string } -var logger = log.New(ioutil.Discard, "bleve mapping ", log.LstdFlags) +var logger = log.New(io.Discard, "bleve mapping ", log.LstdFlags) // SetLog sets the logger used for logging -// by default log messages are sent to ioutil.Discard +// by default log messages are sent to io.Discard func SetLog(l *log.Logger) { logger = l } diff --git a/vendor/github.com/blevesearch/bleve/v2/query.go b/vendor/github.com/blevesearch/bleve/v2/query.go index 3385ed779d..3af750a066 100644 --- a/vendor/github.com/blevesearch/bleve/v2/query.go +++ b/vendor/github.com/blevesearch/bleve/v2/query.go @@ -48,7 +48,9 @@ func NewConjunctionQuery(conjuncts ...query.Query) *query.ConjunctionQuery { // NewDateRangeQuery creates a new Query for ranges // of date values. // Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser +// +// top-level config.QueryDateTimeParser +// // Either, but not both endpoints can be nil. func NewDateRangeQuery(start, end time.Time) *query.DateRangeQuery { return query.NewDateRangeQuery(start, end) @@ -57,13 +59,49 @@ func NewDateRangeQuery(start, end time.Time) *query.DateRangeQuery { // NewDateRangeInclusiveQuery creates a new Query for ranges // of date values. // Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser +// +// top-level config.QueryDateTimeParser +// // Either, but not both endpoints can be nil. // startInclusive and endInclusive control inclusion of the endpoints. func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *query.DateRangeQuery { return query.NewDateRangeInclusiveQuery(start, end, startInclusive, endInclusive) } +// NewDateRangeStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser set using +// +// the DateRangeStringQuery.SetDateTimeParser() method. +// +// If no DateTimeParser is set, then the +// +// top-level config.QueryDateTimeParser +// +// is used. +func NewDateRangeStringQuery(start, end string) *query.DateRangeStringQuery { + return query.NewDateRangeStringQuery(start, end) +} + +// NewDateRangeStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser set using +// +// the DateRangeStringQuery.SetDateTimeParser() method. +// +// this DateTimeParser is a custom date time parser defined in the index mapping, +// using AddCustomDateTimeParser() method. +// If no DateTimeParser is set, then the +// +// top-level config.QueryDateTimeParser +// +// is used. +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeInclusiveStringQuery(start, end string, startInclusive, endInclusive *bool) *query.DateRangeStringQuery { + return query.NewDateRangeStringInclusiveQuery(start, end, startInclusive, endInclusive) +} + // NewDisjunctionQuery creates a new compound Query. // Result documents satisfy at least one Query. func NewDisjunctionQuery(disjuncts ...query.Query) *query.DisjunctionQuery { diff --git a/vendor/github.com/blevesearch/bleve/v2/search.go b/vendor/github.com/blevesearch/bleve/v2/search.go index fe426164aa..8ca0310fbf 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search.go +++ b/vendor/github.com/blevesearch/bleve/v2/search.go @@ -29,63 +29,65 @@ import ( "github.com/blevesearch/bleve/v2/search/collector" "github.com/blevesearch/bleve/v2/search/query" "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" ) -var reflectStaticSizeSearchResult int -var reflectStaticSizeSearchStatus int - -func init() { - var sr SearchResult - reflectStaticSizeSearchResult = int(reflect.TypeOf(sr).Size()) - var ss SearchStatus - reflectStaticSizeSearchStatus = int(reflect.TypeOf(ss).Size()) -} - -var cache = registry.NewCache() - const defaultDateTimeParser = optional.Name -type numericRange struct { - Name string `json:"name,omitempty"` - Min *float64 `json:"min,omitempty"` - Max *float64 `json:"max,omitempty"` +var cache = registry.NewCache() + +var ( + reflectStaticSizeSearchResult int + reflectStaticSizeSearchStatus int +) + +func init() { + reflectStaticSizeSearchResult = int(reflect.TypeOf(SearchResult{}).Size()) + reflectStaticSizeSearchStatus = int(reflect.TypeOf(SearchStatus{}).Size()) } type dateTimeRange struct { - Name string `json:"name,omitempty"` - Start time.Time `json:"start,omitempty"` - End time.Time `json:"end,omitempty"` - startString *string - endString *string + Name string `json:"name,omitempty"` + Start time.Time `json:"start,omitempty"` + End time.Time `json:"end,omitempty"` + DateTimeParser string `json:"datetime_parser,omitempty"` + startString *string + endString *string } -func (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) (start, end time.Time) { +func (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) (start, end time.Time, startLayout, endLayout string, err error) { start = dr.Start + startLayout = time.RFC3339Nano if dr.Start.IsZero() && dr.startString != nil { - s, err := dateTimeParser.ParseDateTime(*dr.startString) - if err == nil { - start = s + s, layout, parseError := dateTimeParser.ParseDateTime(*dr.startString) + if parseError != nil { + return start, end, startLayout, endLayout, fmt.Errorf("error parsing start date '%s' for date range name '%s': %v", *dr.startString, dr.Name, parseError) } + start = s + startLayout = layout } end = dr.End + endLayout = time.RFC3339Nano if dr.End.IsZero() && dr.endString != nil { - e, err := dateTimeParser.ParseDateTime(*dr.endString) - if err == nil { - end = e + e, layout, parseError := dateTimeParser.ParseDateTime(*dr.endString) + if parseError != nil { + return start, end, startLayout, endLayout, fmt.Errorf("error parsing end date '%s' for date range name '%s': %v", *dr.endString, dr.Name, parseError) } + end = e + endLayout = layout } - return start, end + return start, end, startLayout, endLayout, err } func (dr *dateTimeRange) UnmarshalJSON(input []byte) error { var temp struct { - Name string `json:"name,omitempty"` - Start *string `json:"start,omitempty"` - End *string `json:"end,omitempty"` + Name string `json:"name,omitempty"` + Start *string `json:"start,omitempty"` + End *string `json:"end,omitempty"` + DateTimeParser string `json:"datetime_parser,omitempty"` } - err := json.Unmarshal(input, &temp) - if err != nil { + if err := util.UnmarshalJSON(input, &temp); err != nil { return err } @@ -96,23 +98,40 @@ func (dr *dateTimeRange) UnmarshalJSON(input []byte) error { if temp.End != nil { dr.endString = temp.End } + if temp.DateTimeParser != "" { + dr.DateTimeParser = temp.DateTimeParser + } return nil } func (dr *dateTimeRange) MarshalJSON() ([]byte, error) { rv := map[string]interface{}{ - "name": dr.Name, - "start": dr.Start, - "end": dr.End, + "name": dr.Name, } - if dr.Start.IsZero() && dr.startString != nil { + + if !dr.Start.IsZero() { + rv["start"] = dr.Start + } else if dr.startString != nil { rv["start"] = dr.startString } - if dr.End.IsZero() && dr.endString != nil { + + if !dr.End.IsZero() { + rv["end"] = dr.End + } else if dr.endString != nil { rv["end"] = dr.endString } - return json.Marshal(rv) + + if dr.DateTimeParser != "" { + rv["datetime_parser"] = dr.DateTimeParser + } + return util.MarshalJSON(rv) +} + +type numericRange struct { + Name string `json:"name,omitempty"` + Min *float64 `json:"min,omitempty"` + Max *float64 `json:"max,omitempty"` } // A FacetRequest describes a facet or aggregation @@ -125,11 +144,21 @@ type FacetRequest struct { DateTimeRanges []*dateTimeRange `json:"date_ranges,omitempty"` } +// NewFacetRequest creates a facet on the specified +// field that limits the number of entries to the +// specified size. +func NewFacetRequest(field string, size int) *FacetRequest { + return &FacetRequest{ + Field: field, + Size: size, + } +} + func (fr *FacetRequest) Validate() error { nrCount := len(fr.NumericRanges) drCount := len(fr.DateTimeRanges) if nrCount > 0 && drCount > 0 { - return fmt.Errorf("facet can only conain numeric ranges or date ranges, not both") + return fmt.Errorf("facet can only contain numeric ranges or date ranges, not both") } if nrCount > 0 { @@ -155,23 +184,21 @@ func (fr *FacetRequest) Validate() error { return fmt.Errorf("date ranges contains duplicate name '%s'", dr.Name) } drNames[dr.Name] = struct{}{} - start, end := dr.ParseDates(dateTimeParser) - if start.IsZero() && end.IsZero() { - return fmt.Errorf("date range query must specify either start, end or both for range name '%s'", dr.Name) + if dr.DateTimeParser == "" { + // cannot parse the date range dates as the defaultDateTimeParser is overridden + // so perform this validation at query time + start, end, _, _, err := dr.ParseDates(dateTimeParser) + if err != nil { + return fmt.Errorf("ParseDates err: %v, using date time parser named %s", err, defaultDateTimeParser) + } + if start.IsZero() && end.IsZero() { + return fmt.Errorf("date range query must specify either start, end or both for range name '%s'", dr.Name) + } } } } - return nil -} -// NewFacetRequest creates a facet on the specified -// field that limits the number of entries to the -// specified size. -func NewFacetRequest(field string, size int) *FacetRequest { - return &FacetRequest{ - Field: field, - Size: size, - } + return nil } // AddDateTimeRange adds a bucket to a field @@ -186,7 +213,7 @@ func (fr *FacetRequest) AddDateTimeRange(name string, start, end time.Time) { } // AddDateTimeRangeString adds a bucket to a field -// containing date values. +// containing date values. Uses defaultDateTimeParser to parse the date strings. func (fr *FacetRequest) AddDateTimeRangeString(name string, start, end *string) { if fr.DateTimeRanges == nil { fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) @@ -195,6 +222,17 @@ func (fr *FacetRequest) AddDateTimeRangeString(name string, start, end *string) &dateTimeRange{Name: name, startString: start, endString: end}) } +// AddDateTimeRangeString adds a bucket to a field +// containing date values. Uses the specified parser to parse the date strings. +// provided the parser is registered in the index mapping. +func (fr *FacetRequest) AddDateTimeRangeStringWithParser(name string, start, end *string, parser string) { + if fr.DateTimeRanges == nil { + fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) + } + fr.DateTimeRanges = append(fr.DateTimeRanges, + &dateTimeRange{Name: name, startString: start, endString: end, DateTimeParser: parser}) +} + // AddNumericRange adds a bucket to a field // containing numeric values. Documents with a // numeric value falling into this range are @@ -212,8 +250,7 @@ type FacetsRequest map[string]*FacetRequest func (fr FacetsRequest) Validate() error { for _, v := range fr { - err := v.Validate() - if err != nil { + if err := v.Validate(); err != nil { return err } } @@ -269,6 +306,7 @@ func (h *HighlightRequest) AddField(field string) { // // A special field named "*" can be used to return all fields. type SearchRequest struct { + ClientContextID string `json:"client_context_id,omitempty"` Query query.Query `json:"query"` Size int `json:"size"` From int `json:"from"` @@ -285,10 +323,13 @@ type SearchRequest struct { sortFunc func(sort.Interface) } +func (r *SearchRequest) SetClientContextID(id string) { + r.ClientContextID = id +} + func (r *SearchRequest) Validate() error { if srq, ok := r.Query.(query.ValidatableQuery); ok { - err := srq.Validate() - if err != nil { + if err := srq.Validate(); err != nil { return err } } @@ -355,23 +396,26 @@ func (r *SearchRequest) SetSearchBefore(before []string) { // UnmarshalJSON deserializes a JSON representation of // a SearchRequest func (r *SearchRequest) UnmarshalJSON(input []byte) error { - var temp struct { - Q json.RawMessage `json:"query"` - Size *int `json:"size"` - From int `json:"from"` - Highlight *HighlightRequest `json:"highlight"` - Fields []string `json:"fields"` - Facets FacetsRequest `json:"facets"` - Explain bool `json:"explain"` - Sort []json.RawMessage `json:"sort"` - IncludeLocations bool `json:"includeLocations"` - Score string `json:"score"` - SearchAfter []string `json:"search_after"` - SearchBefore []string `json:"search_before"` - } + var ( + temp struct { + ClientContextID string `json:"client_context_id"` + Q json.RawMessage `json:"query"` + Size *int `json:"size"` + From int `json:"from"` + Highlight *HighlightRequest `json:"highlight"` + Fields []string `json:"fields"` + Facets FacetsRequest `json:"facets"` + Explain bool `json:"explain"` + Sort []json.RawMessage `json:"sort"` + IncludeLocations bool `json:"includeLocations"` + Score string `json:"score"` + SearchAfter []string `json:"search_after"` + SearchBefore []string `json:"search_before"` + } + err error + ) - err := json.Unmarshal(input, &temp) - if err != nil { + if err = util.UnmarshalJSON(input, &temp); err != nil { return err } @@ -383,11 +427,11 @@ func (r *SearchRequest) UnmarshalJSON(input []byte) error { if temp.Sort == nil { r.Sort = search.SortOrder{&search.SortScore{Desc: true}} } else { - r.Sort, err = search.ParseSortOrderJSON(temp.Sort) - if err != nil { + if r.Sort, err = search.ParseSortOrderJSON(temp.Sort); err != nil { return err } } + r.ClientContextID = temp.ClientContextID r.From = temp.From r.Explain = temp.Explain r.Highlight = temp.Highlight @@ -397,8 +441,7 @@ func (r *SearchRequest) UnmarshalJSON(input []byte) error { r.Score = temp.Score r.SearchAfter = temp.SearchAfter r.SearchBefore = temp.SearchBefore - r.Query, err = query.ParseQuery(temp.Q) - if err != nil { + if r.Query, err = query.ParseQuery(temp.Q); err != nil { return err } @@ -443,13 +486,12 @@ func (iem IndexErrMap) MarshalJSON() ([]byte, error) { for k, v := range iem { tmp[k] = v.Error() } - return json.Marshal(tmp) + return util.MarshalJSON(tmp) } func (iem IndexErrMap) UnmarshalJSON(data []byte) error { var tmp map[string]string - err := json.Unmarshal(data, &tmp) - if err != nil { + if err := util.UnmarshalJSON(data, &tmp); err != nil { return err } for k, v := range tmp { diff --git a/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go index ff5167f21b..c272396b71 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go @@ -17,6 +17,7 @@ package facet import ( "reflect" "sort" + "strconv" "time" "github.com/blevesearch/bleve/v2/numeric" @@ -35,8 +36,10 @@ func init() { } type dateTimeRange struct { - start time.Time - end time.Time + start time.Time + end time.Time + startLayout string + endLayout string } type DateTimeFacetBuilder struct { @@ -75,10 +78,12 @@ func (fb *DateTimeFacetBuilder) Size() int { return sizeInBytes } -func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time) { +func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time, startLayout string, endLayout string) { r := dateTimeRange{ - start: start, - end: end, + start: start, + end: end, + startLayout: startLayout, + endLayout: endLayout, } fb.ranges[name] = &r } @@ -134,11 +139,23 @@ func (fb *DateTimeFacetBuilder) Result() *search.FacetResult { Count: count, } if !dateRange.start.IsZero() { - start := dateRange.start.Format(time.RFC3339Nano) + var start string + if dateRange.startLayout == "" { + // layout not set probably means it is probably a timestamp + start = strconv.FormatInt(dateRange.start.UnixNano(), 10) + } else { + start = dateRange.start.Format(dateRange.startLayout) + } tf.Start = &start } if !dateRange.end.IsZero() { - end := dateRange.end.Format(time.RFC3339Nano) + var end string + if dateRange.endLayout == "" { + // layout not set probably means it is probably a timestamp + end = strconv.FormatInt(dateRange.end.UnixNano(), 10) + } else { + end = dateRange.end.Format(dateRange.endLayout) + } tf.End = &end } rv.DateRanges = append(rv.DateRanges, tf) diff --git a/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go b/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go index de5a157486..ebe785c02d 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go @@ -15,11 +15,11 @@ package search import ( - "encoding/json" "reflect" "sort" "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -176,12 +176,12 @@ func (tf *TermFacets) Less(i, j int) bool { // To maintain backwards compatibility, we have to implement custom // JSON marshalling. func (tf *TermFacets) MarshalJSON() ([]byte, error) { - return json.Marshal(tf.termFacets) + return util.MarshalJSON(tf.termFacets) } func (tf *TermFacets) UnmarshalJSON(b []byte) error { termFacets := []*TermFacet{} - err := json.Unmarshal(b, &termFacets) + err := util.UnmarshalJSON(b, &termFacets) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/boolean.go b/vendor/github.com/blevesearch/bleve/v2/search/query/boolean.go index b5e1fdc40d..026a58688f 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/boolean.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/boolean.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -205,7 +206,7 @@ func (q *BooleanQuery) UnmarshalJSON(data []byte) error { MustNot json.RawMessage `json:"must_not,omitempty"` Boost *Boost `json:"boost,omitempty"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go index 27bec7d61c..0565e18f75 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/conjunction.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -96,7 +97,7 @@ func (q *ConjunctionQuery) UnmarshalJSON(data []byte) error { Conjuncts []json.RawMessage `json:"conjuncts"` Boost *Boost `json:"boost,omitempty"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/date_range.go b/vendor/github.com/blevesearch/bleve/v2/search/query/date_range.go index ef18f2fb8d..47012fb184 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/date_range.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/date_range.go @@ -16,7 +16,6 @@ package query import ( "context" - "encoding/json" "fmt" "math" "time" @@ -27,13 +26,14 @@ import ( "github.com/blevesearch/bleve/v2/registry" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) -// QueryDateTimeParser controls the default query date time parser +// QueryDateTimeParser controls the default query date time parser. var QueryDateTimeParser = optional.Name -// QueryDateTimeFormat controls the format when Marshaling to JSON +// QueryDateTimeFormat controls the format when Marshaling to JSON. var QueryDateTimeFormat = time.RFC3339 var cache = registry.NewCache() @@ -55,7 +55,7 @@ func queryTimeFromString(t string) (time.Time, error) { if err != nil { return time.Time{}, err } - rv, err := dateTimeParser.ParseDateTime(t) + rv, _, err := dateTimeParser.ParseDateTime(t) if err != nil { return time.Time{}, err } @@ -69,7 +69,7 @@ func (t *BleveQueryTime) MarshalJSON() ([]byte, error) { func (t *BleveQueryTime) UnmarshalJSON(data []byte) error { var timeString string - err := json.Unmarshal(data, &timeString) + err := util.UnmarshalJSON(data, &timeString) if err != nil { return err } @@ -77,7 +77,7 @@ func (t *BleveQueryTime) UnmarshalJSON(data []byte) error { if err != nil { return err } - t.Time, err = dateTimeParser.ParseDateTime(timeString) + t.Time, _, err = dateTimeParser.ParseDateTime(timeString) if err != nil { return err } @@ -96,7 +96,7 @@ type DateRangeQuery struct { // NewDateRangeQuery creates a new Query for ranges // of date values. // Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser +// top-level config.QueryDateTimeParser // Either, but not both endpoints can be nil. func NewDateRangeQuery(start, end time.Time) *DateRangeQuery { return NewDateRangeInclusiveQuery(start, end, nil, nil) @@ -105,7 +105,7 @@ func NewDateRangeQuery(start, end time.Time) *DateRangeQuery { // NewDateRangeInclusiveQuery creates a new Query for ranges // of date values. // Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser +// top-level config.QueryDateTimeParser // Either, but not both endpoints can be nil. // startInclusive and endInclusive control inclusion of the endpoints. func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *DateRangeQuery { diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/date_range_string.go b/vendor/github.com/blevesearch/bleve/v2/search/query/date_range_string.go new file mode 100644 index 0000000000..b5e5c17015 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/date_range_string.go @@ -0,0 +1,176 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +// DateRangeStringQuery represents a query for a range of date values. +// Start and End are the range endpoints, as strings. +// Start and End are parsed using DateTimeParser, which is a custom date time parser +// defined in the index mapping. If DateTimeParser is not specified, then the +// top-level config.QueryDateTimeParser is used. +type DateRangeStringQuery struct { + Start string `json:"start,omitempty"` + End string `json:"end,omitempty"` + InclusiveStart *bool `json:"inclusive_start,omitempty"` + InclusiveEnd *bool `json:"inclusive_end,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + DateTimeParser string `json:"datetime_parser,omitempty"` +} + +// NewDateRangeStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser field of the query struct, +// which is a custom date time parser defined in the index mapping. +// if DateTimeParser is not specified, then the +// top-level config.QueryDateTimeParser is used. +// Either, but not both endpoints can be nil. +func NewDateRangeStringQuery(start, end string) *DateRangeStringQuery { + return NewDateRangeStringInclusiveQuery(start, end, nil, nil) +} + +// NewDateRangeStringQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser field of the query struct, +// which is a custom date time parser defined in the index mapping. +// if DateTimeParser is not specified, then the +// top-level config.QueryDateTimeParser is used. +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeStringInclusiveQuery(start, end string, startInclusive, endInclusive *bool) *DateRangeStringQuery { + return &DateRangeStringQuery{ + Start: start, + End: end, + InclusiveStart: startInclusive, + InclusiveEnd: endInclusive, + } +} + +func (q *DateRangeStringQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *DateRangeStringQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *DateRangeStringQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *DateRangeStringQuery) Field() string { + return q.FieldVal +} + +func (q *DateRangeStringQuery) SetDateTimeParser(d string) { + q.DateTimeParser = d +} + +func (q *DateRangeStringQuery) DateTimeParserName() string { + return q.DateTimeParser +} + +func (q *DateRangeStringQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + dateTimeParserName := QueryDateTimeParser + if q.DateTimeParser != "" { + dateTimeParserName = q.DateTimeParser + } + dateTimeParser := m.DateTimeParserNamed(dateTimeParserName) + if dateTimeParser == nil { + return nil, fmt.Errorf("no dateTimeParser named '%s' registered", dateTimeParserName) + } + + var startTime, endTime time.Time + var err error + if q.Start != "" { + startTime, _, err = dateTimeParser.ParseDateTime(q.Start) + if err != nil { + return nil, fmt.Errorf("%v, date time parser name: %s", err, dateTimeParserName) + } + } + if q.End != "" { + endTime, _, err = dateTimeParser.ParseDateTime(q.End) + if err != nil { + return nil, fmt.Errorf("%v, date time parser name: %s", err, dateTimeParserName) + } + } + + min, max, err := q.parseEndpoints(startTime, endTime) + if err != nil { + return nil, err + } + return searcher.NewNumericRangeSearcher(ctx, i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options) +} + +func (q *DateRangeStringQuery) parseEndpoints(startTime, endTime time.Time) (*float64, *float64, error) { + min := math.Inf(-1) + max := math.Inf(1) + + if startTime.IsZero() && endTime.IsZero() { + return nil, nil, fmt.Errorf("date range query must specify at least one of start/end") + } + + if !startTime.IsZero() { + if !isDateTimeWithinRange(startTime) { + // overflow + return nil, nil, fmt.Errorf("invalid/unsupported date range, start: %v", q.Start) + } + startInt64 := startTime.UnixNano() + min = numeric.Int64ToFloat64(startInt64) + } + if !endTime.IsZero() { + if !isDateTimeWithinRange(endTime) { + // overflow + return nil, nil, fmt.Errorf("invalid/unsupported date range, end: %v", q.End) + } + endInt64 := endTime.UnixNano() + max = numeric.Int64ToFloat64(endInt64) + } + + return &min, &max, nil +} + +func (q *DateRangeStringQuery) Validate() error { + // either start or end must be specified + if q.Start == "" && q.End == "" { + return fmt.Errorf("date range query must specify at least one of start/end") + } + return nil +} + +func isDateTimeWithinRange(t time.Time) bool { + if t.Before(MinRFC3339CompatibleTime) || t.After(MaxRFC3339CompatibleTime) { + return false + } + return true +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go b/vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go index c6cc0d737e..f8573d081f 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/disjunction.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -107,7 +108,7 @@ func (q *DisjunctionQuery) UnmarshalJSON(data []byte) error { Boost *Boost `json:"boost,omitempty"` Min float64 `json:"min"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go index 1397c77996..feb45d314c 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingbox.go @@ -16,13 +16,13 @@ package query import ( "context" - "encoding/json" "fmt" "github.com/blevesearch/bleve/v2/geo" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -95,7 +95,7 @@ func (q *GeoBoundingBoxQuery) UnmarshalJSON(data []byte) error { FieldVal string `json:"field,omitempty"` BoostVal *Boost `json:"boost,omitempty"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingpolygon.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingpolygon.go index baae514d96..7f81a7cac6 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingpolygon.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_boundingpolygon.go @@ -16,13 +16,13 @@ package query import ( "context" - "encoding/json" "fmt" "github.com/blevesearch/bleve/v2/geo" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -76,7 +76,7 @@ func (q *GeoBoundingPolygonQuery) UnmarshalJSON(data []byte) error { FieldVal string `json:"field,omitempty"` BoostVal *Boost `json:"boost,omitempty"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/geo_distance.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_distance.go index 7977d1538b..2ca0964247 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/geo_distance.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_distance.go @@ -16,13 +16,13 @@ package query import ( "context" - "encoding/json" "fmt" "github.com/blevesearch/bleve/v2/geo" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -86,7 +86,7 @@ func (q *GeoDistanceQuery) UnmarshalJSON(data []byte) error { FieldVal string `json:"field,omitempty"` BoostVal *Boost `json:"boost,omitempty"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/geo_shape.go b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_shape.go index 2229dbe9c6..686f486fd5 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/geo_shape.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/geo_shape.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -123,7 +124,7 @@ func (q *Geometry) UnmarshalJSON(data []byte) error { Relation string `json:"relation"` }{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/match.go b/vendor/github.com/blevesearch/bleve/v2/search/query/match.go index 61c00a0033..074d11d34f 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/match.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/match.go @@ -16,11 +16,11 @@ package query import ( "context" - "encoding/json" "fmt" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -46,9 +46,9 @@ const ( func (o MatchQueryOperator) MarshalJSON() ([]byte, error) { switch o { case MatchQueryOperatorOr: - return json.Marshal("or") + return util.MarshalJSON("or") case MatchQueryOperatorAnd: - return json.Marshal("and") + return util.MarshalJSON("and") default: return nil, fmt.Errorf("cannot marshal match operator %d to JSON", o) } @@ -56,7 +56,7 @@ func (o MatchQueryOperator) MarshalJSON() ([]byte, error) { func (o *MatchQueryOperator) UnmarshalJSON(data []byte) error { var operatorString string - err := json.Unmarshal(data, &operatorString) + err := util.UnmarshalJSON(data, &operatorString) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/match_phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/query/match_phrase.go index fa8ac720b3..63a16a5346 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/match_phrase.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/match_phrase.go @@ -29,6 +29,7 @@ type MatchPhraseQuery struct { FieldVal string `json:"field,omitempty"` Analyzer string `json:"analyzer,omitempty"` BoostVal *Boost `json:"boost,omitempty"` + Fuzziness int `json:"fuzziness"` } // NewMatchPhraseQuery creates a new Query object @@ -58,6 +59,10 @@ func (q *MatchPhraseQuery) SetField(f string) { q.FieldVal = f } +func (q *MatchPhraseQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + func (q *MatchPhraseQuery) Field() string { return q.FieldVal } @@ -84,6 +89,7 @@ func (q *MatchPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m phrase := tokenStreamToPhrase(tokens) phraseQuery := NewMultiPhraseQuery(phrase, field) phraseQuery.SetBoost(q.BoostVal.Value()) + phraseQuery.SetFuzziness(q.Fuzziness) return phraseQuery.Searcher(ctx, i, m, options) } noneQuery := NewMatchNoneQuery() diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/multi_phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/query/multi_phrase.go index 2887be16a4..d1144d9086 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/multi_phrase.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/multi_phrase.go @@ -16,19 +16,20 @@ package query import ( "context" - "encoding/json" "fmt" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) type MultiPhraseQuery struct { - Terms [][]string `json:"terms"` - Field string `json:"field,omitempty"` - BoostVal *Boost `json:"boost,omitempty"` + Terms [][]string `json:"terms"` + Field string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Fuzziness int `json:"fuzziness"` } // NewMultiPhraseQuery creates a new Query for finding @@ -47,6 +48,10 @@ func NewMultiPhraseQuery(terms [][]string, field string) *MultiPhraseQuery { } } +func (q *MultiPhraseQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + func (q *MultiPhraseQuery) SetBoost(b float64) { boost := Boost(b) q.BoostVal = &boost @@ -57,7 +62,7 @@ func (q *MultiPhraseQuery) Boost() float64 { } func (q *MultiPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - return searcher.NewMultiPhraseSearcher(ctx, i, q.Terms, q.Field, options) + return searcher.NewMultiPhraseSearcher(ctx, i, q.Terms, q.Fuzziness, q.Field, q.BoostVal.Value(), options) } func (q *MultiPhraseQuery) Validate() error { @@ -70,12 +75,13 @@ func (q *MultiPhraseQuery) Validate() error { func (q *MultiPhraseQuery) UnmarshalJSON(data []byte) error { type _mphraseQuery MultiPhraseQuery tmp := _mphraseQuery{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } q.Terms = tmp.Terms q.Field = tmp.Field q.BoostVal = tmp.BoostVal + q.Fuzziness = tmp.Fuzziness return nil } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/query/phrase.go index 207e66b176..9092e72d03 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/phrase.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/phrase.go @@ -16,19 +16,20 @@ package query import ( "context" - "encoding/json" "fmt" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) type PhraseQuery struct { - Terms []string `json:"terms"` - Field string `json:"field,omitempty"` - BoostVal *Boost `json:"boost,omitempty"` + Terms []string `json:"terms"` + Field string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Fuzziness int `json:"fuzziness"` } // NewPhraseQuery creates a new Query for finding @@ -49,12 +50,16 @@ func (q *PhraseQuery) SetBoost(b float64) { q.BoostVal = &boost } +func (q *PhraseQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + func (q *PhraseQuery) Boost() float64 { return q.BoostVal.Value() } func (q *PhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - return searcher.NewPhraseSearcher(ctx, i, q.Terms, q.Field, options) + return searcher.NewPhraseSearcher(ctx, i, q.Terms, q.Fuzziness, q.Field, q.BoostVal.Value(), options) } func (q *PhraseQuery) Validate() error { @@ -67,12 +72,13 @@ func (q *PhraseQuery) Validate() error { func (q *PhraseQuery) UnmarshalJSON(data []byte) error { type _phraseQuery PhraseQuery tmp := _phraseQuery{} - err := json.Unmarshal(data, &tmp) + err := util.UnmarshalJSON(data, &tmp) if err != nil { return err } q.Terms = tmp.Terms q.Field = tmp.Field q.BoostVal = tmp.BoostVal + q.Fuzziness = tmp.Fuzziness return nil } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/query.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query.go index df560534b4..eb7b34adbd 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/query.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query.go @@ -18,18 +18,19 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "log" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) -var logger = log.New(ioutil.Discard, "bleve mapping ", log.LstdFlags) +var logger = log.New(io.Discard, "bleve mapping ", log.LstdFlags) // SetLog sets the logger used for logging -// by default log messages are sent to ioutil.Discard +// by default log messages are sent to io.Discard func SetLog(l *log.Logger) { logger = l } @@ -68,24 +69,17 @@ type ValidatableQuery interface { // a Query object. func ParseQuery(input []byte) (Query, error) { var tmp map[string]interface{} - err := json.Unmarshal(input, &tmp) + err := util.UnmarshalJSON(input, &tmp) if err != nil { return nil, err } - _, isMatchQuery := tmp["match"] _, hasFuzziness := tmp["fuzziness"] - if hasFuzziness && !isMatchQuery { + _, isMatchQuery := tmp["match"] + _, isMatchPhraseQuery := tmp["match_phrase"] + _, hasTerms := tmp["terms"] + if hasFuzziness && !isMatchQuery && !isMatchPhraseQuery && !hasTerms { var rv FuzzyQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, isTermQuery := tmp["term"] - if isTermQuery { - var rv TermQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -93,16 +87,38 @@ func ParseQuery(input []byte) (Query, error) { } if isMatchQuery { var rv MatchQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } return &rv, nil } - _, isMatchPhraseQuery := tmp["match_phrase"] if isMatchPhraseQuery { var rv MatchPhraseQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + if hasTerms { + var rv PhraseQuery + err := util.UnmarshalJSON(input, &rv) + if err != nil { + // now try multi-phrase + var rv2 MultiPhraseQuery + err = util.UnmarshalJSON(input, &rv2) + if err != nil { + return nil, err + } + return &rv2, nil + } + return &rv, nil + } + _, isTermQuery := tmp["term"] + if isTermQuery { + var rv TermQuery + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -113,31 +129,16 @@ func ParseQuery(input []byte) (Query, error) { _, hasMustNot := tmp["must_not"] if hasMust || hasShould || hasMustNot { var rv BooleanQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } return &rv, nil } - _, hasTerms := tmp["terms"] - if hasTerms { - var rv PhraseQuery - err := json.Unmarshal(input, &rv) - if err != nil { - // now try multi-phrase - var rv2 MultiPhraseQuery - err = json.Unmarshal(input, &rv2) - if err != nil { - return nil, err - } - return &rv2, nil - } - return &rv, nil - } _, hasConjuncts := tmp["conjuncts"] if hasConjuncts { var rv ConjunctionQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -146,7 +147,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasDisjuncts := tmp["disjuncts"] if hasDisjuncts { var rv DisjunctionQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -156,7 +157,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasSyntaxQuery := tmp["query"] if hasSyntaxQuery { var rv QueryStringQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -166,7 +167,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasMax := tmp["max"].(float64) if hasMin || hasMax { var rv NumericRangeQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -176,7 +177,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasMaxStr := tmp["max"].(string) if hasMinStr || hasMaxStr { var rv TermRangeQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -185,8 +186,8 @@ func ParseQuery(input []byte) (Query, error) { _, hasStart := tmp["start"] _, hasEnd := tmp["end"] if hasStart || hasEnd { - var rv DateRangeQuery - err := json.Unmarshal(input, &rv) + var rv DateRangeStringQuery + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -195,7 +196,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasPrefix := tmp["prefix"] if hasPrefix { var rv PrefixQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -204,7 +205,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasRegexp := tmp["regexp"] if hasRegexp { var rv RegexpQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -213,7 +214,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasWildcard := tmp["wildcard"] if hasWildcard { var rv WildcardQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -222,7 +223,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasMatchAll := tmp["match_all"] if hasMatchAll { var rv MatchAllQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -231,7 +232,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasMatchNone := tmp["match_none"] if hasMatchNone { var rv MatchNoneQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -240,7 +241,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasDocIds := tmp["ids"] if hasDocIds { var rv DocIDQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -249,7 +250,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasBool := tmp["bool"] if hasBool { var rv BoolFieldQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -259,7 +260,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasBottomRight := tmp["bottom_right"] if hasTopLeft && hasBottomRight { var rv GeoBoundingBoxQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -268,7 +269,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasDistance := tmp["distance"] if hasDistance { var rv GeoDistanceQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -277,7 +278,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasPoints := tmp["polygon_points"] if hasPoints { var rv GeoBoundingPolygonQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -287,7 +288,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasGeo := tmp["geometry"] if hasGeo { var rv GeoShapeQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } @@ -297,7 +298,7 @@ func ParseQuery(input []byte) (Query, error) { _, hasCIDR := tmp["cidr"] if hasCIDR { var rv IPRangeQuery - err := json.Unmarshal(input, &rv) + err := util.UnmarshalJSON(input, &rv) if err != nil { return nil, err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/search.go b/vendor/github.com/blevesearch/bleve/v2/search/search.go index d2dd33712e..b7a3c42ae2 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/search.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/search.go @@ -155,7 +155,7 @@ type DocumentMatch struct { // Fields contains the values for document fields listed in // SearchRequest.Fields. Text fields are returned as strings, numeric - // fields as float64s and date fields as time.RFC3339 formatted strings. + // fields as float64s and date fields as strings. Fields map[string]interface{} `json:"fields,omitempty"` // used to maintain natural index order @@ -166,6 +166,13 @@ type DocumentMatch struct { // be later incorporated into the Locations map when search // results are completed FieldTermLocations []FieldTermLocation `json:"-"` + + // used to indicate if this match is a partial match + // in the case of a disjunction search + // this means that the match is partial because + // not all sub-queries matched + // if false, all the sub-queries matched + PartialMatch bool `json:"partial_match,omitempty"` } func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) { diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go index 0235838325..d36e301314 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go @@ -197,8 +197,10 @@ func (s *DisjunctionHeapSearcher) Next(ctx *search.SearchContext) ( for !found && len(s.matching) > 0 { if len(s.matching) >= s.min { found = true + partialMatch := len(s.matching) != len(s.searchers) // score this match rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) + rv.PartialMatch = partialMatch } // invoke next on all the matching searchers diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go index 6958cf492f..0969c8cf3c 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go @@ -197,8 +197,10 @@ func (s *DisjunctionSliceSearcher) Next(ctx *search.SearchContext) ( for !found && len(s.matching) > 0 { if len(s.matching) >= s.min { found = true + partialMatch := len(s.matching) != len(s.searchers) // score this match rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) + rv.PartialMatch = partialMatch } // invoke next on all the matching searchers diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go index 5345c272bc..1957168bb3 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go @@ -61,6 +61,10 @@ func NewFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term s if ctx != nil { reportIOStats(ctx, dictBytesRead) search.RecordSearchCost(ctx, search.AddM, dictBytesRead) + fuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey) + if fuzzyTermMatches != nil { + fuzzyTermMatches.(map[string][]string)[term] = candidates + } } return NewMultiTermSearcher(ctx, indexReader, candidates, field, diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoshape.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoshape.go index 1107c94386..ae113107d7 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoshape.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoshape.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/search" index "github.com/blevesearch/bleve_index_api" "github.com/blevesearch/geo/geojson" + "github.com/blevesearch/geo/s2" ) func NewGeoShapeSearcher(ctx context.Context, indexReader index.IndexReader, shape index.GeoJSON, @@ -70,6 +71,12 @@ func buildRelationFilterOnShapes(ctx context.Context, dvReader index.DocValueRea var dvShapeValue []byte var startReading, finishReading bool var reader *bytes.Reader + + var bufPool *s2.GeoBufferPool + if ctx != nil { + bufPool = ctx.Value(search.GeoBufferPoolCallbackKey).(search.GeoBufferPoolCallbackFunc)() + } + return func(d *search.DocumentMatch) bool { var found bool @@ -104,7 +111,7 @@ func buildRelationFilterOnShapes(ctx context.Context, dvReader index.DocValueRea // apply the filter once the entire docvalue is finished reading. if finishReading { v, err := geojson.FilterGeoShapesOnRelation(shape, - dvShapeValue, relation, &reader) + dvShapeValue, relation, &reader, bufPool) if err == nil && v { found = true } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go index 087ad768cb..a7bdb2c81c 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go @@ -41,6 +41,8 @@ type PhraseSearcher struct { paths []phrasePath locations []search.Location initialized bool + // map a term to a list of fuzzy terms that match it + fuzzyTermMatches map[string][]string } func (s *PhraseSearcher) Size() int { @@ -64,22 +66,42 @@ func (s *PhraseSearcher) Size() int { return sizeInBytes } -func NewPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms []string, field string, options search.SearcherOptions) (*PhraseSearcher, error) { +func NewPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms []string, + fuzziness int, field string, boost float64, options search.SearcherOptions) (*PhraseSearcher, error) { + // turn flat terms []string into [][]string mterms := make([][]string, len(terms)) for i, term := range terms { mterms[i] = []string{term} } - return NewMultiPhraseSearcher(ctx, indexReader, mterms, field, options) + return NewMultiPhraseSearcher(ctx, indexReader, mterms, fuzziness, field, boost, options) } -func NewMultiPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms [][]string, field string, options search.SearcherOptions) (*PhraseSearcher, error) { +func NewMultiPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms [][]string, + fuzziness int, field string, boost float64, options search.SearcherOptions) (*PhraseSearcher, error) { + options.IncludeTermVectors = true var termPositionSearchers []search.Searcher + var err error + var ts search.Searcher + var fuzzyTermMatches map[string][]string + if fuzziness > 0 { + fuzzyTermMatches = make(map[string][]string) + ctx = context.WithValue(ctx, search.FuzzyMatchPhraseKey, fuzzyTermMatches) + } + // in case of fuzzy multi-phrase, phrase and match-phrase queries we hardcode the + // prefix length to 0, as setting a per word matching prefix length would not + // make sense from a user perspective. for _, termPos := range terms { if len(termPos) == 1 && termPos[0] != "" { // single term - ts, err := NewTermSearcher(ctx, indexReader, termPos[0], field, 1.0, options) + if fuzziness > 0 { + // fuzzy + ts, err = NewFuzzySearcher(ctx, indexReader, termPos[0], 0, fuzziness, field, boost, options) + } else { + // non-fuzzy + ts, err = NewTermSearcher(ctx, indexReader, termPos[0], field, boost, options) + } if err != nil { // close any searchers already opened for _, ts := range termPositionSearchers { @@ -95,7 +117,13 @@ func NewMultiPhraseSearcher(ctx context.Context, indexReader index.IndexReader, if term == "" { continue } - ts, err := NewTermSearcher(ctx, indexReader, term, field, 1.0, options) + if fuzziness > 0 { + // fuzzy + ts, err = NewFuzzySearcher(ctx, indexReader, term, 0, fuzziness, field, boost, options) + } else { + // non-fuzzy + ts, err = NewTermSearcher(ctx, indexReader, term, field, boost, options) + } if err != nil { // close any searchers already opened for _, ts := range termPositionSearchers { @@ -128,8 +156,9 @@ func NewMultiPhraseSearcher(ctx context.Context, indexReader index.IndexReader, // build our searcher rv := PhraseSearcher{ - mustSearcher: mustSearcher, - terms: terms, + mustSearcher: mustSearcher, + terms: terms, + fuzzyTermMatches: fuzzyTermMatches, } rv.computeQueryNorm() return &rv, nil @@ -213,7 +242,7 @@ func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, // checkCurrMustMatch is solely concerned with determining if the DocumentMatch // pointed to by s.currMust (which satisifies the pre-condition searcher) -// also satisfies the phase constraints. if so, it returns a DocumentMatch +// also satisfies the phrase constraints. if so, it returns a DocumentMatch // for this document, otherwise nil func (s *PhraseSearcher) checkCurrMustMatch(ctx *search.SearchContext) *search.DocumentMatch { s.locations = s.currMust.Complete(s.locations) @@ -244,7 +273,7 @@ func (s *PhraseSearcher) checkCurrMustMatch(ctx *search.SearchContext) *search.D // checkCurrMustMatchField is solely concerned with determining if one // particular field within the currMust DocumentMatch Locations -// satisfies the phase constraints (possibly more than once). if so, +// satisfies the phrase constraints (possibly more than once). if so, // the matching field term locations are appended to the provided // slice func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext, @@ -253,7 +282,21 @@ func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext, if s.path == nil { s.path = make(phrasePath, 0, len(s.terms)) } - s.paths = findPhrasePaths(0, nil, s.terms, tlm, s.path[:0], 0, s.paths[:0]) + var tlmPtr *search.TermLocationMap = &tlm + if s.fuzzyTermMatches != nil { + // if fuzzy search, we need to expand the tlm to include all the fuzzy matches + // Example - term is "foo" and fuzzy matches are "foo", "fool", "food" + // the non expanded tlm will be: + // foo -> Locations[foo] + // fool -> Locations[fool] + // food -> Locations[food] + // the expanded tlm will be: + // foo -> [Locations[foo], Locations[fool], Locations[food]] + expandedTlm := make(search.TermLocationMap) + s.expandFuzzyMatches(tlm, expandedTlm) + tlmPtr = &expandedTlm + } + s.paths = findPhrasePaths(0, nil, s.terms, *tlmPtr, s.path[:0], 0, s.paths[:0]) for _, p := range s.paths { for _, pp := range p { ftls = append(ftls, search.FieldTermLocation{ @@ -271,6 +314,16 @@ func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext, return ftls } +func (s *PhraseSearcher) expandFuzzyMatches(tlm search.TermLocationMap, expandedTlm search.TermLocationMap) { + for term, fuzzyMatches := range s.fuzzyTermMatches { + locations := tlm[term] + for _, fuzzyMatch := range fuzzyMatches { + locations = append(locations, tlm[fuzzyMatch]...) + } + expandedTlm[term] = locations + } +} + type phrasePart struct { term string loc *search.Location @@ -300,26 +353,31 @@ func (p phrasePath) String() string { return rv } -// findPhrasePaths is a function to identify phase matches from a set +// findPhrasePaths is a function to identify phrase matches from a set // of known term locations. it recursive so care must be taken with // arguments and return values. // // prevPos - the previous location, 0 on first invocation +// // ap - array positions of the first candidate phrase part to -// which further recursive phrase parts must match, -// nil on initial invocation or when there are no array positions +// which further recursive phrase parts must match, +// nil on initial invocation or when there are no array positions +// // phraseTerms - slice containing the phrase terms, -// may contain empty string as placeholder (don't care) +// may contain empty string as placeholder (don't care) +// // tlm - the Term Location Map containing all relevant term locations +// // p - the current path being explored (appended to in recursive calls) -// this is the primary state being built during the traversal +// this is the primary state being built during the traversal +// // remainingSlop - amount of sloppiness that's allowed, which is the -// sum of the editDistances from each matching phrase part, -// where 0 means no sloppiness allowed (all editDistances must be 0), -// decremented during recursion +// sum of the editDistances from each matching phrase part, where 0 means no +// sloppiness allowed (all editDistances must be 0), decremented during recursion +// // rv - the final result being appended to by all the recursive calls // -// returns slice of paths, or nil if invocation did not find any successul paths +// returns slice of paths, or nil if invocation did not find any successful paths func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]string, tlm search.TermLocationMap, p phrasePath, remainingSlop int, rv []phrasePath) []phrasePath { // no more terms diff --git a/vendor/github.com/blevesearch/bleve/v2/search/sort.go b/vendor/github.com/blevesearch/bleve/v2/search/sort.go index 3a744af992..b13fa16c13 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/sort.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/sort.go @@ -25,6 +25,7 @@ import ( "github.com/blevesearch/bleve/v2/geo" "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/util" ) var HighTerm = strings.Repeat(string(utf8.MaxRune), 3) @@ -164,10 +165,10 @@ func ParseSearchSortString(input string) SearchSort { func ParseSearchSortJSON(input json.RawMessage) (SearchSort, error) { // first try to parse it as string var sortString string - err := json.Unmarshal(input, &sortString) + err := util.UnmarshalJSON(input, &sortString) if err != nil { var sortObj map[string]interface{} - err = json.Unmarshal(input, &sortObj) + err = util.UnmarshalJSON(input, &sortObj) if err != nil { return nil, err } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/util.go b/vendor/github.com/blevesearch/bleve/v2/search/util.go index 7a946868e1..b2cb62a2d1 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/util.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/util.go @@ -14,7 +14,11 @@ package search -import "context" +import ( + "context" + + "github.com/blevesearch/geo/s2" +) func MergeLocations(locations []FieldTermLocationMap) FieldTermLocationMap { rv := locations[0] @@ -101,6 +105,7 @@ const ( const SearchIncrementalCostKey = "_search_incremental_cost_key" const QueryTypeKey = "_query_type_key" +const FuzzyMatchPhraseKey = "_fuzzy_match_phrase_key" func RecordSearchCost(ctx context.Context, msg SearchIncrementalCostCallbackMsg, bytes uint64) { @@ -118,3 +123,13 @@ func RecordSearchCost(ctx context.Context, } } } + +const GeoBufferPoolCallbackKey = "_geo_buffer_pool_callback_key" + +// Assigning the size of the largest buffer in the pool to 24KB and +// the smallest buffer to 24 bytes. The pools are used to read a +// sequence of vertices which are always 24 bytes each. +const MaxGeoBufPoolSize = 24 * 1024 +const MinGeoBufPoolSize = 24 + +type GeoBufferPoolCallbackFunc func() *s2.GeoBufferPool diff --git a/vendor/github.com/blevesearch/bleve/v2/util/json.go b/vendor/github.com/blevesearch/bleve/v2/util/json.go new file mode 100644 index 0000000000..8745dd8a80 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/util/json.go @@ -0,0 +1,25 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "encoding/json" +) + +// Should only be overwritten during process init()'ialization. +var ( + MarshalJSON = json.Marshal + UnmarshalJSON = json.Unmarshal +) diff --git a/vendor/github.com/blevesearch/bleve_index_api/document.go b/vendor/github.com/blevesearch/bleve_index_api/document.go index 416d701297..0f9012fd18 100644 --- a/vendor/github.com/blevesearch/bleve_index_api/document.go +++ b/vendor/github.com/blevesearch/bleve_index_api/document.go @@ -67,7 +67,7 @@ type NumericField interface { } type DateTimeField interface { - DateTime() (time.Time, error) + DateTime() (time.Time, string, error) } type BooleanField interface { diff --git a/vendor/github.com/blevesearch/geo/geojson/geojson_shapes_util.go b/vendor/github.com/blevesearch/geo/geojson/geojson_shapes_util.go index a62f685cd3..c61912e033 100644 --- a/vendor/github.com/blevesearch/geo/geojson/geojson_shapes_util.go +++ b/vendor/github.com/blevesearch/geo/geojson/geojson_shapes_util.go @@ -31,9 +31,9 @@ var jsoniter = jsoniterator.ConfigCompatibleWithStandardLibrary // the `relation` filter and confirms whether the shape in the document // satisfies the given relation. func FilterGeoShapesOnRelation(shape index.GeoJSON, targetShapeBytes []byte, - relation string, reader **bytes.Reader) (bool, error) { + relation string, reader **bytes.Reader, bufPool *s2.GeoBufferPool) (bool, error) { - shapeInDoc, err := extractShapesFromBytes(targetShapeBytes, reader) + shapeInDoc, err := extractShapesFromBytes(targetShapeBytes, reader, bufPool) if err != nil { return false, err } @@ -43,7 +43,7 @@ func FilterGeoShapesOnRelation(shape index.GeoJSON, targetShapeBytes []byte, // extractShapesFromBytes unmarshal the bytes to retrieve the // embedded geojson shape. -func extractShapesFromBytes(targetShapeBytes []byte, r **bytes.Reader) ( +func extractShapesFromBytes(targetShapeBytes []byte, r **bytes.Reader, bufPool *s2.GeoBufferPool) ( index.GeoJSON, error) { if (*r) == nil { *r = bytes.NewReader(targetShapeBytes[1:]) @@ -109,7 +109,7 @@ func extractShapesFromBytes(targetShapeBytes []byte, r **bytes.Reader) ( return mls, nil case PolygonTypePrefix: - pgn := &Polygon{s2pgn: &s2.Polygon{}} + pgn := &Polygon{s2pgn: &s2.Polygon{BufPool: bufPool}} err := pgn.s2pgn.Decode(*r) if err != nil { return nil, err @@ -156,7 +156,7 @@ func extractShapesFromBytes(targetShapeBytes []byte, r **bytes.Reader) ( gc := &GeometryCollection{Shapes: make([]index.GeoJSON, numShapes)} for i := int32(0); i < numShapes; i++ { - shape, err := extractShapesFromBytes(inputBytes[:lengths[i]], r) + shape, err := extractShapesFromBytes(inputBytes[:lengths[i]], r, nil) if err != nil { return nil, err } diff --git a/vendor/github.com/blevesearch/geo/s2/buffer_pool.go b/vendor/github.com/blevesearch/geo/s2/buffer_pool.go new file mode 100644 index 0000000000..90fb20c373 --- /dev/null +++ b/vendor/github.com/blevesearch/geo/s2/buffer_pool.go @@ -0,0 +1,69 @@ +// Copyright (c) 2023 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s2 + +// GeoBufferPool represents a pool of buffers ranging from a given +// max size to a min size in steps of 2. It uses a lazy approach only allocating +// the buffers when it is needed. + +type GeoBufferPool struct { + buffers [][]byte + maxSize int + minSize int +} + +func NewGeoBufferPool(maxSize int, minSize int) *GeoBufferPool { + // Calculating the number of buffers required. Assuming that + // the value of minSize is correct, the buffers will be of size + // minSize, 2 * minSize, 4 * minSize and so on till it is less + // than or equal to the maxSize. If it is not equal to maxSize, + // then a suitable value less than maxSize will be set as maxSize + length := 0 + temp := minSize + for temp <= maxSize { + length = length + 1 + temp = temp * 2 + } + maxSize = temp / 2 + + return &GeoBufferPool{ + buffers: make([][]byte, length), + maxSize: maxSize, + minSize: minSize, + } +} + +func (b *GeoBufferPool) Get(size int) ([]byte) { + if b == nil { + // GeoBufferPool not setup + return make([]byte, size) + } + + bufSize := b.minSize + + for i := range b.buffers { + if size <= bufSize || i == len(b.buffers) - 1 { + if b.buffers[i] == nil { + b.buffers[i] = make([]byte, bufSize) + } + + return b.buffers[i] + } + + bufSize = bufSize * 2 + } + + return nil +} diff --git a/vendor/github.com/blevesearch/geo/s2/encode.go b/vendor/github.com/blevesearch/geo/s2/encode.go index 00d0adc719..66eaf98a52 100644 --- a/vendor/github.com/blevesearch/geo/s2/encode.go +++ b/vendor/github.com/blevesearch/geo/s2/encode.go @@ -222,3 +222,17 @@ func (d *decoder) readUvarint() (x uint64) { x, d.err = binary.ReadUvarint(d.r) return } + +func (d *decoder) readFloat64Array(size int, buf []byte) int { + if d.err != nil || buf == nil { + return 0 + } + + if size >= len(buf) { + _, d.err = io.ReadFull(d.r, buf) + return len(buf) + } + + _, d.err = io.ReadFull(d.r, buf[0:size]) + return size +} diff --git a/vendor/github.com/blevesearch/geo/s2/loop.go b/vendor/github.com/blevesearch/geo/s2/loop.go index d0035dfa93..95cd17e4b7 100644 --- a/vendor/github.com/blevesearch/geo/s2/loop.go +++ b/vendor/github.com/blevesearch/geo/s2/loop.go @@ -15,9 +15,11 @@ package s2 import ( + "encoding/binary" "fmt" "io" "math" + "reflect" "github.com/golang/geo/r1" "github.com/golang/geo/r3" @@ -66,6 +68,9 @@ type Loop struct { // index is the spatial index for this Loop. index *ShapeIndex + + // A buffer pool to be used while decoding the polygon + BufPool *GeoBufferPool } // LoopFromPoints constructs a loop from the given points. @@ -1258,6 +1263,15 @@ func (l Loop) encode(e *encoder) { l.bound.encode(e) } +func init() { + var f64 float64 + sizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + sizeOfVertex = 3 * sizeOfFloat64 +} + +var sizeOfFloat64 int +var sizeOfVertex int + // Decode decodes a loop. func (l *Loop) Decode(r io.Reader) error { *l = Loop{} @@ -1287,11 +1301,37 @@ func (l *Loop) decode(d *decoder) { return } l.vertices = make([]Point, nvertices) - for i := range l.vertices { - l.vertices[i].X = d.readFloat64() - l.vertices[i].Y = d.readFloat64() - l.vertices[i].Z = d.readFloat64() + + // Each vertex requires 24 bytes of storage + numBytesNeeded := int(nvertices) * sizeOfVertex + + i := 0 + + for numBytesNeeded > 0 { + arr := l.BufPool.Get(numBytesNeeded) + numBytesRead := d.readFloat64Array(numBytesNeeded, arr) + + if numBytesRead == 0 { + break + } + + numBytesNeeded -= numBytesRead + + // Parsing one vertex at a time into the vertex array of the loop + // by going through the buffer in steps of sizeOfVertex and converting + // floatSize worth of bytes into the float values + for j := 0; j < int(numBytesRead/sizeOfVertex); j++ { + l.vertices[i+j].X = math.Float64frombits( + binary.LittleEndian.Uint64(arr[sizeOfFloat64*(j*3) : sizeOfFloat64*(j*3+1)])) + l.vertices[i+j].Y = math.Float64frombits( + binary.LittleEndian.Uint64(arr[sizeOfFloat64*(j*3+1) : sizeOfFloat64*(j*3+2)])) + l.vertices[i+j].Z = math.Float64frombits( + binary.LittleEndian.Uint64(arr[sizeOfFloat64*(j*3+2) : sizeOfFloat64*(j*3+3)])) + } + + i += int(numBytesRead/sizeOfVertex) } + l.index = NewShapeIndex() l.originInside = d.readBool() l.depth = int(d.readUint32()) diff --git a/vendor/github.com/blevesearch/geo/s2/polygon.go b/vendor/github.com/blevesearch/geo/s2/polygon.go index db740070a6..2ec84d2dfd 100644 --- a/vendor/github.com/blevesearch/geo/s2/polygon.go +++ b/vendor/github.com/blevesearch/geo/s2/polygon.go @@ -77,6 +77,9 @@ type Polygon struct { // preceding loops in the polygon. This field is used for polygons that // have a large number of loops, and may be empty for polygons with few loops. cumulativeEdges []int + + // A buffer pool to be used while decoding the polygon + BufPool *GeoBufferPool } // PolygonFromLoops constructs a polygon from the given set of loops. The polygon @@ -1133,7 +1136,7 @@ func (p *Polygon) Decode(r io.Reader) error { const maxEncodedLoops = 10000000 func (p *Polygon) decode(d *decoder) { - *p = Polygon{} + *p = Polygon{BufPool: p.BufPool} d.readUint8() // Ignore irrelevant serialized owns_loops_ value. p.hasHoles = d.readBool() @@ -1151,6 +1154,7 @@ func (p *Polygon) decode(d *decoder) { p.loops = make([]*Loop, nloops) for i := range p.loops { p.loops[i] = new(Loop) + p.loops[i].BufPool = p.BufPool p.loops[i].decode(d) p.numVertices += len(p.loops[i].vertices) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 96401c4dae..5e0965c39b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -157,7 +157,7 @@ github.com/bitly/go-simplejson # github.com/bits-and-blooms/bitset v1.2.1 ## explicit; go 1.14 github.com/bits-and-blooms/bitset -# github.com/blevesearch/bleve/v2 v2.3.9 +# github.com/blevesearch/bleve/v2 v2.3.10 ## explicit; go 1.19 github.com/blevesearch/bleve/v2 github.com/blevesearch/bleve/v2/analysis @@ -194,10 +194,11 @@ github.com/blevesearch/bleve/v2/search/query github.com/blevesearch/bleve/v2/search/scorer github.com/blevesearch/bleve/v2/search/searcher github.com/blevesearch/bleve/v2/size -# github.com/blevesearch/bleve_index_api v1.0.5 +github.com/blevesearch/bleve/v2/util +# github.com/blevesearch/bleve_index_api v1.0.6 ## explicit; go 1.18 github.com/blevesearch/bleve_index_api -# github.com/blevesearch/geo v0.1.17 +# github.com/blevesearch/geo v0.1.18 ## explicit; go 1.18 github.com/blevesearch/geo/geojson github.com/blevesearch/geo/s2 @@ -210,7 +211,7 @@ github.com/blevesearch/gtreap # github.com/blevesearch/mmap-go v1.0.4 ## explicit; go 1.13 github.com/blevesearch/mmap-go -# github.com/blevesearch/scorch_segment_api/v2 v2.1.5 +# github.com/blevesearch/scorch_segment_api/v2 v2.1.6 ## explicit; go 1.19 github.com/blevesearch/scorch_segment_api/v2 # github.com/blevesearch/segment v0.9.1 @@ -229,19 +230,19 @@ github.com/blevesearch/vellum github.com/blevesearch/vellum/levenshtein github.com/blevesearch/vellum/regexp github.com/blevesearch/vellum/utf8 -# github.com/blevesearch/zapx/v11 v11.3.9 +# github.com/blevesearch/zapx/v11 v11.3.10 ## explicit; go 1.19 github.com/blevesearch/zapx/v11 -# github.com/blevesearch/zapx/v12 v12.3.9 +# github.com/blevesearch/zapx/v12 v12.3.10 ## explicit; go 1.19 github.com/blevesearch/zapx/v12 -# github.com/blevesearch/zapx/v13 v13.3.9 +# github.com/blevesearch/zapx/v13 v13.3.10 ## explicit; go 1.19 github.com/blevesearch/zapx/v13 -# github.com/blevesearch/zapx/v14 v14.3.9 +# github.com/blevesearch/zapx/v14 v14.3.10 ## explicit; go 1.19 github.com/blevesearch/zapx/v14 -# github.com/blevesearch/zapx/v15 v15.3.12 +# github.com/blevesearch/zapx/v15 v15.3.13 ## explicit; go 1.19 github.com/blevesearch/zapx/v15 # github.com/bluele/gcache v0.0.2