mirror/opencloud
1
0 Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-10-06 10:52:08 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(search): add range aggregations

Browse Source 
Numeric and date ranges on both engines: bleve folds them from doc values, OpenSearch uses range and date_range aggregations. The proto gains BucketRange and bucketDefinition.ranges, the aggregation package the range parser (bounds are numbers or RFC3339 dates, one kind per aggregation, anything else is rejected), the kind of an option and the validation of ranges against the field type. Every requested range is answered in request order, an empty one with a count of zero, with no space answering too; size does not cut ranges, keyAsNumber sorts them by their lower bound. Range keys read from..to. AGG-04, 05 and 07 to 09 in the parity suite.
This commit is contained in:
Dominik Schmidt committed 2026-10-05 12:08:51 +02:00
1 parent 8f85f33423
commit 38ec0bcafa
20 files changed
+829 -186
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

protogen/gen/opencloud/services/search/v0/search.pb.go
+231 -140
View File
@@ -411,8 +411,9 @@ type AggregationOption struct {
// Required. The indexed field to aggregate on.
Field string `protobuf:"bytes,1,opt,name=field,proto3" json:"field,omitempty"`
// Optional. Maximum number of buckets to return, applied after the
// cross-space merge. 0 returns every bucket.
// Optional. Maximum number of buckets to return for a terms aggregation,
// applied after the cross-space merge. 0 returns every bucket. Ignored for
// range aggregations.
Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
// Optional. Controls bucket selection, ordering and filtering.
BucketDefinition *BucketDefinition `protobuf:"bytes,3,opt,name=bucket_definition,json=bucketDefinition,proto3" json:"bucket_definition,omitempty"`
@@ -481,6 +482,9 @@ type BucketDefinition struct {
SortBy BucketSortBy `protobuf:"varint,1,opt,name=sort_by,json=sortBy,proto3,enum=opencloud.services.search.v0.BucketSortBy" json:"sort_by,omitempty"`
IsDescending bool `protobuf:"varint,2,opt,name=is_descending,json=isDescending,proto3" json:"is_descending,omitempty"`
MinimumCount int32 `protobuf:"varint,3,opt,name=minimum_count,json=minimumCount,proto3" json:"minimum_count,omitempty"`
// Optional. When non-empty, the aggregation is computed over these numeric
// or date ranges instead of as a terms aggregation.
Ranges []*BucketRange `protobuf:"bytes,4,rep,name=ranges,proto3" json:"ranges,omitempty"`
}
func (x *BucketDefinition) Reset() {
@@ -536,6 +540,71 @@ func (x *BucketDefinition) GetMinimumCount() int32 {
return 0
}
func (x *BucketDefinition) GetRanges() []*BucketRange {
if x != nil {
return x.Ranges
}
return nil
}
// A range is from-inclusive and to-exclusive.
type BucketRange struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// At least one of `from` or `to` must be set. Both are string-encoded and
// of one kind per aggregation: decimal numbers or RFC3339 dates.
From string `protobuf:"bytes,1,opt,name=from,proto3" json:"from,omitempty"`
To string `protobuf:"bytes,2,opt,name=to,proto3" json:"to,omitempty"`
}
func (x *BucketRange) Reset() {
*x = BucketRange{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *BucketRange) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*BucketRange) ProtoMessage() {}
func (x *BucketRange) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[6]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use BucketRange.ProtoReflect.Descriptor instead.
func (*BucketRange) Descriptor() ([]byte, []int) {
return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{6}
}
func (x *BucketRange) GetFrom() string {
if x != nil {
return x.From
}
return ""
}
func (x *BucketRange) GetTo() string {
if x != nil {
return x.To
}
return ""
}
// One result per AggregationOption, at the same position.
type AggregationResult struct {
state protoimpl.MessageState
@@ -549,7 +618,7 @@ type AggregationResult struct {
func (x *AggregationResult) Reset() {
*x = AggregationResult{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[6]
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -562,7 +631,7 @@ func (x *AggregationResult) String() string {
func (*AggregationResult) ProtoMessage() {}
func (x *AggregationResult) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[6]
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[7]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -575,7 +644,7 @@ func (x *AggregationResult) ProtoReflect() protoreflect.Message {
// Deprecated: Use AggregationResult.ProtoReflect.Descriptor instead.
func (*AggregationResult) Descriptor() ([]byte, []int) {
return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{6}
return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{7}
}
func (x *AggregationResult) GetField() string {
@@ -604,7 +673,7 @@ type Bucket struct {
func (x *Bucket) Reset() {
*x = Bucket{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[7]
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -617,7 +686,7 @@ func (x *Bucket) String() string {
func (*Bucket) ProtoMessage() {}
func (x *Bucket) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[7]
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[8]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -630,7 +699,7 @@ func (x *Bucket) ProtoReflect() protoreflect.Message {
// Deprecated: Use Bucket.ProtoReflect.Descriptor instead.
func (*Bucket) Descriptor() ([]byte, []int) {
return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{7}
return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{8}
}
func (x *Bucket) GetKey() string {
@@ -661,7 +730,7 @@ type IndexSpaceRequest struct {
func (x *IndexSpaceRequest) Reset() {
*x = IndexSpaceRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[8]
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -674,7 +743,7 @@ func (x *IndexSpaceRequest) String() string {
func (*IndexSpaceRequest) ProtoMessage() {}
func (x *IndexSpaceRequest) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[8]
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[9]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -687,7 +756,7 @@ func (x *IndexSpaceRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use IndexSpaceRequest.ProtoReflect.Descriptor instead.
func (*IndexSpaceRequest) Descriptor() ([]byte, []int) {
return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{8}
return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{9}
}
func (x *IndexSpaceRequest) GetSpaceId() string {
@@ -738,7 +807,7 @@ type IndexSpaceResponse struct {
func (x *IndexSpaceResponse) Reset() {
*x = IndexSpaceResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[9]
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -751,7 +820,7 @@ func (x *IndexSpaceResponse) String() string {
func (*IndexSpaceResponse) ProtoMessage() {}
func (x *IndexSpaceResponse) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[9]
mi := &file_opencloud_services_search_v0_search_proto_msgTypes[10]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -764,7 +833,7 @@ func (x *IndexSpaceResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use IndexSpaceResponse.ProtoReflect.Descriptor instead.
func (*IndexSpaceResponse) Descriptor() ([]byte, []int) {
return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{9}
return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{10}
}
func (x *IndexSpaceResponse) GetSpaceId() string {
@@ -902,7 +971,7 @@ var file_opencloud_services_search_v0_search_proto_rawDesc = []byte{
0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e,
0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e,
0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x10, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x44, 0x65,
0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xa1, 0x01, 0x0a, 0x10, 0x42, 0x75, 0x63,
0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xe4, 0x01, 0x0a, 0x10, 0x42, 0x75, 0x63,
0x6b, 0x65, 0x74, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x43, 0x0a,
0x07, 0x73, 0x6f, 0x72, 0x74, 0x5f, 0x62, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a,
0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
@@ -912,102 +981,110 @@ var file_opencloud_services_search_v0_search_proto_rawDesc = []byte{
0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x73, 0x44, 0x65, 0x73,
0x63, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x69, 0x6e, 0x69, 0x6d,
0x75, 0x6d, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c,
0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x69, 0x0a, 0x11,
0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c,
0x74, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x3e, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65,
0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63,
0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65,
0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x07,
0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, 0x30, 0x0a, 0x06, 0x42, 0x75, 0x63, 0x6b, 0x65,
0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01,
0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x11, 0x49, 0x6e,
0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x19, 0x0a, 0x08, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x07, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73,
0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65,
0x72, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x69,
0x6e, 0x64, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x66, 0x6f, 0x72, 0x63,
0x65, 0x52, 0x65, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x26, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63,
0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2,
0x41, 0x01, 0x01, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79,
0x22, 0xd1, 0x01, 0x0a, 0x12, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x70, 0x61, 0x63, 0x65,
0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x70, 0x61, 0x63, 0x65,
0x49, 0x64, 0x12, 0x40, 0x0a, 0x0e, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x44, 0x75, 0x72, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x64, 0x5f,
0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x69, 0x6e,
0x64, 0x65, 0x78, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74,
0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28,
0x03, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x14,
0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65,
0x72, 0x72, 0x6f, 0x72, 0x2a, 0x8c, 0x01, 0x0a, 0x0c, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53,
0x6f, 0x72, 0x74, 0x42, 0x79, 0x12, 0x1e, 0x0a, 0x1a, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f,
0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46,
0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f,
0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x10, 0x01, 0x12,
0x20, 0x0a, 0x1c, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42,
0x59, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x41, 0x53, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10,
0x02, 0x12, 0x20, 0x0a, 0x1c, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54,
0x5f, 0x42, 0x59, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x41, 0x53, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45,
0x52, 0x10, 0x03, 0x32, 0xb3, 0x02, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x50, 0x72,
0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x85, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63,
0x68, 0x12, 0x2b, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30,
0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c,
0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x41, 0x0a, 0x06,
0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f,
0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x42, 0x75, 0x63, 0x6b,
0x65, 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22,
0x31, 0x0a, 0x0b, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x12,
0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x72,
0x6f, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x74, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02,
0x74, 0x6f, 0x22, 0x69, 0x0a, 0x11, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x3e, 0x0a,
0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24,
0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x42, 0x75,
0x63, 0x6b, 0x65, 0x74, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, 0x30, 0x0a,
0x06, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75,
0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22,
0x94, 0x01, 0x0a, 0x11, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69,
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64,
0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x6f, 0x72,
0x63, 0x65, 0x5f, 0x72, 0x65, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08,
0x52, 0x0c, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x52, 0x65, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x26,
0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x04, 0x20,
0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75,
0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x22, 0xd1, 0x01, 0x0a, 0x12, 0x49, 0x6e, 0x64, 0x65, 0x78,
0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a,
0x08, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x07, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x40, 0x0a, 0x0e, 0x73, 0x70, 0x61, 0x63,
0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x73, 0x70, 0x61,
0x63, 0x65, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e,
0x64, 0x65, 0x78, 0x65, 0x64, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01,
0x28, 0x03, 0x52, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65,
0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65,
0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x70,
0x61, 0x63, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20,
0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2a, 0x8c, 0x01, 0x0a, 0x0c, 0x42,
0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x6f, 0x72, 0x74, 0x42, 0x79, 0x12, 0x1e, 0x0a, 0x1a, 0x42,
0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x55, 0x4e,
0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x42,
0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x43, 0x4f,
0x55, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x20, 0x0a, 0x1c, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f,
0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x41, 0x53, 0x5f, 0x53,
0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x20, 0x0a, 0x1c, 0x42, 0x55, 0x43, 0x4b, 0x45,
0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x41, 0x53,
0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x03, 0x32, 0xb3, 0x02, 0x0a, 0x0e, 0x53, 0x65,
0x61, 0x72, 0x63, 0x68, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x85, 0x01, 0x0a,
0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x2b, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61,
0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x22, 0x20, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1a, 0x3a, 0x01, 0x2a, 0x22, 0x15, 0x2f,
0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x73, 0x65,
0x61, 0x72, 0x63, 0x68, 0x12, 0x98, 0x01, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70,
0x61, 0x63, 0x65, 0x12, 0x2f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e,
0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x25, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1f, 0x3a, 0x01,
0x2a, 0x22, 0x1a, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63,
0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x30, 0x01, 0x32,
0xa7, 0x01, 0x0a, 0x0d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65,
0x72, 0x12, 0x95, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x30, 0x2e, 0x6f,
0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72,
0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31,
0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65,
0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x20, 0x82, 0xd3,
0xe4, 0x93, 0x02, 0x1a, 0x3a, 0x01, 0x2a, 0x22, 0x15, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30,
0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x98,
0x01, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x12, 0x2f, 0x2e,
0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64,
0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30,
0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e,
0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x22, 0x25, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1f, 0x3a, 0x01, 0x2a, 0x22, 0x1a, 0x2f, 0x61, 0x70,
0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65,
0x78, 0x2d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x30, 0x01, 0x32, 0xa7, 0x01, 0x0a, 0x0d, 0x49, 0x6e,
0x64, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x95, 0x01, 0x0a, 0x06,
0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x30, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f,
0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72,
0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65,
0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63,
0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65,
0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e,
0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4,
0x93, 0x02, 0x20, 0x3a, 0x01, 0x2a, 0x22, 0x1b, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f,
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x73, 0x65, 0x61,
0x72, 0x63, 0x68, 0x42, 0xf2, 0x02, 0x5a, 0x4a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f,
0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67,
0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f,
0x76, 0x30, 0x92, 0x41, 0xa2, 0x02, 0x12, 0xb7, 0x01, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x6e, 0x43,
0x6c, 0x6f, 0x75, 0x64, 0x20, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x22, 0x51, 0x0a, 0x0e, 0x4f,
0x70, 0x65, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, 0x48, 0x12, 0x29, 0x68,
0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f,
0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x1a, 0x14, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72,
0x74, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x65, 0x75, 0x2a, 0x49,
0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2d, 0x32, 0x2e, 0x30, 0x12, 0x3b, 0x68, 0x74,
0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70,
0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x69,
0x6e, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, 0x2e, 0x30, 0x2e, 0x30,
0x2a, 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3e, 0x0a, 0x10, 0x44, 0x65, 0x76, 0x65,
0x6c, 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x12, 0x2a, 0x68, 0x74,
0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x64, 0x6f, 0x63, 0x73, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63,
0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x65, 0x75, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x20, 0x3a, 0x01, 0x2a, 0x22, 0x1b, 0x2f, 0x61,
0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64,
0x65, 0x78, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x42, 0xf2, 0x02, 0x5a, 0x4a, 0x67, 0x69,
0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f,
0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70, 0x65,
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x73,
0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x92, 0x41, 0xa2, 0x02, 0x12, 0xb7, 0x01, 0x0a,
0x10, 0x4f, 0x70, 0x65, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x73, 0x65, 0x61, 0x72, 0x63,
0x68, 0x22, 0x51, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47,
0x6d, 0x62, 0x48, 0x12, 0x29, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74,
0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75,
0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x1a, 0x14,
0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75,
0x64, 0x2e, 0x65, 0x75, 0x2a, 0x49, 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2d, 0x32,
0x2e, 0x30, 0x12, 0x3b, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68,
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x62, 0x6c,
0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32,
0x05, 0x31, 0x2e, 0x30, 0x2e, 0x30, 0x2a, 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c,
0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70,
0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3e,
0x0a, 0x10, 0x44, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75,
0x61, 0x6c, 0x12, 0x2a, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x64, 0x6f, 0x63, 0x73,
0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x65, 0x75, 0x2f, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -1023,7 +1100,7 @@ func file_opencloud_services_search_v0_search_proto_rawDescGZIP() []byte {
}
var file_opencloud_services_search_v0_search_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_opencloud_services_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
var file_opencloud_services_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 11)
var file_opencloud_services_search_v0_search_proto_goTypes = []interface{}{
(BucketSortBy)(0), // 0: opencloud.services.search.v0.BucketSortBy
(*SearchRequest)(nil), // 1: opencloud.services.search.v0.SearchRequest
@@ -1032,38 +1109,40 @@ var file_opencloud_services_search_v0_search_proto_goTypes = []interface{}{
(*SearchIndexResponse)(nil), // 4: opencloud.services.search.v0.SearchIndexResponse
(*AggregationOption)(nil), // 5: opencloud.services.search.v0.AggregationOption
(*BucketDefinition)(nil), // 6: opencloud.services.search.v0.BucketDefinition
(*AggregationResult)(nil), // 7: opencloud.services.search.v0.AggregationResult
(*Bucket)(nil), // 8: opencloud.services.search.v0.Bucket
(*IndexSpaceRequest)(nil), // 9: opencloud.services.search.v0.IndexSpaceRequest
(*IndexSpaceResponse)(nil), // 10: opencloud.services.search.v0.IndexSpaceResponse
(*v0.Reference)(nil), // 11: opencloud.messages.search.v0.Reference
(*v0.Match)(nil), // 12: opencloud.messages.search.v0.Match
(*durationpb.Duration)(nil), // 13: google.protobuf.Duration
(*BucketRange)(nil), // 7: opencloud.services.search.v0.BucketRange
(*AggregationResult)(nil), // 8: opencloud.services.search.v0.AggregationResult
(*Bucket)(nil), // 9: opencloud.services.search.v0.Bucket
(*IndexSpaceRequest)(nil), // 10: opencloud.services.search.v0.IndexSpaceRequest
(*IndexSpaceResponse)(nil), // 11: opencloud.services.search.v0.IndexSpaceResponse
(*v0.Reference)(nil), // 12: opencloud.messages.search.v0.Reference
(*v0.Match)(nil), // 13: opencloud.messages.search.v0.Match
(*durationpb.Duration)(nil), // 14: google.protobuf.Duration
}
var file_opencloud_services_search_v0_search_proto_depIdxs = []int32{
11, // 0: opencloud.services.search.v0.SearchRequest.ref:type_name -> opencloud.messages.search.v0.Reference
12, // 0: opencloud.services.search.v0.SearchRequest.ref:type_name -> opencloud.messages.search.v0.Reference
5, // 1: opencloud.services.search.v0.SearchRequest.aggregations:type_name -> opencloud.services.search.v0.AggregationOption
12, // 2: opencloud.services.search.v0.SearchResponse.matches:type_name -> opencloud.messages.search.v0.Match
7, // 3: opencloud.services.search.v0.SearchResponse.aggregations:type_name -> opencloud.services.search.v0.AggregationResult
11, // 4: opencloud.services.search.v0.SearchIndexRequest.ref:type_name -> opencloud.messages.search.v0.Reference
13, // 2: opencloud.services.search.v0.SearchResponse.matches:type_name -> opencloud.messages.search.v0.Match
8, // 3: opencloud.services.search.v0.SearchResponse.aggregations:type_name -> opencloud.services.search.v0.AggregationResult
12, // 4: opencloud.services.search.v0.SearchIndexRequest.ref:type_name -> opencloud.messages.search.v0.Reference
5, // 5: opencloud.services.search.v0.SearchIndexRequest.aggregations:type_name -> opencloud.services.search.v0.AggregationOption
12, // 6: opencloud.services.search.v0.SearchIndexResponse.matches:type_name -> opencloud.messages.search.v0.Match
7, // 7: opencloud.services.search.v0.SearchIndexResponse.aggregations:type_name -> opencloud.services.search.v0.AggregationResult
13, // 6: opencloud.services.search.v0.SearchIndexResponse.matches:type_name -> opencloud.messages.search.v0.Match
8, // 7: opencloud.services.search.v0.SearchIndexResponse.aggregations:type_name -> opencloud.services.search.v0.AggregationResult
6, // 8: opencloud.services.search.v0.AggregationOption.bucket_definition:type_name -> opencloud.services.search.v0.BucketDefinition
0, // 9: opencloud.services.search.v0.BucketDefinition.sort_by:type_name -> opencloud.services.search.v0.BucketSortBy
8, // 10: opencloud.services.search.v0.AggregationResult.buckets:type_name -> opencloud.services.search.v0.Bucket
13, // 11: opencloud.services.search.v0.IndexSpaceResponse.space_duration:type_name -> google.protobuf.Duration
1, // 12: opencloud.services.search.v0.SearchProvider.Search:input_type -> opencloud.services.search.v0.SearchRequest
9, // 13: opencloud.services.search.v0.SearchProvider.IndexSpace:input_type -> opencloud.services.search.v0.IndexSpaceRequest
3, // 14: opencloud.services.search.v0.IndexProvider.Search:input_type -> opencloud.services.search.v0.SearchIndexRequest
2, // 15: opencloud.services.search.v0.SearchProvider.Search:output_type -> opencloud.services.search.v0.SearchResponse
10, // 16: opencloud.services.search.v0.SearchProvider.IndexSpace:output_type -> opencloud.services.search.v0.IndexSpaceResponse
4, // 17: opencloud.services.search.v0.IndexProvider.Search:output_type -> opencloud.services.search.v0.SearchIndexResponse
15, // [15:18] is the sub-list for method output_type
12, // [12:15] is the sub-list for method input_type
12, // [12:12] is the sub-list for extension type_name
12, // [12:12] is the sub-list for extension extendee
0, // [0:12] is the sub-list for field type_name
7, // 10: opencloud.services.search.v0.BucketDefinition.ranges:type_name -> opencloud.services.search.v0.BucketRange
9, // 11: opencloud.services.search.v0.AggregationResult.buckets:type_name -> opencloud.services.search.v0.Bucket
14, // 12: opencloud.services.search.v0.IndexSpaceResponse.space_duration:type_name -> google.protobuf.Duration
1, // 13: opencloud.services.search.v0.SearchProvider.Search:input_type -> opencloud.services.search.v0.SearchRequest
10, // 14: opencloud.services.search.v0.SearchProvider.IndexSpace:input_type -> opencloud.services.search.v0.IndexSpaceRequest
3, // 15: opencloud.services.search.v0.IndexProvider.Search:input_type -> opencloud.services.search.v0.SearchIndexRequest
2, // 16: opencloud.services.search.v0.SearchProvider.Search:output_type -> opencloud.services.search.v0.SearchResponse
11, // 17: opencloud.services.search.v0.SearchProvider.IndexSpace:output_type -> opencloud.services.search.v0.IndexSpaceResponse
4, // 18: opencloud.services.search.v0.IndexProvider.Search:output_type -> opencloud.services.search.v0.SearchIndexResponse
16, // [16:19] is the sub-list for method output_type
13, // [13:16] is the sub-list for method input_type
13, // [13:13] is the sub-list for extension type_name
13, // [13:13] is the sub-list for extension extendee
0, // [0:13] is the sub-list for field type_name
}
func init() { file_opencloud_services_search_v0_search_proto_init() }
@@ -1145,7 +1224,7 @@ func file_opencloud_services_search_v0_search_proto_init() {
}
}
file_opencloud_services_search_v0_search_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*AggregationResult); i {
switch v := v.(*BucketRange); i {
case 0:
return &v.state
case 1:
@@ -1157,7 +1236,7 @@ func file_opencloud_services_search_v0_search_proto_init() {
}
}
file_opencloud_services_search_v0_search_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Bucket); i {
switch v := v.(*AggregationResult); i {
case 0:
return &v.state
case 1:
@@ -1169,7 +1248,7 @@ func file_opencloud_services_search_v0_search_proto_init() {
}
}
file_opencloud_services_search_v0_search_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*IndexSpaceRequest); i {
switch v := v.(*Bucket); i {
case 0:
return &v.state
case 1:
@@ -1181,6 +1260,18 @@ func file_opencloud_services_search_v0_search_proto_init() {
}
}
file_opencloud_services_search_v0_search_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*IndexSpaceRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_opencloud_services_search_v0_search_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*IndexSpaceResponse); i {
case 0:
return &v.state
@@ -1201,7 +1292,7 @@ func file_opencloud_services_search_v0_search_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_opencloud_services_search_v0_search_proto_rawDesc,
NumEnums: 1,
NumMessages: 10,
NumMessages: 11,
NumExtensions: 0,
NumServices: 2,
},
protogen/gen/opencloud/services/search/v0/search.swagger.json
+21 -1
View File
@@ -177,7 +177,7 @@
"size": {
"type": "integer",
"format": "int32",
"description": "Optional. Maximum number of buckets to return, applied after the\ncross-space merge. 0 returns every bucket."
"description": "Optional. Maximum number of buckets to return for a terms aggregation,\napplied after the cross-space merge. 0 returns every bucket. Ignored for\nrange aggregations."
},
"bucketDefinition": {
"$ref": "#/definitions/v0BucketDefinition",
@@ -285,9 +285,29 @@
"minimumCount": {
"type": "integer",
"format": "int32"
},
"ranges": {
"type": "array",
"items": {
"$ref": "#/definitions/v0BucketRange"
},
"description": "Optional. When non-empty, the aggregation is computed over these numeric\nor date ranges instead of as a terms aggregation."
}
}
},
"v0BucketRange": {
"type": "object",
"properties": {
"from": {
"type": "string",
"description": "At least one of `from` or `to` must be set. Both are string-encoded and\nof one kind per aggregation: decimal numbers or RFC3339 dates."
},
"to": {
"type": "string"
}
},
"description": "A range is from-inclusive and to-exclusive."
},
"v0BucketSortBy": {
"type": "string",
"enum": [
protogen/proto/opencloud/services/search/v0/search.proto
+14 -2
View File
@@ -126,8 +126,9 @@ message SearchIndexResponse {
message AggregationOption {
// Required. The indexed field to aggregate on.
string field = 1;
// Optional. Maximum number of buckets to return, applied after the
// cross-space merge. 0 returns every bucket.
// Optional. Maximum number of buckets to return for a terms aggregation,
// applied after the cross-space merge. 0 returns every bucket. Ignored for
// range aggregations.
int32 size = 2 [(google.api.field_behavior) = OPTIONAL];
// Optional. Controls bucket selection, ordering and filtering.
BucketDefinition bucket_definition = 3 [(google.api.field_behavior) = OPTIONAL];
@@ -139,6 +140,9 @@ message BucketDefinition {
BucketSortBy sort_by = 1;
bool is_descending = 2;
int32 minimum_count = 3;
// Optional. When non-empty, the aggregation is computed over these numeric
// or date ranges instead of as a terms aggregation.
repeated BucketRange ranges = 4;
}
enum BucketSortBy {
@@ -148,6 +152,14 @@ enum BucketSortBy {
BUCKET_SORT_BY_KEY_AS_NUMBER = 3;
}
// A range is from-inclusive and to-exclusive.
message BucketRange {
// At least one of `from` or `to` must be set. Both are string-encoded and
// of one kind per aggregation: decimal numbers or RFC3339 dates.
string from = 1;
string to = 2;
}
// One result per AggregationOption, at the same position.
message AggregationResult {
string field = 1;
services/graph/pkg/service/v0/searchquery.go
-2
View File
@@ -100,8 +100,6 @@ func unsupportedProperty(sr libregraph.SearchRequest) string {
return "@libre.graph.subAggregations"
case a.LibreGraphMetricDefinition != nil:
return "@libre.graph.metricDefinition"
case len(a.BucketDefinition.GetRanges()) > 0:
return "bucketDefinition.ranges"
}
}
return ""
services/graph/pkg/service/v0/searchquery_mapping.go
+12 -1
View File
@@ -67,7 +67,10 @@ func libregraphAggregationsToSearch(in []libregraph.AggregationOption) []*search
}
func libregraphBucketDefinitionToSearch(in libregraph.BucketDefinition) *searchsvc.BucketDefinition {
bd := &searchsvc.BucketDefinition{SortBy: bucketSortBy[in.SortBy]}
bd := &searchsvc.BucketDefinition{
SortBy: bucketSortBy[in.SortBy],
Ranges: libregraphRangesToSearch(in.Ranges),
}
if in.IsDescending != nil {
bd.IsDescending = *in.IsDescending
}
@@ -77,6 +80,14 @@ func libregraphBucketDefinitionToSearch(in libregraph.BucketDefinition) *searchs
return bd
}
func libregraphRangesToSearch(in []libregraph.BucketAggregationRange) []*searchsvc.BucketRange {
out := make([]*searchsvc.BucketRange, 0, len(in))
for _, r := range in {
out = append(out, &searchsvc.BucketRange{From: r.GetFrom(), To: r.GetTo()})
}
return out
}
// searchAggregationsToLibregraph maps the results to their definitions by
// position: the search service answers one result per aggregation, in request
// order.
services/graph/pkg/service/v0/searchquery_test.go
+15 -1
View File
@@ -303,18 +303,21 @@ var _ = ginkgo.Describe("SearchQuery", func() {
ginkgo.Entry("a size below one", `"aggregations": [{"field": "audio.artist", "size": 0}]`),
ginkgo.Entry("a negative minimumCount", `"aggregations": [{"field": "audio.artist", "bucketDefinition": {"sortBy": "count", "minimumCount": -1}}]`),
ginkgo.Entry("a field the index does not know", `"aggregations": [{"field": "audio.nonexistent"}]`),
ginkgo.Entry("a range bound that is no number", `"aggregations": [{"field": "audio.year", "bucketDefinition": {"sortBy": "count", "ranges": [{"from": "1970", "to": "198o"}]}}]`),
)
ginkgo.It("translates the bucket definition for the search service", func() {
g, captured := graphWithAggregations()
rr := postSearchQuery(g, searchQueryBody(`"aggregations": [
{"field": "audio.artist", "size": 5, "bucketDefinition": {"sortBy": "keyAsString", "isDescending": true, "minimumCount": 2}},
{"field": "audio.year", "bucketDefinition": {"sortBy": "keyAsNumber", "ranges": [{"to": "1980"}, {"from": "1980", "to": "1990"}]}},
{"field": "audio.genre", "bucketDefinition": {"sortBy": "count"}}
]`))
Expect(rr.Code).To(Equal(http.StatusOK), rr.Body.String())
Expect(captured().GetAggregations()).To(BeComparableTo([]*searchsvc.AggregationOption{
{Field: "audio.artist", Size: 5, BucketDefinition: &searchsvc.BucketDefinition{SortBy: searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_STRING, IsDescending: true, MinimumCount: 2}},
{Field: "audio.year", BucketDefinition: &searchsvc.BucketDefinition{SortBy: searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, Ranges: []*searchsvc.BucketRange{{To: "1980"}, {From: "1980", To: "1990"}}}},
{Field: "audio.genre", BucketDefinition: &searchsvc.BucketDefinition{SortBy: searchsvc.BucketSortBy_BUCKET_SORT_BY_COUNT}},
}, protocmp.Transform()))
})
@@ -353,6 +356,18 @@ var _ = ginkgo.Describe("SearchQuery", func() {
ginkgo.Entry("a property the index rules refuse for terms", "lastModifiedDateTime"),
)
ginkgo.It("resolves lastModifiedDateTime to the modification time of the index", func() {
g, captured := graphWithAggregations(&searchsvc.AggregationResult{
Field: "Mtime",
Buckets: []*searchsvc.Bucket{{Key: "2026-01-01T00:00:00Z..", Count: 3}},
})
rr := postSearchQuery(g, searchQueryBody(`"aggregations": [{"field": "lastModifiedDateTime", "bucketDefinition": {"sortBy": "count", "ranges": [{"from": "2026-01-01T00:00:00Z"}]}}]`))
Expect(rr.Code).To(Equal(http.StatusOK), rr.Body.String())
Expect(captured().GetAggregations()[0].GetField()).To(Equal("Mtime"))
Expect(hitsContainer(rr).Aggregations[0].Field).To(Equal("lastModifiedDateTime"))
})
ginkgo.DescribeTable("answers a property it does not evaluate with 501 instead of ignoring it",
func(fragment, property string) {
rr := postSearchQuery(graphWithoutSearch(), searchQueryBody(fragment))
@@ -364,7 +379,6 @@ var _ = ginkgo.Describe("SearchQuery", func() {
ginkgo.Entry("aggregationFilters", `"aggregationFilters": ["audio.artist:\"ǂǂ5361786f6e\""]`, "aggregationFilters"),
ginkgo.Entry("a geohash aggregation",
`"aggregations": [{"field": "location", "@libre.graph.geohashDefinition": {"precision": 5}}]`, "geohashDefinition"),
ginkgo.Entry("ranges", `"aggregations": [{"field": "audio.year", "bucketDefinition": {"sortBy": "count", "ranges": [{"from": "1980"}]}}]`, "ranges"),
ginkgo.Entry("a metric", `"aggregations": [{"field": "audio.year", "@libre.graph.metricDefinition": {"kind": "sum"}}]`, "metricDefinition"),
ginkgo.Entry("sub-aggregations", `"aggregations": [{"field": "audio.artist", "@libre.graph.subAggregations": [{"field": "audio.album"}]}]`, "subAggregations"),
)
services/search/pkg/aggregation/aggregation.go
+18 -2
View File
@@ -1,6 +1,6 @@
// Package aggregation holds what the engines and the service layer share
// about aggregations: the bucket limit, the spelling of keys and the
// cross-space merge.
// about aggregations: the kind of an option, range bounds and the cross-space
// merge.
package aggregation
import (
@@ -11,6 +11,22 @@ import (
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
)
// Kind is what an aggregation option asks for.
type Kind int
const (
KindTerms Kind = iota
KindRange
)
// KindOf tells the kind from the definitions an option carries.
func KindOf(opt *searchsvc.AggregationOption) Kind {
if len(opt.GetBucketDefinition().GetRanges()) > 0 {
return KindRange
}
return KindTerms
}
// MaxBuckets is how many buckets the aggregations of one request may have in
// one space: the search.max_buckets default of OpenSearch, held on every
// engine.
services/search/pkg/aggregation/merge.go
+55 -7
View File
@@ -2,13 +2,15 @@ package aggregation
import (
"cmp"
"math"
"slices"
"strconv"
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
)
// Empty returns the results of a search without matches: one per option.
// Empty returns the results of a search without matches: one per option, a
// range aggregation with every range at a count of zero.
func Empty(opts []*searchsvc.AggregationOption) []*searchsvc.AggregationResult {
if len(opts) == 0 {
return nil
@@ -16,6 +18,24 @@ func Empty(opts []*searchsvc.AggregationOption) []*searchsvc.AggregationResult {
out := make([]*searchsvc.AggregationResult, len(opts))
for i, opt := range opts {
out[i] = &searchsvc.AggregationResult{Field: opt.GetField()}
if KindOf(opt) == KindRange {
out[i].Buckets = RangeBuckets(opt, nil)
}
}
return out
}
// RangeBuckets lists every range of a range aggregation in request order: the
// answered bucket where there is one, a count of zero otherwise.
func RangeBuckets(opt *searchsvc.AggregationOption, answered map[string]*searchsvc.Bucket) []*searchsvc.Bucket {
ranges := opt.GetBucketDefinition().GetRanges()
out := make([]*searchsvc.Bucket, 0, len(ranges))
for _, r := range ranges {
bucket, ok := answered[RangeKey(r)]
if !ok {
bucket = &searchsvc.Bucket{Key: RangeKey(r)}
}
out = append(out, bucket)
}
return out
}
@@ -64,7 +84,9 @@ func shapeBuckets(opt *searchsvc.AggregationOption, buckets []*searchsvc.Bucket)
slices.SortFunc(buckets, bucketOrder(opt))
if size := int(opt.GetSize()); size > 0 && len(buckets) > size {
// size shapes terms aggregations only, a range aggregation returns the
// ranges it was asked for
if size := int(opt.GetSize()); size > 0 && KindOf(opt) == KindTerms && len(buckets) > size {
buckets = buckets[:size]
}
return buckets
@@ -88,9 +110,10 @@ func bucketOrder(opt *searchsvc.AggregationOption) func(a, b *searchsvc.Bucket)
return func(a, b *searchsvc.Bucket) int { return direction * byKey(a, b) }
case searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER:
// keys that are no number come last, whatever the direction
number := keyNumbers(opt)
return func(a, b *searchsvc.Bucket) int {
av, aok := keyNumber(a.GetKey())
bv, bok := keyNumber(b.GetKey())
av, aok := number(a.GetKey())
bv, bok := number(b.GetKey())
return cmp.Or(cmp.Compare(boolInt(!aok), boolInt(!bok)), direction*cmp.Compare(av, bv), byKey(a, b))
}
}
@@ -106,7 +129,32 @@ func boolInt(b bool) int {
return 0
}
func keyNumber(key string) (float64, bool) {
v, err := strconv.ParseFloat(key, 64)
return v, err == nil
// keyNumbers reads a terms key as a number and a range key as its lower
// bound, an open one as the smallest.
func keyNumbers(opt *searchsvc.AggregationOption) func(key string) (float64, bool) {
if KindOf(opt) != KindRange {
return func(key string) (float64, bool) {
v, err := strconv.ParseFloat(key, 64)
return v, err == nil
}
}
lower := map[string]float64{}
if ranges, err := ParseRanges(opt.GetField(), opt.GetBucketDefinition().GetRanges()); err == nil {
for _, r := range ranges.Numeric {
lower[r.Key] = math.Inf(-1)
if r.From != nil {
lower[r.Key] = *r.From
}
}
for _, r := range ranges.Dates {
lower[r.Key] = math.Inf(-1)
if !r.From.IsZero() {
lower[r.Key] = float64(r.From.UnixNano())
}
}
}
return func(key string) (float64, bool) {
v, ok := lower[key]
return v, ok
}
}
services/search/pkg/aggregation/merge_test.go
+52 -3
View File
@@ -18,6 +18,10 @@ func bucket(key string, count int64) *searchsvc.Bucket {
return &searchsvc.Bucket{Key: key, Count: count}
}
func rangesOf(sortBy searchsvc.BucketSortBy, rs ...*searchsvc.BucketRange) *searchsvc.BucketDefinition {
return &searchsvc.BucketDefinition{SortBy: sortBy, Ranges: rs}
}
// render flattens results like the parity suite: one line per bucket.
func render(results []*searchsvc.AggregationResult) []string {
out := []string{}
@@ -40,13 +44,16 @@ func merged(opts []*searchsvc.AggregationOption, spaces ...[]*searchsvc.Aggregat
}
var _ = Describe("Merge and Finalize", func() {
It("answers one result per option without any space", func() {
opts := []*searchsvc.AggregationOption{{Field: "audio.artist"}, {Field: "audio.album"}}
It("answers one result per option without any space, every range at zero", func() {
opts := []*searchsvc.AggregationOption{
{Field: "audio.artist"},
{Field: "audio.year", BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, &searchsvc.BucketRange{To: "1980"}, &searchsvc.BucketRange{From: "1980"})},
}
got := merged(opts)
Expect(got).To(HaveLen(2))
Expect(got[0].GetField()).To(Equal("audio.artist"))
Expect(got[0].GetBuckets()).To(BeEmpty())
Expect(got[1].GetField()).To(Equal("audio.album"))
Expect(render(got)).To(Equal([]string{"audio.year ..1980=0", "audio.year 1980..=0"}))
})
It("adds up the counts of a bucket across spaces", func() {
@@ -77,6 +84,24 @@ var _ = Describe("Merge and Finalize", func() {
}))
})
It("keeps two range aggregations on one field apart", func() {
opts := []*searchsvc.AggregationOption{
{Field: "audio.year", BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, &searchsvc.BucketRange{To: "1980"}, &searchsvc.BucketRange{From: "1980"})},
{Field: "audio.year", BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, &searchsvc.BucketRange{To: "2000"}, &searchsvc.BucketRange{From: "2000"})},
}
space := func(below1980, from1980, below2000, from2000 int64) []*searchsvc.AggregationResult {
return []*searchsvc.AggregationResult{
buckets("audio.year", bucket("..1980", below1980), bucket("1980..", from1980)),
buckets("audio.year", bucket("..2000", below2000), bucket("2000..", from2000)),
}
}
got := merged(opts, space(1, 1, 2, 0), space(0, 2, 1, 1))
Expect(render(got)).To(Equal([]string{
"audio.year ..1980=1", "audio.year 1980..=3",
"audio.year ..2000=3", "audio.year 2000..=1",
}))
})
Describe("bucket order", func() {
space := []*searchsvc.AggregationResult{buckets("audio.track",
bucket("10", 2), bucket("9", 2), bucket("2", 5), bucket("b-side", 1),
@@ -108,6 +133,22 @@ var _ = Describe("Merge and Finalize", func() {
Entry("keyAsNumber ascending, no number last", searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, false, "2", "9", "10", "b-side"),
Entry("keyAsNumber descending, no number still last", searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, true, "10", "9", "2", "b-side"),
)
It("sorts range buckets by their lower bound with keyAsNumber, an open one first", func() {
opts := []*searchsvc.AggregationOption{{Field: "audio.year", BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER,
&searchsvc.BucketRange{From: "2000"}, &searchsvc.BucketRange{From: "990", To: "2000"}, &searchsvc.BucketRange{To: "990"},
)}}
got := merged(opts, []*searchsvc.AggregationResult{buckets("audio.year", bucket("2000..", 1), bucket("990..2000", 1), bucket("..990", 1))})
Expect(render(got)).To(Equal([]string{"audio.year ..990=1", "audio.year 990..2000=1", "audio.year 2000..=1"}))
})
It("sorts date range buckets by their lower bound with keyAsNumber", func() {
opts := []*searchsvc.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER,
&searchsvc.BucketRange{From: "2019-01-01T00:00:00Z"}, &searchsvc.BucketRange{To: "2019-01-01T00:00:00Z"},
)}}
got := merged(opts, []*searchsvc.AggregationResult{buckets("photo.takenDateTime", bucket("2019-01-01T00:00:00Z..", 1), bucket("..2019-01-01T00:00:00Z", 3))})
Expect(render(got)).To(Equal([]string{"photo.takenDateTime ..2019-01-01T00:00:00Z=3", "photo.takenDateTime 2019-01-01T00:00:00Z..=1"}))
})
})
Describe("bucket selection", func() {
@@ -120,6 +161,14 @@ var _ = Describe("Merge and Finalize", func() {
Expect(render(got)).To(Equal([]string{"audio.artist Saxon=4"}))
})
It("returns every range of a range aggregation, whatever the size", func() {
opts := []*searchsvc.AggregationOption{{Field: "audio.year", Size: 1, BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER,
&searchsvc.BucketRange{To: "1980"}, &searchsvc.BucketRange{From: "1980"},
)}}
got := merged(opts, []*searchsvc.AggregationResult{buckets("audio.year", bucket("..1980", 0), bucket("1980..", 3))})
Expect(render(got)).To(Equal([]string{"audio.year ..1980=0", "audio.year 1980..=3"}))
})
It("drops buckets below the minimum count, counted across spaces", func() {
opts := []*searchsvc.AggregationOption{{Field: "audio.artist", BucketDefinition: &searchsvc.BucketDefinition{MinimumCount: 2, IsDescending: true}}}
got := merged(opts,
services/search/pkg/aggregation/ranges.go
+101
View File
@@ -0,0 +1,101 @@
package aggregation
import (
"fmt"
"math"
"strconv"
"time"
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
)
// Ranges are the parsed ranges of one aggregation or filter, all numeric or
// all dates. A range is from-inclusive and to-exclusive.
type Ranges struct {
Numeric []NumericRange
Dates []DateRange
}
// NumericRange has a nil bound on an open side.
type NumericRange struct {
Key string
From, To *float64
}
// DateRange has a zero time on an open side.
type DateRange struct {
Key string
From, To time.Time
}
// Contains is from-inclusive and to-exclusive.
func (r NumericRange) Contains(v float64) bool {
return (r.From == nil || v >= *r.From) && (r.To == nil || v < *r.To)
}
// Contains is from-inclusive and to-exclusive.
func (r DateRange) Contains(t time.Time) bool {
return (r.From.IsZero() || !t.Before(r.From)) && (r.To.IsZero() || t.Before(r.To))
}
// RangeKey is the bucket key of a range, "from..to" with an open side left
// empty; the dots keep a negative bound readable.
func RangeKey(r *searchsvc.BucketRange) string {
return r.GetFrom() + ".." + r.GetTo()
}
// ParseRanges accepts decimal numbers or RFC3339 dates as bounds, one kind for
// all ranges, and at least one bound per range.
func ParseRanges(field string, ranges []*searchsvc.BucketRange) (Ranges, error) {
var out Ranges
seen := make(map[string]struct{}, len(ranges))
for _, r := range ranges {
key := RangeKey(r)
if r.GetFrom() == "" && r.GetTo() == "" {
return Ranges{}, fmt.Errorf("range on field %q needs a from or a to bound", field)
}
if _, ok := seen[key]; ok {
return Ranges{}, fmt.Errorf("duplicate range %q on field %q", key, field)
}
seen[key] = struct{}{}
from, fromErr := parseNumber(r.GetFrom())
to, toErr := parseNumber(r.GetTo())
if fromErr == nil && toErr == nil {
out.Numeric = append(out.Numeric, NumericRange{Key: key, From: from, To: to})
continue
}
start, startErr := parseDate(r.GetFrom())
end, endErr := parseDate(r.GetTo())
if startErr == nil && endErr == nil {
out.Dates = append(out.Dates, DateRange{Key: key, From: start, To: end})
continue
}
return Ranges{}, fmt.Errorf("invalid range %q on field %q: bounds are numbers or RFC3339 dates", key, field)
}
if len(out.Numeric) > 0 && len(out.Dates) > 0 {
return Ranges{}, fmt.Errorf("ranges on field %q mix numbers and dates", field)
}
return out, nil
}
func parseNumber(s string) (*float64, error) {
if s == "" {
return nil, nil
}
v, err := strconv.ParseFloat(s, 64)
if err != nil {
return nil, err
}
if math.IsNaN(v) || math.IsInf(v, 0) {
return nil, fmt.Errorf("%q is not a finite number", s)
}
return &v, nil
}
func parseDate(s string) (time.Time, error) {
if s == "" {
return time.Time{}, nil
}
return time.Parse(time.RFC3339, s)
}
services/search/pkg/aggregation/ranges_test.go
+65
View File
@@ -0,0 +1,65 @@
package aggregation_test
import (
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/aggregation"
)
var _ = Describe("ParseRanges", func() {
It("parses numeric ranges, an open side has no bound", func() {
got, err := aggregation.ParseRanges("audio.year", []*searchsvc.BucketRange{{To: "1980"}, {From: "1980", To: "1990.5"}, {From: "-3"}})
Expect(err).ToNot(HaveOccurred())
Expect(got.Dates).To(BeEmpty())
Expect(got.Numeric).To(HaveLen(3))
Expect(got.Numeric[0].Key).To(Equal("..1980"))
Expect(got.Numeric[0].From).To(BeNil())
Expect(*got.Numeric[0].To).To(Equal(1980.0))
Expect(got.Numeric[1].Key).To(Equal("1980..1990.5"))
Expect(*got.Numeric[1].To).To(Equal(1990.5))
Expect(got.Numeric[2].Key).To(Equal("-3.."))
Expect(*got.Numeric[2].From).To(Equal(-3.0))
Expect(got.Numeric[2].To).To(BeNil())
})
It("parses RFC3339 date ranges", func() {
got, err := aggregation.ParseRanges("photo.takenDateTime", []*searchsvc.BucketRange{{From: "2018-08-11T00:00:00Z", To: "2018-08-12T00:00:00+02:00"}, {From: "2019-01-01T00:00:00Z"}})
Expect(err).ToNot(HaveOccurred())
Expect(got.Numeric).To(BeEmpty())
Expect(got.Dates).To(HaveLen(2))
Expect(got.Dates[0].From).To(BeTemporally("==", time.Date(2018, 8, 11, 0, 0, 0, 0, time.UTC)))
Expect(got.Dates[0].To).To(BeTemporally("==", time.Date(2018, 8, 11, 22, 0, 0, 0, time.UTC)))
Expect(got.Dates[1].To.IsZero()).To(BeTrue())
})
It("is from-inclusive and to-exclusive", func() {
got, err := aggregation.ParseRanges("audio.year", []*searchsvc.BucketRange{{From: "1980", To: "1990"}})
Expect(err).ToNot(HaveOccurred())
r := got.Numeric[0]
Expect(r.Contains(1979.9)).To(BeFalse())
Expect(r.Contains(1980)).To(BeTrue())
Expect(r.Contains(1989.9)).To(BeTrue())
Expect(r.Contains(1990)).To(BeFalse())
})
DescribeTable("rejects",
func(ranges ...*searchsvc.BucketRange) {
_, err := aggregation.ParseRanges("field", ranges)
Expect(err).To(HaveOccurred())
},
Entry("a range without a bound", &searchsvc.BucketRange{}),
Entry("a bound that is no number", &searchsvc.BucketRange{From: "1970", To: "198o"}),
Entry("a bound that is no date", &searchsvc.BucketRange{From: "2018-08-11T00:00:00Z", To: "not-a-date"}),
Entry("a date without a time", &searchsvc.BucketRange{From: "2018-08-11"}),
Entry("a number and a date in one range", &searchsvc.BucketRange{From: "1970", To: "2018-08-11T00:00:00Z"}),
Entry("numeric and date ranges mixed", &searchsvc.BucketRange{From: "1970"}, &searchsvc.BucketRange{From: "2018-08-11T00:00:00Z"}),
Entry("the same range twice", &searchsvc.BucketRange{From: "1970"}, &searchsvc.BucketRange{From: "1970"}),
Entry("NaN", &searchsvc.BucketRange{From: "NaN"}),
Entry("infinity", &searchsvc.BucketRange{To: "Inf"}),
Entry("query syntax", &searchsvc.BucketRange{From: "1980 OR Name:*", To: "1990"}),
)
})
services/search/pkg/aggregation/validate.go
+26 -5
View File
@@ -8,7 +8,7 @@ import (
)
// ValidateOptions holds aggregations to what the index can answer: a field it
// knows, of a type whose values are buckets. fieldType returns the mapping
// knows, a kind that fits the type of the field. fieldType returns the mapping
// type of a field, empty for one a client may not aggregate on.
func ValidateOptions(opts []*searchsvc.AggregationOption, fieldType func(string) string) error {
for _, opt := range opts {
@@ -16,21 +16,42 @@ func ValidateOptions(opts []*searchsvc.AggregationOption, fieldType func(string)
if t == "" {
return fmt.Errorf("unknown aggregation field %q", field)
}
if err := validateTermsField(field, t); err != nil {
return err
switch KindOf(opt) {
case KindRange:
parsed, err := ParseRanges(field, opt.GetBucketDefinition().GetRanges())
if err != nil {
return err
}
if err := validateRangeField(field, t, parsed); err != nil {
return err
}
case KindTerms:
if err := validateTermsField(field, t); err != nil {
return err
}
}
}
return nil
}
// validateTermsField allows the types whose values are buckets as they are.
// The values of a date are instants, no bucket each.
// The values of a date are instants, they aggregate in ranges.
func validateTermsField(field, fieldType string) error {
switch fieldType {
case mapping.TypeKeyword, mapping.TypeBool, mapping.TypeNumeric:
return nil
case mapping.TypeDatetime:
return fmt.Errorf("terms aggregation is not supported on date field %q", field)
return fmt.Errorf("terms aggregation is not supported on date field %q; use bucketDefinition.ranges", field)
}
return fmt.Errorf("terms aggregation is not supported on field %q", field)
}
func validateRangeField(field, fieldType string, ranges Ranges) error {
if len(ranges.Numeric) > 0 && fieldType != mapping.TypeNumeric {
return fmt.Errorf("numeric ranges need a numeric field, %q is none", field)
}
if len(ranges.Dates) > 0 && fieldType != mapping.TypeDatetime {
return fmt.Errorf("date ranges need a date field, %q is none", field)
}
return nil
}
services/search/pkg/aggregation/validate_test.go
+7
View File
@@ -25,6 +25,8 @@ func fieldTypes(field string) string {
}
var _ = Describe("ValidateOptions", func() {
numeric := &searchsvc.BucketDefinition{Ranges: []*searchsvc.BucketRange{{From: "1980", To: "1990"}}}
dates := &searchsvc.BucketDefinition{Ranges: []*searchsvc.BucketRange{{From: "2018-08-11T00:00:00Z"}}}
DescribeTable("accepts an aggregation that fits its field",
func(opt *searchsvc.AggregationOption) {
@@ -33,6 +35,8 @@ var _ = Describe("ValidateOptions", func() {
Entry("terms on a keyword", &searchsvc.AggregationOption{Field: "audio.artist"}),
Entry("terms on a bool", &searchsvc.AggregationOption{Field: "audio.hasDrm"}),
Entry("terms on a number", &searchsvc.AggregationOption{Field: "audio.year"}),
Entry("numeric ranges on a number", &searchsvc.AggregationOption{Field: "audio.year", BucketDefinition: numeric}),
Entry("date ranges on a date", &searchsvc.AggregationOption{Field: "photo.takenDateTime", BucketDefinition: dates}),
)
DescribeTable("rejects an aggregation the index cannot answer",
@@ -45,5 +49,8 @@ var _ = Describe("ValidateOptions", func() {
Entry("terms on a path", &searchsvc.AggregationOption{Field: "Path"}),
Entry("terms on a facet object", &searchsvc.AggregationOption{Field: "audio"}),
Entry("terms on a geopoint", &searchsvc.AggregationOption{Field: "location"}),
Entry("numeric ranges on a keyword", &searchsvc.AggregationOption{Field: "audio.artist", BucketDefinition: numeric}),
Entry("numeric ranges on a date", &searchsvc.AggregationOption{Field: "photo.takenDateTime", BucketDefinition: numeric}),
Entry("date ranges on a number", &searchsvc.AggregationOption{Field: "audio.year", BucketDefinition: dates}),
)
})
services/search/pkg/bleve/aggregations.go
+54 -20
View File
@@ -3,6 +3,7 @@ package bleve
import (
"context"
"errors"
"time"
"github.com/blevesearch/bleve/v2/numeric"
bleveSearch "github.com/blevesearch/bleve/v2/search"
@@ -35,17 +36,27 @@ func termKey(fieldType, term string) (string, bool) {
type aggLevel struct {
opt *searchService.AggregationOption
kind aggregation.Kind
fieldType string
ranges aggregation.Ranges
}
// numeric values are prefix-coded in the index, their terms come from the
// decoded doc values
func (l *aggLevel) numericTerms() bool {
return l.fieldType == mapping.TypeNumeric
return l.kind == aggregation.KindTerms && l.fieldType == mapping.TypeNumeric
}
func newAggLevel(opt *searchService.AggregationOption) *aggLevel {
return &aggLevel{opt: opt, fieldType: search.FieldType(opt.GetField())}
func newAggLevel(opt *searchService.AggregationOption) (*aggLevel, error) {
l := &aggLevel{opt: opt, kind: aggregation.KindOf(opt), fieldType: search.FieldType(opt.GetField())}
if l.kind == aggregation.KindRange {
ranges, err := aggregation.ParseRanges(opt.GetField(), opt.GetBucketDefinition().GetRanges())
if err != nil {
return nil, err
}
l.ranges = ranges
}
return l, nil
}
// fieldValues is per-document scratch, reused across documents.
@@ -76,17 +87,20 @@ type aggRoot struct {
acc *bucketAcc
}
func newAggCollector(aggs []*searchService.AggregationOption) *aggCollector {
func newAggCollector(aggs []*searchService.AggregationOption) (*aggCollector, error) {
if len(aggs) == 0 {
return nil
return nil, nil
}
c := &aggCollector{fields: map[string]*fieldValues{}}
for _, agg := range aggs {
l := newAggLevel(agg)
l, err := newAggLevel(agg)
if err != nil {
return nil, err
}
c.register(l)
c.roots = append(c.roots, &aggRoot{level: l, acc: newBucketAcc()})
}
return c
return c, nil
}
func (c *aggCollector) register(l *aggLevel) {
@@ -96,10 +110,10 @@ func (c *aggCollector) register(l *aggLevel) {
c.fields[l.opt.GetField()] = fv
c.fieldNames = append(c.fieldNames, l.opt.GetField())
}
if l.numericTerms() {
fv.asNumbers = true
} else {
if l.kind == aggregation.KindTerms && !l.numericTerms() {
fv.asTerms = true
} else {
fv.asNumbers = true
}
}
@@ -143,8 +157,8 @@ func (c *aggCollector) collect(dvr index.DocValueReader, d *bleveSearch.Document
return nil
}
// Numeric doc values are prefix-coded at several precisions; only shift 0
// carries the exact value.
// Numeric and date doc values are prefix-coded at several precisions; only
// shift 0 carries the exact value.
func (c *aggCollector) visit(field string, term []byte) {
fv, ok := c.fields[field]
if !ok {
@@ -165,15 +179,31 @@ func (c *aggCollector) visit(field string, term []byte) {
func (c *aggCollector) fold(a *bucketAcc, l *aggLevel) {
fv := c.fields[l.opt.GetField()]
if l.numericTerms() {
for _, raw := range fv.numbers {
c.foldBucket(a, l, aggregation.NumberKey(numeric.Int64ToFloat64(raw)))
switch l.kind {
case aggregation.KindTerms:
if l.numericTerms() {
for _, raw := range fv.numbers {
c.foldBucket(a, l, aggregation.NumberKey(numeric.Int64ToFloat64(raw)))
}
return
}
return
}
for _, term := range fv.terms {
if key, ok := termKey(l.fieldType, term); ok {
c.foldBucket(a, l, key)
for _, term := range fv.terms {
if key, ok := termKey(l.fieldType, term); ok {
c.foldBucket(a, l, key)
}
}
case aggregation.KindRange:
for _, raw := range fv.numbers {
for _, r := range l.ranges.Numeric {
if r.Contains(numeric.Int64ToFloat64(raw)) {
c.foldBucket(a, l, r.Key)
}
}
for _, r := range l.ranges.Dates {
if r.Contains(time.Unix(0, raw)) {
c.foldBucket(a, l, r.Key)
}
}
}
}
}
@@ -198,6 +228,10 @@ func (a *bucketAcc) result(l *aggLevel) *searchService.AggregationResult {
for key, count := range a.counts {
counted[key] = &searchService.Bucket{Key: key, Count: count}
}
if l.kind == aggregation.KindRange {
r.Buckets = aggregation.RangeBuckets(l.opt, counted)
return r
}
r.Buckets = make([]*searchService.Bucket, 0, len(counted))
for _, b := range counted {
r.Buckets = append(r.Buckets, b)
services/search/pkg/bleve/backend.go
+4 -1
View File
@@ -94,7 +94,10 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
bleveReq.SortBy([]string{"-_score", "_id"})
bleveReq.Size = size
collector := newAggCollector(sir.GetAggregations())
collector, err := newAggCollector(sir.GetAggregations())
if err != nil {
return nil, errtypes.BadRequest(err.Error())
}
if collector != nil {
ctx = collector.withContext(ctx)
}
services/search/pkg/opensearch/internal/aggs/aggs.go
+50 -1
View File
@@ -6,6 +6,7 @@ package aggs
import (
"encoding/json"
"fmt"
"time"
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/aggregation"
@@ -35,7 +36,50 @@ func aggName(position int) string {
}
func build(opt *searchsvc.AggregationOption) (map[string]any, error) {
return map[string]any{"terms": map[string]any{"field": opt.GetField(), "size": aggregation.MaxBuckets}}, nil
var entry map[string]any
switch aggregation.KindOf(opt) {
case aggregation.KindRange:
ranges, err := buildRanges(opt.GetField(), opt.GetBucketDefinition().GetRanges())
if err != nil {
return nil, err
}
entry = ranges
default:
entry = map[string]any{"terms": map[string]any{"field": opt.GetField(), "size": aggregation.MaxBuckets}}
}
return entry, nil
}
// buildRanges renders the numeric "range" or the "date_range" aggregation.
func buildRanges(field string, ranges []*searchsvc.BucketRange) (map[string]any, error) {
parsed, err := aggregation.ParseRanges(field, ranges)
if err != nil {
return nil, err
}
kind := "range"
out := make([]map[string]any, 0, len(ranges))
for _, r := range parsed.Numeric {
entry := map[string]any{"key": r.Key}
if r.From != nil {
entry["from"] = *r.From
}
if r.To != nil {
entry["to"] = *r.To
}
out = append(out, entry)
}
for _, r := range parsed.Dates {
kind = "date_range"
entry := map[string]any{"key": r.Key}
if !r.From.IsZero() {
entry["from"] = r.From.Format(time.RFC3339Nano)
}
if !r.To.IsZero() {
entry["to"] = r.To.Format(time.RFC3339Nano)
}
out = append(out, entry)
}
return map[string]any{kind: map[string]any{"field": field, "ranges": out}}, nil
}
// Parse reads the aggregation block of a response into one result per option,
@@ -84,6 +128,7 @@ func result(opt *searchsvc.AggregationOption, raw json.RawMessage) (*searchsvc.A
}
res := &searchsvc.AggregationResult{Field: opt.GetField(), Buckets: []*searchsvc.Bucket{}}
byKey := make(map[string]*searchsvc.Bucket, len(body.Buckets))
for _, rawBucket := range body.Buckets {
bucket, err := parseBucket(opt, rawBucket)
if err != nil {
@@ -92,8 +137,12 @@ func result(opt *searchsvc.AggregationOption, raw json.RawMessage) (*searchsvc.A
// a field without a value has no bucket, an empty one neither
if bucket.GetKey() != "" {
res.Buckets = append(res.Buckets, bucket)
byKey[bucket.GetKey()] = bucket
}
}
if aggregation.KindOf(opt) == aggregation.KindRange {
res.Buckets = aggregation.RangeBuckets(opt, byKey)
}
return res, nil
}
services/search/pkg/opensearch/internal/aggs/aggs_test.go
+37
View File
@@ -86,6 +86,43 @@ var _ = Describe("Aggregations", func() {
}
})
DescribeTable("builds a range or date_range aggregation by the kind of the bounds, open ones without",
func(field, kind string, ranges []*searchsvc.BucketRange, want []map[string]any) {
r := build(&searchsvc.AggregationOption{Field: field, BucketDefinition: &searchsvc.BucketDefinition{Ranges: ranges}})["a_0"].(map[string]any)[kind].(map[string]any)
Expect(r["field"]).To(Equal(field))
Expect(r["ranges"]).To(Equal(want))
},
Entry("numbers", "audio.year", "range",
[]*searchsvc.BucketRange{{From: "1970", To: "1980"}, {To: "1970"}, {From: "2020"}},
[]map[string]any{{"key": "1970..1980", "from": 1970.0, "to": 1980.0}, {"key": "..1970", "to": 1970.0}, {"key": "2020..", "from": 2020.0}}),
Entry("dates", "photo.takenDateTime", "date_range",
[]*searchsvc.BucketRange{{From: "2018-08-01T00:00:00Z", To: "2018-09-01T00:00:00Z"}, {From: "2018-08-11T00:00:00Z"}},
[]map[string]any{{"key": "2018-08-01T00:00:00Z..2018-09-01T00:00:00Z", "from": "2018-08-01T00:00:00Z", "to": "2018-09-01T00:00:00Z"}, {"key": "2018-08-11T00:00:00Z..", "from": "2018-08-11T00:00:00Z"}}),
)
malformed := &searchsvc.AggregationOption{
Field: "photo.takenDateTime",
BucketDefinition: &searchsvc.BucketDefinition{Ranges: []*searchsvc.BucketRange{{From: "2018-08-11T00:00:00Z", To: "not-a-date"}}},
}
It("rejects a malformed range", func() {
_, err := aggs.Build([]*searchsvc.AggregationOption{malformed})
Expect(err).To(HaveOccurred())
})
It("lists every requested range in request order, empty ones included", func() {
out := parse(`{"a_0": {"buckets": [
{"key": "1970..1980", "doc_count": 4},
{"key": "1980..1990", "doc_count": 0}
]}}`, &searchsvc.AggregationOption{
Field: "audio.year",
BucketDefinition: &searchsvc.BucketDefinition{
Ranges: []*searchsvc.BucketRange{{From: "1980", To: "1990"}, {From: "1970", To: "1980"}, {From: "1990"}},
},
})
Expect(keys(out[0])).To(Equal([]string{"1980..1990=0", "1970..1980=4", "1990..=0"}))
})
DescribeTable("reports a result it cannot decode instead of dropping it",
func(opt *searchsvc.AggregationOption, response string) {
_, err := aggs.Parse([]*searchsvc.AggregationOption{opt}, json.RawMessage(response))
services/search/pkg/parity/README.md
+9
View File
@@ -729,12 +729,21 @@ Fixtures:
- `e.mp3`, MimeType = audio/mpeg
- `f.mp3`, MimeType = audio/mpeg
- `g.mp3`, MimeType = audio/mpeg
- `a.jpg`, MimeType = image/jpeg
- `b.jpg`, MimeType = image/jpeg
- `c.jpg`, MimeType = image/jpeg
- `d.jpg`, MimeType = image/jpeg
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| AGG-01 | `mediatype:audio` reads `term buckets on audio.artist` | audio.artist Motörhead=3, audio.artist Saxon=2 | audio.artist Motörhead=3, audio.artist Saxon=2 | audio.artist Motörhead=3, audio.artist Saxon=2 | ✅ |
| AGG-02 | `mediatype:audio` reads `no aggregations requested` | no match | no match | no match | ✅ |
| AGG-03 | `mediatype:audio` reads `artist and album buckets in one request` | audio.albu... Steel=2, audio.albu...Spades=1, audio.album Bomber=2, audio.artist Motörhead=3, audio.artist Saxon=2 | audio.albu... Steel=2, audio.albu...Spades=1, audio.album Bomber=2, audio.artist Motörhead=3, audio.artist Saxon=2 | audio.albu... Steel=2, audio.albu...Spades=1, audio.album Bomber=2, audio.artist Motörhead=3, audio.artist Saxon=2 | ✅ |
| AGG-04 | `mediatype:audio` reads `audio.year buckets per decade` | audio.year 1970..1980=2, audio.year 1980..1990=1, audio.year 1990..2000=1, audio.year 2000..2010=3 | audio.year 1970..1980=2, audio.year 1980..1990=1, audio.year 1990..2000=1, audio.year 2000..2010=3 | audio.year 1970..1980=2, audio.year 1980..1990=1, audio.year 1990..2000=1, audio.year 2000..2010=3 | ✅ |
| AGG-05 | `mediatype:audio` reads `open-ended audio.year ranges` | audio.year ..1990=3, audio.year 2000..=3 | audio.year ..1990=3, audio.year 2000..=3 | audio.year ..1990=3, audio.year 2000..=3 | ✅ |
| AGG-07 | `mediatype:image` reads `photo.takenDateTime buckets per date range, an empty range counts zero` | photo.take...00:00Z=0, photo.take...00:00Z=1, photo.take...00:00Z=2, photo.take...00:00Z=2 | photo.take...00:00Z=0, photo.take...00:00Z=1, photo.take...00:00Z=2, photo.take...00:00Z=2 | photo.take...00:00Z=0, photo.take...00:00Z=1, photo.take...00:00Z=2, photo.take...00:00Z=2 | ✅ |
| AGG-08 | `mediatype:image` reads `open-ended date ranges` | photo.take...00:00Z=3, photo.take...:00Z..=1 | photo.take...00:00Z=3, photo.take...:00Z..=1 | photo.take...00:00Z=3, photo.take...:00Z..=1 | ✅ |
| AGG-09 | `mediatype:image` reads `malformed date range bound` | bad request | bad request | bad request | ✅ |
| AGG-34 | `mediatype:audio` reads `term buckets on the numeric audio.year` | audio.year 1971=1, audio.year 1975=1, audio.year 1982=1, audio.year 1999=1, audio.year 2001=1, audio.year 2005=1, audio.year 2009=1 | audio.year 1971=1, audio.year 1975=1, audio.year 1982=1, audio.year 1999=1, audio.year 2001=1, audio.year 2005=1, audio.year 2009=1 | audio.year 1971=1, audio.year 1975=1, audio.year 1982=1, audio.year 1999=1, audio.year 2001=1, audio.year 2005=1, audio.year 2009=1 | ✅ |
| AGG-37 | `mediatype:audio` reads `term buckets on the bool audio.hasDrm, spelled true and false` | audio.hasDrm false=1, audio.hasDrm true=1 | audio.hasDrm false=1, audio.hasDrm true=1 | audio.hasDrm false=1, audio.hasDrm true=1 | ✅ |
| AGG-39 | `mediatype:audio` reads `no bucket for the empty Title of every match` | no match | no match | no match | ✅ |
services/search/pkg/parity/aggregations_test.go
+54
View File
@@ -3,6 +3,7 @@ package parity
import (
"context"
"fmt"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -53,6 +54,14 @@ func untaggedSong(name string, year int32) search.Resource {
return fixtureDoc(name, withMime("audio/mpeg"), withAudio(&libregraph.Audio{Year: libregraph.PtrInt32(year)}))
}
func photo(name, taken string) search.Resource {
t, err := time.Parse(time.RFC3339, taken)
if err != nil {
panic(err)
}
return fixtureDoc(name, withMime("image/jpeg"), withPhoto(&libregraph.Photo{TakenDateTime: &t}))
}
func song(name, artist, album string, year int32, opts ...fixtureOption) search.Resource {
return fixtureDoc(name, append([]fixtureOption{withMime("audio/mpeg"), withAudio(&libregraph.Audio{
Artist: libregraph.PtrString(artist),
@@ -76,10 +85,17 @@ func aggregationFixtures() []search.Resource {
song("e.mp3", "Motörhead", "Ace of Spades", 2001),
untaggedSong("f.mp3", 2005),
untaggedSong("g.mp3", 2009),
photo("a.jpg", "2018-08-11T09:15:00Z"),
photo("b.jpg", "2018-08-11T19:42:00Z"),
photo("c.jpg", "2018-09-01T12:00:00Z"),
photo("d.jpg", "2021-08-11T08:00:00Z"),
}
}
func aggregationCases() []aggCase {
ranges := func(rs ...*searchService.BucketRange) *searchService.BucketDefinition {
return &searchService.BucketDefinition{Ranges: rs}
}
return []aggCase{
{id: 1, query: "mediatype:audio", reads: "term buckets on audio.artist",
aggs: []*searchService.AggregationOption{{Field: "audio.artist", Size: 10}},
@@ -91,6 +107,44 @@ func aggregationCases() []aggCase {
"audio.artist Saxon=2", "audio.artist Motörhead=3",
"audio.album Wheels of Steel=2", "audio.album Bomber=2", "audio.album Ace of Spades=1",
}},
{id: 4, query: "mediatype:audio", reads: "audio.year buckets per decade",
aggs: []*searchService.AggregationOption{{Field: "audio.year", BucketDefinition: ranges(
&searchService.BucketRange{From: "1970", To: "1980"},
&searchService.BucketRange{From: "1980", To: "1990"},
&searchService.BucketRange{From: "1990", To: "2000"},
&searchService.BucketRange{From: "2000", To: "2010"},
)}},
want: []string{"audio.year 1970..1980=2", "audio.year 1980..1990=1", "audio.year 1990..2000=1", "audio.year 2000..2010=3"}},
{id: 5, query: "mediatype:audio", reads: "open-ended audio.year ranges",
aggs: []*searchService.AggregationOption{{Field: "audio.year", BucketDefinition: ranges(
&searchService.BucketRange{To: "1990"},
&searchService.BucketRange{From: "2000"},
)}},
want: []string{"audio.year ..1990=3", "audio.year 2000..=3"}},
{id: 7, query: "mediatype:image", reads: "photo.takenDateTime buckets per date range, an empty range counts zero",
aggs: []*searchService.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: ranges(
&searchService.BucketRange{From: "2018-08-11T00:00:00Z", To: "2018-08-12T00:00:00Z"},
&searchService.BucketRange{From: "2018-08-01T00:00:00Z", To: "2018-09-01T00:00:00Z"},
&searchService.BucketRange{From: "2021-01-01T00:00:00Z", To: "2022-01-01T00:00:00Z"},
&searchService.BucketRange{From: "2023-01-01T00:00:00Z", To: "2024-01-01T00:00:00Z"},
)}},
want: []string{
"photo.takenDateTime 2018-08-11T00:00:00Z..2018-08-12T00:00:00Z=2",
"photo.takenDateTime 2018-08-01T00:00:00Z..2018-09-01T00:00:00Z=2",
"photo.takenDateTime 2021-01-01T00:00:00Z..2022-01-01T00:00:00Z=1",
"photo.takenDateTime 2023-01-01T00:00:00Z..2024-01-01T00:00:00Z=0",
}},
{id: 8, query: "mediatype:image", reads: "open-ended date ranges",
aggs: []*searchService.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: ranges(
&searchService.BucketRange{To: "2019-01-01T00:00:00Z"},
&searchService.BucketRange{From: "2019-01-01T00:00:00Z"},
)}},
want: []string{"photo.takenDateTime ..2019-01-01T00:00:00Z=3", "photo.takenDateTime 2019-01-01T00:00:00Z..=1"}},
{id: 9, query: "mediatype:image", reads: "malformed date range bound",
aggs: []*searchService.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: ranges(
&searchService.BucketRange{From: "2018-08-11T00:00:00Z", To: "not-a-date"},
)}},
wantBadRequest: true},
{id: 34, query: "mediatype:audio", reads: "term buckets on the numeric audio.year",
aggs: []*searchService.AggregationOption{{Field: "audio.year"}},
want: []string{
services/search/pkg/parity/fixtures_test.go
+4
View File
@@ -50,6 +50,10 @@ func withAudio(audio *libregraph.Audio) fixtureOption {
return func(r *search.Resource) { r.Audio = audio }
}
func withPhoto(photo *libregraph.Photo) fixtureOption {
return func(r *search.Resource) { r.Photo = photo }
}
func withLocation(location *libregraph.GeoCoordinates) fixtureOption {
return func(r *search.Resource) { r.Location = location }
}