diff --git a/protogen/gen/opencloud/services/search/v0/search.pb.go b/protogen/gen/opencloud/services/search/v0/search.pb.go index bf3eeacc11..5099749fe9 100644 --- a/protogen/gen/opencloud/services/search/v0/search.pb.go +++ b/protogen/gen/opencloud/services/search/v0/search.pb.go @@ -411,8 +411,9 @@ type AggregationOption struct { // Required. The indexed field to aggregate on. Field string `protobuf:"bytes,1,opt,name=field,proto3" json:"field,omitempty"` - // Optional. Maximum number of buckets to return, applied after the - // cross-space merge. 0 returns every bucket. + // Optional. Maximum number of buckets to return for a terms aggregation, + // applied after the cross-space merge. 0 returns every bucket. Ignored for + // range aggregations. Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` // Optional. Controls bucket selection, ordering and filtering. BucketDefinition *BucketDefinition `protobuf:"bytes,3,opt,name=bucket_definition,json=bucketDefinition,proto3" json:"bucket_definition,omitempty"` @@ -481,6 +482,9 @@ type BucketDefinition struct { SortBy BucketSortBy `protobuf:"varint,1,opt,name=sort_by,json=sortBy,proto3,enum=opencloud.services.search.v0.BucketSortBy" json:"sort_by,omitempty"` IsDescending bool `protobuf:"varint,2,opt,name=is_descending,json=isDescending,proto3" json:"is_descending,omitempty"` MinimumCount int32 `protobuf:"varint,3,opt,name=minimum_count,json=minimumCount,proto3" json:"minimum_count,omitempty"` + // Optional. When non-empty, the aggregation is computed over these numeric + // or date ranges instead of as a terms aggregation. + Ranges []*BucketRange `protobuf:"bytes,4,rep,name=ranges,proto3" json:"ranges,omitempty"` } func (x *BucketDefinition) Reset() { @@ -536,6 +540,71 @@ func (x *BucketDefinition) GetMinimumCount() int32 { return 0 } +func (x *BucketDefinition) GetRanges() []*BucketRange { + if x != nil { + return x.Ranges + } + return nil +} + +// A range is from-inclusive and to-exclusive. +type BucketRange struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // At least one of `from` or `to` must be set. Both are string-encoded and + // of one kind per aggregation: decimal numbers or RFC3339 dates. + From string `protobuf:"bytes,1,opt,name=from,proto3" json:"from,omitempty"` + To string `protobuf:"bytes,2,opt,name=to,proto3" json:"to,omitempty"` +} + +func (x *BucketRange) Reset() { + *x = BucketRange{} + if protoimpl.UnsafeEnabled { + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BucketRange) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BucketRange) ProtoMessage() {} + +func (x *BucketRange) ProtoReflect() protoreflect.Message { + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BucketRange.ProtoReflect.Descriptor instead. +func (*BucketRange) Descriptor() ([]byte, []int) { + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{6} +} + +func (x *BucketRange) GetFrom() string { + if x != nil { + return x.From + } + return "" +} + +func (x *BucketRange) GetTo() string { + if x != nil { + return x.To + } + return "" +} + // One result per AggregationOption, at the same position. type AggregationResult struct { state protoimpl.MessageState @@ -549,7 +618,7 @@ type AggregationResult struct { func (x *AggregationResult) Reset() { *x = AggregationResult{} if protoimpl.UnsafeEnabled { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[6] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -562,7 +631,7 @@ func (x *AggregationResult) String() string { func (*AggregationResult) ProtoMessage() {} func (x *AggregationResult) ProtoReflect() protoreflect.Message { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[6] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -575,7 +644,7 @@ func (x *AggregationResult) ProtoReflect() protoreflect.Message { // Deprecated: Use AggregationResult.ProtoReflect.Descriptor instead. func (*AggregationResult) Descriptor() ([]byte, []int) { - return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{6} + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{7} } func (x *AggregationResult) GetField() string { @@ -604,7 +673,7 @@ type Bucket struct { func (x *Bucket) Reset() { *x = Bucket{} if protoimpl.UnsafeEnabled { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[7] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -617,7 +686,7 @@ func (x *Bucket) String() string { func (*Bucket) ProtoMessage() {} func (x *Bucket) ProtoReflect() protoreflect.Message { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[7] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -630,7 +699,7 @@ func (x *Bucket) ProtoReflect() protoreflect.Message { // Deprecated: Use Bucket.ProtoReflect.Descriptor instead. func (*Bucket) Descriptor() ([]byte, []int) { - return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{7} + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{8} } func (x *Bucket) GetKey() string { @@ -661,7 +730,7 @@ type IndexSpaceRequest struct { func (x *IndexSpaceRequest) Reset() { *x = IndexSpaceRequest{} if protoimpl.UnsafeEnabled { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[8] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -674,7 +743,7 @@ func (x *IndexSpaceRequest) String() string { func (*IndexSpaceRequest) ProtoMessage() {} func (x *IndexSpaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[8] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -687,7 +756,7 @@ func (x *IndexSpaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use IndexSpaceRequest.ProtoReflect.Descriptor instead. func (*IndexSpaceRequest) Descriptor() ([]byte, []int) { - return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{8} + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{9} } func (x *IndexSpaceRequest) GetSpaceId() string { @@ -738,7 +807,7 @@ type IndexSpaceResponse struct { func (x *IndexSpaceResponse) Reset() { *x = IndexSpaceResponse{} if protoimpl.UnsafeEnabled { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[9] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -751,7 +820,7 @@ func (x *IndexSpaceResponse) String() string { func (*IndexSpaceResponse) ProtoMessage() {} func (x *IndexSpaceResponse) ProtoReflect() protoreflect.Message { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[9] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -764,7 +833,7 @@ func (x *IndexSpaceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use IndexSpaceResponse.ProtoReflect.Descriptor instead. func (*IndexSpaceResponse) Descriptor() ([]byte, []int) { - return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{9} + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{10} } func (x *IndexSpaceResponse) GetSpaceId() string { @@ -902,7 +971,7 @@ var file_opencloud_services_search_v0_search_proto_rawDesc = []byte{ 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x10, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x44, 0x65, - 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xa1, 0x01, 0x0a, 0x10, 0x42, 0x75, 0x63, + 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xe4, 0x01, 0x0a, 0x10, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x43, 0x0a, 0x07, 0x73, 0x6f, 0x72, 0x74, 0x5f, 0x62, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, @@ -912,102 +981,110 @@ var file_opencloud_services_search_v0_search_proto_rawDesc = []byte{ 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x73, 0x44, 0x65, 0x73, 0x63, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, - 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x69, 0x0a, 0x11, - 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x3e, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, - 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x07, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, 0x30, 0x0a, 0x06, 0x42, 0x75, 0x63, 0x6b, 0x65, - 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x11, 0x49, 0x6e, - 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x19, 0x0a, 0x08, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, - 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, - 0x72, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x69, - 0x6e, 0x64, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x66, 0x6f, 0x72, 0x63, - 0x65, 0x52, 0x65, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x26, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, - 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, - 0x41, 0x01, 0x01, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, - 0x22, 0xd1, 0x01, 0x0a, 0x12, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x49, 0x64, 0x12, 0x40, 0x0a, 0x0e, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x44, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x64, 0x5f, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x14, - 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x2a, 0x8c, 0x01, 0x0a, 0x0c, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, - 0x6f, 0x72, 0x74, 0x42, 0x79, 0x12, 0x1e, 0x0a, 0x1a, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, - 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, - 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, - 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x10, 0x01, 0x12, - 0x20, 0x0a, 0x1c, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, - 0x59, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x41, 0x53, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, - 0x02, 0x12, 0x20, 0x0a, 0x1c, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, - 0x5f, 0x42, 0x59, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x41, 0x53, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, - 0x52, 0x10, 0x03, 0x32, 0xb3, 0x02, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x50, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x85, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x12, 0x2b, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, - 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, + 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x41, 0x0a, 0x06, + 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, + 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x42, 0x75, 0x63, 0x6b, + 0x65, 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, + 0x31, 0x0a, 0x0b, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x72, + 0x6f, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x74, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, + 0x74, 0x6f, 0x22, 0x69, 0x0a, 0x11, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x3e, 0x0a, + 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, + 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x42, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, 0x30, 0x0a, + 0x06, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, + 0x94, 0x01, 0x0a, 0x11, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, + 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x6f, 0x72, + 0x63, 0x65, 0x5f, 0x72, 0x65, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0c, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x52, 0x65, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x26, + 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, + 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x22, 0xd1, 0x01, 0x0a, 0x12, 0x49, 0x6e, 0x64, 0x65, 0x78, + 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, + 0x08, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x40, 0x0a, 0x0e, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x65, 0x64, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, + 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x70, + 0x61, 0x63, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x2a, 0x8c, 0x01, 0x0a, 0x0c, 0x42, + 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x6f, 0x72, 0x74, 0x42, 0x79, 0x12, 0x1e, 0x0a, 0x1a, 0x42, + 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x42, + 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x43, 0x4f, + 0x55, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x20, 0x0a, 0x1c, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, + 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x41, 0x53, 0x5f, 0x53, + 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x20, 0x0a, 0x1c, 0x42, 0x55, 0x43, 0x4b, 0x45, + 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x41, 0x53, + 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x03, 0x32, 0xb3, 0x02, 0x0a, 0x0e, 0x53, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x85, 0x01, 0x0a, + 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x2b, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, + 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x20, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1a, 0x3a, 0x01, 0x2a, 0x22, 0x15, 0x2f, + 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x73, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x12, 0x98, 0x01, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, + 0x61, 0x63, 0x65, 0x12, 0x2f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, + 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x25, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1f, 0x3a, 0x01, + 0x2a, 0x22, 0x1a, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x30, 0x01, 0x32, + 0xa7, 0x01, 0x0a, 0x0d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x12, 0x95, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x30, 0x2e, 0x6f, + 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x20, 0x82, 0xd3, - 0xe4, 0x93, 0x02, 0x1a, 0x3a, 0x01, 0x2a, 0x22, 0x15, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, - 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x98, - 0x01, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x12, 0x2f, 0x2e, - 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64, - 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, - 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, - 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x25, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1f, 0x3a, 0x01, 0x2a, 0x22, 0x1a, 0x2f, 0x61, 0x70, - 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, - 0x78, 0x2d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x30, 0x01, 0x32, 0xa7, 0x01, 0x0a, 0x0d, 0x49, 0x6e, - 0x64, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x95, 0x01, 0x0a, 0x06, - 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x30, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, - 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, - 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x20, 0x3a, 0x01, 0x2a, 0x22, 0x1b, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, - 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x73, 0x65, 0x61, - 0x72, 0x63, 0x68, 0x42, 0xf2, 0x02, 0x5a, 0x4a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, - 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, - 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, - 0x76, 0x30, 0x92, 0x41, 0xa2, 0x02, 0x12, 0xb7, 0x01, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x6e, 0x43, - 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x22, 0x51, 0x0a, 0x0e, 0x4f, - 0x70, 0x65, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, 0x48, 0x12, 0x29, 0x68, - 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, - 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x1a, 0x14, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, - 0x74, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x65, 0x75, 0x2a, 0x49, - 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2d, 0x32, 0x2e, 0x30, 0x12, 0x3b, 0x68, 0x74, - 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, - 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x69, - 0x6e, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, 0x2e, 0x30, 0x2e, 0x30, - 0x2a, 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3e, 0x0a, 0x10, 0x44, 0x65, 0x76, 0x65, - 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x12, 0x2a, 0x68, 0x74, - 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x64, 0x6f, 0x63, 0x73, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x65, 0x75, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x20, 0x3a, 0x01, 0x2a, 0x22, 0x1b, 0x2f, 0x61, + 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x42, 0xf2, 0x02, 0x5a, 0x4a, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70, 0x65, + 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x73, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x92, 0x41, 0xa2, 0x02, 0x12, 0xb7, 0x01, 0x0a, + 0x10, 0x4f, 0x70, 0x65, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x73, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x22, 0x51, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, + 0x6d, 0x62, 0x48, 0x12, 0x29, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, + 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x1a, 0x14, + 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, + 0x64, 0x2e, 0x65, 0x75, 0x2a, 0x49, 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2d, 0x32, + 0x2e, 0x30, 0x12, 0x3b, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x62, 0x6c, + 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, + 0x05, 0x31, 0x2e, 0x30, 0x2e, 0x30, 0x2a, 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3e, + 0x0a, 0x10, 0x44, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, + 0x61, 0x6c, 0x12, 0x2a, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x64, 0x6f, 0x63, 0x73, + 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x65, 0x75, 0x2f, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1023,7 +1100,7 @@ func file_opencloud_services_search_v0_search_proto_rawDescGZIP() []byte { } var file_opencloud_services_search_v0_search_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_opencloud_services_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_opencloud_services_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_opencloud_services_search_v0_search_proto_goTypes = []interface{}{ (BucketSortBy)(0), // 0: opencloud.services.search.v0.BucketSortBy (*SearchRequest)(nil), // 1: opencloud.services.search.v0.SearchRequest @@ -1032,38 +1109,40 @@ var file_opencloud_services_search_v0_search_proto_goTypes = []interface{}{ (*SearchIndexResponse)(nil), // 4: opencloud.services.search.v0.SearchIndexResponse (*AggregationOption)(nil), // 5: opencloud.services.search.v0.AggregationOption (*BucketDefinition)(nil), // 6: opencloud.services.search.v0.BucketDefinition - (*AggregationResult)(nil), // 7: opencloud.services.search.v0.AggregationResult - (*Bucket)(nil), // 8: opencloud.services.search.v0.Bucket - (*IndexSpaceRequest)(nil), // 9: opencloud.services.search.v0.IndexSpaceRequest - (*IndexSpaceResponse)(nil), // 10: opencloud.services.search.v0.IndexSpaceResponse - (*v0.Reference)(nil), // 11: opencloud.messages.search.v0.Reference - (*v0.Match)(nil), // 12: opencloud.messages.search.v0.Match - (*durationpb.Duration)(nil), // 13: google.protobuf.Duration + (*BucketRange)(nil), // 7: opencloud.services.search.v0.BucketRange + (*AggregationResult)(nil), // 8: opencloud.services.search.v0.AggregationResult + (*Bucket)(nil), // 9: opencloud.services.search.v0.Bucket + (*IndexSpaceRequest)(nil), // 10: opencloud.services.search.v0.IndexSpaceRequest + (*IndexSpaceResponse)(nil), // 11: opencloud.services.search.v0.IndexSpaceResponse + (*v0.Reference)(nil), // 12: opencloud.messages.search.v0.Reference + (*v0.Match)(nil), // 13: opencloud.messages.search.v0.Match + (*durationpb.Duration)(nil), // 14: google.protobuf.Duration } var file_opencloud_services_search_v0_search_proto_depIdxs = []int32{ - 11, // 0: opencloud.services.search.v0.SearchRequest.ref:type_name -> opencloud.messages.search.v0.Reference + 12, // 0: opencloud.services.search.v0.SearchRequest.ref:type_name -> opencloud.messages.search.v0.Reference 5, // 1: opencloud.services.search.v0.SearchRequest.aggregations:type_name -> opencloud.services.search.v0.AggregationOption - 12, // 2: opencloud.services.search.v0.SearchResponse.matches:type_name -> opencloud.messages.search.v0.Match - 7, // 3: opencloud.services.search.v0.SearchResponse.aggregations:type_name -> opencloud.services.search.v0.AggregationResult - 11, // 4: opencloud.services.search.v0.SearchIndexRequest.ref:type_name -> opencloud.messages.search.v0.Reference + 13, // 2: opencloud.services.search.v0.SearchResponse.matches:type_name -> opencloud.messages.search.v0.Match + 8, // 3: opencloud.services.search.v0.SearchResponse.aggregations:type_name -> opencloud.services.search.v0.AggregationResult + 12, // 4: opencloud.services.search.v0.SearchIndexRequest.ref:type_name -> opencloud.messages.search.v0.Reference 5, // 5: opencloud.services.search.v0.SearchIndexRequest.aggregations:type_name -> opencloud.services.search.v0.AggregationOption - 12, // 6: opencloud.services.search.v0.SearchIndexResponse.matches:type_name -> opencloud.messages.search.v0.Match - 7, // 7: opencloud.services.search.v0.SearchIndexResponse.aggregations:type_name -> opencloud.services.search.v0.AggregationResult + 13, // 6: opencloud.services.search.v0.SearchIndexResponse.matches:type_name -> opencloud.messages.search.v0.Match + 8, // 7: opencloud.services.search.v0.SearchIndexResponse.aggregations:type_name -> opencloud.services.search.v0.AggregationResult 6, // 8: opencloud.services.search.v0.AggregationOption.bucket_definition:type_name -> opencloud.services.search.v0.BucketDefinition 0, // 9: opencloud.services.search.v0.BucketDefinition.sort_by:type_name -> opencloud.services.search.v0.BucketSortBy - 8, // 10: opencloud.services.search.v0.AggregationResult.buckets:type_name -> opencloud.services.search.v0.Bucket - 13, // 11: opencloud.services.search.v0.IndexSpaceResponse.space_duration:type_name -> google.protobuf.Duration - 1, // 12: opencloud.services.search.v0.SearchProvider.Search:input_type -> opencloud.services.search.v0.SearchRequest - 9, // 13: opencloud.services.search.v0.SearchProvider.IndexSpace:input_type -> opencloud.services.search.v0.IndexSpaceRequest - 3, // 14: opencloud.services.search.v0.IndexProvider.Search:input_type -> opencloud.services.search.v0.SearchIndexRequest - 2, // 15: opencloud.services.search.v0.SearchProvider.Search:output_type -> opencloud.services.search.v0.SearchResponse - 10, // 16: opencloud.services.search.v0.SearchProvider.IndexSpace:output_type -> opencloud.services.search.v0.IndexSpaceResponse - 4, // 17: opencloud.services.search.v0.IndexProvider.Search:output_type -> opencloud.services.search.v0.SearchIndexResponse - 15, // [15:18] is the sub-list for method output_type - 12, // [12:15] is the sub-list for method input_type - 12, // [12:12] is the sub-list for extension type_name - 12, // [12:12] is the sub-list for extension extendee - 0, // [0:12] is the sub-list for field type_name + 7, // 10: opencloud.services.search.v0.BucketDefinition.ranges:type_name -> opencloud.services.search.v0.BucketRange + 9, // 11: opencloud.services.search.v0.AggregationResult.buckets:type_name -> opencloud.services.search.v0.Bucket + 14, // 12: opencloud.services.search.v0.IndexSpaceResponse.space_duration:type_name -> google.protobuf.Duration + 1, // 13: opencloud.services.search.v0.SearchProvider.Search:input_type -> opencloud.services.search.v0.SearchRequest + 10, // 14: opencloud.services.search.v0.SearchProvider.IndexSpace:input_type -> opencloud.services.search.v0.IndexSpaceRequest + 3, // 15: opencloud.services.search.v0.IndexProvider.Search:input_type -> opencloud.services.search.v0.SearchIndexRequest + 2, // 16: opencloud.services.search.v0.SearchProvider.Search:output_type -> opencloud.services.search.v0.SearchResponse + 11, // 17: opencloud.services.search.v0.SearchProvider.IndexSpace:output_type -> opencloud.services.search.v0.IndexSpaceResponse + 4, // 18: opencloud.services.search.v0.IndexProvider.Search:output_type -> opencloud.services.search.v0.SearchIndexResponse + 16, // [16:19] is the sub-list for method output_type + 13, // [13:16] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name } func init() { file_opencloud_services_search_v0_search_proto_init() } @@ -1145,7 +1224,7 @@ func file_opencloud_services_search_v0_search_proto_init() { } } file_opencloud_services_search_v0_search_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AggregationResult); i { + switch v := v.(*BucketRange); i { case 0: return &v.state case 1: @@ -1157,7 +1236,7 @@ func file_opencloud_services_search_v0_search_proto_init() { } } file_opencloud_services_search_v0_search_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Bucket); i { + switch v := v.(*AggregationResult); i { case 0: return &v.state case 1: @@ -1169,7 +1248,7 @@ func file_opencloud_services_search_v0_search_proto_init() { } } file_opencloud_services_search_v0_search_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*IndexSpaceRequest); i { + switch v := v.(*Bucket); i { case 0: return &v.state case 1: @@ -1181,6 +1260,18 @@ func file_opencloud_services_search_v0_search_proto_init() { } } file_opencloud_services_search_v0_search_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IndexSpaceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_opencloud_services_search_v0_search_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*IndexSpaceResponse); i { case 0: return &v.state @@ -1201,7 +1292,7 @@ func file_opencloud_services_search_v0_search_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_opencloud_services_search_v0_search_proto_rawDesc, NumEnums: 1, - NumMessages: 10, + NumMessages: 11, NumExtensions: 0, NumServices: 2, }, diff --git a/protogen/gen/opencloud/services/search/v0/search.swagger.json b/protogen/gen/opencloud/services/search/v0/search.swagger.json index 793fdbf4fe..c07a8d5d23 100644 --- a/protogen/gen/opencloud/services/search/v0/search.swagger.json +++ b/protogen/gen/opencloud/services/search/v0/search.swagger.json @@ -177,7 +177,7 @@ "size": { "type": "integer", "format": "int32", - "description": "Optional. Maximum number of buckets to return, applied after the\ncross-space merge. 0 returns every bucket." + "description": "Optional. Maximum number of buckets to return for a terms aggregation,\napplied after the cross-space merge. 0 returns every bucket. Ignored for\nrange aggregations." }, "bucketDefinition": { "$ref": "#/definitions/v0BucketDefinition", @@ -285,9 +285,29 @@ "minimumCount": { "type": "integer", "format": "int32" + }, + "ranges": { + "type": "array", + "items": { + "$ref": "#/definitions/v0BucketRange" + }, + "description": "Optional. When non-empty, the aggregation is computed over these numeric\nor date ranges instead of as a terms aggregation." } } }, + "v0BucketRange": { + "type": "object", + "properties": { + "from": { + "type": "string", + "description": "At least one of `from` or `to` must be set. Both are string-encoded and\nof one kind per aggregation: decimal numbers or RFC3339 dates." + }, + "to": { + "type": "string" + } + }, + "description": "A range is from-inclusive and to-exclusive." + }, "v0BucketSortBy": { "type": "string", "enum": [ diff --git a/protogen/proto/opencloud/services/search/v0/search.proto b/protogen/proto/opencloud/services/search/v0/search.proto index 2a5d743af0..859b618f96 100644 --- a/protogen/proto/opencloud/services/search/v0/search.proto +++ b/protogen/proto/opencloud/services/search/v0/search.proto @@ -126,8 +126,9 @@ message SearchIndexResponse { message AggregationOption { // Required. The indexed field to aggregate on. string field = 1; - // Optional. Maximum number of buckets to return, applied after the - // cross-space merge. 0 returns every bucket. + // Optional. Maximum number of buckets to return for a terms aggregation, + // applied after the cross-space merge. 0 returns every bucket. Ignored for + // range aggregations. int32 size = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. Controls bucket selection, ordering and filtering. BucketDefinition bucket_definition = 3 [(google.api.field_behavior) = OPTIONAL]; @@ -139,6 +140,9 @@ message BucketDefinition { BucketSortBy sort_by = 1; bool is_descending = 2; int32 minimum_count = 3; + // Optional. When non-empty, the aggregation is computed over these numeric + // or date ranges instead of as a terms aggregation. + repeated BucketRange ranges = 4; } enum BucketSortBy { @@ -148,6 +152,14 @@ enum BucketSortBy { BUCKET_SORT_BY_KEY_AS_NUMBER = 3; } +// A range is from-inclusive and to-exclusive. +message BucketRange { + // At least one of `from` or `to` must be set. Both are string-encoded and + // of one kind per aggregation: decimal numbers or RFC3339 dates. + string from = 1; + string to = 2; +} + // One result per AggregationOption, at the same position. message AggregationResult { string field = 1; diff --git a/services/graph/pkg/service/v0/searchquery.go b/services/graph/pkg/service/v0/searchquery.go index d12f040a23..f9b8fae479 100644 --- a/services/graph/pkg/service/v0/searchquery.go +++ b/services/graph/pkg/service/v0/searchquery.go @@ -100,8 +100,6 @@ func unsupportedProperty(sr libregraph.SearchRequest) string { return "@libre.graph.subAggregations" case a.LibreGraphMetricDefinition != nil: return "@libre.graph.metricDefinition" - case len(a.BucketDefinition.GetRanges()) > 0: - return "bucketDefinition.ranges" } } return "" diff --git a/services/graph/pkg/service/v0/searchquery_mapping.go b/services/graph/pkg/service/v0/searchquery_mapping.go index 088e6a4d44..f96636ca9b 100644 --- a/services/graph/pkg/service/v0/searchquery_mapping.go +++ b/services/graph/pkg/service/v0/searchquery_mapping.go @@ -67,7 +67,10 @@ func libregraphAggregationsToSearch(in []libregraph.AggregationOption) []*search } func libregraphBucketDefinitionToSearch(in libregraph.BucketDefinition) *searchsvc.BucketDefinition { - bd := &searchsvc.BucketDefinition{SortBy: bucketSortBy[in.SortBy]} + bd := &searchsvc.BucketDefinition{ + SortBy: bucketSortBy[in.SortBy], + Ranges: libregraphRangesToSearch(in.Ranges), + } if in.IsDescending != nil { bd.IsDescending = *in.IsDescending } @@ -77,6 +80,14 @@ func libregraphBucketDefinitionToSearch(in libregraph.BucketDefinition) *searchs return bd } +func libregraphRangesToSearch(in []libregraph.BucketAggregationRange) []*searchsvc.BucketRange { + out := make([]*searchsvc.BucketRange, 0, len(in)) + for _, r := range in { + out = append(out, &searchsvc.BucketRange{From: r.GetFrom(), To: r.GetTo()}) + } + return out +} + // searchAggregationsToLibregraph maps the results to their definitions by // position: the search service answers one result per aggregation, in request // order. diff --git a/services/graph/pkg/service/v0/searchquery_test.go b/services/graph/pkg/service/v0/searchquery_test.go index c9765db192..f620d2837d 100644 --- a/services/graph/pkg/service/v0/searchquery_test.go +++ b/services/graph/pkg/service/v0/searchquery_test.go @@ -303,18 +303,21 @@ var _ = ginkgo.Describe("SearchQuery", func() { ginkgo.Entry("a size below one", `"aggregations": [{"field": "audio.artist", "size": 0}]`), ginkgo.Entry("a negative minimumCount", `"aggregations": [{"field": "audio.artist", "bucketDefinition": {"sortBy": "count", "minimumCount": -1}}]`), ginkgo.Entry("a field the index does not know", `"aggregations": [{"field": "audio.nonexistent"}]`), + ginkgo.Entry("a range bound that is no number", `"aggregations": [{"field": "audio.year", "bucketDefinition": {"sortBy": "count", "ranges": [{"from": "1970", "to": "198o"}]}}]`), ) ginkgo.It("translates the bucket definition for the search service", func() { g, captured := graphWithAggregations() rr := postSearchQuery(g, searchQueryBody(`"aggregations": [ {"field": "audio.artist", "size": 5, "bucketDefinition": {"sortBy": "keyAsString", "isDescending": true, "minimumCount": 2}}, + {"field": "audio.year", "bucketDefinition": {"sortBy": "keyAsNumber", "ranges": [{"to": "1980"}, {"from": "1980", "to": "1990"}]}}, {"field": "audio.genre", "bucketDefinition": {"sortBy": "count"}} ]`)) Expect(rr.Code).To(Equal(http.StatusOK), rr.Body.String()) Expect(captured().GetAggregations()).To(BeComparableTo([]*searchsvc.AggregationOption{ {Field: "audio.artist", Size: 5, BucketDefinition: &searchsvc.BucketDefinition{SortBy: searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_STRING, IsDescending: true, MinimumCount: 2}}, + {Field: "audio.year", BucketDefinition: &searchsvc.BucketDefinition{SortBy: searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, Ranges: []*searchsvc.BucketRange{{To: "1980"}, {From: "1980", To: "1990"}}}}, {Field: "audio.genre", BucketDefinition: &searchsvc.BucketDefinition{SortBy: searchsvc.BucketSortBy_BUCKET_SORT_BY_COUNT}}, }, protocmp.Transform())) }) @@ -353,6 +356,18 @@ var _ = ginkgo.Describe("SearchQuery", func() { ginkgo.Entry("a property the index rules refuse for terms", "lastModifiedDateTime"), ) + ginkgo.It("resolves lastModifiedDateTime to the modification time of the index", func() { + g, captured := graphWithAggregations(&searchsvc.AggregationResult{ + Field: "Mtime", + Buckets: []*searchsvc.Bucket{{Key: "2026-01-01T00:00:00Z..", Count: 3}}, + }) + rr := postSearchQuery(g, searchQueryBody(`"aggregations": [{"field": "lastModifiedDateTime", "bucketDefinition": {"sortBy": "count", "ranges": [{"from": "2026-01-01T00:00:00Z"}]}}]`)) + Expect(rr.Code).To(Equal(http.StatusOK), rr.Body.String()) + + Expect(captured().GetAggregations()[0].GetField()).To(Equal("Mtime")) + Expect(hitsContainer(rr).Aggregations[0].Field).To(Equal("lastModifiedDateTime")) + }) + ginkgo.DescribeTable("answers a property it does not evaluate with 501 instead of ignoring it", func(fragment, property string) { rr := postSearchQuery(graphWithoutSearch(), searchQueryBody(fragment)) @@ -364,7 +379,6 @@ var _ = ginkgo.Describe("SearchQuery", func() { ginkgo.Entry("aggregationFilters", `"aggregationFilters": ["audio.artist:\"ǂǂ5361786f6e\""]`, "aggregationFilters"), ginkgo.Entry("a geohash aggregation", `"aggregations": [{"field": "location", "@libre.graph.geohashDefinition": {"precision": 5}}]`, "geohashDefinition"), - ginkgo.Entry("ranges", `"aggregations": [{"field": "audio.year", "bucketDefinition": {"sortBy": "count", "ranges": [{"from": "1980"}]}}]`, "ranges"), ginkgo.Entry("a metric", `"aggregations": [{"field": "audio.year", "@libre.graph.metricDefinition": {"kind": "sum"}}]`, "metricDefinition"), ginkgo.Entry("sub-aggregations", `"aggregations": [{"field": "audio.artist", "@libre.graph.subAggregations": [{"field": "audio.album"}]}]`, "subAggregations"), ) diff --git a/services/search/pkg/aggregation/aggregation.go b/services/search/pkg/aggregation/aggregation.go index 46454c7a06..f5d6d16c87 100644 --- a/services/search/pkg/aggregation/aggregation.go +++ b/services/search/pkg/aggregation/aggregation.go @@ -1,6 +1,6 @@ // Package aggregation holds what the engines and the service layer share -// about aggregations: the bucket limit, the spelling of keys and the -// cross-space merge. +// about aggregations: the kind of an option, range bounds and the cross-space +// merge. package aggregation import ( @@ -11,6 +11,22 @@ import ( searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" ) +// Kind is what an aggregation option asks for. +type Kind int + +const ( + KindTerms Kind = iota + KindRange +) + +// KindOf tells the kind from the definitions an option carries. +func KindOf(opt *searchsvc.AggregationOption) Kind { + if len(opt.GetBucketDefinition().GetRanges()) > 0 { + return KindRange + } + return KindTerms +} + // MaxBuckets is how many buckets the aggregations of one request may have in // one space: the search.max_buckets default of OpenSearch, held on every // engine. diff --git a/services/search/pkg/aggregation/merge.go b/services/search/pkg/aggregation/merge.go index d64f01d79f..511cc664b6 100644 --- a/services/search/pkg/aggregation/merge.go +++ b/services/search/pkg/aggregation/merge.go @@ -2,13 +2,15 @@ package aggregation import ( "cmp" + "math" "slices" "strconv" searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" ) -// Empty returns the results of a search without matches: one per option. +// Empty returns the results of a search without matches: one per option, a +// range aggregation with every range at a count of zero. func Empty(opts []*searchsvc.AggregationOption) []*searchsvc.AggregationResult { if len(opts) == 0 { return nil @@ -16,6 +18,24 @@ func Empty(opts []*searchsvc.AggregationOption) []*searchsvc.AggregationResult { out := make([]*searchsvc.AggregationResult, len(opts)) for i, opt := range opts { out[i] = &searchsvc.AggregationResult{Field: opt.GetField()} + if KindOf(opt) == KindRange { + out[i].Buckets = RangeBuckets(opt, nil) + } + } + return out +} + +// RangeBuckets lists every range of a range aggregation in request order: the +// answered bucket where there is one, a count of zero otherwise. +func RangeBuckets(opt *searchsvc.AggregationOption, answered map[string]*searchsvc.Bucket) []*searchsvc.Bucket { + ranges := opt.GetBucketDefinition().GetRanges() + out := make([]*searchsvc.Bucket, 0, len(ranges)) + for _, r := range ranges { + bucket, ok := answered[RangeKey(r)] + if !ok { + bucket = &searchsvc.Bucket{Key: RangeKey(r)} + } + out = append(out, bucket) } return out } @@ -64,7 +84,9 @@ func shapeBuckets(opt *searchsvc.AggregationOption, buckets []*searchsvc.Bucket) slices.SortFunc(buckets, bucketOrder(opt)) - if size := int(opt.GetSize()); size > 0 && len(buckets) > size { + // size shapes terms aggregations only, a range aggregation returns the + // ranges it was asked for + if size := int(opt.GetSize()); size > 0 && KindOf(opt) == KindTerms && len(buckets) > size { buckets = buckets[:size] } return buckets @@ -88,9 +110,10 @@ func bucketOrder(opt *searchsvc.AggregationOption) func(a, b *searchsvc.Bucket) return func(a, b *searchsvc.Bucket) int { return direction * byKey(a, b) } case searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER: // keys that are no number come last, whatever the direction + number := keyNumbers(opt) return func(a, b *searchsvc.Bucket) int { - av, aok := keyNumber(a.GetKey()) - bv, bok := keyNumber(b.GetKey()) + av, aok := number(a.GetKey()) + bv, bok := number(b.GetKey()) return cmp.Or(cmp.Compare(boolInt(!aok), boolInt(!bok)), direction*cmp.Compare(av, bv), byKey(a, b)) } } @@ -106,7 +129,32 @@ func boolInt(b bool) int { return 0 } -func keyNumber(key string) (float64, bool) { - v, err := strconv.ParseFloat(key, 64) - return v, err == nil +// keyNumbers reads a terms key as a number and a range key as its lower +// bound, an open one as the smallest. +func keyNumbers(opt *searchsvc.AggregationOption) func(key string) (float64, bool) { + if KindOf(opt) != KindRange { + return func(key string) (float64, bool) { + v, err := strconv.ParseFloat(key, 64) + return v, err == nil + } + } + lower := map[string]float64{} + if ranges, err := ParseRanges(opt.GetField(), opt.GetBucketDefinition().GetRanges()); err == nil { + for _, r := range ranges.Numeric { + lower[r.Key] = math.Inf(-1) + if r.From != nil { + lower[r.Key] = *r.From + } + } + for _, r := range ranges.Dates { + lower[r.Key] = math.Inf(-1) + if !r.From.IsZero() { + lower[r.Key] = float64(r.From.UnixNano()) + } + } + } + return func(key string) (float64, bool) { + v, ok := lower[key] + return v, ok + } } diff --git a/services/search/pkg/aggregation/merge_test.go b/services/search/pkg/aggregation/merge_test.go index 1a36ceddc2..7a63e644ff 100644 --- a/services/search/pkg/aggregation/merge_test.go +++ b/services/search/pkg/aggregation/merge_test.go @@ -18,6 +18,10 @@ func bucket(key string, count int64) *searchsvc.Bucket { return &searchsvc.Bucket{Key: key, Count: count} } +func rangesOf(sortBy searchsvc.BucketSortBy, rs ...*searchsvc.BucketRange) *searchsvc.BucketDefinition { + return &searchsvc.BucketDefinition{SortBy: sortBy, Ranges: rs} +} + // render flattens results like the parity suite: one line per bucket. func render(results []*searchsvc.AggregationResult) []string { out := []string{} @@ -40,13 +44,16 @@ func merged(opts []*searchsvc.AggregationOption, spaces ...[]*searchsvc.Aggregat } var _ = Describe("Merge and Finalize", func() { - It("answers one result per option without any space", func() { - opts := []*searchsvc.AggregationOption{{Field: "audio.artist"}, {Field: "audio.album"}} + It("answers one result per option without any space, every range at zero", func() { + opts := []*searchsvc.AggregationOption{ + {Field: "audio.artist"}, + {Field: "audio.year", BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, &searchsvc.BucketRange{To: "1980"}, &searchsvc.BucketRange{From: "1980"})}, + } got := merged(opts) Expect(got).To(HaveLen(2)) Expect(got[0].GetField()).To(Equal("audio.artist")) Expect(got[0].GetBuckets()).To(BeEmpty()) - Expect(got[1].GetField()).To(Equal("audio.album")) + Expect(render(got)).To(Equal([]string{"audio.year ..1980=0", "audio.year 1980..=0"})) }) It("adds up the counts of a bucket across spaces", func() { @@ -77,6 +84,24 @@ var _ = Describe("Merge and Finalize", func() { })) }) + It("keeps two range aggregations on one field apart", func() { + opts := []*searchsvc.AggregationOption{ + {Field: "audio.year", BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, &searchsvc.BucketRange{To: "1980"}, &searchsvc.BucketRange{From: "1980"})}, + {Field: "audio.year", BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, &searchsvc.BucketRange{To: "2000"}, &searchsvc.BucketRange{From: "2000"})}, + } + space := func(below1980, from1980, below2000, from2000 int64) []*searchsvc.AggregationResult { + return []*searchsvc.AggregationResult{ + buckets("audio.year", bucket("..1980", below1980), bucket("1980..", from1980)), + buckets("audio.year", bucket("..2000", below2000), bucket("2000..", from2000)), + } + } + got := merged(opts, space(1, 1, 2, 0), space(0, 2, 1, 1)) + Expect(render(got)).To(Equal([]string{ + "audio.year ..1980=1", "audio.year 1980..=3", + "audio.year ..2000=3", "audio.year 2000..=1", + })) + }) + Describe("bucket order", func() { space := []*searchsvc.AggregationResult{buckets("audio.track", bucket("10", 2), bucket("9", 2), bucket("2", 5), bucket("b-side", 1), @@ -108,6 +133,22 @@ var _ = Describe("Merge and Finalize", func() { Entry("keyAsNumber ascending, no number last", searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, false, "2", "9", "10", "b-side"), Entry("keyAsNumber descending, no number still last", searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, true, "10", "9", "2", "b-side"), ) + + It("sorts range buckets by their lower bound with keyAsNumber, an open one first", func() { + opts := []*searchsvc.AggregationOption{{Field: "audio.year", BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, + &searchsvc.BucketRange{From: "2000"}, &searchsvc.BucketRange{From: "990", To: "2000"}, &searchsvc.BucketRange{To: "990"}, + )}} + got := merged(opts, []*searchsvc.AggregationResult{buckets("audio.year", bucket("2000..", 1), bucket("990..2000", 1), bucket("..990", 1))}) + Expect(render(got)).To(Equal([]string{"audio.year ..990=1", "audio.year 990..2000=1", "audio.year 2000..=1"})) + }) + + It("sorts date range buckets by their lower bound with keyAsNumber", func() { + opts := []*searchsvc.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, + &searchsvc.BucketRange{From: "2019-01-01T00:00:00Z"}, &searchsvc.BucketRange{To: "2019-01-01T00:00:00Z"}, + )}} + got := merged(opts, []*searchsvc.AggregationResult{buckets("photo.takenDateTime", bucket("2019-01-01T00:00:00Z..", 1), bucket("..2019-01-01T00:00:00Z", 3))}) + Expect(render(got)).To(Equal([]string{"photo.takenDateTime ..2019-01-01T00:00:00Z=3", "photo.takenDateTime 2019-01-01T00:00:00Z..=1"})) + }) }) Describe("bucket selection", func() { @@ -120,6 +161,14 @@ var _ = Describe("Merge and Finalize", func() { Expect(render(got)).To(Equal([]string{"audio.artist Saxon=4"})) }) + It("returns every range of a range aggregation, whatever the size", func() { + opts := []*searchsvc.AggregationOption{{Field: "audio.year", Size: 1, BucketDefinition: rangesOf(searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, + &searchsvc.BucketRange{To: "1980"}, &searchsvc.BucketRange{From: "1980"}, + )}} + got := merged(opts, []*searchsvc.AggregationResult{buckets("audio.year", bucket("..1980", 0), bucket("1980..", 3))}) + Expect(render(got)).To(Equal([]string{"audio.year ..1980=0", "audio.year 1980..=3"})) + }) + It("drops buckets below the minimum count, counted across spaces", func() { opts := []*searchsvc.AggregationOption{{Field: "audio.artist", BucketDefinition: &searchsvc.BucketDefinition{MinimumCount: 2, IsDescending: true}}} got := merged(opts, diff --git a/services/search/pkg/aggregation/ranges.go b/services/search/pkg/aggregation/ranges.go new file mode 100644 index 0000000000..cc5a90102d --- /dev/null +++ b/services/search/pkg/aggregation/ranges.go @@ -0,0 +1,101 @@ +package aggregation + +import ( + "fmt" + "math" + "strconv" + "time" + + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" +) + +// Ranges are the parsed ranges of one aggregation or filter, all numeric or +// all dates. A range is from-inclusive and to-exclusive. +type Ranges struct { + Numeric []NumericRange + Dates []DateRange +} + +// NumericRange has a nil bound on an open side. +type NumericRange struct { + Key string + From, To *float64 +} + +// DateRange has a zero time on an open side. +type DateRange struct { + Key string + From, To time.Time +} + +// Contains is from-inclusive and to-exclusive. +func (r NumericRange) Contains(v float64) bool { + return (r.From == nil || v >= *r.From) && (r.To == nil || v < *r.To) +} + +// Contains is from-inclusive and to-exclusive. +func (r DateRange) Contains(t time.Time) bool { + return (r.From.IsZero() || !t.Before(r.From)) && (r.To.IsZero() || t.Before(r.To)) +} + +// RangeKey is the bucket key of a range, "from..to" with an open side left +// empty; the dots keep a negative bound readable. +func RangeKey(r *searchsvc.BucketRange) string { + return r.GetFrom() + ".." + r.GetTo() +} + +// ParseRanges accepts decimal numbers or RFC3339 dates as bounds, one kind for +// all ranges, and at least one bound per range. +func ParseRanges(field string, ranges []*searchsvc.BucketRange) (Ranges, error) { + var out Ranges + seen := make(map[string]struct{}, len(ranges)) + for _, r := range ranges { + key := RangeKey(r) + if r.GetFrom() == "" && r.GetTo() == "" { + return Ranges{}, fmt.Errorf("range on field %q needs a from or a to bound", field) + } + if _, ok := seen[key]; ok { + return Ranges{}, fmt.Errorf("duplicate range %q on field %q", key, field) + } + seen[key] = struct{}{} + + from, fromErr := parseNumber(r.GetFrom()) + to, toErr := parseNumber(r.GetTo()) + if fromErr == nil && toErr == nil { + out.Numeric = append(out.Numeric, NumericRange{Key: key, From: from, To: to}) + continue + } + start, startErr := parseDate(r.GetFrom()) + end, endErr := parseDate(r.GetTo()) + if startErr == nil && endErr == nil { + out.Dates = append(out.Dates, DateRange{Key: key, From: start, To: end}) + continue + } + return Ranges{}, fmt.Errorf("invalid range %q on field %q: bounds are numbers or RFC3339 dates", key, field) + } + if len(out.Numeric) > 0 && len(out.Dates) > 0 { + return Ranges{}, fmt.Errorf("ranges on field %q mix numbers and dates", field) + } + return out, nil +} + +func parseNumber(s string) (*float64, error) { + if s == "" { + return nil, nil + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, err + } + if math.IsNaN(v) || math.IsInf(v, 0) { + return nil, fmt.Errorf("%q is not a finite number", s) + } + return &v, nil +} + +func parseDate(s string) (time.Time, error) { + if s == "" { + return time.Time{}, nil + } + return time.Parse(time.RFC3339, s) +} diff --git a/services/search/pkg/aggregation/ranges_test.go b/services/search/pkg/aggregation/ranges_test.go new file mode 100644 index 0000000000..96fa33f4cd --- /dev/null +++ b/services/search/pkg/aggregation/ranges_test.go @@ -0,0 +1,65 @@ +package aggregation_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" +) + +var _ = Describe("ParseRanges", func() { + It("parses numeric ranges, an open side has no bound", func() { + got, err := aggregation.ParseRanges("audio.year", []*searchsvc.BucketRange{{To: "1980"}, {From: "1980", To: "1990.5"}, {From: "-3"}}) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Dates).To(BeEmpty()) + Expect(got.Numeric).To(HaveLen(3)) + Expect(got.Numeric[0].Key).To(Equal("..1980")) + Expect(got.Numeric[0].From).To(BeNil()) + Expect(*got.Numeric[0].To).To(Equal(1980.0)) + Expect(got.Numeric[1].Key).To(Equal("1980..1990.5")) + Expect(*got.Numeric[1].To).To(Equal(1990.5)) + Expect(got.Numeric[2].Key).To(Equal("-3..")) + Expect(*got.Numeric[2].From).To(Equal(-3.0)) + Expect(got.Numeric[2].To).To(BeNil()) + }) + + It("parses RFC3339 date ranges", func() { + got, err := aggregation.ParseRanges("photo.takenDateTime", []*searchsvc.BucketRange{{From: "2018-08-11T00:00:00Z", To: "2018-08-12T00:00:00+02:00"}, {From: "2019-01-01T00:00:00Z"}}) + Expect(err).ToNot(HaveOccurred()) + Expect(got.Numeric).To(BeEmpty()) + Expect(got.Dates).To(HaveLen(2)) + Expect(got.Dates[0].From).To(BeTemporally("==", time.Date(2018, 8, 11, 0, 0, 0, 0, time.UTC))) + Expect(got.Dates[0].To).To(BeTemporally("==", time.Date(2018, 8, 11, 22, 0, 0, 0, time.UTC))) + Expect(got.Dates[1].To.IsZero()).To(BeTrue()) + }) + + It("is from-inclusive and to-exclusive", func() { + got, err := aggregation.ParseRanges("audio.year", []*searchsvc.BucketRange{{From: "1980", To: "1990"}}) + Expect(err).ToNot(HaveOccurred()) + r := got.Numeric[0] + Expect(r.Contains(1979.9)).To(BeFalse()) + Expect(r.Contains(1980)).To(BeTrue()) + Expect(r.Contains(1989.9)).To(BeTrue()) + Expect(r.Contains(1990)).To(BeFalse()) + }) + + DescribeTable("rejects", + func(ranges ...*searchsvc.BucketRange) { + _, err := aggregation.ParseRanges("field", ranges) + Expect(err).To(HaveOccurred()) + }, + Entry("a range without a bound", &searchsvc.BucketRange{}), + Entry("a bound that is no number", &searchsvc.BucketRange{From: "1970", To: "198o"}), + Entry("a bound that is no date", &searchsvc.BucketRange{From: "2018-08-11T00:00:00Z", To: "not-a-date"}), + Entry("a date without a time", &searchsvc.BucketRange{From: "2018-08-11"}), + Entry("a number and a date in one range", &searchsvc.BucketRange{From: "1970", To: "2018-08-11T00:00:00Z"}), + Entry("numeric and date ranges mixed", &searchsvc.BucketRange{From: "1970"}, &searchsvc.BucketRange{From: "2018-08-11T00:00:00Z"}), + Entry("the same range twice", &searchsvc.BucketRange{From: "1970"}, &searchsvc.BucketRange{From: "1970"}), + Entry("NaN", &searchsvc.BucketRange{From: "NaN"}), + Entry("infinity", &searchsvc.BucketRange{To: "Inf"}), + Entry("query syntax", &searchsvc.BucketRange{From: "1980 OR Name:*", To: "1990"}), + ) +}) diff --git a/services/search/pkg/aggregation/validate.go b/services/search/pkg/aggregation/validate.go index d0767d58a5..93fc50c3d0 100644 --- a/services/search/pkg/aggregation/validate.go +++ b/services/search/pkg/aggregation/validate.go @@ -8,7 +8,7 @@ import ( ) // ValidateOptions holds aggregations to what the index can answer: a field it -// knows, of a type whose values are buckets. fieldType returns the mapping +// knows, a kind that fits the type of the field. fieldType returns the mapping // type of a field, empty for one a client may not aggregate on. func ValidateOptions(opts []*searchsvc.AggregationOption, fieldType func(string) string) error { for _, opt := range opts { @@ -16,21 +16,42 @@ func ValidateOptions(opts []*searchsvc.AggregationOption, fieldType func(string) if t == "" { return fmt.Errorf("unknown aggregation field %q", field) } - if err := validateTermsField(field, t); err != nil { - return err + switch KindOf(opt) { + case KindRange: + parsed, err := ParseRanges(field, opt.GetBucketDefinition().GetRanges()) + if err != nil { + return err + } + if err := validateRangeField(field, t, parsed); err != nil { + return err + } + case KindTerms: + if err := validateTermsField(field, t); err != nil { + return err + } } } return nil } // validateTermsField allows the types whose values are buckets as they are. -// The values of a date are instants, no bucket each. +// The values of a date are instants, they aggregate in ranges. func validateTermsField(field, fieldType string) error { switch fieldType { case mapping.TypeKeyword, mapping.TypeBool, mapping.TypeNumeric: return nil case mapping.TypeDatetime: - return fmt.Errorf("terms aggregation is not supported on date field %q", field) + return fmt.Errorf("terms aggregation is not supported on date field %q; use bucketDefinition.ranges", field) } return fmt.Errorf("terms aggregation is not supported on field %q", field) } + +func validateRangeField(field, fieldType string, ranges Ranges) error { + if len(ranges.Numeric) > 0 && fieldType != mapping.TypeNumeric { + return fmt.Errorf("numeric ranges need a numeric field, %q is none", field) + } + if len(ranges.Dates) > 0 && fieldType != mapping.TypeDatetime { + return fmt.Errorf("date ranges need a date field, %q is none", field) + } + return nil +} diff --git a/services/search/pkg/aggregation/validate_test.go b/services/search/pkg/aggregation/validate_test.go index b9feb964f0..a22fad545f 100644 --- a/services/search/pkg/aggregation/validate_test.go +++ b/services/search/pkg/aggregation/validate_test.go @@ -25,6 +25,8 @@ func fieldTypes(field string) string { } var _ = Describe("ValidateOptions", func() { + numeric := &searchsvc.BucketDefinition{Ranges: []*searchsvc.BucketRange{{From: "1980", To: "1990"}}} + dates := &searchsvc.BucketDefinition{Ranges: []*searchsvc.BucketRange{{From: "2018-08-11T00:00:00Z"}}} DescribeTable("accepts an aggregation that fits its field", func(opt *searchsvc.AggregationOption) { @@ -33,6 +35,8 @@ var _ = Describe("ValidateOptions", func() { Entry("terms on a keyword", &searchsvc.AggregationOption{Field: "audio.artist"}), Entry("terms on a bool", &searchsvc.AggregationOption{Field: "audio.hasDrm"}), Entry("terms on a number", &searchsvc.AggregationOption{Field: "audio.year"}), + Entry("numeric ranges on a number", &searchsvc.AggregationOption{Field: "audio.year", BucketDefinition: numeric}), + Entry("date ranges on a date", &searchsvc.AggregationOption{Field: "photo.takenDateTime", BucketDefinition: dates}), ) DescribeTable("rejects an aggregation the index cannot answer", @@ -45,5 +49,8 @@ var _ = Describe("ValidateOptions", func() { Entry("terms on a path", &searchsvc.AggregationOption{Field: "Path"}), Entry("terms on a facet object", &searchsvc.AggregationOption{Field: "audio"}), Entry("terms on a geopoint", &searchsvc.AggregationOption{Field: "location"}), + Entry("numeric ranges on a keyword", &searchsvc.AggregationOption{Field: "audio.artist", BucketDefinition: numeric}), + Entry("numeric ranges on a date", &searchsvc.AggregationOption{Field: "photo.takenDateTime", BucketDefinition: numeric}), + Entry("date ranges on a number", &searchsvc.AggregationOption{Field: "audio.year", BucketDefinition: dates}), ) }) diff --git a/services/search/pkg/bleve/aggregations.go b/services/search/pkg/bleve/aggregations.go index b73f1e7186..142121eda1 100644 --- a/services/search/pkg/bleve/aggregations.go +++ b/services/search/pkg/bleve/aggregations.go @@ -3,6 +3,7 @@ package bleve import ( "context" "errors" + "time" "github.com/blevesearch/bleve/v2/numeric" bleveSearch "github.com/blevesearch/bleve/v2/search" @@ -35,17 +36,27 @@ func termKey(fieldType, term string) (string, bool) { type aggLevel struct { opt *searchService.AggregationOption + kind aggregation.Kind fieldType string + ranges aggregation.Ranges } // numeric values are prefix-coded in the index, their terms come from the // decoded doc values func (l *aggLevel) numericTerms() bool { - return l.fieldType == mapping.TypeNumeric + return l.kind == aggregation.KindTerms && l.fieldType == mapping.TypeNumeric } -func newAggLevel(opt *searchService.AggregationOption) *aggLevel { - return &aggLevel{opt: opt, fieldType: search.FieldType(opt.GetField())} +func newAggLevel(opt *searchService.AggregationOption) (*aggLevel, error) { + l := &aggLevel{opt: opt, kind: aggregation.KindOf(opt), fieldType: search.FieldType(opt.GetField())} + if l.kind == aggregation.KindRange { + ranges, err := aggregation.ParseRanges(opt.GetField(), opt.GetBucketDefinition().GetRanges()) + if err != nil { + return nil, err + } + l.ranges = ranges + } + return l, nil } // fieldValues is per-document scratch, reused across documents. @@ -76,17 +87,20 @@ type aggRoot struct { acc *bucketAcc } -func newAggCollector(aggs []*searchService.AggregationOption) *aggCollector { +func newAggCollector(aggs []*searchService.AggregationOption) (*aggCollector, error) { if len(aggs) == 0 { - return nil + return nil, nil } c := &aggCollector{fields: map[string]*fieldValues{}} for _, agg := range aggs { - l := newAggLevel(agg) + l, err := newAggLevel(agg) + if err != nil { + return nil, err + } c.register(l) c.roots = append(c.roots, &aggRoot{level: l, acc: newBucketAcc()}) } - return c + return c, nil } func (c *aggCollector) register(l *aggLevel) { @@ -96,10 +110,10 @@ func (c *aggCollector) register(l *aggLevel) { c.fields[l.opt.GetField()] = fv c.fieldNames = append(c.fieldNames, l.opt.GetField()) } - if l.numericTerms() { - fv.asNumbers = true - } else { + if l.kind == aggregation.KindTerms && !l.numericTerms() { fv.asTerms = true + } else { + fv.asNumbers = true } } @@ -143,8 +157,8 @@ func (c *aggCollector) collect(dvr index.DocValueReader, d *bleveSearch.Document return nil } -// Numeric doc values are prefix-coded at several precisions; only shift 0 -// carries the exact value. +// Numeric and date doc values are prefix-coded at several precisions; only +// shift 0 carries the exact value. func (c *aggCollector) visit(field string, term []byte) { fv, ok := c.fields[field] if !ok { @@ -165,15 +179,31 @@ func (c *aggCollector) visit(field string, term []byte) { func (c *aggCollector) fold(a *bucketAcc, l *aggLevel) { fv := c.fields[l.opt.GetField()] - if l.numericTerms() { - for _, raw := range fv.numbers { - c.foldBucket(a, l, aggregation.NumberKey(numeric.Int64ToFloat64(raw))) + switch l.kind { + case aggregation.KindTerms: + if l.numericTerms() { + for _, raw := range fv.numbers { + c.foldBucket(a, l, aggregation.NumberKey(numeric.Int64ToFloat64(raw))) + } + return } - return - } - for _, term := range fv.terms { - if key, ok := termKey(l.fieldType, term); ok { - c.foldBucket(a, l, key) + for _, term := range fv.terms { + if key, ok := termKey(l.fieldType, term); ok { + c.foldBucket(a, l, key) + } + } + case aggregation.KindRange: + for _, raw := range fv.numbers { + for _, r := range l.ranges.Numeric { + if r.Contains(numeric.Int64ToFloat64(raw)) { + c.foldBucket(a, l, r.Key) + } + } + for _, r := range l.ranges.Dates { + if r.Contains(time.Unix(0, raw)) { + c.foldBucket(a, l, r.Key) + } + } } } } @@ -198,6 +228,10 @@ func (a *bucketAcc) result(l *aggLevel) *searchService.AggregationResult { for key, count := range a.counts { counted[key] = &searchService.Bucket{Key: key, Count: count} } + if l.kind == aggregation.KindRange { + r.Buckets = aggregation.RangeBuckets(l.opt, counted) + return r + } r.Buckets = make([]*searchService.Bucket, 0, len(counted)) for _, b := range counted { r.Buckets = append(r.Buckets, b) diff --git a/services/search/pkg/bleve/backend.go b/services/search/pkg/bleve/backend.go index 1ac9191819..45f778e774 100644 --- a/services/search/pkg/bleve/backend.go +++ b/services/search/pkg/bleve/backend.go @@ -94,7 +94,10 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ bleveReq.SortBy([]string{"-_score", "_id"}) bleveReq.Size = size - collector := newAggCollector(sir.GetAggregations()) + collector, err := newAggCollector(sir.GetAggregations()) + if err != nil { + return nil, errtypes.BadRequest(err.Error()) + } if collector != nil { ctx = collector.withContext(ctx) } diff --git a/services/search/pkg/opensearch/internal/aggs/aggs.go b/services/search/pkg/opensearch/internal/aggs/aggs.go index 41146ac36f..dfbff02d6e 100644 --- a/services/search/pkg/opensearch/internal/aggs/aggs.go +++ b/services/search/pkg/opensearch/internal/aggs/aggs.go @@ -6,6 +6,7 @@ package aggs import ( "encoding/json" "fmt" + "time" searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" @@ -35,7 +36,50 @@ func aggName(position int) string { } func build(opt *searchsvc.AggregationOption) (map[string]any, error) { - return map[string]any{"terms": map[string]any{"field": opt.GetField(), "size": aggregation.MaxBuckets}}, nil + var entry map[string]any + switch aggregation.KindOf(opt) { + case aggregation.KindRange: + ranges, err := buildRanges(opt.GetField(), opt.GetBucketDefinition().GetRanges()) + if err != nil { + return nil, err + } + entry = ranges + default: + entry = map[string]any{"terms": map[string]any{"field": opt.GetField(), "size": aggregation.MaxBuckets}} + } + return entry, nil +} + +// buildRanges renders the numeric "range" or the "date_range" aggregation. +func buildRanges(field string, ranges []*searchsvc.BucketRange) (map[string]any, error) { + parsed, err := aggregation.ParseRanges(field, ranges) + if err != nil { + return nil, err + } + kind := "range" + out := make([]map[string]any, 0, len(ranges)) + for _, r := range parsed.Numeric { + entry := map[string]any{"key": r.Key} + if r.From != nil { + entry["from"] = *r.From + } + if r.To != nil { + entry["to"] = *r.To + } + out = append(out, entry) + } + for _, r := range parsed.Dates { + kind = "date_range" + entry := map[string]any{"key": r.Key} + if !r.From.IsZero() { + entry["from"] = r.From.Format(time.RFC3339Nano) + } + if !r.To.IsZero() { + entry["to"] = r.To.Format(time.RFC3339Nano) + } + out = append(out, entry) + } + return map[string]any{kind: map[string]any{"field": field, "ranges": out}}, nil } // Parse reads the aggregation block of a response into one result per option, @@ -84,6 +128,7 @@ func result(opt *searchsvc.AggregationOption, raw json.RawMessage) (*searchsvc.A } res := &searchsvc.AggregationResult{Field: opt.GetField(), Buckets: []*searchsvc.Bucket{}} + byKey := make(map[string]*searchsvc.Bucket, len(body.Buckets)) for _, rawBucket := range body.Buckets { bucket, err := parseBucket(opt, rawBucket) if err != nil { @@ -92,8 +137,12 @@ func result(opt *searchsvc.AggregationOption, raw json.RawMessage) (*searchsvc.A // a field without a value has no bucket, an empty one neither if bucket.GetKey() != "" { res.Buckets = append(res.Buckets, bucket) + byKey[bucket.GetKey()] = bucket } } + if aggregation.KindOf(opt) == aggregation.KindRange { + res.Buckets = aggregation.RangeBuckets(opt, byKey) + } return res, nil } diff --git a/services/search/pkg/opensearch/internal/aggs/aggs_test.go b/services/search/pkg/opensearch/internal/aggs/aggs_test.go index 6602ebc816..c7925a06ba 100644 --- a/services/search/pkg/opensearch/internal/aggs/aggs_test.go +++ b/services/search/pkg/opensearch/internal/aggs/aggs_test.go @@ -86,6 +86,43 @@ var _ = Describe("Aggregations", func() { } }) + DescribeTable("builds a range or date_range aggregation by the kind of the bounds, open ones without", + func(field, kind string, ranges []*searchsvc.BucketRange, want []map[string]any) { + r := build(&searchsvc.AggregationOption{Field: field, BucketDefinition: &searchsvc.BucketDefinition{Ranges: ranges}})["a_0"].(map[string]any)[kind].(map[string]any) + Expect(r["field"]).To(Equal(field)) + Expect(r["ranges"]).To(Equal(want)) + }, + Entry("numbers", "audio.year", "range", + []*searchsvc.BucketRange{{From: "1970", To: "1980"}, {To: "1970"}, {From: "2020"}}, + []map[string]any{{"key": "1970..1980", "from": 1970.0, "to": 1980.0}, {"key": "..1970", "to": 1970.0}, {"key": "2020..", "from": 2020.0}}), + Entry("dates", "photo.takenDateTime", "date_range", + []*searchsvc.BucketRange{{From: "2018-08-01T00:00:00Z", To: "2018-09-01T00:00:00Z"}, {From: "2018-08-11T00:00:00Z"}}, + []map[string]any{{"key": "2018-08-01T00:00:00Z..2018-09-01T00:00:00Z", "from": "2018-08-01T00:00:00Z", "to": "2018-09-01T00:00:00Z"}, {"key": "2018-08-11T00:00:00Z..", "from": "2018-08-11T00:00:00Z"}}), + ) + + malformed := &searchsvc.AggregationOption{ + Field: "photo.takenDateTime", + BucketDefinition: &searchsvc.BucketDefinition{Ranges: []*searchsvc.BucketRange{{From: "2018-08-11T00:00:00Z", To: "not-a-date"}}}, + } + + It("rejects a malformed range", func() { + _, err := aggs.Build([]*searchsvc.AggregationOption{malformed}) + Expect(err).To(HaveOccurred()) + }) + + It("lists every requested range in request order, empty ones included", func() { + out := parse(`{"a_0": {"buckets": [ + {"key": "1970..1980", "doc_count": 4}, + {"key": "1980..1990", "doc_count": 0} + ]}}`, &searchsvc.AggregationOption{ + Field: "audio.year", + BucketDefinition: &searchsvc.BucketDefinition{ + Ranges: []*searchsvc.BucketRange{{From: "1980", To: "1990"}, {From: "1970", To: "1980"}, {From: "1990"}}, + }, + }) + Expect(keys(out[0])).To(Equal([]string{"1980..1990=0", "1970..1980=4", "1990..=0"})) + }) + DescribeTable("reports a result it cannot decode instead of dropping it", func(opt *searchsvc.AggregationOption, response string) { _, err := aggs.Parse([]*searchsvc.AggregationOption{opt}, json.RawMessage(response)) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index 38781e7926..3cb800df32 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -729,12 +729,21 @@ Fixtures: - `e.mp3`, MimeType = audio/mpeg - `f.mp3`, MimeType = audio/mpeg - `g.mp3`, MimeType = audio/mpeg +- `a.jpg`, MimeType = image/jpeg +- `b.jpg`, MimeType = image/jpeg +- `c.jpg`, MimeType = image/jpeg +- `d.jpg`, MimeType = image/jpeg | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | AGG-01 | `mediatype:audio` reads `term buckets on audio.artist` | audio.artist Motörhead=3, audio.artist Saxon=2 | audio.artist Motörhead=3, audio.artist Saxon=2 | audio.artist Motörhead=3, audio.artist Saxon=2 | ✅ | | AGG-02 | `mediatype:audio` reads `no aggregations requested` | no match | no match | no match | ✅ | | AGG-03 | `mediatype:audio` reads `artist and album buckets in one request` | audio.albu... Steel=2, audio.albu...Spades=1, audio.album Bomber=2, audio.artist Motörhead=3, audio.artist Saxon=2 | audio.albu... Steel=2, audio.albu...Spades=1, audio.album Bomber=2, audio.artist Motörhead=3, audio.artist Saxon=2 | audio.albu... Steel=2, audio.albu...Spades=1, audio.album Bomber=2, audio.artist Motörhead=3, audio.artist Saxon=2 | ✅ | +| AGG-04 | `mediatype:audio` reads `audio.year buckets per decade` | audio.year 1970..1980=2, audio.year 1980..1990=1, audio.year 1990..2000=1, audio.year 2000..2010=3 | audio.year 1970..1980=2, audio.year 1980..1990=1, audio.year 1990..2000=1, audio.year 2000..2010=3 | audio.year 1970..1980=2, audio.year 1980..1990=1, audio.year 1990..2000=1, audio.year 2000..2010=3 | ✅ | +| AGG-05 | `mediatype:audio` reads `open-ended audio.year ranges` | audio.year ..1990=3, audio.year 2000..=3 | audio.year ..1990=3, audio.year 2000..=3 | audio.year ..1990=3, audio.year 2000..=3 | ✅ | +| AGG-07 | `mediatype:image` reads `photo.takenDateTime buckets per date range, an empty range counts zero` | photo.take...00:00Z=0, photo.take...00:00Z=1, photo.take...00:00Z=2, photo.take...00:00Z=2 | photo.take...00:00Z=0, photo.take...00:00Z=1, photo.take...00:00Z=2, photo.take...00:00Z=2 | photo.take...00:00Z=0, photo.take...00:00Z=1, photo.take...00:00Z=2, photo.take...00:00Z=2 | ✅ | +| AGG-08 | `mediatype:image` reads `open-ended date ranges` | photo.take...00:00Z=3, photo.take...:00Z..=1 | photo.take...00:00Z=3, photo.take...:00Z..=1 | photo.take...00:00Z=3, photo.take...:00Z..=1 | ✅ | +| AGG-09 | `mediatype:image` reads `malformed date range bound` | bad request | bad request | bad request | ✅ | | AGG-34 | `mediatype:audio` reads `term buckets on the numeric audio.year` | audio.year 1971=1, audio.year 1975=1, audio.year 1982=1, audio.year 1999=1, audio.year 2001=1, audio.year 2005=1, audio.year 2009=1 | audio.year 1971=1, audio.year 1975=1, audio.year 1982=1, audio.year 1999=1, audio.year 2001=1, audio.year 2005=1, audio.year 2009=1 | audio.year 1971=1, audio.year 1975=1, audio.year 1982=1, audio.year 1999=1, audio.year 2001=1, audio.year 2005=1, audio.year 2009=1 | ✅ | | AGG-37 | `mediatype:audio` reads `term buckets on the bool audio.hasDrm, spelled true and false` | audio.hasDrm false=1, audio.hasDrm true=1 | audio.hasDrm false=1, audio.hasDrm true=1 | audio.hasDrm false=1, audio.hasDrm true=1 | ✅ | | AGG-39 | `mediatype:audio` reads `no bucket for the empty Title of every match` | no match | no match | no match | ✅ | diff --git a/services/search/pkg/parity/aggregations_test.go b/services/search/pkg/parity/aggregations_test.go index 727bc9f150..f25c62563c 100644 --- a/services/search/pkg/parity/aggregations_test.go +++ b/services/search/pkg/parity/aggregations_test.go @@ -3,6 +3,7 @@ package parity import ( "context" "fmt" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -53,6 +54,14 @@ func untaggedSong(name string, year int32) search.Resource { return fixtureDoc(name, withMime("audio/mpeg"), withAudio(&libregraph.Audio{Year: libregraph.PtrInt32(year)})) } +func photo(name, taken string) search.Resource { + t, err := time.Parse(time.RFC3339, taken) + if err != nil { + panic(err) + } + return fixtureDoc(name, withMime("image/jpeg"), withPhoto(&libregraph.Photo{TakenDateTime: &t})) +} + func song(name, artist, album string, year int32, opts ...fixtureOption) search.Resource { return fixtureDoc(name, append([]fixtureOption{withMime("audio/mpeg"), withAudio(&libregraph.Audio{ Artist: libregraph.PtrString(artist), @@ -76,10 +85,17 @@ func aggregationFixtures() []search.Resource { song("e.mp3", "Motörhead", "Ace of Spades", 2001), untaggedSong("f.mp3", 2005), untaggedSong("g.mp3", 2009), + photo("a.jpg", "2018-08-11T09:15:00Z"), + photo("b.jpg", "2018-08-11T19:42:00Z"), + photo("c.jpg", "2018-09-01T12:00:00Z"), + photo("d.jpg", "2021-08-11T08:00:00Z"), } } func aggregationCases() []aggCase { + ranges := func(rs ...*searchService.BucketRange) *searchService.BucketDefinition { + return &searchService.BucketDefinition{Ranges: rs} + } return []aggCase{ {id: 1, query: "mediatype:audio", reads: "term buckets on audio.artist", aggs: []*searchService.AggregationOption{{Field: "audio.artist", Size: 10}}, @@ -91,6 +107,44 @@ func aggregationCases() []aggCase { "audio.artist Saxon=2", "audio.artist Motörhead=3", "audio.album Wheels of Steel=2", "audio.album Bomber=2", "audio.album Ace of Spades=1", }}, + {id: 4, query: "mediatype:audio", reads: "audio.year buckets per decade", + aggs: []*searchService.AggregationOption{{Field: "audio.year", BucketDefinition: ranges( + &searchService.BucketRange{From: "1970", To: "1980"}, + &searchService.BucketRange{From: "1980", To: "1990"}, + &searchService.BucketRange{From: "1990", To: "2000"}, + &searchService.BucketRange{From: "2000", To: "2010"}, + )}}, + want: []string{"audio.year 1970..1980=2", "audio.year 1980..1990=1", "audio.year 1990..2000=1", "audio.year 2000..2010=3"}}, + {id: 5, query: "mediatype:audio", reads: "open-ended audio.year ranges", + aggs: []*searchService.AggregationOption{{Field: "audio.year", BucketDefinition: ranges( + &searchService.BucketRange{To: "1990"}, + &searchService.BucketRange{From: "2000"}, + )}}, + want: []string{"audio.year ..1990=3", "audio.year 2000..=3"}}, + {id: 7, query: "mediatype:image", reads: "photo.takenDateTime buckets per date range, an empty range counts zero", + aggs: []*searchService.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: ranges( + &searchService.BucketRange{From: "2018-08-11T00:00:00Z", To: "2018-08-12T00:00:00Z"}, + &searchService.BucketRange{From: "2018-08-01T00:00:00Z", To: "2018-09-01T00:00:00Z"}, + &searchService.BucketRange{From: "2021-01-01T00:00:00Z", To: "2022-01-01T00:00:00Z"}, + &searchService.BucketRange{From: "2023-01-01T00:00:00Z", To: "2024-01-01T00:00:00Z"}, + )}}, + want: []string{ + "photo.takenDateTime 2018-08-11T00:00:00Z..2018-08-12T00:00:00Z=2", + "photo.takenDateTime 2018-08-01T00:00:00Z..2018-09-01T00:00:00Z=2", + "photo.takenDateTime 2021-01-01T00:00:00Z..2022-01-01T00:00:00Z=1", + "photo.takenDateTime 2023-01-01T00:00:00Z..2024-01-01T00:00:00Z=0", + }}, + {id: 8, query: "mediatype:image", reads: "open-ended date ranges", + aggs: []*searchService.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: ranges( + &searchService.BucketRange{To: "2019-01-01T00:00:00Z"}, + &searchService.BucketRange{From: "2019-01-01T00:00:00Z"}, + )}}, + want: []string{"photo.takenDateTime ..2019-01-01T00:00:00Z=3", "photo.takenDateTime 2019-01-01T00:00:00Z..=1"}}, + {id: 9, query: "mediatype:image", reads: "malformed date range bound", + aggs: []*searchService.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: ranges( + &searchService.BucketRange{From: "2018-08-11T00:00:00Z", To: "not-a-date"}, + )}}, + wantBadRequest: true}, {id: 34, query: "mediatype:audio", reads: "term buckets on the numeric audio.year", aggs: []*searchService.AggregationOption{{Field: "audio.year"}}, want: []string{ diff --git a/services/search/pkg/parity/fixtures_test.go b/services/search/pkg/parity/fixtures_test.go index 7f1569ebbe..9e75f037a1 100644 --- a/services/search/pkg/parity/fixtures_test.go +++ b/services/search/pkg/parity/fixtures_test.go @@ -50,6 +50,10 @@ func withAudio(audio *libregraph.Audio) fixtureOption { return func(r *search.Resource) { r.Audio = audio } } +func withPhoto(photo *libregraph.Photo) fixtureOption { + return func(r *search.Resource) { r.Photo = photo } +} + func withLocation(location *libregraph.GeoCoordinates) fixtureOption { return func(r *search.Resource) { r.Location = location } }