diff --git a/protogen/gen/opencloud/services/search/v0/search.pb.go b/protogen/gen/opencloud/services/search/v0/search.pb.go index 5f053ad955..bf3eeacc11 100644 --- a/protogen/gen/opencloud/services/search/v0/search.pb.go +++ b/protogen/gen/opencloud/services/search/v0/search.pb.go @@ -25,6 +25,58 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type BucketSortBy int32 + +const ( + BucketSortBy_BUCKET_SORT_BY_UNSPECIFIED BucketSortBy = 0 + BucketSortBy_BUCKET_SORT_BY_COUNT BucketSortBy = 1 + BucketSortBy_BUCKET_SORT_BY_KEY_AS_STRING BucketSortBy = 2 + BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER BucketSortBy = 3 +) + +// Enum value maps for BucketSortBy. +var ( + BucketSortBy_name = map[int32]string{ + 0: "BUCKET_SORT_BY_UNSPECIFIED", + 1: "BUCKET_SORT_BY_COUNT", + 2: "BUCKET_SORT_BY_KEY_AS_STRING", + 3: "BUCKET_SORT_BY_KEY_AS_NUMBER", + } + BucketSortBy_value = map[string]int32{ + "BUCKET_SORT_BY_UNSPECIFIED": 0, + "BUCKET_SORT_BY_COUNT": 1, + "BUCKET_SORT_BY_KEY_AS_STRING": 2, + "BUCKET_SORT_BY_KEY_AS_NUMBER": 3, + } +) + +func (x BucketSortBy) Enum() *BucketSortBy { + p := new(BucketSortBy) + *p = x + return p +} + +func (x BucketSortBy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (BucketSortBy) Descriptor() protoreflect.EnumDescriptor { + return file_opencloud_services_search_v0_search_proto_enumTypes[0].Descriptor() +} + +func (BucketSortBy) Type() protoreflect.EnumType { + return &file_opencloud_services_search_v0_search_proto_enumTypes[0] +} + +func (x BucketSortBy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use BucketSortBy.Descriptor instead. +func (BucketSortBy) EnumDescriptor() ([]byte, []int) { + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{0} +} + type SearchRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -37,6 +89,8 @@ type SearchRequest struct { PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` Query string `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` Ref *v0.Reference `protobuf:"bytes,4,opt,name=ref,proto3" json:"ref,omitempty"` + // Optional. Aggregations (facets) to compute alongside the matches. + Aggregations []*AggregationOption `protobuf:"bytes,5,rep,name=aggregations,proto3" json:"aggregations,omitempty"` // Optional. Number of leading matches to skip in the globally sorted, // cross-space merged result list. From int32 `protobuf:"varint,7,opt,name=from,proto3" json:"from,omitempty"` @@ -102,6 +156,13 @@ func (x *SearchRequest) GetRef() *v0.Reference { return nil } +func (x *SearchRequest) GetAggregations() []*AggregationOption { + if x != nil { + return x.Aggregations + } + return nil +} + func (x *SearchRequest) GetFrom() int32 { if x != nil { return x.From @@ -119,6 +180,9 @@ type SearchResponse struct { // more results in the list NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` TotalMatches int32 `protobuf:"varint,3,opt,name=total_matches,json=totalMatches,proto3" json:"total_matches,omitempty"` + // Aggregation results, one entry per requested aggregation, in request + // order. + Aggregations []*AggregationResult `protobuf:"bytes,4,rep,name=aggregations,proto3" json:"aggregations,omitempty"` } func (x *SearchResponse) Reset() { @@ -174,6 +238,13 @@ func (x *SearchResponse) GetTotalMatches() int32 { return 0 } +func (x *SearchResponse) GetAggregations() []*AggregationResult { + if x != nil { + return x.Aggregations + } + return nil +} + type SearchIndexRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -186,6 +257,8 @@ type SearchIndexRequest struct { PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` Query string `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` Ref *v0.Reference `protobuf:"bytes,4,opt,name=ref,proto3" json:"ref,omitempty"` + // Optional. Per-space aggregations (facets) to compute alongside the matches. + Aggregations []*AggregationOption `protobuf:"bytes,5,rep,name=aggregations,proto3" json:"aggregations,omitempty"` } func (x *SearchIndexRequest) Reset() { @@ -248,6 +321,13 @@ func (x *SearchIndexRequest) GetRef() *v0.Reference { return nil } +func (x *SearchIndexRequest) GetAggregations() []*AggregationOption { + if x != nil { + return x.Aggregations + } + return nil +} + type SearchIndexResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -258,6 +338,10 @@ type SearchIndexResponse struct { // more results in the list NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` TotalMatches int32 `protobuf:"varint,3,opt,name=total_matches,json=totalMatches,proto3" json:"total_matches,omitempty"` + // Per-space aggregation results, one entry per requested aggregation, in + // request order. The service layer merges these across spaces before + // returning them to the caller. + Aggregations []*AggregationResult `protobuf:"bytes,4,rep,name=aggregations,proto3" json:"aggregations,omitempty"` } func (x *SearchIndexResponse) Reset() { @@ -313,6 +397,256 @@ func (x *SearchIndexResponse) GetTotalMatches() int32 { return 0 } +func (x *SearchIndexResponse) GetAggregations() []*AggregationResult { + if x != nil { + return x.Aggregations + } + return nil +} + +type AggregationOption struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Required. The indexed field to aggregate on. + Field string `protobuf:"bytes,1,opt,name=field,proto3" json:"field,omitempty"` + // Optional. Maximum number of buckets to return, applied after the + // cross-space merge. 0 returns every bucket. + Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + // Optional. Controls bucket selection, ordering and filtering. + BucketDefinition *BucketDefinition `protobuf:"bytes,3,opt,name=bucket_definition,json=bucketDefinition,proto3" json:"bucket_definition,omitempty"` +} + +func (x *AggregationOption) Reset() { + *x = AggregationOption{} + if protoimpl.UnsafeEnabled { + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AggregationOption) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AggregationOption) ProtoMessage() {} + +func (x *AggregationOption) ProtoReflect() protoreflect.Message { + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AggregationOption.ProtoReflect.Descriptor instead. +func (*AggregationOption) Descriptor() ([]byte, []int) { + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{4} +} + +func (x *AggregationOption) GetField() string { + if x != nil { + return x.Field + } + return "" +} + +func (x *AggregationOption) GetSize() int32 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *AggregationOption) GetBucketDefinition() *BucketDefinition { + if x != nil { + return x.BucketDefinition + } + return nil +} + +type BucketDefinition struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Unspecified orders like a facet, by count descending, whatever + // is_descending says. + SortBy BucketSortBy `protobuf:"varint,1,opt,name=sort_by,json=sortBy,proto3,enum=opencloud.services.search.v0.BucketSortBy" json:"sort_by,omitempty"` + IsDescending bool `protobuf:"varint,2,opt,name=is_descending,json=isDescending,proto3" json:"is_descending,omitempty"` + MinimumCount int32 `protobuf:"varint,3,opt,name=minimum_count,json=minimumCount,proto3" json:"minimum_count,omitempty"` +} + +func (x *BucketDefinition) Reset() { + *x = BucketDefinition{} + if protoimpl.UnsafeEnabled { + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BucketDefinition) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BucketDefinition) ProtoMessage() {} + +func (x *BucketDefinition) ProtoReflect() protoreflect.Message { + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BucketDefinition.ProtoReflect.Descriptor instead. +func (*BucketDefinition) Descriptor() ([]byte, []int) { + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{5} +} + +func (x *BucketDefinition) GetSortBy() BucketSortBy { + if x != nil { + return x.SortBy + } + return BucketSortBy_BUCKET_SORT_BY_UNSPECIFIED +} + +func (x *BucketDefinition) GetIsDescending() bool { + if x != nil { + return x.IsDescending + } + return false +} + +func (x *BucketDefinition) GetMinimumCount() int32 { + if x != nil { + return x.MinimumCount + } + return 0 +} + +// One result per AggregationOption, at the same position. +type AggregationResult struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Field string `protobuf:"bytes,1,opt,name=field,proto3" json:"field,omitempty"` + Buckets []*Bucket `protobuf:"bytes,2,rep,name=buckets,proto3" json:"buckets,omitempty"` +} + +func (x *AggregationResult) Reset() { + *x = AggregationResult{} + if protoimpl.UnsafeEnabled { + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AggregationResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AggregationResult) ProtoMessage() {} + +func (x *AggregationResult) ProtoReflect() protoreflect.Message { + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AggregationResult.ProtoReflect.Descriptor instead. +func (*AggregationResult) Descriptor() ([]byte, []int) { + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{6} +} + +func (x *AggregationResult) GetField() string { + if x != nil { + return x.Field + } + return "" +} + +func (x *AggregationResult) GetBuckets() []*Bucket { + if x != nil { + return x.Buckets + } + return nil +} + +type Bucket struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Count int64 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"` +} + +func (x *Bucket) Reset() { + *x = Bucket{} + if protoimpl.UnsafeEnabled { + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Bucket) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Bucket) ProtoMessage() {} + +func (x *Bucket) ProtoReflect() protoreflect.Message { + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Bucket.ProtoReflect.Descriptor instead. +func (*Bucket) Descriptor() ([]byte, []int) { + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{7} +} + +func (x *Bucket) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *Bucket) GetCount() int64 { + if x != nil { + return x.Count + } + return 0 +} + type IndexSpaceRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -327,7 +661,7 @@ type IndexSpaceRequest struct { func (x *IndexSpaceRequest) Reset() { *x = IndexSpaceRequest{} if protoimpl.UnsafeEnabled { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[4] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -340,7 +674,7 @@ func (x *IndexSpaceRequest) String() string { func (*IndexSpaceRequest) ProtoMessage() {} func (x *IndexSpaceRequest) ProtoReflect() protoreflect.Message { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[4] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -353,7 +687,7 @@ func (x *IndexSpaceRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use IndexSpaceRequest.ProtoReflect.Descriptor instead. func (*IndexSpaceRequest) Descriptor() ([]byte, []int) { - return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{4} + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{8} } func (x *IndexSpaceRequest) GetSpaceId() string { @@ -404,7 +738,7 @@ type IndexSpaceResponse struct { func (x *IndexSpaceResponse) Reset() { *x = IndexSpaceResponse{} if protoimpl.UnsafeEnabled { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[5] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -417,7 +751,7 @@ func (x *IndexSpaceResponse) String() string { func (*IndexSpaceResponse) ProtoMessage() {} func (x *IndexSpaceResponse) ProtoReflect() protoreflect.Message { - mi := &file_opencloud_services_search_v0_search_proto_msgTypes[5] + mi := &file_opencloud_services_search_v0_search_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -430,7 +764,7 @@ func (x *IndexSpaceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use IndexSpaceResponse.ProtoReflect.Descriptor instead. func (*IndexSpaceResponse) Descriptor() ([]byte, []int) { - return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{5} + return file_opencloud_services_search_v0_search_proto_rawDescGZIP(), []int{9} } func (x *IndexSpaceResponse) GetSpaceId() string { @@ -489,7 +823,7 @@ var file_opencloud_services_search_v0_search_proto_rawDesc = []byte{ 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xdb, 0x01, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb6, 0x02, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x09, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x48, 0x00, 0x52, 0x08, 0x70, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x88, 0x01, 0x01, 0x12, @@ -500,20 +834,31 @@ var file_opencloud_services_search_v0_search_proto_rawDesc = []byte{ 0x66, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, - 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x18, 0x0a, 0x04, 0x66, - 0x72, 0x6f, 0x6d, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, - 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x73, - 0x69, 0x7a, 0x65, 0x22, 0x9c, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, - 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, - 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, - 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x0a, - 0x0d, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x61, 0x74, 0x63, 0x68, - 0x65, 0x73, 0x22, 0xc6, 0x01, 0x0a, 0x12, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, + 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x59, 0x0a, 0x0c, 0x61, + 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, + 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x0c, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x6d, + 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x22, 0xf1, + 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x3d, 0x0a, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, + 0x30, 0x2e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, + 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, 0x50, + 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x74, 0x61, + 0x6c, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x53, 0x0a, + 0x0c, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, + 0x76, 0x30, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x52, 0x0c, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x22, 0xa1, 0x02, 0x0a, 0x12, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x09, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x48, 0x00, 0x52, 0x08, 0x70, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x88, 0x01, @@ -524,95 +869,145 @@ var file_opencloud_services_search_v0_search_proto_rawDesc = []byte{ 0x72, 0x65, 0x66, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x03, 0x72, 0x65, 0x66, 0x42, 0x0c, 0x0a, - 0x0a, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x22, 0xa1, 0x01, 0x0a, 0x13, - 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x2e, 0x76, 0x30, 0x2e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, - 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, - 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, - 0x74, 0x61, 0x6c, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x22, - 0x94, 0x01, 0x0a, 0x11, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, - 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x6f, 0x72, - 0x63, 0x65, 0x5f, 0x72, 0x65, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0c, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x52, 0x65, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x26, - 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, - 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x22, 0xd1, 0x01, 0x0a, 0x12, 0x49, 0x6e, 0x64, 0x65, 0x78, - 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, - 0x08, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x40, 0x0a, 0x0e, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x65, 0x64, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x0d, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, - 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x70, - 0x61, 0x63, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x32, 0xb3, 0x02, 0x0a, 0x0e, 0x53, - 0x65, 0x61, 0x72, 0x63, 0x68, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x85, 0x01, - 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x2b, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, + 0x63, 0x65, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x59, 0x0a, + 0x0c, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, + 0x76, 0x30, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x0c, 0x61, 0x67, 0x67, 0x72, + 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x70, 0x61, 0x67, + 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x22, 0xf6, 0x01, 0x0a, 0x13, 0x53, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, + 0x0a, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x26, 0x0a, + 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6d, + 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x74, 0x6f, + 0x74, 0x61, 0x6c, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x53, 0x0a, 0x0c, 0x61, 0x67, + 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x2f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, + 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x52, 0x0c, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, + 0xa6, 0x01, 0x0a, 0x11, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x18, 0x0a, 0x04, 0x73, + 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, + 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x61, 0x0a, 0x11, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x5f, + 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2e, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, + 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x10, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x44, 0x65, + 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xa1, 0x01, 0x0a, 0x10, 0x42, 0x75, 0x63, + 0x6b, 0x65, 0x74, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x43, 0x0a, + 0x07, 0x73, 0x6f, 0x72, 0x74, 0x5f, 0x62, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, + 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x42, 0x75, + 0x63, 0x6b, 0x65, 0x74, 0x53, 0x6f, 0x72, 0x74, 0x42, 0x79, 0x52, 0x06, 0x73, 0x6f, 0x72, 0x74, + 0x42, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x73, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x65, 0x6e, 0x64, + 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x73, 0x44, 0x65, 0x73, + 0x63, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x69, 0x6e, 0x69, 0x6d, + 0x75, 0x6d, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, + 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x69, 0x0a, 0x11, + 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x3e, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x20, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1a, 0x3a, 0x01, 0x2a, 0x22, 0x15, - 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x73, - 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x98, 0x01, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, - 0x70, 0x61, 0x63, 0x65, 0x12, 0x2f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x25, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1f, 0x3a, - 0x01, 0x2a, 0x22, 0x1a, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, - 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x30, 0x01, - 0x32, 0xa7, 0x01, 0x0a, 0x0d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x12, 0x95, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x30, 0x2e, + 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x07, + 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x22, 0x30, 0x0a, 0x06, 0x42, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x11, 0x49, 0x6e, + 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x19, 0x0a, 0x08, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, + 0x72, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x69, + 0x6e, 0x64, 0x65, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x66, 0x6f, 0x72, 0x63, + 0x65, 0x52, 0x65, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x26, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, + 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, + 0x41, 0x01, 0x01, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, + 0x22, 0xd1, 0x01, 0x0a, 0x12, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x49, 0x64, 0x12, 0x40, 0x0a, 0x0e, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x64, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x64, 0x5f, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, + 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x2a, 0x8c, 0x01, 0x0a, 0x0c, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, + 0x6f, 0x72, 0x74, 0x42, 0x79, 0x12, 0x1e, 0x0a, 0x1a, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, + 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, + 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, 0x59, 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x10, 0x01, 0x12, + 0x20, 0x0a, 0x1c, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, 0x5f, 0x42, + 0x59, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x41, 0x53, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, + 0x02, 0x12, 0x20, 0x0a, 0x1c, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x5f, 0x53, 0x4f, 0x52, 0x54, + 0x5f, 0x42, 0x59, 0x5f, 0x4b, 0x45, 0x59, 0x5f, 0x41, 0x53, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, + 0x52, 0x10, 0x03, 0x32, 0xb3, 0x02, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x50, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x85, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x12, 0x2b, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, + 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, + 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x20, 0x82, 0xd3, + 0xe4, 0x93, 0x02, 0x1a, 0x3a, 0x01, 0x2a, 0x22, 0x15, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, + 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x98, + 0x01, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x12, 0x2f, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, - 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x31, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, - 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x20, 0x3a, 0x01, 0x2a, 0x22, 0x1b, 0x2f, - 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, - 0x64, 0x65, 0x78, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x42, 0xf2, 0x02, 0x5a, 0x4a, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70, - 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, - 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x92, 0x41, 0xa2, 0x02, 0x12, 0xb7, 0x01, - 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x73, 0x65, 0x61, 0x72, - 0x63, 0x68, 0x22, 0x51, 0x0a, 0x0e, 0x4f, 0x70, 0x65, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, - 0x47, 0x6d, 0x62, 0x48, 0x12, 0x29, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x1a, - 0x14, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x2e, 0x65, 0x75, 0x2a, 0x49, 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2d, - 0x32, 0x2e, 0x30, 0x12, 0x3b, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x62, - 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, - 0x32, 0x05, 0x31, 0x2e, 0x30, 0x2e, 0x30, 0x2a, 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, - 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, - 0x3e, 0x0a, 0x10, 0x44, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, - 0x75, 0x61, 0x6c, 0x12, 0x2a, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x64, 0x6f, 0x63, - 0x73, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x65, 0x75, 0x2f, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64, + 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, + 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, + 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x25, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1f, 0x3a, 0x01, 0x2a, 0x22, 0x1a, 0x2f, 0x61, 0x70, + 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, + 0x78, 0x2d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x30, 0x01, 0x32, 0xa7, 0x01, 0x0a, 0x0d, 0x49, 0x6e, + 0x64, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x95, 0x01, 0x0a, 0x06, + 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x30, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, + 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, + 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4, + 0x93, 0x02, 0x20, 0x3a, 0x01, 0x2a, 0x22, 0x1b, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, + 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x73, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x42, 0xf2, 0x02, 0x5a, 0x4a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, + 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, + 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, + 0x76, 0x30, 0x92, 0x41, 0xa2, 0x02, 0x12, 0xb7, 0x01, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x6e, 0x43, + 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x22, 0x51, 0x0a, 0x0e, 0x4f, + 0x70, 0x65, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, 0x48, 0x12, 0x29, 0x68, + 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, + 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x1a, 0x14, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, + 0x74, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x65, 0x75, 0x2a, 0x49, + 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2d, 0x32, 0x2e, 0x30, 0x12, 0x3b, 0x68, 0x74, + 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, + 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x69, + 0x6e, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, 0x2e, 0x30, 0x2e, 0x30, + 0x2a, 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3e, 0x0a, 0x10, 0x44, 0x65, 0x76, 0x65, + 0x6c, 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x12, 0x2a, 0x68, 0x74, + 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x64, 0x6f, 0x63, 0x73, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, + 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x65, 0x75, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -627,35 +1022,48 @@ func file_opencloud_services_search_v0_search_proto_rawDescGZIP() []byte { return file_opencloud_services_search_v0_search_proto_rawDescData } -var file_opencloud_services_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_opencloud_services_search_v0_search_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_opencloud_services_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 10) var file_opencloud_services_search_v0_search_proto_goTypes = []interface{}{ - (*SearchRequest)(nil), // 0: opencloud.services.search.v0.SearchRequest - (*SearchResponse)(nil), // 1: opencloud.services.search.v0.SearchResponse - (*SearchIndexRequest)(nil), // 2: opencloud.services.search.v0.SearchIndexRequest - (*SearchIndexResponse)(nil), // 3: opencloud.services.search.v0.SearchIndexResponse - (*IndexSpaceRequest)(nil), // 4: opencloud.services.search.v0.IndexSpaceRequest - (*IndexSpaceResponse)(nil), // 5: opencloud.services.search.v0.IndexSpaceResponse - (*v0.Reference)(nil), // 6: opencloud.messages.search.v0.Reference - (*v0.Match)(nil), // 7: opencloud.messages.search.v0.Match - (*durationpb.Duration)(nil), // 8: google.protobuf.Duration + (BucketSortBy)(0), // 0: opencloud.services.search.v0.BucketSortBy + (*SearchRequest)(nil), // 1: opencloud.services.search.v0.SearchRequest + (*SearchResponse)(nil), // 2: opencloud.services.search.v0.SearchResponse + (*SearchIndexRequest)(nil), // 3: opencloud.services.search.v0.SearchIndexRequest + (*SearchIndexResponse)(nil), // 4: opencloud.services.search.v0.SearchIndexResponse + (*AggregationOption)(nil), // 5: opencloud.services.search.v0.AggregationOption + (*BucketDefinition)(nil), // 6: opencloud.services.search.v0.BucketDefinition + (*AggregationResult)(nil), // 7: opencloud.services.search.v0.AggregationResult + (*Bucket)(nil), // 8: opencloud.services.search.v0.Bucket + (*IndexSpaceRequest)(nil), // 9: opencloud.services.search.v0.IndexSpaceRequest + (*IndexSpaceResponse)(nil), // 10: opencloud.services.search.v0.IndexSpaceResponse + (*v0.Reference)(nil), // 11: opencloud.messages.search.v0.Reference + (*v0.Match)(nil), // 12: opencloud.messages.search.v0.Match + (*durationpb.Duration)(nil), // 13: google.protobuf.Duration } var file_opencloud_services_search_v0_search_proto_depIdxs = []int32{ - 6, // 0: opencloud.services.search.v0.SearchRequest.ref:type_name -> opencloud.messages.search.v0.Reference - 7, // 1: opencloud.services.search.v0.SearchResponse.matches:type_name -> opencloud.messages.search.v0.Match - 6, // 2: opencloud.services.search.v0.SearchIndexRequest.ref:type_name -> opencloud.messages.search.v0.Reference - 7, // 3: opencloud.services.search.v0.SearchIndexResponse.matches:type_name -> opencloud.messages.search.v0.Match - 8, // 4: opencloud.services.search.v0.IndexSpaceResponse.space_duration:type_name -> google.protobuf.Duration - 0, // 5: opencloud.services.search.v0.SearchProvider.Search:input_type -> opencloud.services.search.v0.SearchRequest - 4, // 6: opencloud.services.search.v0.SearchProvider.IndexSpace:input_type -> opencloud.services.search.v0.IndexSpaceRequest - 2, // 7: opencloud.services.search.v0.IndexProvider.Search:input_type -> opencloud.services.search.v0.SearchIndexRequest - 1, // 8: opencloud.services.search.v0.SearchProvider.Search:output_type -> opencloud.services.search.v0.SearchResponse - 5, // 9: opencloud.services.search.v0.SearchProvider.IndexSpace:output_type -> opencloud.services.search.v0.IndexSpaceResponse - 3, // 10: opencloud.services.search.v0.IndexProvider.Search:output_type -> opencloud.services.search.v0.SearchIndexResponse - 8, // [8:11] is the sub-list for method output_type - 5, // [5:8] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 11, // 0: opencloud.services.search.v0.SearchRequest.ref:type_name -> opencloud.messages.search.v0.Reference + 5, // 1: opencloud.services.search.v0.SearchRequest.aggregations:type_name -> opencloud.services.search.v0.AggregationOption + 12, // 2: opencloud.services.search.v0.SearchResponse.matches:type_name -> opencloud.messages.search.v0.Match + 7, // 3: opencloud.services.search.v0.SearchResponse.aggregations:type_name -> opencloud.services.search.v0.AggregationResult + 11, // 4: opencloud.services.search.v0.SearchIndexRequest.ref:type_name -> opencloud.messages.search.v0.Reference + 5, // 5: opencloud.services.search.v0.SearchIndexRequest.aggregations:type_name -> opencloud.services.search.v0.AggregationOption + 12, // 6: opencloud.services.search.v0.SearchIndexResponse.matches:type_name -> opencloud.messages.search.v0.Match + 7, // 7: opencloud.services.search.v0.SearchIndexResponse.aggregations:type_name -> opencloud.services.search.v0.AggregationResult + 6, // 8: opencloud.services.search.v0.AggregationOption.bucket_definition:type_name -> opencloud.services.search.v0.BucketDefinition + 0, // 9: opencloud.services.search.v0.BucketDefinition.sort_by:type_name -> opencloud.services.search.v0.BucketSortBy + 8, // 10: opencloud.services.search.v0.AggregationResult.buckets:type_name -> opencloud.services.search.v0.Bucket + 13, // 11: opencloud.services.search.v0.IndexSpaceResponse.space_duration:type_name -> google.protobuf.Duration + 1, // 12: opencloud.services.search.v0.SearchProvider.Search:input_type -> opencloud.services.search.v0.SearchRequest + 9, // 13: opencloud.services.search.v0.SearchProvider.IndexSpace:input_type -> opencloud.services.search.v0.IndexSpaceRequest + 3, // 14: opencloud.services.search.v0.IndexProvider.Search:input_type -> opencloud.services.search.v0.SearchIndexRequest + 2, // 15: opencloud.services.search.v0.SearchProvider.Search:output_type -> opencloud.services.search.v0.SearchResponse + 10, // 16: opencloud.services.search.v0.SearchProvider.IndexSpace:output_type -> opencloud.services.search.v0.IndexSpaceResponse + 4, // 17: opencloud.services.search.v0.IndexProvider.Search:output_type -> opencloud.services.search.v0.SearchIndexResponse + 15, // [15:18] is the sub-list for method output_type + 12, // [12:15] is the sub-list for method input_type + 12, // [12:12] is the sub-list for extension type_name + 12, // [12:12] is the sub-list for extension extendee + 0, // [0:12] is the sub-list for field type_name } func init() { file_opencloud_services_search_v0_search_proto_init() } @@ -713,7 +1121,7 @@ func file_opencloud_services_search_v0_search_proto_init() { } } file_opencloud_services_search_v0_search_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*IndexSpaceRequest); i { + switch v := v.(*AggregationOption); i { case 0: return &v.state case 1: @@ -725,6 +1133,54 @@ func file_opencloud_services_search_v0_search_proto_init() { } } file_opencloud_services_search_v0_search_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BucketDefinition); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_opencloud_services_search_v0_search_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AggregationResult); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_opencloud_services_search_v0_search_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Bucket); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_opencloud_services_search_v0_search_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IndexSpaceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_opencloud_services_search_v0_search_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*IndexSpaceResponse); i { case 0: return &v.state @@ -744,13 +1200,14 @@ func file_opencloud_services_search_v0_search_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_opencloud_services_search_v0_search_proto_rawDesc, - NumEnums: 0, - NumMessages: 6, + NumEnums: 1, + NumMessages: 10, NumExtensions: 0, NumServices: 2, }, GoTypes: file_opencloud_services_search_v0_search_proto_goTypes, DependencyIndexes: file_opencloud_services_search_v0_search_proto_depIdxs, + EnumInfos: file_opencloud_services_search_v0_search_proto_enumTypes, MessageInfos: file_opencloud_services_search_v0_search_proto_msgTypes, }.Build() File_opencloud_services_search_v0_search_proto = out.File diff --git a/protogen/gen/opencloud/services/search/v0/search.swagger.json b/protogen/gen/opencloud/services/search/v0/search.swagger.json index 11ebb9da16..793fdbf4fe 100644 --- a/protogen/gen/opencloud/services/search/v0/search.swagger.json +++ b/protogen/gen/opencloud/services/search/v0/search.swagger.json @@ -167,6 +167,39 @@ } } }, + "v0AggregationOption": { + "type": "object", + "properties": { + "field": { + "type": "string", + "description": "Required. The indexed field to aggregate on." + }, + "size": { + "type": "integer", + "format": "int32", + "description": "Optional. Maximum number of buckets to return, applied after the\ncross-space merge. 0 returns every bucket." + }, + "bucketDefinition": { + "$ref": "#/definitions/v0BucketDefinition", + "description": "Optional. Controls bucket selection, ordering and filtering." + } + } + }, + "v0AggregationResult": { + "type": "object", + "properties": { + "field": { + "type": "string" + }, + "buckets": { + "type": "array", + "items": { + "$ref": "#/definitions/v0Bucket" + } + } + }, + "description": "One result per AggregationOption, at the same position." + }, "v0Audio": { "type": "object", "properties": { @@ -227,6 +260,44 @@ } } }, + "v0Bucket": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "count": { + "type": "string", + "format": "int64" + } + } + }, + "v0BucketDefinition": { + "type": "object", + "properties": { + "sortBy": { + "$ref": "#/definitions/v0BucketSortBy", + "description": "Unspecified orders like a facet, by count descending, whatever\nis_descending says." + }, + "isDescending": { + "type": "boolean" + }, + "minimumCount": { + "type": "integer", + "format": "int32" + } + } + }, + "v0BucketSortBy": { + "type": "string", + "enum": [ + "BUCKET_SORT_BY_UNSPECIFIED", + "BUCKET_SORT_BY_COUNT", + "BUCKET_SORT_BY_KEY_AS_STRING", + "BUCKET_SORT_BY_KEY_AS_NUMBER" + ], + "default": "BUCKET_SORT_BY_UNSPECIFIED" + }, "v0Entity": { "type": "object", "properties": { @@ -527,6 +598,13 @@ }, "ref": { "$ref": "#/definitions/v0Reference" + }, + "aggregations": { + "type": "array", + "items": { + "$ref": "#/definitions/v0AggregationOption" + }, + "description": "Optional. Per-space aggregations (facets) to compute alongside the matches." } } }, @@ -546,6 +624,13 @@ "totalMatches": { "type": "integer", "format": "int32" + }, + "aggregations": { + "type": "array", + "items": { + "$ref": "#/definitions/v0AggregationResult" + }, + "description": "Per-space aggregation results, one entry per requested aggregation, in\nrequest order. The service layer merges these across spaces before\nreturning them to the caller." } } }, @@ -567,6 +652,13 @@ "ref": { "$ref": "#/definitions/v0Reference" }, + "aggregations": { + "type": "array", + "items": { + "$ref": "#/definitions/v0AggregationOption" + }, + "description": "Optional. Aggregations (facets) to compute alongside the matches." + }, "from": { "type": "integer", "format": "int32", @@ -590,6 +682,13 @@ "totalMatches": { "type": "integer", "format": "int32" + }, + "aggregations": { + "type": "array", + "items": { + "$ref": "#/definitions/v0AggregationResult" + }, + "description": "Aggregation results, one entry per requested aggregation, in request\norder." } } }, diff --git a/protogen/proto/opencloud/services/search/v0/search.proto b/protogen/proto/opencloud/services/search/v0/search.proto index 2ab1fb8a5a..2a5d743af0 100644 --- a/protogen/proto/opencloud/services/search/v0/search.proto +++ b/protogen/proto/opencloud/services/search/v0/search.proto @@ -73,6 +73,9 @@ message SearchRequest { string query = 3; opencloud.messages.search.v0.Reference ref = 4 [(google.api.field_behavior) = OPTIONAL]; + // Optional. Aggregations (facets) to compute alongside the matches. + repeated AggregationOption aggregations = 5 [(google.api.field_behavior) = OPTIONAL]; + // Optional. Number of leading matches to skip in the globally sorted, // cross-space merged result list. int32 from = 7 [(google.api.field_behavior) = OPTIONAL]; @@ -85,6 +88,10 @@ message SearchResponse { // more results in the list string next_page_token = 2; int32 total_matches = 3; + + // Aggregation results, one entry per requested aggregation, in request + // order. + repeated AggregationResult aggregations = 4; } message SearchIndexRequest { @@ -97,6 +104,9 @@ message SearchIndexRequest { string query = 3; opencloud.messages.search.v0.Reference ref = 4 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Per-space aggregations (facets) to compute alongside the matches. + repeated AggregationOption aggregations = 5 [(google.api.field_behavior) = OPTIONAL]; } message SearchIndexResponse { @@ -106,6 +116,47 @@ message SearchIndexResponse { // more results in the list string next_page_token = 2; int32 total_matches = 3; + + // Per-space aggregation results, one entry per requested aggregation, in + // request order. The service layer merges these across spaces before + // returning them to the caller. + repeated AggregationResult aggregations = 4; +} + +message AggregationOption { + // Required. The indexed field to aggregate on. + string field = 1; + // Optional. Maximum number of buckets to return, applied after the + // cross-space merge. 0 returns every bucket. + int32 size = 2 [(google.api.field_behavior) = OPTIONAL]; + // Optional. Controls bucket selection, ordering and filtering. + BucketDefinition bucket_definition = 3 [(google.api.field_behavior) = OPTIONAL]; +} + +message BucketDefinition { + // Unspecified orders like a facet, by count descending, whatever + // is_descending says. + BucketSortBy sort_by = 1; + bool is_descending = 2; + int32 minimum_count = 3; +} + +enum BucketSortBy { + BUCKET_SORT_BY_UNSPECIFIED = 0; + BUCKET_SORT_BY_COUNT = 1; + BUCKET_SORT_BY_KEY_AS_STRING = 2; + BUCKET_SORT_BY_KEY_AS_NUMBER = 3; +} + +// One result per AggregationOption, at the same position. +message AggregationResult { + string field = 1; + repeated Bucket buckets = 2; +} + +message Bucket { + string key = 1; + int64 count = 2; } message IndexSpaceRequest { diff --git a/services/graph/pkg/service/v0/searchquery.go b/services/graph/pkg/service/v0/searchquery.go index 38f31972f9..d12f040a23 100644 --- a/services/graph/pkg/service/v0/searchquery.go +++ b/services/graph/pkg/service/v0/searchquery.go @@ -14,6 +14,7 @@ import ( searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) @@ -88,16 +89,27 @@ func unsupportedProperty(sr libregraph.SearchRequest) string { if len(sr.SortProperties) > 0 { return "sortProperties" } - if len(sr.Aggregations) > 0 { - return "aggregations" - } if len(sr.AggregationFilters) > 0 { return "aggregationFilters" } + for _, a := range sr.Aggregations { + switch { + case a.LibreGraphGeohashDefinition != nil: + return "@libre.graph.geohashDefinition" + case len(a.LibreGraphSubAggregations) > 0: + return "@libre.graph.subAggregations" + case a.LibreGraphMetricDefinition != nil: + return "@libre.graph.metricDefinition" + case len(a.BucketDefinition.GetRanges()) > 0: + return "bucketDefinition.ranges" + } + } return "" } -// searchRequestOf validates one request against the spec and translates it. +// searchRequestOf validates one request against the spec and the index rules +// (the search service checks them again) and translates it. Fields are +// validated under the request's names so an error reads like the request. func searchRequestOf(sr libregraph.SearchRequest) (*searchsvc.SearchRequest, error) { if err := validateEntityTypes(sr.EntityTypes); err != nil { return nil, err @@ -105,11 +117,22 @@ func searchRequestOf(sr libregraph.SearchRequest) (*searchsvc.SearchRequest, err if err := validatePagination(sr.From, sr.Size); err != nil { return nil, err } + if err := validateAggregations(sr.Aggregations); err != nil { + return nil, err + } + aggregations := libregraphAggregationsToSearch(sr.Aggregations) + requestFieldType := func(field string) string { return search.AggregatableFieldType(indexField(field)) } + if err := aggregation.ValidateOptions(aggregations, requestFieldType); err != nil { + return nil, err + } + resolveFields(aggregations) + from, size := pagination(sr.From, sr.Size) return &searchsvc.SearchRequest{ - Query: sr.Query.QueryString, - From: from, - PageSize: &size, + Query: sr.Query.QueryString, + From: from, + PageSize: &size, + Aggregations: aggregations, }, nil } @@ -163,6 +186,7 @@ func (g Graph) runSingleSearch(ctx context.Context, sr libregraph.SearchRequest, Hits: hits, Total: &total, MoreResultsAvailable: &more, + Aggregations: searchAggregationsToLibregraph(rsp.Aggregations, sr.Aggregations), }}, }, nil } @@ -196,6 +220,29 @@ func validatePagination(fromP, sizeP *int32) error { return search.CheckResultWindow(from, size) } +// openapi-generator enforces neither the enums nor the bounds of the spec. +// Whether an aggregation fits its field is for aggregation.ValidateOptions to +// say. +func validateAggregations(aggs []libregraph.AggregationOption) error { + for _, a := range aggs { + if a.Field == "" { + return fmt.Errorf("aggregation field must not be empty") + } + if a.Size != nil && *a.Size < 1 { + return fmt.Errorf("size of the aggregation on %q must be at least 1", a.Field) + } + if bd := a.BucketDefinition; bd != nil { + if _, ok := bucketSortBy[bd.SortBy]; !ok { + return fmt.Errorf("unsupported sortBy %q on field %q", bd.SortBy, a.Field) + } + if bd.MinimumCount != nil && *bd.MinimumCount < 0 { + return fmt.Errorf("minimumCount of the aggregation on %q must not be negative", a.Field) + } + } + } + return nil +} + // renderSearchError answers with the status the search service failed with. func (g Graph) renderSearchError(w http.ResponseWriter, r *http.Request, err error) { e := merrors.Parse(err.Error()) diff --git a/services/graph/pkg/service/v0/searchquery_mapping.go b/services/graph/pkg/service/v0/searchquery_mapping.go index 6f536db692..088e6a4d44 100644 --- a/services/graph/pkg/service/v0/searchquery_mapping.go +++ b/services/graph/pkg/service/v0/searchquery_mapping.go @@ -3,6 +3,7 @@ package svc import ( "path" "slices" + "strings" "time" storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -10,9 +11,97 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storagespace" searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" ) +var bucketSortBy = map[string]searchsvc.BucketSortBy{ + "count": searchsvc.BucketSortBy_BUCKET_SORT_BY_COUNT, + "keyAsString": searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_STRING, + "keyAsNumber": searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, +} + +// driveItemFields are the scalar driveItem properties a hit carries and the +// index fields behind them; a facet property (audio.artist) is its index +// field already. +var driveItemFields = map[string]string{ + "name": "Name", + "size": "Size", + "lastModifiedDateTime": "Mtime", + "mimeType": "MimeType", + "@libre.graph.tags": "Tags", +} + +// indexField resolves a field spelled like the driveItem property, and only +// so: a scalar property by name, a facet property as it is. Any other +// spelling resolves to no field and fails validation. +func indexField(field string) string { + if index, ok := driveItemFields[field]; ok { + return index + } + if strings.Contains(field, ".") { + return field + } + return "" +} + +func resolveFields(opts []*searchsvc.AggregationOption) { + for _, opt := range opts { + opt.Field = indexField(opt.Field) + } +} + +func libregraphAggregationsToSearch(in []libregraph.AggregationOption) []*searchsvc.AggregationOption { + out := make([]*searchsvc.AggregationOption, 0, len(in)) + for _, a := range in { + agg := &searchsvc.AggregationOption{Field: a.Field} + if a.Size != nil { + agg.Size = *a.Size + } + if a.BucketDefinition != nil { + agg.BucketDefinition = libregraphBucketDefinitionToSearch(*a.BucketDefinition) + } + out = append(out, agg) + } + return out +} + +func libregraphBucketDefinitionToSearch(in libregraph.BucketDefinition) *searchsvc.BucketDefinition { + bd := &searchsvc.BucketDefinition{SortBy: bucketSortBy[in.SortBy]} + if in.IsDescending != nil { + bd.IsDescending = *in.IsDescending + } + if in.MinimumCount != nil { + bd.MinimumCount = *in.MinimumCount + } + return bd +} + +// searchAggregationsToLibregraph maps the results to their definitions by +// position: the search service answers one result per aggregation, in request +// order. +func searchAggregationsToLibregraph(in []*searchsvc.AggregationResult, defs []libregraph.AggregationOption) []libregraph.SearchAggregation { + if len(in) == 0 { + return nil + } + out := make([]libregraph.SearchAggregation, 0, len(defs)) + for i, def := range defs { + if i >= len(in) { + break + } + agg := libregraph.SearchAggregation{Field: libregraph.PtrString(def.Field)} + agg.Buckets = make([]libregraph.SearchBucket, 0, len(in[i].GetBuckets())) + for _, b := range in[i].GetBuckets() { + agg.Buckets = append(agg.Buckets, libregraph.SearchBucket{ + Key: libregraph.PtrString(b.GetKey()), + Count: libregraph.PtrInt64(b.GetCount()), + }) + } + out = append(out, agg) + } + return out +} + func searchResourceID(id *searchmsg.ResourceID) string { return storagespace.FormatResourceID(&storageprovider.ResourceId{ StorageId: id.GetStorageId(), diff --git a/services/graph/pkg/service/v0/searchquery_test.go b/services/graph/pkg/service/v0/searchquery_test.go index 67a49f6470..c9765db192 100644 --- a/services/graph/pkg/service/v0/searchquery_test.go +++ b/services/graph/pkg/service/v0/searchquery_test.go @@ -16,6 +16,7 @@ import ( libregraph "github.com/opencloud-eu/libre-graph-api-go" "go-micro.dev/v4/client" merrors "go-micro.dev/v4/errors" + "google.golang.org/protobuf/testing/protocmp" "github.com/opencloud-eu/opencloud/pkg/log" searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" @@ -55,6 +56,10 @@ func graphWithSearchAnswer(resp *searchsvc.SearchResponse) (Graph, func() *searc return g, func() *searchsvc.SearchRequest { return captured } } +func graphWithAggregations(aggregations ...*searchsvc.AggregationResult) (Graph, func() *searchsvc.SearchRequest) { + return graphWithSearchAnswer(&searchsvc.SearchResponse{Aggregations: aggregations}) +} + func graphWithoutSearch() Graph { return graphWithSearch(stubSearchService{ search: func(*searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) { @@ -80,6 +85,14 @@ func oneMatch(entity *searchmsg.Entity) *searchsvc.SearchResponse { return &searchsvc.SearchResponse{TotalMatches: 1, Matches: []*searchmsg.Match{{Entity: entity}}} } +type searchAggregationJSON struct { + Field string `json:"field"` + Buckets []struct { + Key string `json:"key"` + Count int64 `json:"count"` + } `json:"buckets"` +} + type searchHitsContainerJSON struct { Hits []struct { Rank int32 `json:"rank"` @@ -96,8 +109,9 @@ type searchHitsContainerJSON struct { } `json:"parentReference"` } `json:"resource"` } `json:"hits"` - Total int64 `json:"total"` - MoreResultsAvailable bool `json:"moreResultsAvailable"` + Total int64 `json:"total"` + MoreResultsAvailable bool `json:"moreResultsAvailable"` + Aggregations []searchAggregationJSON `json:"aggregations"` } // hitsContainers decodes the one hits container of every request. @@ -137,8 +151,8 @@ var _ = ginkgo.Describe("SearchQuery", func() { `"message":"from and size reach beyond`), ginkgo.Entry("a bad second request, every request is validated before the first one runs", `{"requests": [ {"entityTypes": ["driveItem"], "query": {"queryString": "notes"}}, - {"entityTypes": ["driveItem"], "query": {"queryString": "notes"}, "size": 1000} - ]}`, "size must be"), + {"entityTypes": ["driveItem"], "query": {"queryString": "notes"}, "aggregations": [{"field": "audio.nonexistent"}]} + ]}`, "audio.nonexistent"), ) ginkgo.It("answers one hits container per request, in request order", func() { @@ -276,6 +290,69 @@ var _ = ginkgo.Describe("SearchQuery", func() { Expect(hc.Hits).To(BeEmpty()) }) + // the rules of the index are pinned in the aggregation package; one entry + // per kind proves the handler asks them before the search service + ginkgo.DescribeTable("rejects an aggregation the spec or the index does not allow with 400", + func(fragment string) { + rr := postSearchQuery(graphWithoutSearch(), searchQueryBody(fragment)) + Expect(rr.Code).To(Equal(http.StatusBadRequest), rr.Body.String()) + Expect(rr.Body.String()).To(ContainSubstring("invalidRequest")) + }, + ginkgo.Entry("an empty field", `"aggregations": [{"field": ""}]`), + ginkgo.Entry("an unknown sortBy", `"aggregations": [{"field": "audio.artist", "bucketDefinition": {"sortBy": "relevance"}}]`), + ginkgo.Entry("a size below one", `"aggregations": [{"field": "audio.artist", "size": 0}]`), + ginkgo.Entry("a negative minimumCount", `"aggregations": [{"field": "audio.artist", "bucketDefinition": {"sortBy": "count", "minimumCount": -1}}]`), + ginkgo.Entry("a field the index does not know", `"aggregations": [{"field": "audio.nonexistent"}]`), + ) + + ginkgo.It("translates the bucket definition for the search service", func() { + g, captured := graphWithAggregations() + rr := postSearchQuery(g, searchQueryBody(`"aggregations": [ + {"field": "audio.artist", "size": 5, "bucketDefinition": {"sortBy": "keyAsString", "isDescending": true, "minimumCount": 2}}, + {"field": "audio.genre", "bucketDefinition": {"sortBy": "count"}} + ]`)) + Expect(rr.Code).To(Equal(http.StatusOK), rr.Body.String()) + + Expect(captured().GetAggregations()).To(BeComparableTo([]*searchsvc.AggregationOption{ + {Field: "audio.artist", Size: 5, BucketDefinition: &searchsvc.BucketDefinition{SortBy: searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_STRING, IsDescending: true, MinimumCount: 2}}, + {Field: "audio.genre", BucketDefinition: &searchsvc.BucketDefinition{SortBy: searchsvc.BucketSortBy_BUCKET_SORT_BY_COUNT}}, + }, protocmp.Transform())) + }) + + ginkgo.DescribeTable("resolves the driveItem spelling of a field to the index field and answers in the spelling of the request", + func(requested, indexed string) { + g, captured := graphWithAggregations(&searchsvc.AggregationResult{ + Field: indexed, + Buckets: []*searchsvc.Bucket{{Key: "audio/mpeg", Count: 3}}, + }) + rr := postSearchQuery(g, searchQueryBody(fmt.Sprintf(`"aggregations": [{"field": %q}]`, requested))) + Expect(rr.Code).To(Equal(http.StatusOK), rr.Body.String()) + + Expect(captured().GetAggregations()[0].GetField()).To(Equal(indexed)) + Expect(hitsContainer(rr).Aggregations[0].Field).To(Equal(requested)) + }, + ginkgo.Entry("mimeType", "mimeType", "MimeType"), + ginkgo.Entry("name", "name", "Name"), + ginkgo.Entry("tags", "@libre.graph.tags", "Tags"), + ginkgo.Entry("a facet property", "audio.artist", "audio.artist"), + ) + + ginkgo.DescribeTable("rejects a field not spelled like the driveItem property with 400, naming it as the request did", + func(field string) { + rr := postSearchQuery(graphWithoutSearch(), searchQueryBody(fmt.Sprintf(`"aggregations": [{"field": %q}]`, field))) + Expect(rr.Code).To(Equal(http.StatusBadRequest), rr.Body.String()) + Expect(rr.Body.String()).To(ContainSubstring(field)) + Expect(rr.Body.String()).ToNot(ContainSubstring("Mtime"), "no index names") + }, + ginkgo.Entry("the index spelling of a property", "MimeType"), + ginkgo.Entry("another case", "MIMETYPE"), + ginkgo.Entry("the index spelling of the modification time", "mtime"), + ginkgo.Entry("a KQL alias", "tag"), + ginkgo.Entry("a facet in another case", "Audio.Artist"), + ginkgo.Entry("the nested driveItem path of the mime type", "file.mimeType"), + ginkgo.Entry("a property the index rules refuse for terms", "lastModifiedDateTime"), + ) + ginkgo.DescribeTable("answers a property it does not evaluate with 501 instead of ignoring it", func(fragment, property string) { rr := postSearchQuery(graphWithoutSearch(), searchQueryBody(fragment)) @@ -284,8 +361,12 @@ var _ = ginkgo.Describe("SearchQuery", func() { Expect(rr.Body.String()).To(ContainSubstring(property)) }, ginkgo.Entry("sortProperties", `"sortProperties": [{"name": "name"}]`, "sortProperties"), - ginkgo.Entry("aggregations", `"aggregations": [{"field": "audio.artist"}]`, "aggregations"), ginkgo.Entry("aggregationFilters", `"aggregationFilters": ["audio.artist:\"ǂǂ5361786f6e\""]`, "aggregationFilters"), + ginkgo.Entry("a geohash aggregation", + `"aggregations": [{"field": "location", "@libre.graph.geohashDefinition": {"precision": 5}}]`, "geohashDefinition"), + ginkgo.Entry("ranges", `"aggregations": [{"field": "audio.year", "bucketDefinition": {"sortBy": "count", "ranges": [{"from": "1980"}]}}]`, "ranges"), + ginkgo.Entry("a metric", `"aggregations": [{"field": "audio.year", "@libre.graph.metricDefinition": {"kind": "sum"}}]`, "metricDefinition"), + ginkgo.Entry("sub-aggregations", `"aggregations": [{"field": "audio.artist", "@libre.graph.subAggregations": [{"field": "audio.album"}]}]`, "subAggregations"), ) ginkgo.It("rejects an $expand it does not know with 400", func() { @@ -301,7 +382,7 @@ var _ = ginkgo.Describe("SearchQuery", func() { g := graphWithSearch(stubSearchService{ search: func(*searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) { return nil, serviceErr }, }) - rr := postSearchQuery(g, searchQueryBody(`"from": 0`)) + rr := postSearchQuery(g, searchQueryBody(`"aggregations": [{"field": "audio.artist"}]`)) Expect(rr.Code).To(Equal(status), rr.Body.String()) var decoded struct { diff --git a/services/search/pkg/aggregation/aggregation.go b/services/search/pkg/aggregation/aggregation.go new file mode 100644 index 0000000000..46454c7a06 --- /dev/null +++ b/services/search/pkg/aggregation/aggregation.go @@ -0,0 +1,47 @@ +// Package aggregation holds what the engines and the service layer share +// about aggregations: the bucket limit, the spelling of keys and the +// cross-space merge. +package aggregation + +import ( + "fmt" + "math" + "strconv" + + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" +) + +// MaxBuckets is how many buckets the aggregations of one request may have in +// one space: the search.max_buckets default of OpenSearch, held on every +// engine. +const MaxBuckets = math.MaxUint16 + +// ErrTooManyBuckets is what an engine answers beyond MaxBuckets; the service +// layer turns it into a bad request. +var ErrTooManyBuckets = fmt.Errorf("the aggregations have more than %d buckets, narrow the query or the aggregations", MaxBuckets) + +// CheckBuckets holds the results of one space to MaxBuckets. +func CheckBuckets(results []*searchsvc.AggregationResult) error { + if countBuckets(results) > MaxBuckets { + return ErrTooManyBuckets + } + return nil +} + +func countBuckets(results []*searchsvc.AggregationResult) int { + n := 0 + for _, r := range results { + n += len(r.GetBuckets()) + } + return n +} + +// NumberKey and BoolKey spell the bucket key of a numeric and a bool value, +// the same on every engine. +func NumberKey(v float64) string { + return strconv.FormatFloat(v, 'f', -1, 64) +} + +func BoolKey(v bool) string { + return strconv.FormatBool(v) +} diff --git a/services/search/pkg/aggregation/aggregation_suite_test.go b/services/search/pkg/aggregation/aggregation_suite_test.go new file mode 100644 index 0000000000..60b9affb61 --- /dev/null +++ b/services/search/pkg/aggregation/aggregation_suite_test.go @@ -0,0 +1,13 @@ +package aggregation_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAggregation(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Aggregation Suite") +} diff --git a/services/search/pkg/aggregation/aggregation_test.go b/services/search/pkg/aggregation/aggregation_test.go new file mode 100644 index 0000000000..d5746314ce --- /dev/null +++ b/services/search/pkg/aggregation/aggregation_test.go @@ -0,0 +1,28 @@ +package aggregation_test + +import ( + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" +) + +var _ = Describe("CheckBuckets", func() { + artists := func(count int) []*searchsvc.AggregationResult { + res := &searchsvc.AggregationResult{Field: "audio.artist"} + for i := range count { + res.Buckets = append(res.Buckets, &searchsvc.Bucket{Key: strconv.Itoa(i), Count: 1}) + } + return []*searchsvc.AggregationResult{res} + } + + It("counts the buckets against the limit, none is fine", func() { + Expect(aggregation.MaxBuckets).To(Equal(65535)) + Expect(aggregation.CheckBuckets(nil)).To(Succeed()) + Expect(aggregation.CheckBuckets(artists(65535))).To(Succeed()) + Expect(aggregation.CheckBuckets(artists(65536))).To(MatchError(aggregation.ErrTooManyBuckets)) + }) +}) diff --git a/services/search/pkg/aggregation/merge.go b/services/search/pkg/aggregation/merge.go new file mode 100644 index 0000000000..d64f01d79f --- /dev/null +++ b/services/search/pkg/aggregation/merge.go @@ -0,0 +1,112 @@ +package aggregation + +import ( + "cmp" + "slices" + "strconv" + + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" +) + +// Empty returns the results of a search without matches: one per option. +func Empty(opts []*searchsvc.AggregationOption) []*searchsvc.AggregationResult { + if len(opts) == 0 { + return nil + } + out := make([]*searchsvc.AggregationResult, len(opts)) + for i, opt := range opts { + out[i] = &searchsvc.AggregationResult{Field: opt.GetField()} + } + return out +} + +// Merge folds the results of one space into acc and returns it. Results +// belong to the option at their position, so several aggregations on one +// field stay apart. +func Merge(opts []*searchsvc.AggregationOption, acc, results []*searchsvc.AggregationResult) []*searchsvc.AggregationResult { + if acc == nil { + acc = Empty(opts) + } + for i := range opts { + if i >= len(results) || results[i] == nil { + continue + } + byKey := make(map[string]*searchsvc.Bucket, len(acc[i].GetBuckets())) + for _, b := range acc[i].GetBuckets() { + byKey[b.GetKey()] = b + } + for _, b := range results[i].GetBuckets() { + merged, ok := byKey[b.GetKey()] + if !ok { + merged = &searchsvc.Bucket{Key: b.GetKey()} + byKey[b.GetKey()] = merged + acc[i].Buckets = append(acc[i].Buckets, merged) + } + merged.Count += b.GetCount() + } + } + return acc +} + +// Finalize shapes merged results per their options: buckets get their minimum +// count, order and size. +func Finalize(opts []*searchsvc.AggregationOption, results []*searchsvc.AggregationResult) { + for i, opt := range opts { + results[i].Buckets = shapeBuckets(opt, results[i].GetBuckets()) + } +} + +func shapeBuckets(opt *searchsvc.AggregationOption, buckets []*searchsvc.Bucket) []*searchsvc.Bucket { + bd := opt.GetBucketDefinition() + if minCount := int64(bd.GetMinimumCount()); minCount > 0 { + buckets = slices.DeleteFunc(buckets, func(b *searchsvc.Bucket) bool { return b.GetCount() < minCount }) + } + + slices.SortFunc(buckets, bucketOrder(opt)) + + if size := int(opt.GetSize()); size > 0 && len(buckets) > size { + buckets = buckets[:size] + } + return buckets +} + +// bucketOrder follows sortBy and isDescending, ties by key. Without a sortBy +// the buckets come like a facet, by count descending. +func bucketOrder(opt *searchsvc.AggregationOption) func(a, b *searchsvc.Bucket) int { + bd := opt.GetBucketDefinition() + direction := 1 + if bd.GetIsDescending() { + direction = -1 + } + byKey := func(a, b *searchsvc.Bucket) int { return cmp.Compare(a.GetKey(), b.GetKey()) } + switch bd.GetSortBy() { + case searchsvc.BucketSortBy_BUCKET_SORT_BY_COUNT: + return func(a, b *searchsvc.Bucket) int { + return cmp.Or(direction*cmp.Compare(a.GetCount(), b.GetCount()), byKey(a, b)) + } + case searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_STRING: + return func(a, b *searchsvc.Bucket) int { return direction * byKey(a, b) } + case searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER: + // keys that are no number come last, whatever the direction + return func(a, b *searchsvc.Bucket) int { + av, aok := keyNumber(a.GetKey()) + bv, bok := keyNumber(b.GetKey()) + return cmp.Or(cmp.Compare(boolInt(!aok), boolInt(!bok)), direction*cmp.Compare(av, bv), byKey(a, b)) + } + } + return func(a, b *searchsvc.Bucket) int { + return cmp.Or(cmp.Compare(b.GetCount(), a.GetCount()), byKey(a, b)) + } +} + +func boolInt(b bool) int { + if b { + return 1 + } + return 0 +} + +func keyNumber(key string) (float64, bool) { + v, err := strconv.ParseFloat(key, 64) + return v, err == nil +} diff --git a/services/search/pkg/aggregation/merge_test.go b/services/search/pkg/aggregation/merge_test.go new file mode 100644 index 0000000000..1a36ceddc2 --- /dev/null +++ b/services/search/pkg/aggregation/merge_test.go @@ -0,0 +1,132 @@ +package aggregation_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" +) + +func buckets(field string, bs ...*searchsvc.Bucket) *searchsvc.AggregationResult { + return &searchsvc.AggregationResult{Field: field, Buckets: bs} +} + +func bucket(key string, count int64) *searchsvc.Bucket { + return &searchsvc.Bucket{Key: key, Count: count} +} + +// render flattens results like the parity suite: one line per bucket. +func render(results []*searchsvc.AggregationResult) []string { + out := []string{} + for _, r := range results { + for _, b := range r.GetBuckets() { + out = append(out, fmt.Sprintf("%s %s=%d", r.GetField(), b.GetKey(), b.GetCount())) + } + } + return out +} + +// merged runs the service layer's fold over the answers of several spaces. +func merged(opts []*searchsvc.AggregationOption, spaces ...[]*searchsvc.AggregationResult) []*searchsvc.AggregationResult { + acc := aggregation.Empty(opts) + for _, results := range spaces { + acc = aggregation.Merge(opts, acc, results) + } + aggregation.Finalize(opts, acc) + return acc +} + +var _ = Describe("Merge and Finalize", func() { + It("answers one result per option without any space", func() { + opts := []*searchsvc.AggregationOption{{Field: "audio.artist"}, {Field: "audio.album"}} + got := merged(opts) + Expect(got).To(HaveLen(2)) + Expect(got[0].GetField()).To(Equal("audio.artist")) + Expect(got[0].GetBuckets()).To(BeEmpty()) + Expect(got[1].GetField()).To(Equal("audio.album")) + }) + + It("adds up the counts of a bucket across spaces", func() { + opts := []*searchsvc.AggregationOption{{Field: "audio.artist"}} + got := merged(opts, + []*searchsvc.AggregationResult{buckets("audio.artist", bucket("Saxon", 2), bucket("Motörhead", 1))}, + []*searchsvc.AggregationResult{buckets("audio.artist", bucket("Saxon", 3), bucket("Led Zeppelin", 1))}, + ) + Expect(render(got)).To(Equal([]string{"audio.artist Saxon=5", "audio.artist Led Zeppelin=1", "audio.artist Motörhead=1"})) + }) + + It("leaves the answers of the spaces untouched", func() { + opts := []*searchsvc.AggregationOption{{Field: "audio.artist"}} + space := []*searchsvc.AggregationResult{buckets("audio.artist", bucket("Saxon", 2))} + merged(opts, space, space) + Expect(space[0].GetBuckets()[0].GetCount()).To(Equal(int64(2))) + }) + + It("keeps two terms aggregations on one field apart", func() { + opts := []*searchsvc.AggregationOption{{Field: "audio.artist", Size: 1}, {Field: "audio.artist", Size: 2}} + space := []*searchsvc.AggregationResult{ + buckets("audio.artist", bucket("Saxon", 2), bucket("Motörhead", 1)), + buckets("audio.artist", bucket("Saxon", 2), bucket("Motörhead", 1)), + } + Expect(render(merged(opts, space, space))).To(Equal([]string{ + "audio.artist Saxon=4", + "audio.artist Saxon=4", "audio.artist Motörhead=2", + })) + }) + + Describe("bucket order", func() { + space := []*searchsvc.AggregationResult{buckets("audio.track", + bucket("10", 2), bucket("9", 2), bucket("2", 5), bucket("b-side", 1), + )} + keys := func(sortBy searchsvc.BucketSortBy, descending bool) []string { + opts := []*searchsvc.AggregationOption{{Field: "audio.track", BucketDefinition: &searchsvc.BucketDefinition{SortBy: sortBy, IsDescending: descending}}} + out := []string{} + for _, b := range merged(opts, space)[0].GetBuckets() { + out = append(out, b.GetKey()) + } + return out + } + + It("defaults to count descending, ties by key", func() { + opts := []*searchsvc.AggregationOption{{Field: "audio.track"}} + Expect(render(merged(opts, space))).To(Equal([]string{"audio.track 2=5", "audio.track 10=2", "audio.track 9=2", "audio.track b-side=1"})) + }) + + DescribeTable("follows sortBy and isDescending, ties by key", + func(sortBy searchsvc.BucketSortBy, descending bool, want ...string) { + Expect(keys(sortBy, descending)).To(Equal(want)) + }, + Entry("count ascending", searchsvc.BucketSortBy_BUCKET_SORT_BY_COUNT, false, "b-side", "10", "9", "2"), + Entry("count descending", searchsvc.BucketSortBy_BUCKET_SORT_BY_COUNT, true, "2", "10", "9", "b-side"), + Entry("unspecified is the facet order", searchsvc.BucketSortBy_BUCKET_SORT_BY_UNSPECIFIED, true, "2", "10", "9", "b-side"), + Entry("unspecified ignores isDescending", searchsvc.BucketSortBy_BUCKET_SORT_BY_UNSPECIFIED, false, "2", "10", "9", "b-side"), + Entry("keyAsString ascending", searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_STRING, false, "10", "2", "9", "b-side"), + Entry("keyAsString descending", searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_STRING, true, "b-side", "9", "2", "10"), + Entry("keyAsNumber ascending, no number last", searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, false, "2", "9", "10", "b-side"), + Entry("keyAsNumber descending, no number still last", searchsvc.BucketSortBy_BUCKET_SORT_BY_KEY_AS_NUMBER, true, "10", "9", "2", "b-side"), + ) + }) + + Describe("bucket selection", func() { + It("trims a terms aggregation to its size after the merge", func() { + opts := []*searchsvc.AggregationOption{{Field: "audio.artist", Size: 1}} + got := merged(opts, + []*searchsvc.AggregationResult{buckets("audio.artist", bucket("Motörhead", 3), bucket("Saxon", 2))}, + []*searchsvc.AggregationResult{buckets("audio.artist", bucket("Saxon", 2))}, + ) + Expect(render(got)).To(Equal([]string{"audio.artist Saxon=4"})) + }) + + It("drops buckets below the minimum count, counted across spaces", func() { + opts := []*searchsvc.AggregationOption{{Field: "audio.artist", BucketDefinition: &searchsvc.BucketDefinition{MinimumCount: 2, IsDescending: true}}} + got := merged(opts, + []*searchsvc.AggregationResult{buckets("audio.artist", bucket("Saxon", 1), bucket("Motörhead", 1))}, + []*searchsvc.AggregationResult{buckets("audio.artist", bucket("Saxon", 1))}, + ) + Expect(render(got)).To(Equal([]string{"audio.artist Saxon=2"})) + }) + }) +}) diff --git a/services/search/pkg/aggregation/validate.go b/services/search/pkg/aggregation/validate.go new file mode 100644 index 0000000000..d0767d58a5 --- /dev/null +++ b/services/search/pkg/aggregation/validate.go @@ -0,0 +1,36 @@ +package aggregation + +import ( + "fmt" + + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" +) + +// ValidateOptions holds aggregations to what the index can answer: a field it +// knows, of a type whose values are buckets. fieldType returns the mapping +// type of a field, empty for one a client may not aggregate on. +func ValidateOptions(opts []*searchsvc.AggregationOption, fieldType func(string) string) error { + for _, opt := range opts { + field, t := opt.GetField(), fieldType(opt.GetField()) + if t == "" { + return fmt.Errorf("unknown aggregation field %q", field) + } + if err := validateTermsField(field, t); err != nil { + return err + } + } + return nil +} + +// validateTermsField allows the types whose values are buckets as they are. +// The values of a date are instants, no bucket each. +func validateTermsField(field, fieldType string) error { + switch fieldType { + case mapping.TypeKeyword, mapping.TypeBool, mapping.TypeNumeric: + return nil + case mapping.TypeDatetime: + return fmt.Errorf("terms aggregation is not supported on date field %q", field) + } + return fmt.Errorf("terms aggregation is not supported on field %q", field) +} diff --git a/services/search/pkg/aggregation/validate_test.go b/services/search/pkg/aggregation/validate_test.go new file mode 100644 index 0000000000..b9feb964f0 --- /dev/null +++ b/services/search/pkg/aggregation/validate_test.go @@ -0,0 +1,49 @@ +package aggregation_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" +) + +// fieldTypes stands in for the index mapping. +func fieldTypes(field string) string { + return map[string]string{ + "audio.artist": mapping.TypeKeyword, + "audio.album": mapping.TypeKeyword, + "audio.hasDrm": mapping.TypeBool, + "audio.year": mapping.TypeNumeric, + "photo.takenDateTime": mapping.TypeDatetime, + "Content": mapping.TypeFulltext, + "Path": mapping.TypePath, + "audio": mapping.TypeObject, + "location": mapping.TypeGeopoint, + }[field] +} + +var _ = Describe("ValidateOptions", func() { + + DescribeTable("accepts an aggregation that fits its field", + func(opt *searchsvc.AggregationOption) { + Expect(aggregation.ValidateOptions([]*searchsvc.AggregationOption{opt}, fieldTypes)).To(Succeed()) + }, + Entry("terms on a keyword", &searchsvc.AggregationOption{Field: "audio.artist"}), + Entry("terms on a bool", &searchsvc.AggregationOption{Field: "audio.hasDrm"}), + Entry("terms on a number", &searchsvc.AggregationOption{Field: "audio.year"}), + ) + + DescribeTable("rejects an aggregation the index cannot answer", + func(opt *searchsvc.AggregationOption) { + Expect(aggregation.ValidateOptions([]*searchsvc.AggregationOption{opt}, fieldTypes)).ToNot(Succeed()) + }, + Entry("an unknown field", &searchsvc.AggregationOption{Field: "audio.nonexistent"}), + Entry("terms on a date", &searchsvc.AggregationOption{Field: "photo.takenDateTime"}), + Entry("terms on full text", &searchsvc.AggregationOption{Field: "Content"}), + Entry("terms on a path", &searchsvc.AggregationOption{Field: "Path"}), + Entry("terms on a facet object", &searchsvc.AggregationOption{Field: "audio"}), + Entry("terms on a geopoint", &searchsvc.AggregationOption{Field: "location"}), + ) +}) diff --git a/services/search/pkg/bleve/aggregations.go b/services/search/pkg/bleve/aggregations.go new file mode 100644 index 0000000000..b73f1e7186 --- /dev/null +++ b/services/search/pkg/bleve/aggregations.go @@ -0,0 +1,206 @@ +package bleve + +import ( + "context" + "errors" + + "github.com/blevesearch/bleve/v2/numeric" + bleveSearch "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + index "github.com/blevesearch/bleve_index_api" + + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +// Bleve facets count the indexed terms of one field (numbers among them as +// prefix-coded terms), so every aggregation is folded from doc values by +// aggCollector, hooked into the collector walk through bleve's +// document-match-handler context key. Loading hits for them instead costs a +// stored-document decode per match. + +// termKey spells an indexed term as a bucket key: bleve indexes a bool as T +// or F. A field without a value has no bucket, an empty one neither. +func termKey(fieldType, term string) (string, bool) { + if term == "" { + return "", false + } + if fieldType == mapping.TypeBool { + return aggregation.BoolKey(term == "T"), true + } + return term, true +} + +type aggLevel struct { + opt *searchService.AggregationOption + fieldType string +} + +// numeric values are prefix-coded in the index, their terms come from the +// decoded doc values +func (l *aggLevel) numericTerms() bool { + return l.fieldType == mapping.TypeNumeric +} + +func newAggLevel(opt *searchService.AggregationOption) *aggLevel { + return &aggLevel{opt: opt, fieldType: search.FieldType(opt.GetField())} +} + +// fieldValues is per-document scratch, reused across documents. +type fieldValues struct { + asTerms bool + asNumbers bool + terms []string + numbers []int64 +} + +type bucketAcc struct { + counts map[string]int64 +} + +func newBucketAcc() *bucketAcc { + return &bucketAcc{counts: map[string]int64{}} +} + +// aggCollector serves one search; bleve's collector is single-threaded. +type aggCollector struct { + fields map[string]*fieldValues + fieldNames []string + roots []*aggRoot +} + +type aggRoot struct { + level *aggLevel + acc *bucketAcc +} + +func newAggCollector(aggs []*searchService.AggregationOption) *aggCollector { + if len(aggs) == 0 { + return nil + } + c := &aggCollector{fields: map[string]*fieldValues{}} + for _, agg := range aggs { + l := newAggLevel(agg) + c.register(l) + c.roots = append(c.roots, &aggRoot{level: l, acc: newBucketAcc()}) + } + return c +} + +func (c *aggCollector) register(l *aggLevel) { + fv, ok := c.fields[l.opt.GetField()] + if !ok { + fv = &fieldValues{} + c.fields[l.opt.GetField()] = fv + c.fieldNames = append(c.fieldNames, l.opt.GetField()) + } + if l.numericTerms() { + fv.asNumbers = true + } else { + fv.asTerms = true + } +} + +// The handler runs for every match, before the top-n cut. +func (c *aggCollector) withContext(ctx context.Context) context.Context { + maker := bleveSearch.MakeDocumentMatchHandler(func(sc *bleveSearch.SearchContext) (bleveSearch.DocumentMatchHandler, bool, error) { + inner, loadID, err := collector.MakeTopNDocumentMatchHandler(sc) + if err != nil { + return nil, false, err + } + if inner == nil { + return nil, false, errors.New("aggregations need the top-n collector") + } + dvr, err := sc.IndexReader.DocValueReader(c.fieldNames) + if err != nil { + return nil, false, err + } + return func(d *bleveSearch.DocumentMatch) error { + if d != nil { + if err := c.collect(dvr, d); err != nil { + return err + } + } + return inner(d) + }, loadID, nil + }) + return context.WithValue(ctx, bleveSearch.MakeDocumentMatchHandlerKey, maker) +} + +func (c *aggCollector) collect(dvr index.DocValueReader, d *bleveSearch.DocumentMatch) error { + for _, fv := range c.fields { + fv.terms = fv.terms[:0] + fv.numbers = fv.numbers[:0] + } + if err := dvr.VisitDocValues(d.IndexInternalID, c.visit); err != nil { + return err + } + for _, root := range c.roots { + c.fold(root.acc, root.level) + } + return nil +} + +// Numeric doc values are prefix-coded at several precisions; only shift 0 +// carries the exact value. +func (c *aggCollector) visit(field string, term []byte) { + fv, ok := c.fields[field] + if !ok { + return + } + if fv.asTerms { + fv.terms = append(fv.terms, string(term)) + } + if fv.asNumbers { + pc := numeric.PrefixCoded(term) + if shift, err := pc.Shift(); err == nil && shift == 0 { + if v, err := pc.Int64(); err == nil { + fv.numbers = append(fv.numbers, v) + } + } + } +} + +func (c *aggCollector) fold(a *bucketAcc, l *aggLevel) { + fv := c.fields[l.opt.GetField()] + if l.numericTerms() { + for _, raw := range fv.numbers { + c.foldBucket(a, l, aggregation.NumberKey(numeric.Int64ToFloat64(raw))) + } + return + } + for _, term := range fv.terms { + if key, ok := termKey(l.fieldType, term); ok { + c.foldBucket(a, l, key) + } + } +} + +func (c *aggCollector) foldBucket(a *bucketAcc, _ *aggLevel, key string) { + a.counts[key]++ +} + +// results returns one result per aggregation, in request order. +func (c *aggCollector) results() []*searchService.AggregationResult { + out := make([]*searchService.AggregationResult, 0, len(c.roots)) + for _, root := range c.roots { + out = append(out, root.acc.result(root.level)) + } + return out +} + +func (a *bucketAcc) result(l *aggLevel) *searchService.AggregationResult { + r := &searchService.AggregationResult{Field: l.opt.GetField()} + + counted := make(map[string]*searchService.Bucket, len(a.counts)) + for key, count := range a.counts { + counted[key] = &searchService.Bucket{Key: key, Count: count} + } + r.Buckets = make([]*searchService.Bucket, 0, len(counted)) + for _, b := range counted { + r.Buckets = append(r.Buckets, b) + } + return r +} diff --git a/services/search/pkg/bleve/backend.go b/services/search/pkg/bleve/backend.go index 5d0b364708..1ac9191819 100644 --- a/services/search/pkg/bleve/backend.go +++ b/services/search/pkg/bleve/backend.go @@ -15,6 +15,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/kql" "github.com/opencloud-eu/opencloud/pkg/log" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" "github.com/opencloud-eu/opencloud/services/search/pkg/search" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" @@ -42,7 +43,7 @@ func NewBackend(index bleve.Index, queryCreator searchQuery.Creator[query.Query] // Search executes a search request operation within the index. // Returns a SearchIndexResponse object or an error. -func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) { +func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) { createdQuery, err := b.queryCreator.Create(sir.Query) if err != nil { if kql.IsValidationError(err) { @@ -93,8 +94,13 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques bleveReq.SortBy([]string{"-_score", "_id"}) bleveReq.Size = size + collector := newAggCollector(sir.GetAggregations()) + if collector != nil { + ctx = collector.withContext(ctx) + } + bleveReq.Fields = []string{"*"} - res, err := b.index.Search(bleveReq) + res, err := b.index.SearchInContext(ctx, bleveReq) if err != nil { return nil, err } @@ -147,9 +153,18 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques matches = append(matches, match) } + var aggregations []*searchService.AggregationResult + if collector != nil { + aggregations = collector.results() + if err := aggregation.CheckBuckets(aggregations); err != nil { + return nil, errtypes.BadRequest(err.Error()) + } + } + return &searchService.SearchIndexResponse{ Matches: matches, TotalMatches: int32(totalMatches), + Aggregations: aggregations, }, nil } diff --git a/services/search/pkg/mapping/fieldindex.go b/services/search/pkg/mapping/fieldindex.go index 5bc5d21b66..97f91420d9 100644 --- a/services/search/pkg/mapping/fieldindex.go +++ b/services/search/pkg/mapping/fieldindex.go @@ -34,3 +34,17 @@ func FieldNameIndex(t reflect.Type, overrides map[string]FieldOpts) map[string]s walk(t, "") return out } + +// FieldTypes maps every field of t (json names, nested as parent.child) to +// its effective mapping type: the override, or the type inferred from the Go +// field. +func FieldTypes(t reflect.Type, overrides map[string]FieldOpts) map[string]string { + out := map[string]string{} + for key, goType := range collectFields(t, "") { + out[key] = overrides[key].Type + if out[key] == "" { + out[key] = inferType(goType) + } + } + return out +} diff --git a/services/search/pkg/mapping/opts.go b/services/search/pkg/mapping/opts.go index 07ae324b04..5094d22737 100644 --- a/services/search/pkg/mapping/opts.go +++ b/services/search/pkg/mapping/opts.go @@ -56,6 +56,10 @@ type FieldOpts struct { // IncludeInAll controls bleve's _all field inclusion. Nil means "use the // bleve default for this field type". Has no effect on OpenSearch. IncludeInAll *bool + + // Internal marks index housekeeping no client names: not aggregated on, + // not filtered to. KQL still searches it. + Internal bool } func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive == nil || *o.CaseInsensitive } diff --git a/services/search/pkg/opensearch/backend.go b/services/search/pkg/opensearch/backend.go index b3ff91fb2b..da9e1115d4 100644 --- a/services/search/pkg/opensearch/backend.go +++ b/services/search/pkg/opensearch/backend.go @@ -2,10 +2,12 @@ package opensearch import ( "context" + "errors" "fmt" "time" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opensearch-project/opensearch-go/v4" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/opencloud-eu/reva/v2/pkg/errtypes" @@ -17,6 +19,8 @@ import ( "github.com/opencloud-eu/opencloud/pkg/log" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/aggs" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" "github.com/opencloud-eu/opencloud/services/search/pkg/search" @@ -124,6 +128,11 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ Size: conversions.ToPointer(size), } + aggregationsBody, err := aggs.Build(sir.GetAggregations()) + if err != nil { + return nil, errtypes.BadRequest(err.Error()) + } + req, err := osu.BuildSearchReq(&opensearchgoAPI.SearchReq{ Indices: []string{b.index}, Params: searchParams, @@ -142,6 +151,7 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ }, }, }, + Aggs: aggregationsBody, }, ) if err != nil { @@ -149,7 +159,10 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ } resp, err := b.client.Search(ctx, req) - if err != nil { + switch { + case tooManyBuckets(err): + return nil, errtypes.BadRequest(aggregation.ErrTooManyBuckets.Error()) + case err != nil: return nil, fmt.Errorf("failed to search: %w", err) } @@ -164,12 +177,40 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ matches = append(matches, match) } + aggregations, err := aggs.Parse(sir.GetAggregations(), resp.Aggregations) + if err == nil { + err = aggregation.CheckBuckets(aggregations) + } + switch { + case errors.Is(err, aggregation.ErrTooManyBuckets): + return nil, errtypes.BadRequest(err.Error()) + case err != nil: + return nil, fmt.Errorf("failed to parse aggregations: %w", err) + } + return &searchService.SearchIndexResponse{ Matches: matches, TotalMatches: int32(totalMatches), + Aggregations: aggregations, }, nil } +// tooManyBuckets tells whether OpenSearch refused the aggregations for their +// bucket count (search.max_buckets); the cause sits below the search phase +// error. +func tooManyBuckets(err error) bool { + var structErr *opensearch.StructError + if !errors.As(err, &structErr) { + return false + } + for cause := structErr.Err.CausedBy; cause != nil; cause = cause.CausedBy { + if cause.Type == "too_many_buckets_exception" { + return true + } + } + return false +} + func (b *Backend) DocCount() (uint64, error) { req, err := osu.BuildIndicesCountReq( &opensearchgoAPI.IndicesCountReq{ diff --git a/services/search/pkg/opensearch/internal/aggs/aggs.go b/services/search/pkg/opensearch/internal/aggs/aggs.go new file mode 100644 index 0000000000..41146ac36f --- /dev/null +++ b/services/search/pkg/opensearch/internal/aggs/aggs.go @@ -0,0 +1,126 @@ +// Package aggs translates proto aggregation options into the OpenSearch +// aggregation DSL and parses the response. Internal subpackage so its unit +// tests skip the parent package's Docker OpenSearch container. +package aggs + +import ( + "encoding/json" + "fmt" + + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" +) + +// Build renders the aggregations as native OpenSearch aggregations, each +// named by its position. A terms aggregation asks for every bucket up to the +// limit: the service layer cuts to the requested size after the cross-space +// merge, a cut here would make that merge inexact. +func Build(opts []*searchsvc.AggregationOption) (map[string]any, error) { + if len(opts) == 0 { + return nil, nil + } + out := make(map[string]any, len(opts)) + for i, opt := range opts { + entry, err := build(opt) + if err != nil { + return nil, err + } + out[aggName(i)] = entry + } + return out, nil +} + +func aggName(position int) string { + return fmt.Sprintf("a_%d", position) +} + +func build(opt *searchsvc.AggregationOption) (map[string]any, error) { + return map[string]any{"terms": map[string]any{"field": opt.GetField(), "size": aggregation.MaxBuckets}}, nil +} + +// Parse reads the aggregation block of a response into one result per option, +// in request order. +func Parse(opts []*searchsvc.AggregationOption, raw json.RawMessage) ([]*searchsvc.AggregationResult, error) { + node := aggNode{} + if len(raw) > 0 { + if err := json.Unmarshal(raw, &node); err != nil { + return nil, fmt.Errorf("decode opensearch aggregations: %w", err) + } + } + return parse(opts, node) +} + +// aggNode is one level of the aggregation response, by aggregation name. +type aggNode map[string]json.RawMessage + +func parse(opts []*searchsvc.AggregationOption, node aggNode) ([]*searchsvc.AggregationResult, error) { + if len(opts) == 0 { + return nil, nil + } + out := make([]*searchsvc.AggregationResult, 0, len(opts)) + for i, opt := range opts { + res, err := result(opt, node[aggName(i)]) + if err != nil { + return nil, err + } + out = append(out, res) + } + return out, nil +} + +func result(opt *searchsvc.AggregationOption, raw json.RawMessage) (*searchsvc.AggregationResult, error) { + var body struct { + Buckets []json.RawMessage `json:"buckets"` + SumOtherDocCount int64 `json:"sum_other_doc_count"` + } + if len(raw) > 0 { + if err := json.Unmarshal(raw, &body); err != nil { + return nil, fmt.Errorf("decode aggregation on %s: %w", opt.GetField(), err) + } + } + // matches OpenSearch left out of the buckets: there are more than asked for + if body.SumOtherDocCount > 0 { + return nil, aggregation.ErrTooManyBuckets + } + + res := &searchsvc.AggregationResult{Field: opt.GetField(), Buckets: []*searchsvc.Bucket{}} + for _, rawBucket := range body.Buckets { + bucket, err := parseBucket(opt, rawBucket) + if err != nil { + return nil, err + } + // a field without a value has no bucket, an empty one neither + if bucket.GetKey() != "" { + res.Buckets = append(res.Buckets, bucket) + } + } + return res, nil +} + +func parseBucket(opt *searchsvc.AggregationOption, raw json.RawMessage) (*searchsvc.Bucket, error) { + var head struct { + Key any `json:"key"` + KeyAsString *string `json:"key_as_string"` + DocCount int64 `json:"doc_count"` + } + if err := json.Unmarshal(raw, &head); err != nil { + return nil, fmt.Errorf("decode bucket of %s: %w", opt.GetField(), err) + } + return &searchsvc.Bucket{Key: bucketKey(head.Key, head.KeyAsString), Count: head.DocCount}, nil +} + +// bucketKey spells a response key like the bleve backend does: a number +// without trailing zeros, a bool as true or false. OpenSearch keys a bool as +// 1 or 0 and spells it out next to it. +func bucketKey(key any, asString *string) string { + if asString != nil { + return *asString + } + switch k := key.(type) { + case string: + return k + case float64: + return aggregation.NumberKey(k) + } + return "" +} diff --git a/services/search/pkg/opensearch/internal/aggs/aggs_suite_test.go b/services/search/pkg/opensearch/internal/aggs/aggs_suite_test.go new file mode 100644 index 0000000000..11337ad5d3 --- /dev/null +++ b/services/search/pkg/opensearch/internal/aggs/aggs_suite_test.go @@ -0,0 +1,13 @@ +package aggs_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAggs(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Aggs Suite") +} diff --git a/services/search/pkg/opensearch/internal/aggs/aggs_test.go b/services/search/pkg/opensearch/internal/aggs/aggs_test.go new file mode 100644 index 0000000000..6602ebc816 --- /dev/null +++ b/services/search/pkg/opensearch/internal/aggs/aggs_test.go @@ -0,0 +1,98 @@ +package aggs_test + +import ( + "encoding/json" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/aggs" +) + +var _ = Describe("Aggregations", func() { + build := func(opts ...*searchsvc.AggregationOption) map[string]any { + body, err := aggs.Build(opts) + Expect(err).ToNot(HaveOccurred()) + return body + } + + parse := func(response string, opts ...*searchsvc.AggregationOption) []*searchsvc.AggregationResult { + out, err := aggs.Parse(opts, json.RawMessage(response)) + Expect(err).ToNot(HaveOccurred()) + return out + } + + keys := func(res *searchsvc.AggregationResult) []string { + out := []string{} + for _, b := range res.GetBuckets() { + out = append(out, fmt.Sprintf("%s=%d", b.GetKey(), b.GetCount())) + } + return out + } + + It("builds nothing without aggregations", func() { + Expect(build()).To(BeNil()) + }) + + It("asks for every bucket of a terms aggregation up to the limit, whatever its size", func() { + terms := build(&searchsvc.AggregationOption{Field: "audio.artist", Size: 10})["a_0"].(map[string]any)["terms"].(map[string]any) + Expect(terms["field"]).To(Equal("audio.artist")) + Expect(terms["size"]).To(Equal(aggregation.MaxBuckets), "the size cuts after the cross-space merge") + }) + + It("reads the buckets of a terms aggregation", func() { + out := parse(`{"a_0": {"sum_other_doc_count": 0, "buckets": [ + {"key": "Motörhead", "doc_count": 3}, + {"key": "Accept", "doc_count": 2} + ]}}`, &searchsvc.AggregationOption{Field: "audio.artist"}) + Expect(out).To(HaveLen(1)) + Expect(out[0].GetField()).To(Equal("audio.artist")) + Expect(keys(out[0])).To(Equal([]string{"Motörhead=3", "Accept=2"})) + }) + + It("spells numeric and bool keys like the bleve backend", func() { + out := parse(`{ + "a_0": {"buckets": [{"key": 9, "doc_count": 3}, {"key": 4.5, "doc_count": 1}]}, + "a_1": {"buckets": [{"key": 1, "key_as_string": "true", "doc_count": 2}, {"key": 0, "key_as_string": "false", "doc_count": 1}]} + }`, &searchsvc.AggregationOption{Field: "audio.track"}, &searchsvc.AggregationOption{Field: "audio.hasDrm"}) + Expect(keys(out[0])).To(Equal([]string{"9=3", "4.5=1"})) + Expect(keys(out[1])).To(Equal([]string{"true=2", "false=1"})) + }) + + It("leaves out the bucket of an empty value", func() { + out := parse(`{"a_0": {"buckets": [ + {"key": "", "doc_count": 40}, + {"key": "Wheels of Steel", "doc_count": 2} + ]}}`, &searchsvc.AggregationOption{Field: "Title"}) + Expect(keys(out[0])).To(Equal([]string{"Wheels of Steel=2"})) + }) + + It("refuses a terms aggregation with more buckets than the limit instead of cutting it short", func() { + _, err := aggs.Parse([]*searchsvc.AggregationOption{{Field: "audio.artist"}}, json.RawMessage(`{"a_0": { + "sum_other_doc_count": 4465, + "buckets": [{"key": "Accept", "doc_count": 2}] + }}`)) + Expect(err).To(MatchError(aggregation.ErrTooManyBuckets)) + }) + + It("answers an aggregation without buckets for an empty response block", func() { + for _, response := range []string{``, `{}`} { + out := parse(response, &searchsvc.AggregationOption{Field: "audio.artist"}) + Expect(out).To(HaveLen(1)) + Expect(out[0].GetBuckets()).To(BeEmpty()) + } + }) + + DescribeTable("reports a result it cannot decode instead of dropping it", + func(opt *searchsvc.AggregationOption, response string) { + _, err := aggs.Parse([]*searchsvc.AggregationOption{opt}, json.RawMessage(response)) + Expect(err).To(HaveOccurred()) + }, + Entry("malformed json", &searchsvc.AggregationOption{Field: "audio.artist"}, `{`), + Entry("a terms aggregation", &searchsvc.AggregationOption{Field: "audio.artist"}, `{"a_0": {"buckets": "none"}}`), + Entry("a bucket", &searchsvc.AggregationOption{Field: "audio.artist"}, `{"a_0": {"buckets": [{"key": "Saxon", "doc_count": "many"}]}}`), + ) +}) diff --git a/services/search/pkg/opensearch/internal/osu/request.go b/services/search/pkg/opensearch/internal/osu/request.go index 927da7ac54..f6f4e238ee 100644 --- a/services/search/pkg/opensearch/internal/osu/request.go +++ b/services/search/pkg/opensearch/internal/osu/request.go @@ -95,6 +95,7 @@ func BuildSearchReq(req *opensearchgoAPI.SearchReq, q Builder, p ...SearchBodyPa type SearchBodyParams struct { Highlight *BodyParamHighlight `json:"highlight,omitempty"` + Aggs map[string]any `json:"aggs,omitempty"` } //----------------------------------------------------------------------------// diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index 186a48badb..38781e7926 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -715,3 +715,49 @@ Fixtures: | METADATA-01 | `*song*` reads `Audio` | all 16 fields unchanged | all 16 fields unchanged | all 16 fields unchanged | ✅ | | METADATA-02 | `*team*` reads `Location` | all 3 fields unchanged | all 3 fields unchanged | all 3 fields unchanged | ✅ | | METADATA-03 | `*team*` reads `Audio` | none | none | none | ✅ | + +## Aggregations + +### aggregations + +Fixtures: + +- `a.mp3`, MimeType = audio/mpeg +- `b.mp3`, MimeType = audio/mpeg +- `c.mp3`, MimeType = audio/mpeg +- `d.mp3`, MimeType = audio/mpeg +- `e.mp3`, MimeType = audio/mpeg +- `f.mp3`, MimeType = audio/mpeg +- `g.mp3`, MimeType = audio/mpeg + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| AGG-01 | `mediatype:audio` reads `term buckets on audio.artist` | audio.artist Motörhead=3, audio.artist Saxon=2 | audio.artist Motörhead=3, audio.artist Saxon=2 | audio.artist Motörhead=3, audio.artist Saxon=2 | ✅ | +| AGG-02 | `mediatype:audio` reads `no aggregations requested` | no match | no match | no match | ✅ | +| AGG-03 | `mediatype:audio` reads `artist and album buckets in one request` | audio.albu... Steel=2, audio.albu...Spades=1, audio.album Bomber=2, audio.artist Motörhead=3, audio.artist Saxon=2 | audio.albu... Steel=2, audio.albu...Spades=1, audio.album Bomber=2, audio.artist Motörhead=3, audio.artist Saxon=2 | audio.albu... Steel=2, audio.albu...Spades=1, audio.album Bomber=2, audio.artist Motörhead=3, audio.artist Saxon=2 | ✅ | +| AGG-34 | `mediatype:audio` reads `term buckets on the numeric audio.year` | audio.year 1971=1, audio.year 1975=1, audio.year 1982=1, audio.year 1999=1, audio.year 2001=1, audio.year 2005=1, audio.year 2009=1 | audio.year 1971=1, audio.year 1975=1, audio.year 1982=1, audio.year 1999=1, audio.year 2001=1, audio.year 2005=1, audio.year 2009=1 | audio.year 1971=1, audio.year 1975=1, audio.year 1982=1, audio.year 1999=1, audio.year 2001=1, audio.year 2005=1, audio.year 2009=1 | ✅ | +| AGG-37 | `mediatype:audio` reads `term buckets on the bool audio.hasDrm, spelled true and false` | audio.hasDrm false=1, audio.hasDrm true=1 | audio.hasDrm false=1, audio.hasDrm true=1 | audio.hasDrm false=1, audio.hasDrm true=1 | ✅ | +| AGG-39 | `mediatype:audio` reads `no bucket for the empty Title of every match` | no match | no match | no match | ✅ | +| AGG-41 | `mediatype:audio` reads `matches of the same score in the order of their ids, page after page` | 3 of 7 matches, a.mp3, b.mp3, c.mp3 | 3 of 7 matches, a.mp3, b.mp3, c.mp3 | 3 of 7 matches, a.mp3, b.mp3, c.mp3 | ✅ | + +### cardinality + +Fixtures: + +- `card-00000.mp3`, MimeType = audio/mpeg +- `card-00001.mp3`, MimeType = audio/mpeg +- `card-00002.mp3`, MimeType = audio/mpeg +- `card-00003.mp3`, MimeType = audio/mpeg +- `card-00004.mp3`, MimeType = audio/mpeg +- `card-00005.mp3`, MimeType = audio/mpeg +- `card-00006.mp3`, MimeType = audio/mpeg +- `card-00007.mp3`, MimeType = audio/mpeg +- `card-00008.mp3`, MimeType = audio/mpeg +- `card-00009.mp3`, MimeType = audio/mpeg +- ... and 10040 more of the same + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| AGG-17 | `mediatype:audio` reads `one bucket per artist, cardinality above the page size` | 10050 items | audio.arti...-00000=1, audio.arti...-00001=1, audio.arti...-00002=1, audio.arti...-00003=1, audio.arti...-00004=1, audio.arti...-00005=1, audio.arti...-00006=1, audio.arti...-00007=1, … +10042 more | audio.arti...-00000=1, audio.arti...-00001=1, audio.arti...-00002=1, audio.arti...-00003=1, audio.arti...-00004=1, audio.arti...-00005=1, audio.arti...-00006=1, audio.arti...-00007=1, … +10042 more | ✅ | +| AGG-42 | `mediatype:audio` reads `the total counts every match` | 1 of 10050 matches | 1 of 10050 matches | 1 of 10050 matches | ✅ | +| AGG-43 | `mediatype:audio` reads `a page reaching beyond the first 10000 matches is refused` | bad request | bad request | bad request | ✅ | diff --git a/services/search/pkg/parity/aggregations_test.go b/services/search/pkg/parity/aggregations_test.go new file mode 100644 index 0000000000..727bc9f150 --- /dev/null +++ b/services/search/pkg/parity/aggregations_test.go @@ -0,0 +1,227 @@ +package parity + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/reva/v2/pkg/errtypes" + + "github.com/opencloud-eu/opencloud/pkg/conversions" + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +// aggCase is one aggregation request both engines have to answer alike. The +// answer is rendered to strings (one per non-empty bucket or metric), so the +// matrix machinery can carry it like any query answer. +type aggCase struct { + id int + query string + aggs []*searchService.AggregationOption + reads string + want []string + wantCount *int + wantBadRequest bool + engineOverrides map[string]override + // pageSize, when set, prefixes the answer with " of matches" + pageSize int32 + // listMatches puts the names of the matches into the answer and holds the + // engines to their order + listMatches bool +} + +// aggGroup is a set of cases over one set of fixtures. +type aggGroup struct { + name string + fixtures []search.Resource + cases []aggCase +} + +func aggregationGroups() []aggGroup { + return []aggGroup{ + {name: "aggregations", fixtures: aggregationFixtures(), cases: aggregationCases()}, + {name: "cardinality", fixtures: cardinalityFixtures(), cases: cardinalityCases()}, + } +} + +func (c aggCase) label() string { return fmt.Sprintf("AGG-%02d", c.id) } + +func untaggedSong(name string, year int32) search.Resource { + return fixtureDoc(name, withMime("audio/mpeg"), withAudio(&libregraph.Audio{Year: libregraph.PtrInt32(year)})) +} + +func song(name, artist, album string, year int32, opts ...fixtureOption) search.Resource { + return fixtureDoc(name, append([]fixtureOption{withMime("audio/mpeg"), withAudio(&libregraph.Audio{ + Artist: libregraph.PtrString(artist), + Album: libregraph.PtrString(album), + Year: libregraph.PtrInt32(year), + })}, opts...)...) +} + +func withDrm(r search.Resource, drm bool) search.Resource { + r.Audio.HasDrm = &drm + return r +} + +func aggregationFixtures() []search.Resource { + return []search.Resource{ + // years: 1971, 1975, 1982, 1999, 2001, 2005, 2009 + withDrm(song("a.mp3", "Saxon", "Wheels of Steel", 1971), true), + withDrm(song("b.mp3", "Saxon", "Wheels of Steel", 1975), false), + song("c.mp3", "Motörhead", "Bomber", 1982), + song("d.mp3", "Motörhead", "Bomber", 1999), + song("e.mp3", "Motörhead", "Ace of Spades", 2001), + untaggedSong("f.mp3", 2005), + untaggedSong("g.mp3", 2009), + } +} + +func aggregationCases() []aggCase { + return []aggCase{ + {id: 1, query: "mediatype:audio", reads: "term buckets on audio.artist", + aggs: []*searchService.AggregationOption{{Field: "audio.artist", Size: 10}}, + want: []string{"audio.artist Saxon=2", "audio.artist Motörhead=3"}}, + {id: 2, query: "mediatype:audio", reads: "no aggregations requested"}, + {id: 3, query: "mediatype:audio", reads: "artist and album buckets in one request", + aggs: []*searchService.AggregationOption{{Field: "audio.artist"}, {Field: "audio.album"}}, + want: []string{ + "audio.artist Saxon=2", "audio.artist Motörhead=3", + "audio.album Wheels of Steel=2", "audio.album Bomber=2", "audio.album Ace of Spades=1", + }}, + {id: 34, query: "mediatype:audio", reads: "term buckets on the numeric audio.year", + aggs: []*searchService.AggregationOption{{Field: "audio.year"}}, + want: []string{ + "audio.year 1971=1", "audio.year 1975=1", "audio.year 1982=1", "audio.year 1999=1", + "audio.year 2001=1", "audio.year 2005=1", "audio.year 2009=1", + }}, + {id: 37, query: "mediatype:audio", reads: "term buckets on the bool audio.hasDrm, spelled true and false", + aggs: []*searchService.AggregationOption{{Field: "audio.hasDrm"}}, + want: []string{"audio.hasDrm true=1", "audio.hasDrm false=1"}}, + {id: 39, query: "mediatype:audio", reads: "no bucket for the empty Title of every match", + aggs: []*searchService.AggregationOption{{Field: "Title"}}}, + {id: 41, query: "mediatype:audio", reads: "matches of the same score in the order of their ids, page after page", + pageSize: 3, listMatches: true, + want: []string{"3 of 7 matches", "a.mp3", "b.mp3", "c.mp3"}}, + } +} + +// renderAggregations flattens an answer into comparable strings, one per +// bucket. +func renderAggregations(resp *searchService.SearchIndexResponse, err error) []string { + if err != nil { + if _, ok := err.(errtypes.BadRequest); ok { + return []string{"bad request"} + } + return []string{"error"} + } + + out := []string{} + for _, a := range resp.Aggregations { + out = append(out, renderAggregation("", a)...) + } + + return out +} + +func renderAggregation(prefix string, a *searchService.AggregationResult) []string { + out := []string{} + for _, b := range a.Buckets { + out = append(out, prefix+fmt.Sprintf("%s %s=%d", a.Field, b.Key, b.Count)) + } + + return out +} + +// cardinalityFixtures exceed the composite page size and what OpenSearch +// counts and returns by default (10000 matches). +func cardinalityFixtures() []search.Resource { + docs := make([]search.Resource, 0, 10050) + for i := 0; i < 10050; i++ { + docs = append(docs, song(fmt.Sprintf("card-%05d.mp3", i), fmt.Sprintf("artist-%05d", i), "Singles", int32(1950+i%50))) + } + return docs +} + +func cardinalityCases() []aggCase { + count := 10050 + return []aggCase{ + {id: 17, query: "mediatype:audio", reads: "one bucket per artist, cardinality above the page size", + aggs: []*searchService.AggregationOption{{Field: "audio.artist"}}, + wantCount: &count}, + {id: 42, query: "mediatype:audio", reads: "the total counts every match", + pageSize: 1, + want: []string{"1 of 10050 matches"}}, + {id: 43, query: "mediatype:audio", reads: "a page reaching beyond the first 10000 matches is refused", + pageSize: 10001, + wantBadRequest: true}, + } +} + +var _ = Describe("Aggregations", func() { + for groupAt, group := range aggregationGroups() { + Describe(group.name, Ordered, ContinueOnFailure, func() { + var engines []testEngine + + BeforeAll(func() { + engines = newEngines("opencloud-test-engine-parity-agg-"+group.name, group.fixtures) + }) + + for caseAt, c := range group.cases { + row := matrixRow{ + Section: "Aggregations", Group: group.name, ID: c.label(), + Query: c.query, Reads: c.reads, + Want: c.want, WantCount: c.wantCount, WantBadRequest: c.wantBadRequest, Overrides: renderOverrides(c.engineOverrides), + GroupAt: 100 + groupAt, CaseAt: caseAt, + } + planRow(row) + + Describe(c.label()+" "+c.reads, func() { + for _, name := range engineNames { + It("on "+name, func() { + e := engineNamed(engines, name) + if e.unavailable != "" { + recordSkip(row, name) + Skip(e.unavailable) + } + + request := &searchService.SearchIndexRequest{ + Query: c.query, + Aggregations: c.aggs, + } + if c.pageSize != 0 { + request.PageSize = conversions.ToPointer(c.pageSize) + } + resp, err := e.backend.Search(context.Background(), request) + answer := renderAggregations(resp, err) + if err == nil && c.listMatches { + names := make([]string, 0, len(resp.Matches)) + for _, m := range resp.Matches { + names = append(names, m.GetEntity().GetName()) + } + answer = append(names, answer...) + } + if err == nil && c.pageSize > 0 { + answer = append([]string{fmt.Sprintf("%d of %d matches", len(resp.Matches), resp.TotalMatches)}, answer...) + } + recordAnswer(row, name, answer) + + _, overridden := c.engineOverrides[name] + if !overridden && !c.wantBadRequest { + Expect(err).NotTo(HaveOccurred(), "the aggregation has to answer") + } + + if c.listMatches { + Expect(answer).To(Equal(c.want)) + return + } + expectAnswer(name, answer, override{want: c.want, wantCount: c.wantCount, wantBadRequest: c.wantBadRequest}, c.engineOverrides) + }) + } + }) + } + }) + } +}) diff --git a/services/search/pkg/parity/matrix_test.go b/services/search/pkg/parity/matrix_test.go index 9e10a3b8a3..a0ceb88350 100644 --- a/services/search/pkg/parity/matrix_test.go +++ b/services/search/pkg/parity/matrix_test.go @@ -278,6 +278,12 @@ func matrixFixtures(group string) string { fixtures = g.fixtures } + for _, g := range aggregationGroups() { + if g.name == group { + fixtures = g.fixtures + } + } + if len(fixtures) == 0 { return "Fixtures: none" } @@ -379,6 +385,10 @@ func matrixScope(ref *searchMessage.Reference) string { return "in " + space } +// matrixNamesHead keeps a table row readable: a huge answer list renders as +// its head plus a count instead of flooding the README. +const matrixNamesHead = 8 + func matrixNames(names []string) string { if len(names) == 0 { return "no match" @@ -390,6 +400,9 @@ func matrixNames(names []string) string { } sort.Strings(shortened) + if len(shortened) > matrixNamesHead { + return strings.Join(shortened[:matrixNamesHead], ", ") + fmt.Sprintf(", … +%d more", len(shortened)-matrixNamesHead) + } return strings.Join(shortened, ", ") } diff --git a/services/search/pkg/search/fields.go b/services/search/pkg/search/fields.go new file mode 100644 index 0000000000..3e94d3f541 --- /dev/null +++ b/services/search/pkg/search/fields.go @@ -0,0 +1,27 @@ +package search + +import ( + "reflect" + "sync" + + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" +) + +var fieldTypes = sync.OnceValue(func() map[string]string { + return mapping.FieldTypes(reflect.TypeFor[Resource](), Resource{}.SearchFieldOverrides()) +}) + +// FieldType returns the mapping type of an index field (one of the +// mapping.Type* constants), empty for a field the index does not know. +func FieldType(field string) string { + return fieldTypes()[field] +} + +// AggregatableFieldType is FieldType for the fields a client may name, empty +// for the internal ones. +func AggregatableFieldType(field string) string { + if resourceFieldOverrides()[field].Internal { + return "" + } + return FieldType(field) +} diff --git a/services/search/pkg/search/fields_test.go b/services/search/pkg/search/fields_test.go new file mode 100644 index 0000000000..57002e6138 --- /dev/null +++ b/services/search/pkg/search/fields_test.go @@ -0,0 +1,56 @@ +package search_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +var _ = Describe("FieldType", func() { + DescribeTable("reports the mapping type of a field", + func(field, want string) { + Expect(search.FieldType(field)).To(Equal(want)) + }, + Entry("embedded document field", "Size", mapping.TypeNumeric), + Entry("resource field", "Type", mapping.TypeNumeric), + Entry("keyword", "Name", mapping.TypeKeyword), + Entry("overridden type", "Path", mapping.TypePath), + Entry("full text", "Content", mapping.TypeFulltext), + Entry("time", "Mtime", mapping.TypeDatetime), + Entry("facet keyword", "audio.artist", mapping.TypeKeyword), + Entry("facet int", "audio.year", mapping.TypeNumeric), + Entry("facet bool", "audio.hasDrm", mapping.TypeBool), + Entry("facet float", "photo.focalLength", mapping.TypeNumeric), + Entry("facet time", "photo.takenDateTime", mapping.TypeDatetime), + Entry("facet object", "audio", mapping.TypeObject), + Entry("geopoint", "location", mapping.TypeGeopoint), + Entry("unknown", "audio.nonexistent", ""), + ) +}) + +var _ = Describe("AggregatableFieldType", func() { + DescribeTable("leaves out the internal fields", + func(field, want string) { + Expect(search.AggregatableFieldType(field)).To(Equal(want)) + }, + Entry("name", "Name", mapping.TypeKeyword), + Entry("size", "Size", mapping.TypeNumeric), + Entry("modification time", "Mtime", mapping.TypeDatetime), + Entry("mime type", "MimeType", mapping.TypeKeyword), + Entry("tags", "Tags", mapping.TypeKeyword), + Entry("a facet", "audio.artist", mapping.TypeKeyword), + Entry("who favorited", "Favorites", ""), + Entry("an id", "ID", ""), + Entry("the parent id", "ParentID", ""), + Entry("the space root", "RootID", ""), + Entry("the path", "Path", ""), + Entry("the content", "Content", ""), + Entry("a flag", "Hidden", ""), + Entry("the resource type", "Type", ""), + Entry("the title", "Title", ""), + Entry("the deleted flag", "Deleted", ""), + Entry("unknown", "audio.nonexistent", ""), + ) +}) diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index ae2db60310..8401f1a098 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -79,16 +79,22 @@ var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts return map[string]mapping.FieldOpts{ // every keyword field searches case-insensitively and by word (name, // title, the facets) unless opted out: ids are opaque, paths are POSIX, - // the mime type is normalized already, a tag is one label - "ID": {CaseInsensitive: &False, NoWordBreaker: &True}, - "RootID": {CaseInsensitive: &False, NoWordBreaker: &True}, - "ParentID": {CaseInsensitive: &False, NoWordBreaker: &True}, - "Path": {Type: mapping.TypePath, CaseInsensitive: &False}, + // the mime type is normalized already, a tag is one label. + // Internal fields are no driveItem property: ids, flags, the content, + // and Favorites, which would list who favorited what. + "ID": {CaseInsensitive: &False, NoWordBreaker: &True, Internal: true}, + "RootID": {CaseInsensitive: &False, NoWordBreaker: &True, Internal: true}, + "ParentID": {CaseInsensitive: &False, NoWordBreaker: &True, Internal: true}, + "Path": {Type: mapping.TypePath, CaseInsensitive: &False, Internal: true}, "MimeType": {CaseInsensitive: &False, NoWordBreaker: &True}, - "Content": {Type: mapping.TypeFulltext}, + "Content": {Type: mapping.TypeFulltext, Internal: true}, + "Title": {Internal: true}, + "Type": {Internal: true}, + "Deleted": {Internal: true}, + "Hidden": {Internal: true}, "Tags": {NoWordBreaker: &True, IncludeInAll: &False}, - "Favorites": {NoWordBreaker: &True, IncludeInAll: &False, CaseInsensitive: &False}, // opaque user ids - "livePhoto.contentId": {NoWordBreaker: &True, CaseInsensitive: &False}, // opaque pairing uuid + "Favorites": {NoWordBreaker: &True, IncludeInAll: &False, CaseInsensitive: &False, Internal: true}, // opaque user ids + "livePhoto.contentId": {NoWordBreaker: &True, CaseInsensitive: &False}, // opaque pairing uuid "location": {Type: mapping.TypeGeopoint}, } }) diff --git a/services/search/pkg/search/service.go b/services/search/pkg/search/service.go index 482f21ee3d..fae6814535 100644 --- a/services/search/pkg/search/service.go +++ b/services/search/pkg/search/service.go @@ -36,6 +36,7 @@ import ( searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/graph/pkg/unifiedrole" + "github.com/opencloud-eu/opencloud/services/search/pkg/aggregation" "github.com/opencloud-eu/opencloud/services/search/pkg/config" "github.com/opencloud-eu/opencloud/services/search/pkg/content" "github.com/opencloud-eu/opencloud/services/search/pkg/metrics" @@ -102,6 +103,9 @@ func NewService(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], eng E // Search processes a search request and passes it down to the engine. func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) { + if err := aggregation.ValidateOptions(req.GetAggregations(), AggregatableFieldType); err != nil { + return nil, errtypes.BadRequest(err.Error()) + } s.logger.Debug().Str("query", req.Query).Msg("performing a search") // collect metrics @@ -293,6 +297,7 @@ func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*se return nil, err } + aggregations := aggregation.Empty(req.GetAggregations()) for _, res := range responses { if res == nil { continue @@ -301,7 +306,9 @@ func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*se for _, match := range res.Matches { matches = append(matches, match) } + aggregations = aggregation.Merge(req.GetAggregations(), aggregations, res.GetAggregations()) } + aggregation.Finalize(req.GetAggregations(), aggregations) // compile one sorted list of matches from all spaces and apply from/limit if needed sort.Sort(matches) @@ -321,6 +328,7 @@ func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*se return &searchsvc.SearchResponse{ Matches: matches, TotalMatches: total, + Aggregations: aggregations, }, nil } @@ -455,7 +463,8 @@ func (s *Service) searchIndex(ctx context.Context, req *searchsvc.SearchRequest, } searchRequest := &searchsvc.SearchIndexRequest{ - Query: req.Query, + Query: req.Query, + Aggregations: req.GetAggregations(), Ref: &searchmsg.Reference{ ResourceId: searchRootID, Path: searchPathPrefix, diff --git a/services/search/pkg/search/service_test.go b/services/search/pkg/search/service_test.go index 8448620925..d68323e9a5 100644 --- a/services/search/pkg/search/service_test.go +++ b/services/search/pkg/search/service_test.go @@ -12,11 +12,13 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" revactx "github.com/opencloud-eu/reva/v2/pkg/ctx" + "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/rgrpc/status" "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" cs3mocks "github.com/opencloud-eu/reva/v2/tests/cs3mocks/mocks" "github.com/stretchr/testify/mock" "google.golang.org/grpc" + "google.golang.org/protobuf/testing/protocmp" "github.com/opencloud-eu/opencloud/pkg/conversions" "github.com/opencloud-eu/opencloud/pkg/log" @@ -260,8 +262,35 @@ var _ = Describe("Searchprovider", func() { Expect(match.Entity.Ref.ResourceId.OpaqueId).To(Equal(personalSpace.Root.OpaqueId)) Expect(match.Entity.Ref.Path).To(Equal("./path/to/Foo.pdf")) }) + + It("forwards aggregations to the engine", func() { + _, err := s.Search(ctx, &searchsvc.SearchRequest{ + Query: "foo", + Aggregations: []*searchsvc.AggregationOption{ + {Field: "audio.artist", Size: 10}, + }, + }) + Expect(err).ToNot(HaveOccurred()) + indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool { + return len(req.Aggregations) == 1 && + req.Aggregations[0].Field == "audio.artist" && + req.Aggregations[0].Size == 10 + })) + }) }) + DescribeTable("rejects a request the index cannot answer without asking the engine", + func(req *searchsvc.SearchRequest) { + _, err := s.Search(ctx, req) + Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest(""))) + indexClient.AssertNotCalled(GinkgoT(), "Search", mock.Anything, mock.Anything) + gatewayClient.AssertNotCalled(GinkgoT(), "ListStorageSpaces", mock.Anything, mock.Anything) + }, + // the rules are pinned in the aggregation package + Entry("an aggregation on an internal field", &searchsvc.SearchRequest{Query: "foo", + Aggregations: []*searchsvc.AggregationOption{{Field: "Favorites"}}}), + ) + // two personal spaces, the engine answers each by its space id var ( spaceA = &sprovider.StorageSpace{ @@ -287,6 +316,33 @@ var _ = Describe("Searchprovider", func() { })) } + Context("with two personal spaces returning aggregations", func() { + BeforeEach(func() { + listsSpacesAB() + searchOfSpace("a").Return(&searchsvc.SearchIndexResponse{ + TotalMatches: 2, + Aggregations: []*searchsvc.AggregationResult{{Field: "audio.artist", Buckets: []*searchsvc.Bucket{{Key: "Saxon", Count: 2}, {Key: "Motörhead", Count: 1}}}}, + }, nil) + searchOfSpace("b").Return(&searchsvc.SearchIndexResponse{ + TotalMatches: 3, + Aggregations: []*searchsvc.AggregationResult{{Field: "audio.artist", Buckets: []*searchsvc.Bucket{{Key: "Saxon", Count: 3}, {Key: "Led Zeppelin", Count: 1}}}}, + }, nil) + }) + + It("merges bucket counts across spaces", func() { + res, err := s.Search(ctx, &searchsvc.SearchRequest{ + Query: "mediatype:audio", + Aggregations: []*searchsvc.AggregationOption{ + {Field: "audio.artist", Size: 10}, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Aggregations).To(BeComparableTo([]*searchsvc.AggregationResult{{Field: "audio.artist", Buckets: []*searchsvc.Bucket{ + {Key: "Saxon", Count: 5}, {Key: "Led Zeppelin", Count: 1}, {Key: "Motörhead", Count: 1}, + }}}, protocmp.Transform())) + }) + }) + Context("with two personal spaces returning matches of the same score", func() { match := func(space, name string) *searchmsg.Match { return &searchmsg.Match{Score: 1, Entity: &searchmsg.Entity{ diff --git a/services/search/pkg/service/grpc/v0/service.go b/services/search/pkg/service/grpc/v0/service.go index ff8bb0f4b1..5a762c7143 100644 --- a/services/search/pkg/service/grpc/v0/service.go +++ b/services/search/pkg/service/grpc/v0/service.go @@ -113,6 +113,7 @@ func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *s out.Matches = res.Matches out.TotalMatches = res.TotalMatches out.NextPageToken = res.NextPageToken + out.Aggregations = res.Aggregations return nil }