mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-08 19:58:46 -04:00
Compare commits
28
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1bafa4bd18 | ||
|
|
b18618ab4a | ||
|
|
65f19b5d71 | ||
|
|
7f687ea288 | ||
|
|
be83ed19e3 | ||
|
|
b5cb1cd83b | ||
|
|
9797ee903f | ||
|
|
973dbbdf1a | ||
|
|
1ce7f07523 | ||
|
|
3b83221a87 | ||
|
|
391705b017 | ||
|
|
8875d062b0 | ||
|
|
d1d20edbd2 | ||
|
|
b50c5e54d6 | ||
|
|
c2a34e8c45 | ||
|
|
cfefb83d26 | ||
|
|
9357124a91 | ||
|
|
c36af05d14 | ||
|
|
7947c1c98a | ||
|
|
d58c1e3dab | ||
|
|
5a609c98dc | ||
|
|
29913e1d7f | ||
|
|
5028364e2c | ||
|
|
b2b15d44f7 | ||
|
|
6ea363b0b9 | ||
|
|
9594313243 | ||
|
|
62d815abe9 | ||
|
|
5923aa5e5e |
No files matched your search
@@ -95,7 +95,7 @@ require (
|
||||
go-micro.dev/v4 v4.11.0
|
||||
go.etcd.io/bbolt v1.5.0
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0
|
||||
go.opentelemetry.io/contrib/zpages v0.71.0
|
||||
go.opentelemetry.io/otel v1.46.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
|
||||
|
||||
@@ -1293,8 +1293,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
|
||||
go.opentelemetry.io/contrib/zpages v0.71.0 h1:OrQSEmdlfknIusEyGj/t30uBR7MnbmGcCRBt5ov8/rA=
|
||||
go.opentelemetry.io/contrib/zpages v0.71.0/go.mod h1:C3IIpqkQbjswcSREqxQLyEapF0SI9bqwJpxpjwtzFrE=
|
||||
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
|
||||
|
||||
@@ -824,10 +824,6 @@ type Entity struct {
|
||||
MotionPhoto *MotionPhoto `protobuf:"bytes,21,opt,name=motionPhoto,proto3" json:"motionPhoto,omitempty"`
|
||||
Video *Video `protobuf:"bytes,22,opt,name=video,proto3" json:"video,omitempty"`
|
||||
LivePhoto *LivePhoto `protobuf:"bytes,23,opt,name=livePhoto,proto3" json:"livePhoto,omitempty"`
|
||||
// The effective permission actions of the caller, projected from the space
|
||||
// root permission set at query time (the same source as `permissions`), for
|
||||
// the driveItem `@libre.graph.permissions.actions.allowedValues` facet.
|
||||
PermissionsActionsAllowedValues []string `protobuf:"bytes,24,rep,name=permissionsActionsAllowedValues,proto3" json:"permissionsActionsAllowedValues,omitempty"`
|
||||
}
|
||||
|
||||
func (x *Entity) Reset() {
|
||||
@@ -1023,13 +1019,6 @@ func (x *Entity) GetLivePhoto() *LivePhoto {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Entity) GetPermissionsActionsAllowedValues() []string {
|
||||
if x != nil {
|
||||
return x.PermissionsActionsAllowedValues
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Match struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -1270,7 +1259,7 @@ var file_opencloud_messages_search_v0_search_proto_rawDesc = []byte{
|
||||
0x5f, 0x61, 0x75, 0x74, 0x6f, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x76, 0x69, 0x74, 0x61, 0x6c, 0x69,
|
||||
0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x19, 0x0a, 0x17, 0x5f, 0x76, 0x69, 0x74, 0x61,
|
||||
0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x65, 0x72, 0x73, 0x69,
|
||||
0x6f, 0x6e, 0x22, 0x93, 0x09, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a,
|
||||
0x6f, 0x6e, 0x22, 0xc9, 0x08, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a,
|
||||
0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65,
|
||||
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
|
||||
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65,
|
||||
@@ -1338,23 +1327,19 @@ var file_opencloud_messages_search_v0_search_proto_rawDesc = []byte{
|
||||
0x68, 0x6f, 0x74, 0x6f, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65,
|
||||
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
|
||||
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x4c, 0x69, 0x76, 0x65, 0x50, 0x68,
|
||||
0x6f, 0x74, 0x6f, 0x52, 0x09, 0x6c, 0x69, 0x76, 0x65, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x12, 0x48,
|
||||
0x0a, 0x1f, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x41, 0x63, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0x73, 0x41, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65,
|
||||
0x73, 0x18, 0x18, 0x20, 0x03, 0x28, 0x09, 0x52, 0x1f, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73,
|
||||
0x69, 0x6f, 0x6e, 0x73, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x41, 0x6c, 0x6c, 0x6f, 0x77,
|
||||
0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63,
|
||||
0x68, 0x12, 0x3c, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
|
||||
0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65,
|
||||
0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30,
|
||||
0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12,
|
||||
0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05,
|
||||
0x73, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x4d, 0x5a, 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
|
||||
0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75,
|
||||
0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
|
||||
0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75,
|
||||
0x64, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63,
|
||||
0x68, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x6f, 0x74, 0x6f, 0x52, 0x09, 0x6c, 0x69, 0x76, 0x65, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x22, 0x5b,
|
||||
0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x3c, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74,
|
||||
0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
|
||||
0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61,
|
||||
0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65,
|
||||
0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02,
|
||||
0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x4d, 0x5a, 0x4b, 0x67,
|
||||
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
|
||||
0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
|
||||
0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70,
|
||||
0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73,
|
||||
0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -167,67 +167,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0AggregationOption": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"field": {
|
||||
"type": "string",
|
||||
"description": "Required. The indexed field to aggregate on (for terms/range\naggregations) or to reduce (for metric aggregations)."
|
||||
},
|
||||
"size": {
|
||||
"type": "integer",
|
||||
"format": "int32",
|
||||
"description": "Optional. Maximum number of buckets to return for a terms aggregation.\nIgnored for range and metric aggregations."
|
||||
},
|
||||
"bucketDefinition": {
|
||||
"$ref": "#/definitions/v0BucketDefinition",
|
||||
"description": "Optional. Controls bucket selection, ordering and filtering.\nIgnored for metric aggregations."
|
||||
},
|
||||
"subAggregations": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/v0AggregationOption"
|
||||
},
|
||||
"description": "Optional. Nested aggregations computed within each bucket of this\naggregation. On bleve, sub-aggregations are emulated by walking the\nmatched result set; on OpenSearch they translate to native composite\naggregations."
|
||||
},
|
||||
"metricKind": {
|
||||
"$ref": "#/definitions/v0MetricKind",
|
||||
"description": "Optional. When set, this aggregation is a scalar metric over `field`\nrather than a bucket aggregation; the corresponding AggregationResult\ncarries `value` instead of `buckets`."
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0AggregationResult": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"field": {
|
||||
"type": "string"
|
||||
},
|
||||
"buckets": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/v0Bucket"
|
||||
}
|
||||
},
|
||||
"value": {
|
||||
"type": "number",
|
||||
"format": "double",
|
||||
"description": "Scalar value for metric aggregations (metric_kind set on the\ncorresponding AggregationOption). Unset / zero for terms/range\naggregations."
|
||||
},
|
||||
"metricKind": {
|
||||
"$ref": "#/definitions/v0MetricKind",
|
||||
"description": "Echoes the metric_kind of the corresponding AggregationOption. Lets\nthe cross-space merge layer pick the right reducer."
|
||||
},
|
||||
"sum": {
|
||||
"type": "number",
|
||||
"format": "double",
|
||||
"description": "Accumulators used exclusively for AVG during cross-space merges.\nA backend computes (sum, count) per bucket so the service layer can\nmerge them additively and emit `value = sum/count` only at the\noutermost collapse. Other metric kinds leave these unset."
|
||||
},
|
||||
"count": {
|
||||
"type": "string",
|
||||
"format": "int64"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0Audio": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -288,63 +227,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0Bucket": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"count": {
|
||||
"type": "string",
|
||||
"format": "int64"
|
||||
},
|
||||
"aggregationFilterToken": {
|
||||
"type": "string"
|
||||
},
|
||||
"subAggregations": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/v0AggregationResult"
|
||||
},
|
||||
"description": "Nested aggregation results, one entry per sub_aggregation requested\non the parent AggregationOption."
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0BucketDefinition": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sortBy": {
|
||||
"type": "string",
|
||||
"description": "One of \"count\", \"keyAsString\", \"keyAsNumber\"."
|
||||
},
|
||||
"isDescending": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"minimumCount": {
|
||||
"type": "integer",
|
||||
"format": "int32"
|
||||
},
|
||||
"ranges": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/v0BucketRange"
|
||||
},
|
||||
"description": "Optional. When non-empty, the aggregation is computed over these numeric\nor date ranges instead of as a terms aggregation."
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0BucketRange": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"from": {
|
||||
"type": "string",
|
||||
"description": "At least one of `from` or `to` must be set. Both are string-encoded;\nnumeric bounds are decimal strings, dates use RFC3339."
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0Entity": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -425,13 +307,6 @@
|
||||
},
|
||||
"livePhoto": {
|
||||
"$ref": "#/definitions/v0LivePhoto"
|
||||
},
|
||||
"permissionsActionsAllowedValues": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "The effective permission actions of the caller, projected from the space\nroot permission set at query time (the same source as `permissions`), for\nthe driveItem `@libre.graph.permissions.actions.allowedValues` facet."
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -547,17 +422,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0MetricKind": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"METRIC_KIND_UNSPECIFIED",
|
||||
"METRIC_KIND_SUM",
|
||||
"METRIC_KIND_MIN",
|
||||
"METRIC_KIND_MAX",
|
||||
"METRIC_KIND_AVG"
|
||||
],
|
||||
"default": "METRIC_KIND_UNSPECIFIED"
|
||||
},
|
||||
"v0MotionPhoto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -656,27 +520,6 @@
|
||||
},
|
||||
"ref": {
|
||||
"$ref": "#/definitions/v0Reference"
|
||||
},
|
||||
"aggregations": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/v0AggregationOption"
|
||||
},
|
||||
"description": "Optional. Per-space aggregations (facets) to compute alongside the matches."
|
||||
},
|
||||
"aggregationFilters": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "Optional. Decoded aggregation filters as KQL fragments; the engine parses\neach, forces exact/case-sensitive matching, and ANDs them with `query`."
|
||||
},
|
||||
"orderBy": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/v0SortProperty"
|
||||
},
|
||||
"description": "Optional. Fields to sort the matches by, in order of precedence. When\nempty, matches are sorted by relevance score. Each backend translates\nthis to its native sort (bleve: SortBy, OpenSearch: sort clause)."
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -696,13 +539,6 @@
|
||||
"totalMatches": {
|
||||
"type": "integer",
|
||||
"format": "int32"
|
||||
},
|
||||
"aggregations": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/v0AggregationResult"
|
||||
},
|
||||
"description": "Per-space aggregation results. The service layer merges these across\nspaces before returning them to the caller."
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -723,27 +559,6 @@
|
||||
},
|
||||
"ref": {
|
||||
"$ref": "#/definitions/v0Reference"
|
||||
},
|
||||
"aggregations": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/v0AggregationOption"
|
||||
},
|
||||
"description": "Optional. Aggregations (facets) to compute alongside the matches."
|
||||
},
|
||||
"aggregationFilters": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "Optional. Decoded aggregation filters, one per selected bucket, as KQL\nfragments (e.g. `audio.artist:\"Pink Floyd\"`). Combined with `query` via AND\nand matched case-sensitively/exactly. Passed through from the graph layer."
|
||||
},
|
||||
"orderBy": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/v0SortProperty"
|
||||
},
|
||||
"description": "Optional. Fields to sort the matches by, in order of precedence. When\nempty, matches are sorted by relevance score. Only a subset of the\nindexed fields is sortable; the graph service validates this before\nforwarding the request."
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -763,26 +578,6 @@
|
||||
"totalMatches": {
|
||||
"type": "integer",
|
||||
"format": "int32"
|
||||
},
|
||||
"aggregations": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/v0AggregationResult"
|
||||
},
|
||||
"description": "Aggregation results, one entry per requested aggregation."
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0SortProperty": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "Required. The field to sort on, in graph notation (\"name\", \"size\",\n\"lastModifiedDateTime\", \"mimeType\" or a scalar facet field such as\n\"photo.takenDateTime\" or \"audio.artist\"). A field is sortable when it is\nindexed as a scalar in both backends AND carried on the Match entity\n(the service layer needs the sort key to merge per-space result\nstreams); see the search package's IsSortableField."
|
||||
},
|
||||
"isDescending": {
|
||||
"type": "boolean",
|
||||
"description": "Optional. Sort in descending order. Defaults to ascending."
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -110,10 +110,6 @@ message Entity {
|
||||
MotionPhoto motionPhoto = 21;
|
||||
Video video = 22;
|
||||
LivePhoto livePhoto = 23;
|
||||
// The effective permission actions of the caller, projected from the space
|
||||
// root permission set at query time (the same source as `permissions`), for
|
||||
// the driveItem `@libre.graph.permissions.actions.allowedValues` facet.
|
||||
repeated string permissionsActionsAllowedValues = 24;
|
||||
}
|
||||
|
||||
message Match {
|
||||
|
||||
@@ -72,19 +72,6 @@ message SearchRequest {
|
||||
|
||||
string query = 3;
|
||||
opencloud.messages.search.v0.Reference ref = 4 [(google.api.field_behavior) = OPTIONAL];
|
||||
|
||||
// Optional. Aggregations (facets) to compute alongside the matches.
|
||||
repeated AggregationOption aggregations = 5 [(google.api.field_behavior) = OPTIONAL];
|
||||
|
||||
// Optional. Decoded aggregation filters, one per selected bucket, as KQL
|
||||
// fragments (e.g. `audio.artist:"Pink Floyd"`). Combined with `query` via AND
|
||||
// and matched case-sensitively/exactly. Passed through from the graph layer.
|
||||
repeated string aggregation_filters = 6 [(google.api.field_behavior) = OPTIONAL];
|
||||
// Optional. Fields to sort the matches by, in order of precedence. When
|
||||
// empty, matches are sorted by relevance score. Only a subset of the
|
||||
// indexed fields is sortable; the graph service validates this before
|
||||
// forwarding the request.
|
||||
repeated SortProperty order_by = 7 [(google.api.field_behavior) = OPTIONAL];
|
||||
}
|
||||
|
||||
message SearchResponse {
|
||||
@@ -94,9 +81,6 @@ message SearchResponse {
|
||||
// more results in the list
|
||||
string next_page_token = 2;
|
||||
int32 total_matches = 3;
|
||||
|
||||
// Aggregation results, one entry per requested aggregation.
|
||||
repeated AggregationResult aggregations = 4;
|
||||
}
|
||||
|
||||
message SearchIndexRequest {
|
||||
@@ -109,17 +93,6 @@ message SearchIndexRequest {
|
||||
|
||||
string query = 3;
|
||||
opencloud.messages.search.v0.Reference ref = 4 [(google.api.field_behavior) = OPTIONAL];
|
||||
|
||||
// Optional. Per-space aggregations (facets) to compute alongside the matches.
|
||||
repeated AggregationOption aggregations = 5 [(google.api.field_behavior) = OPTIONAL];
|
||||
|
||||
// Optional. Decoded aggregation filters as KQL fragments; the engine parses
|
||||
// each, forces exact/case-sensitive matching, and ANDs them with `query`.
|
||||
repeated string aggregation_filters = 6 [(google.api.field_behavior) = OPTIONAL];
|
||||
// Optional. Fields to sort the matches by, in order of precedence. When
|
||||
// empty, matches are sorted by relevance score. Each backend translates
|
||||
// this to its native sort (bleve: SortBy, OpenSearch: sort clause).
|
||||
repeated SortProperty order_by = 7 [(google.api.field_behavior) = OPTIONAL];
|
||||
}
|
||||
|
||||
message SearchIndexResponse {
|
||||
@@ -129,95 +102,6 @@ message SearchIndexResponse {
|
||||
// more results in the list
|
||||
string next_page_token = 2;
|
||||
int32 total_matches = 3;
|
||||
|
||||
// Per-space aggregation results. The service layer merges these across
|
||||
// spaces before returning them to the caller.
|
||||
repeated AggregationResult aggregations = 4;
|
||||
}
|
||||
|
||||
message AggregationOption {
|
||||
// Required. The indexed field to aggregate on (for terms/range
|
||||
// aggregations) or to reduce (for metric aggregations).
|
||||
string field = 1;
|
||||
// Optional. Maximum number of buckets to return for a terms aggregation.
|
||||
// Ignored for range and metric aggregations.
|
||||
int32 size = 2 [(google.api.field_behavior) = OPTIONAL];
|
||||
// Optional. Controls bucket selection, ordering and filtering.
|
||||
// Ignored for metric aggregations.
|
||||
BucketDefinition bucket_definition = 3 [(google.api.field_behavior) = OPTIONAL];
|
||||
// Optional. Nested aggregations computed within each bucket of this
|
||||
// aggregation. On bleve, sub-aggregations are emulated by walking the
|
||||
// matched result set; on OpenSearch they translate to native composite
|
||||
// aggregations.
|
||||
repeated AggregationOption sub_aggregations = 4 [(google.api.field_behavior) = OPTIONAL];
|
||||
// Optional. When set, this aggregation is a scalar metric over `field`
|
||||
// rather than a bucket aggregation; the corresponding AggregationResult
|
||||
// carries `value` instead of `buckets`.
|
||||
MetricKind metric_kind = 5 [(google.api.field_behavior) = OPTIONAL];
|
||||
}
|
||||
|
||||
message SortProperty {
|
||||
// Required. The field to sort on, in graph notation ("name", "size",
|
||||
// "lastModifiedDateTime", "mimeType" or a scalar facet field such as
|
||||
// "photo.takenDateTime" or "audio.artist"). A field is sortable when it is
|
||||
// indexed as a scalar in both backends AND carried on the Match entity
|
||||
// (the service layer needs the sort key to merge per-space result
|
||||
// streams); see the search package's IsSortableField.
|
||||
string name = 1;
|
||||
// Optional. Sort in descending order. Defaults to ascending.
|
||||
bool is_descending = 2;
|
||||
}
|
||||
|
||||
enum MetricKind {
|
||||
METRIC_KIND_UNSPECIFIED = 0;
|
||||
METRIC_KIND_SUM = 1;
|
||||
METRIC_KIND_MIN = 2;
|
||||
METRIC_KIND_MAX = 3;
|
||||
METRIC_KIND_AVG = 4;
|
||||
}
|
||||
|
||||
message BucketDefinition {
|
||||
// One of "count", "keyAsString", "keyAsNumber".
|
||||
string sort_by = 1;
|
||||
bool is_descending = 2;
|
||||
int32 minimum_count = 3;
|
||||
// Optional. When non-empty, the aggregation is computed over these numeric
|
||||
// or date ranges instead of as a terms aggregation.
|
||||
repeated BucketRange ranges = 4;
|
||||
}
|
||||
|
||||
message BucketRange {
|
||||
// At least one of `from` or `to` must be set. Both are string-encoded;
|
||||
// numeric bounds are decimal strings, dates use RFC3339.
|
||||
string from = 1;
|
||||
string to = 2;
|
||||
}
|
||||
|
||||
message AggregationResult {
|
||||
string field = 1;
|
||||
repeated Bucket buckets = 2;
|
||||
// Scalar value for metric aggregations (metric_kind set on the
|
||||
// corresponding AggregationOption). Unset / zero for terms/range
|
||||
// aggregations.
|
||||
double value = 3;
|
||||
// Echoes the metric_kind of the corresponding AggregationOption. Lets
|
||||
// the cross-space merge layer pick the right reducer.
|
||||
MetricKind metric_kind = 4;
|
||||
// Accumulators used exclusively for AVG during cross-space merges.
|
||||
// A backend computes (sum, count) per bucket so the service layer can
|
||||
// merge them additively and emit `value = sum/count` only at the
|
||||
// outermost collapse. Other metric kinds leave these unset.
|
||||
double sum = 5;
|
||||
int64 count = 6;
|
||||
}
|
||||
|
||||
message Bucket {
|
||||
string key = 1;
|
||||
int64 count = 2;
|
||||
string aggregation_filter_token = 3;
|
||||
// Nested aggregation results, one entry per sub_aggregation requested
|
||||
// on the parent AggregationOption.
|
||||
repeated AggregationResult sub_aggregations = 4;
|
||||
}
|
||||
|
||||
message IndexSpaceRequest {
|
||||
|
||||
@@ -3,17 +3,18 @@ package command
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/pkg/runner"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/olekukonko/errors"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/pkg/generators"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
natspkg "github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/registry"
|
||||
"github.com/opencloud-eu/opencloud/pkg/runner"
|
||||
ogrpc "github.com/opencloud-eu/opencloud/pkg/service/grpc"
|
||||
"github.com/opencloud-eu/opencloud/pkg/tracing"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
@@ -24,6 +25,12 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/metrics"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/server/debug"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/server/http"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
svcEvents "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events"
|
||||
svcHttp "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
var _registeredEvents = []events.Unmarshaller{
|
||||
@@ -62,19 +69,11 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
|
||||
gr := runner.NewGroup()
|
||||
ctx, cancel := context.WithCancel(cmd.Context())
|
||||
defer cancel()
|
||||
|
||||
mtrcs := metrics.New()
|
||||
mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1)
|
||||
|
||||
defer cancel()
|
||||
|
||||
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
|
||||
evStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig(cfg.Events))
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to initialize event stream")
|
||||
return err
|
||||
}
|
||||
|
||||
tm, err := pool.StringToTLSMode(cfg.GRPCClientTLS.Mode)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to parse tls mode")
|
||||
@@ -99,28 +98,101 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
return err
|
||||
}
|
||||
|
||||
hClient := ehsvc.NewEventHistoryService("eu.opencloud.api.eventhistory", grpcClient)
|
||||
vClient := settingssvc.NewValueService("eu.opencloud.api.settings", grpcClient)
|
||||
kv, err := ConnectNatsKV(cfg.Store)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
activityLog, err := activitylog.New(kv,
|
||||
activitylog.Logger(logger),
|
||||
activitylog.MaxActivities(cfg.MaxActivities),
|
||||
activitylog.WriteBufferDuration(cfg.WriteBufferDuration),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to initialize activity log")
|
||||
return err
|
||||
}
|
||||
|
||||
{
|
||||
svc, err := http.Server(
|
||||
http.Logger(logger),
|
||||
http.Config(cfg),
|
||||
http.Context(ctx), // NOTE: not passing this "option" leads to a panic in go-micro
|
||||
http.TraceProvider(tracerProvider),
|
||||
http.Stream(evStream),
|
||||
http.GatewaySelector(gatewaySelector),
|
||||
http.HistoryClient(hClient),
|
||||
http.ValueClient(vClient),
|
||||
http.RegisteredEvents(_registeredEvents),
|
||||
if !cfg.HTTP.Disabled {
|
||||
|
||||
hClient := ehsvc.NewEventHistoryService("eu.opencloud.api.eventhistory", grpcClient)
|
||||
|
||||
svc, err := svcHttp.New(
|
||||
activityLog,
|
||||
svcHttp.Logger(logger),
|
||||
svcHttp.GatewaySelector(gatewaySelector),
|
||||
svcHttp.RegisteredEvents(_registeredEvents),
|
||||
//svcHttp.TraceProvider(tracerProvider),
|
||||
svcHttp.HistoryClient(hClient),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Str("transport", "http").Msg("Failed to initialize server")
|
||||
logger.Error().Err(err).Msg("handler init")
|
||||
return err
|
||||
}
|
||||
// TODO svc = service.NewInstrument(svc, metrics)
|
||||
// TODO svc = service.NewLogging(svc, logger) // this logs service specific data
|
||||
// TODO svc = service.NewTracing(svc, traceProvider)
|
||||
vClient := settingssvc.NewValueService("eu.opencloud.api.settings", grpcClient)
|
||||
|
||||
server, err := http.Server(
|
||||
http.ValueClient(vClient),
|
||||
http.Logger(logger),
|
||||
http.Context(ctx),
|
||||
http.Config(cfg),
|
||||
http.Service(svc),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Info().
|
||||
Err(err).
|
||||
Str("transport", "http").
|
||||
Msg("Failed to initialize server")
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", svc))
|
||||
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", server))
|
||||
} else {
|
||||
logger.Info().Msg("HTTP server disabled, not starting HTTP service")
|
||||
}
|
||||
|
||||
if !cfg.Events.Disabled {
|
||||
|
||||
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
|
||||
evStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig{
|
||||
Endpoint: cfg.Events.Endpoint,
|
||||
Cluster: cfg.Events.Cluster,
|
||||
EnableTLS: cfg.Events.EnableTLS,
|
||||
TLSInsecure: cfg.Events.TLSInsecure,
|
||||
TLSRootCACertificate: cfg.Events.TLSRootCACertificate,
|
||||
AuthUsername: cfg.Events.AuthUsername,
|
||||
AuthPassword: cfg.Events.AuthPassword,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to initialize event stream")
|
||||
return err
|
||||
}
|
||||
|
||||
eventSvc, err := svcEvents.New(
|
||||
activityLog,
|
||||
evStream,
|
||||
svcEvents.Context(ctx),
|
||||
svcEvents.Logger(logger),
|
||||
svcEvents.ServiceAccount(cfg.ServiceAccount),
|
||||
svcEvents.GatewaySelector(gatewaySelector),
|
||||
svcEvents.RegisteredEvents(_registeredEvents),
|
||||
svcEvents.NumConsumers(cfg.NumConsumers),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Str("transport", "event").Msg("Failed to initialize server")
|
||||
return err
|
||||
}
|
||||
|
||||
gr.Add(runner.New(cfg.Service.Name+".svc", func() error {
|
||||
return eventSvc.Run()
|
||||
}, func() {
|
||||
eventSvc.Close()
|
||||
}))
|
||||
} else {
|
||||
logger.Info().Msg("event listening disabled, not starting event service")
|
||||
}
|
||||
|
||||
{
|
||||
@@ -149,3 +221,33 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func ConnectNatsKV(cfg config.Store) (nats.KeyValue, error) {
|
||||
// Connect to NATS servers
|
||||
secureOption := natspkg.Secure(cfg.EnableTLS, cfg.TLSInsecure, cfg.TLSRootCACertificate)
|
||||
conn, err := nats.Connect(strings.Join(cfg.Nodes, ","), secureOption, nats.UserInfo(cfg.AuthUsername, cfg.AuthPassword))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
kv, err := js.KeyValue(cfg.Database)
|
||||
if err != nil {
|
||||
if !errors.Is(err, nats.ErrBucketNotFound) {
|
||||
return nil, errors.Wrapf(err, "Failed to get bucket (%s)", cfg.Database)
|
||||
}
|
||||
|
||||
kv, err = js.CreateKeyValue(&nats.KeyValueConfig{
|
||||
Bucket: cfg.Database,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to create bucket (%s)", cfg.Database)
|
||||
}
|
||||
}
|
||||
|
||||
return kv, nil
|
||||
}
|
||||
@@ -35,10 +35,12 @@ type Config struct {
|
||||
|
||||
WriteBufferDuration time.Duration `yaml:"write_buffer_duration" env:"ACTIVITYLOG_WRITE_BUFFER_DURATION" desc:"The duration to wait before flushing the write buffer. This is used to reduce the number of writes to the store." introductionVersion:"4.0.0"`
|
||||
MaxActivities int `yaml:"max_activities" env:"ACTIVITYLOG_MAX_ACTIVITIES" desc:"The maximum number of activities to keep in the store per resource. If the number of activities exceeds this value, the oldest activities will be removed." introductionVersion:"4.0.0"`
|
||||
NumConsumers int `yaml:"num_consumers" env:"ACTIVITYLOG_NUM_CONSUMERS" desc:"The amount of concurrent event consumers to start. Event consumers are used for updating the list of activities. Multiple consumers increase parallelisation, but will also increase CPU and memory demands." introductionVersion:"%NEXT%"`
|
||||
}
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
Disabled bool `yaml:"disabled" env:"ACTIVITYLOG_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%NEXT%"`
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
|
||||
@@ -77,6 +79,7 @@ type CORS struct {
|
||||
|
||||
// HTTP defines the available http configuration.
|
||||
type HTTP struct {
|
||||
Disabled bool `yaml:"disabled" env:"ACTIVITYLOG_HTTP_DISABLED" desc:"Disables the HTTP service. Set this to true if the service should only handle events." introductionVersion:"1.0.0"`
|
||||
Addr string `yaml:"addr" env:"ACTIVITYLOG_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
Root string `yaml:"root" env:"ACTIVITYLOG_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -52,7 +52,9 @@ func DefaultConfig() *config.Config {
|
||||
},
|
||||
},
|
||||
WriteBufferDuration: 10 * time.Second,
|
||||
MaxActivities: 6000,
|
||||
// Nats runs into max payload exceeded errors at around 7k activities. Let's keep a buffer.
|
||||
MaxActivities: 6000,
|
||||
NumConsumers: 1,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
|
||||
occfg "github.com/opencloud-eu/opencloud/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/pkg/shared"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config/defaults"
|
||||
|
||||
@@ -34,5 +35,8 @@ func ParseConfig(cfg *config.Config) error {
|
||||
|
||||
// Validate validates the config
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.Events.Disabled && cfg.HTTP.Disabled {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -3,18 +3,12 @@ package http
|
||||
import (
|
||||
"context"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
"go-micro.dev/v4/store"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
@@ -22,19 +16,15 @@ type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Metrics *metrics.Metrics
|
||||
Flags []pflag.Flag
|
||||
Namespace string
|
||||
Store store.Store
|
||||
Stream events.Stream
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
TraceProvider trace.TracerProvider
|
||||
HistoryClient ehsvc.EventHistoryService
|
||||
ValueClient settingssvc.ValueService
|
||||
RegisteredEvents []events.Unmarshaller
|
||||
Name string
|
||||
Namespace string
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Flags []pflag.Flag
|
||||
Service ActivityLogService
|
||||
TraceProvider trace.TracerProvider
|
||||
ValueClient settingssvc.ValueService
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
@@ -69,10 +59,10 @@ func Config(val *config.Config) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// Metrics provides a function to set the metrics option.
|
||||
func Metrics(val *metrics.Metrics) Option {
|
||||
// Service provides a function to set the service option.
|
||||
func Service(val ActivityLogService) Option {
|
||||
return func(o *Options) {
|
||||
o.Metrics = val
|
||||
o.Service = val
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,58 +73,20 @@ func Flags(flags ...pflag.Flag) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// Namespace provides a function to set the Namespace option.
|
||||
func Namespace(val string) Option {
|
||||
// TraceProvider provides a function to configure the trace provider
|
||||
func TraceProvider(traceProvider trace.TracerProvider) Option {
|
||||
return func(o *Options) {
|
||||
o.Namespace = val
|
||||
if traceProvider != nil {
|
||||
o.TraceProvider = traceProvider
|
||||
} else {
|
||||
o.TraceProvider = noop.NewTracerProvider()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Store provides a function to configure the store
|
||||
func Store(store store.Store) Option {
|
||||
// ValueClient adds a grpc client for the value service
|
||||
func ValueClient(vs settingssvc.ValueService) Option {
|
||||
return func(o *Options) {
|
||||
o.Store = store
|
||||
}
|
||||
}
|
||||
|
||||
// Stream provides a function to configure the stream
|
||||
func Stream(stream events.Stream) Option {
|
||||
return func(o *Options) {
|
||||
o.Stream = stream
|
||||
}
|
||||
}
|
||||
|
||||
// GatewaySelector provides a function to configure the gateway client selector
|
||||
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
|
||||
return func(o *Options) {
|
||||
o.GatewaySelector = gatewaySelector
|
||||
}
|
||||
}
|
||||
|
||||
// HistoryClient provides a function to configure the event history client
|
||||
func HistoryClient(h ehsvc.EventHistoryService) Option {
|
||||
return func(o *Options) {
|
||||
o.HistoryClient = h
|
||||
}
|
||||
}
|
||||
|
||||
// RegisteredEvents provides a function to register events
|
||||
func RegisteredEvents(evs []events.Unmarshaller) Option {
|
||||
return func(o *Options) {
|
||||
o.RegisteredEvents = evs
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider provides a function to set the TracerProvider option
|
||||
func TraceProvider(val trace.TracerProvider) Option {
|
||||
return func(o *Options) {
|
||||
o.TraceProvider = val
|
||||
}
|
||||
}
|
||||
|
||||
// ValueClient provides a function to set the ValueClient options
|
||||
func ValueClient(val settingssvc.ValueService) Option {
|
||||
return func(o *Options) {
|
||||
o.ValueClient = val
|
||||
o.ValueClient = vs
|
||||
}
|
||||
}
|
||||
@@ -1,49 +1,63 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
stdhttp "net/http"
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
chimiddleware "github.com/go-chi/chi/v5/middleware"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/opencloud/pkg/account"
|
||||
"github.com/opencloud-eu/opencloud/pkg/cors"
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/pkg/middleware"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/http"
|
||||
"github.com/opencloud-eu/opencloud/pkg/tracing"
|
||||
ohttp "github.com/opencloud-eu/opencloud/pkg/service/http"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
svc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"
|
||||
"github.com/riandyrn/otelchi"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
activityloghttp "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"go-micro.dev/v4"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
// Service is the service interface
|
||||
type Service any
|
||||
var (
|
||||
//go:embed l10n/locale
|
||||
_localeFS embed.FS
|
||||
|
||||
// subfolder where the translation files are stored
|
||||
_localeSubPath = "l10n/locale"
|
||||
|
||||
// domain of the activitylog service (transifex)
|
||||
_domain = "activitylog"
|
||||
)
|
||||
|
||||
// Server initializes the http service and server.
|
||||
func Server(opts ...Option) (http.Service, error) {
|
||||
func Server(opts ...Option) (ohttp.Service, error) {
|
||||
options := newOptions(opts...)
|
||||
service := options.Service
|
||||
|
||||
service, err := http.NewService(
|
||||
http.TLSConfig(options.Config.HTTP.TLS),
|
||||
http.Logger(options.Logger),
|
||||
http.Namespace(options.Config.HTTP.Namespace),
|
||||
http.Name(options.Config.Service.Name),
|
||||
http.Version(version.GetString()),
|
||||
http.Address(options.Config.HTTP.Addr),
|
||||
http.Context(options.Context),
|
||||
http.Flags(options.Flags...),
|
||||
http.TraceProvider(options.TraceProvider),
|
||||
newService, err := ohttp.NewService(
|
||||
ohttp.TLSConfig(options.Config.HTTP.TLS),
|
||||
ohttp.Logger(options.Logger),
|
||||
ohttp.Namespace(options.Config.HTTP.Namespace),
|
||||
ohttp.Name(options.Config.Service.Name),
|
||||
ohttp.Version(version.GetString()),
|
||||
ohttp.Address(options.Config.HTTP.Addr),
|
||||
ohttp.Context(options.Context),
|
||||
ohttp.Flags(options.Flags...),
|
||||
)
|
||||
if err != nil {
|
||||
options.Logger.Error().
|
||||
Err(err).
|
||||
Msg("Error initializing http service")
|
||||
return http.Service{}, fmt.Errorf("could not initialize http service: %w", err)
|
||||
return ohttp.Service{}, err
|
||||
}
|
||||
|
||||
middlewares := []func(stdhttp.Handler) stdhttp.Handler{
|
||||
middlewares := []func(http.Handler) http.Handler{
|
||||
chimiddleware.RequestID,
|
||||
middleware.Version(
|
||||
options.Config.Service.Name,
|
||||
@@ -52,6 +66,7 @@ func Server(opts ...Option) (http.Service, error) {
|
||||
middleware.Logger(
|
||||
options.Logger,
|
||||
),
|
||||
middleware.TraceContext,
|
||||
middleware.ExtractAccountUUID(
|
||||
account.Logger(options.Logger),
|
||||
account.JWTSecret(options.Config.TokenManager.JWTSecret),
|
||||
@@ -68,33 +83,75 @@ func Server(opts ...Option) (http.Service, error) {
|
||||
mux := chi.NewMux()
|
||||
mux.Use(middlewares...)
|
||||
|
||||
mux.Use(
|
||||
otelchi.Middleware(
|
||||
"actitivylog",
|
||||
otelchi.WithChiRoutes(mux),
|
||||
otelchi.WithTracerProvider(options.TraceProvider),
|
||||
otelchi.WithPropagators(tracing.GetPropagator()),
|
||||
),
|
||||
)
|
||||
t := l10n.NewTranslatorFromCommonConfig(options.Config.DefaultLanguage, _domain, options.Config.TranslationPath, _localeFS, _localeSubPath)
|
||||
mux.Route(options.Config.HTTP.Root, func(r chi.Router) {
|
||||
r.Get("/graph/v1beta1/extensions/org.libregraph/activities", GetItemActivitiesHandler(options.Logger, service, options.ValueClient, t))
|
||||
})
|
||||
|
||||
handle, err := svc.New(
|
||||
svc.Logger(options.Logger),
|
||||
svc.Stream(options.Stream),
|
||||
svc.Mux(mux),
|
||||
svc.Config(options.Config),
|
||||
svc.GatewaySelector(options.GatewaySelector),
|
||||
svc.TraceProvider(options.TraceProvider),
|
||||
svc.HistoryClient(options.HistoryClient),
|
||||
svc.ValueClient(options.ValueClient),
|
||||
svc.RegisteredEvents(options.RegisteredEvents),
|
||||
)
|
||||
err = micro.RegisterHandler(newService.Server(), mux)
|
||||
if err != nil {
|
||||
return http.Service{}, err
|
||||
options.Logger.Fatal().Err(err).Msg("failed to register the handler")
|
||||
}
|
||||
|
||||
if err := micro.RegisterHandler(service.Server(), handle); err != nil {
|
||||
return http.Service{}, err
|
||||
}
|
||||
newService.Init()
|
||||
return newService, nil
|
||||
|
||||
return service, nil
|
||||
}
|
||||
|
||||
// Service defines the business logic implementations need to provide.
|
||||
type ActivityLogService interface {
|
||||
GetItemActivities(ctx context.Context, query, loc string, t l10n.Translator) ([]libregraph.Activity, error)
|
||||
}
|
||||
|
||||
// GetActivitiesResponse is the response on GET activities requests
|
||||
type GetActivitiesResponse struct {
|
||||
Activities []libregraph.Activity `json:"value"`
|
||||
}
|
||||
|
||||
func GetItemActivitiesHandler(log log.Logger, s ActivityLogService, vc settingssvc.ValueService, t l10n.Translator) func(w http.ResponseWriter, r *http.Request) {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, r.Header.Get(revactx.TokenHeader))
|
||||
|
||||
activeUser, ok := revactx.ContextGetUser(ctx)
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
loc := l10n.MustGetUserLocale(ctx, activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), vc)
|
||||
|
||||
activities, err := s.GetItemActivities(ctx, r.URL.Query().Get("kql"), loc, t)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, activityloghttp.ErrBadRequest):
|
||||
log.Debug().Str("query", r.URL.Query().Get("kql")).Err(err).Msg("error getting activities")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
case errors.Is(err, activityloghttp.ErrForbidden):
|
||||
log.Debug().Err(err).Msg("error getting activities")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
default:
|
||||
log.Error().Err(err).Msg("error getting activities")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
res := GetActivitiesResponse{
|
||||
Activities: activities,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; odata.metadata=minimal")
|
||||
w.Header().Set("OData-Version", "4.0")
|
||||
if reqID := chimiddleware.GetReqID(ctx); reqID != "" {
|
||||
w.Header().Set("request-id", reqID)
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
if err := json.NewEncoder(w).Encode(res); err != nil {
|
||||
log.Error().Err(err).Msg("error encoding activities")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
package activitylog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base32"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/jellydator/ttlcache/v2"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog")
|
||||
}
|
||||
|
||||
var (
|
||||
_maxActivitiesDefault = 6000
|
||||
_writeBufferDuration = 10 * time.Second
|
||||
)
|
||||
|
||||
// Activitylog stores and retrieves activities for resources and their parents from a nats kv
|
||||
type ActivityLog struct {
|
||||
log log.Logger
|
||||
// FIXME the lock does not protect agains concurrent resource activities on multiple instances
|
||||
// known since https://github.com/owncloud/ocis/pull/9361#pullrequestreview-2135350157
|
||||
// current ocis discussion in https://github.com/owncloud/ocis/issues/12475
|
||||
lock sync.RWMutex
|
||||
debouncer *Debouncer
|
||||
parentIdCache *ttlcache.Cache
|
||||
natskv nats.KeyValue
|
||||
|
||||
maxActivities int
|
||||
}
|
||||
|
||||
type batchInfo struct {
|
||||
key string
|
||||
count int
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
// New creates a new ActivitylogService
|
||||
func New(kv nats.KeyValue, opts ...Option) (*ActivityLog, error) {
|
||||
o := &Options{
|
||||
MaxActivities: _maxActivitiesDefault,
|
||||
WriteBufferDuration: _writeBufferDuration,
|
||||
Logger: log.NopLogger(),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
|
||||
cache := ttlcache.NewCache()
|
||||
err := cache.SetTTL(30 * time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &ActivityLog{
|
||||
log: o.Logger,
|
||||
lock: sync.RWMutex{},
|
||||
parentIdCache: cache,
|
||||
maxActivities: o.MaxActivities,
|
||||
natskv: kv,
|
||||
}
|
||||
s.debouncer = NewDebouncer(o.WriteBufferDuration, s.StoreActivity)
|
||||
|
||||
// run migrations
|
||||
err = s.runMigrations(context.Background(), kv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// RemoveResource removes the resource from the store
|
||||
func (a *ActivityLog) RemoveResource(rid *provider.ResourceId) error {
|
||||
if rid == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
err := a.natskv.Delete(storagespace.FormatResourceID(rid))
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not delete resource %s: %w", rid.OpaqueId, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RawActivity represents an activity as it is stored in the activitylog store
|
||||
type RawActivity struct {
|
||||
EventID string `json:"event_id"`
|
||||
Depth int `json:"depth"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
func (a *ActivityLog) AddActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time, getResource func(context.Context, *provider.Reference) (*provider.ResourceInfo, error)) error {
|
||||
var (
|
||||
err error
|
||||
depth int
|
||||
ref = initRef
|
||||
)
|
||||
ctx, span := tracer.Start(ctx, "AddActivity")
|
||||
defer span.End()
|
||||
for {
|
||||
var info *provider.ResourceInfo
|
||||
id := ref.GetResourceId()
|
||||
if ref.Path != "" {
|
||||
// Path based reference, we need to resolve the resource id
|
||||
ctx, span = tracer.Start(ctx, "AddActivity.getResource")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil {
|
||||
// TODO If the resource was deleted should we still log an activity in the parent?
|
||||
return fmt.Errorf("could not get resource info for reference %v: %w", ref, err)
|
||||
}
|
||||
id = info.GetId()
|
||||
}
|
||||
if id == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
key := storagespace.FormatResourceID(id)
|
||||
a.debouncer.Debounce(key, RawActivity{
|
||||
EventID: eventID,
|
||||
Depth: depth,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
|
||||
if id.OpaqueId == id.SpaceId {
|
||||
// we are at the root of the space, no need to go further
|
||||
break
|
||||
}
|
||||
|
||||
// check if parent id is cached
|
||||
// parent id is cached in the format <storageid>$<spaceid>!<resourceid>
|
||||
// if it is not cached, get the resource info and cache it
|
||||
if parentId == nil {
|
||||
if v, err := a.parentIdCache.Get(key); err != nil {
|
||||
if info == nil {
|
||||
ctx, span := tracer.Start(ctx, "AddActivity.getResource parent")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil || info.GetParentId() == nil || info.GetParentId().GetOpaqueId() == "" {
|
||||
return fmt.Errorf("could not get parent id: %w", err)
|
||||
}
|
||||
}
|
||||
parentId = info.GetParentId()
|
||||
a.parentIdCache.Set(key, parentId)
|
||||
} else {
|
||||
parentId = v.(*provider.ResourceId)
|
||||
}
|
||||
} else {
|
||||
a.log.Debug().Msg("parent id is cached")
|
||||
}
|
||||
|
||||
depth++
|
||||
ref = &provider.Reference{ResourceId: parentId}
|
||||
parentId = nil // reset parent id so it's not reused in the next iteration
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivityLog) StoreActivity(resourceID string, activities []RawActivity) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
ctx, span := tracer.Start(context.Background(), "storeActivity")
|
||||
defer span.End()
|
||||
|
||||
_, subspan := tracer.Start(ctx, "storeActivity.Marshal")
|
||||
b, err := msgpack.Marshal(activities)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = tracer.Start(ctx, "storeActivity.natskv.Put")
|
||||
key := natsKey(resourceID, len(activities))
|
||||
_, err = a.natskv.Put(key, b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
ctx, subspan = tracer.Start(ctx, "storeActivity.enforceMaxActivities")
|
||||
a.enforceMaxActivities(ctx, resourceID)
|
||||
subspan.End()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivityLog) enforceMaxActivities(ctx context.Context, resourceID string) {
|
||||
if a.maxActivities <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
_, subspan := tracer.Start(ctx, "enforceMaxActivities.watch")
|
||||
watcher, err := a.natskv.Watch(key, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("resourceID", resourceID).Msg("could not watch")
|
||||
return
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var keys []string
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
|
||||
}
|
||||
keys = append(keys, update.Key())
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = tracer.Start(ctx, "enforceMaxActivities.compile")
|
||||
// Parse keys into batches
|
||||
batches := make([]batchInfo, 0)
|
||||
var activitiesCount int
|
||||
for _, k := range keys {
|
||||
parts := strings.SplitN(k, ".", 3)
|
||||
if len(parts) < 3 {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, not enough parts")
|
||||
continue
|
||||
}
|
||||
|
||||
c, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse count")
|
||||
continue
|
||||
}
|
||||
|
||||
// parse timestamp
|
||||
nano, err := strconv.ParseInt(parts[2], 10, 64)
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse timestamp")
|
||||
continue
|
||||
}
|
||||
|
||||
batches = append(batches, batchInfo{
|
||||
key: k,
|
||||
count: c,
|
||||
timestamp: time.Unix(0, nano),
|
||||
})
|
||||
activitiesCount += c
|
||||
}
|
||||
|
||||
// sort batches by timestamp
|
||||
sort.Slice(batches, func(i, j int) bool {
|
||||
return batches[i].timestamp.Before(batches[j].timestamp)
|
||||
})
|
||||
subspan.End()
|
||||
|
||||
_, subspan = tracer.Start(ctx, "enforceMaxActivities.delete")
|
||||
// remove oldest keys until we are at max activities
|
||||
for _, b := range batches {
|
||||
if activitiesCount-b.count < a.maxActivities {
|
||||
break
|
||||
}
|
||||
|
||||
activitiesCount -= b.count
|
||||
err = a.natskv.Delete(b.key)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("key", b.key).Msg("could not delete key")
|
||||
break
|
||||
}
|
||||
}
|
||||
subspan.End()
|
||||
}
|
||||
|
||||
func (a *ActivityLog) InvalidateCachedParentID(purgeId *provider.ResourceId) {
|
||||
// The parent id cache is populated lazily and its entries expire, so a
|
||||
// missing key is the expected case rather than an error.
|
||||
if err := a.parentIdCache.Remove(storagespace.FormatResourceID(purgeId)); err != nil {
|
||||
a.log.Debug().Interface("event", purgeId).Err(err).Msg("could not delete parent id cache")
|
||||
}
|
||||
}
|
||||
|
||||
func natsKey(resourceID string, activitiesCount int) string {
|
||||
return fmt.Sprintf("%s.%d.%d",
|
||||
base32.StdEncoding.EncodeToString([]byte(resourceID)),
|
||||
activitiesCount,
|
||||
time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func (a *ActivityLog) Activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
a.lock.RLock()
|
||||
defer a.lock.RUnlock()
|
||||
|
||||
return a.activities(rid)
|
||||
}
|
||||
|
||||
func (a *ActivityLog) activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
resourceID := storagespace.FormatResourceID(rid)
|
||||
|
||||
glob := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
watcher, err := a.natskv.Watch(glob, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var activities []RawActivity
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack")
|
||||
}
|
||||
activities = append(activities, batchActivities...)
|
||||
}
|
||||
|
||||
return activities, nil
|
||||
}
|
||||
|
||||
// RemoveActivities removes the activities from the given resource
|
||||
func (a *ActivityLog) RemoveActivities(rid *provider.ResourceId, toDelete map[string]struct{}) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
curActivities, err := a.activities(rid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var acts []RawActivity
|
||||
for _, a := range curActivities {
|
||||
if _, ok := toDelete[a.EventID]; !ok {
|
||||
acts = append(acts, a)
|
||||
}
|
||||
}
|
||||
|
||||
b, err := json.Marshal(acts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = a.natskv.Put(storagespace.FormatResourceID(rid), b)
|
||||
return err
|
||||
}
|
||||
+3
-3
@@ -1,4 +1,4 @@
|
||||
package aggregation_test
|
||||
package activitylog_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestAggregation(t *testing.T) {
|
||||
func TestActivitylog(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Aggregation Suite")
|
||||
RunSpecs(t, "Activitylog Suite")
|
||||
}
|
||||
+34
-81
@@ -1,7 +1,6 @@
|
||||
package service
|
||||
package activitylog_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
@@ -9,17 +8,13 @@ import (
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
nserver "github.com/nats-io/nats-server/v2/server"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/command"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
eventsmocks "github.com/opencloud-eu/reva/v2/pkg/events/mocks"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/test-go/testify/mock"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -68,31 +63,25 @@ var _ = SynchronizedAfterSuite(func() {
|
||||
|
||||
var _ = Describe("ActivitylogService", func() {
|
||||
var (
|
||||
alog *ActivitylogService
|
||||
alog *activitylog.ActivityLog
|
||||
getResource func(_ context.Context, ref *provider.Reference) (*provider.ResourceInfo, error)
|
||||
writebufferduration = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
JustBeforeEach(func() {
|
||||
var err error
|
||||
stream := &eventsmocks.Stream{}
|
||||
stream.EXPECT().Consume(mock.Anything, mock.Anything).Return(nil, nil)
|
||||
alog, err = New(
|
||||
Config(&config.Config{
|
||||
Service: config.Service{
|
||||
Name: "activitylog-test",
|
||||
},
|
||||
Store: config.Store{
|
||||
Store: "nats-js-kv",
|
||||
Nodes: []string{server.Addr().String()},
|
||||
Database: "activitylog-test-" + uuid.New().String(),
|
||||
},
|
||||
MaxActivities: 4,
|
||||
WriteBufferDuration: writebufferduration,
|
||||
}),
|
||||
Stream(stream),
|
||||
TraceProvider(noop.NewTracerProvider()),
|
||||
Mux(chi.NewMux()),
|
||||
db := "activitylog-test-" + uuid.New().String()
|
||||
|
||||
kv, err := command.ConnectNatsKV(config.Store{
|
||||
Nodes: []string{server.Addr().String()},
|
||||
Database: db,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
alog, err = activitylog.New(
|
||||
kv,
|
||||
activitylog.MaxActivities(4),
|
||||
activitylog.WriteBufferDuration(writebufferduration),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
@@ -107,7 +96,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
Name string
|
||||
Tree map[string]*provider.ResourceInfo
|
||||
Activities map[string]string
|
||||
Expected map[string][]RawActivity
|
||||
Expected map[string][]activitylog.RawActivity
|
||||
}
|
||||
|
||||
testCases := []testCase{
|
||||
@@ -121,7 +110,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
Activities: map[string]string{
|
||||
"activity": "base",
|
||||
},
|
||||
Expected: map[string][]RawActivity{
|
||||
Expected: map[string][]activitylog.RawActivity{
|
||||
"base": activitites("activity", 0),
|
||||
"parent": activitites("activity", 1),
|
||||
"spaceid": activitites("activity", 2),
|
||||
@@ -138,7 +127,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
"activity1": "base",
|
||||
"activity2": "base",
|
||||
},
|
||||
Expected: map[string][]RawActivity{
|
||||
Expected: map[string][]activitylog.RawActivity{
|
||||
"base": activitites("activity1", 0, "activity2", 0),
|
||||
"parent": activitites("activity1", 1, "activity2", 1),
|
||||
"spaceid": activitites("activity1", 2, "activity2", 2),
|
||||
@@ -155,7 +144,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
}
|
||||
|
||||
for k, v := range tc.Activities {
|
||||
err := alog.addActivity(context.Background(), reference(v), nil, k, time.Time{}, getResource)
|
||||
err := alog.AddActivity(context.Background(), reference(v), nil, k, time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
})
|
||||
@@ -194,9 +183,9 @@ var _ = Describe("ActivitylogService", func() {
|
||||
|
||||
It("debounces activities", func() {
|
||||
|
||||
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -207,7 +196,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
})
|
||||
|
||||
It("adheres to the MaxActivities setting", func() {
|
||||
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Eventually(func(g Gomega) {
|
||||
activities, err := alog.Activities(resourceID("base"))
|
||||
@@ -215,7 +204,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
g.Expect(len(activities)).To(Equal(1))
|
||||
}).Should(Succeed())
|
||||
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Eventually(func(g Gomega) {
|
||||
activities, err := alog.Activities(resourceID("base"))
|
||||
@@ -223,11 +212,11 @@ var _ = Describe("ActivitylogService", func() {
|
||||
g.Expect(len(activities)).To(Equal(2))
|
||||
}).Should(Succeed())
|
||||
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity5", time.Time{}, getResource)
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity5", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -244,9 +233,9 @@ var _ = Describe("ActivitylogService", func() {
|
||||
return tree[ref.GetResourceId().GetOpaqueId()], nil
|
||||
}
|
||||
|
||||
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -255,9 +244,9 @@ var _ = Describe("ActivitylogService", func() {
|
||||
g.Expect(activities).To(ConsistOf(activitites("activity1", 0, "activity2", 0)))
|
||||
}).Should(Succeed())
|
||||
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -268,47 +257,11 @@ var _ = Describe("ActivitylogService", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("removeCachedParentID", func() {
|
||||
var logBuffer *bytes.Buffer
|
||||
|
||||
newLoggerAtLevel := func(level string) log.Logger {
|
||||
logBuffer = &bytes.Buffer{}
|
||||
return log.Logger{Logger: log.NewLogger(log.Level(level)).Output(logBuffer)}
|
||||
}
|
||||
|
||||
It("does not log an error when the entry was never cached", func() {
|
||||
alog.log = newLoggerAtLevel("error")
|
||||
|
||||
alog.removeCachedParentID(reference("never-cached"))
|
||||
|
||||
Expect(logBuffer.String()).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("logs a missing entry at debug level", func() {
|
||||
alog.log = newLoggerAtLevel("debug")
|
||||
|
||||
alog.removeCachedParentID(reference("never-cached"))
|
||||
|
||||
Expect(logBuffer.String()).To(ContainSubstring("could not delete parent id cache"))
|
||||
Expect(logBuffer.String()).To(ContainSubstring(`"level":"debug"`))
|
||||
})
|
||||
|
||||
It("does not log at all when the entry was cached", func() {
|
||||
alog.log = newLoggerAtLevel("debug")
|
||||
ref := reference("cached")
|
||||
Expect(alog.parentIdCache.Set(storagespace.FormatResourceID(ref.GetResourceId()), resourceID("parent"))).To(Succeed())
|
||||
|
||||
alog.removeCachedParentID(ref)
|
||||
|
||||
Expect(logBuffer.String()).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func activitites(acts ...any) []RawActivity {
|
||||
var activities []RawActivity
|
||||
act := RawActivity{}
|
||||
func activitites(acts ...any) []activitylog.RawActivity {
|
||||
var activities []activitylog.RawActivity
|
||||
act := activitylog.RawActivity{}
|
||||
for _, a := range acts {
|
||||
switch v := a.(type) {
|
||||
case string:
|
||||
@@ -0,0 +1,76 @@
|
||||
package activitylog
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Debouncer is used to debounce writes to the activity log store.
|
||||
type Debouncer struct {
|
||||
after time.Duration
|
||||
f func(id string, ra []RawActivity) error
|
||||
pending sync.Map
|
||||
inProgress sync.Map
|
||||
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
type queueItem struct {
|
||||
activities []RawActivity
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
// NewDebouncer returns a new Debouncer instance.
|
||||
func NewDebouncer(d time.Duration, f func(id string, ra []RawActivity) error) *Debouncer {
|
||||
return &Debouncer{
|
||||
after: d,
|
||||
f: f,
|
||||
pending: sync.Map{},
|
||||
inProgress: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
// Debounce restarts the debounce timer for the given space.
|
||||
func (d *Debouncer) Debounce(id string, ra RawActivity) {
|
||||
if d.after == 0 {
|
||||
d.f(id, []RawActivity{ra})
|
||||
return
|
||||
}
|
||||
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
|
||||
item := &queueItem{
|
||||
activities: []RawActivity{ra},
|
||||
}
|
||||
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
// if the item is already in the queue, append the new activities
|
||||
item, ok = i.(*queueItem)
|
||||
if ok {
|
||||
item.activities = append(item.activities, ra)
|
||||
}
|
||||
}
|
||||
|
||||
if item.timer == nil {
|
||||
item.timer = time.AfterFunc(d.after, func() {
|
||||
if _, ok := d.inProgress.Load(id); ok {
|
||||
// Reschedule this run for when the previous run has finished
|
||||
d.mutex.Lock()
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
i.(*queueItem).timer.Reset(d.after)
|
||||
}
|
||||
|
||||
d.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
d.pending.Delete(id)
|
||||
d.inProgress.Store(id, true)
|
||||
defer d.inProgress.Delete(id)
|
||||
d.f(id, item.activities)
|
||||
})
|
||||
}
|
||||
|
||||
d.pending.Store(id, item)
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package activitylog_test
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
)
|
||||
|
||||
var _ = Describe("Debouncer", func() {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
callbacks []activitylog.RawActivity
|
||||
newCallback func(id string, ra []activitylog.RawActivity) error
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
mu.Lock()
|
||||
callbacks = nil
|
||||
mu.Unlock()
|
||||
newCallback = func(id string, ra []activitylog.RawActivity) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
callbacks = append(callbacks, ra...)
|
||||
return nil
|
||||
}
|
||||
})
|
||||
|
||||
Context("with zero duration", func() {
|
||||
It("calls the callback immediately", func() {
|
||||
d := activitylog.NewDebouncer(0, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
Expect(callbacks).To(HaveLen(1))
|
||||
Expect(callbacks[0].EventID).To(Equal("activity1"))
|
||||
})
|
||||
|
||||
It("calls the callback immediately for each event", func() {
|
||||
d := activitylog.NewDebouncer(0, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
d.Debounce("space2", activitylog.RawActivity{EventID: "activity2"})
|
||||
Expect(callbacks).To(HaveLen(2))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with non-zero duration", func() {
|
||||
It("batches activities with the same id", func() {
|
||||
d := activitylog.NewDebouncer(10*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity3"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(3))
|
||||
})
|
||||
|
||||
It("handles different ids independently", func() {
|
||||
d := activitylog.NewDebouncer(10*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
d.Debounce("space2", activitylog.RawActivity{EventID: "activity2"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(2))
|
||||
})
|
||||
|
||||
It("batches activities that arrive within the debounce window", func() {
|
||||
d := activitylog.NewDebouncer(100*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity3"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(3))
|
||||
})
|
||||
|
||||
It("processes new batch after previous completes", func() {
|
||||
d := activitylog.NewDebouncer(5*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
time.Sleep(20 * time.Millisecond) // let first batch complete
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(2))
|
||||
})
|
||||
|
||||
It("skips duplicate write when timer fires during in-progress callback", func() {
|
||||
slowCallback := func(id string, ra []activitylog.RawActivity) error {
|
||||
time.Sleep(50 * time.Millisecond) // simulate slow write
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
callbacks = append(callbacks, ra...)
|
||||
return nil
|
||||
}
|
||||
|
||||
d := activitylog.NewDebouncer(10*time.Millisecond, slowCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
time.Sleep(20 * time.Millisecond) // timer fires while callback is running
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(1))
|
||||
})
|
||||
})
|
||||
})
|
||||
+3
-3
@@ -1,4 +1,4 @@
|
||||
package service
|
||||
package activitylog
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -15,7 +15,7 @@ const currentMigrationVersion = "1"
|
||||
|
||||
// RunMigrations checks the activitylog data version and runs migrations if necessary.
|
||||
// It should be called during service startup, after the NATS KeyValue store is initialized.
|
||||
func (a *ActivitylogService) runMigrations(ctx context.Context, kv nats.KeyValue) error {
|
||||
func (a *ActivityLog) runMigrations(ctx context.Context, kv nats.KeyValue) error {
|
||||
entry, err := kv.Get(activitylogVersionKey)
|
||||
if err == nats.ErrKeyNotFound {
|
||||
a.log.Info().Msg("activitylog version key not found. Running migration to V1...")
|
||||
@@ -40,7 +40,7 @@ func (a *ActivitylogService) runMigrations(ctx context.Context, kv nats.KeyValue
|
||||
// For each such key, it creates a new key in the format "originalKey.count.timestamp"
|
||||
// and stores the original list of strings (re-marshalled to messagepack) as its value.
|
||||
// Finally, it sets the activitylog.version key to "1".
|
||||
func (a *ActivitylogService) migrateToV1(_ context.Context, kv nats.KeyValue) error {
|
||||
func (a *ActivityLog) migrateToV1(_ context.Context, kv nats.KeyValue) error {
|
||||
lister, err := kv.ListKeys()
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrateToV1: failed to list keys from NATS KV store: %w", err)
|
||||
@@ -0,0 +1,35 @@
|
||||
package activitylog
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
)
|
||||
|
||||
// Option for the activitylog service
|
||||
type Option func(*Options)
|
||||
|
||||
// Options for the activitylog service
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
MaxActivities int
|
||||
WriteBufferDuration time.Duration
|
||||
}
|
||||
|
||||
// Logger configures a logger for the activitylog service
|
||||
func Logger(log log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = log
|
||||
}
|
||||
}
|
||||
|
||||
func MaxActivities(max int) Option {
|
||||
return func(o *Options) {
|
||||
o.MaxActivities = max
|
||||
}
|
||||
}
|
||||
func WriteBufferDuration(d time.Duration) Option {
|
||||
return func(o *Options) {
|
||||
o.WriteBufferDuration = d
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -1,4 +1,4 @@
|
||||
package service_test
|
||||
package events_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestService(t *testing.T) {
|
||||
func TestEvents(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Service Suite")
|
||||
RunSpecs(t, "Events Suite")
|
||||
}
|
||||
+16
-38
@@ -1,17 +1,14 @@
|
||||
package service
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// Option for the activitylog service
|
||||
@@ -19,17 +16,20 @@ type Option func(*Options)
|
||||
|
||||
// Options for the activitylog service
|
||||
type Options struct {
|
||||
Context context.Context
|
||||
Logger log.Logger
|
||||
Config *config.Config
|
||||
TraceProvider trace.TracerProvider
|
||||
ServiceAccount config.ServiceAccount
|
||||
Stream events.Stream
|
||||
RegisteredEvents []events.Unmarshaller
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
Mux *chi.Mux
|
||||
HistoryClient ehsvc.EventHistoryService
|
||||
ValueClient settingssvc.ValueService
|
||||
WriteBufferDuration time.Duration
|
||||
MaxActivities int
|
||||
NumConsumers int
|
||||
}
|
||||
|
||||
func Context(ctx context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = ctx
|
||||
}
|
||||
}
|
||||
|
||||
// Logger configures a logger for the activitylog service
|
||||
@@ -39,17 +39,10 @@ func Logger(log log.Logger) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// Config adds the config for the activitylog service
|
||||
func Config(c *config.Config) Option {
|
||||
// ServiceAccount configures a service account for the activitylog service
|
||||
func ServiceAccount(sa config.ServiceAccount) Option {
|
||||
return func(o *Options) {
|
||||
o.Config = c
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider adds a tracer provider for the activitylog service
|
||||
func TraceProvider(tp trace.TracerProvider) Option {
|
||||
return func(o *Options) {
|
||||
o.TraceProvider = tp
|
||||
o.ServiceAccount = sa
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,23 +67,8 @@ func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient])
|
||||
}
|
||||
}
|
||||
|
||||
// Mux defines the muxer for the service
|
||||
func Mux(m *chi.Mux) Option {
|
||||
func NumConsumers(num int) Option {
|
||||
return func(o *Options) {
|
||||
o.Mux = m
|
||||
}
|
||||
}
|
||||
|
||||
// HistoryClient adds a grpc client for the eventhistory service
|
||||
func HistoryClient(hc ehsvc.EventHistoryService) Option {
|
||||
return func(o *Options) {
|
||||
o.HistoryClient = hc
|
||||
}
|
||||
}
|
||||
|
||||
// ValueClient adds a grpc client for the value service
|
||||
func ValueClient(vs settingssvc.ValueService) Option {
|
||||
return func(o *Options) {
|
||||
o.ValueClient = vs
|
||||
o.NumConsumers = num
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events")
|
||||
}
|
||||
|
||||
var (
|
||||
_numConsumersDefault = 1
|
||||
)
|
||||
|
||||
// ActivitylogService logs events per resource
|
||||
type ActivitylogService struct {
|
||||
ctx context.Context
|
||||
sa config.ServiceAccount
|
||||
log log.Logger
|
||||
stream events.Stream
|
||||
gws pool.Selectable[gateway.GatewayAPIClient]
|
||||
al *activitylog.ActivityLog
|
||||
|
||||
numConsumers int
|
||||
|
||||
events []events.Unmarshaller
|
||||
|
||||
stopCh chan struct{}
|
||||
stopped *atomic.Bool
|
||||
}
|
||||
|
||||
// New creates a new ActivitylogService
|
||||
func New(al *activitylog.ActivityLog, stream events.Stream, opts ...Option) (*ActivitylogService, error) {
|
||||
o := &Options{
|
||||
NumConsumers: _numConsumersDefault,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
|
||||
s := &ActivitylogService{
|
||||
ctx: o.Context,
|
||||
log: o.Logger,
|
||||
sa: o.ServiceAccount,
|
||||
stream: stream,
|
||||
gws: o.GatewaySelector,
|
||||
events: o.RegisteredEvents,
|
||||
numConsumers: o.NumConsumers,
|
||||
al: al,
|
||||
stopCh: make(chan struct{}, 1),
|
||||
stopped: new(atomic.Bool),
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Run to fulfil Runner interface
|
||||
func (s *ActivitylogService) Run() error {
|
||||
ch, err := events.Consume(s.stream, "activitylog", s.events...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
ctx, cancel := context.WithCancel(s.ctx)
|
||||
defer cancel()
|
||||
|
||||
s.log.Debug().Int("worker.count", s.numConsumers).
|
||||
Str("messaging.consumer.group.name", "activitylog").
|
||||
Str("messaging.system", "nats").
|
||||
Str("messaging.operation.name", "receive").
|
||||
Msg("starting event processing workers")
|
||||
|
||||
// start workers
|
||||
for i := 0; i < s.numConsumers; i++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := s.processEvent(e); err != nil {
|
||||
s.log.Error().Err(err).
|
||||
Int("worker", workerID).
|
||||
Interface("event", e).
|
||||
Msg("failed to process event")
|
||||
}
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// wait for stop signal
|
||||
<-s.stopCh
|
||||
cancel() // signal workers to stop
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close will make the service to stop processing, so the `Run`
|
||||
// method can finish.
|
||||
func (s *ActivitylogService) Close() {
|
||||
if s.stopped.CompareAndSwap(false, true) {
|
||||
close(s.stopCh)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ActivitylogService) processEvent(e events.Event) error {
|
||||
ctx := e.GetTraceContext(s.ctx)
|
||||
ctx, span := tracer.Start(ctx, "processEvent")
|
||||
defer span.End()
|
||||
|
||||
s.log.Debug().Interface("event", e).Msg("updating activitylog")
|
||||
|
||||
switch ev := e.Event.(type) {
|
||||
case events.UploadReady:
|
||||
return s.AddActivity(ctx, ev.FileRef, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.FileTouched:
|
||||
return s.AddActivity(ctx, ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
// Disabled https://github.com/owncloud/ocis/issues/10293
|
||||
//case events.FileDownloaded:
|
||||
// we are only interested in public link downloads - so no need to store others.
|
||||
//if ev.ImpersonatingUser.GetDisplayName() == "Public" {
|
||||
// err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
//}
|
||||
case events.ContainerCreated:
|
||||
return s.AddActivity(ctx, ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemTrashed:
|
||||
return s.AddActivityTrashed(ctx, ev.ID, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemPurged:
|
||||
return s.al.RemoveResource(ev.ID)
|
||||
case events.ItemMoved:
|
||||
// remove the cached parent id for this resource
|
||||
s.removeCachedParentID(ctx, ev.Ref)
|
||||
|
||||
return s.AddActivity(ctx, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ShareCreated:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.ShareUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.ShareRemoved:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, ev.Timestamp)
|
||||
case events.LinkCreated:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.LinkUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.LinkRemoved:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.SpaceShared:
|
||||
return s.AddSpaceActivity(ctx, ev.ID, e.ID, ev.Timestamp)
|
||||
case events.SpaceUnshared:
|
||||
return s.AddSpaceActivity(ctx, ev.ID, e.ID, ev.Timestamp)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddActivity adds the activity to the given resource and all its parents
|
||||
func (a *ActivitylogService) AddActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
ctx, span := tracer.Start(ctx, "AddActivity")
|
||||
defer span.End()
|
||||
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
|
||||
}
|
||||
return a.al.AddActivity(ctx, initRef, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
|
||||
func (a *ActivitylogService) AddActivityTrashed(ctx context.Context, resourceID *provider.ResourceId, reference *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
ctx, span := tracer.Start(ctx, "AddActivityTrashed")
|
||||
defer span.End()
|
||||
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
}
|
||||
|
||||
// store activity on trashed item
|
||||
if err := a.al.StoreActivity(storagespace.FormatResourceID(resourceID), []activitylog.RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("could not store activity: %w", err)
|
||||
}
|
||||
|
||||
// get previous parent
|
||||
ref := &provider.Reference{
|
||||
ResourceId: reference.GetResourceId(),
|
||||
Path: filepath.Dir(reference.GetPath()),
|
||||
}
|
||||
|
||||
return a.al.AddActivity(ctx, ref, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddSpaceActivity adds the activity to the given spaceroot
|
||||
func (a *ActivitylogService) AddSpaceActivity(ctx context.Context, spaceID *provider.StorageSpaceId, eventID string, timestamp time.Time) error {
|
||||
_, span := tracer.Start(ctx, "AddSpaceActivity")
|
||||
defer span.End()
|
||||
// spaceID is in format <providerid>$<spaceid>
|
||||
// activitylog service uses format <providerid>$<spaceid>!<resourceid>
|
||||
// lets do some converting, shall we?
|
||||
rid, err := storagespace.ParseID(spaceID.GetOpaqueId())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse space id: %w", err)
|
||||
}
|
||||
rid.OpaqueId = rid.GetSpaceId()
|
||||
err = a.al.StoreActivity(storagespace.FormatResourceID(&rid), []activitylog.RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not store activity: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func toRef(r *provider.ResourceId) *provider.Reference {
|
||||
return &provider.Reference{
|
||||
ResourceId: r,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) removeCachedParentID(ctx context.Context, ref *provider.Reference) {
|
||||
var span trace.Span
|
||||
ctx, span = tracer.Start(ctx, "removeCachedParentID")
|
||||
defer span.End()
|
||||
|
||||
purgeId := ref.GetResourceId()
|
||||
if ref.GetPath() != "" {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get gateway client")
|
||||
return
|
||||
}
|
||||
|
||||
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get service user context")
|
||||
return
|
||||
}
|
||||
|
||||
info, err := utils.GetResource(ctx, ref, gwc)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get resource info")
|
||||
return
|
||||
}
|
||||
purgeId = info.GetId()
|
||||
}
|
||||
a.al.InvalidateCachedParentID(purgeId)
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package events_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
eventssvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
var _ = Describe("ActivitylogService", func() {
|
||||
Describe("New", func() {
|
||||
var (
|
||||
al *activitylog.ActivityLog
|
||||
stream events.Stream
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
al = &activitylog.ActivityLog{}
|
||||
stream = nil
|
||||
})
|
||||
|
||||
It("creates a service with minimal options", func() {
|
||||
svc, err := eventssvc.New(al, stream)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with context option", func() {
|
||||
ctx := context.Background()
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.Context(ctx))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with logger option", func() {
|
||||
logger := log.NopLogger()
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.Logger(logger))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with service account option", func() {
|
||||
sa := config.ServiceAccount{
|
||||
ServiceAccountID: "sa-id",
|
||||
ServiceAccountSecret: "sa-secret",
|
||||
}
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.ServiceAccount(sa))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with registered events", func() {
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}, &events.FileTouched{}}
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.RegisteredEvents(evts))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with gateway selector", func() {
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.GatewaySelector(gs))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with num consumers option", func() {
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.NumConsumers(5))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with all options", func() {
|
||||
ctx := context.Background()
|
||||
logger := log.NopLogger()
|
||||
sa := config.ServiceAccount{
|
||||
ServiceAccountID: "sa-id",
|
||||
ServiceAccountSecret: "sa-secret",
|
||||
}
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}, &events.ContainerCreated{}}
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
|
||||
svc, err := eventssvc.New(
|
||||
al,
|
||||
stream,
|
||||
eventssvc.Context(ctx),
|
||||
eventssvc.Logger(logger),
|
||||
eventssvc.ServiceAccount(sa),
|
||||
eventssvc.RegisteredEvents(evts),
|
||||
eventssvc.GatewaySelector(gs),
|
||||
eventssvc.NumConsumers(3),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Close", func() {
|
||||
It("can be called without panic on a new service", func() {
|
||||
al := &activitylog.ActivityLog{}
|
||||
svc, err := eventssvc.New(al, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(func() { svc.Close() }).ToNot(Panic())
|
||||
})
|
||||
|
||||
It("can be called multiple times without panic", func() {
|
||||
al := &activitylog.ActivityLog{}
|
||||
svc, err := eventssvc.New(al, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
svc.Close()
|
||||
svc.Close()
|
||||
svc.Close()
|
||||
})
|
||||
})
|
||||
})
|
||||
+3
-3
@@ -1,4 +1,4 @@
|
||||
package aggs_test
|
||||
package http_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestAggs(t *testing.T) {
|
||||
func TestHTTP(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Aggs Suite")
|
||||
RunSpecs(t, "HTTP Suite")
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
RegisteredEvents []events.Unmarshaller
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
HistoryClient ehsvc.EventHistoryService
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// RegisteredEvents registers the events the service should listen to
|
||||
func RegisteredEvents(e []events.Unmarshaller) Option {
|
||||
return func(o *Options) {
|
||||
o.RegisteredEvents = e
|
||||
}
|
||||
}
|
||||
|
||||
// GatewaySelector adds a grpc client selector for the gateway service
|
||||
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
|
||||
return func(o *Options) {
|
||||
o.GatewaySelector = gatewaySelector
|
||||
}
|
||||
}
|
||||
|
||||
// HistoryClient adds a grpc client for the eventhistory service
|
||||
func HistoryClient(hc ehsvc.EventHistoryService) Option {
|
||||
return func(o *Options) {
|
||||
o.HistoryClient = hc
|
||||
}
|
||||
}
|
||||
+8
-7
@@ -1,4 +1,4 @@
|
||||
package service
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -44,11 +44,6 @@ var (
|
||||
StrDescription = l10n.Template("description")
|
||||
)
|
||||
|
||||
// GetActivitiesResponse is the response on GET activities requests
|
||||
type GetActivitiesResponse struct {
|
||||
Activities []libregraph.Activity `json:"value"`
|
||||
}
|
||||
|
||||
// Resource represents an item such as a file or folder
|
||||
type Resource struct {
|
||||
ID string `json:"id"`
|
||||
@@ -311,7 +306,7 @@ func NewActivity(message string, ts time.Time, eventID string, vars map[string]a
|
||||
}
|
||||
|
||||
// GetVars calls other service to gather the required data for the activity variables
|
||||
func (s *ActivitylogService) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]any, error) {
|
||||
func (s *svc) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]any, error) {
|
||||
gwc, err := s.gws.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -327,6 +322,12 @@ func (s *ActivitylogService) GetVars(ctx context.Context, opts ...ActivityOption
|
||||
return vars, nil
|
||||
}
|
||||
|
||||
func toSpace(r *provider.Reference) *provider.StorageSpaceId {
|
||||
return &provider.StorageSpaceId{
|
||||
OpaqueId: storagespace.FormatStorageID(r.GetResourceId().GetStorageId(), r.GetResourceId().GetSpaceId()),
|
||||
}
|
||||
}
|
||||
|
||||
func getFolderName(ctx context.Context, gwc gateway.GatewayAPIClient, ref *provider.Reference) string {
|
||||
n := filepath.Base(filepath.Dir(ref.GetPath()))
|
||||
if n == "." || n == "/" {
|
||||
@@ -0,0 +1,150 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
httpsvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
)
|
||||
|
||||
var _ = Describe("Response", func() {
|
||||
Describe("NewActivity", func() {
|
||||
It("creates an activity with the given parameters", func() {
|
||||
ts := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
|
||||
vars := map[string]any{
|
||||
"user": "testuser",
|
||||
"resource": "testfile.txt",
|
||||
}
|
||||
|
||||
act := httpsvc.NewActivity("Test message", ts, "event-123", vars)
|
||||
|
||||
Expect(act.Id).To(Equal("event-123"))
|
||||
Expect(act.Times.RecordedTime).To(Equal(ts))
|
||||
Expect(act.Template.Message).To(Equal("Test message"))
|
||||
Expect(act.Template.Variables).To(HaveKeyWithValue("user", "testuser"))
|
||||
Expect(act.Template.Variables).To(HaveKeyWithValue("resource", "testfile.txt"))
|
||||
})
|
||||
|
||||
It("handles empty variables map", func() {
|
||||
act := httpsvc.NewActivity("", time.Time{}, "", map[string]any{})
|
||||
|
||||
Expect(act.Id).To(BeEmpty())
|
||||
Expect(act.Times.RecordedTime).To(Equal(time.Time{}))
|
||||
Expect(act.Template.Message).To(BeEmpty())
|
||||
Expect(act.Template.Variables).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithOldResource", func() {
|
||||
It("sets the oldResource variable from reference path", func() {
|
||||
ref := &provider.Reference{
|
||||
Path: "/old/path/oldname.txt",
|
||||
}
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithOldResource(ref)
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("oldResource"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithUser", func() {
|
||||
It("returns error when no user is provided", func() {
|
||||
opt := httpsvc.WithUser(nil, nil, nil)
|
||||
err := opt(context.Background(), nil, make(map[string]any))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("no user provided"))
|
||||
})
|
||||
|
||||
It("uses impersonator when provided", func() {
|
||||
impersonator := &user.User{
|
||||
Id: &user.UserId{
|
||||
OpaqueId: "imp-user-id",
|
||||
},
|
||||
DisplayName: "Impersonated User",
|
||||
}
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithUser(nil, nil, impersonator)
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("user"))
|
||||
})
|
||||
|
||||
It("uses executing user when no impersonator", func() {
|
||||
execUser := &user.User{
|
||||
Id: &user.UserId{
|
||||
OpaqueId: "exec-user-id",
|
||||
},
|
||||
DisplayName: "Executing User",
|
||||
}
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithUser(nil, execUser, nil)
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("user"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithVar", func() {
|
||||
It("sets a simple key-value variable", func() {
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithVar("token", "id123", "My Token")
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("token"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithTranslation", func() {
|
||||
It("sets translated field variable", func() {
|
||||
var t l10n.Translator
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithTranslation(&t, "en", "field", []string{"permission"})
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("field"))
|
||||
})
|
||||
|
||||
It("handles empty values slice", func() {
|
||||
var t l10n.Translator
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithTranslation(&t, "en", "field", []string{})
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("field"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("ActivityOption type", func() {
|
||||
It("allows composing multiple options", func() {
|
||||
vars := make(map[string]any)
|
||||
ctx := context.Background()
|
||||
var gwc gateway.GatewayAPIClient
|
||||
|
||||
options := []httpsvc.ActivityOption{
|
||||
httpsvc.WithVar("key1", "id1", "name1"),
|
||||
httpsvc.WithVar("key2", "id2", "name2"),
|
||||
}
|
||||
|
||||
for _, opt := range options {
|
||||
err := opt(ctx, gwc, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
|
||||
Expect(vars).To(HaveKey("key1"))
|
||||
Expect(vars).To(HaveKey("key2"))
|
||||
})
|
||||
})
|
||||
})
|
||||
+75
-76
@@ -1,89 +1,97 @@
|
||||
package service
|
||||
package http
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"context"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"google.golang.org/grpc/metadata"
|
||||
|
||||
"github.com/olekukonko/errors"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/eventhistory/v0"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var (
|
||||
//go:embed l10n/locale
|
||||
_localeFS embed.FS
|
||||
var tracer trace.Tracer
|
||||
|
||||
// subfolder where the translation files are stored
|
||||
_localeSubPath = "l10n/locale"
|
||||
|
||||
// domain of the activitylog service (transifex)
|
||||
_domain = "activitylog"
|
||||
)
|
||||
|
||||
// ServeHTTP implements the http.Handler interface.
|
||||
func (s *ActivitylogService) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.mux.ServeHTTP(w, r)
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http")
|
||||
}
|
||||
|
||||
// HandleGetItemActivities handles the request to get the activities of an item.
|
||||
func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, r.Header.Get(revactx.TokenHeader))
|
||||
// New returns a new instance of Service
|
||||
func New(al *activitylog.ActivityLog, opts ...Option) (*svc, error) {
|
||||
o := newOptions(opts...)
|
||||
|
||||
activeUser, ok := revactx.ContextGetUser(ctx)
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
registeredEvents := make(map[string]events.Unmarshaller)
|
||||
for _, e := range o.RegisteredEvents {
|
||||
typ := reflect.TypeOf(e)
|
||||
registeredEvents[typ.String()] = e
|
||||
}
|
||||
|
||||
return &svc{
|
||||
log: o.Logger,
|
||||
evHistory: o.HistoryClient,
|
||||
al: al,
|
||||
registeredEvents: registeredEvents,
|
||||
gws: o.GatewaySelector,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type svc struct {
|
||||
log log.Logger
|
||||
evHistory ehsvc.EventHistoryService
|
||||
gws pool.Selectable[gateway.GatewayAPIClient]
|
||||
al *activitylog.ActivityLog
|
||||
registeredEvents map[string]events.Unmarshaller
|
||||
}
|
||||
|
||||
var (
|
||||
ErrBadRequest = errors.New("bad request")
|
||||
ErrForbidden = errors.New("forbidden")
|
||||
)
|
||||
|
||||
func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.Translator) ([]libregraph.Activity, error) {
|
||||
gwc, err := s.gws.Next()
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rid, limit, rawActivityAccepted, activityAccepted, sort, err := s.getFilters(r.URL.Query().Get("kql"))
|
||||
rid, limit, rawActivityAccepted, activityAccepted, sort, err := s.getFilters(query)
|
||||
if err != nil {
|
||||
s.log.Info().Str("query", r.URL.Query().Get("kql")).Err(err).Msg("error getting filters")
|
||||
_, _ = w.Write([]byte(err.Error()))
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
s.log.Info().Str("query", query).Err(err).Msg("error getting filters")
|
||||
return nil, ErrBadRequest
|
||||
}
|
||||
|
||||
info, err := utils.GetResourceByID(ctx, rid, gwc)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
return nil, ErrForbidden
|
||||
}
|
||||
|
||||
// you need ListGrants to see activities
|
||||
if !info.GetPermissionSet().GetListGrants() {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
return nil, ErrForbidden
|
||||
}
|
||||
|
||||
raw, err := s.Activities(rid)
|
||||
raw, err := s.al.Activities(rid)
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error getting activities")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(raw))
|
||||
@@ -96,21 +104,21 @@ func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *h
|
||||
toDelete[a.EventID] = struct{}{}
|
||||
}
|
||||
|
||||
evRes, err := s.evHistory.GetEvents(r.Context(), &ehsvc.GetEventsRequest{Ids: ids})
|
||||
evRes, err := s.evHistory.GetEvents(ctx, &ehsvc.GetEventsRequest{Ids: ids})
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error getting events")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
return nil, err
|
||||
}
|
||||
|
||||
evs := evRes.GetEvents()
|
||||
sort(evs)
|
||||
|
||||
resp := GetActivitiesResponse{Activities: make([]libregraph.Activity, 0, len(evRes.GetEvents()))}
|
||||
// TODO cut the interface here?
|
||||
activities := make([]libregraph.Activity, 0, len(evRes.GetEvents()))
|
||||
for _, e := range evs {
|
||||
delete(toDelete, e.GetId())
|
||||
|
||||
if limit > 0 && limit <= len(resp.Activities) {
|
||||
if limit > 0 && limit <= len(activities) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -124,9 +132,6 @@ func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *h
|
||||
vars map[string]any
|
||||
)
|
||||
|
||||
loc := l10n.MustGetUserLocale(r.Context(), activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), s.valService)
|
||||
t := l10n.NewTranslatorFromCommonConfig(s.cfg.DefaultLanguage, _domain, s.cfg.TranslationPath, _localeFS, _localeSubPath)
|
||||
|
||||
switch ev := s.unwrapEvent(e).(type) {
|
||||
case nil:
|
||||
// error already logged in unwrapEvent
|
||||
@@ -224,35 +229,29 @@ func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *h
|
||||
continue
|
||||
}
|
||||
|
||||
resp.Activities = append(resp.Activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars))
|
||||
activities = append(activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars))
|
||||
}
|
||||
|
||||
// delete activities in separate go routine
|
||||
if len(toDelete) > 0 {
|
||||
go func() {
|
||||
err := s.RemoveActivities(rid, toDelete)
|
||||
err := s.al.RemoveActivities(rid, toDelete)
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error removing activities")
|
||||
}
|
||||
}()
|
||||
}
|
||||
return activities, nil
|
||||
|
||||
b, err := json.Marshal(resp)
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error marshalling activities")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := w.Write(b); err != nil {
|
||||
s.log.Error().Err(err).Msg("error writing response")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (s *ActivitylogService) unwrapEvent(e *ehmsg.Event) any {
|
||||
func toRef(r *provider.ResourceId) *provider.Reference {
|
||||
return &provider.Reference{
|
||||
ResourceId: r,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *svc) unwrapEvent(e *ehmsg.Event) any {
|
||||
etype, ok := s.registeredEvents[e.GetType()]
|
||||
if !ok {
|
||||
s.log.Error().Str("eventid", e.GetId()).Str("eventtype", e.GetType()).Msg("event not registered")
|
||||
@@ -268,13 +267,13 @@ func (s *ActivitylogService) unwrapEvent(e *ehmsg.Event) any {
|
||||
return einterface
|
||||
}
|
||||
|
||||
func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int, func(RawActivity) bool, func(*ehmsg.Event) bool, func([]*ehmsg.Event), error) {
|
||||
func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activitylog.RawActivity) bool, func(*ehmsg.Event) bool, func([]*ehmsg.Event), error) {
|
||||
qast, err := kql.Builder{}.Build(query)
|
||||
if err != nil {
|
||||
return nil, 0, nil, nil, nil, err
|
||||
}
|
||||
|
||||
prefilters := make([]func(RawActivity) bool, 0)
|
||||
prefilters := make([]func(activitylog.RawActivity) bool, 0)
|
||||
postfilters := make([]func(*ehmsg.Event) bool, 0)
|
||||
|
||||
sortby := func(_ []*ehmsg.Event) {}
|
||||
@@ -299,7 +298,7 @@ func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int
|
||||
break
|
||||
}
|
||||
|
||||
prefilters = append(prefilters, func(a RawActivity) bool {
|
||||
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
|
||||
return a.Depth <= depth
|
||||
})
|
||||
case "limit":
|
||||
@@ -322,11 +321,11 @@ func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int
|
||||
case *ast.DateTimeNode:
|
||||
switch v.Operator.Value {
|
||||
case "<", "<=":
|
||||
prefilters = append(prefilters, func(a RawActivity) bool {
|
||||
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
|
||||
return a.Timestamp.Before(v.Value)
|
||||
})
|
||||
case ">", ">=":
|
||||
prefilters = append(prefilters, func(a RawActivity) bool {
|
||||
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
|
||||
return a.Timestamp.After(v.Value)
|
||||
})
|
||||
}
|
||||
@@ -345,7 +344,7 @@ func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int
|
||||
// space root requested - fix format
|
||||
rid.OpaqueId = rid.GetSpaceId()
|
||||
}
|
||||
pref := func(a RawActivity) bool {
|
||||
pref := func(a activitylog.RawActivity) bool {
|
||||
for _, f := range prefilters {
|
||||
if !f(a) {
|
||||
return false
|
||||
@@ -0,0 +1,76 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
httpsvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
var _ = Describe("Service", func() {
|
||||
Describe("New", func() {
|
||||
var (
|
||||
al *activitylog.ActivityLog
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
al = &activitylog.ActivityLog{}
|
||||
})
|
||||
|
||||
It("creates a service with default options", func() {
|
||||
svc, err := httpsvc.New(al)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with logger option", func() {
|
||||
logger := log.NopLogger()
|
||||
svc, err := httpsvc.New(al, httpsvc.Logger(logger))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with registered events", func() {
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}}
|
||||
svc, err := httpsvc.New(al, httpsvc.RegisteredEvents(evts))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with gateway selector", func() {
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
svc, err := httpsvc.New(al, httpsvc.GatewaySelector(gs))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with history client", func() {
|
||||
var hc ehsvc.EventHistoryService
|
||||
svc, err := httpsvc.New(al, httpsvc.HistoryClient(hc))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with all options", func() {
|
||||
logger := log.NopLogger()
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}, &events.FileTouched{}}
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
var hc ehsvc.EventHistoryService
|
||||
|
||||
svc, err := httpsvc.New(
|
||||
al,
|
||||
httpsvc.Logger(logger),
|
||||
httpsvc.RegisteredEvents(evts),
|
||||
httpsvc.GatewaySelector(gs),
|
||||
httpsvc.HistoryClient(hc),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,676 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base32"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/jellydator/ttlcache/v2"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
)
|
||||
|
||||
// Nats runs into max payload exceeded errors at around 7k activities. Let's keep a buffer.
|
||||
var _maxActivitiesDefault = 6000
|
||||
|
||||
// RawActivity represents an activity as it is stored in the activitylog store
|
||||
type RawActivity struct {
|
||||
EventID string `json:"event_id"`
|
||||
Depth int `json:"depth"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
// ActivitylogService logs events per resource
|
||||
type ActivitylogService struct {
|
||||
cfg *config.Config
|
||||
log log.Logger
|
||||
events <-chan events.Event
|
||||
gws pool.Selectable[gateway.GatewayAPIClient]
|
||||
mux *chi.Mux
|
||||
evHistory ehsvc.EventHistoryService
|
||||
valService settingssvc.ValueService
|
||||
lock sync.RWMutex
|
||||
tp trace.TracerProvider
|
||||
tracer trace.Tracer
|
||||
debouncer *Debouncer
|
||||
parentIdCache *ttlcache.Cache
|
||||
natskv nats.KeyValue
|
||||
|
||||
maxActivities int
|
||||
|
||||
registeredEvents map[string]events.Unmarshaller
|
||||
}
|
||||
|
||||
type Debouncer struct {
|
||||
after time.Duration
|
||||
f func(id string, ra []RawActivity) error
|
||||
pending sync.Map
|
||||
inProgress sync.Map
|
||||
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
type queueItem struct {
|
||||
activities []RawActivity
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
type batchInfo struct {
|
||||
key string
|
||||
count int
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
// NewDebouncer returns a new Debouncer instance
|
||||
func NewDebouncer(d time.Duration, f func(id string, ra []RawActivity) error) *Debouncer {
|
||||
return &Debouncer{
|
||||
after: d,
|
||||
f: f,
|
||||
pending: sync.Map{},
|
||||
inProgress: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
// Debounce restarts the debounce timer for the given space
|
||||
func (d *Debouncer) Debounce(id string, ra RawActivity) {
|
||||
if d.after == 0 {
|
||||
d.f(id, []RawActivity{ra})
|
||||
return
|
||||
}
|
||||
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
|
||||
activities := []RawActivity{ra}
|
||||
item := &queueItem{
|
||||
activities: activities,
|
||||
}
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
// if the item is already in the queue, append the new activities
|
||||
item, ok = i.(*queueItem)
|
||||
if ok {
|
||||
item.activities = append(item.activities, ra)
|
||||
}
|
||||
}
|
||||
|
||||
if item.timer == nil {
|
||||
item.timer = time.AfterFunc(d.after, func() {
|
||||
if _, ok := d.inProgress.Load(id); ok {
|
||||
// Reschedule this run for when the previous run has finished
|
||||
d.mutex.Lock()
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
i.(*queueItem).timer.Reset(d.after)
|
||||
}
|
||||
|
||||
d.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
d.pending.Delete(id)
|
||||
d.inProgress.Store(id, true)
|
||||
defer d.inProgress.Delete(id)
|
||||
d.f(id, item.activities)
|
||||
})
|
||||
}
|
||||
|
||||
d.pending.Store(id, item)
|
||||
}
|
||||
|
||||
// New creates a new ActivitylogService
|
||||
func New(opts ...Option) (*ActivitylogService, error) {
|
||||
o := &Options{
|
||||
MaxActivities: _maxActivitiesDefault,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
|
||||
if o.Stream == nil {
|
||||
return nil, errors.New("stream is required")
|
||||
}
|
||||
|
||||
ch, err := events.Consume(o.Stream, o.Config.Service.Name, o.RegisteredEvents...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cache := ttlcache.NewCache()
|
||||
err = cache.SetTTL(30 * time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Connect to NATS servers
|
||||
natsOptions := nats.Options{
|
||||
Servers: o.Config.Store.Nodes,
|
||||
}
|
||||
if o.Config.Store.EnableTLS {
|
||||
if o.Config.Store.TLSRootCACertificate != "" {
|
||||
// when root ca is configured use it. an insecure flag is ignored.
|
||||
nats.RootCAs(o.Config.Store.TLSRootCACertificate)(&natsOptions)
|
||||
} else {
|
||||
// enable tls and use insecure flag
|
||||
nats.Secure(&tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: o.Config.Store.TLSInsecure})(&natsOptions)
|
||||
}
|
||||
}
|
||||
if o.Config.Store.AuthUsername != "" && o.Config.Store.AuthPassword != "" {
|
||||
nats.UserInfo(o.Config.Store.AuthUsername, o.Config.Store.AuthPassword)(&natsOptions)
|
||||
}
|
||||
conn, err := natsOptions.Connect()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
kv, err := js.KeyValue(o.Config.Store.Database)
|
||||
if err != nil {
|
||||
if !errors.Is(err, nats.ErrBucketNotFound) {
|
||||
return nil, errors.Wrapf(err, "Failed to get bucket (%s)", o.Config.Store.Database)
|
||||
}
|
||||
|
||||
kv, err = js.CreateKeyValue(&nats.KeyValueConfig{
|
||||
Bucket: o.Config.Store.Database,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to create bucket (%s)", o.Config.Store.Database)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &ActivitylogService{
|
||||
log: o.Logger,
|
||||
cfg: o.Config,
|
||||
events: ch,
|
||||
gws: o.GatewaySelector,
|
||||
mux: o.Mux,
|
||||
evHistory: o.HistoryClient,
|
||||
valService: o.ValueClient,
|
||||
lock: sync.RWMutex{},
|
||||
registeredEvents: make(map[string]events.Unmarshaller),
|
||||
tp: o.TraceProvider,
|
||||
tracer: o.TraceProvider.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"),
|
||||
parentIdCache: cache,
|
||||
maxActivities: o.Config.MaxActivities,
|
||||
natskv: kv,
|
||||
}
|
||||
s.debouncer = NewDebouncer(o.Config.WriteBufferDuration, s.storeActivity)
|
||||
|
||||
// run migrations
|
||||
err = s.runMigrations(context.Background(), kv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.mux.Get("/graph/v1beta1/extensions/org.libregraph/activities", s.HandleGetItemActivities)
|
||||
|
||||
for _, e := range o.RegisteredEvents {
|
||||
typ := reflect.TypeOf(e)
|
||||
s.registeredEvents[typ.String()] = e
|
||||
}
|
||||
|
||||
go s.Run()
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Run runs the service
|
||||
func (a *ActivitylogService) Run() {
|
||||
for e := range a.events {
|
||||
var err error
|
||||
switch ev := e.Event.(type) {
|
||||
case events.UploadReady:
|
||||
err = a.AddActivity(ev.FileRef, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.FileTouched:
|
||||
err = a.AddActivity(ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
// Disabled https://github.com/owncloud/ocis/issues/10293
|
||||
//case events.FileDownloaded:
|
||||
// we are only interested in public link downloads - so no need to store others.
|
||||
//if ev.ImpersonatingUser.GetDisplayName() == "Public" {
|
||||
// err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
//}
|
||||
case events.ContainerCreated:
|
||||
err = a.AddActivity(ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemTrashed:
|
||||
err = a.AddActivityTrashed(ev.ID, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemPurged:
|
||||
err = a.RemoveResource(ev.ID)
|
||||
case events.ItemMoved:
|
||||
// remove the cached parent id for this resource
|
||||
a.removeCachedParentID(ev.Ref)
|
||||
|
||||
err = a.AddActivity(ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ShareCreated:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.ShareUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.ShareRemoved:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, ev.Timestamp)
|
||||
case events.LinkCreated:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.LinkUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.LinkRemoved:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.SpaceShared:
|
||||
err = a.AddSpaceActivity(ev.ID, e.ID, ev.Timestamp)
|
||||
case events.SpaceUnshared:
|
||||
err = a.AddSpaceActivity(ev.ID, e.ID, ev.Timestamp)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Interface("event", e).Msg("could not process event")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddActivity adds the activity to the given resource and all its parents
|
||||
func (a *ActivitylogService) AddActivity(initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
}
|
||||
var span trace.Span
|
||||
ctx, span = a.tracer.Start(ctx, "AddActivity")
|
||||
defer span.End()
|
||||
|
||||
return a.addActivity(ctx, initRef, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
|
||||
func (a *ActivitylogService) AddActivityTrashed(resourceID *provider.ResourceId, reference *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
}
|
||||
|
||||
// store activity on trashed item
|
||||
if err := a.storeActivity(storagespace.FormatResourceID(resourceID), []RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("could not store activity: %w", err)
|
||||
}
|
||||
|
||||
// get previous parent
|
||||
ref := &provider.Reference{
|
||||
ResourceId: reference.GetResourceId(),
|
||||
Path: filepath.Dir(reference.GetPath()),
|
||||
}
|
||||
|
||||
var span trace.Span
|
||||
ctx, span = a.tracer.Start(ctx, "AddActivityTrashed")
|
||||
defer span.End()
|
||||
|
||||
return a.addActivity(ctx, ref, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddSpaceActivity adds the activity to the given spaceroot
|
||||
func (a *ActivitylogService) AddSpaceActivity(spaceID *provider.StorageSpaceId, eventID string, timestamp time.Time) error {
|
||||
// spaceID is in format <providerid>$<spaceid>
|
||||
// activitylog service uses format <providerid>$<spaceid>!<resourceid>
|
||||
// lets do some converting, shall we?
|
||||
rid, err := storagespace.ParseID(spaceID.GetOpaqueId())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse space id: %w", err)
|
||||
}
|
||||
rid.OpaqueId = rid.GetSpaceId()
|
||||
return a.storeActivity(storagespace.FormatResourceID(&rid), []RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// Activities returns the activities for the given resource
|
||||
func (a *ActivitylogService) Activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
a.lock.RLock()
|
||||
defer a.lock.RUnlock()
|
||||
|
||||
return a.activities(rid)
|
||||
}
|
||||
|
||||
// RemoveActivities removes the activities from the given resource
|
||||
func (a *ActivitylogService) RemoveActivities(rid *provider.ResourceId, toDelete map[string]struct{}) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
curActivities, err := a.activities(rid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var acts []RawActivity
|
||||
for _, a := range curActivities {
|
||||
if _, ok := toDelete[a.EventID]; !ok {
|
||||
acts = append(acts, a)
|
||||
}
|
||||
}
|
||||
|
||||
b, err := json.Marshal(acts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = a.natskv.Put(storagespace.FormatResourceID(rid), b)
|
||||
return err
|
||||
}
|
||||
|
||||
// RemoveResource removes the resource from the store
|
||||
func (a *ActivitylogService) RemoveResource(rid *provider.ResourceId) error {
|
||||
if rid == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
return a.natskv.Delete(storagespace.FormatResourceID(rid))
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
resourceID := storagespace.FormatResourceID(rid)
|
||||
|
||||
glob := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
watcher, err := a.natskv.Watch(glob, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var activities []RawActivity
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
|
||||
}
|
||||
activities = append(activities, batchActivities...)
|
||||
}
|
||||
|
||||
return activities, nil
|
||||
}
|
||||
|
||||
// note: getResource is abstracted to allow unit testing, in general this will just be utils.GetResource
|
||||
func (a *ActivitylogService) addActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time, getResource func(context.Context, *provider.Reference) (*provider.ResourceInfo, error)) error {
|
||||
var (
|
||||
err error
|
||||
depth int
|
||||
ref = initRef
|
||||
)
|
||||
ctx, span := a.tracer.Start(ctx, "addActivity")
|
||||
defer span.End()
|
||||
for {
|
||||
var info *provider.ResourceInfo
|
||||
id := ref.GetResourceId()
|
||||
if ref.Path != "" {
|
||||
// Path based reference, we need to resolve the resource id
|
||||
ctx, span = a.tracer.Start(ctx, "addActivity.getResource")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get resource info: %w", err)
|
||||
}
|
||||
id = info.GetId()
|
||||
}
|
||||
if id == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
key := storagespace.FormatResourceID(id)
|
||||
a.debouncer.Debounce(key, RawActivity{
|
||||
EventID: eventID,
|
||||
Depth: depth,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
|
||||
if id.OpaqueId == id.SpaceId {
|
||||
// we are at the root of the space, no need to go further
|
||||
break
|
||||
}
|
||||
|
||||
// check if parent id is cached
|
||||
// parent id is cached in the format <storageid>$<spaceid>!<resourceid>
|
||||
// if it is not cached, get the resource info and cache it
|
||||
if parentId == nil {
|
||||
if v, err := a.parentIdCache.Get(key); err != nil {
|
||||
if info == nil {
|
||||
ctx, span := a.tracer.Start(ctx, "addActivity.getResource parent")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil || info.GetParentId() == nil || info.GetParentId().GetOpaqueId() == "" {
|
||||
return fmt.Errorf("could not get parent id: %w", err)
|
||||
}
|
||||
}
|
||||
parentId = info.GetParentId()
|
||||
a.parentIdCache.Set(key, parentId)
|
||||
} else {
|
||||
parentId = v.(*provider.ResourceId)
|
||||
}
|
||||
} else {
|
||||
a.log.Debug().Msg("parent id is cached")
|
||||
}
|
||||
|
||||
depth++
|
||||
ref = &provider.Reference{ResourceId: parentId}
|
||||
parentId = nil // reset parent id so it's not reused in the next iteration
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) storeActivity(resourceID string, activities []RawActivity) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
ctx, span := a.tracer.Start(context.Background(), "storeActivity")
|
||||
defer span.End()
|
||||
|
||||
_, subspan := a.tracer.Start(ctx, "storeActivity.Marshal")
|
||||
b, err := msgpack.Marshal(activities)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = a.tracer.Start(ctx, "storeActivity.natskv.Put")
|
||||
key := natsKey(resourceID, len(activities))
|
||||
_, err = a.natskv.Put(key, b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
ctx, subspan = a.tracer.Start(ctx, "storeActivity.enforceMaxActivities")
|
||||
a.enforceMaxActivities(ctx, resourceID)
|
||||
subspan.End()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) enforceMaxActivities(ctx context.Context, resourceID string) {
|
||||
if a.maxActivities <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
_, subspan := a.tracer.Start(ctx, "enforceMaxActivities.watch")
|
||||
watcher, err := a.natskv.Watch(key, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("resourceID", resourceID).Msg("could not watch")
|
||||
return
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var keys []string
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
|
||||
}
|
||||
keys = append(keys, update.Key())
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = a.tracer.Start(ctx, "enforceMaxActivities.compile")
|
||||
// Parse keys into batches
|
||||
batches := make([]batchInfo, 0)
|
||||
var activitiesCount int
|
||||
for _, k := range keys {
|
||||
parts := strings.SplitN(k, ".", 3)
|
||||
if len(parts) < 3 {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, not enough parts")
|
||||
continue
|
||||
}
|
||||
|
||||
c, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse count")
|
||||
continue
|
||||
}
|
||||
|
||||
// parse timestamp
|
||||
nano, err := strconv.ParseInt(parts[2], 10, 64)
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse timestamp")
|
||||
continue
|
||||
}
|
||||
|
||||
batches = append(batches, batchInfo{
|
||||
key: k,
|
||||
count: c,
|
||||
timestamp: time.Unix(0, nano),
|
||||
})
|
||||
activitiesCount += c
|
||||
}
|
||||
|
||||
// sort batches by timestamp
|
||||
sort.Slice(batches, func(i, j int) bool {
|
||||
return batches[i].timestamp.Before(batches[j].timestamp)
|
||||
})
|
||||
subspan.End()
|
||||
|
||||
_, subspan = a.tracer.Start(ctx, "enforceMaxActivities.delete")
|
||||
// remove oldest keys until we are at max activities
|
||||
for _, b := range batches {
|
||||
if activitiesCount-b.count < a.maxActivities {
|
||||
break
|
||||
}
|
||||
|
||||
activitiesCount -= b.count
|
||||
err = a.natskv.Delete(b.key)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("key", b.key).Msg("could not delete key")
|
||||
break
|
||||
}
|
||||
}
|
||||
subspan.End()
|
||||
}
|
||||
|
||||
func toRef(r *provider.ResourceId) *provider.Reference {
|
||||
return &provider.Reference{
|
||||
ResourceId: r,
|
||||
}
|
||||
}
|
||||
|
||||
func toSpace(r *provider.Reference) *provider.StorageSpaceId {
|
||||
return &provider.StorageSpaceId{
|
||||
OpaqueId: storagespace.FormatStorageID(r.GetResourceId().GetStorageId(), r.GetResourceId().GetSpaceId()),
|
||||
}
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) removeCachedParentID(ref *provider.Reference) {
|
||||
purgeId := ref.GetResourceId()
|
||||
if ref.GetPath() != "" {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get gateway client")
|
||||
return
|
||||
}
|
||||
|
||||
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get service user context")
|
||||
return
|
||||
}
|
||||
|
||||
info, err := utils.GetResource(ctx, ref, gwc)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get resource info")
|
||||
return
|
||||
}
|
||||
purgeId = info.GetId()
|
||||
}
|
||||
// The parent id cache is populated lazily and its entries expire, so a
|
||||
// missing key is the expected case rather than an error.
|
||||
if err := a.parentIdCache.Remove(storagespace.FormatResourceID(purgeId)); err != nil {
|
||||
a.log.Debug().Interface("event", ref).Err(err).Msg("could not delete parent id cache")
|
||||
}
|
||||
}
|
||||
|
||||
func natsKey(resourceID string, activitiesCount int) string {
|
||||
return fmt.Sprintf("%s.%d.%d",
|
||||
base32.StdEncoding.EncodeToString([]byte(resourceID)),
|
||||
activitiesCount,
|
||||
time.Now().UnixNano())
|
||||
}
|
||||
@@ -63,10 +63,6 @@ Store specific notes:
|
||||
- When using `nats-js-kv` it is recommended to set `OC_CACHE_STORE_NODES` to the same value as `OC_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OC_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
### Auto-Accept Shares
|
||||
|
||||
When setting the `SHARING_AUTO_ACCEPT_SHARES` to `true` (sharing service), all incoming shares will be accepted automatically. Users can overwrite this setting individually in their profile. The deprecated `FRONTEND_AUTO_ACCEPT_SHARES` is still supported for backwards compatibility.
|
||||
|
||||
## Passwords
|
||||
|
||||
### The Password Policy
|
||||
|
||||
@@ -61,7 +61,7 @@ type Spaces struct {
|
||||
}
|
||||
|
||||
type LDAPMetrics struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_LDAP_METRICS_DISABLE" desc:"Disables the metrics for outbound LDAP operations." introductionVersion:"%NEXT%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_LDAP_METRICS_DISABLE" desc:"Disables the metrics for outbound LDAP operations." introductionVersion:"%%NEXT%%"`
|
||||
}
|
||||
|
||||
type LDAP struct {
|
||||
@@ -121,7 +121,7 @@ type LDAPEducationConfig struct {
|
||||
}
|
||||
|
||||
type IdentityMetrics struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_IDENTITY_BACKEND_METRICS_DISABLE" desc:"Disables the metrics for inbound identity backend operations." introductionVersion:"%NEXT%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_IDENTITY_BACKEND_METRICS_DISABLE" desc:"Disables the metrics for inbound identity backend operations." introductionVersion:"%%NEXT%%"`
|
||||
}
|
||||
|
||||
type Identity struct {
|
||||
@@ -141,7 +141,7 @@ type API struct {
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
DisabledConsumer bool `yaml:"disabled_consumer" env:"GRAPH_EVENTS_DISABLE_CONSUMER" desc:"Disables consuming events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%NEXT%"`
|
||||
DisabledConsumer bool `yaml:"disabled_consumer" env:"GRAPH_EVENTS_DISABLE_CONSUMER" desc:"Disables consuming events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%%NEXT%%"`
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;GRAPH_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Set to a empty string to disable emitting events." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;GRAPH_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;GRAPH_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -3,12 +3,12 @@ package config
|
||||
import "github.com/opencloud-eu/opencloud/pkg/shared"
|
||||
|
||||
type HTTPMetrics struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_METRICS_DISABLE" desc:"Disables the metrics for the HTTP service." introductionVersion:"%NEXT%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_METRICS_DISABLE" desc:"Disables the metrics for the HTTP service." introductionVersion:"%%NEXT%%"`
|
||||
}
|
||||
|
||||
// HTTP defines the available http configuration.
|
||||
type HTTP struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_DISABLE" desc:"Disables the HTTP service. Set this to true if the service should only consume events." introductionVersion:"%NEXT%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_DISABLE" desc:"Disables the HTTP service. Set this to true if the service should only consume events." introductionVersion:"%%NEXT%%"`
|
||||
Addr string `yaml:"addr" env:"GRAPH_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
Root string `yaml:"root" env:"GRAPH_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -40,11 +40,7 @@ func ParseConfig(cfg *config.Config) error {
|
||||
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.HTTP.Disabled && cfg.Events.DisabledConsumer {
|
||||
// might be debatable, but this situation should be treated as an error,
|
||||
// as the process wouldn't be able to serve either API and would thus be
|
||||
// completely useless -- in that case, just don't start this service
|
||||
// in the first place (especially since it's optional)
|
||||
return shared.AllComponentsDisabledError("graph")
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
}
|
||||
|
||||
if cfg.TokenManager.JWTSecret == "" {
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
# SOME DESCRIPTIVE TITLE.
|
||||
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
|
||||
# This file is distributed under the same license as the PACKAGE package.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
|
||||
#
|
||||
# Translators:
|
||||
# Roy-Magne Mo, 2026
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: \n"
|
||||
"Report-Msgid-Bugs-To: EMAIL\n"
|
||||
"POT-Creation-Date: 2026-09-07 23:16+0000\n"
|
||||
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
|
||||
"Last-Translator: Roy-Magne Mo, 2026\n"
|
||||
"Language-Team: Norwegian (https://app.transifex.com/opencloud-eu/teams/204053/no/)\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Language: no\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
||||
|
||||
#. UnifiedRole Editor, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole Editor, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole EditorListGrants, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole SpaseEditor, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole FileEditor, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole FileEditorWithVersions, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole FileEditorListGrants, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:137 pkg/unifiedrole/roles.go:143
|
||||
#: pkg/unifiedrole/roles.go:149 pkg/unifiedrole/roles.go:155
|
||||
#: pkg/unifiedrole/roles.go:167 pkg/unifiedrole/roles.go:173
|
||||
#: pkg/unifiedrole/roles.go:179
|
||||
msgid "Can edit"
|
||||
msgstr "Kan redigere"
|
||||
|
||||
#. UnifiedRole SpaseEditorWithoutVersions, Role DisplayName (resolves
|
||||
#. directly)
|
||||
#: pkg/unifiedrole/roles.go:161
|
||||
msgid "Can edit without versions"
|
||||
msgstr "Kan redigere uten versjoner"
|
||||
|
||||
#. UnifiedRole Manager, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:191
|
||||
msgid "Can manage"
|
||||
msgstr "Kan håndtere"
|
||||
|
||||
#. UnifiedRole EditorLite, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:185
|
||||
msgid "Can upload"
|
||||
msgstr "Kan laste opp"
|
||||
|
||||
#. UnifiedRole Viewer, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole ViewerWithVersions, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole Viewer, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole SpaseViewer, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole SpaseViewer, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:107 pkg/unifiedrole/roles.go:113
|
||||
#: pkg/unifiedrole/roles.go:119 pkg/unifiedrole/roles.go:125
|
||||
#: pkg/unifiedrole/roles.go:131
|
||||
msgid "Can view"
|
||||
msgstr "Kan se"
|
||||
|
||||
#. UnifiedRole SecureViewer, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:197
|
||||
msgid "Can view (secure)"
|
||||
msgstr "Kan vise (sikker)"
|
||||
|
||||
#. UnifiedRole FullDenial, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:203
|
||||
msgid "Cannot access"
|
||||
msgstr "Kan ikke få tilgang"
|
||||
|
||||
#. UnifiedRole FullDenial, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:200
|
||||
msgid "Deny all access."
|
||||
msgstr "Avvis all tilgang."
|
||||
|
||||
#. default description for new spaces
|
||||
#: pkg/service/v0/spacetemplates.go:32
|
||||
msgid "Here you can add a description for this Space."
|
||||
msgstr "Her kan du legge til en beskrivelse for dette rommet."
|
||||
|
||||
#. UnifiedRole ViewerWithVersions, Role Description (resolves directly)
|
||||
#. UnifiedRole SpaceViewer, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:110 pkg/unifiedrole/roles.go:128
|
||||
msgid "View and download including the history."
|
||||
msgstr "Vis og last ned, inkludert historikken."
|
||||
|
||||
#. UnifiedRole Viewer, Role Description (resolves directly)
|
||||
#. UnifiedRole SpaceViewer, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:104 pkg/unifiedrole/roles.go:122
|
||||
msgid "View and download."
|
||||
msgstr "Vis og last ned."
|
||||
|
||||
#. UnifiedRole SecureViewer, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:194
|
||||
msgid "View only documents, images and PDFs. Watermarks will be applied."
|
||||
msgstr "Vis kun dokumenter, bilder og PDF-er. Vannmerker vil bli brukt."
|
||||
|
||||
#. UnifiedRole FileEditorWithVErsions, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:176
|
||||
msgid "View, download and edit including the history."
|
||||
msgstr "Vis, last ned og rediger, inkludert historikken."
|
||||
|
||||
#. UnifiedRole FileEditor, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:164
|
||||
msgid "View, download and edit."
|
||||
msgstr "Vis, last ned og rediger."
|
||||
|
||||
#. UnifiedRole ViewerListGrants, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:116
|
||||
msgid "View, download and show all invited people."
|
||||
msgstr "Vis, last ned og vis alle inviterte personer."
|
||||
|
||||
#. UnifiedRole EditorLite, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:182
|
||||
msgid "View, download and upload."
|
||||
msgstr "Vis, last ned og last opp."
|
||||
|
||||
#. UnifiedRole FileEditorListGrants, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:170
|
||||
msgid "View, download, edit and show all invited people."
|
||||
msgstr "Vis, last ned, rediger og vis alle inviterte personer."
|
||||
|
||||
#. UnifiedRole Editor, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:140
|
||||
msgid "View, download, upload, edit, add and delete including the history."
|
||||
msgstr ""
|
||||
"Vis, last ned, last opp, rediger, legg til og slett, inkludert historikken."
|
||||
|
||||
#. UnifiedRole Editor, Role Description (resolves directly)
|
||||
#. UnifiedRole SpaseEditorWithoutVersions, Role Description (resolves
|
||||
#. directly)
|
||||
#: pkg/unifiedrole/roles.go:134 pkg/unifiedrole/roles.go:158
|
||||
msgid "View, download, upload, edit, add and delete."
|
||||
msgstr "Vis, last ned, last opp, rediger, legg til og slett."
|
||||
|
||||
#. UnifiedRole Manager, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:188
|
||||
msgid "View, download, upload, edit, add, delete and manage members."
|
||||
msgstr ""
|
||||
"Vis, last ned, last opp, rediger, legg til, slett og administrer medlemmer."
|
||||
|
||||
#. UnifiedRoleListGrants Editor, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:146
|
||||
msgid "View, download, upload, edit, add, delete and show all invited people."
|
||||
msgstr ""
|
||||
"Se, last ned, last opp, rediger, legg til, slett og vis alle inviterte "
|
||||
"personer."
|
||||
|
||||
#. UnifiedRole SpaseEditor, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:152
|
||||
msgid "View, download, upload, edit, add, delete including the history."
|
||||
msgstr ""
|
||||
"Vis, last ned, last opp, rediger, legg til, slett inkludert historikken."
|
||||
@@ -500,18 +500,6 @@ func cs3TimestampToTime(t *types.Timestamp) time.Time {
|
||||
return time.Unix(int64(t.GetSeconds()), int64(t.GetNanos()))
|
||||
}
|
||||
|
||||
// webURLForID builds an item's private link, {publicBaseURL}/f/{id}, which the
|
||||
// web client reads as privateLink. Used by the drive item listing and the
|
||||
// search hits so the field is identical in both.
|
||||
func webURLForID(publicBaseURL *url.URL, id string) *string {
|
||||
if publicBaseURL == nil {
|
||||
return nil
|
||||
}
|
||||
u := *publicBaseURL
|
||||
u.Path = path.Join(u.Path, "f", id)
|
||||
return libregraph.PtrString(u.String())
|
||||
}
|
||||
|
||||
func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *storageprovider.ResourceInfo) (*libregraph.DriveItem, error) {
|
||||
size := new(int64)
|
||||
*size = int64(res.GetSize()) // TODO lurking overflow: make size of libregraph drive item use uint64
|
||||
@@ -521,7 +509,9 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
|
||||
Size: size,
|
||||
}
|
||||
|
||||
driveItem.WebUrl = webURLForID(publicBaseURL, storagespace.FormatResourceID(res.GetId()))
|
||||
webURL := *publicBaseURL
|
||||
webURL.Path = path.Join(webURL.Path, "f", storagespace.FormatResourceID(res.GetId()))
|
||||
driveItem.WebUrl = libregraph.PtrString(webURL.String())
|
||||
|
||||
if name := path.Base(res.GetPath()); name != "" {
|
||||
driveItem.Name = &name
|
||||
|
||||
@@ -1,586 +0,0 @@
|
||||
package svc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/go-chi/render"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
revaCtx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
merrors "go-micro.dev/v4/errors"
|
||||
"go-micro.dev/v4/metadata"
|
||||
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/aggregation"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// SearchQuery runs the search requests and returns results grouped by request
|
||||
// (MS Graph searchQuery).
|
||||
func (g Graph) SearchQuery(w http.ResponseWriter, r *http.Request) {
|
||||
var req libregraph.SearchQueryRequest
|
||||
if err := StrictJSONUnmarshal(r.Body, &req); err != nil {
|
||||
g.logger.Debug().Err(err).Msg("could not decode search query request")
|
||||
errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, "invalid body schema definition")
|
||||
return
|
||||
}
|
||||
|
||||
if len(req.Requests) == 0 {
|
||||
errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, "requests array must not be empty")
|
||||
return
|
||||
}
|
||||
|
||||
for _, sr := range req.Requests {
|
||||
if err := validateAggregations(sr.Aggregations); err != nil {
|
||||
errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if err := validateSortProperties(sr.SortProperties); err != nil {
|
||||
errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
th := r.Header.Get(revaCtx.TokenHeader)
|
||||
ctx := revaCtx.ContextSetToken(r.Context(), th)
|
||||
ctx = metadata.Set(ctx, revaCtx.TokenHeader, th)
|
||||
|
||||
expandThumbnails := driveItemRelationExpanded(r, _expandThumbnails)
|
||||
|
||||
responses := make([]libregraph.SearchResponse, 0, len(req.Requests))
|
||||
for _, sr := range req.Requests {
|
||||
sresp, err := g.runSingleSearch(ctx, sr, expandThumbnails)
|
||||
if err != nil {
|
||||
g.renderSearchError(w, r, err)
|
||||
return
|
||||
}
|
||||
responses = append(responses, sresp)
|
||||
}
|
||||
|
||||
render.Status(r, http.StatusOK)
|
||||
render.JSON(w, r, libregraph.SearchQuery200Response{Value: responses})
|
||||
}
|
||||
|
||||
func (g Graph) runSingleSearch(ctx context.Context, sr libregraph.SearchRequest, expandThumbnails bool) (libregraph.SearchResponse, error) {
|
||||
from, size := clampPagination(sr.From, sr.Size)
|
||||
|
||||
// The gRPC layer has no from field: request from+size matches and slice
|
||||
// client-side. int64 avoids int32 overflow.
|
||||
pageSize := int32(int64(from) + int64(size))
|
||||
if size == 0 {
|
||||
pageSize = 0
|
||||
}
|
||||
|
||||
rsp, err := g.searchService.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: sr.Query.QueryString,
|
||||
PageSize: pageSize,
|
||||
Aggregations: libregraphAggregationsToSearch(sr.Aggregations),
|
||||
AggregationFilters: sr.AggregationFilters,
|
||||
OrderBy: libregraphSortToSearch(sr.SortProperties),
|
||||
})
|
||||
if err != nil {
|
||||
return libregraph.SearchResponse{}, err
|
||||
}
|
||||
|
||||
// the current user id decides the @libre.graph.me.following (favorite) flag,
|
||||
// mirroring the WebDAV report's oc:favorite (favorited by the caller).
|
||||
uid := ""
|
||||
if u, ok := revaCtx.ContextGetUser(ctx); ok {
|
||||
uid = u.GetId().GetOpaqueId()
|
||||
}
|
||||
|
||||
hits := make([]libregraph.SearchHit, 0)
|
||||
if size > 0 {
|
||||
start := min(int(from), len(rsp.Matches))
|
||||
end := min(start+int(size), len(rsp.Matches))
|
||||
for i := start; i < end; i++ {
|
||||
hit := matchToSearchHit(rsp.Matches[i], int32(i+1), uid)
|
||||
hit.Resource.WebUrl = webURLForID(g.publicBaseURL, hit.Resource.GetId())
|
||||
if expandThumbnails {
|
||||
setDriveItemThumbnailsByID(hit.Resource, hit.Resource.GetId(), g.config.Commons.OpenCloudURL)
|
||||
}
|
||||
hits = append(hits, hit)
|
||||
}
|
||||
}
|
||||
|
||||
total := int64(rsp.TotalMatches)
|
||||
more := int64(from+size) < total
|
||||
return libregraph.SearchResponse{
|
||||
SearchTerms: []string{sr.Query.QueryString},
|
||||
HitsContainers: []libregraph.SearchHitsContainer{{
|
||||
Hits: hits,
|
||||
Total: &total,
|
||||
MoreResultsAvailable: &more,
|
||||
Aggregations: searchAggregationsToLibregraph(rsp.Aggregations, sr.Aggregations),
|
||||
}},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// maxPageSize mirrors the openapi spec's upper bound on SearchRequest.size.
|
||||
const maxPageSize = 500
|
||||
|
||||
// clampPagination normalises the from/size JSON pointers into safe non-negative
|
||||
// int32s; openapi-generator does not enforce the spec's [0,500]/[0,inf) bounds.
|
||||
func clampPagination(fromP, sizeP *int32) (int32, int32) {
|
||||
from := int32(0)
|
||||
if fromP != nil && *fromP > 0 {
|
||||
from = *fromP
|
||||
}
|
||||
size := int32(25)
|
||||
if sizeP != nil {
|
||||
size = *sizeP
|
||||
if size < 0 {
|
||||
size = 0
|
||||
}
|
||||
if size > maxPageSize {
|
||||
size = maxPageSize
|
||||
}
|
||||
}
|
||||
// from+size is sent as a single int32 PageSize; guard against overflow.
|
||||
const maxInt32 = int32(1<<31 - 1)
|
||||
if int64(from)+int64(size) > int64(maxInt32) {
|
||||
if from > maxInt32-size {
|
||||
from = maxInt32 - size
|
||||
}
|
||||
}
|
||||
return from, size
|
||||
}
|
||||
|
||||
// validateAggregations rejects terms aggregations on numeric/time fields: bleve
|
||||
// indexes them as prefix-coded binary, so term buckets are meaningless. Ranges
|
||||
// are the supported alternative. Classification via search.IsNumericField.
|
||||
func validateAggregations(aggs []libregraph.AggregationOption) error {
|
||||
for _, a := range aggs {
|
||||
if !search.IsNumericField(a.Field) {
|
||||
continue
|
||||
}
|
||||
if a.LibreGraphMetricDefinition != nil {
|
||||
// metrics reduce numeric values, no term buckets involved
|
||||
continue
|
||||
}
|
||||
hasRanges := a.BucketDefinition != nil && len(a.BucketDefinition.Ranges) > 0
|
||||
if hasRanges {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("terms aggregation is not supported on numeric field %q; use bucketDefinition.ranges", a.Field)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateSortProperties rejects sorting by unknown or multivalued fields.
|
||||
// Sortable are scalar fields carried on the search hit: name, size,
|
||||
// lastModifiedDateTime, mimeType and the facet fields (photo.takenDateTime,
|
||||
// audio.artist, image.width, ...); see search.IsSortableField.
|
||||
func validateSortProperties(sortProperties []libregraph.SortProperty) error {
|
||||
for _, sp := range sortProperties {
|
||||
if !search.IsSortableField(sp.Name) {
|
||||
return fmt.Errorf("field %q is not sortable; sortable are scalar hit fields such as name, size, lastModifiedDateTime, mimeType or photo.takenDateTime", sp.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func libregraphSortToSearch(in []libregraph.SortProperty) []*searchsvc.SortProperty {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]*searchsvc.SortProperty, 0, len(in))
|
||||
for _, sp := range in {
|
||||
p := &searchsvc.SortProperty{Name: sp.Name}
|
||||
if sp.IsDescending != nil {
|
||||
p.IsDescending = *sp.IsDescending
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func libregraphAggregationsToSearch(in []libregraph.AggregationOption) []*searchsvc.AggregationOption {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]*searchsvc.AggregationOption, 0, len(in))
|
||||
for _, a := range in {
|
||||
agg := &searchsvc.AggregationOption{Field: a.Field}
|
||||
if a.Size != nil {
|
||||
agg.Size = *a.Size
|
||||
}
|
||||
if a.BucketDefinition != nil {
|
||||
agg.BucketDefinition = libregraphBucketDefinitionToSearch(*a.BucketDefinition)
|
||||
}
|
||||
if len(a.LibreGraphSubAggregations) > 0 {
|
||||
agg.SubAggregations = libregraphAggregationsToSearch(a.LibreGraphSubAggregations)
|
||||
}
|
||||
if a.LibreGraphMetricDefinition != nil {
|
||||
agg.MetricKind = metricKindFromLibregraph(a.LibreGraphMetricDefinition.Kind)
|
||||
}
|
||||
out = append(out, agg)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// metricKindFromLibregraph maps the OpenAPI string enum to the proto enum;
|
||||
// unknown values degrade to UNSPECIFIED.
|
||||
func metricKindFromLibregraph(kind string) searchsvc.MetricKind {
|
||||
switch kind {
|
||||
case "sum":
|
||||
return searchsvc.MetricKind_METRIC_KIND_SUM
|
||||
case "min":
|
||||
return searchsvc.MetricKind_METRIC_KIND_MIN
|
||||
case "max":
|
||||
return searchsvc.MetricKind_METRIC_KIND_MAX
|
||||
case "avg":
|
||||
return searchsvc.MetricKind_METRIC_KIND_AVG
|
||||
}
|
||||
return searchsvc.MetricKind_METRIC_KIND_UNSPECIFIED
|
||||
}
|
||||
|
||||
// metricKindToLibregraph is the inverse of metricKindFromLibregraph.
|
||||
func metricKindToLibregraph(kind searchsvc.MetricKind) *string {
|
||||
var s string
|
||||
switch kind {
|
||||
case searchsvc.MetricKind_METRIC_KIND_SUM:
|
||||
s = "sum"
|
||||
case searchsvc.MetricKind_METRIC_KIND_MIN:
|
||||
s = "min"
|
||||
case searchsvc.MetricKind_METRIC_KIND_MAX:
|
||||
s = "max"
|
||||
case searchsvc.MetricKind_METRIC_KIND_AVG:
|
||||
s = "avg"
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return &s
|
||||
}
|
||||
|
||||
func libregraphBucketDefinitionToSearch(in libregraph.BucketDefinition) *searchsvc.BucketDefinition {
|
||||
bd := &searchsvc.BucketDefinition{SortBy: in.SortBy}
|
||||
if in.IsDescending != nil {
|
||||
bd.IsDescending = *in.IsDescending
|
||||
}
|
||||
if in.MinimumCount != nil {
|
||||
bd.MinimumCount = *in.MinimumCount
|
||||
}
|
||||
if len(in.Ranges) > 0 {
|
||||
bd.Ranges = make([]*searchsvc.BucketRange, 0, len(in.Ranges))
|
||||
for _, r := range in.Ranges {
|
||||
br := &searchsvc.BucketRange{}
|
||||
if r.From != nil {
|
||||
br.From = *r.From
|
||||
}
|
||||
if r.To != nil {
|
||||
br.To = *r.To
|
||||
}
|
||||
bd.Ranges = append(bd.Ranges, br)
|
||||
}
|
||||
}
|
||||
return bd
|
||||
}
|
||||
|
||||
func searchAggregationsToLibregraph(in []*searchsvc.AggregationResult, defs []libregraph.AggregationOption) []libregraph.SearchAggregation {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
defsByField := make(map[string]libregraph.AggregationOption, len(defs))
|
||||
for _, d := range defs {
|
||||
defsByField[d.Field] = d
|
||||
}
|
||||
out := make([]libregraph.SearchAggregation, 0, len(in))
|
||||
for _, a := range in {
|
||||
field := a.GetField()
|
||||
def := defsByField[field]
|
||||
// Metric result: a scalar, no buckets. For AVG the backend
|
||||
// transported (sum, count); collapse to the average here.
|
||||
if kind := a.GetMetricKind(); kind != searchsvc.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
value := a.GetValue()
|
||||
if kind == searchsvc.MetricKind_METRIC_KIND_AVG && a.GetCount() > 0 {
|
||||
value = a.GetSum() / float64(a.GetCount())
|
||||
}
|
||||
out = append(out, libregraph.SearchAggregation{
|
||||
Field: &field,
|
||||
LibreGraphMetric: &libregraph.SearchMetric{Kind: metricKindToLibregraph(kind), Value: &value},
|
||||
})
|
||||
continue
|
||||
}
|
||||
buckets := make([]libregraph.SearchBucket, 0, len(a.GetBuckets()))
|
||||
for _, b := range a.GetBuckets() {
|
||||
key := b.GetKey()
|
||||
count := b.GetCount()
|
||||
lb := libregraph.SearchBucket{
|
||||
Key: &key,
|
||||
Count: &count,
|
||||
}
|
||||
if token := aggregationTokenForBucket(key, def); token != "" {
|
||||
lb.AggregationFilterToken = &token
|
||||
}
|
||||
if subs := b.GetSubAggregations(); len(subs) > 0 {
|
||||
lb.LibreGraphSubAggregations = searchAggregationsToLibregraph(subs, def.LibreGraphSubAggregations)
|
||||
}
|
||||
buckets = append(buckets, lb)
|
||||
}
|
||||
out = append(out, libregraph.SearchAggregation{
|
||||
Field: &field,
|
||||
Buckets: buckets,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// aggregationTokenForBucket returns the aggregationFilterToken for a bucket: a
|
||||
// range token when the aggregation defines ranges (matched to the range whose
|
||||
// from-to key produced this bucket), otherwise a terms token for the key.
|
||||
func aggregationTokenForBucket(key string, def libregraph.AggregationOption) string {
|
||||
if def.BucketDefinition != nil && len(def.BucketDefinition.Ranges) > 0 {
|
||||
for _, r := range def.BucketDefinition.Ranges {
|
||||
from, to := ptrStr(r.From), ptrStr(r.To)
|
||||
if from+"-"+to == key {
|
||||
return aggregation.EncodeRangeToken(from, to)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return aggregation.EncodeTermsToken(key)
|
||||
}
|
||||
|
||||
func ptrStr(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
}
|
||||
return *s
|
||||
}
|
||||
|
||||
func (g Graph) renderSearchError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
e := merrors.Parse(err.Error())
|
||||
switch e.Code {
|
||||
case http.StatusBadRequest:
|
||||
errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, e.Detail)
|
||||
default:
|
||||
g.logger.Error().Err(err).Msg("search service call failed")
|
||||
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func matchToSearchHit(m *searchmsg.Match, rank int32, uid string) libregraph.SearchHit {
|
||||
hit := libregraph.SearchHit{
|
||||
HitId: libregraph.PtrString(searchEntityHitID(m.GetEntity())),
|
||||
Rank: &rank,
|
||||
}
|
||||
if h := m.GetEntity().GetHighlights(); h != "" {
|
||||
hit.Summary = libregraph.PtrString(h)
|
||||
}
|
||||
di := searchEntityToDriveItem(m.GetEntity(), uid)
|
||||
hit.Resource = di
|
||||
return hit
|
||||
}
|
||||
|
||||
func searchEntityHitID(e *searchmsg.Entity) string {
|
||||
return storagespace.FormatResourceID(&storageprovider.ResourceId{
|
||||
StorageId: e.GetId().GetStorageId(),
|
||||
SpaceId: e.GetId().GetSpaceId(),
|
||||
OpaqueId: e.GetId().GetOpaqueId(),
|
||||
})
|
||||
}
|
||||
|
||||
func searchEntityToDriveItem(e *searchmsg.Entity, uid string) *libregraph.DriveItem {
|
||||
size := int64(e.GetSize())
|
||||
di := &libregraph.DriveItem{
|
||||
Id: libregraph.PtrString(searchEntityHitID(e)),
|
||||
Name: libregraph.PtrString(e.GetName()),
|
||||
Size: &size,
|
||||
}
|
||||
if etag := e.GetEtag(); etag != "" {
|
||||
di.ETag = &etag
|
||||
}
|
||||
if mt := e.GetLastModifiedTime(); mt != nil {
|
||||
lm := time.Unix(mt.GetSeconds(), int64(mt.GetNanos())).UTC()
|
||||
di.LastModifiedDateTime = &lm
|
||||
}
|
||||
if e.GetType() == uint64(storageprovider.ResourceType_RESOURCE_TYPE_FILE) && e.GetMimeType() != "" {
|
||||
mt := e.GetMimeType()
|
||||
di.File = &libregraph.OpenGraphFile{MimeType: &mt}
|
||||
}
|
||||
if e.GetType() == uint64(storageprovider.ResourceType_RESOURCE_TYPE_CONTAINER) {
|
||||
di.Folder = &libregraph.Folder{}
|
||||
}
|
||||
if p := e.GetParentId(); p != nil {
|
||||
ref := libregraph.NewItemReference()
|
||||
ref.SetDriveId(storagespace.FormatStorageID(p.GetStorageId(), p.GetSpaceId()))
|
||||
ref.SetId(storagespace.FormatResourceID(&storageprovider.ResourceId{
|
||||
StorageId: p.GetStorageId(),
|
||||
SpaceId: p.GetSpaceId(),
|
||||
OpaqueId: p.GetOpaqueId(),
|
||||
}))
|
||||
if refPath := e.GetRef().GetPath(); refPath != "" {
|
||||
ref.SetName(path.Base(path.Dir(refPath)))
|
||||
ref.SetPath(path.Dir(refPath))
|
||||
}
|
||||
di.ParentReference = ref
|
||||
}
|
||||
di.RemoteItem = searchEntityToRemoteItem(e)
|
||||
di.Audio = searchAudioToLibregraph(e.GetAudio())
|
||||
di.Image = searchImageToLibregraph(e.GetImage())
|
||||
di.Photo = searchPhotoToLibregraph(e.GetPhoto())
|
||||
di.Location = searchLocationToLibregraph(e.GetLocation())
|
||||
di.Video = searchVideoToLibregraph(e.GetVideo())
|
||||
di.LibreGraphMotionPhoto = searchMotionPhotoToLibregraph(e.GetMotionPhoto())
|
||||
di.LibreGraphLivePhoto = searchLivePhotoToLibregraph(e.GetLivePhoto())
|
||||
if tags := e.GetTags(); len(tags) > 0 {
|
||||
di.LibreGraphTags = tags
|
||||
}
|
||||
if av := e.GetPermissionsActionsAllowedValues(); len(av) > 0 {
|
||||
di.LibreGraphPermissionsActionsAllowedValues = av
|
||||
}
|
||||
// @libre.graph.me.following mirrors the WebDAV report's oc:favorite: the
|
||||
// report emits it only when the current user has favorited the item, so set
|
||||
// it to true only in that case and leave it unset otherwise.
|
||||
if uid != "" && slices.Contains(e.GetFavorites(), uid) {
|
||||
di.LibreGraphMeFollowing = libregraph.PtrBool(true)
|
||||
}
|
||||
return di
|
||||
}
|
||||
|
||||
// searchEntityToRemoteItem describes a hit that lives in a space shared with the
|
||||
// caller: the item id in the owner's drive and the mountpoint it is reached
|
||||
// through. Absent for hits from the caller's own spaces.
|
||||
func searchEntityToRemoteItem(e *searchmsg.Entity) *libregraph.RemoteItem {
|
||||
id := e.GetRemoteItemId()
|
||||
if id == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
item := libregraph.NewRemoteItem()
|
||||
item.SetId(storagespace.FormatResourceID(&storageprovider.ResourceId{
|
||||
StorageId: id.GetStorageId(),
|
||||
SpaceId: id.GetSpaceId(),
|
||||
OpaqueId: id.GetOpaqueId(),
|
||||
}))
|
||||
|
||||
if root := e.GetShareRootName(); root != "" {
|
||||
item.SetPath(root)
|
||||
item.SetName(path.Base(root))
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
|
||||
func searchAudioToLibregraph(a *searchmsg.Audio) *libregraph.Audio {
|
||||
if a == nil {
|
||||
return nil
|
||||
}
|
||||
out := &libregraph.Audio{
|
||||
Album: a.Album,
|
||||
AlbumArtist: a.AlbumArtist,
|
||||
Artist: a.Artist,
|
||||
Bitrate: a.Bitrate,
|
||||
Composers: a.Composers,
|
||||
Copyright: a.Copyright,
|
||||
Disc: a.Disc,
|
||||
DiscCount: a.DiscCount,
|
||||
Duration: a.Duration,
|
||||
Genre: a.Genre,
|
||||
HasDrm: a.HasDrm,
|
||||
IsVariableBitrate: a.IsVariableBitrate,
|
||||
Title: a.Title,
|
||||
Track: a.Track,
|
||||
TrackCount: a.TrackCount,
|
||||
Year: a.Year,
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func searchImageToLibregraph(i *searchmsg.Image) *libregraph.Image {
|
||||
if i == nil {
|
||||
return nil
|
||||
}
|
||||
return &libregraph.Image{Width: i.Width, Height: i.Height}
|
||||
}
|
||||
|
||||
func searchPhotoToLibregraph(p *searchmsg.Photo) *libregraph.Photo {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
out := &libregraph.Photo{
|
||||
CameraMake: p.CameraMake,
|
||||
CameraModel: p.CameraModel,
|
||||
ExposureDenominator: f32ToF64(p.ExposureDenominator),
|
||||
ExposureNumerator: f32ToF64(p.ExposureNumerator),
|
||||
FNumber: f32ToF64(p.FNumber),
|
||||
FocalLength: f32ToF64(p.FocalLength),
|
||||
Iso: p.Iso,
|
||||
Orientation: p.Orientation,
|
||||
}
|
||||
if p.TakenDateTime != nil {
|
||||
t := time.Unix(p.TakenDateTime.GetSeconds(), int64(p.TakenDateTime.GetNanos())).UTC()
|
||||
out.TakenDateTime = &t
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func searchLocationToLibregraph(l *searchmsg.GeoCoordinates) *libregraph.GeoCoordinates {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
return &libregraph.GeoCoordinates{
|
||||
Altitude: l.Altitude,
|
||||
Latitude: l.Latitude,
|
||||
Longitude: l.Longitude,
|
||||
}
|
||||
}
|
||||
|
||||
func searchVideoToLibregraph(v *searchmsg.Video) *libregraph.Video {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return &libregraph.Video{
|
||||
AudioBitsPerSample: v.AudioBitsPerSample,
|
||||
AudioChannels: v.AudioChannels,
|
||||
AudioFormat: v.AudioFormat,
|
||||
AudioSamplesPerSecond: v.AudioSamplesPerSecond,
|
||||
Bitrate: v.Bitrate,
|
||||
Duration: v.Duration,
|
||||
FourCC: v.FourCC,
|
||||
FrameRate: v.FrameRate,
|
||||
Height: v.Height,
|
||||
Width: v.Width,
|
||||
}
|
||||
}
|
||||
|
||||
func searchMotionPhotoToLibregraph(m *searchmsg.MotionPhoto) *libregraph.MotionPhoto {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
return &libregraph.MotionPhoto{
|
||||
Version: m.Version,
|
||||
PresentationTimestampUs: m.PresentationTimestampUs,
|
||||
VideoSize: m.VideoSize,
|
||||
}
|
||||
}
|
||||
|
||||
func searchLivePhotoToLibregraph(l *searchmsg.LivePhoto) *libregraph.LivePhoto {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
return &libregraph.LivePhoto{
|
||||
ContentId: l.GetContentId(),
|
||||
StillImageTimeUs: l.StillImageTimeUs,
|
||||
Auto: l.Auto,
|
||||
VitalityScore: l.VitalityScore,
|
||||
VitalityScoringVersion: l.VitalityScoringVersion,
|
||||
}
|
||||
}
|
||||
|
||||
func f32ToF64(v *float32) *float64 {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
f := float64(*v)
|
||||
return &f
|
||||
}
|
||||
@@ -1,322 +0,0 @@
|
||||
package svc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
||||
// ginkgo qualified: the svc package declares Context (option.go), which
|
||||
// would collide with a dot-import.
|
||||
ginkgo "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"go-micro.dev/v4/client"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
)
|
||||
|
||||
type stubSearchService struct {
|
||||
search func(*searchsvc.SearchRequest) (*searchsvc.SearchResponse, error)
|
||||
}
|
||||
|
||||
func (s stubSearchService) Search(_ context.Context, req *searchsvc.SearchRequest, _ ...client.CallOption) (*searchsvc.SearchResponse, error) {
|
||||
return s.search(req)
|
||||
}
|
||||
|
||||
func (s stubSearchService) IndexSpace(_ context.Context, _ *searchsvc.IndexSpaceRequest, _ ...client.CallOption) (searchsvc.SearchProvider_IndexSpaceService, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func graphWithSearch(stub stubSearchService) Graph {
|
||||
logger := log.NewLogger()
|
||||
return Graph{
|
||||
BaseGraphService: BaseGraphService{logger: &logger},
|
||||
searchService: stub,
|
||||
}
|
||||
}
|
||||
|
||||
func postSearchQuery(g Graph, body string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodPost, "/search/query", bytes.NewBufferString(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
rr := httptest.NewRecorder()
|
||||
g.SearchQuery(rr, req)
|
||||
return rr
|
||||
}
|
||||
|
||||
func int32Ptr(v int32) *int32 { return &v }
|
||||
|
||||
// searchHitRemoteItem decodes the remoteItem of the first hit, nil when absent.
|
||||
func searchHitRemoteItem(rr *httptest.ResponseRecorder) *struct {
|
||||
Id *string `json:"id"`
|
||||
Name *string `json:"name"`
|
||||
Path *string `json:"path"`
|
||||
} {
|
||||
var decoded struct {
|
||||
Value []struct {
|
||||
HitsContainers []struct {
|
||||
Hits []struct {
|
||||
Resource struct {
|
||||
RemoteItem *struct {
|
||||
Id *string `json:"id"`
|
||||
Name *string `json:"name"`
|
||||
Path *string `json:"path"`
|
||||
} `json:"remoteItem"`
|
||||
} `json:"resource"`
|
||||
} `json:"hits"`
|
||||
} `json:"hitsContainers"`
|
||||
} `json:"value"`
|
||||
}
|
||||
Expect(json.Unmarshal(rr.Body.Bytes(), &decoded)).To(Succeed())
|
||||
Expect(decoded.Value).To(HaveLen(1))
|
||||
Expect(decoded.Value[0].HitsContainers).To(HaveLen(1))
|
||||
Expect(decoded.Value[0].HitsContainers[0].Hits).To(HaveLen(1))
|
||||
|
||||
return decoded.Value[0].HitsContainers[0].Hits[0].Resource.RemoteItem
|
||||
}
|
||||
|
||||
var _ = ginkgo.Describe("SearchQuery", func() {
|
||||
ginkgo.It("forwards aggregations to the search service and groups results by request", func() {
|
||||
var captured *searchsvc.SearchRequest
|
||||
g := graphWithSearch(stubSearchService{
|
||||
search: func(req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
|
||||
captured = req
|
||||
return &searchsvc.SearchResponse{
|
||||
TotalMatches: 10,
|
||||
Aggregations: []*searchsvc.AggregationResult{{
|
||||
Field: "audio.artist",
|
||||
Buckets: []*searchsvc.Bucket{
|
||||
{Key: "Pink Floyd", Count: 7},
|
||||
{Key: "Motörhead", Count: 3},
|
||||
},
|
||||
}},
|
||||
}, nil
|
||||
},
|
||||
})
|
||||
|
||||
rr := postSearchQuery(g, `{
|
||||
"requests": [{
|
||||
"entityTypes": ["driveItem"],
|
||||
"query": {"queryString": "mediatype:audio"},
|
||||
"size": 0,
|
||||
"aggregations": [{
|
||||
"field": "audio.artist",
|
||||
"size": 5,
|
||||
"bucketDefinition": {"sortBy": "count", "isDescending": true}
|
||||
}]
|
||||
}]
|
||||
}`)
|
||||
|
||||
Expect(rr.Code).To(Equal(http.StatusOK), rr.Body.String())
|
||||
Expect(captured).ToNot(BeNil())
|
||||
Expect(captured.Aggregations).To(HaveLen(1))
|
||||
Expect(captured.Aggregations[0].Field).To(Equal("audio.artist"))
|
||||
|
||||
var decoded struct {
|
||||
Value []struct {
|
||||
HitsContainers []struct {
|
||||
Aggregations []struct {
|
||||
Field *string `json:"field"`
|
||||
Buckets []struct {
|
||||
Key *string `json:"key"`
|
||||
Count *int64 `json:"count"`
|
||||
} `json:"buckets"`
|
||||
} `json:"aggregations"`
|
||||
} `json:"hitsContainers"`
|
||||
} `json:"value"`
|
||||
}
|
||||
Expect(json.Unmarshal(rr.Body.Bytes(), &decoded)).To(Succeed())
|
||||
Expect(decoded.Value).To(HaveLen(1))
|
||||
Expect(decoded.Value[0].HitsContainers).To(HaveLen(1))
|
||||
|
||||
aggs := decoded.Value[0].HitsContainers[0].Aggregations
|
||||
Expect(aggs).To(HaveLen(1))
|
||||
Expect(aggs[0].Field).To(HaveValue(Equal("audio.artist")))
|
||||
Expect(aggs[0].Buckets).To(HaveLen(2))
|
||||
})
|
||||
|
||||
ginkgo.It("describes a hit from a shared space as a remote item", func() {
|
||||
g := graphWithSearch(stubSearchService{
|
||||
search: func(_ *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
|
||||
return &searchsvc.SearchResponse{
|
||||
TotalMatches: 1,
|
||||
Matches: []*searchmsg.Match{{
|
||||
Entity: &searchmsg.Entity{
|
||||
Id: &searchmsg.ResourceID{StorageId: "1", SpaceId: "2", OpaqueId: "3"},
|
||||
Name: "contract.pdf",
|
||||
ShareRootName: "/Project X",
|
||||
RemoteItemId: &searchmsg.ResourceID{StorageId: "4", SpaceId: "5", OpaqueId: "6"},
|
||||
},
|
||||
}},
|
||||
}, nil
|
||||
},
|
||||
})
|
||||
|
||||
rr := postSearchQuery(g, `{"requests": [{"entityTypes": ["driveItem"], "query": {"queryString": "contract"}}]}`)
|
||||
Expect(rr.Code).To(Equal(http.StatusOK))
|
||||
|
||||
remote := searchHitRemoteItem(rr)
|
||||
Expect(remote).ToNot(BeNil())
|
||||
Expect(remote.Id).To(HaveValue(Equal("4$5!6")))
|
||||
Expect(remote.Path).To(HaveValue(Equal("/Project X")))
|
||||
Expect(remote.Name).To(HaveValue(Equal("Project X")), "the mountpoint name the caller sees")
|
||||
})
|
||||
|
||||
ginkgo.It("leaves the remote item out for hits from the caller's own spaces", func() {
|
||||
g := graphWithSearch(stubSearchService{
|
||||
search: func(_ *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
|
||||
return &searchsvc.SearchResponse{
|
||||
TotalMatches: 1,
|
||||
Matches: []*searchmsg.Match{{
|
||||
Entity: &searchmsg.Entity{
|
||||
Id: &searchmsg.ResourceID{StorageId: "1", SpaceId: "2", OpaqueId: "3"},
|
||||
Name: "notes.txt",
|
||||
},
|
||||
}},
|
||||
}, nil
|
||||
},
|
||||
})
|
||||
|
||||
rr := postSearchQuery(g, `{"requests": [{"entityTypes": ["driveItem"], "query": {"queryString": "notes"}}]}`)
|
||||
Expect(rr.Code).To(Equal(http.StatusOK))
|
||||
Expect(searchHitRemoteItem(rr)).To(BeNil())
|
||||
})
|
||||
|
||||
ginkgo.DescribeTable("clampPagination keeps from/size within valid bounds",
|
||||
func(from, size *int32, wantFrom, wantSize int32) {
|
||||
gotFrom, gotSize := clampPagination(from, size)
|
||||
Expect(gotFrom).To(Equal(wantFrom))
|
||||
Expect(gotSize).To(Equal(wantSize))
|
||||
},
|
||||
ginkgo.Entry("defaults", nil, nil, int32(0), int32(25)),
|
||||
ginkgo.Entry("zero size", int32Ptr(5), int32Ptr(0), int32(5), int32(0)),
|
||||
ginkgo.Entry("negative from clamps to zero", int32Ptr(-10), int32Ptr(5), int32(0), int32(5)),
|
||||
ginkgo.Entry("negative size clamps to zero", int32Ptr(10), int32Ptr(-1), int32(10), int32(0)),
|
||||
ginkgo.Entry("oversized size clamps to max", int32Ptr(0), int32Ptr(1000), int32(0), int32(500)),
|
||||
ginkgo.Entry("from+size overflow collapses", int32Ptr(1<<31-1), int32Ptr(500), int32(1<<31-1-500), int32(500)),
|
||||
)
|
||||
|
||||
ginkgo.It("forwards sortProperties to the search service as order_by", func() {
|
||||
var captured *searchsvc.SearchRequest
|
||||
g := graphWithSearch(stubSearchService{
|
||||
search: func(req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
|
||||
captured = req
|
||||
return &searchsvc.SearchResponse{}, nil
|
||||
},
|
||||
})
|
||||
rr := postSearchQuery(g, `{
|
||||
"requests": [{
|
||||
"entityTypes": ["driveItem"],
|
||||
"query": {"queryString": "mediatype:image"},
|
||||
"sortProperties": [
|
||||
{"name": "photo.takenDateTime", "isDescending": true},
|
||||
{"name": "name"}
|
||||
]
|
||||
}]
|
||||
}`)
|
||||
Expect(rr.Code).To(Equal(http.StatusOK), rr.Body.String())
|
||||
Expect(captured).ToNot(BeNil())
|
||||
Expect(captured.OrderBy).To(HaveLen(2))
|
||||
Expect(captured.OrderBy[0].Name).To(Equal("photo.takenDateTime"))
|
||||
Expect(captured.OrderBy[0].IsDescending).To(BeTrue())
|
||||
Expect(captured.OrderBy[1].Name).To(Equal("name"))
|
||||
Expect(captured.OrderBy[1].IsDescending).To(BeFalse())
|
||||
})
|
||||
|
||||
ginkgo.DescribeTable("accepts sorting by scalar hit fields",
|
||||
func(field string) {
|
||||
g := graphWithSearch(stubSearchService{
|
||||
search: func(*searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
|
||||
return &searchsvc.SearchResponse{}, nil
|
||||
},
|
||||
})
|
||||
rr := postSearchQuery(g, `{
|
||||
"requests": [{
|
||||
"entityTypes": ["driveItem"],
|
||||
"query": {"queryString": "*"},
|
||||
"sortProperties": [{"name": "`+field+`"}]
|
||||
}]
|
||||
}`)
|
||||
Expect(rr.Code).To(Equal(http.StatusOK), rr.Body.String())
|
||||
},
|
||||
ginkgo.Entry("name", "name"),
|
||||
ginkgo.Entry("size", "size"),
|
||||
ginkgo.Entry("lastModifiedDateTime", "lastModifiedDateTime"),
|
||||
ginkgo.Entry("mimeType", "mimeType"),
|
||||
ginkgo.Entry("photo.takenDateTime", "photo.takenDateTime"),
|
||||
ginkgo.Entry("photo.iso", "photo.iso"),
|
||||
ginkgo.Entry("audio.artist", "audio.artist"),
|
||||
ginkgo.Entry("image.width", "image.width"),
|
||||
)
|
||||
|
||||
ginkgo.DescribeTable("rejects sorting by unsortable fields with 400",
|
||||
func(field string) {
|
||||
g := graphWithSearch(stubSearchService{
|
||||
search: func(*searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
|
||||
ginkgo.Fail("search service must not be called when validation fails")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
rr := postSearchQuery(g, `{
|
||||
"requests": [{
|
||||
"entityTypes": ["driveItem"],
|
||||
"query": {"queryString": "mediatype:image"},
|
||||
"sortProperties": [{"name": "`+field+`"}]
|
||||
}]
|
||||
}`)
|
||||
Expect(rr.Code).To(Equal(http.StatusBadRequest), rr.Body.String())
|
||||
Expect(rr.Body.String()).To(ContainSubstring(field))
|
||||
},
|
||||
ginkgo.Entry("unknown field", "definitelyNotAField"),
|
||||
ginkgo.Entry("multivalued field", "tags"),
|
||||
ginkgo.Entry("internal index field name", "Mtime"),
|
||||
ginkgo.Entry("bare audio facet", "audio"),
|
||||
ginkgo.Entry("bare location facet", "location"),
|
||||
)
|
||||
|
||||
ginkgo.It("rejects a terms aggregation on a numeric field with 400", func() {
|
||||
g := graphWithSearch(stubSearchService{
|
||||
search: func(*searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
|
||||
ginkgo.Fail("search service must not be called when validation fails")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
rr := postSearchQuery(g, `{
|
||||
"requests": [{
|
||||
"entityTypes": ["driveItem"],
|
||||
"query": {"queryString": "mediatype:audio"},
|
||||
"size": 0,
|
||||
"aggregations": [{"field": "audio.year"}]
|
||||
}]
|
||||
}`)
|
||||
Expect(rr.Code).To(Equal(http.StatusBadRequest), rr.Body.String())
|
||||
})
|
||||
|
||||
ginkgo.It("allows a range aggregation on a numeric field", func() {
|
||||
called := false
|
||||
g := graphWithSearch(stubSearchService{
|
||||
search: func(*searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
|
||||
called = true
|
||||
return &searchsvc.SearchResponse{}, nil
|
||||
},
|
||||
})
|
||||
rr := postSearchQuery(g, `{
|
||||
"requests": [{
|
||||
"entityTypes": ["driveItem"],
|
||||
"query": {"queryString": "mediatype:audio"},
|
||||
"size": 0,
|
||||
"aggregations": [{
|
||||
"field": "audio.year",
|
||||
"bucketDefinition": {
|
||||
"sortBy": "keyAsString",
|
||||
"ranges": [{"from": "1970", "to": "1980"}]
|
||||
}
|
||||
}]
|
||||
}]
|
||||
}`)
|
||||
Expect(rr.Code).To(Equal(http.StatusOK), rr.Body.String())
|
||||
Expect(called).To(BeTrue())
|
||||
})
|
||||
})
|
||||
@@ -110,8 +110,6 @@ type Service interface { //nolint:interfacebloat
|
||||
GetTags(w http.ResponseWriter, r *http.Request)
|
||||
AssignTags(w http.ResponseWriter, r *http.Request)
|
||||
UnassignTags(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
SearchQuery(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
// NewService returns a service implementation for Service.
|
||||
@@ -290,7 +288,6 @@ func NewService(opts ...Option) (Graph, error) { //nolint:maintidx
|
||||
r.Get("/", svc.GetRoleDefinitions)
|
||||
r.Get("/{roleID}", svc.GetRoleDefinition)
|
||||
})
|
||||
r.Post("/search/query", svc.SearchQuery)
|
||||
})
|
||||
r.Route("/v1.0", func(r chi.Router) {
|
||||
r.Route("/extensions/org.libregraph", func(r chi.Router) {
|
||||
|
||||
@@ -42,7 +42,7 @@ func (g Graph) GetSharedByMe(w http.ResponseWriter, r *http.Request) {
|
||||
expandThumbnails := strings.Contains(expand, "thumbnails")
|
||||
if expandThumbnails {
|
||||
for k, item := range driveItems {
|
||||
setDriveItemThumbnailsByID(&item, item.GetId(), g.config.Commons.OpenCloudURL)
|
||||
setShareThumbnails(&item, item.GetId(), g.config.Commons.OpenCloudURL)
|
||||
driveItems[k] = item
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,7 +71,7 @@ func (g Graph) listSharedWithMe(ctx context.Context, expandThumbnails bool) ([]l
|
||||
|
||||
if expandThumbnails {
|
||||
for k, item := range driveItems {
|
||||
setDriveItemThumbnailsByID(&item, item.RemoteItem.GetId(), g.config.Commons.OpenCloudURL)
|
||||
setShareThumbnails(&item, item.RemoteItem.GetId(), g.config.Commons.OpenCloudURL)
|
||||
driveItems[k] = item
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,10 +57,9 @@ func previewThumbnail(base string, box int32) *libregraph.Thumbnail {
|
||||
return &libregraph.Thumbnail{Url: &url}
|
||||
}
|
||||
|
||||
// setDriveItemThumbnailsByID works off the driveItem itself (mime type from the
|
||||
// item, id passed separately), for callers that have no CS3 resource info: the
|
||||
// share listings and the search results.
|
||||
func setDriveItemThumbnailsByID(item *libregraph.DriveItem, itemID, baseURL string) {
|
||||
// setShareThumbnails works off the driveItem, the share listings have no resource
|
||||
// info. The id comes separately, a received share carries it on its remote item.
|
||||
func setShareThumbnails(item *libregraph.DriveItem, itemID, baseURL string) {
|
||||
mimeType := item.GetFile().MimeType
|
||||
if itemID == "" || mimeType == nil || !thumbnail.IsMimeTypeSupported(*mimeType) {
|
||||
return
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
# SOME DESCRIPTIVE TITLE.
|
||||
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
|
||||
# This file is distributed under the same license as the PACKAGE package.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
|
||||
#
|
||||
# Translators:
|
||||
# Roy-Magne Mo, 2026
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: \n"
|
||||
"Report-Msgid-Bugs-To: EMAIL\n"
|
||||
"POT-Creation-Date: 2026-09-07 23:16+0000\n"
|
||||
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
|
||||
"Last-Translator: Roy-Magne Mo, 2026\n"
|
||||
"Language-Team: Norwegian (https://app.transifex.com/opencloud-eu/teams/204053/no/)\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Language: no\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
||||
|
||||
#. UnsharedSpace email template, resolves via {{ .CallToAction }}
|
||||
#: pkg/email/templates.go:78
|
||||
msgid "Click here to check it: {ShareLink}"
|
||||
msgstr "Klikk her for å sjekke det: {ShareLink}"
|
||||
|
||||
#. ShareCreated email template, resolves via {{ .CallToAction }}
|
||||
#. SharedSpace email template, resolves via {{ .CallToAction }}
|
||||
#: pkg/email/templates.go:23 pkg/email/templates.go:63
|
||||
msgid "Click here to view it: {ShareLink}"
|
||||
msgstr "Klikk her for å se den: {ShareLink}"
|
||||
|
||||
#: pkg/email/templates.go:139
|
||||
msgid "Hello {RecipientName},"
|
||||
msgstr "Hei {RecipientName},"
|
||||
|
||||
#. ShareCreated email template, resolves via {{ .Greeting }}
|
||||
#: pkg/email/templates.go:19
|
||||
msgid "Hello {ShareGrantee}"
|
||||
msgstr "Hei {ShareGrantee}"
|
||||
|
||||
#. ShareRemoved email template, resolves via {{ .Greeting }}
|
||||
#. ShareExpired email template, resolves via {{ .Greeting }}
|
||||
#: pkg/email/templates.go:32 pkg/email/templates.go:45
|
||||
msgid "Hello {ShareGrantee},"
|
||||
msgstr "Hei {ShareGrantee},"
|
||||
|
||||
#. SharedSpace email template, resolves via {{ .Greeting }}
|
||||
#. UnsharedSpace email template, resolves via {{ .Greeting }}
|
||||
#. MembershipExpired email template, resolves via {{ .Greeting }}
|
||||
#: pkg/email/templates.go:59 pkg/email/templates.go:72
|
||||
#: pkg/email/templates.go:87
|
||||
msgid "Hello {SpaceGrantee},"
|
||||
msgstr "Hei {SpaceGrantee},"
|
||||
|
||||
#. Grouped email template, resolves via {{ .Greeting }}
|
||||
#: pkg/email/templates.go:131
|
||||
msgid "Hi {DisplayName},"
|
||||
msgstr "Hei {DisplayName},"
|
||||
|
||||
#. ScienceMeshInviteTokenGenerated email template, resolves via {{ .Greeting
|
||||
#. }}
|
||||
#. ScienceMeshInviteTokenGeneratedWithoutShareLink email template, resolves
|
||||
#. via {{ .Greeting }}
|
||||
#: pkg/email/templates.go:100 pkg/email/templates.go:117
|
||||
msgid "Hi,"
|
||||
msgstr "Hei,"
|
||||
|
||||
#. MembershipExpired email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:85
|
||||
msgid "Membership of '{SpaceName}' expired at {ExpiredAt}"
|
||||
msgstr "Medlemskap i '{SpaceName}' utløpt {ExpiredAt}"
|
||||
|
||||
#. Grouped email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:129
|
||||
msgid "Report"
|
||||
msgstr "Rapport"
|
||||
|
||||
#. ScienceMeshInviteTokenGenerated email template, Subject field (resolves
|
||||
#. directly)
|
||||
#. ScienceMeshInviteTokenGeneratedWithoutShareLink email template, Subject
|
||||
#. field (resolves directly)
|
||||
#: pkg/email/templates.go:98 pkg/email/templates.go:115
|
||||
msgid "ScienceMesh: {InitiatorName} wants to collaborate with you"
|
||||
msgstr "ScienceMesh: {InitiatorName} ønsker å samarbeide med deg"
|
||||
|
||||
#. ShareExpired email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:43
|
||||
msgid "Share to '{ShareFolder}' expired at {ExpiredAt}"
|
||||
msgstr "Deling til '{ShareFolder}' utløpt {ExpiredAt}"
|
||||
|
||||
#: pkg/email/templates.go:141
|
||||
msgid "You can view the mention here: {ResourceLink}"
|
||||
msgstr "Du kan se omtalen her: {ResourceLink}"
|
||||
|
||||
#: pkg/email/templates.go:138
|
||||
msgid "You were mentioned in '{ResourceName}'"
|
||||
msgstr "Du ble nevnt i «{ResourceName}»"
|
||||
|
||||
#. MembershipExpired email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:89
|
||||
msgid ""
|
||||
"Your membership of space {SpaceName} has expired at {ExpiredAt}\n"
|
||||
"\n"
|
||||
"Even though this membership has expired you still might have access through other shares and/or space memberships"
|
||||
msgstr ""
|
||||
"Medlemskapet ditt i rom {SpaceName} utløp {ExpiredAt}\n"
|
||||
"\n"
|
||||
"Selv om dette medlemskapet har utløpt, kan du fortsatt ha tilgang gjennom andre delinger og/eller rommedlemskap"
|
||||
|
||||
#. ShareExpired email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:47
|
||||
msgid ""
|
||||
"Your share to {ShareFolder} has expired at {ExpiredAt}\n"
|
||||
"\n"
|
||||
"Even though this share has been revoked you still might have access through other shares and/or space memberships."
|
||||
msgstr ""
|
||||
"Deltakelsen din i {ShareFolder} utløp {ExpiredAt}\n"
|
||||
"\n"
|
||||
"Selv om denne deltakelsen er trukket tilbake, kan du fortsatt ha tilgang gjennom andre deltakelser og/eller plassmedlemskap."
|
||||
|
||||
#: pkg/email/templates.go:140
|
||||
msgid "{AuthorName} mentioned you in \"{ResourceName}\"."
|
||||
msgstr "{AuthorName} nevnte deg i «{ResourceName}»."
|
||||
|
||||
#. ScienceMeshInviteTokenGeneratedWithoutShareLink email template, resolves
|
||||
#. via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:119
|
||||
msgid ""
|
||||
"{ShareSharer} ({ShareSharerMail}) wants to start sharing collaboration resources with you.\n"
|
||||
"Please visit your federation settings and use the following details:\n"
|
||||
" Token: {Token}\n"
|
||||
" ProviderDomain: {ProviderDomain}"
|
||||
msgstr ""
|
||||
"{ShareSharer} ({ShareSharerMail}) ønsker å begynne å dele samarbeidsressurser med deg.\n"
|
||||
"Vennligst besøk føderasjonsinnstillingene dine og bruk følgende detaljer:\n"
|
||||
" Token: {Token}\n"
|
||||
" ProviderDomain: {ProviderDomain}"
|
||||
|
||||
#. ScienceMeshInviteTokenGenerated email template, resolves via {{
|
||||
#. .MessageBody }}
|
||||
#: pkg/email/templates.go:102
|
||||
msgid ""
|
||||
"{ShareSharer} ({ShareSharerMail}) wants to start sharing collaboration resources with you.\n"
|
||||
"To accept the invite, please visit the following URL:\n"
|
||||
"{ShareLink}\n"
|
||||
"\n"
|
||||
"Alternatively, you can visit your federation settings and use the following details:\n"
|
||||
" Token: {Token}\n"
|
||||
" ProviderDomain: {ProviderDomain}"
|
||||
msgstr ""
|
||||
"{ShareSharer} ({ShareSharerMail}) ønsker å begynne å dele samarbeidsressurser med deg.\n"
|
||||
"For å akseptere invitasjonen, besøk følgende URL:\n"
|
||||
"{ShareLink}\n"
|
||||
"\n"
|
||||
"Alternativt kan du besøke dine føderasjonsinnstillinger og bruke følgende detaljer:\n"
|
||||
" Token: {Token}\n"
|
||||
" ProviderDomain: {ProviderDomain}"
|
||||
|
||||
#. ShareCreated email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:21
|
||||
msgid "{ShareSharer} has shared \"{ShareFolder}\" with you."
|
||||
msgstr "{ShareSharer} har delt «{ShareFolder}» med deg."
|
||||
|
||||
#. ShareRemoved email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:34
|
||||
msgid ""
|
||||
"{ShareSharer} has unshared '{ShareFolder}' with you.\n"
|
||||
"\n"
|
||||
"Even though this share has been revoked you still might have access through other shares and/or space memberships."
|
||||
msgstr ""
|
||||
"{ShareSharer} har fjernet deling '{ShareFolder}' til deg.\n"
|
||||
"\n"
|
||||
"Selv om denne delingen er trukket tilbake, kan du fortsatt ha tilgang gjennom andre delinger og/eller medlemskap i rommet."
|
||||
|
||||
#. ShareCreated email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:17
|
||||
msgid "{ShareSharer} shared '{ShareFolder}' with you"
|
||||
msgstr "{ShareSharer} delte '{ShareFolder}' med deg"
|
||||
|
||||
#. ShareRemoved email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:30
|
||||
msgid "{ShareSharer} unshared '{ShareFolder}' with you"
|
||||
msgstr "{ShareSharer} deler ikke '{ShareFolder}' med deg"
|
||||
|
||||
#. SharedSpace email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:61
|
||||
msgid "{SpaceSharer} has invited you to join \"{SpaceName}\"."
|
||||
msgstr "{SpaceSharer} har invitert deg til å bli med i «{SpaceName}»."
|
||||
|
||||
#. UnsharedSpace email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:74
|
||||
msgid ""
|
||||
"{SpaceSharer} has removed you from \"{SpaceName}\".\n"
|
||||
"\n"
|
||||
"You might still have access through your other groups or direct membership."
|
||||
msgstr ""
|
||||
"{SpaceSharer} har fjernet deg fra «{SpaceName}».\n"
|
||||
"\n"
|
||||
"Du kan fortsatt ha tilgang gjennom andre grupper eller direkte medlemskap."
|
||||
|
||||
#. SharedSpace email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:57
|
||||
msgid "{SpaceSharer} invited you to join {SpaceName}"
|
||||
msgstr "{SpaceSharer} inviterte deg til å bli med i {SpaceName}"
|
||||
|
||||
#. UnsharedSpace email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:70
|
||||
msgid "{SpaceSharer} removed you from {SpaceName}"
|
||||
msgstr "{SpaceSharer} fjernet deg fra {SpaceName}"
|
||||
@@ -29,7 +29,7 @@ type Service struct {
|
||||
|
||||
// GRPC defines the available grpc configuration.
|
||||
type GRPC struct {
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_GRPC_DISABLED" desc:"Disables listening for GRPC API calls. Set this to true if the service should only handle requests through events." introductionVersion:"%NEXT%"`
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_GRPC_DISABLED" desc:"Disables listening for GRPC API calls. Set this to true if the service should only handle requests through events." introductionVersion:"%%NEXT%%"`
|
||||
Addr string `yaml:"addr" env:"POLICIES_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
TLS *shared.GRPCServiceTLS `yaml:"tls"`
|
||||
@@ -50,7 +50,7 @@ type Postprocessing struct {
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle GRPC requests." introductionVersion:"%NEXT%"`
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle GRPC requests." introductionVersion:"%%NEXT%%"`
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;POLICIES_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;POLICIES_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;POLICIES_EVENTS_TLS_INSECURE" desc:"Whether the server should skip the client certificate verification during the TLS handshake." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
|
||||
occfg "github.com/opencloud-eu/opencloud/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/pkg/shared"
|
||||
"github.com/opencloud-eu/opencloud/services/policies/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/policies/pkg/config/defaults"
|
||||
|
||||
@@ -33,12 +34,8 @@ func ParseConfig(cfg *config.Config) error {
|
||||
}
|
||||
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.GRPC.Disabled && cfg.Events.Disabled {
|
||||
// might be debatable, but this situation should be treated as an error,
|
||||
// as the process wouldn't be able to serve either API and would thus be
|
||||
// completely useless -- in that case, just don't start this service
|
||||
// in the first place (especially since it's optional)
|
||||
return errors.New("both gRPC and events APIs are disabled by configuration; at least one must be enabled")
|
||||
if cfg.Events.Disabled && cfg.GRPC.Disabled {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"go-micro.dev/v4/store"
|
||||
@@ -186,6 +187,13 @@ func verifyExpiresAt(claims map[string]any, cmp time.Time) bool {
|
||||
return cmp.Before(expiry)
|
||||
}
|
||||
|
||||
// isExpectedTokenTimeError reports whether the error is caused by a token that
|
||||
// failed its expiry (or not-before) validation. These are expected as part of
|
||||
// the normal token renewal cycle and should not be logged at error level.
|
||||
func isExpectedTokenTimeError(err error) bool {
|
||||
return errors.Is(err, jwt.ErrTokenExpired) || errors.Is(err, jwt.ErrTokenNotValidYet)
|
||||
}
|
||||
|
||||
func (m OIDCAuthenticator) shouldServe(req *http.Request) bool {
|
||||
if m.OIDCIss == "" {
|
||||
return false
|
||||
@@ -212,7 +220,17 @@ func (m *OIDCAuthenticator) Authenticate(r *http.Request) (*http.Request, bool)
|
||||
claims, newSession, err := m.getClaims(token, r)
|
||||
if err != nil {
|
||||
host, port, _ := net.SplitHostPort(r.RemoteAddr)
|
||||
m.Logger.Error().
|
||||
|
||||
// Expired (or not yet valid) tokens are a normal part of the token
|
||||
// renewal cycle: the client hits the proxy with a token that is just
|
||||
// past its expiry, receives a 401 challenge and refreshes. This is not
|
||||
// a failure of the proxy, so we log it at debug level to avoid noise.
|
||||
logEvent := m.Logger.Debug()
|
||||
if !isExpectedTokenTimeError(err) {
|
||||
logEvent = m.Logger.Error()
|
||||
}
|
||||
|
||||
logEvent.
|
||||
Err(err).
|
||||
Str("authenticator", "oidc").
|
||||
Str("path", r.URL.Path).
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"time"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/pkg/oidc"
|
||||
oidcmocks "github.com/opencloud-eu/opencloud/pkg/oidc/mocks"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"go-micro.dev/v4/store"
|
||||
)
|
||||
@@ -106,4 +108,69 @@ var _ = Describe("Authenticating requests", Label("OIDCAuthenticator"), func() {
|
||||
Expect(req2).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
When("the request contains an expired token", func() {
|
||||
var buf bytes.Buffer
|
||||
expiredAuthenticator := &OIDCAuthenticator{
|
||||
OIDCIss: "http://idp.example.com",
|
||||
Logger: log.Logger{zerolog.New(&buf).Level(zerolog.DebugLevel)},
|
||||
oidcClient: &oidcmocks.OIDCClient{},
|
||||
userInfoCache: store.NewMemoryStore(),
|
||||
skipUserInfo: true,
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
expiredAuthenticator.oidcClient.(*oidcmocks.OIDCClient).On("VerifyAccessToken", mock.Anything, mock.Anything).Return(
|
||||
oidc.RegClaimsWithSID{}, jwt.MapClaims{}, jwt.ErrTokenExpired,
|
||||
)
|
||||
})
|
||||
|
||||
It("should reject the request", func() {
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/example/path", http.NoBody)
|
||||
req.Header.Set(_headerAuthorization, "Bearer jwt.token.sig")
|
||||
|
||||
req2, valid := expiredAuthenticator.Authenticate(req)
|
||||
|
||||
Expect(valid).To(Equal(false))
|
||||
Expect(req2).To(BeNil())
|
||||
})
|
||||
|
||||
It("should log the authentication failure at debug level", func() {
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/example/path", http.NoBody)
|
||||
req.Header.Set(_headerAuthorization, "Bearer jwt.token.sig")
|
||||
|
||||
expiredAuthenticator.Authenticate(req)
|
||||
|
||||
Expect(buf.String()).To(ContainSubstring("failed to authenticate the request"))
|
||||
Expect(buf.String()).To(ContainSubstring(`"level":"debug"`))
|
||||
Expect(buf.String()).ToNot(ContainSubstring(`"level":"error"`))
|
||||
})
|
||||
})
|
||||
|
||||
When("the request contains a token that cannot be verified", func() {
|
||||
var buf bytes.Buffer
|
||||
failingAuthenticator := &OIDCAuthenticator{
|
||||
OIDCIss: "http://idp.example.com",
|
||||
Logger: log.Logger{zerolog.New(&buf).Level(zerolog.DebugLevel)},
|
||||
oidcClient: &oidcmocks.OIDCClient{},
|
||||
userInfoCache: store.NewMemoryStore(),
|
||||
skipUserInfo: true,
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
failingAuthenticator.oidcClient.(*oidcmocks.OIDCClient).On("VerifyAccessToken", mock.Anything, mock.Anything).Return(
|
||||
oidc.RegClaimsWithSID{}, jwt.MapClaims{}, jwt.ErrTokenMalformed,
|
||||
)
|
||||
})
|
||||
|
||||
It("should log the authentication failure at error level", func() {
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/example/path", http.NoBody)
|
||||
req.Header.Set(_headerAuthorization, "Bearer jwt.token.sig")
|
||||
|
||||
failingAuthenticator.Authenticate(req)
|
||||
|
||||
Expect(buf.String()).To(ContainSubstring("failed to authenticate the request"))
|
||||
Expect(buf.String()).To(ContainSubstring(`"level":"error"`))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,148 +0,0 @@
|
||||
// Package aggregation encodes and decodes the aggregationFilterToken exchanged
|
||||
// with clients. A terms token is the bucket key as lowercase hex of its UTF-8
|
||||
// bytes, prefixed with U+01C2 twice and wrapped in double quotes (the same
|
||||
// encoding MS Graph uses); a range token is range(from,to) with min/max for open
|
||||
// bounds. Decoding turns a {field}:{token} filter into a KQL fragment the search
|
||||
// engines parse and then force to an exact, case-sensitive match.
|
||||
package aggregation
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// termPrefix marks a hex-encoded terms token (U+01C2 LATIN LETTER ALVEOLAR
|
||||
// CLICK, twice).
|
||||
const termPrefix = "ǂǂ"
|
||||
|
||||
// EncodeTermsToken encodes a terms bucket key as an aggregationFilterToken: the
|
||||
// key as lowercase hex of its UTF-8 bytes, prefixed with termPrefix and wrapped
|
||||
// in double quotes. The quotes are part of the token value.
|
||||
func EncodeTermsToken(key string) string {
|
||||
return `"` + termPrefix + hex.EncodeToString([]byte(key)) + `"`
|
||||
}
|
||||
|
||||
// EncodeRangeToken encodes a range bucket as range(from,to). An empty bound is
|
||||
// open and written as min (lower) or max (upper).
|
||||
func EncodeRangeToken(from, to string) string {
|
||||
if from == "" {
|
||||
from = "min"
|
||||
}
|
||||
if to == "" {
|
||||
to = "max"
|
||||
}
|
||||
return "range(" + from + "," + to + ")"
|
||||
}
|
||||
|
||||
// DecodeAggregationFilter turns a {field}:{token} aggregation filter into a KQL
|
||||
// fragment. Terms and or() tokens become field:"value" restrictions (to be
|
||||
// matched exactly and case-sensitively by the caller); range() becomes a
|
||||
// numeric/date range. Tokens that are not server-shaped are rejected.
|
||||
func DecodeAggregationFilter(filter string) (string, error) {
|
||||
field, token, ok := strings.Cut(filter, ":")
|
||||
if !ok || field == "" || token == "" {
|
||||
return "", fmt.Errorf("invalid aggregation filter %q", filter)
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(token, "or("):
|
||||
return decodeOr(field, token)
|
||||
case strings.HasPrefix(token, "range("):
|
||||
return decodeRange(field, token)
|
||||
default:
|
||||
v, err := decodeTerm(token)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
frag, err := kqlTerm(field, v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return frag, nil
|
||||
}
|
||||
}
|
||||
|
||||
// decodeTerm strips the quotes and termPrefix and hex-decodes a terms token.
|
||||
func decodeTerm(token string) (string, error) {
|
||||
if len(token) < 2 || token[0] != '"' || token[len(token)-1] != '"' {
|
||||
return "", fmt.Errorf("invalid terms token %q", token)
|
||||
}
|
||||
inner := token[1 : len(token)-1]
|
||||
if !strings.HasPrefix(inner, termPrefix) {
|
||||
return "", fmt.Errorf("invalid terms token %q", token)
|
||||
}
|
||||
b, err := hex.DecodeString(strings.TrimPrefix(inner, termPrefix))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid terms token %q: %w", token, err)
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
// decodeRange turns range(from,to) into a KQL comparison; open bounds (min/max)
|
||||
// are dropped.
|
||||
func decodeRange(field, token string) (string, error) {
|
||||
inner, ok := trimCall(token, "range")
|
||||
if !ok {
|
||||
return "", fmt.Errorf("invalid range token %q", token)
|
||||
}
|
||||
from, to, ok := strings.Cut(inner, ",")
|
||||
if !ok {
|
||||
return "", fmt.Errorf("invalid range token %q", token)
|
||||
}
|
||||
from, to = strings.TrimSpace(from), strings.TrimSpace(to)
|
||||
var parts []string
|
||||
if from != "" && from != "min" {
|
||||
parts = append(parts, field+">="+from)
|
||||
}
|
||||
if to != "" && to != "max" {
|
||||
parts = append(parts, field+"<="+to)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "", fmt.Errorf("range token %q has no bounds", token)
|
||||
}
|
||||
return "(" + strings.Join(parts, " AND ") + ")", nil
|
||||
}
|
||||
|
||||
// decodeOr turns or("token","token",...) into an OR group of terms.
|
||||
func decodeOr(field, token string) (string, error) {
|
||||
inner, ok := trimCall(token, "or")
|
||||
if !ok {
|
||||
return "", fmt.Errorf("invalid or token %q", token)
|
||||
}
|
||||
// terms tokens are quote-wrapped lowercase hex, so they never contain a
|
||||
// comma; a plain split is safe.
|
||||
parts := make([]string, 0)
|
||||
for _, t := range strings.Split(inner, ",") {
|
||||
v, err := decodeTerm(strings.TrimSpace(t))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
frag, err := kqlTerm(field, v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts = append(parts, frag)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "", fmt.Errorf("empty or token %q", token)
|
||||
}
|
||||
return "(" + strings.Join(parts, " OR ") + ")", nil
|
||||
}
|
||||
|
||||
// trimCall unwraps name(inner); ok is false when token is not name(...).
|
||||
func trimCall(token, name string) (string, bool) {
|
||||
if !strings.HasPrefix(token, name+"(") || !strings.HasSuffix(token, ")") {
|
||||
return "", false
|
||||
}
|
||||
return token[len(name)+1 : len(token)-1], true
|
||||
}
|
||||
|
||||
// kqlTerm builds a field:"value" restriction. KQL quoted strings have no escape
|
||||
// syntax, so a value containing a double quote cannot be expressed and is
|
||||
// rejected rather than emitted as broken KQL.
|
||||
func kqlTerm(field, value string) (string, error) {
|
||||
if strings.Contains(value, `"`) {
|
||||
return "", fmt.Errorf("aggregation value %q contains an unsupported double quote", value)
|
||||
}
|
||||
return field + `:"` + value + `"`, nil
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package aggregation_test
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/aggregation"
|
||||
)
|
||||
|
||||
var _ = Describe("Token", func() {
|
||||
Describe("EncodeTermsToken", func() {
|
||||
It("encodes the key as quoted ǂǂ-prefixed lowercase hex", func() {
|
||||
Expect(aggregation.EncodeTermsToken("And the Bands Played On")).To(Equal(`"ǂǂ416e64207468652042616e647320506c61796564204f6e"`))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("EncodeRangeToken", func() {
|
||||
DescribeTable("bounds",
|
||||
func(from, to, want string) {
|
||||
Expect(aggregation.EncodeRangeToken(from, to)).To(Equal(want))
|
||||
},
|
||||
Entry("closed", "0", "100", "range(0,100)"),
|
||||
Entry("open lower", "", "100", "range(min,100)"),
|
||||
Entry("open upper", "0", "", "range(0,max)"),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("DecodeAggregationFilter", func() {
|
||||
DescribeTable("valid tokens",
|
||||
func(filter, want string) {
|
||||
got, err := aggregation.DecodeAggregationFilter(filter)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(Equal(want))
|
||||
},
|
||||
Entry("terms with a space", `audio.artist:"ǂǂ5361786f6e"`, `audio.artist:"Saxon"`),
|
||||
Entry("closed range", "Size:range(0,100)", "(Size>=0 AND Size<=100)"),
|
||||
Entry("open lower range", "Size:range(min,100)", "(Size<=100)"),
|
||||
Entry("open upper range", "Size:range(0,max)", "(Size>=0)"),
|
||||
Entry("or of two terms",
|
||||
`audio.artist:or("ǂǂ5361786f6e","ǂǂ49726f6e204d616964656e")`,
|
||||
`(audio.artist:"Saxon" OR audio.artist:"Iron Maiden")`),
|
||||
)
|
||||
|
||||
DescribeTable("rejected tokens",
|
||||
func(filter string) {
|
||||
_, err := aggregation.DecodeAggregationFilter(filter)
|
||||
Expect(err).To(HaveOccurred())
|
||||
},
|
||||
Entry("no colon", `audio.artist"ǂǂ00"`),
|
||||
Entry("empty field", `:"ǂǂ00"`),
|
||||
Entry("missing ǂǂ prefix", `audio.artist:"deadbeef"`),
|
||||
Entry("odd hex", `audio.artist:"ǂǂabc"`),
|
||||
Entry("range without bounds", "Size:range(min,max)"),
|
||||
)
|
||||
|
||||
It("round-trips a terms key through encode+decode", func() {
|
||||
got, err := aggregation.DecodeAggregationFilter("audio.artist:" + aggregation.EncodeTermsToken("AC/DC"))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(Equal(`audio.artist:"AC/DC"`))
|
||||
})
|
||||
|
||||
It("rejects a decoded value containing a double quote", func() {
|
||||
// 22 is a double quote; it cannot be expressed in a KQL string.
|
||||
_, err := aggregation.DecodeAggregationFilter(`audio.artist:"ǂǂ22"`)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -2,13 +2,10 @@ package bleve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
bleveSearch "github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
@@ -46,7 +43,7 @@ func NewBackend(index bleve.Index, queryCreator searchQuery.Creator[query.Query]
|
||||
// Search executes a search request operation within the index.
|
||||
// Returns a SearchIndexResponse object or an error.
|
||||
func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
|
||||
createdQuery, err := b.queryCreator.CreateWithFilters(sir.Query, sir.GetAggregationFilters())
|
||||
createdQuery, err := b.queryCreator.Create(sir.Query)
|
||||
if err != nil {
|
||||
if kql.IsValidationError(err) {
|
||||
return nil, errtypes.BadRequest(err.Error())
|
||||
@@ -92,24 +89,6 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
|
||||
bleveReq := bleve.NewSearchRequest(q)
|
||||
bleveReq.Highlight = bleve.NewHighlight()
|
||||
|
||||
// Sort natively in the index; the service layer re-establishes this order
|
||||
// when merging matches across spaces. Score sorting (bleve's default)
|
||||
// stays in place when no order_by is given.
|
||||
if orderBy := sir.GetOrderBy(); len(orderBy) > 0 {
|
||||
sortOrder := make([]string, 0, len(orderBy)+1)
|
||||
for _, sp := range orderBy {
|
||||
field, ok := search.SortIndexField(sp.GetName())
|
||||
if !ok {
|
||||
return nil, errtypes.BadRequest(fmt.Sprintf("field %q is not sortable", sp.GetName()))
|
||||
}
|
||||
if sp.GetIsDescending() {
|
||||
field = "-" + field
|
||||
}
|
||||
sortOrder = append(sortOrder, field)
|
||||
}
|
||||
bleveReq.SortBy(append(sortOrder, "-_score"))
|
||||
}
|
||||
|
||||
switch {
|
||||
case sir.PageSize == -1:
|
||||
bleveReq.Size = math.MaxInt
|
||||
@@ -119,26 +98,6 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
|
||||
bleveReq.Size = int(sir.PageSize)
|
||||
}
|
||||
|
||||
for _, agg := range sir.GetAggregations() {
|
||||
// Top-level metrics are computed by scanning the matched hits, they
|
||||
// have no facet representation.
|
||||
if agg.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
continue
|
||||
}
|
||||
fr, err := newBleveFacetRequest(agg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bleveReq.AddFacet(agg.GetField(), fr)
|
||||
}
|
||||
|
||||
// Sub-aggregations and top-level metrics need the matched hit set, not just
|
||||
// count facets: widen the page so the emulator has enough docs. The caller's
|
||||
// larger PageSize wins.
|
||||
if needsSubAggScan(sir.GetAggregations()) && bleveReq.Size < subAggScanSize {
|
||||
bleveReq.Size = subAggScanSize
|
||||
}
|
||||
|
||||
bleveReq.Fields = []string{"*"}
|
||||
res, err := b.index.Search(bleveReq)
|
||||
if err != nil {
|
||||
@@ -196,356 +155,9 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
|
||||
return &searchService.SearchIndexResponse{
|
||||
Matches: matches,
|
||||
TotalMatches: int32(totalMatches),
|
||||
Aggregations: extractBleveAggregations(res, sir.GetAggregations()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// subAggScanSize caps how many hits we walk when emulating sub-aggregations;
|
||||
// math.MaxInt returns everything.
|
||||
const subAggScanSize = math.MaxInt
|
||||
|
||||
func needsSubAggScan(aggs []*searchService.AggregationOption) bool {
|
||||
for _, agg := range aggs {
|
||||
if len(agg.GetSubAggregations()) > 0 {
|
||||
return true
|
||||
}
|
||||
if agg.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// defaultFacetSize is used when no size is requested; the service layer trims
|
||||
// after cross-space merge.
|
||||
const defaultFacetSize = 1000
|
||||
|
||||
func newBleveFacetRequest(agg *searchService.AggregationOption) (*bleve.FacetRequest, error) {
|
||||
size := int(agg.GetSize())
|
||||
if size <= 0 {
|
||||
size = defaultFacetSize
|
||||
}
|
||||
fr := bleve.NewFacetRequest(agg.GetField(), size)
|
||||
ranges := aggregationRanges(agg)
|
||||
if rangesAreDates(ranges) {
|
||||
// bleve facets cannot mix numeric and date ranges, so one date-looking
|
||||
// bound switches the whole aggregation to date mode.
|
||||
for _, r := range ranges {
|
||||
start, err := parseRangeTime(r.GetFrom())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid date range bound %q on field %q", r.GetFrom(), agg.GetField())
|
||||
}
|
||||
end, err := parseRangeTime(r.GetTo())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid date range bound %q on field %q", r.GetTo(), agg.GetField())
|
||||
}
|
||||
fr.AddDateTimeRange(rangeBucketKey(r), start, end)
|
||||
}
|
||||
return fr, nil
|
||||
}
|
||||
for _, r := range ranges {
|
||||
minP := parseFloatPtr(r.GetFrom())
|
||||
maxP := parseFloatPtr(r.GetTo())
|
||||
fr.AddNumericRange(rangeBucketKey(r), minP, maxP)
|
||||
}
|
||||
return fr, nil
|
||||
}
|
||||
|
||||
// rangeTimeLayouts are the accepted formats for date range bounds, tried in order.
|
||||
var rangeTimeLayouts = []string{time.RFC3339, "2006-01-02"}
|
||||
|
||||
// rangesAreDates reports whether the ranges should be treated as datetime
|
||||
// ranges: at least one bound parses as a date rather than a number.
|
||||
func rangesAreDates(ranges []*searchService.BucketRange) bool {
|
||||
for _, r := range ranges {
|
||||
for _, s := range []string{r.GetFrom(), r.GetTo()} {
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := strconv.ParseFloat(s, 64); err == nil {
|
||||
continue
|
||||
}
|
||||
if _, err := parseRangeTime(s); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parseRangeTime parses a range bound; the zero time marks an open bound.
|
||||
func parseRangeTime(s string) (time.Time, error) {
|
||||
if s == "" {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
for _, layout := range rangeTimeLayouts {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("unsupported time format %q", s)
|
||||
}
|
||||
|
||||
func extractBleveAggregations(res *bleve.SearchResult, aggs []*searchService.AggregationOption) []*searchService.AggregationResult {
|
||||
if len(aggs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]*searchService.AggregationResult, 0, len(aggs))
|
||||
for _, agg := range aggs {
|
||||
// Top-level metric: fold the matched hits through the sub-agg
|
||||
// accumulator, there is no facet to read from.
|
||||
if agg.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
acc := newSubAcc(agg)
|
||||
for _, hit := range res.Hits {
|
||||
accumulateHit(acc, agg, hit)
|
||||
}
|
||||
if r := emitAcc(acc, agg); r != nil {
|
||||
out = append(out, r)
|
||||
}
|
||||
continue
|
||||
}
|
||||
fr, ok := res.Facets[agg.GetField()]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
buckets := make([]*searchService.Bucket, 0)
|
||||
if len(aggregationRanges(agg)) > 0 {
|
||||
for _, nr := range fr.NumericRanges {
|
||||
buckets = append(buckets, &searchService.Bucket{
|
||||
Key: nr.Name,
|
||||
Count: int64(nr.Count),
|
||||
})
|
||||
}
|
||||
for _, dr := range fr.DateRanges {
|
||||
buckets = append(buckets, &searchService.Bucket{
|
||||
Key: dr.Name,
|
||||
Count: int64(dr.Count),
|
||||
})
|
||||
}
|
||||
} else {
|
||||
for _, t := range fr.Terms.Terms() {
|
||||
buckets = append(buckets, &searchService.Bucket{
|
||||
Key: t.Term,
|
||||
Count: int64(t.Count),
|
||||
})
|
||||
}
|
||||
}
|
||||
if subAggs := agg.GetSubAggregations(); len(subAggs) > 0 {
|
||||
attachSubAggregations(res, agg.GetField(), subAggs, buckets)
|
||||
}
|
||||
out = append(out, &searchService.AggregationResult{
|
||||
Field: agg.GetField(),
|
||||
Buckets: buckets,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// subAcc is the recursive accumulator emulating composite aggregations: one
|
||||
// node per sub-aggregation under a parent bucket.
|
||||
type subAcc struct {
|
||||
// terms: count + recursive accumulators per child value
|
||||
termCount map[string]int64
|
||||
termSubs map[string][]*subAcc
|
||||
|
||||
// metric
|
||||
metricVal float64 // SUM/MIN/MAX
|
||||
sum float64 // AVG transport: numerator
|
||||
count int64 // AVG transport: denominator
|
||||
|
||||
seen bool // at least one hit contributed
|
||||
}
|
||||
|
||||
// newSubAcc allocates an accumulator for the given sub-agg.
|
||||
func newSubAcc(sa *searchService.AggregationOption) *subAcc {
|
||||
a := &subAcc{}
|
||||
if sa.GetMetricKind() == searchService.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
a.termCount = map[string]int64{}
|
||||
if len(sa.GetSubAggregations()) > 0 {
|
||||
a.termSubs = map[string][]*subAcc{}
|
||||
}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// accumulateHit folds one hit into a sub-agg accumulator, recursing into
|
||||
// grand-sub-aggregations.
|
||||
func accumulateHit(a *subAcc, sa *searchService.AggregationOption, hit *bleveSearch.DocumentMatch) {
|
||||
switch sa.GetMetricKind() {
|
||||
case searchService.MetricKind_METRIC_KIND_UNSPECIFIED:
|
||||
val, ok := hit.Fields[sa.GetField()].(string)
|
||||
if !ok || val == "" {
|
||||
return
|
||||
}
|
||||
a.termCount[val]++
|
||||
a.seen = true
|
||||
if subs := sa.GetSubAggregations(); len(subs) > 0 {
|
||||
childAccs, ok := a.termSubs[val]
|
||||
if !ok {
|
||||
childAccs = make([]*subAcc, len(subs))
|
||||
for i, ssa := range subs {
|
||||
childAccs[i] = newSubAcc(ssa)
|
||||
}
|
||||
a.termSubs[val] = childAccs
|
||||
}
|
||||
for i, ssa := range subs {
|
||||
accumulateHit(childAccs[i], ssa, hit)
|
||||
}
|
||||
}
|
||||
default:
|
||||
v, ok := numericFieldValue(hit.Fields[sa.GetField()])
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch sa.GetMetricKind() {
|
||||
case searchService.MetricKind_METRIC_KIND_SUM:
|
||||
a.metricVal += v
|
||||
case searchService.MetricKind_METRIC_KIND_MIN:
|
||||
if !a.seen || v < a.metricVal {
|
||||
a.metricVal = v
|
||||
}
|
||||
case searchService.MetricKind_METRIC_KIND_MAX:
|
||||
if !a.seen || v > a.metricVal {
|
||||
a.metricVal = v
|
||||
}
|
||||
case searchService.MetricKind_METRIC_KIND_AVG:
|
||||
a.sum += v
|
||||
a.count++
|
||||
}
|
||||
a.seen = true
|
||||
}
|
||||
}
|
||||
|
||||
// emitAcc materialises a sub-agg accumulator into the proto result.
|
||||
func emitAcc(a *subAcc, sa *searchService.AggregationOption) *searchService.AggregationResult {
|
||||
if sa.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
if !a.seen {
|
||||
return nil
|
||||
}
|
||||
r := &searchService.AggregationResult{
|
||||
Field: sa.GetField(),
|
||||
MetricKind: sa.GetMetricKind(),
|
||||
}
|
||||
if sa.GetMetricKind() == searchService.MetricKind_METRIC_KIND_AVG {
|
||||
r.Sum = a.sum
|
||||
r.Count = a.count
|
||||
} else {
|
||||
r.Value = a.metricVal
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
subs := sa.GetSubAggregations()
|
||||
childBuckets := make([]*searchService.Bucket, 0, len(a.termCount))
|
||||
for term, count := range a.termCount {
|
||||
b := &searchService.Bucket{Key: term, Count: count}
|
||||
if len(subs) > 0 {
|
||||
if childAccs, ok := a.termSubs[term]; ok {
|
||||
for i, ssa := range subs {
|
||||
if sub := emitAcc(childAccs[i], ssa); sub != nil {
|
||||
b.SubAggregations = append(b.SubAggregations, sub)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
childBuckets = append(childBuckets, b)
|
||||
}
|
||||
if sz := int(sa.GetSize()); sz > 0 && len(childBuckets) > sz {
|
||||
childBuckets = childBuckets[:sz]
|
||||
}
|
||||
return &searchService.AggregationResult{
|
||||
Field: sa.GetField(),
|
||||
Buckets: childBuckets,
|
||||
}
|
||||
}
|
||||
|
||||
// attachSubAggregations folds the matched hits into nested aggregation results
|
||||
// per parent bucket, via a single hit walk dispatched through the accumulator tree.
|
||||
func attachSubAggregations(res *bleve.SearchResult, parentField string, subAggs []*searchService.AggregationOption, buckets []*searchService.Bucket) {
|
||||
bucketByKey := make(map[string]*searchService.Bucket, len(buckets))
|
||||
for _, b := range buckets {
|
||||
bucketByKey[b.GetKey()] = b
|
||||
}
|
||||
|
||||
perParent := make(map[string][]*subAcc, len(buckets))
|
||||
for _, b := range buckets {
|
||||
accs := make([]*subAcc, len(subAggs))
|
||||
for i, sa := range subAggs {
|
||||
accs[i] = newSubAcc(sa)
|
||||
}
|
||||
perParent[b.GetKey()] = accs
|
||||
}
|
||||
|
||||
for _, hit := range res.Hits {
|
||||
parentVal, ok := hit.Fields[parentField].(string)
|
||||
if !ok || parentVal == "" {
|
||||
continue
|
||||
}
|
||||
accs, ok := perParent[parentVal]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for i, sa := range subAggs {
|
||||
accumulateHit(accs[i], sa, hit)
|
||||
}
|
||||
}
|
||||
|
||||
for key, accs := range perParent {
|
||||
b := bucketByKey[key]
|
||||
for i, sa := range subAggs {
|
||||
if r := emitAcc(accs[i], sa); r != nil {
|
||||
b.SubAggregations = append(b.SubAggregations, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// numericFieldValue coerces a bleve stored value to float64 (also accepts
|
||||
// string forms).
|
||||
func numericFieldValue(raw interface{}) (float64, bool) {
|
||||
switch v := raw.(type) {
|
||||
case float64:
|
||||
return v, true
|
||||
case int64:
|
||||
return float64(v), true
|
||||
case int32:
|
||||
return float64(v), true
|
||||
case string:
|
||||
f, err := strconv.ParseFloat(v, 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return f, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func aggregationRanges(agg *searchService.AggregationOption) []*searchService.BucketRange {
|
||||
bd := agg.GetBucketDefinition()
|
||||
if bd == nil {
|
||||
return nil
|
||||
}
|
||||
return bd.GetRanges()
|
||||
}
|
||||
|
||||
// rangeBucketKey formats a range as "from-to" for stable merge keys; open sides
|
||||
// render as "-N" or "N-".
|
||||
func rangeBucketKey(r *searchService.BucketRange) string {
|
||||
return r.GetFrom() + "-" + r.GetTo()
|
||||
}
|
||||
|
||||
func parseFloatPtr(s string) *float64 {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
v, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return &v
|
||||
}
|
||||
|
||||
func (b *Backend) DocCount() (uint64, error) {
|
||||
return b.index.DocCount()
|
||||
}
|
||||
|
||||
@@ -34,6 +34,10 @@ func ParseConfig(cfg *config.Config) error {
|
||||
}
|
||||
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.Events.Disabled && cfg.GRPC.Disabled {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
}
|
||||
|
||||
if cfg.TokenManager.JWTSecret == "" {
|
||||
return shared.MissingJWTTokenError(cfg.Service.Name)
|
||||
}
|
||||
|
||||
@@ -82,6 +82,9 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
|
||||
if err != nil {
|
||||
return doc, err
|
||||
}
|
||||
if len(metas) == 0 {
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
for _, meta := range metas {
|
||||
title, err := getFirstValue(meta, "dc:title")
|
||||
@@ -98,40 +101,21 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
|
||||
} else if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil {
|
||||
doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content))
|
||||
}
|
||||
|
||||
// keep facets from earlier entries, an embedded resource's meta
|
||||
// (e.g. cover art) must not reset them
|
||||
if v := t.getLocation(meta); v != nil {
|
||||
doc.Location = v
|
||||
}
|
||||
if v := t.getImage(meta); v != nil {
|
||||
doc.Image = v
|
||||
}
|
||||
if v := t.getPhoto(meta); v != nil {
|
||||
doc.Photo = v
|
||||
}
|
||||
if v := t.getAudio(meta); v != nil {
|
||||
doc.Audio = v
|
||||
}
|
||||
if v := t.getLivePhoto(meta); v != nil {
|
||||
doc.LivePhoto = v
|
||||
}
|
||||
}
|
||||
|
||||
if len(metas) > 0 {
|
||||
// the video facet says the file is a video, so it comes from the file
|
||||
// itself: the clip tika extracts from a motion photo must not make its
|
||||
// image look like one
|
||||
doc.Video = t.getVideo(metas[0])
|
||||
}
|
||||
// facets describe the file itself, not its embedded parts (cover art, clips)
|
||||
m0 := metas[0]
|
||||
doc.Location = t.getLocation(m0)
|
||||
doc.Image = t.getImage(m0)
|
||||
doc.Photo = t.getPhoto(m0)
|
||||
doc.Audio = t.getAudio(m0)
|
||||
doc.LivePhoto = t.getLivePhoto(m0)
|
||||
doc.Video = t.getVideo(m0)
|
||||
|
||||
// a motion photo is the xmp on the file itself plus the video tika extracted
|
||||
// from it. The xmp alone proves nothing: a share can keep it and strip the
|
||||
// appended video.
|
||||
if len(metas) > 0 {
|
||||
if i := slices.IndexFunc(metas[1:], isVideo); i >= 0 {
|
||||
doc.MotionPhoto = t.getMotionPhoto(metas[0], metas[i+1])
|
||||
}
|
||||
// a motion photo is the file's own xmp plus the video tika extracted from
|
||||
// it; the xmp alone proves nothing, a share can strip the appended clip
|
||||
if i := slices.IndexFunc(metas[1:], isVideo); i >= 0 {
|
||||
doc.MotionPhoto = t.getMotionPhoto(m0, metas[i+1])
|
||||
}
|
||||
|
||||
if langCode := t.detectLanguage(ctx, doc.Content); langCode != "" && t.CleanStopWords {
|
||||
|
||||
@@ -170,7 +170,9 @@ var _ = Describe("Tika", func() {
|
||||
Expect(doc.Content).To(Equal("body test stop words!!!"))
|
||||
})
|
||||
|
||||
It("keeps the audio facet when an embedded resource follows", func() {
|
||||
It("takes facets from the main document, not from an embedded resource", func() {
|
||||
// metas[0] is the file (audio), metas[1] its embedded cover art. The
|
||||
// cover must not give the track an image facet.
|
||||
fullResponse = `[{"Content-Type": "audio/mpeg", "dc:title": "Sucker", "tk:content": "lyrics"}, {"Content-Type": "image/jpeg", "tiff:ImageWidth": "500"}]`
|
||||
|
||||
doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
|
||||
@@ -180,7 +182,7 @@ var _ = Describe("Tika", func() {
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(doc.Audio).ToNot(BeNil())
|
||||
Expect(doc.Audio.Title).To(Equal(libregraph.PtrString("Sucker")))
|
||||
Expect(doc.Image).ToNot(BeNil())
|
||||
Expect(doc.Image).To(BeNil())
|
||||
})
|
||||
|
||||
It("adds no audio facet to non-audio documents", func() {
|
||||
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/aggs"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
@@ -73,7 +72,7 @@ func NewBackend(ctx context.Context, name string, client *opensearchgoAPI.Client
|
||||
}
|
||||
|
||||
func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
|
||||
boolQuery, err := convert.KQLToOpenSearchBoolQueryWithFilters(sir.Query, sir.GetAggregationFilters())
|
||||
boolQuery, err := convert.KQLToOpenSearchBoolQuery(sir.Query)
|
||||
switch {
|
||||
case kql.IsValidationError(err):
|
||||
return nil, errtypes.BadRequest(err.Error())
|
||||
@@ -123,31 +122,6 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
|
||||
searchParams.Size = conversions.ToPointer(int(sir.PageSize))
|
||||
}
|
||||
|
||||
builtAggs, err := aggs.Build(sir.GetAggregations())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Sort natively in the index; the service layer re-establishes this order
|
||||
// when merging matches across spaces. Score sorting (the default) stays in
|
||||
// place when no order_by is given. Missing values sort last in both
|
||||
// directions, matching the cross-space merge.
|
||||
var sortClause []map[string]any
|
||||
if orderBy := sir.GetOrderBy(); len(orderBy) > 0 {
|
||||
sortClause = make([]map[string]any, 0, len(orderBy)+1)
|
||||
for _, sp := range orderBy {
|
||||
field, ok := search.SortIndexField(sp.GetName())
|
||||
if !ok {
|
||||
return nil, errtypes.BadRequest(fmt.Sprintf("field %q is not sortable", sp.GetName()))
|
||||
}
|
||||
order := "asc"
|
||||
if sp.GetIsDescending() {
|
||||
order = "desc"
|
||||
}
|
||||
sortClause = append(sortClause, map[string]any{field: map[string]any{"order": order, "missing": "_last"}})
|
||||
}
|
||||
sortClause = append(sortClause, map[string]any{"_score": map[string]any{"order": "desc"}})
|
||||
}
|
||||
|
||||
req, err := osu.BuildSearchReq(&opensearchgoAPI.SearchReq{
|
||||
Indices: []string{b.index},
|
||||
Params: searchParams,
|
||||
@@ -166,8 +140,6 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
|
||||
},
|
||||
},
|
||||
},
|
||||
Aggs: builtAggs,
|
||||
Sort: sortClause,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
@@ -190,15 +162,9 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
|
||||
matches = append(matches, match)
|
||||
}
|
||||
|
||||
aggResults, err := aggs.Parse(resp.Aggregations, sir.GetAggregations())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse aggregations: %w", err)
|
||||
}
|
||||
|
||||
return &searchService.SearchIndexResponse{
|
||||
Matches: matches,
|
||||
TotalMatches: int32(totalMatches),
|
||||
Aggregations: aggResults,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -74,6 +74,7 @@ func buildResourceMapping() ([]byte, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
index := map[string]any{
|
||||
"settings": map[string]any{
|
||||
"number_of_shards": "1",
|
||||
|
||||
@@ -1,315 +0,0 @@
|
||||
// Package aggs translates proto aggregation options into the OpenSearch
|
||||
// aggregation DSL and parses the response. Internal subpackage so its unit
|
||||
// tests skip the parent package's Docker OpenSearch container.
|
||||
package aggs
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
)
|
||||
|
||||
// DefaultFacetSize matches the bleve backend: pull a generous bucket count per
|
||||
// space, the service layer trims to top N after cross-space merge.
|
||||
const DefaultFacetSize = 1000
|
||||
|
||||
// Build translates AggregationOptions into the OpenSearch aggregation DSL
|
||||
// (terms, range, date_range, metric, nested). Entries get an index-derived
|
||||
// name so repeated aggs on one field don't collide. A range bound that is
|
||||
// neither a number nor a date is an error.
|
||||
func Build(opts []*searchsvc.AggregationOption) (map[string]any, error) {
|
||||
return buildLevel(opts, "a")
|
||||
}
|
||||
|
||||
func buildLevel(opts []*searchsvc.AggregationOption, prefix string) (map[string]any, error) {
|
||||
if len(opts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
aggs := map[string]any{}
|
||||
for i, opt := range opts {
|
||||
name := fmt.Sprintf("%s_%d", prefix, i)
|
||||
entry, err := buildOne(opt, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if entry != nil {
|
||||
aggs[name] = entry
|
||||
}
|
||||
}
|
||||
if len(aggs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return aggs, nil
|
||||
}
|
||||
|
||||
func buildOne(opt *searchsvc.AggregationOption, name string) (map[string]any, error) {
|
||||
field := opt.GetField()
|
||||
if mk := opt.GetMetricKind(); mk != searchsvc.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
return buildMetric(field, mk), nil
|
||||
}
|
||||
var entry map[string]any
|
||||
if ranges := rangesOf(opt); len(ranges) > 0 {
|
||||
built, kind, err := buildRanges(field, ranges)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry = map[string]any{
|
||||
kind: map[string]any{
|
||||
"field": field,
|
||||
"ranges": built,
|
||||
},
|
||||
}
|
||||
} else {
|
||||
size := int(opt.GetSize())
|
||||
if size <= 0 {
|
||||
size = DefaultFacetSize
|
||||
}
|
||||
entry = map[string]any{
|
||||
"terms": map[string]any{
|
||||
"field": field,
|
||||
"size": size,
|
||||
},
|
||||
}
|
||||
}
|
||||
if subs := opt.GetSubAggregations(); len(subs) > 0 {
|
||||
nested, err := buildLevel(subs, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if nested != nil {
|
||||
entry["aggs"] = nested
|
||||
}
|
||||
}
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
// buildMetric emits the sum/min/max metric. AVG uses a stats agg to transport
|
||||
// (sum, count) for the cross-space merge; the service layer collapses to the average.
|
||||
func buildMetric(field string, kind searchsvc.MetricKind) map[string]any {
|
||||
switch kind {
|
||||
case searchsvc.MetricKind_METRIC_KIND_SUM:
|
||||
return map[string]any{"sum": map[string]any{"field": field}}
|
||||
case searchsvc.MetricKind_METRIC_KIND_MIN:
|
||||
return map[string]any{"min": map[string]any{"field": field}}
|
||||
case searchsvc.MetricKind_METRIC_KIND_MAX:
|
||||
return map[string]any{"max": map[string]any{"field": field}}
|
||||
case searchsvc.MetricKind_METRIC_KIND_AVG:
|
||||
return map[string]any{"stats": map[string]any{"field": field}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rangesOf(opt *searchsvc.AggregationOption) []*searchsvc.BucketRange {
|
||||
bd := opt.GetBucketDefinition()
|
||||
if bd == nil {
|
||||
return nil
|
||||
}
|
||||
return bd.GetRanges()
|
||||
}
|
||||
|
||||
// rangeTimeLayouts mirrors the bleve backend's accepted date bound formats.
|
||||
var rangeTimeLayouts = []string{time.RFC3339, "2006-01-02"}
|
||||
|
||||
func boundIsDate(s string) bool {
|
||||
for _, layout := range rangeTimeLayouts {
|
||||
if _, err := time.Parse(layout, s); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// buildRanges renders the ranges and decides between the numeric "range" and
|
||||
// the "date_range" aggregation: one date-looking bound switches the whole
|
||||
// aggregation to date mode, exactly like the bleve backend.
|
||||
func buildRanges(field string, ranges []*searchsvc.BucketRange) ([]map[string]any, string, error) {
|
||||
dates := false
|
||||
for _, r := range ranges {
|
||||
for _, s := range []string{r.GetFrom(), r.GetTo()} {
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := strconv.ParseFloat(s, 64); err != nil {
|
||||
dates = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]map[string]any, 0, len(ranges))
|
||||
for _, r := range ranges {
|
||||
entry := map[string]any{
|
||||
"key": RangeKey(r),
|
||||
}
|
||||
for side, s := range map[string]string{"from": r.GetFrom(), "to": r.GetTo()} {
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if dates {
|
||||
if !boundIsDate(s) {
|
||||
return nil, "", fmt.Errorf("invalid date range bound %q on field %q", s, field)
|
||||
}
|
||||
entry[side] = s
|
||||
continue
|
||||
}
|
||||
v, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("invalid range bound %q on field %q", s, field)
|
||||
}
|
||||
entry[side] = v
|
||||
}
|
||||
out = append(out, entry)
|
||||
}
|
||||
|
||||
kind := "range"
|
||||
if dates {
|
||||
kind = "date_range"
|
||||
}
|
||||
return out, kind, nil
|
||||
}
|
||||
|
||||
// RangeKey mirrors the bleve backend so cross-space merging keys match.
|
||||
func RangeKey(r *searchsvc.BucketRange) string {
|
||||
return r.GetFrom() + "-" + r.GetTo()
|
||||
}
|
||||
|
||||
// Parse converts the response aggregations block into proto results, preserving
|
||||
// request order and recursing into sub-aggregations. Empty input yields
|
||||
// (nil, nil); invalid JSON yields an error.
|
||||
func Parse(raw json.RawMessage, opts []*searchsvc.AggregationOption) ([]*searchsvc.AggregationResult, error) {
|
||||
if len(raw) == 0 || len(opts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
node, err := parseNode(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseLevel(node, opts, "a"), nil
|
||||
}
|
||||
|
||||
// aggNode is a lazily-decoded cursor over one level of the aggs response.
|
||||
type aggNode map[string]json.RawMessage
|
||||
|
||||
func parseNode(raw json.RawMessage) (aggNode, error) {
|
||||
var m aggNode
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("decode opensearch aggregations: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func parseLevel(node aggNode, opts []*searchsvc.AggregationOption, prefix string) []*searchsvc.AggregationResult {
|
||||
out := make([]*searchsvc.AggregationResult, 0, len(opts))
|
||||
for i, opt := range opts {
|
||||
name := fmt.Sprintf("%s_%d", prefix, i)
|
||||
raw, ok := node[name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if res := parseOne(raw, opt, name); res != nil {
|
||||
out = append(out, res)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseOne(raw json.RawMessage, opt *searchsvc.AggregationOption, name string) *searchsvc.AggregationResult {
|
||||
field := opt.GetField()
|
||||
if mk := opt.GetMetricKind(); mk != searchsvc.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
return parseMetric(raw, field, mk)
|
||||
}
|
||||
var body struct {
|
||||
Buckets []json.RawMessage `json:"buckets"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &body); err != nil {
|
||||
return nil
|
||||
}
|
||||
buckets := make([]*searchsvc.Bucket, 0, len(body.Buckets))
|
||||
for _, b := range body.Buckets {
|
||||
if bucket := parseBucket(b, opt.GetSubAggregations(), name); bucket != nil {
|
||||
buckets = append(buckets, bucket)
|
||||
}
|
||||
}
|
||||
return &searchsvc.AggregationResult{
|
||||
Field: field,
|
||||
Buckets: buckets,
|
||||
}
|
||||
}
|
||||
|
||||
func parseBucket(raw json.RawMessage, subs []*searchsvc.AggregationOption, prefix string) *searchsvc.Bucket {
|
||||
var head struct {
|
||||
Key any `json:"key"`
|
||||
DocCount int64 `json:"doc_count"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &head); err != nil {
|
||||
return nil
|
||||
}
|
||||
b := &searchsvc.Bucket{
|
||||
Key: bucketKeyToString(head.Key),
|
||||
Count: head.DocCount,
|
||||
}
|
||||
if len(subs) > 0 {
|
||||
node, err := parseNode(raw)
|
||||
if err == nil {
|
||||
b.SubAggregations = parseLevel(node, subs, prefix)
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func parseMetric(raw json.RawMessage, field string, kind searchsvc.MetricKind) *searchsvc.AggregationResult {
|
||||
switch kind {
|
||||
case searchsvc.MetricKind_METRIC_KIND_SUM,
|
||||
searchsvc.MetricKind_METRIC_KIND_MIN,
|
||||
searchsvc.MetricKind_METRIC_KIND_MAX:
|
||||
var body struct {
|
||||
Value *float64 `json:"value"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &body); err != nil {
|
||||
return nil
|
||||
}
|
||||
res := &searchsvc.AggregationResult{
|
||||
Field: field,
|
||||
MetricKind: kind,
|
||||
}
|
||||
if body.Value != nil {
|
||||
res.Value = *body.Value
|
||||
}
|
||||
return res
|
||||
case searchsvc.MetricKind_METRIC_KIND_AVG:
|
||||
var body struct {
|
||||
Sum float64 `json:"sum"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &body); err != nil {
|
||||
return nil
|
||||
}
|
||||
return &searchsvc.AggregationResult{
|
||||
Field: field,
|
||||
Sum: body.Sum,
|
||||
Count: body.Count,
|
||||
MetricKind: kind,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bucketKeyToString normalises a response key to a string (terms are strings,
|
||||
// ranges use our "from-to" key, numeric terms come back as JSON numbers).
|
||||
func bucketKeyToString(v any) string {
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
return x
|
||||
case float64:
|
||||
// format without trailing zeros so keys match filter values
|
||||
return strconv.FormatFloat(x, 'f', -1, 64)
|
||||
case bool:
|
||||
return strconv.FormatBool(x)
|
||||
case nil:
|
||||
return ""
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -1,268 +0,0 @@
|
||||
package aggs_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/aggs"
|
||||
)
|
||||
|
||||
var _ = Describe("Build", func() {
|
||||
build := func(opts []*searchsvc.AggregationOption) map[string]any {
|
||||
res, err := aggs.Build(opts)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return res
|
||||
}
|
||||
|
||||
It("builds a terms aggregation", func() {
|
||||
res := build([]*searchsvc.AggregationOption{
|
||||
{Field: "audio.artist", Size: 10},
|
||||
})
|
||||
Expect(res).ToNot(BeNil())
|
||||
entry, ok := res["a_0"].(map[string]any)
|
||||
Expect(ok).To(BeTrue())
|
||||
terms, ok := entry["terms"].(map[string]any)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(terms["field"]).To(Equal("audio.artist"))
|
||||
Expect(terms["size"]).To(Equal(10))
|
||||
})
|
||||
|
||||
It("builds a date_range aggregation for date bounds", func() {
|
||||
res := build([]*searchsvc.AggregationOption{{
|
||||
Field: "photo.takenDateTime",
|
||||
BucketDefinition: &searchsvc.BucketDefinition{
|
||||
Ranges: []*searchsvc.BucketRange{
|
||||
{From: "2018-08-01", To: "2018-09-01"},
|
||||
{From: "2018-08-11T00:00:00Z"},
|
||||
},
|
||||
},
|
||||
}})
|
||||
r := res["a_0"].(map[string]any)["date_range"].(map[string]any)
|
||||
Expect(r["field"]).To(Equal("photo.takenDateTime"))
|
||||
ranges := r["ranges"].([]map[string]any)
|
||||
Expect(ranges).To(HaveLen(2))
|
||||
Expect(ranges[0]).To(SatisfyAll(
|
||||
HaveKeyWithValue("key", "2018-08-01-2018-09-01"),
|
||||
HaveKeyWithValue("from", "2018-08-01"),
|
||||
HaveKeyWithValue("to", "2018-09-01"),
|
||||
))
|
||||
Expect(ranges[1]).To(HaveKeyWithValue("from", "2018-08-11T00:00:00Z"))
|
||||
Expect(ranges[1]).ToNot(HaveKey("to"))
|
||||
})
|
||||
|
||||
It("rejects a bound that is neither number nor date", func() {
|
||||
_, err := aggs.Build([]*searchsvc.AggregationOption{{
|
||||
Field: "photo.takenDateTime",
|
||||
BucketDefinition: &searchsvc.BucketDefinition{
|
||||
Ranges: []*searchsvc.BucketRange{
|
||||
{From: "2018-08-11T00:00:00Z", To: "not-a-date"},
|
||||
},
|
||||
},
|
||||
}})
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
It("builds a range aggregation with open-ended bounds", func() {
|
||||
res := build([]*searchsvc.AggregationOption{{
|
||||
Field: "audio.year",
|
||||
BucketDefinition: &searchsvc.BucketDefinition{
|
||||
Ranges: []*searchsvc.BucketRange{
|
||||
{From: "1970", To: "1980"},
|
||||
{To: "1970"},
|
||||
{From: "2020"},
|
||||
},
|
||||
},
|
||||
}})
|
||||
r := res["a_0"].(map[string]any)["range"].(map[string]any)
|
||||
Expect(r["field"]).To(Equal("audio.year"))
|
||||
ranges := r["ranges"].([]map[string]any)
|
||||
Expect(ranges).To(HaveLen(3))
|
||||
Expect(ranges[0]).To(SatisfyAll(
|
||||
HaveKeyWithValue("key", "1970-1980"),
|
||||
HaveKeyWithValue("from", 1970.0),
|
||||
HaveKeyWithValue("to", 1980.0),
|
||||
))
|
||||
Expect(ranges[1]).ToNot(HaveKey("from")) // open lower bound
|
||||
Expect(ranges[2]).ToNot(HaveKey("to")) // open upper bound
|
||||
})
|
||||
|
||||
DescribeTable("builds single-value metric aggregations",
|
||||
func(kind searchsvc.MetricKind, esKind string) {
|
||||
res := build([]*searchsvc.AggregationOption{
|
||||
{Field: "audio.duration", MetricKind: kind},
|
||||
})
|
||||
body, ok := res["a_0"].(map[string]any)[esKind].(map[string]any)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(body["field"]).To(Equal("audio.duration"))
|
||||
},
|
||||
Entry("sum", searchsvc.MetricKind_METRIC_KIND_SUM, "sum"),
|
||||
Entry("min", searchsvc.MetricKind_METRIC_KIND_MIN, "min"),
|
||||
Entry("max", searchsvc.MetricKind_METRIC_KIND_MAX, "max"),
|
||||
)
|
||||
|
||||
It("uses a stats aggregation for AVG", func() {
|
||||
res := build([]*searchsvc.AggregationOption{
|
||||
{Field: "audio.duration", MetricKind: searchsvc.MetricKind_METRIC_KIND_AVG},
|
||||
})
|
||||
stats, ok := res["a_0"].(map[string]any)["stats"].(map[string]any)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(stats["field"]).To(Equal("audio.duration"))
|
||||
})
|
||||
|
||||
It("nests sub-aggregations under their parent bucket", func() {
|
||||
res := build([]*searchsvc.AggregationOption{{
|
||||
Field: "audio.artist", Size: 5,
|
||||
SubAggregations: []*searchsvc.AggregationOption{{
|
||||
Field: "audio.album", Size: 7,
|
||||
SubAggregations: []*searchsvc.AggregationOption{
|
||||
{Field: "audio.duration", MetricKind: searchsvc.MetricKind_METRIC_KIND_SUM},
|
||||
},
|
||||
}},
|
||||
}})
|
||||
album := res["a_0"].(map[string]any)["aggs"].(map[string]any)["a_0_0"].(map[string]any)
|
||||
albumTerms := album["terms"].(map[string]any)
|
||||
Expect(albumTerms["field"]).To(Equal("audio.album"))
|
||||
Expect(albumTerms["size"]).To(Equal(7))
|
||||
metric := album["aggs"].(map[string]any)["a_0_0_0"].(map[string]any)
|
||||
Expect(metric["sum"].(map[string]any)["field"]).To(Equal("audio.duration"))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("Parse", func() {
|
||||
It("parses flat term and range buckets, stringifying numeric keys", func() {
|
||||
raw := json.RawMessage(`{
|
||||
"a_0": {"buckets": [
|
||||
{"key": "Pink Floyd", "doc_count": 42},
|
||||
{"key": "Motörhead", "doc_count": 35}
|
||||
]},
|
||||
"a_1": {"buckets": [
|
||||
{"key": "1970-1980", "from": 1970.0, "to": 1980.0, "doc_count": 12}
|
||||
]},
|
||||
"a_2": {"buckets": [
|
||||
{"key": 9, "doc_count": 3}
|
||||
]}
|
||||
}`)
|
||||
out, err := aggs.Parse(raw, []*searchsvc.AggregationOption{
|
||||
{Field: "audio.artist"},
|
||||
{Field: "audio.year"},
|
||||
{Field: "audio.track"},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(out).To(HaveLen(3))
|
||||
|
||||
Expect(out[0].Field).To(Equal("audio.artist"))
|
||||
Expect(out[0].Buckets).To(HaveLen(2))
|
||||
Expect(out[0].Buckets[0].Key).To(Equal("Pink Floyd"))
|
||||
Expect(out[0].Buckets[0].Count).To(Equal(int64(42)))
|
||||
|
||||
Expect(out[1].Buckets[0].Key).To(Equal("1970-1980"))
|
||||
Expect(out[1].Buckets[0].Count).To(Equal(int64(12)))
|
||||
|
||||
// numeric term key stringified without trailing zeros
|
||||
Expect(out[2].Buckets[0].Key).To(Equal("9"))
|
||||
})
|
||||
|
||||
It("parses nested buckets carrying a metric", func() {
|
||||
raw := json.RawMessage(`{
|
||||
"a_0": {"buckets": [{
|
||||
"key": "Iron Maiden", "doc_count": 300,
|
||||
"a_0_0": {"buckets": [
|
||||
{"key": "The Number of the Beast", "doc_count": 8, "a_0_0_0": {"value": 2756000.0}},
|
||||
{"key": "Powerslave", "doc_count": 8, "a_0_0_0": {"value": 3061000.0}}
|
||||
]}
|
||||
}]}
|
||||
}`)
|
||||
out, err := aggs.Parse(raw, []*searchsvc.AggregationOption{{
|
||||
Field: "audio.artist",
|
||||
SubAggregations: []*searchsvc.AggregationOption{{
|
||||
Field: "audio.album",
|
||||
SubAggregations: []*searchsvc.AggregationOption{
|
||||
{Field: "audio.duration", MetricKind: searchsvc.MetricKind_METRIC_KIND_SUM},
|
||||
},
|
||||
}},
|
||||
}})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(out).To(HaveLen(1))
|
||||
Expect(out[0].Field).To(Equal("audio.artist"))
|
||||
Expect(out[0].Buckets).To(HaveLen(1))
|
||||
|
||||
artistBucket := out[0].Buckets[0]
|
||||
Expect(artistBucket.Key).To(Equal("Iron Maiden"))
|
||||
Expect(artistBucket.Count).To(Equal(int64(300)))
|
||||
Expect(artistBucket.SubAggregations).To(HaveLen(1))
|
||||
|
||||
albumAgg := artistBucket.SubAggregations[0]
|
||||
Expect(albumAgg.Field).To(Equal("audio.album"))
|
||||
Expect(albumAgg.Buckets).To(HaveLen(2))
|
||||
|
||||
nob := albumAgg.Buckets[0]
|
||||
Expect(nob.Key).To(Equal("The Number of the Beast"))
|
||||
Expect(nob.Count).To(Equal(int64(8)))
|
||||
Expect(nob.SubAggregations).To(HaveLen(1))
|
||||
|
||||
metric := nob.SubAggregations[0]
|
||||
Expect(metric.Field).To(Equal("audio.duration"))
|
||||
Expect(metric.MetricKind).To(Equal(searchsvc.MetricKind_METRIC_KIND_SUM))
|
||||
Expect(metric.Value).To(Equal(2756000.0))
|
||||
})
|
||||
|
||||
DescribeTable("parses single-value metrics",
|
||||
func(kind searchsvc.MetricKind, value float64) {
|
||||
raw := json.RawMessage(fmt.Sprintf(`{"a_0": {"value": %g}}`, value))
|
||||
out, err := aggs.Parse(raw, []*searchsvc.AggregationOption{
|
||||
{Field: "audio.duration", MetricKind: kind},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(out).To(HaveLen(1))
|
||||
Expect(out[0].MetricKind).To(Equal(kind))
|
||||
Expect(out[0].Value).To(Equal(value))
|
||||
},
|
||||
Entry("sum", searchsvc.MetricKind_METRIC_KIND_SUM, 1234.5),
|
||||
Entry("min", searchsvc.MetricKind_METRIC_KIND_MIN, 10.0),
|
||||
Entry("max", searchsvc.MetricKind_METRIC_KIND_MAX, 99.0),
|
||||
)
|
||||
|
||||
It("decodes a null metric value to zero", func() {
|
||||
// OpenSearch returns value: null when a metric has no matching docs.
|
||||
out, err := aggs.Parse(json.RawMessage(`{"a_0": {"value": null}}`),
|
||||
[]*searchsvc.AggregationOption{{Field: "audio.duration", MetricKind: searchsvc.MetricKind_METRIC_KIND_SUM}})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(out).To(HaveLen(1))
|
||||
Expect(out[0].Value).To(BeZero())
|
||||
Expect(out[0].MetricKind).To(Equal(searchsvc.MetricKind_METRIC_KIND_SUM))
|
||||
})
|
||||
|
||||
It("carries avg transport (sum + count) from a stats response", func() {
|
||||
raw := json.RawMessage(`{
|
||||
"a_0": {"count": 100, "min": 30000.0, "max": 500000.0, "avg": 245000.0, "sum": 24500000.0}
|
||||
}`)
|
||||
out, err := aggs.Parse(raw, []*searchsvc.AggregationOption{
|
||||
{Field: "audio.duration", MetricKind: searchsvc.MetricKind_METRIC_KIND_AVG},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(out).To(HaveLen(1))
|
||||
Expect(out[0].MetricKind).To(Equal(searchsvc.MetricKind_METRIC_KIND_AVG))
|
||||
Expect(out[0].Sum).To(Equal(24500000.0))
|
||||
Expect(out[0].Count).To(Equal(int64(100)))
|
||||
})
|
||||
|
||||
It("returns nil for empty raw or empty options", func() {
|
||||
got, err := aggs.Parse(nil, []*searchsvc.AggregationOption{{Field: "x"}})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(BeNil())
|
||||
|
||||
got, err = aggs.Parse(json.RawMessage(`{}`), nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(BeNil())
|
||||
})
|
||||
|
||||
It("errors on malformed json and returns no result", func() {
|
||||
got, err := aggs.Parse(json.RawMessage(`not-json`), []*searchsvc.AggregationOption{{Field: "x"}})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(got).To(BeNil())
|
||||
})
|
||||
})
|
||||
@@ -13,19 +13,14 @@ var (
|
||||
)
|
||||
|
||||
func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
|
||||
return KQLToOpenSearchBoolQueryWithFilters(kqlQuery, nil)
|
||||
}
|
||||
|
||||
// KQLToOpenSearchBoolQueryWithFilters compiles the query together with decoded
|
||||
// aggregation filters, which are ANDed in as exact case-sensitive matches.
|
||||
func KQLToOpenSearchBoolQueryWithFilters(kqlQuery string, filters []string) (*osu.BoolQuery, error) {
|
||||
// shared lowering (field resolution, media-type expansion, value lowercasing)
|
||||
// plus the filters, forced to exact case-sensitive matches, ANDed in.
|
||||
kqlAst, err := query.MergeFilters(kql.Builder{}, kqlQuery, filters)
|
||||
kqlAst, err := kql.Builder{}.Build(kqlQuery)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// shared lowering: field resolution, media-type expansion, value lowercasing.
|
||||
kqlAst = query.Normalize(kqlAst, query.ResolveField)
|
||||
|
||||
builder, err := TranspileKQLToOpenSearch(kqlAst.Nodes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile query: %w", err)
|
||||
|
||||
@@ -95,8 +95,6 @@ func BuildSearchReq(req *opensearchgoAPI.SearchReq, q Builder, p ...SearchBodyPa
|
||||
|
||||
type SearchBodyParams struct {
|
||||
Highlight *BodyParamHighlight `json:"highlight,omitempty"`
|
||||
Aggs map[string]any `json:"aggs,omitempty"`
|
||||
Sort []map[string]any `json:"sort,omitempty"`
|
||||
}
|
||||
|
||||
//----------------------------------------------------------------------------//
|
||||
|
||||
@@ -693,33 +693,3 @@ Fixtures:
|
||||
| METADATA-01 | `*song*` reads `Audio` | all 16 fields unchanged | all 16 fields unchanged | all 16 fields unchanged | ✅ |
|
||||
| METADATA-02 | `*team*` reads `Location` | all 3 fields unchanged | all 3 fields unchanged | all 3 fields unchanged | ✅ |
|
||||
| METADATA-03 | `*team*` reads `Audio` | none | none | none | ✅ |
|
||||
|
||||
## Aggregations
|
||||
|
||||
### aggregations
|
||||
|
||||
Fixtures:
|
||||
|
||||
- `a.mp3`, MimeType = audio/mpeg
|
||||
- `b.mp3`, MimeType = audio/mpeg
|
||||
- `c.mp3`, MimeType = audio/mpeg
|
||||
- `d.mp3`, MimeType = audio/mpeg
|
||||
- `e.mp3`, MimeType = audio/mpeg
|
||||
- `f.mp3`, MimeType = audio/mpeg
|
||||
- `g.mp3`, MimeType = audio/mpeg
|
||||
- `a.jpg`, MimeType = image/jpeg
|
||||
- `b.jpg`, MimeType = image/jpeg
|
||||
- `c.jpg`, MimeType = image/jpeg
|
||||
- `d.jpg`, MimeType = image/jpeg
|
||||
|
||||
| Case | Query | expected | bleve | OpenSearch | same? |
|
||||
|---|---|---|---|---|---|
|
||||
| AGG-01 | `mediatype:audio` reads `term buckets on audio.artist` | audio.arti... Floyd=2, audio.arti...¶rhead=3 | audio.arti... Floyd=2, audio.arti...¶rhead=3 | audio.arti... Floyd=2, audio.arti...¶rhead=3 | ✅ |
|
||||
| AGG-02 | `mediatype:audio` reads `no aggregations requested` | no match | no match | no match | ✅ |
|
||||
| AGG-03 | `mediatype:audio` reads `artist and album buckets in one request` | audio.albu...Spades=1, audio.album Bomber=2, audio.album The Wall=2, audio.arti... Floyd=2, audio.arti...¶rhead=3 | audio.albu...Spades=1, audio.album Bomber=2, audio.album The Wall=2, audio.arti... Floyd=2, audio.arti...¶rhead=3 | audio.albu...Spades=1, audio.album Bomber=2, audio.album The Wall=2, audio.arti... Floyd=2, audio.arti...¶rhead=3 | ✅ |
|
||||
| AGG-04 | `mediatype:audio` reads `audio.year buckets per decade` | audio.year 1970-1980=2, audio.year 1980-1990=1, audio.year 1990-2000=1, audio.year 2000-2010=3 | audio.year 1970-1980=2, audio.year 1980-1990=1, audio.year 1990-2000=1, audio.year 2000-2010=3 | audio.year 1970-1980=2, audio.year 1980-1990=1, audio.year 1990-2000=1, audio.year 2000-2010=3 | ✅ |
|
||||
| AGG-05 | `mediatype:audio` reads `open-ended audio.year ranges` | audio.year -1990=3, audio.year 2000-=3 | audio.year -1990=3, audio.year 2000-=3 | audio.year -1990=3, audio.year 2000-=3 | ✅ |
|
||||
| AGG-06 | `mediatype:audio` reads `top-level metrics on audio.year` | audio.year max=2009, audio.year min=1971, audio.year sum=13942, audio.year... count=7 | audio.year max=2009, audio.year min=1971, audio.year sum=13942, audio.year... count=7 | audio.year max=2009, audio.year min=1971, audio.year sum=13942, audio.year... count=7 | ✅ |
|
||||
| AGG-07 | `mediatype:image` reads `photo.takenDateTime buckets per date range` | photo.take...-01-01=1, photo.take...-09-01=2, photo.take...00:00Z=2 | photo.take...-01-01=1, photo.take...-09-01=2, photo.take...00:00Z=2 | photo.take...-01-01=1, photo.take...-09-01=2, photo.take...00:00Z=2 | ✅ |
|
||||
| AGG-08 | `mediatype:image` reads `open-ended date ranges` | photo.take...-01-01=3, photo.take...01-01-=1 | photo.take...-01-01=3, photo.take...01-01-=1 | photo.take...-01-01=3, photo.take...01-01-=1 | ✅ |
|
||||
| AGG-09 | `mediatype:image` reads `malformed date range bound` | error | error | error | ✅ |
|
||||
@@ -1,207 +0,0 @@
|
||||
package parity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// aggCase is one aggregation request both engines have to answer alike. The
|
||||
// answer is rendered to strings (one per non-empty bucket or metric), so the
|
||||
// matrix machinery can carry it like any query answer.
|
||||
type aggCase struct {
|
||||
id int
|
||||
query string
|
||||
aggs []*searchService.AggregationOption
|
||||
reads string
|
||||
want []string
|
||||
wantError bool
|
||||
engineOverrides map[string]override
|
||||
}
|
||||
|
||||
func (c aggCase) label() string { return fmt.Sprintf("AGG-%02d", c.id) }
|
||||
|
||||
func withYear(name string, year int32) search.Resource {
|
||||
return fixtureDoc(name, withMime("audio/mpeg"), withAudio(&libregraph.Audio{Year: libregraph.PtrInt32(year)}))
|
||||
}
|
||||
|
||||
func withTaken(name, taken string) search.Resource {
|
||||
t, err := time.Parse(time.RFC3339, taken)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return fixtureDoc(name, withMime("image/jpeg"), withPhoto(&libregraph.Photo{TakenDateTime: &t}))
|
||||
}
|
||||
|
||||
func song(name, artist, album string, year int32) search.Resource {
|
||||
return fixtureDoc(name, withMime("audio/mpeg"), withAudio(&libregraph.Audio{
|
||||
Artist: libregraph.PtrString(artist),
|
||||
Album: libregraph.PtrString(album),
|
||||
Year: libregraph.PtrInt32(year),
|
||||
}))
|
||||
}
|
||||
|
||||
func aggregationFixtures() []search.Resource {
|
||||
return []search.Resource{
|
||||
// years: 1971, 1975, 1982, 1999, 2001, 2005, 2009
|
||||
song("a.mp3", "Pink Floyd", "The Wall", 1971),
|
||||
song("b.mp3", "Pink Floyd", "The Wall", 1975),
|
||||
song("c.mp3", "Motörhead", "Bomber", 1982),
|
||||
song("d.mp3", "Motörhead", "Bomber", 1999),
|
||||
song("e.mp3", "Motörhead", "Ace of Spades", 2001),
|
||||
withYear("f.mp3", 2005),
|
||||
withYear("g.mp3", 2009),
|
||||
withTaken("a.jpg", "2018-08-11T09:15:00Z"),
|
||||
withTaken("b.jpg", "2018-08-11T19:42:00Z"),
|
||||
withTaken("c.jpg", "2018-09-01T12:00:00Z"),
|
||||
withTaken("d.jpg", "2021-08-11T08:00:00Z"),
|
||||
}
|
||||
}
|
||||
|
||||
func aggregationCases() []aggCase {
|
||||
ranges := func(rs ...*searchService.BucketRange) *searchService.BucketDefinition {
|
||||
return &searchService.BucketDefinition{Ranges: rs}
|
||||
}
|
||||
|
||||
return []aggCase{
|
||||
{id: 1, query: "mediatype:audio", reads: "term buckets on audio.artist",
|
||||
aggs: []*searchService.AggregationOption{{Field: "audio.artist", Size: 10}},
|
||||
want: []string{"audio.artist Pink Floyd=2", "audio.artist Motörhead=3"}},
|
||||
{id: 2, query: "mediatype:audio", reads: "no aggregations requested"},
|
||||
{id: 3, query: "mediatype:audio", reads: "artist and album buckets in one request",
|
||||
aggs: []*searchService.AggregationOption{{Field: "audio.artist"}, {Field: "audio.album"}},
|
||||
want: []string{
|
||||
"audio.artist Pink Floyd=2", "audio.artist Motörhead=3",
|
||||
"audio.album The Wall=2", "audio.album Bomber=2", "audio.album Ace of Spades=1",
|
||||
}},
|
||||
{id: 4, query: "mediatype:audio", reads: "audio.year buckets per decade",
|
||||
aggs: []*searchService.AggregationOption{{Field: "audio.year", BucketDefinition: ranges(
|
||||
&searchService.BucketRange{From: "1970", To: "1980"},
|
||||
&searchService.BucketRange{From: "1980", To: "1990"},
|
||||
&searchService.BucketRange{From: "1990", To: "2000"},
|
||||
&searchService.BucketRange{From: "2000", To: "2010"},
|
||||
)}},
|
||||
want: []string{"audio.year 1970-1980=2", "audio.year 1980-1990=1", "audio.year 1990-2000=1", "audio.year 2000-2010=3"}},
|
||||
{id: 5, query: "mediatype:audio", reads: "open-ended audio.year ranges",
|
||||
aggs: []*searchService.AggregationOption{{Field: "audio.year", BucketDefinition: ranges(
|
||||
&searchService.BucketRange{To: "1990"},
|
||||
&searchService.BucketRange{From: "2000"},
|
||||
)}},
|
||||
want: []string{"audio.year -1990=3", "audio.year 2000-=3"}},
|
||||
{id: 6, query: "mediatype:audio", reads: "top-level metrics on audio.year",
|
||||
aggs: []*searchService.AggregationOption{
|
||||
{Field: "audio.year", MetricKind: searchService.MetricKind_METRIC_KIND_SUM},
|
||||
{Field: "audio.year", MetricKind: searchService.MetricKind_METRIC_KIND_MIN},
|
||||
{Field: "audio.year", MetricKind: searchService.MetricKind_METRIC_KIND_MAX},
|
||||
{Field: "audio.year", MetricKind: searchService.MetricKind_METRIC_KIND_AVG},
|
||||
},
|
||||
want: []string{"audio.year sum=13942", "audio.year min=1971", "audio.year max=2009", "audio.year avg sum=13942 count=7"}},
|
||||
{id: 7, query: "mediatype:image", reads: "photo.takenDateTime buckets per date range",
|
||||
aggs: []*searchService.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: ranges(
|
||||
&searchService.BucketRange{From: "2018-08-11T00:00:00Z", To: "2018-08-12T00:00:00Z"},
|
||||
&searchService.BucketRange{From: "2018-08-01", To: "2018-09-01"},
|
||||
&searchService.BucketRange{From: "2021-01-01", To: "2022-01-01"},
|
||||
&searchService.BucketRange{From: "2023-01-01", To: "2024-01-01"},
|
||||
)}},
|
||||
want: []string{
|
||||
"photo.takenDateTime 2018-08-11T00:00:00Z-2018-08-12T00:00:00Z=2",
|
||||
"photo.takenDateTime 2018-08-01-2018-09-01=2",
|
||||
"photo.takenDateTime 2021-01-01-2022-01-01=1",
|
||||
}},
|
||||
{id: 8, query: "mediatype:image", reads: "open-ended date ranges",
|
||||
aggs: []*searchService.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: ranges(
|
||||
&searchService.BucketRange{To: "2019-01-01"},
|
||||
&searchService.BucketRange{From: "2019-01-01"},
|
||||
)}},
|
||||
want: []string{"photo.takenDateTime -2019-01-01=3", "photo.takenDateTime 2019-01-01-=1"}},
|
||||
{id: 9, query: "mediatype:image", reads: "malformed date range bound",
|
||||
aggs: []*searchService.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: ranges(
|
||||
&searchService.BucketRange{From: "2018-08-11T00:00:00Z", To: "not-a-date"},
|
||||
)}},
|
||||
wantError: true, want: []string{"error"}},
|
||||
}
|
||||
}
|
||||
|
||||
// renderAggregations flattens an answer into comparable strings; buckets with
|
||||
// no hits are dropped, the engines differ in whether they emit them at all.
|
||||
func renderAggregations(resp *searchService.SearchIndexResponse, err error) []string {
|
||||
if err != nil {
|
||||
return []string{"error"}
|
||||
}
|
||||
|
||||
out := []string{}
|
||||
for _, a := range resp.Aggregations {
|
||||
if a.MetricKind != searchService.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
kind := strings.ToLower(strings.TrimPrefix(a.MetricKind.String(), "METRIC_KIND_"))
|
||||
if a.MetricKind == searchService.MetricKind_METRIC_KIND_AVG {
|
||||
out = append(out, fmt.Sprintf("%s avg sum=%v count=%d", a.Field, a.Sum, a.Count))
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s %s=%v", a.Field, kind, a.Value))
|
||||
continue
|
||||
}
|
||||
|
||||
for _, b := range a.Buckets {
|
||||
if b.Count == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s %s=%d", a.Field, b.Key, b.Count))
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
var _ = Describe("Aggregations", func() {
|
||||
Describe("aggregations", Ordered, ContinueOnFailure, func() {
|
||||
var engines []testEngine
|
||||
|
||||
BeforeAll(func() {
|
||||
engines = newEngines("opencloud-test-engine-parity-aggregations", aggregationFixtures())
|
||||
})
|
||||
|
||||
for caseAt, c := range aggregationCases() {
|
||||
row := matrixRow{
|
||||
Section: "Aggregations", Group: "aggregations", ID: c.label(),
|
||||
Query: c.query, Reads: c.reads,
|
||||
Want: c.want, Overrides: renderOverrides(c.engineOverrides),
|
||||
GroupAt: 100, CaseAt: caseAt,
|
||||
}
|
||||
planRow(row)
|
||||
|
||||
Describe(c.label()+" "+c.reads, func() {
|
||||
for _, name := range engineNames {
|
||||
It("on "+name, func() {
|
||||
e := engineNamed(engines, name)
|
||||
if e.unavailable != "" {
|
||||
recordSkip(row, name)
|
||||
Skip(e.unavailable)
|
||||
}
|
||||
|
||||
resp, err := e.backend.Search(context.Background(), &searchService.SearchIndexRequest{
|
||||
Query: c.query,
|
||||
Aggregations: c.aggs,
|
||||
})
|
||||
answer := renderAggregations(resp, err)
|
||||
recordAnswer(row, name, answer)
|
||||
|
||||
_, overridden := c.engineOverrides[name]
|
||||
if !overridden && !c.wantError {
|
||||
Expect(err).NotTo(HaveOccurred(), "the aggregation has to answer")
|
||||
}
|
||||
|
||||
expectAnswer(name, answer, override{want: c.want}, c.engineOverrides)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -50,10 +50,6 @@ func withAudio(audio *libregraph.Audio) fixtureOption {
|
||||
return func(r *search.Resource) { r.Audio = audio }
|
||||
}
|
||||
|
||||
func withPhoto(photo *libregraph.Photo) fixtureOption {
|
||||
return func(r *search.Resource) { r.Photo = photo }
|
||||
}
|
||||
|
||||
func withLocation(location *libregraph.GeoCoordinates) fixtureOption {
|
||||
return func(r *search.Resource) { r.Location = location }
|
||||
}
|
||||
|
||||
@@ -278,10 +278,6 @@ func matrixFixtures(group string) string {
|
||||
fixtures = g.fixtures
|
||||
}
|
||||
|
||||
if group == "aggregations" {
|
||||
fixtures = aggregationFixtures()
|
||||
}
|
||||
|
||||
if len(fixtures) == 0 {
|
||||
return "Fixtures: none"
|
||||
}
|
||||
|
||||
@@ -34,16 +34,5 @@ func (c Creator[T]) Create(qs string) (T, error) {
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// CreateWithFilters compiles the query together with decoded aggregation
|
||||
// filters, ANDing them in as exact case-sensitive matches.
|
||||
func (c Creator[T]) CreateWithFilters(qs string, filters []string) (T, error) {
|
||||
var t T
|
||||
merged, err := query.MergeFilters(c.builder, qs, filters)
|
||||
if err != nil {
|
||||
return t, err
|
||||
}
|
||||
return c.compiler.Compile(merged)
|
||||
}
|
||||
|
||||
// DefaultCreator exposes a kql to bleve query creator.
|
||||
var DefaultCreator = Creator[bQuery.Query]{kql.Builder{}, Compiler{}}
|
||||
@@ -1,27 +0,0 @@
|
||||
package query
|
||||
|
||||
import "github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
|
||||
// ForceCaseSensitive marks every string restriction in the tree as an exact,
|
||||
// case-sensitive match. It is applied to a decoded aggregation filter after
|
||||
// Normalize: the filter values are exact bucket keys the server issued, so they
|
||||
// must match the case-preserving base field, not the lowercased sibling.
|
||||
func ForceCaseSensitive(a *ast.Ast) *ast.Ast {
|
||||
if a == nil {
|
||||
return a
|
||||
}
|
||||
forceCaseSensitiveNodes(a.Nodes)
|
||||
return a
|
||||
}
|
||||
|
||||
func forceCaseSensitiveNodes(nodes []ast.Node) {
|
||||
for _, n := range nodes {
|
||||
switch node := n.(type) {
|
||||
case *ast.StringNode:
|
||||
node.Exact = true
|
||||
node.CaseInsensitive = false
|
||||
case *ast.GroupNode:
|
||||
forceCaseSensitiveNodes(node.Nodes)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
)
|
||||
|
||||
// MergeFilters parses and normalizes the main query and each decoded aggregation
|
||||
// filter, forces the filters to exact case-sensitive matches, and ANDs
|
||||
// everything into one AST ready to compile. The main query and every filter are
|
||||
// wrapped in their own group so the AND binds across the whole query rather than
|
||||
// tangling with the query's own operator precedence. With no filters the
|
||||
// normalized main AST is returned unchanged.
|
||||
func MergeFilters(b Builder, qs string, filters []string) (*ast.Ast, error) {
|
||||
main, err := b.Build(qs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
main = Normalize(main, ResolveField)
|
||||
if len(filters) == 0 {
|
||||
return main, nil
|
||||
}
|
||||
|
||||
nodes := make([]ast.Node, 0, 2*len(filters)+1)
|
||||
if len(main.Nodes) > 0 {
|
||||
nodes = append(nodes, &ast.GroupNode{Base: &ast.Base{}, Nodes: main.Nodes})
|
||||
}
|
||||
for _, f := range filters {
|
||||
fa, err := b.Build(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fa = Normalize(fa, ResolveField)
|
||||
ForceCaseSensitive(fa)
|
||||
if len(fa.Nodes) == 0 {
|
||||
continue
|
||||
}
|
||||
if len(nodes) > 0 {
|
||||
nodes = append(nodes, &ast.OperatorNode{Value: kql.BoolAND})
|
||||
}
|
||||
nodes = append(nodes, &ast.GroupNode{Base: &ast.Base{}, Nodes: fa.Nodes})
|
||||
}
|
||||
return &ast.Ast{Nodes: nodes}, nil
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package query_test
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
|
||||
)
|
||||
|
||||
func collectStringNodes(nodes []ast.Node) []*ast.StringNode {
|
||||
var out []*ast.StringNode
|
||||
for _, n := range nodes {
|
||||
switch node := n.(type) {
|
||||
case *ast.StringNode:
|
||||
out = append(out, node)
|
||||
case *ast.GroupNode:
|
||||
out = append(out, collectStringNodes(node.Nodes)...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var _ = Describe("MergeFilters", func() {
|
||||
It("returns the normalized main query unchanged when there are no filters", func() {
|
||||
a, err := query.MergeFilters(kql.Builder{}, `name:"hello"`, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(a.Nodes).To(HaveLen(1))
|
||||
})
|
||||
|
||||
It("ANDs a filter in as an exact, case-sensitive match", func() {
|
||||
a, err := query.MergeFilters(kql.Builder{}, `name:"hello"`, []string{`Tags:"Pink Floyd"`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
strs := collectStringNodes(a.Nodes)
|
||||
var forced *ast.StringNode
|
||||
for _, s := range strs {
|
||||
if s.Value == "Pink Floyd" {
|
||||
forced = s
|
||||
}
|
||||
}
|
||||
Expect(forced).ToNot(BeNil(), "the decoded filter node should be present")
|
||||
Expect(forced.Exact).To(BeTrue())
|
||||
Expect(forced.CaseInsensitive).To(BeFalse())
|
||||
})
|
||||
|
||||
It("forces every node of an OR filter", func() {
|
||||
a, err := query.MergeFilters(kql.Builder{}, `name:"hello"`, []string{`(Tags:"a" OR Tags:"b")`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
forced := 0
|
||||
for _, s := range collectStringNodes(a.Nodes) {
|
||||
if s.Value != "a" && s.Value != "b" {
|
||||
continue
|
||||
}
|
||||
forced++
|
||||
Expect(s.Exact).To(BeTrue())
|
||||
Expect(s.CaseInsensitive).To(BeFalse())
|
||||
}
|
||||
Expect(forced).To(Equal(2))
|
||||
})
|
||||
})
|
||||
@@ -16,7 +16,4 @@ type Compiler[T any] interface {
|
||||
// Creator is the interface that wraps the basic Create method.
|
||||
type Creator[T any] interface {
|
||||
Create(qs string) (T, error)
|
||||
// CreateWithFilters compiles the query together with decoded aggregation
|
||||
// filters, which are ANDed in as exact case-sensitive matches.
|
||||
CreateWithFilters(qs string, filters []string) (T, error)
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// IsNumericField reports whether the indexed field at the dotted path holds a
|
||||
// numeric or time value. Terms aggregations on those are rejected: bleve stores
|
||||
// them as prefix-coded binary, so term buckets are meaningless. The set is built
|
||||
// by walking the Resource type, so new facet fields are picked up automatically.
|
||||
func IsNumericField(dottedPath string) bool {
|
||||
return numericFields[dottedPath]
|
||||
}
|
||||
|
||||
var numericFields = buildNumericFieldSet()
|
||||
|
||||
var timeType = reflect.TypeOf(time.Time{})
|
||||
|
||||
func buildNumericFieldSet() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
walkStruct(out, "", reflect.TypeOf(Resource{}))
|
||||
return out
|
||||
}
|
||||
|
||||
func walkStruct(out map[string]bool, prefix string, t reflect.Type) {
|
||||
for t.Kind() == reflect.Ptr {
|
||||
t = t.Elem()
|
||||
}
|
||||
if t.Kind() != reflect.Struct {
|
||||
return
|
||||
}
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
f := t.Field(i)
|
||||
if !f.IsExported() {
|
||||
continue
|
||||
}
|
||||
if f.Anonymous {
|
||||
// embedded: promote fields into the current prefix, like encoding/json.
|
||||
walkStruct(out, prefix, f.Type)
|
||||
continue
|
||||
}
|
||||
path := prefix + jsonFieldName(f)
|
||||
ft := f.Type
|
||||
for ft.Kind() == reflect.Ptr {
|
||||
ft = ft.Elem()
|
||||
}
|
||||
switch ft.Kind() {
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
|
||||
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
|
||||
reflect.Float32, reflect.Float64:
|
||||
out[path] = true
|
||||
case reflect.Struct:
|
||||
if ft == timeType {
|
||||
// time.Time round-trips as RFC3339; treat as numeric.
|
||||
out[path] = true
|
||||
continue
|
||||
}
|
||||
walkStruct(out, path+".", ft)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func jsonFieldName(f reflect.StructField) string {
|
||||
tag := f.Tag.Get("json")
|
||||
if tag == "" {
|
||||
return f.Name
|
||||
}
|
||||
return strings.Split(tag, ",")[0]
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package search_test
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
var _ = Describe("IsNumericField", func() {
|
||||
DescribeTable("reports whether a field maps to a numeric type",
|
||||
func(field string, numeric bool) {
|
||||
Expect(search.IsNumericField(field)).To(Equal(numeric))
|
||||
},
|
||||
// top-level numeric fields on Resource / Document
|
||||
Entry("Size (uint64 via embedded Document)", "Size", true),
|
||||
Entry("Type (uint64 on Resource)", "Type", true),
|
||||
// top-level string fields
|
||||
Entry("Name", "Name", false),
|
||||
Entry("Path", "Path", false),
|
||||
Entry("MimeType", "MimeType", false),
|
||||
// nested audio
|
||||
Entry("audio.artist", "audio.artist", false),
|
||||
Entry("audio.album", "audio.album", false),
|
||||
Entry("audio.year", "audio.year", true),
|
||||
Entry("audio.bitrate", "audio.bitrate", true),
|
||||
Entry("audio.track", "audio.track", true),
|
||||
Entry("audio.hasDrm (bool, not numeric)", "audio.hasDrm", false),
|
||||
// nested image
|
||||
Entry("image.width", "image.width", true),
|
||||
Entry("image.height", "image.height", true),
|
||||
// nested photo
|
||||
Entry("photo.cameraMake", "photo.cameraMake", false),
|
||||
Entry("photo.iso", "photo.iso", true),
|
||||
Entry("photo.focalLength (float32)", "photo.focalLength", true),
|
||||
Entry("photo.exposureDenominator (float32)", "photo.exposureDenominator", true),
|
||||
Entry("photo.takenDateTime (time.Time, treated as numeric)", "photo.takenDateTime", true),
|
||||
// nested location
|
||||
Entry("location.altitude", "location.altitude", true),
|
||||
Entry("location.latitude", "location.latitude", true),
|
||||
Entry("location.longitude", "location.longitude", true),
|
||||
// unknown fields the caller may still aggregate on
|
||||
Entry("nonexistent", "nonexistent", false),
|
||||
Entry("audio.nonexistent", "audio.nonexistent", false),
|
||||
)
|
||||
})
|
||||
@@ -128,6 +128,16 @@ func ResolveReference(ctx context.Context, ref *provider.Reference, ri *provider
|
||||
|
||||
type matchArray []*searchmsg.Match
|
||||
|
||||
func (ma matchArray) Len() int {
|
||||
return len(ma)
|
||||
}
|
||||
func (ma matchArray) Swap(i, j int) {
|
||||
ma[i], ma[j] = ma[j], ma[i]
|
||||
}
|
||||
func (ma matchArray) Less(i, j int) bool {
|
||||
return ma[i].GetScore() > ma[j].GetScore()
|
||||
}
|
||||
|
||||
func logDocCount(engine Engine, logger log.Logger) {
|
||||
c, err := engine.DocCount()
|
||||
if err != nil {
|
||||
|
||||
@@ -30,8 +30,6 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/unifiedrole"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/aggregation"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/metrics"
|
||||
@@ -136,22 +134,6 @@ func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*se
|
||||
return nil, errtypes.BadRequest("empty query provided")
|
||||
}
|
||||
req.Query = query
|
||||
|
||||
// Decode the aggregation filters (opaque bucket tokens) into KQL fragments
|
||||
// once, up front, so a malformed token fails the whole request rather than
|
||||
// silently dropping. The engines force these to exact case-sensitive matches.
|
||||
if raw := req.GetAggregationFilters(); len(raw) > 0 {
|
||||
decoded := make([]string, 0, len(raw))
|
||||
for _, f := range raw {
|
||||
frag, err := aggregation.DecodeAggregationFilter(f)
|
||||
if err != nil {
|
||||
return nil, errtypes.BadRequest(err.Error())
|
||||
}
|
||||
decoded = append(decoded, frag)
|
||||
}
|
||||
req.AggregationFilters = decoded
|
||||
}
|
||||
|
||||
if len(scope) > 0 {
|
||||
scopedID, err := storagespace.ParseID(scope)
|
||||
if err != nil {
|
||||
@@ -304,8 +286,6 @@ func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*se
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mergedAggregations := map[string]map[string]*searchmsgBucket{}
|
||||
mergedMetrics := map[string]*searchsvc.AggregationResult{}
|
||||
for _, res := range responses {
|
||||
if res == nil {
|
||||
continue
|
||||
@@ -314,68 +294,10 @@ func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*se
|
||||
for _, match := range res.Matches {
|
||||
matches = append(matches, match)
|
||||
}
|
||||
for _, agg := range res.GetAggregations() {
|
||||
// Top-level metric: reduce across spaces; keyed by field+kind so
|
||||
// several metrics on the same field stay separate.
|
||||
if kind := agg.GetMetricKind(); kind != searchsvc.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
key := agg.GetField() + "|" + kind.String()
|
||||
existing, ok := mergedMetrics[key]
|
||||
if !ok {
|
||||
mergedMetrics[key] = agg
|
||||
continue
|
||||
}
|
||||
if kind == searchsvc.MetricKind_METRIC_KIND_AVG {
|
||||
existing.Sum += agg.GetSum()
|
||||
existing.Count += agg.GetCount()
|
||||
} else {
|
||||
existing.Value = reduceMetric(kind, existing.GetValue(), agg.GetValue())
|
||||
}
|
||||
continue
|
||||
}
|
||||
field := agg.GetField()
|
||||
if _, ok := mergedAggregations[field]; !ok {
|
||||
mergedAggregations[field] = map[string]*searchmsgBucket{}
|
||||
}
|
||||
for _, b := range agg.GetBuckets() {
|
||||
if existing, ok := mergedAggregations[field][b.GetKey()]; ok {
|
||||
existing.Count += b.GetCount()
|
||||
// union child buckets per sub-aggregation so counts stay
|
||||
// right when a key spans multiple spaces
|
||||
existing.SubAggregations = mergeSubAggregations(existing.GetSubAggregations(), b.GetSubAggregations())
|
||||
continue
|
||||
}
|
||||
mergedAggregations[field][b.GetKey()] = &searchsvc.Bucket{
|
||||
Key: b.GetKey(),
|
||||
Count: b.GetCount(),
|
||||
SubAggregations: b.GetSubAggregations(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// compile one sorted list of matches from all spaces and apply the limit if needed
|
||||
//
|
||||
// NOTE(perf): every space was searched with the caller's full page size,
|
||||
// so serving one page costs O(spaces x page_size) fetched matches. With
|
||||
// offset pagination (the graph layer maps from/size onto a single
|
||||
// page_size) each deeper page re-fetches everything before it on top.
|
||||
// Accepted for now. The known fix is field-sorted cursor pagination via
|
||||
// the currently unused page_token request/response fields: each space
|
||||
// then serves "sort key < cursor, limit size" and page cost becomes
|
||||
// independent of depth. Pushing plain offsets down into the engines
|
||||
// would only trim the transfer, not the per-space overfetch, so it is
|
||||
// not worth doing on its own.
|
||||
//
|
||||
// Each engine already returns its matches in order_by order (or by score
|
||||
// when no order_by is given); this merge re-establishes that order across
|
||||
// spaces, with the score as tiebreaker.
|
||||
orderBy := req.GetOrderBy()
|
||||
sort.SliceStable(matches, func(i, j int) bool {
|
||||
if c := CompareMatches(matches[i], matches[j], orderBy); c != 0 {
|
||||
return c < 0
|
||||
}
|
||||
return matches[i].GetScore() > matches[j].GetScore()
|
||||
})
|
||||
sort.Sort(matches)
|
||||
limit := req.PageSize
|
||||
if limit == 0 {
|
||||
limit = 200
|
||||
@@ -384,163 +306,13 @@ func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*se
|
||||
matches = matches[0:limit]
|
||||
}
|
||||
|
||||
aggregations := make([]*searchsvc.AggregationResult, 0, len(req.GetAggregations()))
|
||||
for _, opt := range req.GetAggregations() {
|
||||
field := opt.GetField()
|
||||
if kind := opt.GetMetricKind(); kind != searchsvc.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
if m, ok := mergedMetrics[field+"|"+kind.String()]; ok {
|
||||
aggregations = append(aggregations, m)
|
||||
}
|
||||
continue
|
||||
}
|
||||
bucketMap := mergedAggregations[field]
|
||||
buckets := make([]*searchsvc.Bucket, 0, len(bucketMap))
|
||||
for _, b := range bucketMap {
|
||||
buckets = append(buckets, b)
|
||||
}
|
||||
aggregations = append(aggregations, &searchsvc.AggregationResult{
|
||||
Field: field,
|
||||
Buckets: postProcessBuckets(buckets, opt),
|
||||
})
|
||||
}
|
||||
|
||||
success = true
|
||||
return &searchsvc.SearchResponse{
|
||||
Matches: matches,
|
||||
TotalMatches: total,
|
||||
Aggregations: aggregations,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// searchmsgBucket aliases the bucket type for the map-of-maps below.
|
||||
type searchmsgBucket = searchsvc.Bucket
|
||||
|
||||
// mergeSubAggregations unions two nested-aggregation lists by field: terms
|
||||
// union child buckets by key (summing, recursing); metrics apply their reducer
|
||||
// (sum/min/max).
|
||||
func mergeSubAggregations(a, b []*searchsvc.AggregationResult) []*searchsvc.AggregationResult {
|
||||
if len(a) == 0 {
|
||||
return b
|
||||
}
|
||||
if len(b) == 0 {
|
||||
return a
|
||||
}
|
||||
byField := make(map[string]*searchsvc.AggregationResult, len(a))
|
||||
for _, r := range a {
|
||||
byField[r.GetField()] = r
|
||||
}
|
||||
for _, r := range b {
|
||||
existing, ok := byField[r.GetField()]
|
||||
if !ok {
|
||||
byField[r.GetField()] = r
|
||||
continue
|
||||
}
|
||||
if existing.GetMetricKind() != searchsvc.MetricKind_METRIC_KIND_UNSPECIFIED ||
|
||||
r.GetMetricKind() != searchsvc.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
// Metric result: apply the kind's reducer; prefer existing's kind.
|
||||
kind := existing.GetMetricKind()
|
||||
if kind == searchsvc.MetricKind_METRIC_KIND_UNSPECIFIED {
|
||||
kind = r.GetMetricKind()
|
||||
}
|
||||
existing.MetricKind = kind
|
||||
if kind == searchsvc.MetricKind_METRIC_KIND_AVG {
|
||||
existing.Sum += r.GetSum()
|
||||
existing.Count += r.GetCount()
|
||||
} else {
|
||||
existing.Value = reduceMetric(kind, existing.GetValue(), r.GetValue())
|
||||
}
|
||||
continue
|
||||
}
|
||||
byKey := make(map[string]*searchsvc.Bucket, len(existing.Buckets))
|
||||
for _, bk := range existing.Buckets {
|
||||
byKey[bk.GetKey()] = bk
|
||||
}
|
||||
for _, bk := range r.GetBuckets() {
|
||||
if prev, ok := byKey[bk.GetKey()]; ok {
|
||||
prev.Count += bk.GetCount()
|
||||
prev.SubAggregations = mergeSubAggregations(prev.GetSubAggregations(), bk.GetSubAggregations())
|
||||
} else {
|
||||
existing.Buckets = append(existing.Buckets, bk)
|
||||
byKey[bk.GetKey()] = bk
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]*searchsvc.AggregationResult, 0, len(byField))
|
||||
for _, r := range byField {
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reduceMetric applies the metric's cross-shard reducer; only called when both
|
||||
// sides carry a value.
|
||||
func reduceMetric(kind searchsvc.MetricKind, a, b float64) float64 {
|
||||
switch kind {
|
||||
case searchsvc.MetricKind_METRIC_KIND_SUM:
|
||||
return a + b
|
||||
case searchsvc.MetricKind_METRIC_KIND_MIN:
|
||||
if b < a {
|
||||
return b
|
||||
}
|
||||
return a
|
||||
case searchsvc.MetricKind_METRIC_KIND_MAX:
|
||||
if b > a {
|
||||
return b
|
||||
}
|
||||
return a
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// postProcessBuckets applies the BucketDefinition (minimumCount filter, sort by
|
||||
// count/keyAsString/keyAsNumber, trim to Size). Defaults to count-descending.
|
||||
func postProcessBuckets(buckets []*searchsvc.Bucket, opt *searchsvc.AggregationOption) []*searchsvc.Bucket {
|
||||
bd := opt.GetBucketDefinition()
|
||||
sortBy := "count"
|
||||
desc := true
|
||||
var minCount int64
|
||||
if bd != nil {
|
||||
if bd.GetSortBy() != "" {
|
||||
sortBy = bd.GetSortBy()
|
||||
}
|
||||
desc = bd.GetIsDescending()
|
||||
minCount = int64(bd.GetMinimumCount())
|
||||
}
|
||||
|
||||
if minCount > 0 {
|
||||
filtered := buckets[:0]
|
||||
for _, b := range buckets {
|
||||
if b.GetCount() >= minCount {
|
||||
filtered = append(filtered, b)
|
||||
}
|
||||
}
|
||||
buckets = filtered
|
||||
}
|
||||
|
||||
sort.SliceStable(buckets, func(i, j int) bool {
|
||||
less := false
|
||||
switch sortBy {
|
||||
case "keyAsString":
|
||||
less = buckets[i].GetKey() < buckets[j].GetKey()
|
||||
case "keyAsNumber":
|
||||
iv, _ := strconv.ParseFloat(buckets[i].GetKey(), 64)
|
||||
jv, _ := strconv.ParseFloat(buckets[j].GetKey(), 64)
|
||||
less = iv < jv
|
||||
default: // "count"
|
||||
less = buckets[i].GetCount() < buckets[j].GetCount()
|
||||
}
|
||||
if desc {
|
||||
return !less
|
||||
}
|
||||
return less
|
||||
})
|
||||
|
||||
if size := opt.GetSize(); size > 0 && int32(len(buckets)) > size {
|
||||
buckets = buckets[:size]
|
||||
}
|
||||
return buckets
|
||||
}
|
||||
|
||||
func (s *Service) searchIndex(ctx context.Context, req *searchsvc.SearchRequest, space *provider.StorageSpace, mountpointID string) (*searchsvc.SearchIndexResponse, error) {
|
||||
if req.Ref != nil &&
|
||||
(req.Ref.ResourceId.StorageId != space.Root.StorageId ||
|
||||
@@ -638,10 +410,7 @@ func (s *Service) searchIndex(ctx context.Context, req *searchsvc.SearchRequest,
|
||||
}
|
||||
|
||||
searchRequest := &searchsvc.SearchIndexRequest{
|
||||
Query: req.Query,
|
||||
Aggregations: req.GetAggregations(),
|
||||
AggregationFilters: req.GetAggregationFilters(),
|
||||
OrderBy: req.GetOrderBy(),
|
||||
Query: req.Query,
|
||||
Ref: &searchmsg.Reference{
|
||||
ResourceId: searchRootID,
|
||||
Path: searchPathPrefix,
|
||||
@@ -677,9 +446,6 @@ func (s *Service) searchIndex(ctx context.Context, req *searchsvc.SearchRequest,
|
||||
isMountpoint := isShared && match.GetEntity().GetRef().GetPath() == "."
|
||||
isDir := match.GetEntity().GetMimeType() == "httpd/unix-directory"
|
||||
match.Entity.Permissions = convertToWebDAVPermissions(isShared, isMountpoint, isDir, permissions)
|
||||
// allowedValues is the same effective permission set the WebDAV report's
|
||||
// oc:permissions string projects, in libregraph action notation.
|
||||
match.Entity.PermissionsActionsAllowedValues = unifiedrole.CS3ResourcePermissionsToLibregraphActions(permissions)
|
||||
|
||||
if req.Ref != nil && searchPathPrefix == "/"+match.Entity.Name {
|
||||
continue
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
cs3mocks "github.com/opencloud-eu/reva/v2/tests/cs3mocks/mocks"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"google.golang.org/grpc"
|
||||
typespb "google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
@@ -259,256 +258,6 @@ var _ = Describe("Searchprovider", func() {
|
||||
Expect(match.Entity.Ref.ResourceId.OpaqueId).To(Equal(personalSpace.Root.OpaqueId))
|
||||
Expect(match.Entity.Ref.Path).To(Equal("./path/to/Foo.pdf"))
|
||||
})
|
||||
|
||||
It("forwards aggregations to the engine", func() {
|
||||
_, err := s.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "foo",
|
||||
Aggregations: []*searchsvc.AggregationOption{
|
||||
{Field: "audio.artist", Size: 10},
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool {
|
||||
return len(req.Aggregations) == 1 &&
|
||||
req.Aggregations[0].Field == "audio.artist" &&
|
||||
req.Aggregations[0].Size == 10
|
||||
}))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with two personal spaces returning matches", func() {
|
||||
var (
|
||||
spaceA = &sprovider.StorageSpace{
|
||||
Id: &sprovider.StorageSpaceId{OpaqueId: "storageid$a!a"},
|
||||
Root: &sprovider.ResourceId{StorageId: "storageid", SpaceId: "a", OpaqueId: "a"},
|
||||
Name: "space-a",
|
||||
SpaceType: "personal",
|
||||
}
|
||||
spaceB = &sprovider.StorageSpace{
|
||||
Id: &sprovider.StorageSpaceId{OpaqueId: "storageid$b!b"},
|
||||
Root: &sprovider.ResourceId{StorageId: "storageid", SpaceId: "b", OpaqueId: "b"},
|
||||
Name: "space-b",
|
||||
SpaceType: "personal",
|
||||
}
|
||||
|
||||
photoMatch = func(space string, name string, taken int64, score float32) *searchmsg.Match {
|
||||
return &searchmsg.Match{
|
||||
Score: score,
|
||||
Entity: &searchmsg.Entity{
|
||||
Ref: &searchmsg.Reference{
|
||||
ResourceId: &searchmsg.ResourceID{StorageId: "storageid", SpaceId: space, OpaqueId: space},
|
||||
Path: "./" + name,
|
||||
},
|
||||
Id: &searchmsg.ResourceID{StorageId: "storageid", SpaceId: space, OpaqueId: name},
|
||||
Name: name,
|
||||
Photo: &searchmsg.Photo{
|
||||
TakenDateTime: &typespb.Timestamp{Seconds: taken},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
gatewayClient.On("ListStorageSpaces", mock.Anything, mock.Anything).Return(&sprovider.ListStorageSpacesResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
StorageSpaces: []*sprovider.StorageSpace{spaceA, spaceB},
|
||||
}, nil)
|
||||
indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool {
|
||||
return req.Ref != nil && req.Ref.ResourceId.SpaceId == "a"
|
||||
})).Return(&searchsvc.SearchIndexResponse{
|
||||
TotalMatches: 2,
|
||||
Matches: []*searchmsg.Match{
|
||||
photoMatch("a", "a-old.jpg", 100, 0.9),
|
||||
photoMatch("a", "a-new.jpg", 300, 0.1),
|
||||
},
|
||||
}, nil)
|
||||
indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool {
|
||||
return req.Ref != nil && req.Ref.ResourceId.SpaceId == "b"
|
||||
})).Return(&searchsvc.SearchIndexResponse{
|
||||
TotalMatches: 2,
|
||||
Matches: []*searchmsg.Match{
|
||||
photoMatch("b", "b-newest.jpg", 400, 0.5),
|
||||
photoMatch("b", "b-mid.jpg", 200, 0.4),
|
||||
},
|
||||
}, nil)
|
||||
})
|
||||
|
||||
It("forwards order_by to the engine", func() {
|
||||
_, err := s.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:image",
|
||||
OrderBy: []*searchsvc.SortProperty{{Name: "photo.takenDateTime", IsDescending: true}},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool {
|
||||
return len(req.OrderBy) == 1 &&
|
||||
req.OrderBy[0].Name == "photo.takenDateTime" &&
|
||||
req.OrderBy[0].IsDescending
|
||||
}))
|
||||
})
|
||||
|
||||
It("merges matches across spaces in sort order", func() {
|
||||
res, err := s.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:image",
|
||||
OrderBy: []*searchsvc.SortProperty{{Name: "photo.takenDateTime", IsDescending: true}},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
names := []string{}
|
||||
for _, m := range res.Matches {
|
||||
names = append(names, m.Entity.Name)
|
||||
}
|
||||
Expect(names).To(Equal([]string{"b-newest.jpg", "a-new.jpg", "b-mid.jpg", "a-old.jpg"}))
|
||||
})
|
||||
|
||||
It("merges matches by score when no order_by is given", func() {
|
||||
res, err := s.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:image",
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
names := []string{}
|
||||
for _, m := range res.Matches {
|
||||
names = append(names, m.Entity.Name)
|
||||
}
|
||||
Expect(names).To(Equal([]string{"a-old.jpg", "b-newest.jpg", "b-mid.jpg", "a-new.jpg"}))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with two personal spaces returning aggregations", func() {
|
||||
var (
|
||||
spaceA = &sprovider.StorageSpace{
|
||||
Id: &sprovider.StorageSpaceId{OpaqueId: "storageid$a!a"},
|
||||
Root: &sprovider.ResourceId{StorageId: "storageid", SpaceId: "a", OpaqueId: "a"},
|
||||
Name: "space-a",
|
||||
SpaceType: "personal",
|
||||
}
|
||||
spaceB = &sprovider.StorageSpace{
|
||||
Id: &sprovider.StorageSpaceId{OpaqueId: "storageid$b!b"},
|
||||
Root: &sprovider.ResourceId{StorageId: "storageid", SpaceId: "b", OpaqueId: "b"},
|
||||
Name: "space-b",
|
||||
SpaceType: "personal",
|
||||
}
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
gatewayClient.On("ListStorageSpaces", mock.Anything, mock.Anything).Return(&sprovider.ListStorageSpacesResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
StorageSpaces: []*sprovider.StorageSpace{spaceA, spaceB},
|
||||
}, nil)
|
||||
indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool {
|
||||
return req.Ref != nil && req.Ref.ResourceId.SpaceId == "a"
|
||||
})).Return(&searchsvc.SearchIndexResponse{
|
||||
TotalMatches: 2,
|
||||
Aggregations: []*searchsvc.AggregationResult{{
|
||||
Field: "audio.artist",
|
||||
Buckets: []*searchsvc.Bucket{
|
||||
{Key: "Pink Floyd", Count: 2},
|
||||
{Key: "Motörhead", Count: 1},
|
||||
},
|
||||
}},
|
||||
}, nil)
|
||||
indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool {
|
||||
return req.Ref != nil && req.Ref.ResourceId.SpaceId == "b"
|
||||
})).Return(&searchsvc.SearchIndexResponse{
|
||||
TotalMatches: 3,
|
||||
Aggregations: []*searchsvc.AggregationResult{{
|
||||
Field: "audio.artist",
|
||||
Buckets: []*searchsvc.Bucket{
|
||||
{Key: "Pink Floyd", Count: 3},
|
||||
{Key: "Led Zeppelin", Count: 1},
|
||||
},
|
||||
}},
|
||||
}, nil)
|
||||
})
|
||||
|
||||
It("merges bucket counts across spaces", func() {
|
||||
res, err := s.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:audio",
|
||||
Aggregations: []*searchsvc.AggregationOption{
|
||||
{Field: "audio.artist", Size: 10},
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Aggregations).To(HaveLen(1))
|
||||
agg := res.Aggregations[0]
|
||||
Expect(agg.Field).To(Equal("audio.artist"))
|
||||
|
||||
counts := map[string]int64{}
|
||||
for _, b := range agg.Buckets {
|
||||
counts[b.Key] = b.Count
|
||||
}
|
||||
Expect(counts).To(HaveKeyWithValue("Pink Floyd", int64(5)))
|
||||
Expect(counts).To(HaveKeyWithValue("Motörhead", int64(1)))
|
||||
Expect(counts).To(HaveKeyWithValue("Led Zeppelin", int64(1)))
|
||||
})
|
||||
|
||||
It("sorts buckets by count descending by default", func() {
|
||||
res, err := s.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:audio",
|
||||
Aggregations: []*searchsvc.AggregationOption{
|
||||
{Field: "audio.artist"},
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
keys := []string{}
|
||||
for _, b := range res.Aggregations[0].Buckets {
|
||||
keys = append(keys, b.Key)
|
||||
}
|
||||
// Pink Floyd:5, Motörhead:1, Led Zeppelin:1 (count desc)
|
||||
Expect(keys[0]).To(Equal("Pink Floyd"))
|
||||
})
|
||||
|
||||
It("sorts buckets alphabetically ascending with sortBy keyAsString", func() {
|
||||
res, err := s.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:audio",
|
||||
Aggregations: []*searchsvc.AggregationOption{
|
||||
{
|
||||
Field: "audio.artist",
|
||||
BucketDefinition: &searchsvc.BucketDefinition{
|
||||
SortBy: "keyAsString",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
keys := []string{}
|
||||
for _, b := range res.Aggregations[0].Buckets {
|
||||
keys = append(keys, b.Key)
|
||||
}
|
||||
Expect(keys).To(Equal([]string{"Led Zeppelin", "Motörhead", "Pink Floyd"}))
|
||||
})
|
||||
|
||||
It("applies minimumCount filter and size cap", func() {
|
||||
res, err := s.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:audio",
|
||||
Aggregations: []*searchsvc.AggregationOption{
|
||||
{
|
||||
Field: "audio.artist",
|
||||
Size: 5,
|
||||
BucketDefinition: &searchsvc.BucketDefinition{
|
||||
SortBy: "count",
|
||||
IsDescending: true,
|
||||
MinimumCount: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// only Pink Floyd has count >= 2
|
||||
Expect(res.Aggregations[0].Buckets).To(HaveLen(1))
|
||||
Expect(res.Aggregations[0].Buckets[0].Key).To(Equal("Pink Floyd"))
|
||||
})
|
||||
|
||||
It("trims the bucket list to Size", func() {
|
||||
res, err := s.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:audio",
|
||||
Aggregations: []*searchsvc.AggregationOption{
|
||||
{Field: "audio.artist", Size: 1},
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Aggregations[0].Buckets).To(HaveLen(1))
|
||||
Expect(res.Aggregations[0].Buckets[0].Key).To(Equal("Pink Floyd"))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with a personal space with a filter", func() {
|
||||
|
||||
@@ -1,257 +0,0 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"google.golang.org/protobuf/reflect/protoreflect"
|
||||
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
)
|
||||
|
||||
// Sorting support for search results (graph sortProperties / proto order_by).
|
||||
//
|
||||
// A field is sortable when both of the following hold:
|
||||
// - it is indexed as a scalar (string, number, bool or time), so the
|
||||
// engines can sort on it natively; multivalued fields like Tags are not
|
||||
// sortable
|
||||
// - it is carried on the match entity, so the service layer can read the
|
||||
// sort key when merging the per-space result streams
|
||||
//
|
||||
// Both sets are derived by reflection (index side: the Resource type, match
|
||||
// side: the Entity proto), so new facet fields become sortable automatically.
|
||||
|
||||
// sortIndexAliases maps the graph-facing names of top-level fields to their
|
||||
// index field names. Facet fields (photo.*, audio.*, ...) share the same
|
||||
// dotted names in both worlds and need no alias. Top-level fields are only
|
||||
// exposed under these graph names; internal fields like RootID or Deleted
|
||||
// stay unsortable.
|
||||
var sortIndexAliases = map[string]string{
|
||||
"name": "Name",
|
||||
"size": "Size",
|
||||
"lastModifiedDateTime": "Mtime",
|
||||
"mimeType": "MimeType",
|
||||
}
|
||||
|
||||
// entityJSONAliases maps graph-facing names to the Entity proto's JSON names
|
||||
// where the two disagree.
|
||||
var entityJSONAliases = map[string]string{
|
||||
"lastModifiedDateTime": "lastModifiedTime",
|
||||
}
|
||||
|
||||
// IsSortableField reports whether results can be sorted by the field.
|
||||
func IsSortableField(name string) bool {
|
||||
_, ok := SortIndexField(name)
|
||||
return ok
|
||||
}
|
||||
|
||||
// SortIndexField translates a graph sortProperties name into the index field
|
||||
// name to sort on, reporting whether the field is sortable at all.
|
||||
func SortIndexField(name string) (string, bool) {
|
||||
field := name
|
||||
if alias, ok := sortIndexAliases[name]; ok {
|
||||
field = alias
|
||||
} else if !strings.Contains(name, ".") {
|
||||
return "", false
|
||||
}
|
||||
if !sortableIndexFields[field] {
|
||||
return "", false
|
||||
}
|
||||
if !entityFieldResolvable(name) {
|
||||
return "", false
|
||||
}
|
||||
return field, true
|
||||
}
|
||||
|
||||
// CompareMatches orders match a relative to b according to orderBy: -1 when a
|
||||
// comes first, 1 when b comes first, 0 when the sort keys tie (callers fall
|
||||
// back to the score). Matches missing a sort key sort after those that have
|
||||
// it, regardless of direction.
|
||||
func CompareMatches(a, b *searchmsg.Match, orderBy []*searchsvc.SortProperty) int {
|
||||
for _, sp := range orderBy {
|
||||
ka := matchSortKey(a, sp.GetName())
|
||||
kb := matchSortKey(b, sp.GetName())
|
||||
if !ka.present && !kb.present {
|
||||
continue
|
||||
}
|
||||
if !ka.present {
|
||||
return 1
|
||||
}
|
||||
if !kb.present {
|
||||
return -1
|
||||
}
|
||||
c := 0
|
||||
switch {
|
||||
case ka.isString:
|
||||
c = strings.Compare(ka.str, kb.str)
|
||||
case ka.num < kb.num:
|
||||
c = -1
|
||||
case ka.num > kb.num:
|
||||
c = 1
|
||||
}
|
||||
if c == 0 {
|
||||
continue
|
||||
}
|
||||
if sp.GetIsDescending() {
|
||||
c = -c
|
||||
}
|
||||
return c
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// sortableIndexFields is the set of scalar indexed fields, keyed by index
|
||||
// field name.
|
||||
var sortableIndexFields = buildSortableFieldSet()
|
||||
|
||||
func buildSortableFieldSet() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
collectScalarFields(out, "", reflect.TypeOf(Resource{}))
|
||||
return out
|
||||
}
|
||||
|
||||
func collectScalarFields(out map[string]bool, prefix string, t reflect.Type) {
|
||||
for t.Kind() == reflect.Ptr {
|
||||
t = t.Elem()
|
||||
}
|
||||
if t.Kind() != reflect.Struct {
|
||||
return
|
||||
}
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
f := t.Field(i)
|
||||
if !f.IsExported() {
|
||||
continue
|
||||
}
|
||||
if f.Anonymous {
|
||||
collectScalarFields(out, prefix, f.Type)
|
||||
continue
|
||||
}
|
||||
path := prefix + jsonFieldName(f)
|
||||
ft := f.Type
|
||||
for ft.Kind() == reflect.Ptr {
|
||||
ft = ft.Elem()
|
||||
}
|
||||
switch ft.Kind() {
|
||||
case reflect.String, reflect.Bool,
|
||||
reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
|
||||
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
|
||||
reflect.Float32, reflect.Float64:
|
||||
out[path] = true
|
||||
case reflect.Struct:
|
||||
if ft == timeType {
|
||||
out[path] = true
|
||||
continue
|
||||
}
|
||||
collectScalarFields(out, path+".", ft)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func entityPath(name string) []string {
|
||||
if alias, ok := entityJSONAliases[name]; ok {
|
||||
name = alias
|
||||
}
|
||||
return strings.Split(name, ".")
|
||||
}
|
||||
|
||||
const timestampFullName = protoreflect.FullName("google.protobuf.Timestamp")
|
||||
|
||||
// entityFieldResolvable reports whether the graph field name resolves to a
|
||||
// scalar (or timestamp) field on the match entity.
|
||||
func entityFieldResolvable(name string) bool {
|
||||
md := (&searchmsg.Entity{}).ProtoReflect().Descriptor()
|
||||
segments := entityPath(name)
|
||||
for i, seg := range segments {
|
||||
fd := md.Fields().ByJSONName(seg)
|
||||
if fd == nil || fd.IsList() || fd.IsMap() {
|
||||
return false
|
||||
}
|
||||
if i < len(segments)-1 {
|
||||
if fd.Kind() != protoreflect.MessageKind {
|
||||
return false
|
||||
}
|
||||
md = fd.Message()
|
||||
continue
|
||||
}
|
||||
switch fd.Kind() {
|
||||
case protoreflect.StringKind, protoreflect.BoolKind,
|
||||
protoreflect.Int32Kind, protoreflect.Int64Kind,
|
||||
protoreflect.Sint32Kind, protoreflect.Sint64Kind,
|
||||
protoreflect.Sfixed32Kind, protoreflect.Sfixed64Kind,
|
||||
protoreflect.Uint32Kind, protoreflect.Uint64Kind,
|
||||
protoreflect.Fixed32Kind, protoreflect.Fixed64Kind,
|
||||
protoreflect.FloatKind, protoreflect.DoubleKind:
|
||||
return true
|
||||
case protoreflect.MessageKind:
|
||||
return fd.Message().FullName() == timestampFullName
|
||||
}
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sortKey is the comparable value of a sort field on a concrete match.
|
||||
type sortKey struct {
|
||||
present bool
|
||||
isString bool
|
||||
str string
|
||||
num float64
|
||||
}
|
||||
|
||||
// matchSortKey extracts the sort key for the graph field name from a match by
|
||||
// walking the entity proto along the field's JSON names.
|
||||
func matchSortKey(m *searchmsg.Match, name string) sortKey {
|
||||
entity := m.GetEntity()
|
||||
if entity == nil {
|
||||
return sortKey{}
|
||||
}
|
||||
msg := entity.ProtoReflect()
|
||||
segments := entityPath(name)
|
||||
for i, seg := range segments {
|
||||
fd := msg.Descriptor().Fields().ByJSONName(seg)
|
||||
if fd == nil || fd.IsList() || fd.IsMap() {
|
||||
return sortKey{}
|
||||
}
|
||||
if i < len(segments)-1 {
|
||||
if fd.Kind() != protoreflect.MessageKind || !msg.Has(fd) {
|
||||
return sortKey{}
|
||||
}
|
||||
msg = msg.Get(fd).Message()
|
||||
continue
|
||||
}
|
||||
if fd.HasPresence() && !msg.Has(fd) {
|
||||
return sortKey{}
|
||||
}
|
||||
v := msg.Get(fd)
|
||||
switch fd.Kind() {
|
||||
case protoreflect.StringKind:
|
||||
return sortKey{present: true, isString: true, str: v.String()}
|
||||
case protoreflect.BoolKind:
|
||||
num := 0.0
|
||||
if v.Bool() {
|
||||
num = 1.0
|
||||
}
|
||||
return sortKey{present: true, num: num}
|
||||
case protoreflect.Int32Kind, protoreflect.Int64Kind,
|
||||
protoreflect.Sint32Kind, protoreflect.Sint64Kind,
|
||||
protoreflect.Sfixed32Kind, protoreflect.Sfixed64Kind:
|
||||
return sortKey{present: true, num: float64(v.Int())}
|
||||
case protoreflect.Uint32Kind, protoreflect.Uint64Kind,
|
||||
protoreflect.Fixed32Kind, protoreflect.Fixed64Kind:
|
||||
return sortKey{present: true, num: float64(v.Uint())}
|
||||
case protoreflect.FloatKind, protoreflect.DoubleKind:
|
||||
return sortKey{present: true, num: v.Float()}
|
||||
case protoreflect.MessageKind:
|
||||
if fd.Message().FullName() != timestampFullName {
|
||||
return sortKey{}
|
||||
}
|
||||
ts := v.Message()
|
||||
seconds := ts.Get(ts.Descriptor().Fields().ByName("seconds")).Int()
|
||||
nanos := ts.Get(ts.Descriptor().Fields().ByName("nanos")).Int()
|
||||
return sortKey{present: true, num: float64(seconds) + float64(nanos)/1e9}
|
||||
}
|
||||
return sortKey{}
|
||||
}
|
||||
return sortKey{}
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
package search_test
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
var _ = Describe("SortIndexField", func() {
|
||||
DescribeTable("maps graph field names to index fields",
|
||||
func(name string, wantField string, wantOK bool) {
|
||||
field, ok := search.SortIndexField(name)
|
||||
Expect(ok).To(Equal(wantOK))
|
||||
Expect(field).To(Equal(wantField))
|
||||
},
|
||||
// top-level fields are exposed under their graph names
|
||||
Entry("name", "name", "Name", true),
|
||||
Entry("size", "size", "Size", true),
|
||||
Entry("lastModifiedDateTime", "lastModifiedDateTime", "Mtime", true),
|
||||
Entry("mimeType", "mimeType", "MimeType", true),
|
||||
// facet fields keep their dotted names
|
||||
Entry("photo.takenDateTime", "photo.takenDateTime", "photo.takenDateTime", true),
|
||||
Entry("photo.iso", "photo.iso", "photo.iso", true),
|
||||
Entry("photo.cameraModel", "photo.cameraModel", "photo.cameraModel", true),
|
||||
Entry("audio.artist", "audio.artist", "audio.artist", true),
|
||||
Entry("audio.year", "audio.year", "audio.year", true),
|
||||
Entry("image.width", "image.width", "image.width", true),
|
||||
Entry("location.latitude", "location.latitude", "location.latitude", true),
|
||||
// bare facets (message-typed, no scalar value) are not sortable
|
||||
Entry("audio (bare facet)", "audio", "", false),
|
||||
Entry("photo (bare facet)", "photo", "", false),
|
||||
Entry("location (bare facet)", "location", "", false),
|
||||
Entry("location (dotted but not scalar)", "location.", "", false),
|
||||
// multivalued fields are not sortable
|
||||
Entry("tags (repeated)", "tags", "", false),
|
||||
Entry("Tags (index name, repeated)", "Tags", "", false),
|
||||
// internal index fields are not exposed under their index names
|
||||
Entry("Name (index name)", "Name", "", false),
|
||||
Entry("Mtime (index name)", "Mtime", "", false),
|
||||
Entry("RootID", "RootID", "", false),
|
||||
Entry("Deleted", "Deleted", "", false),
|
||||
// unknown fields
|
||||
Entry("unknown", "definitelyNotAField", "", false),
|
||||
Entry("unknown facet field", "photo.definitelyNotAField", "", false),
|
||||
Entry("empty", "", "", false),
|
||||
)
|
||||
})
|
||||
|
||||
var _ = Describe("CompareMatches", func() {
|
||||
match := func(mutate func(e *searchmsg.Entity)) *searchmsg.Match {
|
||||
e := &searchmsg.Entity{}
|
||||
mutate(e)
|
||||
return &searchmsg.Match{Entity: e}
|
||||
}
|
||||
asc := func(name string) []*searchsvc.SortProperty {
|
||||
return []*searchsvc.SortProperty{{Name: name}}
|
||||
}
|
||||
desc := func(name string) []*searchsvc.SortProperty {
|
||||
return []*searchsvc.SortProperty{{Name: name, IsDescending: true}}
|
||||
}
|
||||
|
||||
It("compares string fields lexicographically", func() {
|
||||
a := match(func(e *searchmsg.Entity) { e.Name = "a.jpg" })
|
||||
b := match(func(e *searchmsg.Entity) { e.Name = "b.jpg" })
|
||||
Expect(search.CompareMatches(a, b, asc("name"))).To(Equal(-1))
|
||||
Expect(search.CompareMatches(b, a, asc("name"))).To(Equal(1))
|
||||
Expect(search.CompareMatches(a, b, desc("name"))).To(Equal(1))
|
||||
})
|
||||
|
||||
It("compares numeric fields numerically", func() {
|
||||
small := match(func(e *searchmsg.Entity) { e.Size = 9 })
|
||||
big := match(func(e *searchmsg.Entity) { e.Size = 10 })
|
||||
Expect(search.CompareMatches(small, big, asc("size"))).To(Equal(-1))
|
||||
Expect(search.CompareMatches(small, big, desc("size"))).To(Equal(1))
|
||||
})
|
||||
|
||||
It("compares timestamps", func() {
|
||||
older := match(func(e *searchmsg.Entity) {
|
||||
e.Photo = &searchmsg.Photo{TakenDateTime: ×tamppb.Timestamp{Seconds: 100}}
|
||||
})
|
||||
newer := match(func(e *searchmsg.Entity) {
|
||||
e.Photo = &searchmsg.Photo{TakenDateTime: ×tamppb.Timestamp{Seconds: 200}}
|
||||
})
|
||||
Expect(search.CompareMatches(older, newer, asc("photo.takenDateTime"))).To(Equal(-1))
|
||||
Expect(search.CompareMatches(older, newer, desc("photo.takenDateTime"))).To(Equal(1))
|
||||
})
|
||||
|
||||
It("compares lastModifiedDateTime via the entity's lastModifiedTime", func() {
|
||||
older := match(func(e *searchmsg.Entity) {
|
||||
e.LastModifiedTime = ×tamppb.Timestamp{Seconds: 100}
|
||||
})
|
||||
newer := match(func(e *searchmsg.Entity) {
|
||||
e.LastModifiedTime = ×tamppb.Timestamp{Seconds: 200}
|
||||
})
|
||||
Expect(search.CompareMatches(older, newer, asc("lastModifiedDateTime"))).To(Equal(-1))
|
||||
})
|
||||
|
||||
It("sorts matches missing the field after those that have it, in both directions", func() {
|
||||
has := match(func(e *searchmsg.Entity) {
|
||||
e.Photo = &searchmsg.Photo{TakenDateTime: ×tamppb.Timestamp{Seconds: 100}}
|
||||
})
|
||||
missing := match(func(e *searchmsg.Entity) {})
|
||||
Expect(search.CompareMatches(has, missing, asc("photo.takenDateTime"))).To(Equal(-1))
|
||||
Expect(search.CompareMatches(missing, has, asc("photo.takenDateTime"))).To(Equal(1))
|
||||
Expect(search.CompareMatches(has, missing, desc("photo.takenDateTime"))).To(Equal(-1))
|
||||
})
|
||||
|
||||
It("falls through to the next sort property on ties", func() {
|
||||
a := match(func(e *searchmsg.Entity) { e.Size = 5; e.Name = "a" })
|
||||
b := match(func(e *searchmsg.Entity) { e.Size = 5; e.Name = "b" })
|
||||
orderBy := []*searchsvc.SortProperty{{Name: "size"}, {Name: "name"}}
|
||||
Expect(search.CompareMatches(a, b, orderBy)).To(Equal(-1))
|
||||
})
|
||||
|
||||
It("returns 0 for full ties and empty orderBy", func() {
|
||||
a := match(func(e *searchmsg.Entity) { e.Size = 5 })
|
||||
b := match(func(e *searchmsg.Entity) { e.Size = 5 })
|
||||
Expect(search.CompareMatches(a, b, asc("size"))).To(Equal(0))
|
||||
Expect(search.CompareMatches(a, b, nil)).To(Equal(0))
|
||||
})
|
||||
})
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"go-micro.dev/v4/metadata"
|
||||
"golang.org/x/sync/errgroup"
|
||||
grpcmetadata "google.golang.org/grpc/metadata"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/durationpb"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
@@ -95,16 +94,14 @@ func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *s
|
||||
}
|
||||
ctx = revactx.ContextSetUser(ctx, u)
|
||||
|
||||
key := cacheKey(in.Query, in.PageSize, in.Ref, u, in.Aggregations, in.OrderBy)
|
||||
key := cacheKey(in.Query, in.PageSize, in.Ref, u)
|
||||
res, ok := s.FromCache(key)
|
||||
if !ok {
|
||||
var err error
|
||||
res, err = s.searcher.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: in.Query,
|
||||
PageSize: in.PageSize,
|
||||
Ref: in.Ref,
|
||||
Aggregations: in.Aggregations,
|
||||
OrderBy: in.OrderBy,
|
||||
Query: in.Query,
|
||||
PageSize: in.PageSize,
|
||||
Ref: in.Ref,
|
||||
})
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
@@ -121,7 +118,6 @@ func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *s
|
||||
out.Matches = res.Matches
|
||||
out.TotalMatches = res.TotalMatches
|
||||
out.NextPageToken = res.NextPageToken
|
||||
out.Aggregations = res.Aggregations
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -246,18 +242,6 @@ func (s Service) Cache(key string, res *searchsvc.SearchResponse) {
|
||||
_ = s.cache.Set(key, res)
|
||||
}
|
||||
|
||||
// cacheKey builds the cache identity for a search. Every result-affecting field
|
||||
// must be in the key, including aggregations and order_by (serialised via
|
||||
// deterministic proto marshalling). If those protos ever gain a map field,
|
||||
// determinism requires all writers to set Deterministic=true.
|
||||
func cacheKey(query string, pagesize int32, ref *v0.Reference, user *user.User, aggs []*searchsvc.AggregationOption, orderBy []*searchsvc.SortProperty) string {
|
||||
protoPart := ""
|
||||
if len(aggs) > 0 || len(orderBy) > 0 {
|
||||
b, _ := proto.MarshalOptions{Deterministic: true}.Marshal(&searchsvc.SearchRequest{Aggregations: aggs, OrderBy: orderBy})
|
||||
protoPart = string(b)
|
||||
}
|
||||
return fmt.Sprintf("%s|%d|%s$%s!%s/%s|%s|%s",
|
||||
query, pagesize,
|
||||
ref.GetResourceId().GetStorageId(), ref.GetResourceId().GetSpaceId(), ref.GetResourceId().GetOpaqueId(),
|
||||
ref.GetPath(), user.GetId().GetOpaqueId(), protoPart)
|
||||
func cacheKey(query string, pagesize int32, ref *v0.Reference, user *user.User) string {
|
||||
return fmt.Sprintf("%s|%d|%s$%s!%s/%s|%s", query, pagesize, ref.GetResourceId().GetStorageId(), ref.GetResourceId().GetSpaceId(), ref.GetResourceId().GetOpaqueId(), ref.GetPath(), user.GetId().GetOpaqueId())
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
"github.com/jellydator/ttlcache/v2"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/auth/scope"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/token/manager/jwt"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go-micro.dev/v4/metadata"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
searchmocks "github.com/opencloud-eu/opencloud/services/search/pkg/search/mocks"
|
||||
)
|
||||
|
||||
func newTestService(t *testing.T, searcher *searchmocks.Searcher) (Service, context.Context) {
|
||||
t.Helper()
|
||||
|
||||
tm, err := jwt.New(map[string]interface{}{"secret": "test-secret"})
|
||||
require.NoError(t, err)
|
||||
|
||||
u := &user.User{Id: &user.UserId{OpaqueId: "test-user", Idp: "idp"}, Username: "test"}
|
||||
scopes, err := scope.AddOwnerScope(nil)
|
||||
require.NoError(t, err)
|
||||
tok, err := tm.MintToken(context.Background(), u, scopes)
|
||||
require.NoError(t, err)
|
||||
ctx := metadata.Set(context.Background(), revactx.TokenHeader, tok)
|
||||
|
||||
cache := ttlcache.NewCache()
|
||||
require.NoError(t, cache.SetTTL(30*time.Second))
|
||||
|
||||
logger := log.NopLogger()
|
||||
return Service{
|
||||
log: &logger,
|
||||
searcher: searcher,
|
||||
cache: cache,
|
||||
tokenManager: tm,
|
||||
}, ctx
|
||||
}
|
||||
|
||||
func TestServiceSearchForwardsOrderBy(t *testing.T) {
|
||||
searcher := searchmocks.NewSearcher(t)
|
||||
svc, ctx := newTestService(t, searcher)
|
||||
|
||||
var captured *searchsvc.SearchRequest
|
||||
searcher.EXPECT().
|
||||
Search(mock.Anything, mock.Anything).
|
||||
Run(func(_ context.Context, req *searchsvc.SearchRequest) {
|
||||
captured = req
|
||||
}).
|
||||
Return(&searchsvc.SearchResponse{}, nil).
|
||||
Once()
|
||||
|
||||
err := svc.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:image",
|
||||
OrderBy: []*searchsvc.SortProperty{
|
||||
{Name: "photo.takenDateTime", IsDescending: true},
|
||||
},
|
||||
}, &searchsvc.SearchResponse{})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, captured)
|
||||
require.Len(t, captured.OrderBy, 1)
|
||||
require.Equal(t, "photo.takenDateTime", captured.OrderBy[0].Name)
|
||||
require.True(t, captured.OrderBy[0].IsDescending)
|
||||
}
|
||||
|
||||
func TestServiceSearchCacheDistinguishesOrderBy(t *testing.T) {
|
||||
searcher := searchmocks.NewSearcher(t)
|
||||
svc, ctx := newTestService(t, searcher)
|
||||
|
||||
responseFor := func(name string) *searchsvc.SearchResponse {
|
||||
return &searchsvc.SearchResponse{
|
||||
TotalMatches: 1,
|
||||
Matches: []*searchmsg.Match{{Entity: &searchmsg.Entity{Name: name}}},
|
||||
}
|
||||
}
|
||||
searcher.EXPECT().
|
||||
Search(mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchRequest) bool {
|
||||
return len(req.OrderBy) > 0 && req.OrderBy[0].IsDescending
|
||||
})).
|
||||
Return(responseFor("newest.jpg"), nil).
|
||||
Once()
|
||||
searcher.EXPECT().
|
||||
Search(mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchRequest) bool {
|
||||
return len(req.OrderBy) > 0 && !req.OrderBy[0].IsDescending
|
||||
})).
|
||||
Return(responseFor("oldest.jpg"), nil).
|
||||
Once()
|
||||
|
||||
descOut := &searchsvc.SearchResponse{}
|
||||
require.NoError(t, svc.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:image",
|
||||
OrderBy: []*searchsvc.SortProperty{{Name: "photo.takenDateTime", IsDescending: true}},
|
||||
}, descOut))
|
||||
|
||||
ascOut := &searchsvc.SearchResponse{}
|
||||
require.NoError(t, svc.Search(ctx, &searchsvc.SearchRequest{
|
||||
Query: "mediatype:image",
|
||||
OrderBy: []*searchsvc.SortProperty{{Name: "photo.takenDateTime"}},
|
||||
}, ascOut))
|
||||
|
||||
require.Equal(t, "newest.jpg", descOut.Matches[0].Entity.Name)
|
||||
require.Equal(t, "oldest.jpg", ascOut.Matches[0].Entity.Name)
|
||||
}
|
||||
@@ -11,7 +11,7 @@ msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: \n"
|
||||
"Report-Msgid-Bugs-To: EMAIL\n"
|
||||
"POT-Creation-Date: 2026-08-17 23:18+0000\n"
|
||||
"POT-Creation-Date: 2026-09-07 23:16+0000\n"
|
||||
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
|
||||
"Last-Translator: Jiri Grönroos <jiri.gronroos@iki.fi>, 2026\n"
|
||||
"Language-Team: Finnish (https://app.transifex.com/opencloud-eu/teams/204053/fi/)\n"
|
||||
|
||||
@@ -35,7 +35,12 @@ Share behavior can be configured via environment variables:
|
||||
- Auto-acceptance of shares
|
||||
- Share permissions and restrictions
|
||||
|
||||
See the `frontend` service README for more details on share-related configuration options.
|
||||
### Auto-Accept Shares
|
||||
|
||||
When setting the `SHARING_AUTO_ACCEPT_SHARES` to `true` (sharing service), all
|
||||
incoming shares will be accepted automatically. Users can overwrite this
|
||||
setting individually in their profile. The deprecated
|
||||
`FRONTEND_AUTO_ACCEPT_SHARES` is still supported for backwards compatibility.
|
||||
|
||||
## Scalability
|
||||
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
//go:build !enable_vips
|
||||
|
||||
package preprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
thumbnailerErrors "github.com/opencloud-eu/opencloud/services/thumbnails/pkg/errors"
|
||||
)
|
||||
|
||||
// craftDimensionBomb encodes a tiny valid grayscale JPEG, then overwrites the
|
||||
// SOF0 width/height so the header declares huge dimensions while the payload
|
||||
// stays tiny.
|
||||
func craftDimensionBomb(width, height uint16) []byte {
|
||||
var buf bytes.Buffer
|
||||
Expect(jpeg.Encode(&buf, image.NewGray(image.Rect(0, 0, 8, 8)), &jpeg.Options{Quality: 10})).To(Succeed())
|
||||
b := buf.Bytes()
|
||||
for i := 0; i+9 < len(b); i++ {
|
||||
if b[i] == 0xff && b[i+1] == 0xc0 {
|
||||
b[i+5], b[i+6] = byte(height>>8), byte(height)
|
||||
b[i+7], b[i+8] = byte(width>>8), byte(width)
|
||||
return b
|
||||
}
|
||||
}
|
||||
Fail("no SOF0 marker in encoded jpeg")
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ = Describe("ImageDecoder dimension guard", func() {
|
||||
It("rejects a source whose declared dimensions exceed the limit before decoding", func() {
|
||||
dec := ImageDecoder{limit: decodeLimit{maxWidth: 7680, maxHeight: 7680}}
|
||||
_, err := dec.Convert(bytes.NewReader(craftDimensionBomb(20000, 20000)))
|
||||
Expect(err).To(MatchError(thumbnailerErrors.ErrImageTooLarge))
|
||||
})
|
||||
|
||||
It("decodes an image within the limit", func() {
|
||||
var buf bytes.Buffer
|
||||
Expect(jpeg.Encode(&buf, image.NewGray(image.Rect(0, 0, 800, 600)), nil)).To(Succeed())
|
||||
dec := ImageDecoder{limit: decodeLimit{maxWidth: 7680, maxHeight: 7680}}
|
||||
img, err := dec.Convert(bytes.NewReader(buf.Bytes()))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(img).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("applies no limit when the bounds are zero", func() {
|
||||
dec := ImageDecoder{}
|
||||
var buf bytes.Buffer
|
||||
Expect(jpeg.Encode(&buf, image.NewGray(image.Rect(0, 0, 16, 16)), nil)).To(Succeed())
|
||||
_, err := dec.Convert(bytes.NewReader(buf.Bytes()))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("rejects an oversized gif before decoding", func() {
|
||||
dec := GifDecoder{limit: decodeLimit{maxWidth: 7680, maxHeight: 7680}}
|
||||
// LSD declares a 20000x20000 logical screen
|
||||
g := []byte("GIF89a")
|
||||
g = append(g, 0x20, 0x4e, 0x20, 0x4e, 0xf0, 0x00, 0x00) // 20000x20000, gct flag
|
||||
g = append(g, bytes.Repeat([]byte{0}, 6)...) // minimal gct + terminator-ish
|
||||
_, err := dec.Convert(bytes.NewReader(g))
|
||||
Expect(err).To(MatchError(thumbnailerErrors.ErrImageTooLarge))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("dimension limit propagation", func() {
|
||||
limit := decodeLimit{maxWidth: 7680, maxHeight: 7680}
|
||||
opts := map[string]any{"maxInputWidth": 7680, "maxInputHeight": 7680}
|
||||
|
||||
It("threads the limit into decoders that recurse into ForType", func() {
|
||||
Expect(ForType("audio/mpeg", opts)).To(Equal(AudioDecoder{limit: limit}))
|
||||
Expect(ForType("application/vnd.geogebra.pinboard", opts)).To(Equal(GgpDecoder{limit: limit}))
|
||||
g, ok := ForType("application/vnd.geogebra.slides", opts).(GgsDecoder)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(g.limit).To(Equal(limit))
|
||||
})
|
||||
|
||||
It("rejects an oversized cover image embedded in a ggp file", func() {
|
||||
bomb := craftDimensionBomb(20000, 20000)
|
||||
payload := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(bomb)
|
||||
ggp := GGPStruct{}
|
||||
ggp.Sections = append(ggp.Sections, struct {
|
||||
Cards []struct {
|
||||
Element struct {
|
||||
Image struct{ Base64Image string }
|
||||
}
|
||||
}
|
||||
}{Cards: []struct {
|
||||
Element struct {
|
||||
Image struct{ Base64Image string }
|
||||
}
|
||||
}{{Element: struct {
|
||||
Image struct{ Base64Image string }
|
||||
}{Image: struct{ Base64Image string }{Base64Image: payload}}}}})
|
||||
raw, err := json.Marshal(ggp)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = GgpDecoder{limit: limit}.Convert(bytes.NewReader(raw))
|
||||
Expect(err).To(MatchError(thumbnailerErrors.ErrImageTooLarge))
|
||||
})
|
||||
})
|
||||
Loaded 100 of 130 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user