mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-08 11:53:07 -04:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Dominik Schmidt
|
4993c9c487 |
No files matched your search
@@ -62,8 +62,8 @@ require (
|
||||
github.com/onsi/gomega v1.42.1
|
||||
github.com/open-policy-agent/opa v1.19.1
|
||||
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067
|
||||
github.com/opencloud-eu/reva/v2 v2.49.1-0.20260903122659-26f34ec05774
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260901070651-08a5330ce57d
|
||||
github.com/opencloud-eu/reva/v2 v2.49.0
|
||||
github.com/opensearch-project/opensearch-go/v4 v4.7.3
|
||||
github.com/orcaman/concurrent-map v1.0.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
@@ -95,7 +95,7 @@ require (
|
||||
go-micro.dev/v4 v4.11.0
|
||||
go.etcd.io/bbolt v1.5.0
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
|
||||
go.opentelemetry.io/contrib/zpages v0.71.0
|
||||
go.opentelemetry.io/otel v1.46.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
|
||||
@@ -324,6 +324,7 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/pquerna/cachecontrol v0.2.0 // indirect
|
||||
github.com/prometheus/alertmanager v0.33.1 // indirect
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/procfs v0.21.1 // indirect
|
||||
github.com/prometheus/statsd_exporter v0.22.8 // indirect
|
||||
|
||||
@@ -940,10 +940,10 @@ github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-202505121527
|
||||
github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-20250512152754-23325793059a/go.mod h1:pjcozWijkNPbEtX5SIQaxEW/h8VAVZYTLx+70bmB3LY=
|
||||
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+lP5lUUIzjRGDg93WrQfZJZCaV1ZP3KeyXi8bzY=
|
||||
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067 h1:UkNMKauyJAzY6RE6mmthz9bQZLYkbvBuApm7ZDCparE=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU=
|
||||
github.com/opencloud-eu/reva/v2 v2.49.1-0.20260903122659-26f34ec05774 h1:8N42Pwzyafdxq66zYP6kJ1CH/QBRpxovQXq/0Kt6bJY=
|
||||
github.com/opencloud-eu/reva/v2 v2.49.1-0.20260903122659-26f34ec05774/go.mod h1:Frg+UWnVcSy+412UB3l2LcD0KY8ZNu1samimwkNywbg=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260901070651-08a5330ce57d h1:3zbb31655ZvqwLMWMIiUAhYer6YcFnv/oguujnX7aOU=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260901070651-08a5330ce57d/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU=
|
||||
github.com/opencloud-eu/reva/v2 v2.49.0 h1:AwECMDDth3NUaihZRf9bI9HNpWutvaRijOCvQyQfTT4=
|
||||
github.com/opencloud-eu/reva/v2 v2.49.0/go.mod h1:Frg+UWnVcSy+412UB3l2LcD0KY8ZNu1samimwkNywbg=
|
||||
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
|
||||
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
@@ -1001,6 +1001,8 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:Om
|
||||
github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k=
|
||||
github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI=
|
||||
github.com/pquerna/otp v1.3.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
|
||||
github.com/prometheus/alertmanager v0.33.1 h1:PJHGvTdb8Q0ZEpJnWff120WxB2kxIScLa707AaIudTo=
|
||||
github.com/prometheus/alertmanager v0.33.1/go.mod h1:V06Uc8EZ5X5wLOJRGhtXx+EE2LgrinFIADbKWMVm1RY=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
|
||||
@@ -1293,8 +1295,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
|
||||
go.opentelemetry.io/contrib/zpages v0.71.0 h1:OrQSEmdlfknIusEyGj/t30uBR7MnbmGcCRBt5ov8/rA=
|
||||
go.opentelemetry.io/contrib/zpages v0.71.0/go.mod h1:C3IIpqkQbjswcSREqxQLyEapF0SI9bqwJpxpjwtzFrE=
|
||||
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
|
||||
|
||||
@@ -7,6 +7,12 @@ ifdef ENABLE_VIPS
|
||||
TAGS := ${TAGS},enable_vips
|
||||
endif
|
||||
|
||||
# bleve KNN support; needs libfaiss_c at build and run time (see the vectors
|
||||
# stage in docker/Dockerfile.multiarch)
|
||||
ifdef ENABLE_VECTORS
|
||||
TAGS := ${TAGS},vectors
|
||||
endif
|
||||
|
||||
ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI
|
||||
include ../.bingo/Variables.mk
|
||||
endif
|
||||
|
||||
@@ -1,3 +1,24 @@
|
||||
FROM quay.io/opencloudeu/golang-ci:1.25 AS faiss
|
||||
# pinned to the last bleve-branch commit before the size-API change in
|
||||
# Index_c_ex.h, matching the vendored go-faiss v1.1.0
|
||||
ARG FAISS_REF=3ea23cd9bfdc9b0ca6d48beca0c85cb208a18119
|
||||
RUN apk add --no-cache cmake g++ make git openblas-dev && \
|
||||
git clone https://github.com/blevesearch/faiss.git /faiss && \
|
||||
git -C /faiss checkout --detach "$FAISS_REF" && \
|
||||
cmake -S /faiss -B /faiss/build \
|
||||
-DFAISS_ENABLE_GPU=OFF \
|
||||
-DFAISS_ENABLE_PYTHON=OFF \
|
||||
-DFAISS_ENABLE_C_API=ON \
|
||||
-DBUILD_SHARED_LIBS=ON \
|
||||
-DBUILD_TESTING=OFF \
|
||||
-DFAISS_OPT_LEVEL=generic \
|
||||
-DCMAKE_BUILD_TYPE=Release && \
|
||||
make -C /faiss/build -j"$(nproc)" faiss faiss_c && \
|
||||
make -C /faiss/build install && \
|
||||
cp /faiss/build/c_api/libfaiss_c.so /usr/local/lib/ && \
|
||||
mkdir -p /usr/local/include/faiss/c_api && \
|
||||
cd /faiss/c_api && find . -name '*.h' -exec cp --parents {} /usr/local/include/faiss/c_api/ \;
|
||||
|
||||
FROM quay.io/opencloudeu/golang-ci:1.25 AS build
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
@@ -6,13 +27,18 @@ ARG STRING
|
||||
ARG EDITION="dev"
|
||||
ARG SRCDIR
|
||||
|
||||
# openblas resolves the BLAS/LAPACK symbols libfaiss references at link time
|
||||
RUN apk add --no-cache openblas
|
||||
COPY --from=faiss /usr/local/lib/libfaiss.so /usr/local/lib/libfaiss_c.so /usr/local/lib/
|
||||
COPY --from=faiss /usr/local/include/faiss /usr/local/include/faiss
|
||||
|
||||
WORKDIR /build
|
||||
RUN --mount=type=bind,target=/build,rw \
|
||||
--mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache \
|
||||
GOOS="${TARGETOS:-linux}" GOARCH="${TARGETARCH:-amd64}" ; \
|
||||
make -C ${SRCDIR:-.}/opencloud release-linux-docker-${TARGETARCH} \
|
||||
ENABLE_VIPS=true DIST=/dist \
|
||||
ENABLE_VIPS=true ENABLE_VECTORS=true DIST=/dist \
|
||||
VERSION="${VERSION}" EDITION="${EDITION}" \
|
||||
${STRING:+STRING="${STRING}"}
|
||||
|
||||
@@ -23,10 +49,12 @@ ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN apk add --no-cache attr bash ca-certificates curl imagemagick \
|
||||
inotify-tools libc6-compat mailcap tree vips \
|
||||
vips-magick patch && \
|
||||
inotify-tools libc6-compat libgomp libstdc++ mailcap \
|
||||
openblas tree vips vips-magick patch && \
|
||||
echo 'hosts: files dns' >| /etc/nsswitch.conf
|
||||
|
||||
COPY --from=faiss /usr/local/lib/libfaiss.so /usr/local/lib/libfaiss_c.so /usr/lib/
|
||||
|
||||
LABEL maintainer="openCloud GmbH <devops@opencloud.eu>" \
|
||||
org.opencontainers.image.title="OpenCloud" \
|
||||
org.opencontainers.image.vendor="OpenCloud GmbH" \
|
||||
|
||||
@@ -717,85 +717,6 @@ func (x *MotionPhoto) GetVideoSize() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
type LivePhoto struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
ContentId *string `protobuf:"bytes,1,opt,name=contentId,proto3,oneof" json:"contentId,omitempty"`
|
||||
StillImageTimeUs *int64 `protobuf:"varint,2,opt,name=stillImageTimeUs,proto3,oneof" json:"stillImageTimeUs,omitempty"`
|
||||
Auto *bool `protobuf:"varint,3,opt,name=auto,proto3,oneof" json:"auto,omitempty"`
|
||||
VitalityScore *float64 `protobuf:"fixed64,4,opt,name=vitalityScore,proto3,oneof" json:"vitalityScore,omitempty"`
|
||||
VitalityScoringVersion *int64 `protobuf:"varint,5,opt,name=vitalityScoringVersion,proto3,oneof" json:"vitalityScoringVersion,omitempty"`
|
||||
}
|
||||
|
||||
func (x *LivePhoto) Reset() {
|
||||
*x = LivePhoto{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[8]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *LivePhoto) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*LivePhoto) ProtoMessage() {}
|
||||
|
||||
func (x *LivePhoto) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[8]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use LivePhoto.ProtoReflect.Descriptor instead.
|
||||
func (*LivePhoto) Descriptor() ([]byte, []int) {
|
||||
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{8}
|
||||
}
|
||||
|
||||
func (x *LivePhoto) GetContentId() string {
|
||||
if x != nil && x.ContentId != nil {
|
||||
return *x.ContentId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *LivePhoto) GetStillImageTimeUs() int64 {
|
||||
if x != nil && x.StillImageTimeUs != nil {
|
||||
return *x.StillImageTimeUs
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *LivePhoto) GetAuto() bool {
|
||||
if x != nil && x.Auto != nil {
|
||||
return *x.Auto
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *LivePhoto) GetVitalityScore() float64 {
|
||||
if x != nil && x.VitalityScore != nil {
|
||||
return *x.VitalityScore
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *LivePhoto) GetVitalityScoringVersion() int64 {
|
||||
if x != nil && x.VitalityScoringVersion != nil {
|
||||
return *x.VitalityScoringVersion
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type Entity struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -823,13 +744,12 @@ type Entity struct {
|
||||
Favorites []string `protobuf:"bytes,20,rep,name=favorites,proto3" json:"favorites,omitempty"`
|
||||
MotionPhoto *MotionPhoto `protobuf:"bytes,21,opt,name=motionPhoto,proto3" json:"motionPhoto,omitempty"`
|
||||
Video *Video `protobuf:"bytes,22,opt,name=video,proto3" json:"video,omitempty"`
|
||||
LivePhoto *LivePhoto `protobuf:"bytes,23,opt,name=livePhoto,proto3" json:"livePhoto,omitempty"`
|
||||
}
|
||||
|
||||
func (x *Entity) Reset() {
|
||||
*x = Entity{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[9]
|
||||
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[8]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -842,7 +762,7 @@ func (x *Entity) String() string {
|
||||
func (*Entity) ProtoMessage() {}
|
||||
|
||||
func (x *Entity) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[9]
|
||||
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[8]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -855,7 +775,7 @@ func (x *Entity) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Entity.ProtoReflect.Descriptor instead.
|
||||
func (*Entity) Descriptor() ([]byte, []int) {
|
||||
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{9}
|
||||
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{8}
|
||||
}
|
||||
|
||||
func (x *Entity) GetRef() *Reference {
|
||||
@@ -1012,13 +932,6 @@ func (x *Entity) GetVideo() *Video {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Entity) GetLivePhoto() *LivePhoto {
|
||||
if x != nil {
|
||||
return x.LivePhoto
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Match struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -1033,7 +946,7 @@ type Match struct {
|
||||
func (x *Match) Reset() {
|
||||
*x = Match{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[10]
|
||||
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[9]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1046,7 +959,7 @@ func (x *Match) String() string {
|
||||
func (*Match) ProtoMessage() {}
|
||||
|
||||
func (x *Match) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[10]
|
||||
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[9]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1059,7 +972,7 @@ func (x *Match) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Match.ProtoReflect.Descriptor instead.
|
||||
func (*Match) Descriptor() ([]byte, []int) {
|
||||
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{10}
|
||||
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{9}
|
||||
}
|
||||
|
||||
func (x *Match) GetEntity() *Entity {
|
||||
@@ -1239,107 +1152,82 @@ var file_opencloud_messages_search_v0_search_proto_rawDesc = []byte{
|
||||
0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x1a, 0x0a, 0x18,
|
||||
0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d,
|
||||
0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x55, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x76, 0x69, 0x64,
|
||||
0x65, 0x6f, 0x53, 0x69, 0x7a, 0x65, 0x22, 0xb9, 0x02, 0x0a, 0x09, 0x4c, 0x69, 0x76, 0x65, 0x50,
|
||||
0x68, 0x6f, 0x74, 0x6f, 0x12, 0x21, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x49,
|
||||
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65,
|
||||
0x6e, 0x74, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x2f, 0x0a, 0x10, 0x73, 0x74, 0x69, 0x6c, 0x6c,
|
||||
0x49, 0x6d, 0x61, 0x67, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28,
|
||||
0x03, 0x48, 0x01, 0x52, 0x10, 0x73, 0x74, 0x69, 0x6c, 0x6c, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x54,
|
||||
0x69, 0x6d, 0x65, 0x55, 0x73, 0x88, 0x01, 0x01, 0x12, 0x17, 0x0a, 0x04, 0x61, 0x75, 0x74, 0x6f,
|
||||
0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x48, 0x02, 0x52, 0x04, 0x61, 0x75, 0x74, 0x6f, 0x88, 0x01,
|
||||
0x01, 0x12, 0x29, 0x0a, 0x0d, 0x76, 0x69, 0x74, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f,
|
||||
0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x48, 0x03, 0x52, 0x0d, 0x76, 0x69, 0x74, 0x61,
|
||||
0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x88, 0x01, 0x01, 0x12, 0x3b, 0x0a, 0x16,
|
||||
0x76, 0x69, 0x74, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x56,
|
||||
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x48, 0x04, 0x52, 0x16,
|
||||
0x76, 0x69, 0x74, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x56,
|
||||
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, 0x6f,
|
||||
0x6e, 0x74, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x73, 0x74, 0x69, 0x6c,
|
||||
0x6c, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x42, 0x07, 0x0a, 0x05,
|
||||
0x5f, 0x61, 0x75, 0x74, 0x6f, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x76, 0x69, 0x74, 0x61, 0x6c, 0x69,
|
||||
0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x19, 0x0a, 0x17, 0x5f, 0x76, 0x69, 0x74, 0x61,
|
||||
0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x65, 0x72, 0x73, 0x69,
|
||||
0x6f, 0x6e, 0x22, 0xc9, 0x08, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a,
|
||||
0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65,
|
||||
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
|
||||
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65,
|
||||
0x6e, 0x63, 0x65, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x38, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
|
||||
0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
|
||||
0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x02,
|
||||
0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
|
||||
0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x74, 0x61, 0x67, 0x18, 0x04,
|
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x65, 0x74, 0x61, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69,
|
||||
0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x48,
|
||||
0x0a, 0x12, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f,
|
||||
0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f,
|
||||
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d,
|
||||
0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69,
|
||||
0x66, 0x69, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x69, 0x6d, 0x65,
|
||||
0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d,
|
||||
0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73,
|
||||
0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x65, 0x72, 0x6d,
|
||||
0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18,
|
||||
0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64,
|
||||
0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x64, 0x65,
|
||||
0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x65, 0x52, 0x6f,
|
||||
0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x68,
|
||||
0x61, 0x72, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x45, 0x0a, 0x09, 0x70,
|
||||
0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28,
|
||||
0x65, 0x6f, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x82, 0x08, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74,
|
||||
0x79, 0x12, 0x39, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27,
|
||||
0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61,
|
||||
0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65,
|
||||
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74,
|
||||
0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x09,
|
||||
0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x68, 0x69, 0x67, 0x68, 0x6c, 0x69,
|
||||
0x67, 0x68, 0x74, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x68, 0x69, 0x67, 0x68,
|
||||
0x6c, 0x69, 0x67, 0x68, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x05, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x18,
|
||||
0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75,
|
||||
0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63,
|
||||
0x68, 0x2e, 0x76, 0x30, 0x2e, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x52, 0x05, 0x61, 0x75, 0x64, 0x69,
|
||||
0x6f, 0x12, 0x48, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x10, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e,
|
||||
0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e,
|
||||
0x76, 0x30, 0x2e, 0x47, 0x65, 0x6f, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x65,
|
||||
0x73, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4e, 0x0a, 0x0e, 0x72,
|
||||
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x11, 0x20,
|
||||
0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e,
|
||||
0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e,
|
||||
0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x0c, 0x72,
|
||||
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x12, 0x39, 0x0a, 0x05, 0x69,
|
||||
0x6d, 0x61, 0x67, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65,
|
||||
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
|
||||
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52,
|
||||
0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x05, 0x70, 0x68, 0x6f, 0x74, 0x6f, 0x18,
|
||||
0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75,
|
||||
0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63,
|
||||
0x68, 0x2e, 0x76, 0x30, 0x2e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x52, 0x05, 0x70, 0x68, 0x6f, 0x74,
|
||||
0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74, 0x65, 0x73, 0x18, 0x14,
|
||||
0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74, 0x65, 0x73, 0x12,
|
||||
0x4b, 0x0a, 0x0b, 0x6d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x18, 0x15,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
|
||||
0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
|
||||
0x2e, 0x76, 0x30, 0x2e, 0x4d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x52,
|
||||
0x0b, 0x6d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x12, 0x39, 0x0a, 0x05,
|
||||
0x76, 0x69, 0x64, 0x65, 0x6f, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70,
|
||||
0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73,
|
||||
0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x56, 0x69, 0x64, 0x65, 0x6f,
|
||||
0x52, 0x05, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x45, 0x0a, 0x09, 0x6c, 0x69, 0x76, 0x65, 0x50,
|
||||
0x68, 0x6f, 0x74, 0x6f, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65,
|
||||
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
|
||||
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x4c, 0x69, 0x76, 0x65, 0x50, 0x68,
|
||||
0x6f, 0x74, 0x6f, 0x52, 0x09, 0x6c, 0x69, 0x76, 0x65, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x22, 0x5b,
|
||||
0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x3c, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74,
|
||||
0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
|
||||
0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x38, 0x0a, 0x02,
|
||||
0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63,
|
||||
0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65,
|
||||
0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
|
||||
0x49, 0x44, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03,
|
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x74,
|
||||
0x61, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x65, 0x74, 0x61, 0x67, 0x12, 0x12,
|
||||
0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69,
|
||||
0x7a, 0x65, 0x12, 0x48, 0x0a, 0x12, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66,
|
||||
0x69, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a,
|
||||
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
|
||||
0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74,
|
||||
0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09,
|
||||
0x6d, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x08, 0x6d, 0x69, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x65, 0x72,
|
||||
0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b,
|
||||
0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74,
|
||||
0x79, 0x70, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12,
|
||||
0x18, 0x0a, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08,
|
||||
0x52, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x68, 0x61,
|
||||
0x72, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09,
|
||||
0x52, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12,
|
||||
0x45, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d,
|
||||
0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76,
|
||||
0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x08, 0x70, 0x61,
|
||||
0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x0d,
|
||||
0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x68, 0x69,
|
||||
0x67, 0x68, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a,
|
||||
0x68, 0x69, 0x67, 0x68, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x05, 0x61, 0x75,
|
||||
0x64, 0x69, 0x6f, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e,
|
||||
0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73,
|
||||
0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x52, 0x05,
|
||||
0x61, 0x75, 0x64, 0x69, 0x6f, 0x12, 0x48, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f,
|
||||
0x6e, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
|
||||
0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61,
|
||||
0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65,
|
||||
0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02,
|
||||
0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x4d, 0x5a, 0x4b, 0x67,
|
||||
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
|
||||
0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
|
||||
0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70,
|
||||
0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73,
|
||||
0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x33,
|
||||
0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x47, 0x65, 0x6f, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69,
|
||||
0x6e, 0x61, 0x74, 0x65, 0x73, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12,
|
||||
0x4e, 0x0a, 0x0e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69,
|
||||
0x64, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
|
||||
0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61,
|
||||
0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49,
|
||||
0x44, 0x52, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x12,
|
||||
0x39, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23,
|
||||
0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61,
|
||||
0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6d,
|
||||
0x61, 0x67, 0x65, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x05, 0x70, 0x68,
|
||||
0x6f, 0x74, 0x6f, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e,
|
||||
0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73,
|
||||
0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x52, 0x05,
|
||||
0x70, 0x68, 0x6f, 0x74, 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74,
|
||||
0x65, 0x73, 0x18, 0x14, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69,
|
||||
0x74, 0x65, 0x73, 0x12, 0x4b, 0x0a, 0x0b, 0x6d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f,
|
||||
0x74, 0x6f, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63,
|
||||
0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65,
|
||||
0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x4d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68,
|
||||
0x6f, 0x74, 0x6f, 0x52, 0x0b, 0x6d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f,
|
||||
0x12, 0x39, 0x0a, 0x05, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32,
|
||||
0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73,
|
||||
0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x56,
|
||||
0x69, 0x64, 0x65, 0x6f, 0x52, 0x05, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x22, 0x5b, 0x0a, 0x05, 0x4d,
|
||||
0x61, 0x74, 0x63, 0x68, 0x12, 0x3c, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
|
||||
0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
|
||||
0x2e, 0x76, 0x30, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69,
|
||||
0x74, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
|
||||
0x02, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x4d, 0x5a, 0x4b, 0x67, 0x69, 0x74, 0x68,
|
||||
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
|
||||
0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63,
|
||||
0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65,
|
||||
0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -1354,7 +1242,7 @@ func file_opencloud_messages_search_v0_search_proto_rawDescGZIP() []byte {
|
||||
return file_opencloud_messages_search_v0_search_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_opencloud_messages_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 11)
|
||||
var file_opencloud_messages_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
|
||||
var file_opencloud_messages_search_v0_search_proto_goTypes = []interface{}{
|
||||
(*ResourceID)(nil), // 0: opencloud.messages.search.v0.ResourceID
|
||||
(*Reference)(nil), // 1: opencloud.messages.search.v0.Reference
|
||||
@@ -1364,17 +1252,16 @@ var file_opencloud_messages_search_v0_search_proto_goTypes = []interface{}{
|
||||
(*Photo)(nil), // 5: opencloud.messages.search.v0.Photo
|
||||
(*Video)(nil), // 6: opencloud.messages.search.v0.Video
|
||||
(*MotionPhoto)(nil), // 7: opencloud.messages.search.v0.MotionPhoto
|
||||
(*LivePhoto)(nil), // 8: opencloud.messages.search.v0.LivePhoto
|
||||
(*Entity)(nil), // 9: opencloud.messages.search.v0.Entity
|
||||
(*Match)(nil), // 10: opencloud.messages.search.v0.Match
|
||||
(*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp
|
||||
(*Entity)(nil), // 8: opencloud.messages.search.v0.Entity
|
||||
(*Match)(nil), // 9: opencloud.messages.search.v0.Match
|
||||
(*timestamppb.Timestamp)(nil), // 10: google.protobuf.Timestamp
|
||||
}
|
||||
var file_opencloud_messages_search_v0_search_proto_depIdxs = []int32{
|
||||
0, // 0: opencloud.messages.search.v0.Reference.resource_id:type_name -> opencloud.messages.search.v0.ResourceID
|
||||
11, // 1: opencloud.messages.search.v0.Photo.takenDateTime:type_name -> google.protobuf.Timestamp
|
||||
10, // 1: opencloud.messages.search.v0.Photo.takenDateTime:type_name -> google.protobuf.Timestamp
|
||||
1, // 2: opencloud.messages.search.v0.Entity.ref:type_name -> opencloud.messages.search.v0.Reference
|
||||
0, // 3: opencloud.messages.search.v0.Entity.id:type_name -> opencloud.messages.search.v0.ResourceID
|
||||
11, // 4: opencloud.messages.search.v0.Entity.last_modified_time:type_name -> google.protobuf.Timestamp
|
||||
10, // 4: opencloud.messages.search.v0.Entity.last_modified_time:type_name -> google.protobuf.Timestamp
|
||||
0, // 5: opencloud.messages.search.v0.Entity.parent_id:type_name -> opencloud.messages.search.v0.ResourceID
|
||||
2, // 6: opencloud.messages.search.v0.Entity.audio:type_name -> opencloud.messages.search.v0.Audio
|
||||
4, // 7: opencloud.messages.search.v0.Entity.location:type_name -> opencloud.messages.search.v0.GeoCoordinates
|
||||
@@ -1383,13 +1270,12 @@ var file_opencloud_messages_search_v0_search_proto_depIdxs = []int32{
|
||||
5, // 10: opencloud.messages.search.v0.Entity.photo:type_name -> opencloud.messages.search.v0.Photo
|
||||
7, // 11: opencloud.messages.search.v0.Entity.motionPhoto:type_name -> opencloud.messages.search.v0.MotionPhoto
|
||||
6, // 12: opencloud.messages.search.v0.Entity.video:type_name -> opencloud.messages.search.v0.Video
|
||||
8, // 13: opencloud.messages.search.v0.Entity.livePhoto:type_name -> opencloud.messages.search.v0.LivePhoto
|
||||
9, // 14: opencloud.messages.search.v0.Match.entity:type_name -> opencloud.messages.search.v0.Entity
|
||||
15, // [15:15] is the sub-list for method output_type
|
||||
15, // [15:15] is the sub-list for method input_type
|
||||
15, // [15:15] is the sub-list for extension type_name
|
||||
15, // [15:15] is the sub-list for extension extendee
|
||||
0, // [0:15] is the sub-list for field type_name
|
||||
8, // 13: opencloud.messages.search.v0.Match.entity:type_name -> opencloud.messages.search.v0.Entity
|
||||
14, // [14:14] is the sub-list for method output_type
|
||||
14, // [14:14] is the sub-list for method input_type
|
||||
14, // [14:14] is the sub-list for extension type_name
|
||||
14, // [14:14] is the sub-list for extension extendee
|
||||
0, // [0:14] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_opencloud_messages_search_v0_search_proto_init() }
|
||||
@@ -1495,18 +1381,6 @@ func file_opencloud_messages_search_v0_search_proto_init() {
|
||||
}
|
||||
}
|
||||
file_opencloud_messages_search_v0_search_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*LivePhoto); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_opencloud_messages_search_v0_search_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*Entity); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
@@ -1518,7 +1392,7 @@ func file_opencloud_messages_search_v0_search_proto_init() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_opencloud_messages_search_v0_search_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
|
||||
file_opencloud_messages_search_v0_search_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*Match); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
@@ -1537,14 +1411,13 @@ func file_opencloud_messages_search_v0_search_proto_init() {
|
||||
file_opencloud_messages_search_v0_search_proto_msgTypes[5].OneofWrappers = []interface{}{}
|
||||
file_opencloud_messages_search_v0_search_proto_msgTypes[6].OneofWrappers = []interface{}{}
|
||||
file_opencloud_messages_search_v0_search_proto_msgTypes[7].OneofWrappers = []interface{}{}
|
||||
file_opencloud_messages_search_v0_search_proto_msgTypes[8].OneofWrappers = []interface{}{}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
File: protoimpl.DescBuilder{
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_opencloud_messages_search_v0_search_proto_rawDesc,
|
||||
NumEnums: 0,
|
||||
NumMessages: 11,
|
||||
NumMessages: 10,
|
||||
NumExtensions: 0,
|
||||
NumServices: 0,
|
||||
},
|
||||
|
||||
@@ -304,9 +304,6 @@
|
||||
},
|
||||
"video": {
|
||||
"$ref": "#/definitions/v0Video"
|
||||
},
|
||||
"livePhoto": {
|
||||
"$ref": "#/definitions/v0LivePhoto"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -385,29 +382,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0LivePhoto": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"contentId": {
|
||||
"type": "string"
|
||||
},
|
||||
"stillImageTimeUs": {
|
||||
"type": "string",
|
||||
"format": "int64"
|
||||
},
|
||||
"auto": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"vitalityScore": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
},
|
||||
"vitalityScoringVersion": {
|
||||
"type": "string",
|
||||
"format": "int64"
|
||||
}
|
||||
}
|
||||
},
|
||||
"v0Match": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
@@ -78,14 +78,6 @@ message MotionPhoto {
|
||||
optional int64 videoSize = 3;
|
||||
}
|
||||
|
||||
message LivePhoto {
|
||||
optional string contentId = 1;
|
||||
optional int64 stillImageTimeUs = 2;
|
||||
optional bool auto = 3;
|
||||
optional double vitalityScore = 4;
|
||||
optional int64 vitalityScoringVersion = 5;
|
||||
}
|
||||
|
||||
message Entity {
|
||||
Reference ref = 1;
|
||||
ResourceID id = 2;
|
||||
@@ -109,7 +101,6 @@ message Entity {
|
||||
repeated string favorites = 20;
|
||||
MotionPhoto motionPhoto = 21;
|
||||
Video video = 22;
|
||||
LivePhoto livePhoto = 23;
|
||||
}
|
||||
|
||||
message Match {
|
||||
|
||||
@@ -3,18 +3,17 @@ package command
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/olekukonko/errors"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/pkg/runner"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/pkg/generators"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
natspkg "github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/registry"
|
||||
"github.com/opencloud-eu/opencloud/pkg/runner"
|
||||
ogrpc "github.com/opencloud-eu/opencloud/pkg/service/grpc"
|
||||
"github.com/opencloud-eu/opencloud/pkg/tracing"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
@@ -25,12 +24,6 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/metrics"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/server/debug"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/server/http"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
svcEvents "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events"
|
||||
svcHttp "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
var _registeredEvents = []events.Unmarshaller{
|
||||
@@ -69,11 +62,19 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
|
||||
gr := runner.NewGroup()
|
||||
ctx, cancel := context.WithCancel(cmd.Context())
|
||||
defer cancel()
|
||||
|
||||
mtrcs := metrics.New()
|
||||
mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1)
|
||||
|
||||
defer cancel()
|
||||
|
||||
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
|
||||
evStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig(cfg.Events))
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to initialize event stream")
|
||||
return err
|
||||
}
|
||||
|
||||
tm, err := pool.StringToTLSMode(cfg.GRPCClientTLS.Mode)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to parse tls mode")
|
||||
@@ -98,101 +99,28 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
return err
|
||||
}
|
||||
|
||||
kv, err := ConnectNatsKV(cfg.Store)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
activityLog, err := activitylog.New(kv,
|
||||
activitylog.Logger(logger),
|
||||
activitylog.MaxActivities(cfg.MaxActivities),
|
||||
activitylog.WriteBufferDuration(cfg.WriteBufferDuration),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to initialize activity log")
|
||||
return err
|
||||
}
|
||||
hClient := ehsvc.NewEventHistoryService("eu.opencloud.api.eventhistory", grpcClient)
|
||||
vClient := settingssvc.NewValueService("eu.opencloud.api.settings", grpcClient)
|
||||
|
||||
if !cfg.HTTP.Disabled {
|
||||
|
||||
hClient := ehsvc.NewEventHistoryService("eu.opencloud.api.eventhistory", grpcClient)
|
||||
|
||||
svc, err := svcHttp.New(
|
||||
activityLog,
|
||||
svcHttp.Logger(logger),
|
||||
svcHttp.GatewaySelector(gatewaySelector),
|
||||
svcHttp.RegisteredEvents(_registeredEvents),
|
||||
//svcHttp.TraceProvider(tracerProvider),
|
||||
svcHttp.HistoryClient(hClient),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("handler init")
|
||||
return err
|
||||
}
|
||||
// TODO svc = service.NewInstrument(svc, metrics)
|
||||
// TODO svc = service.NewLogging(svc, logger) // this logs service specific data
|
||||
// TODO svc = service.NewTracing(svc, traceProvider)
|
||||
vClient := settingssvc.NewValueService("eu.opencloud.api.settings", grpcClient)
|
||||
|
||||
server, err := http.Server(
|
||||
http.ValueClient(vClient),
|
||||
{
|
||||
svc, err := http.Server(
|
||||
http.Logger(logger),
|
||||
http.Context(ctx),
|
||||
http.Config(cfg),
|
||||
http.Service(svc),
|
||||
http.Context(ctx), // NOTE: not passing this "option" leads to a panic in go-micro
|
||||
http.TraceProvider(tracerProvider),
|
||||
http.Stream(evStream),
|
||||
http.GatewaySelector(gatewaySelector),
|
||||
http.HistoryClient(hClient),
|
||||
http.ValueClient(vClient),
|
||||
http.RegisteredEvents(_registeredEvents),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Info().
|
||||
Err(err).
|
||||
Str("transport", "http").
|
||||
Msg("Failed to initialize server")
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Str("transport", "http").Msg("Failed to initialize server")
|
||||
return err
|
||||
}
|
||||
|
||||
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", server))
|
||||
} else {
|
||||
logger.Info().Msg("HTTP server disabled, not starting HTTP service")
|
||||
}
|
||||
|
||||
if !cfg.Events.Disabled {
|
||||
|
||||
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
|
||||
evStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig{
|
||||
Endpoint: cfg.Events.Endpoint,
|
||||
Cluster: cfg.Events.Cluster,
|
||||
EnableTLS: cfg.Events.EnableTLS,
|
||||
TLSInsecure: cfg.Events.TLSInsecure,
|
||||
TLSRootCACertificate: cfg.Events.TLSRootCACertificate,
|
||||
AuthUsername: cfg.Events.AuthUsername,
|
||||
AuthPassword: cfg.Events.AuthPassword,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to initialize event stream")
|
||||
return err
|
||||
}
|
||||
|
||||
eventSvc, err := svcEvents.New(
|
||||
activityLog,
|
||||
evStream,
|
||||
svcEvents.Context(ctx),
|
||||
svcEvents.Logger(logger),
|
||||
svcEvents.ServiceAccount(cfg.ServiceAccount),
|
||||
svcEvents.GatewaySelector(gatewaySelector),
|
||||
svcEvents.RegisteredEvents(_registeredEvents),
|
||||
svcEvents.NumConsumers(cfg.NumConsumers),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Str("transport", "event").Msg("Failed to initialize server")
|
||||
return err
|
||||
}
|
||||
|
||||
gr.Add(runner.New(cfg.Service.Name+".svc", func() error {
|
||||
return eventSvc.Run()
|
||||
}, func() {
|
||||
eventSvc.Close()
|
||||
}))
|
||||
} else {
|
||||
logger.Info().Msg("event listening disabled, not starting event service")
|
||||
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", svc))
|
||||
}
|
||||
|
||||
{
|
||||
@@ -221,33 +149,3 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func ConnectNatsKV(cfg config.Store) (nats.KeyValue, error) {
|
||||
// Connect to NATS servers
|
||||
secureOption := natspkg.Secure(cfg.EnableTLS, cfg.TLSInsecure, cfg.TLSRootCACertificate)
|
||||
conn, err := nats.Connect(strings.Join(cfg.Nodes, ","), secureOption, nats.UserInfo(cfg.AuthUsername, cfg.AuthPassword))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
kv, err := js.KeyValue(cfg.Database)
|
||||
if err != nil {
|
||||
if !errors.Is(err, nats.ErrBucketNotFound) {
|
||||
return nil, errors.Wrapf(err, "Failed to get bucket (%s)", cfg.Database)
|
||||
}
|
||||
|
||||
kv, err = js.CreateKeyValue(&nats.KeyValueConfig{
|
||||
Bucket: cfg.Database,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to create bucket (%s)", cfg.Database)
|
||||
}
|
||||
}
|
||||
|
||||
return kv, nil
|
||||
}
|
||||
@@ -35,12 +35,10 @@ type Config struct {
|
||||
|
||||
WriteBufferDuration time.Duration `yaml:"write_buffer_duration" env:"ACTIVITYLOG_WRITE_BUFFER_DURATION" desc:"The duration to wait before flushing the write buffer. This is used to reduce the number of writes to the store." introductionVersion:"4.0.0"`
|
||||
MaxActivities int `yaml:"max_activities" env:"ACTIVITYLOG_MAX_ACTIVITIES" desc:"The maximum number of activities to keep in the store per resource. If the number of activities exceeds this value, the oldest activities will be removed." introductionVersion:"4.0.0"`
|
||||
NumConsumers int `yaml:"num_consumers" env:"ACTIVITYLOG_NUM_CONSUMERS" desc:"The amount of concurrent event consumers to start. Event consumers are used for updating the list of activities. Multiple consumers increase parallelisation, but will also increase CPU and memory demands." introductionVersion:"%NEXT%"`
|
||||
}
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
Disabled bool `yaml:"disabled" env:"ACTIVITYLOG_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%NEXT%"`
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
|
||||
@@ -79,7 +77,6 @@ type CORS struct {
|
||||
|
||||
// HTTP defines the available http configuration.
|
||||
type HTTP struct {
|
||||
Disabled bool `yaml:"disabled" env:"ACTIVITYLOG_HTTP_DISABLED" desc:"Disables the HTTP service. Set this to true if the service should only handle events." introductionVersion:"1.0.0"`
|
||||
Addr string `yaml:"addr" env:"ACTIVITYLOG_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
Root string `yaml:"root" env:"ACTIVITYLOG_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -52,9 +52,7 @@ func DefaultConfig() *config.Config {
|
||||
},
|
||||
},
|
||||
WriteBufferDuration: 10 * time.Second,
|
||||
// Nats runs into max payload exceeded errors at around 7k activities. Let's keep a buffer.
|
||||
MaxActivities: 6000,
|
||||
NumConsumers: 1,
|
||||
MaxActivities: 6000,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"errors"
|
||||
|
||||
occfg "github.com/opencloud-eu/opencloud/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/pkg/shared"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config/defaults"
|
||||
|
||||
@@ -35,8 +34,5 @@ func ParseConfig(cfg *config.Config) error {
|
||||
|
||||
// Validate validates the config
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.Events.Disabled && cfg.HTTP.Disabled {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -3,12 +3,18 @@ package http
|
||||
import (
|
||||
"context"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
"go-micro.dev/v4/store"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
@@ -16,15 +22,19 @@ type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Name string
|
||||
Namespace string
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Flags []pflag.Flag
|
||||
Service ActivityLogService
|
||||
TraceProvider trace.TracerProvider
|
||||
ValueClient settingssvc.ValueService
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Metrics *metrics.Metrics
|
||||
Flags []pflag.Flag
|
||||
Namespace string
|
||||
Store store.Store
|
||||
Stream events.Stream
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
TraceProvider trace.TracerProvider
|
||||
HistoryClient ehsvc.EventHistoryService
|
||||
ValueClient settingssvc.ValueService
|
||||
RegisteredEvents []events.Unmarshaller
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
@@ -59,10 +69,10 @@ func Config(val *config.Config) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// Service provides a function to set the service option.
|
||||
func Service(val ActivityLogService) Option {
|
||||
// Metrics provides a function to set the metrics option.
|
||||
func Metrics(val *metrics.Metrics) Option {
|
||||
return func(o *Options) {
|
||||
o.Service = val
|
||||
o.Metrics = val
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,20 +83,58 @@ func Flags(flags ...pflag.Flag) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider provides a function to configure the trace provider
|
||||
func TraceProvider(traceProvider trace.TracerProvider) Option {
|
||||
// Namespace provides a function to set the Namespace option.
|
||||
func Namespace(val string) Option {
|
||||
return func(o *Options) {
|
||||
if traceProvider != nil {
|
||||
o.TraceProvider = traceProvider
|
||||
} else {
|
||||
o.TraceProvider = noop.NewTracerProvider()
|
||||
}
|
||||
o.Namespace = val
|
||||
}
|
||||
}
|
||||
|
||||
// ValueClient adds a grpc client for the value service
|
||||
func ValueClient(vs settingssvc.ValueService) Option {
|
||||
// Store provides a function to configure the store
|
||||
func Store(store store.Store) Option {
|
||||
return func(o *Options) {
|
||||
o.ValueClient = vs
|
||||
o.Store = store
|
||||
}
|
||||
}
|
||||
|
||||
// Stream provides a function to configure the stream
|
||||
func Stream(stream events.Stream) Option {
|
||||
return func(o *Options) {
|
||||
o.Stream = stream
|
||||
}
|
||||
}
|
||||
|
||||
// GatewaySelector provides a function to configure the gateway client selector
|
||||
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
|
||||
return func(o *Options) {
|
||||
o.GatewaySelector = gatewaySelector
|
||||
}
|
||||
}
|
||||
|
||||
// HistoryClient provides a function to configure the event history client
|
||||
func HistoryClient(h ehsvc.EventHistoryService) Option {
|
||||
return func(o *Options) {
|
||||
o.HistoryClient = h
|
||||
}
|
||||
}
|
||||
|
||||
// RegisteredEvents provides a function to register events
|
||||
func RegisteredEvents(evs []events.Unmarshaller) Option {
|
||||
return func(o *Options) {
|
||||
o.RegisteredEvents = evs
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider provides a function to set the TracerProvider option
|
||||
func TraceProvider(val trace.TracerProvider) Option {
|
||||
return func(o *Options) {
|
||||
o.TraceProvider = val
|
||||
}
|
||||
}
|
||||
|
||||
// ValueClient provides a function to set the ValueClient options
|
||||
func ValueClient(val settingssvc.ValueService) Option {
|
||||
return func(o *Options) {
|
||||
o.ValueClient = val
|
||||
}
|
||||
}
|
||||
@@ -1,63 +1,49 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"fmt"
|
||||
|
||||
stdhttp "net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
chimiddleware "github.com/go-chi/chi/v5/middleware"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/opencloud/pkg/account"
|
||||
"github.com/opencloud-eu/opencloud/pkg/cors"
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/pkg/middleware"
|
||||
ohttp "github.com/opencloud-eu/opencloud/pkg/service/http"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/http"
|
||||
"github.com/opencloud-eu/opencloud/pkg/tracing"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
activityloghttp "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
svc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"
|
||||
"github.com/riandyrn/otelchi"
|
||||
"go-micro.dev/v4"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
var (
|
||||
//go:embed l10n/locale
|
||||
_localeFS embed.FS
|
||||
|
||||
// subfolder where the translation files are stored
|
||||
_localeSubPath = "l10n/locale"
|
||||
|
||||
// domain of the activitylog service (transifex)
|
||||
_domain = "activitylog"
|
||||
)
|
||||
// Service is the service interface
|
||||
type Service any
|
||||
|
||||
// Server initializes the http service and server.
|
||||
func Server(opts ...Option) (ohttp.Service, error) {
|
||||
func Server(opts ...Option) (http.Service, error) {
|
||||
options := newOptions(opts...)
|
||||
service := options.Service
|
||||
|
||||
newService, err := ohttp.NewService(
|
||||
ohttp.TLSConfig(options.Config.HTTP.TLS),
|
||||
ohttp.Logger(options.Logger),
|
||||
ohttp.Namespace(options.Config.HTTP.Namespace),
|
||||
ohttp.Name(options.Config.Service.Name),
|
||||
ohttp.Version(version.GetString()),
|
||||
ohttp.Address(options.Config.HTTP.Addr),
|
||||
ohttp.Context(options.Context),
|
||||
ohttp.Flags(options.Flags...),
|
||||
service, err := http.NewService(
|
||||
http.TLSConfig(options.Config.HTTP.TLS),
|
||||
http.Logger(options.Logger),
|
||||
http.Namespace(options.Config.HTTP.Namespace),
|
||||
http.Name(options.Config.Service.Name),
|
||||
http.Version(version.GetString()),
|
||||
http.Address(options.Config.HTTP.Addr),
|
||||
http.Context(options.Context),
|
||||
http.Flags(options.Flags...),
|
||||
http.TraceProvider(options.TraceProvider),
|
||||
)
|
||||
if err != nil {
|
||||
options.Logger.Error().
|
||||
Err(err).
|
||||
Msg("Error initializing http service")
|
||||
return ohttp.Service{}, err
|
||||
return http.Service{}, fmt.Errorf("could not initialize http service: %w", err)
|
||||
}
|
||||
|
||||
middlewares := []func(http.Handler) http.Handler{
|
||||
middlewares := []func(stdhttp.Handler) stdhttp.Handler{
|
||||
chimiddleware.RequestID,
|
||||
middleware.Version(
|
||||
options.Config.Service.Name,
|
||||
@@ -66,7 +52,6 @@ func Server(opts ...Option) (ohttp.Service, error) {
|
||||
middleware.Logger(
|
||||
options.Logger,
|
||||
),
|
||||
middleware.TraceContext,
|
||||
middleware.ExtractAccountUUID(
|
||||
account.Logger(options.Logger),
|
||||
account.JWTSecret(options.Config.TokenManager.JWTSecret),
|
||||
@@ -83,75 +68,33 @@ func Server(opts ...Option) (ohttp.Service, error) {
|
||||
mux := chi.NewMux()
|
||||
mux.Use(middlewares...)
|
||||
|
||||
t := l10n.NewTranslatorFromCommonConfig(options.Config.DefaultLanguage, _domain, options.Config.TranslationPath, _localeFS, _localeSubPath)
|
||||
mux.Route(options.Config.HTTP.Root, func(r chi.Router) {
|
||||
r.Get("/graph/v1beta1/extensions/org.libregraph/activities", GetItemActivitiesHandler(options.Logger, service, options.ValueClient, t))
|
||||
})
|
||||
mux.Use(
|
||||
otelchi.Middleware(
|
||||
"actitivylog",
|
||||
otelchi.WithChiRoutes(mux),
|
||||
otelchi.WithTracerProvider(options.TraceProvider),
|
||||
otelchi.WithPropagators(tracing.GetPropagator()),
|
||||
),
|
||||
)
|
||||
|
||||
err = micro.RegisterHandler(newService.Server(), mux)
|
||||
handle, err := svc.New(
|
||||
svc.Logger(options.Logger),
|
||||
svc.Stream(options.Stream),
|
||||
svc.Mux(mux),
|
||||
svc.Config(options.Config),
|
||||
svc.GatewaySelector(options.GatewaySelector),
|
||||
svc.TraceProvider(options.TraceProvider),
|
||||
svc.HistoryClient(options.HistoryClient),
|
||||
svc.ValueClient(options.ValueClient),
|
||||
svc.RegisteredEvents(options.RegisteredEvents),
|
||||
)
|
||||
if err != nil {
|
||||
options.Logger.Fatal().Err(err).Msg("failed to register the handler")
|
||||
return http.Service{}, err
|
||||
}
|
||||
|
||||
newService.Init()
|
||||
return newService, nil
|
||||
|
||||
}
|
||||
|
||||
// Service defines the business logic implementations need to provide.
|
||||
type ActivityLogService interface {
|
||||
GetItemActivities(ctx context.Context, query, loc string, t l10n.Translator) ([]libregraph.Activity, error)
|
||||
}
|
||||
|
||||
// GetActivitiesResponse is the response on GET activities requests
|
||||
type GetActivitiesResponse struct {
|
||||
Activities []libregraph.Activity `json:"value"`
|
||||
}
|
||||
|
||||
func GetItemActivitiesHandler(log log.Logger, s ActivityLogService, vc settingssvc.ValueService, t l10n.Translator) func(w http.ResponseWriter, r *http.Request) {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, r.Header.Get(revactx.TokenHeader))
|
||||
|
||||
activeUser, ok := revactx.ContextGetUser(ctx)
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
loc := l10n.MustGetUserLocale(ctx, activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), vc)
|
||||
|
||||
activities, err := s.GetItemActivities(ctx, r.URL.Query().Get("kql"), loc, t)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, activityloghttp.ErrBadRequest):
|
||||
log.Debug().Str("query", r.URL.Query().Get("kql")).Err(err).Msg("error getting activities")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
case errors.Is(err, activityloghttp.ErrForbidden):
|
||||
log.Debug().Err(err).Msg("error getting activities")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
default:
|
||||
log.Error().Err(err).Msg("error getting activities")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
res := GetActivitiesResponse{
|
||||
Activities: activities,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; odata.metadata=minimal")
|
||||
w.Header().Set("OData-Version", "4.0")
|
||||
if reqID := chimiddleware.GetReqID(ctx); reqID != "" {
|
||||
w.Header().Set("request-id", reqID)
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
if err := json.NewEncoder(w).Encode(res); err != nil {
|
||||
log.Error().Err(err).Msg("error encoding activities")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if err := micro.RegisterHandler(service.Server(), handle); err != nil {
|
||||
return http.Service{}, err
|
||||
}
|
||||
|
||||
return service, nil
|
||||
}
|
||||
@@ -1,366 +0,0 @@
|
||||
package activitylog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base32"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/jellydator/ttlcache/v2"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog")
|
||||
}
|
||||
|
||||
var (
|
||||
_maxActivitiesDefault = 6000
|
||||
_writeBufferDuration = 10 * time.Second
|
||||
)
|
||||
|
||||
// Activitylog stores and retrieves activities for resources and their parents from a nats kv
|
||||
type ActivityLog struct {
|
||||
log log.Logger
|
||||
// FIXME the lock does not protect agains concurrent resource activities on multiple instances
|
||||
// known since https://github.com/owncloud/ocis/pull/9361#pullrequestreview-2135350157
|
||||
// current ocis discussion in https://github.com/owncloud/ocis/issues/12475
|
||||
lock sync.RWMutex
|
||||
debouncer *Debouncer
|
||||
parentIdCache *ttlcache.Cache
|
||||
natskv nats.KeyValue
|
||||
|
||||
maxActivities int
|
||||
}
|
||||
|
||||
type batchInfo struct {
|
||||
key string
|
||||
count int
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
// New creates a new ActivitylogService
|
||||
func New(kv nats.KeyValue, opts ...Option) (*ActivityLog, error) {
|
||||
o := &Options{
|
||||
MaxActivities: _maxActivitiesDefault,
|
||||
WriteBufferDuration: _writeBufferDuration,
|
||||
Logger: log.NopLogger(),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
|
||||
cache := ttlcache.NewCache()
|
||||
err := cache.SetTTL(30 * time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &ActivityLog{
|
||||
log: o.Logger,
|
||||
lock: sync.RWMutex{},
|
||||
parentIdCache: cache,
|
||||
maxActivities: o.MaxActivities,
|
||||
natskv: kv,
|
||||
}
|
||||
s.debouncer = NewDebouncer(o.WriteBufferDuration, s.StoreActivity)
|
||||
|
||||
// run migrations
|
||||
err = s.runMigrations(context.Background(), kv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// RemoveResource removes the resource from the store
|
||||
func (a *ActivityLog) RemoveResource(rid *provider.ResourceId) error {
|
||||
if rid == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
err := a.natskv.Delete(storagespace.FormatResourceID(rid))
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not delete resource %s: %w", rid.OpaqueId, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RawActivity represents an activity as it is stored in the activitylog store
|
||||
type RawActivity struct {
|
||||
EventID string `json:"event_id"`
|
||||
Depth int `json:"depth"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
func (a *ActivityLog) AddActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time, getResource func(context.Context, *provider.Reference) (*provider.ResourceInfo, error)) error {
|
||||
var (
|
||||
err error
|
||||
depth int
|
||||
ref = initRef
|
||||
)
|
||||
ctx, span := tracer.Start(ctx, "AddActivity")
|
||||
defer span.End()
|
||||
for {
|
||||
var info *provider.ResourceInfo
|
||||
id := ref.GetResourceId()
|
||||
if ref.Path != "" {
|
||||
// Path based reference, we need to resolve the resource id
|
||||
ctx, span = tracer.Start(ctx, "AddActivity.getResource")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil {
|
||||
// TODO If the resource was deleted should we still log an activity in the parent?
|
||||
return fmt.Errorf("could not get resource info for reference %v: %w", ref, err)
|
||||
}
|
||||
id = info.GetId()
|
||||
}
|
||||
if id == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
key := storagespace.FormatResourceID(id)
|
||||
a.debouncer.Debounce(key, RawActivity{
|
||||
EventID: eventID,
|
||||
Depth: depth,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
|
||||
if id.OpaqueId == id.SpaceId {
|
||||
// we are at the root of the space, no need to go further
|
||||
break
|
||||
}
|
||||
|
||||
// check if parent id is cached
|
||||
// parent id is cached in the format <storageid>$<spaceid>!<resourceid>
|
||||
// if it is not cached, get the resource info and cache it
|
||||
if parentId == nil {
|
||||
if v, err := a.parentIdCache.Get(key); err != nil {
|
||||
if info == nil {
|
||||
ctx, span := tracer.Start(ctx, "AddActivity.getResource parent")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil || info.GetParentId() == nil || info.GetParentId().GetOpaqueId() == "" {
|
||||
return fmt.Errorf("could not get parent id: %w", err)
|
||||
}
|
||||
}
|
||||
parentId = info.GetParentId()
|
||||
a.parentIdCache.Set(key, parentId)
|
||||
} else {
|
||||
parentId = v.(*provider.ResourceId)
|
||||
}
|
||||
} else {
|
||||
a.log.Debug().Msg("parent id is cached")
|
||||
}
|
||||
|
||||
depth++
|
||||
ref = &provider.Reference{ResourceId: parentId}
|
||||
parentId = nil // reset parent id so it's not reused in the next iteration
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivityLog) StoreActivity(resourceID string, activities []RawActivity) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
ctx, span := tracer.Start(context.Background(), "storeActivity")
|
||||
defer span.End()
|
||||
|
||||
_, subspan := tracer.Start(ctx, "storeActivity.Marshal")
|
||||
b, err := msgpack.Marshal(activities)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = tracer.Start(ctx, "storeActivity.natskv.Put")
|
||||
key := natsKey(resourceID, len(activities))
|
||||
_, err = a.natskv.Put(key, b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
ctx, subspan = tracer.Start(ctx, "storeActivity.enforceMaxActivities")
|
||||
a.enforceMaxActivities(ctx, resourceID)
|
||||
subspan.End()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivityLog) enforceMaxActivities(ctx context.Context, resourceID string) {
|
||||
if a.maxActivities <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
_, subspan := tracer.Start(ctx, "enforceMaxActivities.watch")
|
||||
watcher, err := a.natskv.Watch(key, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("resourceID", resourceID).Msg("could not watch")
|
||||
return
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var keys []string
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
|
||||
}
|
||||
keys = append(keys, update.Key())
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = tracer.Start(ctx, "enforceMaxActivities.compile")
|
||||
// Parse keys into batches
|
||||
batches := make([]batchInfo, 0)
|
||||
var activitiesCount int
|
||||
for _, k := range keys {
|
||||
parts := strings.SplitN(k, ".", 3)
|
||||
if len(parts) < 3 {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, not enough parts")
|
||||
continue
|
||||
}
|
||||
|
||||
c, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse count")
|
||||
continue
|
||||
}
|
||||
|
||||
// parse timestamp
|
||||
nano, err := strconv.ParseInt(parts[2], 10, 64)
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse timestamp")
|
||||
continue
|
||||
}
|
||||
|
||||
batches = append(batches, batchInfo{
|
||||
key: k,
|
||||
count: c,
|
||||
timestamp: time.Unix(0, nano),
|
||||
})
|
||||
activitiesCount += c
|
||||
}
|
||||
|
||||
// sort batches by timestamp
|
||||
sort.Slice(batches, func(i, j int) bool {
|
||||
return batches[i].timestamp.Before(batches[j].timestamp)
|
||||
})
|
||||
subspan.End()
|
||||
|
||||
_, subspan = tracer.Start(ctx, "enforceMaxActivities.delete")
|
||||
// remove oldest keys until we are at max activities
|
||||
for _, b := range batches {
|
||||
if activitiesCount-b.count < a.maxActivities {
|
||||
break
|
||||
}
|
||||
|
||||
activitiesCount -= b.count
|
||||
err = a.natskv.Delete(b.key)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("key", b.key).Msg("could not delete key")
|
||||
break
|
||||
}
|
||||
}
|
||||
subspan.End()
|
||||
}
|
||||
|
||||
func (a *ActivityLog) InvalidateCachedParentID(purgeId *provider.ResourceId) {
|
||||
// The parent id cache is populated lazily and its entries expire, so a
|
||||
// missing key is the expected case rather than an error.
|
||||
if err := a.parentIdCache.Remove(storagespace.FormatResourceID(purgeId)); err != nil {
|
||||
a.log.Debug().Interface("event", purgeId).Err(err).Msg("could not delete parent id cache")
|
||||
}
|
||||
}
|
||||
|
||||
func natsKey(resourceID string, activitiesCount int) string {
|
||||
return fmt.Sprintf("%s.%d.%d",
|
||||
base32.StdEncoding.EncodeToString([]byte(resourceID)),
|
||||
activitiesCount,
|
||||
time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func (a *ActivityLog) Activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
a.lock.RLock()
|
||||
defer a.lock.RUnlock()
|
||||
|
||||
return a.activities(rid)
|
||||
}
|
||||
|
||||
func (a *ActivityLog) activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
resourceID := storagespace.FormatResourceID(rid)
|
||||
|
||||
glob := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
watcher, err := a.natskv.Watch(glob, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var activities []RawActivity
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack")
|
||||
}
|
||||
activities = append(activities, batchActivities...)
|
||||
}
|
||||
|
||||
return activities, nil
|
||||
}
|
||||
|
||||
// RemoveActivities removes the activities from the given resource
|
||||
func (a *ActivityLog) RemoveActivities(rid *provider.ResourceId, toDelete map[string]struct{}) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
curActivities, err := a.activities(rid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var acts []RawActivity
|
||||
for _, a := range curActivities {
|
||||
if _, ok := toDelete[a.EventID]; !ok {
|
||||
acts = append(acts, a)
|
||||
}
|
||||
}
|
||||
|
||||
b, err := json.Marshal(acts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = a.natskv.Put(storagespace.FormatResourceID(rid), b)
|
||||
return err
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package activitylog_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestActivitylog(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Activitylog Suite")
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package activitylog
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Debouncer is used to debounce writes to the activity log store.
|
||||
type Debouncer struct {
|
||||
after time.Duration
|
||||
f func(id string, ra []RawActivity) error
|
||||
pending sync.Map
|
||||
inProgress sync.Map
|
||||
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
type queueItem struct {
|
||||
activities []RawActivity
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
// NewDebouncer returns a new Debouncer instance.
|
||||
func NewDebouncer(d time.Duration, f func(id string, ra []RawActivity) error) *Debouncer {
|
||||
return &Debouncer{
|
||||
after: d,
|
||||
f: f,
|
||||
pending: sync.Map{},
|
||||
inProgress: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
// Debounce restarts the debounce timer for the given space.
|
||||
func (d *Debouncer) Debounce(id string, ra RawActivity) {
|
||||
if d.after == 0 {
|
||||
d.f(id, []RawActivity{ra})
|
||||
return
|
||||
}
|
||||
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
|
||||
item := &queueItem{
|
||||
activities: []RawActivity{ra},
|
||||
}
|
||||
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
// if the item is already in the queue, append the new activities
|
||||
item, ok = i.(*queueItem)
|
||||
if ok {
|
||||
item.activities = append(item.activities, ra)
|
||||
}
|
||||
}
|
||||
|
||||
if item.timer == nil {
|
||||
item.timer = time.AfterFunc(d.after, func() {
|
||||
if _, ok := d.inProgress.Load(id); ok {
|
||||
// Reschedule this run for when the previous run has finished
|
||||
d.mutex.Lock()
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
i.(*queueItem).timer.Reset(d.after)
|
||||
}
|
||||
|
||||
d.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
d.pending.Delete(id)
|
||||
d.inProgress.Store(id, true)
|
||||
defer d.inProgress.Delete(id)
|
||||
d.f(id, item.activities)
|
||||
})
|
||||
}
|
||||
|
||||
d.pending.Store(id, item)
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
package activitylog_test
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
)
|
||||
|
||||
var _ = Describe("Debouncer", func() {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
callbacks []activitylog.RawActivity
|
||||
newCallback func(id string, ra []activitylog.RawActivity) error
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
mu.Lock()
|
||||
callbacks = nil
|
||||
mu.Unlock()
|
||||
newCallback = func(id string, ra []activitylog.RawActivity) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
callbacks = append(callbacks, ra...)
|
||||
return nil
|
||||
}
|
||||
})
|
||||
|
||||
Context("with zero duration", func() {
|
||||
It("calls the callback immediately", func() {
|
||||
d := activitylog.NewDebouncer(0, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
Expect(callbacks).To(HaveLen(1))
|
||||
Expect(callbacks[0].EventID).To(Equal("activity1"))
|
||||
})
|
||||
|
||||
It("calls the callback immediately for each event", func() {
|
||||
d := activitylog.NewDebouncer(0, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
d.Debounce("space2", activitylog.RawActivity{EventID: "activity2"})
|
||||
Expect(callbacks).To(HaveLen(2))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with non-zero duration", func() {
|
||||
It("batches activities with the same id", func() {
|
||||
d := activitylog.NewDebouncer(10*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity3"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(3))
|
||||
})
|
||||
|
||||
It("handles different ids independently", func() {
|
||||
d := activitylog.NewDebouncer(10*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
d.Debounce("space2", activitylog.RawActivity{EventID: "activity2"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(2))
|
||||
})
|
||||
|
||||
It("batches activities that arrive within the debounce window", func() {
|
||||
d := activitylog.NewDebouncer(100*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity3"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(3))
|
||||
})
|
||||
|
||||
It("processes new batch after previous completes", func() {
|
||||
d := activitylog.NewDebouncer(5*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
time.Sleep(20 * time.Millisecond) // let first batch complete
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(2))
|
||||
})
|
||||
|
||||
It("skips duplicate write when timer fires during in-progress callback", func() {
|
||||
slowCallback := func(id string, ra []activitylog.RawActivity) error {
|
||||
time.Sleep(50 * time.Millisecond) // simulate slow write
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
callbacks = append(callbacks, ra...)
|
||||
return nil
|
||||
}
|
||||
|
||||
d := activitylog.NewDebouncer(10*time.Millisecond, slowCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
time.Sleep(20 * time.Millisecond) // timer fires while callback is running
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(1))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,35 +0,0 @@
|
||||
package activitylog
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
)
|
||||
|
||||
// Option for the activitylog service
|
||||
type Option func(*Options)
|
||||
|
||||
// Options for the activitylog service
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
MaxActivities int
|
||||
WriteBufferDuration time.Duration
|
||||
}
|
||||
|
||||
// Logger configures a logger for the activitylog service
|
||||
func Logger(log log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = log
|
||||
}
|
||||
}
|
||||
|
||||
func MaxActivities(max int) Option {
|
||||
return func(o *Options) {
|
||||
o.MaxActivities = max
|
||||
}
|
||||
}
|
||||
func WriteBufferDuration(d time.Duration) Option {
|
||||
return func(o *Options) {
|
||||
o.WriteBufferDuration = d
|
||||
}
|
||||
}
|
||||
@@ -1,302 +0,0 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events")
|
||||
}
|
||||
|
||||
var (
|
||||
_numConsumersDefault = 1
|
||||
)
|
||||
|
||||
// ActivitylogService logs events per resource
|
||||
type ActivitylogService struct {
|
||||
ctx context.Context
|
||||
sa config.ServiceAccount
|
||||
log log.Logger
|
||||
stream events.Stream
|
||||
gws pool.Selectable[gateway.GatewayAPIClient]
|
||||
al *activitylog.ActivityLog
|
||||
|
||||
numConsumers int
|
||||
|
||||
events []events.Unmarshaller
|
||||
|
||||
stopCh chan struct{}
|
||||
stopped *atomic.Bool
|
||||
}
|
||||
|
||||
// New creates a new ActivitylogService
|
||||
func New(al *activitylog.ActivityLog, stream events.Stream, opts ...Option) (*ActivitylogService, error) {
|
||||
o := &Options{
|
||||
NumConsumers: _numConsumersDefault,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
|
||||
s := &ActivitylogService{
|
||||
ctx: o.Context,
|
||||
log: o.Logger,
|
||||
sa: o.ServiceAccount,
|
||||
stream: stream,
|
||||
gws: o.GatewaySelector,
|
||||
events: o.RegisteredEvents,
|
||||
numConsumers: o.NumConsumers,
|
||||
al: al,
|
||||
stopCh: make(chan struct{}, 1),
|
||||
stopped: new(atomic.Bool),
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Run to fulfil Runner interface
|
||||
func (s *ActivitylogService) Run() error {
|
||||
ch, err := events.Consume(s.stream, "activitylog", s.events...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
ctx, cancel := context.WithCancel(s.ctx)
|
||||
defer cancel()
|
||||
|
||||
s.log.Debug().Int("worker.count", s.numConsumers).
|
||||
Str("messaging.consumer.group.name", "activitylog").
|
||||
Str("messaging.system", "nats").
|
||||
Str("messaging.operation.name", "receive").
|
||||
Msg("starting event processing workers")
|
||||
|
||||
// start workers
|
||||
for i := 0; i < s.numConsumers; i++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := s.processEvent(e); err != nil {
|
||||
s.log.Error().Err(err).
|
||||
Int("worker", workerID).
|
||||
Interface("event", e).
|
||||
Msg("failed to process event")
|
||||
}
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// wait for stop signal
|
||||
<-s.stopCh
|
||||
cancel() // signal workers to stop
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close will make the service to stop processing, so the `Run`
|
||||
// method can finish.
|
||||
func (s *ActivitylogService) Close() {
|
||||
if s.stopped.CompareAndSwap(false, true) {
|
||||
close(s.stopCh)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ActivitylogService) processEvent(e events.Event) error {
|
||||
ctx := e.GetTraceContext(s.ctx)
|
||||
ctx, span := tracer.Start(ctx, "processEvent")
|
||||
defer span.End()
|
||||
|
||||
s.log.Debug().Interface("event", e).Msg("updating activitylog")
|
||||
|
||||
switch ev := e.Event.(type) {
|
||||
case events.UploadReady:
|
||||
return s.AddActivity(ctx, ev.FileRef, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.FileTouched:
|
||||
return s.AddActivity(ctx, ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
// Disabled https://github.com/owncloud/ocis/issues/10293
|
||||
//case events.FileDownloaded:
|
||||
// we are only interested in public link downloads - so no need to store others.
|
||||
//if ev.ImpersonatingUser.GetDisplayName() == "Public" {
|
||||
// err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
//}
|
||||
case events.ContainerCreated:
|
||||
return s.AddActivity(ctx, ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemTrashed:
|
||||
return s.AddActivityTrashed(ctx, ev.ID, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemPurged:
|
||||
return s.al.RemoveResource(ev.ID)
|
||||
case events.ItemMoved:
|
||||
// remove the cached parent id for this resource
|
||||
s.removeCachedParentID(ctx, ev.Ref)
|
||||
|
||||
return s.AddActivity(ctx, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ShareCreated:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.ShareUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.ShareRemoved:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, ev.Timestamp)
|
||||
case events.LinkCreated:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.LinkUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.LinkRemoved:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.SpaceShared:
|
||||
return s.AddSpaceActivity(ctx, ev.ID, e.ID, ev.Timestamp)
|
||||
case events.SpaceUnshared:
|
||||
return s.AddSpaceActivity(ctx, ev.ID, e.ID, ev.Timestamp)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddActivity adds the activity to the given resource and all its parents
|
||||
func (a *ActivitylogService) AddActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
ctx, span := tracer.Start(ctx, "AddActivity")
|
||||
defer span.End()
|
||||
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
|
||||
}
|
||||
return a.al.AddActivity(ctx, initRef, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
|
||||
func (a *ActivitylogService) AddActivityTrashed(ctx context.Context, resourceID *provider.ResourceId, reference *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
ctx, span := tracer.Start(ctx, "AddActivityTrashed")
|
||||
defer span.End()
|
||||
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
}
|
||||
|
||||
// store activity on trashed item
|
||||
if err := a.al.StoreActivity(storagespace.FormatResourceID(resourceID), []activitylog.RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("could not store activity: %w", err)
|
||||
}
|
||||
|
||||
// get previous parent
|
||||
ref := &provider.Reference{
|
||||
ResourceId: reference.GetResourceId(),
|
||||
Path: filepath.Dir(reference.GetPath()),
|
||||
}
|
||||
|
||||
return a.al.AddActivity(ctx, ref, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddSpaceActivity adds the activity to the given spaceroot
|
||||
func (a *ActivitylogService) AddSpaceActivity(ctx context.Context, spaceID *provider.StorageSpaceId, eventID string, timestamp time.Time) error {
|
||||
_, span := tracer.Start(ctx, "AddSpaceActivity")
|
||||
defer span.End()
|
||||
// spaceID is in format <providerid>$<spaceid>
|
||||
// activitylog service uses format <providerid>$<spaceid>!<resourceid>
|
||||
// lets do some converting, shall we?
|
||||
rid, err := storagespace.ParseID(spaceID.GetOpaqueId())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse space id: %w", err)
|
||||
}
|
||||
rid.OpaqueId = rid.GetSpaceId()
|
||||
err = a.al.StoreActivity(storagespace.FormatResourceID(&rid), []activitylog.RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not store activity: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func toRef(r *provider.ResourceId) *provider.Reference {
|
||||
return &provider.Reference{
|
||||
ResourceId: r,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) removeCachedParentID(ctx context.Context, ref *provider.Reference) {
|
||||
var span trace.Span
|
||||
ctx, span = tracer.Start(ctx, "removeCachedParentID")
|
||||
defer span.End()
|
||||
|
||||
purgeId := ref.GetResourceId()
|
||||
if ref.GetPath() != "" {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get gateway client")
|
||||
return
|
||||
}
|
||||
|
||||
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get service user context")
|
||||
return
|
||||
}
|
||||
|
||||
info, err := utils.GetResource(ctx, ref, gwc)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get resource info")
|
||||
return
|
||||
}
|
||||
purgeId = info.GetId()
|
||||
}
|
||||
a.al.InvalidateCachedParentID(purgeId)
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
package events_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
eventssvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
var _ = Describe("ActivitylogService", func() {
|
||||
Describe("New", func() {
|
||||
var (
|
||||
al *activitylog.ActivityLog
|
||||
stream events.Stream
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
al = &activitylog.ActivityLog{}
|
||||
stream = nil
|
||||
})
|
||||
|
||||
It("creates a service with minimal options", func() {
|
||||
svc, err := eventssvc.New(al, stream)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with context option", func() {
|
||||
ctx := context.Background()
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.Context(ctx))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with logger option", func() {
|
||||
logger := log.NopLogger()
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.Logger(logger))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with service account option", func() {
|
||||
sa := config.ServiceAccount{
|
||||
ServiceAccountID: "sa-id",
|
||||
ServiceAccountSecret: "sa-secret",
|
||||
}
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.ServiceAccount(sa))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with registered events", func() {
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}, &events.FileTouched{}}
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.RegisteredEvents(evts))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with gateway selector", func() {
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.GatewaySelector(gs))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with num consumers option", func() {
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.NumConsumers(5))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with all options", func() {
|
||||
ctx := context.Background()
|
||||
logger := log.NopLogger()
|
||||
sa := config.ServiceAccount{
|
||||
ServiceAccountID: "sa-id",
|
||||
ServiceAccountSecret: "sa-secret",
|
||||
}
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}, &events.ContainerCreated{}}
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
|
||||
svc, err := eventssvc.New(
|
||||
al,
|
||||
stream,
|
||||
eventssvc.Context(ctx),
|
||||
eventssvc.Logger(logger),
|
||||
eventssvc.ServiceAccount(sa),
|
||||
eventssvc.RegisteredEvents(evts),
|
||||
eventssvc.GatewaySelector(gs),
|
||||
eventssvc.NumConsumers(3),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Close", func() {
|
||||
It("can be called without panic on a new service", func() {
|
||||
al := &activitylog.ActivityLog{}
|
||||
svc, err := eventssvc.New(al, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(func() { svc.Close() }).ToNot(Panic())
|
||||
})
|
||||
|
||||
It("can be called multiple times without panic", func() {
|
||||
al := &activitylog.ActivityLog{}
|
||||
svc, err := eventssvc.New(al, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
svc.Close()
|
||||
svc.Close()
|
||||
svc.Close()
|
||||
})
|
||||
})
|
||||
})
|
||||
+80
-79
@@ -1,97 +1,89 @@
|
||||
package http
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/olekukonko/errors"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"google.golang.org/grpc/metadata"
|
||||
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/eventhistory/v0"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http")
|
||||
}
|
||||
|
||||
// New returns a new instance of Service
|
||||
func New(al *activitylog.ActivityLog, opts ...Option) (*svc, error) {
|
||||
o := newOptions(opts...)
|
||||
|
||||
registeredEvents := make(map[string]events.Unmarshaller)
|
||||
for _, e := range o.RegisteredEvents {
|
||||
typ := reflect.TypeOf(e)
|
||||
registeredEvents[typ.String()] = e
|
||||
}
|
||||
|
||||
return &svc{
|
||||
log: o.Logger,
|
||||
evHistory: o.HistoryClient,
|
||||
al: al,
|
||||
registeredEvents: registeredEvents,
|
||||
gws: o.GatewaySelector,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type svc struct {
|
||||
log log.Logger
|
||||
evHistory ehsvc.EventHistoryService
|
||||
gws pool.Selectable[gateway.GatewayAPIClient]
|
||||
al *activitylog.ActivityLog
|
||||
registeredEvents map[string]events.Unmarshaller
|
||||
}
|
||||
|
||||
var (
|
||||
ErrBadRequest = errors.New("bad request")
|
||||
ErrForbidden = errors.New("forbidden")
|
||||
//go:embed l10n/locale
|
||||
_localeFS embed.FS
|
||||
|
||||
// subfolder where the translation files are stored
|
||||
_localeSubPath = "l10n/locale"
|
||||
|
||||
// domain of the activitylog service (transifex)
|
||||
_domain = "activitylog"
|
||||
)
|
||||
|
||||
func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.Translator) ([]libregraph.Activity, error) {
|
||||
gwc, err := s.gws.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// ServeHTTP implements the http.Handler interface.
|
||||
func (s *ActivitylogService) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.mux.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// HandleGetItemActivities handles the request to get the activities of an item.
|
||||
func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, r.Header.Get(revactx.TokenHeader))
|
||||
|
||||
activeUser, ok := revactx.ContextGetUser(ctx)
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
rid, limit, rawActivityAccepted, activityAccepted, sort, err := s.getFilters(query)
|
||||
gwc, err := s.gws.Next()
|
||||
if err != nil {
|
||||
s.log.Info().Str("query", query).Err(err).Msg("error getting filters")
|
||||
return nil, ErrBadRequest
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
rid, limit, rawActivityAccepted, activityAccepted, sort, err := s.getFilters(r.URL.Query().Get("kql"))
|
||||
if err != nil {
|
||||
s.log.Info().Str("query", r.URL.Query().Get("kql")).Err(err).Msg("error getting filters")
|
||||
_, _ = w.Write([]byte(err.Error()))
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
info, err := utils.GetResourceByID(ctx, rid, gwc)
|
||||
if err != nil {
|
||||
return nil, ErrForbidden
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
||||
// you need ListGrants to see activities
|
||||
if !info.GetPermissionSet().GetListGrants() {
|
||||
return nil, ErrForbidden
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
||||
raw, err := s.al.Activities(rid)
|
||||
raw, err := s.Activities(rid)
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error getting activities")
|
||||
return nil, err
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(raw))
|
||||
@@ -104,21 +96,21 @@ func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.T
|
||||
toDelete[a.EventID] = struct{}{}
|
||||
}
|
||||
|
||||
evRes, err := s.evHistory.GetEvents(ctx, &ehsvc.GetEventsRequest{Ids: ids})
|
||||
evRes, err := s.evHistory.GetEvents(r.Context(), &ehsvc.GetEventsRequest{Ids: ids})
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error getting events")
|
||||
return nil, err
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
evs := evRes.GetEvents()
|
||||
sort(evs)
|
||||
|
||||
// TODO cut the interface here?
|
||||
activities := make([]libregraph.Activity, 0, len(evRes.GetEvents()))
|
||||
resp := GetActivitiesResponse{Activities: make([]libregraph.Activity, 0, len(evRes.GetEvents()))}
|
||||
for _, e := range evs {
|
||||
delete(toDelete, e.GetId())
|
||||
|
||||
if limit > 0 && limit <= len(activities) {
|
||||
if limit > 0 && limit <= len(resp.Activities) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -132,6 +124,9 @@ func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.T
|
||||
vars map[string]any
|
||||
)
|
||||
|
||||
loc := l10n.MustGetUserLocale(r.Context(), activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), s.valService)
|
||||
t := l10n.NewTranslatorFromCommonConfig(s.cfg.DefaultLanguage, _domain, s.cfg.TranslationPath, _localeFS, _localeSubPath)
|
||||
|
||||
switch ev := s.unwrapEvent(e).(type) {
|
||||
case nil:
|
||||
// error already logged in unwrapEvent
|
||||
@@ -229,29 +224,35 @@ func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.T
|
||||
continue
|
||||
}
|
||||
|
||||
activities = append(activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars))
|
||||
resp.Activities = append(resp.Activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars))
|
||||
}
|
||||
|
||||
// delete activities in separate go routine
|
||||
if len(toDelete) > 0 {
|
||||
go func() {
|
||||
err := s.al.RemoveActivities(rid, toDelete)
|
||||
err := s.RemoveActivities(rid, toDelete)
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error removing activities")
|
||||
}
|
||||
}()
|
||||
}
|
||||
return activities, nil
|
||||
|
||||
}
|
||||
|
||||
func toRef(r *provider.ResourceId) *provider.Reference {
|
||||
return &provider.Reference{
|
||||
ResourceId: r,
|
||||
b, err := json.Marshal(resp)
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error marshalling activities")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := w.Write(b); err != nil {
|
||||
s.log.Error().Err(err).Msg("error writing response")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (s *svc) unwrapEvent(e *ehmsg.Event) any {
|
||||
func (s *ActivitylogService) unwrapEvent(e *ehmsg.Event) any {
|
||||
etype, ok := s.registeredEvents[e.GetType()]
|
||||
if !ok {
|
||||
s.log.Error().Str("eventid", e.GetId()).Str("eventtype", e.GetType()).Msg("event not registered")
|
||||
@@ -267,13 +268,13 @@ func (s *svc) unwrapEvent(e *ehmsg.Event) any {
|
||||
return einterface
|
||||
}
|
||||
|
||||
func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activitylog.RawActivity) bool, func(*ehmsg.Event) bool, func([]*ehmsg.Event), error) {
|
||||
func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int, func(RawActivity) bool, func(*ehmsg.Event) bool, func([]*ehmsg.Event), error) {
|
||||
qast, err := kql.Builder{}.Build(query)
|
||||
if err != nil {
|
||||
return nil, 0, nil, nil, nil, err
|
||||
}
|
||||
|
||||
prefilters := make([]func(activitylog.RawActivity) bool, 0)
|
||||
prefilters := make([]func(RawActivity) bool, 0)
|
||||
postfilters := make([]func(*ehmsg.Event) bool, 0)
|
||||
|
||||
sortby := func(_ []*ehmsg.Event) {}
|
||||
@@ -298,7 +299,7 @@ func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activity
|
||||
break
|
||||
}
|
||||
|
||||
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
|
||||
prefilters = append(prefilters, func(a RawActivity) bool {
|
||||
return a.Depth <= depth
|
||||
})
|
||||
case "limit":
|
||||
@@ -321,11 +322,11 @@ func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activity
|
||||
case *ast.DateTimeNode:
|
||||
switch v.Operator.Value {
|
||||
case "<", "<=":
|
||||
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
|
||||
prefilters = append(prefilters, func(a RawActivity) bool {
|
||||
return a.Timestamp.Before(v.Value)
|
||||
})
|
||||
case ">", ">=":
|
||||
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
|
||||
prefilters = append(prefilters, func(a RawActivity) bool {
|
||||
return a.Timestamp.After(v.Value)
|
||||
})
|
||||
}
|
||||
@@ -344,7 +345,7 @@ func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activity
|
||||
// space root requested - fix format
|
||||
rid.OpaqueId = rid.GetSpaceId()
|
||||
}
|
||||
pref := func(a activitylog.RawActivity) bool {
|
||||
pref := func(a RawActivity) bool {
|
||||
for _, f := range prefilters {
|
||||
if !f(a) {
|
||||
return false
|
||||
@@ -1,13 +0,0 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestHTTP(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "HTTP Suite")
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
RegisteredEvents []events.Unmarshaller
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
HistoryClient ehsvc.EventHistoryService
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// RegisteredEvents registers the events the service should listen to
|
||||
func RegisteredEvents(e []events.Unmarshaller) Option {
|
||||
return func(o *Options) {
|
||||
o.RegisteredEvents = e
|
||||
}
|
||||
}
|
||||
|
||||
// GatewaySelector adds a grpc client selector for the gateway service
|
||||
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
|
||||
return func(o *Options) {
|
||||
o.GatewaySelector = gatewaySelector
|
||||
}
|
||||
}
|
||||
|
||||
// HistoryClient adds a grpc client for the eventhistory service
|
||||
func HistoryClient(hc ehsvc.EventHistoryService) Option {
|
||||
return func(o *Options) {
|
||||
o.HistoryClient = hc
|
||||
}
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
httpsvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
)
|
||||
|
||||
var _ = Describe("Response", func() {
|
||||
Describe("NewActivity", func() {
|
||||
It("creates an activity with the given parameters", func() {
|
||||
ts := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
|
||||
vars := map[string]any{
|
||||
"user": "testuser",
|
||||
"resource": "testfile.txt",
|
||||
}
|
||||
|
||||
act := httpsvc.NewActivity("Test message", ts, "event-123", vars)
|
||||
|
||||
Expect(act.Id).To(Equal("event-123"))
|
||||
Expect(act.Times.RecordedTime).To(Equal(ts))
|
||||
Expect(act.Template.Message).To(Equal("Test message"))
|
||||
Expect(act.Template.Variables).To(HaveKeyWithValue("user", "testuser"))
|
||||
Expect(act.Template.Variables).To(HaveKeyWithValue("resource", "testfile.txt"))
|
||||
})
|
||||
|
||||
It("handles empty variables map", func() {
|
||||
act := httpsvc.NewActivity("", time.Time{}, "", map[string]any{})
|
||||
|
||||
Expect(act.Id).To(BeEmpty())
|
||||
Expect(act.Times.RecordedTime).To(Equal(time.Time{}))
|
||||
Expect(act.Template.Message).To(BeEmpty())
|
||||
Expect(act.Template.Variables).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithOldResource", func() {
|
||||
It("sets the oldResource variable from reference path", func() {
|
||||
ref := &provider.Reference{
|
||||
Path: "/old/path/oldname.txt",
|
||||
}
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithOldResource(ref)
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("oldResource"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithUser", func() {
|
||||
It("returns error when no user is provided", func() {
|
||||
opt := httpsvc.WithUser(nil, nil, nil)
|
||||
err := opt(context.Background(), nil, make(map[string]any))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("no user provided"))
|
||||
})
|
||||
|
||||
It("uses impersonator when provided", func() {
|
||||
impersonator := &user.User{
|
||||
Id: &user.UserId{
|
||||
OpaqueId: "imp-user-id",
|
||||
},
|
||||
DisplayName: "Impersonated User",
|
||||
}
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithUser(nil, nil, impersonator)
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("user"))
|
||||
})
|
||||
|
||||
It("uses executing user when no impersonator", func() {
|
||||
execUser := &user.User{
|
||||
Id: &user.UserId{
|
||||
OpaqueId: "exec-user-id",
|
||||
},
|
||||
DisplayName: "Executing User",
|
||||
}
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithUser(nil, execUser, nil)
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("user"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithVar", func() {
|
||||
It("sets a simple key-value variable", func() {
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithVar("token", "id123", "My Token")
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("token"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithTranslation", func() {
|
||||
It("sets translated field variable", func() {
|
||||
var t l10n.Translator
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithTranslation(&t, "en", "field", []string{"permission"})
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("field"))
|
||||
})
|
||||
|
||||
It("handles empty values slice", func() {
|
||||
var t l10n.Translator
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithTranslation(&t, "en", "field", []string{})
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("field"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("ActivityOption type", func() {
|
||||
It("allows composing multiple options", func() {
|
||||
vars := make(map[string]any)
|
||||
ctx := context.Background()
|
||||
var gwc gateway.GatewayAPIClient
|
||||
|
||||
options := []httpsvc.ActivityOption{
|
||||
httpsvc.WithVar("key1", "id1", "name1"),
|
||||
httpsvc.WithVar("key2", "id2", "name2"),
|
||||
}
|
||||
|
||||
for _, opt := range options {
|
||||
err := opt(ctx, gwc, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
|
||||
Expect(vars).To(HaveKey("key1"))
|
||||
Expect(vars).To(HaveKey("key2"))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,76 +0,0 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
httpsvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
var _ = Describe("Service", func() {
|
||||
Describe("New", func() {
|
||||
var (
|
||||
al *activitylog.ActivityLog
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
al = &activitylog.ActivityLog{}
|
||||
})
|
||||
|
||||
It("creates a service with default options", func() {
|
||||
svc, err := httpsvc.New(al)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with logger option", func() {
|
||||
logger := log.NopLogger()
|
||||
svc, err := httpsvc.New(al, httpsvc.Logger(logger))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with registered events", func() {
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}}
|
||||
svc, err := httpsvc.New(al, httpsvc.RegisteredEvents(evts))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with gateway selector", func() {
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
svc, err := httpsvc.New(al, httpsvc.GatewaySelector(gs))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with history client", func() {
|
||||
var hc ehsvc.EventHistoryService
|
||||
svc, err := httpsvc.New(al, httpsvc.HistoryClient(hc))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with all options", func() {
|
||||
logger := log.NopLogger()
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}, &events.FileTouched{}}
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
var hc ehsvc.EventHistoryService
|
||||
|
||||
svc, err := httpsvc.New(
|
||||
al,
|
||||
httpsvc.Logger(logger),
|
||||
httpsvc.RegisteredEvents(evts),
|
||||
httpsvc.GatewaySelector(gs),
|
||||
httpsvc.HistoryClient(hc),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
})
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -1,102 +0,0 @@
|
||||
# SOME DESCRIPTIVE TITLE.
|
||||
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
|
||||
# This file is distributed under the same license as the PACKAGE package.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
|
||||
#
|
||||
# Translators:
|
||||
# Roy-Magne Mo, 2026
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: \n"
|
||||
"Report-Msgid-Bugs-To: EMAIL\n"
|
||||
"POT-Creation-Date: 2026-09-06 23:15+0000\n"
|
||||
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
|
||||
"Last-Translator: Roy-Magne Mo, 2026\n"
|
||||
"Language-Team: Norwegian (https://app.transifex.com/opencloud-eu/teams/204053/no/)\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Language: no\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
||||
|
||||
#: pkg/service/response.go:44
|
||||
msgid "description"
|
||||
msgstr "beskrivelse"
|
||||
|
||||
#: pkg/service/response.go:43
|
||||
msgid "display name"
|
||||
msgstr "visningsnavn"
|
||||
|
||||
#: pkg/service/response.go:42
|
||||
msgid "expiration date"
|
||||
msgstr "utløpsdato"
|
||||
|
||||
#: pkg/service/response.go:41
|
||||
msgid "password"
|
||||
msgstr "passord"
|
||||
|
||||
#: pkg/service/response.go:40
|
||||
msgid "permission"
|
||||
msgstr "tillatelse"
|
||||
|
||||
#: pkg/service/response.go:39
|
||||
msgid "some field"
|
||||
msgstr "et felt"
|
||||
|
||||
#: pkg/service/response.go:26
|
||||
msgid "{resource} was downloaded via public link {token}"
|
||||
msgstr "{resource} ble lastet ned via offentlig lenke {token}"
|
||||
|
||||
#: pkg/service/response.go:24
|
||||
msgid "{user} added {resource} to {folder}"
|
||||
msgstr "{user} la til {resource} i {folder}"
|
||||
|
||||
#: pkg/service/response.go:36
|
||||
msgid "{user} added {sharee} as member of {space}"
|
||||
msgstr "{user} la til {sharee} som medlem av {space}"
|
||||
|
||||
#: pkg/service/response.go:27
|
||||
msgid "{user} deleted {resource} from {folder}"
|
||||
msgstr "{user} slettet {resource} fra {folder}"
|
||||
|
||||
#: pkg/service/response.go:28
|
||||
msgid "{user} moved {resource} to {folder}"
|
||||
msgstr "{user} flyttet {resource} til {folder}"
|
||||
|
||||
#: pkg/service/response.go:35
|
||||
msgid "{user} removed link to {resource}"
|
||||
msgstr "{user} fjernet lenke til {resource}"
|
||||
|
||||
#: pkg/service/response.go:32
|
||||
msgid "{user} removed {sharee} from {resource}"
|
||||
msgstr "{user} fjernet {sharee} fra {resource}"
|
||||
|
||||
#: pkg/service/response.go:37
|
||||
msgid "{user} removed {sharee} from {space}"
|
||||
msgstr "{user} fjernet {sharee} fra {space}"
|
||||
|
||||
#: pkg/service/response.go:29
|
||||
msgid "{user} renamed {oldResource} to {resource}"
|
||||
msgstr "{user} ga nytt navn til {oldResource} som {resource}"
|
||||
|
||||
#: pkg/service/response.go:33
|
||||
msgid "{user} shared {resource} via link"
|
||||
msgstr "{user} delte {resource} via lenke"
|
||||
|
||||
#: pkg/service/response.go:30
|
||||
msgid "{user} shared {resource} with {sharee}"
|
||||
msgstr "{user} delte {resource} med {sharee}"
|
||||
|
||||
#: pkg/service/response.go:34
|
||||
msgid "{user} updated {field} for a link {token} on {resource}"
|
||||
msgstr "{user} oppdaterte {field} for en lenke {token} på {resource}"
|
||||
|
||||
#: pkg/service/response.go:31
|
||||
msgid "{user} updated {field} for the {resource}"
|
||||
msgstr "{user} oppdaterte {field} for {resource}"
|
||||
|
||||
#: pkg/service/response.go:25
|
||||
msgid "{user} updated {resource} in {folder}"
|
||||
msgstr "{user} oppdaterte {resource} i {folder}"
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+3
-3
@@ -1,4 +1,4 @@
|
||||
package activitylog
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -15,7 +15,7 @@ const currentMigrationVersion = "1"
|
||||
|
||||
// RunMigrations checks the activitylog data version and runs migrations if necessary.
|
||||
// It should be called during service startup, after the NATS KeyValue store is initialized.
|
||||
func (a *ActivityLog) runMigrations(ctx context.Context, kv nats.KeyValue) error {
|
||||
func (a *ActivitylogService) runMigrations(ctx context.Context, kv nats.KeyValue) error {
|
||||
entry, err := kv.Get(activitylogVersionKey)
|
||||
if err == nats.ErrKeyNotFound {
|
||||
a.log.Info().Msg("activitylog version key not found. Running migration to V1...")
|
||||
@@ -40,7 +40,7 @@ func (a *ActivityLog) runMigrations(ctx context.Context, kv nats.KeyValue) error
|
||||
// For each such key, it creates a new key in the format "originalKey.count.timestamp"
|
||||
// and stores the original list of strings (re-marshalled to messagepack) as its value.
|
||||
// Finally, it sets the activitylog.version key to "1".
|
||||
func (a *ActivityLog) migrateToV1(_ context.Context, kv nats.KeyValue) error {
|
||||
func (a *ActivitylogService) migrateToV1(_ context.Context, kv nats.KeyValue) error {
|
||||
lister, err := kv.ListKeys()
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrateToV1: failed to list keys from NATS KV store: %w", err)
|
||||
+38
-16
@@ -1,14 +1,17 @@
|
||||
package events
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// Option for the activitylog service
|
||||
@@ -16,20 +19,17 @@ type Option func(*Options)
|
||||
|
||||
// Options for the activitylog service
|
||||
type Options struct {
|
||||
Context context.Context
|
||||
Logger log.Logger
|
||||
ServiceAccount config.ServiceAccount
|
||||
Config *config.Config
|
||||
TraceProvider trace.TracerProvider
|
||||
Stream events.Stream
|
||||
RegisteredEvents []events.Unmarshaller
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
Mux *chi.Mux
|
||||
HistoryClient ehsvc.EventHistoryService
|
||||
ValueClient settingssvc.ValueService
|
||||
WriteBufferDuration time.Duration
|
||||
NumConsumers int
|
||||
}
|
||||
|
||||
func Context(ctx context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = ctx
|
||||
}
|
||||
MaxActivities int
|
||||
}
|
||||
|
||||
// Logger configures a logger for the activitylog service
|
||||
@@ -39,10 +39,17 @@ func Logger(log log.Logger) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// ServiceAccount configures a service account for the activitylog service
|
||||
func ServiceAccount(sa config.ServiceAccount) Option {
|
||||
// Config adds the config for the activitylog service
|
||||
func Config(c *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.ServiceAccount = sa
|
||||
o.Config = c
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider adds a tracer provider for the activitylog service
|
||||
func TraceProvider(tp trace.TracerProvider) Option {
|
||||
return func(o *Options) {
|
||||
o.TraceProvider = tp
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,8 +74,23 @@ func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient])
|
||||
}
|
||||
}
|
||||
|
||||
func NumConsumers(num int) Option {
|
||||
// Mux defines the muxer for the service
|
||||
func Mux(m *chi.Mux) Option {
|
||||
return func(o *Options) {
|
||||
o.NumConsumers = num
|
||||
o.Mux = m
|
||||
}
|
||||
}
|
||||
|
||||
// HistoryClient adds a grpc client for the eventhistory service
|
||||
func HistoryClient(hc ehsvc.EventHistoryService) Option {
|
||||
return func(o *Options) {
|
||||
o.HistoryClient = hc
|
||||
}
|
||||
}
|
||||
|
||||
// ValueClient adds a grpc client for the value service
|
||||
func ValueClient(vs settingssvc.ValueService) Option {
|
||||
return func(o *Options) {
|
||||
o.ValueClient = vs
|
||||
}
|
||||
}
|
||||
+7
-8
@@ -1,4 +1,4 @@
|
||||
package http
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -44,6 +44,11 @@ var (
|
||||
StrDescription = l10n.Template("description")
|
||||
)
|
||||
|
||||
// GetActivitiesResponse is the response on GET activities requests
|
||||
type GetActivitiesResponse struct {
|
||||
Activities []libregraph.Activity `json:"value"`
|
||||
}
|
||||
|
||||
// Resource represents an item such as a file or folder
|
||||
type Resource struct {
|
||||
ID string `json:"id"`
|
||||
@@ -306,7 +311,7 @@ func NewActivity(message string, ts time.Time, eventID string, vars map[string]a
|
||||
}
|
||||
|
||||
// GetVars calls other service to gather the required data for the activity variables
|
||||
func (s *svc) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]any, error) {
|
||||
func (s *ActivitylogService) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]any, error) {
|
||||
gwc, err := s.gws.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -322,12 +327,6 @@ func (s *svc) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]a
|
||||
return vars, nil
|
||||
}
|
||||
|
||||
func toSpace(r *provider.Reference) *provider.StorageSpaceId {
|
||||
return &provider.StorageSpaceId{
|
||||
OpaqueId: storagespace.FormatStorageID(r.GetResourceId().GetStorageId(), r.GetResourceId().GetSpaceId()),
|
||||
}
|
||||
}
|
||||
|
||||
func getFolderName(ctx context.Context, gwc gateway.GatewayAPIClient, ref *provider.Reference) string {
|
||||
n := filepath.Base(filepath.Dir(ref.GetPath()))
|
||||
if n == "." || n == "/" {
|
||||
@@ -0,0 +1,676 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base32"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/jellydator/ttlcache/v2"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
)
|
||||
|
||||
// Nats runs into max payload exceeded errors at around 7k activities. Let's keep a buffer.
|
||||
var _maxActivitiesDefault = 6000
|
||||
|
||||
// RawActivity represents an activity as it is stored in the activitylog store
|
||||
type RawActivity struct {
|
||||
EventID string `json:"event_id"`
|
||||
Depth int `json:"depth"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
// ActivitylogService logs events per resource
|
||||
type ActivitylogService struct {
|
||||
cfg *config.Config
|
||||
log log.Logger
|
||||
events <-chan events.Event
|
||||
gws pool.Selectable[gateway.GatewayAPIClient]
|
||||
mux *chi.Mux
|
||||
evHistory ehsvc.EventHistoryService
|
||||
valService settingssvc.ValueService
|
||||
lock sync.RWMutex
|
||||
tp trace.TracerProvider
|
||||
tracer trace.Tracer
|
||||
debouncer *Debouncer
|
||||
parentIdCache *ttlcache.Cache
|
||||
natskv nats.KeyValue
|
||||
|
||||
maxActivities int
|
||||
|
||||
registeredEvents map[string]events.Unmarshaller
|
||||
}
|
||||
|
||||
type Debouncer struct {
|
||||
after time.Duration
|
||||
f func(id string, ra []RawActivity) error
|
||||
pending sync.Map
|
||||
inProgress sync.Map
|
||||
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
type queueItem struct {
|
||||
activities []RawActivity
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
type batchInfo struct {
|
||||
key string
|
||||
count int
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
// NewDebouncer returns a new Debouncer instance
|
||||
func NewDebouncer(d time.Duration, f func(id string, ra []RawActivity) error) *Debouncer {
|
||||
return &Debouncer{
|
||||
after: d,
|
||||
f: f,
|
||||
pending: sync.Map{},
|
||||
inProgress: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
// Debounce restarts the debounce timer for the given space
|
||||
func (d *Debouncer) Debounce(id string, ra RawActivity) {
|
||||
if d.after == 0 {
|
||||
d.f(id, []RawActivity{ra})
|
||||
return
|
||||
}
|
||||
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
|
||||
activities := []RawActivity{ra}
|
||||
item := &queueItem{
|
||||
activities: activities,
|
||||
}
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
// if the item is already in the queue, append the new activities
|
||||
item, ok = i.(*queueItem)
|
||||
if ok {
|
||||
item.activities = append(item.activities, ra)
|
||||
}
|
||||
}
|
||||
|
||||
if item.timer == nil {
|
||||
item.timer = time.AfterFunc(d.after, func() {
|
||||
if _, ok := d.inProgress.Load(id); ok {
|
||||
// Reschedule this run for when the previous run has finished
|
||||
d.mutex.Lock()
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
i.(*queueItem).timer.Reset(d.after)
|
||||
}
|
||||
|
||||
d.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
d.pending.Delete(id)
|
||||
d.inProgress.Store(id, true)
|
||||
defer d.inProgress.Delete(id)
|
||||
d.f(id, item.activities)
|
||||
})
|
||||
}
|
||||
|
||||
d.pending.Store(id, item)
|
||||
}
|
||||
|
||||
// New creates a new ActivitylogService
|
||||
func New(opts ...Option) (*ActivitylogService, error) {
|
||||
o := &Options{
|
||||
MaxActivities: _maxActivitiesDefault,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
|
||||
if o.Stream == nil {
|
||||
return nil, errors.New("stream is required")
|
||||
}
|
||||
|
||||
ch, err := events.Consume(o.Stream, o.Config.Service.Name, o.RegisteredEvents...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cache := ttlcache.NewCache()
|
||||
err = cache.SetTTL(30 * time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Connect to NATS servers
|
||||
natsOptions := nats.Options{
|
||||
Servers: o.Config.Store.Nodes,
|
||||
}
|
||||
if o.Config.Store.EnableTLS {
|
||||
if o.Config.Store.TLSRootCACertificate != "" {
|
||||
// when root ca is configured use it. an insecure flag is ignored.
|
||||
nats.RootCAs(o.Config.Store.TLSRootCACertificate)(&natsOptions)
|
||||
} else {
|
||||
// enable tls and use insecure flag
|
||||
nats.Secure(&tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: o.Config.Store.TLSInsecure})(&natsOptions)
|
||||
}
|
||||
}
|
||||
if o.Config.Store.AuthUsername != "" && o.Config.Store.AuthPassword != "" {
|
||||
nats.UserInfo(o.Config.Store.AuthUsername, o.Config.Store.AuthPassword)(&natsOptions)
|
||||
}
|
||||
conn, err := natsOptions.Connect()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
kv, err := js.KeyValue(o.Config.Store.Database)
|
||||
if err != nil {
|
||||
if !errors.Is(err, nats.ErrBucketNotFound) {
|
||||
return nil, errors.Wrapf(err, "Failed to get bucket (%s)", o.Config.Store.Database)
|
||||
}
|
||||
|
||||
kv, err = js.CreateKeyValue(&nats.KeyValueConfig{
|
||||
Bucket: o.Config.Store.Database,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to create bucket (%s)", o.Config.Store.Database)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &ActivitylogService{
|
||||
log: o.Logger,
|
||||
cfg: o.Config,
|
||||
events: ch,
|
||||
gws: o.GatewaySelector,
|
||||
mux: o.Mux,
|
||||
evHistory: o.HistoryClient,
|
||||
valService: o.ValueClient,
|
||||
lock: sync.RWMutex{},
|
||||
registeredEvents: make(map[string]events.Unmarshaller),
|
||||
tp: o.TraceProvider,
|
||||
tracer: o.TraceProvider.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"),
|
||||
parentIdCache: cache,
|
||||
maxActivities: o.Config.MaxActivities,
|
||||
natskv: kv,
|
||||
}
|
||||
s.debouncer = NewDebouncer(o.Config.WriteBufferDuration, s.storeActivity)
|
||||
|
||||
// run migrations
|
||||
err = s.runMigrations(context.Background(), kv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.mux.Get("/graph/v1beta1/extensions/org.libregraph/activities", s.HandleGetItemActivities)
|
||||
|
||||
for _, e := range o.RegisteredEvents {
|
||||
typ := reflect.TypeOf(e)
|
||||
s.registeredEvents[typ.String()] = e
|
||||
}
|
||||
|
||||
go s.Run()
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Run runs the service
|
||||
func (a *ActivitylogService) Run() {
|
||||
for e := range a.events {
|
||||
var err error
|
||||
switch ev := e.Event.(type) {
|
||||
case events.UploadReady:
|
||||
err = a.AddActivity(ev.FileRef, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.FileTouched:
|
||||
err = a.AddActivity(ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
// Disabled https://github.com/owncloud/ocis/issues/10293
|
||||
//case events.FileDownloaded:
|
||||
// we are only interested in public link downloads - so no need to store others.
|
||||
//if ev.ImpersonatingUser.GetDisplayName() == "Public" {
|
||||
// err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
//}
|
||||
case events.ContainerCreated:
|
||||
err = a.AddActivity(ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemTrashed:
|
||||
err = a.AddActivityTrashed(ev.ID, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemPurged:
|
||||
err = a.RemoveResource(ev.ID)
|
||||
case events.ItemMoved:
|
||||
// remove the cached parent id for this resource
|
||||
a.removeCachedParentID(ev.Ref)
|
||||
|
||||
err = a.AddActivity(ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ShareCreated:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.ShareUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.ShareRemoved:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, ev.Timestamp)
|
||||
case events.LinkCreated:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.LinkUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.LinkRemoved:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.SpaceShared:
|
||||
err = a.AddSpaceActivity(ev.ID, e.ID, ev.Timestamp)
|
||||
case events.SpaceUnshared:
|
||||
err = a.AddSpaceActivity(ev.ID, e.ID, ev.Timestamp)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Interface("event", e).Msg("could not process event")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddActivity adds the activity to the given resource and all its parents
|
||||
func (a *ActivitylogService) AddActivity(initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
}
|
||||
var span trace.Span
|
||||
ctx, span = a.tracer.Start(ctx, "AddActivity")
|
||||
defer span.End()
|
||||
|
||||
return a.addActivity(ctx, initRef, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
|
||||
func (a *ActivitylogService) AddActivityTrashed(resourceID *provider.ResourceId, reference *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
}
|
||||
|
||||
// store activity on trashed item
|
||||
if err := a.storeActivity(storagespace.FormatResourceID(resourceID), []RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("could not store activity: %w", err)
|
||||
}
|
||||
|
||||
// get previous parent
|
||||
ref := &provider.Reference{
|
||||
ResourceId: reference.GetResourceId(),
|
||||
Path: filepath.Dir(reference.GetPath()),
|
||||
}
|
||||
|
||||
var span trace.Span
|
||||
ctx, span = a.tracer.Start(ctx, "AddActivityTrashed")
|
||||
defer span.End()
|
||||
|
||||
return a.addActivity(ctx, ref, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddSpaceActivity adds the activity to the given spaceroot
|
||||
func (a *ActivitylogService) AddSpaceActivity(spaceID *provider.StorageSpaceId, eventID string, timestamp time.Time) error {
|
||||
// spaceID is in format <providerid>$<spaceid>
|
||||
// activitylog service uses format <providerid>$<spaceid>!<resourceid>
|
||||
// lets do some converting, shall we?
|
||||
rid, err := storagespace.ParseID(spaceID.GetOpaqueId())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse space id: %w", err)
|
||||
}
|
||||
rid.OpaqueId = rid.GetSpaceId()
|
||||
return a.storeActivity(storagespace.FormatResourceID(&rid), []RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// Activities returns the activities for the given resource
|
||||
func (a *ActivitylogService) Activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
a.lock.RLock()
|
||||
defer a.lock.RUnlock()
|
||||
|
||||
return a.activities(rid)
|
||||
}
|
||||
|
||||
// RemoveActivities removes the activities from the given resource
|
||||
func (a *ActivitylogService) RemoveActivities(rid *provider.ResourceId, toDelete map[string]struct{}) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
curActivities, err := a.activities(rid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var acts []RawActivity
|
||||
for _, a := range curActivities {
|
||||
if _, ok := toDelete[a.EventID]; !ok {
|
||||
acts = append(acts, a)
|
||||
}
|
||||
}
|
||||
|
||||
b, err := json.Marshal(acts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = a.natskv.Put(storagespace.FormatResourceID(rid), b)
|
||||
return err
|
||||
}
|
||||
|
||||
// RemoveResource removes the resource from the store
|
||||
func (a *ActivitylogService) RemoveResource(rid *provider.ResourceId) error {
|
||||
if rid == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
return a.natskv.Delete(storagespace.FormatResourceID(rid))
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
resourceID := storagespace.FormatResourceID(rid)
|
||||
|
||||
glob := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
watcher, err := a.natskv.Watch(glob, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var activities []RawActivity
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
|
||||
}
|
||||
activities = append(activities, batchActivities...)
|
||||
}
|
||||
|
||||
return activities, nil
|
||||
}
|
||||
|
||||
// note: getResource is abstracted to allow unit testing, in general this will just be utils.GetResource
|
||||
func (a *ActivitylogService) addActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time, getResource func(context.Context, *provider.Reference) (*provider.ResourceInfo, error)) error {
|
||||
var (
|
||||
err error
|
||||
depth int
|
||||
ref = initRef
|
||||
)
|
||||
ctx, span := a.tracer.Start(ctx, "addActivity")
|
||||
defer span.End()
|
||||
for {
|
||||
var info *provider.ResourceInfo
|
||||
id := ref.GetResourceId()
|
||||
if ref.Path != "" {
|
||||
// Path based reference, we need to resolve the resource id
|
||||
ctx, span = a.tracer.Start(ctx, "addActivity.getResource")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get resource info: %w", err)
|
||||
}
|
||||
id = info.GetId()
|
||||
}
|
||||
if id == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
key := storagespace.FormatResourceID(id)
|
||||
a.debouncer.Debounce(key, RawActivity{
|
||||
EventID: eventID,
|
||||
Depth: depth,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
|
||||
if id.OpaqueId == id.SpaceId {
|
||||
// we are at the root of the space, no need to go further
|
||||
break
|
||||
}
|
||||
|
||||
// check if parent id is cached
|
||||
// parent id is cached in the format <storageid>$<spaceid>!<resourceid>
|
||||
// if it is not cached, get the resource info and cache it
|
||||
if parentId == nil {
|
||||
if v, err := a.parentIdCache.Get(key); err != nil {
|
||||
if info == nil {
|
||||
ctx, span := a.tracer.Start(ctx, "addActivity.getResource parent")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil || info.GetParentId() == nil || info.GetParentId().GetOpaqueId() == "" {
|
||||
return fmt.Errorf("could not get parent id: %w", err)
|
||||
}
|
||||
}
|
||||
parentId = info.GetParentId()
|
||||
a.parentIdCache.Set(key, parentId)
|
||||
} else {
|
||||
parentId = v.(*provider.ResourceId)
|
||||
}
|
||||
} else {
|
||||
a.log.Debug().Msg("parent id is cached")
|
||||
}
|
||||
|
||||
depth++
|
||||
ref = &provider.Reference{ResourceId: parentId}
|
||||
parentId = nil // reset parent id so it's not reused in the next iteration
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) storeActivity(resourceID string, activities []RawActivity) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
ctx, span := a.tracer.Start(context.Background(), "storeActivity")
|
||||
defer span.End()
|
||||
|
||||
_, subspan := a.tracer.Start(ctx, "storeActivity.Marshal")
|
||||
b, err := msgpack.Marshal(activities)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = a.tracer.Start(ctx, "storeActivity.natskv.Put")
|
||||
key := natsKey(resourceID, len(activities))
|
||||
_, err = a.natskv.Put(key, b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
ctx, subspan = a.tracer.Start(ctx, "storeActivity.enforceMaxActivities")
|
||||
a.enforceMaxActivities(ctx, resourceID)
|
||||
subspan.End()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) enforceMaxActivities(ctx context.Context, resourceID string) {
|
||||
if a.maxActivities <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
_, subspan := a.tracer.Start(ctx, "enforceMaxActivities.watch")
|
||||
watcher, err := a.natskv.Watch(key, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("resourceID", resourceID).Msg("could not watch")
|
||||
return
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var keys []string
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
|
||||
}
|
||||
keys = append(keys, update.Key())
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = a.tracer.Start(ctx, "enforceMaxActivities.compile")
|
||||
// Parse keys into batches
|
||||
batches := make([]batchInfo, 0)
|
||||
var activitiesCount int
|
||||
for _, k := range keys {
|
||||
parts := strings.SplitN(k, ".", 3)
|
||||
if len(parts) < 3 {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, not enough parts")
|
||||
continue
|
||||
}
|
||||
|
||||
c, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse count")
|
||||
continue
|
||||
}
|
||||
|
||||
// parse timestamp
|
||||
nano, err := strconv.ParseInt(parts[2], 10, 64)
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse timestamp")
|
||||
continue
|
||||
}
|
||||
|
||||
batches = append(batches, batchInfo{
|
||||
key: k,
|
||||
count: c,
|
||||
timestamp: time.Unix(0, nano),
|
||||
})
|
||||
activitiesCount += c
|
||||
}
|
||||
|
||||
// sort batches by timestamp
|
||||
sort.Slice(batches, func(i, j int) bool {
|
||||
return batches[i].timestamp.Before(batches[j].timestamp)
|
||||
})
|
||||
subspan.End()
|
||||
|
||||
_, subspan = a.tracer.Start(ctx, "enforceMaxActivities.delete")
|
||||
// remove oldest keys until we are at max activities
|
||||
for _, b := range batches {
|
||||
if activitiesCount-b.count < a.maxActivities {
|
||||
break
|
||||
}
|
||||
|
||||
activitiesCount -= b.count
|
||||
err = a.natskv.Delete(b.key)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("key", b.key).Msg("could not delete key")
|
||||
break
|
||||
}
|
||||
}
|
||||
subspan.End()
|
||||
}
|
||||
|
||||
func toRef(r *provider.ResourceId) *provider.Reference {
|
||||
return &provider.Reference{
|
||||
ResourceId: r,
|
||||
}
|
||||
}
|
||||
|
||||
func toSpace(r *provider.Reference) *provider.StorageSpaceId {
|
||||
return &provider.StorageSpaceId{
|
||||
OpaqueId: storagespace.FormatStorageID(r.GetResourceId().GetStorageId(), r.GetResourceId().GetSpaceId()),
|
||||
}
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) removeCachedParentID(ref *provider.Reference) {
|
||||
purgeId := ref.GetResourceId()
|
||||
if ref.GetPath() != "" {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get gateway client")
|
||||
return
|
||||
}
|
||||
|
||||
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get service user context")
|
||||
return
|
||||
}
|
||||
|
||||
info, err := utils.GetResource(ctx, ref, gwc)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get resource info")
|
||||
return
|
||||
}
|
||||
purgeId = info.GetId()
|
||||
}
|
||||
// The parent id cache is populated lazily and its entries expire, so a
|
||||
// missing key is the expected case rather than an error.
|
||||
if err := a.parentIdCache.Remove(storagespace.FormatResourceID(purgeId)); err != nil {
|
||||
a.log.Debug().Interface("event", ref).Err(err).Msg("could not delete parent id cache")
|
||||
}
|
||||
}
|
||||
|
||||
func natsKey(resourceID string, activitiesCount int) string {
|
||||
return fmt.Sprintf("%s.%d.%d",
|
||||
base32.StdEncoding.EncodeToString([]byte(resourceID)),
|
||||
activitiesCount,
|
||||
time.Now().UnixNano())
|
||||
}
|
||||
+3
-3
@@ -1,4 +1,4 @@
|
||||
package events_test
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestEvents(t *testing.T) {
|
||||
func TestService(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Events Suite")
|
||||
RunSpecs(t, "Service Suite")
|
||||
}
|
||||
+81
-34
@@ -1,6 +1,7 @@
|
||||
package activitylog_test
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
@@ -8,13 +9,17 @@ import (
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
nserver "github.com/nats-io/nats-server/v2/server"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/command"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
eventsmocks "github.com/opencloud-eu/reva/v2/pkg/events/mocks"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/test-go/testify/mock"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -63,25 +68,31 @@ var _ = SynchronizedAfterSuite(func() {
|
||||
|
||||
var _ = Describe("ActivitylogService", func() {
|
||||
var (
|
||||
alog *activitylog.ActivityLog
|
||||
alog *ActivitylogService
|
||||
getResource func(_ context.Context, ref *provider.Reference) (*provider.ResourceInfo, error)
|
||||
writebufferduration = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
JustBeforeEach(func() {
|
||||
var err error
|
||||
db := "activitylog-test-" + uuid.New().String()
|
||||
|
||||
kv, err := command.ConnectNatsKV(config.Store{
|
||||
Nodes: []string{server.Addr().String()},
|
||||
Database: db,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
alog, err = activitylog.New(
|
||||
kv,
|
||||
activitylog.MaxActivities(4),
|
||||
activitylog.WriteBufferDuration(writebufferduration),
|
||||
stream := &eventsmocks.Stream{}
|
||||
stream.EXPECT().Consume(mock.Anything, mock.Anything).Return(nil, nil)
|
||||
alog, err = New(
|
||||
Config(&config.Config{
|
||||
Service: config.Service{
|
||||
Name: "activitylog-test",
|
||||
},
|
||||
Store: config.Store{
|
||||
Store: "nats-js-kv",
|
||||
Nodes: []string{server.Addr().String()},
|
||||
Database: "activitylog-test-" + uuid.New().String(),
|
||||
},
|
||||
MaxActivities: 4,
|
||||
WriteBufferDuration: writebufferduration,
|
||||
}),
|
||||
Stream(stream),
|
||||
TraceProvider(noop.NewTracerProvider()),
|
||||
Mux(chi.NewMux()),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
@@ -96,7 +107,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
Name string
|
||||
Tree map[string]*provider.ResourceInfo
|
||||
Activities map[string]string
|
||||
Expected map[string][]activitylog.RawActivity
|
||||
Expected map[string][]RawActivity
|
||||
}
|
||||
|
||||
testCases := []testCase{
|
||||
@@ -110,7 +121,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
Activities: map[string]string{
|
||||
"activity": "base",
|
||||
},
|
||||
Expected: map[string][]activitylog.RawActivity{
|
||||
Expected: map[string][]RawActivity{
|
||||
"base": activitites("activity", 0),
|
||||
"parent": activitites("activity", 1),
|
||||
"spaceid": activitites("activity", 2),
|
||||
@@ -127,7 +138,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
"activity1": "base",
|
||||
"activity2": "base",
|
||||
},
|
||||
Expected: map[string][]activitylog.RawActivity{
|
||||
Expected: map[string][]RawActivity{
|
||||
"base": activitites("activity1", 0, "activity2", 0),
|
||||
"parent": activitites("activity1", 1, "activity2", 1),
|
||||
"spaceid": activitites("activity1", 2, "activity2", 2),
|
||||
@@ -144,7 +155,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
}
|
||||
|
||||
for k, v := range tc.Activities {
|
||||
err := alog.AddActivity(context.Background(), reference(v), nil, k, time.Time{}, getResource)
|
||||
err := alog.addActivity(context.Background(), reference(v), nil, k, time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
})
|
||||
@@ -183,9 +194,9 @@ var _ = Describe("ActivitylogService", func() {
|
||||
|
||||
It("debounces activities", func() {
|
||||
|
||||
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -196,7 +207,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
})
|
||||
|
||||
It("adheres to the MaxActivities setting", func() {
|
||||
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Eventually(func(g Gomega) {
|
||||
activities, err := alog.Activities(resourceID("base"))
|
||||
@@ -204,7 +215,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
g.Expect(len(activities)).To(Equal(1))
|
||||
}).Should(Succeed())
|
||||
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Eventually(func(g Gomega) {
|
||||
activities, err := alog.Activities(resourceID("base"))
|
||||
@@ -212,11 +223,11 @@ var _ = Describe("ActivitylogService", func() {
|
||||
g.Expect(len(activities)).To(Equal(2))
|
||||
}).Should(Succeed())
|
||||
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity5", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity5", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -233,9 +244,9 @@ var _ = Describe("ActivitylogService", func() {
|
||||
return tree[ref.GetResourceId().GetOpaqueId()], nil
|
||||
}
|
||||
|
||||
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -244,9 +255,9 @@ var _ = Describe("ActivitylogService", func() {
|
||||
g.Expect(activities).To(ConsistOf(activitites("activity1", 0, "activity2", 0)))
|
||||
}).Should(Succeed())
|
||||
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -257,11 +268,47 @@ var _ = Describe("ActivitylogService", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("removeCachedParentID", func() {
|
||||
var logBuffer *bytes.Buffer
|
||||
|
||||
newLoggerAtLevel := func(level string) log.Logger {
|
||||
logBuffer = &bytes.Buffer{}
|
||||
return log.Logger{Logger: log.NewLogger(log.Level(level)).Output(logBuffer)}
|
||||
}
|
||||
|
||||
It("does not log an error when the entry was never cached", func() {
|
||||
alog.log = newLoggerAtLevel("error")
|
||||
|
||||
alog.removeCachedParentID(reference("never-cached"))
|
||||
|
||||
Expect(logBuffer.String()).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("logs a missing entry at debug level", func() {
|
||||
alog.log = newLoggerAtLevel("debug")
|
||||
|
||||
alog.removeCachedParentID(reference("never-cached"))
|
||||
|
||||
Expect(logBuffer.String()).To(ContainSubstring("could not delete parent id cache"))
|
||||
Expect(logBuffer.String()).To(ContainSubstring(`"level":"debug"`))
|
||||
})
|
||||
|
||||
It("does not log at all when the entry was cached", func() {
|
||||
alog.log = newLoggerAtLevel("debug")
|
||||
ref := reference("cached")
|
||||
Expect(alog.parentIdCache.Set(storagespace.FormatResourceID(ref.GetResourceId()), resourceID("parent"))).To(Succeed())
|
||||
|
||||
alog.removeCachedParentID(ref)
|
||||
|
||||
Expect(logBuffer.String()).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func activitites(acts ...any) []activitylog.RawActivity {
|
||||
var activities []activitylog.RawActivity
|
||||
act := activitylog.RawActivity{}
|
||||
func activitites(acts ...any) []RawActivity {
|
||||
var activities []RawActivity
|
||||
act := RawActivity{}
|
||||
for _, a := range acts {
|
||||
switch v := a.(type) {
|
||||
case string:
|
||||
@@ -63,6 +63,10 @@ Store specific notes:
|
||||
- When using `nats-js-kv` it is recommended to set `OC_CACHE_STORE_NODES` to the same value as `OC_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OC_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
### Auto-Accept Shares
|
||||
|
||||
When setting the `SHARING_AUTO_ACCEPT_SHARES` to `true` (sharing service), all incoming shares will be accepted automatically. Users can overwrite this setting individually in their profile. The deprecated `FRONTEND_AUTO_ACCEPT_SHARES` is still supported for backwards compatibility.
|
||||
|
||||
## Passwords
|
||||
|
||||
### The Password Policy
|
||||
|
||||
@@ -61,7 +61,7 @@ type Spaces struct {
|
||||
}
|
||||
|
||||
type LDAPMetrics struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_LDAP_METRICS_DISABLE" desc:"Disables the metrics for outbound LDAP operations." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_LDAP_METRICS_DISABLE" desc:"Disables the metrics for outbound LDAP operations." introductionVersion:"%NEXT%"`
|
||||
}
|
||||
|
||||
type LDAP struct {
|
||||
@@ -121,7 +121,7 @@ type LDAPEducationConfig struct {
|
||||
}
|
||||
|
||||
type IdentityMetrics struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_IDENTITY_BACKEND_METRICS_DISABLE" desc:"Disables the metrics for inbound identity backend operations." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_IDENTITY_BACKEND_METRICS_DISABLE" desc:"Disables the metrics for inbound identity backend operations." introductionVersion:"%NEXT%"`
|
||||
}
|
||||
|
||||
type Identity struct {
|
||||
@@ -141,7 +141,7 @@ type API struct {
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
DisabledConsumer bool `yaml:"disabled_consumer" env:"GRAPH_EVENTS_DISABLE_CONSUMER" desc:"Disables consuming events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%%NEXT%%"`
|
||||
DisabledConsumer bool `yaml:"disabled_consumer" env:"GRAPH_EVENTS_DISABLE_CONSUMER" desc:"Disables consuming events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%NEXT%"`
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;GRAPH_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Set to a empty string to disable emitting events." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;GRAPH_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;GRAPH_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -3,12 +3,12 @@ package config
|
||||
import "github.com/opencloud-eu/opencloud/pkg/shared"
|
||||
|
||||
type HTTPMetrics struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_METRICS_DISABLE" desc:"Disables the metrics for the HTTP service." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_METRICS_DISABLE" desc:"Disables the metrics for the HTTP service." introductionVersion:"%NEXT%"`
|
||||
}
|
||||
|
||||
// HTTP defines the available http configuration.
|
||||
type HTTP struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_DISABLE" desc:"Disables the HTTP service. Set this to true if the service should only consume events." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_DISABLE" desc:"Disables the HTTP service. Set this to true if the service should only consume events." introductionVersion:"%NEXT%"`
|
||||
Addr string `yaml:"addr" env:"GRAPH_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
Root string `yaml:"root" env:"GRAPH_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -40,7 +40,11 @@ func ParseConfig(cfg *config.Config) error {
|
||||
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.HTTP.Disabled && cfg.Events.DisabledConsumer {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
// might be debatable, but this situation should be treated as an error,
|
||||
// as the process wouldn't be able to serve either API and would thus be
|
||||
// completely useless -- in that case, just don't start this service
|
||||
// in the first place (especially since it's optional)
|
||||
return shared.AllComponentsDisabledError("graph")
|
||||
}
|
||||
|
||||
if cfg.TokenManager.JWTSecret == "" {
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
# SOME DESCRIPTIVE TITLE.
|
||||
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
|
||||
# This file is distributed under the same license as the PACKAGE package.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
|
||||
#
|
||||
# Translators:
|
||||
# Roy-Magne Mo, 2026
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: \n"
|
||||
"Report-Msgid-Bugs-To: EMAIL\n"
|
||||
"POT-Creation-Date: 2026-09-07 23:16+0000\n"
|
||||
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
|
||||
"Last-Translator: Roy-Magne Mo, 2026\n"
|
||||
"Language-Team: Norwegian (https://app.transifex.com/opencloud-eu/teams/204053/no/)\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Language: no\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
||||
|
||||
#. UnifiedRole Editor, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole Editor, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole EditorListGrants, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole SpaseEditor, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole FileEditor, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole FileEditorWithVersions, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole FileEditorListGrants, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:137 pkg/unifiedrole/roles.go:143
|
||||
#: pkg/unifiedrole/roles.go:149 pkg/unifiedrole/roles.go:155
|
||||
#: pkg/unifiedrole/roles.go:167 pkg/unifiedrole/roles.go:173
|
||||
#: pkg/unifiedrole/roles.go:179
|
||||
msgid "Can edit"
|
||||
msgstr "Kan redigere"
|
||||
|
||||
#. UnifiedRole SpaseEditorWithoutVersions, Role DisplayName (resolves
|
||||
#. directly)
|
||||
#: pkg/unifiedrole/roles.go:161
|
||||
msgid "Can edit without versions"
|
||||
msgstr "Kan redigere uten versjoner"
|
||||
|
||||
#. UnifiedRole Manager, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:191
|
||||
msgid "Can manage"
|
||||
msgstr "Kan håndtere"
|
||||
|
||||
#. UnifiedRole EditorLite, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:185
|
||||
msgid "Can upload"
|
||||
msgstr "Kan laste opp"
|
||||
|
||||
#. UnifiedRole Viewer, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole ViewerWithVersions, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole Viewer, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole SpaseViewer, Role DisplayName (resolves directly)
|
||||
#. UnifiedRole SpaseViewer, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:107 pkg/unifiedrole/roles.go:113
|
||||
#: pkg/unifiedrole/roles.go:119 pkg/unifiedrole/roles.go:125
|
||||
#: pkg/unifiedrole/roles.go:131
|
||||
msgid "Can view"
|
||||
msgstr "Kan se"
|
||||
|
||||
#. UnifiedRole SecureViewer, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:197
|
||||
msgid "Can view (secure)"
|
||||
msgstr "Kan vise (sikker)"
|
||||
|
||||
#. UnifiedRole FullDenial, Role DisplayName (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:203
|
||||
msgid "Cannot access"
|
||||
msgstr "Kan ikke få tilgang"
|
||||
|
||||
#. UnifiedRole FullDenial, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:200
|
||||
msgid "Deny all access."
|
||||
msgstr "Avvis all tilgang."
|
||||
|
||||
#. default description for new spaces
|
||||
#: pkg/service/v0/spacetemplates.go:32
|
||||
msgid "Here you can add a description for this Space."
|
||||
msgstr "Her kan du legge til en beskrivelse for dette rommet."
|
||||
|
||||
#. UnifiedRole ViewerWithVersions, Role Description (resolves directly)
|
||||
#. UnifiedRole SpaceViewer, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:110 pkg/unifiedrole/roles.go:128
|
||||
msgid "View and download including the history."
|
||||
msgstr "Vis og last ned, inkludert historikken."
|
||||
|
||||
#. UnifiedRole Viewer, Role Description (resolves directly)
|
||||
#. UnifiedRole SpaceViewer, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:104 pkg/unifiedrole/roles.go:122
|
||||
msgid "View and download."
|
||||
msgstr "Vis og last ned."
|
||||
|
||||
#. UnifiedRole SecureViewer, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:194
|
||||
msgid "View only documents, images and PDFs. Watermarks will be applied."
|
||||
msgstr "Vis kun dokumenter, bilder og PDF-er. Vannmerker vil bli brukt."
|
||||
|
||||
#. UnifiedRole FileEditorWithVErsions, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:176
|
||||
msgid "View, download and edit including the history."
|
||||
msgstr "Vis, last ned og rediger, inkludert historikken."
|
||||
|
||||
#. UnifiedRole FileEditor, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:164
|
||||
msgid "View, download and edit."
|
||||
msgstr "Vis, last ned og rediger."
|
||||
|
||||
#. UnifiedRole ViewerListGrants, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:116
|
||||
msgid "View, download and show all invited people."
|
||||
msgstr "Vis, last ned og vis alle inviterte personer."
|
||||
|
||||
#. UnifiedRole EditorLite, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:182
|
||||
msgid "View, download and upload."
|
||||
msgstr "Vis, last ned og last opp."
|
||||
|
||||
#. UnifiedRole FileEditorListGrants, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:170
|
||||
msgid "View, download, edit and show all invited people."
|
||||
msgstr "Vis, last ned, rediger og vis alle inviterte personer."
|
||||
|
||||
#. UnifiedRole Editor, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:140
|
||||
msgid "View, download, upload, edit, add and delete including the history."
|
||||
msgstr ""
|
||||
"Vis, last ned, last opp, rediger, legg til og slett, inkludert historikken."
|
||||
|
||||
#. UnifiedRole Editor, Role Description (resolves directly)
|
||||
#. UnifiedRole SpaseEditorWithoutVersions, Role Description (resolves
|
||||
#. directly)
|
||||
#: pkg/unifiedrole/roles.go:134 pkg/unifiedrole/roles.go:158
|
||||
msgid "View, download, upload, edit, add and delete."
|
||||
msgstr "Vis, last ned, last opp, rediger, legg til og slett."
|
||||
|
||||
#. UnifiedRole Manager, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:188
|
||||
msgid "View, download, upload, edit, add, delete and manage members."
|
||||
msgstr ""
|
||||
"Vis, last ned, last opp, rediger, legg til, slett og administrer medlemmer."
|
||||
|
||||
#. UnifiedRoleListGrants Editor, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:146
|
||||
msgid "View, download, upload, edit, add, delete and show all invited people."
|
||||
msgstr ""
|
||||
"Se, last ned, last opp, rediger, legg til, slett og vis alle inviterte "
|
||||
"personer."
|
||||
|
||||
#. UnifiedRole SpaseEditor, Role Description (resolves directly)
|
||||
#: pkg/unifiedrole/roles.go:152
|
||||
msgid "View, download, upload, edit, add, delete including the history."
|
||||
msgstr ""
|
||||
"Vis, last ned, last opp, rediger, legg til, slett inkludert historikken."
|
||||
@@ -15,16 +15,13 @@ import (
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
cs3rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
|
||||
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
|
||||
storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
|
||||
"github.com/go-chi/render"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"golang.org/x/crypto/sha3"
|
||||
"google.golang.org/protobuf/types/known/fieldmaskpb"
|
||||
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/publicshare"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/tags"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
@@ -36,19 +33,10 @@ import (
|
||||
)
|
||||
|
||||
// opt-in driveItem instance annotations, returned only when requested via $select
|
||||
const (
|
||||
_selectAllowedValues = "@libre.graph.permissions.actions.allowedValues"
|
||||
_selectShareTypes = "@libre.graph.shareTypes"
|
||||
)
|
||||
|
||||
// without it the provider leaves the share-types opaque empty
|
||||
var shareTypesFieldMask = &fieldmaskpb.FieldMask{Paths: []string{"share-types"}}
|
||||
const _selectAllowedValues = "@libre.graph.permissions.actions.allowedValues"
|
||||
|
||||
// opt-in driveItem relations, returned only when requested via $expand
|
||||
const (
|
||||
_expandChildren = "children"
|
||||
_expandThumbnails = "thumbnails"
|
||||
)
|
||||
const _expandChildren = "children"
|
||||
|
||||
func odataListContains(r *http.Request, parameter, value string) bool {
|
||||
for _, values := range r.URL.Query()[parameter] {
|
||||
@@ -222,14 +210,9 @@ func (g Graph) GetRootDriveChildren(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
listRequest := &storageprovider.ListContainerRequest{
|
||||
lRes, err := gatewayClient.ListContainer(ctx, &storageprovider.ListContainerRequest{
|
||||
Ref: &storageprovider.Reference{ResourceId: space.GetRoot()},
|
||||
}
|
||||
if driveItemPropertySelected(r, _selectShareTypes) {
|
||||
listRequest.FieldMask = shareTypesFieldMask
|
||||
}
|
||||
|
||||
lRes, err := gatewayClient.ListContainer(ctx, listRequest)
|
||||
})
|
||||
switch {
|
||||
case err != nil:
|
||||
g.logger.Error().Err(err).Msg("error making ListContainer grpc call")
|
||||
@@ -256,7 +239,6 @@ func (g Graph) GetRootDriveChildren(w http.ResponseWriter, r *http.Request) {
|
||||
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
g.setDriveItemsThumbnails(r, files, lRes.GetInfos())
|
||||
|
||||
if driveItemPropertySelected(r, _selectAllowedValues) {
|
||||
for i, info := range lRes.GetInfos() {
|
||||
@@ -264,10 +246,6 @@ func (g Graph) GetRootDriveChildren(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
if driveItemPropertySelected(r, _selectShareTypes) {
|
||||
g.addShareTypes(ctx, files, lRes.GetInfos())
|
||||
}
|
||||
|
||||
render.Status(r, http.StatusOK)
|
||||
render.JSON(w, r, &ListResponse{Value: files})
|
||||
}
|
||||
@@ -345,15 +323,6 @@ func (g Graph) GetDriveItem(w http.ResponseWriter, r *http.Request) {
|
||||
driveItem.Children = children
|
||||
}
|
||||
|
||||
if driveItemPropertySelected(r, _selectShareTypes) {
|
||||
infos := []*storageprovider.ResourceInfo{res.GetInfo()}
|
||||
driveItem.LibreGraphShareTypes = shareTypesOf(res.GetInfo(), g.listLinkShares(ctx, infos))
|
||||
}
|
||||
|
||||
if driveItemRelationExpanded(r, _expandThumbnails) {
|
||||
setDriveItemThumbnails(driveItem, res.GetInfo(), g.config.Commons.OpenCloudURL)
|
||||
}
|
||||
|
||||
render.Status(r, http.StatusOK)
|
||||
render.JSON(w, r, &driveItem)
|
||||
}
|
||||
@@ -402,14 +371,9 @@ func (g Graph) listDriveItemChildren(w http.ResponseWriter, r *http.Request, dri
|
||||
return nil, false
|
||||
}
|
||||
|
||||
childrenRequest := &storageprovider.ListContainerRequest{
|
||||
res, err := gatewayClient.ListContainer(r.Context(), &storageprovider.ListContainerRequest{
|
||||
Ref: &storageprovider.Reference{ResourceId: driveItemID},
|
||||
}
|
||||
if driveItemPropertySelected(r, _selectShareTypes) {
|
||||
childrenRequest.FieldMask = shareTypesFieldMask
|
||||
}
|
||||
|
||||
res, err := gatewayClient.ListContainer(r.Context(), childrenRequest)
|
||||
})
|
||||
switch {
|
||||
case err != nil:
|
||||
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
|
||||
@@ -436,12 +400,6 @@ func (g Graph) listDriveItemChildren(w http.ResponseWriter, r *http.Request, dri
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if driveItemPropertySelected(r, _selectShareTypes) {
|
||||
g.addShareTypes(r.Context(), files, res.GetInfos())
|
||||
}
|
||||
|
||||
g.setDriveItemsThumbnails(r, files, res.GetInfos())
|
||||
|
||||
return files, true
|
||||
}
|
||||
|
||||
@@ -557,15 +515,12 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
|
||||
driveItem.Photo = metadataToFacet[libregraph.Photo](metadata, "photo")
|
||||
driveItem.Video = metadataToFacet[libregraph.Video](metadata, "video")
|
||||
driveItem.LibreGraphMotionPhoto = metadataToFacet[libregraph.MotionPhoto](metadata, "motionPhoto")
|
||||
driveItem.LibreGraphLivePhoto = metadataToFacet[libregraph.LivePhoto](metadata, "livePhoto")
|
||||
driveItem.LibreGraphMeFollowing = libregraph.PtrBool(metadata[_favoriteMetadataKey] == "1")
|
||||
if t := metadata["tags"]; t != "" {
|
||||
driveItem.LibreGraphTags = tags.New(t).AsSlice()
|
||||
}
|
||||
}
|
||||
|
||||
driveItem.LockInfo = lockToFacet(res.GetLock())
|
||||
|
||||
if utils.IsProcessing(res) {
|
||||
// queuedDateTime stays absent, we do not track when postprocessing started
|
||||
driveItem.PendingOperations = &libregraph.PendingOperations{
|
||||
@@ -576,103 +531,6 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
|
||||
return driveItem, nil
|
||||
}
|
||||
|
||||
// lockToFacet maps a CS3 lock; WebDAV renders the same state as d:lockdiscovery.
|
||||
// Only exclusive locks are reported, that is all OpenCloud issues.
|
||||
func lockToFacet(lock *storageprovider.Lock) *libregraph.LockInfo {
|
||||
switch lock.GetType() {
|
||||
case storageprovider.LockType_LOCK_TYPE_EXCL, storageprovider.LockType_LOCK_TYPE_WRITE:
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
info := libregraph.NewLockInfo()
|
||||
info.SetLockType("exclusive")
|
||||
|
||||
if appName := lock.GetAppName(); appName != "" {
|
||||
info.SetLibreGraphAppName(appName)
|
||||
}
|
||||
|
||||
if user := lock.GetUser(); user != nil {
|
||||
info.SetOwners([]libregraph.Identity{{
|
||||
Id: libregraph.PtrString(user.GetOpaqueId()),
|
||||
DisplayName: utils.ReadPlainFromOpaque(lock.GetOpaque(), "lockownername"),
|
||||
}})
|
||||
}
|
||||
|
||||
if lockTime, err := time.Parse(time.RFC3339, utils.ReadPlainFromOpaque(lock.GetOpaque(), "locktime")); err == nil {
|
||||
info.SetCreatedDateTime(lockTime.UTC())
|
||||
}
|
||||
|
||||
if expiration := lock.GetExpiration(); expiration != nil {
|
||||
info.SetExpirationDateTime(time.Unix(int64(expiration.GetSeconds()), 0).UTC())
|
||||
}
|
||||
|
||||
return info
|
||||
}
|
||||
|
||||
// addShareTypes reads user and group shares off the grants, links from the share
|
||||
// manager. Links are not stored as grants, so they need one filter per item,
|
||||
// which is why the whole annotation is only built when it is selected.
|
||||
func (g Graph) addShareTypes(ctx context.Context, items []libregraph.DriveItem, infos []*storageprovider.ResourceInfo) {
|
||||
linkShares := g.listLinkShares(ctx, infos)
|
||||
|
||||
for i, info := range infos {
|
||||
if i >= len(items) {
|
||||
break
|
||||
}
|
||||
|
||||
items[i].LibreGraphShareTypes = shareTypesOf(info, linkShares)
|
||||
}
|
||||
}
|
||||
|
||||
// shareTypesOf maps the grants on a resource, plus a hit in the link lookup, to
|
||||
// the annotation values.
|
||||
func shareTypesOf(info *storageprovider.ResourceInfo, linkShares map[string]struct{}) []string {
|
||||
var types []string
|
||||
for _, grant := range strings.Split(utils.ReadPlainFromOpaque(info.GetOpaque(), "share-types"), ",") {
|
||||
switch grant {
|
||||
case strconv.Itoa(int(storageprovider.GranteeType_GRANTEE_TYPE_USER)):
|
||||
types = append(types, "user")
|
||||
case strconv.Itoa(int(storageprovider.GranteeType_GRANTEE_TYPE_GROUP)):
|
||||
types = append(types, "group")
|
||||
}
|
||||
}
|
||||
|
||||
if _, ok := linkShares[info.GetId().GetOpaqueId()]; ok {
|
||||
types = append(types, "link")
|
||||
}
|
||||
|
||||
return types
|
||||
}
|
||||
|
||||
// listLinkShares returns the resource ids that carry a public link. A failed
|
||||
// lookup is logged and treated as "no links", same as the WebDAV PROPFIND.
|
||||
func (g Graph) listLinkShares(ctx context.Context, infos []*storageprovider.ResourceInfo) map[string]struct{} {
|
||||
gatewayClient, err := g.gatewaySelector.Next()
|
||||
if err != nil {
|
||||
g.logger.Error().Err(err).Msg("could not select gateway client for public shares")
|
||||
return nil
|
||||
}
|
||||
|
||||
filters := make([]*link.ListPublicSharesRequest_Filter, 0, len(infos))
|
||||
for _, info := range infos {
|
||||
filters = append(filters, publicshare.ResourceIDFilter(info.GetId()))
|
||||
}
|
||||
|
||||
res, err := gatewayClient.ListPublicShares(ctx, &link.ListPublicSharesRequest{Filters: filters})
|
||||
if err != nil || res.GetStatus().GetCode() != cs3rpc.Code_CODE_OK {
|
||||
g.logger.Error().Err(err).Msg("could not list public shares")
|
||||
return nil
|
||||
}
|
||||
|
||||
linkShares := make(map[string]struct{}, len(res.GetShare()))
|
||||
for _, share := range res.GetShare() {
|
||||
linkShares[share.GetResourceId().GetOpaqueId()] = struct{}{}
|
||||
}
|
||||
|
||||
return linkShares
|
||||
}
|
||||
|
||||
// metadataToFacet builds a DriveItem facet *T from CS3 arbitrary metadata under
|
||||
// the "libre.graph.<facet>." key prefix. Nil when no such keys are present.
|
||||
func metadataToFacet[T any](metadata map[string]string, facet string) *T {
|
||||
|
||||
@@ -11,9 +11,7 @@ import (
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
|
||||
"github.com/go-chi/chi/v5"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -208,40 +206,6 @@ var _ = Describe("Driveitems", func() {
|
||||
Expect(res.Value[0].LibreGraphPermissionsActionsAllowedValues).To(BeNil())
|
||||
})
|
||||
|
||||
It("returns the thumbnails when requested via $expand", func() {
|
||||
cfg.Commons.OpenCloudURL = "https://cloud.test"
|
||||
gatewayClient.On("ListStorageSpaces", mock.Anything, mock.Anything).Return(&provider.ListStorageSpacesResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
StorageSpaces: []*provider.StorageSpace{{Owner: currentUser, Root: &provider.ResourceId{}}},
|
||||
}, nil)
|
||||
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
Infos: []*provider.ResourceInfo{
|
||||
{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
|
||||
MimeType: "image/jpeg",
|
||||
Mtime: utils.TimeToTS(time.Now()),
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
r := httptest.NewRequest(http.MethodGet, "/graph/v1.0/me/drive/root/children?$expand=thumbnails", nil)
|
||||
r = r.WithContext(revactx.ContextSetUser(ctx, currentUser))
|
||||
svc.GetRootDriveChildren(rr, r)
|
||||
Expect(rr.Code).To(Equal(http.StatusOK))
|
||||
data, err := io.ReadAll(rr.Body)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
res := itemsList{}
|
||||
Expect(json.Unmarshal(data, &res)).To(Succeed())
|
||||
Expect(len(res.Value)).To(Equal(1))
|
||||
Expect(res.Value[0].Thumbnails).To(HaveLen(1))
|
||||
Expect(res.Value[0].Thumbnails[0].Small.GetUrl()).To(Equal(
|
||||
"https://cloud.test/dav/spaces/storageid$spaceid!opaqueid" +
|
||||
"?scalingup=0&preview=1&processor=thumbnail&x=36&y=36",
|
||||
))
|
||||
})
|
||||
|
||||
It("returns the allowed actions when requested via $select", func() {
|
||||
gatewayClient.On("ListStorageSpaces", mock.Anything, mock.Anything).Return(&provider.ListStorageSpacesResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
@@ -282,7 +246,6 @@ var _ = Describe("Driveitems", func() {
|
||||
Describe("GetDriveItem", func() {
|
||||
var (
|
||||
folderInfo *provider.ResourceInfo
|
||||
childInfo *provider.ResourceInfo
|
||||
mtime = time.Now()
|
||||
)
|
||||
|
||||
@@ -316,15 +279,16 @@ var _ = Describe("Driveitems", func() {
|
||||
Status: status.NewOK(ctx),
|
||||
Info: folderInfo,
|
||||
}, nil)
|
||||
childInfo = &provider.ResourceInfo{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
|
||||
Etag: "etag",
|
||||
Mtime: utils.TimeToTS(mtime),
|
||||
}
|
||||
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
Infos: []*provider.ResourceInfo{childInfo},
|
||||
Infos: []*provider.ResourceInfo{
|
||||
{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
|
||||
Etag: "etag",
|
||||
Mtime: utils.TimeToTS(mtime),
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
})
|
||||
|
||||
@@ -346,49 +310,6 @@ var _ = Describe("Driveitems", func() {
|
||||
Expect(getItem(newRequest("?$expand=children")).Children).To(BeNil())
|
||||
gatewayClient.AssertNotCalled(GinkgoT(), "ListContainer", mock.Anything, mock.Anything)
|
||||
})
|
||||
|
||||
Context("$expand=thumbnails", func() {
|
||||
const previewURL = "https://cloud.test/dav/spaces/storageid$spaceid!nodeid" +
|
||||
"?scalingup=0&preview=1&processor=thumbnail"
|
||||
|
||||
BeforeEach(func() {
|
||||
cfg.Commons.OpenCloudURL = "https://cloud.test"
|
||||
folderInfo.Type = provider.ResourceType_RESOURCE_TYPE_FILE
|
||||
folderInfo.MimeType = "image/jpeg"
|
||||
})
|
||||
|
||||
It("leaves thumbnails unset without $expand", func() {
|
||||
Expect(getItem(newRequest("")).Thumbnails).To(BeNil())
|
||||
})
|
||||
|
||||
It("returns the thumbnail urls when requested", func() {
|
||||
thumbnails := getItem(newRequest("?$expand=thumbnails")).Thumbnails
|
||||
|
||||
Expect(thumbnails).To(HaveLen(1))
|
||||
Expect(thumbnails[0].Small.GetUrl()).To(Equal(previewURL + "&x=36&y=36"))
|
||||
Expect(thumbnails[0].Medium.GetUrl()).To(Equal(previewURL + "&x=48&y=48"))
|
||||
Expect(thumbnails[0].Large.GetUrl()).To(Equal(previewURL + "&x=96&y=96"))
|
||||
})
|
||||
|
||||
It("leaves thumbnails unset for a mime type the thumbnailer cannot render", func() {
|
||||
folderInfo.MimeType = "application/zip"
|
||||
|
||||
Expect(getItem(newRequest("?$expand=thumbnails")).Thumbnails).To(BeNil())
|
||||
})
|
||||
|
||||
It("adds them to expanded children as well", func() {
|
||||
folderInfo.Type = provider.ResourceType_RESOURCE_TYPE_CONTAINER
|
||||
folderInfo.MimeType = ""
|
||||
childInfo.MimeType = "image/jpeg"
|
||||
|
||||
item := getItem(newRequest("?$expand=children,thumbnails"))
|
||||
|
||||
// a folder has no preview of its own
|
||||
Expect(item.Thumbnails).To(BeNil())
|
||||
Expect(item.Children).To(HaveLen(1))
|
||||
Expect(item.Children[0].Thumbnails).To(HaveLen(1))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetDriveItem $expand=children error", func() {
|
||||
@@ -515,158 +436,6 @@ var _ = Describe("Driveitems", func() {
|
||||
Expect(res.Value[0].PendingOperations).To(BeNil())
|
||||
})
|
||||
|
||||
It("omits share types unless they are selected", func() {
|
||||
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
Infos: []*provider.ResourceInfo{
|
||||
{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
|
||||
Etag: "etag",
|
||||
Mtime: utils.TimeToTS(mtime),
|
||||
Opaque: utils.AppendPlainToOpaque(nil, "share-types", "1,2"),
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
res := assertItemsList(1)
|
||||
Expect(res.Value[0].LibreGraphShareTypes).To(BeNil())
|
||||
gatewayClient.AssertNotCalled(GinkgoT(), "ListPublicShares", mock.Anything, mock.Anything)
|
||||
})
|
||||
|
||||
It("returns the share types of an item when selected", func() {
|
||||
r = r.WithContext(r.Context())
|
||||
q := r.URL.Query()
|
||||
q.Add("$select", "@libre.graph.shareTypes")
|
||||
r.URL.RawQuery = q.Encode()
|
||||
|
||||
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
Infos: []*provider.ResourceInfo{
|
||||
{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
|
||||
Etag: "etag",
|
||||
Mtime: utils.TimeToTS(mtime),
|
||||
Opaque: utils.AppendPlainToOpaque(nil, "share-types", "1,2"),
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
gatewayClient.On("ListPublicShares", mock.Anything, mock.Anything).Return(&link.ListPublicSharesResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
Share: []*link.PublicShare{
|
||||
{ResourceId: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"}},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
res := assertItemsList(1)
|
||||
Expect(res.Value[0].LibreGraphShareTypes).To(ConsistOf("user", "group", "link"))
|
||||
})
|
||||
|
||||
It("reports only the link when the item has no grants", func() {
|
||||
q := r.URL.Query()
|
||||
q.Add("$select", "@libre.graph.shareTypes")
|
||||
r.URL.RawQuery = q.Encode()
|
||||
|
||||
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
Infos: []*provider.ResourceInfo{
|
||||
{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
|
||||
Etag: "etag",
|
||||
Mtime: utils.TimeToTS(mtime),
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
gatewayClient.On("ListPublicShares", mock.Anything, mock.Anything).Return(&link.ListPublicSharesResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
Share: []*link.PublicShare{
|
||||
{ResourceId: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"}},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
res := assertItemsList(1)
|
||||
Expect(res.Value[0].LibreGraphShareTypes).To(ConsistOf("link"))
|
||||
})
|
||||
|
||||
It("keeps the grant types when the public share lookup fails", func() {
|
||||
q := r.URL.Query()
|
||||
q.Add("$select", "@libre.graph.shareTypes")
|
||||
r.URL.RawQuery = q.Encode()
|
||||
|
||||
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
Infos: []*provider.ResourceInfo{
|
||||
{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
|
||||
Etag: "etag",
|
||||
Mtime: utils.TimeToTS(mtime),
|
||||
Opaque: utils.AppendPlainToOpaque(nil, "share-types", "1"),
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
gatewayClient.On("ListPublicShares", mock.Anything, mock.Anything).Return(nil, errors.New("nope"))
|
||||
|
||||
res := assertItemsList(1)
|
||||
Expect(res.Value[0].LibreGraphShareTypes).To(ConsistOf("user"))
|
||||
})
|
||||
|
||||
It("returns the lock info of a locked item", func() {
|
||||
// a lock time with a non-UTC offset, the way reva writes it
|
||||
lockTime := time.Now().Truncate(time.Second).In(time.FixedZone("CEST", 2*60*60))
|
||||
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
Infos: []*provider.ResourceInfo{
|
||||
{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
|
||||
Etag: "etag",
|
||||
Mtime: utils.TimeToTS(mtime),
|
||||
Lock: &provider.Lock{
|
||||
Type: provider.LockType_LOCK_TYPE_EXCL,
|
||||
AppName: "Collabora",
|
||||
User: &userpb.UserId{OpaqueId: "user-id"},
|
||||
Expiration: &typesv1beta1.Timestamp{Seconds: uint64(lockTime.Add(time.Hour).Unix())},
|
||||
Opaque: utils.AppendPlainToOpaque(
|
||||
utils.AppendPlainToOpaque(nil, "lockownername", "Alice Hansen"),
|
||||
"locktime", lockTime.Format(time.RFC3339)),
|
||||
},
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
res := assertItemsList(1)
|
||||
lock := res.Value[0].LockInfo
|
||||
Expect(lock).ToNot(BeNil())
|
||||
Expect(lock.GetLockType()).To(Equal("exclusive"))
|
||||
Expect(lock.GetLibreGraphAppName()).To(Equal("Collabora"))
|
||||
Expect(lock.GetCreatedDateTime()).To(BeTemporally("==", lockTime))
|
||||
Expect(lock.GetCreatedDateTime().Location()).To(Equal(time.UTC))
|
||||
Expect(lock.GetExpirationDateTime().Location()).To(Equal(time.UTC))
|
||||
Expect(lock.GetExpirationDateTime()).To(BeTemporally("==", lockTime.Add(time.Hour)))
|
||||
Expect(lock.GetOwners()).To(HaveLen(1))
|
||||
Expect(lock.GetOwners()[0].GetId()).To(Equal("user-id"))
|
||||
Expect(lock.GetOwners()[0].GetDisplayName()).To(Equal("Alice Hansen"))
|
||||
})
|
||||
|
||||
It("omits the lock info for an unlocked item", func() {
|
||||
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
Infos: []*provider.ResourceInfo{
|
||||
{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
|
||||
Etag: "etag",
|
||||
Mtime: utils.TimeToTS(mtime),
|
||||
},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
Expect(assertItemsList(1).Value[0].LockInfo).To(BeNil())
|
||||
})
|
||||
|
||||
It("reports a pending content update while the item is being processed", func() {
|
||||
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
|
||||
Status: status.NewOK(ctx),
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
package svc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
|
||||
"github.com/go-chi/render"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/opencloud/services/thumbnails/pkg/thumbnail"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
|
||||
)
|
||||
@@ -42,8 +44,30 @@ func (g Graph) GetSharedByMe(w http.ResponseWriter, r *http.Request) {
|
||||
expandThumbnails := strings.Contains(expand, "thumbnails")
|
||||
if expandThumbnails {
|
||||
for k, item := range driveItems {
|
||||
setShareThumbnails(&item, item.GetId(), g.config.Commons.OpenCloudURL)
|
||||
driveItems[k] = item
|
||||
mt := item.GetFile().MimeType
|
||||
if mt == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
_, match := thumbnail.SupportedMimeTypes[*mt]
|
||||
if match {
|
||||
baseUrl := fmt.Sprintf("%s/dav/spaces/%s?scalingup=0&preview=1&processor=thumbnail",
|
||||
g.config.Commons.OpenCloudURL,
|
||||
item.GetId())
|
||||
smallUrl := baseUrl + "&x=36&y=36"
|
||||
mediumUrl := baseUrl + "&x=48&y=48"
|
||||
largeUrl := baseUrl + "&x=96&y=96"
|
||||
|
||||
item.SetThumbnails([]libregraph.ThumbnailSet{
|
||||
{
|
||||
Small: &libregraph.Thumbnail{Url: &smallUrl},
|
||||
Medium: &libregraph.Thumbnail{Url: &mediumUrl},
|
||||
Large: &libregraph.Thumbnail{Url: &largeUrl},
|
||||
},
|
||||
})
|
||||
|
||||
driveItems[k] = item // assign modified item back to the map
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package svc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
|
||||
"github.com/opencloud-eu/opencloud/services/thumbnails/pkg/thumbnail"
|
||||
)
|
||||
|
||||
// ListSharedWithMe lists the files shared with the current user.
|
||||
@@ -71,8 +73,30 @@ func (g Graph) listSharedWithMe(ctx context.Context, expandThumbnails bool) ([]l
|
||||
|
||||
if expandThumbnails {
|
||||
for k, item := range driveItems {
|
||||
setShareThumbnails(&item, item.RemoteItem.GetId(), g.config.Commons.OpenCloudURL)
|
||||
driveItems[k] = item
|
||||
mt := item.GetFile().MimeType
|
||||
if mt == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
_, match := thumbnail.SupportedMimeTypes[*mt]
|
||||
if match {
|
||||
baseUrl := fmt.Sprintf("%s/dav/spaces/%s?scalingup=0&preview=1&processor=thumbnail",
|
||||
g.config.Commons.OpenCloudURL,
|
||||
item.RemoteItem.GetId())
|
||||
smallUrl := baseUrl + "&x=36&y=36"
|
||||
mediumUrl := baseUrl + "&x=48&y=48"
|
||||
largeUrl := baseUrl + "&x=96&y=96"
|
||||
|
||||
item.SetThumbnails([]libregraph.ThumbnailSet{
|
||||
{
|
||||
Small: &libregraph.Thumbnail{Url: &smallUrl},
|
||||
Medium: &libregraph.Thumbnail{Url: &mediumUrl},
|
||||
Large: &libregraph.Thumbnail{Url: &largeUrl},
|
||||
},
|
||||
})
|
||||
|
||||
driveItems[k] = item // assign modified item back to the map
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
package svc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/thumbnails/pkg/thumbnail"
|
||||
)
|
||||
|
||||
const (
|
||||
thumbnailBoxSmall = 36
|
||||
thumbnailBoxMedium = 48
|
||||
thumbnailBoxLarge = 96
|
||||
)
|
||||
|
||||
func (g Graph) setDriveItemsThumbnails(r *http.Request, items []libregraph.DriveItem, infos []*provider.ResourceInfo) {
|
||||
if !driveItemRelationExpanded(r, _expandThumbnails) {
|
||||
return
|
||||
}
|
||||
for i := range items {
|
||||
if i < len(infos) {
|
||||
setDriveItemThumbnails(&items[i], infos[i], g.config.Commons.OpenCloudURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func setDriveItemThumbnails(item *libregraph.DriveItem, res *provider.ResourceInfo, baseURL string) {
|
||||
if set := previewThumbnailSet(res, baseURL); set != nil {
|
||||
item.SetThumbnails([]libregraph.ThumbnailSet{*set})
|
||||
}
|
||||
}
|
||||
|
||||
// previewThumbnailSet returns nil when the thumbnailer cannot render the resource.
|
||||
func previewThumbnailSet(res *provider.ResourceInfo, baseURL string) *libregraph.ThumbnailSet {
|
||||
if !thumbnail.IsMimeTypeSupported(res.GetMimeType()) {
|
||||
return nil
|
||||
}
|
||||
return thumbnailSetFor(baseURL, storagespace.FormatResourceID(res.GetId()))
|
||||
}
|
||||
|
||||
// thumbnailSetFor builds the urls of the WebDAV preview endpoint.
|
||||
func thumbnailSetFor(baseURL, itemID string) *libregraph.ThumbnailSet {
|
||||
base := fmt.Sprintf("%s/dav/spaces/%s?scalingup=0&preview=1&processor=thumbnail", baseURL, itemID)
|
||||
return &libregraph.ThumbnailSet{
|
||||
Small: previewThumbnail(base, thumbnailBoxSmall),
|
||||
Medium: previewThumbnail(base, thumbnailBoxMedium),
|
||||
Large: previewThumbnail(base, thumbnailBoxLarge),
|
||||
}
|
||||
}
|
||||
|
||||
func previewThumbnail(base string, box int32) *libregraph.Thumbnail {
|
||||
url := fmt.Sprintf("%s&x=%d&y=%d", base, box, box)
|
||||
return &libregraph.Thumbnail{Url: &url}
|
||||
}
|
||||
|
||||
// setShareThumbnails works off the driveItem, the share listings have no resource
|
||||
// info. The id comes separately, a received share carries it on its remote item.
|
||||
func setShareThumbnails(item *libregraph.DriveItem, itemID, baseURL string) {
|
||||
mimeType := item.GetFile().MimeType
|
||||
if itemID == "" || mimeType == nil || !thumbnail.IsMimeTypeSupported(*mimeType) {
|
||||
return
|
||||
}
|
||||
item.SetThumbnails([]libregraph.ThumbnailSet{*thumbnailSetFor(baseURL, itemID)})
|
||||
}
|
||||
@@ -7,16 +7,15 @@
|
||||
# Jörn Friedrich Dreyer <jfd@butonic.de>, 2025
|
||||
# Jonas, 2025
|
||||
# Jannik Stehle, 2026
|
||||
# Jannick Kuhr, 2026
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: \n"
|
||||
"Report-Msgid-Bugs-To: EMAIL\n"
|
||||
"POT-Creation-Date: 2026-09-03 23:16+0000\n"
|
||||
"POT-Creation-Date: 2026-09-02 23:17+0000\n"
|
||||
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
|
||||
"Last-Translator: Jannick Kuhr, 2026\n"
|
||||
"Last-Translator: Jannik Stehle, 2026\n"
|
||||
"Language-Team: German (https://app.transifex.com/opencloud-eu/teams/204053/de/)\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
@@ -174,9 +173,6 @@ msgid ""
|
||||
"\n"
|
||||
"Even though this share has been revoked you still might have access through other shares and/or space memberships."
|
||||
msgstr ""
|
||||
"{ShareSharer} hat die Freigabe für »{ShareFolder}« für Sie aufgehoben.\n"
|
||||
"\n"
|
||||
"Auch wenn diese Freigabe widerrufen wurde, haben Sie möglicherweise weiterhin Zugriff über andere Freigaben und/oder Mitgliedschaften in Bereichen."
|
||||
|
||||
#. ShareCreated email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:17
|
||||
@@ -187,7 +183,6 @@ msgstr "{ShareSharer} hat \"{ShareFolder}\" mit Ihnen geteilt"
|
||||
#: pkg/email/templates.go:30
|
||||
msgid "{ShareSharer} unshared '{ShareFolder}' with you"
|
||||
msgstr ""
|
||||
"{ShareSharer} hat die Freigabe von »{ShareFolder}« für Sie aufgehoben."
|
||||
|
||||
#. SharedSpace email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:61
|
||||
|
||||
@@ -1,212 +0,0 @@
|
||||
# SOME DESCRIPTIVE TITLE.
|
||||
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
|
||||
# This file is distributed under the same license as the PACKAGE package.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
|
||||
#
|
||||
# Translators:
|
||||
# Roy-Magne Mo, 2026
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: \n"
|
||||
"Report-Msgid-Bugs-To: EMAIL\n"
|
||||
"POT-Creation-Date: 2026-09-07 23:16+0000\n"
|
||||
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
|
||||
"Last-Translator: Roy-Magne Mo, 2026\n"
|
||||
"Language-Team: Norwegian (https://app.transifex.com/opencloud-eu/teams/204053/no/)\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Language: no\n"
|
||||
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
|
||||
|
||||
#. UnsharedSpace email template, resolves via {{ .CallToAction }}
|
||||
#: pkg/email/templates.go:78
|
||||
msgid "Click here to check it: {ShareLink}"
|
||||
msgstr "Klikk her for å sjekke det: {ShareLink}"
|
||||
|
||||
#. ShareCreated email template, resolves via {{ .CallToAction }}
|
||||
#. SharedSpace email template, resolves via {{ .CallToAction }}
|
||||
#: pkg/email/templates.go:23 pkg/email/templates.go:63
|
||||
msgid "Click here to view it: {ShareLink}"
|
||||
msgstr "Klikk her for å se den: {ShareLink}"
|
||||
|
||||
#: pkg/email/templates.go:139
|
||||
msgid "Hello {RecipientName},"
|
||||
msgstr "Hei {RecipientName},"
|
||||
|
||||
#. ShareCreated email template, resolves via {{ .Greeting }}
|
||||
#: pkg/email/templates.go:19
|
||||
msgid "Hello {ShareGrantee}"
|
||||
msgstr "Hei {ShareGrantee}"
|
||||
|
||||
#. ShareRemoved email template, resolves via {{ .Greeting }}
|
||||
#. ShareExpired email template, resolves via {{ .Greeting }}
|
||||
#: pkg/email/templates.go:32 pkg/email/templates.go:45
|
||||
msgid "Hello {ShareGrantee},"
|
||||
msgstr "Hei {ShareGrantee},"
|
||||
|
||||
#. SharedSpace email template, resolves via {{ .Greeting }}
|
||||
#. UnsharedSpace email template, resolves via {{ .Greeting }}
|
||||
#. MembershipExpired email template, resolves via {{ .Greeting }}
|
||||
#: pkg/email/templates.go:59 pkg/email/templates.go:72
|
||||
#: pkg/email/templates.go:87
|
||||
msgid "Hello {SpaceGrantee},"
|
||||
msgstr "Hei {SpaceGrantee},"
|
||||
|
||||
#. Grouped email template, resolves via {{ .Greeting }}
|
||||
#: pkg/email/templates.go:131
|
||||
msgid "Hi {DisplayName},"
|
||||
msgstr "Hei {DisplayName},"
|
||||
|
||||
#. ScienceMeshInviteTokenGenerated email template, resolves via {{ .Greeting
|
||||
#. }}
|
||||
#. ScienceMeshInviteTokenGeneratedWithoutShareLink email template, resolves
|
||||
#. via {{ .Greeting }}
|
||||
#: pkg/email/templates.go:100 pkg/email/templates.go:117
|
||||
msgid "Hi,"
|
||||
msgstr "Hei,"
|
||||
|
||||
#. MembershipExpired email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:85
|
||||
msgid "Membership of '{SpaceName}' expired at {ExpiredAt}"
|
||||
msgstr "Medlemskap i '{SpaceName}' utløpt {ExpiredAt}"
|
||||
|
||||
#. Grouped email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:129
|
||||
msgid "Report"
|
||||
msgstr "Rapport"
|
||||
|
||||
#. ScienceMeshInviteTokenGenerated email template, Subject field (resolves
|
||||
#. directly)
|
||||
#. ScienceMeshInviteTokenGeneratedWithoutShareLink email template, Subject
|
||||
#. field (resolves directly)
|
||||
#: pkg/email/templates.go:98 pkg/email/templates.go:115
|
||||
msgid "ScienceMesh: {InitiatorName} wants to collaborate with you"
|
||||
msgstr "ScienceMesh: {InitiatorName} ønsker å samarbeide med deg"
|
||||
|
||||
#. ShareExpired email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:43
|
||||
msgid "Share to '{ShareFolder}' expired at {ExpiredAt}"
|
||||
msgstr "Deling til '{ShareFolder}' utløpt {ExpiredAt}"
|
||||
|
||||
#: pkg/email/templates.go:141
|
||||
msgid "You can view the mention here: {ResourceLink}"
|
||||
msgstr "Du kan se omtalen her: {ResourceLink}"
|
||||
|
||||
#: pkg/email/templates.go:138
|
||||
msgid "You were mentioned in '{ResourceName}'"
|
||||
msgstr "Du ble nevnt i «{ResourceName}»"
|
||||
|
||||
#. MembershipExpired email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:89
|
||||
msgid ""
|
||||
"Your membership of space {SpaceName} has expired at {ExpiredAt}\n"
|
||||
"\n"
|
||||
"Even though this membership has expired you still might have access through other shares and/or space memberships"
|
||||
msgstr ""
|
||||
"Medlemskapet ditt i rom {SpaceName} utløp {ExpiredAt}\n"
|
||||
"\n"
|
||||
"Selv om dette medlemskapet har utløpt, kan du fortsatt ha tilgang gjennom andre delinger og/eller rommedlemskap"
|
||||
|
||||
#. ShareExpired email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:47
|
||||
msgid ""
|
||||
"Your share to {ShareFolder} has expired at {ExpiredAt}\n"
|
||||
"\n"
|
||||
"Even though this share has been revoked you still might have access through other shares and/or space memberships."
|
||||
msgstr ""
|
||||
"Deltakelsen din i {ShareFolder} utløp {ExpiredAt}\n"
|
||||
"\n"
|
||||
"Selv om denne deltakelsen er trukket tilbake, kan du fortsatt ha tilgang gjennom andre deltakelser og/eller plassmedlemskap."
|
||||
|
||||
#: pkg/email/templates.go:140
|
||||
msgid "{AuthorName} mentioned you in \"{ResourceName}\"."
|
||||
msgstr "{AuthorName} nevnte deg i «{ResourceName}»."
|
||||
|
||||
#. ScienceMeshInviteTokenGeneratedWithoutShareLink email template, resolves
|
||||
#. via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:119
|
||||
msgid ""
|
||||
"{ShareSharer} ({ShareSharerMail}) wants to start sharing collaboration resources with you.\n"
|
||||
"Please visit your federation settings and use the following details:\n"
|
||||
" Token: {Token}\n"
|
||||
" ProviderDomain: {ProviderDomain}"
|
||||
msgstr ""
|
||||
"{ShareSharer} ({ShareSharerMail}) ønsker å begynne å dele samarbeidsressurser med deg.\n"
|
||||
"Vennligst besøk føderasjonsinnstillingene dine og bruk følgende detaljer:\n"
|
||||
" Token: {Token}\n"
|
||||
" ProviderDomain: {ProviderDomain}"
|
||||
|
||||
#. ScienceMeshInviteTokenGenerated email template, resolves via {{
|
||||
#. .MessageBody }}
|
||||
#: pkg/email/templates.go:102
|
||||
msgid ""
|
||||
"{ShareSharer} ({ShareSharerMail}) wants to start sharing collaboration resources with you.\n"
|
||||
"To accept the invite, please visit the following URL:\n"
|
||||
"{ShareLink}\n"
|
||||
"\n"
|
||||
"Alternatively, you can visit your federation settings and use the following details:\n"
|
||||
" Token: {Token}\n"
|
||||
" ProviderDomain: {ProviderDomain}"
|
||||
msgstr ""
|
||||
"{ShareSharer} ({ShareSharerMail}) ønsker å begynne å dele samarbeidsressurser med deg.\n"
|
||||
"For å akseptere invitasjonen, besøk følgende URL:\n"
|
||||
"{ShareLink}\n"
|
||||
"\n"
|
||||
"Alternativt kan du besøke dine føderasjonsinnstillinger og bruke følgende detaljer:\n"
|
||||
" Token: {Token}\n"
|
||||
" ProviderDomain: {ProviderDomain}"
|
||||
|
||||
#. ShareCreated email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:21
|
||||
msgid "{ShareSharer} has shared \"{ShareFolder}\" with you."
|
||||
msgstr "{ShareSharer} har delt «{ShareFolder}» med deg."
|
||||
|
||||
#. ShareRemoved email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:34
|
||||
msgid ""
|
||||
"{ShareSharer} has unshared '{ShareFolder}' with you.\n"
|
||||
"\n"
|
||||
"Even though this share has been revoked you still might have access through other shares and/or space memberships."
|
||||
msgstr ""
|
||||
"{ShareSharer} har fjernet deling '{ShareFolder}' til deg.\n"
|
||||
"\n"
|
||||
"Selv om denne delingen er trukket tilbake, kan du fortsatt ha tilgang gjennom andre delinger og/eller medlemskap i rommet."
|
||||
|
||||
#. ShareCreated email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:17
|
||||
msgid "{ShareSharer} shared '{ShareFolder}' with you"
|
||||
msgstr "{ShareSharer} delte '{ShareFolder}' med deg"
|
||||
|
||||
#. ShareRemoved email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:30
|
||||
msgid "{ShareSharer} unshared '{ShareFolder}' with you"
|
||||
msgstr "{ShareSharer} deler ikke '{ShareFolder}' med deg"
|
||||
|
||||
#. SharedSpace email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:61
|
||||
msgid "{SpaceSharer} has invited you to join \"{SpaceName}\"."
|
||||
msgstr "{SpaceSharer} har invitert deg til å bli med i «{SpaceName}»."
|
||||
|
||||
#. UnsharedSpace email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:74
|
||||
msgid ""
|
||||
"{SpaceSharer} has removed you from \"{SpaceName}\".\n"
|
||||
"\n"
|
||||
"You might still have access through your other groups or direct membership."
|
||||
msgstr ""
|
||||
"{SpaceSharer} har fjernet deg fra «{SpaceName}».\n"
|
||||
"\n"
|
||||
"Du kan fortsatt ha tilgang gjennom andre grupper eller direkte medlemskap."
|
||||
|
||||
#. SharedSpace email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:57
|
||||
msgid "{SpaceSharer} invited you to join {SpaceName}"
|
||||
msgstr "{SpaceSharer} inviterte deg til å bli med i {SpaceName}"
|
||||
|
||||
#. UnsharedSpace email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:70
|
||||
msgid "{SpaceSharer} removed you from {SpaceName}"
|
||||
msgstr "{SpaceSharer} fjernet deg fra {SpaceName}"
|
||||
@@ -12,7 +12,7 @@ msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: \n"
|
||||
"Report-Msgid-Bugs-To: EMAIL\n"
|
||||
"POT-Creation-Date: 2026-09-03 23:16+0000\n"
|
||||
"POT-Creation-Date: 2026-09-02 23:17+0000\n"
|
||||
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
|
||||
"Last-Translator: Daniel Nylander <po@danielnylander.se>, 2026\n"
|
||||
"Language-Team: Swedish (https://app.transifex.com/opencloud-eu/teams/204053/sv/)\n"
|
||||
@@ -172,9 +172,6 @@ msgid ""
|
||||
"\n"
|
||||
"Even though this share has been revoked you still might have access through other shares and/or space memberships."
|
||||
msgstr ""
|
||||
"{ShareSharer} har slutat dela ”{ShareFolder}” med dig.\\n\\nÄven om denna "
|
||||
"delning har återkallats kan du fortfarande ha åtkomst genom andra delningar "
|
||||
"och/eller medlemskap i utrymmen."
|
||||
|
||||
#. ShareCreated email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:17
|
||||
@@ -184,7 +181,7 @@ msgstr "{ShareSharer} delade '{ShareFolder}' med dig"
|
||||
#. ShareRemoved email template, Subject field (resolves directly)
|
||||
#: pkg/email/templates.go:30
|
||||
msgid "{ShareSharer} unshared '{ShareFolder}' with you"
|
||||
msgstr "{ShareSharer} har slutat dela ”{ShareFolder}” med dig"
|
||||
msgstr ""
|
||||
|
||||
#. SharedSpace email template, resolves via {{ .MessageBody }}
|
||||
#: pkg/email/templates.go:61
|
||||
|
||||
@@ -29,7 +29,7 @@ type Service struct {
|
||||
|
||||
// GRPC defines the available grpc configuration.
|
||||
type GRPC struct {
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_GRPC_DISABLED" desc:"Disables listening for GRPC API calls. Set this to true if the service should only handle requests through events." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_GRPC_DISABLED" desc:"Disables listening for GRPC API calls. Set this to true if the service should only handle requests through events." introductionVersion:"%NEXT%"`
|
||||
Addr string `yaml:"addr" env:"POLICIES_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
TLS *shared.GRPCServiceTLS `yaml:"tls"`
|
||||
@@ -50,7 +50,7 @@ type Postprocessing struct {
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle GRPC requests." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle GRPC requests." introductionVersion:"%NEXT%"`
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;POLICIES_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;POLICIES_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;POLICIES_EVENTS_TLS_INSECURE" desc:"Whether the server should skip the client certificate verification during the TLS handshake." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"errors"
|
||||
|
||||
occfg "github.com/opencloud-eu/opencloud/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/pkg/shared"
|
||||
"github.com/opencloud-eu/opencloud/services/policies/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/policies/pkg/config/defaults"
|
||||
|
||||
@@ -34,8 +33,12 @@ func ParseConfig(cfg *config.Config) error {
|
||||
}
|
||||
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.Events.Disabled && cfg.GRPC.Disabled {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
if cfg.GRPC.Disabled && cfg.Events.Disabled {
|
||||
// might be debatable, but this situation should be treated as an error,
|
||||
// as the process wouldn't be able to serve either API and would thus be
|
||||
// completely useless -- in that case, just don't start this service
|
||||
// in the first place (especially since it's optional)
|
||||
return errors.New("both gRPC and events APIs are disabled by configuration; at least one must be enabled")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"go-micro.dev/v4/store"
|
||||
@@ -187,13 +186,6 @@ func verifyExpiresAt(claims map[string]any, cmp time.Time) bool {
|
||||
return cmp.Before(expiry)
|
||||
}
|
||||
|
||||
// isExpectedTokenTimeError reports whether the error is caused by a token that
|
||||
// failed its expiry (or not-before) validation. These are expected as part of
|
||||
// the normal token renewal cycle and should not be logged at error level.
|
||||
func isExpectedTokenTimeError(err error) bool {
|
||||
return errors.Is(err, jwt.ErrTokenExpired) || errors.Is(err, jwt.ErrTokenNotValidYet)
|
||||
}
|
||||
|
||||
func (m OIDCAuthenticator) shouldServe(req *http.Request) bool {
|
||||
if m.OIDCIss == "" {
|
||||
return false
|
||||
@@ -220,17 +212,7 @@ func (m *OIDCAuthenticator) Authenticate(r *http.Request) (*http.Request, bool)
|
||||
claims, newSession, err := m.getClaims(token, r)
|
||||
if err != nil {
|
||||
host, port, _ := net.SplitHostPort(r.RemoteAddr)
|
||||
|
||||
// Expired (or not yet valid) tokens are a normal part of the token
|
||||
// renewal cycle: the client hits the proxy with a token that is just
|
||||
// past its expiry, receives a 401 challenge and refreshes. This is not
|
||||
// a failure of the proxy, so we log it at debug level to avoid noise.
|
||||
logEvent := m.Logger.Debug()
|
||||
if !isExpectedTokenTimeError(err) {
|
||||
logEvent = m.Logger.Error()
|
||||
}
|
||||
|
||||
logEvent.
|
||||
m.Logger.Error().
|
||||
Err(err).
|
||||
Str("authenticator", "oidc").
|
||||
Str("path", r.URL.Path).
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"time"
|
||||
@@ -12,7 +11,6 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/pkg/oidc"
|
||||
oidcmocks "github.com/opencloud-eu/opencloud/pkg/oidc/mocks"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"go-micro.dev/v4/store"
|
||||
)
|
||||
@@ -108,69 +106,4 @@ var _ = Describe("Authenticating requests", Label("OIDCAuthenticator"), func() {
|
||||
Expect(req2).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
When("the request contains an expired token", func() {
|
||||
var buf bytes.Buffer
|
||||
expiredAuthenticator := &OIDCAuthenticator{
|
||||
OIDCIss: "http://idp.example.com",
|
||||
Logger: log.Logger{zerolog.New(&buf).Level(zerolog.DebugLevel)},
|
||||
oidcClient: &oidcmocks.OIDCClient{},
|
||||
userInfoCache: store.NewMemoryStore(),
|
||||
skipUserInfo: true,
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
expiredAuthenticator.oidcClient.(*oidcmocks.OIDCClient).On("VerifyAccessToken", mock.Anything, mock.Anything).Return(
|
||||
oidc.RegClaimsWithSID{}, jwt.MapClaims{}, jwt.ErrTokenExpired,
|
||||
)
|
||||
})
|
||||
|
||||
It("should reject the request", func() {
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/example/path", http.NoBody)
|
||||
req.Header.Set(_headerAuthorization, "Bearer jwt.token.sig")
|
||||
|
||||
req2, valid := expiredAuthenticator.Authenticate(req)
|
||||
|
||||
Expect(valid).To(Equal(false))
|
||||
Expect(req2).To(BeNil())
|
||||
})
|
||||
|
||||
It("should log the authentication failure at debug level", func() {
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/example/path", http.NoBody)
|
||||
req.Header.Set(_headerAuthorization, "Bearer jwt.token.sig")
|
||||
|
||||
expiredAuthenticator.Authenticate(req)
|
||||
|
||||
Expect(buf.String()).To(ContainSubstring("failed to authenticate the request"))
|
||||
Expect(buf.String()).To(ContainSubstring(`"level":"debug"`))
|
||||
Expect(buf.String()).ToNot(ContainSubstring(`"level":"error"`))
|
||||
})
|
||||
})
|
||||
|
||||
When("the request contains a token that cannot be verified", func() {
|
||||
var buf bytes.Buffer
|
||||
failingAuthenticator := &OIDCAuthenticator{
|
||||
OIDCIss: "http://idp.example.com",
|
||||
Logger: log.Logger{zerolog.New(&buf).Level(zerolog.DebugLevel)},
|
||||
oidcClient: &oidcmocks.OIDCClient{},
|
||||
userInfoCache: store.NewMemoryStore(),
|
||||
skipUserInfo: true,
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
failingAuthenticator.oidcClient.(*oidcmocks.OIDCClient).On("VerifyAccessToken", mock.Anything, mock.Anything).Return(
|
||||
oidc.RegClaimsWithSID{}, jwt.MapClaims{}, jwt.ErrTokenMalformed,
|
||||
)
|
||||
})
|
||||
|
||||
It("should log the authentication failure at error level", func() {
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/example/path", http.NoBody)
|
||||
req.Header.Set(_headerAuthorization, "Bearer jwt.token.sig")
|
||||
|
||||
failingAuthenticator.Authenticate(req)
|
||||
|
||||
Expect(buf.String()).To(ContainSubstring("failed to authenticate the request"))
|
||||
Expect(buf.String()).To(ContainSubstring(`"level":"error"`))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -236,16 +236,6 @@ func (m SignedURLAuthenticator) Authenticate(r *http.Request) (*http.Request, bo
|
||||
}
|
||||
|
||||
func (m SignedURLAuthenticator) authenticate(r *http.Request) (*http.Request, bool) {
|
||||
if err := m.requestMethodIsAllowed(r.Method); err != nil {
|
||||
m.Logger.Error().
|
||||
Err(err).
|
||||
Str("authenticator", "signed_url_jwt").
|
||||
Str("path", r.URL.Path).
|
||||
Str("method", r.Method).
|
||||
Msg("Request method not allowed for signed urls")
|
||||
return nil, false
|
||||
}
|
||||
|
||||
u := r.URL.String()
|
||||
if !r.URL.IsAbs() {
|
||||
u = "https://" + r.Host + u
|
||||
|
||||
@@ -2,13 +2,11 @@ package middleware
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/proxy/pkg/config"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/signedurl"
|
||||
@@ -73,27 +71,6 @@ func TestSignedURLAuth_shouldServe(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignedURLAuth_authenticateRejectsDisallowedMethods(t *testing.T) {
|
||||
signer, err := signedurl.NewJWTSignedURL(signedurl.WithSecret("secret"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create signer: %v", err)
|
||||
}
|
||||
signed, err := signer.Sign("https://example.com/dav/spaces/file.txt", "userid", time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to sign url: %v", err)
|
||||
}
|
||||
|
||||
pua := SignedURLAuthenticator{Logger: log.NewLogger(), URLVerifier: signer}
|
||||
pua.PreSignedURLConfig.AllowedHTTPMethods = []string{"GET"}
|
||||
|
||||
for _, method := range []string{http.MethodPost, http.MethodPut, http.MethodDelete, "PROPFIND", "MOVE"} {
|
||||
r := httptest.NewRequest(method, signed, nil)
|
||||
if _, ok := pua.authenticate(r); ok {
|
||||
t.Errorf("expected %s with a signed url to be rejected", method)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignedURLAuth_allRequiredParametersPresent(t *testing.T) {
|
||||
pua := SignedURLAuthenticator{}
|
||||
baseURL := "https://example.com/example.jpg?"
|
||||
|
||||
@@ -2,6 +2,7 @@ package bleve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
@@ -24,45 +25,82 @@ import (
|
||||
|
||||
const defaultBatchSize = 50
|
||||
|
||||
// semanticK picks the number of nearest neighbors for a semantic clause: at
|
||||
// least a fusion-friendly window, at most a sane cap (huge page sizes stand
|
||||
// for "everything", but a similarity ranking beyond 1000 hits carries no
|
||||
// signal, see the KNNRequest semantics: k results per clause, no threshold).
|
||||
func semanticK(size int) int64 {
|
||||
const minK, maxK = 200, 1000
|
||||
switch {
|
||||
case size >= maxK:
|
||||
return maxK
|
||||
case size < minK:
|
||||
return minK
|
||||
default:
|
||||
return int64(size)
|
||||
}
|
||||
}
|
||||
|
||||
var _ search.Engine = (*Backend)(nil) // ensure Backend implements Engine
|
||||
|
||||
type Backend struct {
|
||||
index bleve.Index
|
||||
queryCreator searchQuery.Creator[query.Query]
|
||||
vectorizer search.TextVectorizer
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func NewBackend(index bleve.Index, queryCreator searchQuery.Creator[query.Query], log log.Logger) *Backend {
|
||||
return &Backend{
|
||||
// Option configures a Backend.
|
||||
type Option func(*Backend)
|
||||
|
||||
// WithTextVectorizer enables semantic queries (`semantic:"..."`); without it
|
||||
// they are rejected.
|
||||
func WithTextVectorizer(v search.TextVectorizer) Option {
|
||||
return func(b *Backend) {
|
||||
b.vectorizer = v
|
||||
}
|
||||
}
|
||||
|
||||
func NewBackend(index bleve.Index, queryCreator searchQuery.Creator[query.Query], log log.Logger, opts ...Option) *Backend {
|
||||
b := &Backend{
|
||||
index: index,
|
||||
queryCreator: queryCreator,
|
||||
log: log,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(b)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Search executes a search request operation within the index.
|
||||
// Returns a SearchIndexResponse object or an error.
|
||||
func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
|
||||
createdQuery, err := b.queryCreator.Create(sir.Query)
|
||||
func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
|
||||
// the semantic clause ranks, the remaining query filters: it scopes the
|
||||
// vector search and stays the only source of totals. The creator splits
|
||||
// the clause off the parsed KQL tree.
|
||||
createdQuery, semanticText, err := b.queryCreator.CreateWithSemantic(sir.Query)
|
||||
if err != nil {
|
||||
if kql.IsValidationError(err) {
|
||||
return nil, errtypes.BadRequest(err.Error())
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
pureSemantic := createdQuery == nil && semanticText != ""
|
||||
|
||||
q := bleve.NewConjunctionQuery(
|
||||
// scope holds the restrictions every hit must satisfy; it also pre-filters
|
||||
// the vector search
|
||||
scope := bleve.NewConjunctionQuery(
|
||||
// Skip documents that have been marked as deleted
|
||||
&query.BoolFieldQuery{
|
||||
Bool: false,
|
||||
FieldVal: "Deleted",
|
||||
},
|
||||
createdQuery,
|
||||
)
|
||||
|
||||
if sir.Ref != nil {
|
||||
q.Conjuncts = append(
|
||||
q.Conjuncts,
|
||||
scope.Conjuncts = append(
|
||||
scope.Conjuncts,
|
||||
&query.TermQuery{
|
||||
FieldVal: "RootID",
|
||||
Term: storagespace.FormatResourceID(
|
||||
@@ -79,13 +117,25 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
|
||||
// (paths act as references, /Foo and /foo are distinct), so the exact
|
||||
// folder or the folder prefix matches all of, and only, the scope.
|
||||
if requestedPath := utils.MakeRelativePath(sir.Ref.Path); requestedPath != "." {
|
||||
q.Conjuncts = append(q.Conjuncts, query.NewDisjunctionQuery([]query.Query{
|
||||
scope.Conjuncts = append(scope.Conjuncts, query.NewDisjunctionQuery([]query.Query{
|
||||
&query.TermQuery{FieldVal: "Path", Term: requestedPath},
|
||||
&query.PrefixQuery{FieldVal: "Path", Prefix: requestedPath + "/"},
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
// A purely semantic query has no filter part: the hits come exclusively
|
||||
// from the KNN clause (base match_none), so the reported total is the
|
||||
// number of semantic hits, not a library count (a similarity search has
|
||||
// no result set, only a ranking). With a filter part the hits are the
|
||||
// filter matches, re-ranked by the fusion.
|
||||
var q query.Query
|
||||
if pureSemantic {
|
||||
q = query.NewMatchNoneQuery()
|
||||
} else {
|
||||
q = bleve.NewConjunctionQuery(append([]query.Query{}, append(scope.Conjuncts, createdQuery)...)...)
|
||||
}
|
||||
|
||||
bleveReq := bleve.NewSearchRequest(q)
|
||||
bleveReq.Highlight = bleve.NewHighlight()
|
||||
|
||||
@@ -98,12 +148,43 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
|
||||
bleveReq.Size = int(sir.PageSize)
|
||||
}
|
||||
|
||||
if semanticText != "" {
|
||||
if b.vectorizer == nil {
|
||||
return nil, errtypes.BadRequest("semantic search is not configured")
|
||||
}
|
||||
vector, err := b.vectorizer.VectorizeText(ctx, semanticText)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to vectorize the semantic query: %w", err)
|
||||
}
|
||||
// pre-filter the vector search: scope only for a purely semantic
|
||||
// query, the full filter conjunction otherwise
|
||||
knnFilter := query.Query(scope)
|
||||
if !pureSemantic {
|
||||
knnFilter = q
|
||||
}
|
||||
if err := addSemanticKNN(bleveReq, vector, knnFilter, semanticK(bleveReq.Size), !pureSemantic); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
bleveReq.Fields = []string{"*"}
|
||||
res, err := b.index.Search(bleveReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// hybrid semantic: the fusion reports the fused-hit count as Total, so the
|
||||
// filter-part total (the only meaningful one) needs its own cheap count
|
||||
if semanticText != "" && !pureSemantic {
|
||||
countReq := bleve.NewSearchRequest(q)
|
||||
countReq.Size = 0
|
||||
countRes, err := b.index.Search(countReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Total = countRes.Total
|
||||
}
|
||||
|
||||
matches := make([]*searchMessage.Match, 0, len(res.Hits))
|
||||
totalMatches := res.Total
|
||||
for _, hit := range res.Hits {
|
||||
@@ -141,7 +222,6 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
|
||||
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
|
||||
Video: hitToFacet[searchMessage.Video](hit.Fields, "video"),
|
||||
MotionPhoto: hitToFacet[searchMessage.MotionPhoto](hit.Fields, "motionPhoto"),
|
||||
LivePhoto: hitToFacet[searchMessage.LivePhoto](hit.Fields, "livePhoto"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
//go:build !vectors
|
||||
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
)
|
||||
|
||||
// addSemanticKNN rejects semantic clauses: this build carries no bleve vector
|
||||
// support (see the vectors build tag).
|
||||
func addSemanticKNN(*bleve.SearchRequest, []float32, query.Query, int64, bool) error {
|
||||
return errtypes.BadRequest("semantic search is not supported by this build")
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//go:build vectors
|
||||
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
// addSemanticKNN attaches the semantic clause to the request: a KNN search on
|
||||
// the _faiss sibling, pre-filtered by the regular query. With a lexical part
|
||||
// (hybrid) the two rankings are fused via reciprocal rank fusion; without an
|
||||
// explicit Score bleve would naively sum BM25 and vector scores, whose scales
|
||||
// are not comparable. A purely semantic query keeps the raw similarity
|
||||
// scores: they are comparable across the per-space searches the service layer
|
||||
// merges, while rank-based fusion scores are not.
|
||||
func addSemanticKNN(req *bleve.SearchRequest, vector []float32, filter query.Query, k int64, hybrid bool) error {
|
||||
req.AddKNNWithFilter("imageVector"+mapping.VectorIndexSuffix, vector, k, 1.0, filter)
|
||||
if hybrid {
|
||||
req.Score = "rrf"
|
||||
// the fusion window defaults to From+Size, which would truncate the
|
||||
// ranking depth to the page size
|
||||
window := req.From + req.Size
|
||||
if window < int(k) {
|
||||
window = int(k)
|
||||
}
|
||||
req.Params = &bleve.RequestParams{ScoreWindowSize: window}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -49,6 +49,10 @@ func (b *Batch) indexResource(id string, r search.Resource) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// vectors build: copy vector fields to their indexable _faiss sibling
|
||||
// (the field itself is mapped stored-only, see VectorIndexSuffix);
|
||||
// !vectors build: drop them
|
||||
prepareVectorFields(doc, r.SearchFieldOverrides())
|
||||
return b.batch.Index(id, doc)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
//go:build !vectors
|
||||
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
// prepareVectorFields drops vector fields from the document: this build cannot
|
||||
// search them, so storing them would only cost space. Enabling vectors later
|
||||
// requires a reindex anyway (the faiss part of the index was never populated).
|
||||
func prepareVectorFields(doc map[string]any, overrides map[string]mapping.FieldOpts) {
|
||||
for key, opts := range overrides {
|
||||
if opts.Type == mapping.TypeVector {
|
||||
delete(doc, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//go:build vectors
|
||||
|
||||
package bleve_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
bleveSearch "github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/index/scorch"
|
||||
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// axisVector returns a one-hot vector of the schema dimensionality: distinct
|
||||
// axes are orthogonal (cosine 0), same axes identical (cosine 1).
|
||||
func axisVector(axis int) []float32 {
|
||||
v := make([]float32, content.ImageVectorDims)
|
||||
v[axis] = 1
|
||||
return v
|
||||
}
|
||||
|
||||
// fakeVectorizer maps query texts to fixed vectors.
|
||||
type fakeVectorizer struct{ byText map[string][]float32 }
|
||||
|
||||
func (f fakeVectorizer) VectorizeText(_ context.Context, text string) ([]float32, error) {
|
||||
v, ok := f.byText[strings.ToLower(text)]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no vector for %q", text)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
var _ = Describe("Semantic search (vectors build)", func() {
|
||||
var (
|
||||
eng *bleve.Backend
|
||||
|
||||
doSearch = func(query string) *searchsvc.SearchIndexResponse {
|
||||
res, err := eng.Search(context.Background(), &searchsvc.SearchIndexRequest{
|
||||
Query: query,
|
||||
Ref: &searchmsg.Reference{
|
||||
ResourceId: &searchmsg.ResourceID{StorageId: "1", SpaceId: "2", OpaqueId: "2"},
|
||||
},
|
||||
})
|
||||
ExpectWithOffset(1, err).ToNot(HaveOccurred())
|
||||
return res
|
||||
}
|
||||
|
||||
names = func(res *searchsvc.SearchIndexResponse) []string {
|
||||
out := make([]string, 0, len(res.Matches))
|
||||
for _, m := range res.Matches {
|
||||
out = append(out, m.GetEntity().GetName())
|
||||
}
|
||||
return out
|
||||
}
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
mapping, err := bleve.NewMapping()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "bleve-semantic-test-")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
idx, err := bleveSearch.NewUsing(tmpDir, mapping, scorch.Name, bleveSearch.Config.DefaultKVStore, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// close before removing: the background persister still writes into
|
||||
// the store directory otherwise
|
||||
DeferCleanup(func() error {
|
||||
if err := idx.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.RemoveAll(tmpDir)
|
||||
})
|
||||
|
||||
eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}, bleve.WithTextVectorizer(fakeVectorizer{
|
||||
byText: map[string][]float32{
|
||||
"cat": axisVector(0),
|
||||
"dog": axisVector(1),
|
||||
},
|
||||
}))
|
||||
|
||||
upsert := func(id, path, name string, vector []float32) {
|
||||
r := search.Resource{
|
||||
ID: id,
|
||||
RootID: "1$2!2",
|
||||
ParentID: "1$2!2",
|
||||
Path: path,
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: name, MimeType: "image/jpeg", ImageVector: vector},
|
||||
}
|
||||
Expect(eng.Upsert(id, r)).To(Succeed())
|
||||
}
|
||||
upsert("1$2!10", "./cat.jpg", "cat.jpg", axisVector(0))
|
||||
upsert("1$2!11", "./dog.jpg", "dog.jpg", axisVector(1))
|
||||
Expect(eng.Upsert("1$2!12", search.Resource{
|
||||
ID: "1$2!12", RootID: "1$2!2", ParentID: "1$2!2", Path: "./note.txt",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "note.txt", MimeType: "text/plain"},
|
||||
})).To(Succeed())
|
||||
})
|
||||
|
||||
It("ranks the nearest image first on a purely semantic query", func() {
|
||||
res := doSearch(`semantic:"cat"`)
|
||||
Expect(len(res.Matches)).To(BeNumerically(">=", 1))
|
||||
Expect(res.Matches[0].GetEntity().GetName()).To(Equal("cat.jpg"))
|
||||
})
|
||||
|
||||
It("combines the semantic clause with a filter part", func() {
|
||||
res := doSearch(`Name:dog* AND semantic:"cat"`)
|
||||
Expect(names(res)).To(Equal([]string{"dog.jpg"}))
|
||||
})
|
||||
|
||||
It("keeps the vector across move and restore round-trips", func() {
|
||||
Expect(eng.Move("1$2!10", "1$2!2", "./moved-cat.jpg")).To(Succeed())
|
||||
res := doSearch(`semantic:"cat"`)
|
||||
Expect(res.Matches[0].GetEntity().GetName()).To(Equal("moved-cat.jpg"))
|
||||
|
||||
Expect(eng.Delete("1$2!10")).To(Succeed())
|
||||
res = doSearch(`semantic:"cat"`)
|
||||
for _, n := range names(res) {
|
||||
Expect(n).ToNot(Equal("moved-cat.jpg"))
|
||||
}
|
||||
|
||||
Expect(eng.Restore("1$2!10")).To(Succeed())
|
||||
res = doSearch(`semantic:"cat"`)
|
||||
Expect(res.Matches[0].GetEntity().GetName()).To(Equal("moved-cat.jpg"))
|
||||
})
|
||||
|
||||
It("rejects semantic queries without a vectorizer", func() {
|
||||
bare := bleve.NewBackend(nil, bleveQuery.DefaultCreator, log.Logger{})
|
||||
_, err := bare.Search(context.Background(), &searchsvc.SearchIndexRequest{Query: `semantic:"cat"`})
|
||||
Expect(err).To(MatchError(ContainSubstring("not configured")))
|
||||
})
|
||||
})
|
||||
+6
-69
@@ -702,78 +702,15 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"livePhoto": {
|
||||
"imageVector": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"properties": {
|
||||
"auto": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "boolean",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"contentId": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"analyzer": "keyword",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"stillImageTimeUs": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"vitalityScore": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"vitalityScoringVersion": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"location": {
|
||||
"enabled": true,
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
//go:build vectors
|
||||
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
// prepareVectorFields copies vector fields to their _faiss siblings, which
|
||||
// carry the searchable vector in this build (see mapping.VectorIndexSuffix).
|
||||
func prepareVectorFields(doc map[string]any, overrides map[string]mapping.FieldOpts) {
|
||||
mapping.AddVectorIndexSiblings(doc, overrides)
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// Package clip talks to a CLIP inference service (immich machine-learning API).
|
||||
// Images and texts are embedded into the same vector space, so a text query
|
||||
// vector can rank image vectors by cosine similarity.
|
||||
package clip
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultModel is a multilingual CLIP model: German (and other non-English)
|
||||
// queries match image content without a translation step.
|
||||
const DefaultModel = "XLM-Roberta-Large-Vit-B-32"
|
||||
|
||||
// Client is a minimal client for the immich machine-learning /predict API.
|
||||
type Client struct {
|
||||
url string
|
||||
model string
|
||||
hc *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a Client for the inference service at url. A zero timeout
|
||||
// falls back to 30 seconds, an empty model to DefaultModel.
|
||||
func NewClient(url, model string, timeout time.Duration) *Client {
|
||||
if model == "" {
|
||||
model = DefaultModel
|
||||
}
|
||||
if timeout == 0 {
|
||||
timeout = 30 * time.Second
|
||||
}
|
||||
return &Client{
|
||||
url: url,
|
||||
model: model,
|
||||
hc: &http.Client{Timeout: timeout},
|
||||
}
|
||||
}
|
||||
|
||||
// VectorizeText embeds a text query.
|
||||
func (c *Client) VectorizeText(ctx context.Context, text string) ([]float32, error) {
|
||||
return c.predict(ctx, "textual", func(w *multipart.Writer) error {
|
||||
return w.WriteField("text", text)
|
||||
})
|
||||
}
|
||||
|
||||
// VectorizeImage embeds an image from r.
|
||||
func (c *Client) VectorizeImage(ctx context.Context, r io.Reader) ([]float32, error) {
|
||||
return c.predict(ctx, "visual", func(w *multipart.Writer) error {
|
||||
fw, err := w.CreateFormFile("image", "image")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(fw, r)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// predict posts a multipart request to /predict. The entries field declares
|
||||
// which model runs; the response carries the embedding as a JSON-encoded array
|
||||
// inside a JSON string field.
|
||||
func (c *Client) predict(ctx context.Context, task string, addPayload func(*multipart.Writer) error) ([]float32, error) {
|
||||
var body bytes.Buffer
|
||||
w := multipart.NewWriter(&body)
|
||||
|
||||
entries, err := json.Marshal(map[string]any{
|
||||
"clip": map[string]any{task: map[string]any{"modelName": c.model}},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := w.WriteField("entries", string(entries)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := addPayload(w); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url+"/predict", &body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", w.FormDataContentType())
|
||||
|
||||
res, err := c.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
msg, _ := io.ReadAll(io.LimitReader(res.Body, 512))
|
||||
return nil, fmt.Errorf("clip inference failed: %s: %s", res.Status, msg)
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Clip string `json:"clip"`
|
||||
}
|
||||
if err := json.NewDecoder(res.Body).Decode(&payload); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode clip inference response: %w", err)
|
||||
}
|
||||
var vector []float32
|
||||
if err := json.Unmarshal([]byte(payload.Clip), &vector); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode clip embedding: %w", err)
|
||||
}
|
||||
if len(vector) == 0 {
|
||||
return nil, fmt.Errorf("clip inference returned an empty embedding")
|
||||
}
|
||||
return vector, nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package clip
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
// QueryCache caches text embeddings behind an LRU: users type the same terms
|
||||
// repeatedly, and every miss is a network round-trip to the inference service.
|
||||
// A singleflight collapses the concurrent misses the per-space search fan-out
|
||||
// produces for one and the same query.
|
||||
type QueryCache struct {
|
||||
next interface {
|
||||
VectorizeText(ctx context.Context, text string) ([]float32, error)
|
||||
}
|
||||
group singleflight.Group
|
||||
|
||||
mu sync.Mutex
|
||||
maxSize int
|
||||
entries map[string]*list.Element
|
||||
order *list.List // front = most recently used
|
||||
}
|
||||
|
||||
type cacheEntry struct {
|
||||
key string
|
||||
vector []float32
|
||||
}
|
||||
|
||||
// NewQueryCache wraps a text vectorizer with an LRU of the given size.
|
||||
func NewQueryCache(next *Client, size int) *QueryCache {
|
||||
if size <= 0 {
|
||||
size = 512
|
||||
}
|
||||
return &QueryCache{
|
||||
next: next,
|
||||
maxSize: size,
|
||||
entries: map[string]*list.Element{},
|
||||
order: list.New(),
|
||||
}
|
||||
}
|
||||
|
||||
// VectorizeText returns the cached embedding for text or fetches and caches it.
|
||||
func (c *QueryCache) VectorizeText(ctx context.Context, text string) ([]float32, error) {
|
||||
key := strings.Join(strings.Fields(strings.ToLower(text)), " ")
|
||||
|
||||
c.mu.Lock()
|
||||
if el, ok := c.entries[key]; ok {
|
||||
c.order.MoveToFront(el)
|
||||
vector := el.Value.(*cacheEntry).vector
|
||||
c.mu.Unlock()
|
||||
return vector, nil
|
||||
}
|
||||
c.mu.Unlock()
|
||||
|
||||
v, err, _ := c.group.Do(key, func() (any, error) {
|
||||
vector, err := c.next.VectorizeText(ctx, text)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.entries[key]; ok {
|
||||
c.order.MoveToFront(el)
|
||||
return el.Value.(*cacheEntry).vector, nil
|
||||
}
|
||||
c.entries[key] = c.order.PushFront(&cacheEntry{key: key, vector: vector})
|
||||
if c.order.Len() > c.maxSize {
|
||||
oldest := c.order.Back()
|
||||
c.order.Remove(oldest)
|
||||
delete(c.entries, oldest.Value.(*cacheEntry).key)
|
||||
}
|
||||
return vector, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return v.([]float32), nil
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/pkg/tracing"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/clip"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
@@ -63,6 +64,15 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
mtrcs := metrics.New()
|
||||
mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1)
|
||||
|
||||
// one clip client serves both sides: the extractor embeds images at
|
||||
// index time, the engines embed query texts at search time
|
||||
var clipClient *clip.Client
|
||||
var vectorizer search.TextVectorizer
|
||||
if cfg.Extractor.Clip.URL != "" {
|
||||
clipClient = clip.NewClient(cfg.Extractor.Clip.URL, cfg.Extractor.Clip.Model, time.Duration(cfg.Extractor.Clip.Timeout)*time.Second)
|
||||
vectorizer = clip.NewQueryCache(clipClient, 512)
|
||||
}
|
||||
|
||||
// initialize search engine
|
||||
var eng search.Engine
|
||||
switch cfg.Engine.Type {
|
||||
@@ -78,16 +88,25 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
}
|
||||
}()
|
||||
|
||||
eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, logger)
|
||||
var opts []bleve.Option
|
||||
if vectorizer != nil {
|
||||
opts = append(opts, bleve.WithTextVectorizer(vectorizer))
|
||||
}
|
||||
eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, logger, opts...)
|
||||
case "open-search":
|
||||
client, err := opensearch.NewClient(cfg.Engine.OpenSearch.Client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var opts []opensearch.Option
|
||||
if vectorizer != nil {
|
||||
opts = append(opts, opensearch.WithTextVectorizer(vectorizer))
|
||||
}
|
||||
|
||||
// a hung cluster must fail the start, not block it forever
|
||||
startupCtx, cancelStartup := context.WithTimeout(ctx, time.Minute)
|
||||
openSearchBackend, err := opensearch.NewBackend(startupCtx, cfg.Engine.OpenSearch.ResourceIndex.Name, client, logger)
|
||||
openSearchBackend, err := opensearch.NewBackend(startupCtx, cfg.Engine.OpenSearch.ResourceIndex.Name, client, logger, opts...)
|
||||
cancelStartup()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create OpenSearch backend: %w", err)
|
||||
@@ -120,6 +139,14 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
return fmt.Errorf("unknown search extractor: %s", cfg.Extractor.Type)
|
||||
}
|
||||
|
||||
// vectorization is orthogonal to text extraction: decorate the
|
||||
// chosen extractor instead of adding an exclusive type
|
||||
if clipClient != nil {
|
||||
if extractor, err = content.NewClipExtractor(extractor, clipClient, selector, logger, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
ss := search.NewService(selector, eng, extractor, mtrcs, logger, cfg)
|
||||
|
||||
// setup the servers
|
||||
|
||||
@@ -5,6 +5,7 @@ type Extractor struct {
|
||||
Type string `yaml:"type" env:"SEARCH_EXTRACTOR_TYPE" desc:"Defines the content extraction engine. Defaults to 'basic'. Supported values are: 'basic' and 'tika'." introductionVersion:"1.0.0"`
|
||||
CS3AllowInsecure bool `yaml:"cs3_allow_insecure" env:"OC_INSECURE;SEARCH_EXTRACTOR_CS3SOURCE_INSECURE" desc:"Ignore untrusted SSL certificates when connecting to the CS3 source." introductionVersion:"1.0.0"`
|
||||
Tika ExtractorTika `yaml:"tika"`
|
||||
Clip ExtractorClip `yaml:"clip"`
|
||||
}
|
||||
|
||||
// ExtractorTika configures the Tika extractor
|
||||
@@ -12,3 +13,15 @@ type ExtractorTika struct {
|
||||
TikaURL string `yaml:"tika_url" env:"SEARCH_EXTRACTOR_TIKA_TIKA_URL" desc:"URL of the tika server." introductionVersion:"1.0.0"`
|
||||
CleanStopWords bool `yaml:"clean_stop_words" env:"SEARCH_EXTRACTOR_TIKA_CLEAN_STOP_WORDS" desc:"Defines if stop words should be cleaned or not. See the documentation for more details." introductionVersion:"1.0.0"`
|
||||
}
|
||||
|
||||
// ExtractorClip configures the CLIP inference service used for semantic image
|
||||
// search. It decorates the configured extractor, so it combines with 'basic'
|
||||
// and 'tika'. The vector dimensionality is part of the index schema and
|
||||
// therefore not configurable; the service verifies at startup that the
|
||||
// configured model matches.
|
||||
type ExtractorClip struct {
|
||||
URL string `yaml:"url" env:"SEARCH_EXTRACTOR_CLIP_URL" desc:"URL of the CLIP inference service (immich machine-learning API). When set, image files are embedded during indexing to enable semantic search." introductionVersion:"7.5.0"`
|
||||
Model string `yaml:"model" env:"SEARCH_EXTRACTOR_CLIP_MODEL" desc:"Name of the CLIP model to use. Must be a multilingual model producing 512-dimensional embeddings. Changing the model requires a full reindex." introductionVersion:"7.5.0"`
|
||||
MaxBytes uint64 `yaml:"max_bytes" env:"SEARCH_EXTRACTOR_CLIP_MAX_BYTES" desc:"Maximum image size in bytes to send to the inference service. Larger images are skipped." introductionVersion:"7.5.0"`
|
||||
Timeout int `yaml:"timeout" env:"SEARCH_EXTRACTOR_CLIP_TIMEOUT" desc:"Timeout in seconds for requests to the inference service." introductionVersion:"7.5.0"`
|
||||
}
|
||||
@@ -52,6 +52,11 @@ func DefaultConfig() *config.Config {
|
||||
TikaURL: "http://127.0.0.1:9998",
|
||||
CleanStopWords: false,
|
||||
},
|
||||
Clip: config.ExtractorClip{
|
||||
Model: "XLM-Roberta-Large-Vit-B-32",
|
||||
MaxBytes: 50 * 1024 * 1024,
|
||||
Timeout: 30,
|
||||
},
|
||||
},
|
||||
Events: config.Events{
|
||||
Endpoint: "127.0.0.1:9233",
|
||||
|
||||
@@ -34,10 +34,6 @@ func ParseConfig(cfg *config.Config) error {
|
||||
}
|
||||
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.Events.Disabled && cfg.GRPC.Disabled {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
}
|
||||
|
||||
if cfg.TokenManager.JWTSecret == "" {
|
||||
return shared.MissingJWTTokenError(cfg.Service.Name)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package content
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/clip"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/config"
|
||||
)
|
||||
|
||||
// Clip decorates another extractor with image vectorization: it delegates
|
||||
// Extract and adds an ImageVector to image documents. Vectorization is
|
||||
// orthogonal to text extraction, so it wraps 'basic' as well as 'tika'.
|
||||
type Clip struct {
|
||||
next Extractor
|
||||
Retriever
|
||||
client *clip.Client
|
||||
logger log.Logger
|
||||
maxBytes uint64
|
||||
}
|
||||
|
||||
// NewClipExtractor wraps next with a CLIP vectorization step. It probes the
|
||||
// inference service once to verify that the configured model produces vectors
|
||||
// of the schema dimensionality (ImageVectorDims) and refuses to start
|
||||
// otherwise: mismatched vectors would be dropped silently by the index.
|
||||
func NewClipExtractor(next Extractor, client *clip.Client, gatewaySelector pool.Selectable[gateway.GatewayAPIClient], logger log.Logger, cfg *config.Config) (*Clip, error) {
|
||||
probe, err := client.VectorizeText(context.Background(), "startup probe")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("clip inference service unavailable: %w", err)
|
||||
}
|
||||
if len(probe) != ImageVectorDims {
|
||||
return nil, fmt.Errorf("clip model produces %d-dimensional vectors, the index schema requires %d; use a matching model", len(probe), ImageVectorDims)
|
||||
}
|
||||
logger.Info().Str("url", cfg.Extractor.Clip.URL).Int("dims", len(probe)).Msg("clip inference service connected")
|
||||
|
||||
maxBytes := cfg.Extractor.Clip.MaxBytes
|
||||
if maxBytes == 0 {
|
||||
maxBytes = 50 * 1024 * 1024
|
||||
}
|
||||
|
||||
return &Clip{
|
||||
next: next,
|
||||
Retriever: newCS3Retriever(gatewaySelector, logger, cfg.Extractor.CS3AllowInsecure),
|
||||
client: client,
|
||||
logger: logger,
|
||||
maxBytes: maxBytes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Extract delegates to the wrapped extractor and adds the image embedding.
|
||||
// Vectorization failures are not fatal: the document is indexed without a
|
||||
// vector and the rest of the search keeps working.
|
||||
func (c *Clip) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, error) {
|
||||
doc, err := c.next.Extract(ctx, ri)
|
||||
if err != nil {
|
||||
return doc, err
|
||||
}
|
||||
|
||||
if ri.Type != provider.ResourceType_RESOURCE_TYPE_FILE ||
|
||||
!strings.HasPrefix(ri.GetMimeType(), "image/") ||
|
||||
ri.Size == 0 || ri.Size > c.maxBytes {
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
data, err := c.Retrieve(ctx, ri.Id)
|
||||
if err != nil {
|
||||
c.logger.Warn().Err(err).Interface("ResourceID", ri.Id).Str("Name", ri.Name).Msg("clip: failed to retrieve image, indexing without vector")
|
||||
return doc, nil
|
||||
}
|
||||
defer data.Close()
|
||||
|
||||
vector, err := c.client.VectorizeImage(ctx, data)
|
||||
if err != nil {
|
||||
c.logger.Warn().Err(err).Interface("ResourceID", ri.Id).Str("Name", ri.Name).Msg("clip: vectorization failed, indexing without vector")
|
||||
return doc, nil
|
||||
}
|
||||
if len(vector) != ImageVectorDims {
|
||||
// the index would drop a mismatched vector silently, make it visible
|
||||
c.logger.Warn().Int("dims", len(vector)).Int("want", ImageVectorDims).Interface("ResourceID", ri.Id).Str("Name", ri.Name).Msg("clip: vector dimensionality mismatch, indexing without vector")
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
doc.ImageVector = vector
|
||||
return doc, nil
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package content_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/stretchr/testify/mock"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/clip"
|
||||
conf "github.com/opencloud-eu/opencloud/services/search/pkg/config/defaults"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
contentMocks "github.com/opencloud-eu/opencloud/services/search/pkg/content/mocks"
|
||||
)
|
||||
|
||||
// clipResponse renders the immich-ml /predict response for a vector of n dims.
|
||||
func clipResponse(n int) string {
|
||||
vec := make([]float32, n)
|
||||
for i := range vec {
|
||||
vec[i] = float32(i) / float32(n)
|
||||
}
|
||||
inner, _ := json.Marshal(vec)
|
||||
outer, _ := json.Marshal(map[string]string{"clip": string(inner)})
|
||||
return string(outer)
|
||||
}
|
||||
|
||||
var _ = Describe("Clip", func() {
|
||||
var (
|
||||
srv *httptest.Server
|
||||
dims int
|
||||
failNext atomic.Bool
|
||||
imageCalls atomic.Int32
|
||||
inner *contentMocks.Extractor
|
||||
retriever *contentMocks.Retriever
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
dims = content.ImageVectorDims
|
||||
failNext.Store(false)
|
||||
imageCalls.Store(0)
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
if req.URL.Path != "/predict" {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if failNext.Load() {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
Expect(req.ParseMultipartForm(1 << 20)).To(Succeed())
|
||||
if _, _, err := req.FormFile("image"); err == nil {
|
||||
imageCalls.Add(1)
|
||||
}
|
||||
_, _ = w.Write([]byte(clipResponse(dims)))
|
||||
}))
|
||||
|
||||
inner = &contentMocks.Extractor{}
|
||||
retriever = &contentMocks.Retriever{}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
srv.Close()
|
||||
})
|
||||
|
||||
newClip := func() (*content.Clip, error) {
|
||||
cfg := conf.DefaultConfig()
|
||||
cfg.Extractor.Clip.URL = srv.URL
|
||||
|
||||
client := clip.NewClient(srv.URL, cfg.Extractor.Clip.Model, 0)
|
||||
extractor, err := content.NewClipExtractor(inner, client, nil, log.NewLogger(), cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
extractor.Retriever = retriever
|
||||
return extractor, nil
|
||||
}
|
||||
|
||||
imageResource := func(size uint64) *provider.ResourceInfo {
|
||||
return &provider.ResourceInfo{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Name: "photo.jpg",
|
||||
MimeType: "image/jpeg",
|
||||
Size: size,
|
||||
}
|
||||
}
|
||||
|
||||
Describe("NewClipExtractor", func() {
|
||||
It("fails when the inference service is unreachable", func() {
|
||||
srv.Close()
|
||||
_, err := newClip()
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
It("fails when the model dimensionality does not match the schema", func() {
|
||||
dims = 768
|
||||
_, err := newClip()
|
||||
Expect(err).To(MatchError(ContainSubstring("768")))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Extract", func() {
|
||||
var extractor *content.Clip
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
extractor, err = newClip()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
inner.On("Extract", mock.Anything, mock.Anything).Return(content.Document{Name: "photo.jpg"}, nil)
|
||||
retriever.On("Retrieve", mock.Anything, mock.Anything).Return(io.NopCloser(strings.NewReader("fakeimagebytes")), nil)
|
||||
})
|
||||
|
||||
It("adds a vector to image documents", func() {
|
||||
doc, err := extractor.Extract(context.TODO(), imageResource(1024))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(doc.Name).To(Equal("photo.jpg"))
|
||||
Expect(doc.ImageVector).To(HaveLen(content.ImageVectorDims))
|
||||
Expect(imageCalls.Load()).To(Equal(int32(1)))
|
||||
})
|
||||
|
||||
It("skips non-image files", func() {
|
||||
doc, err := extractor.Extract(context.TODO(), &provider.ResourceInfo{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
Name: "doc.pdf",
|
||||
MimeType: "application/pdf",
|
||||
Size: 1024,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(doc.ImageVector).To(BeNil())
|
||||
Expect(imageCalls.Load()).To(Equal(int32(0)))
|
||||
})
|
||||
|
||||
It("skips directories", func() {
|
||||
doc, err := extractor.Extract(context.TODO(), &provider.ResourceInfo{
|
||||
Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER,
|
||||
MimeType: "image/jpeg",
|
||||
Size: 1024,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(doc.ImageVector).To(BeNil())
|
||||
})
|
||||
|
||||
It("skips images above the size limit", func() {
|
||||
doc, err := extractor.Extract(context.TODO(), imageResource(51*1024*1024))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(doc.ImageVector).To(BeNil())
|
||||
Expect(imageCalls.Load()).To(Equal(int32(0)))
|
||||
})
|
||||
|
||||
It("indexes without a vector when inference fails", func() {
|
||||
failNext.Store(true)
|
||||
doc, err := extractor.Extract(context.TODO(), imageResource(1024))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(doc.Name).To(Equal("photo.jpg"))
|
||||
Expect(doc.ImageVector).To(BeNil())
|
||||
})
|
||||
|
||||
It("indexes without a vector when retrieval fails", func() {
|
||||
failingRetriever := &contentMocks.Retriever{}
|
||||
failingRetriever.On("Retrieve", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("boom"))
|
||||
extractor.Retriever = failingRetriever
|
||||
|
||||
doc, err := extractor.Extract(context.TODO(), imageResource(1024))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(doc.ImageVector).To(BeNil())
|
||||
})
|
||||
|
||||
It("propagates inner extractor errors", func() {
|
||||
failingInner := &contentMocks.Extractor{}
|
||||
failingInner.On("Extract", mock.Anything, mock.Anything).Return(content.Document{}, fmt.Errorf("inner boom"))
|
||||
cfg := conf.DefaultConfig()
|
||||
cfg.Extractor.Clip.URL = srv.URL
|
||||
client := clip.NewClient(srv.URL, cfg.Extractor.Clip.Model, 0)
|
||||
failing, err := content.NewClipExtractor(failingInner, client, nil, log.NewLogger(), cfg)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
failing.Retriever = retriever
|
||||
|
||||
_, err = failing.Extract(context.TODO(), imageResource(1024))
|
||||
Expect(err).To(MatchError(ContainSubstring("inner boom")))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -12,6 +12,13 @@ func init() {
|
||||
stopwords.OverwriteWordSegmenter(`[^ ]+`)
|
||||
}
|
||||
|
||||
// ImageVectorDims is the dimensionality of the ImageVector field. It is part
|
||||
// of the index schema (baked into the index mapping on creation), not a
|
||||
// configuration value: vectors of any other length cannot be indexed, and a
|
||||
// model of another size requires a new index. The CLIP extractor verifies at
|
||||
// startup that the configured model produces vectors of this length.
|
||||
const ImageVectorDims = 512
|
||||
|
||||
// Document wraps all resource meta fields,
|
||||
// it is used as a content extraction result.
|
||||
type Document struct {
|
||||
@@ -29,7 +36,7 @@ type Document struct {
|
||||
Photo *libregraph.Photo `json:"photo,omitempty"`
|
||||
Video *libregraph.Video `json:"video,omitempty"`
|
||||
MotionPhoto *libregraph.MotionPhoto `json:"motionPhoto,omitempty"`
|
||||
LivePhoto *libregraph.LivePhoto `json:"livePhoto,omitempty"`
|
||||
ImageVector []float32 `json:"imageVector,omitempty"`
|
||||
}
|
||||
|
||||
func CleanString(content, langCode string) string {
|
||||
|
||||
@@ -82,9 +82,6 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
|
||||
if err != nil {
|
||||
return doc, err
|
||||
}
|
||||
if len(metas) == 0 {
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
for _, meta := range metas {
|
||||
title, err := getFirstValue(meta, "dc:title")
|
||||
@@ -101,21 +98,37 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
|
||||
} else if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil {
|
||||
doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content))
|
||||
}
|
||||
|
||||
// keep facets from earlier entries, an embedded resource's meta
|
||||
// (e.g. cover art) must not reset them
|
||||
if v := t.getLocation(meta); v != nil {
|
||||
doc.Location = v
|
||||
}
|
||||
if v := t.getImage(meta); v != nil {
|
||||
doc.Image = v
|
||||
}
|
||||
if v := t.getPhoto(meta); v != nil {
|
||||
doc.Photo = v
|
||||
}
|
||||
if v := t.getAudio(meta); v != nil {
|
||||
doc.Audio = v
|
||||
}
|
||||
}
|
||||
|
||||
// facets describe the file itself, not its embedded parts (cover art, clips)
|
||||
m0 := metas[0]
|
||||
doc.Location = t.getLocation(m0)
|
||||
doc.Image = t.getImage(m0)
|
||||
doc.Photo = t.getPhoto(m0)
|
||||
doc.Audio = t.getAudio(m0)
|
||||
doc.LivePhoto = t.getLivePhoto(m0)
|
||||
doc.Video = t.getVideo(m0)
|
||||
if len(metas) > 0 {
|
||||
// the video facet says the file is a video, so it comes from the file
|
||||
// itself: the clip tika extracts from a motion photo must not make its
|
||||
// image look like one
|
||||
doc.Video = t.getVideo(metas[0])
|
||||
}
|
||||
|
||||
// a motion photo is the file's own xmp plus the video tika extracted from
|
||||
// it; the xmp alone proves nothing, a share can strip the appended clip
|
||||
if i := slices.IndexFunc(metas[1:], isVideo); i >= 0 {
|
||||
doc.MotionPhoto = t.getMotionPhoto(m0, metas[i+1])
|
||||
// a motion photo is the xmp on the file itself plus the video tika extracted
|
||||
// from it. The xmp alone proves nothing: a share can keep it and strip the
|
||||
// appended video.
|
||||
if len(metas) > 0 {
|
||||
if i := slices.IndexFunc(metas[1:], isVideo); i >= 0 {
|
||||
doc.MotionPhoto = t.getMotionPhoto(metas[0], metas[i+1])
|
||||
}
|
||||
}
|
||||
|
||||
if langCode := t.detectLanguage(ctx, doc.Content); langCode != "" && t.CleanStopWords {
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
package content
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strconv"
|
||||
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
)
|
||||
|
||||
func (t Tika) getLivePhoto(meta map[string][]string) *libregraph.LivePhoto {
|
||||
// ContentId pairs the two halves, without it there is no live photo. The
|
||||
// video carries it in the QuickTime item list, the still image in the Apple
|
||||
// maker note, which tika surfaces as "Content Identifier". A file is only
|
||||
// ever one half, so both keys are read.
|
||||
contentID, err := getFirstValue(meta, "com.apple.quicktime.content.identifier", "Content Identifier")
|
||||
if err != nil || contentID == "" {
|
||||
return nil
|
||||
}
|
||||
livePhoto := libregraph.NewLivePhoto(contentID)
|
||||
|
||||
// tika emits still-image-time already in microseconds
|
||||
if v, err := getFirstValue(meta, "quicktime:still-image-time"); err == nil {
|
||||
if f, err := strconv.ParseFloat(v, 64); err == nil {
|
||||
livePhoto.SetStillImageTimeUs(int64(math.Round(f)))
|
||||
}
|
||||
}
|
||||
|
||||
if v, err := getFirstValue(meta, "com.apple.quicktime.live-photo.auto"); err == nil {
|
||||
if b, err := strconv.ParseBool(v); err == nil {
|
||||
livePhoto.SetAuto(b)
|
||||
}
|
||||
}
|
||||
|
||||
if v, err := getFirstValue(meta, "com.apple.quicktime.live-photo.vitality-score"); err == nil {
|
||||
if f, err := strconv.ParseFloat(v, 64); err == nil {
|
||||
livePhoto.SetVitalityScore(f)
|
||||
}
|
||||
}
|
||||
|
||||
if v, err := getFirstValue(meta, "com.apple.quicktime.live-photo.vitality-scoring-version"); err == nil {
|
||||
if i, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
livePhoto.SetVitalityScoringVersion(i)
|
||||
}
|
||||
}
|
||||
|
||||
return livePhoto
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
package content
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
)
|
||||
|
||||
var _ = Describe("getLivePhoto", func() {
|
||||
It("maps the video half of a live photo", func() {
|
||||
livePhoto := Tika{}.getLivePhoto(map[string][]string{
|
||||
"Content-Type": {"video/quicktime"},
|
||||
"com.apple.quicktime.content.identifier": {"6F1A2B3C-1234-4E5F-9A8B-0011223344CC"},
|
||||
"quicktime:still-image-time": {"1500000"},
|
||||
"com.apple.quicktime.live-photo.auto": {"1"},
|
||||
"com.apple.quicktime.live-photo.vitality-score": {"0.75"},
|
||||
"com.apple.quicktime.live-photo.vitality-scoring-version": {"4"},
|
||||
})
|
||||
Expect(livePhoto).ToNot(BeNil())
|
||||
|
||||
Expect(livePhoto.ContentId).To(Equal("6F1A2B3C-1234-4E5F-9A8B-0011223344CC"))
|
||||
Expect(livePhoto.StillImageTimeUs).To(Equal(libregraph.PtrInt64(1500000)))
|
||||
Expect(livePhoto.Auto).To(Equal(libregraph.PtrBool(true)))
|
||||
Expect(livePhoto.VitalityScore).To(Equal(libregraph.PtrFloat64(0.75)))
|
||||
Expect(livePhoto.VitalityScoringVersion).To(Equal(libregraph.PtrInt64(4)))
|
||||
})
|
||||
|
||||
It("maps the still half via the Apple maker-note content identifier", func() {
|
||||
livePhoto := Tika{}.getLivePhoto(map[string][]string{
|
||||
"Content-Type": {"image/heic"},
|
||||
"Content Identifier": {"6F1A2B3C-1234-4E5F-9A8B-0011223344CC"},
|
||||
})
|
||||
Expect(livePhoto).ToNot(BeNil())
|
||||
Expect(livePhoto.ContentId).To(Equal("6F1A2B3C-1234-4E5F-9A8B-0011223344CC"))
|
||||
})
|
||||
|
||||
It("returns nil without a content identifier", func() {
|
||||
Expect(Tika{}.getLivePhoto(map[string][]string{
|
||||
"Content-Type": {"image/jpeg"},
|
||||
})).To(BeNil())
|
||||
Expect(Tika{}.getLivePhoto(map[string][]string{
|
||||
"Content-Type": {"image/heic"},
|
||||
"Content Identifier": {""},
|
||||
})).To(BeNil(), "an empty pairing id is no live photo")
|
||||
})
|
||||
|
||||
It("rounds a fractional still-image-time", func() {
|
||||
livePhoto := Tika{}.getLivePhoto(map[string][]string{
|
||||
"com.apple.quicktime.content.identifier": {"6F1A2B3C"},
|
||||
"quicktime:still-image-time": {"1500000.7"},
|
||||
})
|
||||
Expect(livePhoto).ToNot(BeNil())
|
||||
Expect(livePhoto.StillImageTimeUs).To(Equal(libregraph.PtrInt64(1500001)))
|
||||
})
|
||||
|
||||
It("keeps the facet when optional values are malformed", func() {
|
||||
livePhoto := Tika{}.getLivePhoto(map[string][]string{
|
||||
"com.apple.quicktime.content.identifier": {"6F1A2B3C"},
|
||||
"com.apple.quicktime.live-photo.auto": {"maybe"},
|
||||
"com.apple.quicktime.live-photo.vitality-score": {"very"},
|
||||
})
|
||||
Expect(livePhoto).ToNot(BeNil())
|
||||
Expect(livePhoto.Auto).To(BeNil())
|
||||
Expect(livePhoto.VitalityScore).To(BeNil())
|
||||
})
|
||||
|
||||
It("leaves the video-only fields empty on the still half", func() {
|
||||
livePhoto := Tika{}.getLivePhoto(map[string][]string{
|
||||
"Content Identifier": {"6F1A2B3C"},
|
||||
})
|
||||
Expect(livePhoto).ToNot(BeNil())
|
||||
Expect(livePhoto.StillImageTimeUs).To(BeNil())
|
||||
Expect(livePhoto.Auto).To(BeNil())
|
||||
Expect(livePhoto.VitalityScore).To(BeNil())
|
||||
Expect(livePhoto.VitalityScoringVersion).To(BeNil())
|
||||
})
|
||||
})
|
||||
@@ -170,9 +170,7 @@ var _ = Describe("Tika", func() {
|
||||
Expect(doc.Content).To(Equal("body test stop words!!!"))
|
||||
})
|
||||
|
||||
It("takes facets from the main document, not from an embedded resource", func() {
|
||||
// metas[0] is the file (audio), metas[1] its embedded cover art. The
|
||||
// cover must not give the track an image facet.
|
||||
It("keeps the audio facet when an embedded resource follows", func() {
|
||||
fullResponse = `[{"Content-Type": "audio/mpeg", "dc:title": "Sucker", "tk:content": "lyrics"}, {"Content-Type": "image/jpeg", "tiff:ImageWidth": "500"}]`
|
||||
|
||||
doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
|
||||
@@ -182,7 +180,7 @@ var _ = Describe("Tika", func() {
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(doc.Audio).ToNot(BeNil())
|
||||
Expect(doc.Audio.Title).To(Equal(libregraph.PtrString("Sucker")))
|
||||
Expect(doc.Image).To(BeNil())
|
||||
Expect(doc.Image).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("adds no audio facet to non-audio documents", func() {
|
||||
|
||||
@@ -74,6 +74,29 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
|
||||
return nil
|
||||
}
|
||||
|
||||
if fieldType == TypeVector {
|
||||
// The field itself is stored-only: it round-trips through the
|
||||
// regular deserialization (bleve cannot return vector-typed fields
|
||||
// as stored fields, see VectorIndexSuffix). Both builds.
|
||||
nf := bleve.NewNumericFieldMapping()
|
||||
nf.Index = false
|
||||
nf.Store = true
|
||||
nf.IncludeInAll = false
|
||||
nf.DocValues = false
|
||||
doc.AddFieldMappingsAt(fi.Name, nf)
|
||||
// The _faiss sibling carries the searchable vector; without the
|
||||
// vectors build tag bleve cannot index it and the field is left
|
||||
// out of the mapping entirely (nil, nil from the !vectors twin).
|
||||
fm, err := bleveVectorFieldMapping(opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mapping: field %q: %w", key, err)
|
||||
}
|
||||
if fm != nil {
|
||||
doc.AddFieldMappingsAt(fi.Name+VectorIndexSuffix, fm)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
fm, err := bleveFieldMapping(fieldType, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mapping: field %q: %w", key, err)
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
//go:build !vectors
|
||||
|
||||
package mapping
|
||||
|
||||
import (
|
||||
bleveMapping "github.com/blevesearch/bleve/v2/mapping"
|
||||
)
|
||||
|
||||
// bleveVectorFieldMapping is the !vectors twin: bleve has no vector support in
|
||||
// this build, so TypeVector fields are left out of the mapping (nil, nil means
|
||||
// "skip the field").
|
||||
func bleveVectorFieldMapping(FieldOpts) (*bleveMapping.FieldMapping, error) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
//go:build vectors
|
||||
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
bleveMapping "github.com/blevesearch/bleve/v2/mapping"
|
||||
)
|
||||
|
||||
// bleveVectorFieldMapping builds the bleve field mapping for a TypeVector
|
||||
// field. Only compiled in with the vectors build tag (bleve gates its KNN
|
||||
// support behind it); the !vectors twin drops the field from the mapping.
|
||||
func bleveVectorFieldMapping(opts FieldOpts) (*bleveMapping.FieldMapping, error) {
|
||||
if opts.Dims <= 0 {
|
||||
return nil, fmt.Errorf("vector field needs Dims")
|
||||
}
|
||||
fm := bleve.NewVectorFieldMapping()
|
||||
fm.Dims = opts.Dims
|
||||
// cosine: bleve normalizes the vectors at index time itself
|
||||
fm.Similarity = "cosine"
|
||||
fm.VectorIndexOptimizedFor = "recall"
|
||||
return fm, nil
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
|
||||
return nil
|
||||
}
|
||||
|
||||
fm, err := openSearchFieldMapping(fieldType, fi.GoField.Type)
|
||||
fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mapping: field %q: %w", key, err)
|
||||
}
|
||||
@@ -91,7 +91,7 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
|
||||
|
||||
// openSearchFieldMapping handles the non-keyword/path types; keyword and path
|
||||
// are emitted (with their cased forms) by buildOpenSearchProperties directly.
|
||||
func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]any, error) {
|
||||
func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Type) (map[string]any, error) {
|
||||
switch fieldType {
|
||||
case TypeFulltext:
|
||||
return map[string]any{
|
||||
@@ -112,6 +112,23 @@ func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]a
|
||||
return map[string]any{"type": "date"}, nil
|
||||
case TypeGeopoint:
|
||||
return map[string]any{"type": "geo_point"}, nil
|
||||
case TypeVector:
|
||||
if opts.Dims <= 0 {
|
||||
return nil, fmt.Errorf("vector field needs Dims")
|
||||
}
|
||||
// lucene engine: native efficient pre-filtering, no plugin config
|
||||
return map[string]any{
|
||||
"type": "knn_vector",
|
||||
"dimension": opts.Dims,
|
||||
"method": map[string]any{
|
||||
"name": "hnsw",
|
||||
"engine": "lucene",
|
||||
"space_type": "cosinesimil",
|
||||
// OpenSearch adds this on read, emit it so the reconcile
|
||||
// comparison sees local and remote as equal
|
||||
"parameters": map[string]any{},
|
||||
},
|
||||
}, nil
|
||||
case "":
|
||||
return nil, fmt.Errorf("no type inferred and no override")
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ const (
|
||||
TypeBool = "bool"
|
||||
TypeObject = "object"
|
||||
TypeGeopoint = "geopoint"
|
||||
TypeVector = "vector"
|
||||
)
|
||||
|
||||
// LowercaseSuffix names the lowercased sibling of a keyword/path field.
|
||||
@@ -51,6 +52,11 @@ type FieldOpts struct {
|
||||
// IncludeInAll controls bleve's _all field inclusion. Nil means "use the
|
||||
// bleve default for this field type". Has no effect on OpenSearch.
|
||||
IncludeInAll *bool
|
||||
|
||||
// Dims is the dimensionality of a TypeVector field. It is index schema:
|
||||
// vectors of any other length are not indexed (bleve drops them silently),
|
||||
// and changing it requires a new index. Similarity is always cosine.
|
||||
Dims int
|
||||
}
|
||||
|
||||
func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive == nil || *o.CaseInsensitive }
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// VectorIndexSuffix is appended to a TypeVector field's name to produce the
|
||||
// bleve sibling field that faiss actually indexes (e.g. "imageVector" ->
|
||||
// "imageVector_faiss"), mirroring the "location" / "location_geopoint" split.
|
||||
// bleve hands vector fields to faiss and cannot return them as stored fields
|
||||
// (verified: even Store:true yields nothing), so the original field is mapped
|
||||
// stored-only instead and round-trips like any other field, keeping vectors
|
||||
// alive across Move / Delete / Restore. OpenSearch keeps vectors in _source
|
||||
// and indexes the original field directly, no sibling there.
|
||||
const VectorIndexSuffix = "_faiss"
|
||||
|
||||
// AddVectorIndexSiblings writes, for each TypeVector override present in m,
|
||||
// the vector again under the suffixed sibling key. bleve vectors-build upsert
|
||||
// path only: without the vectors tag the sibling has no mapping and bleve's
|
||||
// dynamic mapping would index the raw floats.
|
||||
func AddVectorIndexSiblings(m map[string]any, overrides map[string]FieldOpts) {
|
||||
for key, opts := range overrides {
|
||||
if opts.Type != TypeVector {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(key, ".")
|
||||
parent := m
|
||||
for _, p := range parts[:len(parts)-1] {
|
||||
next, ok := parent[p].(map[string]any)
|
||||
if !ok {
|
||||
parent = nil
|
||||
break
|
||||
}
|
||||
parent = next
|
||||
}
|
||||
if parent == nil {
|
||||
continue
|
||||
}
|
||||
leaf := parts[len(parts)-1]
|
||||
if vec, ok := parent[leaf].([]any); ok && len(vec) > 0 {
|
||||
parent[leaf+VectorIndexSuffix] = vec
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package opensearch
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
@@ -29,12 +30,24 @@ var (
|
||||
)
|
||||
|
||||
type Backend struct {
|
||||
index string
|
||||
client *opensearchgoAPI.Client
|
||||
index string
|
||||
client *opensearchgoAPI.Client
|
||||
vectorizer search.TextVectorizer
|
||||
}
|
||||
|
||||
// Option configures a Backend.
|
||||
type Option func(*Backend)
|
||||
|
||||
// WithTextVectorizer enables semantic queries (`semantic:"..."`); without it
|
||||
// they are rejected.
|
||||
func WithTextVectorizer(v search.TextVectorizer) Option {
|
||||
return func(b *Backend) {
|
||||
b.vectorizer = v
|
||||
}
|
||||
}
|
||||
|
||||
// NewBackend creates a backend on the versioned generation of the named index.
|
||||
func NewBackend(ctx context.Context, name string, client *opensearchgoAPI.Client, logger log.Logger) (*Backend, error) {
|
||||
func NewBackend(ctx context.Context, name string, client *opensearchgoAPI.Client, logger log.Logger, opts ...Option) (*Backend, error) {
|
||||
index := VersionedIndexName(name)
|
||||
|
||||
pingResp, err := client.Ping(ctx, &opensearchgoAPI.PingReq{})
|
||||
@@ -68,17 +81,29 @@ func NewBackend(ctx context.Context, name string, client *opensearchgoAPI.Client
|
||||
return nil, fmt.Errorf("%w, cluster health is not green or yellow: %s", ErrUnhealthyCluster, resp.Status)
|
||||
}
|
||||
|
||||
return &Backend{index: index, client: client}, nil
|
||||
b := &Backend{index: index, client: client}
|
||||
for _, opt := range opts {
|
||||
opt(b)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
|
||||
boolQuery, err := convert.KQLToOpenSearchBoolQuery(sir.Query)
|
||||
// the semantic clause ranks, the remaining query filters: it scopes the
|
||||
// vector search and stays the only source of totals. The converter splits
|
||||
// the clause off the parsed KQL tree.
|
||||
boolQuery, semanticText, err := convert.KQLToOpenSearchBoolQueryWithSemantic(sir.Query, true)
|
||||
switch {
|
||||
case kql.IsValidationError(err):
|
||||
return nil, errtypes.BadRequest(err.Error())
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to convert KQL query to OpenSearch bool query: %w", err)
|
||||
}
|
||||
pureSemantic := semanticText != "" && isEmptyBoolQuery(boolQuery)
|
||||
|
||||
if semanticText != "" && b.vectorizer == nil {
|
||||
return nil, errtypes.BadRequest("semantic search is not configured")
|
||||
}
|
||||
|
||||
// filter out deleted resources
|
||||
boolQuery.Filter(
|
||||
@@ -110,7 +135,9 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
|
||||
}
|
||||
|
||||
searchParams := opensearchgoAPI.SearchParams{
|
||||
SourceExcludes: []string{"Content"}, // Do not send back the full content in the search response, as it is only needed for highlighting and can be large. The highlighted snippets will be sent back in the response instead.
|
||||
// Do not send back the full content (only needed for highlighting, the
|
||||
// snippets come back instead) or the raw image vectors (ranking data).
|
||||
SourceExcludes: []string{"Content", "imageVector"},
|
||||
}
|
||||
|
||||
switch {
|
||||
@@ -122,9 +149,15 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
|
||||
searchParams.Size = conversions.ToPointer(int(sir.PageSize))
|
||||
}
|
||||
|
||||
// the filter request carries the totals and the lexical ranking; for a
|
||||
// purely semantic query it only supplies the totals
|
||||
filterParams := searchParams
|
||||
if pureSemantic {
|
||||
filterParams.Size = conversions.ToPointer(0)
|
||||
}
|
||||
req, err := osu.BuildSearchReq(&opensearchgoAPI.SearchReq{
|
||||
Indices: []string{b.index},
|
||||
Params: searchParams,
|
||||
Params: filterParams,
|
||||
},
|
||||
boolQuery,
|
||||
osu.SearchBodyParams{
|
||||
@@ -151,15 +184,47 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
|
||||
return nil, fmt.Errorf("failed to search: %w", err)
|
||||
}
|
||||
|
||||
matches := make([]*searchMessage.Match, 0, len(resp.Hits.Hits))
|
||||
matches, err := convertHits(resp.Hits.Hits)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
totalMatches := resp.Hits.Total.Value
|
||||
for _, hit := range resp.Hits.Hits {
|
||||
match, err := convert.OpenSearchHitToMatch(hit)
|
||||
|
||||
if semanticText != "" {
|
||||
vector, err := b.vectorizer.VectorizeText(ctx, semanticText)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert hit to match: %w", err)
|
||||
return nil, fmt.Errorf("failed to vectorize the semantic query: %w", err)
|
||||
}
|
||||
k := semanticK(searchParams.Size)
|
||||
knnParams := searchParams
|
||||
knnParams.Size = conversions.ToPointer(k)
|
||||
// the full filter query also pre-filters the neighbor search
|
||||
knnQuery := osu.NewKnnQuery("imageVector").Vector(vector).K(k).Filter(boolQuery)
|
||||
knnReq, err := osu.BuildSearchReq(&opensearchgoAPI.SearchReq{
|
||||
Indices: []string{b.index},
|
||||
Params: knnParams,
|
||||
}, knnQuery)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build knn request: %w", err)
|
||||
}
|
||||
knnResp, err := b.client.Search(ctx, knnReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to run knn search: %w", err)
|
||||
}
|
||||
knnMatches, err := convertHits(knnResp.Hits.Hits)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
matches = append(matches, match)
|
||||
if pureSemantic {
|
||||
// the knn result is the ranking, and the only honest total is the
|
||||
// number of semantic hits (a similarity search has no result set,
|
||||
// only a ranking)
|
||||
matches = knnMatches
|
||||
totalMatches = len(knnMatches)
|
||||
} else {
|
||||
matches = fuseRRF(matches, knnMatches, searchParams.Size)
|
||||
}
|
||||
}
|
||||
|
||||
return &searchService.SearchIndexResponse{
|
||||
@@ -168,6 +233,80 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
|
||||
}, nil
|
||||
}
|
||||
|
||||
// isEmptyBoolQuery reports whether q carries no clauses (yet): an empty map
|
||||
// render means the parsed query had no filter part.
|
||||
func isEmptyBoolQuery(q *osu.BoolQuery) bool {
|
||||
m, err := q.Map()
|
||||
return err == nil && len(m) == 0
|
||||
}
|
||||
|
||||
// convertHits converts OpenSearch hits to matches.
|
||||
func convertHits(hits []opensearchgoAPI.SearchHit) ([]*searchMessage.Match, error) {
|
||||
matches := make([]*searchMessage.Match, 0, len(hits))
|
||||
for _, hit := range hits {
|
||||
match, err := convert.OpenSearchHitToMatch(hit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert hit to match: %w", err)
|
||||
}
|
||||
matches = append(matches, match)
|
||||
}
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
// semanticK picks the number of nearest neighbors for a semantic clause: at
|
||||
// least a fusion-friendly window, at most a sane cap (huge page sizes stand
|
||||
// for "everything", but a similarity ranking beyond 1000 hits carries no
|
||||
// signal).
|
||||
func semanticK(size *int) int {
|
||||
const minK, maxK = 200, 1000
|
||||
switch {
|
||||
case size == nil || *size >= maxK:
|
||||
return maxK
|
||||
case *size < minK:
|
||||
return minK
|
||||
default:
|
||||
return *size
|
||||
}
|
||||
}
|
||||
|
||||
// fuseRRF merges the lexical and the semantic ranking via reciprocal rank
|
||||
// fusion (rank constant 60, like bleve's default) and trims to limit.
|
||||
func fuseRRF(lexical, semantic []*searchMessage.Match, limit *int) []*searchMessage.Match {
|
||||
const rankConst = 60
|
||||
type entry struct {
|
||||
match *searchMessage.Match
|
||||
score float64
|
||||
}
|
||||
byID := map[string]*entry{}
|
||||
order := make([]*entry, 0, len(lexical)+len(semantic))
|
||||
add := func(list []*searchMessage.Match) {
|
||||
for i, m := range list {
|
||||
id := m.GetEntity().GetId()
|
||||
key := id.GetStorageId() + "$" + id.GetSpaceId() + "!" + id.GetOpaqueId()
|
||||
e, ok := byID[key]
|
||||
if !ok {
|
||||
e = &entry{match: m}
|
||||
byID[key] = e
|
||||
order = append(order, e)
|
||||
}
|
||||
e.score += 1.0 / float64(rankConst+i+1)
|
||||
}
|
||||
}
|
||||
add(lexical)
|
||||
add(semantic)
|
||||
sort.SliceStable(order, func(i, j int) bool { return order[i].score > order[j].score })
|
||||
|
||||
out := make([]*searchMessage.Match, 0, len(order))
|
||||
for _, e := range order {
|
||||
e.match.Score = float32(e.score)
|
||||
out = append(out, e.match)
|
||||
}
|
||||
if limit != nil && len(out) > *limit {
|
||||
out = out[:*limit]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *Backend) DocCount() (uint64, error) {
|
||||
req, err := osu.BuildIndicesCountReq(
|
||||
&opensearchgoAPI.IndicesCountReq{
|
||||
|
||||
@@ -2,6 +2,8 @@ package opensearch_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
@@ -10,7 +12,9 @@ import (
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
|
||||
)
|
||||
|
||||
@@ -80,4 +84,92 @@ var _ = Describe("Backend", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Semantic search", func() {
|
||||
const indexName = "opencloud-test-engine-semantic"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
// the backend versions the physical index by schema generation
|
||||
physical := opensearch.VersionedIndexName(indexName)
|
||||
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{physical})
|
||||
deleteIndexOnCleanup(tc, physical)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(context.Background(), indexName, tc.Client(), log.NopLogger(), opensearch.WithTextVectorizer(fakeVectorizer{
|
||||
byText: map[string][]float32{
|
||||
"cat": axisVector(0),
|
||||
"dog": axisVector(1),
|
||||
},
|
||||
}))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
upsert := func(id, path, name string, vector []float32) {
|
||||
r := opensearchtest.Testdata.Resources.File
|
||||
r.ID = id
|
||||
r.Path = path
|
||||
r.Name = name
|
||||
r.ImageVector = vector
|
||||
Expect(backend.Upsert(id, r)).To(Succeed())
|
||||
}
|
||||
upsert("1$1!10", "./cat.jpg", "cat.jpg", axisVector(0))
|
||||
upsert("1$1!11", "./dog.jpg", "dog.jpg", axisVector(1))
|
||||
upsert("1$1!12", "./note.txt", "note.txt", nil)
|
||||
tc.Require.IndicesRefresh([]string{physical}, nil)
|
||||
})
|
||||
|
||||
It("ranks the nearest image first for a purely semantic query", func() {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `semantic:"cat"`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).ToNot(BeEmpty())
|
||||
Expect(resp.Matches[0].Entity.Name).To(Equal("cat.jpg"))
|
||||
// the only honest total is the number of semantic hits
|
||||
Expect(resp.TotalMatches).To(Equal(int32(2)))
|
||||
})
|
||||
|
||||
It("combines a semantic clause with a filter part", func() {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `Name:dog* AND semantic:"cat"`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(1))
|
||||
Expect(resp.Matches[0].Entity.Name).To(Equal("dog.jpg"))
|
||||
Expect(resp.TotalMatches).To(Equal(int32(1)))
|
||||
})
|
||||
|
||||
It("keeps semantic inside a quoted value literal", func() {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `Name:"*semantic:cat*"`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("rejects semantic queries without a vectorizer", func() {
|
||||
bare, err := opensearch.NewBackend(context.Background(), indexName, tc.Client(), log.NopLogger())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = bare.Search(context.Background(), &searchService.SearchIndexRequest{Query: `semantic:"cat"`})
|
||||
Expect(err).To(MatchError(ContainSubstring("not configured")))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// axisVector returns a one-hot vector of the schema dimensionality: distinct
|
||||
// axes are orthogonal (cosine 0), same axes identical (cosine 1).
|
||||
func axisVector(axis int) []float32 {
|
||||
v := make([]float32, content.ImageVectorDims)
|
||||
v[axis] = 1
|
||||
return v
|
||||
}
|
||||
|
||||
// fakeVectorizer maps query texts to fixed vectors.
|
||||
type fakeVectorizer struct{ byText map[string][]float32 }
|
||||
|
||||
func (f fakeVectorizer) VectorizeText(_ context.Context, text string) ([]float32, error) {
|
||||
v, ok := f.byText[strings.ToLower(text)]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no vector for %q", text)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
@@ -79,6 +79,9 @@ func buildResourceMapping() ([]byte, error) {
|
||||
"settings": map[string]any{
|
||||
"number_of_shards": "1",
|
||||
"number_of_replicas": "1",
|
||||
// static setting, must be set at creation time: enables the knn
|
||||
// query type for the imageVector field
|
||||
"knn": "true",
|
||||
"analysis": map[string]any{
|
||||
// path_hierarchy is case-preserving; casing lives in the value.
|
||||
"analyzer": map[string]any{
|
||||
|
||||
Loaded 100 of 216 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user