Compare commits

..
Author SHA1 Message Date
Dominik Schmidt eba5e87047 refactor(config): keep the re-export a function, pin env expansion in the test 2026-09-03 01:41:41 +02:00
Dominik Schmidt b9fd10c202 refactor(config): extract BindSourcesToStructs into pkg/config/binder
New leaf package (only gookit + pkg/config/defaults) so callers can bind a
yaml config file without importing the aggregate service config. pkg/config
keeps a backward-compatible re-export.
2026-09-03 01:41:41 +02:00
209 changed files with 11899 additions and 5711 deletions

No files matched your search

+4 -3
View File
@@ -62,8 +62,8 @@ require (
github.com/onsi/gomega v1.42.1
github.com/open-policy-agent/opa v1.19.1
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067
github.com/opencloud-eu/reva/v2 v2.49.1-0.20260903122659-26f34ec05774
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260901070651-08a5330ce57d
github.com/opencloud-eu/reva/v2 v2.49.0
github.com/opensearch-project/opensearch-go/v4 v4.7.3
github.com/orcaman/concurrent-map v1.0.0
github.com/pkg/errors v0.9.1
@@ -95,7 +95,7 @@ require (
go-micro.dev/v4 v4.11.0
go.etcd.io/bbolt v1.5.0
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
go.opentelemetry.io/contrib/zpages v0.71.0
go.opentelemetry.io/otel v1.46.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
@@ -324,6 +324,7 @@ require (
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/pquerna/cachecontrol v0.2.0 // indirect
github.com/prometheus/alertmanager v0.33.1 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/prometheus/statsd_exporter v0.22.8 // indirect
+8 -6
View File
@@ -940,10 +940,10 @@ github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-202505121527
github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-20250512152754-23325793059a/go.mod h1:pjcozWijkNPbEtX5SIQaxEW/h8VAVZYTLx+70bmB3LY=
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+lP5lUUIzjRGDg93WrQfZJZCaV1ZP3KeyXi8bzY=
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067 h1:UkNMKauyJAzY6RE6mmthz9bQZLYkbvBuApm7ZDCparE=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU=
github.com/opencloud-eu/reva/v2 v2.49.1-0.20260903122659-26f34ec05774 h1:8N42Pwzyafdxq66zYP6kJ1CH/QBRpxovQXq/0Kt6bJY=
github.com/opencloud-eu/reva/v2 v2.49.1-0.20260903122659-26f34ec05774/go.mod h1:Frg+UWnVcSy+412UB3l2LcD0KY8ZNu1samimwkNywbg=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260901070651-08a5330ce57d h1:3zbb31655ZvqwLMWMIiUAhYer6YcFnv/oguujnX7aOU=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260901070651-08a5330ce57d/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU=
github.com/opencloud-eu/reva/v2 v2.49.0 h1:AwECMDDth3NUaihZRf9bI9HNpWutvaRijOCvQyQfTT4=
github.com/opencloud-eu/reva/v2 v2.49.0/go.mod h1:Frg+UWnVcSy+412UB3l2LcD0KY8ZNu1samimwkNywbg=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -1001,6 +1001,8 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:Om
github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k=
github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI=
github.com/pquerna/otp v1.3.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
github.com/prometheus/alertmanager v0.33.1 h1:PJHGvTdb8Q0ZEpJnWff120WxB2kxIScLa707AaIudTo=
github.com/prometheus/alertmanager v0.33.1/go.mod h1:V06Uc8EZ5X5wLOJRGhtXx+EE2LgrinFIADbKWMVm1RY=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
@@ -1293,8 +1295,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
go.opentelemetry.io/contrib/zpages v0.71.0 h1:OrQSEmdlfknIusEyGj/t30uBR7MnbmGcCRBt5ov8/rA=
go.opentelemetry.io/contrib/zpages v0.71.0/go.mod h1:C3IIpqkQbjswcSREqxQLyEapF0SI9bqwJpxpjwtzFrE=
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
+55
View File
@@ -0,0 +1,55 @@
// Package binder binds OpenCloud yaml config files to config structs, as a leaf
// package that avoids importing the aggregate service config in pkg/config.
package binder
import (
"io/fs"
"os"
"path"
"strings"
gofig "github.com/gookit/config/v2"
gooyaml "github.com/gookit/config/v2/yaml"
"github.com/opencloud-eu/opencloud/pkg/config/defaults"
)
// decoderConfigTagName sets the tag name to be used from the config structs
// currently we only support "yaml" because we only support config loading
// from yaml files and the yaml parser has no simple way to set a custom tag name to use
const decoderConfigTagName = "yaml"
// BindSourcesToStructs assigns any config value from a config file / env variable to struct `dst`.
func BindSourcesToStructs(service string, dst any) error {
fileSystem := os.DirFS("/")
filePath := strings.TrimLeft(path.Join(defaults.BaseConfigPath(), service+".yaml"), "/")
return BindSourcesToStructsFS(fileSystem, filePath, service, dst)
}
// BindSourcesToStructsFS is like BindSourcesToStructs but reads from the given fs.FS and path.
func BindSourcesToStructsFS(fileSystem fs.FS, filePath, service string, dst any) error {
cnf := gofig.NewWithOptions(service)
cnf.WithOptions(func(options *gofig.Options) {
options.ParseEnv = true
options.DecoderConfig.TagName = decoderConfigTagName
})
cnf.AddDriver(gooyaml.Driver)
yamlContent, err := fs.ReadFile(fileSystem, filePath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
// the error is ignored on purpose, matching the pre-extraction behavior:
// an unparseable yaml file binds nothing instead of failing the startup
_ = cnf.LoadSources("yaml", yamlContent)
err = cnf.BindStruct("", &dst)
if err != nil {
return err
}
return nil
}
+55
View File
@@ -0,0 +1,55 @@
package binder
import (
"testing"
"testing/fstest"
"gotest.tools/v3/assert"
)
type TestConfig struct {
A string `yaml:"a"`
B string `yaml:"b"`
C string `yaml:"c"`
}
func TestBindSourcesToStructs(t *testing.T) {
// setup test env: one var set to pin env expansion, two deliberately
// unset to pin the defaults
t.Setenv("BINDER_TEST_SET_VAR", "from-env")
yaml := `
a: "${BINDER_TEST_SET_VAR|no-foo}"
b: "${BINDER_TEST_UNSET_VAR|no-bar}"
c: "${BINDER_TEST_OTHER_UNSET_VAR|code}"
`
filePath := "etc/opencloud/foo.yaml"
fs := fstest.MapFS{
filePath: {Data: []byte(yaml)},
}
// perform test
c := TestConfig{}
err := BindSourcesToStructsFS(fs, filePath, "foo", &c)
if err != nil {
t.Error(err)
}
assert.Equal(t, c.A, "from-env")
assert.Equal(t, c.B, "no-bar")
assert.Equal(t, c.C, "code")
}
func TestBindSourcesToStructs_UnknownFile(t *testing.T) {
// setup test env
filePath := "etc/opencloud/foo.yaml"
fs := fstest.MapFS{}
// perform test
c := TestConfig{}
err := BindSourcesToStructsFS(fs, filePath, "foo", &c)
if err != nil {
t.Error(err)
}
assert.Equal(t, c.A, "")
assert.Equal(t, c.B, "")
assert.Equal(t, c.C, "")
}
+3 -42
View File
@@ -1,54 +1,15 @@
package config
import (
"io/fs"
"os"
"path"
"strings"
gofig "github.com/gookit/config/v2"
gooyaml "github.com/gookit/config/v2/yaml"
"github.com/opencloud-eu/opencloud/pkg/config/defaults"
)
var (
// decoderConfigTagName sets the tag name to be used from the config structs
// currently we only support "yaml" because we only support config loading
// from yaml files and the yaml parser has no simple way to set a custom tag name to use
decoderConfigTagName = "yaml"
"github.com/opencloud-eu/opencloud/pkg/config/binder"
)
// BindSourcesToStructs assigns any config value from a config file / env variable to struct `dst`.
// The implementation lives in pkg/config/binder.
func BindSourcesToStructs(service string, dst any) error {
fileSystem := os.DirFS("/")
filePath := strings.TrimLeft(path.Join(defaults.BaseConfigPath(), service+".yaml"), "/")
return bindSourcesToStructs(fileSystem, filePath, service, dst)
}
func bindSourcesToStructs(fileSystem fs.FS, filePath, service string, dst any) error {
cnf := gofig.NewWithOptions(service)
cnf.WithOptions(func(options *gofig.Options) {
options.ParseEnv = true
options.DecoderConfig.TagName = decoderConfigTagName
})
cnf.AddDriver(gooyaml.Driver)
yamlContent, err := fs.ReadFile(fileSystem, filePath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
_ = cnf.LoadSources("yaml", yamlContent)
err = cnf.BindStruct("", &dst)
if err != nil {
return err
}
return nil
return binder.BindSourcesToStructs(service, dst)
}
// LocalEndpoint returns the local endpoint for a given protocol and address.
+2 -46
View File
@@ -4,54 +4,10 @@ import (
"testing"
"testing/fstest"
"github.com/opencloud-eu/opencloud/pkg/config/binder"
"gotest.tools/v3/assert"
)
type TestConfig struct {
A string `yaml:"a"`
B string `yaml:"b"`
C string `yaml:"c"`
}
func TestBindSourcesToStructs(t *testing.T) {
// setup test env
yaml := `
a: "${FOO_VAR|no-foo}"
b: "${BAR_VAR|no-bar}"
c: "${CODE_VAR|code}"
`
filePath := "etc/opencloud/foo.yaml"
fs := fstest.MapFS{
filePath: {Data: []byte(yaml)},
}
// perform test
c := TestConfig{}
err := bindSourcesToStructs(fs, filePath, "foo", &c)
if err != nil {
t.Error(err)
}
assert.Equal(t, c.A, "no-foo")
assert.Equal(t, c.B, "no-bar")
assert.Equal(t, c.C, "code")
}
func TestBindSourcesToStructs_UnknownFile(t *testing.T) {
// setup test env
filePath := "etc/opencloud/foo.yaml"
fs := fstest.MapFS{}
// perform test
c := TestConfig{}
err := bindSourcesToStructs(fs, filePath, "foo", &c)
if err != nil {
t.Error(err)
}
assert.Equal(t, c.A, "")
assert.Equal(t, c.B, "")
assert.Equal(t, c.C, "")
}
func TestBindSourcesToStructs_NoEnvVar(t *testing.T) {
// setup test env
yaml := `
@@ -180,7 +136,7 @@ clientlog:
}
// perform test
c := Config{}
err := bindSourcesToStructs(fs, filePath, "foo", &c)
err := binder.BindSourcesToStructsFS(fs, filePath, "foo", &c)
if err != nil {
t.Error(err)
}
@@ -654,148 +654,6 @@ func (x *Video) GetWidth() int32 {
return 0
}
type MotionPhoto struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Version *int32 `protobuf:"varint,1,opt,name=version,proto3,oneof" json:"version,omitempty"`
PresentationTimestampUs *int64 `protobuf:"varint,2,opt,name=presentationTimestampUs,proto3,oneof" json:"presentationTimestampUs,omitempty"`
VideoSize *int64 `protobuf:"varint,3,opt,name=videoSize,proto3,oneof" json:"videoSize,omitempty"`
}
func (x *MotionPhoto) Reset() {
*x = MotionPhoto{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *MotionPhoto) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MotionPhoto) ProtoMessage() {}
func (x *MotionPhoto) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[7]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use MotionPhoto.ProtoReflect.Descriptor instead.
func (*MotionPhoto) Descriptor() ([]byte, []int) {
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{7}
}
func (x *MotionPhoto) GetVersion() int32 {
if x != nil && x.Version != nil {
return *x.Version
}
return 0
}
func (x *MotionPhoto) GetPresentationTimestampUs() int64 {
if x != nil && x.PresentationTimestampUs != nil {
return *x.PresentationTimestampUs
}
return 0
}
func (x *MotionPhoto) GetVideoSize() int64 {
if x != nil && x.VideoSize != nil {
return *x.VideoSize
}
return 0
}
type LivePhoto struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
ContentId *string `protobuf:"bytes,1,opt,name=contentId,proto3,oneof" json:"contentId,omitempty"`
StillImageTimeUs *int64 `protobuf:"varint,2,opt,name=stillImageTimeUs,proto3,oneof" json:"stillImageTimeUs,omitempty"`
Auto *bool `protobuf:"varint,3,opt,name=auto,proto3,oneof" json:"auto,omitempty"`
VitalityScore *float64 `protobuf:"fixed64,4,opt,name=vitalityScore,proto3,oneof" json:"vitalityScore,omitempty"`
VitalityScoringVersion *int64 `protobuf:"varint,5,opt,name=vitalityScoringVersion,proto3,oneof" json:"vitalityScoringVersion,omitempty"`
}
func (x *LivePhoto) Reset() {
*x = LivePhoto{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *LivePhoto) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*LivePhoto) ProtoMessage() {}
func (x *LivePhoto) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[8]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use LivePhoto.ProtoReflect.Descriptor instead.
func (*LivePhoto) Descriptor() ([]byte, []int) {
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{8}
}
func (x *LivePhoto) GetContentId() string {
if x != nil && x.ContentId != nil {
return *x.ContentId
}
return ""
}
func (x *LivePhoto) GetStillImageTimeUs() int64 {
if x != nil && x.StillImageTimeUs != nil {
return *x.StillImageTimeUs
}
return 0
}
func (x *LivePhoto) GetAuto() bool {
if x != nil && x.Auto != nil {
return *x.Auto
}
return false
}
func (x *LivePhoto) GetVitalityScore() float64 {
if x != nil && x.VitalityScore != nil {
return *x.VitalityScore
}
return 0
}
func (x *LivePhoto) GetVitalityScoringVersion() int64 {
if x != nil && x.VitalityScoringVersion != nil {
return *x.VitalityScoringVersion
}
return 0
}
type Entity struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -821,15 +679,13 @@ type Entity struct {
Image *Image `protobuf:"bytes,18,opt,name=image,proto3" json:"image,omitempty"`
Photo *Photo `protobuf:"bytes,19,opt,name=photo,proto3" json:"photo,omitempty"`
Favorites []string `protobuf:"bytes,20,rep,name=favorites,proto3" json:"favorites,omitempty"`
MotionPhoto *MotionPhoto `protobuf:"bytes,21,opt,name=motionPhoto,proto3" json:"motionPhoto,omitempty"`
Video *Video `protobuf:"bytes,22,opt,name=video,proto3" json:"video,omitempty"`
LivePhoto *LivePhoto `protobuf:"bytes,23,opt,name=livePhoto,proto3" json:"livePhoto,omitempty"`
}
func (x *Entity) Reset() {
*x = Entity{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[9]
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -842,7 +698,7 @@ func (x *Entity) String() string {
func (*Entity) ProtoMessage() {}
func (x *Entity) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[9]
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[7]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -855,7 +711,7 @@ func (x *Entity) ProtoReflect() protoreflect.Message {
// Deprecated: Use Entity.ProtoReflect.Descriptor instead.
func (*Entity) Descriptor() ([]byte, []int) {
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{9}
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{7}
}
func (x *Entity) GetRef() *Reference {
@@ -998,13 +854,6 @@ func (x *Entity) GetFavorites() []string {
return nil
}
func (x *Entity) GetMotionPhoto() *MotionPhoto {
if x != nil {
return x.MotionPhoto
}
return nil
}
func (x *Entity) GetVideo() *Video {
if x != nil {
return x.Video
@@ -1012,13 +861,6 @@ func (x *Entity) GetVideo() *Video {
return nil
}
func (x *Entity) GetLivePhoto() *LivePhoto {
if x != nil {
return x.LivePhoto
}
return nil
}
type Match struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -1033,7 +875,7 @@ type Match struct {
func (x *Match) Reset() {
*x = Match{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[10]
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1046,7 +888,7 @@ func (x *Match) String() string {
func (*Match) ProtoMessage() {}
func (x *Match) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[10]
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[8]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1059,7 +901,7 @@ func (x *Match) ProtoReflect() protoreflect.Message {
// Deprecated: Use Match.ProtoReflect.Descriptor instead.
func (*Match) Descriptor() ([]byte, []int) {
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{10}
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{8}
}
func (x *Match) GetEntity() *Entity {
@@ -1227,119 +1069,77 @@ var file_opencloud_messages_search_v0_search_proto_rawDesc = []byte{
0x6e, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x66, 0x6f, 0x75, 0x72, 0x43, 0x43, 0x42, 0x0c, 0x0a, 0x0a,
0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x52, 0x61, 0x74, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x68,
0x65, 0x69, 0x67, 0x68, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22,
0xc4, 0x01, 0x0a, 0x0b, 0x4d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x12,
0x1d, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05,
0x48, 0x00, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x3d,
0x0a, 0x17, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x55, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x48,
0x01, 0x52, 0x17, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x55, 0x73, 0x88, 0x01, 0x01, 0x12, 0x21, 0x0a,
0x09, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03,
0x48, 0x02, 0x52, 0x09, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x53, 0x69, 0x7a, 0x65, 0x88, 0x01, 0x01,
0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x1a, 0x0a, 0x18,
0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d,
0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x55, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x76, 0x69, 0x64,
0x65, 0x6f, 0x53, 0x69, 0x7a, 0x65, 0x22, 0xb9, 0x02, 0x0a, 0x09, 0x4c, 0x69, 0x76, 0x65, 0x50,
0x68, 0x6f, 0x74, 0x6f, 0x12, 0x21, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x49,
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65,
0x6e, 0x74, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x2f, 0x0a, 0x10, 0x73, 0x74, 0x69, 0x6c, 0x6c,
0x49, 0x6d, 0x61, 0x67, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28,
0x03, 0x48, 0x01, 0x52, 0x10, 0x73, 0x74, 0x69, 0x6c, 0x6c, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x54,
0x69, 0x6d, 0x65, 0x55, 0x73, 0x88, 0x01, 0x01, 0x12, 0x17, 0x0a, 0x04, 0x61, 0x75, 0x74, 0x6f,
0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x48, 0x02, 0x52, 0x04, 0x61, 0x75, 0x74, 0x6f, 0x88, 0x01,
0x01, 0x12, 0x29, 0x0a, 0x0d, 0x76, 0x69, 0x74, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f,
0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x48, 0x03, 0x52, 0x0d, 0x76, 0x69, 0x74, 0x61,
0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x88, 0x01, 0x01, 0x12, 0x3b, 0x0a, 0x16,
0x76, 0x69, 0x74, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x56,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x48, 0x04, 0x52, 0x16,
0x76, 0x69, 0x74, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x56,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, 0x6f,
0x6e, 0x74, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x73, 0x74, 0x69, 0x6c,
0x6c, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x42, 0x07, 0x0a, 0x05,
0x5f, 0x61, 0x75, 0x74, 0x6f, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x76, 0x69, 0x74, 0x61, 0x6c, 0x69,
0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x19, 0x0a, 0x17, 0x5f, 0x76, 0x69, 0x74, 0x61,
0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x65, 0x72, 0x73, 0x69,
0x6f, 0x6e, 0x22, 0xc9, 0x08, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a,
0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65,
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65,
0x6e, 0x63, 0x65, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x38, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x02,
0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x74, 0x61, 0x67, 0x18, 0x04,
0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x65, 0x74, 0x61, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69,
0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x48,
0x0a, 0x12, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f,
0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d,
0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69,
0x66, 0x69, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x69, 0x6d, 0x65,
0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d,
0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73,
0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x65, 0x72, 0x6d,
0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18,
0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64,
0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x64, 0x65,
0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x65, 0x52, 0x6f,
0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x68,
0x61, 0x72, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x45, 0x0a, 0x09, 0x70,
0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28,
0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61,
0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65,
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74,
0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x09,
0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x68, 0x69, 0x67, 0x68, 0x6c, 0x69,
0x67, 0x68, 0x74, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x68, 0x69, 0x67, 0x68,
0x6c, 0x69, 0x67, 0x68, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x05, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x18,
0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75,
0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63,
0x68, 0x2e, 0x76, 0x30, 0x2e, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x52, 0x05, 0x61, 0x75, 0x64, 0x69,
0x6f, 0x12, 0x48, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x10, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e,
0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e,
0x76, 0x30, 0x2e, 0x47, 0x65, 0x6f, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x65,
0x73, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4e, 0x0a, 0x0e, 0x72,
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x11, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e,
0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e,
0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x0c, 0x72,
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x12, 0x39, 0x0a, 0x05, 0x69,
0x6d, 0x61, 0x67, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65,
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52,
0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x05, 0x70, 0x68, 0x6f, 0x74, 0x6f, 0x18,
0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75,
0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63,
0x68, 0x2e, 0x76, 0x30, 0x2e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x52, 0x05, 0x70, 0x68, 0x6f, 0x74,
0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74, 0x65, 0x73, 0x18, 0x14,
0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74, 0x65, 0x73, 0x12,
0x4b, 0x0a, 0x0b, 0x6d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x18, 0x15,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
0x2e, 0x76, 0x30, 0x2e, 0x4d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x52,
0x0b, 0x6d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x12, 0x39, 0x0a, 0x05,
0xb5, 0x07, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a, 0x03, 0x72, 0x65,
0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61,
0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65,
0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x38, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65,
0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30,
0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x02, 0x69, 0x64, 0x12,
0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x74, 0x61, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28,
0x09, 0x52, 0x04, 0x65, 0x74, 0x61, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18,
0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x48, 0x0a, 0x12, 0x6c,
0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d,
0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74,
0x61, 0x6d, 0x70, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65,
0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x79,
0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d, 0x65, 0x54, 0x79,
0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e,
0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73,
0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x09, 0x20, 0x01,
0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x6c, 0x65,
0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74,
0x65, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x4e,
0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x65,
0x52, 0x6f, 0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x45, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65,
0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70,
0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73,
0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75,
0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12,
0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74,
0x61, 0x67, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x68, 0x69, 0x67, 0x68, 0x6c, 0x69, 0x67, 0x68, 0x74,
0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x68, 0x69, 0x67, 0x68, 0x6c, 0x69, 0x67,
0x68, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x05, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x18, 0x0f, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d,
0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76,
0x30, 0x2e, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x52, 0x05, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x12, 0x48,
0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x2c, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73,
0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e,
0x47, 0x65, 0x6f, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x73, 0x52, 0x08,
0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4e, 0x0a, 0x0e, 0x72, 0x65, 0x6d, 0x6f,
0x74, 0x65, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73,
0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e,
0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x0c, 0x72, 0x65, 0x6d, 0x6f,
0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x12, 0x39, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67,
0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61,
0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x05, 0x69, 0x6d,
0x61, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x05, 0x70, 0x68, 0x6f, 0x74, 0x6f, 0x18, 0x13, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d,
0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76,
0x30, 0x2e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x52, 0x05, 0x70, 0x68, 0x6f, 0x74, 0x6f, 0x12, 0x1c,
0x0a, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74, 0x65, 0x73, 0x18, 0x14, 0x20, 0x03, 0x28,
0x09, 0x52, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x05,
0x76, 0x69, 0x64, 0x65, 0x6f, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70,
0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73,
0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x56, 0x69, 0x64, 0x65, 0x6f,
0x52, 0x05, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x45, 0x0a, 0x09, 0x6c, 0x69, 0x76, 0x65, 0x50,
0x68, 0x6f, 0x74, 0x6f, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65,
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x4c, 0x69, 0x76, 0x65, 0x50, 0x68,
0x6f, 0x74, 0x6f, 0x52, 0x09, 0x6c, 0x69, 0x76, 0x65, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x22, 0x5b,
0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x3c, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74,
0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61,
0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65,
0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02,
0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x4d, 0x5a, 0x4b, 0x67,
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70,
0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73,
0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
0x52, 0x05, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x22, 0x5b, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68,
0x12, 0x3c, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x24, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73,
0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e,
0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x14,
0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73,
0x63, 0x6f, 0x72, 0x65, 0x42, 0x4d, 0x5a, 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
0x6f, 0x6d, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x65, 0x75, 0x2f,
0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67,
0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -1354,7 +1154,7 @@ func file_opencloud_messages_search_v0_search_proto_rawDescGZIP() []byte {
return file_opencloud_messages_search_v0_search_proto_rawDescData
}
var file_opencloud_messages_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 11)
var file_opencloud_messages_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
var file_opencloud_messages_search_v0_search_proto_goTypes = []interface{}{
(*ResourceID)(nil), // 0: opencloud.messages.search.v0.ResourceID
(*Reference)(nil), // 1: opencloud.messages.search.v0.Reference
@@ -1363,33 +1163,29 @@ var file_opencloud_messages_search_v0_search_proto_goTypes = []interface{}{
(*GeoCoordinates)(nil), // 4: opencloud.messages.search.v0.GeoCoordinates
(*Photo)(nil), // 5: opencloud.messages.search.v0.Photo
(*Video)(nil), // 6: opencloud.messages.search.v0.Video
(*MotionPhoto)(nil), // 7: opencloud.messages.search.v0.MotionPhoto
(*LivePhoto)(nil), // 8: opencloud.messages.search.v0.LivePhoto
(*Entity)(nil), // 9: opencloud.messages.search.v0.Entity
(*Match)(nil), // 10: opencloud.messages.search.v0.Match
(*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp
(*Entity)(nil), // 7: opencloud.messages.search.v0.Entity
(*Match)(nil), // 8: opencloud.messages.search.v0.Match
(*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp
}
var file_opencloud_messages_search_v0_search_proto_depIdxs = []int32{
0, // 0: opencloud.messages.search.v0.Reference.resource_id:type_name -> opencloud.messages.search.v0.ResourceID
11, // 1: opencloud.messages.search.v0.Photo.takenDateTime:type_name -> google.protobuf.Timestamp
9, // 1: opencloud.messages.search.v0.Photo.takenDateTime:type_name -> google.protobuf.Timestamp
1, // 2: opencloud.messages.search.v0.Entity.ref:type_name -> opencloud.messages.search.v0.Reference
0, // 3: opencloud.messages.search.v0.Entity.id:type_name -> opencloud.messages.search.v0.ResourceID
11, // 4: opencloud.messages.search.v0.Entity.last_modified_time:type_name -> google.protobuf.Timestamp
9, // 4: opencloud.messages.search.v0.Entity.last_modified_time:type_name -> google.protobuf.Timestamp
0, // 5: opencloud.messages.search.v0.Entity.parent_id:type_name -> opencloud.messages.search.v0.ResourceID
2, // 6: opencloud.messages.search.v0.Entity.audio:type_name -> opencloud.messages.search.v0.Audio
4, // 7: opencloud.messages.search.v0.Entity.location:type_name -> opencloud.messages.search.v0.GeoCoordinates
0, // 8: opencloud.messages.search.v0.Entity.remote_item_id:type_name -> opencloud.messages.search.v0.ResourceID
3, // 9: opencloud.messages.search.v0.Entity.image:type_name -> opencloud.messages.search.v0.Image
5, // 10: opencloud.messages.search.v0.Entity.photo:type_name -> opencloud.messages.search.v0.Photo
7, // 11: opencloud.messages.search.v0.Entity.motionPhoto:type_name -> opencloud.messages.search.v0.MotionPhoto
6, // 12: opencloud.messages.search.v0.Entity.video:type_name -> opencloud.messages.search.v0.Video
8, // 13: opencloud.messages.search.v0.Entity.livePhoto:type_name -> opencloud.messages.search.v0.LivePhoto
9, // 14: opencloud.messages.search.v0.Match.entity:type_name -> opencloud.messages.search.v0.Entity
15, // [15:15] is the sub-list for method output_type
15, // [15:15] is the sub-list for method input_type
15, // [15:15] is the sub-list for extension type_name
15, // [15:15] is the sub-list for extension extendee
0, // [0:15] is the sub-list for field type_name
6, // 11: opencloud.messages.search.v0.Entity.video:type_name -> opencloud.messages.search.v0.Video
7, // 12: opencloud.messages.search.v0.Match.entity:type_name -> opencloud.messages.search.v0.Entity
13, // [13:13] is the sub-list for method output_type
13, // [13:13] is the sub-list for method input_type
13, // [13:13] is the sub-list for extension type_name
13, // [13:13] is the sub-list for extension extendee
0, // [0:13] is the sub-list for field type_name
}
func init() { file_opencloud_messages_search_v0_search_proto_init() }
@@ -1483,30 +1279,6 @@ func file_opencloud_messages_search_v0_search_proto_init() {
}
}
file_opencloud_messages_search_v0_search_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*MotionPhoto); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_opencloud_messages_search_v0_search_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*LivePhoto); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_opencloud_messages_search_v0_search_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Entity); i {
case 0:
return &v.state
@@ -1518,7 +1290,7 @@ func file_opencloud_messages_search_v0_search_proto_init() {
return nil
}
}
file_opencloud_messages_search_v0_search_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
file_opencloud_messages_search_v0_search_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Match); i {
case 0:
return &v.state
@@ -1536,15 +1308,13 @@ func file_opencloud_messages_search_v0_search_proto_init() {
file_opencloud_messages_search_v0_search_proto_msgTypes[4].OneofWrappers = []interface{}{}
file_opencloud_messages_search_v0_search_proto_msgTypes[5].OneofWrappers = []interface{}{}
file_opencloud_messages_search_v0_search_proto_msgTypes[6].OneofWrappers = []interface{}{}
file_opencloud_messages_search_v0_search_proto_msgTypes[7].OneofWrappers = []interface{}{}
file_opencloud_messages_search_v0_search_proto_msgTypes[8].OneofWrappers = []interface{}{}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_opencloud_messages_search_v0_search_proto_rawDesc,
NumEnums: 0,
NumMessages: 11,
NumMessages: 9,
NumExtensions: 0,
NumServices: 0,
},
@@ -299,14 +299,8 @@
"type": "string"
}
},
"motionPhoto": {
"$ref": "#/definitions/v0MotionPhoto"
},
"video": {
"$ref": "#/definitions/v0Video"
},
"livePhoto": {
"$ref": "#/definitions/v0LivePhoto"
}
}
},
@@ -385,29 +379,6 @@
}
}
},
"v0LivePhoto": {
"type": "object",
"properties": {
"contentId": {
"type": "string"
},
"stillImageTimeUs": {
"type": "string",
"format": "int64"
},
"auto": {
"type": "boolean"
},
"vitalityScore": {
"type": "number",
"format": "double"
},
"vitalityScoringVersion": {
"type": "string",
"format": "int64"
}
}
},
"v0Match": {
"type": "object",
"properties": {
@@ -422,23 +393,6 @@
}
}
},
"v0MotionPhoto": {
"type": "object",
"properties": {
"version": {
"type": "integer",
"format": "int32"
},
"presentationTimestampUs": {
"type": "string",
"format": "int64"
},
"videoSize": {
"type": "string",
"format": "int64"
}
}
},
"v0Photo": {
"type": "object",
"properties": {
@@ -72,20 +72,6 @@ message Video {
optional int32 width = 10;
}
message MotionPhoto {
optional int32 version = 1;
optional int64 presentationTimestampUs = 2;
optional int64 videoSize = 3;
}
message LivePhoto {
optional string contentId = 1;
optional int64 stillImageTimeUs = 2;
optional bool auto = 3;
optional double vitalityScore = 4;
optional int64 vitalityScoringVersion = 5;
}
message Entity {
Reference ref = 1;
ResourceID id = 2;
@@ -107,9 +93,7 @@ message Entity {
Image image = 18;
Photo photo = 19;
repeated string favorites = 20;
MotionPhoto motionPhoto = 21;
Video video = 22;
LivePhoto livePhoto = 23;
}
message Match {
+28 -130
View File
@@ -3,18 +3,17 @@ package command
import (
"context"
"fmt"
"strings"
"github.com/nats-io/nats.go"
"github.com/olekukonko/errors"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/pkg/runner"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/spf13/cobra"
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
"github.com/opencloud-eu/opencloud/pkg/generators"
"github.com/opencloud-eu/opencloud/pkg/log"
natspkg "github.com/opencloud-eu/opencloud/pkg/nats"
"github.com/opencloud-eu/opencloud/pkg/registry"
"github.com/opencloud-eu/opencloud/pkg/runner"
ogrpc "github.com/opencloud-eu/opencloud/pkg/service/grpc"
"github.com/opencloud-eu/opencloud/pkg/tracing"
"github.com/opencloud-eu/opencloud/pkg/version"
@@ -25,12 +24,6 @@ import (
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/metrics"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/server/debug"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/server/http"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
svcEvents "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events"
svcHttp "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
var _registeredEvents = []events.Unmarshaller{
@@ -69,11 +62,19 @@ func Server(cfg *config.Config) *cobra.Command {
gr := runner.NewGroup()
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
mtrcs := metrics.New()
mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1)
defer cancel()
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
evStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig(cfg.Events))
if err != nil {
logger.Error().Err(err).Msg("Failed to initialize event stream")
return err
}
tm, err := pool.StringToTLSMode(cfg.GRPCClientTLS.Mode)
if err != nil {
logger.Error().Err(err).Msg("Failed to parse tls mode")
@@ -98,101 +99,28 @@ func Server(cfg *config.Config) *cobra.Command {
return err
}
kv, err := ConnectNatsKV(cfg.Store)
if err != nil {
return err
}
activityLog, err := activitylog.New(kv,
activitylog.Logger(logger),
activitylog.MaxActivities(cfg.MaxActivities),
activitylog.WriteBufferDuration(cfg.WriteBufferDuration),
)
if err != nil {
logger.Error().Err(err).Msg("Failed to initialize activity log")
return err
}
hClient := ehsvc.NewEventHistoryService("eu.opencloud.api.eventhistory", grpcClient)
vClient := settingssvc.NewValueService("eu.opencloud.api.settings", grpcClient)
if !cfg.HTTP.Disabled {
hClient := ehsvc.NewEventHistoryService("eu.opencloud.api.eventhistory", grpcClient)
svc, err := svcHttp.New(
activityLog,
svcHttp.Logger(logger),
svcHttp.GatewaySelector(gatewaySelector),
svcHttp.RegisteredEvents(_registeredEvents),
//svcHttp.TraceProvider(tracerProvider),
svcHttp.HistoryClient(hClient),
)
if err != nil {
logger.Error().Err(err).Msg("handler init")
return err
}
// TODO svc = service.NewInstrument(svc, metrics)
// TODO svc = service.NewLogging(svc, logger) // this logs service specific data
// TODO svc = service.NewTracing(svc, traceProvider)
vClient := settingssvc.NewValueService("eu.opencloud.api.settings", grpcClient)
server, err := http.Server(
http.ValueClient(vClient),
{
svc, err := http.Server(
http.Logger(logger),
http.Context(ctx),
http.Config(cfg),
http.Service(svc),
http.Context(ctx), // NOTE: not passing this "option" leads to a panic in go-micro
http.TraceProvider(tracerProvider),
http.Stream(evStream),
http.GatewaySelector(gatewaySelector),
http.HistoryClient(hClient),
http.ValueClient(vClient),
http.RegisteredEvents(_registeredEvents),
)
if err != nil {
logger.Info().
Err(err).
Str("transport", "http").
Msg("Failed to initialize server")
if err != nil {
logger.Error().Err(err).Str("transport", "http").Msg("Failed to initialize server")
return err
}
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", server))
} else {
logger.Info().Msg("HTTP server disabled, not starting HTTP service")
}
if !cfg.Events.Disabled {
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
evStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig{
Endpoint: cfg.Events.Endpoint,
Cluster: cfg.Events.Cluster,
EnableTLS: cfg.Events.EnableTLS,
TLSInsecure: cfg.Events.TLSInsecure,
TLSRootCACertificate: cfg.Events.TLSRootCACertificate,
AuthUsername: cfg.Events.AuthUsername,
AuthPassword: cfg.Events.AuthPassword,
})
if err != nil {
logger.Error().Err(err).Msg("Failed to initialize event stream")
return err
}
eventSvc, err := svcEvents.New(
activityLog,
evStream,
svcEvents.Context(ctx),
svcEvents.Logger(logger),
svcEvents.ServiceAccount(cfg.ServiceAccount),
svcEvents.GatewaySelector(gatewaySelector),
svcEvents.RegisteredEvents(_registeredEvents),
svcEvents.NumConsumers(cfg.NumConsumers),
)
if err != nil {
logger.Error().Err(err).Str("transport", "event").Msg("Failed to initialize server")
return err
}
gr.Add(runner.New(cfg.Service.Name+".svc", func() error {
return eventSvc.Run()
}, func() {
eventSvc.Close()
}))
} else {
logger.Info().Msg("event listening disabled, not starting event service")
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", svc))
}
{
@@ -221,33 +149,3 @@ func Server(cfg *config.Config) *cobra.Command {
},
}
}
func ConnectNatsKV(cfg config.Store) (nats.KeyValue, error) {
// Connect to NATS servers
secureOption := natspkg.Secure(cfg.EnableTLS, cfg.TLSInsecure, cfg.TLSRootCACertificate)
conn, err := nats.Connect(strings.Join(cfg.Nodes, ","), secureOption, nats.UserInfo(cfg.AuthUsername, cfg.AuthPassword))
if err != nil {
return nil, err
}
js, err := conn.JetStream()
if err != nil {
return nil, err
}
kv, err := js.KeyValue(cfg.Database)
if err != nil {
if !errors.Is(err, nats.ErrBucketNotFound) {
return nil, errors.Wrapf(err, "Failed to get bucket (%s)", cfg.Database)
}
kv, err = js.CreateKeyValue(&nats.KeyValueConfig{
Bucket: cfg.Database,
})
if err != nil {
return nil, errors.Wrapf(err, "Failed to create bucket (%s)", cfg.Database)
}
}
return kv, nil
}
@@ -35,12 +35,10 @@ type Config struct {
WriteBufferDuration time.Duration `yaml:"write_buffer_duration" env:"ACTIVITYLOG_WRITE_BUFFER_DURATION" desc:"The duration to wait before flushing the write buffer. This is used to reduce the number of writes to the store." introductionVersion:"4.0.0"`
MaxActivities int `yaml:"max_activities" env:"ACTIVITYLOG_MAX_ACTIVITIES" desc:"The maximum number of activities to keep in the store per resource. If the number of activities exceeds this value, the oldest activities will be removed." introductionVersion:"4.0.0"`
NumConsumers int `yaml:"num_consumers" env:"ACTIVITYLOG_NUM_CONSUMERS" desc:"The amount of concurrent event consumers to start. Event consumers are used for updating the list of activities. Multiple consumers increase parallelisation, but will also increase CPU and memory demands." introductionVersion:"%NEXT%"`
}
// Events combines the configuration options for the event bus.
type Events struct {
Disabled bool `yaml:"disabled" env:"ACTIVITYLOG_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%NEXT%"`
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
@@ -79,7 +77,6 @@ type CORS struct {
// HTTP defines the available http configuration.
type HTTP struct {
Disabled bool `yaml:"disabled" env:"ACTIVITYLOG_HTTP_DISABLED" desc:"Disables the HTTP service. Set this to true if the service should only handle events." introductionVersion:"1.0.0"`
Addr string `yaml:"addr" env:"ACTIVITYLOG_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
Namespace string `yaml:"-"`
Root string `yaml:"root" env:"ACTIVITYLOG_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
@@ -52,9 +52,7 @@ func DefaultConfig() *config.Config {
},
},
WriteBufferDuration: 10 * time.Second,
// Nats runs into max payload exceeded errors at around 7k activities. Let's keep a buffer.
MaxActivities: 6000,
NumConsumers: 1,
MaxActivities: 6000,
}
}
+71 -23
View File
@@ -3,12 +3,18 @@ package http
import (
"context"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/log"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/metrics"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/spf13/pflag"
"go-micro.dev/v4/store"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/noop"
)
// Option defines a single option function.
@@ -16,15 +22,19 @@ type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Name string
Namespace string
Logger log.Logger
Context context.Context
Config *config.Config
Flags []pflag.Flag
Service ActivityLogService
TraceProvider trace.TracerProvider
ValueClient settingssvc.ValueService
Logger log.Logger
Context context.Context
Config *config.Config
Metrics *metrics.Metrics
Flags []pflag.Flag
Namespace string
Store store.Store
Stream events.Stream
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
TraceProvider trace.TracerProvider
HistoryClient ehsvc.EventHistoryService
ValueClient settingssvc.ValueService
RegisteredEvents []events.Unmarshaller
}
// newOptions initializes the available default options.
@@ -59,10 +69,10 @@ func Config(val *config.Config) Option {
}
}
// Service provides a function to set the service option.
func Service(val ActivityLogService) Option {
// Metrics provides a function to set the metrics option.
func Metrics(val *metrics.Metrics) Option {
return func(o *Options) {
o.Service = val
o.Metrics = val
}
}
@@ -73,20 +83,58 @@ func Flags(flags ...pflag.Flag) Option {
}
}
// TraceProvider provides a function to configure the trace provider
func TraceProvider(traceProvider trace.TracerProvider) Option {
// Namespace provides a function to set the Namespace option.
func Namespace(val string) Option {
return func(o *Options) {
if traceProvider != nil {
o.TraceProvider = traceProvider
} else {
o.TraceProvider = noop.NewTracerProvider()
}
o.Namespace = val
}
}
// ValueClient adds a grpc client for the value service
func ValueClient(vs settingssvc.ValueService) Option {
// Store provides a function to configure the store
func Store(store store.Store) Option {
return func(o *Options) {
o.ValueClient = vs
o.Store = store
}
}
// Stream provides a function to configure the stream
func Stream(stream events.Stream) Option {
return func(o *Options) {
o.Stream = stream
}
}
// GatewaySelector provides a function to configure the gateway client selector
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
return func(o *Options) {
o.GatewaySelector = gatewaySelector
}
}
// HistoryClient provides a function to configure the event history client
func HistoryClient(h ehsvc.EventHistoryService) Option {
return func(o *Options) {
o.HistoryClient = h
}
}
// RegisteredEvents provides a function to register events
func RegisteredEvents(evs []events.Unmarshaller) Option {
return func(o *Options) {
o.RegisteredEvents = evs
}
}
// TraceProvider provides a function to set the TracerProvider option
func TraceProvider(val trace.TracerProvider) Option {
return func(o *Options) {
o.TraceProvider = val
}
}
// ValueClient provides a function to set the ValueClient options
func ValueClient(val settingssvc.ValueService) Option {
return func(o *Options) {
o.ValueClient = val
}
}
+46 -103
View File
@@ -1,63 +1,49 @@
package http
import (
"context"
"embed"
"encoding/json"
"errors"
"net/http"
"fmt"
stdhttp "net/http"
"github.com/go-chi/chi/v5"
chimiddleware "github.com/go-chi/chi/v5/middleware"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/pkg/account"
"github.com/opencloud-eu/opencloud/pkg/cors"
"github.com/opencloud-eu/opencloud/pkg/l10n"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/pkg/middleware"
ohttp "github.com/opencloud-eu/opencloud/pkg/service/http"
"github.com/opencloud-eu/opencloud/pkg/service/http"
"github.com/opencloud-eu/opencloud/pkg/tracing"
"github.com/opencloud-eu/opencloud/pkg/version"
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
activityloghttp "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
svc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"
"github.com/riandyrn/otelchi"
"go-micro.dev/v4"
"google.golang.org/grpc/metadata"
)
var (
//go:embed l10n/locale
_localeFS embed.FS
// subfolder where the translation files are stored
_localeSubPath = "l10n/locale"
// domain of the activitylog service (transifex)
_domain = "activitylog"
)
// Service is the service interface
type Service any
// Server initializes the http service and server.
func Server(opts ...Option) (ohttp.Service, error) {
func Server(opts ...Option) (http.Service, error) {
options := newOptions(opts...)
service := options.Service
newService, err := ohttp.NewService(
ohttp.TLSConfig(options.Config.HTTP.TLS),
ohttp.Logger(options.Logger),
ohttp.Namespace(options.Config.HTTP.Namespace),
ohttp.Name(options.Config.Service.Name),
ohttp.Version(version.GetString()),
ohttp.Address(options.Config.HTTP.Addr),
ohttp.Context(options.Context),
ohttp.Flags(options.Flags...),
service, err := http.NewService(
http.TLSConfig(options.Config.HTTP.TLS),
http.Logger(options.Logger),
http.Namespace(options.Config.HTTP.Namespace),
http.Name(options.Config.Service.Name),
http.Version(version.GetString()),
http.Address(options.Config.HTTP.Addr),
http.Context(options.Context),
http.Flags(options.Flags...),
http.TraceProvider(options.TraceProvider),
)
if err != nil {
options.Logger.Error().
Err(err).
Msg("Error initializing http service")
return ohttp.Service{}, err
return http.Service{}, fmt.Errorf("could not initialize http service: %w", err)
}
middlewares := []func(http.Handler) http.Handler{
middlewares := []func(stdhttp.Handler) stdhttp.Handler{
chimiddleware.RequestID,
middleware.Version(
options.Config.Service.Name,
@@ -66,7 +52,6 @@ func Server(opts ...Option) (ohttp.Service, error) {
middleware.Logger(
options.Logger,
),
middleware.TraceContext,
middleware.ExtractAccountUUID(
account.Logger(options.Logger),
account.JWTSecret(options.Config.TokenManager.JWTSecret),
@@ -83,75 +68,33 @@ func Server(opts ...Option) (ohttp.Service, error) {
mux := chi.NewMux()
mux.Use(middlewares...)
t := l10n.NewTranslatorFromCommonConfig(options.Config.DefaultLanguage, _domain, options.Config.TranslationPath, _localeFS, _localeSubPath)
mux.Route(options.Config.HTTP.Root, func(r chi.Router) {
r.Get("/graph/v1beta1/extensions/org.libregraph/activities", GetItemActivitiesHandler(options.Logger, service, options.ValueClient, t))
})
mux.Use(
otelchi.Middleware(
"actitivylog",
otelchi.WithChiRoutes(mux),
otelchi.WithTracerProvider(options.TraceProvider),
otelchi.WithPropagators(tracing.GetPropagator()),
),
)
err = micro.RegisterHandler(newService.Server(), mux)
handle, err := svc.New(
svc.Logger(options.Logger),
svc.Stream(options.Stream),
svc.Mux(mux),
svc.Config(options.Config),
svc.GatewaySelector(options.GatewaySelector),
svc.TraceProvider(options.TraceProvider),
svc.HistoryClient(options.HistoryClient),
svc.ValueClient(options.ValueClient),
svc.RegisteredEvents(options.RegisteredEvents),
)
if err != nil {
options.Logger.Fatal().Err(err).Msg("failed to register the handler")
return http.Service{}, err
}
newService.Init()
return newService, nil
}
// Service defines the business logic implementations need to provide.
type ActivityLogService interface {
GetItemActivities(ctx context.Context, query, loc string, t l10n.Translator) ([]libregraph.Activity, error)
}
// GetActivitiesResponse is the response on GET activities requests
type GetActivitiesResponse struct {
Activities []libregraph.Activity `json:"value"`
}
func GetItemActivitiesHandler(log log.Logger, s ActivityLogService, vc settingssvc.ValueService, t l10n.Translator) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, r.Header.Get(revactx.TokenHeader))
activeUser, ok := revactx.ContextGetUser(ctx)
if !ok {
w.WriteHeader(http.StatusUnauthorized)
return
}
loc := l10n.MustGetUserLocale(ctx, activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), vc)
activities, err := s.GetItemActivities(ctx, r.URL.Query().Get("kql"), loc, t)
if err != nil {
switch {
case errors.Is(err, activityloghttp.ErrBadRequest):
log.Debug().Str("query", r.URL.Query().Get("kql")).Err(err).Msg("error getting activities")
w.WriteHeader(http.StatusBadRequest)
return
case errors.Is(err, activityloghttp.ErrForbidden):
log.Debug().Err(err).Msg("error getting activities")
w.WriteHeader(http.StatusForbidden)
return
default:
log.Error().Err(err).Msg("error getting activities")
w.WriteHeader(http.StatusInternalServerError)
return
}
}
res := GetActivitiesResponse{
Activities: activities,
}
w.Header().Set("Content-Type", "application/json; odata.metadata=minimal")
w.Header().Set("OData-Version", "4.0")
if reqID := chimiddleware.GetReqID(ctx); reqID != "" {
w.Header().Set("request-id", reqID)
}
w.Header().Set("Cache-Control", "no-cache")
if err := json.NewEncoder(w).Encode(res); err != nil {
log.Error().Err(err).Msg("error encoding activities")
w.WriteHeader(http.StatusInternalServerError)
return
}
if err := micro.RegisterHandler(service.Server(), handle); err != nil {
return http.Service{}, err
}
return service, nil
}
@@ -1,366 +0,0 @@
package activitylog
import (
"context"
"encoding/base32"
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/jellydator/ttlcache/v2"
"github.com/nats-io/nats.go"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/vmihailenco/msgpack/v5"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
var tracer trace.Tracer
func init() {
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog")
}
var (
_maxActivitiesDefault = 6000
_writeBufferDuration = 10 * time.Second
)
// Activitylog stores and retrieves activities for resources and their parents from a nats kv
type ActivityLog struct {
log log.Logger
// FIXME the lock does not protect agains concurrent resource activities on multiple instances
// known since https://github.com/owncloud/ocis/pull/9361#pullrequestreview-2135350157
// current ocis discussion in https://github.com/owncloud/ocis/issues/12475
lock sync.RWMutex
debouncer *Debouncer
parentIdCache *ttlcache.Cache
natskv nats.KeyValue
maxActivities int
}
type batchInfo struct {
key string
count int
timestamp time.Time
}
// New creates a new ActivitylogService
func New(kv nats.KeyValue, opts ...Option) (*ActivityLog, error) {
o := &Options{
MaxActivities: _maxActivitiesDefault,
WriteBufferDuration: _writeBufferDuration,
Logger: log.NopLogger(),
}
for _, opt := range opts {
opt(o)
}
cache := ttlcache.NewCache()
err := cache.SetTTL(30 * time.Second)
if err != nil {
return nil, err
}
s := &ActivityLog{
log: o.Logger,
lock: sync.RWMutex{},
parentIdCache: cache,
maxActivities: o.MaxActivities,
natskv: kv,
}
s.debouncer = NewDebouncer(o.WriteBufferDuration, s.StoreActivity)
// run migrations
err = s.runMigrations(context.Background(), kv)
if err != nil {
return nil, err
}
return s, nil
}
// RemoveResource removes the resource from the store
func (a *ActivityLog) RemoveResource(rid *provider.ResourceId) error {
if rid == nil {
return fmt.Errorf("resource id is required")
}
a.lock.Lock()
defer a.lock.Unlock()
err := a.natskv.Delete(storagespace.FormatResourceID(rid))
if err != nil {
return fmt.Errorf("could not delete resource %s: %w", rid.OpaqueId, err)
}
return nil
}
// RawActivity represents an activity as it is stored in the activitylog store
type RawActivity struct {
EventID string `json:"event_id"`
Depth int `json:"depth"`
Timestamp time.Time `json:"timestamp"`
}
func (a *ActivityLog) AddActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time, getResource func(context.Context, *provider.Reference) (*provider.ResourceInfo, error)) error {
var (
err error
depth int
ref = initRef
)
ctx, span := tracer.Start(ctx, "AddActivity")
defer span.End()
for {
var info *provider.ResourceInfo
id := ref.GetResourceId()
if ref.Path != "" {
// Path based reference, we need to resolve the resource id
ctx, span = tracer.Start(ctx, "AddActivity.getResource")
info, err = getResource(ctx, ref)
span.End()
if err != nil {
// TODO If the resource was deleted should we still log an activity in the parent?
return fmt.Errorf("could not get resource info for reference %v: %w", ref, err)
}
id = info.GetId()
}
if id == nil {
return fmt.Errorf("resource id is required")
}
key := storagespace.FormatResourceID(id)
a.debouncer.Debounce(key, RawActivity{
EventID: eventID,
Depth: depth,
Timestamp: timestamp,
})
if id.OpaqueId == id.SpaceId {
// we are at the root of the space, no need to go further
break
}
// check if parent id is cached
// parent id is cached in the format <storageid>$<spaceid>!<resourceid>
// if it is not cached, get the resource info and cache it
if parentId == nil {
if v, err := a.parentIdCache.Get(key); err != nil {
if info == nil {
ctx, span := tracer.Start(ctx, "AddActivity.getResource parent")
info, err = getResource(ctx, ref)
span.End()
if err != nil || info.GetParentId() == nil || info.GetParentId().GetOpaqueId() == "" {
return fmt.Errorf("could not get parent id: %w", err)
}
}
parentId = info.GetParentId()
a.parentIdCache.Set(key, parentId)
} else {
parentId = v.(*provider.ResourceId)
}
} else {
a.log.Debug().Msg("parent id is cached")
}
depth++
ref = &provider.Reference{ResourceId: parentId}
parentId = nil // reset parent id so it's not reused in the next iteration
}
return nil
}
func (a *ActivityLog) StoreActivity(resourceID string, activities []RawActivity) error {
a.lock.Lock()
defer a.lock.Unlock()
ctx, span := tracer.Start(context.Background(), "storeActivity")
defer span.End()
_, subspan := tracer.Start(ctx, "storeActivity.Marshal")
b, err := msgpack.Marshal(activities)
if err != nil {
return err
}
subspan.End()
_, subspan = tracer.Start(ctx, "storeActivity.natskv.Put")
key := natsKey(resourceID, len(activities))
_, err = a.natskv.Put(key, b)
if err != nil {
return err
}
subspan.End()
ctx, subspan = tracer.Start(ctx, "storeActivity.enforceMaxActivities")
a.enforceMaxActivities(ctx, resourceID)
subspan.End()
return nil
}
func (a *ActivityLog) enforceMaxActivities(ctx context.Context, resourceID string) {
if a.maxActivities <= 0 {
return
}
key := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
_, subspan := tracer.Start(ctx, "enforceMaxActivities.watch")
watcher, err := a.natskv.Watch(key, nats.IgnoreDeletes())
if err != nil {
a.log.Error().Err(err).Str("resourceID", resourceID).Msg("could not watch")
return
}
defer watcher.Stop()
var keys []string
for update := range watcher.Updates() {
if update == nil {
break
}
var batchActivities []RawActivity
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
}
keys = append(keys, update.Key())
}
subspan.End()
_, subspan = tracer.Start(ctx, "enforceMaxActivities.compile")
// Parse keys into batches
batches := make([]batchInfo, 0)
var activitiesCount int
for _, k := range keys {
parts := strings.SplitN(k, ".", 3)
if len(parts) < 3 {
a.log.Warn().Str("key", k).Msg("skipping key, not enough parts")
continue
}
c, err := strconv.Atoi(parts[1])
if err != nil {
a.log.Warn().Str("key", k).Msg("skipping key, can not parse count")
continue
}
// parse timestamp
nano, err := strconv.ParseInt(parts[2], 10, 64)
if err != nil {
a.log.Warn().Str("key", k).Msg("skipping key, can not parse timestamp")
continue
}
batches = append(batches, batchInfo{
key: k,
count: c,
timestamp: time.Unix(0, nano),
})
activitiesCount += c
}
// sort batches by timestamp
sort.Slice(batches, func(i, j int) bool {
return batches[i].timestamp.Before(batches[j].timestamp)
})
subspan.End()
_, subspan = tracer.Start(ctx, "enforceMaxActivities.delete")
// remove oldest keys until we are at max activities
for _, b := range batches {
if activitiesCount-b.count < a.maxActivities {
break
}
activitiesCount -= b.count
err = a.natskv.Delete(b.key)
if err != nil {
a.log.Error().Err(err).Str("key", b.key).Msg("could not delete key")
break
}
}
subspan.End()
}
func (a *ActivityLog) InvalidateCachedParentID(purgeId *provider.ResourceId) {
// The parent id cache is populated lazily and its entries expire, so a
// missing key is the expected case rather than an error.
if err := a.parentIdCache.Remove(storagespace.FormatResourceID(purgeId)); err != nil {
a.log.Debug().Interface("event", purgeId).Err(err).Msg("could not delete parent id cache")
}
}
func natsKey(resourceID string, activitiesCount int) string {
return fmt.Sprintf("%s.%d.%d",
base32.StdEncoding.EncodeToString([]byte(resourceID)),
activitiesCount,
time.Now().UnixNano())
}
func (a *ActivityLog) Activities(rid *provider.ResourceId) ([]RawActivity, error) {
a.lock.RLock()
defer a.lock.RUnlock()
return a.activities(rid)
}
func (a *ActivityLog) activities(rid *provider.ResourceId) ([]RawActivity, error) {
resourceID := storagespace.FormatResourceID(rid)
glob := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
watcher, err := a.natskv.Watch(glob, nats.IgnoreDeletes())
if err != nil {
return nil, err
}
defer watcher.Stop()
var activities []RawActivity
for update := range watcher.Updates() {
if update == nil {
break
}
var batchActivities []RawActivity
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack")
}
activities = append(activities, batchActivities...)
}
return activities, nil
}
// RemoveActivities removes the activities from the given resource
func (a *ActivityLog) RemoveActivities(rid *provider.ResourceId, toDelete map[string]struct{}) error {
a.lock.Lock()
defer a.lock.Unlock()
curActivities, err := a.activities(rid)
if err != nil {
return err
}
var acts []RawActivity
for _, a := range curActivities {
if _, ok := toDelete[a.EventID]; !ok {
acts = append(acts, a)
}
}
b, err := json.Marshal(acts)
if err != nil {
return err
}
_, err = a.natskv.Put(storagespace.FormatResourceID(rid), b)
return err
}
@@ -1,13 +0,0 @@
package activitylog_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestActivitylog(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Activitylog Suite")
}
@@ -1,76 +0,0 @@
package activitylog
import (
"sync"
"time"
)
// Debouncer is used to debounce writes to the activity log store.
type Debouncer struct {
after time.Duration
f func(id string, ra []RawActivity) error
pending sync.Map
inProgress sync.Map
mutex sync.Mutex
}
type queueItem struct {
activities []RawActivity
timer *time.Timer
}
// NewDebouncer returns a new Debouncer instance.
func NewDebouncer(d time.Duration, f func(id string, ra []RawActivity) error) *Debouncer {
return &Debouncer{
after: d,
f: f,
pending: sync.Map{},
inProgress: sync.Map{},
}
}
// Debounce restarts the debounce timer for the given space.
func (d *Debouncer) Debounce(id string, ra RawActivity) {
if d.after == 0 {
d.f(id, []RawActivity{ra})
return
}
d.mutex.Lock()
defer d.mutex.Unlock()
item := &queueItem{
activities: []RawActivity{ra},
}
if i, ok := d.pending.Load(id); ok {
// if the item is already in the queue, append the new activities
item, ok = i.(*queueItem)
if ok {
item.activities = append(item.activities, ra)
}
}
if item.timer == nil {
item.timer = time.AfterFunc(d.after, func() {
if _, ok := d.inProgress.Load(id); ok {
// Reschedule this run for when the previous run has finished
d.mutex.Lock()
if i, ok := d.pending.Load(id); ok {
i.(*queueItem).timer.Reset(d.after)
}
d.mutex.Unlock()
return
}
d.pending.Delete(id)
d.inProgress.Store(id, true)
defer d.inProgress.Delete(id)
d.f(id, item.activities)
})
}
d.pending.Store(id, item)
}
@@ -1,121 +0,0 @@
package activitylog_test
import (
"sync"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
)
var _ = Describe("Debouncer", func() {
var (
mu sync.Mutex
callbacks []activitylog.RawActivity
newCallback func(id string, ra []activitylog.RawActivity) error
)
BeforeEach(func() {
mu.Lock()
callbacks = nil
mu.Unlock()
newCallback = func(id string, ra []activitylog.RawActivity) error {
mu.Lock()
defer mu.Unlock()
callbacks = append(callbacks, ra...)
return nil
}
})
Context("with zero duration", func() {
It("calls the callback immediately", func() {
d := activitylog.NewDebouncer(0, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
Expect(callbacks).To(HaveLen(1))
Expect(callbacks[0].EventID).To(Equal("activity1"))
})
It("calls the callback immediately for each event", func() {
d := activitylog.NewDebouncer(0, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
d.Debounce("space2", activitylog.RawActivity{EventID: "activity2"})
Expect(callbacks).To(HaveLen(2))
})
})
Context("with non-zero duration", func() {
It("batches activities with the same id", func() {
d := activitylog.NewDebouncer(10*time.Millisecond, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
d.Debounce("space1", activitylog.RawActivity{EventID: "activity3"})
Eventually(func() int {
mu.Lock()
defer mu.Unlock()
return len(callbacks)
}).Should(Equal(3))
})
It("handles different ids independently", func() {
d := activitylog.NewDebouncer(10*time.Millisecond, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
d.Debounce("space2", activitylog.RawActivity{EventID: "activity2"})
Eventually(func() int {
mu.Lock()
defer mu.Unlock()
return len(callbacks)
}).Should(Equal(2))
})
It("batches activities that arrive within the debounce window", func() {
d := activitylog.NewDebouncer(100*time.Millisecond, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
time.Sleep(20 * time.Millisecond)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
time.Sleep(20 * time.Millisecond)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity3"})
Eventually(func() int {
mu.Lock()
defer mu.Unlock()
return len(callbacks)
}).Should(Equal(3))
})
It("processes new batch after previous completes", func() {
d := activitylog.NewDebouncer(5*time.Millisecond, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
time.Sleep(20 * time.Millisecond) // let first batch complete
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
Eventually(func() int {
mu.Lock()
defer mu.Unlock()
return len(callbacks)
}).Should(Equal(2))
})
It("skips duplicate write when timer fires during in-progress callback", func() {
slowCallback := func(id string, ra []activitylog.RawActivity) error {
time.Sleep(50 * time.Millisecond) // simulate slow write
mu.Lock()
defer mu.Unlock()
callbacks = append(callbacks, ra...)
return nil
}
d := activitylog.NewDebouncer(10*time.Millisecond, slowCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
time.Sleep(20 * time.Millisecond) // timer fires while callback is running
Eventually(func() int {
mu.Lock()
defer mu.Unlock()
return len(callbacks)
}).Should(Equal(1))
})
})
})
@@ -1,35 +0,0 @@
package activitylog
import (
"time"
"github.com/opencloud-eu/opencloud/pkg/log"
)
// Option for the activitylog service
type Option func(*Options)
// Options for the activitylog service
type Options struct {
Logger log.Logger
MaxActivities int
WriteBufferDuration time.Duration
}
// Logger configures a logger for the activitylog service
func Logger(log log.Logger) Option {
return func(o *Options) {
o.Logger = log
}
}
func MaxActivities(max int) Option {
return func(o *Options) {
o.MaxActivities = max
}
}
func WriteBufferDuration(d time.Duration) Option {
return func(o *Options) {
o.WriteBufferDuration = d
}
}
@@ -1,302 +0,0 @@
package events
import (
"context"
"fmt"
"path/filepath"
"sync"
"sync/atomic"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
var tracer trace.Tracer
func init() {
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events")
}
var (
_numConsumersDefault = 1
)
// ActivitylogService logs events per resource
type ActivitylogService struct {
ctx context.Context
sa config.ServiceAccount
log log.Logger
stream events.Stream
gws pool.Selectable[gateway.GatewayAPIClient]
al *activitylog.ActivityLog
numConsumers int
events []events.Unmarshaller
stopCh chan struct{}
stopped *atomic.Bool
}
// New creates a new ActivitylogService
func New(al *activitylog.ActivityLog, stream events.Stream, opts ...Option) (*ActivitylogService, error) {
o := &Options{
NumConsumers: _numConsumersDefault,
}
for _, opt := range opts {
opt(o)
}
s := &ActivitylogService{
ctx: o.Context,
log: o.Logger,
sa: o.ServiceAccount,
stream: stream,
gws: o.GatewaySelector,
events: o.RegisteredEvents,
numConsumers: o.NumConsumers,
al: al,
stopCh: make(chan struct{}, 1),
stopped: new(atomic.Bool),
}
return s, nil
}
// Run to fulfil Runner interface
func (s *ActivitylogService) Run() error {
ch, err := events.Consume(s.stream, "activitylog", s.events...)
if err != nil {
return err
}
var wg sync.WaitGroup
ctx, cancel := context.WithCancel(s.ctx)
defer cancel()
s.log.Debug().Int("worker.count", s.numConsumers).
Str("messaging.consumer.group.name", "activitylog").
Str("messaging.system", "nats").
Str("messaging.operation.name", "receive").
Msg("starting event processing workers")
// start workers
for i := 0; i < s.numConsumers; i++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
for {
select {
case <-ctx.Done():
return
case e, ok := <-ch:
if !ok {
return
}
if err := s.processEvent(e); err != nil {
s.log.Error().Err(err).
Int("worker", workerID).
Interface("event", e).
Msg("failed to process event")
}
}
}
}(i)
}
// wait for stop signal
<-s.stopCh
cancel() // signal workers to stop
wg.Wait()
return nil
}
// Close will make the service to stop processing, so the `Run`
// method can finish.
func (s *ActivitylogService) Close() {
if s.stopped.CompareAndSwap(false, true) {
close(s.stopCh)
}
}
func (s *ActivitylogService) processEvent(e events.Event) error {
ctx := e.GetTraceContext(s.ctx)
ctx, span := tracer.Start(ctx, "processEvent")
defer span.End()
s.log.Debug().Interface("event", e).Msg("updating activitylog")
switch ev := e.Event.(type) {
case events.UploadReady:
return s.AddActivity(ctx, ev.FileRef, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
case events.FileTouched:
return s.AddActivity(ctx, ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
// Disabled https://github.com/owncloud/ocis/issues/10293
//case events.FileDownloaded:
// we are only interested in public link downloads - so no need to store others.
//if ev.ImpersonatingUser.GetDisplayName() == "Public" {
// err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
//}
case events.ContainerCreated:
return s.AddActivity(ctx, ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemTrashed:
return s.AddActivityTrashed(ctx, ev.ID, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemPurged:
return s.al.RemoveResource(ev.ID)
case events.ItemMoved:
// remove the cached parent id for this resource
s.removeCachedParentID(ctx, ev.Ref)
return s.AddActivity(ctx, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.ShareCreated:
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
case events.ShareUpdated:
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
}
case events.ShareRemoved:
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, ev.Timestamp)
case events.LinkCreated:
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
case events.LinkUpdated:
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
}
case events.LinkRemoved:
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.SpaceShared:
return s.AddSpaceActivity(ctx, ev.ID, e.ID, ev.Timestamp)
case events.SpaceUnshared:
return s.AddSpaceActivity(ctx, ev.ID, e.ID, ev.Timestamp)
}
return nil
}
// AddActivity adds the activity to the given resource and all its parents
func (a *ActivitylogService) AddActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
ctx, span := tracer.Start(ctx, "AddActivity")
defer span.End()
gwc, err := a.gws.Next()
if err != nil {
return fmt.Errorf("cant get gateway client: %w", err)
}
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("cant get service user context: %w", err)
}
return a.al.AddActivity(ctx, initRef, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
return utils.GetResource(ctx, ref, gwc)
})
}
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
func (a *ActivitylogService) AddActivityTrashed(ctx context.Context, resourceID *provider.ResourceId, reference *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
ctx, span := tracer.Start(ctx, "AddActivityTrashed")
defer span.End()
gwc, err := a.gws.Next()
if err != nil {
return fmt.Errorf("cant get gateway client: %w", err)
}
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("cant get service user context: %w", err)
}
// store activity on trashed item
if err := a.al.StoreActivity(storagespace.FormatResourceID(resourceID), []activitylog.RawActivity{
{
EventID: eventID,
Depth: 0,
Timestamp: timestamp,
},
}); err != nil {
return fmt.Errorf("could not store activity: %w", err)
}
// get previous parent
ref := &provider.Reference{
ResourceId: reference.GetResourceId(),
Path: filepath.Dir(reference.GetPath()),
}
return a.al.AddActivity(ctx, ref, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
return utils.GetResource(ctx, ref, gwc)
})
}
// AddSpaceActivity adds the activity to the given spaceroot
func (a *ActivitylogService) AddSpaceActivity(ctx context.Context, spaceID *provider.StorageSpaceId, eventID string, timestamp time.Time) error {
_, span := tracer.Start(ctx, "AddSpaceActivity")
defer span.End()
// spaceID is in format <providerid>$<spaceid>
// activitylog service uses format <providerid>$<spaceid>!<resourceid>
// lets do some converting, shall we?
rid, err := storagespace.ParseID(spaceID.GetOpaqueId())
if err != nil {
return fmt.Errorf("could not parse space id: %w", err)
}
rid.OpaqueId = rid.GetSpaceId()
err = a.al.StoreActivity(storagespace.FormatResourceID(&rid), []activitylog.RawActivity{
{
EventID: eventID,
Depth: 0,
Timestamp: timestamp,
},
})
if err != nil {
return fmt.Errorf("could not store activity: %w", err)
}
return nil
}
func toRef(r *provider.ResourceId) *provider.Reference {
return &provider.Reference{
ResourceId: r,
}
}
func (a *ActivitylogService) removeCachedParentID(ctx context.Context, ref *provider.Reference) {
var span trace.Span
ctx, span = tracer.Start(ctx, "removeCachedParentID")
defer span.End()
purgeId := ref.GetResourceId()
if ref.GetPath() != "" {
gwc, err := a.gws.Next()
if err != nil {
a.log.Error().Err(err).Msg("could not get gateway client")
return
}
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
if err != nil {
a.log.Error().Err(err).Msg("could not get service user context")
return
}
info, err := utils.GetResource(ctx, ref, gwc)
if err != nil {
a.log.Error().Err(err).Msg("could not get resource info")
return
}
purgeId = info.GetId()
}
a.al.InvalidateCachedParentID(purgeId)
}
@@ -1,123 +0,0 @@
package events_test
import (
"context"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
eventssvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
var _ = Describe("ActivitylogService", func() {
Describe("New", func() {
var (
al *activitylog.ActivityLog
stream events.Stream
)
BeforeEach(func() {
al = &activitylog.ActivityLog{}
stream = nil
})
It("creates a service with minimal options", func() {
svc, err := eventssvc.New(al, stream)
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with context option", func() {
ctx := context.Background()
svc, err := eventssvc.New(al, stream, eventssvc.Context(ctx))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with logger option", func() {
logger := log.NopLogger()
svc, err := eventssvc.New(al, stream, eventssvc.Logger(logger))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with service account option", func() {
sa := config.ServiceAccount{
ServiceAccountID: "sa-id",
ServiceAccountSecret: "sa-secret",
}
svc, err := eventssvc.New(al, stream, eventssvc.ServiceAccount(sa))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with registered events", func() {
evts := []events.Unmarshaller{&events.UploadReady{}, &events.FileTouched{}}
svc, err := eventssvc.New(al, stream, eventssvc.RegisteredEvents(evts))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with gateway selector", func() {
var gs pool.Selectable[gateway.GatewayAPIClient]
svc, err := eventssvc.New(al, stream, eventssvc.GatewaySelector(gs))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with num consumers option", func() {
svc, err := eventssvc.New(al, stream, eventssvc.NumConsumers(5))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with all options", func() {
ctx := context.Background()
logger := log.NopLogger()
sa := config.ServiceAccount{
ServiceAccountID: "sa-id",
ServiceAccountSecret: "sa-secret",
}
evts := []events.Unmarshaller{&events.UploadReady{}, &events.ContainerCreated{}}
var gs pool.Selectable[gateway.GatewayAPIClient]
svc, err := eventssvc.New(
al,
stream,
eventssvc.Context(ctx),
eventssvc.Logger(logger),
eventssvc.ServiceAccount(sa),
eventssvc.RegisteredEvents(evts),
eventssvc.GatewaySelector(gs),
eventssvc.NumConsumers(3),
)
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
})
Describe("Close", func() {
It("can be called without panic on a new service", func() {
al := &activitylog.ActivityLog{}
svc, err := eventssvc.New(al, nil)
Expect(err).ToNot(HaveOccurred())
Expect(func() { svc.Close() }).ToNot(Panic())
})
It("can be called multiple times without panic", func() {
al := &activitylog.ActivityLog{}
svc, err := eventssvc.New(al, nil)
Expect(err).ToNot(HaveOccurred())
svc.Close()
svc.Close()
svc.Close()
})
})
})
@@ -1,97 +1,89 @@
package http
package service
import (
"context"
"embed"
"encoding/json"
"errors"
"net/http"
"path/filepath"
"reflect"
"slices"
"strconv"
"strings"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/olekukonko/errors"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"google.golang.org/grpc/metadata"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/pkg/l10n"
"github.com/opencloud-eu/opencloud/pkg/log"
ehmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/eventhistory/v0"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
var tracer trace.Tracer
func init() {
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http")
}
// New returns a new instance of Service
func New(al *activitylog.ActivityLog, opts ...Option) (*svc, error) {
o := newOptions(opts...)
registeredEvents := make(map[string]events.Unmarshaller)
for _, e := range o.RegisteredEvents {
typ := reflect.TypeOf(e)
registeredEvents[typ.String()] = e
}
return &svc{
log: o.Logger,
evHistory: o.HistoryClient,
al: al,
registeredEvents: registeredEvents,
gws: o.GatewaySelector,
}, nil
}
type svc struct {
log log.Logger
evHistory ehsvc.EventHistoryService
gws pool.Selectable[gateway.GatewayAPIClient]
al *activitylog.ActivityLog
registeredEvents map[string]events.Unmarshaller
}
var (
ErrBadRequest = errors.New("bad request")
ErrForbidden = errors.New("forbidden")
//go:embed l10n/locale
_localeFS embed.FS
// subfolder where the translation files are stored
_localeSubPath = "l10n/locale"
// domain of the activitylog service (transifex)
_domain = "activitylog"
)
func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.Translator) ([]libregraph.Activity, error) {
gwc, err := s.gws.Next()
if err != nil {
return nil, err
// ServeHTTP implements the http.Handler interface.
func (s *ActivitylogService) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.mux.ServeHTTP(w, r)
}
// HandleGetItemActivities handles the request to get the activities of an item.
func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, r.Header.Get(revactx.TokenHeader))
activeUser, ok := revactx.ContextGetUser(ctx)
if !ok {
w.WriteHeader(http.StatusUnauthorized)
return
}
rid, limit, rawActivityAccepted, activityAccepted, sort, err := s.getFilters(query)
gwc, err := s.gws.Next()
if err != nil {
s.log.Info().Str("query", query).Err(err).Msg("error getting filters")
return nil, ErrBadRequest
w.WriteHeader(http.StatusInternalServerError)
return
}
rid, limit, rawActivityAccepted, activityAccepted, sort, err := s.getFilters(r.URL.Query().Get("kql"))
if err != nil {
s.log.Info().Str("query", r.URL.Query().Get("kql")).Err(err).Msg("error getting filters")
_, _ = w.Write([]byte(err.Error()))
w.WriteHeader(http.StatusBadRequest)
return
}
info, err := utils.GetResourceByID(ctx, rid, gwc)
if err != nil {
return nil, ErrForbidden
w.WriteHeader(http.StatusForbidden)
return
}
// you need ListGrants to see activities
if !info.GetPermissionSet().GetListGrants() {
return nil, ErrForbidden
w.WriteHeader(http.StatusForbidden)
return
}
raw, err := s.al.Activities(rid)
raw, err := s.Activities(rid)
if err != nil {
s.log.Error().Err(err).Msg("error getting activities")
return nil, err
w.WriteHeader(http.StatusInternalServerError)
return
}
ids := make([]string, 0, len(raw))
@@ -104,21 +96,21 @@ func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.T
toDelete[a.EventID] = struct{}{}
}
evRes, err := s.evHistory.GetEvents(ctx, &ehsvc.GetEventsRequest{Ids: ids})
evRes, err := s.evHistory.GetEvents(r.Context(), &ehsvc.GetEventsRequest{Ids: ids})
if err != nil {
s.log.Error().Err(err).Msg("error getting events")
return nil, err
w.WriteHeader(http.StatusInternalServerError)
return
}
evs := evRes.GetEvents()
sort(evs)
// TODO cut the interface here?
activities := make([]libregraph.Activity, 0, len(evRes.GetEvents()))
resp := GetActivitiesResponse{Activities: make([]libregraph.Activity, 0, len(evRes.GetEvents()))}
for _, e := range evs {
delete(toDelete, e.GetId())
if limit > 0 && limit <= len(activities) {
if limit > 0 && limit <= len(resp.Activities) {
continue
}
@@ -132,6 +124,9 @@ func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.T
vars map[string]any
)
loc := l10n.MustGetUserLocale(r.Context(), activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), s.valService)
t := l10n.NewTranslatorFromCommonConfig(s.cfg.DefaultLanguage, _domain, s.cfg.TranslationPath, _localeFS, _localeSubPath)
switch ev := s.unwrapEvent(e).(type) {
case nil:
// error already logged in unwrapEvent
@@ -229,29 +224,35 @@ func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.T
continue
}
activities = append(activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars))
resp.Activities = append(resp.Activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars))
}
// delete activities in separate go routine
if len(toDelete) > 0 {
go func() {
err := s.al.RemoveActivities(rid, toDelete)
err := s.RemoveActivities(rid, toDelete)
if err != nil {
s.log.Error().Err(err).Msg("error removing activities")
}
}()
}
return activities, nil
}
func toRef(r *provider.ResourceId) *provider.Reference {
return &provider.Reference{
ResourceId: r,
b, err := json.Marshal(resp)
if err != nil {
s.log.Error().Err(err).Msg("error marshalling activities")
w.WriteHeader(http.StatusInternalServerError)
return
}
if _, err := w.Write(b); err != nil {
s.log.Error().Err(err).Msg("error writing response")
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
}
func (s *svc) unwrapEvent(e *ehmsg.Event) any {
func (s *ActivitylogService) unwrapEvent(e *ehmsg.Event) any {
etype, ok := s.registeredEvents[e.GetType()]
if !ok {
s.log.Error().Str("eventid", e.GetId()).Str("eventtype", e.GetType()).Msg("event not registered")
@@ -267,13 +268,13 @@ func (s *svc) unwrapEvent(e *ehmsg.Event) any {
return einterface
}
func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activitylog.RawActivity) bool, func(*ehmsg.Event) bool, func([]*ehmsg.Event), error) {
func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int, func(RawActivity) bool, func(*ehmsg.Event) bool, func([]*ehmsg.Event), error) {
qast, err := kql.Builder{}.Build(query)
if err != nil {
return nil, 0, nil, nil, nil, err
}
prefilters := make([]func(activitylog.RawActivity) bool, 0)
prefilters := make([]func(RawActivity) bool, 0)
postfilters := make([]func(*ehmsg.Event) bool, 0)
sortby := func(_ []*ehmsg.Event) {}
@@ -298,7 +299,7 @@ func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activity
break
}
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
prefilters = append(prefilters, func(a RawActivity) bool {
return a.Depth <= depth
})
case "limit":
@@ -321,11 +322,11 @@ func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activity
case *ast.DateTimeNode:
switch v.Operator.Value {
case "<", "<=":
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
prefilters = append(prefilters, func(a RawActivity) bool {
return a.Timestamp.Before(v.Value)
})
case ">", ">=":
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
prefilters = append(prefilters, func(a RawActivity) bool {
return a.Timestamp.After(v.Value)
})
}
@@ -344,7 +345,7 @@ func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activity
// space root requested - fix format
rid.OpaqueId = rid.GetSpaceId()
}
pref := func(a activitylog.RawActivity) bool {
pref := func(a RawActivity) bool {
for _, f := range prefilters {
if !f(a) {
return false
@@ -1,13 +0,0 @@
package http_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestHTTP(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "HTTP Suite")
}
@@ -1,59 +0,0 @@
package http
import (
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/log"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Logger log.Logger
RegisteredEvents []events.Unmarshaller
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
HistoryClient ehsvc.EventHistoryService
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// RegisteredEvents registers the events the service should listen to
func RegisteredEvents(e []events.Unmarshaller) Option {
return func(o *Options) {
o.RegisteredEvents = e
}
}
// GatewaySelector adds a grpc client selector for the gateway service
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
return func(o *Options) {
o.GatewaySelector = gatewaySelector
}
}
// HistoryClient adds a grpc client for the eventhistory service
func HistoryClient(hc ehsvc.EventHistoryService) Option {
return func(o *Options) {
o.HistoryClient = hc
}
}
@@ -1,150 +0,0 @@
package http_test
import (
"context"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/l10n"
httpsvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
)
var _ = Describe("Response", func() {
Describe("NewActivity", func() {
It("creates an activity with the given parameters", func() {
ts := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
vars := map[string]any{
"user": "testuser",
"resource": "testfile.txt",
}
act := httpsvc.NewActivity("Test message", ts, "event-123", vars)
Expect(act.Id).To(Equal("event-123"))
Expect(act.Times.RecordedTime).To(Equal(ts))
Expect(act.Template.Message).To(Equal("Test message"))
Expect(act.Template.Variables).To(HaveKeyWithValue("user", "testuser"))
Expect(act.Template.Variables).To(HaveKeyWithValue("resource", "testfile.txt"))
})
It("handles empty variables map", func() {
act := httpsvc.NewActivity("", time.Time{}, "", map[string]any{})
Expect(act.Id).To(BeEmpty())
Expect(act.Times.RecordedTime).To(Equal(time.Time{}))
Expect(act.Template.Message).To(BeEmpty())
Expect(act.Template.Variables).To(BeEmpty())
})
})
Describe("WithOldResource", func() {
It("sets the oldResource variable from reference path", func() {
ref := &provider.Reference{
Path: "/old/path/oldname.txt",
}
vars := make(map[string]any)
opt := httpsvc.WithOldResource(ref)
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("oldResource"))
})
})
Describe("WithUser", func() {
It("returns error when no user is provided", func() {
opt := httpsvc.WithUser(nil, nil, nil)
err := opt(context.Background(), nil, make(map[string]any))
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("no user provided"))
})
It("uses impersonator when provided", func() {
impersonator := &user.User{
Id: &user.UserId{
OpaqueId: "imp-user-id",
},
DisplayName: "Impersonated User",
}
vars := make(map[string]any)
opt := httpsvc.WithUser(nil, nil, impersonator)
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("user"))
})
It("uses executing user when no impersonator", func() {
execUser := &user.User{
Id: &user.UserId{
OpaqueId: "exec-user-id",
},
DisplayName: "Executing User",
}
vars := make(map[string]any)
opt := httpsvc.WithUser(nil, execUser, nil)
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("user"))
})
})
Describe("WithVar", func() {
It("sets a simple key-value variable", func() {
vars := make(map[string]any)
opt := httpsvc.WithVar("token", "id123", "My Token")
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("token"))
})
})
Describe("WithTranslation", func() {
It("sets translated field variable", func() {
var t l10n.Translator
vars := make(map[string]any)
opt := httpsvc.WithTranslation(&t, "en", "field", []string{"permission"})
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("field"))
})
It("handles empty values slice", func() {
var t l10n.Translator
vars := make(map[string]any)
opt := httpsvc.WithTranslation(&t, "en", "field", []string{})
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("field"))
})
})
Describe("ActivityOption type", func() {
It("allows composing multiple options", func() {
vars := make(map[string]any)
ctx := context.Background()
var gwc gateway.GatewayAPIClient
options := []httpsvc.ActivityOption{
httpsvc.WithVar("key1", "id1", "name1"),
httpsvc.WithVar("key2", "id2", "name2"),
}
for _, opt := range options {
err := opt(ctx, gwc, vars)
Expect(err).ToNot(HaveOccurred())
}
Expect(vars).To(HaveKey("key1"))
Expect(vars).To(HaveKey("key2"))
})
})
})
@@ -1,76 +0,0 @@
package http_test
import (
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/log"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
httpsvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
var _ = Describe("Service", func() {
Describe("New", func() {
var (
al *activitylog.ActivityLog
)
BeforeEach(func() {
al = &activitylog.ActivityLog{}
})
It("creates a service with default options", func() {
svc, err := httpsvc.New(al)
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with logger option", func() {
logger := log.NopLogger()
svc, err := httpsvc.New(al, httpsvc.Logger(logger))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with registered events", func() {
evts := []events.Unmarshaller{&events.UploadReady{}}
svc, err := httpsvc.New(al, httpsvc.RegisteredEvents(evts))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with gateway selector", func() {
var gs pool.Selectable[gateway.GatewayAPIClient]
svc, err := httpsvc.New(al, httpsvc.GatewaySelector(gs))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with history client", func() {
var hc ehsvc.EventHistoryService
svc, err := httpsvc.New(al, httpsvc.HistoryClient(hc))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with all options", func() {
logger := log.NopLogger()
evts := []events.Unmarshaller{&events.UploadReady{}, &events.FileTouched{}}
var gs pool.Selectable[gateway.GatewayAPIClient]
var hc ehsvc.EventHistoryService
svc, err := httpsvc.New(
al,
httpsvc.Logger(logger),
httpsvc.RegisteredEvents(evts),
httpsvc.GatewaySelector(gs),
httpsvc.HistoryClient(hc),
)
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
})
})
@@ -1,102 +0,0 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
# Translators:
# Roy-Magne Mo, 2026
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2026-09-06 23:15+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Roy-Magne Mo, 2026\n"
"Language-Team: Norwegian (https://app.transifex.com/opencloud-eu/teams/204053/no/)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Language: no\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
#: pkg/service/response.go:44
msgid "description"
msgstr "beskrivelse"
#: pkg/service/response.go:43
msgid "display name"
msgstr "visningsnavn"
#: pkg/service/response.go:42
msgid "expiration date"
msgstr "utløpsdato"
#: pkg/service/response.go:41
msgid "password"
msgstr "passord"
#: pkg/service/response.go:40
msgid "permission"
msgstr "tillatelse"
#: pkg/service/response.go:39
msgid "some field"
msgstr "et felt"
#: pkg/service/response.go:26
msgid "{resource} was downloaded via public link {token}"
msgstr "{resource} ble lastet ned via offentlig lenke {token}"
#: pkg/service/response.go:24
msgid "{user} added {resource} to {folder}"
msgstr "{user} la til {resource} i {folder}"
#: pkg/service/response.go:36
msgid "{user} added {sharee} as member of {space}"
msgstr "{user} la til {sharee} som medlem av {space}"
#: pkg/service/response.go:27
msgid "{user} deleted {resource} from {folder}"
msgstr "{user} slettet {resource} fra {folder}"
#: pkg/service/response.go:28
msgid "{user} moved {resource} to {folder}"
msgstr "{user} flyttet {resource} til {folder}"
#: pkg/service/response.go:35
msgid "{user} removed link to {resource}"
msgstr "{user} fjernet lenke til {resource}"
#: pkg/service/response.go:32
msgid "{user} removed {sharee} from {resource}"
msgstr "{user} fjernet {sharee} fra {resource}"
#: pkg/service/response.go:37
msgid "{user} removed {sharee} from {space}"
msgstr "{user} fjernet {sharee} fra {space}"
#: pkg/service/response.go:29
msgid "{user} renamed {oldResource} to {resource}"
msgstr "{user} ga nytt navn til {oldResource} som {resource}"
#: pkg/service/response.go:33
msgid "{user} shared {resource} via link"
msgstr "{user} delte {resource} via lenke"
#: pkg/service/response.go:30
msgid "{user} shared {resource} with {sharee}"
msgstr "{user} delte {resource} med {sharee}"
#: pkg/service/response.go:34
msgid "{user} updated {field} for a link {token} on {resource}"
msgstr "{user} oppdaterte {field} for en lenke {token} på {resource}"
#: pkg/service/response.go:31
msgid "{user} updated {field} for the {resource}"
msgstr "{user} oppdaterte {field} for {resource}"
#: pkg/service/response.go:25
msgid "{user} updated {resource} in {folder}"
msgstr "{user} oppdaterte {resource} i {folder}"
@@ -1,4 +1,4 @@
package activitylog
package service
import (
"context"
@@ -15,7 +15,7 @@ const currentMigrationVersion = "1"
// RunMigrations checks the activitylog data version and runs migrations if necessary.
// It should be called during service startup, after the NATS KeyValue store is initialized.
func (a *ActivityLog) runMigrations(ctx context.Context, kv nats.KeyValue) error {
func (a *ActivitylogService) runMigrations(ctx context.Context, kv nats.KeyValue) error {
entry, err := kv.Get(activitylogVersionKey)
if err == nats.ErrKeyNotFound {
a.log.Info().Msg("activitylog version key not found. Running migration to V1...")
@@ -40,7 +40,7 @@ func (a *ActivityLog) runMigrations(ctx context.Context, kv nats.KeyValue) error
// For each such key, it creates a new key in the format "originalKey.count.timestamp"
// and stores the original list of strings (re-marshalled to messagepack) as its value.
// Finally, it sets the activitylog.version key to "1".
func (a *ActivityLog) migrateToV1(_ context.Context, kv nats.KeyValue) error {
func (a *ActivitylogService) migrateToV1(_ context.Context, kv nats.KeyValue) error {
lister, err := kv.ListKeys()
if err != nil {
return fmt.Errorf("migrateToV1: failed to list keys from NATS KV store: %w", err)
@@ -1,14 +1,17 @@
package events
package service
import (
"context"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/go-chi/chi/v5"
"github.com/opencloud-eu/opencloud/pkg/log"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"go.opentelemetry.io/otel/trace"
)
// Option for the activitylog service
@@ -16,20 +19,17 @@ type Option func(*Options)
// Options for the activitylog service
type Options struct {
Context context.Context
Logger log.Logger
ServiceAccount config.ServiceAccount
Config *config.Config
TraceProvider trace.TracerProvider
Stream events.Stream
RegisteredEvents []events.Unmarshaller
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
Mux *chi.Mux
HistoryClient ehsvc.EventHistoryService
ValueClient settingssvc.ValueService
WriteBufferDuration time.Duration
NumConsumers int
}
func Context(ctx context.Context) Option {
return func(o *Options) {
o.Context = ctx
}
MaxActivities int
}
// Logger configures a logger for the activitylog service
@@ -39,10 +39,17 @@ func Logger(log log.Logger) Option {
}
}
// ServiceAccount configures a service account for the activitylog service
func ServiceAccount(sa config.ServiceAccount) Option {
// Config adds the config for the activitylog service
func Config(c *config.Config) Option {
return func(o *Options) {
o.ServiceAccount = sa
o.Config = c
}
}
// TraceProvider adds a tracer provider for the activitylog service
func TraceProvider(tp trace.TracerProvider) Option {
return func(o *Options) {
o.TraceProvider = tp
}
}
@@ -67,8 +74,23 @@ func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient])
}
}
func NumConsumers(num int) Option {
// Mux defines the muxer for the service
func Mux(m *chi.Mux) Option {
return func(o *Options) {
o.NumConsumers = num
o.Mux = m
}
}
// HistoryClient adds a grpc client for the eventhistory service
func HistoryClient(hc ehsvc.EventHistoryService) Option {
return func(o *Options) {
o.HistoryClient = hc
}
}
// ValueClient adds a grpc client for the value service
func ValueClient(vs settingssvc.ValueService) Option {
return func(o *Options) {
o.ValueClient = vs
}
}
@@ -1,4 +1,4 @@
package http
package service
import (
"context"
@@ -44,6 +44,11 @@ var (
StrDescription = l10n.Template("description")
)
// GetActivitiesResponse is the response on GET activities requests
type GetActivitiesResponse struct {
Activities []libregraph.Activity `json:"value"`
}
// Resource represents an item such as a file or folder
type Resource struct {
ID string `json:"id"`
@@ -306,7 +311,7 @@ func NewActivity(message string, ts time.Time, eventID string, vars map[string]a
}
// GetVars calls other service to gather the required data for the activity variables
func (s *svc) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]any, error) {
func (s *ActivitylogService) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]any, error) {
gwc, err := s.gws.Next()
if err != nil {
return nil, err
@@ -322,12 +327,6 @@ func (s *svc) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]a
return vars, nil
}
func toSpace(r *provider.Reference) *provider.StorageSpaceId {
return &provider.StorageSpaceId{
OpaqueId: storagespace.FormatStorageID(r.GetResourceId().GetStorageId(), r.GetResourceId().GetSpaceId()),
}
}
func getFolderName(ctx context.Context, gwc gateway.GatewayAPIClient, ref *provider.Reference) string {
n := filepath.Base(filepath.Dir(ref.GetPath()))
if n == "." || n == "/" {
+676
View File
@@ -0,0 +1,676 @@
package service
import (
"context"
"crypto/tls"
"encoding/base32"
"encoding/json"
"fmt"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"sync"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/go-chi/chi/v5"
"github.com/jellydator/ttlcache/v2"
"github.com/nats-io/nats.go"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/pkg/errors"
"github.com/vmihailenco/msgpack/v5"
"go.opentelemetry.io/otel/trace"
"github.com/opencloud-eu/opencloud/pkg/log"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
)
// Nats runs into max payload exceeded errors at around 7k activities. Let's keep a buffer.
var _maxActivitiesDefault = 6000
// RawActivity represents an activity as it is stored in the activitylog store
type RawActivity struct {
EventID string `json:"event_id"`
Depth int `json:"depth"`
Timestamp time.Time `json:"timestamp"`
}
// ActivitylogService logs events per resource
type ActivitylogService struct {
cfg *config.Config
log log.Logger
events <-chan events.Event
gws pool.Selectable[gateway.GatewayAPIClient]
mux *chi.Mux
evHistory ehsvc.EventHistoryService
valService settingssvc.ValueService
lock sync.RWMutex
tp trace.TracerProvider
tracer trace.Tracer
debouncer *Debouncer
parentIdCache *ttlcache.Cache
natskv nats.KeyValue
maxActivities int
registeredEvents map[string]events.Unmarshaller
}
type Debouncer struct {
after time.Duration
f func(id string, ra []RawActivity) error
pending sync.Map
inProgress sync.Map
mutex sync.Mutex
}
type queueItem struct {
activities []RawActivity
timer *time.Timer
}
type batchInfo struct {
key string
count int
timestamp time.Time
}
// NewDebouncer returns a new Debouncer instance
func NewDebouncer(d time.Duration, f func(id string, ra []RawActivity) error) *Debouncer {
return &Debouncer{
after: d,
f: f,
pending: sync.Map{},
inProgress: sync.Map{},
}
}
// Debounce restarts the debounce timer for the given space
func (d *Debouncer) Debounce(id string, ra RawActivity) {
if d.after == 0 {
d.f(id, []RawActivity{ra})
return
}
d.mutex.Lock()
defer d.mutex.Unlock()
activities := []RawActivity{ra}
item := &queueItem{
activities: activities,
}
if i, ok := d.pending.Load(id); ok {
// if the item is already in the queue, append the new activities
item, ok = i.(*queueItem)
if ok {
item.activities = append(item.activities, ra)
}
}
if item.timer == nil {
item.timer = time.AfterFunc(d.after, func() {
if _, ok := d.inProgress.Load(id); ok {
// Reschedule this run for when the previous run has finished
d.mutex.Lock()
if i, ok := d.pending.Load(id); ok {
i.(*queueItem).timer.Reset(d.after)
}
d.mutex.Unlock()
return
}
d.pending.Delete(id)
d.inProgress.Store(id, true)
defer d.inProgress.Delete(id)
d.f(id, item.activities)
})
}
d.pending.Store(id, item)
}
// New creates a new ActivitylogService
func New(opts ...Option) (*ActivitylogService, error) {
o := &Options{
MaxActivities: _maxActivitiesDefault,
}
for _, opt := range opts {
opt(o)
}
if o.Stream == nil {
return nil, errors.New("stream is required")
}
ch, err := events.Consume(o.Stream, o.Config.Service.Name, o.RegisteredEvents...)
if err != nil {
return nil, err
}
cache := ttlcache.NewCache()
err = cache.SetTTL(30 * time.Second)
if err != nil {
return nil, err
}
// Connect to NATS servers
natsOptions := nats.Options{
Servers: o.Config.Store.Nodes,
}
if o.Config.Store.EnableTLS {
if o.Config.Store.TLSRootCACertificate != "" {
// when root ca is configured use it. an insecure flag is ignored.
nats.RootCAs(o.Config.Store.TLSRootCACertificate)(&natsOptions)
} else {
// enable tls and use insecure flag
nats.Secure(&tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: o.Config.Store.TLSInsecure})(&natsOptions)
}
}
if o.Config.Store.AuthUsername != "" && o.Config.Store.AuthPassword != "" {
nats.UserInfo(o.Config.Store.AuthUsername, o.Config.Store.AuthPassword)(&natsOptions)
}
conn, err := natsOptions.Connect()
if err != nil {
return nil, err
}
js, err := conn.JetStream()
if err != nil {
return nil, err
}
kv, err := js.KeyValue(o.Config.Store.Database)
if err != nil {
if !errors.Is(err, nats.ErrBucketNotFound) {
return nil, errors.Wrapf(err, "Failed to get bucket (%s)", o.Config.Store.Database)
}
kv, err = js.CreateKeyValue(&nats.KeyValueConfig{
Bucket: o.Config.Store.Database,
})
if err != nil {
return nil, errors.Wrapf(err, "Failed to create bucket (%s)", o.Config.Store.Database)
}
}
if err != nil {
return nil, err
}
s := &ActivitylogService{
log: o.Logger,
cfg: o.Config,
events: ch,
gws: o.GatewaySelector,
mux: o.Mux,
evHistory: o.HistoryClient,
valService: o.ValueClient,
lock: sync.RWMutex{},
registeredEvents: make(map[string]events.Unmarshaller),
tp: o.TraceProvider,
tracer: o.TraceProvider.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"),
parentIdCache: cache,
maxActivities: o.Config.MaxActivities,
natskv: kv,
}
s.debouncer = NewDebouncer(o.Config.WriteBufferDuration, s.storeActivity)
// run migrations
err = s.runMigrations(context.Background(), kv)
if err != nil {
return nil, err
}
s.mux.Get("/graph/v1beta1/extensions/org.libregraph/activities", s.HandleGetItemActivities)
for _, e := range o.RegisteredEvents {
typ := reflect.TypeOf(e)
s.registeredEvents[typ.String()] = e
}
go s.Run()
return s, nil
}
// Run runs the service
func (a *ActivitylogService) Run() {
for e := range a.events {
var err error
switch ev := e.Event.(type) {
case events.UploadReady:
err = a.AddActivity(ev.FileRef, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
case events.FileTouched:
err = a.AddActivity(ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
// Disabled https://github.com/owncloud/ocis/issues/10293
//case events.FileDownloaded:
// we are only interested in public link downloads - so no need to store others.
//if ev.ImpersonatingUser.GetDisplayName() == "Public" {
// err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
//}
case events.ContainerCreated:
err = a.AddActivity(ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemTrashed:
err = a.AddActivityTrashed(ev.ID, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemPurged:
err = a.RemoveResource(ev.ID)
case events.ItemMoved:
// remove the cached parent id for this resource
a.removeCachedParentID(ev.Ref)
err = a.AddActivity(ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.ShareCreated:
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
case events.ShareUpdated:
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
}
case events.ShareRemoved:
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, ev.Timestamp)
case events.LinkCreated:
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
case events.LinkUpdated:
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
}
case events.LinkRemoved:
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.SpaceShared:
err = a.AddSpaceActivity(ev.ID, e.ID, ev.Timestamp)
case events.SpaceUnshared:
err = a.AddSpaceActivity(ev.ID, e.ID, ev.Timestamp)
}
if err != nil {
a.log.Error().Err(err).Interface("event", e).Msg("could not process event")
}
}
}
// AddActivity adds the activity to the given resource and all its parents
func (a *ActivitylogService) AddActivity(initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
gwc, err := a.gws.Next()
if err != nil {
return fmt.Errorf("cant get gateway client: %w", err)
}
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("cant get service user context: %w", err)
}
var span trace.Span
ctx, span = a.tracer.Start(ctx, "AddActivity")
defer span.End()
return a.addActivity(ctx, initRef, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
return utils.GetResource(ctx, ref, gwc)
})
}
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
func (a *ActivitylogService) AddActivityTrashed(resourceID *provider.ResourceId, reference *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
gwc, err := a.gws.Next()
if err != nil {
return fmt.Errorf("cant get gateway client: %w", err)
}
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("cant get service user context: %w", err)
}
// store activity on trashed item
if err := a.storeActivity(storagespace.FormatResourceID(resourceID), []RawActivity{
{
EventID: eventID,
Depth: 0,
Timestamp: timestamp,
},
}); err != nil {
return fmt.Errorf("could not store activity: %w", err)
}
// get previous parent
ref := &provider.Reference{
ResourceId: reference.GetResourceId(),
Path: filepath.Dir(reference.GetPath()),
}
var span trace.Span
ctx, span = a.tracer.Start(ctx, "AddActivityTrashed")
defer span.End()
return a.addActivity(ctx, ref, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
return utils.GetResource(ctx, ref, gwc)
})
}
// AddSpaceActivity adds the activity to the given spaceroot
func (a *ActivitylogService) AddSpaceActivity(spaceID *provider.StorageSpaceId, eventID string, timestamp time.Time) error {
// spaceID is in format <providerid>$<spaceid>
// activitylog service uses format <providerid>$<spaceid>!<resourceid>
// lets do some converting, shall we?
rid, err := storagespace.ParseID(spaceID.GetOpaqueId())
if err != nil {
return fmt.Errorf("could not parse space id: %w", err)
}
rid.OpaqueId = rid.GetSpaceId()
return a.storeActivity(storagespace.FormatResourceID(&rid), []RawActivity{
{
EventID: eventID,
Depth: 0,
Timestamp: timestamp,
},
})
}
// Activities returns the activities for the given resource
func (a *ActivitylogService) Activities(rid *provider.ResourceId) ([]RawActivity, error) {
a.lock.RLock()
defer a.lock.RUnlock()
return a.activities(rid)
}
// RemoveActivities removes the activities from the given resource
func (a *ActivitylogService) RemoveActivities(rid *provider.ResourceId, toDelete map[string]struct{}) error {
a.lock.Lock()
defer a.lock.Unlock()
curActivities, err := a.activities(rid)
if err != nil {
return err
}
var acts []RawActivity
for _, a := range curActivities {
if _, ok := toDelete[a.EventID]; !ok {
acts = append(acts, a)
}
}
b, err := json.Marshal(acts)
if err != nil {
return err
}
_, err = a.natskv.Put(storagespace.FormatResourceID(rid), b)
return err
}
// RemoveResource removes the resource from the store
func (a *ActivitylogService) RemoveResource(rid *provider.ResourceId) error {
if rid == nil {
return fmt.Errorf("resource id is required")
}
a.lock.Lock()
defer a.lock.Unlock()
return a.natskv.Delete(storagespace.FormatResourceID(rid))
}
func (a *ActivitylogService) activities(rid *provider.ResourceId) ([]RawActivity, error) {
resourceID := storagespace.FormatResourceID(rid)
glob := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
watcher, err := a.natskv.Watch(glob, nats.IgnoreDeletes())
if err != nil {
return nil, err
}
defer watcher.Stop()
var activities []RawActivity
for update := range watcher.Updates() {
if update == nil {
break
}
var batchActivities []RawActivity
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
}
activities = append(activities, batchActivities...)
}
return activities, nil
}
// note: getResource is abstracted to allow unit testing, in general this will just be utils.GetResource
func (a *ActivitylogService) addActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time, getResource func(context.Context, *provider.Reference) (*provider.ResourceInfo, error)) error {
var (
err error
depth int
ref = initRef
)
ctx, span := a.tracer.Start(ctx, "addActivity")
defer span.End()
for {
var info *provider.ResourceInfo
id := ref.GetResourceId()
if ref.Path != "" {
// Path based reference, we need to resolve the resource id
ctx, span = a.tracer.Start(ctx, "addActivity.getResource")
info, err = getResource(ctx, ref)
span.End()
if err != nil {
return fmt.Errorf("could not get resource info: %w", err)
}
id = info.GetId()
}
if id == nil {
return fmt.Errorf("resource id is required")
}
key := storagespace.FormatResourceID(id)
a.debouncer.Debounce(key, RawActivity{
EventID: eventID,
Depth: depth,
Timestamp: timestamp,
})
if id.OpaqueId == id.SpaceId {
// we are at the root of the space, no need to go further
break
}
// check if parent id is cached
// parent id is cached in the format <storageid>$<spaceid>!<resourceid>
// if it is not cached, get the resource info and cache it
if parentId == nil {
if v, err := a.parentIdCache.Get(key); err != nil {
if info == nil {
ctx, span := a.tracer.Start(ctx, "addActivity.getResource parent")
info, err = getResource(ctx, ref)
span.End()
if err != nil || info.GetParentId() == nil || info.GetParentId().GetOpaqueId() == "" {
return fmt.Errorf("could not get parent id: %w", err)
}
}
parentId = info.GetParentId()
a.parentIdCache.Set(key, parentId)
} else {
parentId = v.(*provider.ResourceId)
}
} else {
a.log.Debug().Msg("parent id is cached")
}
depth++
ref = &provider.Reference{ResourceId: parentId}
parentId = nil // reset parent id so it's not reused in the next iteration
}
return nil
}
func (a *ActivitylogService) storeActivity(resourceID string, activities []RawActivity) error {
a.lock.Lock()
defer a.lock.Unlock()
ctx, span := a.tracer.Start(context.Background(), "storeActivity")
defer span.End()
_, subspan := a.tracer.Start(ctx, "storeActivity.Marshal")
b, err := msgpack.Marshal(activities)
if err != nil {
return err
}
subspan.End()
_, subspan = a.tracer.Start(ctx, "storeActivity.natskv.Put")
key := natsKey(resourceID, len(activities))
_, err = a.natskv.Put(key, b)
if err != nil {
return err
}
subspan.End()
ctx, subspan = a.tracer.Start(ctx, "storeActivity.enforceMaxActivities")
a.enforceMaxActivities(ctx, resourceID)
subspan.End()
return nil
}
func (a *ActivitylogService) enforceMaxActivities(ctx context.Context, resourceID string) {
if a.maxActivities <= 0 {
return
}
key := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
_, subspan := a.tracer.Start(ctx, "enforceMaxActivities.watch")
watcher, err := a.natskv.Watch(key, nats.IgnoreDeletes())
if err != nil {
a.log.Error().Err(err).Str("resourceID", resourceID).Msg("could not watch")
return
}
defer watcher.Stop()
var keys []string
for update := range watcher.Updates() {
if update == nil {
break
}
var batchActivities []RawActivity
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
}
keys = append(keys, update.Key())
}
subspan.End()
_, subspan = a.tracer.Start(ctx, "enforceMaxActivities.compile")
// Parse keys into batches
batches := make([]batchInfo, 0)
var activitiesCount int
for _, k := range keys {
parts := strings.SplitN(k, ".", 3)
if len(parts) < 3 {
a.log.Warn().Str("key", k).Msg("skipping key, not enough parts")
continue
}
c, err := strconv.Atoi(parts[1])
if err != nil {
a.log.Warn().Str("key", k).Msg("skipping key, can not parse count")
continue
}
// parse timestamp
nano, err := strconv.ParseInt(parts[2], 10, 64)
if err != nil {
a.log.Warn().Str("key", k).Msg("skipping key, can not parse timestamp")
continue
}
batches = append(batches, batchInfo{
key: k,
count: c,
timestamp: time.Unix(0, nano),
})
activitiesCount += c
}
// sort batches by timestamp
sort.Slice(batches, func(i, j int) bool {
return batches[i].timestamp.Before(batches[j].timestamp)
})
subspan.End()
_, subspan = a.tracer.Start(ctx, "enforceMaxActivities.delete")
// remove oldest keys until we are at max activities
for _, b := range batches {
if activitiesCount-b.count < a.maxActivities {
break
}
activitiesCount -= b.count
err = a.natskv.Delete(b.key)
if err != nil {
a.log.Error().Err(err).Str("key", b.key).Msg("could not delete key")
break
}
}
subspan.End()
}
func toRef(r *provider.ResourceId) *provider.Reference {
return &provider.Reference{
ResourceId: r,
}
}
func toSpace(r *provider.Reference) *provider.StorageSpaceId {
return &provider.StorageSpaceId{
OpaqueId: storagespace.FormatStorageID(r.GetResourceId().GetStorageId(), r.GetResourceId().GetSpaceId()),
}
}
func (a *ActivitylogService) removeCachedParentID(ref *provider.Reference) {
purgeId := ref.GetResourceId()
if ref.GetPath() != "" {
gwc, err := a.gws.Next()
if err != nil {
a.log.Error().Err(err).Msg("could not get gateway client")
return
}
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
a.log.Error().Err(err).Msg("could not get service user context")
return
}
info, err := utils.GetResource(ctx, ref, gwc)
if err != nil {
a.log.Error().Err(err).Msg("could not get resource info")
return
}
purgeId = info.GetId()
}
// The parent id cache is populated lazily and its entries expire, so a
// missing key is the expected case rather than an error.
if err := a.parentIdCache.Remove(storagespace.FormatResourceID(purgeId)); err != nil {
a.log.Debug().Interface("event", ref).Err(err).Msg("could not delete parent id cache")
}
}
func natsKey(resourceID string, activitiesCount int) string {
return fmt.Sprintf("%s.%d.%d",
base32.StdEncoding.EncodeToString([]byte(resourceID)),
activitiesCount,
time.Now().UnixNano())
}
@@ -1,4 +1,4 @@
package events_test
package service_test
import (
"testing"
@@ -7,7 +7,7 @@ import (
. "github.com/onsi/gomega"
)
func TestEvents(t *testing.T) {
func TestService(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Events Suite")
RunSpecs(t, "Service Suite")
}
@@ -1,6 +1,7 @@
package activitylog_test
package service
import (
"bytes"
"context"
"net"
"os"
@@ -8,13 +9,17 @@ import (
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
nserver "github.com/nats-io/nats-server/v2/server"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/command"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
eventsmocks "github.com/opencloud-eu/reva/v2/pkg/events/mocks"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/test-go/testify/mock"
"go.opentelemetry.io/otel/trace/noop"
)
var (
@@ -63,25 +68,31 @@ var _ = SynchronizedAfterSuite(func() {
var _ = Describe("ActivitylogService", func() {
var (
alog *activitylog.ActivityLog
alog *ActivitylogService
getResource func(_ context.Context, ref *provider.Reference) (*provider.ResourceInfo, error)
writebufferduration = 100 * time.Millisecond
)
JustBeforeEach(func() {
var err error
db := "activitylog-test-" + uuid.New().String()
kv, err := command.ConnectNatsKV(config.Store{
Nodes: []string{server.Addr().String()},
Database: db,
})
Expect(err).ToNot(HaveOccurred())
alog, err = activitylog.New(
kv,
activitylog.MaxActivities(4),
activitylog.WriteBufferDuration(writebufferduration),
stream := &eventsmocks.Stream{}
stream.EXPECT().Consume(mock.Anything, mock.Anything).Return(nil, nil)
alog, err = New(
Config(&config.Config{
Service: config.Service{
Name: "activitylog-test",
},
Store: config.Store{
Store: "nats-js-kv",
Nodes: []string{server.Addr().String()},
Database: "activitylog-test-" + uuid.New().String(),
},
MaxActivities: 4,
WriteBufferDuration: writebufferduration,
}),
Stream(stream),
TraceProvider(noop.NewTracerProvider()),
Mux(chi.NewMux()),
)
Expect(err).ToNot(HaveOccurred())
})
@@ -96,7 +107,7 @@ var _ = Describe("ActivitylogService", func() {
Name string
Tree map[string]*provider.ResourceInfo
Activities map[string]string
Expected map[string][]activitylog.RawActivity
Expected map[string][]RawActivity
}
testCases := []testCase{
@@ -110,7 +121,7 @@ var _ = Describe("ActivitylogService", func() {
Activities: map[string]string{
"activity": "base",
},
Expected: map[string][]activitylog.RawActivity{
Expected: map[string][]RawActivity{
"base": activitites("activity", 0),
"parent": activitites("activity", 1),
"spaceid": activitites("activity", 2),
@@ -127,7 +138,7 @@ var _ = Describe("ActivitylogService", func() {
"activity1": "base",
"activity2": "base",
},
Expected: map[string][]activitylog.RawActivity{
Expected: map[string][]RawActivity{
"base": activitites("activity1", 0, "activity2", 0),
"parent": activitites("activity1", 1, "activity2", 1),
"spaceid": activitites("activity1", 2, "activity2", 2),
@@ -144,7 +155,7 @@ var _ = Describe("ActivitylogService", func() {
}
for k, v := range tc.Activities {
err := alog.AddActivity(context.Background(), reference(v), nil, k, time.Time{}, getResource)
err := alog.addActivity(context.Background(), reference(v), nil, k, time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
}
})
@@ -183,9 +194,9 @@ var _ = Describe("ActivitylogService", func() {
It("debounces activities", func() {
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
@@ -196,7 +207,7 @@ var _ = Describe("ActivitylogService", func() {
})
It("adheres to the MaxActivities setting", func() {
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
activities, err := alog.Activities(resourceID("base"))
@@ -204,7 +215,7 @@ var _ = Describe("ActivitylogService", func() {
g.Expect(len(activities)).To(Equal(1))
}).Should(Succeed())
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
activities, err := alog.Activities(resourceID("base"))
@@ -212,11 +223,11 @@ var _ = Describe("ActivitylogService", func() {
g.Expect(len(activities)).To(Equal(2))
}).Should(Succeed())
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
err = alog.addActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
err = alog.addActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity5", time.Time{}, getResource)
err = alog.addActivity(context.Background(), reference("base"), nil, "activity5", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
@@ -233,9 +244,9 @@ var _ = Describe("ActivitylogService", func() {
return tree[ref.GetResourceId().GetOpaqueId()], nil
}
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
@@ -244,9 +255,9 @@ var _ = Describe("ActivitylogService", func() {
g.Expect(activities).To(ConsistOf(activitites("activity1", 0, "activity2", 0)))
}).Should(Succeed())
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
err = alog.addActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
err = alog.addActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
@@ -257,11 +268,47 @@ var _ = Describe("ActivitylogService", func() {
})
})
})
Describe("removeCachedParentID", func() {
var logBuffer *bytes.Buffer
newLoggerAtLevel := func(level string) log.Logger {
logBuffer = &bytes.Buffer{}
return log.Logger{Logger: log.NewLogger(log.Level(level)).Output(logBuffer)}
}
It("does not log an error when the entry was never cached", func() {
alog.log = newLoggerAtLevel("error")
alog.removeCachedParentID(reference("never-cached"))
Expect(logBuffer.String()).To(BeEmpty())
})
It("logs a missing entry at debug level", func() {
alog.log = newLoggerAtLevel("debug")
alog.removeCachedParentID(reference("never-cached"))
Expect(logBuffer.String()).To(ContainSubstring("could not delete parent id cache"))
Expect(logBuffer.String()).To(ContainSubstring(`"level":"debug"`))
})
It("does not log at all when the entry was cached", func() {
alog.log = newLoggerAtLevel("debug")
ref := reference("cached")
Expect(alog.parentIdCache.Set(storagespace.FormatResourceID(ref.GetResourceId()), resourceID("parent"))).To(Succeed())
alog.removeCachedParentID(ref)
Expect(logBuffer.String()).To(BeEmpty())
})
})
})
func activitites(acts ...any) []activitylog.RawActivity {
var activities []activitylog.RawActivity
act := activitylog.RawActivity{}
func activitites(acts ...any) []RawActivity {
var activities []RawActivity
act := RawActivity{}
for _, a := range acts {
switch v := a.(type) {
case string:
+3 -3
View File
@@ -61,7 +61,7 @@ type Spaces struct {
}
type LDAPMetrics struct {
Disabled bool `yaml:"disabled" env:"GRAPH_LDAP_METRICS_DISABLE" desc:"Disables the metrics for outbound LDAP operations." introductionVersion:"%%NEXT%%"`
Disabled bool `yaml:"disabled" env:"GRAPH_LDAP_METRICS_DISABLE" desc:"Disables the metrics for outbound LDAP operations." introductionVersion:"%NEXT%"`
}
type LDAP struct {
@@ -121,7 +121,7 @@ type LDAPEducationConfig struct {
}
type IdentityMetrics struct {
Disabled bool `yaml:"disabled" env:"GRAPH_IDENTITY_BACKEND_METRICS_DISABLE" desc:"Disables the metrics for inbound identity backend operations." introductionVersion:"%%NEXT%%"`
Disabled bool `yaml:"disabled" env:"GRAPH_IDENTITY_BACKEND_METRICS_DISABLE" desc:"Disables the metrics for inbound identity backend operations." introductionVersion:"%NEXT%"`
}
type Identity struct {
@@ -141,7 +141,7 @@ type API struct {
// Events combines the configuration options for the event bus.
type Events struct {
DisabledConsumer bool `yaml:"disabled_consumer" env:"GRAPH_EVENTS_DISABLE_CONSUMER" desc:"Disables consuming events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%%NEXT%%"`
DisabledConsumer bool `yaml:"disabled_consumer" env:"GRAPH_EVENTS_DISABLE_CONSUMER" desc:"Disables consuming events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%NEXT%"`
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;GRAPH_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Set to a empty string to disable emitting events." introductionVersion:"1.0.0"`
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;GRAPH_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;GRAPH_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
+2 -2
View File
@@ -3,12 +3,12 @@ package config
import "github.com/opencloud-eu/opencloud/pkg/shared"
type HTTPMetrics struct {
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_METRICS_DISABLE" desc:"Disables the metrics for the HTTP service." introductionVersion:"%%NEXT%%"`
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_METRICS_DISABLE" desc:"Disables the metrics for the HTTP service." introductionVersion:"%NEXT%"`
}
// HTTP defines the available http configuration.
type HTTP struct {
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_DISABLE" desc:"Disables the HTTP service. Set this to true if the service should only consume events." introductionVersion:"%%NEXT%%"`
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_DISABLE" desc:"Disables the HTTP service. Set this to true if the service should only consume events." introductionVersion:"%NEXT%"`
Addr string `yaml:"addr" env:"GRAPH_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
Namespace string `yaml:"-"`
Root string `yaml:"root" env:"GRAPH_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
@@ -1,157 +0,0 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
# Translators:
# Roy-Magne Mo, 2026
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2026-09-07 23:16+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Roy-Magne Mo, 2026\n"
"Language-Team: Norwegian (https://app.transifex.com/opencloud-eu/teams/204053/no/)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Language: no\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
#. UnifiedRole Editor, Role DisplayName (resolves directly)
#. UnifiedRole Editor, Role DisplayName (resolves directly)
#. UnifiedRole EditorListGrants, Role DisplayName (resolves directly)
#. UnifiedRole SpaseEditor, Role DisplayName (resolves directly)
#. UnifiedRole FileEditor, Role DisplayName (resolves directly)
#. UnifiedRole FileEditorWithVersions, Role DisplayName (resolves directly)
#. UnifiedRole FileEditorListGrants, Role DisplayName (resolves directly)
#: pkg/unifiedrole/roles.go:137 pkg/unifiedrole/roles.go:143
#: pkg/unifiedrole/roles.go:149 pkg/unifiedrole/roles.go:155
#: pkg/unifiedrole/roles.go:167 pkg/unifiedrole/roles.go:173
#: pkg/unifiedrole/roles.go:179
msgid "Can edit"
msgstr "Kan redigere"
#. UnifiedRole SpaseEditorWithoutVersions, Role DisplayName (resolves
#. directly)
#: pkg/unifiedrole/roles.go:161
msgid "Can edit without versions"
msgstr "Kan redigere uten versjoner"
#. UnifiedRole Manager, Role DisplayName (resolves directly)
#: pkg/unifiedrole/roles.go:191
msgid "Can manage"
msgstr "Kan håndtere"
#. UnifiedRole EditorLite, Role DisplayName (resolves directly)
#: pkg/unifiedrole/roles.go:185
msgid "Can upload"
msgstr "Kan laste opp"
#. UnifiedRole Viewer, Role DisplayName (resolves directly)
#. UnifiedRole ViewerWithVersions, Role DisplayName (resolves directly)
#. UnifiedRole Viewer, Role DisplayName (resolves directly)
#. UnifiedRole SpaseViewer, Role DisplayName (resolves directly)
#. UnifiedRole SpaseViewer, Role DisplayName (resolves directly)
#: pkg/unifiedrole/roles.go:107 pkg/unifiedrole/roles.go:113
#: pkg/unifiedrole/roles.go:119 pkg/unifiedrole/roles.go:125
#: pkg/unifiedrole/roles.go:131
msgid "Can view"
msgstr "Kan se"
#. UnifiedRole SecureViewer, Role DisplayName (resolves directly)
#: pkg/unifiedrole/roles.go:197
msgid "Can view (secure)"
msgstr "Kan vise (sikker)"
#. UnifiedRole FullDenial, Role DisplayName (resolves directly)
#: pkg/unifiedrole/roles.go:203
msgid "Cannot access"
msgstr "Kan ikke få tilgang"
#. UnifiedRole FullDenial, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:200
msgid "Deny all access."
msgstr "Avvis all tilgang."
#. default description for new spaces
#: pkg/service/v0/spacetemplates.go:32
msgid "Here you can add a description for this Space."
msgstr "Her kan du legge til en beskrivelse for dette rommet."
#. UnifiedRole ViewerWithVersions, Role Description (resolves directly)
#. UnifiedRole SpaceViewer, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:110 pkg/unifiedrole/roles.go:128
msgid "View and download including the history."
msgstr "Vis og last ned, inkludert historikken."
#. UnifiedRole Viewer, Role Description (resolves directly)
#. UnifiedRole SpaceViewer, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:104 pkg/unifiedrole/roles.go:122
msgid "View and download."
msgstr "Vis og last ned."
#. UnifiedRole SecureViewer, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:194
msgid "View only documents, images and PDFs. Watermarks will be applied."
msgstr "Vis kun dokumenter, bilder og PDF-er. Vannmerker vil bli brukt."
#. UnifiedRole FileEditorWithVErsions, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:176
msgid "View, download and edit including the history."
msgstr "Vis, last ned og rediger, inkludert historikken."
#. UnifiedRole FileEditor, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:164
msgid "View, download and edit."
msgstr "Vis, last ned og rediger."
#. UnifiedRole ViewerListGrants, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:116
msgid "View, download and show all invited people."
msgstr "Vis, last ned og vis alle inviterte personer."
#. UnifiedRole EditorLite, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:182
msgid "View, download and upload."
msgstr "Vis, last ned og last opp."
#. UnifiedRole FileEditorListGrants, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:170
msgid "View, download, edit and show all invited people."
msgstr "Vis, last ned, rediger og vis alle inviterte personer."
#. UnifiedRole Editor, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:140
msgid "View, download, upload, edit, add and delete including the history."
msgstr ""
"Vis, last ned, last opp, rediger, legg til og slett, inkludert historikken."
#. UnifiedRole Editor, Role Description (resolves directly)
#. UnifiedRole SpaseEditorWithoutVersions, Role Description (resolves
#. directly)
#: pkg/unifiedrole/roles.go:134 pkg/unifiedrole/roles.go:158
msgid "View, download, upload, edit, add and delete."
msgstr "Vis, last ned, last opp, rediger, legg til og slett."
#. UnifiedRole Manager, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:188
msgid "View, download, upload, edit, add, delete and manage members."
msgstr ""
"Vis, last ned, last opp, rediger, legg til, slett og administrer medlemmer."
#. UnifiedRoleListGrants Editor, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:146
msgid "View, download, upload, edit, add, delete and show all invited people."
msgstr ""
"Se, last ned, last opp, rediger, legg til, slett og vis alle inviterte "
"personer."
#. UnifiedRole SpaseEditor, Role Description (resolves directly)
#: pkg/unifiedrole/roles.go:152
msgid "View, download, upload, edit, add, delete including the history."
msgstr ""
"Vis, last ned, last opp, rediger, legg til, slett inkludert historikken."
+23 -195
View File
@@ -15,16 +15,13 @@ import (
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
cs3rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/go-chi/render"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"golang.org/x/crypto/sha3"
"google.golang.org/protobuf/types/known/fieldmaskpb"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/publicshare"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/tags"
"github.com/opencloud-eu/reva/v2/pkg/utils"
@@ -36,24 +33,13 @@ import (
)
// opt-in driveItem instance annotations, returned only when requested via $select
const (
_selectAllowedValues = "@libre.graph.permissions.actions.allowedValues"
_selectShareTypes = "@libre.graph.shareTypes"
)
const _selectAllowedValues = "@libre.graph.permissions.actions.allowedValues"
// without it the provider leaves the share-types opaque empty
var shareTypesFieldMask = &fieldmaskpb.FieldMask{Paths: []string{"share-types"}}
// opt-in driveItem relations, returned only when requested via $expand
const (
_expandChildren = "children"
_expandThumbnails = "thumbnails"
)
func odataListContains(r *http.Request, parameter, value string) bool {
for _, values := range r.URL.Query()[parameter] {
// driveItemPropertySelected reports whether the given opt-in property was requested via $select
func driveItemPropertySelected(r *http.Request, property string) bool {
for _, values := range r.URL.Query()["$select"] {
for _, v := range strings.Split(values, ",") {
if v == value {
if v == property {
return true
}
}
@@ -61,16 +47,6 @@ func odataListContains(r *http.Request, parameter, value string) bool {
return false
}
// driveItemPropertySelected reports whether the given opt-in property was requested via $select
func driveItemPropertySelected(r *http.Request, property string) bool {
return odataListContains(r, "$select", property)
}
// driveItemRelationExpanded reports whether the relation was requested via $expand
func driveItemRelationExpanded(r *http.Request, relation string) bool {
return odataListContains(r, "$expand", relation)
}
// CreateUploadSession create an upload session to allow your app to upload files up to the maximum file size.
// An upload session allows your app to upload ranges of the file in sequential API requests, which allows the
// transfer to be resumed if a connection is dropped while the upload is in progress.
@@ -222,14 +198,9 @@ func (g Graph) GetRootDriveChildren(w http.ResponseWriter, r *http.Request) {
}
}
listRequest := &storageprovider.ListContainerRequest{
lRes, err := gatewayClient.ListContainer(ctx, &storageprovider.ListContainerRequest{
Ref: &storageprovider.Reference{ResourceId: space.GetRoot()},
}
if driveItemPropertySelected(r, _selectShareTypes) {
listRequest.FieldMask = shareTypesFieldMask
}
lRes, err := gatewayClient.ListContainer(ctx, listRequest)
})
switch {
case err != nil:
g.logger.Error().Err(err).Msg("error making ListContainer grpc call")
@@ -256,7 +227,6 @@ func (g Graph) GetRootDriveChildren(w http.ResponseWriter, r *http.Request) {
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return
}
g.setDriveItemsThumbnails(r, files, lRes.GetInfos())
if driveItemPropertySelected(r, _selectAllowedValues) {
for i, info := range lRes.GetInfos() {
@@ -264,10 +234,6 @@ func (g Graph) GetRootDriveChildren(w http.ResponseWriter, r *http.Request) {
}
}
if driveItemPropertySelected(r, _selectShareTypes) {
g.addShareTypes(ctx, files, lRes.GetInfos())
}
render.Status(r, http.StatusOK)
render.JSON(w, r, &ListResponse{Value: files})
}
@@ -336,24 +302,6 @@ func (g Graph) GetDriveItem(w http.ResponseWriter, r *http.Request) {
driveItem.LibreGraphPermissionsActionsAllowedValues = unifiedrole.CS3ResourcePermissionsToLibregraphActions(res.GetInfo().GetPermissionSet())
}
// only containers have children
if res.GetInfo().GetType() == storageprovider.ResourceType_RESOURCE_TYPE_CONTAINER && driveItemRelationExpanded(r, _expandChildren) {
children, ok := g.listDriveItemChildren(w, r, &driveItemID)
if !ok {
return
}
driveItem.Children = children
}
if driveItemPropertySelected(r, _selectShareTypes) {
infos := []*storageprovider.ResourceInfo{res.GetInfo()}
driveItem.LibreGraphShareTypes = shareTypesOf(res.GetInfo(), g.listLinkShares(ctx, infos))
}
if driveItemRelationExpanded(r, _expandThumbnails) {
setDriveItemThumbnails(driveItem, res.GetInfo(), g.config.Commons.OpenCloudURL)
}
render.Status(r, http.StatusOK)
render.JSON(w, r, &driveItem)
}
@@ -361,6 +309,7 @@ func (g Graph) GetDriveItem(w http.ResponseWriter, r *http.Request) {
// GetDriveItemChildren lists the children of a driveItem
func (g Graph) GetDriveItemChildren(w http.ResponseWriter, r *http.Request) {
g.logger.Info().Msg("Calling GetDriveItemChildren")
ctx := r.Context()
driveID, err := parseIDParam(r, "driveID")
if err != nil {
@@ -386,63 +335,43 @@ func (g Graph) GetDriveItemChildren(w http.ResponseWriter, r *http.Request) {
}
*/
files, ok := g.listDriveItemChildren(w, r, &driveItemID)
if !ok {
return
}
render.Status(r, http.StatusOK)
render.JSON(w, r, &ListResponse{Value: files})
}
func (g Graph) listDriveItemChildren(w http.ResponseWriter, r *http.Request, driveItemID *storageprovider.ResourceId) ([]libregraph.DriveItem, bool) {
gatewayClient, err := g.gatewaySelector.Next()
if err != nil {
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return nil, false
return
}
childrenRequest := &storageprovider.ListContainerRequest{
Ref: &storageprovider.Reference{ResourceId: driveItemID},
}
if driveItemPropertySelected(r, _selectShareTypes) {
childrenRequest.FieldMask = shareTypesFieldMask
}
res, err := gatewayClient.ListContainer(r.Context(), childrenRequest)
res, err := gatewayClient.ListContainer(ctx, &storageprovider.ListContainerRequest{
Ref: &storageprovider.Reference{ResourceId: &driveItemID},
})
switch {
case err != nil:
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return nil, false
return
case res.GetStatus().GetCode() == cs3rpc.Code_CODE_OK:
// ok
case res.GetStatus().GetCode() == cs3rpc.Code_CODE_NOT_FOUND:
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, res.GetStatus().GetMessage())
return nil, false
return
case res.GetStatus().GetCode() == cs3rpc.Code_CODE_PERMISSION_DENIED:
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, res.GetStatus().GetMessage()) // do not leak existence? check what graph does
return nil, false
return
case res.GetStatus().GetCode() == cs3rpc.Code_CODE_UNAUTHENTICATED:
errorcode.Unauthenticated.Render(w, r, http.StatusUnauthorized, res.GetStatus().GetMessage()) // do not leak existence? check what graph does
return nil, false
return
default:
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, res.GetStatus().GetMessage())
return nil, false
return
}
files, err := formatDriveItems(g.logger, g.publicBaseURL, res.GetInfos())
if err != nil {
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return nil, false
return
}
if driveItemPropertySelected(r, _selectShareTypes) {
g.addShareTypes(r.Context(), files, res.GetInfos())
}
g.setDriveItemsThumbnails(r, files, res.GetInfos())
return files, true
render.Status(r, http.StatusOK)
render.JSON(w, r, &ListResponse{Value: files})
}
func (g Graph) getRemoteItem(ctx context.Context, root *storageprovider.ResourceId, baseURL *url.URL) (*libregraph.RemoteItem, error) {
@@ -483,14 +412,14 @@ func (g Graph) getRemoteItem(ctx context.Context, root *storageprovider.Resource
return item, nil
}
func formatDriveItems(logger *log.Logger, publicBaseURL *url.URL, mds []*storageprovider.ResourceInfo) ([]libregraph.DriveItem, error) {
responses := make([]libregraph.DriveItem, 0, len(mds))
func formatDriveItems(logger *log.Logger, publicBaseURL *url.URL, mds []*storageprovider.ResourceInfo) ([]*libregraph.DriveItem, error) {
responses := make([]*libregraph.DriveItem, 0, len(mds))
for i := range mds {
res, err := cs3ResourceToDriveItem(logger, publicBaseURL, mds[i])
if err != nil {
return nil, err
}
responses = append(responses, *res)
responses = append(responses, res)
}
return responses, nil
@@ -556,16 +485,12 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
driveItem.Location = metadataToFacet[libregraph.GeoCoordinates](metadata, "location")
driveItem.Photo = metadataToFacet[libregraph.Photo](metadata, "photo")
driveItem.Video = metadataToFacet[libregraph.Video](metadata, "video")
driveItem.LibreGraphMotionPhoto = metadataToFacet[libregraph.MotionPhoto](metadata, "motionPhoto")
driveItem.LibreGraphLivePhoto = metadataToFacet[libregraph.LivePhoto](metadata, "livePhoto")
driveItem.LibreGraphMeFollowing = libregraph.PtrBool(metadata[_favoriteMetadataKey] == "1")
if t := metadata["tags"]; t != "" {
driveItem.LibreGraphTags = tags.New(t).AsSlice()
}
}
driveItem.LockInfo = lockToFacet(res.GetLock())
if utils.IsProcessing(res) {
// queuedDateTime stays absent, we do not track when postprocessing started
driveItem.PendingOperations = &libregraph.PendingOperations{
@@ -576,103 +501,6 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
return driveItem, nil
}
// lockToFacet maps a CS3 lock; WebDAV renders the same state as d:lockdiscovery.
// Only exclusive locks are reported, that is all OpenCloud issues.
func lockToFacet(lock *storageprovider.Lock) *libregraph.LockInfo {
switch lock.GetType() {
case storageprovider.LockType_LOCK_TYPE_EXCL, storageprovider.LockType_LOCK_TYPE_WRITE:
default:
return nil
}
info := libregraph.NewLockInfo()
info.SetLockType("exclusive")
if appName := lock.GetAppName(); appName != "" {
info.SetLibreGraphAppName(appName)
}
if user := lock.GetUser(); user != nil {
info.SetOwners([]libregraph.Identity{{
Id: libregraph.PtrString(user.GetOpaqueId()),
DisplayName: utils.ReadPlainFromOpaque(lock.GetOpaque(), "lockownername"),
}})
}
if lockTime, err := time.Parse(time.RFC3339, utils.ReadPlainFromOpaque(lock.GetOpaque(), "locktime")); err == nil {
info.SetCreatedDateTime(lockTime.UTC())
}
if expiration := lock.GetExpiration(); expiration != nil {
info.SetExpirationDateTime(time.Unix(int64(expiration.GetSeconds()), 0).UTC())
}
return info
}
// addShareTypes reads user and group shares off the grants, links from the share
// manager. Links are not stored as grants, so they need one filter per item,
// which is why the whole annotation is only built when it is selected.
func (g Graph) addShareTypes(ctx context.Context, items []libregraph.DriveItem, infos []*storageprovider.ResourceInfo) {
linkShares := g.listLinkShares(ctx, infos)
for i, info := range infos {
if i >= len(items) {
break
}
items[i].LibreGraphShareTypes = shareTypesOf(info, linkShares)
}
}
// shareTypesOf maps the grants on a resource, plus a hit in the link lookup, to
// the annotation values.
func shareTypesOf(info *storageprovider.ResourceInfo, linkShares map[string]struct{}) []string {
var types []string
for _, grant := range strings.Split(utils.ReadPlainFromOpaque(info.GetOpaque(), "share-types"), ",") {
switch grant {
case strconv.Itoa(int(storageprovider.GranteeType_GRANTEE_TYPE_USER)):
types = append(types, "user")
case strconv.Itoa(int(storageprovider.GranteeType_GRANTEE_TYPE_GROUP)):
types = append(types, "group")
}
}
if _, ok := linkShares[info.GetId().GetOpaqueId()]; ok {
types = append(types, "link")
}
return types
}
// listLinkShares returns the resource ids that carry a public link. A failed
// lookup is logged and treated as "no links", same as the WebDAV PROPFIND.
func (g Graph) listLinkShares(ctx context.Context, infos []*storageprovider.ResourceInfo) map[string]struct{} {
gatewayClient, err := g.gatewaySelector.Next()
if err != nil {
g.logger.Error().Err(err).Msg("could not select gateway client for public shares")
return nil
}
filters := make([]*link.ListPublicSharesRequest_Filter, 0, len(infos))
for _, info := range infos {
filters = append(filters, publicshare.ResourceIDFilter(info.GetId()))
}
res, err := gatewayClient.ListPublicShares(ctx, &link.ListPublicSharesRequest{Filters: filters})
if err != nil || res.GetStatus().GetCode() != cs3rpc.Code_CODE_OK {
g.logger.Error().Err(err).Msg("could not list public shares")
return nil
}
linkShares := make(map[string]struct{}, len(res.GetShare()))
for _, share := range res.GetShare() {
linkShares[share.GetResourceId().GetOpaqueId()] = struct{}{}
}
return linkShares
}
// metadataToFacet builds a DriveItem facet *T from CS3 arbitrary metadata under
// the "libre.graph.<facet>." key prefix. Nil when no such keys are present.
func metadataToFacet[T any](metadata map[string]string, facet string) *T {
@@ -11,9 +11,7 @@ import (
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/go-chi/chi/v5"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -208,40 +206,6 @@ var _ = Describe("Driveitems", func() {
Expect(res.Value[0].LibreGraphPermissionsActionsAllowedValues).To(BeNil())
})
It("returns the thumbnails when requested via $expand", func() {
cfg.Commons.OpenCloudURL = "https://cloud.test"
gatewayClient.On("ListStorageSpaces", mock.Anything, mock.Anything).Return(&provider.ListStorageSpacesResponse{
Status: status.NewOK(ctx),
StorageSpaces: []*provider.StorageSpace{{Owner: currentUser, Root: &provider.ResourceId{}}},
}, nil)
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
Status: status.NewOK(ctx),
Infos: []*provider.ResourceInfo{
{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
MimeType: "image/jpeg",
Mtime: utils.TimeToTS(time.Now()),
},
},
}, nil)
r := httptest.NewRequest(http.MethodGet, "/graph/v1.0/me/drive/root/children?$expand=thumbnails", nil)
r = r.WithContext(revactx.ContextSetUser(ctx, currentUser))
svc.GetRootDriveChildren(rr, r)
Expect(rr.Code).To(Equal(http.StatusOK))
data, err := io.ReadAll(rr.Body)
Expect(err).ToNot(HaveOccurred())
res := itemsList{}
Expect(json.Unmarshal(data, &res)).To(Succeed())
Expect(len(res.Value)).To(Equal(1))
Expect(res.Value[0].Thumbnails).To(HaveLen(1))
Expect(res.Value[0].Thumbnails[0].Small.GetUrl()).To(Equal(
"https://cloud.test/dav/spaces/storageid$spaceid!opaqueid" +
"?scalingup=0&preview=1&processor=thumbnail&x=36&y=36",
))
})
It("returns the allowed actions when requested via $select", func() {
gatewayClient.On("ListStorageSpaces", mock.Anything, mock.Anything).Return(&provider.ListStorageSpacesResponse{
Status: status.NewOK(ctx),
@@ -279,143 +243,6 @@ var _ = Describe("Driveitems", func() {
})
})
Describe("GetDriveItem", func() {
var (
folderInfo *provider.ResourceInfo
childInfo *provider.ResourceInfo
mtime = time.Now()
)
newRequest := func(query string) *http.Request {
r := httptest.NewRequest(http.MethodGet, "/graph/v1.0/drives/storageid$spaceid/items/storageid$spaceid!nodeid"+query, nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("driveID", "storageid$spaceid")
rctx.URLParams.Add("driveItemID", "storageid$spaceid!nodeid")
return r.WithContext(context.WithValue(revactx.ContextSetUser(ctx, currentUser), chi.RouteCtxKey, rctx))
}
getItem := func(r *http.Request) libregraph.DriveItem {
svc.GetDriveItem(rr, r)
Expect(rr.Code).To(Equal(http.StatusOK))
data, err := io.ReadAll(rr.Body)
Expect(err).ToNot(HaveOccurred())
item := libregraph.DriveItem{}
Expect(json.Unmarshal(data, &item)).To(Succeed())
return item
}
BeforeEach(func() {
folderInfo = &provider.ResourceInfo{
Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER,
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "nodeid"},
Etag: "etag",
Mtime: utils.TimeToTS(mtime),
}
gatewayClient.On("Stat", mock.Anything, mock.Anything).Return(&provider.StatResponse{
Status: status.NewOK(ctx),
Info: folderInfo,
}, nil)
childInfo = &provider.ResourceInfo{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
Etag: "etag",
Mtime: utils.TimeToTS(mtime),
}
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
Status: status.NewOK(ctx),
Infos: []*provider.ResourceInfo{childInfo},
}, nil)
})
It("leaves children unset without $expand", func() {
Expect(getItem(newRequest("")).Children).To(BeNil())
gatewayClient.AssertNotCalled(GinkgoT(), "ListContainer", mock.Anything, mock.Anything)
})
It("returns the children when requested via $expand", func() {
item := getItem(newRequest("?$expand=children"))
Expect(item.Children).To(HaveLen(1))
Expect(item.Children[0].GetId()).To(Equal("storageid$spaceid!opaqueid"))
Expect(item.Children[0].GetETag()).To(Equal("etag"))
})
It("leaves children unset for a file", func() {
folderInfo.Type = provider.ResourceType_RESOURCE_TYPE_FILE
Expect(getItem(newRequest("?$expand=children")).Children).To(BeNil())
gatewayClient.AssertNotCalled(GinkgoT(), "ListContainer", mock.Anything, mock.Anything)
})
Context("$expand=thumbnails", func() {
const previewURL = "https://cloud.test/dav/spaces/storageid$spaceid!nodeid" +
"?scalingup=0&preview=1&processor=thumbnail"
BeforeEach(func() {
cfg.Commons.OpenCloudURL = "https://cloud.test"
folderInfo.Type = provider.ResourceType_RESOURCE_TYPE_FILE
folderInfo.MimeType = "image/jpeg"
})
It("leaves thumbnails unset without $expand", func() {
Expect(getItem(newRequest("")).Thumbnails).To(BeNil())
})
It("returns the thumbnail urls when requested", func() {
thumbnails := getItem(newRequest("?$expand=thumbnails")).Thumbnails
Expect(thumbnails).To(HaveLen(1))
Expect(thumbnails[0].Small.GetUrl()).To(Equal(previewURL + "&x=36&y=36"))
Expect(thumbnails[0].Medium.GetUrl()).To(Equal(previewURL + "&x=48&y=48"))
Expect(thumbnails[0].Large.GetUrl()).To(Equal(previewURL + "&x=96&y=96"))
})
It("leaves thumbnails unset for a mime type the thumbnailer cannot render", func() {
folderInfo.MimeType = "application/zip"
Expect(getItem(newRequest("?$expand=thumbnails")).Thumbnails).To(BeNil())
})
It("adds them to expanded children as well", func() {
folderInfo.Type = provider.ResourceType_RESOURCE_TYPE_CONTAINER
folderInfo.MimeType = ""
childInfo.MimeType = "image/jpeg"
item := getItem(newRequest("?$expand=children,thumbnails"))
// a folder has no preview of its own
Expect(item.Thumbnails).To(BeNil())
Expect(item.Children).To(HaveLen(1))
Expect(item.Children[0].Thumbnails).To(HaveLen(1))
})
})
})
Describe("GetDriveItem $expand=children error", func() {
It("propagates a failing child listing", func() {
gatewayClient.On("Stat", mock.Anything, mock.Anything).Return(&provider.StatResponse{
Status: status.NewOK(ctx),
Info: &provider.ResourceInfo{
Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER,
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "nodeid"},
Mtime: utils.TimeToTS(time.Now()),
},
}, nil)
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
Status: status.NewNotFound(ctx, "not found"),
}, nil)
r := httptest.NewRequest(http.MethodGet, "/graph/v1.0/drives/storageid$spaceid/items/storageid$spaceid!nodeid?$expand=children", nil)
rctx := chi.NewRouteContext()
rctx.URLParams.Add("driveID", "storageid$spaceid")
rctx.URLParams.Add("driveItemID", "storageid$spaceid!nodeid")
r = r.WithContext(context.WithValue(revactx.ContextSetUser(ctx, currentUser), chi.RouteCtxKey, rctx))
svc.GetDriveItem(rr, r)
Expect(rr.Code).To(Equal(http.StatusNotFound))
})
})
Describe("GetDriveItemChildren", func() {
It("handles ListContainer not found", func() {
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
@@ -515,158 +342,6 @@ var _ = Describe("Driveitems", func() {
Expect(res.Value[0].PendingOperations).To(BeNil())
})
It("omits share types unless they are selected", func() {
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
Status: status.NewOK(ctx),
Infos: []*provider.ResourceInfo{
{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
Etag: "etag",
Mtime: utils.TimeToTS(mtime),
Opaque: utils.AppendPlainToOpaque(nil, "share-types", "1,2"),
},
},
}, nil)
res := assertItemsList(1)
Expect(res.Value[0].LibreGraphShareTypes).To(BeNil())
gatewayClient.AssertNotCalled(GinkgoT(), "ListPublicShares", mock.Anything, mock.Anything)
})
It("returns the share types of an item when selected", func() {
r = r.WithContext(r.Context())
q := r.URL.Query()
q.Add("$select", "@libre.graph.shareTypes")
r.URL.RawQuery = q.Encode()
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
Status: status.NewOK(ctx),
Infos: []*provider.ResourceInfo{
{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
Etag: "etag",
Mtime: utils.TimeToTS(mtime),
Opaque: utils.AppendPlainToOpaque(nil, "share-types", "1,2"),
},
},
}, nil)
gatewayClient.On("ListPublicShares", mock.Anything, mock.Anything).Return(&link.ListPublicSharesResponse{
Status: status.NewOK(ctx),
Share: []*link.PublicShare{
{ResourceId: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"}},
},
}, nil)
res := assertItemsList(1)
Expect(res.Value[0].LibreGraphShareTypes).To(ConsistOf("user", "group", "link"))
})
It("reports only the link when the item has no grants", func() {
q := r.URL.Query()
q.Add("$select", "@libre.graph.shareTypes")
r.URL.RawQuery = q.Encode()
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
Status: status.NewOK(ctx),
Infos: []*provider.ResourceInfo{
{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
Etag: "etag",
Mtime: utils.TimeToTS(mtime),
},
},
}, nil)
gatewayClient.On("ListPublicShares", mock.Anything, mock.Anything).Return(&link.ListPublicSharesResponse{
Status: status.NewOK(ctx),
Share: []*link.PublicShare{
{ResourceId: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"}},
},
}, nil)
res := assertItemsList(1)
Expect(res.Value[0].LibreGraphShareTypes).To(ConsistOf("link"))
})
It("keeps the grant types when the public share lookup fails", func() {
q := r.URL.Query()
q.Add("$select", "@libre.graph.shareTypes")
r.URL.RawQuery = q.Encode()
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
Status: status.NewOK(ctx),
Infos: []*provider.ResourceInfo{
{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
Etag: "etag",
Mtime: utils.TimeToTS(mtime),
Opaque: utils.AppendPlainToOpaque(nil, "share-types", "1"),
},
},
}, nil)
gatewayClient.On("ListPublicShares", mock.Anything, mock.Anything).Return(nil, errors.New("nope"))
res := assertItemsList(1)
Expect(res.Value[0].LibreGraphShareTypes).To(ConsistOf("user"))
})
It("returns the lock info of a locked item", func() {
// a lock time with a non-UTC offset, the way reva writes it
lockTime := time.Now().Truncate(time.Second).In(time.FixedZone("CEST", 2*60*60))
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
Status: status.NewOK(ctx),
Infos: []*provider.ResourceInfo{
{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
Etag: "etag",
Mtime: utils.TimeToTS(mtime),
Lock: &provider.Lock{
Type: provider.LockType_LOCK_TYPE_EXCL,
AppName: "Collabora",
User: &userpb.UserId{OpaqueId: "user-id"},
Expiration: &typesv1beta1.Timestamp{Seconds: uint64(lockTime.Add(time.Hour).Unix())},
Opaque: utils.AppendPlainToOpaque(
utils.AppendPlainToOpaque(nil, "lockownername", "Alice Hansen"),
"locktime", lockTime.Format(time.RFC3339)),
},
},
},
}, nil)
res := assertItemsList(1)
lock := res.Value[0].LockInfo
Expect(lock).ToNot(BeNil())
Expect(lock.GetLockType()).To(Equal("exclusive"))
Expect(lock.GetLibreGraphAppName()).To(Equal("Collabora"))
Expect(lock.GetCreatedDateTime()).To(BeTemporally("==", lockTime))
Expect(lock.GetCreatedDateTime().Location()).To(Equal(time.UTC))
Expect(lock.GetExpirationDateTime().Location()).To(Equal(time.UTC))
Expect(lock.GetExpirationDateTime()).To(BeTemporally("==", lockTime.Add(time.Hour)))
Expect(lock.GetOwners()).To(HaveLen(1))
Expect(lock.GetOwners()[0].GetId()).To(Equal("user-id"))
Expect(lock.GetOwners()[0].GetDisplayName()).To(Equal("Alice Hansen"))
})
It("omits the lock info for an unlocked item", func() {
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
Status: status.NewOK(ctx),
Infos: []*provider.ResourceInfo{
{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Id: &provider.ResourceId{StorageId: "storageid", SpaceId: "spaceid", OpaqueId: "opaqueid"},
Etag: "etag",
Mtime: utils.TimeToTS(mtime),
},
},
}, nil)
Expect(assertItemsList(1).Value[0].LockInfo).To(BeNil())
})
It("reports a pending content update while the item is being processed", func() {
gatewayClient.On("ListContainer", mock.Anything, mock.Anything).Return(&provider.ListContainerResponse{
Status: status.NewOK(ctx),
+26 -2
View File
@@ -1,12 +1,14 @@
package svc
import (
"fmt"
"net/http"
"strings"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
"github.com/go-chi/render"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/services/thumbnails/pkg/thumbnail"
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
)
@@ -42,8 +44,30 @@ func (g Graph) GetSharedByMe(w http.ResponseWriter, r *http.Request) {
expandThumbnails := strings.Contains(expand, "thumbnails")
if expandThumbnails {
for k, item := range driveItems {
setShareThumbnails(&item, item.GetId(), g.config.Commons.OpenCloudURL)
driveItems[k] = item
mt := item.GetFile().MimeType
if mt == nil {
continue
}
_, match := thumbnail.SupportedMimeTypes[*mt]
if match {
baseUrl := fmt.Sprintf("%s/dav/spaces/%s?scalingup=0&preview=1&processor=thumbnail",
g.config.Commons.OpenCloudURL,
item.GetId())
smallUrl := baseUrl + "&x=36&y=36"
mediumUrl := baseUrl + "&x=48&y=48"
largeUrl := baseUrl + "&x=96&y=96"
item.SetThumbnails([]libregraph.ThumbnailSet{
{
Small: &libregraph.Thumbnail{Url: &smallUrl},
Medium: &libregraph.Thumbnail{Url: &mediumUrl},
Large: &libregraph.Thumbnail{Url: &largeUrl},
},
})
driveItems[k] = item // assign modified item back to the map
}
}
}
+26 -2
View File
@@ -2,6 +2,7 @@ package svc
import (
"context"
"fmt"
"net/http"
"strings"
@@ -12,6 +13,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/share"
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
"github.com/opencloud-eu/opencloud/services/thumbnails/pkg/thumbnail"
)
// ListSharedWithMe lists the files shared with the current user.
@@ -71,8 +73,30 @@ func (g Graph) listSharedWithMe(ctx context.Context, expandThumbnails bool) ([]l
if expandThumbnails {
for k, item := range driveItems {
setShareThumbnails(&item, item.RemoteItem.GetId(), g.config.Commons.OpenCloudURL)
driveItems[k] = item
mt := item.GetFile().MimeType
if mt == nil {
continue
}
_, match := thumbnail.SupportedMimeTypes[*mt]
if match {
baseUrl := fmt.Sprintf("%s/dav/spaces/%s?scalingup=0&preview=1&processor=thumbnail",
g.config.Commons.OpenCloudURL,
item.RemoteItem.GetId())
smallUrl := baseUrl + "&x=36&y=36"
mediumUrl := baseUrl + "&x=48&y=48"
largeUrl := baseUrl + "&x=96&y=96"
item.SetThumbnails([]libregraph.ThumbnailSet{
{
Small: &libregraph.Thumbnail{Url: &smallUrl},
Medium: &libregraph.Thumbnail{Url: &mediumUrl},
Large: &libregraph.Thumbnail{Url: &largeUrl},
},
})
driveItems[k] = item // assign modified item back to the map
}
}
}
@@ -1,68 +0,0 @@
package svc
import (
"fmt"
"net/http"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/opencloud/services/thumbnails/pkg/thumbnail"
)
const (
thumbnailBoxSmall = 36
thumbnailBoxMedium = 48
thumbnailBoxLarge = 96
)
func (g Graph) setDriveItemsThumbnails(r *http.Request, items []libregraph.DriveItem, infos []*provider.ResourceInfo) {
if !driveItemRelationExpanded(r, _expandThumbnails) {
return
}
for i := range items {
if i < len(infos) {
setDriveItemThumbnails(&items[i], infos[i], g.config.Commons.OpenCloudURL)
}
}
}
func setDriveItemThumbnails(item *libregraph.DriveItem, res *provider.ResourceInfo, baseURL string) {
if set := previewThumbnailSet(res, baseURL); set != nil {
item.SetThumbnails([]libregraph.ThumbnailSet{*set})
}
}
// previewThumbnailSet returns nil when the thumbnailer cannot render the resource.
func previewThumbnailSet(res *provider.ResourceInfo, baseURL string) *libregraph.ThumbnailSet {
if !thumbnail.IsMimeTypeSupported(res.GetMimeType()) {
return nil
}
return thumbnailSetFor(baseURL, storagespace.FormatResourceID(res.GetId()))
}
// thumbnailSetFor builds the urls of the WebDAV preview endpoint.
func thumbnailSetFor(baseURL, itemID string) *libregraph.ThumbnailSet {
base := fmt.Sprintf("%s/dav/spaces/%s?scalingup=0&preview=1&processor=thumbnail", baseURL, itemID)
return &libregraph.ThumbnailSet{
Small: previewThumbnail(base, thumbnailBoxSmall),
Medium: previewThumbnail(base, thumbnailBoxMedium),
Large: previewThumbnail(base, thumbnailBoxLarge),
}
}
func previewThumbnail(base string, box int32) *libregraph.Thumbnail {
url := fmt.Sprintf("%s&x=%d&y=%d", base, box, box)
return &libregraph.Thumbnail{Url: &url}
}
// setShareThumbnails works off the driveItem, the share listings have no resource
// info. The id comes separately, a received share carries it on its remote item.
func setShareThumbnails(item *libregraph.DriveItem, itemID, baseURL string) {
mimeType := item.GetFile().MimeType
if itemID == "" || mimeType == nil || !thumbnail.IsMimeTypeSupported(*mimeType) {
return
}
item.SetThumbnails([]libregraph.ThumbnailSet{*thumbnailSetFor(baseURL, itemID)})
}
@@ -7,16 +7,15 @@
# Jörn Friedrich Dreyer <jfd@butonic.de>, 2025
# Jonas, 2025
# Jannik Stehle, 2026
# Jannick Kuhr, 2026
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2026-09-03 23:16+0000\n"
"POT-Creation-Date: 2026-09-02 23:17+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Jannick Kuhr, 2026\n"
"Last-Translator: Jannik Stehle, 2026\n"
"Language-Team: German (https://app.transifex.com/opencloud-eu/teams/204053/de/)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
@@ -174,9 +173,6 @@ msgid ""
"\n"
"Even though this share has been revoked you still might have access through other shares and/or space memberships."
msgstr ""
"{ShareSharer} hat die Freigabe für »{ShareFolder}« für Sie aufgehoben.\n"
"\n"
"Auch wenn diese Freigabe widerrufen wurde, haben Sie möglicherweise weiterhin Zugriff über andere Freigaben und/oder Mitgliedschaften in Bereichen."
#. ShareCreated email template, Subject field (resolves directly)
#: pkg/email/templates.go:17
@@ -187,7 +183,6 @@ msgstr "{ShareSharer} hat \"{ShareFolder}\" mit Ihnen geteilt"
#: pkg/email/templates.go:30
msgid "{ShareSharer} unshared '{ShareFolder}' with you"
msgstr ""
"{ShareSharer} hat die Freigabe von »{ShareFolder}« für Sie aufgehoben."
#. SharedSpace email template, resolves via {{ .MessageBody }}
#: pkg/email/templates.go:61
@@ -1,212 +0,0 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
# Translators:
# Roy-Magne Mo, 2026
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2026-09-07 23:16+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Roy-Magne Mo, 2026\n"
"Language-Team: Norwegian (https://app.transifex.com/opencloud-eu/teams/204053/no/)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Language: no\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
#. UnsharedSpace email template, resolves via {{ .CallToAction }}
#: pkg/email/templates.go:78
msgid "Click here to check it: {ShareLink}"
msgstr "Klikk her for å sjekke det: {ShareLink}"
#. ShareCreated email template, resolves via {{ .CallToAction }}
#. SharedSpace email template, resolves via {{ .CallToAction }}
#: pkg/email/templates.go:23 pkg/email/templates.go:63
msgid "Click here to view it: {ShareLink}"
msgstr "Klikk her for å se den: {ShareLink}"
#: pkg/email/templates.go:139
msgid "Hello {RecipientName},"
msgstr "Hei {RecipientName},"
#. ShareCreated email template, resolves via {{ .Greeting }}
#: pkg/email/templates.go:19
msgid "Hello {ShareGrantee}"
msgstr "Hei {ShareGrantee}"
#. ShareRemoved email template, resolves via {{ .Greeting }}
#. ShareExpired email template, resolves via {{ .Greeting }}
#: pkg/email/templates.go:32 pkg/email/templates.go:45
msgid "Hello {ShareGrantee},"
msgstr "Hei {ShareGrantee},"
#. SharedSpace email template, resolves via {{ .Greeting }}
#. UnsharedSpace email template, resolves via {{ .Greeting }}
#. MembershipExpired email template, resolves via {{ .Greeting }}
#: pkg/email/templates.go:59 pkg/email/templates.go:72
#: pkg/email/templates.go:87
msgid "Hello {SpaceGrantee},"
msgstr "Hei {SpaceGrantee},"
#. Grouped email template, resolves via {{ .Greeting }}
#: pkg/email/templates.go:131
msgid "Hi {DisplayName},"
msgstr "Hei {DisplayName},"
#. ScienceMeshInviteTokenGenerated email template, resolves via {{ .Greeting
#. }}
#. ScienceMeshInviteTokenGeneratedWithoutShareLink email template, resolves
#. via {{ .Greeting }}
#: pkg/email/templates.go:100 pkg/email/templates.go:117
msgid "Hi,"
msgstr "Hei,"
#. MembershipExpired email template, Subject field (resolves directly)
#: pkg/email/templates.go:85
msgid "Membership of '{SpaceName}' expired at {ExpiredAt}"
msgstr "Medlemskap i '{SpaceName}' utløpt {ExpiredAt}"
#. Grouped email template, Subject field (resolves directly)
#: pkg/email/templates.go:129
msgid "Report"
msgstr "Rapport"
#. ScienceMeshInviteTokenGenerated email template, Subject field (resolves
#. directly)
#. ScienceMeshInviteTokenGeneratedWithoutShareLink email template, Subject
#. field (resolves directly)
#: pkg/email/templates.go:98 pkg/email/templates.go:115
msgid "ScienceMesh: {InitiatorName} wants to collaborate with you"
msgstr "ScienceMesh: {InitiatorName} ønsker å samarbeide med deg"
#. ShareExpired email template, Subject field (resolves directly)
#: pkg/email/templates.go:43
msgid "Share to '{ShareFolder}' expired at {ExpiredAt}"
msgstr "Deling til '{ShareFolder}' utløpt {ExpiredAt}"
#: pkg/email/templates.go:141
msgid "You can view the mention here: {ResourceLink}"
msgstr "Du kan se omtalen her: {ResourceLink}"
#: pkg/email/templates.go:138
msgid "You were mentioned in '{ResourceName}'"
msgstr "Du ble nevnt i «{ResourceName}»"
#. MembershipExpired email template, resolves via {{ .MessageBody }}
#: pkg/email/templates.go:89
msgid ""
"Your membership of space {SpaceName} has expired at {ExpiredAt}\n"
"\n"
"Even though this membership has expired you still might have access through other shares and/or space memberships"
msgstr ""
"Medlemskapet ditt i rom {SpaceName} utløp {ExpiredAt}\n"
"\n"
"Selv om dette medlemskapet har utløpt, kan du fortsatt ha tilgang gjennom andre delinger og/eller rommedlemskap"
#. ShareExpired email template, resolves via {{ .MessageBody }}
#: pkg/email/templates.go:47
msgid ""
"Your share to {ShareFolder} has expired at {ExpiredAt}\n"
"\n"
"Even though this share has been revoked you still might have access through other shares and/or space memberships."
msgstr ""
"Deltakelsen din i {ShareFolder} utløp {ExpiredAt}\n"
"\n"
"Selv om denne deltakelsen er trukket tilbake, kan du fortsatt ha tilgang gjennom andre deltakelser og/eller plassmedlemskap."
#: pkg/email/templates.go:140
msgid "{AuthorName} mentioned you in \"{ResourceName}\"."
msgstr "{AuthorName} nevnte deg i «{ResourceName}»."
#. ScienceMeshInviteTokenGeneratedWithoutShareLink email template, resolves
#. via {{ .MessageBody }}
#: pkg/email/templates.go:119
msgid ""
"{ShareSharer} ({ShareSharerMail}) wants to start sharing collaboration resources with you.\n"
"Please visit your federation settings and use the following details:\n"
" Token: {Token}\n"
" ProviderDomain: {ProviderDomain}"
msgstr ""
"{ShareSharer} ({ShareSharerMail}) ønsker å begynne å dele samarbeidsressurser med deg.\n"
"Vennligst besøk føderasjonsinnstillingene dine og bruk følgende detaljer:\n"
" Token: {Token}\n"
" ProviderDomain: {ProviderDomain}"
#. ScienceMeshInviteTokenGenerated email template, resolves via {{
#. .MessageBody }}
#: pkg/email/templates.go:102
msgid ""
"{ShareSharer} ({ShareSharerMail}) wants to start sharing collaboration resources with you.\n"
"To accept the invite, please visit the following URL:\n"
"{ShareLink}\n"
"\n"
"Alternatively, you can visit your federation settings and use the following details:\n"
" Token: {Token}\n"
" ProviderDomain: {ProviderDomain}"
msgstr ""
"{ShareSharer} ({ShareSharerMail}) ønsker å begynne å dele samarbeidsressurser med deg.\n"
"For å akseptere invitasjonen, besøk følgende URL:\n"
"{ShareLink}\n"
"\n"
"Alternativt kan du besøke dine føderasjonsinnstillinger og bruke følgende detaljer:\n"
" Token: {Token}\n"
" ProviderDomain: {ProviderDomain}"
#. ShareCreated email template, resolves via {{ .MessageBody }}
#: pkg/email/templates.go:21
msgid "{ShareSharer} has shared \"{ShareFolder}\" with you."
msgstr "{ShareSharer} har delt «{ShareFolder}» med deg."
#. ShareRemoved email template, resolves via {{ .MessageBody }}
#: pkg/email/templates.go:34
msgid ""
"{ShareSharer} has unshared '{ShareFolder}' with you.\n"
"\n"
"Even though this share has been revoked you still might have access through other shares and/or space memberships."
msgstr ""
"{ShareSharer} har fjernet deling '{ShareFolder}' til deg.\n"
"\n"
"Selv om denne delingen er trukket tilbake, kan du fortsatt ha tilgang gjennom andre delinger og/eller medlemskap i rommet."
#. ShareCreated email template, Subject field (resolves directly)
#: pkg/email/templates.go:17
msgid "{ShareSharer} shared '{ShareFolder}' with you"
msgstr "{ShareSharer} delte '{ShareFolder}' med deg"
#. ShareRemoved email template, Subject field (resolves directly)
#: pkg/email/templates.go:30
msgid "{ShareSharer} unshared '{ShareFolder}' with you"
msgstr "{ShareSharer} deler ikke '{ShareFolder}' med deg"
#. SharedSpace email template, resolves via {{ .MessageBody }}
#: pkg/email/templates.go:61
msgid "{SpaceSharer} has invited you to join \"{SpaceName}\"."
msgstr "{SpaceSharer} har invitert deg til å bli med i «{SpaceName}»."
#. UnsharedSpace email template, resolves via {{ .MessageBody }}
#: pkg/email/templates.go:74
msgid ""
"{SpaceSharer} has removed you from \"{SpaceName}\".\n"
"\n"
"You might still have access through your other groups or direct membership."
msgstr ""
"{SpaceSharer} har fjernet deg fra «{SpaceName}».\n"
"\n"
"Du kan fortsatt ha tilgang gjennom andre grupper eller direkte medlemskap."
#. SharedSpace email template, Subject field (resolves directly)
#: pkg/email/templates.go:57
msgid "{SpaceSharer} invited you to join {SpaceName}"
msgstr "{SpaceSharer} inviterte deg til å bli med i {SpaceName}"
#. UnsharedSpace email template, Subject field (resolves directly)
#: pkg/email/templates.go:70
msgid "{SpaceSharer} removed you from {SpaceName}"
msgstr "{SpaceSharer} fjernet deg fra {SpaceName}"
@@ -12,7 +12,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2026-09-03 23:16+0000\n"
"POT-Creation-Date: 2026-09-02 23:17+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Daniel Nylander <po@danielnylander.se>, 2026\n"
"Language-Team: Swedish (https://app.transifex.com/opencloud-eu/teams/204053/sv/)\n"
@@ -172,9 +172,6 @@ msgid ""
"\n"
"Even though this share has been revoked you still might have access through other shares and/or space memberships."
msgstr ""
"{ShareSharer} har slutat dela ”{ShareFolder}” med dig.\\n\\nÄven om denna "
"delning har återkallats kan du fortfarande ha åtkomst genom andra delningar "
"och/eller medlemskap i utrymmen."
#. ShareCreated email template, Subject field (resolves directly)
#: pkg/email/templates.go:17
@@ -184,7 +181,7 @@ msgstr "{ShareSharer} delade '{ShareFolder}' med dig"
#. ShareRemoved email template, Subject field (resolves directly)
#: pkg/email/templates.go:30
msgid "{ShareSharer} unshared '{ShareFolder}' with you"
msgstr "{ShareSharer} har slutat dela ”{ShareFolder}” med dig"
msgstr ""
#. SharedSpace email template, resolves via {{ .MessageBody }}
#: pkg/email/templates.go:61
+2 -2
View File
@@ -29,7 +29,7 @@ type Service struct {
// GRPC defines the available grpc configuration.
type GRPC struct {
Disabled bool `yaml:"disabled" env:"POLICIES_GRPC_DISABLED" desc:"Disables listening for GRPC API calls. Set this to true if the service should only handle requests through events." introductionVersion:"%%NEXT%%"`
Disabled bool `yaml:"disabled" env:"POLICIES_GRPC_DISABLED" desc:"Disables listening for GRPC API calls. Set this to true if the service should only handle requests through events." introductionVersion:"%NEXT%"`
Addr string `yaml:"addr" env:"POLICIES_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"1.0.0"`
Namespace string `yaml:"-"`
TLS *shared.GRPCServiceTLS `yaml:"tls"`
@@ -50,7 +50,7 @@ type Postprocessing struct {
// Events combines the configuration options for the event bus.
type Events struct {
Disabled bool `yaml:"disabled" env:"POLICIES_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle GRPC requests." introductionVersion:"%%NEXT%%"`
Disabled bool `yaml:"disabled" env:"POLICIES_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle GRPC requests." introductionVersion:"%NEXT%"`
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;POLICIES_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;POLICIES_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;POLICIES_EVENTS_TLS_INSECURE" desc:"Whether the server should skip the client certificate verification during the TLS handshake." introductionVersion:"1.0.0"`
+1 -19
View File
@@ -8,7 +8,6 @@ import (
"strings"
"time"
"github.com/golang-jwt/jwt/v5"
"github.com/pkg/errors"
"github.com/vmihailenco/msgpack/v5"
"go-micro.dev/v4/store"
@@ -187,13 +186,6 @@ func verifyExpiresAt(claims map[string]any, cmp time.Time) bool {
return cmp.Before(expiry)
}
// isExpectedTokenTimeError reports whether the error is caused by a token that
// failed its expiry (or not-before) validation. These are expected as part of
// the normal token renewal cycle and should not be logged at error level.
func isExpectedTokenTimeError(err error) bool {
return errors.Is(err, jwt.ErrTokenExpired) || errors.Is(err, jwt.ErrTokenNotValidYet)
}
func (m OIDCAuthenticator) shouldServe(req *http.Request) bool {
if m.OIDCIss == "" {
return false
@@ -220,17 +212,7 @@ func (m *OIDCAuthenticator) Authenticate(r *http.Request) (*http.Request, bool)
claims, newSession, err := m.getClaims(token, r)
if err != nil {
host, port, _ := net.SplitHostPort(r.RemoteAddr)
// Expired (or not yet valid) tokens are a normal part of the token
// renewal cycle: the client hits the proxy with a token that is just
// past its expiry, receives a 401 challenge and refreshes. This is not
// a failure of the proxy, so we log it at debug level to avoid noise.
logEvent := m.Logger.Debug()
if !isExpectedTokenTimeError(err) {
logEvent = m.Logger.Error()
}
logEvent.
m.Logger.Error().
Err(err).
Str("authenticator", "oidc").
Str("path", r.URL.Path).
@@ -1,7 +1,6 @@
package middleware
import (
"bytes"
"net/http"
"net/http/httptest"
"time"
@@ -12,7 +11,6 @@ import (
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/pkg/oidc"
oidcmocks "github.com/opencloud-eu/opencloud/pkg/oidc/mocks"
"github.com/rs/zerolog"
"github.com/stretchr/testify/mock"
"go-micro.dev/v4/store"
)
@@ -108,69 +106,4 @@ var _ = Describe("Authenticating requests", Label("OIDCAuthenticator"), func() {
Expect(req2).ToNot(BeNil())
})
})
When("the request contains an expired token", func() {
var buf bytes.Buffer
expiredAuthenticator := &OIDCAuthenticator{
OIDCIss: "http://idp.example.com",
Logger: log.Logger{zerolog.New(&buf).Level(zerolog.DebugLevel)},
oidcClient: &oidcmocks.OIDCClient{},
userInfoCache: store.NewMemoryStore(),
skipUserInfo: true,
}
BeforeEach(func() {
expiredAuthenticator.oidcClient.(*oidcmocks.OIDCClient).On("VerifyAccessToken", mock.Anything, mock.Anything).Return(
oidc.RegClaimsWithSID{}, jwt.MapClaims{}, jwt.ErrTokenExpired,
)
})
It("should reject the request", func() {
req := httptest.NewRequest(http.MethodGet, "http://example.com/example/path", http.NoBody)
req.Header.Set(_headerAuthorization, "Bearer jwt.token.sig")
req2, valid := expiredAuthenticator.Authenticate(req)
Expect(valid).To(Equal(false))
Expect(req2).To(BeNil())
})
It("should log the authentication failure at debug level", func() {
req := httptest.NewRequest(http.MethodGet, "http://example.com/example/path", http.NoBody)
req.Header.Set(_headerAuthorization, "Bearer jwt.token.sig")
expiredAuthenticator.Authenticate(req)
Expect(buf.String()).To(ContainSubstring("failed to authenticate the request"))
Expect(buf.String()).To(ContainSubstring(`"level":"debug"`))
Expect(buf.String()).ToNot(ContainSubstring(`"level":"error"`))
})
})
When("the request contains a token that cannot be verified", func() {
var buf bytes.Buffer
failingAuthenticator := &OIDCAuthenticator{
OIDCIss: "http://idp.example.com",
Logger: log.Logger{zerolog.New(&buf).Level(zerolog.DebugLevel)},
oidcClient: &oidcmocks.OIDCClient{},
userInfoCache: store.NewMemoryStore(),
skipUserInfo: true,
}
BeforeEach(func() {
failingAuthenticator.oidcClient.(*oidcmocks.OIDCClient).On("VerifyAccessToken", mock.Anything, mock.Anything).Return(
oidc.RegClaimsWithSID{}, jwt.MapClaims{}, jwt.ErrTokenMalformed,
)
})
It("should log the authentication failure at error level", func() {
req := httptest.NewRequest(http.MethodGet, "http://example.com/example/path", http.NoBody)
req.Header.Set(_headerAuthorization, "Bearer jwt.token.sig")
failingAuthenticator.Authenticate(req)
Expect(buf.String()).To(ContainSubstring("failed to authenticate the request"))
Expect(buf.String()).To(ContainSubstring(`"level":"error"`))
})
})
})
@@ -236,16 +236,6 @@ func (m SignedURLAuthenticator) Authenticate(r *http.Request) (*http.Request, bo
}
func (m SignedURLAuthenticator) authenticate(r *http.Request) (*http.Request, bool) {
if err := m.requestMethodIsAllowed(r.Method); err != nil {
m.Logger.Error().
Err(err).
Str("authenticator", "signed_url_jwt").
Str("path", r.URL.Path).
Str("method", r.Method).
Msg("Request method not allowed for signed urls")
return nil, false
}
u := r.URL.String()
if !r.URL.IsAbs() {
u = "https://" + r.Host + u
@@ -2,13 +2,11 @@ package middleware
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/proxy/pkg/config"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/signedurl"
@@ -73,27 +71,6 @@ func TestSignedURLAuth_shouldServe(t *testing.T) {
}
}
func TestSignedURLAuth_authenticateRejectsDisallowedMethods(t *testing.T) {
signer, err := signedurl.NewJWTSignedURL(signedurl.WithSecret("secret"))
if err != nil {
t.Fatalf("failed to create signer: %v", err)
}
signed, err := signer.Sign("https://example.com/dav/spaces/file.txt", "userid", time.Minute)
if err != nil {
t.Fatalf("failed to sign url: %v", err)
}
pua := SignedURLAuthenticator{Logger: log.NewLogger(), URLVerifier: signer}
pua.PreSignedURLConfig.AllowedHTTPMethods = []string{"GET"}
for _, method := range []string{http.MethodPost, http.MethodPut, http.MethodDelete, "PROPFIND", "MOVE"} {
r := httptest.NewRequest(method, signed, nil)
if _, ok := pua.authenticate(r); ok {
t.Errorf("expected %s with a signed url to be rejected", method)
}
}
}
func TestSignedURLAuth_allRequiredParametersPresent(t *testing.T) {
pua := SignedURLAuthenticator{}
baseURL := "https://example.com/example.jpg?"
+2 -2
View File
@@ -13,7 +13,7 @@ Fill the new index by indexing all spaces again:
```shell
# the service keeps running while it happens
opencloud search index --all-spaces --insecure
opencloud search index --all-spaces
```
Once the new index is filled, every index but the one with the highest
@@ -31,7 +31,7 @@ The new index is a directory next to the old `bleve` one, both in
bleve index cannot be copied, index all spaces again:
```shell
opencloud search index --all-spaces --insecure
opencloud search index --all-spaces
```
Once the new index is filled, every directory but the one with the highest
+2 -2
View File
@@ -124,14 +124,14 @@ opencloud search index --space $SPACE_ID
It can also be used to re-index all spaces:
```shell
opencloud search index --all-spaces --insecure
opencloud search index --all-spaces
```
Please note that a reindex only picks up new or changed files. Files that have already been indexed are not scanned again, even if the configuration or the whole extractor has been changed. To force a full rescan (re-running the extractor on every file) you need to use the `force-rescan` flag:
```shell
opencloud search index --all-spaces --force-rescan --insecure
opencloud search index --all-spaces --force-rescan
```
## Metrics
+22 -20
View File
@@ -75,10 +75,14 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
},
)
// Scope below the space root: restrict at query level so totals and
// paging respect the path too. The folder term matches the folder and
// its descendants (see PathAnalyzer).
// paging respect the path too. Path is a case-preserving keyword
// (paths act as references, /Foo and /foo are distinct), so the exact
// folder or the folder prefix matches all of, and only, the scope.
if requestedPath := utils.MakeRelativePath(sir.Ref.Path); requestedPath != "." {
q.Conjuncts = append(q.Conjuncts, &query.TermQuery{FieldVal: "Path", Term: requestedPath})
q.Conjuncts = append(q.Conjuncts, query.NewDisjunctionQuery([]query.Query{
&query.TermQuery{FieldVal: "Path", Term: requestedPath},
&query.PrefixQuery{FieldVal: "Path", Prefix: requestedPath + "/"},
}))
}
}
@@ -121,23 +125,21 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
ResourceId: resourceIDtoSearchID(rootID),
Path: getFieldValue[string](hit.Fields, "Path"),
},
Id: resourceIDtoSearchID(rID),
Name: getFieldValue[string](hit.Fields, "Name"),
ParentId: resourceIDtoSearchID(pID),
Size: uint64(getFieldValue[float64](hit.Fields, "Size")),
Type: uint64(getFieldValue[float64](hit.Fields, "Type")),
MimeType: getFieldValue[string](hit.Fields, "MimeType"),
Deleted: getFieldValue[bool](hit.Fields, "Deleted"),
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
Video: hitToFacet[searchMessage.Video](hit.Fields, "video"),
MotionPhoto: hitToFacet[searchMessage.MotionPhoto](hit.Fields, "motionPhoto"),
LivePhoto: hitToFacet[searchMessage.LivePhoto](hit.Fields, "livePhoto"),
Id: resourceIDtoSearchID(rID),
Name: getFieldValue[string](hit.Fields, "Name"),
ParentId: resourceIDtoSearchID(pID),
Size: uint64(getFieldValue[float64](hit.Fields, "Size")),
Type: uint64(getFieldValue[float64](hit.Fields, "Type")),
MimeType: getFieldValue[string](hit.Fields, "MimeType"),
Deleted: getFieldValue[bool](hit.Fields, "Deleted"),
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
Video: hitToFacet[searchMessage.Video](hit.Fields, "video"),
},
}
+8
View File
@@ -1,6 +1,8 @@
package bleve
import (
"regexp"
bleveSearch "github.com/blevesearch/bleve/v2/search"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
@@ -9,6 +11,8 @@ import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
var queryEscape = regexp.MustCompile(`([` + regexp.QuoteMeta(`+=&|><!(){}[]^\"~*?:\/`) + `\-\s])`)
func getFieldValue[T any](m map[string]any, key string) (out T) {
val, ok := m[key]
if !ok {
@@ -80,3 +84,7 @@ func hitToFacet[T any](fields map[string]any, prefix string) *T {
func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource {
return mapping.Deserialize[search.Resource](match.Fields)
}
func escapeQuery(s string) string {
return queryEscape.ReplaceAllString(s, "\\$1")
}
@@ -1,82 +0,0 @@
package hierarchy
import (
"bytes"
"strconv"
"github.com/blevesearch/bleve/v2/analysis"
"github.com/blevesearch/bleve/v2/registry"
)
// emits every prefix up to a level: "./a/b" -> ".", "./a", "./a/b" with
// delimiter "/", one level per byte without. tag_depth prepends "<depth>/".
const Name = "hierarchy"
type Tokenizer struct {
delimiter []byte
tagDepth bool
}
func (t *Tokenizer) Tokenize(input []byte) analysis.TokenStream {
if len(input) == 0 {
return nil
}
var out analysis.TokenStream
emit := func(depth, end int) {
term := input[:end]
if t.tagDepth {
term = strconv.AppendInt(make([]byte, 0, end+4), int64(depth), 10)
term = append(term, '/')
term = append(term, input[:end]...)
}
out = append(out, &analysis.Token{
Term: term,
Position: depth,
Start: 0,
End: end,
Type: analysis.AlphaNumeric,
})
}
if len(t.delimiter) == 0 {
for i := range input {
emit(i+1, i+1)
}
return out
}
depth := 0
for start := 0; start <= len(input); {
i := bytes.Index(input[start:], t.delimiter)
if i < 0 {
if start < len(input) {
depth++
emit(depth, len(input))
}
break
}
if i > 0 {
depth++
emit(depth, start+i)
}
start += i + len(t.delimiter)
}
return out
}
func Constructor(config map[string]interface{}, _ *registry.Cache) (analysis.Tokenizer, error) {
t := &Tokenizer{}
if d, ok := config["delimiter"].(string); ok {
t.delimiter = []byte(d)
}
if v, ok := config["tag_depth"].(bool); ok {
t.tagDepth = v
}
return t, nil
}
func init() {
if err := registry.RegisterTokenizer(Name, Constructor); err != nil {
panic(err)
}
}
@@ -1,13 +0,0 @@
package hierarchy_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestHierarchy(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "hierarchy tokenizer")
}
@@ -1,56 +0,0 @@
package hierarchy_test
import (
"github.com/blevesearch/bleve/v2/analysis"
"github.com/blevesearch/bleve/v2/registry"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve/hierarchy"
)
func terms(ts analysis.TokenStream) []string {
out := make([]string, 0, len(ts))
for _, t := range ts {
out = append(out, string(t.Term))
}
return out
}
func tokenize(config map[string]any, input string) []string {
tok, err := hierarchy.Constructor(config, registry.NewCache())
Expect(err).ToNot(HaveOccurred())
return terms(tok.Tokenize([]byte(input)))
}
var _ = Describe("hierarchy tokenizer", func() {
path := map[string]any{"delimiter": "/"}
geohash := map[string]any{"tag_depth": true}
DescribeTable("emits every prefix up to a level boundary",
func(config map[string]any, input string, want []string) {
Expect(tokenize(config, input)).To(Equal(want))
},
Entry("relative path", path, "./a/b.txt", []string{".", "./a", "./a/b.txt"}),
Entry("space root", path, ".", []string{"."}),
Entry("trailing delimiter is not a level", path, "./a/", []string{".", "./a"}),
Entry("delimiter only", path, "/", []string{}),
Entry("leading delimiter", path, "/abs/x", []string{"/abs", "/abs/x"}),
Entry("double delimiter", path, "./a//b", []string{".", "./a", "./a//b"}),
Entry("spaces and special characters stay literal", path, "./odd name*[1]/f:x?.txt",
[]string{".", "./odd name*[1]", "./odd name*[1]/f:x?.txt"}),
Entry("empty input", path, "", []string{}),
Entry("geohash, one level per byte, depth tagged", geohash, "u4pru",
[]string{"1/u", "2/u4", "3/u4p", "4/u4pr", "5/u4pru"}),
)
It("keeps byte offsets on the source value", func() {
tok, err := hierarchy.Constructor(path, registry.NewCache())
Expect(err).ToNot(HaveOccurred())
ts := tok.Tokenize([]byte("./a/b"))
Expect(ts).To(HaveLen(3))
Expect(ts[2].Start).To(Equal(0))
Expect(ts[2].End).To(Equal(5))
Expect(ts[2].Position).To(Equal(3))
})
})
+5 -50
View File
@@ -19,7 +19,6 @@ import (
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve/hierarchy"
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
@@ -209,42 +208,6 @@ func NewMapping() (mapping.IndexMapping, error) {
if err != nil {
return nil, err
}
// path: every ancestor prefix is a term, so one term query matches a folder
// and all of its descendants
err = indexMapping.AddCustomTokenizer("path_hierarchy", map[string]any{
"type": hierarchy.Name,
"delimiter": "/",
})
if err != nil {
return nil, err
}
err = indexMapping.AddCustomAnalyzer(searchmapping.PathAnalyzer, map[string]any{
"type": custom.Name,
"tokenizer": "path_hierarchy",
})
if err != nil {
return nil, err
}
// geohash: every prefix is a depth-tagged term (1/u, 2/u4, ...), so a terms
// facet with TermPrefix "<precision>/" is a geohash grid at that precision.
// No field uses it yet. It is part of the v5 schema so that #3272 can add
// its geohash field additively: new fields reconcile at startup, a changed
// analysis block does not (classifyStoredMapping), so the names and the
// config below must not change.
err = indexMapping.AddCustomTokenizer("geohash_hierarchy", map[string]any{
"type": hierarchy.Name,
"tag_depth": true,
})
if err != nil {
return nil, err
}
err = indexMapping.AddCustomAnalyzer("geohash", map[string]any{
"type": custom.Name,
"tokenizer": "geohash_hierarchy",
})
if err != nil {
return nil, err
}
return indexMapping, nil
}
@@ -263,15 +226,11 @@ func searchResourceByID(id string, index bleve.Index) (*search.Resource, error)
return matchToResource(res.Hits[0]), nil
}
// searchResourcesByPath returns the descendants of the folder at lookupPath.
// The folder term matches the folder and everything below it in one term
// query (see PathAnalyzer); the folder itself is dropped from the result.
func searchResourcesByPath(rootID string, lookupPath string, index bleve.Index) ([]*search.Resource, error) {
rootQuery := bleve.NewTermQuery(rootID)
rootQuery.SetField("RootID")
pathQuery := bleve.NewTermQuery(lookupPath)
pathQuery.SetField("Path")
q := bleve.NewConjunctionQuery(rootQuery, pathQuery)
q := bleve.NewConjunctionQuery(
bleve.NewQueryStringQuery("RootID:"+rootID),
bleve.NewQueryStringQuery("Path:"+escapeQuery(lookupPath+"/*")),
)
bleveReq := bleve.NewSearchRequest(q)
bleveReq.Size = math.MaxInt
bleveReq.Fields = []string{"*"}
@@ -282,11 +241,7 @@ func searchResourcesByPath(rootID string, lookupPath string, index bleve.Index)
resources := make([]*search.Resource, 0, res.Hits.Len())
for _, match := range res.Hits {
resource := matchToResource(match)
if resource.Path == lookupPath {
continue
}
resources = append(resources, resource)
resources = append(resources, matchToResource(match))
}
return resources, nil
+1 -137
View File
@@ -160,7 +160,7 @@
"fields": [
{
"type": "text",
"analyzer": "path_hierarchy",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
@@ -702,79 +702,6 @@
}
}
},
"livePhoto": {
"enabled": true,
"dynamic": true,
"properties": {
"auto": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "boolean",
"store": true,
"index": true,
"include_in_all": true,
"docvalues": true
}
]
},
"contentId": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"include_in_all": true,
"docvalues": true
}
]
},
"stillImageTimeUs": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "number",
"store": true,
"index": true,
"include_in_all": true,
"docvalues": true
}
]
},
"vitalityScore": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "number",
"store": true,
"index": true,
"include_in_all": true,
"docvalues": true
}
]
},
"vitalityScoringVersion": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "number",
"store": true,
"index": true,
"include_in_all": true,
"docvalues": true
}
]
}
}
},
"location": {
"enabled": true,
"dynamic": true,
@@ -833,51 +760,6 @@
}
]
},
"motionPhoto": {
"enabled": true,
"dynamic": true,
"properties": {
"presentationTimestampUs": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "number",
"store": true,
"index": true,
"include_in_all": true,
"docvalues": true
}
]
},
"version": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "number",
"store": true,
"index": true,
"include_in_all": true,
"docvalues": true
}
]
},
"videoSize": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "number",
"store": true,
"index": true,
"include_in_all": true,
"docvalues": true
}
]
}
}
},
"photo": {
"enabled": true,
"dynamic": true,
@@ -1259,25 +1141,7 @@
"type": "regexp"
}
},
"tokenizers": {
"geohash_hierarchy": {
"tag_depth": true,
"type": "hierarchy"
},
"path_hierarchy": {
"delimiter": "/",
"type": "hierarchy"
}
},
"analyzers": {
"geohash": {
"tokenizer": "geohash_hierarchy",
"type": "custom"
},
"path_hierarchy": {
"tokenizer": "path_hierarchy",
"type": "custom"
},
"words": {
"char_filters": [
"dot_to_space"
+13 -15
View File
@@ -15,21 +15,19 @@ func init() {
// Document wraps all resource meta fields,
// it is used as a content extraction result.
type Document struct {
Title string `json:"Title"`
Name string `json:"Name"`
Content string `json:"Content"`
Size uint64 `json:"Size"`
Mtime *time.Time `json:"Mtime,omitempty"`
MimeType string `json:"MimeType"`
Tags []string `json:"Tags"`
Favorites []string `json:"Favorites"`
Audio *libregraph.Audio `json:"audio,omitempty"`
Image *libregraph.Image `json:"image,omitempty"`
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
Photo *libregraph.Photo `json:"photo,omitempty"`
Video *libregraph.Video `json:"video,omitempty"`
MotionPhoto *libregraph.MotionPhoto `json:"motionPhoto,omitempty"`
LivePhoto *libregraph.LivePhoto `json:"livePhoto,omitempty"`
Title string `json:"Title"`
Name string `json:"Name"`
Content string `json:"Content"`
Size uint64 `json:"Size"`
Mtime *time.Time `json:"Mtime,omitempty"`
MimeType string `json:"MimeType"`
Tags []string `json:"Tags"`
Favorites []string `json:"Favorites"`
Audio *libregraph.Audio `json:"audio,omitempty"`
Image *libregraph.Image `json:"image,omitempty"`
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
Photo *libregraph.Photo `json:"photo,omitempty"`
Video *libregraph.Video `json:"video,omitempty"`
}
func CleanString(content, langCode string) string {
+15 -8
View File
@@ -2,6 +2,7 @@ package content
import (
"context"
"errors"
"fmt"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
@@ -12,13 +13,19 @@ type Extractor interface {
Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, error)
}
// getFirstValue returns the first metadata value present among keys, trying them
// in order. It errors when the map is nil or none of the keys holds a value.
func getFirstValue(m map[string][]string, keys ...string) (string, error) {
for _, key := range keys {
if v, ok := m[key]; ok && len(v) > 0 {
return v[0], nil
}
func getFirstValue(m map[string][]string, key string) (string, error) {
if m == nil {
return "", errors.New("undefined map")
}
return "", fmt.Errorf("no value for keys: %v", keys)
v, ok := m[key]
if !ok {
return "", fmt.Errorf("unknown key: %v", key)
}
if len(v) == 0 {
return "", fmt.Errorf("no values for: %v", key)
}
return v[0], nil
}
+17 -18
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"net/http"
"slices"
"strings"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
@@ -82,9 +81,6 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
if err != nil {
return doc, err
}
if len(metas) == 0 {
return doc, nil
}
for _, meta := range metas {
title, err := getFirstValue(meta, "dc:title")
@@ -101,21 +97,24 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
} else if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil {
doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content))
}
}
// facets describe the file itself, not its embedded parts (cover art, clips)
m0 := metas[0]
doc.Location = t.getLocation(m0)
doc.Image = t.getImage(m0)
doc.Photo = t.getPhoto(m0)
doc.Audio = t.getAudio(m0)
doc.LivePhoto = t.getLivePhoto(m0)
doc.Video = t.getVideo(m0)
// a motion photo is the file's own xmp plus the video tika extracted from
// it; the xmp alone proves nothing, a share can strip the appended clip
if i := slices.IndexFunc(metas[1:], isVideo); i >= 0 {
doc.MotionPhoto = t.getMotionPhoto(m0, metas[i+1])
// keep facets from earlier entries, an embedded resource's meta
// (e.g. cover art) must not reset them
if v := t.getLocation(meta); v != nil {
doc.Location = v
}
if v := t.getImage(meta); v != nil {
doc.Image = v
}
if v := t.getPhoto(meta); v != nil {
doc.Photo = v
}
if v := t.getAudio(meta); v != nil {
doc.Audio = v
}
if v := t.getVideo(meta); v != nil {
doc.Video = v
}
}
if langCode := t.detectLanguage(ctx, doc.Content); langCode != "" && t.CleanStopWords {
@@ -1,47 +0,0 @@
package content
import (
"math"
"strconv"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
func (t Tika) getLivePhoto(meta map[string][]string) *libregraph.LivePhoto {
// ContentId pairs the two halves, without it there is no live photo. The
// video carries it in the QuickTime item list, the still image in the Apple
// maker note, which tika surfaces as "Content Identifier". A file is only
// ever one half, so both keys are read.
contentID, err := getFirstValue(meta, "com.apple.quicktime.content.identifier", "Content Identifier")
if err != nil || contentID == "" {
return nil
}
livePhoto := libregraph.NewLivePhoto(contentID)
// tika emits still-image-time already in microseconds
if v, err := getFirstValue(meta, "quicktime:still-image-time"); err == nil {
if f, err := strconv.ParseFloat(v, 64); err == nil {
livePhoto.SetStillImageTimeUs(int64(math.Round(f)))
}
}
if v, err := getFirstValue(meta, "com.apple.quicktime.live-photo.auto"); err == nil {
if b, err := strconv.ParseBool(v); err == nil {
livePhoto.SetAuto(b)
}
}
if v, err := getFirstValue(meta, "com.apple.quicktime.live-photo.vitality-score"); err == nil {
if f, err := strconv.ParseFloat(v, 64); err == nil {
livePhoto.SetVitalityScore(f)
}
}
if v, err := getFirstValue(meta, "com.apple.quicktime.live-photo.vitality-scoring-version"); err == nil {
if i, err := strconv.ParseInt(v, 10, 64); err == nil {
livePhoto.SetVitalityScoringVersion(i)
}
}
return livePhoto
}
@@ -1,77 +0,0 @@
package content
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
var _ = Describe("getLivePhoto", func() {
It("maps the video half of a live photo", func() {
livePhoto := Tika{}.getLivePhoto(map[string][]string{
"Content-Type": {"video/quicktime"},
"com.apple.quicktime.content.identifier": {"6F1A2B3C-1234-4E5F-9A8B-0011223344CC"},
"quicktime:still-image-time": {"1500000"},
"com.apple.quicktime.live-photo.auto": {"1"},
"com.apple.quicktime.live-photo.vitality-score": {"0.75"},
"com.apple.quicktime.live-photo.vitality-scoring-version": {"4"},
})
Expect(livePhoto).ToNot(BeNil())
Expect(livePhoto.ContentId).To(Equal("6F1A2B3C-1234-4E5F-9A8B-0011223344CC"))
Expect(livePhoto.StillImageTimeUs).To(Equal(libregraph.PtrInt64(1500000)))
Expect(livePhoto.Auto).To(Equal(libregraph.PtrBool(true)))
Expect(livePhoto.VitalityScore).To(Equal(libregraph.PtrFloat64(0.75)))
Expect(livePhoto.VitalityScoringVersion).To(Equal(libregraph.PtrInt64(4)))
})
It("maps the still half via the Apple maker-note content identifier", func() {
livePhoto := Tika{}.getLivePhoto(map[string][]string{
"Content-Type": {"image/heic"},
"Content Identifier": {"6F1A2B3C-1234-4E5F-9A8B-0011223344CC"},
})
Expect(livePhoto).ToNot(BeNil())
Expect(livePhoto.ContentId).To(Equal("6F1A2B3C-1234-4E5F-9A8B-0011223344CC"))
})
It("returns nil without a content identifier", func() {
Expect(Tika{}.getLivePhoto(map[string][]string{
"Content-Type": {"image/jpeg"},
})).To(BeNil())
Expect(Tika{}.getLivePhoto(map[string][]string{
"Content-Type": {"image/heic"},
"Content Identifier": {""},
})).To(BeNil(), "an empty pairing id is no live photo")
})
It("rounds a fractional still-image-time", func() {
livePhoto := Tika{}.getLivePhoto(map[string][]string{
"com.apple.quicktime.content.identifier": {"6F1A2B3C"},
"quicktime:still-image-time": {"1500000.7"},
})
Expect(livePhoto).ToNot(BeNil())
Expect(livePhoto.StillImageTimeUs).To(Equal(libregraph.PtrInt64(1500001)))
})
It("keeps the facet when optional values are malformed", func() {
livePhoto := Tika{}.getLivePhoto(map[string][]string{
"com.apple.quicktime.content.identifier": {"6F1A2B3C"},
"com.apple.quicktime.live-photo.auto": {"maybe"},
"com.apple.quicktime.live-photo.vitality-score": {"very"},
})
Expect(livePhoto).ToNot(BeNil())
Expect(livePhoto.Auto).To(BeNil())
Expect(livePhoto.VitalityScore).To(BeNil())
})
It("leaves the video-only fields empty on the still half", func() {
livePhoto := Tika{}.getLivePhoto(map[string][]string{
"Content Identifier": {"6F1A2B3C"},
})
Expect(livePhoto).ToNot(BeNil())
Expect(livePhoto.StillImageTimeUs).To(BeNil())
Expect(livePhoto.Auto).To(BeNil())
Expect(livePhoto.VitalityScore).To(BeNil())
Expect(livePhoto.VitalityScoringVersion).To(BeNil())
})
})
@@ -1,63 +0,0 @@
package content
import (
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"strconv"
"strings"
)
// getMotionPhoto reads Google Motion Photo XMP, which Tika exposes under the
// canonical Camera/Container prefixes. It covers both the current MotionPhoto
// scheme and the legacy MicroVideo scheme. videoSize (needed to range-fetch the
// video) comes from the video tika extracted, and is required: without it the
// facet is dropped.
func (t Tika) getMotionPhoto(meta, video map[string][]string) *libregraph.MotionPhoto {
// the marker is what makes this a motion photo rather than a picture that
// happens to carry a video: per the spec only a value of 1 counts, every
// other value means "treat as a still image".
if v, err := getFirstValue(meta, "Camera:MotionPhoto", "Camera:MicroVideo"); err != nil || v != "1" {
return nil
}
var motionPhoto *libregraph.MotionPhoto
initMotionPhoto := func() {
if motionPhoto == nil {
motionPhoto = libregraph.NewMotionPhoto()
}
}
if v, err := getFirstValue(meta, "Camera:MotionPhotoVersion", "Camera:MicroVideoVersion"); err == nil {
if i, err := strconv.ParseInt(v, 10, 32); err == nil {
initMotionPhoto()
motionPhoto.SetVersion(int32(i))
}
}
if v, err := getFirstValue(meta, "Camera:MotionPhotoPresentationTimestampUs", "Camera:MicroVideoPresentationTimestampUs"); err == nil {
if i, err := strconv.ParseInt(v, 10, 64); err == nil {
initMotionPhoto()
motionPhoto.SetPresentationTimestampUs(i)
}
}
if v, err := getFirstValue(video, "Content-Length"); err == nil {
if i, err := strconv.ParseInt(v, 10, 64); err == nil {
initMotionPhoto()
motionPhoto.SetVideoSize(i)
}
}
if motionPhoto == nil || !motionPhoto.HasVideoSize() {
return nil
}
return motionPhoto
}
// isVideo reports whether meta describes a video. Tika emits the video appended
// to a motion photo as an embedded document, and it only does so when the bytes
// the xmp advertises are really there: a shared motion photo can keep the xmp
// and lose the video.
func isVideo(meta map[string][]string) bool {
v, err := getFirstValue(meta, "Content-Type")
return err == nil && strings.HasPrefix(v, "video/")
}
@@ -1,85 +0,0 @@
package content
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
)
var _ = Describe("getMotionPhoto", func() {
It("maps the current MotionPhoto XMP scheme", func() {
mp := Tika{}.getMotionPhoto(map[string][]string{
"Camera:MotionPhoto": {"1"},
"Camera:MotionPhotoVersion": {"1"},
"Camera:MotionPhotoPresentationTimestampUs": {"1500000"},
}, map[string][]string{"Content-Length": {"1048576"}, "Content-Type": {"video/mp4"}})
Expect(mp).ToNot(BeNil())
Expect(mp.Version).To(Equal(libregraph.PtrInt32(1)))
Expect(mp.PresentationTimestampUs).To(Equal(libregraph.PtrInt64(1500000)))
Expect(mp.VideoSize).To(Equal(libregraph.PtrInt64(1048576)))
})
It("maps the legacy MicroVideo XMP scheme", func() {
mp := Tika{}.getMotionPhoto(map[string][]string{
"Camera:MicroVideo": {"1"},
"Camera:MicroVideoVersion": {"1"},
"Camera:MicroVideoPresentationTimestampUs": {"1500000"},
}, map[string][]string{"Content-Length": {"1048576"}, "Content-Type": {"video/mp4"}})
Expect(mp).ToNot(BeNil())
Expect(mp.Version).To(Equal(libregraph.PtrInt32(1)))
Expect(mp.PresentationTimestampUs).To(Equal(libregraph.PtrInt64(1500000)))
Expect(mp.VideoSize).To(Equal(libregraph.PtrInt64(1048576)))
})
It("drops the facet when the video reports no length", func() {
Expect(Tika{}.getMotionPhoto(map[string][]string{
"Camera:MotionPhoto": {"1"},
"Camera:MotionPhotoVersion": {"1"},
}, map[string][]string{"Content-Type": {"video/mp4"}})).To(BeNil())
})
It("returns nil without the marker, a picture may just carry a video", func() {
Expect(Tika{}.getMotionPhoto(map[string][]string{}, map[string][]string{"Content-Length": {"1048576"}, "Content-Type": {"video/mp4"}})).To(BeNil())
})
It("treats a zero MotionPhoto marker as a still image", func() {
Expect(Tika{}.getMotionPhoto(map[string][]string{
"Camera:MotionPhoto": {"0"},
"Camera:MotionPhotoVersion": {"1"},
}, map[string][]string{"Content-Length": {"1048576"}, "Content-Type": {"video/mp4"}})).To(BeNil())
Expect(Tika{}.getMotionPhoto(map[string][]string{
"Camera:MicroVideo": {"0"},
}, map[string][]string{"Content-Length": {"1048576"}, "Content-Type": {"video/mp4"}})).To(BeNil())
})
It("treats undefined marker values as a still image", func() {
Expect(Tika{}.getMotionPhoto(map[string][]string{
"Camera:MotionPhoto": {"2"},
}, map[string][]string{"Content-Length": {"1048576"}, "Content-Type": {"video/mp4"}})).To(BeNil())
})
It("prefers the current scheme when both are present", func() {
mp := Tika{}.getMotionPhoto(map[string][]string{
"Camera:MotionPhoto": {"1"},
"Camera:MotionPhotoVersion": {"2"},
"Camera:MicroVideoVersion": {"1"},
}, map[string][]string{"Content-Length": {"1048576"}, "Content-Type": {"video/mp4"}})
Expect(mp).ToNot(BeNil())
Expect(mp.Version).To(Equal(libregraph.PtrInt32(2)))
Expect(mp.VideoSize).To(Equal(libregraph.PtrInt64(1048576)))
})
})
var _ = Describe("isVideo", func() {
DescribeTable("recognizes the video tika extracted from a motion photo",
func(meta map[string][]string, expected bool) {
Expect(isVideo(meta)).To(Equal(expected))
},
Entry("mp4", map[string][]string{"Content-Type": {"video/mp4"}}, true),
Entry("quicktime", map[string][]string{"Content-Type": {"video/quicktime"}}, true),
Entry("with parameters", map[string][]string{"Content-Type": {"video/mp4; codecs=avc1"}}, true),
Entry("the image itself", map[string][]string{"Content-Type": {"image/jpeg"}}, false),
Entry("another attachment", map[string][]string{"Content-Type": {"application/pdf"}}, false),
Entry("no content type", map[string][]string{"Camera:MotionPhoto": {"1"}}, false),
)
})
+2 -27
View File
@@ -170,9 +170,7 @@ var _ = Describe("Tika", func() {
Expect(doc.Content).To(Equal("body test stop words!!!"))
})
It("takes facets from the main document, not from an embedded resource", func() {
// metas[0] is the file (audio), metas[1] its embedded cover art. The
// cover must not give the track an image facet.
It("keeps the audio facet when an embedded resource follows", func() {
fullResponse = `[{"Content-Type": "audio/mpeg", "dc:title": "Sucker", "tk:content": "lyrics"}, {"Content-Type": "image/jpeg", "tiff:ImageWidth": "500"}]`
doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
@@ -182,7 +180,7 @@ var _ = Describe("Tika", func() {
Expect(err).ToNot(HaveOccurred())
Expect(doc.Audio).ToNot(BeNil())
Expect(doc.Audio.Title).To(Equal(libregraph.PtrString("Sucker")))
Expect(doc.Image).To(BeNil())
Expect(doc.Image).ToNot(BeNil())
})
It("adds no audio facet to non-audio documents", func() {
@@ -219,29 +217,6 @@ var _ = Describe("Tika", func() {
Expect(doc.Content).To(Equal("one two"))
})
It("keeps the motion photo facet when tika emits the video", func() {
fullResponse = `[{"Camera:MotionPhoto": "1", "Camera:MotionPhotoVersion": "1"}, {"Content-Type": "video/mp4", "Content-Length": "40"}]`
doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Size: 100,
})
Expect(err).ToNot(HaveOccurred())
Expect(doc.MotionPhoto).ToNot(BeNil())
Expect(doc.MotionPhoto.VideoSize).To(Equal(libregraph.PtrInt64(40)))
})
It("drops the motion photo facet when the advertised video is gone", func() {
fullResponse = `[{"Camera:MotionPhoto": "1", "Camera:MotionPhotoVersion": "1"}]`
doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
Type: provider.ResourceType_RESOURCE_TYPE_FILE,
Size: 100,
})
Expect(err).ToNot(HaveOccurred())
Expect(doc.MotionPhoto).To(BeNil())
})
It("keeps stop words", func() {
body = "body to test stop words!!! against almost everyone"
language = "en"
+3 -6
View File
@@ -60,6 +60,7 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
}
if fieldType == TypeKeyword || fieldType == TypePath {
// bleve has no path tokenizer, so a path is a plain keyword here.
base := bleveKeywordMapping(fieldType, opts)
doc.AddFieldMappingsAt(fi.Name, base)
if opts.caseInsensitive() {
@@ -83,9 +84,8 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
return doc, err
}
// bleveKeywordMapping is a case-preserving keyword field; path fields are
// analyzed into their ancestor prefixes (see PathAnalyzer) and stay out of
// _all by default.
// bleveKeywordMapping is a case-preserving keyword field; path fields stay out
// of _all by default.
func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMapping {
fm := bleve.NewKeywordFieldMapping()
switch {
@@ -94,9 +94,6 @@ func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMa
case fieldType == TypePath:
fm.IncludeInAll = false
}
if fieldType == TypePath {
fm.Analyzer = PathAnalyzer
}
return fm
}
+1 -1
View File
@@ -61,7 +61,7 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
// path_hierarchy is case-preserving here; casing lives in the value.
m := map[string]any{"type": "keyword"}
if fieldType == TypePath {
m = map[string]any{"type": "text", "analyzer": PathAnalyzer}
m = map[string]any{"type": "text", "analyzer": "path_hierarchy"}
}
props[fi.Name] = m
if opts.caseInsensitive() {
@@ -100,8 +100,8 @@ var _ = Describe("OpenSearchBuildMapping", func() {
Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content)
Expect(content["analyzer"]).To(Equal(WordsAnalyzer), "Content uses the words analyzer, like bleve")
// Path: path_hierarchy base + lowercased sibling, both case-preserving.
Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": PathAnalyzer}))
Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": PathAnalyzer}))
Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"}))
Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"}))
mime := props["MimeType"].(map[string]any)
Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime)
})
-5
View File
@@ -26,11 +26,6 @@ const WordsSuffix = "_words"
// WordsAnalyzer names the analyzer both engines register for the words sibling.
const WordsAnalyzer = "words"
// PathAnalyzer names the analyzer both engines register for TypePath fields:
// every ancestor prefix of a path is a term, so one term query matches a
// folder and its descendants.
const PathAnalyzer = "path_hierarchy"
// FieldOpts overrides the default type inference for a struct field. Keys in
// the override map are json-tag names (e.g. "Name", "location", "audio.artist"),
// not Go field names.
+2 -2
View File
@@ -27,7 +27,7 @@ func Reconcile(index string, r SchemaReconciler, logger log.Logger) (Classificat
case VerdictAdditive:
persisted, err := r.ApplyAdditive()
if persisted {
logger.Warn().Strs("fields", classification.NewFields).Str("index", index).Msg("extended the search index mapping with new fields; documents indexed before the upgrade do not contain them and queries on these fields will miss those documents until they are re-indexed; to re-index everything run: opencloud search index --all-spaces --force-rescan --insecure")
logger.Warn().Strs("fields", classification.NewFields).Str("index", index).Msg("extended the search index mapping with new fields; documents indexed before the upgrade do not contain them and queries on these fields will miss those documents until they are re-indexed; to re-index everything run: opencloud search index --all-spaces --force-rescan")
}
if err != nil {
return classification, err
@@ -40,5 +40,5 @@ func Reconcile(index string, r SchemaReconciler, logger log.Logger) (Classificat
// LogNewIndexCreated logs that a fresh, empty index was created and how to
// backfill it. The create path does not run through Reconcile.
func LogNewIndexCreated(logger log.Logger, index string) {
logger.Info().Str("index", index).Msg("created a new empty search index; if this OpenCloud instance already held files, they are not in it yet, index them by running: opencloud search index --all-spaces --force-rescan --insecure")
logger.Info().Str("index", index).Msg("created a new empty search index; if this OpenCloud instance already held files, they are not in it yet, index them by running: opencloud search index --all-spaces --force-rescan")
}
+1 -1
View File
@@ -82,7 +82,7 @@ func buildResourceMapping() ([]byte, error) {
"analysis": map[string]any{
// path_hierarchy is case-preserving; casing lives in the value.
"analyzer": map[string]any{
searchmapping.PathAnalyzer: map[string]any{
"path_hierarchy": map[string]any{
"type": "custom",
"tokenizer": "path_hierarchy",
},
@@ -79,13 +79,11 @@ func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match,
return strings.Join(contentHighlights[:], "; ")
}(),
Audio: copyFacet[searchMessage.Audio](resource.Audio),
Image: copyFacet[searchMessage.Image](resource.Image),
Location: copyFacet[searchMessage.GeoCoordinates](resource.Location),
Photo: copyFacet[searchMessage.Photo](resource.Photo),
Video: copyFacet[searchMessage.Video](resource.Video),
MotionPhoto: copyFacet[searchMessage.MotionPhoto](resource.MotionPhoto),
LivePhoto: copyFacet[searchMessage.LivePhoto](resource.LivePhoto),
Audio: copyFacet[searchMessage.Audio](resource.Audio),
Image: copyFacet[searchMessage.Image](resource.Image),
Location: copyFacet[searchMessage.GeoCoordinates](resource.Location),
Photo: copyFacet[searchMessage.Photo](resource.Photo),
Video: copyFacet[searchMessage.Video](resource.Video),
},
}
@@ -188,25 +188,6 @@
}
}
},
"livePhoto": {
"properties": {
"auto": {
"type": "boolean"
},
"contentId": {
"type": "keyword"
},
"stillImageTimeUs": {
"type": "long"
},
"vitalityScore": {
"type": "double"
},
"vitalityScoringVersion": {
"type": "long"
}
}
},
"location": {
"properties": {
"altitude": {
@@ -223,19 +204,6 @@
"location_geopoint": {
"type": "geo_point"
},
"motionPhoto": {
"properties": {
"presentationTimestampUs": {
"type": "long"
},
"version": {
"type": "integer"
},
"videoSize": {
"type": "long"
}
}
},
"photo": {
"properties": {
"cameraMake": {
-10
View File
@@ -530,22 +530,12 @@ Fixtures:
- `parent`, ID = 1$1!2, folder
- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf
- `big`, ID = 1$1!4, folder
- `f1.txt`, ID = 1$1!5, Path = ./big/f1.txt
- `x.txt`, ID = 1$1!6, Path = ./big2/x.txt
- `odd name (1)`, ID = 1$1!7, folder
- `f:x+y.txt`, ID = 1$1!8, Path = ./odd name (1)/f:x+y.txt
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| MOVE-01 | carries the descendants to the new path, then `path:"./my/newname/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ |
| MOVE-01 | carries the descendants to the new path, then `path:"./parent/child.pdf"` | no match | no match | no match | ✅ |
| MOVE-02 | through the trash and back leaves the flag behind, then `hidden:true` | no match | no match | no match | ✅ |
| MOVE-03 | leaves a sibling folder that shares the prefix alone, then `path:"./moved"` | f1.txt, moved | f1.txt, moved | f1.txt, moved | ✅ |
| MOVE-03 | leaves a sibling folder that shares the prefix alone, then `path:"./big"` | no match | no match | no match | ✅ |
| MOVE-03 | leaves a sibling folder that shares the prefix alone, then `path:"./big2"` | x.txt | x.txt | x.txt | ✅ |
| MOVE-04 | carries the descendants of a path with special characters, then `path:"./odd name (2)"` | f:x+y.txt, odd name (2) | f:x+y.txt, odd name (2) | f:x+y.txt, odd name (2) | ✅ |
| MOVE-04 | carries the descendants of a path with special characters, then `path:"./odd name (1)"` | no match | no match | no match | ✅ |
### rootscope
@@ -6,11 +6,6 @@ import (
func moveLifecycle() lifecycleGroup {
parent, child := fixtureTree()
big := fixtureFolder("big", withID("1$1!4"))
inBig := fixtureDoc("f1.txt", withID("1$1!5"), withParent(big.ID), withPath("./big/f1.txt"))
inSibling := fixtureDoc("x.txt", withID("1$1!6"), withParent("1$1!big2"), withPath("./big2/x.txt"))
odd := fixtureFolder("odd name (1)", withID("1$1!7"))
inOdd := fixtureDoc("f:x+y.txt", withID("1$1!8"), withParent(odd.ID), withPath("./odd name (1)/f:x+y.txt"))
return lifecycleGroup{
name: "move",
@@ -37,25 +32,6 @@ func moveLifecycle() lifecycleGroup {
},
expect: []expectation{{`hidden:true`, nil}},
},
{
id: 3, title: "leaves a sibling folder that shares the prefix alone",
fixtures: []search.Resource{big, inBig, inSibling},
do: func(e search.Engine) error { return e.Move(big.ID, big.ParentID, "./moved") },
expect: []expectation{
{`path:"./moved"`, []string{"moved", "f1.txt"}},
{`path:"./big"`, nil},
{`path:"./big2"`, []string{"x.txt"}},
},
},
{
id: 4, title: "carries the descendants of a path with special characters",
fixtures: []search.Resource{odd, inOdd},
do: func(e search.Engine) error { return e.Move(odd.ID, odd.ParentID, "./odd name (2)") },
expect: []expectation{
{`path:"./odd name (2)"`, []string{"odd name (2)", "f:x+y.txt"}},
{`path:"./odd name (1)"`, nil},
},
},
},
}
}
+11 -7
View File
@@ -123,13 +123,6 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v)
switch {
case searchQuery.FieldIsPath(n.Key) && !isWildcard:
// the folder term matches the folder itself and its descendants
// (see PathAnalyzer); a query string would analyze the value into
// its prefixes and match everything under the root
tq := bleveQuery.NewTermQuery(val)
tq.SetField(k)
q = tq
case n.Exact && !isWildcard:
// = matches the whole value, on the lowercased sibling for
// case-insensitive fields
@@ -147,6 +140,17 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
bq.SetMinShould(1)
q = bq
}
if searchQuery.FieldIsPath(n.Key) {
// bleve has no path hierarchy analyzer, unlike OpenSearch: match the
// folder itself and its descendants (`\/*`). A BooleanQuery keeps
// this atomic; a DisjunctionQuery would be redistributed by an
// enclosing AND (mapBinary treats a left disjunction as an OR-chain).
bq := bleve.NewBooleanQuery()
bq.AddShould(q, bleveQuery.NewQueryStringQuery(k+":"+v+`\/*`))
bq.SetMinShould(1)
q = bq
}
if prev == nil {
prev = q
} else {
@@ -51,17 +51,23 @@ func Test_compile(t *testing.T) {
wantErr: false,
},
{
// one term matches the folder itself and its descendants
// path fields expand to match the folder itself and its descendants,
// since bleve has no path hierarchy analyzer.
name: `path:/Foo`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "path", Value: "/Foo"},
},
},
// a BooleanQuery (should: exact OR descendants), not a DisjunctionQuery,
// so an enclosing AND does not redistribute the folder-itself clause.
want: func() query.Query {
tq := query.NewTermQuery("/Foo")
tq.SetField("Path")
return query.NewConjunctionQuery([]query.Query{tq})
bq := query.NewBooleanQuery(nil, []query.Query{
query.NewQueryStringQuery(`Path:\/Foo`),
query.NewQueryStringQuery(`Path:\/Foo\/*`),
}, nil)
bq.SetMinShould(1)
return query.NewConjunctionQuery([]query.Query{bq})
}(),
wantErr: false,
},
+10 -11
View File
@@ -29,7 +29,7 @@ import (
// on a breaking mapping change: each version gets its own index (OpenSearch name
// suffix, bleve path suffix), so the service builds a fresh index instead of
// colliding with the old one. No migration; reindex to populate.
const SchemaVersion = 5
const SchemaVersion = 4
var scopeRegex = regexp.MustCompile(`scope:\s*([^" "\n\r]*)`)
@@ -79,16 +79,15 @@ var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts
// every keyword field searches case-insensitively and by word (name,
// title, the facets) unless opted out: ids are opaque, paths are POSIX,
// the mime type is normalized already, a tag is one label
"ID": {CaseInsensitive: &False, NoWordBreaker: &True},
"RootID": {CaseInsensitive: &False, NoWordBreaker: &True},
"ParentID": {CaseInsensitive: &False, NoWordBreaker: &True},
"Path": {Type: mapping.TypePath, CaseInsensitive: &False},
"MimeType": {CaseInsensitive: &False, NoWordBreaker: &True},
"Content": {Type: mapping.TypeFulltext},
"Tags": {NoWordBreaker: &True, IncludeInAll: &False},
"Favorites": {NoWordBreaker: &True, IncludeInAll: &False, CaseInsensitive: &False}, // opaque user ids
"livePhoto.contentId": {NoWordBreaker: &True, CaseInsensitive: &False}, // opaque pairing uuid
"location": {Type: mapping.TypeGeopoint},
"ID": {CaseInsensitive: &False, NoWordBreaker: &True},
"RootID": {CaseInsensitive: &False, NoWordBreaker: &True},
"ParentID": {CaseInsensitive: &False, NoWordBreaker: &True},
"Path": {Type: mapping.TypePath, CaseInsensitive: &False},
"MimeType": {CaseInsensitive: &False, NoWordBreaker: &True},
"Content": {Type: mapping.TypeFulltext},
"Tags": {NoWordBreaker: &True, IncludeInAll: &False},
"Favorites": {NoWordBreaker: &True, IncludeInAll: &False, CaseInsensitive: &False}, // opaque user ids
"location": {Type: mapping.TypeGeopoint},
}
})
-2
View File
@@ -686,8 +686,6 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) {
facetToMetadata(metadata, doc.Location, "libre.graph.location.")
facetToMetadata(metadata, doc.Photo, "libre.graph.photo.")
facetToMetadata(metadata, doc.Video, "libre.graph.video.")
facetToMetadata(metadata, doc.MotionPhoto, "libre.graph.motionPhoto.")
facetToMetadata(metadata, doc.LivePhoto, "libre.graph.livePhoto.")
if len(metadata) == 0 {
return
}
@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2026-09-07 23:16+0000\n"
"POT-Creation-Date: 2026-08-17 23:18+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Jiri Grönroos <jiri.gronroos@iki.fi>, 2026\n"
"Language-Team: Finnish (https://app.transifex.com/opencloud-eu/teams/204053/fi/)\n"
Loaded 100 of 209 files, more files were not shown because too many files have changed in this diff. Show more