mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-09 12:19:08 -04:00
Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
359dd2f267 | ||
|
|
b392f43662 | ||
|
|
1e6d2ec168 | ||
|
|
d9613a068f | ||
|
|
6727cec526 | ||
|
|
6d09a83adc | ||
|
|
a0e1d7cbab | ||
|
|
f70d2c45b5 | ||
|
|
56bee0a339 | ||
|
|
bcfccc4dd2 | ||
|
|
5b260877dc | ||
|
|
84f9621659 | ||
|
|
8733428aed | ||
|
|
a9822a7e24 | ||
|
|
f640858427 | ||
|
|
1e29eb9504 | ||
|
|
da421b6747 | ||
|
|
6eb235ab1c | ||
|
|
adb7327779 | ||
|
|
af999345a6 | ||
|
|
b045b2be86 | ||
|
|
d18c1a1edb | ||
|
|
aa03c6c742 | ||
|
|
1d65ce7052 | ||
|
|
40f7aaa327 | ||
|
|
46a7dbfb47 |
No files matched your search
@@ -0,0 +1,27 @@
|
||||
package middleware
|
||||
|
||||
import "net/http"
|
||||
|
||||
const (
|
||||
// PublicLinkTokenName is the query parameter and header carrying a public
|
||||
// link token on a request.
|
||||
PublicLinkTokenName = "public-token"
|
||||
|
||||
// PublicLinkAuthHeader marks the outcome of a failed public link
|
||||
// authentication so a downstream service can tell "password required" from
|
||||
// "wrong password" when it renders the 401. The proxy sets it, the graph
|
||||
// service reads it.
|
||||
PublicLinkAuthHeader = "X-Public-Link-Auth"
|
||||
|
||||
// PublicLinkPasswordRequired means the link is password protected and no
|
||||
// password was provided.
|
||||
PublicLinkPasswordRequired = "password-required"
|
||||
// PublicLinkInvalidPassword means a password was provided but rejected.
|
||||
PublicLinkInvalidPassword = "invalid-password"
|
||||
)
|
||||
|
||||
// HasPublicLinkToken reports whether a public link token rides on the request,
|
||||
// as a query parameter or header.
|
||||
func HasPublicLinkToken(r *http.Request) bool {
|
||||
return r.URL.Query().Get(PublicLinkTokenName) != "" || r.Header.Get(PublicLinkTokenName) != ""
|
||||
}
|
||||
@@ -3,18 +3,17 @@ package command
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/olekukonko/errors"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/pkg/runner"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/pkg/generators"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
natspkg "github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/registry"
|
||||
"github.com/opencloud-eu/opencloud/pkg/runner"
|
||||
ogrpc "github.com/opencloud-eu/opencloud/pkg/service/grpc"
|
||||
"github.com/opencloud-eu/opencloud/pkg/tracing"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
@@ -25,12 +24,6 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/metrics"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/server/debug"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/server/http"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
svcEvents "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events"
|
||||
svcHttp "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
var _registeredEvents = []events.Unmarshaller{
|
||||
@@ -69,11 +62,19 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
|
||||
gr := runner.NewGroup()
|
||||
ctx, cancel := context.WithCancel(cmd.Context())
|
||||
defer cancel()
|
||||
|
||||
mtrcs := metrics.New()
|
||||
mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1)
|
||||
|
||||
defer cancel()
|
||||
|
||||
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
|
||||
evStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig(cfg.Events))
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to initialize event stream")
|
||||
return err
|
||||
}
|
||||
|
||||
tm, err := pool.StringToTLSMode(cfg.GRPCClientTLS.Mode)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to parse tls mode")
|
||||
@@ -98,101 +99,28 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
return err
|
||||
}
|
||||
|
||||
kv, err := ConnectNatsKV(cfg.Store)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
activityLog, err := activitylog.New(kv,
|
||||
activitylog.Logger(logger),
|
||||
activitylog.MaxActivities(cfg.MaxActivities),
|
||||
activitylog.WriteBufferDuration(cfg.WriteBufferDuration),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to initialize activity log")
|
||||
return err
|
||||
}
|
||||
hClient := ehsvc.NewEventHistoryService("eu.opencloud.api.eventhistory", grpcClient)
|
||||
vClient := settingssvc.NewValueService("eu.opencloud.api.settings", grpcClient)
|
||||
|
||||
if !cfg.HTTP.Disabled {
|
||||
|
||||
hClient := ehsvc.NewEventHistoryService("eu.opencloud.api.eventhistory", grpcClient)
|
||||
|
||||
svc, err := svcHttp.New(
|
||||
activityLog,
|
||||
svcHttp.Logger(logger),
|
||||
svcHttp.GatewaySelector(gatewaySelector),
|
||||
svcHttp.RegisteredEvents(_registeredEvents),
|
||||
//svcHttp.TraceProvider(tracerProvider),
|
||||
svcHttp.HistoryClient(hClient),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("handler init")
|
||||
return err
|
||||
}
|
||||
// TODO svc = service.NewInstrument(svc, metrics)
|
||||
// TODO svc = service.NewLogging(svc, logger) // this logs service specific data
|
||||
// TODO svc = service.NewTracing(svc, traceProvider)
|
||||
vClient := settingssvc.NewValueService("eu.opencloud.api.settings", grpcClient)
|
||||
|
||||
server, err := http.Server(
|
||||
http.ValueClient(vClient),
|
||||
{
|
||||
svc, err := http.Server(
|
||||
http.Logger(logger),
|
||||
http.Context(ctx),
|
||||
http.Config(cfg),
|
||||
http.Service(svc),
|
||||
http.Context(ctx), // NOTE: not passing this "option" leads to a panic in go-micro
|
||||
http.TraceProvider(tracerProvider),
|
||||
http.Stream(evStream),
|
||||
http.GatewaySelector(gatewaySelector),
|
||||
http.HistoryClient(hClient),
|
||||
http.ValueClient(vClient),
|
||||
http.RegisteredEvents(_registeredEvents),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Info().
|
||||
Err(err).
|
||||
Str("transport", "http").
|
||||
Msg("Failed to initialize server")
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Str("transport", "http").Msg("Failed to initialize server")
|
||||
return err
|
||||
}
|
||||
|
||||
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", server))
|
||||
} else {
|
||||
logger.Info().Msg("HTTP server disabled, not starting HTTP service")
|
||||
}
|
||||
|
||||
if !cfg.Events.Disabled {
|
||||
|
||||
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
|
||||
evStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig{
|
||||
Endpoint: cfg.Events.Endpoint,
|
||||
Cluster: cfg.Events.Cluster,
|
||||
EnableTLS: cfg.Events.EnableTLS,
|
||||
TLSInsecure: cfg.Events.TLSInsecure,
|
||||
TLSRootCACertificate: cfg.Events.TLSRootCACertificate,
|
||||
AuthUsername: cfg.Events.AuthUsername,
|
||||
AuthPassword: cfg.Events.AuthPassword,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to initialize event stream")
|
||||
return err
|
||||
}
|
||||
|
||||
eventSvc, err := svcEvents.New(
|
||||
activityLog,
|
||||
evStream,
|
||||
svcEvents.Context(ctx),
|
||||
svcEvents.Logger(logger),
|
||||
svcEvents.ServiceAccount(cfg.ServiceAccount),
|
||||
svcEvents.GatewaySelector(gatewaySelector),
|
||||
svcEvents.RegisteredEvents(_registeredEvents),
|
||||
svcEvents.NumConsumers(cfg.NumConsumers),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Str("transport", "event").Msg("Failed to initialize server")
|
||||
return err
|
||||
}
|
||||
|
||||
gr.Add(runner.New(cfg.Service.Name+".svc", func() error {
|
||||
return eventSvc.Run()
|
||||
}, func() {
|
||||
eventSvc.Close()
|
||||
}))
|
||||
} else {
|
||||
logger.Info().Msg("event listening disabled, not starting event service")
|
||||
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", svc))
|
||||
}
|
||||
|
||||
{
|
||||
@@ -221,33 +149,3 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func ConnectNatsKV(cfg config.Store) (nats.KeyValue, error) {
|
||||
// Connect to NATS servers
|
||||
secureOption := natspkg.Secure(cfg.EnableTLS, cfg.TLSInsecure, cfg.TLSRootCACertificate)
|
||||
conn, err := nats.Connect(strings.Join(cfg.Nodes, ","), secureOption, nats.UserInfo(cfg.AuthUsername, cfg.AuthPassword))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
kv, err := js.KeyValue(cfg.Database)
|
||||
if err != nil {
|
||||
if !errors.Is(err, nats.ErrBucketNotFound) {
|
||||
return nil, errors.Wrapf(err, "Failed to get bucket (%s)", cfg.Database)
|
||||
}
|
||||
|
||||
kv, err = js.CreateKeyValue(&nats.KeyValueConfig{
|
||||
Bucket: cfg.Database,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to create bucket (%s)", cfg.Database)
|
||||
}
|
||||
}
|
||||
|
||||
return kv, nil
|
||||
}
|
||||
@@ -35,12 +35,10 @@ type Config struct {
|
||||
|
||||
WriteBufferDuration time.Duration `yaml:"write_buffer_duration" env:"ACTIVITYLOG_WRITE_BUFFER_DURATION" desc:"The duration to wait before flushing the write buffer. This is used to reduce the number of writes to the store." introductionVersion:"4.0.0"`
|
||||
MaxActivities int `yaml:"max_activities" env:"ACTIVITYLOG_MAX_ACTIVITIES" desc:"The maximum number of activities to keep in the store per resource. If the number of activities exceeds this value, the oldest activities will be removed." introductionVersion:"4.0.0"`
|
||||
NumConsumers int `yaml:"num_consumers" env:"ACTIVITYLOG_NUM_CONSUMERS" desc:"The amount of concurrent event consumers to start. Event consumers are used for updating the list of activities. Multiple consumers increase parallelisation, but will also increase CPU and memory demands." introductionVersion:"%NEXT%"`
|
||||
}
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
Disabled bool `yaml:"disabled" env:"ACTIVITYLOG_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%NEXT%"`
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
|
||||
@@ -79,7 +77,6 @@ type CORS struct {
|
||||
|
||||
// HTTP defines the available http configuration.
|
||||
type HTTP struct {
|
||||
Disabled bool `yaml:"disabled" env:"ACTIVITYLOG_HTTP_DISABLED" desc:"Disables the HTTP service. Set this to true if the service should only handle events." introductionVersion:"1.0.0"`
|
||||
Addr string `yaml:"addr" env:"ACTIVITYLOG_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
Root string `yaml:"root" env:"ACTIVITYLOG_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -52,9 +52,7 @@ func DefaultConfig() *config.Config {
|
||||
},
|
||||
},
|
||||
WriteBufferDuration: 10 * time.Second,
|
||||
// Nats runs into max payload exceeded errors at around 7k activities. Let's keep a buffer.
|
||||
MaxActivities: 6000,
|
||||
NumConsumers: 1,
|
||||
MaxActivities: 6000,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"errors"
|
||||
|
||||
occfg "github.com/opencloud-eu/opencloud/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/pkg/shared"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config/defaults"
|
||||
|
||||
@@ -35,8 +34,5 @@ func ParseConfig(cfg *config.Config) error {
|
||||
|
||||
// Validate validates the config
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.Events.Disabled && cfg.HTTP.Disabled {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -3,12 +3,18 @@ package http
|
||||
import (
|
||||
"context"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
"go-micro.dev/v4/store"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
@@ -16,15 +22,19 @@ type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Name string
|
||||
Namespace string
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Flags []pflag.Flag
|
||||
Service ActivityLogService
|
||||
TraceProvider trace.TracerProvider
|
||||
ValueClient settingssvc.ValueService
|
||||
Logger log.Logger
|
||||
Context context.Context
|
||||
Config *config.Config
|
||||
Metrics *metrics.Metrics
|
||||
Flags []pflag.Flag
|
||||
Namespace string
|
||||
Store store.Store
|
||||
Stream events.Stream
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
TraceProvider trace.TracerProvider
|
||||
HistoryClient ehsvc.EventHistoryService
|
||||
ValueClient settingssvc.ValueService
|
||||
RegisteredEvents []events.Unmarshaller
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
@@ -59,10 +69,10 @@ func Config(val *config.Config) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// Service provides a function to set the service option.
|
||||
func Service(val ActivityLogService) Option {
|
||||
// Metrics provides a function to set the metrics option.
|
||||
func Metrics(val *metrics.Metrics) Option {
|
||||
return func(o *Options) {
|
||||
o.Service = val
|
||||
o.Metrics = val
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,20 +83,58 @@ func Flags(flags ...pflag.Flag) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider provides a function to configure the trace provider
|
||||
func TraceProvider(traceProvider trace.TracerProvider) Option {
|
||||
// Namespace provides a function to set the Namespace option.
|
||||
func Namespace(val string) Option {
|
||||
return func(o *Options) {
|
||||
if traceProvider != nil {
|
||||
o.TraceProvider = traceProvider
|
||||
} else {
|
||||
o.TraceProvider = noop.NewTracerProvider()
|
||||
}
|
||||
o.Namespace = val
|
||||
}
|
||||
}
|
||||
|
||||
// ValueClient adds a grpc client for the value service
|
||||
func ValueClient(vs settingssvc.ValueService) Option {
|
||||
// Store provides a function to configure the store
|
||||
func Store(store store.Store) Option {
|
||||
return func(o *Options) {
|
||||
o.ValueClient = vs
|
||||
o.Store = store
|
||||
}
|
||||
}
|
||||
|
||||
// Stream provides a function to configure the stream
|
||||
func Stream(stream events.Stream) Option {
|
||||
return func(o *Options) {
|
||||
o.Stream = stream
|
||||
}
|
||||
}
|
||||
|
||||
// GatewaySelector provides a function to configure the gateway client selector
|
||||
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
|
||||
return func(o *Options) {
|
||||
o.GatewaySelector = gatewaySelector
|
||||
}
|
||||
}
|
||||
|
||||
// HistoryClient provides a function to configure the event history client
|
||||
func HistoryClient(h ehsvc.EventHistoryService) Option {
|
||||
return func(o *Options) {
|
||||
o.HistoryClient = h
|
||||
}
|
||||
}
|
||||
|
||||
// RegisteredEvents provides a function to register events
|
||||
func RegisteredEvents(evs []events.Unmarshaller) Option {
|
||||
return func(o *Options) {
|
||||
o.RegisteredEvents = evs
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider provides a function to set the TracerProvider option
|
||||
func TraceProvider(val trace.TracerProvider) Option {
|
||||
return func(o *Options) {
|
||||
o.TraceProvider = val
|
||||
}
|
||||
}
|
||||
|
||||
// ValueClient provides a function to set the ValueClient options
|
||||
func ValueClient(val settingssvc.ValueService) Option {
|
||||
return func(o *Options) {
|
||||
o.ValueClient = val
|
||||
}
|
||||
}
|
||||
@@ -1,63 +1,49 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"fmt"
|
||||
|
||||
stdhttp "net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
chimiddleware "github.com/go-chi/chi/v5/middleware"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/opencloud/pkg/account"
|
||||
"github.com/opencloud-eu/opencloud/pkg/cors"
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/pkg/middleware"
|
||||
ohttp "github.com/opencloud-eu/opencloud/pkg/service/http"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/http"
|
||||
"github.com/opencloud-eu/opencloud/pkg/tracing"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
activityloghttp "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
svc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"
|
||||
"github.com/riandyrn/otelchi"
|
||||
"go-micro.dev/v4"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
var (
|
||||
//go:embed l10n/locale
|
||||
_localeFS embed.FS
|
||||
|
||||
// subfolder where the translation files are stored
|
||||
_localeSubPath = "l10n/locale"
|
||||
|
||||
// domain of the activitylog service (transifex)
|
||||
_domain = "activitylog"
|
||||
)
|
||||
// Service is the service interface
|
||||
type Service any
|
||||
|
||||
// Server initializes the http service and server.
|
||||
func Server(opts ...Option) (ohttp.Service, error) {
|
||||
func Server(opts ...Option) (http.Service, error) {
|
||||
options := newOptions(opts...)
|
||||
service := options.Service
|
||||
|
||||
newService, err := ohttp.NewService(
|
||||
ohttp.TLSConfig(options.Config.HTTP.TLS),
|
||||
ohttp.Logger(options.Logger),
|
||||
ohttp.Namespace(options.Config.HTTP.Namespace),
|
||||
ohttp.Name(options.Config.Service.Name),
|
||||
ohttp.Version(version.GetString()),
|
||||
ohttp.Address(options.Config.HTTP.Addr),
|
||||
ohttp.Context(options.Context),
|
||||
ohttp.Flags(options.Flags...),
|
||||
service, err := http.NewService(
|
||||
http.TLSConfig(options.Config.HTTP.TLS),
|
||||
http.Logger(options.Logger),
|
||||
http.Namespace(options.Config.HTTP.Namespace),
|
||||
http.Name(options.Config.Service.Name),
|
||||
http.Version(version.GetString()),
|
||||
http.Address(options.Config.HTTP.Addr),
|
||||
http.Context(options.Context),
|
||||
http.Flags(options.Flags...),
|
||||
http.TraceProvider(options.TraceProvider),
|
||||
)
|
||||
if err != nil {
|
||||
options.Logger.Error().
|
||||
Err(err).
|
||||
Msg("Error initializing http service")
|
||||
return ohttp.Service{}, err
|
||||
return http.Service{}, fmt.Errorf("could not initialize http service: %w", err)
|
||||
}
|
||||
|
||||
middlewares := []func(http.Handler) http.Handler{
|
||||
middlewares := []func(stdhttp.Handler) stdhttp.Handler{
|
||||
chimiddleware.RequestID,
|
||||
middleware.Version(
|
||||
options.Config.Service.Name,
|
||||
@@ -66,7 +52,6 @@ func Server(opts ...Option) (ohttp.Service, error) {
|
||||
middleware.Logger(
|
||||
options.Logger,
|
||||
),
|
||||
middleware.TraceContext,
|
||||
middleware.ExtractAccountUUID(
|
||||
account.Logger(options.Logger),
|
||||
account.JWTSecret(options.Config.TokenManager.JWTSecret),
|
||||
@@ -83,75 +68,33 @@ func Server(opts ...Option) (ohttp.Service, error) {
|
||||
mux := chi.NewMux()
|
||||
mux.Use(middlewares...)
|
||||
|
||||
t := l10n.NewTranslatorFromCommonConfig(options.Config.DefaultLanguage, _domain, options.Config.TranslationPath, _localeFS, _localeSubPath)
|
||||
mux.Route(options.Config.HTTP.Root, func(r chi.Router) {
|
||||
r.Get("/graph/v1beta1/extensions/org.libregraph/activities", GetItemActivitiesHandler(options.Logger, service, options.ValueClient, t))
|
||||
})
|
||||
mux.Use(
|
||||
otelchi.Middleware(
|
||||
"actitivylog",
|
||||
otelchi.WithChiRoutes(mux),
|
||||
otelchi.WithTracerProvider(options.TraceProvider),
|
||||
otelchi.WithPropagators(tracing.GetPropagator()),
|
||||
),
|
||||
)
|
||||
|
||||
err = micro.RegisterHandler(newService.Server(), mux)
|
||||
handle, err := svc.New(
|
||||
svc.Logger(options.Logger),
|
||||
svc.Stream(options.Stream),
|
||||
svc.Mux(mux),
|
||||
svc.Config(options.Config),
|
||||
svc.GatewaySelector(options.GatewaySelector),
|
||||
svc.TraceProvider(options.TraceProvider),
|
||||
svc.HistoryClient(options.HistoryClient),
|
||||
svc.ValueClient(options.ValueClient),
|
||||
svc.RegisteredEvents(options.RegisteredEvents),
|
||||
)
|
||||
if err != nil {
|
||||
options.Logger.Fatal().Err(err).Msg("failed to register the handler")
|
||||
return http.Service{}, err
|
||||
}
|
||||
|
||||
newService.Init()
|
||||
return newService, nil
|
||||
|
||||
}
|
||||
|
||||
// Service defines the business logic implementations need to provide.
|
||||
type ActivityLogService interface {
|
||||
GetItemActivities(ctx context.Context, query, loc string, t l10n.Translator) ([]libregraph.Activity, error)
|
||||
}
|
||||
|
||||
// GetActivitiesResponse is the response on GET activities requests
|
||||
type GetActivitiesResponse struct {
|
||||
Activities []libregraph.Activity `json:"value"`
|
||||
}
|
||||
|
||||
func GetItemActivitiesHandler(log log.Logger, s ActivityLogService, vc settingssvc.ValueService, t l10n.Translator) func(w http.ResponseWriter, r *http.Request) {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, r.Header.Get(revactx.TokenHeader))
|
||||
|
||||
activeUser, ok := revactx.ContextGetUser(ctx)
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
loc := l10n.MustGetUserLocale(ctx, activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), vc)
|
||||
|
||||
activities, err := s.GetItemActivities(ctx, r.URL.Query().Get("kql"), loc, t)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, activityloghttp.ErrBadRequest):
|
||||
log.Debug().Str("query", r.URL.Query().Get("kql")).Err(err).Msg("error getting activities")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
case errors.Is(err, activityloghttp.ErrForbidden):
|
||||
log.Debug().Err(err).Msg("error getting activities")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
default:
|
||||
log.Error().Err(err).Msg("error getting activities")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
res := GetActivitiesResponse{
|
||||
Activities: activities,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json; odata.metadata=minimal")
|
||||
w.Header().Set("OData-Version", "4.0")
|
||||
if reqID := chimiddleware.GetReqID(ctx); reqID != "" {
|
||||
w.Header().Set("request-id", reqID)
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
if err := json.NewEncoder(w).Encode(res); err != nil {
|
||||
log.Error().Err(err).Msg("error encoding activities")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if err := micro.RegisterHandler(service.Server(), handle); err != nil {
|
||||
return http.Service{}, err
|
||||
}
|
||||
|
||||
return service, nil
|
||||
}
|
||||
@@ -1,366 +0,0 @@
|
||||
package activitylog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base32"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/jellydator/ttlcache/v2"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog")
|
||||
}
|
||||
|
||||
var (
|
||||
_maxActivitiesDefault = 6000
|
||||
_writeBufferDuration = 10 * time.Second
|
||||
)
|
||||
|
||||
// Activitylog stores and retrieves activities for resources and their parents from a nats kv
|
||||
type ActivityLog struct {
|
||||
log log.Logger
|
||||
// FIXME the lock does not protect agains concurrent resource activities on multiple instances
|
||||
// known since https://github.com/owncloud/ocis/pull/9361#pullrequestreview-2135350157
|
||||
// current ocis discussion in https://github.com/owncloud/ocis/issues/12475
|
||||
lock sync.RWMutex
|
||||
debouncer *Debouncer
|
||||
parentIdCache *ttlcache.Cache
|
||||
natskv nats.KeyValue
|
||||
|
||||
maxActivities int
|
||||
}
|
||||
|
||||
type batchInfo struct {
|
||||
key string
|
||||
count int
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
// New creates a new ActivitylogService
|
||||
func New(kv nats.KeyValue, opts ...Option) (*ActivityLog, error) {
|
||||
o := &Options{
|
||||
MaxActivities: _maxActivitiesDefault,
|
||||
WriteBufferDuration: _writeBufferDuration,
|
||||
Logger: log.NopLogger(),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
|
||||
cache := ttlcache.NewCache()
|
||||
err := cache.SetTTL(30 * time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &ActivityLog{
|
||||
log: o.Logger,
|
||||
lock: sync.RWMutex{},
|
||||
parentIdCache: cache,
|
||||
maxActivities: o.MaxActivities,
|
||||
natskv: kv,
|
||||
}
|
||||
s.debouncer = NewDebouncer(o.WriteBufferDuration, s.StoreActivity)
|
||||
|
||||
// run migrations
|
||||
err = s.runMigrations(context.Background(), kv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// RemoveResource removes the resource from the store
|
||||
func (a *ActivityLog) RemoveResource(rid *provider.ResourceId) error {
|
||||
if rid == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
err := a.natskv.Delete(storagespace.FormatResourceID(rid))
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not delete resource %s: %w", rid.OpaqueId, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RawActivity represents an activity as it is stored in the activitylog store
|
||||
type RawActivity struct {
|
||||
EventID string `json:"event_id"`
|
||||
Depth int `json:"depth"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
func (a *ActivityLog) AddActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time, getResource func(context.Context, *provider.Reference) (*provider.ResourceInfo, error)) error {
|
||||
var (
|
||||
err error
|
||||
depth int
|
||||
ref = initRef
|
||||
)
|
||||
ctx, span := tracer.Start(ctx, "AddActivity")
|
||||
defer span.End()
|
||||
for {
|
||||
var info *provider.ResourceInfo
|
||||
id := ref.GetResourceId()
|
||||
if ref.Path != "" {
|
||||
// Path based reference, we need to resolve the resource id
|
||||
ctx, span = tracer.Start(ctx, "AddActivity.getResource")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil {
|
||||
// TODO If the resource was deleted should we still log an activity in the parent?
|
||||
return fmt.Errorf("could not get resource info for reference %v: %w", ref, err)
|
||||
}
|
||||
id = info.GetId()
|
||||
}
|
||||
if id == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
key := storagespace.FormatResourceID(id)
|
||||
a.debouncer.Debounce(key, RawActivity{
|
||||
EventID: eventID,
|
||||
Depth: depth,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
|
||||
if id.OpaqueId == id.SpaceId {
|
||||
// we are at the root of the space, no need to go further
|
||||
break
|
||||
}
|
||||
|
||||
// check if parent id is cached
|
||||
// parent id is cached in the format <storageid>$<spaceid>!<resourceid>
|
||||
// if it is not cached, get the resource info and cache it
|
||||
if parentId == nil {
|
||||
if v, err := a.parentIdCache.Get(key); err != nil {
|
||||
if info == nil {
|
||||
ctx, span := tracer.Start(ctx, "AddActivity.getResource parent")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil || info.GetParentId() == nil || info.GetParentId().GetOpaqueId() == "" {
|
||||
return fmt.Errorf("could not get parent id: %w", err)
|
||||
}
|
||||
}
|
||||
parentId = info.GetParentId()
|
||||
a.parentIdCache.Set(key, parentId)
|
||||
} else {
|
||||
parentId = v.(*provider.ResourceId)
|
||||
}
|
||||
} else {
|
||||
a.log.Debug().Msg("parent id is cached")
|
||||
}
|
||||
|
||||
depth++
|
||||
ref = &provider.Reference{ResourceId: parentId}
|
||||
parentId = nil // reset parent id so it's not reused in the next iteration
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivityLog) StoreActivity(resourceID string, activities []RawActivity) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
ctx, span := tracer.Start(context.Background(), "storeActivity")
|
||||
defer span.End()
|
||||
|
||||
_, subspan := tracer.Start(ctx, "storeActivity.Marshal")
|
||||
b, err := msgpack.Marshal(activities)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = tracer.Start(ctx, "storeActivity.natskv.Put")
|
||||
key := natsKey(resourceID, len(activities))
|
||||
_, err = a.natskv.Put(key, b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
ctx, subspan = tracer.Start(ctx, "storeActivity.enforceMaxActivities")
|
||||
a.enforceMaxActivities(ctx, resourceID)
|
||||
subspan.End()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivityLog) enforceMaxActivities(ctx context.Context, resourceID string) {
|
||||
if a.maxActivities <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
_, subspan := tracer.Start(ctx, "enforceMaxActivities.watch")
|
||||
watcher, err := a.natskv.Watch(key, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("resourceID", resourceID).Msg("could not watch")
|
||||
return
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var keys []string
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
|
||||
}
|
||||
keys = append(keys, update.Key())
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = tracer.Start(ctx, "enforceMaxActivities.compile")
|
||||
// Parse keys into batches
|
||||
batches := make([]batchInfo, 0)
|
||||
var activitiesCount int
|
||||
for _, k := range keys {
|
||||
parts := strings.SplitN(k, ".", 3)
|
||||
if len(parts) < 3 {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, not enough parts")
|
||||
continue
|
||||
}
|
||||
|
||||
c, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse count")
|
||||
continue
|
||||
}
|
||||
|
||||
// parse timestamp
|
||||
nano, err := strconv.ParseInt(parts[2], 10, 64)
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse timestamp")
|
||||
continue
|
||||
}
|
||||
|
||||
batches = append(batches, batchInfo{
|
||||
key: k,
|
||||
count: c,
|
||||
timestamp: time.Unix(0, nano),
|
||||
})
|
||||
activitiesCount += c
|
||||
}
|
||||
|
||||
// sort batches by timestamp
|
||||
sort.Slice(batches, func(i, j int) bool {
|
||||
return batches[i].timestamp.Before(batches[j].timestamp)
|
||||
})
|
||||
subspan.End()
|
||||
|
||||
_, subspan = tracer.Start(ctx, "enforceMaxActivities.delete")
|
||||
// remove oldest keys until we are at max activities
|
||||
for _, b := range batches {
|
||||
if activitiesCount-b.count < a.maxActivities {
|
||||
break
|
||||
}
|
||||
|
||||
activitiesCount -= b.count
|
||||
err = a.natskv.Delete(b.key)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("key", b.key).Msg("could not delete key")
|
||||
break
|
||||
}
|
||||
}
|
||||
subspan.End()
|
||||
}
|
||||
|
||||
func (a *ActivityLog) InvalidateCachedParentID(purgeId *provider.ResourceId) {
|
||||
// The parent id cache is populated lazily and its entries expire, so a
|
||||
// missing key is the expected case rather than an error.
|
||||
if err := a.parentIdCache.Remove(storagespace.FormatResourceID(purgeId)); err != nil {
|
||||
a.log.Debug().Interface("event", purgeId).Err(err).Msg("could not delete parent id cache")
|
||||
}
|
||||
}
|
||||
|
||||
func natsKey(resourceID string, activitiesCount int) string {
|
||||
return fmt.Sprintf("%s.%d.%d",
|
||||
base32.StdEncoding.EncodeToString([]byte(resourceID)),
|
||||
activitiesCount,
|
||||
time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func (a *ActivityLog) Activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
a.lock.RLock()
|
||||
defer a.lock.RUnlock()
|
||||
|
||||
return a.activities(rid)
|
||||
}
|
||||
|
||||
func (a *ActivityLog) activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
resourceID := storagespace.FormatResourceID(rid)
|
||||
|
||||
glob := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
watcher, err := a.natskv.Watch(glob, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var activities []RawActivity
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack")
|
||||
}
|
||||
activities = append(activities, batchActivities...)
|
||||
}
|
||||
|
||||
return activities, nil
|
||||
}
|
||||
|
||||
// RemoveActivities removes the activities from the given resource
|
||||
func (a *ActivityLog) RemoveActivities(rid *provider.ResourceId, toDelete map[string]struct{}) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
curActivities, err := a.activities(rid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var acts []RawActivity
|
||||
for _, a := range curActivities {
|
||||
if _, ok := toDelete[a.EventID]; !ok {
|
||||
acts = append(acts, a)
|
||||
}
|
||||
}
|
||||
|
||||
b, err := json.Marshal(acts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = a.natskv.Put(storagespace.FormatResourceID(rid), b)
|
||||
return err
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package activitylog_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestActivitylog(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Activitylog Suite")
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package activitylog
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Debouncer is used to debounce writes to the activity log store.
|
||||
type Debouncer struct {
|
||||
after time.Duration
|
||||
f func(id string, ra []RawActivity) error
|
||||
pending sync.Map
|
||||
inProgress sync.Map
|
||||
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
type queueItem struct {
|
||||
activities []RawActivity
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
// NewDebouncer returns a new Debouncer instance.
|
||||
func NewDebouncer(d time.Duration, f func(id string, ra []RawActivity) error) *Debouncer {
|
||||
return &Debouncer{
|
||||
after: d,
|
||||
f: f,
|
||||
pending: sync.Map{},
|
||||
inProgress: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
// Debounce restarts the debounce timer for the given space.
|
||||
func (d *Debouncer) Debounce(id string, ra RawActivity) {
|
||||
if d.after == 0 {
|
||||
d.f(id, []RawActivity{ra})
|
||||
return
|
||||
}
|
||||
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
|
||||
item := &queueItem{
|
||||
activities: []RawActivity{ra},
|
||||
}
|
||||
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
// if the item is already in the queue, append the new activities
|
||||
item, ok = i.(*queueItem)
|
||||
if ok {
|
||||
item.activities = append(item.activities, ra)
|
||||
}
|
||||
}
|
||||
|
||||
if item.timer == nil {
|
||||
item.timer = time.AfterFunc(d.after, func() {
|
||||
if _, ok := d.inProgress.Load(id); ok {
|
||||
// Reschedule this run for when the previous run has finished
|
||||
d.mutex.Lock()
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
i.(*queueItem).timer.Reset(d.after)
|
||||
}
|
||||
|
||||
d.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
d.pending.Delete(id)
|
||||
d.inProgress.Store(id, true)
|
||||
defer d.inProgress.Delete(id)
|
||||
d.f(id, item.activities)
|
||||
})
|
||||
}
|
||||
|
||||
d.pending.Store(id, item)
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
package activitylog_test
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
)
|
||||
|
||||
var _ = Describe("Debouncer", func() {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
callbacks []activitylog.RawActivity
|
||||
newCallback func(id string, ra []activitylog.RawActivity) error
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
mu.Lock()
|
||||
callbacks = nil
|
||||
mu.Unlock()
|
||||
newCallback = func(id string, ra []activitylog.RawActivity) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
callbacks = append(callbacks, ra...)
|
||||
return nil
|
||||
}
|
||||
})
|
||||
|
||||
Context("with zero duration", func() {
|
||||
It("calls the callback immediately", func() {
|
||||
d := activitylog.NewDebouncer(0, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
Expect(callbacks).To(HaveLen(1))
|
||||
Expect(callbacks[0].EventID).To(Equal("activity1"))
|
||||
})
|
||||
|
||||
It("calls the callback immediately for each event", func() {
|
||||
d := activitylog.NewDebouncer(0, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
d.Debounce("space2", activitylog.RawActivity{EventID: "activity2"})
|
||||
Expect(callbacks).To(HaveLen(2))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with non-zero duration", func() {
|
||||
It("batches activities with the same id", func() {
|
||||
d := activitylog.NewDebouncer(10*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity3"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(3))
|
||||
})
|
||||
|
||||
It("handles different ids independently", func() {
|
||||
d := activitylog.NewDebouncer(10*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
d.Debounce("space2", activitylog.RawActivity{EventID: "activity2"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(2))
|
||||
})
|
||||
|
||||
It("batches activities that arrive within the debounce window", func() {
|
||||
d := activitylog.NewDebouncer(100*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity3"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(3))
|
||||
})
|
||||
|
||||
It("processes new batch after previous completes", func() {
|
||||
d := activitylog.NewDebouncer(5*time.Millisecond, newCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
time.Sleep(20 * time.Millisecond) // let first batch complete
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(2))
|
||||
})
|
||||
|
||||
It("skips duplicate write when timer fires during in-progress callback", func() {
|
||||
slowCallback := func(id string, ra []activitylog.RawActivity) error {
|
||||
time.Sleep(50 * time.Millisecond) // simulate slow write
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
callbacks = append(callbacks, ra...)
|
||||
return nil
|
||||
}
|
||||
|
||||
d := activitylog.NewDebouncer(10*time.Millisecond, slowCallback)
|
||||
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
|
||||
time.Sleep(20 * time.Millisecond) // timer fires while callback is running
|
||||
|
||||
Eventually(func() int {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(callbacks)
|
||||
}).Should(Equal(1))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,35 +0,0 @@
|
||||
package activitylog
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
)
|
||||
|
||||
// Option for the activitylog service
|
||||
type Option func(*Options)
|
||||
|
||||
// Options for the activitylog service
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
MaxActivities int
|
||||
WriteBufferDuration time.Duration
|
||||
}
|
||||
|
||||
// Logger configures a logger for the activitylog service
|
||||
func Logger(log log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = log
|
||||
}
|
||||
}
|
||||
|
||||
func MaxActivities(max int) Option {
|
||||
return func(o *Options) {
|
||||
o.MaxActivities = max
|
||||
}
|
||||
}
|
||||
func WriteBufferDuration(d time.Duration) Option {
|
||||
return func(o *Options) {
|
||||
o.WriteBufferDuration = d
|
||||
}
|
||||
}
|
||||
@@ -1,302 +0,0 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events")
|
||||
}
|
||||
|
||||
var (
|
||||
_numConsumersDefault = 1
|
||||
)
|
||||
|
||||
// ActivitylogService logs events per resource
|
||||
type ActivitylogService struct {
|
||||
ctx context.Context
|
||||
sa config.ServiceAccount
|
||||
log log.Logger
|
||||
stream events.Stream
|
||||
gws pool.Selectable[gateway.GatewayAPIClient]
|
||||
al *activitylog.ActivityLog
|
||||
|
||||
numConsumers int
|
||||
|
||||
events []events.Unmarshaller
|
||||
|
||||
stopCh chan struct{}
|
||||
stopped *atomic.Bool
|
||||
}
|
||||
|
||||
// New creates a new ActivitylogService
|
||||
func New(al *activitylog.ActivityLog, stream events.Stream, opts ...Option) (*ActivitylogService, error) {
|
||||
o := &Options{
|
||||
NumConsumers: _numConsumersDefault,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
|
||||
s := &ActivitylogService{
|
||||
ctx: o.Context,
|
||||
log: o.Logger,
|
||||
sa: o.ServiceAccount,
|
||||
stream: stream,
|
||||
gws: o.GatewaySelector,
|
||||
events: o.RegisteredEvents,
|
||||
numConsumers: o.NumConsumers,
|
||||
al: al,
|
||||
stopCh: make(chan struct{}, 1),
|
||||
stopped: new(atomic.Bool),
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Run to fulfil Runner interface
|
||||
func (s *ActivitylogService) Run() error {
|
||||
ch, err := events.Consume(s.stream, "activitylog", s.events...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
ctx, cancel := context.WithCancel(s.ctx)
|
||||
defer cancel()
|
||||
|
||||
s.log.Debug().Int("worker.count", s.numConsumers).
|
||||
Str("messaging.consumer.group.name", "activitylog").
|
||||
Str("messaging.system", "nats").
|
||||
Str("messaging.operation.name", "receive").
|
||||
Msg("starting event processing workers")
|
||||
|
||||
// start workers
|
||||
for i := 0; i < s.numConsumers; i++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := s.processEvent(e); err != nil {
|
||||
s.log.Error().Err(err).
|
||||
Int("worker", workerID).
|
||||
Interface("event", e).
|
||||
Msg("failed to process event")
|
||||
}
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// wait for stop signal
|
||||
<-s.stopCh
|
||||
cancel() // signal workers to stop
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close will make the service to stop processing, so the `Run`
|
||||
// method can finish.
|
||||
func (s *ActivitylogService) Close() {
|
||||
if s.stopped.CompareAndSwap(false, true) {
|
||||
close(s.stopCh)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ActivitylogService) processEvent(e events.Event) error {
|
||||
ctx := e.GetTraceContext(s.ctx)
|
||||
ctx, span := tracer.Start(ctx, "processEvent")
|
||||
defer span.End()
|
||||
|
||||
s.log.Debug().Interface("event", e).Msg("updating activitylog")
|
||||
|
||||
switch ev := e.Event.(type) {
|
||||
case events.UploadReady:
|
||||
return s.AddActivity(ctx, ev.FileRef, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.FileTouched:
|
||||
return s.AddActivity(ctx, ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
// Disabled https://github.com/owncloud/ocis/issues/10293
|
||||
//case events.FileDownloaded:
|
||||
// we are only interested in public link downloads - so no need to store others.
|
||||
//if ev.ImpersonatingUser.GetDisplayName() == "Public" {
|
||||
// err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
//}
|
||||
case events.ContainerCreated:
|
||||
return s.AddActivity(ctx, ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemTrashed:
|
||||
return s.AddActivityTrashed(ctx, ev.ID, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemPurged:
|
||||
return s.al.RemoveResource(ev.ID)
|
||||
case events.ItemMoved:
|
||||
// remove the cached parent id for this resource
|
||||
s.removeCachedParentID(ctx, ev.Ref)
|
||||
|
||||
return s.AddActivity(ctx, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ShareCreated:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.ShareUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.ShareRemoved:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, ev.Timestamp)
|
||||
case events.LinkCreated:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.LinkUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.LinkRemoved:
|
||||
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.SpaceShared:
|
||||
return s.AddSpaceActivity(ctx, ev.ID, e.ID, ev.Timestamp)
|
||||
case events.SpaceUnshared:
|
||||
return s.AddSpaceActivity(ctx, ev.ID, e.ID, ev.Timestamp)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddActivity adds the activity to the given resource and all its parents
|
||||
func (a *ActivitylogService) AddActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
ctx, span := tracer.Start(ctx, "AddActivity")
|
||||
defer span.End()
|
||||
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
|
||||
}
|
||||
return a.al.AddActivity(ctx, initRef, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
|
||||
func (a *ActivitylogService) AddActivityTrashed(ctx context.Context, resourceID *provider.ResourceId, reference *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
ctx, span := tracer.Start(ctx, "AddActivityTrashed")
|
||||
defer span.End()
|
||||
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
}
|
||||
|
||||
// store activity on trashed item
|
||||
if err := a.al.StoreActivity(storagespace.FormatResourceID(resourceID), []activitylog.RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("could not store activity: %w", err)
|
||||
}
|
||||
|
||||
// get previous parent
|
||||
ref := &provider.Reference{
|
||||
ResourceId: reference.GetResourceId(),
|
||||
Path: filepath.Dir(reference.GetPath()),
|
||||
}
|
||||
|
||||
return a.al.AddActivity(ctx, ref, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddSpaceActivity adds the activity to the given spaceroot
|
||||
func (a *ActivitylogService) AddSpaceActivity(ctx context.Context, spaceID *provider.StorageSpaceId, eventID string, timestamp time.Time) error {
|
||||
_, span := tracer.Start(ctx, "AddSpaceActivity")
|
||||
defer span.End()
|
||||
// spaceID is in format <providerid>$<spaceid>
|
||||
// activitylog service uses format <providerid>$<spaceid>!<resourceid>
|
||||
// lets do some converting, shall we?
|
||||
rid, err := storagespace.ParseID(spaceID.GetOpaqueId())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse space id: %w", err)
|
||||
}
|
||||
rid.OpaqueId = rid.GetSpaceId()
|
||||
err = a.al.StoreActivity(storagespace.FormatResourceID(&rid), []activitylog.RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not store activity: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func toRef(r *provider.ResourceId) *provider.Reference {
|
||||
return &provider.Reference{
|
||||
ResourceId: r,
|
||||
}
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) removeCachedParentID(ctx context.Context, ref *provider.Reference) {
|
||||
var span trace.Span
|
||||
ctx, span = tracer.Start(ctx, "removeCachedParentID")
|
||||
defer span.End()
|
||||
|
||||
purgeId := ref.GetResourceId()
|
||||
if ref.GetPath() != "" {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get gateway client")
|
||||
return
|
||||
}
|
||||
|
||||
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get service user context")
|
||||
return
|
||||
}
|
||||
|
||||
info, err := utils.GetResource(ctx, ref, gwc)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get resource info")
|
||||
return
|
||||
}
|
||||
purgeId = info.GetId()
|
||||
}
|
||||
a.al.InvalidateCachedParentID(purgeId)
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
package events_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
eventssvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
var _ = Describe("ActivitylogService", func() {
|
||||
Describe("New", func() {
|
||||
var (
|
||||
al *activitylog.ActivityLog
|
||||
stream events.Stream
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
al = &activitylog.ActivityLog{}
|
||||
stream = nil
|
||||
})
|
||||
|
||||
It("creates a service with minimal options", func() {
|
||||
svc, err := eventssvc.New(al, stream)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with context option", func() {
|
||||
ctx := context.Background()
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.Context(ctx))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with logger option", func() {
|
||||
logger := log.NopLogger()
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.Logger(logger))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with service account option", func() {
|
||||
sa := config.ServiceAccount{
|
||||
ServiceAccountID: "sa-id",
|
||||
ServiceAccountSecret: "sa-secret",
|
||||
}
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.ServiceAccount(sa))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with registered events", func() {
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}, &events.FileTouched{}}
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.RegisteredEvents(evts))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with gateway selector", func() {
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.GatewaySelector(gs))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with num consumers option", func() {
|
||||
svc, err := eventssvc.New(al, stream, eventssvc.NumConsumers(5))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with all options", func() {
|
||||
ctx := context.Background()
|
||||
logger := log.NopLogger()
|
||||
sa := config.ServiceAccount{
|
||||
ServiceAccountID: "sa-id",
|
||||
ServiceAccountSecret: "sa-secret",
|
||||
}
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}, &events.ContainerCreated{}}
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
|
||||
svc, err := eventssvc.New(
|
||||
al,
|
||||
stream,
|
||||
eventssvc.Context(ctx),
|
||||
eventssvc.Logger(logger),
|
||||
eventssvc.ServiceAccount(sa),
|
||||
eventssvc.RegisteredEvents(evts),
|
||||
eventssvc.GatewaySelector(gs),
|
||||
eventssvc.NumConsumers(3),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Close", func() {
|
||||
It("can be called without panic on a new service", func() {
|
||||
al := &activitylog.ActivityLog{}
|
||||
svc, err := eventssvc.New(al, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(func() { svc.Close() }).ToNot(Panic())
|
||||
})
|
||||
|
||||
It("can be called multiple times without panic", func() {
|
||||
al := &activitylog.ActivityLog{}
|
||||
svc, err := eventssvc.New(al, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
svc.Close()
|
||||
svc.Close()
|
||||
svc.Close()
|
||||
})
|
||||
})
|
||||
})
|
||||
+80
-79
@@ -1,97 +1,89 @@
|
||||
package http
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/olekukonko/errors"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"google.golang.org/grpc/metadata"
|
||||
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/eventhistory/v0"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http")
|
||||
}
|
||||
|
||||
// New returns a new instance of Service
|
||||
func New(al *activitylog.ActivityLog, opts ...Option) (*svc, error) {
|
||||
o := newOptions(opts...)
|
||||
|
||||
registeredEvents := make(map[string]events.Unmarshaller)
|
||||
for _, e := range o.RegisteredEvents {
|
||||
typ := reflect.TypeOf(e)
|
||||
registeredEvents[typ.String()] = e
|
||||
}
|
||||
|
||||
return &svc{
|
||||
log: o.Logger,
|
||||
evHistory: o.HistoryClient,
|
||||
al: al,
|
||||
registeredEvents: registeredEvents,
|
||||
gws: o.GatewaySelector,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type svc struct {
|
||||
log log.Logger
|
||||
evHistory ehsvc.EventHistoryService
|
||||
gws pool.Selectable[gateway.GatewayAPIClient]
|
||||
al *activitylog.ActivityLog
|
||||
registeredEvents map[string]events.Unmarshaller
|
||||
}
|
||||
|
||||
var (
|
||||
ErrBadRequest = errors.New("bad request")
|
||||
ErrForbidden = errors.New("forbidden")
|
||||
//go:embed l10n/locale
|
||||
_localeFS embed.FS
|
||||
|
||||
// subfolder where the translation files are stored
|
||||
_localeSubPath = "l10n/locale"
|
||||
|
||||
// domain of the activitylog service (transifex)
|
||||
_domain = "activitylog"
|
||||
)
|
||||
|
||||
func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.Translator) ([]libregraph.Activity, error) {
|
||||
gwc, err := s.gws.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// ServeHTTP implements the http.Handler interface.
|
||||
func (s *ActivitylogService) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.mux.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// HandleGetItemActivities handles the request to get the activities of an item.
|
||||
func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, r.Header.Get(revactx.TokenHeader))
|
||||
|
||||
activeUser, ok := revactx.ContextGetUser(ctx)
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
|
||||
rid, limit, rawActivityAccepted, activityAccepted, sort, err := s.getFilters(query)
|
||||
gwc, err := s.gws.Next()
|
||||
if err != nil {
|
||||
s.log.Info().Str("query", query).Err(err).Msg("error getting filters")
|
||||
return nil, ErrBadRequest
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
rid, limit, rawActivityAccepted, activityAccepted, sort, err := s.getFilters(r.URL.Query().Get("kql"))
|
||||
if err != nil {
|
||||
s.log.Info().Str("query", r.URL.Query().Get("kql")).Err(err).Msg("error getting filters")
|
||||
_, _ = w.Write([]byte(err.Error()))
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
info, err := utils.GetResourceByID(ctx, rid, gwc)
|
||||
if err != nil {
|
||||
return nil, ErrForbidden
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
||||
// you need ListGrants to see activities
|
||||
if !info.GetPermissionSet().GetListGrants() {
|
||||
return nil, ErrForbidden
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
|
||||
raw, err := s.al.Activities(rid)
|
||||
raw, err := s.Activities(rid)
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error getting activities")
|
||||
return nil, err
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(raw))
|
||||
@@ -104,21 +96,21 @@ func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.T
|
||||
toDelete[a.EventID] = struct{}{}
|
||||
}
|
||||
|
||||
evRes, err := s.evHistory.GetEvents(ctx, &ehsvc.GetEventsRequest{Ids: ids})
|
||||
evRes, err := s.evHistory.GetEvents(r.Context(), &ehsvc.GetEventsRequest{Ids: ids})
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error getting events")
|
||||
return nil, err
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
evs := evRes.GetEvents()
|
||||
sort(evs)
|
||||
|
||||
// TODO cut the interface here?
|
||||
activities := make([]libregraph.Activity, 0, len(evRes.GetEvents()))
|
||||
resp := GetActivitiesResponse{Activities: make([]libregraph.Activity, 0, len(evRes.GetEvents()))}
|
||||
for _, e := range evs {
|
||||
delete(toDelete, e.GetId())
|
||||
|
||||
if limit > 0 && limit <= len(activities) {
|
||||
if limit > 0 && limit <= len(resp.Activities) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -132,6 +124,9 @@ func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.T
|
||||
vars map[string]any
|
||||
)
|
||||
|
||||
loc := l10n.MustGetUserLocale(r.Context(), activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), s.valService)
|
||||
t := l10n.NewTranslatorFromCommonConfig(s.cfg.DefaultLanguage, _domain, s.cfg.TranslationPath, _localeFS, _localeSubPath)
|
||||
|
||||
switch ev := s.unwrapEvent(e).(type) {
|
||||
case nil:
|
||||
// error already logged in unwrapEvent
|
||||
@@ -229,29 +224,35 @@ func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.T
|
||||
continue
|
||||
}
|
||||
|
||||
activities = append(activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars))
|
||||
resp.Activities = append(resp.Activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars))
|
||||
}
|
||||
|
||||
// delete activities in separate go routine
|
||||
if len(toDelete) > 0 {
|
||||
go func() {
|
||||
err := s.al.RemoveActivities(rid, toDelete)
|
||||
err := s.RemoveActivities(rid, toDelete)
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error removing activities")
|
||||
}
|
||||
}()
|
||||
}
|
||||
return activities, nil
|
||||
|
||||
}
|
||||
|
||||
func toRef(r *provider.ResourceId) *provider.Reference {
|
||||
return &provider.Reference{
|
||||
ResourceId: r,
|
||||
b, err := json.Marshal(resp)
|
||||
if err != nil {
|
||||
s.log.Error().Err(err).Msg("error marshalling activities")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := w.Write(b); err != nil {
|
||||
s.log.Error().Err(err).Msg("error writing response")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (s *svc) unwrapEvent(e *ehmsg.Event) any {
|
||||
func (s *ActivitylogService) unwrapEvent(e *ehmsg.Event) any {
|
||||
etype, ok := s.registeredEvents[e.GetType()]
|
||||
if !ok {
|
||||
s.log.Error().Str("eventid", e.GetId()).Str("eventtype", e.GetType()).Msg("event not registered")
|
||||
@@ -267,13 +268,13 @@ func (s *svc) unwrapEvent(e *ehmsg.Event) any {
|
||||
return einterface
|
||||
}
|
||||
|
||||
func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activitylog.RawActivity) bool, func(*ehmsg.Event) bool, func([]*ehmsg.Event), error) {
|
||||
func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int, func(RawActivity) bool, func(*ehmsg.Event) bool, func([]*ehmsg.Event), error) {
|
||||
qast, err := kql.Builder{}.Build(query)
|
||||
if err != nil {
|
||||
return nil, 0, nil, nil, nil, err
|
||||
}
|
||||
|
||||
prefilters := make([]func(activitylog.RawActivity) bool, 0)
|
||||
prefilters := make([]func(RawActivity) bool, 0)
|
||||
postfilters := make([]func(*ehmsg.Event) bool, 0)
|
||||
|
||||
sortby := func(_ []*ehmsg.Event) {}
|
||||
@@ -298,7 +299,7 @@ func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activity
|
||||
break
|
||||
}
|
||||
|
||||
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
|
||||
prefilters = append(prefilters, func(a RawActivity) bool {
|
||||
return a.Depth <= depth
|
||||
})
|
||||
case "limit":
|
||||
@@ -321,11 +322,11 @@ func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activity
|
||||
case *ast.DateTimeNode:
|
||||
switch v.Operator.Value {
|
||||
case "<", "<=":
|
||||
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
|
||||
prefilters = append(prefilters, func(a RawActivity) bool {
|
||||
return a.Timestamp.Before(v.Value)
|
||||
})
|
||||
case ">", ">=":
|
||||
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
|
||||
prefilters = append(prefilters, func(a RawActivity) bool {
|
||||
return a.Timestamp.After(v.Value)
|
||||
})
|
||||
}
|
||||
@@ -344,7 +345,7 @@ func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activity
|
||||
// space root requested - fix format
|
||||
rid.OpaqueId = rid.GetSpaceId()
|
||||
}
|
||||
pref := func(a activitylog.RawActivity) bool {
|
||||
pref := func(a RawActivity) bool {
|
||||
for _, f := range prefilters {
|
||||
if !f(a) {
|
||||
return false
|
||||
@@ -1,13 +0,0 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestHTTP(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "HTTP Suite")
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
Logger log.Logger
|
||||
RegisteredEvents []events.Unmarshaller
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
HistoryClient ehsvc.EventHistoryService
|
||||
}
|
||||
|
||||
// newOptions initializes the available default options.
|
||||
func newOptions(opts ...Option) Options {
|
||||
opt := Options{}
|
||||
|
||||
for _, o := range opts {
|
||||
o(&opt)
|
||||
}
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
// Logger provides a function to set the logger option.
|
||||
func Logger(val log.Logger) Option {
|
||||
return func(o *Options) {
|
||||
o.Logger = val
|
||||
}
|
||||
}
|
||||
|
||||
// RegisteredEvents registers the events the service should listen to
|
||||
func RegisteredEvents(e []events.Unmarshaller) Option {
|
||||
return func(o *Options) {
|
||||
o.RegisteredEvents = e
|
||||
}
|
||||
}
|
||||
|
||||
// GatewaySelector adds a grpc client selector for the gateway service
|
||||
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
|
||||
return func(o *Options) {
|
||||
o.GatewaySelector = gatewaySelector
|
||||
}
|
||||
}
|
||||
|
||||
// HistoryClient adds a grpc client for the eventhistory service
|
||||
func HistoryClient(hc ehsvc.EventHistoryService) Option {
|
||||
return func(o *Options) {
|
||||
o.HistoryClient = hc
|
||||
}
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
httpsvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
)
|
||||
|
||||
var _ = Describe("Response", func() {
|
||||
Describe("NewActivity", func() {
|
||||
It("creates an activity with the given parameters", func() {
|
||||
ts := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
|
||||
vars := map[string]any{
|
||||
"user": "testuser",
|
||||
"resource": "testfile.txt",
|
||||
}
|
||||
|
||||
act := httpsvc.NewActivity("Test message", ts, "event-123", vars)
|
||||
|
||||
Expect(act.Id).To(Equal("event-123"))
|
||||
Expect(act.Times.RecordedTime).To(Equal(ts))
|
||||
Expect(act.Template.Message).To(Equal("Test message"))
|
||||
Expect(act.Template.Variables).To(HaveKeyWithValue("user", "testuser"))
|
||||
Expect(act.Template.Variables).To(HaveKeyWithValue("resource", "testfile.txt"))
|
||||
})
|
||||
|
||||
It("handles empty variables map", func() {
|
||||
act := httpsvc.NewActivity("", time.Time{}, "", map[string]any{})
|
||||
|
||||
Expect(act.Id).To(BeEmpty())
|
||||
Expect(act.Times.RecordedTime).To(Equal(time.Time{}))
|
||||
Expect(act.Template.Message).To(BeEmpty())
|
||||
Expect(act.Template.Variables).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithOldResource", func() {
|
||||
It("sets the oldResource variable from reference path", func() {
|
||||
ref := &provider.Reference{
|
||||
Path: "/old/path/oldname.txt",
|
||||
}
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithOldResource(ref)
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("oldResource"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithUser", func() {
|
||||
It("returns error when no user is provided", func() {
|
||||
opt := httpsvc.WithUser(nil, nil, nil)
|
||||
err := opt(context.Background(), nil, make(map[string]any))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("no user provided"))
|
||||
})
|
||||
|
||||
It("uses impersonator when provided", func() {
|
||||
impersonator := &user.User{
|
||||
Id: &user.UserId{
|
||||
OpaqueId: "imp-user-id",
|
||||
},
|
||||
DisplayName: "Impersonated User",
|
||||
}
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithUser(nil, nil, impersonator)
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("user"))
|
||||
})
|
||||
|
||||
It("uses executing user when no impersonator", func() {
|
||||
execUser := &user.User{
|
||||
Id: &user.UserId{
|
||||
OpaqueId: "exec-user-id",
|
||||
},
|
||||
DisplayName: "Executing User",
|
||||
}
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithUser(nil, execUser, nil)
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("user"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithVar", func() {
|
||||
It("sets a simple key-value variable", func() {
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithVar("token", "id123", "My Token")
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("token"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WithTranslation", func() {
|
||||
It("sets translated field variable", func() {
|
||||
var t l10n.Translator
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithTranslation(&t, "en", "field", []string{"permission"})
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("field"))
|
||||
})
|
||||
|
||||
It("handles empty values slice", func() {
|
||||
var t l10n.Translator
|
||||
vars := make(map[string]any)
|
||||
|
||||
opt := httpsvc.WithTranslation(&t, "en", "field", []string{})
|
||||
err := opt(context.Background(), nil, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(vars).To(HaveKey("field"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("ActivityOption type", func() {
|
||||
It("allows composing multiple options", func() {
|
||||
vars := make(map[string]any)
|
||||
ctx := context.Background()
|
||||
var gwc gateway.GatewayAPIClient
|
||||
|
||||
options := []httpsvc.ActivityOption{
|
||||
httpsvc.WithVar("key1", "id1", "name1"),
|
||||
httpsvc.WithVar("key2", "id2", "name2"),
|
||||
}
|
||||
|
||||
for _, opt := range options {
|
||||
err := opt(ctx, gwc, vars)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
|
||||
Expect(vars).To(HaveKey("key1"))
|
||||
Expect(vars).To(HaveKey("key2"))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,76 +0,0 @@
|
||||
package http_test
|
||||
|
||||
import (
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
httpsvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
)
|
||||
|
||||
var _ = Describe("Service", func() {
|
||||
Describe("New", func() {
|
||||
var (
|
||||
al *activitylog.ActivityLog
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
al = &activitylog.ActivityLog{}
|
||||
})
|
||||
|
||||
It("creates a service with default options", func() {
|
||||
svc, err := httpsvc.New(al)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with logger option", func() {
|
||||
logger := log.NopLogger()
|
||||
svc, err := httpsvc.New(al, httpsvc.Logger(logger))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with registered events", func() {
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}}
|
||||
svc, err := httpsvc.New(al, httpsvc.RegisteredEvents(evts))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with gateway selector", func() {
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
svc, err := httpsvc.New(al, httpsvc.GatewaySelector(gs))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with history client", func() {
|
||||
var hc ehsvc.EventHistoryService
|
||||
svc, err := httpsvc.New(al, httpsvc.HistoryClient(hc))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("creates a service with all options", func() {
|
||||
logger := log.NopLogger()
|
||||
evts := []events.Unmarshaller{&events.UploadReady{}, &events.FileTouched{}}
|
||||
var gs pool.Selectable[gateway.GatewayAPIClient]
|
||||
var hc ehsvc.EventHistoryService
|
||||
|
||||
svc, err := httpsvc.New(
|
||||
al,
|
||||
httpsvc.Logger(logger),
|
||||
httpsvc.RegisteredEvents(evts),
|
||||
httpsvc.GatewaySelector(gs),
|
||||
httpsvc.HistoryClient(hc),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(svc).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
})
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+3
-3
@@ -1,4 +1,4 @@
|
||||
package activitylog
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -15,7 +15,7 @@ const currentMigrationVersion = "1"
|
||||
|
||||
// RunMigrations checks the activitylog data version and runs migrations if necessary.
|
||||
// It should be called during service startup, after the NATS KeyValue store is initialized.
|
||||
func (a *ActivityLog) runMigrations(ctx context.Context, kv nats.KeyValue) error {
|
||||
func (a *ActivitylogService) runMigrations(ctx context.Context, kv nats.KeyValue) error {
|
||||
entry, err := kv.Get(activitylogVersionKey)
|
||||
if err == nats.ErrKeyNotFound {
|
||||
a.log.Info().Msg("activitylog version key not found. Running migration to V1...")
|
||||
@@ -40,7 +40,7 @@ func (a *ActivityLog) runMigrations(ctx context.Context, kv nats.KeyValue) error
|
||||
// For each such key, it creates a new key in the format "originalKey.count.timestamp"
|
||||
// and stores the original list of strings (re-marshalled to messagepack) as its value.
|
||||
// Finally, it sets the activitylog.version key to "1".
|
||||
func (a *ActivityLog) migrateToV1(_ context.Context, kv nats.KeyValue) error {
|
||||
func (a *ActivitylogService) migrateToV1(_ context.Context, kv nats.KeyValue) error {
|
||||
lister, err := kv.ListKeys()
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrateToV1: failed to list keys from NATS KV store: %w", err)
|
||||
+38
-16
@@ -1,14 +1,17 @@
|
||||
package events
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// Option for the activitylog service
|
||||
@@ -16,20 +19,17 @@ type Option func(*Options)
|
||||
|
||||
// Options for the activitylog service
|
||||
type Options struct {
|
||||
Context context.Context
|
||||
Logger log.Logger
|
||||
ServiceAccount config.ServiceAccount
|
||||
Config *config.Config
|
||||
TraceProvider trace.TracerProvider
|
||||
Stream events.Stream
|
||||
RegisteredEvents []events.Unmarshaller
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
Mux *chi.Mux
|
||||
HistoryClient ehsvc.EventHistoryService
|
||||
ValueClient settingssvc.ValueService
|
||||
WriteBufferDuration time.Duration
|
||||
NumConsumers int
|
||||
}
|
||||
|
||||
func Context(ctx context.Context) Option {
|
||||
return func(o *Options) {
|
||||
o.Context = ctx
|
||||
}
|
||||
MaxActivities int
|
||||
}
|
||||
|
||||
// Logger configures a logger for the activitylog service
|
||||
@@ -39,10 +39,17 @@ func Logger(log log.Logger) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// ServiceAccount configures a service account for the activitylog service
|
||||
func ServiceAccount(sa config.ServiceAccount) Option {
|
||||
// Config adds the config for the activitylog service
|
||||
func Config(c *config.Config) Option {
|
||||
return func(o *Options) {
|
||||
o.ServiceAccount = sa
|
||||
o.Config = c
|
||||
}
|
||||
}
|
||||
|
||||
// TraceProvider adds a tracer provider for the activitylog service
|
||||
func TraceProvider(tp trace.TracerProvider) Option {
|
||||
return func(o *Options) {
|
||||
o.TraceProvider = tp
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,8 +74,23 @@ func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient])
|
||||
}
|
||||
}
|
||||
|
||||
func NumConsumers(num int) Option {
|
||||
// Mux defines the muxer for the service
|
||||
func Mux(m *chi.Mux) Option {
|
||||
return func(o *Options) {
|
||||
o.NumConsumers = num
|
||||
o.Mux = m
|
||||
}
|
||||
}
|
||||
|
||||
// HistoryClient adds a grpc client for the eventhistory service
|
||||
func HistoryClient(hc ehsvc.EventHistoryService) Option {
|
||||
return func(o *Options) {
|
||||
o.HistoryClient = hc
|
||||
}
|
||||
}
|
||||
|
||||
// ValueClient adds a grpc client for the value service
|
||||
func ValueClient(vs settingssvc.ValueService) Option {
|
||||
return func(o *Options) {
|
||||
o.ValueClient = vs
|
||||
}
|
||||
}
|
||||
+7
-8
@@ -1,4 +1,4 @@
|
||||
package http
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -44,6 +44,11 @@ var (
|
||||
StrDescription = l10n.Template("description")
|
||||
)
|
||||
|
||||
// GetActivitiesResponse is the response on GET activities requests
|
||||
type GetActivitiesResponse struct {
|
||||
Activities []libregraph.Activity `json:"value"`
|
||||
}
|
||||
|
||||
// Resource represents an item such as a file or folder
|
||||
type Resource struct {
|
||||
ID string `json:"id"`
|
||||
@@ -306,7 +311,7 @@ func NewActivity(message string, ts time.Time, eventID string, vars map[string]a
|
||||
}
|
||||
|
||||
// GetVars calls other service to gather the required data for the activity variables
|
||||
func (s *svc) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]any, error) {
|
||||
func (s *ActivitylogService) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]any, error) {
|
||||
gwc, err := s.gws.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -322,12 +327,6 @@ func (s *svc) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]a
|
||||
return vars, nil
|
||||
}
|
||||
|
||||
func toSpace(r *provider.Reference) *provider.StorageSpaceId {
|
||||
return &provider.StorageSpaceId{
|
||||
OpaqueId: storagespace.FormatStorageID(r.GetResourceId().GetStorageId(), r.GetResourceId().GetSpaceId()),
|
||||
}
|
||||
}
|
||||
|
||||
func getFolderName(ctx context.Context, gwc gateway.GatewayAPIClient, ref *provider.Reference) string {
|
||||
n := filepath.Base(filepath.Dir(ref.GetPath()))
|
||||
if n == "." || n == "/" {
|
||||
@@ -0,0 +1,676 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base32"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/jellydator/ttlcache/v2"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
|
||||
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
)
|
||||
|
||||
// Nats runs into max payload exceeded errors at around 7k activities. Let's keep a buffer.
|
||||
var _maxActivitiesDefault = 6000
|
||||
|
||||
// RawActivity represents an activity as it is stored in the activitylog store
|
||||
type RawActivity struct {
|
||||
EventID string `json:"event_id"`
|
||||
Depth int `json:"depth"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
// ActivitylogService logs events per resource
|
||||
type ActivitylogService struct {
|
||||
cfg *config.Config
|
||||
log log.Logger
|
||||
events <-chan events.Event
|
||||
gws pool.Selectable[gateway.GatewayAPIClient]
|
||||
mux *chi.Mux
|
||||
evHistory ehsvc.EventHistoryService
|
||||
valService settingssvc.ValueService
|
||||
lock sync.RWMutex
|
||||
tp trace.TracerProvider
|
||||
tracer trace.Tracer
|
||||
debouncer *Debouncer
|
||||
parentIdCache *ttlcache.Cache
|
||||
natskv nats.KeyValue
|
||||
|
||||
maxActivities int
|
||||
|
||||
registeredEvents map[string]events.Unmarshaller
|
||||
}
|
||||
|
||||
type Debouncer struct {
|
||||
after time.Duration
|
||||
f func(id string, ra []RawActivity) error
|
||||
pending sync.Map
|
||||
inProgress sync.Map
|
||||
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
type queueItem struct {
|
||||
activities []RawActivity
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
type batchInfo struct {
|
||||
key string
|
||||
count int
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
// NewDebouncer returns a new Debouncer instance
|
||||
func NewDebouncer(d time.Duration, f func(id string, ra []RawActivity) error) *Debouncer {
|
||||
return &Debouncer{
|
||||
after: d,
|
||||
f: f,
|
||||
pending: sync.Map{},
|
||||
inProgress: sync.Map{},
|
||||
}
|
||||
}
|
||||
|
||||
// Debounce restarts the debounce timer for the given space
|
||||
func (d *Debouncer) Debounce(id string, ra RawActivity) {
|
||||
if d.after == 0 {
|
||||
d.f(id, []RawActivity{ra})
|
||||
return
|
||||
}
|
||||
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
|
||||
activities := []RawActivity{ra}
|
||||
item := &queueItem{
|
||||
activities: activities,
|
||||
}
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
// if the item is already in the queue, append the new activities
|
||||
item, ok = i.(*queueItem)
|
||||
if ok {
|
||||
item.activities = append(item.activities, ra)
|
||||
}
|
||||
}
|
||||
|
||||
if item.timer == nil {
|
||||
item.timer = time.AfterFunc(d.after, func() {
|
||||
if _, ok := d.inProgress.Load(id); ok {
|
||||
// Reschedule this run for when the previous run has finished
|
||||
d.mutex.Lock()
|
||||
if i, ok := d.pending.Load(id); ok {
|
||||
i.(*queueItem).timer.Reset(d.after)
|
||||
}
|
||||
|
||||
d.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
d.pending.Delete(id)
|
||||
d.inProgress.Store(id, true)
|
||||
defer d.inProgress.Delete(id)
|
||||
d.f(id, item.activities)
|
||||
})
|
||||
}
|
||||
|
||||
d.pending.Store(id, item)
|
||||
}
|
||||
|
||||
// New creates a new ActivitylogService
|
||||
func New(opts ...Option) (*ActivitylogService, error) {
|
||||
o := &Options{
|
||||
MaxActivities: _maxActivitiesDefault,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(o)
|
||||
}
|
||||
|
||||
if o.Stream == nil {
|
||||
return nil, errors.New("stream is required")
|
||||
}
|
||||
|
||||
ch, err := events.Consume(o.Stream, o.Config.Service.Name, o.RegisteredEvents...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cache := ttlcache.NewCache()
|
||||
err = cache.SetTTL(30 * time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Connect to NATS servers
|
||||
natsOptions := nats.Options{
|
||||
Servers: o.Config.Store.Nodes,
|
||||
}
|
||||
if o.Config.Store.EnableTLS {
|
||||
if o.Config.Store.TLSRootCACertificate != "" {
|
||||
// when root ca is configured use it. an insecure flag is ignored.
|
||||
nats.RootCAs(o.Config.Store.TLSRootCACertificate)(&natsOptions)
|
||||
} else {
|
||||
// enable tls and use insecure flag
|
||||
nats.Secure(&tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: o.Config.Store.TLSInsecure})(&natsOptions)
|
||||
}
|
||||
}
|
||||
if o.Config.Store.AuthUsername != "" && o.Config.Store.AuthPassword != "" {
|
||||
nats.UserInfo(o.Config.Store.AuthUsername, o.Config.Store.AuthPassword)(&natsOptions)
|
||||
}
|
||||
conn, err := natsOptions.Connect()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
js, err := conn.JetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
kv, err := js.KeyValue(o.Config.Store.Database)
|
||||
if err != nil {
|
||||
if !errors.Is(err, nats.ErrBucketNotFound) {
|
||||
return nil, errors.Wrapf(err, "Failed to get bucket (%s)", o.Config.Store.Database)
|
||||
}
|
||||
|
||||
kv, err = js.CreateKeyValue(&nats.KeyValueConfig{
|
||||
Bucket: o.Config.Store.Database,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to create bucket (%s)", o.Config.Store.Database)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &ActivitylogService{
|
||||
log: o.Logger,
|
||||
cfg: o.Config,
|
||||
events: ch,
|
||||
gws: o.GatewaySelector,
|
||||
mux: o.Mux,
|
||||
evHistory: o.HistoryClient,
|
||||
valService: o.ValueClient,
|
||||
lock: sync.RWMutex{},
|
||||
registeredEvents: make(map[string]events.Unmarshaller),
|
||||
tp: o.TraceProvider,
|
||||
tracer: o.TraceProvider.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"),
|
||||
parentIdCache: cache,
|
||||
maxActivities: o.Config.MaxActivities,
|
||||
natskv: kv,
|
||||
}
|
||||
s.debouncer = NewDebouncer(o.Config.WriteBufferDuration, s.storeActivity)
|
||||
|
||||
// run migrations
|
||||
err = s.runMigrations(context.Background(), kv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.mux.Get("/graph/v1beta1/extensions/org.libregraph/activities", s.HandleGetItemActivities)
|
||||
|
||||
for _, e := range o.RegisteredEvents {
|
||||
typ := reflect.TypeOf(e)
|
||||
s.registeredEvents[typ.String()] = e
|
||||
}
|
||||
|
||||
go s.Run()
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Run runs the service
|
||||
func (a *ActivitylogService) Run() {
|
||||
for e := range a.events {
|
||||
var err error
|
||||
switch ev := e.Event.(type) {
|
||||
case events.UploadReady:
|
||||
err = a.AddActivity(ev.FileRef, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.FileTouched:
|
||||
err = a.AddActivity(ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
// Disabled https://github.com/owncloud/ocis/issues/10293
|
||||
//case events.FileDownloaded:
|
||||
// we are only interested in public link downloads - so no need to store others.
|
||||
//if ev.ImpersonatingUser.GetDisplayName() == "Public" {
|
||||
// err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
//}
|
||||
case events.ContainerCreated:
|
||||
err = a.AddActivity(ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemTrashed:
|
||||
err = a.AddActivityTrashed(ev.ID, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ItemPurged:
|
||||
err = a.RemoveResource(ev.ID)
|
||||
case events.ItemMoved:
|
||||
// remove the cached parent id for this resource
|
||||
a.removeCachedParentID(ev.Ref)
|
||||
|
||||
err = a.AddActivity(ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.ShareCreated:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.ShareUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.ShareRemoved:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, ev.Timestamp)
|
||||
case events.LinkCreated:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
|
||||
case events.LinkUpdated:
|
||||
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
|
||||
}
|
||||
case events.LinkRemoved:
|
||||
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.Timestamp))
|
||||
case events.SpaceShared:
|
||||
err = a.AddSpaceActivity(ev.ID, e.ID, ev.Timestamp)
|
||||
case events.SpaceUnshared:
|
||||
err = a.AddSpaceActivity(ev.ID, e.ID, ev.Timestamp)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Interface("event", e).Msg("could not process event")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddActivity adds the activity to the given resource and all its parents
|
||||
func (a *ActivitylogService) AddActivity(initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
}
|
||||
var span trace.Span
|
||||
ctx, span = a.tracer.Start(ctx, "AddActivity")
|
||||
defer span.End()
|
||||
|
||||
return a.addActivity(ctx, initRef, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
|
||||
func (a *ActivitylogService) AddActivityTrashed(resourceID *provider.ResourceId, reference *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get gateway client: %w", err)
|
||||
}
|
||||
|
||||
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cant get service user context: %w", err)
|
||||
}
|
||||
|
||||
// store activity on trashed item
|
||||
if err := a.storeActivity(storagespace.FormatResourceID(resourceID), []RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("could not store activity: %w", err)
|
||||
}
|
||||
|
||||
// get previous parent
|
||||
ref := &provider.Reference{
|
||||
ResourceId: reference.GetResourceId(),
|
||||
Path: filepath.Dir(reference.GetPath()),
|
||||
}
|
||||
|
||||
var span trace.Span
|
||||
ctx, span = a.tracer.Start(ctx, "AddActivityTrashed")
|
||||
defer span.End()
|
||||
|
||||
return a.addActivity(ctx, ref, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
|
||||
return utils.GetResource(ctx, ref, gwc)
|
||||
})
|
||||
}
|
||||
|
||||
// AddSpaceActivity adds the activity to the given spaceroot
|
||||
func (a *ActivitylogService) AddSpaceActivity(spaceID *provider.StorageSpaceId, eventID string, timestamp time.Time) error {
|
||||
// spaceID is in format <providerid>$<spaceid>
|
||||
// activitylog service uses format <providerid>$<spaceid>!<resourceid>
|
||||
// lets do some converting, shall we?
|
||||
rid, err := storagespace.ParseID(spaceID.GetOpaqueId())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse space id: %w", err)
|
||||
}
|
||||
rid.OpaqueId = rid.GetSpaceId()
|
||||
return a.storeActivity(storagespace.FormatResourceID(&rid), []RawActivity{
|
||||
{
|
||||
EventID: eventID,
|
||||
Depth: 0,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// Activities returns the activities for the given resource
|
||||
func (a *ActivitylogService) Activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
a.lock.RLock()
|
||||
defer a.lock.RUnlock()
|
||||
|
||||
return a.activities(rid)
|
||||
}
|
||||
|
||||
// RemoveActivities removes the activities from the given resource
|
||||
func (a *ActivitylogService) RemoveActivities(rid *provider.ResourceId, toDelete map[string]struct{}) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
curActivities, err := a.activities(rid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var acts []RawActivity
|
||||
for _, a := range curActivities {
|
||||
if _, ok := toDelete[a.EventID]; !ok {
|
||||
acts = append(acts, a)
|
||||
}
|
||||
}
|
||||
|
||||
b, err := json.Marshal(acts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = a.natskv.Put(storagespace.FormatResourceID(rid), b)
|
||||
return err
|
||||
}
|
||||
|
||||
// RemoveResource removes the resource from the store
|
||||
func (a *ActivitylogService) RemoveResource(rid *provider.ResourceId) error {
|
||||
if rid == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
return a.natskv.Delete(storagespace.FormatResourceID(rid))
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) activities(rid *provider.ResourceId) ([]RawActivity, error) {
|
||||
resourceID := storagespace.FormatResourceID(rid)
|
||||
|
||||
glob := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
watcher, err := a.natskv.Watch(glob, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var activities []RawActivity
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
|
||||
}
|
||||
activities = append(activities, batchActivities...)
|
||||
}
|
||||
|
||||
return activities, nil
|
||||
}
|
||||
|
||||
// note: getResource is abstracted to allow unit testing, in general this will just be utils.GetResource
|
||||
func (a *ActivitylogService) addActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time, getResource func(context.Context, *provider.Reference) (*provider.ResourceInfo, error)) error {
|
||||
var (
|
||||
err error
|
||||
depth int
|
||||
ref = initRef
|
||||
)
|
||||
ctx, span := a.tracer.Start(ctx, "addActivity")
|
||||
defer span.End()
|
||||
for {
|
||||
var info *provider.ResourceInfo
|
||||
id := ref.GetResourceId()
|
||||
if ref.Path != "" {
|
||||
// Path based reference, we need to resolve the resource id
|
||||
ctx, span = a.tracer.Start(ctx, "addActivity.getResource")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get resource info: %w", err)
|
||||
}
|
||||
id = info.GetId()
|
||||
}
|
||||
if id == nil {
|
||||
return fmt.Errorf("resource id is required")
|
||||
}
|
||||
|
||||
key := storagespace.FormatResourceID(id)
|
||||
a.debouncer.Debounce(key, RawActivity{
|
||||
EventID: eventID,
|
||||
Depth: depth,
|
||||
Timestamp: timestamp,
|
||||
})
|
||||
|
||||
if id.OpaqueId == id.SpaceId {
|
||||
// we are at the root of the space, no need to go further
|
||||
break
|
||||
}
|
||||
|
||||
// check if parent id is cached
|
||||
// parent id is cached in the format <storageid>$<spaceid>!<resourceid>
|
||||
// if it is not cached, get the resource info and cache it
|
||||
if parentId == nil {
|
||||
if v, err := a.parentIdCache.Get(key); err != nil {
|
||||
if info == nil {
|
||||
ctx, span := a.tracer.Start(ctx, "addActivity.getResource parent")
|
||||
info, err = getResource(ctx, ref)
|
||||
span.End()
|
||||
if err != nil || info.GetParentId() == nil || info.GetParentId().GetOpaqueId() == "" {
|
||||
return fmt.Errorf("could not get parent id: %w", err)
|
||||
}
|
||||
}
|
||||
parentId = info.GetParentId()
|
||||
a.parentIdCache.Set(key, parentId)
|
||||
} else {
|
||||
parentId = v.(*provider.ResourceId)
|
||||
}
|
||||
} else {
|
||||
a.log.Debug().Msg("parent id is cached")
|
||||
}
|
||||
|
||||
depth++
|
||||
ref = &provider.Reference{ResourceId: parentId}
|
||||
parentId = nil // reset parent id so it's not reused in the next iteration
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) storeActivity(resourceID string, activities []RawActivity) error {
|
||||
a.lock.Lock()
|
||||
defer a.lock.Unlock()
|
||||
|
||||
ctx, span := a.tracer.Start(context.Background(), "storeActivity")
|
||||
defer span.End()
|
||||
|
||||
_, subspan := a.tracer.Start(ctx, "storeActivity.Marshal")
|
||||
b, err := msgpack.Marshal(activities)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = a.tracer.Start(ctx, "storeActivity.natskv.Put")
|
||||
key := natsKey(resourceID, len(activities))
|
||||
_, err = a.natskv.Put(key, b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
ctx, subspan = a.tracer.Start(ctx, "storeActivity.enforceMaxActivities")
|
||||
a.enforceMaxActivities(ctx, resourceID)
|
||||
subspan.End()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) enforceMaxActivities(ctx context.Context, resourceID string) {
|
||||
if a.maxActivities <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
|
||||
|
||||
_, subspan := a.tracer.Start(ctx, "enforceMaxActivities.watch")
|
||||
watcher, err := a.natskv.Watch(key, nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("resourceID", resourceID).Msg("could not watch")
|
||||
return
|
||||
}
|
||||
defer watcher.Stop()
|
||||
|
||||
var keys []string
|
||||
for update := range watcher.Updates() {
|
||||
if update == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var batchActivities []RawActivity
|
||||
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
|
||||
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
|
||||
}
|
||||
keys = append(keys, update.Key())
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = a.tracer.Start(ctx, "enforceMaxActivities.compile")
|
||||
// Parse keys into batches
|
||||
batches := make([]batchInfo, 0)
|
||||
var activitiesCount int
|
||||
for _, k := range keys {
|
||||
parts := strings.SplitN(k, ".", 3)
|
||||
if len(parts) < 3 {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, not enough parts")
|
||||
continue
|
||||
}
|
||||
|
||||
c, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse count")
|
||||
continue
|
||||
}
|
||||
|
||||
// parse timestamp
|
||||
nano, err := strconv.ParseInt(parts[2], 10, 64)
|
||||
if err != nil {
|
||||
a.log.Warn().Str("key", k).Msg("skipping key, can not parse timestamp")
|
||||
continue
|
||||
}
|
||||
|
||||
batches = append(batches, batchInfo{
|
||||
key: k,
|
||||
count: c,
|
||||
timestamp: time.Unix(0, nano),
|
||||
})
|
||||
activitiesCount += c
|
||||
}
|
||||
|
||||
// sort batches by timestamp
|
||||
sort.Slice(batches, func(i, j int) bool {
|
||||
return batches[i].timestamp.Before(batches[j].timestamp)
|
||||
})
|
||||
subspan.End()
|
||||
|
||||
_, subspan = a.tracer.Start(ctx, "enforceMaxActivities.delete")
|
||||
// remove oldest keys until we are at max activities
|
||||
for _, b := range batches {
|
||||
if activitiesCount-b.count < a.maxActivities {
|
||||
break
|
||||
}
|
||||
|
||||
activitiesCount -= b.count
|
||||
err = a.natskv.Delete(b.key)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Str("key", b.key).Msg("could not delete key")
|
||||
break
|
||||
}
|
||||
}
|
||||
subspan.End()
|
||||
}
|
||||
|
||||
func toRef(r *provider.ResourceId) *provider.Reference {
|
||||
return &provider.Reference{
|
||||
ResourceId: r,
|
||||
}
|
||||
}
|
||||
|
||||
func toSpace(r *provider.Reference) *provider.StorageSpaceId {
|
||||
return &provider.StorageSpaceId{
|
||||
OpaqueId: storagespace.FormatStorageID(r.GetResourceId().GetStorageId(), r.GetResourceId().GetSpaceId()),
|
||||
}
|
||||
}
|
||||
|
||||
func (a *ActivitylogService) removeCachedParentID(ref *provider.Reference) {
|
||||
purgeId := ref.GetResourceId()
|
||||
if ref.GetPath() != "" {
|
||||
gwc, err := a.gws.Next()
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get gateway client")
|
||||
return
|
||||
}
|
||||
|
||||
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get service user context")
|
||||
return
|
||||
}
|
||||
|
||||
info, err := utils.GetResource(ctx, ref, gwc)
|
||||
if err != nil {
|
||||
a.log.Error().Err(err).Msg("could not get resource info")
|
||||
return
|
||||
}
|
||||
purgeId = info.GetId()
|
||||
}
|
||||
// The parent id cache is populated lazily and its entries expire, so a
|
||||
// missing key is the expected case rather than an error.
|
||||
if err := a.parentIdCache.Remove(storagespace.FormatResourceID(purgeId)); err != nil {
|
||||
a.log.Debug().Interface("event", ref).Err(err).Msg("could not delete parent id cache")
|
||||
}
|
||||
}
|
||||
|
||||
func natsKey(resourceID string, activitiesCount int) string {
|
||||
return fmt.Sprintf("%s.%d.%d",
|
||||
base32.StdEncoding.EncodeToString([]byte(resourceID)),
|
||||
activitiesCount,
|
||||
time.Now().UnixNano())
|
||||
}
|
||||
+3
-3
@@ -1,4 +1,4 @@
|
||||
package events_test
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestEvents(t *testing.T) {
|
||||
func TestService(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Events Suite")
|
||||
RunSpecs(t, "Service Suite")
|
||||
}
|
||||
+81
-34
@@ -1,6 +1,7 @@
|
||||
package activitylog_test
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
@@ -8,13 +9,17 @@ import (
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
nserver "github.com/nats-io/nats-server/v2/server"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/command"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
|
||||
eventsmocks "github.com/opencloud-eu/reva/v2/pkg/events/mocks"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/test-go/testify/mock"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -63,25 +68,31 @@ var _ = SynchronizedAfterSuite(func() {
|
||||
|
||||
var _ = Describe("ActivitylogService", func() {
|
||||
var (
|
||||
alog *activitylog.ActivityLog
|
||||
alog *ActivitylogService
|
||||
getResource func(_ context.Context, ref *provider.Reference) (*provider.ResourceInfo, error)
|
||||
writebufferduration = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
JustBeforeEach(func() {
|
||||
var err error
|
||||
db := "activitylog-test-" + uuid.New().String()
|
||||
|
||||
kv, err := command.ConnectNatsKV(config.Store{
|
||||
Nodes: []string{server.Addr().String()},
|
||||
Database: db,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
alog, err = activitylog.New(
|
||||
kv,
|
||||
activitylog.MaxActivities(4),
|
||||
activitylog.WriteBufferDuration(writebufferduration),
|
||||
stream := &eventsmocks.Stream{}
|
||||
stream.EXPECT().Consume(mock.Anything, mock.Anything).Return(nil, nil)
|
||||
alog, err = New(
|
||||
Config(&config.Config{
|
||||
Service: config.Service{
|
||||
Name: "activitylog-test",
|
||||
},
|
||||
Store: config.Store{
|
||||
Store: "nats-js-kv",
|
||||
Nodes: []string{server.Addr().String()},
|
||||
Database: "activitylog-test-" + uuid.New().String(),
|
||||
},
|
||||
MaxActivities: 4,
|
||||
WriteBufferDuration: writebufferduration,
|
||||
}),
|
||||
Stream(stream),
|
||||
TraceProvider(noop.NewTracerProvider()),
|
||||
Mux(chi.NewMux()),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
@@ -96,7 +107,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
Name string
|
||||
Tree map[string]*provider.ResourceInfo
|
||||
Activities map[string]string
|
||||
Expected map[string][]activitylog.RawActivity
|
||||
Expected map[string][]RawActivity
|
||||
}
|
||||
|
||||
testCases := []testCase{
|
||||
@@ -110,7 +121,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
Activities: map[string]string{
|
||||
"activity": "base",
|
||||
},
|
||||
Expected: map[string][]activitylog.RawActivity{
|
||||
Expected: map[string][]RawActivity{
|
||||
"base": activitites("activity", 0),
|
||||
"parent": activitites("activity", 1),
|
||||
"spaceid": activitites("activity", 2),
|
||||
@@ -127,7 +138,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
"activity1": "base",
|
||||
"activity2": "base",
|
||||
},
|
||||
Expected: map[string][]activitylog.RawActivity{
|
||||
Expected: map[string][]RawActivity{
|
||||
"base": activitites("activity1", 0, "activity2", 0),
|
||||
"parent": activitites("activity1", 1, "activity2", 1),
|
||||
"spaceid": activitites("activity1", 2, "activity2", 2),
|
||||
@@ -144,7 +155,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
}
|
||||
|
||||
for k, v := range tc.Activities {
|
||||
err := alog.AddActivity(context.Background(), reference(v), nil, k, time.Time{}, getResource)
|
||||
err := alog.addActivity(context.Background(), reference(v), nil, k, time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
})
|
||||
@@ -183,9 +194,9 @@ var _ = Describe("ActivitylogService", func() {
|
||||
|
||||
It("debounces activities", func() {
|
||||
|
||||
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -196,7 +207,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
})
|
||||
|
||||
It("adheres to the MaxActivities setting", func() {
|
||||
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Eventually(func(g Gomega) {
|
||||
activities, err := alog.Activities(resourceID("base"))
|
||||
@@ -204,7 +215,7 @@ var _ = Describe("ActivitylogService", func() {
|
||||
g.Expect(len(activities)).To(Equal(1))
|
||||
}).Should(Succeed())
|
||||
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Eventually(func(g Gomega) {
|
||||
activities, err := alog.Activities(resourceID("base"))
|
||||
@@ -212,11 +223,11 @@ var _ = Describe("ActivitylogService", func() {
|
||||
g.Expect(len(activities)).To(Equal(2))
|
||||
}).Should(Succeed())
|
||||
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity5", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity5", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -233,9 +244,9 @@ var _ = Describe("ActivitylogService", func() {
|
||||
return tree[ref.GetResourceId().GetOpaqueId()], nil
|
||||
}
|
||||
|
||||
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -244,9 +255,9 @@ var _ = Describe("ActivitylogService", func() {
|
||||
g.Expect(activities).To(ConsistOf(activitites("activity1", 0, "activity2", 0)))
|
||||
}).Should(Succeed())
|
||||
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
err = alog.addActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
Eventually(func(g Gomega) {
|
||||
@@ -257,11 +268,47 @@ var _ = Describe("ActivitylogService", func() {
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Describe("removeCachedParentID", func() {
|
||||
var logBuffer *bytes.Buffer
|
||||
|
||||
newLoggerAtLevel := func(level string) log.Logger {
|
||||
logBuffer = &bytes.Buffer{}
|
||||
return log.Logger{Logger: log.NewLogger(log.Level(level)).Output(logBuffer)}
|
||||
}
|
||||
|
||||
It("does not log an error when the entry was never cached", func() {
|
||||
alog.log = newLoggerAtLevel("error")
|
||||
|
||||
alog.removeCachedParentID(reference("never-cached"))
|
||||
|
||||
Expect(logBuffer.String()).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("logs a missing entry at debug level", func() {
|
||||
alog.log = newLoggerAtLevel("debug")
|
||||
|
||||
alog.removeCachedParentID(reference("never-cached"))
|
||||
|
||||
Expect(logBuffer.String()).To(ContainSubstring("could not delete parent id cache"))
|
||||
Expect(logBuffer.String()).To(ContainSubstring(`"level":"debug"`))
|
||||
})
|
||||
|
||||
It("does not log at all when the entry was cached", func() {
|
||||
alog.log = newLoggerAtLevel("debug")
|
||||
ref := reference("cached")
|
||||
Expect(alog.parentIdCache.Set(storagespace.FormatResourceID(ref.GetResourceId()), resourceID("parent"))).To(Succeed())
|
||||
|
||||
alog.removeCachedParentID(ref)
|
||||
|
||||
Expect(logBuffer.String()).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
func activitites(acts ...any) []activitylog.RawActivity {
|
||||
var activities []activitylog.RawActivity
|
||||
act := activitylog.RawActivity{}
|
||||
func activitites(acts ...any) []RawActivity {
|
||||
var activities []RawActivity
|
||||
act := RawActivity{}
|
||||
for _, a := range acts {
|
||||
switch v := a.(type) {
|
||||
case string:
|
||||
@@ -63,6 +63,10 @@ Store specific notes:
|
||||
- When using `nats-js-kv` it is recommended to set `OC_CACHE_STORE_NODES` to the same value as `OC_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
|
||||
- When using the `nats-js-kv` store, it is possible to set `OC_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
|
||||
|
||||
### Auto-Accept Shares
|
||||
|
||||
When setting the `SHARING_AUTO_ACCEPT_SHARES` to `true` (sharing service), all incoming shares will be accepted automatically. Users can overwrite this setting individually in their profile. The deprecated `FRONTEND_AUTO_ACCEPT_SHARES` is still supported for backwards compatibility.
|
||||
|
||||
## Passwords
|
||||
|
||||
### The Password Policy
|
||||
|
||||
@@ -61,7 +61,7 @@ type Spaces struct {
|
||||
}
|
||||
|
||||
type LDAPMetrics struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_LDAP_METRICS_DISABLE" desc:"Disables the metrics for outbound LDAP operations." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_LDAP_METRICS_DISABLE" desc:"Disables the metrics for outbound LDAP operations." introductionVersion:"%NEXT%"`
|
||||
}
|
||||
|
||||
type LDAP struct {
|
||||
@@ -121,7 +121,7 @@ type LDAPEducationConfig struct {
|
||||
}
|
||||
|
||||
type IdentityMetrics struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_IDENTITY_BACKEND_METRICS_DISABLE" desc:"Disables the metrics for inbound identity backend operations." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_IDENTITY_BACKEND_METRICS_DISABLE" desc:"Disables the metrics for inbound identity backend operations." introductionVersion:"%NEXT%"`
|
||||
}
|
||||
|
||||
type Identity struct {
|
||||
@@ -141,7 +141,7 @@ type API struct {
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
DisabledConsumer bool `yaml:"disabled_consumer" env:"GRAPH_EVENTS_DISABLE_CONSUMER" desc:"Disables consuming events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%%NEXT%%"`
|
||||
DisabledConsumer bool `yaml:"disabled_consumer" env:"GRAPH_EVENTS_DISABLE_CONSUMER" desc:"Disables consuming events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%NEXT%"`
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;GRAPH_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Set to a empty string to disable emitting events." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;GRAPH_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;GRAPH_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -3,12 +3,12 @@ package config
|
||||
import "github.com/opencloud-eu/opencloud/pkg/shared"
|
||||
|
||||
type HTTPMetrics struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_METRICS_DISABLE" desc:"Disables the metrics for the HTTP service." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_METRICS_DISABLE" desc:"Disables the metrics for the HTTP service." introductionVersion:"%NEXT%"`
|
||||
}
|
||||
|
||||
// HTTP defines the available http configuration.
|
||||
type HTTP struct {
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_DISABLE" desc:"Disables the HTTP service. Set this to true if the service should only consume events." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_DISABLE" desc:"Disables the HTTP service. Set this to true if the service should only consume events." introductionVersion:"%NEXT%"`
|
||||
Addr string `yaml:"addr" env:"GRAPH_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
Root string `yaml:"root" env:"GRAPH_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -40,7 +40,11 @@ func ParseConfig(cfg *config.Config) error {
|
||||
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.HTTP.Disabled && cfg.Events.DisabledConsumer {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
// might be debatable, but this situation should be treated as an error,
|
||||
// as the process wouldn't be able to serve either API and would thus be
|
||||
// completely useless -- in that case, just don't start this service
|
||||
// in the first place (especially since it's optional)
|
||||
return shared.AllComponentsDisabledError("graph")
|
||||
}
|
||||
|
||||
if cfg.TokenManager.JWTSecret == "" {
|
||||
|
||||
@@ -76,6 +76,10 @@ const (
|
||||
PreconditionFailed
|
||||
// ItemIsLocked The item is locked by another process. Try again later.
|
||||
ItemIsLocked
|
||||
// PublicLinkPasswordRequired the public link is password protected and no password was provided.
|
||||
PublicLinkPasswordRequired
|
||||
// PublicLinkPasswordInvalid a password was provided for the public link but it was rejected.
|
||||
PublicLinkPasswordInvalid
|
||||
)
|
||||
|
||||
var errorCodes = [...]string{
|
||||
@@ -99,6 +103,8 @@ var errorCodes = [...]string{
|
||||
"unauthenticated",
|
||||
"preconditionFailed",
|
||||
"itemIsLocked",
|
||||
"publicLinkPasswordRequired",
|
||||
"publicLinkPasswordInvalid",
|
||||
}
|
||||
|
||||
// New constructs a new errorcode.Error
|
||||
@@ -151,6 +157,8 @@ func (e Error) Render(w http.ResponseWriter, r *http.Request) {
|
||||
status = http.StatusMethodNotAllowed
|
||||
case ItemIsLocked:
|
||||
status = http.StatusLocked
|
||||
case PublicLinkPasswordRequired, PublicLinkPasswordInvalid:
|
||||
status = http.StatusUnauthorized
|
||||
case PreconditionFailed:
|
||||
status = http.StatusPreconditionFailed
|
||||
default:
|
||||
|
||||
@@ -44,6 +44,23 @@ func Auth(opts ...account.Option) func(http.Handler) http.Handler {
|
||||
ctx := r.Context()
|
||||
t := r.Header.Get(revactx.TokenHeader)
|
||||
if t == "" {
|
||||
// a public link request that failed the share auth carries a
|
||||
// hint (set by the proxy) so we can tell the two cases apart;
|
||||
// only trust it when a share token is actually on the request
|
||||
if hint := r.Header.Get(opkgm.PublicLinkAuthHeader); hint != "" && opkgm.HasPublicLinkToken(r) {
|
||||
switch hint {
|
||||
// distinguish via the body only, never WWW-Authenticate: a
|
||||
// Basic challenge would pop the browser's native auth dialog
|
||||
// instead of the app's password field (the proxy strips it
|
||||
// on public paths for the same reason)
|
||||
case opkgm.PublicLinkPasswordRequired:
|
||||
errorcode.PublicLinkPasswordRequired.Render(w, r, http.StatusUnauthorized, "This public link is password protected.")
|
||||
return
|
||||
case opkgm.PublicLinkInvalidPassword:
|
||||
errorcode.PublicLinkPasswordInvalid.Render(w, r, http.StatusUnauthorized, "The password is incorrect.")
|
||||
return
|
||||
}
|
||||
}
|
||||
errorcode.InvalidAuthenticationToken.Render(w, r, http.StatusUnauthorized, "Access token is empty.")
|
||||
/* msgraph error for GET https://graph.microsoft.com/v1.0/me
|
||||
{
|
||||
@@ -67,7 +84,8 @@ func Auth(opts ...account.Option) func(http.Handler) http.Handler {
|
||||
errorcode.InvalidAuthenticationToken.Render(w, r, http.StatusUnauthorized, "invalid token")
|
||||
return
|
||||
}
|
||||
if ok, err := scope.VerifyScope(ctx, tokenScope, r); err != nil || !ok {
|
||||
// scope handlers judge CS3 requests and url paths, not *http.Request
|
||||
if ok, err := scope.VerifyScope(ctx, tokenScope, r.URL.Path); err != nil || !ok {
|
||||
opt.Logger.Error().Str(log.RequestIDString, r.Header.Get("X-Request-ID")).Err(err).Msg("verifying scope failed")
|
||||
errorcode.InvalidAuthenticationToken.Render(w, r, http.StatusUnauthorized, "verifying scope failed")
|
||||
return
|
||||
|
||||
@@ -203,7 +203,9 @@ func rewriteColonPath(
|
||||
logger.Debug().Err(err).Str("driveID", driveID).Msg("invalid drive id in colon path")
|
||||
return "", errInvalidRequest
|
||||
}
|
||||
if drive.GetStorageId() != anchor.GetStorageId() || drive.GetSpaceId() != anchor.GetSpaceId() {
|
||||
// items below the public share drive keep their real ids; the scope guards access
|
||||
isPublicDrive := drive.GetStorageId() == utils.PublicStorageProviderID && drive.GetSpaceId() == utils.PublicStorageSpaceID
|
||||
if !isPublicDrive && (drive.GetStorageId() != anchor.GetStorageId() || drive.GetSpaceId() != anchor.GetSpaceId()) {
|
||||
logger.Debug().
|
||||
Str("driveID", driveID).
|
||||
Str("itemID", anchorIDStr).
|
||||
|
||||
@@ -61,6 +61,76 @@ func odataListContains(r *http.Request, parameter, value string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// driveItemInDrive reports whether an item id may be addressed below a drive.
|
||||
// Below the public share drive items keep their real storage ids, so the
|
||||
// prefix can never match; the token scope checks containment instead.
|
||||
func driveItemInDrive(driveID, driveItemID *storageprovider.ResourceId) bool {
|
||||
if driveID.GetStorageId() == utils.PublicStorageProviderID && driveID.GetSpaceId() == utils.PublicStorageSpaceID {
|
||||
return true
|
||||
}
|
||||
return driveID.GetStorageId() == driveItemID.GetStorageId() && driveID.GetSpaceId() == driveItemID.GetSpaceId()
|
||||
}
|
||||
|
||||
// publicDriveRequest reports whether the request addresses the public share drive.
|
||||
func publicDriveRequest(r *http.Request) bool {
|
||||
driveID, err := parseIDParam(r, "driveID")
|
||||
return err == nil &&
|
||||
driveID.GetStorageId() == utils.PublicStorageProviderID &&
|
||||
driveID.GetSpaceId() == utils.PublicStorageSpaceID
|
||||
}
|
||||
|
||||
// sanitizePublicDriveInfos applies the publicstorageprovider's reduction to
|
||||
// infos that bypassed it (navigation by id).
|
||||
func (g Graph) sanitizePublicDriveInfos(ctx context.Context, r *http.Request, infos ...*storageprovider.ResourceInfo) error {
|
||||
shareRoot, grant, err := g.publicLinkOfRequest(ctx, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, info := range infos {
|
||||
if info == nil {
|
||||
continue
|
||||
}
|
||||
publicshare.FilterResourceInfo(info, shareRoot, grant)
|
||||
// the favorite flag is the owner's, not the visitor's
|
||||
delete(info.GetArbitraryMetadata().GetMetadata(), _favoriteMetadataKey)
|
||||
// the share root's parent lies outside the share
|
||||
if utils.ResourceIDEqual(info.GetId(), shareRoot.GetId()) {
|
||||
info.ParentId = nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// publicLinkOfRequest resolves the link the request runs in; the token is the
|
||||
// public drive's opaque id.
|
||||
func (g Graph) publicLinkOfRequest(ctx context.Context, r *http.Request) (*storageprovider.ResourceInfo, *storageprovider.ResourcePermissions, error) {
|
||||
driveID, err := parseIDParam(r, "driveID")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
gatewayClient, err := g.gatewaySelector.Next()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
shareResp, err := gatewayClient.GetPublicShare(ctx, &link.GetPublicShareRequest{
|
||||
Ref: &link.PublicShareReference{
|
||||
Spec: &link.PublicShareReference_Token{Token: driveID.GetOpaqueId()},
|
||||
},
|
||||
})
|
||||
if err := errorcode.FromCS3Status(shareResp.GetStatus(), err); err != nil {
|
||||
g.logger.Error().Err(err).Msg("could not resolve the public link of the request")
|
||||
return nil, nil, err
|
||||
}
|
||||
statResp, err := gatewayClient.Stat(ctx, &storageprovider.StatRequest{
|
||||
Ref: &storageprovider.Reference{ResourceId: shareResp.GetShare().GetResourceId()},
|
||||
})
|
||||
if err := errorcode.FromCS3Status(statResp.GetStatus(), err); err != nil {
|
||||
g.logger.Error().Err(err).Msg("could not stat the public link root")
|
||||
return nil, nil, err
|
||||
}
|
||||
return statResp.GetInfo(), shareResp.GetShare().GetPermissions().GetPermissions(), nil
|
||||
}
|
||||
|
||||
// driveItemPropertySelected reports whether the given opt-in property was requested via $select
|
||||
func driveItemPropertySelected(r *http.Request, property string) bool {
|
||||
return odataListContains(r, "$select", property)
|
||||
@@ -98,7 +168,7 @@ func (g Graph) CreateUploadSession(w http.ResponseWriter, r *http.Request) {
|
||||
errorcode.RenderError(w, r, err)
|
||||
return
|
||||
}
|
||||
if driveID.GetStorageId() != driveItemID.GetStorageId() || driveID.GetSpaceId() != driveItemID.GetSpaceId() {
|
||||
if !driveItemInDrive(&driveID, &driveItemID) {
|
||||
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, "Item does not exist")
|
||||
return
|
||||
}
|
||||
@@ -287,7 +357,7 @@ func (g Graph) GetDriveItem(w http.ResponseWriter, r *http.Request) {
|
||||
errorcode.RenderError(w, r, err)
|
||||
return
|
||||
}
|
||||
if driveID.GetStorageId() != driveItemID.GetStorageId() || driveID.GetSpaceId() != driveItemID.GetSpaceId() {
|
||||
if !driveItemInDrive(&driveID, &driveItemID) {
|
||||
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, "Item does not exist")
|
||||
return
|
||||
}
|
||||
@@ -312,7 +382,12 @@ func (g Graph) GetDriveItem(w http.ResponseWriter, r *http.Request) {
|
||||
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
case res.GetStatus().GetCode() == cs3rpc.Code_CODE_OK:
|
||||
// ok
|
||||
if publicDriveRequest(r) {
|
||||
if err := g.sanitizePublicDriveInfos(ctx, r, res.GetInfo()); err != nil {
|
||||
errorcode.RenderError(w, r, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
case res.GetStatus().GetCode() == cs3rpc.Code_CODE_NOT_FOUND:
|
||||
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, res.GetStatus().GetMessage())
|
||||
return
|
||||
@@ -345,7 +420,7 @@ func (g Graph) GetDriveItem(w http.ResponseWriter, r *http.Request) {
|
||||
driveItem.Children = children
|
||||
}
|
||||
|
||||
if driveItemPropertySelected(r, _selectShareTypes) {
|
||||
if driveItemPropertySelected(r, _selectShareTypes) && !publicDriveRequest(r) {
|
||||
infos := []*storageprovider.ResourceInfo{res.GetInfo()}
|
||||
driveItem.LibreGraphShareTypes = shareTypesOf(res.GetInfo(), g.listLinkShares(ctx, infos))
|
||||
}
|
||||
@@ -372,7 +447,7 @@ func (g Graph) GetDriveItemChildren(w http.ResponseWriter, r *http.Request) {
|
||||
errorcode.RenderError(w, r, err)
|
||||
return
|
||||
}
|
||||
if driveID.GetStorageId() != driveItemID.GetStorageId() || driveID.GetSpaceId() != driveItemID.GetSpaceId() {
|
||||
if !driveItemInDrive(&driveID, &driveItemID) {
|
||||
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, "Item does not exist")
|
||||
return
|
||||
}
|
||||
@@ -405,7 +480,7 @@ func (g Graph) listDriveItemChildren(w http.ResponseWriter, r *http.Request, dri
|
||||
childrenRequest := &storageprovider.ListContainerRequest{
|
||||
Ref: &storageprovider.Reference{ResourceId: driveItemID},
|
||||
}
|
||||
if driveItemPropertySelected(r, _selectShareTypes) {
|
||||
if driveItemPropertySelected(r, _selectShareTypes) && !publicDriveRequest(r) {
|
||||
childrenRequest.FieldMask = shareTypesFieldMask
|
||||
}
|
||||
|
||||
@@ -430,13 +505,21 @@ func (g Graph) listDriveItemChildren(w http.ResponseWriter, r *http.Request, dri
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if publicDriveRequest(r) {
|
||||
if err := g.sanitizePublicDriveInfos(r.Context(), r, res.GetInfos()...); err != nil {
|
||||
errorcode.RenderError(w, r, err)
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
files, err := formatDriveItems(g.logger, g.publicBaseURL, res.GetInfos())
|
||||
if err != nil {
|
||||
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if driveItemPropertySelected(r, _selectShareTypes) {
|
||||
// collaborative grants are not for public link visitors
|
||||
if driveItemPropertySelected(r, _selectShareTypes) && !publicDriveRequest(r) {
|
||||
g.addShareTypes(r.Context(), files, res.GetInfos())
|
||||
}
|
||||
|
||||
|
||||
@@ -881,6 +881,19 @@ func (g Graph) cs3StorageSpaceToDrive(ctx context.Context, baseURL *url.URL, spa
|
||||
// DisplayName: , TODO read and cache from users provider
|
||||
},
|
||||
}
|
||||
} else if space.GetRoot().GetStorageId() == utils.PublicStorageProviderID {
|
||||
// a public share mountpoint carries no space owner; the request runs as
|
||||
// the share creator (publicshares auth), so the context user is who
|
||||
// shared it, the same source webdav fills oc:owner-display-name from.
|
||||
if u, ok := revactx.ContextGetUser(ctx); ok && u.GetId().GetOpaqueId() != "" {
|
||||
id := u.GetId().GetOpaqueId()
|
||||
drive.Owner = &libregraph.IdentitySet{
|
||||
User: &libregraph.Identity{
|
||||
Id: &id,
|
||||
DisplayName: u.GetDisplayName(),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
if space.Mtime != nil {
|
||||
lastModified := cs3TimestampToTime(space.Mtime)
|
||||
|
||||
@@ -29,7 +29,7 @@ type Service struct {
|
||||
|
||||
// GRPC defines the available grpc configuration.
|
||||
type GRPC struct {
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_GRPC_DISABLED" desc:"Disables listening for GRPC API calls. Set this to true if the service should only handle requests through events." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_GRPC_DISABLED" desc:"Disables listening for GRPC API calls. Set this to true if the service should only handle requests through events." introductionVersion:"%NEXT%"`
|
||||
Addr string `yaml:"addr" env:"POLICIES_GRPC_ADDR" desc:"The bind address of the GRPC service." introductionVersion:"1.0.0"`
|
||||
Namespace string `yaml:"-"`
|
||||
TLS *shared.GRPCServiceTLS `yaml:"tls"`
|
||||
@@ -50,7 +50,7 @@ type Postprocessing struct {
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
type Events struct {
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle GRPC requests." introductionVersion:"%%NEXT%%"`
|
||||
Disabled bool `yaml:"disabled" env:"POLICIES_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle GRPC requests." introductionVersion:"%NEXT%"`
|
||||
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;POLICIES_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
|
||||
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;POLICIES_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;POLICIES_EVENTS_TLS_INSECURE" desc:"Whether the server should skip the client certificate verification during the TLS handshake." introductionVersion:"1.0.0"`
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"errors"
|
||||
|
||||
occfg "github.com/opencloud-eu/opencloud/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/pkg/shared"
|
||||
"github.com/opencloud-eu/opencloud/services/policies/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/policies/pkg/config/defaults"
|
||||
|
||||
@@ -34,8 +33,12 @@ func ParseConfig(cfg *config.Config) error {
|
||||
}
|
||||
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.Events.Disabled && cfg.GRPC.Disabled {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
if cfg.GRPC.Disabled && cfg.Events.Disabled {
|
||||
// might be debatable, but this situation should be treated as an error,
|
||||
// as the process wouldn't be able to serve either API and would thus be
|
||||
// completely useless -- in that case, just don't start this service
|
||||
// in the first place (especially since it's optional)
|
||||
return errors.New("both gRPC and events APIs are disabled by configuration; at least one must be enabled")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -18,7 +18,7 @@ type BasicAuthenticator struct {
|
||||
|
||||
// Authenticate implements the authenticator interface to authenticate requests via basic auth.
|
||||
func (m BasicAuthenticator) Authenticate(r *http.Request) (*http.Request, bool) {
|
||||
if isPublicPath(r.URL.Path) && isPublicWithShareToken(r) {
|
||||
if (isPublicPath(r.URL.Path) && isPublicWithShareToken(r)) || isPublicShareGraphRequest(r) {
|
||||
// The authentication of public path requests is handled by another authenticator.
|
||||
// Since we can't guarantee the order of execution of the authenticators, we better
|
||||
// implement an early return here for paths we can't authenticate in this authenticator.
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"strings"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
ocmw "github.com/opencloud-eu/opencloud/pkg/middleware"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
@@ -14,7 +16,7 @@ import (
|
||||
|
||||
const (
|
||||
headerRevaAccessToken = revactx.TokenHeader
|
||||
headerShareToken = "public-token"
|
||||
headerShareToken = ocmw.PublicLinkTokenName
|
||||
basicAuthPasswordPrefix = "password|"
|
||||
authenticationType = "publicshares"
|
||||
|
||||
@@ -55,12 +57,30 @@ func isPublicShareAppOpen(r *http.Request) bool {
|
||||
// the BasicAuthenticator needs to ignore the request when the headerShareToken exist.
|
||||
func isPublicWithShareToken(r *http.Request) bool {
|
||||
return (strings.HasPrefix(r.URL.Path, "/dav/public-files") || strings.HasPrefix(r.URL.Path, "/remote.php/dav/public-files")) &&
|
||||
(r.URL.Query().Get(headerShareToken) != "" || r.Header.Get(headerShareToken) != "")
|
||||
hasShareToken(r)
|
||||
}
|
||||
|
||||
// A graph request carrying a share token runs in the public share context,
|
||||
// like public-files.
|
||||
func isPublicShareGraphRequest(r *http.Request) bool {
|
||||
return strings.HasPrefix(r.URL.Path, "/graph/") && hasShareToken(r)
|
||||
}
|
||||
|
||||
func hasShareToken(r *http.Request) bool {
|
||||
return r.URL.Query().Get(headerShareToken) != "" || r.Header.Get(headerShareToken) != ""
|
||||
}
|
||||
|
||||
// shareTokenHint identifies a token in logs without spelling it out.
|
||||
func shareTokenHint(token string) string {
|
||||
if len(token) <= 4 {
|
||||
return token
|
||||
}
|
||||
return token[:4] + "..."
|
||||
}
|
||||
|
||||
// Authenticate implements the authenticator interface to authenticate requests via public share auth.
|
||||
func (a PublicShareAuthenticator) Authenticate(r *http.Request) (*http.Request, bool) {
|
||||
if !isPublicPath(r.URL.Path) && !isPublicShareArchive(r) && !isPublicShareAppOpen(r) {
|
||||
if !isPublicPath(r.URL.Path) && !isPublicShareArchive(r) && !isPublicShareAppOpen(r) && !isPublicShareGraphRequest(r) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
@@ -99,7 +119,7 @@ func (a PublicShareAuthenticator) Authenticate(r *http.Request) (*http.Request,
|
||||
a.Logger.Error().
|
||||
Err(err).
|
||||
Str("authenticator", "public_share").
|
||||
Str("public_share_token", shareToken).
|
||||
Str("public_share_token", shareTokenHint(shareToken)).
|
||||
Str("path", r.URL.Path).
|
||||
Msg("could not select next gateway client")
|
||||
return nil, false
|
||||
@@ -115,12 +135,29 @@ func (a PublicShareAuthenticator) Authenticate(r *http.Request) (*http.Request,
|
||||
a.Logger.Error().
|
||||
Err(err).
|
||||
Str("authenticator", "public_share").
|
||||
Str("public_share_token", shareToken).
|
||||
Str("public_share_token", shareTokenHint(shareToken)).
|
||||
Str("path", r.URL.Path).
|
||||
Msg("failed to authenticate request")
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if authResp.GetStatus().GetCode() != rpc.Code_CODE_OK {
|
||||
// A graph request cannot render its own 401 from here (no writer), and
|
||||
// the generic one cannot tell the two password cases apart. Mark the
|
||||
// outcome and let the graph auth middleware render it. Other surfaces
|
||||
// (webdav) are handled by their own backend, so they just fail here.
|
||||
if isPublicShareGraphRequest(r) {
|
||||
_, password, ok := r.BasicAuth()
|
||||
if ok && password != "" {
|
||||
r.Header.Set(ocmw.PublicLinkAuthHeader, ocmw.PublicLinkInvalidPassword)
|
||||
} else {
|
||||
r.Header.Set(ocmw.PublicLinkAuthHeader, ocmw.PublicLinkPasswordRequired)
|
||||
}
|
||||
return r, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
r.Header.Add(headerRevaAccessToken, authResp.Token)
|
||||
|
||||
trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("enduser.id", "public"))
|
||||
|
||||
@@ -34,10 +34,6 @@ func ParseConfig(cfg *config.Config) error {
|
||||
}
|
||||
|
||||
func Validate(cfg *config.Config) error {
|
||||
if cfg.Events.Disabled && cfg.GRPC.Disabled {
|
||||
return shared.AllComponentsDisabledError(cfg.Service.Name)
|
||||
}
|
||||
|
||||
if cfg.TokenManager.JWTSecret == "" {
|
||||
return shared.MissingJWTTokenError(cfg.Service.Name)
|
||||
}
|
||||
|
||||
@@ -82,9 +82,6 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
|
||||
if err != nil {
|
||||
return doc, err
|
||||
}
|
||||
if len(metas) == 0 {
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
for _, meta := range metas {
|
||||
title, err := getFirstValue(meta, "dc:title")
|
||||
@@ -101,21 +98,40 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document,
|
||||
} else if content, err := getFirstValue(meta, "X-TIKA:content"); err == nil {
|
||||
doc.Content = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Content, content))
|
||||
}
|
||||
|
||||
// keep facets from earlier entries, an embedded resource's meta
|
||||
// (e.g. cover art) must not reset them
|
||||
if v := t.getLocation(meta); v != nil {
|
||||
doc.Location = v
|
||||
}
|
||||
if v := t.getImage(meta); v != nil {
|
||||
doc.Image = v
|
||||
}
|
||||
if v := t.getPhoto(meta); v != nil {
|
||||
doc.Photo = v
|
||||
}
|
||||
if v := t.getAudio(meta); v != nil {
|
||||
doc.Audio = v
|
||||
}
|
||||
if v := t.getLivePhoto(meta); v != nil {
|
||||
doc.LivePhoto = v
|
||||
}
|
||||
}
|
||||
|
||||
// facets describe the file itself, not its embedded parts (cover art, clips)
|
||||
m0 := metas[0]
|
||||
doc.Location = t.getLocation(m0)
|
||||
doc.Image = t.getImage(m0)
|
||||
doc.Photo = t.getPhoto(m0)
|
||||
doc.Audio = t.getAudio(m0)
|
||||
doc.LivePhoto = t.getLivePhoto(m0)
|
||||
doc.Video = t.getVideo(m0)
|
||||
if len(metas) > 0 {
|
||||
// the video facet says the file is a video, so it comes from the file
|
||||
// itself: the clip tika extracts from a motion photo must not make its
|
||||
// image look like one
|
||||
doc.Video = t.getVideo(metas[0])
|
||||
}
|
||||
|
||||
// a motion photo is the file's own xmp plus the video tika extracted from
|
||||
// it; the xmp alone proves nothing, a share can strip the appended clip
|
||||
if i := slices.IndexFunc(metas[1:], isVideo); i >= 0 {
|
||||
doc.MotionPhoto = t.getMotionPhoto(m0, metas[i+1])
|
||||
// a motion photo is the xmp on the file itself plus the video tika extracted
|
||||
// from it. The xmp alone proves nothing: a share can keep it and strip the
|
||||
// appended video.
|
||||
if len(metas) > 0 {
|
||||
if i := slices.IndexFunc(metas[1:], isVideo); i >= 0 {
|
||||
doc.MotionPhoto = t.getMotionPhoto(metas[0], metas[i+1])
|
||||
}
|
||||
}
|
||||
|
||||
if langCode := t.detectLanguage(ctx, doc.Content); langCode != "" && t.CleanStopWords {
|
||||
|
||||
@@ -170,9 +170,7 @@ var _ = Describe("Tika", func() {
|
||||
Expect(doc.Content).To(Equal("body test stop words!!!"))
|
||||
})
|
||||
|
||||
It("takes facets from the main document, not from an embedded resource", func() {
|
||||
// metas[0] is the file (audio), metas[1] its embedded cover art. The
|
||||
// cover must not give the track an image facet.
|
||||
It("keeps the audio facet when an embedded resource follows", func() {
|
||||
fullResponse = `[{"Content-Type": "audio/mpeg", "dc:title": "Sucker", "tk:content": "lyrics"}, {"Content-Type": "image/jpeg", "tiff:ImageWidth": "500"}]`
|
||||
|
||||
doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{
|
||||
@@ -182,7 +180,7 @@ var _ = Describe("Tika", func() {
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(doc.Audio).ToNot(BeNil())
|
||||
Expect(doc.Audio.Title).To(Equal(libregraph.PtrString("Sucker")))
|
||||
Expect(doc.Image).To(BeNil())
|
||||
Expect(doc.Image).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("adds no audio facet to non-audio documents", func() {
|
||||
|
||||
@@ -35,12 +35,7 @@ Share behavior can be configured via environment variables:
|
||||
- Auto-acceptance of shares
|
||||
- Share permissions and restrictions
|
||||
|
||||
### Auto-Accept Shares
|
||||
|
||||
When setting the `SHARING_AUTO_ACCEPT_SHARES` to `true` (sharing service), all
|
||||
incoming shares will be accepted automatically. Users can overwrite this
|
||||
setting individually in their profile. The deprecated
|
||||
`FRONTEND_AUTO_ACCEPT_SHARES` is still supported for backwards compatibility.
|
||||
See the `frontend` service README for more details on share-related configuration options.
|
||||
|
||||
## Scalability
|
||||
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
//go:build !enable_vips
|
||||
|
||||
package preprocessor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
thumbnailerErrors "github.com/opencloud-eu/opencloud/services/thumbnails/pkg/errors"
|
||||
)
|
||||
|
||||
// craftDimensionBomb encodes a tiny valid grayscale JPEG, then overwrites the
|
||||
// SOF0 width/height so the header declares huge dimensions while the payload
|
||||
// stays tiny.
|
||||
func craftDimensionBomb(width, height uint16) []byte {
|
||||
var buf bytes.Buffer
|
||||
Expect(jpeg.Encode(&buf, image.NewGray(image.Rect(0, 0, 8, 8)), &jpeg.Options{Quality: 10})).To(Succeed())
|
||||
b := buf.Bytes()
|
||||
for i := 0; i+9 < len(b); i++ {
|
||||
if b[i] == 0xff && b[i+1] == 0xc0 {
|
||||
b[i+5], b[i+6] = byte(height>>8), byte(height)
|
||||
b[i+7], b[i+8] = byte(width>>8), byte(width)
|
||||
return b
|
||||
}
|
||||
}
|
||||
Fail("no SOF0 marker in encoded jpeg")
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ = Describe("ImageDecoder dimension guard", func() {
|
||||
It("rejects a source whose declared dimensions exceed the limit before decoding", func() {
|
||||
dec := ImageDecoder{limit: decodeLimit{maxWidth: 7680, maxHeight: 7680}}
|
||||
_, err := dec.Convert(bytes.NewReader(craftDimensionBomb(20000, 20000)))
|
||||
Expect(err).To(MatchError(thumbnailerErrors.ErrImageTooLarge))
|
||||
})
|
||||
|
||||
It("decodes an image within the limit", func() {
|
||||
var buf bytes.Buffer
|
||||
Expect(jpeg.Encode(&buf, image.NewGray(image.Rect(0, 0, 800, 600)), nil)).To(Succeed())
|
||||
dec := ImageDecoder{limit: decodeLimit{maxWidth: 7680, maxHeight: 7680}}
|
||||
img, err := dec.Convert(bytes.NewReader(buf.Bytes()))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(img).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("applies no limit when the bounds are zero", func() {
|
||||
dec := ImageDecoder{}
|
||||
var buf bytes.Buffer
|
||||
Expect(jpeg.Encode(&buf, image.NewGray(image.Rect(0, 0, 16, 16)), nil)).To(Succeed())
|
||||
_, err := dec.Convert(bytes.NewReader(buf.Bytes()))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("rejects an oversized gif before decoding", func() {
|
||||
dec := GifDecoder{limit: decodeLimit{maxWidth: 7680, maxHeight: 7680}}
|
||||
// LSD declares a 20000x20000 logical screen
|
||||
g := []byte("GIF89a")
|
||||
g = append(g, 0x20, 0x4e, 0x20, 0x4e, 0xf0, 0x00, 0x00) // 20000x20000, gct flag
|
||||
g = append(g, bytes.Repeat([]byte{0}, 6)...) // minimal gct + terminator-ish
|
||||
_, err := dec.Convert(bytes.NewReader(g))
|
||||
Expect(err).To(MatchError(thumbnailerErrors.ErrImageTooLarge))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("dimension limit propagation", func() {
|
||||
limit := decodeLimit{maxWidth: 7680, maxHeight: 7680}
|
||||
opts := map[string]any{"maxInputWidth": 7680, "maxInputHeight": 7680}
|
||||
|
||||
It("threads the limit into decoders that recurse into ForType", func() {
|
||||
Expect(ForType("audio/mpeg", opts)).To(Equal(AudioDecoder{limit: limit}))
|
||||
Expect(ForType("application/vnd.geogebra.pinboard", opts)).To(Equal(GgpDecoder{limit: limit}))
|
||||
g, ok := ForType("application/vnd.geogebra.slides", opts).(GgsDecoder)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(g.limit).To(Equal(limit))
|
||||
})
|
||||
|
||||
It("rejects an oversized cover image embedded in a ggp file", func() {
|
||||
bomb := craftDimensionBomb(20000, 20000)
|
||||
payload := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(bomb)
|
||||
ggp := GGPStruct{}
|
||||
ggp.Sections = append(ggp.Sections, struct {
|
||||
Cards []struct {
|
||||
Element struct {
|
||||
Image struct{ Base64Image string }
|
||||
}
|
||||
}
|
||||
}{Cards: []struct {
|
||||
Element struct {
|
||||
Image struct{ Base64Image string }
|
||||
}
|
||||
}{{Element: struct {
|
||||
Image struct{ Base64Image string }
|
||||
}{Image: struct{ Base64Image string }{Base64Image: payload}}}}})
|
||||
raw, err := json.Marshal(ggp)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = GgpDecoder{limit: limit}.Convert(bytes.NewReader(raw))
|
||||
Expect(err).To(MatchError(thumbnailerErrors.ErrImageTooLarge))
|
||||
})
|
||||
})
|
||||
@@ -30,14 +30,10 @@ type FileConverter interface {
|
||||
}
|
||||
|
||||
// GifDecoder is a converter for the gif file
|
||||
type GifDecoder struct{ limit decodeLimit }
|
||||
type GifDecoder struct{}
|
||||
|
||||
// Convert reads the gif file and returns the thumbnail image
|
||||
func (i GifDecoder) Convert(r io.Reader) (any, error) {
|
||||
r, err := i.limit.guardDimensions(r, gif.DecodeConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
img, err := gif.DecodeAll(r)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, `could not decode the image`)
|
||||
@@ -46,10 +42,7 @@ func (i GifDecoder) Convert(r io.Reader) (any, error) {
|
||||
}
|
||||
|
||||
// GgsDecoder is a converter for the geogebra slides file
|
||||
type GgsDecoder struct {
|
||||
thumbnailpath string
|
||||
limit decodeLimit
|
||||
}
|
||||
type GgsDecoder struct{ thumbnailpath string }
|
||||
|
||||
// Convert reads the ggs file and returns the thumbnail image
|
||||
func (g GgsDecoder) Convert(r io.Reader) (any, error) {
|
||||
@@ -68,7 +61,7 @@ func (g GgsDecoder) Convert(r io.Reader) (any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
converter := ForType("image/png", g.limit.opts())
|
||||
converter := ForType("image/png", nil)
|
||||
if converter == nil {
|
||||
return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromGgsFile
|
||||
}
|
||||
@@ -83,7 +76,7 @@ func (g GgsDecoder) Convert(r io.Reader) (any, error) {
|
||||
}
|
||||
|
||||
// AudioDecoder is a converter for the audio file
|
||||
type AudioDecoder struct{ limit decodeLimit }
|
||||
type AudioDecoder struct{}
|
||||
|
||||
// Convert reads the audio file and extracts the thumbnail image from the id3 tag
|
||||
func (i AudioDecoder) Convert(r io.Reader) (any, error) {
|
||||
@@ -101,7 +94,7 @@ func (i AudioDecoder) Convert(r io.Reader) (any, error) {
|
||||
return nil, thumbnailerErrors.ErrNoImageFromAudioFile
|
||||
}
|
||||
|
||||
converter := ForType(picture.MIMEType, i.limit.opts())
|
||||
converter := ForType(picture.MIMEType, nil)
|
||||
if converter == nil {
|
||||
return nil, thumbnailerErrors.ErrNoConverterForExtractedImageFromAudioFile
|
||||
}
|
||||
@@ -207,7 +200,7 @@ type GGPStruct struct {
|
||||
}
|
||||
|
||||
// GgpDecoder is a converter for the geogebra pinboard file
|
||||
type GgpDecoder struct{ limit decodeLimit }
|
||||
type GgpDecoder struct{}
|
||||
|
||||
// Convert reads the ggp file and returns the first thumbnail image
|
||||
func (j GgpDecoder) Convert(r io.Reader) (any, error) {
|
||||
@@ -227,14 +220,7 @@ func (j GgpDecoder) Convert(r io.Reader) (any, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
r2, err := j.limit.guardDimensions(bytes.NewReader(b), func(rr io.Reader) (image.Config, error) {
|
||||
cfg, _, err := image.DecodeConfig(rr)
|
||||
return cfg, err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
img, _, err := image.Decode(r2)
|
||||
img, _, err := image.Decode(bytes.NewReader(b))
|
||||
return img, err
|
||||
}
|
||||
|
||||
@@ -311,61 +297,11 @@ func drawWord(canvas *font.Drawer, word string, minX, maxX, incY, maxY fixed.Int
|
||||
}
|
||||
}
|
||||
|
||||
// decodeLimit bounds the source image dimensions a decoder accepts. A zero
|
||||
// value on an axis disables the limit for that axis.
|
||||
type decodeLimit struct {
|
||||
maxWidth int
|
||||
maxHeight int
|
||||
}
|
||||
|
||||
func decodeLimitFromOpts(opts map[string]any) decodeLimit {
|
||||
l := decodeLimit{}
|
||||
if v, ok := opts["maxInputWidth"].(int); ok {
|
||||
l.maxWidth = v
|
||||
}
|
||||
if v, ok := opts["maxInputHeight"].(int); ok {
|
||||
l.maxHeight = v
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func (l decodeLimit) exceeded(width, height int) bool {
|
||||
return (l.maxWidth > 0 && width > l.maxWidth) || (l.maxHeight > 0 && height > l.maxHeight)
|
||||
}
|
||||
|
||||
// opts renders the limit back into an options map so decoders that recurse
|
||||
// into ForType can forward it to the nested image decoder.
|
||||
func (l decodeLimit) opts() map[string]any {
|
||||
return map[string]any{"maxInputWidth": l.maxWidth, "maxInputHeight": l.maxHeight}
|
||||
}
|
||||
|
||||
// guardDimensions reads only the image header (no pixel allocation) and
|
||||
// rejects a source whose declared dimensions exceed the limit, before the
|
||||
// full bitmap is decoded. It returns a reader that replays the consumed
|
||||
// header so the caller can still decode from the start. A header that cannot
|
||||
// be parsed is passed through unchecked, letting the real decoder report it.
|
||||
func (l decodeLimit) guardDimensions(r io.Reader, config func(io.Reader) (image.Config, error)) (io.Reader, error) {
|
||||
if l.maxWidth <= 0 && l.maxHeight <= 0 {
|
||||
return r, nil
|
||||
}
|
||||
var head bytes.Buffer
|
||||
cfg, err := config(io.TeeReader(r, &head))
|
||||
replay := io.MultiReader(&head, r)
|
||||
if err != nil {
|
||||
return replay, nil
|
||||
}
|
||||
if l.exceeded(cfg.Width, cfg.Height) {
|
||||
return nil, thumbnailerErrors.ErrImageTooLarge
|
||||
}
|
||||
return replay, nil
|
||||
}
|
||||
|
||||
// ForType returns the converter for the specified mimeType
|
||||
func ForType(mimeType string, opts map[string]any) FileConverter {
|
||||
// We can ignore the error here because we parse it in IsMimeTypeSupported before and if it fails
|
||||
// return the service call. So we should only get here when the mimeType parses fine.
|
||||
mimeType, _, _ = mime.ParseMediaType(mimeType)
|
||||
limit := decodeLimitFromOpts(opts)
|
||||
switch mimeType {
|
||||
case "text/plain":
|
||||
fontFileMap := ""
|
||||
@@ -397,18 +333,18 @@ func ForType(mimeType string, opts map[string]any) FileConverter {
|
||||
fontLoader: fontLoader,
|
||||
}
|
||||
case "application/vnd.geogebra.slides":
|
||||
return GgsDecoder{thumbnailpath: "_slide0/geogebra_thumbnail.png", limit: limit}
|
||||
return GgsDecoder{"_slide0/geogebra_thumbnail.png"}
|
||||
case "application/vnd.geogebra.pinboard":
|
||||
return GgpDecoder{limit: limit}
|
||||
return GgpDecoder{}
|
||||
case "image/gif":
|
||||
return GifDecoder{limit: limit}
|
||||
return GifDecoder{}
|
||||
case "audio/flac":
|
||||
fallthrough
|
||||
case "audio/mpeg":
|
||||
fallthrough
|
||||
case "audio/ogg":
|
||||
return AudioDecoder{limit: limit}
|
||||
return AudioDecoder{}
|
||||
default:
|
||||
return ImageDecoder{limit: limit}
|
||||
return ImageDecoder{}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@
|
||||
package preprocessor
|
||||
|
||||
import (
|
||||
"image"
|
||||
"io"
|
||||
|
||||
"github.com/kovidgoyal/imaging"
|
||||
@@ -11,20 +10,10 @@ import (
|
||||
)
|
||||
|
||||
// ImageDecoder is a converter for the image file
|
||||
type ImageDecoder struct{ limit decodeLimit }
|
||||
type ImageDecoder struct{}
|
||||
|
||||
// Convert reads the image file and returns the thumbnail image
|
||||
func (i ImageDecoder) Convert(r io.Reader) (any, error) {
|
||||
// bound the declared dimensions before imaging.Decode allocates the full
|
||||
// pixel buffer: a crafted header (e.g. 65535x65535) would otherwise OOM
|
||||
// the worker, the downstream dimension guard only runs after the decode
|
||||
r, err := i.limit.guardDimensions(r, func(rr io.Reader) (image.Config, error) {
|
||||
cfg, _, err := image.DecodeConfig(rr)
|
||||
return cfg, err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
img, err := imaging.Decode(r, imaging.AutoOrientation(true))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, `could not decode the image`)
|
||||
|
||||
@@ -81,14 +81,14 @@ var _ = Describe("ImageDecoder", func() {
|
||||
})
|
||||
|
||||
It("should decode a ggs", func() {
|
||||
decoder := GgsDecoder{thumbnailpath: "_slide0/geogebra_thumbnail.png"}
|
||||
decoder := GgsDecoder{"_slide0/geogebra_thumbnail.png"}
|
||||
img, err := decoder.Convert(fileReader)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(img).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("should return an error if the ggs is invalid", func() {
|
||||
decoder := GgsDecoder{thumbnailpath: "_slide0/geogebra_thumbnail.png"}
|
||||
decoder := GgsDecoder{"_slide0/geogebra_thumbnail.png"}
|
||||
img, err := decoder.Convert(bytes.NewReader([]byte("not a ggs")))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(img).To(BeNil())
|
||||
|
||||
@@ -6,26 +6,15 @@ import (
|
||||
"io"
|
||||
|
||||
"github.com/davidbyttow/govips/v2/vips"
|
||||
|
||||
thumbnailerErrors "github.com/opencloud-eu/opencloud/services/thumbnails/pkg/errors"
|
||||
)
|
||||
|
||||
func init() {
|
||||
vips.LoggingSettings(nil, vips.LogLevelError)
|
||||
}
|
||||
|
||||
type ImageDecoder struct{ limit decodeLimit }
|
||||
type ImageDecoder struct{}
|
||||
|
||||
func (v ImageDecoder) Convert(r io.Reader) (interface{}, error) {
|
||||
// NewImageFromReader is header-lazy, Width/Height read the header without
|
||||
// materializing pixels: reject oversized sources before ThumbnailWithSize
|
||||
img, err := vips.NewImageFromReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v.limit.exceeded(img.Width(), img.Height()) {
|
||||
img.Close()
|
||||
return nil, thumbnailerErrors.ErrImageTooLarge
|
||||
}
|
||||
return img, nil
|
||||
return img, err
|
||||
}
|
||||
@@ -54,8 +54,6 @@ func NewService(opts ...Option) decorators.DecoratedService {
|
||||
selector: options.GatewaySelector,
|
||||
preprocessorOpts: PreprocessorOpts{
|
||||
TxtFontFileMap: options.Config.Thumbnail.FontMapFile,
|
||||
MaxInputWidth: options.Config.Thumbnail.MaxInputWidth,
|
||||
MaxInputHeight: options.Config.Thumbnail.MaxInputHeight,
|
||||
},
|
||||
dataEndpoint: options.Config.Thumbnail.DataEndpoint,
|
||||
transferSecret: options.Config.Thumbnail.TransferSecret,
|
||||
@@ -80,8 +78,6 @@ type Thumbnail struct {
|
||||
// PreprocessorOpts holds the options for the preprocessor
|
||||
type PreprocessorOpts struct {
|
||||
TxtFontFileMap string
|
||||
MaxInputWidth int
|
||||
MaxInputHeight int
|
||||
}
|
||||
|
||||
// GetThumbnail retrieves a thumbnail for an image
|
||||
@@ -170,15 +166,10 @@ func (g Thumbnail) handleCS3Source(ctx context.Context, req *thumbnailssvc.GetTh
|
||||
|
||||
defer r.Close()
|
||||
ppOpts := map[string]any{
|
||||
"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
|
||||
"maxInputWidth": g.preprocessorOpts.MaxInputWidth,
|
||||
"maxInputHeight": g.preprocessorOpts.MaxInputHeight,
|
||||
"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
|
||||
}
|
||||
pp := preprocessor.ForType(sRes.GetInfo().GetMimeType(), ppOpts)
|
||||
img, err := pp.Convert(r)
|
||||
if errors.Is(err, terrors.ErrImageTooLarge) {
|
||||
return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
|
||||
}
|
||||
if err != nil {
|
||||
g.logger.Error().Err(err).Msg("failed to convert image")
|
||||
}
|
||||
@@ -271,15 +262,10 @@ func (g Thumbnail) handleWebdavSource(ctx context.Context, req *thumbnailssvc.Ge
|
||||
}
|
||||
defer r.Close()
|
||||
ppOpts := map[string]any{
|
||||
"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
|
||||
"maxInputWidth": g.preprocessorOpts.MaxInputWidth,
|
||||
"maxInputHeight": g.preprocessorOpts.MaxInputHeight,
|
||||
"fontFileMap": g.preprocessorOpts.TxtFontFileMap,
|
||||
}
|
||||
pp := preprocessor.ForType(sRes.GetInfo().GetMimeType(), ppOpts)
|
||||
img, err := pp.Convert(r)
|
||||
if errors.Is(err, terrors.ErrImageTooLarge) {
|
||||
return "", merrors.Forbidden(g.serviceID, "%s", err.Error())
|
||||
}
|
||||
if img == nil || err != nil {
|
||||
return "", merrors.NotFound(g.serviceID, "could not get image")
|
||||
}
|
||||
|
||||
@@ -3490,4 +3490,380 @@ class GraphContext implements Context {
|
||||
$url = "/graph/$apiVersion/drives/$driveId/root:/$encoded";
|
||||
$this->sendGraphRequestAndCaptureResponse($user, "GET", $url);
|
||||
}
|
||||
|
||||
/**
|
||||
* The virtual drive a public link is addressed under: the public storage
|
||||
* provider/space id pair with the link token as opaque id.
|
||||
*
|
||||
* @param string $token
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
private function publicLinkDriveId(string $token): string {
|
||||
$publicStorageId = "7993447f-687f-490d-875c-ac95e89a62a4";
|
||||
return "$publicStorageId\$$publicStorageId!$token";
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an anonymous Graph request in the context of the last created
|
||||
* public link: token via the public-token query parameter, an optional
|
||||
* password as basic auth for user "public".
|
||||
*
|
||||
* @param string $urlSuffix part below /graph/v1.0/drives/{publicDriveId}
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function publicSendsGraphDriveRequest(string $urlSuffix, ?string $password = null): void {
|
||||
$token = $this->featureContext->shareNgGetLastCreatedLinkShareToken();
|
||||
$driveId = $this->publicLinkDriveId($token);
|
||||
$url = $this->featureContext->getBaseUrl()
|
||||
. "/graph/v1.0/drives/$driveId$urlSuffix"
|
||||
. (\str_contains($urlSuffix, "?") ? "&" : "?") . "public-token=$token";
|
||||
$response = HttpRequestHelper::get(
|
||||
$url,
|
||||
$this->featureContext->getStepLineRef(),
|
||||
$password === null ? null : "public",
|
||||
$this->featureContext->getActualPassword($password)
|
||||
);
|
||||
$this->featureContext->setResponse($response);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public lists the children of the last created public link using the Graph API')]
|
||||
#[When('the public lists the children of the last created public link with password :password using the Graph API')]
|
||||
public function thePublicListsTheChildrenOfTheLastCreatedPublicLink(?string $password = null): void {
|
||||
$token = $this->featureContext->shareNgGetLastCreatedLinkShareToken();
|
||||
$rootId = $this->publicLinkDriveId($token);
|
||||
$this->publicSendsGraphDriveRequest("/items/$rootId/children", $password);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $path
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public lists the children of path :path of the last created public link using the Graph API')]
|
||||
#[When('the public lists the children of path :path of the last created public link with password :password using the Graph API')]
|
||||
public function thePublicListsTheChildrenOfPathOfTheLastCreatedPublicLink(
|
||||
string $path,
|
||||
?string $password = null
|
||||
): void {
|
||||
$encoded = $this->encodeColonPathSegment($path);
|
||||
$this->publicSendsGraphDriveRequest("/root:/$encoded:/children", $password);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public gets the root of the last created public link expanding its children using the Graph API')]
|
||||
#[When('the public gets the root of the last created public link expanding its children with password :password using the Graph API')]
|
||||
public function thePublicGetsTheRootOfTheLastCreatedPublicLinkExpandingItsChildren(?string $password = null): void {
|
||||
$token = $this->featureContext->shareNgGetLastCreatedLinkShareToken();
|
||||
$rootId = $this->publicLinkDriveId($token);
|
||||
$this->publicSendsGraphDriveRequest("/items/$rootId?\$expand=children", $password);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $path
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public gets the drive item of path :path of the last created public link selecting the allowed actions with password :password using the Graph API')]
|
||||
public function thePublicGetsTheDriveItemOfPathSelectingTheAllowedActions(
|
||||
string $path,
|
||||
?string $password = null
|
||||
): void {
|
||||
$encoded = $this->encodeColonPathSegment($path);
|
||||
$select = "%24select=%40libre.graph.permissions.actions.allowedValues";
|
||||
$this->publicSendsGraphDriveRequest("/root:/$encoded?$select", $password);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $urlSuffix raw suffix below /graph/v1.0, may contain :spaceOfUser
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function publicSendsRawGraphRequest(string $urlSuffix, ?string $password = null): void {
|
||||
$token = $this->featureContext->shareNgGetLastCreatedLinkShareToken();
|
||||
$url = $this->featureContext->getBaseUrl() . "/graph/v1.0" . $urlSuffix
|
||||
. (\str_contains($urlSuffix, "?") ? "&" : "?") . "public-token=$token";
|
||||
$response = HttpRequestHelper::get(
|
||||
$url,
|
||||
$this->featureContext->getStepLineRef(),
|
||||
$password === null ? null : "public",
|
||||
$this->featureContext->getActualPassword($password)
|
||||
);
|
||||
$this->featureContext->setResponse($response);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public lists the children of the last created public link selecting the share types with password :password using the Graph API')]
|
||||
public function thePublicListsTheChildrenSelectingTheShareTypes(?string $password = null): void {
|
||||
$token = $this->featureContext->shareNgGetLastCreatedLinkShareToken();
|
||||
$rootId = $this->publicLinkDriveId($token);
|
||||
$select = "%24select=%40libre.graph.shareTypes";
|
||||
$this->publicSendsGraphDriveRequest("/items/$rootId/children?$select", $password);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public tries to list the drives using the token of the last created public link with password :password using the Graph API')]
|
||||
public function thePublicTriesToListTheDrives(?string $password = null): void {
|
||||
$this->publicSendsRawGraphRequest("/drives/", $password);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $user
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public tries to get the personal drive of user :user through the last created public link with password :password using the Graph API')]
|
||||
public function thePublicTriesToGetThePersonalDriveOfUser(string $user, ?string $password = null): void {
|
||||
$user = $this->featureContext->getActualUsername($user);
|
||||
$driveId = $this->spacesContext->getSpaceIdByName($user, "Personal");
|
||||
$this->publicSendsRawGraphRequest("/drives/$driveId", $password);
|
||||
}
|
||||
|
||||
/**
|
||||
* Addressing an in-share item through its REAL drive id must be rejected:
|
||||
* only the token's public drive is authorized.
|
||||
*
|
||||
* @param string $child
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public tries to get the child :child of the last created public link through its real drive id with password :password using the Graph API')]
|
||||
public function thePublicTriesToGetTheChildThroughItsRealDriveId(string $child, ?string $password = null): void {
|
||||
$token = $this->featureContext->shareNgGetLastCreatedLinkShareToken();
|
||||
$rootId = $this->publicLinkDriveId($token);
|
||||
$url = $this->featureContext->getBaseUrl()
|
||||
. "/graph/v1.0/drives/$rootId/items/$rootId/children?public-token=$token";
|
||||
$listing = HttpRequestHelper::get(
|
||||
$url,
|
||||
$this->featureContext->getStepLineRef(),
|
||||
$password === null ? null : "public",
|
||||
$this->featureContext->getActualPassword($password)
|
||||
);
|
||||
$children = \json_decode($listing->getBody()->getContents(), true)["value"] ?? [];
|
||||
$childId = null;
|
||||
foreach ($children as $entry) {
|
||||
if ($entry["name"] === $child) {
|
||||
$childId = $entry["id"];
|
||||
}
|
||||
}
|
||||
Assert::assertNotNull($childId, "child '$child' not found in the public link listing");
|
||||
$realDriveId = \explode("!", $childId)[0];
|
||||
$this->publicSendsRawGraphRequest("/drives/$realDriveId/items/$childId", $password);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public tries to list the children of a foreign public link drive using the last created token with password :password using the Graph API')]
|
||||
public function thePublicTriesToListAForeignPublicLinkDrive(?string $password = null): void {
|
||||
$foreign = $this->publicLinkDriveId("notthetokenofthislink");
|
||||
$this->publicSendsRawGraphRequest("/drives/$foreign/items/$foreign/children", $password);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mutation probes: the public link surface is read only, every write on the
|
||||
* beta routes has to be rejected.
|
||||
*
|
||||
* @param string $action
|
||||
* @param string $child
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('/^the public tries to (create a link for|delete|rename|list the permissions of) the child "([^"]*)" of the last created public link with password "([^"]*)" using the Graph API$/')]
|
||||
public function thePublicTriesToMutateTheChildOfTheLastCreatedPublicLink(
|
||||
string $action,
|
||||
string $child,
|
||||
?string $password = null
|
||||
): void {
|
||||
$token = $this->featureContext->shareNgGetLastCreatedLinkShareToken();
|
||||
$rootId = $this->publicLinkDriveId($token);
|
||||
$listing = HttpRequestHelper::get(
|
||||
$this->featureContext->getBaseUrl()
|
||||
. "/graph/v1.0/drives/$rootId/items/$rootId/children?public-token=$token",
|
||||
$this->featureContext->getStepLineRef(),
|
||||
"public",
|
||||
$this->featureContext->getActualPassword($password)
|
||||
);
|
||||
$children = \json_decode($listing->getBody()->getContents(), true)["value"] ?? [];
|
||||
$childId = null;
|
||||
foreach ($children as $entry) {
|
||||
if ($entry["name"] === $child) {
|
||||
$childId = $entry["id"];
|
||||
}
|
||||
}
|
||||
Assert::assertNotNull($childId, "child '$child' not found in the public link listing");
|
||||
|
||||
$base = "/graph/v1beta1/drives/$rootId/items/$childId";
|
||||
switch ($action) {
|
||||
case "create a link for":
|
||||
$method = "POST";
|
||||
$url = "$base/createLink";
|
||||
$body = \json_encode(["type" => "view", "password" => "Sup3rS3cret!x"]);
|
||||
break;
|
||||
case "delete":
|
||||
$method = "DELETE";
|
||||
$url = $base;
|
||||
$body = null;
|
||||
break;
|
||||
case "rename":
|
||||
$method = "PATCH";
|
||||
$url = $base;
|
||||
$body = \json_encode(["name" => "renamed.txt"]);
|
||||
break;
|
||||
case "list the permissions of":
|
||||
$method = "GET";
|
||||
$url = "$base/permissions";
|
||||
$body = null;
|
||||
break;
|
||||
default:
|
||||
throw new \Exception("unknown mutation action '$action'");
|
||||
}
|
||||
$response = HttpRequestHelper::sendRequest(
|
||||
$this->featureContext->getBaseUrl() . $url
|
||||
. (\str_contains($url, "?") ? "&" : "?") . "public-token=$token",
|
||||
$this->featureContext->getStepLineRef(),
|
||||
$method,
|
||||
"public",
|
||||
$this->featureContext->getActualPassword($password),
|
||||
["Content-Type" => "application/json"],
|
||||
$body
|
||||
);
|
||||
$this->featureContext->setResponse($response);
|
||||
}
|
||||
|
||||
/**
|
||||
* The public link drive endpoint rejects the token as a query parameter,
|
||||
* so it rides in the header here.
|
||||
*
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public gets the drive of the last created public link with password :password using the Graph API')]
|
||||
public function thePublicGetsTheDriveOfTheLastCreatedPublicLink(?string $password = null): void {
|
||||
$token = $this->featureContext->shareNgGetLastCreatedLinkShareToken();
|
||||
$driveId = $this->publicLinkDriveId($token);
|
||||
$response = HttpRequestHelper::get(
|
||||
$this->featureContext->getBaseUrl() . "/graph/v1.0/drives/$driveId",
|
||||
$this->featureContext->getStepLineRef(),
|
||||
$password === null ? null : "public",
|
||||
$this->featureContext->getActualPassword($password),
|
||||
["public-token" => $token]
|
||||
);
|
||||
$this->featureContext->setResponse($response);
|
||||
}
|
||||
|
||||
/**
|
||||
* Item anchored colon path: the anchor id is resolved through the public
|
||||
* children listing, so the step stays within the public API.
|
||||
*
|
||||
* @param string $path
|
||||
* @param string $child
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public gets the drive item :path below the child :child of the last created public link with password :password using the Graph API')]
|
||||
public function thePublicGetsTheDriveItemBelowTheChildOfTheLastCreatedPublicLink(
|
||||
string $path,
|
||||
string $child,
|
||||
?string $password = null
|
||||
): void {
|
||||
$token = $this->featureContext->shareNgGetLastCreatedLinkShareToken();
|
||||
$rootId = $this->publicLinkDriveId($token);
|
||||
$url = $this->featureContext->getBaseUrl()
|
||||
. "/graph/v1.0/drives/$rootId/items/$rootId/children?public-token=$token";
|
||||
$response = HttpRequestHelper::get(
|
||||
$url,
|
||||
$this->featureContext->getStepLineRef(),
|
||||
$password === null ? null : "public",
|
||||
$this->featureContext->getActualPassword($password)
|
||||
);
|
||||
$children = \json_decode($response->getBody()->getContents(), true)["value"] ?? [];
|
||||
$childId = null;
|
||||
foreach ($children as $entry) {
|
||||
if ($entry["name"] === $child) {
|
||||
$childId = $entry["id"];
|
||||
}
|
||||
}
|
||||
Assert::assertNotNull($childId, "child '$child' not found in the public link listing");
|
||||
$encoded = $this->encodeColonPathSegment($path);
|
||||
$this->publicSendsGraphDriveRequest("/items/$childId:/$encoded", $password);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $name
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public creates an upload session for :name in the last created public link using the Graph API')]
|
||||
#[When('the public creates an upload session for :name in the last created public link with password :password using the Graph API')]
|
||||
public function thePublicCreatesAnUploadSessionForInTheLastCreatedPublicLink(
|
||||
string $name,
|
||||
?string $password = null
|
||||
): void {
|
||||
$token = $this->featureContext->shareNgGetLastCreatedLinkShareToken();
|
||||
$rootId = $this->publicLinkDriveId($token);
|
||||
$url = $this->featureContext->getBaseUrl()
|
||||
. "/graph/v1.0/drives/$rootId/items/$rootId/createUploadSession?public-token=$token";
|
||||
$response = HttpRequestHelper::post(
|
||||
$url,
|
||||
$this->featureContext->getStepLineRef(),
|
||||
$password === null ? null : "public",
|
||||
$this->featureContext->getActualPassword($password),
|
||||
["Content-Type" => "application/json"],
|
||||
\json_encode(["item" => ["name" => $name, "fileSize" => 6]])
|
||||
);
|
||||
$this->featureContext->setResponse($response);
|
||||
}
|
||||
|
||||
/**
|
||||
* The security probe: an id of a resource that is NOT inside the public
|
||||
* link must not be readable through the link's token.
|
||||
*
|
||||
* @param string $path
|
||||
* @param string $user
|
||||
* @param string|null $password
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the public tries to get the resource :path of user :user through the last created public link using the Graph API')]
|
||||
#[When('the public tries to get the resource :path of user :user through the last created public link with password :password using the Graph API')]
|
||||
public function thePublicTriesToGetTheResourceOfUserThroughTheLastCreatedPublicLink(
|
||||
string $path,
|
||||
string $user,
|
||||
?string $password = null
|
||||
): void {
|
||||
$user = $this->featureContext->getActualUsername($user);
|
||||
$resourceId = $this->featureContext->getFileIdForPath($user, $path);
|
||||
$this->publicSendsGraphDriveRequest("/items/$resourceId", $password);
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,6 @@ use GuzzleHttp\Exception\GuzzleException;
|
||||
use PHPUnit\Framework\Assert;
|
||||
use Psr\Http\Message\ResponseInterface;
|
||||
use TestHelpers\WebDavHelper;
|
||||
use TestHelpers\WaitHelper;
|
||||
use TestHelpers\HttpRequestHelper;
|
||||
use TestHelpers\BehatHelper;
|
||||
use Behat\Step\Then;
|
||||
@@ -38,7 +37,6 @@ require_once 'bootstrap.php';
|
||||
*/
|
||||
class SearchContext implements Context {
|
||||
private FeatureContext $featureContext;
|
||||
private array $lastSearchQuery = [];
|
||||
|
||||
/**
|
||||
* @param string $user
|
||||
@@ -151,13 +149,6 @@ class SearchContext implements Context {
|
||||
// NOTE: because indexing of newly uploaded files or directories with OpenCloud is decoupled and occurs asynchronously
|
||||
// short wait is necessary before searching
|
||||
sleep(10);
|
||||
// remember the query so "should eventually contain" steps can re-search
|
||||
$this->lastSearchQuery = [
|
||||
"user" => $user,
|
||||
"pattern" => $pattern,
|
||||
"limit" => $limit,
|
||||
"properties" => $properties,
|
||||
];
|
||||
$response = $this->searchFiles($user, $pattern, $limit, null, null, null, $properties);
|
||||
$this->featureContext->setResponse($response);
|
||||
}
|
||||
@@ -288,83 +279,4 @@ class SearchContext implements Context {
|
||||
$response = $this-> searchFiles($user, $pattern, null, $scopeType, $scope, $spaceName);
|
||||
$this->featureContext->setResponse($response);
|
||||
}
|
||||
|
||||
/**
|
||||
* re-run the last WebDAV search until the assertion passes or the WaitHelper
|
||||
* timeout elapses, leaving the last response set for a final assertion by the
|
||||
* caller. Indexing of newly uploaded resources is asynchronous, so a wanted
|
||||
* file can be missing from an early search; OpenSearch never returns a partial
|
||||
* document, so once the expected entries are present the result is complete.
|
||||
*
|
||||
* @param callable $assert
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function retrySearchUntilSatisfied(callable $assert): void {
|
||||
Assert::assertNotEmpty(
|
||||
$this->lastSearchQuery,
|
||||
'No search to retry. Use a "searches for ... using the WebDAV API" step first.'
|
||||
);
|
||||
$query = $this->lastSearchQuery;
|
||||
$response = WaitHelper::waitUntil(
|
||||
fn () => $this->searchFiles(
|
||||
$query["user"],
|
||||
$query["pattern"],
|
||||
$query["limit"],
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
$query["properties"]
|
||||
),
|
||||
function ($response) use ($assert) {
|
||||
$this->featureContext->setResponse($response);
|
||||
try {
|
||||
$assert();
|
||||
return true;
|
||||
} catch (\Throwable $e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
);
|
||||
$this->featureContext->setResponse($response);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $user
|
||||
* @param TableNode $expectedFiles
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[Then('/^the search result of user "([^"]*)" should eventually contain only these (?:files|entries):$/')]
|
||||
public function theSearchResultShouldEventuallyContainOnlyEntries(string $user, TableNode $expectedFiles): void {
|
||||
$assert = fn () => $this->featureContext->thePropfindResultShouldContainOnlyEntries($user, $expectedFiles);
|
||||
$this->retrySearchUntilSatisfied($assert);
|
||||
$assert();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $user
|
||||
* @param TableNode $expectedFiles
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[Then('/^the search result of user "([^"]*)" should eventually contain these (?:files|entries):$/')]
|
||||
public function theSearchResultShouldEventuallyContainEntries(string $user, TableNode $expectedFiles): void {
|
||||
$assert = fn () => $this->featureContext->thePropfindResultShouldContainEntries($user, '', $expectedFiles);
|
||||
$this->retrySearchUntilSatisfied($assert);
|
||||
$assert();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param TableNode $expectedFiles
|
||||
* @param string $expectedContent
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[Then('/^the search result should eventually contain these (?:files|entries) with highlight on keyword "([^"]*)"$/')]
|
||||
public function theSearchResultShouldEventuallyContainEntriesWithHighlight(TableNode $expectedFiles, string $expectedContent): void {
|
||||
$assert = fn () => $this->theSearchResultShouldContainEntriesWithHighlight($expectedFiles, $expectedContent);
|
||||
$this->retrySearchUntilSatisfied($assert);
|
||||
$assert();
|
||||
}
|
||||
}
|
||||
@@ -55,9 +55,6 @@ class SpacesContext implements Context {
|
||||
* key is space name and value is the username that created the space
|
||||
*/
|
||||
private array $createdSpaces = [];
|
||||
// the request of the last driveItem GET, so a later "should eventually
|
||||
// match" step can re-run it (the counterpart of the stored response)
|
||||
private array $lastDriveItemRequest = [];
|
||||
private string $ocsApiUrl = '/ocs/v2.php/apps/files_sharing/api/v1/shares';
|
||||
|
||||
/**
|
||||
@@ -4536,16 +4533,17 @@ class SpacesContext implements Context {
|
||||
}
|
||||
|
||||
/**
|
||||
* resolve the Graph URL of a file in a space
|
||||
*
|
||||
* @param string $user
|
||||
* @param string $file
|
||||
* @param string $space
|
||||
*
|
||||
* @return string
|
||||
* @return void
|
||||
*/
|
||||
private function getDriveItemUrl(string $user, string $file, string $space): string {
|
||||
#[When('user :user gets the file :file from space :space using the Graph API')]
|
||||
public function userGetsTheDriveItemInSpace(string $user, string $file, string $space): void {
|
||||
$spaceId = ($this->getSpaceByName($user, $space))["id"];
|
||||
$itemId = '';
|
||||
if ($space === "Shares") {
|
||||
$itemId = GraphHelper::getShareMountId(
|
||||
$this->featureContext->getBaseUrl(),
|
||||
@@ -4557,51 +4555,10 @@ class SpacesContext implements Context {
|
||||
} else {
|
||||
$itemId = $this->getFileId($user, $space, $file);
|
||||
}
|
||||
return $this->featureContext->getBaseUrl() . "/graph/v1.0/drives/$spaceId/items/$itemId";
|
||||
}
|
||||
$url = $this->featureContext->getBaseUrl() . "/graph/v1.0/drives/$spaceId/items/$itemId";
|
||||
|
||||
/**
|
||||
*
|
||||
* @param string $user
|
||||
* @param string $file
|
||||
* @param string $space
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('user :user gets the file :file from space :space using the Graph API')]
|
||||
public function userGetsTheDriveItemInSpace(string $user, string $file, string $space): void {
|
||||
$this->lastDriveItemRequest = [
|
||||
"url" => $this->getDriveItemUrl($user, $file, $space),
|
||||
"user" => $user,
|
||||
];
|
||||
$response = HttpRequestHelper::get(
|
||||
$this->lastDriveItemRequest["url"],
|
||||
$this->featureContext->getStepLineRef(),
|
||||
$user,
|
||||
$this->featureContext->getPasswordForUser($user),
|
||||
);
|
||||
$this->featureContext->setResponse($response);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param PyStringNode $schemaString
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[Then('the JSON data of the response should eventually match')]
|
||||
public function theJsonDataOfTheResponseShouldEventuallyMatch(PyStringNode $schemaString): void {
|
||||
Assert::assertNotEmpty(
|
||||
$this->lastDriveItemRequest,
|
||||
'no driveItem request to repeat, get the file using the Graph API first'
|
||||
);
|
||||
$url = $this->lastDriveItemRequest["url"];
|
||||
$user = $this->lastDriveItemRequest["user"];
|
||||
$schema = $this->featureContext->getJSONSchema($schemaString);
|
||||
|
||||
// Extraction is asynchronous, so re-fetch until the response satisfies the
|
||||
// expected schema (a partial payload never matches) or the WaitHelper
|
||||
// timeout elapses.
|
||||
// NOTE: extracting properties occurs asynchronously after upload, so we need to wait until the properties are available
|
||||
$extractionFacets = ["image", "photo", "location", "audio", "video"];
|
||||
$response = WaitHelper::waitUntil(
|
||||
fn () => HttpRequestHelper::get(
|
||||
$url,
|
||||
@@ -4609,24 +4566,16 @@ class SpacesContext implements Context {
|
||||
$user,
|
||||
$this->featureContext->getPasswordForUser($user),
|
||||
),
|
||||
function ($response) use ($schema) {
|
||||
function ($response) use ($extractionFacets) {
|
||||
if ($response->getStatusCode() !== 200) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
$body = $this->featureContext->getJsonDecodedResponseBodyContent($response);
|
||||
$this->featureContext->assertJsonDocumentMatchesSchema($body, $schema);
|
||||
return true;
|
||||
} catch (\Throwable $e) {
|
||||
return false;
|
||||
}
|
||||
$body = $this->featureContext->getJsonDecodedResponseBodyContent($response);
|
||||
return \is_object($body)
|
||||
&& !empty(\array_intersect($extractionFacets, \array_keys((array) $body)));
|
||||
}
|
||||
);
|
||||
|
||||
$this->featureContext->setResponse($response);
|
||||
$this->featureContext->assertJsonDocumentMatchesSchema(
|
||||
$this->featureContext->getJsonDecodedResponseBodyContent($response),
|
||||
$schema
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
Feature: listing the content of a public link via the Graph API
|
||||
As an anonymous visitor of a public link
|
||||
I want to list the shared folder through the Graph API
|
||||
So that clients can browse public links without WebDAV
|
||||
|
||||
Background:
|
||||
Given user "Alice" has been created with default attributes
|
||||
And user "Alice" has created folder "publicfolder"
|
||||
And user "Alice" has created folder "publicfolder/sub"
|
||||
And user "Alice" has uploaded file with content "hello public" to "publicfolder/a.txt"
|
||||
And user "Alice" has uploaded file with content "nested" to "publicfolder/sub/b.txt"
|
||||
And user "Alice" has uploaded file with content "not shared" to "private.txt"
|
||||
And user "Alice" has created the following resource link share:
|
||||
| resource | publicfolder |
|
||||
| space | Personal |
|
||||
| permissionsRole | view |
|
||||
| password | %public% |
|
||||
|
||||
|
||||
Scenario: the public lists the children of a public link
|
||||
When the public lists the children of the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["value"],
|
||||
"properties": {
|
||||
"value": {
|
||||
"type": "array",
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["name", "folder"],
|
||||
"properties": {
|
||||
"name": { "const": "sub" }
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["name", "file", "size"],
|
||||
"properties": {
|
||||
"name": { "const": "a.txt" },
|
||||
"size": { "const": 12 }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
Scenario: the public lists a subfolder of a public link by path
|
||||
When the public lists the children of path "sub" of the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["value"],
|
||||
"properties": {
|
||||
"value": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"maxItems": 1,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["name", "file"],
|
||||
"properties": {
|
||||
"name": { "const": "b.txt" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
Scenario: the public expands the children of the public link root
|
||||
When the public gets the root of the last created public link expanding its children with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["folder", "children"],
|
||||
"properties": {
|
||||
"children": {
|
||||
"type": "array",
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["name"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
Scenario: the public gets a file through an item anchored colon path
|
||||
When the public gets the drive item "b.txt" below the child "sub" of the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["name", "file", "parentReference"],
|
||||
"properties": {
|
||||
"name": { "const": "b.txt" },
|
||||
"parentReference": {
|
||||
"type": "object",
|
||||
"required": ["path"],
|
||||
"properties": {
|
||||
"path": { "const": "/sub" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
Scenario: the public must not see the owner's paths above the share
|
||||
Given user "Alice" has created folder "deep"
|
||||
And user "Alice" has created folder "deep/shared"
|
||||
And user "Alice" has uploaded file with content "x" to "deep/shared/c.txt"
|
||||
And user "Alice" has created the following resource link share:
|
||||
| resource | deep/shared |
|
||||
| space | Personal |
|
||||
| permissionsRole | view |
|
||||
| password | %public% |
|
||||
When the public lists the children of the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["value"],
|
||||
"properties": {
|
||||
"value": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"maxItems": 1,
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"parentReference": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {
|
||||
"not": { "pattern": "deep" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
Scenario: advertised permissions inside a public link stay within the link role
|
||||
When the public gets the drive item of path "sub" of the last created public link selecting the allowed actions with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["@libre.graph.permissions.actions.allowedValues"],
|
||||
"properties": {
|
||||
"@libre.graph.permissions.actions.allowedValues": {
|
||||
"type": "array",
|
||||
"minItems": 6,
|
||||
"maxItems": 6,
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"type": "string",
|
||||
"not": { "pattern": "/(delete|create|update|deny)$" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
Scenario: listing a password protected public link without the password reports that a password is required
|
||||
When the public lists the children of the last created public link using the Graph API
|
||||
Then the HTTP status code should be "401"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["error"],
|
||||
"properties": {
|
||||
"error": {
|
||||
"type": "object",
|
||||
"required": ["code"],
|
||||
"properties": {
|
||||
"code": { "const": "publicLinkPasswordRequired" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
Scenario: listing a password protected public link with a wrong password reports an invalid password
|
||||
When the public lists the children of the last created public link with password "wrong" using the Graph API
|
||||
Then the HTTP status code should be "401"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["error"],
|
||||
"properties": {
|
||||
"error": {
|
||||
"type": "object",
|
||||
"required": ["code"],
|
||||
"properties": {
|
||||
"code": { "const": "publicLinkPasswordInvalid" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
Scenario: an editable public link grants an upload session
|
||||
Given user "Alice" has created the following resource link share:
|
||||
| resource | publicfolder |
|
||||
| space | Personal |
|
||||
| permissionsRole | edit |
|
||||
| password | %public% |
|
||||
When the public creates an upload session for "up.txt" in the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["UploadURL"],
|
||||
"properties": {
|
||||
"UploadURL": {
|
||||
"type": "string",
|
||||
"pattern": "/data/"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
Scenario: a view only public link does not grant an upload session
|
||||
When the public creates an upload session for "up.txt" in the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "404"
|
||||
|
||||
|
||||
Scenario: a resource outside the public link is not readable through its token
|
||||
When the public tries to get the resource "private.txt" of user "Alice" through the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "404"
|
||||
|
||||
|
||||
Scenario: a public token cannot list drives
|
||||
When the public tries to list the drives using the token of the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "401"
|
||||
|
||||
|
||||
Scenario: a public token cannot read the owner's personal drive
|
||||
When the public tries to get the personal drive of user "Alice" through the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "401"
|
||||
|
||||
|
||||
Scenario: an in-share item is not addressable through its real drive id
|
||||
When the public tries to get the child "sub" of the last created public link through its real drive id with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "401"
|
||||
|
||||
|
||||
Scenario: a token does not open another link's drive
|
||||
When the public tries to list the children of a foreign public link drive using the last created token with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "401"
|
||||
|
||||
|
||||
Scenario: collaborative share types are not disclosed to the public
|
||||
Given user "Brian" has been created with default attributes
|
||||
And user "Alice" has sent the following resource share invitation:
|
||||
| resource | publicfolder/sub |
|
||||
| space | Personal |
|
||||
| sharee | Brian |
|
||||
| shareType | user |
|
||||
| permissionsRole | Viewer |
|
||||
When the public lists the children of the last created public link selecting the share types with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["value"],
|
||||
"properties": {
|
||||
"value": {
|
||||
"type": "array",
|
||||
"minItems": 2,
|
||||
"maxItems": 2,
|
||||
"uniqueItems": true,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"not": { "required": ["@libre.graph.shareTypes"] }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
Scenario Outline: writes through the public link surface are rejected
|
||||
Given user "Alice" has created the following resource link share:
|
||||
| resource | publicfolder |
|
||||
| space | Personal |
|
||||
| permissionsRole | edit |
|
||||
| password | %public% |
|
||||
When the public tries to <action> the child "a.txt" of the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "<code>"
|
||||
And as "Alice" file "publicfolder/a.txt" should exist
|
||||
|
||||
Examples:
|
||||
| action | code |
|
||||
| create a link for | 404 |
|
||||
| delete | 400 |
|
||||
| rename | 400 |
|
||||
| list the permissions of | 404 |
|
||||
|
||||
|
||||
Scenario: the public sees who shared the link on the drive
|
||||
When the public gets the drive of the last created public link with password "%public%" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["driveType", "owner"],
|
||||
"properties": {
|
||||
"driveType": { "const": "mountpoint" },
|
||||
"owner": {
|
||||
"type": "object",
|
||||
"required": ["user"],
|
||||
"properties": {
|
||||
"user": {
|
||||
"type": "object",
|
||||
"required": ["id", "displayName"],
|
||||
"properties": {
|
||||
"displayName": { "const": "Alice Hansen" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
@@ -16,7 +16,7 @@ Feature: content search
|
||||
And user "Alice" has uploaded file with content "namaste from nepal" to "hello.txt"
|
||||
When user "Alice" searches for "Content:hello" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| keywordAtStart.txt |
|
||||
| keywordAtMiddle.txt |
|
||||
| keywordAtLast.txt |
|
||||
@@ -34,15 +34,15 @@ Feature: content search
|
||||
And user "Alice" has uploaded file with content "alan@example.org want to say hello" to "findByEmail.docs"
|
||||
When user "Alice" searches for "Content:k6" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| wordWithNumber.md |
|
||||
When user "Alice" searches for "Content:https://opencloud.eu/" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| findByWebSite.txt |
|
||||
When user "Alice" searches for "Content:alan@" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| findByEmail.docs |
|
||||
Examples:
|
||||
| dav-path-version |
|
||||
@@ -71,11 +71,11 @@ Feature: content search
|
||||
And user "Alice" has uploaded file with content "He has expirience, we must to have, I have to find ...." to "fileWithStopWords.txt"
|
||||
When user "Alice" searches for 'Content:"he has"' using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| fileWithStopWords.txt |
|
||||
When user "Alice" searches for 'Content:"I have"' using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| fileWithStopWords.txt |
|
||||
Examples:
|
||||
| dav-path-version |
|
||||
@@ -101,7 +101,7 @@ Feature: content search
|
||||
And user "Brian" has a share "uploadFolder" synced
|
||||
When user "Brian" searches for "Content:hello" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Brian" should eventually contain only these files:
|
||||
And the search result of user "Brian" should contain only these files:
|
||||
| keywordAtStart.txt |
|
||||
| keywordAtMiddle.txt |
|
||||
| keywordAtLast.txt |
|
||||
@@ -121,7 +121,7 @@ Feature: content search
|
||||
And user "Alice" has deleted file "keywordAtLast.txt"
|
||||
When user "Alice" searches for "Content:hello" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| keywordAtStart.txt |
|
||||
| keywordAtMiddle.txt |
|
||||
Examples:
|
||||
@@ -139,7 +139,7 @@ Feature: content search
|
||||
And user "Alice" has restored the file with original path "keywordAtStart.txt"
|
||||
When user "Alice" searches for "Content:hello" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| keywordAtStart.txt |
|
||||
Examples:
|
||||
| dav-path-version |
|
||||
@@ -154,7 +154,7 @@ Feature: content search
|
||||
And user "Alice" has restored version index "1" of file "test.txt"
|
||||
When user "Alice" searches for "Content:hello" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| test.txt |
|
||||
Examples:
|
||||
| dav-path-version |
|
||||
@@ -175,7 +175,7 @@ Feature: content search
|
||||
And using <dav-path-version> DAV path
|
||||
When user "Alice" searches for "Content:hello" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| keywordAtStart.txt |
|
||||
| keywordAtMiddle.txt |
|
||||
| keywordAtLast.txt |
|
||||
@@ -204,7 +204,7 @@ Feature: content search
|
||||
And using <dav-path-version> DAV path
|
||||
When user "Brian" searches for "Content:hello" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain only these files:
|
||||
And the search result of user "Alice" should contain only these files:
|
||||
| keywordAtStart.txt |
|
||||
| keywordAtMiddle.txt |
|
||||
| keywordAtLast.txt |
|
||||
@@ -224,10 +224,10 @@ Feature: content search
|
||||
| technical task.txt | test |
|
||||
When user "Alice" searches for '<pattern>' using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result of user "Alice" should eventually contain these entries:
|
||||
And the search result should contain "<result-count>" entries
|
||||
And the search result of user "Alice" should contain these entries:
|
||||
| <search-result-1> |
|
||||
| <search-result-2> |
|
||||
And the search result should contain "<result-count>" entries
|
||||
Examples:
|
||||
| pattern | result-count | search-result-1 | search-result-2 |
|
||||
| Content:hello | 1 | technical task.txt | |
|
||||
@@ -251,7 +251,7 @@ Feature: content search
|
||||
And user "Alice" has uploaded a file inside space "project-space" with content "this is a simple odt file" to "test-odt-file.odt"
|
||||
When user "Alice" searches for "Content:simple" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result should eventually contain these entries with highlight on keyword "simple"
|
||||
And the search result should contain these entries with highlight on keyword "simple"
|
||||
| test-text-file.txt |
|
||||
| test-pdf-file.pdf |
|
||||
| test-cpp-file.cpp |
|
||||
|
||||
@@ -131,7 +131,7 @@ Feature: propfind extracted props
|
||||
Given user "Alice" has uploaded a file "filesForUpload/testaudio.mp3" to "testaudio.mp3" in space "Personal"
|
||||
When user "Alice" gets the file "testaudio.mp3" from space "Personal" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should eventually match
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
@@ -179,7 +179,7 @@ Feature: propfind extracted props
|
||||
Given user "Alice" has uploaded a file "filesForUpload/testavatar.jpg" to "testavatar.jpg" in space "Personal"
|
||||
When user "Alice" gets the file "testavatar.jpg" from space "Personal" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should eventually match
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
@@ -251,7 +251,7 @@ Feature: propfind extracted props
|
||||
And user "Alice" has uploaded a file "filesForUpload/testaudio.mp3" to "testaudio.mp3" in space "new-space"
|
||||
When user "Alice" gets the file "testaudio.mp3" from space "new-space" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should eventually match
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
@@ -301,7 +301,7 @@ Feature: propfind extracted props
|
||||
And user "Alice" has uploaded a file "filesForUpload/testavatar.jpg" to "testavatar.jpg" in space "new-space"
|
||||
When user "Alice" gets the file "testavatar.jpg" from space "new-space" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should eventually match
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
@@ -379,7 +379,7 @@ Feature: propfind extracted props
|
||||
And user "Brian" has a share "testaudio.mp3" synced
|
||||
When user "Brian" gets the file "testaudio.mp3" from space "Shares" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should eventually match
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
@@ -435,7 +435,7 @@ Feature: propfind extracted props
|
||||
And user "Brian" has a share "testavatar.jpg" synced
|
||||
When user "Brian" gets the file "testavatar.jpg" from space "Shares" using the Graph API
|
||||
Then the HTTP status code should be "200"
|
||||
And the JSON data of the response should eventually match
|
||||
And the JSON data of the response should match
|
||||
"""
|
||||
{
|
||||
"type": "object",
|
||||
|
||||
Generated
Vendored
+3
-32
@@ -23,7 +23,6 @@ package publicstorageprovider
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
@@ -37,6 +36,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/publicshare"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/status"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
@@ -769,15 +769,7 @@ func (s *service) augmentStatResponse(ctx context.Context, statInfo *provider.Re
|
||||
appctx.GetLogger(ctx).Error().Err(err).Interface("share", share).Interface("info", statInfo).Msg("error when adding share")
|
||||
}
|
||||
|
||||
var sharePath string
|
||||
if shareInfo.Type == provider.ResourceType_RESOURCE_TYPE_FILE {
|
||||
sharePath = path.Base(shareInfo.Path)
|
||||
} else {
|
||||
sharePath = strings.TrimPrefix(statInfo.Path, shareInfo.Path)
|
||||
}
|
||||
|
||||
statInfo.Path = path.Join("/", sharePath)
|
||||
filterPermissions(statInfo.PermissionSet, shareInfo.PermissionSet)
|
||||
publicshare.FilterResourceInfo(statInfo, shareInfo, shareInfo.GetPermissionSet())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -843,7 +835,7 @@ func (s *service) ListContainer(ctx context.Context, req *provider.ListContainer
|
||||
for i := range listContainerR.Infos {
|
||||
// FIXME how do we reduce permissions to what is granted by the public link?
|
||||
// only a problem for id based access -> middleware
|
||||
filterPermissions(listContainerR.Infos[i].PermissionSet, info.PermissionSet)
|
||||
publicshare.FilterPermissions(listContainerR.Infos[i].PermissionSet, info.PermissionSet)
|
||||
if err := addShare(listContainerR.Infos[i], share); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Interface("share", share).Interface("info", listContainerR.Infos[i]).Msg("error when adding share")
|
||||
}
|
||||
@@ -852,27 +844,6 @@ func (s *service) ListContainer(ctx context.Context, req *provider.ListContainer
|
||||
return listContainerR, nil
|
||||
}
|
||||
|
||||
func filterPermissions(l *provider.ResourcePermissions, r *provider.ResourcePermissions) {
|
||||
l.AddGrant = l.AddGrant && r.AddGrant
|
||||
l.CreateContainer = l.CreateContainer && r.CreateContainer
|
||||
l.Delete = l.Delete && r.Delete
|
||||
l.GetPath = l.GetPath && r.GetPath
|
||||
l.GetQuota = l.GetQuota && r.GetQuota
|
||||
l.InitiateFileDownload = l.InitiateFileDownload && r.InitiateFileDownload
|
||||
l.InitiateFileUpload = l.InitiateFileUpload && r.InitiateFileUpload
|
||||
l.ListContainer = l.ListContainer && r.ListContainer
|
||||
l.ListFileVersions = l.ListFileVersions && r.ListFileVersions
|
||||
l.ListGrants = l.ListGrants && r.ListGrants
|
||||
l.ListRecycle = l.ListRecycle && r.ListRecycle
|
||||
l.Move = l.Move && r.Move
|
||||
l.PurgeRecycle = l.PurgeRecycle && r.PurgeRecycle
|
||||
l.RemoveGrant = l.RemoveGrant && r.RemoveGrant
|
||||
l.RestoreFileVersion = l.RestoreFileVersion && r.RestoreFileVersion
|
||||
l.RestoreRecycleItem = l.RestoreRecycleItem && r.RestoreRecycleItem
|
||||
l.Stat = l.Stat && r.Stat
|
||||
l.UpdateGrant = l.UpdateGrant && r.UpdateGrant
|
||||
}
|
||||
|
||||
func (s *service) ListFileVersions(ctx context.Context, req *provider.ListFileVersionsRequest) (*provider.ListFileVersionsResponse, error) {
|
||||
return nil, gstatus.Errorf(codes.Unimplemented, "method not implemented")
|
||||
}
|
||||
|
||||
+19
-1
@@ -20,6 +20,7 @@ package scope
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1"
|
||||
@@ -143,7 +144,7 @@ func publicshareScope(ctx context.Context, scope *authpb.Scope, resource interfa
|
||||
// public links must not leak info about collaborative shares
|
||||
return false, nil
|
||||
case string:
|
||||
return checkResourcePath(v), nil
|
||||
return checkResourcePath(v) || checkGraphDrivesPath(v, share.Token), nil
|
||||
}
|
||||
|
||||
msg := "public resource type assertion failed"
|
||||
@@ -151,6 +152,23 @@ func publicshareScope(ctx context.Context, scope *authpb.Scope, resource interfa
|
||||
return false, errtypes.InternalError(msg)
|
||||
}
|
||||
|
||||
// checkGraphDrivesPath opens the graph drive routes of exactly the link's own
|
||||
// public drive; every other drive stays closed, notably the drives collection
|
||||
// and real space ids. Per CS3 request checks enforce what may be read below it.
|
||||
func checkGraphDrivesPath(p, token string) bool {
|
||||
if token == "" {
|
||||
return false
|
||||
}
|
||||
p = path.Clean(p)
|
||||
drive := PublicStorageProviderID + "$" + PublicStorageProviderID + "!" + token
|
||||
for _, prefix := range []string{"/graph/v1.0/drives/", "/graph/v1beta1/drives/"} {
|
||||
if p == prefix+drive || strings.HasPrefix(p, prefix+drive+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func checkStorageRef(ctx context.Context, s *link.PublicShare, r *provider.Reference) bool {
|
||||
// r: <resource_id:<storage_id:$storageID space_id:$spaceID opaque_id:$opaqueID> path:$path > >
|
||||
if utils.ResourceIDEqual(s.ResourceId, r.GetResourceId()) {
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
// Copyright 2018-2026 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package publicshare
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
)
|
||||
|
||||
// FilterResourceInfo rewrites a resource info for a public link consumer: the
|
||||
// path becomes share-root relative, the permissions are cut to the grant.
|
||||
func FilterResourceInfo(info, shareRoot *provider.ResourceInfo, grant *provider.ResourcePermissions) {
|
||||
if info == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var sharePath string
|
||||
if shareRoot.GetType() == provider.ResourceType_RESOURCE_TYPE_FILE {
|
||||
sharePath = path.Base(shareRoot.GetPath())
|
||||
} else {
|
||||
sharePath = strings.TrimPrefix(info.GetPath(), shareRoot.GetPath())
|
||||
}
|
||||
info.Path = path.Join("/", sharePath)
|
||||
|
||||
if info.PermissionSet != nil {
|
||||
FilterPermissions(info.PermissionSet, grant)
|
||||
}
|
||||
}
|
||||
|
||||
// FilterPermissions reduces l to what r also grants. A nil r clears l.
|
||||
func FilterPermissions(l, r *provider.ResourcePermissions) {
|
||||
l.AddGrant = l.AddGrant && r.GetAddGrant()
|
||||
l.CreateContainer = l.CreateContainer && r.GetCreateContainer()
|
||||
l.Delete = l.Delete && r.GetDelete()
|
||||
l.DenyGrant = l.DenyGrant && r.GetDenyGrant()
|
||||
l.GetPath = l.GetPath && r.GetGetPath()
|
||||
l.GetQuota = l.GetQuota && r.GetGetQuota()
|
||||
l.InitiateFileDownload = l.InitiateFileDownload && r.GetInitiateFileDownload()
|
||||
l.InitiateFileUpload = l.InitiateFileUpload && r.GetInitiateFileUpload()
|
||||
l.ListContainer = l.ListContainer && r.GetListContainer()
|
||||
l.ListFileVersions = l.ListFileVersions && r.GetListFileVersions()
|
||||
l.ListGrants = l.ListGrants && r.GetListGrants()
|
||||
l.ListRecycle = l.ListRecycle && r.GetListRecycle()
|
||||
l.Move = l.Move && r.GetMove()
|
||||
l.PurgeRecycle = l.PurgeRecycle && r.GetPurgeRecycle()
|
||||
l.RemoveGrant = l.RemoveGrant && r.GetRemoveGrant()
|
||||
l.RestoreFileVersion = l.RestoreFileVersion && r.GetRestoreFileVersion()
|
||||
l.RestoreRecycleItem = l.RestoreRecycleItem && r.GetRestoreRecycleItem()
|
||||
l.Stat = l.Stat && r.GetStat()
|
||||
l.UpdateGrant = l.UpdateGrant && r.GetUpdateGrant()
|
||||
}
|
||||
Reference in new issue
Block a user