From 3099d4a821415ef476b640dd90473086c67e0099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 8 Apr 2022 10:49:29 +0200 Subject: [PATCH 01/49] Add an initial version of a search service. It's still incomplete and isn't working yet. --- .../proto/ocis/messages/search/search.proto | 16 + .../proto/ocis/services/search/search.proto | 84 ++++ search/pkg/command/health.go | 53 +++ search/pkg/command/root.go | 64 +++ search/pkg/command/server.go | 93 ++++ search/pkg/command/version.go | 50 +++ search/pkg/config/config.go | 25 ++ search/pkg/config/debug.go | 9 + search/pkg/config/defaults/defaultconfig.go | 62 +++ search/pkg/config/http.go | 8 + search/pkg/config/log.go | 9 + search/pkg/config/parser/parse.go | 33 ++ search/pkg/config/reva.go | 11 + search/pkg/config/service.go | 6 + search/pkg/config/tracing.go | 9 + search/pkg/logging/logging.go | 17 + search/pkg/metrics/metrics.go | 33 ++ search/pkg/search/index/index.go | 122 +++++ search/pkg/search/index/index_suite_test.go | 13 + search/pkg/search/index/index_test.go | 123 ++++++ search/pkg/search/index/mocks/BleveIndex.go | 415 ++++++++++++++++++ search/pkg/search/mocks/IndexClient.go | 38 ++ search/pkg/search/mocks/ProviderClient.go | 38 ++ .../search/provider/provider_suite_test.go | 31 ++ search/pkg/search/provider/searchprovider.go | 99 +++++ .../search/provider/searchprovider_test.go | 258 +++++++++++ search/pkg/search/search.go | 61 +++ search/pkg/search/search_suite_test.go | 31 ++ search/pkg/server/debug/option.go | 50 +++ search/pkg/server/debug/server.go | 63 +++ search/pkg/server/grpc/option.go | 85 ++++ search/pkg/server/grpc/server.go | 36 ++ search/pkg/service/v0/option.go | 57 +++ search/pkg/service/v0/service.go | 71 +++ search/pkg/tracing/tracing.go | 23 + 35 files changed, 2196 insertions(+) create mode 100644 protogen/proto/ocis/messages/search/search.proto create mode 100644 protogen/proto/ocis/services/search/search.proto create mode 100644 search/pkg/command/health.go create mode 100644 search/pkg/command/root.go create mode 100644 search/pkg/command/server.go create mode 100644 search/pkg/command/version.go create mode 100644 search/pkg/config/config.go create mode 100644 search/pkg/config/debug.go create mode 100644 search/pkg/config/defaults/defaultconfig.go create mode 100644 search/pkg/config/http.go create mode 100644 search/pkg/config/log.go create mode 100644 search/pkg/config/parser/parse.go create mode 100644 search/pkg/config/reva.go create mode 100644 search/pkg/config/service.go create mode 100644 search/pkg/config/tracing.go create mode 100644 search/pkg/logging/logging.go create mode 100644 search/pkg/metrics/metrics.go create mode 100644 search/pkg/search/index/index.go create mode 100644 search/pkg/search/index/index_suite_test.go create mode 100644 search/pkg/search/index/index_test.go create mode 100644 search/pkg/search/index/mocks/BleveIndex.go create mode 100644 search/pkg/search/mocks/IndexClient.go create mode 100644 search/pkg/search/mocks/ProviderClient.go create mode 100644 search/pkg/search/provider/provider_suite_test.go create mode 100644 search/pkg/search/provider/searchprovider.go create mode 100644 search/pkg/search/provider/searchprovider_test.go create mode 100644 search/pkg/search/search.go create mode 100644 search/pkg/search/search_suite_test.go create mode 100644 search/pkg/server/debug/option.go create mode 100644 search/pkg/server/debug/server.go create mode 100644 search/pkg/server/grpc/option.go create mode 100644 search/pkg/server/grpc/server.go create mode 100644 search/pkg/service/v0/option.go create mode 100644 search/pkg/service/v0/service.go create mode 100644 search/pkg/tracing/tracing.go diff --git a/protogen/proto/ocis/messages/search/search.proto b/protogen/proto/ocis/messages/search/search.proto new file mode 100644 index 000000000..948e5fd2a --- /dev/null +++ b/protogen/proto/ocis/messages/search/search.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package ocis.messages.search.v0; + +option go_package = "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0"; + +message Match { + // key of the recorda + string key = 1; + // value in the record + bytes value = 2; + // time.Duration (signed int64 nanoseconds) + int64 expiry = 3; + // the associated metadata + map metadata = 4; +} \ No newline at end of file diff --git a/protogen/proto/ocis/services/search/search.proto b/protogen/proto/ocis/services/search/search.proto new file mode 100644 index 000000000..7f95a226d --- /dev/null +++ b/protogen/proto/ocis/services/search/search.proto @@ -0,0 +1,84 @@ +syntax = "proto3"; + +package ocis.services.search.v0; + +option go_package = "github.com/owncloud/ocis/protogen/gen/ocis/service/search/v0"; + +import "ocis/messages/search/v0/search.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; +import "cs3/storage/provider/v1beta1/resources.proto"; + +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + title: "ownCloud Infinite Scale search"; + version: "1.0.0"; + contact: { + name: "ownCloud GmbH"; + url: "https://github.com/owncloud/ocis"; + email: "support@owncloud.com"; + }; + license: { + name: "Apache-2.0"; + url: "https://github.com/owncloud/ocis/blob/master/LICENSE"; + }; + }; + schemes: HTTP; + schemes: HTTPS; + consumes: "application/json"; + produces: "application/json"; + external_docs: { + description: "Developer Manual"; + url: "https://owncloud.dev/extensions/search/"; + }; +}; + +service SearchProvider { + rpc Search(SearchRequest) returns (SearchResponse) {}; +} + +service IndexProvider { + rpc Search(SearchIndexRequest) returns (SearchIndexResponse) {}; + rpc Index(IndexRequest) returns (IndexResponse) {}; + rpc Remove(RemoveRequest) returns (RemoveResponse) {}; +} + +message SearchRequest { + // Optional. The maximum number of entries to return in the response + int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to `Get` + // that indicates from where search should continue + string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Used to specify a subset of fields that should be + // returned by a get operation or modified by an update operation. + google.protobuf.FieldMask field_mask = 3; + string query = 4; +} + +message SearchResponse { + repeated ocis.messages.search.v0.Match matches = 1; + + // Token to retrieve the next page of results, or empty if there are no + // more results in the list + string next_page_token = 2; +} + +message SearchIndexRequest { + // Optional. The maximum number of entries to return in the response + int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to `Get` + // that indicates from where search should continue + string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Used to specify a subset of fields that should be + // returned by a get operation or modified by an update operation. + google.protobuf.FieldMask field_mask = 3; + string query = 4; + string +} + +message SearchIndexResponse { + repeated ocis.messages.search.v0.Record records = 1; +} diff --git a/search/pkg/command/health.go b/search/pkg/command/health.go new file mode 100644 index 000000000..d0389cc9c --- /dev/null +++ b/search/pkg/command/health.go @@ -0,0 +1,53 @@ +package command + +import ( + "fmt" + "net/http" + + "github.com/owncloud/ocis/search/pkg/config" + "github.com/owncloud/ocis/search/pkg/config/parser" + "github.com/owncloud/ocis/search/pkg/logging" + "github.com/urfave/cli/v2" +) + +// Health is the entrypoint for the health command. +func Health(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "health", + Usage: "check health status", + Category: "info", + Before: func(c *cli.Context) error { + return parser.ParseConfig(cfg) + }, + Action: func(c *cli.Context) error { + logger := logging.Configure(cfg.Service.Name, cfg.Log) + + resp, err := http.Get( + fmt.Sprintf( + "http://%s/healthz", + cfg.Debug.Addr, + ), + ) + + if err != nil { + logger.Fatal(). + Err(err). + Msg("Failed to request health check") + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + logger.Fatal(). + Int("code", resp.StatusCode). + Msg("Health seems to be in bad state") + } + + logger.Debug(). + Int("code", resp.StatusCode). + Msg("Health got a good state") + + return nil + }, + } +} diff --git a/search/pkg/command/root.go b/search/pkg/command/root.go new file mode 100644 index 000000000..f4578fb2d --- /dev/null +++ b/search/pkg/command/root.go @@ -0,0 +1,64 @@ +package command + +import ( + "context" + "os" + + "github.com/owncloud/ocis/ocis-pkg/clihelper" + "github.com/thejerf/suture/v4" + + ociscfg "github.com/owncloud/ocis/ocis-pkg/config" + "github.com/owncloud/ocis/search/pkg/config" + "github.com/urfave/cli/v2" +) + +// GetCommands provides all commands for this service +func GetCommands(cfg *config.Config) cli.Commands { + return []*cli.Command{ + // start this service + Server(cfg), + + // interaction with this service + + // infos about this service + Health(cfg), + Version(cfg), + } +} + +// Execute is the entry point for the ocis-search command. +func Execute(cfg *config.Config) error { + app := clihelper.DefaultApp(&cli.App{ + Name: "ocis-search", + Usage: "Serve search API for oCIS", + Commands: GetCommands(cfg), + }) + cli.HelpFlag = &cli.BoolFlag{ + Name: "help,h", + Usage: "Show the help", + } + + return app.Run(os.Args) +} + +// SutureService allows for the search command to be embedded and supervised by a suture supervisor tree. +type SutureService struct { + cfg *config.Config +} + +// NewSutureService creates a new search.SutureService +func NewSutureService(cfg *ociscfg.Config) suture.Service { + cfg.Search.Commons = cfg.Commons + return SutureService{ + cfg: cfg.Search, + } +} + +func (s SutureService) Serve(ctx context.Context) error { + s.cfg.Context = ctx + if err := Execute(s.cfg); err != nil { + return err + } + + return nil +} diff --git a/search/pkg/command/server.go b/search/pkg/command/server.go new file mode 100644 index 000000000..cff21ee66 --- /dev/null +++ b/search/pkg/command/server.go @@ -0,0 +1,93 @@ +package command + +import ( + "context" + "fmt" + + "github.com/oklog/run" + "github.com/owncloud/ocis/idp/pkg/server/http" + "github.com/owncloud/ocis/ocis-pkg/version" + "github.com/owncloud/ocis/search/pkg/config" + "github.com/owncloud/ocis/search/pkg/config/parser" + "github.com/owncloud/ocis/search/pkg/logging" + "github.com/owncloud/ocis/search/pkg/metrics" + "github.com/owncloud/ocis/search/pkg/server/debug" + "github.com/owncloud/ocis/search/pkg/tracing" + "github.com/urfave/cli/v2" +) + +// Server is the entrypoint for the server command. +func Server(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "server", + Usage: fmt.Sprintf("start %s extension without runtime (unsupervised mode)", cfg.Service.Name), + Category: "server", + Before: func(c *cli.Context) error { + return parser.ParseConfig(cfg) + }, + Action: func(c *cli.Context) error { + logger := logging.Configure(cfg.Service.Name, cfg.Log) + err := tracing.Configure(cfg) + if err != nil { + return err + } + + gr := run.Group{} + ctx, cancel := func() (context.Context, context.CancelFunc) { + if cfg.Context == nil { + return context.WithCancel(context.Background()) + } + return context.WithCancel(cfg.Context) + }() + mtrcs := metrics.New() + + defer cancel() + + mtrcs.BuildInfo.WithLabelValues(version.String).Set(1) + + { + server, err := http.Server( + http.Logger(logger), + http.Context(ctx), + http.Config(cfg), + http.Metrics(mtrcs), + ) + + if err != nil { + logger.Info().Err(err).Str("transport", "http").Msg("Failed to initialize server") + return err + } + + gr.Add(func() error { + return server.Run() + }, func(_ error) { + logger.Info(). + Str("transport", "http"). + Msg("Shutting down server") + + cancel() + }) + } + + { + server, err := debug.Server( + debug.Logger(logger), + debug.Context(ctx), + debug.Config(cfg), + ) + + if err != nil { + logger.Info().Err(err).Str("transport", "debug").Msg("Failed to initialize server") + return err + } + + gr.Add(server.ListenAndServe, func(_ error) { + _ = server.Shutdown(ctx) + cancel() + }) + } + + return gr.Run() + }, + } +} diff --git a/search/pkg/command/version.go b/search/pkg/command/version.go new file mode 100644 index 000000000..07a4959c8 --- /dev/null +++ b/search/pkg/command/version.go @@ -0,0 +1,50 @@ +package command + +import ( + "fmt" + "os" + + "github.com/owncloud/ocis/ocis-pkg/registry" + "github.com/owncloud/ocis/ocis-pkg/version" + + tw "github.com/olekukonko/tablewriter" + "github.com/owncloud/ocis/search/pkg/config" + "github.com/urfave/cli/v2" +) + +// Version prints the service versions of all running instances. +func Version(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "version", + Usage: "print the version of this binary and the running extension instances", + Category: "info", + Action: func(c *cli.Context) error { + fmt.Println("Version: " + version.String) + fmt.Printf("Compiled: %s\n", version.Compiled()) + fmt.Println("") + + reg := registry.GetRegistry() + services, err := reg.GetService(cfg.HTTP.Namespace + "." + cfg.Service.Name) + if err != nil { + fmt.Println(fmt.Errorf("could not get %s services from the registry: %v", cfg.Service.Name, err)) + return err + } + + if len(services) == 0 { + fmt.Println("No running " + cfg.Service.Name + " service found.") + return nil + } + + table := tw.NewWriter(os.Stdout) + table.SetHeader([]string{"Version", "Address", "Id"}) + table.SetAutoFormatHeaders(false) + for _, s := range services { + for _, n := range s.Nodes { + table.Append([]string{s.Version, n.Address, n.Id}) + } + } + table.Render() + return nil + }, + } +} diff --git a/search/pkg/config/config.go b/search/pkg/config/config.go new file mode 100644 index 000000000..2cc26d055 --- /dev/null +++ b/search/pkg/config/config.go @@ -0,0 +1,25 @@ +package config + +import ( + "context" + + "github.com/owncloud/ocis/ocis-pkg/shared" +) + +// Config combines all available configuration parts. +type Config struct { + *shared.Commons `ocisConfig:"-" yaml:"-"` + + Service Service `ocisConfig:"-" yaml:"-"` + + Tracing *Tracing `ocisConfig:"tracing"` + Log *Log `ocisConfig:"log"` + Debug Debug `ocisConfig:"debug"` + + HTTP HTTP `ocisConfig:"http"` + + Reva Reva `ocisConfig:"reva"` + TokenManager TokenManager `ocisConfig:"token_manager"` + + Context context.Context `ocisConfig:"-" yaml:"-"` +} diff --git a/search/pkg/config/debug.go b/search/pkg/config/debug.go new file mode 100644 index 000000000..a6ab80cbd --- /dev/null +++ b/search/pkg/config/debug.go @@ -0,0 +1,9 @@ +package config + +// Debug defines the available debug configuration. +type Debug struct { + Addr string `ocisConfig:"addr" env:"SEARCH_DEBUG_ADDR"` + Token string `ocisConfig:"token" env:"SEARCH_DEBUG_TOKEN"` + Pprof bool `ocisConfig:"pprof" env:"SEARCH_DEBUG_PPROF"` + Zpages bool `ocisConfig:"zpages" env:"SEARCH_DEBUG_ZPAGES"` +} diff --git a/search/pkg/config/defaults/defaultconfig.go b/search/pkg/config/defaults/defaultconfig.go new file mode 100644 index 000000000..8fc349789 --- /dev/null +++ b/search/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,62 @@ +package defaults + +import ( + "strings" + + "github.com/owncloud/ocis/search/pkg/config" +) + +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9124", + Token: "", + }, + HTTP: config.HTTP{ + Addr: "127.0.0.1:9120", + Namespace: "com.owncloud.search", + Root: "/search", + }, + Service: config.Service{ + Name: "search", + }, + Reva: config.Reva{ + Address: "127.0.0.1:9142", + }, + TokenManager: config.TokenManager{ + JWTSecret: "Pive-Fumkiu4", + }, + } +} + +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for BindEnv. + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + // provide with defaults for shared tracing, since we need a valid destination address for BindEnv. + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +func Sanitize(cfg *config.Config) { + // sanitize config + if cfg.HTTP.Root != "/" { + cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") + } +} diff --git a/search/pkg/config/http.go b/search/pkg/config/http.go new file mode 100644 index 000000000..018f8c551 --- /dev/null +++ b/search/pkg/config/http.go @@ -0,0 +1,8 @@ +package config + +// HTTP defines the available http configuration. +type HTTP struct { + Addr string `ocisConfig:"addr" env:"SEARCH_HTTP_ADDR"` + Namespace string `ocisConfig:"-" yaml:"-"` + Root string `ocisConfig:"root" env:"SEARCH_HTTP_ROOT"` +} diff --git a/search/pkg/config/log.go b/search/pkg/config/log.go new file mode 100644 index 000000000..6e4d2fa93 --- /dev/null +++ b/search/pkg/config/log.go @@ -0,0 +1,9 @@ +package config + +// Log defines the available log configuration. +type Log struct { + Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;SEARCH_LOG_LEVEL"` + Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;SEARCH_LOG_PRETTY"` + Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;SEARCH_LOG_COLOR"` + File string `mapstructure:"file" env:"OCIS_LOG_FILE;SEARCH_LOG_FILE"` +} diff --git a/search/pkg/config/parser/parse.go b/search/pkg/config/parser/parse.go new file mode 100644 index 000000000..b467524d7 --- /dev/null +++ b/search/pkg/config/parser/parse.go @@ -0,0 +1,33 @@ +package parser + +import ( + "errors" + + ociscfg "github.com/owncloud/ocis/ocis-pkg/config" + "github.com/owncloud/ocis/search/pkg/config" + "github.com/owncloud/ocis/search/pkg/config/defaults" + + "github.com/owncloud/ocis/ocis-pkg/config/envdecode" +) + +// ParseConfig loads accounts configuration from known paths. +func ParseConfig(cfg *config.Config) error { + _, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg) + if err != nil { + return err + } + + defaults.EnsureDefaults(cfg) + + // load all env variables relevant to the config in the current context. + if err := envdecode.Decode(cfg); err != nil { + // no environment variable set for this config is an expected "error" + if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) { + return err + } + } + + defaults.Sanitize(cfg) + + return nil +} diff --git a/search/pkg/config/reva.go b/search/pkg/config/reva.go new file mode 100644 index 000000000..68bfd2f62 --- /dev/null +++ b/search/pkg/config/reva.go @@ -0,0 +1,11 @@ +package config + +// Reva defines all available REVA configuration. +type Reva struct { + Address string `ocisConfig:"address" env:"REVA_GATEWAY"` +} + +// TokenManager is the config for using the reva token manager +type TokenManager struct { + JWTSecret string `ocisConfig:"jwt_secret" env:"OCIS_JWT_SECRET;SEARCH_JWT_SECRET"` +} diff --git a/search/pkg/config/service.go b/search/pkg/config/service.go new file mode 100644 index 000000000..c019b7304 --- /dev/null +++ b/search/pkg/config/service.go @@ -0,0 +1,6 @@ +package config + +// Service defines the available service configuration. +type Service struct { + Name string `ocisConfig:"-" yaml:"-"` +} diff --git a/search/pkg/config/tracing.go b/search/pkg/config/tracing.go new file mode 100644 index 000000000..50c49234d --- /dev/null +++ b/search/pkg/config/tracing.go @@ -0,0 +1,9 @@ +package config + +// Tracing defines the available tracing configuration. +type Tracing struct { + Enabled bool `ocisConfig:"enabled" env:"OCIS_TRACING_ENABLED;SEARCH_TRACING_ENABLED"` + Type string `ocisConfig:"type" env:"OCIS_TRACING_TYPE;SEARCH_TRACING_TYPE"` + Endpoint string `ocisConfig:"endpoint" env:"OCIS_TRACING_ENDPOINT;SEARCH_TRACING_ENDPOINT"` + Collector string `ocisConfig:"collector" env:"OCIS_TRACING_COLLECTOR;SEARCH_TRACING_COLLECTOR"` +} diff --git a/search/pkg/logging/logging.go b/search/pkg/logging/logging.go new file mode 100644 index 000000000..eeb49456f --- /dev/null +++ b/search/pkg/logging/logging.go @@ -0,0 +1,17 @@ +package logging + +import ( + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/search/pkg/config" +) + +// LoggerFromConfig initializes a service-specific logger instance. +func Configure(name string, cfg *config.Log) log.Logger { + return log.NewLogger( + log.Name(name), + log.Level(cfg.Level), + log.Pretty(cfg.Pretty), + log.Color(cfg.Color), + log.File(cfg.File), + ) +} diff --git a/search/pkg/metrics/metrics.go b/search/pkg/metrics/metrics.go new file mode 100644 index 000000000..c3076bb01 --- /dev/null +++ b/search/pkg/metrics/metrics.go @@ -0,0 +1,33 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +var ( + // Namespace defines the namespace for the defines metrics. + Namespace = "ocis" + + // Subsystem defines the subsystem for the defines metrics. + Subsystem = "search" +) + +// Metrics defines the available metrics of this service. +type Metrics struct { + // Counter *prometheus.CounterVec + BuildInfo *prometheus.GaugeVec +} + +// New initializes the available metrics. +func New() *Metrics { + m := &Metrics{ + BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "build_info", + Help: "Build information", + }, []string{"version"}), + } + + _ = prometheus.Register(m.BuildInfo) + // TODO: implement metrics + return m +} diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go new file mode 100644 index 000000000..78f01ec26 --- /dev/null +++ b/search/pkg/search/index/index.go @@ -0,0 +1,122 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package index + +import ( + "context" + "strings" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/owncloud/ocis/search/pkg/search" + + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +type Index struct { + bleveIndex bleve.Index +} + +type Entity struct { + RootID string + ID string + + Name string + Size uint64 +} + +func NewPersisted(path string) (*Index, error) { + bi, err := bleve.New(path, BuildMapping()) + if err != nil { + return nil, err + } + return &Index{ + bleveIndex: bi, + }, nil +} + +func New(bleveIndex bleve.Index) (*Index, error) { + return &Index{ + bleveIndex: bleveIndex, + }, nil +} + +func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error { + entity := toEntity(ref, ri) + return i.bleveIndex.Index(entity.ID, entity) +} + +func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*search.SearchIndexResult, error) { + bleveReq := bleve.NewSearchRequest(bleve.NewMatchQuery(req.Query)) + bleveReq.Fields = []string{"*"} + res, err := i.bleveIndex.Search(bleveReq) + if err != nil { + return nil, err + } + + matches := []search.Match{} + for _, h := range res.Hits { + match, err := fromFields(h.Fields) + if err != nil { + return nil, err + } + matches = append(matches, match) + } + + return &search.SearchIndexResult{ + Matches: matches, + }, nil +} + +func BuildMapping() mapping.IndexMapping { + indexMapping := bleve.NewIndexMapping() + indexMapping.DefaultAnalyzer = keyword.Name + return indexMapping +} + +func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *Entity { + return &Entity{ + RootID: ref.ResourceId.GetStorageId() + ":" + ref.ResourceId.GetOpaqueId(), + ID: ri.Id.GetStorageId() + ":" + ri.Id.GetOpaqueId(), + Name: ri.Path, + Size: ri.Size, + } +} + +func fromFields(fields map[string]interface{}) (search.Match, error) { + rootIDParts := strings.SplitN(fields["RootID"].(string), ":", 2) + IDParts := strings.SplitN(fields["ID"].(string), ":", 2) + + return search.Match{ + Reference: &sprovider.Reference{ + ResourceId: &sprovider.ResourceId{ + StorageId: rootIDParts[0], + OpaqueId: rootIDParts[1], + }, + }, + Info: &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: IDParts[0], + OpaqueId: IDParts[1], + }, + Path: fields["Name"].(string), + }, + }, nil +} diff --git a/search/pkg/search/index/index_suite_test.go b/search/pkg/search/index/index_suite_test.go new file mode 100644 index 000000000..09099db1a --- /dev/null +++ b/search/pkg/search/index/index_suite_test.go @@ -0,0 +1,13 @@ +package index_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestIndex(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Index Suite") +} diff --git a/search/pkg/search/index/index_test.go b/search/pkg/search/index/index_test.go new file mode 100644 index 000000000..5887c58a6 --- /dev/null +++ b/search/pkg/search/index/index_test.go @@ -0,0 +1,123 @@ +package index_test + +import ( + "context" + + "github.com/blevesearch/bleve/v2" + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/owncloud/ocis/search/pkg/search" + "github.com/owncloud/ocis/search/pkg/search/index" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Index", func() { + var ( + i *index.Index + bleveIndex bleve.Index + ref *sprovider.Reference + ri *sprovider.ResourceInfo + + ctx context.Context + ) + + BeforeEach(func() { + var err error + bleveIndex, err = bleve.NewMemOnly(index.BuildMapping()) + Expect(err).ToNot(HaveOccurred()) + + i, err = index.New(bleveIndex) + Expect(err).ToNot(HaveOccurred()) + + ref = &sprovider.Reference{ + ResourceId: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "rootopaqueid", + }, + } + ri = &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "opaqueid", + }, + Path: "foo.pdf", + } + }) + + Describe("New", func() { + It("returns a new index instance", func() { + i, err := index.New(bleveIndex) + Expect(err).ToNot(HaveOccurred()) + Expect(i).ToNot(BeNil()) + }) + }) + + Describe("NewPersisted", func() { + It("returns a new index instance", func() { + i, err := index.NewPersisted("") + Expect(err).ToNot(HaveOccurred()) + Expect(i).ToNot(BeNil()) + }) + }) + + Describe("Search", func() { + It("finds files by prefix", func() { + err := i.Add(ref, ri) + Expect(err).ToNot(HaveOccurred()) + + res, err := i.Search(ctx, &search.SearchIndexRequest{ + Reference: &sprovider.Reference{ + ResourceId: ref.ResourceId, + }, + Query: "foo.pdf", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(1)) + Expect(res.Matches[0].Reference.ResourceId).To(Equal(ref.ResourceId)) + Expect(res.Matches[0].Info.Id).To(Equal(ri.Id)) + Expect(res.Matches[0].Info.Path).To(Equal(ri.Path)) + }) + + PIt("finds files living deeper in the tree by prefix") + PIt("finds directories by prefix") + PIt("finds directories living deeper in the tree by prefix") + }) + + Describe("Scan", func() { + PIt("adds the given resource recursively") + }) + + Describe("Index", func() { + It("adds a resourceInfo to the index", func() { + err := i.Add(ref, ri) + Expect(err).ToNot(HaveOccurred()) + + count, err := bleveIndex.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1))) + + query := bleve.NewMatchQuery("foo.pdf") + res, err := bleveIndex.Search(bleve.NewSearchRequest(query)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits.Len()).To(Equal(1)) + }) + + It("updates an existing resource in the index", func() { + err := i.Add(ref, ri) + Expect(err).ToNot(HaveOccurred()) + count, _ := bleveIndex.DocCount() + Expect(count).To(Equal(uint64(1))) + + err = i.Add(ref, ri) + Expect(err).ToNot(HaveOccurred()) + count, _ = bleveIndex.DocCount() + Expect(count).To(Equal(uint64(1))) + }) + }) + + Describe("Remove", func() { + PIt("removes a resource from the index") + }) +}) diff --git a/search/pkg/search/index/mocks/BleveIndex.go b/search/pkg/search/index/mocks/BleveIndex.go new file mode 100644 index 000000000..1bb4f1b28 --- /dev/null +++ b/search/pkg/search/index/mocks/BleveIndex.go @@ -0,0 +1,415 @@ +// Code generated by mockery v2.10.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + bleve "github.com/blevesearch/bleve/v2" + + index "github.com/blevesearch/bleve_index_api" + + mapping "github.com/blevesearch/bleve/v2/mapping" + + mock "github.com/stretchr/testify/mock" +) + +// BleveIndex is an autogenerated mock type for the BleveIndex type +type BleveIndex struct { + mock.Mock +} + +// Advanced provides a mock function with given fields: +func (_m *BleveIndex) Advanced() (index.Index, error) { + ret := _m.Called() + + var r0 index.Index + if rf, ok := ret.Get(0).(func() index.Index); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(index.Index) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Batch provides a mock function with given fields: b +func (_m *BleveIndex) Batch(b *bleve.Batch) error { + ret := _m.Called(b) + + var r0 error + if rf, ok := ret.Get(0).(func(*bleve.Batch) error); ok { + r0 = rf(b) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Close provides a mock function with given fields: +func (_m *BleveIndex) Close() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Delete provides a mock function with given fields: id +func (_m *BleveIndex) Delete(id string) error { + ret := _m.Called(id) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DeleteInternal provides a mock function with given fields: key +func (_m *BleveIndex) DeleteInternal(key []byte) error { + ret := _m.Called(key) + + var r0 error + if rf, ok := ret.Get(0).(func([]byte) error); ok { + r0 = rf(key) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DocCount provides a mock function with given fields: +func (_m *BleveIndex) DocCount() (uint64, error) { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Document provides a mock function with given fields: id +func (_m *BleveIndex) Document(id string) (index.Document, error) { + ret := _m.Called(id) + + var r0 index.Document + if rf, ok := ret.Get(0).(func(string) index.Document); ok { + r0 = rf(id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(index.Document) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FieldDict provides a mock function with given fields: field +func (_m *BleveIndex) FieldDict(field string) (index.FieldDict, error) { + ret := _m.Called(field) + + var r0 index.FieldDict + if rf, ok := ret.Get(0).(func(string) index.FieldDict); ok { + r0 = rf(field) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(index.FieldDict) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(field) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FieldDictPrefix provides a mock function with given fields: field, termPrefix +func (_m *BleveIndex) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) { + ret := _m.Called(field, termPrefix) + + var r0 index.FieldDict + if rf, ok := ret.Get(0).(func(string, []byte) index.FieldDict); ok { + r0 = rf(field, termPrefix) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(index.FieldDict) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, []byte) error); ok { + r1 = rf(field, termPrefix) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// FieldDictRange provides a mock function with given fields: field, startTerm, endTerm +func (_m *BleveIndex) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { + ret := _m.Called(field, startTerm, endTerm) + + var r0 index.FieldDict + if rf, ok := ret.Get(0).(func(string, []byte, []byte) index.FieldDict); ok { + r0 = rf(field, startTerm, endTerm) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(index.FieldDict) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string, []byte, []byte) error); ok { + r1 = rf(field, startTerm, endTerm) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Fields provides a mock function with given fields: +func (_m *BleveIndex) Fields() ([]string, error) { + ret := _m.Called() + + var r0 []string + if rf, ok := ret.Get(0).(func() []string); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetInternal provides a mock function with given fields: key +func (_m *BleveIndex) GetInternal(key []byte) ([]byte, error) { + ret := _m.Called(key) + + var r0 []byte + if rf, ok := ret.Get(0).(func([]byte) []byte); ok { + r0 = rf(key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func([]byte) error); ok { + r1 = rf(key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Index provides a mock function with given fields: id, data +func (_m *BleveIndex) Index(id string, data interface{}) error { + ret := _m.Called(id, data) + + var r0 error + if rf, ok := ret.Get(0).(func(string, interface{}) error); ok { + r0 = rf(id, data) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Mapping provides a mock function with given fields: +func (_m *BleveIndex) Mapping() mapping.IndexMapping { + ret := _m.Called() + + var r0 mapping.IndexMapping + if rf, ok := ret.Get(0).(func() mapping.IndexMapping); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(mapping.IndexMapping) + } + } + + return r0 +} + +// Name provides a mock function with given fields: +func (_m *BleveIndex) Name() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// NewBatch provides a mock function with given fields: +func (_m *BleveIndex) NewBatch() *bleve.Batch { + ret := _m.Called() + + var r0 *bleve.Batch + if rf, ok := ret.Get(0).(func() *bleve.Batch); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*bleve.Batch) + } + } + + return r0 +} + +// Search provides a mock function with given fields: req +func (_m *BleveIndex) Search(req *bleve.SearchRequest) (*bleve.SearchResult, error) { + ret := _m.Called(req) + + var r0 *bleve.SearchResult + if rf, ok := ret.Get(0).(func(*bleve.SearchRequest) *bleve.SearchResult); ok { + r0 = rf(req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*bleve.SearchResult) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*bleve.SearchRequest) error); ok { + r1 = rf(req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SearchInContext provides a mock function with given fields: ctx, req +func (_m *BleveIndex) SearchInContext(ctx context.Context, req *bleve.SearchRequest) (*bleve.SearchResult, error) { + ret := _m.Called(ctx, req) + + var r0 *bleve.SearchResult + if rf, ok := ret.Get(0).(func(context.Context, *bleve.SearchRequest) *bleve.SearchResult); ok { + r0 = rf(ctx, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*bleve.SearchResult) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *bleve.SearchRequest) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SetInternal provides a mock function with given fields: key, val +func (_m *BleveIndex) SetInternal(key []byte, val []byte) error { + ret := _m.Called(key, val) + + var r0 error + if rf, ok := ret.Get(0).(func([]byte, []byte) error); ok { + r0 = rf(key, val) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SetName provides a mock function with given fields: _a0 +func (_m *BleveIndex) SetName(_a0 string) { + _m.Called(_a0) +} + +// Stats provides a mock function with given fields: +func (_m *BleveIndex) Stats() *bleve.IndexStat { + ret := _m.Called() + + var r0 *bleve.IndexStat + if rf, ok := ret.Get(0).(func() *bleve.IndexStat); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*bleve.IndexStat) + } + } + + return r0 +} + +// StatsMap provides a mock function with given fields: +func (_m *BleveIndex) StatsMap() map[string]interface{} { + ret := _m.Called() + + var r0 map[string]interface{} + if rf, ok := ret.Get(0).(func() map[string]interface{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]interface{}) + } + } + + return r0 +} diff --git a/search/pkg/search/mocks/IndexClient.go b/search/pkg/search/mocks/IndexClient.go new file mode 100644 index 000000000..45f1c25e3 --- /dev/null +++ b/search/pkg/search/mocks/IndexClient.go @@ -0,0 +1,38 @@ +// Code generated by mockery v2.10.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + search "github.com/owncloud/ocis/search/pkg/search" + mock "github.com/stretchr/testify/mock" +) + +// IndexClient is an autogenerated mock type for the IndexClient type +type IndexClient struct { + mock.Mock +} + +// Search provides a mock function with given fields: ctx, req +func (_m *IndexClient) Search(ctx context.Context, req *search.SearchIndexRequest) (*search.SearchIndexResult, error) { + ret := _m.Called(ctx, req) + + var r0 *search.SearchIndexResult + if rf, ok := ret.Get(0).(func(context.Context, *search.SearchIndexRequest) *search.SearchIndexResult); ok { + r0 = rf(ctx, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*search.SearchIndexResult) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *search.SearchIndexRequest) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/search/pkg/search/mocks/ProviderClient.go b/search/pkg/search/mocks/ProviderClient.go new file mode 100644 index 000000000..0e097d54e --- /dev/null +++ b/search/pkg/search/mocks/ProviderClient.go @@ -0,0 +1,38 @@ +// Code generated by mockery v2.10.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + search "github.com/owncloud/ocis/search/pkg/search" + mock "github.com/stretchr/testify/mock" +) + +// ProviderClient is an autogenerated mock type for the ProviderClient type +type ProviderClient struct { + mock.Mock +} + +// Search provides a mock function with given fields: ctx, req +func (_m *ProviderClient) Search(ctx context.Context, req *search.SearchRequest) (*search.SearchResult, error) { + ret := _m.Called(ctx, req) + + var r0 *search.SearchResult + if rf, ok := ret.Get(0).(func(context.Context, *search.SearchRequest) *search.SearchResult); ok { + r0 = rf(ctx, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*search.SearchResult) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *search.SearchRequest) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/search/pkg/search/provider/provider_suite_test.go b/search/pkg/search/provider/provider_suite_test.go new file mode 100644 index 000000000..abe3eb217 --- /dev/null +++ b/search/pkg/search/provider/provider_suite_test.go @@ -0,0 +1,31 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package provider_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestProvider(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Provider Suite") +} diff --git a/search/pkg/search/provider/searchprovider.go b/search/pkg/search/provider/searchprovider.go new file mode 100644 index 000000000..0afdc4524 --- /dev/null +++ b/search/pkg/search/provider/searchprovider.go @@ -0,0 +1,99 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package provider + +import ( + "context" + "strings" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/utils" + "github.com/owncloud/ocis/search/pkg/search" +) + +type Provider struct { + gwClient gateway.GatewayAPIClient + indexClient search.IndexClient +} + +func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient) *Provider { + return &Provider{ + gwClient: gwClient, + indexClient: indexClient, + } +} + +func (p *Provider) Search(ctx context.Context, req *search.SearchRequest) (*search.SearchResult, error) { + if req.Query == "" { + return nil, errtypes.PreconditionFailed("empty query provided") + } + + listSpacesRes, err := p.gwClient.ListStorageSpaces(ctx, &providerv1beta1.ListStorageSpacesRequest{ + Opaque: &typesv1beta1.Opaque{Map: map[string]*typesv1beta1.OpaqueEntry{ + "path": { + Decoder: "plain", + Value: []byte("/"), + }, + }}, + }) + if err != nil { + return nil, err + } + + matches := []search.Match{} + for _, space := range listSpacesRes.StorageSpaces { + pathPrefix := "" + if space.SpaceType == "grant" { + gpRes, err := p.gwClient.GetPath(ctx, &providerv1beta1.GetPathRequest{ + ResourceId: space.Root, + }) + if err != nil { + return nil, err + } + if gpRes.Status.Code != rpcv1beta1.Code_CODE_OK { + return nil, errtypes.NewErrtypeFromStatus(gpRes.Status) + } + pathPrefix = utils.MakeRelativePath(gpRes.Path) + } + + res, err := p.indexClient.Search(ctx, &search.SearchIndexRequest{ + Query: req.Query, + Reference: &providerv1beta1.Reference{ + ResourceId: space.Root, + Path: pathPrefix, + }, + }) + if err != nil { + return nil, err + } + + for _, match := range res.Matches { + if pathPrefix != "" { + match.Reference.Path = utils.MakeRelativePath(strings.TrimPrefix(match.Reference.Path, pathPrefix)) + } + matches = append(matches, match) + } + } + + return &search.SearchResult{Matches: matches}, nil +} diff --git a/search/pkg/search/provider/searchprovider_test.go b/search/pkg/search/provider/searchprovider_test.go new file mode 100644 index 000000000..6b3ee5a17 --- /dev/null +++ b/search/pkg/search/provider/searchprovider_test.go @@ -0,0 +1,258 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package provider_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/v2/pkg/rgrpc/status" + cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" + "github.com/owncloud/ocis/search/pkg/search" + "github.com/owncloud/ocis/search/pkg/search/mocks" + provider "github.com/owncloud/ocis/search/pkg/search/provider" +) + +var _ = Describe("Searchprovider", func() { + var ( + p *provider.Provider + gwClient *cs3mocks.GatewayAPIClient + indexClient *mocks.IndexClient + + ctx context.Context + + otherUser = &userv1beta1.User{ + Id: &userv1beta1.UserId{ + OpaqueId: "otheruser", + }, + } + personalSpace = &sprovider.StorageSpace{ + Opaque: &typesv1beta1.Opaque{ + Map: map[string]*typesv1beta1.OpaqueEntry{ + "path": { + Decoder: "plain", + Value: []byte("/foo"), + }, + }, + }, + Id: &sprovider.StorageSpaceId{OpaqueId: "personalspace"}, + Root: &sprovider.ResourceId{OpaqueId: "personalspaceroot"}, + Name: "personalspace", + } + ) + + BeforeEach(func() { + ctx = context.Background() + gwClient = &cs3mocks.GatewayAPIClient{} + indexClient = &mocks.IndexClient{} + + p = provider.New(gwClient, indexClient) + }) + + Describe("New", func() { + It("returns a new instance", func() { + p := provider.New(gwClient, indexClient) + Expect(p).ToNot(BeNil()) + }) + }) + + Describe("Search", func() { + It("fails when an empty query is given", func() { + res, err := p.Search(ctx, &search.SearchRequest{ + Query: "", + }) + Expect(err).To(HaveOccurred()) + Expect(res).To(BeNil()) + }) + + Context("with a personal space", func() { + BeforeEach(func() { + gwClient.On("ListStorageSpaces", mock.Anything, mock.MatchedBy(func(req *sprovider.ListStorageSpacesRequest) bool { + p := string(req.Opaque.Map["path"].Value) + return p == "/" + })).Return(&sprovider.ListStorageSpacesResponse{ + Status: status.NewOK(ctx), + StorageSpaces: []*sprovider.StorageSpace{personalSpace}, + }, nil) + indexClient.On("Search", mock.Anything, mock.Anything).Return(&search.SearchIndexResult{ + Matches: []search.Match{ + { + Reference: &sprovider.Reference{ + ResourceId: personalSpace.Root, + Path: "./path/to/Foo.pdf", + }, + Info: &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: personalSpace.Root.StorageId, + OpaqueId: "foo-id", + }, + Path: "Foo.pdf", + }, + }, + }, + }, nil) + }) + + It("searches the personal user space", func() { + res, err := p.Search(ctx, &search.SearchRequest{ + Query: "foo", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(1)) + match := res.Matches[0] + Expect(match.Info.Id.OpaqueId).To(Equal("foo-id")) + Expect(match.Info.Path).To(Equal("Foo.pdf")) + Expect(match.Reference.ResourceId).To(Equal(personalSpace.Root)) + Expect(match.Reference.Path).To(Equal("./path/to/Foo.pdf")) + + indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *search.SearchIndexRequest) bool { + return req.Query == "foo" && req.Reference.ResourceId == personalSpace.Root && req.Reference.Path == "" + })) + }) + }) + + Context("with received shares", func() { + var ( + grantSpace *sprovider.StorageSpace + ) + + BeforeEach(func() { + grantSpace = &sprovider.StorageSpace{ + SpaceType: "grant", + Owner: otherUser, + Id: &sprovider.StorageSpaceId{OpaqueId: "otherspaceroot!otherspacegrant"}, + Root: &sprovider.ResourceId{StorageId: "otherspaceroot", OpaqueId: "otherspacegrant"}, + Name: "grantspace", + } + gwClient.On("GetPath", mock.Anything, mock.Anything).Return(&sprovider.GetPathResponse{ + Status: status.NewOK(ctx), + Path: "/grant/path", + }, nil) + }) + + It("searches the received spaces (grants)", func() { + gwClient.On("ListStorageSpaces", mock.Anything, mock.MatchedBy(func(req *sprovider.ListStorageSpacesRequest) bool { + p := string(req.Opaque.Map["path"].Value) + return p == "/" + })).Return(&sprovider.ListStorageSpacesResponse{ + Status: status.NewOK(ctx), + StorageSpaces: []*sprovider.StorageSpace{grantSpace}, + }, nil) + indexClient.On("Search", mock.Anything, mock.Anything).Return(&search.SearchIndexResult{ + Matches: []search.Match{ + search.Match{ + Reference: &sprovider.Reference{ + ResourceId: grantSpace.Root, + Path: "./grant/path/to/Foo.pdf", + }, + Info: &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: grantSpace.Root.StorageId, + OpaqueId: "grant-foo-id", + }, + Path: "Foo.pdf", + }, + }, + }, + }, nil) + + res, err := p.Search(ctx, &search.SearchRequest{ + Query: "foo", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(1)) + match := res.Matches[0] + Expect(match.Info.Id.OpaqueId).To(Equal("grant-foo-id")) + Expect(match.Info.Path).To(Equal("Foo.pdf")) + Expect(match.Reference.ResourceId).To(Equal(grantSpace.Root)) + Expect(match.Reference.Path).To(Equal("./to/Foo.pdf")) + + indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *search.SearchIndexRequest) bool { + return req.Query == "foo" && req.Reference.ResourceId == grantSpace.Root && req.Reference.Path == "./grant/path" + })) + }) + + It("finds matches in both the personal space AND the grant", func() { + gwClient.On("ListStorageSpaces", mock.Anything, mock.MatchedBy(func(req *sprovider.ListStorageSpacesRequest) bool { + p := string(req.Opaque.Map["path"].Value) + return p == "/" + })).Return(&sprovider.ListStorageSpacesResponse{ + Status: status.NewOK(ctx), + StorageSpaces: []*sprovider.StorageSpace{personalSpace, grantSpace}, + }, nil) + indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *search.SearchIndexRequest) bool { + return req.Reference.ResourceId == grantSpace.Root + })).Return(&search.SearchIndexResult{ + Matches: []search.Match{ + search.Match{ + Reference: &sprovider.Reference{ + ResourceId: grantSpace.Root, + Path: "./grant/path/to/Foo.pdf", + }, + Info: &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: grantSpace.Root.StorageId, + OpaqueId: "grant-foo-id", + }, + Path: "Foo.pdf", + }, + }, + }, + }, nil) + indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *search.SearchIndexRequest) bool { + return req.Reference.ResourceId == personalSpace.Root + })).Return(&search.SearchIndexResult{ + Matches: []search.Match{ + search.Match{ + Reference: &sprovider.Reference{ + ResourceId: personalSpace.Root, + Path: "./path/to/Foo.pdf", + }, + Info: &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: personalSpace.Root.StorageId, + OpaqueId: "foo-id", + }, + Path: "Foo.pdf", + }, + }, + }, + }, nil) + + res, err := p.Search(ctx, &search.SearchRequest{ + Query: "foo", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(2)) + ids := []string{res.Matches[0].Info.Id.OpaqueId, res.Matches[1].Info.Id.OpaqueId} + Expect(ids).To(ConsistOf("foo-id", "grant-foo-id")) + + }) + }) + }) +}) diff --git a/search/pkg/search/search.go b/search/pkg/search/search.go new file mode 100644 index 000000000..28b3f592f --- /dev/null +++ b/search/pkg/search/search.go @@ -0,0 +1,61 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package search + +import ( + "context" + + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +//go:generate mockery --name=ProviderClient +//go:generate mockery --name=IndexClient + +type SearchRequest struct { + Query string +} + +type Match struct { + Reference *sprovider.Reference + Info *sprovider.ResourceInfo +} + +type SearchResult struct { + Matches []Match +} + +type SearchIndexRequest struct { + // Reference is not a list because the Path is used as a filter which is + // cut off in the matches by the provider. Multiple paths would not be + // distinguishable. + Reference *sprovider.Reference + Query string +} + +type SearchIndexResult struct { + Matches []Match +} + +type ProviderClient interface { + Search(ctx context.Context, req *SearchRequest) (*SearchResult, error) +} + +type IndexClient interface { + Search(ctx context.Context, req *SearchIndexRequest) (*SearchIndexResult, error) +} diff --git a/search/pkg/search/search_suite_test.go b/search/pkg/search/search_suite_test.go new file mode 100644 index 000000000..46c3962a2 --- /dev/null +++ b/search/pkg/search/search_suite_test.go @@ -0,0 +1,31 @@ +// Copyright 2018-2022 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package search_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSearch(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Search Suite") +} diff --git a/search/pkg/server/debug/option.go b/search/pkg/server/debug/option.go new file mode 100644 index 000000000..1a88ca113 --- /dev/null +++ b/search/pkg/server/debug/option.go @@ -0,0 +1,50 @@ +package debug + +import ( + "context" + + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/search/pkg/config" +) + +// Option defines a single option function. +type Option func(o *Options) + +// Options defines the available options for this package. +type Options struct { + Logger log.Logger + Context context.Context + Config *config.Config +} + +// newOptions initializes the available default options. +func newOptions(opts ...Option) Options { + opt := Options{} + + for _, o := range opts { + o(&opt) + } + + return opt +} + +// Logger provides a function to set the logger option. +func Logger(val log.Logger) Option { + return func(o *Options) { + o.Logger = val + } +} + +// Context provides a function to set the context option. +func Context(val context.Context) Option { + return func(o *Options) { + o.Context = val + } +} + +// Config provides a function to set the config option. +func Config(val *config.Config) Option { + return func(o *Options) { + o.Config = val + } +} diff --git a/search/pkg/server/debug/server.go b/search/pkg/server/debug/server.go new file mode 100644 index 000000000..9b69aa0fe --- /dev/null +++ b/search/pkg/server/debug/server.go @@ -0,0 +1,63 @@ +package debug + +import ( + "io" + "net/http" + + "github.com/owncloud/ocis/ocis-pkg/service/debug" + "github.com/owncloud/ocis/ocis-pkg/version" + "github.com/owncloud/ocis/search/pkg/config" +) + +// Server initializes the debug service and server. +func Server(opts ...Option) (*http.Server, error) { + options := newOptions(opts...) + + return debug.NewService( + debug.Logger(options.Logger), + debug.Name(options.Config.Service.Name), + debug.Version(version.String), + debug.Address(options.Config.Debug.Addr), + debug.Token(options.Config.Debug.Token), + debug.Pprof(options.Config.Debug.Pprof), + debug.Zpages(options.Config.Debug.Zpages), + debug.Health(health(options.Config)), + debug.Ready(ready(options.Config)), + debug.CorsAllowedOrigins(options.Config.HTTP.CORS.AllowedOrigins), + debug.CorsAllowedMethods(options.Config.HTTP.CORS.AllowedMethods), + debug.CorsAllowedHeaders(options.Config.HTTP.CORS.AllowedHeaders), + debug.CorsAllowCredentials(options.Config.HTTP.CORS.AllowCredentials), + ), nil +} + +// health implements the health check. +func health(cfg *config.Config) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + + // TODO: check if services are up and running + + _, err := io.WriteString(w, http.StatusText(http.StatusOK)) + // io.WriteString should not fail but if it does we want to know. + if err != nil { + panic(err) + } + } +} + +// ready implements the ready check. +func ready(cfg *config.Config) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + + // TODO: check if services are up and running + + _, err := io.WriteString(w, http.StatusText(http.StatusOK)) + // io.WriteString should not fail but if it does we want to know. + if err != nil { + panic(err) + } + } +} diff --git a/search/pkg/server/grpc/option.go b/search/pkg/server/grpc/option.go new file mode 100644 index 000000000..b1b969f71 --- /dev/null +++ b/search/pkg/server/grpc/option.go @@ -0,0 +1,85 @@ +package grpc + +import ( + "context" + + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/search/pkg/config" + "github.com/owncloud/ocis/search/pkg/metrics" + svc "github.com/owncloud/ocis/search/pkg/service/v0" + "github.com/urfave/cli/v2" +) + +// Option defines a single option function. +type Option func(o *Options) + +// Options defines the available options for this package. +type Options struct { + Name string + Logger log.Logger + Context context.Context + Config *config.Config + Metrics *metrics.Metrics + Flags []cli.Flag + Handler *svc.Service +} + +// newOptions initializes the available default options. +func newOptions(opts ...Option) Options { + opt := Options{} + + for _, o := range opts { + o(&opt) + } + + return opt +} + +// Name provides a name for the service. +func Name(val string) Option { + return func(o *Options) { + o.Name = val + } +} + +// Logger provides a function to set the logger option. +func Logger(val log.Logger) Option { + return func(o *Options) { + o.Logger = val + } +} + +// Context provides a function to set the context option. +func Context(val context.Context) Option { + return func(o *Options) { + o.Context = val + } +} + +// Config provides a function to set the config option. +func Config(val *config.Config) Option { + return func(o *Options) { + o.Config = val + } +} + +// Metrics provides a function to set the metrics option. +func Metrics(val *metrics.Metrics) Option { + return func(o *Options) { + o.Metrics = val + } +} + +// Flags provides a function to set the flags option. +func Flags(val []cli.Flag) Option { + return func(o *Options) { + o.Flags = append(o.Flags, val...) + } +} + +// Handler provides a function to set the handler option. +func Handler(val *svc.Service) Option { + return func(o *Options) { + o.Handler = val + } +} diff --git a/search/pkg/server/grpc/server.go b/search/pkg/server/grpc/server.go new file mode 100644 index 000000000..48def49bd --- /dev/null +++ b/search/pkg/server/grpc/server.go @@ -0,0 +1,36 @@ +package grpc + +import ( + accountssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/accounts/v0" + + "github.com/owncloud/ocis/ocis-pkg/service/grpc" + "github.com/owncloud/ocis/ocis-pkg/version" +) + +// Server initializes a new go-micro service ready to run +func Server(opts ...Option) grpc.Service { + options := newOptions(opts...) + handler := options.Handler + + service := grpc.NewService( + grpc.Name(options.Config.Service.Name), + grpc.Context(options.Context), + grpc.Address(options.Config.GRPC.Addr), + grpc.Namespace(options.Config.GRPC.Namespace), + grpc.Logger(options.Logger), + grpc.Flags(options.Flags...), + grpc.Version(version.String), + ) + + if err := accountssvc.RegisterAccountsServiceHandler(service.Server(), handler); err != nil { + options.Logger.Fatal().Err(err).Msg("could not register service handler") + } + if err := accountssvc.RegisterGroupsServiceHandler(service.Server(), handler); err != nil { + options.Logger.Fatal().Err(err).Msg("could not register groups handler") + } + if err := accountssvc.RegisterIndexServiceHandler(service.Server(), handler); err != nil { + options.Logger.Fatal().Err(err).Msg("could not register index handler") + } + + return service +} diff --git a/search/pkg/service/v0/option.go b/search/pkg/service/v0/option.go new file mode 100644 index 000000000..4658ba8b0 --- /dev/null +++ b/search/pkg/service/v0/option.go @@ -0,0 +1,57 @@ +package service + +import ( + "github.com/owncloud/ocis/accounts/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/ocis-pkg/roles" + settingssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/settings/v0" +) + +// Option defines a single option function. +type Option func(o *Options) + +// Options defines the available options for this package. +type Options struct { + Logger log.Logger + Config *config.Config + RoleService settingssvc.RoleService + RoleManager *roles.Manager +} + +func newOptions(opts ...Option) Options { + opt := Options{} + + for _, o := range opts { + o(&opt) + } + + return opt +} + +// Logger provides a function to set the Logger option. +func Logger(val log.Logger) Option { + return func(o *Options) { + o.Logger = val + } +} + +// Config provides a function to set the Config option. +func Config(val *config.Config) Option { + return func(o *Options) { + o.Config = val + } +} + +// RoleService provides a function to set the RoleService option. +func RoleService(val settingssvc.RoleService) Option { + return func(o *Options) { + o.RoleService = val + } +} + +// RoleManager provides a function to set the RoleManager option. +func RoleManager(val *roles.Manager) Option { + return func(o *Options) { + o.RoleManager = val + } +} diff --git a/search/pkg/service/v0/service.go b/search/pkg/service/v0/service.go new file mode 100644 index 000000000..b2d479f74 --- /dev/null +++ b/search/pkg/service/v0/service.go @@ -0,0 +1,71 @@ +package service + +import ( + "time" + + "github.com/pkg/errors" + + "github.com/owncloud/ocis/ocis-pkg/service/grpc" + + "github.com/owncloud/ocis/ocis-pkg/indexer" + + "github.com/owncloud/ocis/ocis-pkg/log" + oreg "github.com/owncloud/ocis/ocis-pkg/registry" + "github.com/owncloud/ocis/ocis-pkg/roles" + settingssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/settings/v0" + "github.com/owncloud/ocis/search/pkg/config" +) + +// userDefaultGID is the default integer representing the "users" group. +const userDefaultGID = 30000 + +// New returns a new instance of Service +func New(opts ...Option) (s *Service, err error) { + options := newOptions(opts...) + logger := options.Logger + cfg := options.Config + + roleService := options.RoleService + if roleService == nil { + roleService = settingssvc.NewRoleService("com.owncloud.api.settings", grpc.DefaultClient) + } + roleManager := options.RoleManager + if roleManager == nil { + m := roles.NewManager( + roles.CacheSize(1024), + roles.CacheTTL(time.Hour*24*7), + roles.Logger(options.Logger), + roles.RoleService(roleService), + ) + roleManager = &m + } + + storage, err := createMetadataStorage(cfg, logger) + if err != nil { + return nil, errors.Wrap(err, "could not create metadata storage") + } + + s = &Service{ + id: cfg.GRPC.Namespace + "." + cfg.Service.Name, + log: logger, + Config: cfg, + } + + r := oreg.GetRegistry() + if cfg.Repo.Backend == "cs3" { + if _, err := r.GetService("com.owncloud.storage.metadata"); err != nil { + logger.Error().Err(err).Msg("index: storage-metadata service not present") + return nil, err + } + } + + return +} + +// Service implements the searchServiceHandler interface +type Service struct { + id string + log log.Logger + Config *config.Config + index *indexer.Indexer +} diff --git a/search/pkg/tracing/tracing.go b/search/pkg/tracing/tracing.go new file mode 100644 index 000000000..6d9692684 --- /dev/null +++ b/search/pkg/tracing/tracing.go @@ -0,0 +1,23 @@ +package tracing + +import ( + pkgtrace "github.com/owncloud/ocis/ocis-pkg/tracing" + "github.com/owncloud/ocis/search/pkg/config" + "go.opentelemetry.io/otel/trace" +) + +var ( + // TraceProvider is the global trace provider for the proxy service. + TraceProvider = trace.NewNoopTracerProvider() +) + +func Configure(cfg *config.Config) error { + var err error + if cfg.Tracing.Enabled { + if TraceProvider, err = pkgtrace.GetTraceProvider(cfg.Tracing.Endpoint, cfg.Tracing.Collector, cfg.Service.Name, cfg.Tracing.Type); err != nil { + return err + } + } + + return nil +} From 6cc04872eef4a1c667ff9f390649bcb908f30d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 8 Apr 2022 10:57:44 +0200 Subject: [PATCH 02/49] Also store (and return) the Size in the index --- search/pkg/search/index/index.go | 1 + search/pkg/search/index/index_test.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index 78f01ec26..bc74c29de 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -117,6 +117,7 @@ func fromFields(fields map[string]interface{}) (search.Match, error) { OpaqueId: IDParts[1], }, Path: fields["Name"].(string), + Size: uint64(fields["Size"].(float64)), }, }, nil } diff --git a/search/pkg/search/index/index_test.go b/search/pkg/search/index/index_test.go index 5887c58a6..f1f73850d 100644 --- a/search/pkg/search/index/index_test.go +++ b/search/pkg/search/index/index_test.go @@ -42,6 +42,7 @@ var _ = Describe("Index", func() { OpaqueId: "opaqueid", }, Path: "foo.pdf", + Size: 12345, } }) @@ -78,6 +79,7 @@ var _ = Describe("Index", func() { Expect(res.Matches[0].Reference.ResourceId).To(Equal(ref.ResourceId)) Expect(res.Matches[0].Info.Id).To(Equal(ri.Id)) Expect(res.Matches[0].Info.Path).To(Equal(ri.Path)) + Expect(res.Matches[0].Info.Size).To(Equal(ri.Size)) }) PIt("finds files living deeper in the tree by prefix") From 0768aef1b51547569d75b817fbfe8dd9391ec43f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 11 Apr 2022 08:46:48 +0200 Subject: [PATCH 03/49] Use NewQueryStringQuery to allow for using wildcards etc. --- search/pkg/search/index/index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index bc74c29de..292288c4b 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -64,7 +64,7 @@ func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error } func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*search.SearchIndexResult, error) { - bleveReq := bleve.NewSearchRequest(bleve.NewMatchQuery(req.Query)) + bleveReq := bleve.NewSearchRequest(bleve.NewQueryStringQuery(req.Query)) bleveReq.Fields = []string{"*"} res, err := i.bleveIndex.Search(bleveReq) if err != nil { From 79035baaf044b596897a66c5a13be7a526727639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 11 Apr 2022 08:46:14 +0200 Subject: [PATCH 04/49] Store and return the path. Add more tests --- search/pkg/search/index/index.go | 3 + search/pkg/search/index/index_test.go | 106 +++++++++++++++++++++----- 2 files changed, 90 insertions(+), 19 deletions(-) diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index 292288c4b..0d945b11f 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -36,6 +36,7 @@ type Index struct { type Entity struct { RootID string + Path string ID string Name string @@ -94,6 +95,7 @@ func BuildMapping() mapping.IndexMapping { func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *Entity { return &Entity{ RootID: ref.ResourceId.GetStorageId() + ":" + ref.ResourceId.GetOpaqueId(), + Path: ref.Path, ID: ri.Id.GetStorageId() + ":" + ri.Id.GetOpaqueId(), Name: ri.Path, Size: ri.Size, @@ -110,6 +112,7 @@ func fromFields(fields map[string]interface{}) (search.Match, error) { StorageId: rootIDParts[0], OpaqueId: rootIDParts[1], }, + Path: fields["Path"].(string), }, Info: &sprovider.ResourceInfo{ Id: &sprovider.ResourceId{ diff --git a/search/pkg/search/index/index_test.go b/search/pkg/search/index/index_test.go index f1f73850d..19d51364e 100644 --- a/search/pkg/search/index/index_test.go +++ b/search/pkg/search/index/index_test.go @@ -35,6 +35,7 @@ var _ = Describe("Index", func() { StorageId: "storageid", OpaqueId: "rootopaqueid", }, + Path: "./foo.pdf", } ri = &sprovider.ResourceInfo{ Id: &sprovider.ResourceId{ @@ -63,28 +64,95 @@ var _ = Describe("Index", func() { }) Describe("Search", func() { - It("finds files by prefix", func() { - err := i.Add(ref, ri) - Expect(err).ToNot(HaveOccurred()) - - res, err := i.Search(ctx, &search.SearchIndexRequest{ - Reference: &sprovider.Reference{ - ResourceId: ref.ResourceId, - }, - Query: "foo.pdf", + Context("with a file in the root of the space", func() { + BeforeEach(func() { + err := i.Add(ref, ri) + Expect(err).ToNot(HaveOccurred()) }) - Expect(err).ToNot(HaveOccurred()) - Expect(res).ToNot(BeNil()) - Expect(len(res.Matches)).To(Equal(1)) - Expect(res.Matches[0].Reference.ResourceId).To(Equal(ref.ResourceId)) - Expect(res.Matches[0].Info.Id).To(Equal(ri.Id)) - Expect(res.Matches[0].Info.Path).To(Equal(ri.Path)) - Expect(res.Matches[0].Info.Size).To(Equal(ri.Size)) + + It("finds files by filename", func() { + res, err := i.Search(ctx, &search.SearchIndexRequest{ + Reference: &sprovider.Reference{ + ResourceId: ref.ResourceId, + }, + Query: "foo.pdf", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(1)) + Expect(res.Matches[0].Reference.ResourceId).To(Equal(ref.ResourceId)) + Expect(res.Matches[0].Reference.Path).To(Equal(ref.Path)) + Expect(res.Matches[0].Info.Id).To(Equal(ri.Id)) + Expect(res.Matches[0].Info.Path).To(Equal(ri.Path)) + Expect(res.Matches[0].Info.Size).To(Equal(ri.Size)) + }) + + It("finds files by filename prefix", func() { + res, err := i.Search(ctx, &search.SearchIndexRequest{ + Reference: &sprovider.Reference{ + ResourceId: ref.ResourceId, + }, + Query: "foo*", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(1)) + Expect(res.Matches[0].Reference.ResourceId).To(Equal(ref.ResourceId)) + Expect(res.Matches[0].Reference.Path).To(Equal(ref.Path)) + Expect(res.Matches[0].Info.Id).To(Equal(ri.Id)) + Expect(res.Matches[0].Info.Path).To(Equal(ri.Path)) + Expect(res.Matches[0].Info.Size).To(Equal(ri.Size)) + }) + + PIt("finds directories by prefix") + }) + + Context("with a file in a subdirectory", func() { + var ( + nestedRef *sprovider.Reference + nestedRI *sprovider.ResourceInfo + ) + + BeforeEach(func() { + nestedRef = &sprovider.Reference{ + ResourceId: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "rootopaqueid", + }, + Path: "./nested/nestedpdf.pdf", + } + nestedRI = &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "opaqueid", + }, + Path: "nestedpdf.pdf", + Size: 12345, + } + err := i.Add(nestedRef, nestedRI) + Expect(err).ToNot(HaveOccurred()) + }) + + It("finds files living deeper in the tree by filename", func() { + res, err := i.Search(ctx, &search.SearchIndexRequest{ + Reference: &sprovider.Reference{ + ResourceId: ref.ResourceId, + }, + Query: "nestedpdf.pdf", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(1)) + Expect(res.Matches[0].Reference.ResourceId).To(Equal(nestedRef.ResourceId)) + Expect(res.Matches[0].Reference.Path).To(Equal(nestedRef.Path)) + Expect(res.Matches[0].Info.Id).To(Equal(nestedRI.Id)) + Expect(res.Matches[0].Info.Path).To(Equal(nestedRI.Path)) + Expect(res.Matches[0].Info.Size).To(Equal(nestedRI.Size)) + }) + + PIt("finds directories living deeper in the tree by prefix") }) - PIt("finds files living deeper in the tree by prefix") - PIt("finds directories by prefix") - PIt("finds directories living deeper in the tree by prefix") }) Describe("Scan", func() { From 099a89a523f12496b60d407a619b549279e7be61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 11 Apr 2022 09:13:44 +0200 Subject: [PATCH 05/49] Limit the search to the directory specified in the reference --- search/pkg/search/index/index.go | 6 +- search/pkg/search/index/index_test.go | 149 +++++++++++++------------- 2 files changed, 79 insertions(+), 76 deletions(-) diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index 0d945b11f..eccc581ef 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -65,7 +65,11 @@ func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error } func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*search.SearchIndexResult, error) { - bleveReq := bleve.NewSearchRequest(bleve.NewQueryStringQuery(req.Query)) + query := bleve.NewConjunctionQuery( + bleve.NewQueryStringQuery(req.Query), + bleve.NewQueryStringQuery("Path:"+req.Reference.Path+"*"), // Limit search to this directory in the space + ) + bleveReq := bleve.NewSearchRequest(query) bleveReq.Fields = []string{"*"} res, err := i.bleveIndex.Search(bleveReq) if err != nil { diff --git a/search/pkg/search/index/index_test.go b/search/pkg/search/index/index_test.go index 19d51364e..556499ce0 100644 --- a/search/pkg/search/index/index_test.go +++ b/search/pkg/search/index/index_test.go @@ -70,89 +70,88 @@ var _ = Describe("Index", func() { Expect(err).ToNot(HaveOccurred()) }) - It("finds files by filename", func() { - res, err := i.Search(ctx, &search.SearchIndexRequest{ - Reference: &sprovider.Reference{ - ResourceId: ref.ResourceId, - }, - Query: "foo.pdf", - }) - Expect(err).ToNot(HaveOccurred()) - Expect(res).ToNot(BeNil()) - Expect(len(res.Matches)).To(Equal(1)) - Expect(res.Matches[0].Reference.ResourceId).To(Equal(ref.ResourceId)) - Expect(res.Matches[0].Reference.Path).To(Equal(ref.Path)) - Expect(res.Matches[0].Info.Id).To(Equal(ri.Id)) - Expect(res.Matches[0].Info.Path).To(Equal(ri.Path)) - Expect(res.Matches[0].Info.Size).To(Equal(ri.Size)) - }) - - It("finds files by filename prefix", func() { - res, err := i.Search(ctx, &search.SearchIndexRequest{ - Reference: &sprovider.Reference{ - ResourceId: ref.ResourceId, - }, - Query: "foo*", - }) - Expect(err).ToNot(HaveOccurred()) - Expect(res).ToNot(BeNil()) - Expect(len(res.Matches)).To(Equal(1)) - Expect(res.Matches[0].Reference.ResourceId).To(Equal(ref.ResourceId)) - Expect(res.Matches[0].Reference.Path).To(Equal(ref.Path)) - Expect(res.Matches[0].Info.Id).To(Equal(ri.Id)) - Expect(res.Matches[0].Info.Path).To(Equal(ri.Path)) - Expect(res.Matches[0].Info.Size).To(Equal(ri.Size)) + It("finds files by name, prefix or substring match", func() { + queries := []string{"foo.pdf", "foo*", "*oo.p*"} + for _, query := range queries { + res, err := i.Search(ctx, &search.SearchIndexRequest{ + Reference: &sprovider.Reference{ + ResourceId: ref.ResourceId, + }, + Query: query, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(1), "query returned no result: "+query) + Expect(res.Matches[0].Reference.ResourceId).To(Equal(ref.ResourceId)) + Expect(res.Matches[0].Reference.Path).To(Equal(ref.Path)) + Expect(res.Matches[0].Info.Id).To(Equal(ri.Id)) + Expect(res.Matches[0].Info.Path).To(Equal(ri.Path)) + Expect(res.Matches[0].Info.Size).To(Equal(ri.Size)) + } }) PIt("finds directories by prefix") - }) - Context("with a file in a subdirectory", func() { - var ( - nestedRef *sprovider.Reference - nestedRI *sprovider.ResourceInfo - ) + Context("and an additional file in a subdirectory", func() { + var ( + nestedRef *sprovider.Reference + nestedRI *sprovider.ResourceInfo + ) - BeforeEach(func() { - nestedRef = &sprovider.Reference{ - ResourceId: &sprovider.ResourceId{ - StorageId: "storageid", - OpaqueId: "rootopaqueid", - }, - Path: "./nested/nestedpdf.pdf", - } - nestedRI = &sprovider.ResourceInfo{ - Id: &sprovider.ResourceId{ - StorageId: "storageid", - OpaqueId: "opaqueid", - }, - Path: "nestedpdf.pdf", - Size: 12345, - } - err := i.Add(nestedRef, nestedRI) - Expect(err).ToNot(HaveOccurred()) - }) - - It("finds files living deeper in the tree by filename", func() { - res, err := i.Search(ctx, &search.SearchIndexRequest{ - Reference: &sprovider.Reference{ - ResourceId: ref.ResourceId, - }, - Query: "nestedpdf.pdf", + BeforeEach(func() { + nestedRef = &sprovider.Reference{ + ResourceId: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "rootopaqueid", + }, + Path: "./nested/nestedpdf.pdf", + } + nestedRI = &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "nestedopaqueid", + }, + Path: "nestedpdf.pdf", + Size: 12345, + } + err := i.Add(nestedRef, nestedRI) + Expect(err).ToNot(HaveOccurred()) + }) + + It("finds files living deeper in the tree by filename, prefix or substring match", func() { + queries := []string{"nestedpdf.pdf", "nested*", "*tedpdf.*"} + for _, query := range queries { + res, err := i.Search(ctx, &search.SearchIndexRequest{ + Reference: &sprovider.Reference{ + ResourceId: ref.ResourceId, + }, + Query: query, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(1), "query returned no result: "+query) + Expect(res.Matches[0].Reference.ResourceId).To(Equal(nestedRef.ResourceId)) + Expect(res.Matches[0].Reference.Path).To(Equal(nestedRef.Path)) + Expect(res.Matches[0].Info.Id).To(Equal(nestedRI.Id)) + Expect(res.Matches[0].Info.Path).To(Equal(nestedRI.Path)) + Expect(res.Matches[0].Info.Size).To(Equal(nestedRI.Size)) + } + }) + + It("does not find the higher levels when limiting the searched directory", func() { + res, err := i.Search(ctx, &search.SearchIndexRequest{ + Reference: &sprovider.Reference{ + ResourceId: ref.ResourceId, + Path: "./nested/", + }, + Query: "foo.pdf", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(0)) }) - Expect(err).ToNot(HaveOccurred()) - Expect(res).ToNot(BeNil()) - Expect(len(res.Matches)).To(Equal(1)) - Expect(res.Matches[0].Reference.ResourceId).To(Equal(nestedRef.ResourceId)) - Expect(res.Matches[0].Reference.Path).To(Equal(nestedRef.Path)) - Expect(res.Matches[0].Info.Id).To(Equal(nestedRI.Id)) - Expect(res.Matches[0].Info.Path).To(Equal(nestedRI.Path)) - Expect(res.Matches[0].Info.Size).To(Equal(nestedRI.Size)) }) - - PIt("finds directories living deeper in the tree by prefix") }) - }) Describe("Scan", func() { From a0890eb04371e5b4f9e37be7582e9f3ad6a1cbbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 11 Apr 2022 09:26:48 +0200 Subject: [PATCH 06/49] Implement Remove for removing entries from the index --- search/pkg/search/index/index.go | 4 ++++ search/pkg/search/index/index_test.go | 14 +++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index eccc581ef..4e2c43aac 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -64,6 +64,10 @@ func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error return i.bleveIndex.Index(entity.ID, entity) } +func (i *Index) Remove(ri *sprovider.ResourceInfo) error { + return i.bleveIndex.Delete(ri.Id.GetStorageId() + ":" + ri.Id.GetOpaqueId()) +} + func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*search.SearchIndexResult, error) { query := bleve.NewConjunctionQuery( bleve.NewQueryStringQuery(req.Query), diff --git a/search/pkg/search/index/index_test.go b/search/pkg/search/index/index_test.go index 556499ce0..795938528 100644 --- a/search/pkg/search/index/index_test.go +++ b/search/pkg/search/index/index_test.go @@ -90,8 +90,6 @@ var _ = Describe("Index", func() { } }) - PIt("finds directories by prefix") - Context("and an additional file in a subdirectory", func() { var ( nestedRef *sprovider.Reference @@ -187,6 +185,16 @@ var _ = Describe("Index", func() { }) Describe("Remove", func() { - PIt("removes a resource from the index") + It("removes a resource from the index", func() { + err := i.Add(ref, ri) + Expect(err).ToNot(HaveOccurred()) + count, _ := bleveIndex.DocCount() + Expect(count).To(Equal(uint64(1))) + + err = i.Remove(ri) + Expect(err).ToNot(HaveOccurred()) + count, _ = bleveIndex.DocCount() + Expect(count).To(Equal(uint64(0))) + }) }) }) From 6f593956115ac24b5c16bb4609b78f97a9b21d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 11 Apr 2022 10:00:28 +0200 Subject: [PATCH 07/49] Cleanup --- search/pkg/search/index/index.go | 12 ++++++-- search/pkg/search/index/index_test.go | 4 --- .../search/provider/searchprovider_test.go | 27 +++++++++--------- search/pkg/search/search.go | 28 +++++++++++-------- 4 files changed, 38 insertions(+), 33 deletions(-) diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index 4e2c43aac..ec8480c2d 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -30,10 +30,12 @@ import ( sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ) +// Index represents a bleve based search index type Index struct { bleveIndex bleve.Index } +// Entity describes an Entity stored in the index type Entity struct { RootID string Path string @@ -43,31 +45,34 @@ type Entity struct { Size uint64 } +// NewPersisted returns a new instance of Index with the data being persisted in the given directory func NewPersisted(path string) (*Index, error) { bi, err := bleve.New(path, BuildMapping()) if err != nil { return nil, err } - return &Index{ - bleveIndex: bi, - }, nil + return New(bi) } +// New returns a new instance of Index using the given bleve Index as the backend func New(bleveIndex bleve.Index) (*Index, error) { return &Index{ bleveIndex: bleveIndex, }, nil } +// Add adds a new entity to the Index func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error { entity := toEntity(ref, ri) return i.bleveIndex.Index(entity.ID, entity) } +// Remove removes an entity from the index func (i *Index) Remove(ri *sprovider.ResourceInfo) error { return i.bleveIndex.Delete(ri.Id.GetStorageId() + ":" + ri.Id.GetOpaqueId()) } +// Search searches the index according to the criteria specified in the given SearchIndexRequest func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*search.SearchIndexResult, error) { query := bleve.NewConjunctionQuery( bleve.NewQueryStringQuery(req.Query), @@ -94,6 +99,7 @@ func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*se }, nil } +// BuildMapping builds a bleve index mapping which can be used for indexing func BuildMapping() mapping.IndexMapping { indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = keyword.Name diff --git a/search/pkg/search/index/index_test.go b/search/pkg/search/index/index_test.go index 795938528..37bf24468 100644 --- a/search/pkg/search/index/index_test.go +++ b/search/pkg/search/index/index_test.go @@ -152,10 +152,6 @@ var _ = Describe("Index", func() { }) }) - Describe("Scan", func() { - PIt("adds the given resource recursively") - }) - Describe("Index", func() { It("adds a resourceInfo to the index", func() { err := i.Add(ref, ri) diff --git a/search/pkg/search/provider/searchprovider_test.go b/search/pkg/search/provider/searchprovider_test.go index 6b3ee5a17..9f57fc45a 100644 --- a/search/pkg/search/provider/searchprovider_test.go +++ b/search/pkg/search/provider/searchprovider_test.go @@ -163,17 +163,17 @@ var _ = Describe("Searchprovider", func() { }, nil) indexClient.On("Search", mock.Anything, mock.Anything).Return(&search.SearchIndexResult{ Matches: []search.Match{ - search.Match{ + { Reference: &sprovider.Reference{ ResourceId: grantSpace.Root, - Path: "./grant/path/to/Foo.pdf", + Path: "./grant/path/to/Shared.pdf", }, Info: &sprovider.ResourceInfo{ Id: &sprovider.ResourceId{ StorageId: grantSpace.Root.StorageId, - OpaqueId: "grant-foo-id", + OpaqueId: "grant-shared-id", }, - Path: "Foo.pdf", + Path: "Shared.pdf", }, }, }, @@ -186,10 +186,10 @@ var _ = Describe("Searchprovider", func() { Expect(res).ToNot(BeNil()) Expect(len(res.Matches)).To(Equal(1)) match := res.Matches[0] - Expect(match.Info.Id.OpaqueId).To(Equal("grant-foo-id")) - Expect(match.Info.Path).To(Equal("Foo.pdf")) + Expect(match.Info.Id.OpaqueId).To(Equal("grant-shared-id")) + Expect(match.Info.Path).To(Equal("Shared.pdf")) Expect(match.Reference.ResourceId).To(Equal(grantSpace.Root)) - Expect(match.Reference.Path).To(Equal("./to/Foo.pdf")) + Expect(match.Reference.Path).To(Equal("./to/Shared.pdf")) indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *search.SearchIndexRequest) bool { return req.Query == "foo" && req.Reference.ResourceId == grantSpace.Root && req.Reference.Path == "./grant/path" @@ -208,17 +208,17 @@ var _ = Describe("Searchprovider", func() { return req.Reference.ResourceId == grantSpace.Root })).Return(&search.SearchIndexResult{ Matches: []search.Match{ - search.Match{ + { Reference: &sprovider.Reference{ ResourceId: grantSpace.Root, - Path: "./grant/path/to/Foo.pdf", + Path: "./grant/path/to/Shared.pdf", }, Info: &sprovider.ResourceInfo{ Id: &sprovider.ResourceId{ StorageId: grantSpace.Root.StorageId, - OpaqueId: "grant-foo-id", + OpaqueId: "grant-shared-id", }, - Path: "Foo.pdf", + Path: "Shared.pdf", }, }, }, @@ -227,7 +227,7 @@ var _ = Describe("Searchprovider", func() { return req.Reference.ResourceId == personalSpace.Root })).Return(&search.SearchIndexResult{ Matches: []search.Match{ - search.Match{ + { Reference: &sprovider.Reference{ ResourceId: personalSpace.Root, Path: "./path/to/Foo.pdf", @@ -250,8 +250,7 @@ var _ = Describe("Searchprovider", func() { Expect(res).ToNot(BeNil()) Expect(len(res.Matches)).To(Equal(2)) ids := []string{res.Matches[0].Info.Id.OpaqueId, res.Matches[1].Info.Id.OpaqueId} - Expect(ids).To(ConsistOf("foo-id", "grant-foo-id")) - + Expect(ids).To(ConsistOf("foo-id", "grant-shared-id")) }) }) }) diff --git a/search/pkg/search/search.go b/search/pkg/search/search.go index 28b3f592f..27acf38b0 100644 --- a/search/pkg/search/search.go +++ b/search/pkg/search/search.go @@ -27,35 +27,39 @@ import ( //go:generate mockery --name=ProviderClient //go:generate mockery --name=IndexClient +// SearchRequest represents a search request from a user to the search provider type SearchRequest struct { Query string } +// Match holds the information of a matched resource in a search type Match struct { Reference *sprovider.Reference Info *sprovider.ResourceInfo } +// SearchResult contains the matches being returned for a search type SearchResult struct { Matches []Match } -type SearchIndexRequest struct { - // Reference is not a list because the Path is used as a filter which is - // cut off in the matches by the provider. Multiple paths would not be - // distinguishable. - Reference *sprovider.Reference - Query string -} - -type SearchIndexResult struct { - Matches []Match -} - +// ProviderClient is the interface to the search provider service type ProviderClient interface { Search(ctx context.Context, req *SearchRequest) (*SearchResult, error) } +// SearchIndexRequest represents a search request to the index +type SearchIndexRequest struct { + Reference *sprovider.Reference + Query string +} + +// SearchResult contains the matches in the index being returned for a search +type SearchIndexResult struct { + Matches []Match +} + +// IndexClient is the interface to the search index type IndexClient interface { Search(ctx context.Context, req *SearchIndexRequest) (*SearchIndexResult, error) } From 6bb9af5a24399aa3af5e317c8a1a4615f8ebd494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 12 Apr 2022 14:51:16 +0000 Subject: [PATCH 08/49] allow proxy to route based on request method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- proxy/pkg/config/config.go | 10 +++--- proxy/pkg/config/defaults/defaultconfig.go | 9 +++++ proxy/pkg/proxy/proxy.go | 34 +++++++++++++----- proxy/pkg/proxy/proxy_test.go | 40 ++++++++++++++++++++-- 4 files changed, 78 insertions(+), 15 deletions(-) diff --git a/proxy/pkg/config/config.go b/proxy/pkg/config/config.go index 6cda66b8f..d5c316b1c 100644 --- a/proxy/pkg/config/config.go +++ b/proxy/pkg/config/config.go @@ -45,10 +45,12 @@ type Policy struct { // Route define forwarding routes type Route struct { - Type RouteType `ocisConfig:"type"` - Endpoint string `ocisConfig:"endpoint"` - Backend string `ocisConfig:"backend"` - ApacheVHost bool `ocisConfig:"apache-vhost"` + Type RouteType `ocisConfig:"type"` + // Method optionally limits the route to this HTTP method + Method string `ocisConfig:"method"` + Endpoint string `ocisConfig:"endpoint"` + Backend string `ocisConfig:"backend"` + ApacheVHost bool `ocisConfig:"apache-vhost"` } // RouteType defines the type of a route diff --git a/proxy/pkg/config/defaults/defaultconfig.go b/proxy/pkg/config/defaults/defaultconfig.go index eaa328b5a..776941930 100644 --- a/proxy/pkg/config/defaults/defaultconfig.go +++ b/proxy/pkg/config/defaults/defaultconfig.go @@ -96,6 +96,15 @@ func DefaultPolicies() []config.Policy { Endpoint: "/remote.php/?preview=1", Backend: "http://localhost:9115", }, + { + // TODO the actual REPORT goes to /dav/files/{username}, which is user specific ... how would this work in a spaces world? + // TODO what paths are returned? the href contains the full path so it should be possible to return urls from other spaces? + // TODO or we allow a REPORT on /dav/spaces to search all spaces and /dav/space/{spaceid} to search a specific space + // send webdav REPORT requests to search service + Method: "REPORT", + Endpoint: "/dav/", + Backend: "http://localhost:????", // TODO use registry? + }, { Endpoint: "/remote.php/", Backend: "http://localhost:9140", diff --git a/proxy/pkg/proxy/proxy.go b/proxy/pkg/proxy/proxy.go index 7681defb8..89ff7b3a7 100644 --- a/proxy/pkg/proxy/proxy.go +++ b/proxy/pkg/proxy/proxy.go @@ -27,7 +27,8 @@ import ( // MultiHostReverseProxy extends "httputil" to support multiple hosts with different policies type MultiHostReverseProxy struct { httputil.ReverseProxy - Directors map[string]map[config.RouteType]map[string]func(req *http.Request) + // Directors holds policy route type method endpoint Director + Directors map[string]map[config.RouteType]map[string]map[string]func(req *http.Request) PolicySelector policy.Selector logger log.Logger config *config.Config @@ -38,7 +39,7 @@ func NewMultiHostReverseProxy(opts ...Option) *MultiHostReverseProxy { options := newOptions(opts...) rp := &MultiHostReverseProxy{ - Directors: make(map[string]map[config.RouteType]map[string]func(req *http.Request)), + Directors: make(map[string]map[config.RouteType]map[string]map[string]func(req *http.Request)), logger: options.Logger, config: options.Config, } @@ -117,6 +118,7 @@ func (p *MultiHostReverseProxy) directorSelectionDirector(r *http.Request) { return } + method := "" // find matching director for _, rt := range config.RouteTypes { var handler func(string, url.URL) bool @@ -130,25 +132,36 @@ func (p *MultiHostReverseProxy) directorSelectionDirector(r *http.Request) { default: handler = p.prefixRouteMatcher } - for endpoint := range p.Directors[pol][rt] { + if p.Directors[pol][rt][r.Method] != nil { + // use specific method + method = r.Method + } + for endpoint := range p.Directors[pol][rt][method] { if handler(endpoint, *r.URL) { p.logger.Debug(). Str("policy", pol). + Str("method", r.Method). Str("prefix", endpoint). Str("path", r.URL.Path). Str("routeType", string(rt)). Msg("director found") - p.Directors[pol][rt][endpoint](r) + p.Directors[pol][rt][method][endpoint](r) return } } } // override default director with root. If any - if p.Directors[pol][config.PrefixRoute]["/"] != nil { - p.Directors[pol][config.PrefixRoute]["/"](r) + switch { + case p.Directors[pol][config.PrefixRoute][method]["/"] != nil: + // try specific method + p.Directors[pol][config.PrefixRoute][method]["/"](r) + return + case p.Directors[pol][config.PrefixRoute][""]["/"] != nil: + // fallback to unspecific method + p.Directors[pol][config.PrefixRoute][""]["/"](r) return } @@ -175,16 +188,19 @@ func singleJoiningSlash(a, b string) string { func (p *MultiHostReverseProxy) AddHost(policy string, target *url.URL, rt config.Route) { targetQuery := target.RawQuery if p.Directors[policy] == nil { - p.Directors[policy] = make(map[config.RouteType]map[string]func(req *http.Request)) + p.Directors[policy] = make(map[config.RouteType]map[string]map[string]func(req *http.Request)) } routeType := config.DefaultRouteType if rt.Type != "" { routeType = rt.Type } if p.Directors[policy][routeType] == nil { - p.Directors[policy][routeType] = make(map[string]func(req *http.Request)) + p.Directors[policy][routeType] = make(map[string]map[string]func(req *http.Request)) } - p.Directors[policy][routeType][rt.Endpoint] = func(req *http.Request) { + if p.Directors[policy][routeType][rt.Method] == nil { + p.Directors[policy][routeType][rt.Method] = make(map[string]func(req *http.Request)) + } + p.Directors[policy][routeType][rt.Method][rt.Endpoint] = func(req *http.Request) { req.URL.Scheme = target.Scheme req.URL.Host = target.Host // Apache deployments host addresses need to match on req.Host and req.URL.Host diff --git a/proxy/pkg/proxy/proxy_test.go b/proxy/pkg/proxy/proxy_test.go index f64362c49..213882fb9 100644 --- a/proxy/pkg/proxy/proxy_test.go +++ b/proxy/pkg/proxy/proxy_test.go @@ -1,15 +1,19 @@ package proxy import ( + "fmt" + "net/http" + "net/http/httptest" "net/url" "testing" + "github.com/owncloud/ocis/proxy/pkg/config" "github.com/owncloud/ocis/proxy/pkg/config/defaults" ) type matchertest struct { - endpoint, target string - matches bool + method, endpoint, target string + matches bool } func TestPrefixRouteMatcher(t *testing.T) { @@ -99,3 +103,35 @@ func TestSingleJoiningSlash(t *testing.T) { } } } + +func TestDirectorSelectionDirector(t *testing.T) { + + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "ok") + })) + defer svr.Close() + + p := NewMultiHostReverseProxy(Config(&config.Config{ + PolicySelector: &config.PolicySelector{ + Static: &config.StaticSelectorConf{ + Policy: "default", + }, + }, + })) + p.AddHost("default", &url.URL{Host: "ocdav"}, config.Route{Type: config.PrefixRoute, Method: "", Endpoint: "/dav", Backend: "ocdav"}) + p.AddHost("default", &url.URL{Host: "ocis-webdav"}, config.Route{Type: config.PrefixRoute, Method: "REPORT", Endpoint: "/dav", Backend: "ocis-webdav"}) + + table := []matchertest{ + {method: "PROPFIND", endpoint: "/dav/files/demo/", target: "ocdav"}, + {method: "REPORT", endpoint: "/dav/files/demo/", target: "ocis-webdav"}, + } + + for _, test := range table { + r := httptest.NewRequest(http.MethodGet, "/dav/files/demo/", nil) + p.directorSelectionDirector(r) + if r.Host != test.target { + t.Errorf("TestDirectorSelectionDirector got host %s expected %s", r.Host, test.target) + + } + } +} From 6be2d8c0c505be55f913eb1ea21cc842ec60cba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 12 Apr 2022 09:23:39 +0200 Subject: [PATCH 09/49] Add protobuf definitions and use those instead of our structs --- .../gen/ocis/messages/search/v0/search.pb.go | 405 +++++++++++++++ .../messages/search/v0/search.pb.micro.go | 15 + .../ocis/messages/search/v0/search.pb.web.go | 155 ++++++ .../messages/search/v0/search.swagger.json | 43 ++ .../gen/ocis/services/search/v0/search.pb.go | 485 ++++++++++++++++++ .../services/search/v0/search.pb.micro.go | 180 +++++++ .../ocis/services/search/v0/search.pb.web.go | 238 +++++++++ .../services/search/v0/search.swagger.json | 254 +++++++++ .../proto/ocis/messages/search/search.proto | 16 - .../ocis/messages/search/v0/search.proto | 29 ++ .../proto/ocis/services/search/search.proto | 84 --- .../ocis/services/search/v0/search.proto | 95 ++++ search/pkg/command/server.go | 68 ++- search/pkg/config/config.go | 1 + search/pkg/config/defaults/defaultconfig.go | 4 + search/pkg/config/grpc.go | 7 + search/pkg/search/index/index.go | 10 - search/pkg/search/mocks/IndexClient.go | 13 +- search/pkg/search/mocks/ProviderClient.go | 13 +- search/pkg/search/provider/searchprovider.go | 24 +- .../search/provider/searchprovider_test.go | 129 ++--- search/pkg/search/search.go | 33 +- search/pkg/server/grpc/server.go | 10 +- search/pkg/service/v0/option.go | 24 +- search/pkg/service/v0/service.go | 70 +-- 25 files changed, 2076 insertions(+), 329 deletions(-) create mode 100644 protogen/gen/ocis/messages/search/v0/search.pb.go create mode 100644 protogen/gen/ocis/messages/search/v0/search.pb.micro.go create mode 100644 protogen/gen/ocis/messages/search/v0/search.pb.web.go create mode 100644 protogen/gen/ocis/messages/search/v0/search.swagger.json create mode 100644 protogen/gen/ocis/services/search/v0/search.pb.go create mode 100644 protogen/gen/ocis/services/search/v0/search.pb.micro.go create mode 100644 protogen/gen/ocis/services/search/v0/search.pb.web.go create mode 100644 protogen/gen/ocis/services/search/v0/search.swagger.json delete mode 100644 protogen/proto/ocis/messages/search/search.proto create mode 100644 protogen/proto/ocis/messages/search/v0/search.proto delete mode 100644 protogen/proto/ocis/services/search/search.proto create mode 100644 protogen/proto/ocis/services/search/v0/search.proto create mode 100644 search/pkg/config/grpc.go diff --git a/protogen/gen/ocis/messages/search/v0/search.pb.go b/protogen/gen/ocis/messages/search/v0/search.pb.go new file mode 100644 index 000000000..9e22a09d8 --- /dev/null +++ b/protogen/gen/ocis/messages/search/v0/search.pb.go @@ -0,0 +1,405 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.1 +// protoc v3.17.3 +// source: ocis/messages/search/v0/search.proto + +package v0 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ResourceID struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + StorageId string `protobuf:"bytes,1,opt,name=storage_id,json=storageId,proto3" json:"storage_id,omitempty"` + OpaqueId string `protobuf:"bytes,2,opt,name=opaque_id,json=opaqueId,proto3" json:"opaque_id,omitempty"` +} + +func (x *ResourceID) Reset() { + *x = ResourceID{} + if protoimpl.UnsafeEnabled { + mi := &file_ocis_messages_search_v0_search_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResourceID) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResourceID) ProtoMessage() {} + +func (x *ResourceID) ProtoReflect() protoreflect.Message { + mi := &file_ocis_messages_search_v0_search_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResourceID.ProtoReflect.Descriptor instead. +func (*ResourceID) Descriptor() ([]byte, []int) { + return file_ocis_messages_search_v0_search_proto_rawDescGZIP(), []int{0} +} + +func (x *ResourceID) GetStorageId() string { + if x != nil { + return x.StorageId + } + return "" +} + +func (x *ResourceID) GetOpaqueId() string { + if x != nil { + return x.OpaqueId + } + return "" +} + +type Reference struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ResourceId *ResourceID `protobuf:"bytes,1,opt,name=resource_id,json=resourceId,proto3" json:"resource_id,omitempty"` + Path string `protobuf:"bytes,2,opt,name=path,proto3" json:"path,omitempty"` +} + +func (x *Reference) Reset() { + *x = Reference{} + if protoimpl.UnsafeEnabled { + mi := &file_ocis_messages_search_v0_search_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Reference) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Reference) ProtoMessage() {} + +func (x *Reference) ProtoReflect() protoreflect.Message { + mi := &file_ocis_messages_search_v0_search_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Reference.ProtoReflect.Descriptor instead. +func (*Reference) Descriptor() ([]byte, []int) { + return file_ocis_messages_search_v0_search_proto_rawDescGZIP(), []int{1} +} + +func (x *Reference) GetResourceId() *ResourceID { + if x != nil { + return x.ResourceId + } + return nil +} + +func (x *Reference) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +type Entity struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Ref *Reference `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` + Id *ResourceID `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + Size int64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` +} + +func (x *Entity) Reset() { + *x = Entity{} + if protoimpl.UnsafeEnabled { + mi := &file_ocis_messages_search_v0_search_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Entity) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Entity) ProtoMessage() {} + +func (x *Entity) ProtoReflect() protoreflect.Message { + mi := &file_ocis_messages_search_v0_search_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Entity.ProtoReflect.Descriptor instead. +func (*Entity) Descriptor() ([]byte, []int) { + return file_ocis_messages_search_v0_search_proto_rawDescGZIP(), []int{2} +} + +func (x *Entity) GetRef() *Reference { + if x != nil { + return x.Ref + } + return nil +} + +func (x *Entity) GetId() *ResourceID { + if x != nil { + return x.Id + } + return nil +} + +func (x *Entity) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Entity) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +type Match struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // the matched entity + Entity *Entity `protobuf:"bytes,1,opt,name=entity,proto3" json:"entity,omitempty"` + // the match score + Score float32 `protobuf:"fixed32,2,opt,name=score,proto3" json:"score,omitempty"` +} + +func (x *Match) Reset() { + *x = Match{} + if protoimpl.UnsafeEnabled { + mi := &file_ocis_messages_search_v0_search_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Match) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Match) ProtoMessage() {} + +func (x *Match) ProtoReflect() protoreflect.Message { + mi := &file_ocis_messages_search_v0_search_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Match.ProtoReflect.Descriptor instead. +func (*Match) Descriptor() ([]byte, []int) { + return file_ocis_messages_search_v0_search_proto_rawDescGZIP(), []int{3} +} + +func (x *Match) GetEntity() *Entity { + if x != nil { + return x.Entity + } + return nil +} + +func (x *Match) GetScore() float32 { + if x != nil { + return x.Score + } + return 0 +} + +var File_ocis_messages_search_v0_search_proto protoreflect.FileDescriptor + +var file_ocis_messages_search_v0_search_proto_rawDesc = []byte{ + 0x0a, 0x24, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, + 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x17, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x22, + 0x48, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x12, 0x1d, 0x0a, + 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, + 0x6f, 0x70, 0x61, 0x71, 0x75, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x6f, 0x70, 0x61, 0x71, 0x75, 0x65, 0x49, 0x64, 0x22, 0x65, 0x0a, 0x09, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x63, + 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, + 0x52, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, + 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x22, 0x9b, 0x01, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x03, 0x72, + 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, + 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x03, 0x72, 0x65, + 0x66, 0x12, 0x33, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x49, 0x44, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, + 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x56, + 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x37, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, + 0x30, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, + 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, + 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, + 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_ocis_messages_search_v0_search_proto_rawDescOnce sync.Once + file_ocis_messages_search_v0_search_proto_rawDescData = file_ocis_messages_search_v0_search_proto_rawDesc +) + +func file_ocis_messages_search_v0_search_proto_rawDescGZIP() []byte { + file_ocis_messages_search_v0_search_proto_rawDescOnce.Do(func() { + file_ocis_messages_search_v0_search_proto_rawDescData = protoimpl.X.CompressGZIP(file_ocis_messages_search_v0_search_proto_rawDescData) + }) + return file_ocis_messages_search_v0_search_proto_rawDescData +} + +var file_ocis_messages_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_ocis_messages_search_v0_search_proto_goTypes = []interface{}{ + (*ResourceID)(nil), // 0: ocis.messages.search.v0.ResourceID + (*Reference)(nil), // 1: ocis.messages.search.v0.Reference + (*Entity)(nil), // 2: ocis.messages.search.v0.Entity + (*Match)(nil), // 3: ocis.messages.search.v0.Match +} +var file_ocis_messages_search_v0_search_proto_depIdxs = []int32{ + 0, // 0: ocis.messages.search.v0.Reference.resource_id:type_name -> ocis.messages.search.v0.ResourceID + 1, // 1: ocis.messages.search.v0.Entity.ref:type_name -> ocis.messages.search.v0.Reference + 0, // 2: ocis.messages.search.v0.Entity.id:type_name -> ocis.messages.search.v0.ResourceID + 2, // 3: ocis.messages.search.v0.Match.entity:type_name -> ocis.messages.search.v0.Entity + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_ocis_messages_search_v0_search_proto_init() } +func file_ocis_messages_search_v0_search_proto_init() { + if File_ocis_messages_search_v0_search_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_ocis_messages_search_v0_search_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ResourceID); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ocis_messages_search_v0_search_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Reference); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ocis_messages_search_v0_search_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Entity); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ocis_messages_search_v0_search_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Match); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_ocis_messages_search_v0_search_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_ocis_messages_search_v0_search_proto_goTypes, + DependencyIndexes: file_ocis_messages_search_v0_search_proto_depIdxs, + MessageInfos: file_ocis_messages_search_v0_search_proto_msgTypes, + }.Build() + File_ocis_messages_search_v0_search_proto = out.File + file_ocis_messages_search_v0_search_proto_rawDesc = nil + file_ocis_messages_search_v0_search_proto_goTypes = nil + file_ocis_messages_search_v0_search_proto_depIdxs = nil +} diff --git a/protogen/gen/ocis/messages/search/v0/search.pb.micro.go b/protogen/gen/ocis/messages/search/v0/search.pb.micro.go new file mode 100644 index 000000000..5f507f0ae --- /dev/null +++ b/protogen/gen/ocis/messages/search/v0/search.pb.micro.go @@ -0,0 +1,15 @@ +// Code generated by protoc-gen-micro. DO NOT EDIT. +// source: ocis/messages/search/v0/search.proto + +package v0 + +import ( + fmt "fmt" + proto "google.golang.org/protobuf/proto" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf diff --git a/protogen/gen/ocis/messages/search/v0/search.pb.web.go b/protogen/gen/ocis/messages/search/v0/search.pb.web.go new file mode 100644 index 000000000..7055e8349 --- /dev/null +++ b/protogen/gen/ocis/messages/search/v0/search.pb.web.go @@ -0,0 +1,155 @@ +// Code generated by protoc-gen-microweb. DO NOT EDIT. +// source: v0.proto + +package v0 + +import ( + "bytes" + "encoding/json" + + "github.com/golang/protobuf/jsonpb" +) + +// ResourceIDJSONMarshaler describes the default jsonpb.Marshaler used by all +// instances of ResourceID. This struct is safe to replace or modify but +// should not be done so concurrently. +var ResourceIDJSONMarshaler = new(jsonpb.Marshaler) + +// MarshalJSON satisfies the encoding/json Marshaler interface. This method +// uses the more correct jsonpb package to correctly marshal the message. +func (m *ResourceID) MarshalJSON() ([]byte, error) { + if m == nil { + return json.Marshal(nil) + } + + buf := &bytes.Buffer{} + + if err := ResourceIDJSONMarshaler.Marshal(buf, m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +var _ json.Marshaler = (*ResourceID)(nil) + +// ResourceIDJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all +// instances of ResourceID. This struct is safe to replace or modify but +// should not be done so concurrently. +var ResourceIDJSONUnmarshaler = new(jsonpb.Unmarshaler) + +// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method +// uses the more correct jsonpb package to correctly unmarshal the message. +func (m *ResourceID) UnmarshalJSON(b []byte) error { + return ResourceIDJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m) +} + +var _ json.Unmarshaler = (*ResourceID)(nil) + +// ReferenceJSONMarshaler describes the default jsonpb.Marshaler used by all +// instances of Reference. This struct is safe to replace or modify but +// should not be done so concurrently. +var ReferenceJSONMarshaler = new(jsonpb.Marshaler) + +// MarshalJSON satisfies the encoding/json Marshaler interface. This method +// uses the more correct jsonpb package to correctly marshal the message. +func (m *Reference) MarshalJSON() ([]byte, error) { + if m == nil { + return json.Marshal(nil) + } + + buf := &bytes.Buffer{} + + if err := ReferenceJSONMarshaler.Marshal(buf, m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +var _ json.Marshaler = (*Reference)(nil) + +// ReferenceJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all +// instances of Reference. This struct is safe to replace or modify but +// should not be done so concurrently. +var ReferenceJSONUnmarshaler = new(jsonpb.Unmarshaler) + +// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method +// uses the more correct jsonpb package to correctly unmarshal the message. +func (m *Reference) UnmarshalJSON(b []byte) error { + return ReferenceJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m) +} + +var _ json.Unmarshaler = (*Reference)(nil) + +// EntityJSONMarshaler describes the default jsonpb.Marshaler used by all +// instances of Entity. This struct is safe to replace or modify but +// should not be done so concurrently. +var EntityJSONMarshaler = new(jsonpb.Marshaler) + +// MarshalJSON satisfies the encoding/json Marshaler interface. This method +// uses the more correct jsonpb package to correctly marshal the message. +func (m *Entity) MarshalJSON() ([]byte, error) { + if m == nil { + return json.Marshal(nil) + } + + buf := &bytes.Buffer{} + + if err := EntityJSONMarshaler.Marshal(buf, m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +var _ json.Marshaler = (*Entity)(nil) + +// EntityJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all +// instances of Entity. This struct is safe to replace or modify but +// should not be done so concurrently. +var EntityJSONUnmarshaler = new(jsonpb.Unmarshaler) + +// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method +// uses the more correct jsonpb package to correctly unmarshal the message. +func (m *Entity) UnmarshalJSON(b []byte) error { + return EntityJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m) +} + +var _ json.Unmarshaler = (*Entity)(nil) + +// MatchJSONMarshaler describes the default jsonpb.Marshaler used by all +// instances of Match. This struct is safe to replace or modify but +// should not be done so concurrently. +var MatchJSONMarshaler = new(jsonpb.Marshaler) + +// MarshalJSON satisfies the encoding/json Marshaler interface. This method +// uses the more correct jsonpb package to correctly marshal the message. +func (m *Match) MarshalJSON() ([]byte, error) { + if m == nil { + return json.Marshal(nil) + } + + buf := &bytes.Buffer{} + + if err := MatchJSONMarshaler.Marshal(buf, m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +var _ json.Marshaler = (*Match)(nil) + +// MatchJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all +// instances of Match. This struct is safe to replace or modify but +// should not be done so concurrently. +var MatchJSONUnmarshaler = new(jsonpb.Unmarshaler) + +// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method +// uses the more correct jsonpb package to correctly unmarshal the message. +func (m *Match) UnmarshalJSON(b []byte) error { + return MatchJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m) +} + +var _ json.Unmarshaler = (*Match)(nil) diff --git a/protogen/gen/ocis/messages/search/v0/search.swagger.json b/protogen/gen/ocis/messages/search/v0/search.swagger.json new file mode 100644 index 000000000..dd7fdc358 --- /dev/null +++ b/protogen/gen/ocis/messages/search/v0/search.swagger.json @@ -0,0 +1,43 @@ +{ + "swagger": "2.0", + "info": { + "title": "ocis/messages/search/v0/search.proto", + "version": "version not set" + }, + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "paths": {}, + "definitions": { + "protobufAny": { + "type": "object", + "properties": { + "@type": { + "type": "string" + } + }, + "additionalProperties": {} + }, + "rpcStatus": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + }, + "details": { + "type": "array", + "items": { + "$ref": "#/definitions/protobufAny" + } + } + } + } + } +} diff --git a/protogen/gen/ocis/services/search/v0/search.pb.go b/protogen/gen/ocis/services/search/v0/search.pb.go new file mode 100644 index 000000000..73d450a6b --- /dev/null +++ b/protogen/gen/ocis/services/search/v0/search.pb.go @@ -0,0 +1,485 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.1 +// protoc v3.17.3 +// source: ocis/services/search/v0/search.proto + +package v0 + +import ( + _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2/options" + v0 "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" + _ "google.golang.org/genproto/googleapis/api/annotations" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + _ "google.golang.org/protobuf/types/known/fieldmaskpb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SearchRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Optional. The maximum number of entries to return in the response + PageSize int32 `protobuf:"varint,1,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + // Optional. A pagination token returned from a previous call to `Get` + // that indicates from where search should continue + PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` + Query string `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` +} + +func (x *SearchRequest) Reset() { + *x = SearchRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SearchRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchRequest) ProtoMessage() {} + +func (x *SearchRequest) ProtoReflect() protoreflect.Message { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead. +func (*SearchRequest) Descriptor() ([]byte, []int) { + return file_ocis_services_search_v0_search_proto_rawDescGZIP(), []int{0} +} + +func (x *SearchRequest) GetPageSize() int32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *SearchRequest) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + +func (x *SearchRequest) GetQuery() string { + if x != nil { + return x.Query + } + return "" +} + +type SearchResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Matches []*v0.Match `protobuf:"bytes,1,rep,name=matches,proto3" json:"matches,omitempty"` + // Token to retrieve the next page of results, or empty if there are no + // more results in the list + NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` +} + +func (x *SearchResponse) Reset() { + *x = SearchResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SearchResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchResponse) ProtoMessage() {} + +func (x *SearchResponse) ProtoReflect() protoreflect.Message { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchResponse.ProtoReflect.Descriptor instead. +func (*SearchResponse) Descriptor() ([]byte, []int) { + return file_ocis_services_search_v0_search_proto_rawDescGZIP(), []int{1} +} + +func (x *SearchResponse) GetMatches() []*v0.Match { + if x != nil { + return x.Matches + } + return nil +} + +func (x *SearchResponse) GetNextPageToken() string { + if x != nil { + return x.NextPageToken + } + return "" +} + +type SearchIndexRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Optional. The maximum number of entries to return in the response + PageSize int32 `protobuf:"varint,1,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + // Optional. A pagination token returned from a previous call to `Get` + // that indicates from where search should continue + PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` + Query string `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + Ref *v0.Reference `protobuf:"bytes,4,opt,name=ref,proto3" json:"ref,omitempty"` +} + +func (x *SearchIndexRequest) Reset() { + *x = SearchIndexRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SearchIndexRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchIndexRequest) ProtoMessage() {} + +func (x *SearchIndexRequest) ProtoReflect() protoreflect.Message { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchIndexRequest.ProtoReflect.Descriptor instead. +func (*SearchIndexRequest) Descriptor() ([]byte, []int) { + return file_ocis_services_search_v0_search_proto_rawDescGZIP(), []int{2} +} + +func (x *SearchIndexRequest) GetPageSize() int32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *SearchIndexRequest) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + +func (x *SearchIndexRequest) GetQuery() string { + if x != nil { + return x.Query + } + return "" +} + +func (x *SearchIndexRequest) GetRef() *v0.Reference { + if x != nil { + return x.Ref + } + return nil +} + +type SearchIndexResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Matches []*v0.Match `protobuf:"bytes,1,rep,name=matches,proto3" json:"matches,omitempty"` + // Token to retrieve the next page of results, or empty if there are no + // more results in the list + NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` +} + +func (x *SearchIndexResponse) Reset() { + *x = SearchIndexResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SearchIndexResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchIndexResponse) ProtoMessage() {} + +func (x *SearchIndexResponse) ProtoReflect() protoreflect.Message { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchIndexResponse.ProtoReflect.Descriptor instead. +func (*SearchIndexResponse) Descriptor() ([]byte, []int) { + return file_ocis_services_search_v0_search_proto_rawDescGZIP(), []int{3} +} + +func (x *SearchIndexResponse) GetMatches() []*v0.Match { + if x != nil { + return x.Matches + } + return nil +} + +func (x *SearchIndexResponse) GetNextPageToken() string { + if x != nil { + return x.NextPageToken + } + return "" +} + +var File_ocis_services_search_v0_search_proto protoreflect.FileDescriptor + +var file_ocis_services_search_v0_search_proto_rawDesc = []byte{ + 0x0a, 0x24, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, + 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x17, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x1a, + 0x24, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x2d, 0x67, 0x65, + 0x6e, 0x2d, 0x6f, 0x70, 0x65, 0x6e, 0x61, 0x70, 0x69, 0x76, 0x32, 0x2f, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, + 0x69, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x62, 0x65, 0x68, 0x61, 0x76, 0x69, 0x6f, 0x72, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, + 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x73, 0x6b, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6d, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x09, 0x70, 0x61, 0x67, 0x65, 0x5f, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, + 0x52, 0x08, 0x70, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x23, 0x0a, 0x0a, 0x70, 0x61, + 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, + 0xe2, 0x41, 0x01, 0x01, 0x52, 0x09, 0x70, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, + 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x72, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, + 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, + 0x76, 0x30, 0x2e, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, + 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, + 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xae, 0x01, 0x0a, 0x12, 0x53, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x21, 0x0a, 0x09, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x08, 0x70, 0x61, 0x67, 0x65, 0x53, + 0x69, 0x7a, 0x65, 0x12, 0x23, 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x09, 0x70, + 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x3a, + 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x63, + 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x42, + 0x04, 0xe2, 0x41, 0x01, 0x01, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0x77, 0x0a, 0x13, 0x53, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x38, 0x0a, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, + 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x32, 0x8d, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x50, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x7b, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x12, 0x26, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, + 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x20, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1a, 0x22, 0x15, 0x2f, 0x61, 0x70, 0x69, 0x2f, + 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x3a, 0x01, 0x2a, 0x32, 0x9d, 0x01, 0x0a, 0x0d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x50, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x8b, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x12, 0x2b, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, + 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, + 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4, + 0x93, 0x02, 0x20, 0x22, 0x1b, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x3a, 0x01, 0x2a, 0x42, 0xde, 0x02, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63, + 0x69, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x2f, 0x76, 0x30, 0x92, 0x41, 0x9c, 0x02, 0x12, 0xb4, 0x01, 0x0a, 0x1e, 0x6f, 0x77, 0x6e, + 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x65, 0x20, 0x53, + 0x63, 0x61, 0x6c, 0x65, 0x20, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x22, 0x47, 0x0a, 0x0d, 0x6f, + 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, 0x48, 0x12, 0x20, 0x68, 0x74, + 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x1a, 0x14, + 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x2e, 0x63, 0x6f, 0x6d, 0x2a, 0x42, 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2d, 0x32, + 0x2e, 0x30, 0x12, 0x34, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, + 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, + 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, 0x2e, 0x30, 0x2e, 0x30, 0x2a, + 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3b, 0x0a, 0x10, 0x44, 0x65, 0x76, 0x65, 0x6c, + 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x12, 0x27, 0x68, 0x74, 0x74, + 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x64, 0x65, + 0x76, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x73, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_ocis_services_search_v0_search_proto_rawDescOnce sync.Once + file_ocis_services_search_v0_search_proto_rawDescData = file_ocis_services_search_v0_search_proto_rawDesc +) + +func file_ocis_services_search_v0_search_proto_rawDescGZIP() []byte { + file_ocis_services_search_v0_search_proto_rawDescOnce.Do(func() { + file_ocis_services_search_v0_search_proto_rawDescData = protoimpl.X.CompressGZIP(file_ocis_services_search_v0_search_proto_rawDescData) + }) + return file_ocis_services_search_v0_search_proto_rawDescData +} + +var file_ocis_services_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_ocis_services_search_v0_search_proto_goTypes = []interface{}{ + (*SearchRequest)(nil), // 0: ocis.services.search.v0.SearchRequest + (*SearchResponse)(nil), // 1: ocis.services.search.v0.SearchResponse + (*SearchIndexRequest)(nil), // 2: ocis.services.search.v0.SearchIndexRequest + (*SearchIndexResponse)(nil), // 3: ocis.services.search.v0.SearchIndexResponse + (*v0.Match)(nil), // 4: ocis.messages.search.v0.Match + (*v0.Reference)(nil), // 5: ocis.messages.search.v0.Reference +} +var file_ocis_services_search_v0_search_proto_depIdxs = []int32{ + 4, // 0: ocis.services.search.v0.SearchResponse.matches:type_name -> ocis.messages.search.v0.Match + 5, // 1: ocis.services.search.v0.SearchIndexRequest.ref:type_name -> ocis.messages.search.v0.Reference + 4, // 2: ocis.services.search.v0.SearchIndexResponse.matches:type_name -> ocis.messages.search.v0.Match + 0, // 3: ocis.services.search.v0.SearchProvider.Search:input_type -> ocis.services.search.v0.SearchRequest + 2, // 4: ocis.services.search.v0.IndexProvider.Search:input_type -> ocis.services.search.v0.SearchIndexRequest + 1, // 5: ocis.services.search.v0.SearchProvider.Search:output_type -> ocis.services.search.v0.SearchResponse + 3, // 6: ocis.services.search.v0.IndexProvider.Search:output_type -> ocis.services.search.v0.SearchIndexResponse + 5, // [5:7] is the sub-list for method output_type + 3, // [3:5] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_ocis_services_search_v0_search_proto_init() } +func file_ocis_services_search_v0_search_proto_init() { + if File_ocis_services_search_v0_search_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_ocis_services_search_v0_search_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SearchRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ocis_services_search_v0_search_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SearchResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ocis_services_search_v0_search_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SearchIndexRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ocis_services_search_v0_search_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SearchIndexResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_ocis_services_search_v0_search_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 2, + }, + GoTypes: file_ocis_services_search_v0_search_proto_goTypes, + DependencyIndexes: file_ocis_services_search_v0_search_proto_depIdxs, + MessageInfos: file_ocis_services_search_v0_search_proto_msgTypes, + }.Build() + File_ocis_services_search_v0_search_proto = out.File + file_ocis_services_search_v0_search_proto_rawDesc = nil + file_ocis_services_search_v0_search_proto_goTypes = nil + file_ocis_services_search_v0_search_proto_depIdxs = nil +} diff --git a/protogen/gen/ocis/services/search/v0/search.pb.micro.go b/protogen/gen/ocis/services/search/v0/search.pb.micro.go new file mode 100644 index 000000000..2c45e6a7e --- /dev/null +++ b/protogen/gen/ocis/services/search/v0/search.pb.micro.go @@ -0,0 +1,180 @@ +// Code generated by protoc-gen-micro. DO NOT EDIT. +// source: ocis/services/search/v0/search.proto + +package v0 + +import ( + fmt "fmt" + _ "github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2/options" + _ "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" + _ "google.golang.org/genproto/googleapis/api/annotations" + proto "google.golang.org/protobuf/proto" + _ "google.golang.org/protobuf/types/known/fieldmaskpb" + math "math" +) + +import ( + context "context" + api "go-micro.dev/v4/api" + client "go-micro.dev/v4/client" + server "go-micro.dev/v4/server" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// Reference imports to suppress errors if they are not otherwise used. +var _ api.Endpoint +var _ context.Context +var _ client.Option +var _ server.Option + +// Api Endpoints for SearchProvider service + +func NewSearchProviderEndpoints() []*api.Endpoint { + return []*api.Endpoint{ + { + Name: "SearchProvider.Search", + Path: []string{"/api/v0/search/search"}, + Method: []string{"POST"}, + Body: "*", + Handler: "rpc", + }, + } +} + +// Client API for SearchProvider service + +type SearchProviderService interface { + Search(ctx context.Context, in *SearchRequest, opts ...client.CallOption) (*SearchResponse, error) +} + +type searchProviderService struct { + c client.Client + name string +} + +func NewSearchProviderService(name string, c client.Client) SearchProviderService { + return &searchProviderService{ + c: c, + name: name, + } +} + +func (c *searchProviderService) Search(ctx context.Context, in *SearchRequest, opts ...client.CallOption) (*SearchResponse, error) { + req := c.c.NewRequest(c.name, "SearchProvider.Search", in) + out := new(SearchResponse) + err := c.c.Call(ctx, req, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for SearchProvider service + +type SearchProviderHandler interface { + Search(context.Context, *SearchRequest, *SearchResponse) error +} + +func RegisterSearchProviderHandler(s server.Server, hdlr SearchProviderHandler, opts ...server.HandlerOption) error { + type searchProvider interface { + Search(ctx context.Context, in *SearchRequest, out *SearchResponse) error + } + type SearchProvider struct { + searchProvider + } + h := &searchProviderHandler{hdlr} + opts = append(opts, api.WithEndpoint(&api.Endpoint{ + Name: "SearchProvider.Search", + Path: []string{"/api/v0/search/search"}, + Method: []string{"POST"}, + Body: "*", + Handler: "rpc", + })) + return s.Handle(s.NewHandler(&SearchProvider{h}, opts...)) +} + +type searchProviderHandler struct { + SearchProviderHandler +} + +func (h *searchProviderHandler) Search(ctx context.Context, in *SearchRequest, out *SearchResponse) error { + return h.SearchProviderHandler.Search(ctx, in, out) +} + +// Api Endpoints for IndexProvider service + +func NewIndexProviderEndpoints() []*api.Endpoint { + return []*api.Endpoint{ + { + Name: "IndexProvider.Search", + Path: []string{"/api/v0/search/index/search"}, + Method: []string{"POST"}, + Body: "*", + Handler: "rpc", + }, + } +} + +// Client API for IndexProvider service + +type IndexProviderService interface { + Search(ctx context.Context, in *SearchIndexRequest, opts ...client.CallOption) (*SearchIndexResponse, error) +} + +type indexProviderService struct { + c client.Client + name string +} + +func NewIndexProviderService(name string, c client.Client) IndexProviderService { + return &indexProviderService{ + c: c, + name: name, + } +} + +func (c *indexProviderService) Search(ctx context.Context, in *SearchIndexRequest, opts ...client.CallOption) (*SearchIndexResponse, error) { + req := c.c.NewRequest(c.name, "IndexProvider.Search", in) + out := new(SearchIndexResponse) + err := c.c.Call(ctx, req, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for IndexProvider service + +type IndexProviderHandler interface { + Search(context.Context, *SearchIndexRequest, *SearchIndexResponse) error +} + +func RegisterIndexProviderHandler(s server.Server, hdlr IndexProviderHandler, opts ...server.HandlerOption) error { + type indexProvider interface { + Search(ctx context.Context, in *SearchIndexRequest, out *SearchIndexResponse) error + } + type IndexProvider struct { + indexProvider + } + h := &indexProviderHandler{hdlr} + opts = append(opts, api.WithEndpoint(&api.Endpoint{ + Name: "IndexProvider.Search", + Path: []string{"/api/v0/search/index/search"}, + Method: []string{"POST"}, + Body: "*", + Handler: "rpc", + })) + return s.Handle(s.NewHandler(&IndexProvider{h}, opts...)) +} + +type indexProviderHandler struct { + IndexProviderHandler +} + +func (h *indexProviderHandler) Search(ctx context.Context, in *SearchIndexRequest, out *SearchIndexResponse) error { + return h.IndexProviderHandler.Search(ctx, in, out) +} diff --git a/protogen/gen/ocis/services/search/v0/search.pb.web.go b/protogen/gen/ocis/services/search/v0/search.pb.web.go new file mode 100644 index 000000000..46105d330 --- /dev/null +++ b/protogen/gen/ocis/services/search/v0/search.pb.web.go @@ -0,0 +1,238 @@ +// Code generated by protoc-gen-microweb. DO NOT EDIT. +// source: v0.proto + +package v0 + +import ( + "bytes" + "encoding/json" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/render" + "github.com/golang/protobuf/jsonpb" +) + +type webSearchProviderHandler struct { + r chi.Router + h SearchProviderHandler +} + +func (h *webSearchProviderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.r.ServeHTTP(w, r) +} + +func (h *webSearchProviderHandler) Search(w http.ResponseWriter, r *http.Request) { + req := &SearchRequest{} + resp := &SearchResponse{} + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusPreconditionFailed) + return + } + + if err := h.h.Search( + r.Context(), + req, + resp, + ); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + render.Status(r, http.StatusCreated) + render.JSON(w, r, resp) +} + +func RegisterSearchProviderWeb(r chi.Router, i SearchProviderHandler, middlewares ...func(http.Handler) http.Handler) { + handler := &webSearchProviderHandler{ + r: r, + h: i, + } + + r.MethodFunc("POST", "/api/v0/search/search", handler.Search) +} + +type webIndexProviderHandler struct { + r chi.Router + h IndexProviderHandler +} + +func (h *webIndexProviderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.r.ServeHTTP(w, r) +} + +func (h *webIndexProviderHandler) Search(w http.ResponseWriter, r *http.Request) { + req := &SearchIndexRequest{} + resp := &SearchIndexResponse{} + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusPreconditionFailed) + return + } + + if err := h.h.Search( + r.Context(), + req, + resp, + ); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + render.Status(r, http.StatusCreated) + render.JSON(w, r, resp) +} + +func RegisterIndexProviderWeb(r chi.Router, i IndexProviderHandler, middlewares ...func(http.Handler) http.Handler) { + handler := &webIndexProviderHandler{ + r: r, + h: i, + } + + r.MethodFunc("POST", "/api/v0/search/index/search", handler.Search) +} + +// SearchRequestJSONMarshaler describes the default jsonpb.Marshaler used by all +// instances of SearchRequest. This struct is safe to replace or modify but +// should not be done so concurrently. +var SearchRequestJSONMarshaler = new(jsonpb.Marshaler) + +// MarshalJSON satisfies the encoding/json Marshaler interface. This method +// uses the more correct jsonpb package to correctly marshal the message. +func (m *SearchRequest) MarshalJSON() ([]byte, error) { + if m == nil { + return json.Marshal(nil) + } + + buf := &bytes.Buffer{} + + if err := SearchRequestJSONMarshaler.Marshal(buf, m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +var _ json.Marshaler = (*SearchRequest)(nil) + +// SearchRequestJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all +// instances of SearchRequest. This struct is safe to replace or modify but +// should not be done so concurrently. +var SearchRequestJSONUnmarshaler = new(jsonpb.Unmarshaler) + +// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method +// uses the more correct jsonpb package to correctly unmarshal the message. +func (m *SearchRequest) UnmarshalJSON(b []byte) error { + return SearchRequestJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m) +} + +var _ json.Unmarshaler = (*SearchRequest)(nil) + +// SearchResponseJSONMarshaler describes the default jsonpb.Marshaler used by all +// instances of SearchResponse. This struct is safe to replace or modify but +// should not be done so concurrently. +var SearchResponseJSONMarshaler = new(jsonpb.Marshaler) + +// MarshalJSON satisfies the encoding/json Marshaler interface. This method +// uses the more correct jsonpb package to correctly marshal the message. +func (m *SearchResponse) MarshalJSON() ([]byte, error) { + if m == nil { + return json.Marshal(nil) + } + + buf := &bytes.Buffer{} + + if err := SearchResponseJSONMarshaler.Marshal(buf, m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +var _ json.Marshaler = (*SearchResponse)(nil) + +// SearchResponseJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all +// instances of SearchResponse. This struct is safe to replace or modify but +// should not be done so concurrently. +var SearchResponseJSONUnmarshaler = new(jsonpb.Unmarshaler) + +// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method +// uses the more correct jsonpb package to correctly unmarshal the message. +func (m *SearchResponse) UnmarshalJSON(b []byte) error { + return SearchResponseJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m) +} + +var _ json.Unmarshaler = (*SearchResponse)(nil) + +// SearchIndexRequestJSONMarshaler describes the default jsonpb.Marshaler used by all +// instances of SearchIndexRequest. This struct is safe to replace or modify but +// should not be done so concurrently. +var SearchIndexRequestJSONMarshaler = new(jsonpb.Marshaler) + +// MarshalJSON satisfies the encoding/json Marshaler interface. This method +// uses the more correct jsonpb package to correctly marshal the message. +func (m *SearchIndexRequest) MarshalJSON() ([]byte, error) { + if m == nil { + return json.Marshal(nil) + } + + buf := &bytes.Buffer{} + + if err := SearchIndexRequestJSONMarshaler.Marshal(buf, m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +var _ json.Marshaler = (*SearchIndexRequest)(nil) + +// SearchIndexRequestJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all +// instances of SearchIndexRequest. This struct is safe to replace or modify but +// should not be done so concurrently. +var SearchIndexRequestJSONUnmarshaler = new(jsonpb.Unmarshaler) + +// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method +// uses the more correct jsonpb package to correctly unmarshal the message. +func (m *SearchIndexRequest) UnmarshalJSON(b []byte) error { + return SearchIndexRequestJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m) +} + +var _ json.Unmarshaler = (*SearchIndexRequest)(nil) + +// SearchIndexResponseJSONMarshaler describes the default jsonpb.Marshaler used by all +// instances of SearchIndexResponse. This struct is safe to replace or modify but +// should not be done so concurrently. +var SearchIndexResponseJSONMarshaler = new(jsonpb.Marshaler) + +// MarshalJSON satisfies the encoding/json Marshaler interface. This method +// uses the more correct jsonpb package to correctly marshal the message. +func (m *SearchIndexResponse) MarshalJSON() ([]byte, error) { + if m == nil { + return json.Marshal(nil) + } + + buf := &bytes.Buffer{} + + if err := SearchIndexResponseJSONMarshaler.Marshal(buf, m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +var _ json.Marshaler = (*SearchIndexResponse)(nil) + +// SearchIndexResponseJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all +// instances of SearchIndexResponse. This struct is safe to replace or modify but +// should not be done so concurrently. +var SearchIndexResponseJSONUnmarshaler = new(jsonpb.Unmarshaler) + +// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method +// uses the more correct jsonpb package to correctly unmarshal the message. +func (m *SearchIndexResponse) UnmarshalJSON(b []byte) error { + return SearchIndexResponseJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m) +} + +var _ json.Unmarshaler = (*SearchIndexResponse)(nil) diff --git a/protogen/gen/ocis/services/search/v0/search.swagger.json b/protogen/gen/ocis/services/search/v0/search.swagger.json new file mode 100644 index 000000000..8b582b7ad --- /dev/null +++ b/protogen/gen/ocis/services/search/v0/search.swagger.json @@ -0,0 +1,254 @@ +{ + "swagger": "2.0", + "info": { + "title": "ownCloud Infinite Scale search", + "version": "1.0.0", + "contact": { + "name": "ownCloud GmbH", + "url": "https://github.com/owncloud/ocis", + "email": "support@owncloud.com" + }, + "license": { + "name": "Apache-2.0", + "url": "https://github.com/owncloud/ocis/blob/master/LICENSE" + } + }, + "tags": [ + { + "name": "SearchProvider" + }, + { + "name": "IndexProvider" + } + ], + "schemes": [ + "http", + "https" + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "paths": { + "/api/v0/search/index/search": { + "post": { + "operationId": "IndexProvider_Search", + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/v0SearchIndexResponse" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/rpcStatus" + } + } + }, + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v0SearchIndexRequest" + } + } + ], + "tags": [ + "IndexProvider" + ] + } + }, + "/api/v0/search/search": { + "post": { + "operationId": "SearchProvider_Search", + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/v0SearchResponse" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/rpcStatus" + } + } + }, + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v0SearchRequest" + } + } + ], + "tags": [ + "SearchProvider" + ] + } + } + }, + "definitions": { + "protobufAny": { + "type": "object", + "properties": { + "@type": { + "type": "string" + } + }, + "additionalProperties": {} + }, + "rpcStatus": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + }, + "details": { + "type": "array", + "items": { + "$ref": "#/definitions/protobufAny" + } + } + } + }, + "v0Entity": { + "type": "object", + "properties": { + "ref": { + "$ref": "#/definitions/v0Reference" + }, + "id": { + "$ref": "#/definitions/v0ResourceID" + }, + "name": { + "type": "string" + }, + "size": { + "type": "string", + "format": "int64" + } + } + }, + "v0Match": { + "type": "object", + "properties": { + "entity": { + "$ref": "#/definitions/v0Entity", + "title": "the matched entity" + }, + "score": { + "type": "number", + "format": "float", + "title": "the match score" + } + } + }, + "v0Reference": { + "type": "object", + "properties": { + "resourceId": { + "$ref": "#/definitions/v0ResourceID" + }, + "path": { + "type": "string" + } + } + }, + "v0ResourceID": { + "type": "object", + "properties": { + "storageId": { + "type": "string" + }, + "opaqueId": { + "type": "string" + } + } + }, + "v0SearchIndexRequest": { + "type": "object", + "properties": { + "pageSize": { + "type": "integer", + "format": "int32", + "title": "Optional. The maximum number of entries to return in the response" + }, + "pageToken": { + "type": "string", + "title": "Optional. A pagination token returned from a previous call to `Get`\nthat indicates from where search should continue" + }, + "query": { + "type": "string" + }, + "ref": { + "$ref": "#/definitions/v0Reference" + } + } + }, + "v0SearchIndexResponse": { + "type": "object", + "properties": { + "matches": { + "type": "array", + "items": { + "$ref": "#/definitions/v0Match" + } + }, + "nextPageToken": { + "type": "string", + "title": "Token to retrieve the next page of results, or empty if there are no\nmore results in the list" + } + } + }, + "v0SearchRequest": { + "type": "object", + "properties": { + "pageSize": { + "type": "integer", + "format": "int32", + "title": "Optional. The maximum number of entries to return in the response" + }, + "pageToken": { + "type": "string", + "title": "Optional. A pagination token returned from a previous call to `Get`\nthat indicates from where search should continue" + }, + "query": { + "type": "string" + } + } + }, + "v0SearchResponse": { + "type": "object", + "properties": { + "matches": { + "type": "array", + "items": { + "$ref": "#/definitions/v0Match" + } + }, + "nextPageToken": { + "type": "string", + "title": "Token to retrieve the next page of results, or empty if there are no\nmore results in the list" + } + } + } + }, + "externalDocs": { + "description": "Developer Manual", + "url": "https://owncloud.dev/extensions/search/" + } +} diff --git a/protogen/proto/ocis/messages/search/search.proto b/protogen/proto/ocis/messages/search/search.proto deleted file mode 100644 index 948e5fd2a..000000000 --- a/protogen/proto/ocis/messages/search/search.proto +++ /dev/null @@ -1,16 +0,0 @@ -syntax = "proto3"; - -package ocis.messages.search.v0; - -option go_package = "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0"; - -message Match { - // key of the recorda - string key = 1; - // value in the record - bytes value = 2; - // time.Duration (signed int64 nanoseconds) - int64 expiry = 3; - // the associated metadata - map metadata = 4; -} \ No newline at end of file diff --git a/protogen/proto/ocis/messages/search/v0/search.proto b/protogen/proto/ocis/messages/search/v0/search.proto new file mode 100644 index 000000000..cf46dd348 --- /dev/null +++ b/protogen/proto/ocis/messages/search/v0/search.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package ocis.messages.search.v0; + +option go_package = "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0"; + +message ResourceID { + string storage_id = 1; + string opaque_id = 2; +} + +message Reference { + ResourceID resource_id = 1; + string path = 2; +} + +message Entity { + Reference ref = 1; + ResourceID id = 2; + string name = 3; + int64 size = 4; +} + +message Match { + // the matched entity + Entity entity = 1; + // the match score + float score = 2; +} \ No newline at end of file diff --git a/protogen/proto/ocis/services/search/search.proto b/protogen/proto/ocis/services/search/search.proto deleted file mode 100644 index 7f95a226d..000000000 --- a/protogen/proto/ocis/services/search/search.proto +++ /dev/null @@ -1,84 +0,0 @@ -syntax = "proto3"; - -package ocis.services.search.v0; - -option go_package = "github.com/owncloud/ocis/protogen/gen/ocis/service/search/v0"; - -import "ocis/messages/search/v0/search.proto"; -import "protoc-gen-openapiv2/options/annotations.proto"; -import "cs3/storage/provider/v1beta1/resources.proto"; - -option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { - info: { - title: "ownCloud Infinite Scale search"; - version: "1.0.0"; - contact: { - name: "ownCloud GmbH"; - url: "https://github.com/owncloud/ocis"; - email: "support@owncloud.com"; - }; - license: { - name: "Apache-2.0"; - url: "https://github.com/owncloud/ocis/blob/master/LICENSE"; - }; - }; - schemes: HTTP; - schemes: HTTPS; - consumes: "application/json"; - produces: "application/json"; - external_docs: { - description: "Developer Manual"; - url: "https://owncloud.dev/extensions/search/"; - }; -}; - -service SearchProvider { - rpc Search(SearchRequest) returns (SearchResponse) {}; -} - -service IndexProvider { - rpc Search(SearchIndexRequest) returns (SearchIndexResponse) {}; - rpc Index(IndexRequest) returns (IndexResponse) {}; - rpc Remove(RemoveRequest) returns (RemoveResponse) {}; -} - -message SearchRequest { - // Optional. The maximum number of entries to return in the response - int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A pagination token returned from a previous call to `Get` - // that indicates from where search should continue - string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Used to specify a subset of fields that should be - // returned by a get operation or modified by an update operation. - google.protobuf.FieldMask field_mask = 3; - string query = 4; -} - -message SearchResponse { - repeated ocis.messages.search.v0.Match matches = 1; - - // Token to retrieve the next page of results, or empty if there are no - // more results in the list - string next_page_token = 2; -} - -message SearchIndexRequest { - // Optional. The maximum number of entries to return in the response - int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A pagination token returned from a previous call to `Get` - // that indicates from where search should continue - string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Used to specify a subset of fields that should be - // returned by a get operation or modified by an update operation. - google.protobuf.FieldMask field_mask = 3; - string query = 4; - string -} - -message SearchIndexResponse { - repeated ocis.messages.search.v0.Record records = 1; -} diff --git a/protogen/proto/ocis/services/search/v0/search.proto b/protogen/proto/ocis/services/search/v0/search.proto new file mode 100644 index 000000000..8bc8fb392 --- /dev/null +++ b/protogen/proto/ocis/services/search/v0/search.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + +package ocis.services.search.v0; + +option go_package = "github.com/owncloud/ocis/protogen/gen/ocis/service/search/v0"; + +import "ocis/messages/search/v0/search.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; +import "google/api/field_behavior.proto"; +import "google/api/annotations.proto"; +import "google/protobuf/field_mask.proto"; + +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + title: "ownCloud Infinite Scale search"; + version: "1.0.0"; + contact: { + name: "ownCloud GmbH"; + url: "https://github.com/owncloud/ocis"; + email: "support@owncloud.com"; + }; + license: { + name: "Apache-2.0"; + url: "https://github.com/owncloud/ocis/blob/master/LICENSE"; + }; + }; + schemes: HTTP; + schemes: HTTPS; + consumes: "application/json"; + produces: "application/json"; + external_docs: { + description: "Developer Manual"; + url: "https://owncloud.dev/extensions/search/"; + }; +}; + +service SearchProvider { + rpc Search(SearchRequest) returns (SearchResponse) { + // List method maps to HTTP GET + option (google.api.http) = { + post: "/api/v0/search/search", + body: "*" + }; + }; +} + +service IndexProvider { + rpc Search(SearchIndexRequest) returns (SearchIndexResponse) { + // List method maps to HTTP GET + option (google.api.http) = { + post: "/api/v0/search/index/search", + body: "*" + }; + }; + // rpc Remove(RemoveRequest) returns (RemoveResponse) {}; +} + +message SearchRequest { + // Optional. The maximum number of entries to return in the response + int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to `Get` + // that indicates from where search should continue + string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + + string query = 3; +} + +message SearchResponse { + repeated ocis.messages.search.v0.Match matches = 1; + + // Token to retrieve the next page of results, or empty if there are no + // more results in the list + string next_page_token = 2; +} + +message SearchIndexRequest { + // Optional. The maximum number of entries to return in the response + int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. A pagination token returned from a previous call to `Get` + // that indicates from where search should continue + string page_token = 2 [(google.api.field_behavior) = OPTIONAL]; + + string query = 3; + ocis.messages.search.v0.Reference ref = 4 [(google.api.field_behavior) = OPTIONAL]; +} + +message SearchIndexResponse { + repeated ocis.messages.search.v0.Match matches = 1; + + // Token to retrieve the next page of results, or empty if there are no + // more results in the list + string next_page_token = 2; +} \ No newline at end of file diff --git a/search/pkg/command/server.go b/search/pkg/command/server.go index cff21ee66..c09ba5196 100644 --- a/search/pkg/command/server.go +++ b/search/pkg/command/server.go @@ -5,13 +5,14 @@ import ( "fmt" "github.com/oklog/run" - "github.com/owncloud/ocis/idp/pkg/server/http" "github.com/owncloud/ocis/ocis-pkg/version" "github.com/owncloud/ocis/search/pkg/config" "github.com/owncloud/ocis/search/pkg/config/parser" "github.com/owncloud/ocis/search/pkg/logging" "github.com/owncloud/ocis/search/pkg/metrics" "github.com/owncloud/ocis/search/pkg/server/debug" + "github.com/owncloud/ocis/search/pkg/server/grpc" + svc "github.com/owncloud/ocis/search/pkg/service/v0" "github.com/owncloud/ocis/search/pkg/tracing" "github.com/urfave/cli/v2" ) @@ -45,47 +46,40 @@ func Server(cfg *config.Config) *cli.Command { mtrcs.BuildInfo.WithLabelValues(version.String).Set(1) - { - server, err := http.Server( - http.Logger(logger), - http.Context(ctx), - http.Config(cfg), - http.Metrics(mtrcs), - ) + handler, err := svc.New(svc.Logger(logger), svc.Config(cfg)) + if err != nil { + logger.Error().Err(err).Msg("handler init") + return err + } + grpcServer := grpc.Server( + grpc.Config(cfg), + grpc.Logger(logger), + grpc.Name(cfg.Service.Name), + grpc.Context(ctx), + grpc.Metrics(mtrcs), + grpc.Handler(handler), + ) - if err != nil { - logger.Info().Err(err).Str("transport", "http").Msg("Failed to initialize server") - return err - } + gr.Add(grpcServer.Run, func(_ error) { + logger.Info().Str("server", "grpc").Msg("shutting down server") + cancel() + }) - gr.Add(func() error { - return server.Run() - }, func(_ error) { - logger.Info(). - Str("transport", "http"). - Msg("Shutting down server") + server, err := debug.Server( + debug.Logger(logger), + debug.Context(ctx), + debug.Config(cfg), + ) - cancel() - }) + if err != nil { + logger.Info().Err(err).Str("transport", "debug").Msg("Failed to initialize server") + return err } - { - server, err := debug.Server( - debug.Logger(logger), - debug.Context(ctx), - debug.Config(cfg), - ) - - if err != nil { - logger.Info().Err(err).Str("transport", "debug").Msg("Failed to initialize server") - return err - } - - gr.Add(server.ListenAndServe, func(_ error) { - _ = server.Shutdown(ctx) - cancel() - }) - } + gr.Add(server.ListenAndServe, func(_ error) { + _ = server.Shutdown(ctx) + cancel() + }) return gr.Run() }, diff --git a/search/pkg/config/config.go b/search/pkg/config/config.go index 2cc26d055..65e85f028 100644 --- a/search/pkg/config/config.go +++ b/search/pkg/config/config.go @@ -17,6 +17,7 @@ type Config struct { Debug Debug `ocisConfig:"debug"` HTTP HTTP `ocisConfig:"http"` + GRPC GRPC `ocisConfig:"grpc"` Reva Reva `ocisConfig:"reva"` TokenManager TokenManager `ocisConfig:"token_manager"` diff --git a/search/pkg/config/defaults/defaultconfig.go b/search/pkg/config/defaults/defaultconfig.go index 8fc349789..b770cabc1 100644 --- a/search/pkg/config/defaults/defaultconfig.go +++ b/search/pkg/config/defaults/defaultconfig.go @@ -17,6 +17,10 @@ func DefaultConfig() *config.Config { Namespace: "com.owncloud.search", Root: "/search", }, + GRPC: config.GRPC{ + Addr: "127.0.0.1:9180", + Namespace: "com.owncloud.api", + }, Service: config.Service{ Name: "search", }, diff --git a/search/pkg/config/grpc.go b/search/pkg/config/grpc.go new file mode 100644 index 000000000..148fda925 --- /dev/null +++ b/search/pkg/config/grpc.go @@ -0,0 +1,7 @@ +package config + +// GRPC defines the available grpc configuration. +type GRPC struct { + Addr string `ocisConfig:"addr" env:"ACCOUNTS_GRPC_ADDR" desc:"The address of the grpc service."` + Namespace string `ocisConfig:"-" yaml:"-"` +} diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index ec8480c2d..5b6b80465 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -35,16 +35,6 @@ type Index struct { bleveIndex bleve.Index } -// Entity describes an Entity stored in the index -type Entity struct { - RootID string - Path string - ID string - - Name string - Size uint64 -} - // NewPersisted returns a new instance of Index with the data being persisted in the given directory func NewPersisted(path string) (*Index, error) { bi, err := bleve.New(path, BuildMapping()) diff --git a/search/pkg/search/mocks/IndexClient.go b/search/pkg/search/mocks/IndexClient.go index 45f1c25e3..e8a62ceca 100644 --- a/search/pkg/search/mocks/IndexClient.go +++ b/search/pkg/search/mocks/IndexClient.go @@ -5,8 +5,9 @@ package mocks import ( context "context" - search "github.com/owncloud/ocis/search/pkg/search" mock "github.com/stretchr/testify/mock" + + v0 "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) // IndexClient is an autogenerated mock type for the IndexClient type @@ -15,20 +16,20 @@ type IndexClient struct { } // Search provides a mock function with given fields: ctx, req -func (_m *IndexClient) Search(ctx context.Context, req *search.SearchIndexRequest) (*search.SearchIndexResult, error) { +func (_m *IndexClient) Search(ctx context.Context, req *v0.SearchIndexRequest) (*v0.SearchIndexResponse, error) { ret := _m.Called(ctx, req) - var r0 *search.SearchIndexResult - if rf, ok := ret.Get(0).(func(context.Context, *search.SearchIndexRequest) *search.SearchIndexResult); ok { + var r0 *v0.SearchIndexResponse + if rf, ok := ret.Get(0).(func(context.Context, *v0.SearchIndexRequest) *v0.SearchIndexResponse); ok { r0 = rf(ctx, req) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*search.SearchIndexResult) + r0 = ret.Get(0).(*v0.SearchIndexResponse) } } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *search.SearchIndexRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *v0.SearchIndexRequest) error); ok { r1 = rf(ctx, req) } else { r1 = ret.Error(1) diff --git a/search/pkg/search/mocks/ProviderClient.go b/search/pkg/search/mocks/ProviderClient.go index 0e097d54e..be426b753 100644 --- a/search/pkg/search/mocks/ProviderClient.go +++ b/search/pkg/search/mocks/ProviderClient.go @@ -5,8 +5,9 @@ package mocks import ( context "context" - search "github.com/owncloud/ocis/search/pkg/search" mock "github.com/stretchr/testify/mock" + + v0 "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) // ProviderClient is an autogenerated mock type for the ProviderClient type @@ -15,20 +16,20 @@ type ProviderClient struct { } // Search provides a mock function with given fields: ctx, req -func (_m *ProviderClient) Search(ctx context.Context, req *search.SearchRequest) (*search.SearchResult, error) { +func (_m *ProviderClient) Search(ctx context.Context, req *v0.SearchRequest) (*v0.SearchResponse, error) { ret := _m.Called(ctx, req) - var r0 *search.SearchResult - if rf, ok := ret.Get(0).(func(context.Context, *search.SearchRequest) *search.SearchResult); ok { + var r0 *v0.SearchResponse + if rf, ok := ret.Get(0).(func(context.Context, *v0.SearchRequest) *v0.SearchResponse); ok { r0 = rf(ctx, req) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*search.SearchResult) + r0 = ret.Get(0).(*v0.SearchResponse) } } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *search.SearchRequest) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *v0.SearchRequest) error); ok { r1 = rf(ctx, req) } else { r1 = ret.Error(1) diff --git a/search/pkg/search/provider/searchprovider.go b/search/pkg/search/provider/searchprovider.go index 0afdc4524..b96358b90 100644 --- a/search/pkg/search/provider/searchprovider.go +++ b/search/pkg/search/provider/searchprovider.go @@ -29,6 +29,9 @@ import ( "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/utils" "github.com/owncloud/ocis/search/pkg/search" + + searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) type Provider struct { @@ -43,7 +46,7 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient) *Pro } } -func (p *Provider) Search(ctx context.Context, req *search.SearchRequest) (*search.SearchResult, error) { +func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) { if req.Query == "" { return nil, errtypes.PreconditionFailed("empty query provided") } @@ -60,7 +63,7 @@ func (p *Provider) Search(ctx context.Context, req *search.SearchRequest) (*sear return nil, err } - matches := []search.Match{} + matches := []*searchmsg.Match{} for _, space := range listSpacesRes.StorageSpaces { pathPrefix := "" if space.SpaceType == "grant" { @@ -76,11 +79,14 @@ func (p *Provider) Search(ctx context.Context, req *search.SearchRequest) (*sear pathPrefix = utils.MakeRelativePath(gpRes.Path) } - res, err := p.indexClient.Search(ctx, &search.SearchIndexRequest{ + res, err := p.indexClient.Search(ctx, &searchsvc.SearchIndexRequest{ Query: req.Query, - Reference: &providerv1beta1.Reference{ - ResourceId: space.Root, - Path: pathPrefix, + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: space.Root.StorageId, + OpaqueId: space.Root.OpaqueId, + }, + Path: pathPrefix, }, }) if err != nil { @@ -89,11 +95,13 @@ func (p *Provider) Search(ctx context.Context, req *search.SearchRequest) (*sear for _, match := range res.Matches { if pathPrefix != "" { - match.Reference.Path = utils.MakeRelativePath(strings.TrimPrefix(match.Reference.Path, pathPrefix)) + match.Entity.Ref.Path = utils.MakeRelativePath(strings.TrimPrefix(match.Entity.Ref.Path, pathPrefix)) } matches = append(matches, match) } } - return &search.SearchResult{Matches: matches}, nil + return &searchsvc.SearchResponse{ + Matches: matches, + }, nil } diff --git a/search/pkg/search/provider/searchprovider_test.go b/search/pkg/search/provider/searchprovider_test.go index 9f57fc45a..575d5d56d 100644 --- a/search/pkg/search/provider/searchprovider_test.go +++ b/search/pkg/search/provider/searchprovider_test.go @@ -30,7 +30,8 @@ import ( typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/v2/pkg/rgrpc/status" cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" - "github.com/owncloud/ocis/search/pkg/search" + searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" "github.com/owncloud/ocis/search/pkg/search/mocks" provider "github.com/owncloud/ocis/search/pkg/search/provider" ) @@ -80,7 +81,7 @@ var _ = Describe("Searchprovider", func() { Describe("Search", func() { It("fails when an empty query is given", func() { - res, err := p.Search(ctx, &search.SearchRequest{ + res, err := p.Search(ctx, &searchsvc.SearchRequest{ Query: "", }) Expect(err).To(HaveOccurred()) @@ -96,19 +97,22 @@ var _ = Describe("Searchprovider", func() { Status: status.NewOK(ctx), StorageSpaces: []*sprovider.StorageSpace{personalSpace}, }, nil) - indexClient.On("Search", mock.Anything, mock.Anything).Return(&search.SearchIndexResult{ - Matches: []search.Match{ + indexClient.On("Search", mock.Anything, mock.Anything).Return(&searchsvc.SearchIndexResponse{ + Matches: []*searchmsg.Match{ { - Reference: &sprovider.Reference{ - ResourceId: personalSpace.Root, - Path: "./path/to/Foo.pdf", - }, - Info: &sprovider.ResourceInfo{ - Id: &sprovider.ResourceId{ + Entity: &searchmsg.Entity{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: personalSpace.Root.StorageId, + OpaqueId: personalSpace.Root.OpaqueId, + }, + Path: "./path/to/Foo.pdf", + }, + Id: &searchmsg.ResourceID{ StorageId: personalSpace.Root.StorageId, OpaqueId: "foo-id", }, - Path: "Foo.pdf", + Name: "Foo.pdf", }, }, }, @@ -116,20 +120,20 @@ var _ = Describe("Searchprovider", func() { }) It("searches the personal user space", func() { - res, err := p.Search(ctx, &search.SearchRequest{ + res, err := p.Search(ctx, &searchsvc.SearchRequest{ Query: "foo", }) Expect(err).ToNot(HaveOccurred()) Expect(res).ToNot(BeNil()) Expect(len(res.Matches)).To(Equal(1)) match := res.Matches[0] - Expect(match.Info.Id.OpaqueId).To(Equal("foo-id")) - Expect(match.Info.Path).To(Equal("Foo.pdf")) - Expect(match.Reference.ResourceId).To(Equal(personalSpace.Root)) - Expect(match.Reference.Path).To(Equal("./path/to/Foo.pdf")) + Expect(match.Entity.Id.OpaqueId).To(Equal("foo-id")) + Expect(match.Entity.Name).To(Equal("Foo.pdf")) + Expect(match.Entity.Ref.ResourceId.OpaqueId).To(Equal(personalSpace.Root.OpaqueId)) + Expect(match.Entity.Ref.Path).To(Equal("./path/to/Foo.pdf")) - indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *search.SearchIndexRequest) bool { - return req.Query == "foo" && req.Reference.ResourceId == personalSpace.Root && req.Reference.Path == "" + indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool { + return req.Query == "foo" && req.Ref.ResourceId.OpaqueId == personalSpace.Root.OpaqueId && req.Ref.Path == "" })) }) }) @@ -161,38 +165,41 @@ var _ = Describe("Searchprovider", func() { Status: status.NewOK(ctx), StorageSpaces: []*sprovider.StorageSpace{grantSpace}, }, nil) - indexClient.On("Search", mock.Anything, mock.Anything).Return(&search.SearchIndexResult{ - Matches: []search.Match{ + indexClient.On("Search", mock.Anything, mock.Anything).Return(&searchsvc.SearchIndexResponse{ + Matches: []*searchmsg.Match{ { - Reference: &sprovider.Reference{ - ResourceId: grantSpace.Root, - Path: "./grant/path/to/Shared.pdf", - }, - Info: &sprovider.ResourceInfo{ - Id: &sprovider.ResourceId{ + Entity: &searchmsg.Entity{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: grantSpace.Root.StorageId, + OpaqueId: grantSpace.Root.OpaqueId, + }, + Path: "./grant/path/to/Shared.pdf", + }, + Id: &searchmsg.ResourceID{ StorageId: grantSpace.Root.StorageId, OpaqueId: "grant-shared-id", }, - Path: "Shared.pdf", + Name: "Shared.pdf", }, }, }, }, nil) - res, err := p.Search(ctx, &search.SearchRequest{ + res, err := p.Search(ctx, &searchsvc.SearchRequest{ Query: "foo", }) Expect(err).ToNot(HaveOccurred()) Expect(res).ToNot(BeNil()) Expect(len(res.Matches)).To(Equal(1)) match := res.Matches[0] - Expect(match.Info.Id.OpaqueId).To(Equal("grant-shared-id")) - Expect(match.Info.Path).To(Equal("Shared.pdf")) - Expect(match.Reference.ResourceId).To(Equal(grantSpace.Root)) - Expect(match.Reference.Path).To(Equal("./to/Shared.pdf")) + Expect(match.Entity.Id.OpaqueId).To(Equal("grant-shared-id")) + Expect(match.Entity.Name).To(Equal("Shared.pdf")) + Expect(match.Entity.Ref.ResourceId.OpaqueId).To(Equal(grantSpace.Root.OpaqueId)) + Expect(match.Entity.Ref.Path).To(Equal("./to/Shared.pdf")) - indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *search.SearchIndexRequest) bool { - return req.Query == "foo" && req.Reference.ResourceId == grantSpace.Root && req.Reference.Path == "./grant/path" + indexClient.AssertCalled(GinkgoT(), "Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool { + return req.Query == "foo" && req.Ref.ResourceId.OpaqueId == grantSpace.Root.OpaqueId && req.Ref.Path == "./grant/path" })) }) @@ -204,52 +211,58 @@ var _ = Describe("Searchprovider", func() { Status: status.NewOK(ctx), StorageSpaces: []*sprovider.StorageSpace{personalSpace, grantSpace}, }, nil) - indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *search.SearchIndexRequest) bool { - return req.Reference.ResourceId == grantSpace.Root - })).Return(&search.SearchIndexResult{ - Matches: []search.Match{ + indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool { + return req.Ref.ResourceId.OpaqueId == grantSpace.Root.OpaqueId + })).Return(&searchsvc.SearchIndexResponse{ + Matches: []*searchmsg.Match{ { - Reference: &sprovider.Reference{ - ResourceId: grantSpace.Root, - Path: "./grant/path/to/Shared.pdf", - }, - Info: &sprovider.ResourceInfo{ - Id: &sprovider.ResourceId{ + Entity: &searchmsg.Entity{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: grantSpace.Root.StorageId, + OpaqueId: grantSpace.Root.OpaqueId, + }, + Path: "./grant/path/to/Shared.pdf", + }, + Id: &searchmsg.ResourceID{ StorageId: grantSpace.Root.StorageId, OpaqueId: "grant-shared-id", }, - Path: "Shared.pdf", + Name: "Shared.pdf", }, }, }, }, nil) - indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *search.SearchIndexRequest) bool { - return req.Reference.ResourceId == personalSpace.Root - })).Return(&search.SearchIndexResult{ - Matches: []search.Match{ + indexClient.On("Search", mock.Anything, mock.MatchedBy(func(req *searchsvc.SearchIndexRequest) bool { + return req.Ref.ResourceId.OpaqueId == personalSpace.Root.OpaqueId + })).Return(&searchsvc.SearchIndexResponse{ + Matches: []*searchmsg.Match{ { - Reference: &sprovider.Reference{ - ResourceId: personalSpace.Root, - Path: "./path/to/Foo.pdf", - }, - Info: &sprovider.ResourceInfo{ - Id: &sprovider.ResourceId{ + Entity: &searchmsg.Entity{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: personalSpace.Root.StorageId, + OpaqueId: personalSpace.Root.OpaqueId, + }, + Path: "./path/to/Foo.pdf", + }, + Id: &searchmsg.ResourceID{ StorageId: personalSpace.Root.StorageId, OpaqueId: "foo-id", }, - Path: "Foo.pdf", + Name: "Foo.pdf", }, }, }, }, nil) - res, err := p.Search(ctx, &search.SearchRequest{ + res, err := p.Search(ctx, &searchsvc.SearchRequest{ Query: "foo", }) Expect(err).ToNot(HaveOccurred()) Expect(res).ToNot(BeNil()) Expect(len(res.Matches)).To(Equal(2)) - ids := []string{res.Matches[0].Info.Id.OpaqueId, res.Matches[1].Info.Id.OpaqueId} + ids := []string{res.Matches[0].Entity.Id.OpaqueId, res.Matches[1].Entity.Id.OpaqueId} Expect(ids).To(ConsistOf("foo-id", "grant-shared-id")) }) }) diff --git a/search/pkg/search/search.go b/search/pkg/search/search.go index 27acf38b0..b28563b77 100644 --- a/search/pkg/search/search.go +++ b/search/pkg/search/search.go @@ -21,45 +21,18 @@ package search import ( "context" - sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) //go:generate mockery --name=ProviderClient //go:generate mockery --name=IndexClient -// SearchRequest represents a search request from a user to the search provider -type SearchRequest struct { - Query string -} - -// Match holds the information of a matched resource in a search -type Match struct { - Reference *sprovider.Reference - Info *sprovider.ResourceInfo -} - -// SearchResult contains the matches being returned for a search -type SearchResult struct { - Matches []Match -} - // ProviderClient is the interface to the search provider service type ProviderClient interface { - Search(ctx context.Context, req *SearchRequest) (*SearchResult, error) -} - -// SearchIndexRequest represents a search request to the index -type SearchIndexRequest struct { - Reference *sprovider.Reference - Query string -} - -// SearchResult contains the matches in the index being returned for a search -type SearchIndexResult struct { - Matches []Match + Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) } // IndexClient is the interface to the search index type IndexClient interface { - Search(ctx context.Context, req *SearchIndexRequest) (*SearchIndexResult, error) + Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) } diff --git a/search/pkg/server/grpc/server.go b/search/pkg/server/grpc/server.go index 48def49bd..d8adea91d 100644 --- a/search/pkg/server/grpc/server.go +++ b/search/pkg/server/grpc/server.go @@ -1,8 +1,6 @@ package grpc import ( - accountssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/accounts/v0" - "github.com/owncloud/ocis/ocis-pkg/service/grpc" "github.com/owncloud/ocis/ocis-pkg/version" ) @@ -22,15 +20,9 @@ func Server(opts ...Option) grpc.Service { grpc.Version(version.String), ) - if err := accountssvc.RegisterAccountsServiceHandler(service.Server(), handler); err != nil { + if err := searchsvc.RegisterSearchServiceHandler(service.Server(), handler); err != nil { options.Logger.Fatal().Err(err).Msg("could not register service handler") } - if err := accountssvc.RegisterGroupsServiceHandler(service.Server(), handler); err != nil { - options.Logger.Fatal().Err(err).Msg("could not register groups handler") - } - if err := accountssvc.RegisterIndexServiceHandler(service.Server(), handler); err != nil { - options.Logger.Fatal().Err(err).Msg("could not register index handler") - } return service } diff --git a/search/pkg/service/v0/option.go b/search/pkg/service/v0/option.go index 4658ba8b0..fca7bd5fc 100644 --- a/search/pkg/service/v0/option.go +++ b/search/pkg/service/v0/option.go @@ -1,10 +1,8 @@ package service import ( - "github.com/owncloud/ocis/accounts/pkg/config" "github.com/owncloud/ocis/ocis-pkg/log" - "github.com/owncloud/ocis/ocis-pkg/roles" - settingssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/settings/v0" + "github.com/owncloud/ocis/search/pkg/config" ) // Option defines a single option function. @@ -12,10 +10,8 @@ type Option func(o *Options) // Options defines the available options for this package. type Options struct { - Logger log.Logger - Config *config.Config - RoleService settingssvc.RoleService - RoleManager *roles.Manager + Logger log.Logger + Config *config.Config } func newOptions(opts ...Option) Options { @@ -41,17 +37,3 @@ func Config(val *config.Config) Option { o.Config = val } } - -// RoleService provides a function to set the RoleService option. -func RoleService(val settingssvc.RoleService) Option { - return func(o *Options) { - o.RoleService = val - } -} - -// RoleManager provides a function to set the RoleManager option. -func RoleManager(val *roles.Manager) Option { - return func(o *Options) { - o.RoleManager = val - } -} diff --git a/search/pkg/service/v0/service.go b/search/pkg/service/v0/service.go index b2d479f74..951d48311 100644 --- a/search/pkg/service/v0/service.go +++ b/search/pkg/service/v0/service.go @@ -1,71 +1,53 @@ package service import ( - "time" - - "github.com/pkg/errors" - - "github.com/owncloud/ocis/ocis-pkg/service/grpc" - - "github.com/owncloud/ocis/ocis-pkg/indexer" + "github.com/blevesearch/bleve/v2" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" "github.com/owncloud/ocis/ocis-pkg/log" - oreg "github.com/owncloud/ocis/ocis-pkg/registry" - "github.com/owncloud/ocis/ocis-pkg/roles" - settingssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/settings/v0" "github.com/owncloud/ocis/search/pkg/config" + "github.com/owncloud/ocis/search/pkg/search" + "github.com/owncloud/ocis/search/pkg/search/index" + searchprovider "github.com/owncloud/ocis/search/pkg/search/provider" ) // userDefaultGID is the default integer representing the "users" group. const userDefaultGID = 30000 // New returns a new instance of Service -func New(opts ...Option) (s *Service, err error) { +func New(opts ...Option) (*Service, error) { options := newOptions(opts...) logger := options.Logger cfg := options.Config - roleService := options.RoleService - if roleService == nil { - roleService = settingssvc.NewRoleService("com.owncloud.api.settings", grpc.DefaultClient) - } - roleManager := options.RoleManager - if roleManager == nil { - m := roles.NewManager( - roles.CacheSize(1024), - roles.CacheTTL(time.Hour*24*7), - roles.Logger(options.Logger), - roles.RoleService(roleService), - ) - roleManager = &m - } - - storage, err := createMetadataStorage(cfg, logger) + bleveIndex, err := bleve.NewMemOnly(index.BuildMapping()) if err != nil { - return nil, errors.Wrap(err, "could not create metadata storage") + return nil, err + } + index, err := index.New(bleveIndex) + if err != nil { + return nil, err } - s = &Service{ - id: cfg.GRPC.Namespace + "." + cfg.Service.Name, - log: logger, - Config: cfg, + gwclient, err := pool.GetGatewayServiceClient(cfg.Reva.Address) + if err != nil { + logger.Fatal().Msgf("could not get reva client at address %s", cfg.Reva.Address) } - r := oreg.GetRegistry() - if cfg.Repo.Backend == "cs3" { - if _, err := r.GetService("com.owncloud.storage.metadata"); err != nil { - logger.Error().Err(err).Msg("index: storage-metadata service not present") - return nil, err - } - } + provider := searchprovider.New(gwclient, index) - return + return &Service{ + id: cfg.GRPC.Namespace + "." + cfg.Service.Name, + log: logger, + Config: cfg, + provider: provider, + }, nil } // Service implements the searchServiceHandler interface type Service struct { - id string - log log.Logger - Config *config.Config - index *indexer.Indexer + id string + log log.Logger + Config *config.Config + provider search.ProviderClient } From f232923d606c0c842984ca75aef736f68c9f5224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 13 Apr 2022 11:47:15 +0000 Subject: [PATCH 10/49] webdav: handle REPORT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- webdav/pkg/service/v0/search.go | 33 ++++++++++++++++++++++++++++++++ webdav/pkg/service/v0/service.go | 5 +++++ 2 files changed, 38 insertions(+) create mode 100644 webdav/pkg/service/v0/search.go diff --git a/webdav/pkg/service/v0/search.go b/webdav/pkg/service/v0/search.go new file mode 100644 index 000000000..3a299db02 --- /dev/null +++ b/webdav/pkg/service/v0/search.go @@ -0,0 +1,33 @@ +package svc + +import ( + "net/http" + + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" + merrors "go-micro.dev/v4/errors" +) + +// Search is the endpoint for retrieving search results for REPORT requests +func (g Webdav) Search(w http.ResponseWriter, r *http.Request) { + + rsp, err := g.searchClient.Search(r.Context(), &searchsvc.SearchRequest{}) + if err != nil { + e := merrors.Parse(err.Error()) + switch e.Code { + case http.StatusBadRequest: + renderError(w, r, errBadRequest(err.Error())) + default: + renderError(w, r, errInternalError(err.Error())) + } + g.log.Error().Err(err).Msg("could not get search results") + return + } + + g.sendSearchResponse(rsp, w, r) +} + +func (g Webdav) sendSearchResponse(rsp *searchsvc.SearchResponse, w http.ResponseWriter, r *http.Request) { + + w.WriteHeader(http.StatusOK) + +} diff --git a/webdav/pkg/service/v0/service.go b/webdav/pkg/service/v0/service.go index c98a3b3af..8d08ac1cf 100644 --- a/webdav/pkg/service/v0/service.go +++ b/webdav/pkg/service/v0/service.go @@ -21,6 +21,7 @@ import ( "github.com/go-chi/chi/v5" thumbnailsmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/thumbnails/v0" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" thumbnailssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/thumbnails/v0" "github.com/owncloud/ocis/webdav/pkg/config" "github.com/owncloud/ocis/webdav/pkg/dav/requests" @@ -62,6 +63,7 @@ func NewService(opts ...Option) (Service, error) { config: conf, log: options.Logger, mux: m, + searchClient: searchsvc.NewSearchProviderService("search", grpc.DefaultClient), thumbnailsClient: thumbnailssvc.NewThumbnailService("com.owncloud.api.thumbnails", grpc.DefaultClient), revaClient: gwc, } @@ -71,6 +73,8 @@ func NewService(opts ...Option) (Service, error) { r.Get("/remote.php/dav/files/{id}/*", svc.Thumbnail) r.Get("/remote.php/dav/public-files/{token}/*", svc.PublicThumbnail) r.Head("/remote.php/dav/public-files/{token}/*", svc.PublicThumbnailHead) + + r.MethodFunc("REPORT", "/remote.php/dav/files/{id}/*", svc.Search) }) return svc, nil @@ -81,6 +85,7 @@ type Webdav struct { config *config.Config log log.Logger mux *chi.Mux + searchClient searchsvc.SearchProviderService thumbnailsClient thumbnailssvc.ThumbnailService revaClient gatewayv1beta1.GatewayAPIClient } From 4c23ced637f603f4961b4762c2fe678c2d7552f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 13 Apr 2022 11:10:09 +0200 Subject: [PATCH 11/49] Fix the index after the protobuf refactoring --- search/pkg/search/index/index.go | 54 +++++++++++++++++---------- search/pkg/search/index/index_test.go | 52 +++++++++++++++----------- 2 files changed, 65 insertions(+), 41 deletions(-) diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index 5b6b80465..5bdd48259 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -25,11 +25,21 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/mapping" - "github.com/owncloud/ocis/search/pkg/search" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) +type indexDocument struct { + RootID string + Path string + ID string + + Name string + Size uint64 +} + // Index represents a bleve based search index type Index struct { bleveIndex bleve.Index @@ -54,19 +64,19 @@ func New(bleveIndex bleve.Index) (*Index, error) { // Add adds a new entity to the Index func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error { entity := toEntity(ref, ri) - return i.bleveIndex.Index(entity.ID, entity) + return i.bleveIndex.Index(idToBleveId(ri.Id), entity) } // Remove removes an entity from the index func (i *Index) Remove(ri *sprovider.ResourceInfo) error { - return i.bleveIndex.Delete(ri.Id.GetStorageId() + ":" + ri.Id.GetOpaqueId()) + return i.bleveIndex.Delete(idToBleveId(ri.Id)) } // Search searches the index according to the criteria specified in the given SearchIndexRequest -func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*search.SearchIndexResult, error) { +func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) { query := bleve.NewConjunctionQuery( bleve.NewQueryStringQuery(req.Query), - bleve.NewQueryStringQuery("Path:"+req.Reference.Path+"*"), // Limit search to this directory in the space + bleve.NewQueryStringQuery("Path:"+req.Ref.Path+"*"), // Limit search to this directory in the space ) bleveReq := bleve.NewSearchRequest(query) bleveReq.Fields = []string{"*"} @@ -75,7 +85,7 @@ func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*se return nil, err } - matches := []search.Match{} + matches := []*searchmsg.Match{} for _, h := range res.Hits { match, err := fromFields(h.Fields) if err != nil { @@ -84,7 +94,7 @@ func (i *Index) Search(ctx context.Context, req *search.SearchIndexRequest) (*se matches = append(matches, match) } - return &search.SearchIndexResult{ + return &searchsvc.SearchIndexResponse{ Matches: matches, }, nil } @@ -96,8 +106,8 @@ func BuildMapping() mapping.IndexMapping { return indexMapping } -func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *Entity { - return &Entity{ +func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocument { + return &indexDocument{ RootID: ref.ResourceId.GetStorageId() + ":" + ref.ResourceId.GetOpaqueId(), Path: ref.Path, ID: ri.Id.GetStorageId() + ":" + ri.Id.GetOpaqueId(), @@ -106,25 +116,29 @@ func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *Entity { } } -func fromFields(fields map[string]interface{}) (search.Match, error) { +func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) { rootIDParts := strings.SplitN(fields["RootID"].(string), ":", 2) IDParts := strings.SplitN(fields["ID"].(string), ":", 2) - return search.Match{ - Reference: &sprovider.Reference{ - ResourceId: &sprovider.ResourceId{ - StorageId: rootIDParts[0], - OpaqueId: rootIDParts[1], + return &searchmsg.Match{ + Entity: &searchmsg.Entity{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rootIDParts[0], + OpaqueId: rootIDParts[1], + }, + Path: fields["Path"].(string), }, - Path: fields["Path"].(string), - }, - Info: &sprovider.ResourceInfo{ - Id: &sprovider.ResourceId{ + Id: &searchmsg.ResourceID{ StorageId: IDParts[0], OpaqueId: IDParts[1], }, - Path: fields["Name"].(string), + Name: fields["Name"].(string), Size: uint64(fields["Size"].(float64)), }, }, nil } + +func idToBleveId(id *sprovider.ResourceId) string { + return id.StorageId + ":" + id.OpaqueId +} diff --git a/search/pkg/search/index/index_test.go b/search/pkg/search/index/index_test.go index 37bf24468..5e11cab2e 100644 --- a/search/pkg/search/index/index_test.go +++ b/search/pkg/search/index/index_test.go @@ -5,7 +5,8 @@ import ( "github.com/blevesearch/bleve/v2" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/owncloud/ocis/search/pkg/search" + searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" "github.com/owncloud/ocis/search/pkg/search/index" . "github.com/onsi/ginkgo/v2" @@ -73,20 +74,23 @@ var _ = Describe("Index", func() { It("finds files by name, prefix or substring match", func() { queries := []string{"foo.pdf", "foo*", "*oo.p*"} for _, query := range queries { - res, err := i.Search(ctx, &search.SearchIndexRequest{ - Reference: &sprovider.Reference{ - ResourceId: ref.ResourceId, + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: ref.ResourceId.StorageId, + OpaqueId: ref.ResourceId.OpaqueId, + }, }, Query: query, }) Expect(err).ToNot(HaveOccurred()) Expect(res).ToNot(BeNil()) Expect(len(res.Matches)).To(Equal(1), "query returned no result: "+query) - Expect(res.Matches[0].Reference.ResourceId).To(Equal(ref.ResourceId)) - Expect(res.Matches[0].Reference.Path).To(Equal(ref.Path)) - Expect(res.Matches[0].Info.Id).To(Equal(ri.Id)) - Expect(res.Matches[0].Info.Path).To(Equal(ri.Path)) - Expect(res.Matches[0].Info.Size).To(Equal(ri.Size)) + Expect(res.Matches[0].Entity.Ref.ResourceId.OpaqueId).To(Equal(ref.ResourceId.OpaqueId)) + Expect(res.Matches[0].Entity.Ref.Path).To(Equal(ref.Path)) + Expect(res.Matches[0].Entity.Id.OpaqueId).To(Equal(ri.Id.OpaqueId)) + Expect(res.Matches[0].Entity.Name).To(Equal(ri.Path)) + Expect(res.Matches[0].Entity.Size).To(Equal(ri.Size)) } }) @@ -119,28 +123,34 @@ var _ = Describe("Index", func() { It("finds files living deeper in the tree by filename, prefix or substring match", func() { queries := []string{"nestedpdf.pdf", "nested*", "*tedpdf.*"} for _, query := range queries { - res, err := i.Search(ctx, &search.SearchIndexRequest{ - Reference: &sprovider.Reference{ - ResourceId: ref.ResourceId, + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: ref.ResourceId.StorageId, + OpaqueId: ref.ResourceId.OpaqueId, + }, }, Query: query, }) Expect(err).ToNot(HaveOccurred()) Expect(res).ToNot(BeNil()) Expect(len(res.Matches)).To(Equal(1), "query returned no result: "+query) - Expect(res.Matches[0].Reference.ResourceId).To(Equal(nestedRef.ResourceId)) - Expect(res.Matches[0].Reference.Path).To(Equal(nestedRef.Path)) - Expect(res.Matches[0].Info.Id).To(Equal(nestedRI.Id)) - Expect(res.Matches[0].Info.Path).To(Equal(nestedRI.Path)) - Expect(res.Matches[0].Info.Size).To(Equal(nestedRI.Size)) + Expect(res.Matches[0].Entity.Ref.ResourceId.OpaqueId).To(Equal(nestedRef.ResourceId.OpaqueId)) + Expect(res.Matches[0].Entity.Ref.Path).To(Equal(nestedRef.Path)) + Expect(res.Matches[0].Entity.Id.OpaqueId).To(Equal(nestedRI.Id.OpaqueId)) + Expect(res.Matches[0].Entity.Name).To(Equal(nestedRI.Path)) + Expect(res.Matches[0].Entity.Size).To(Equal(nestedRI.Size)) } }) It("does not find the higher levels when limiting the searched directory", func() { - res, err := i.Search(ctx, &search.SearchIndexRequest{ - Reference: &sprovider.Reference{ - ResourceId: ref.ResourceId, - Path: "./nested/", + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: ref.ResourceId.StorageId, + OpaqueId: ref.ResourceId.OpaqueId, + }, + Path: "./nested/", }, Query: "foo.pdf", }) From a737f4e0a7d8bc0630fee6ee8f9cd49269c36fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 13 Apr 2022 15:17:08 +0200 Subject: [PATCH 12/49] Fix type, hook up proper service --- protogen/gen/ocis/messages/search/v0/search.pb.go | 6 +++--- protogen/gen/ocis/services/search/v0/search.swagger.json | 2 +- protogen/proto/ocis/messages/search/v0/search.proto | 2 +- search/pkg/server/grpc/server.go | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/protogen/gen/ocis/messages/search/v0/search.pb.go b/protogen/gen/ocis/messages/search/v0/search.pb.go index 9e22a09d8..5e27039f5 100644 --- a/protogen/gen/ocis/messages/search/v0/search.pb.go +++ b/protogen/gen/ocis/messages/search/v0/search.pb.go @@ -138,7 +138,7 @@ type Entity struct { Ref *Reference `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` Id *ResourceID `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` - Size int64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` + Size uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` } func (x *Entity) Reset() { @@ -194,7 +194,7 @@ func (x *Entity) GetName() string { return "" } -func (x *Entity) GetSize() int64 { +func (x *Entity) GetSize() uint64 { if x != nil { return x.Size } @@ -285,7 +285,7 @@ var file_ocis_messages_search_v0_search_proto_rawDesc = []byte{ 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, - 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x56, + 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x56, 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x37, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, diff --git a/protogen/gen/ocis/services/search/v0/search.swagger.json b/protogen/gen/ocis/services/search/v0/search.swagger.json index 8b582b7ad..dab8aa3ac 100644 --- a/protogen/gen/ocis/services/search/v0/search.swagger.json +++ b/protogen/gen/ocis/services/search/v0/search.swagger.json @@ -139,7 +139,7 @@ }, "size": { "type": "string", - "format": "int64" + "format": "uint64" } } }, diff --git a/protogen/proto/ocis/messages/search/v0/search.proto b/protogen/proto/ocis/messages/search/v0/search.proto index cf46dd348..45b40df24 100644 --- a/protogen/proto/ocis/messages/search/v0/search.proto +++ b/protogen/proto/ocis/messages/search/v0/search.proto @@ -18,7 +18,7 @@ message Entity { Reference ref = 1; ResourceID id = 2; string name = 3; - int64 size = 4; + uint64 size = 4; } message Match { diff --git a/search/pkg/server/grpc/server.go b/search/pkg/server/grpc/server.go index d8adea91d..1685b7423 100644 --- a/search/pkg/server/grpc/server.go +++ b/search/pkg/server/grpc/server.go @@ -3,6 +3,7 @@ package grpc import ( "github.com/owncloud/ocis/ocis-pkg/service/grpc" "github.com/owncloud/ocis/ocis-pkg/version" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) // Server initializes a new go-micro service ready to run @@ -20,7 +21,7 @@ func Server(opts ...Option) grpc.Service { grpc.Version(version.String), ) - if err := searchsvc.RegisterSearchServiceHandler(service.Server(), handler); err != nil { + if err := searchsvc.RegisterSearchProviderHandler(service.Server(), handler); err != nil { options.Logger.Fatal().Err(err).Msg("could not register service handler") } From b4017a04510f60523a628a8b58270fba43466709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 13 Apr 2022 15:22:02 +0000 Subject: [PATCH 13/49] minimal report handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- docs/extensions/port-ranges.md | 2 +- docs/extensions/storage/ports.md | 7 +- ocis-pkg/config/config.go | 2 + ocis-pkg/config/defaultconfig.go | 2 + ocis/pkg/runtime/service/service.go | 2 + .../gen/ocis/messages/search/v0/search.pb.go | 4 +- .../gen/ocis/messages/store/v0/store.pb.go | 4 +- .../ocis/services/accounts/v0/accounts.pb.go | 4 +- search/pkg/command/server.go | 10 +- search/pkg/command/version.go | 2 +- search/pkg/config/config.go | 1 - search/pkg/config/defaults/defaultconfig.go | 16 +- search/pkg/search/index/index.go | 10 +- search/pkg/server/debug/server.go | 4 - search/pkg/server/grpc/server.go | 18 +- search/pkg/service/v0/service.go | 16 +- webdav/pkg/service/v0/search.go | 164 +++++++++++++++++- webdav/pkg/service/v0/service.go | 7 +- 18 files changed, 217 insertions(+), 58 deletions(-) diff --git a/docs/extensions/port-ranges.md b/docs/extensions/port-ranges.md index ac28f54c5..bc3143c08 100644 --- a/docs/extensions/port-ranges.md +++ b/docs/extensions/port-ranges.md @@ -42,7 +42,7 @@ We also suggest to use the last port in your extensions' range as a debug/metric | 9205-9209 | [markdown-editor](https://github.com/owncloud/ocis-markdown-editor) | | 9210-9214 | [reva](https://github.com/owncloud/ocis-reva) unused? | | 9215-9219 | reva metadata storage | -| 9220-9224 | FREE | +| 9220-9224 | search | | 9225-9229 | photoprism (state: PoC) | | 9230-9234 | [nats](https://github.com/owncloud/ocis/tree/master/nats) | | 9235-9239 | idm TBD | diff --git a/docs/extensions/storage/ports.md b/docs/extensions/storage/ports.md index 2ab34f5dd..62bd6ac82 100644 --- a/docs/extensions/storage/ports.md +++ b/docs/extensions/storage/ports.md @@ -34,10 +34,13 @@ For now, the storage service uses these ports to preconfigure those services: | 9159 | storage users debug | | 9160 | groups | | 9161 | groups debug | -| 9164 | storage appprovider | -| 9165 | storage appprovider debug | +| 9164 | storage appprovider | +| 9165 | storage appprovider debug | | 9178 | storage public link | | 9179 | storage public link data | +| 9180 | accounts grpc | +| 9181 | accounts http | +| 9182 | accounts debug | | 9215 | storage meta grpc | | 9216 | storage meta http | | 9217 | storage meta debug | diff --git a/ocis-pkg/config/config.go b/ocis-pkg/config/config.go index e708b9d18..60544abe9 100644 --- a/ocis-pkg/config/config.go +++ b/ocis-pkg/config/config.go @@ -14,6 +14,7 @@ import ( notifications "github.com/owncloud/ocis/notifications/pkg/config" ocs "github.com/owncloud/ocis/ocs/pkg/config" proxy "github.com/owncloud/ocis/proxy/pkg/config" + search "github.com/owncloud/ocis/search/pkg/config" settings "github.com/owncloud/ocis/settings/pkg/config" storage "github.com/owncloud/ocis/storage/pkg/config" store "github.com/owncloud/ocis/store/pkg/config" @@ -71,6 +72,7 @@ type Config struct { OCS *ocs.Config `yaml:"ocs"` Web *web.Config `yaml:"web"` Proxy *proxy.Config `yaml:"proxy"` + Search *search.Config `yaml:"search"` Settings *settings.Config `yaml:"settings"` Storage *storage.Config `yaml:"storage"` Store *store.Config `yaml:"store"` diff --git a/ocis-pkg/config/defaultconfig.go b/ocis-pkg/config/defaultconfig.go index 07ce28407..2ea23c401 100644 --- a/ocis-pkg/config/defaultconfig.go +++ b/ocis-pkg/config/defaultconfig.go @@ -12,6 +12,7 @@ import ( notifications "github.com/owncloud/ocis/notifications/pkg/config/defaults" ocs "github.com/owncloud/ocis/ocs/pkg/config/defaults" proxy "github.com/owncloud/ocis/proxy/pkg/config/defaults" + search "github.com/owncloud/ocis/search/pkg/config/defaults" settings "github.com/owncloud/ocis/settings/pkg/config/defaults" storage "github.com/owncloud/ocis/storage/pkg/config/defaults" store "github.com/owncloud/ocis/store/pkg/config/defaults" @@ -40,6 +41,7 @@ func DefaultConfig() *Config { Proxy: proxy.DefaultConfig(), GraphExplorer: graphExplorer.DefaultConfig(), OCS: ocs.DefaultConfig(), + Search: search.DefaultConfig(), Settings: settings.DefaultConfig(), Web: web.DefaultConfig(), Store: store.DefaultConfig(), diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index 20fcf21f5..db5b9f72b 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -32,6 +32,7 @@ import ( "github.com/owncloud/ocis/ocis-pkg/log" ocs "github.com/owncloud/ocis/ocs/pkg/command" proxy "github.com/owncloud/ocis/proxy/pkg/command" + search "github.com/owncloud/ocis/search/pkg/command" settings "github.com/owncloud/ocis/settings/pkg/command" storage "github.com/owncloud/ocis/storage/pkg/command" store "github.com/owncloud/ocis/store/pkg/command" @@ -120,6 +121,7 @@ func NewService(options ...Option) (*Service, error) { s.ServicesRegistry["storage-public-link"] = storage.NewStoragePublicLink s.ServicesRegistry["storage-appprovider"] = storage.NewAppProvider s.ServicesRegistry["notifications"] = notifications.NewSutureService + s.ServicesRegistry["search"] = search.NewSutureService // populate delayed services s.Delayed["storage-sharing"] = storage.NewSharing diff --git a/protogen/gen/ocis/messages/search/v0/search.pb.go b/protogen/gen/ocis/messages/search/v0/search.pb.go index 5e27039f5..66f5d25dd 100644 --- a/protogen/gen/ocis/messages/search/v0/search.pb.go +++ b/protogen/gen/ocis/messages/search/v0/search.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc (unknown) // source: ocis/messages/search/v0/search.proto package v0 diff --git a/protogen/gen/ocis/messages/store/v0/store.pb.go b/protogen/gen/ocis/messages/store/v0/store.pb.go index 1471dda61..0783db6ec 100644 --- a/protogen/gen/ocis/messages/store/v0/store.pb.go +++ b/protogen/gen/ocis/messages/store/v0/store.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc (unknown) // source: ocis/messages/store/v0/store.proto package v0 diff --git a/protogen/gen/ocis/services/accounts/v0/accounts.pb.go b/protogen/gen/ocis/services/accounts/v0/accounts.pb.go index bc4f5c54d..e5ff1ca4c 100644 --- a/protogen/gen/ocis/services/accounts/v0/accounts.pb.go +++ b/protogen/gen/ocis/services/accounts/v0/accounts.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc (unknown) // source: ocis/services/accounts/v0/accounts.proto package v0 diff --git a/search/pkg/command/server.go b/search/pkg/command/server.go index c09ba5196..03fff3956 100644 --- a/search/pkg/command/server.go +++ b/search/pkg/command/server.go @@ -12,7 +12,6 @@ import ( "github.com/owncloud/ocis/search/pkg/metrics" "github.com/owncloud/ocis/search/pkg/server/debug" "github.com/owncloud/ocis/search/pkg/server/grpc" - svc "github.com/owncloud/ocis/search/pkg/service/v0" "github.com/owncloud/ocis/search/pkg/tracing" "github.com/urfave/cli/v2" ) @@ -40,24 +39,17 @@ func Server(cfg *config.Config) *cli.Command { } return context.WithCancel(cfg.Context) }() - mtrcs := metrics.New() - defer cancel() + mtrcs := metrics.New() mtrcs.BuildInfo.WithLabelValues(version.String).Set(1) - handler, err := svc.New(svc.Logger(logger), svc.Config(cfg)) - if err != nil { - logger.Error().Err(err).Msg("handler init") - return err - } grpcServer := grpc.Server( grpc.Config(cfg), grpc.Logger(logger), grpc.Name(cfg.Service.Name), grpc.Context(ctx), grpc.Metrics(mtrcs), - grpc.Handler(handler), ) gr.Add(grpcServer.Run, func(_ error) { diff --git a/search/pkg/command/version.go b/search/pkg/command/version.go index 07a4959c8..86a4c7ee9 100644 --- a/search/pkg/command/version.go +++ b/search/pkg/command/version.go @@ -24,7 +24,7 @@ func Version(cfg *config.Config) *cli.Command { fmt.Println("") reg := registry.GetRegistry() - services, err := reg.GetService(cfg.HTTP.Namespace + "." + cfg.Service.Name) + services, err := reg.GetService(cfg.GRPC.Namespace + "." + cfg.Service.Name) if err != nil { fmt.Println(fmt.Errorf("could not get %s services from the registry: %v", cfg.Service.Name, err)) return err diff --git a/search/pkg/config/config.go b/search/pkg/config/config.go index 65e85f028..706a09cd9 100644 --- a/search/pkg/config/config.go +++ b/search/pkg/config/config.go @@ -16,7 +16,6 @@ type Config struct { Log *Log `ocisConfig:"log"` Debug Debug `ocisConfig:"debug"` - HTTP HTTP `ocisConfig:"http"` GRPC GRPC `ocisConfig:"grpc"` Reva Reva `ocisConfig:"reva"` diff --git a/search/pkg/config/defaults/defaultconfig.go b/search/pkg/config/defaults/defaultconfig.go index b770cabc1..d1f212bbe 100644 --- a/search/pkg/config/defaults/defaultconfig.go +++ b/search/pkg/config/defaults/defaultconfig.go @@ -1,24 +1,17 @@ package defaults import ( - "strings" - "github.com/owncloud/ocis/search/pkg/config" ) func DefaultConfig() *config.Config { return &config.Config{ Debug: config.Debug{ - Addr: "127.0.0.1:9124", + Addr: "127.0.0.1:9224", Token: "", }, - HTTP: config.HTTP{ - Addr: "127.0.0.1:9120", - Namespace: "com.owncloud.search", - Root: "/search", - }, GRPC: config.GRPC{ - Addr: "127.0.0.1:9180", + Addr: "127.0.0.1:9220", Namespace: "com.owncloud.api", }, Service: config.Service{ @@ -59,8 +52,5 @@ func EnsureDefaults(cfg *config.Config) { } func Sanitize(cfg *config.Config) { - // sanitize config - if cfg.HTTP.Root != "/" { - cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/") - } + // no http endpoint to be sanitized } diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index 5bdd48259..ed6d004e4 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -108,17 +108,17 @@ func BuildMapping() mapping.IndexMapping { func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocument { return &indexDocument{ - RootID: ref.ResourceId.GetStorageId() + ":" + ref.ResourceId.GetOpaqueId(), + RootID: ref.ResourceId.GetStorageId() + "!" + ref.ResourceId.GetOpaqueId(), Path: ref.Path, - ID: ri.Id.GetStorageId() + ":" + ri.Id.GetOpaqueId(), + ID: ri.Id.GetStorageId() + "!" + ri.Id.GetOpaqueId(), Name: ri.Path, Size: ri.Size, } } func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) { - rootIDParts := strings.SplitN(fields["RootID"].(string), ":", 2) - IDParts := strings.SplitN(fields["ID"].(string), ":", 2) + rootIDParts := strings.SplitN(fields["RootID"].(string), "!", 2) + IDParts := strings.SplitN(fields["ID"].(string), "!", 2) return &searchmsg.Match{ Entity: &searchmsg.Entity{ @@ -140,5 +140,5 @@ func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) { } func idToBleveId(id *sprovider.ResourceId) string { - return id.StorageId + ":" + id.OpaqueId + return id.StorageId + "!" + id.OpaqueId } diff --git a/search/pkg/server/debug/server.go b/search/pkg/server/debug/server.go index 9b69aa0fe..88b51662b 100644 --- a/search/pkg/server/debug/server.go +++ b/search/pkg/server/debug/server.go @@ -23,10 +23,6 @@ func Server(opts ...Option) (*http.Server, error) { debug.Zpages(options.Config.Debug.Zpages), debug.Health(health(options.Config)), debug.Ready(ready(options.Config)), - debug.CorsAllowedOrigins(options.Config.HTTP.CORS.AllowedOrigins), - debug.CorsAllowedMethods(options.Config.HTTP.CORS.AllowedMethods), - debug.CorsAllowedHeaders(options.Config.HTTP.CORS.AllowedHeaders), - debug.CorsAllowCredentials(options.Config.HTTP.CORS.AllowCredentials), ), nil } diff --git a/search/pkg/server/grpc/server.go b/search/pkg/server/grpc/server.go index 1685b7423..e2fe642b1 100644 --- a/search/pkg/server/grpc/server.go +++ b/search/pkg/server/grpc/server.go @@ -4,12 +4,12 @@ import ( "github.com/owncloud/ocis/ocis-pkg/service/grpc" "github.com/owncloud/ocis/ocis-pkg/version" searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" + svc "github.com/owncloud/ocis/search/pkg/service/v0" ) // Server initializes a new go-micro service ready to run func Server(opts ...Option) grpc.Service { options := newOptions(opts...) - handler := options.Handler service := grpc.NewService( grpc.Name(options.Config.Service.Name), @@ -21,9 +21,19 @@ func Server(opts ...Option) grpc.Service { grpc.Version(version.String), ) - if err := searchsvc.RegisterSearchProviderHandler(service.Server(), handler); err != nil { - options.Logger.Fatal().Err(err).Msg("could not register service handler") + handle, err := svc.NewHandler( + svc.Config(options.Config), + svc.Logger(options.Logger), + ) + if err != nil { + options.Logger.Error(). + Err(err). + Msg("Error initializing search service") + return grpc.Service{} } - + _ = searchsvc.RegisterSearchProviderHandler( + service.Server(), + handle, + ) return service } diff --git a/search/pkg/service/v0/service.go b/search/pkg/service/v0/service.go index 951d48311..319d97112 100644 --- a/search/pkg/service/v0/service.go +++ b/search/pkg/service/v0/service.go @@ -1,21 +1,21 @@ package service import ( + "context" + "github.com/blevesearch/bleve/v2" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" "github.com/owncloud/ocis/ocis-pkg/log" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" "github.com/owncloud/ocis/search/pkg/config" "github.com/owncloud/ocis/search/pkg/search" "github.com/owncloud/ocis/search/pkg/search/index" searchprovider "github.com/owncloud/ocis/search/pkg/search/provider" ) -// userDefaultGID is the default integer representing the "users" group. -const userDefaultGID = 30000 - -// New returns a new instance of Service -func New(opts ...Option) (*Service, error) { +// NewHandler returns a service implementation for Service. +func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, error) { options := newOptions(opts...) logger := options.Logger cfg := options.Config @@ -31,7 +31,7 @@ func New(opts ...Option) (*Service, error) { gwclient, err := pool.GetGatewayServiceClient(cfg.Reva.Address) if err != nil { - logger.Fatal().Msgf("could not get reva client at address %s", cfg.Reva.Address) + logger.Fatal().Err(err).Str("addr", cfg.Reva.Address).Msg("could not get reva client") } provider := searchprovider.New(gwclient, index) @@ -51,3 +51,7 @@ type Service struct { Config *config.Config provider search.ProviderClient } + +func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *searchsvc.SearchResponse) error { + return nil +} diff --git a/webdav/pkg/service/v0/search.go b/webdav/pkg/service/v0/search.go index 3a299db02..cfa305a6c 100644 --- a/webdav/pkg/service/v0/search.go +++ b/webdav/pkg/service/v0/search.go @@ -1,16 +1,38 @@ package svc import ( + "context" + "encoding/xml" + "io" "net/http" + "strconv" + searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" + "github.com/owncloud/ocis/webdav/pkg/net" + "github.com/owncloud/ocis/webdav/pkg/prop" + "github.com/owncloud/ocis/webdav/pkg/propfind" merrors "go-micro.dev/v4/errors" ) +const ( + elementNameSearchFiles = "search-files" + // TODO elementNameFilterFiles = "filter-files" +) + // Search is the endpoint for retrieving search results for REPORT requests func (g Webdav) Search(w http.ResponseWriter, r *http.Request) { - rsp, err := g.searchClient.Search(r.Context(), &searchsvc.SearchRequest{}) + rep, err := readReport(r.Body) + if err != nil { + renderError(w, r, errBadRequest(err.Error())) + g.log.Error().Err(err).Msg("error reading report") + return + } + + rsp, err := g.searchClient.Search(r.Context(), &searchsvc.SearchRequest{ + Query: rep.SearchFiles.Search.Pattern, + }) if err != nil { e := merrors.Parse(err.Error()) switch e.Code { @@ -28,6 +50,142 @@ func (g Webdav) Search(w http.ResponseWriter, r *http.Request) { func (g Webdav) sendSearchResponse(rsp *searchsvc.SearchResponse, w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - + responsesXML, err := multistatusResponse(r.Context(), rsp.Matches) + if err != nil { + g.log.Error().Err(err).Msg("error formatting propfind") + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set(net.HeaderDav, "1, 3, extended-mkcol") + w.Header().Set(net.HeaderContentType, "application/xml; charset=utf-8") + w.WriteHeader(http.StatusMultiStatus) + if _, err := w.Write(responsesXML); err != nil { + g.log.Err(err).Msg("error writing response") + } +} + +// multistatusResponse converts a list of matches into a multistatus response string +func multistatusResponse(ctx context.Context, matches []*searchmsg.Match) ([]byte, error) { + responses := make([]*propfind.ResponseXML, 0, len(matches)) + for i := range matches { + res, err := matchToPropResponse(ctx, matches[i]) + if err != nil { + return nil, err + } + responses = append(responses, res) + } + + msr := propfind.NewMultiStatusResponseXML() + msr.Responses = responses + msg, err := xml.Marshal(msr) + if err != nil { + return nil, err + } + return msg, nil +} + +func matchToPropResponse(ctx context.Context, match *searchmsg.Match) (*propfind.ResponseXML, error) { + + response := propfind.ResponseXML{ + Href: net.EncodePath(match.Entity.Ref.Path), + Propstat: []propfind.PropstatXML{}, + } + + propstatOK := propfind.PropstatXML{ + Status: "HTTP/1.1 200 OK", + Prop: []prop.PropertyXML{}, + } + + propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:id", match.Entity.Id.StorageId+"!"+match.Entity.Id.OpaqueId)) + + size := strconv.FormatUint(match.Entity.Size, 10) + propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:size", size)) + + // TODO find name for score property + score := strconv.FormatFloat(float64(match.Score), 'f', -1, 64) + propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:score", score)) + + if len(propstatOK.Prop) > 0 { + response.Propstat = append(response.Propstat, propstatOK) + } + + return &response, nil +} + +type report struct { + SearchFiles *reportSearchFiles + // FilterFiles TODO add this for tag based search + FilterFiles *reportFilterFiles `xml:"filter-files"` +} +type reportSearchFiles struct { + XMLName xml.Name `xml:"search-files"` + Lang string `xml:"xml:lang,attr,omitempty"` + Prop Props `xml:"DAV: prop"` + Search reportSearchFilesSearch `xml:"search"` +} +type reportSearchFilesSearch struct { + Pattern string `xml:"search"` + Limit int `xml:"limit"` + Offset int `xml:"offset"` +} + +type reportFilterFiles struct { + XMLName xml.Name `xml:"filter-files"` + Lang string `xml:"xml:lang,attr,omitempty"` + Prop Props `xml:"DAV: prop"` + Rules reportFilterFilesRules `xml:"filter-rules"` +} + +type reportFilterFilesRules struct { + Favorite bool `xml:"favorite"` + SystemTag int `xml:"systemtag"` +} + +// Props represents properties related to a resource +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_prop (for propfind) +type Props []xml.Name + +// XML holds the xml representation of a propfind +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_propfind +type XML struct { + XMLName xml.Name `xml:"DAV: propfind"` + Allprop *struct{} `xml:"DAV: allprop"` + Propname *struct{} `xml:"DAV: propname"` + Prop Props `xml:"DAV: prop"` + Include Props `xml:"DAV: include"` +} + +func readReport(r io.Reader) (rep *report, err error) { + decoder := xml.NewDecoder(r) + rep = &report{} + for { + t, err := decoder.Token() + if err == io.EOF { + // io.EOF is a successful end + return rep, nil + } + if err != nil { + return nil, err + } + + if v, ok := t.(xml.StartElement); ok { + if v.Name.Local == elementNameSearchFiles { + var repSF reportSearchFiles + err = decoder.DecodeElement(&repSF, &v) + if err != nil { + return nil, err + } + rep.SearchFiles = &repSF + /* + } else if v.Name.Local == elementNameFilterFiles { + var repFF reportFilterFiles + err = decoder.DecodeElement(&repFF, &v) + if err != nil { + return nil, http.StatusBadRequest, err + } + rep.FilterFiles = &repFF + */ + } + } + } } diff --git a/webdav/pkg/service/v0/service.go b/webdav/pkg/service/v0/service.go index 8d08ac1cf..2d7c541f1 100644 --- a/webdav/pkg/service/v0/service.go +++ b/webdav/pkg/service/v0/service.go @@ -52,6 +52,7 @@ func NewService(opts ...Option) (Service, error) { conf := options.Config m := chi.NewMux() + chi.RegisterMethod("REPORT") m.Use(options.Middleware...) gwc, err := pool.GetGatewayServiceClient(conf.RevaGateway) @@ -63,7 +64,7 @@ func NewService(opts ...Option) (Service, error) { config: conf, log: options.Logger, mux: m, - searchClient: searchsvc.NewSearchProviderService("search", grpc.DefaultClient), + searchClient: searchsvc.NewSearchProviderService("com.owncloud.api.search", grpc.DefaultClient), thumbnailsClient: thumbnailssvc.NewThumbnailService("com.owncloud.api.thumbnails", grpc.DefaultClient), revaClient: gwc, } @@ -74,13 +75,13 @@ func NewService(opts ...Option) (Service, error) { r.Get("/remote.php/dav/public-files/{token}/*", svc.PublicThumbnail) r.Head("/remote.php/dav/public-files/{token}/*", svc.PublicThumbnailHead) - r.MethodFunc("REPORT", "/remote.php/dav/files/{id}/*", svc.Search) + r.MethodFunc("REPORT", "/remote.php/dav/files/{id}", svc.Search) }) return svc, nil } -// Webdav defines implements the business logic for Service. +// Webdav implements the business logic for Service. type Webdav struct { config *config.Config log log.Logger From ab2a4e7068afc79378e600430f01f7c7ae8f5a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 13 Apr 2022 16:19:05 +0000 Subject: [PATCH 14/49] fix nil MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- webdav/pkg/service/v0/search.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/webdav/pkg/service/v0/search.go b/webdav/pkg/service/v0/search.go index cfa305a6c..aa2a6fcb9 100644 --- a/webdav/pkg/service/v0/search.go +++ b/webdav/pkg/service/v0/search.go @@ -30,6 +30,12 @@ func (g Webdav) Search(w http.ResponseWriter, r *http.Request) { return } + if rep.SearchFiles == nil { + renderError(w, r, errBadRequest("missing search-files tag")) + g.log.Error().Err(err).Msg("error reading report") + return + } + rsp, err := g.searchClient.Search(r.Context(), &searchsvc.SearchRequest{ Query: rep.SearchFiles.Search.Pattern, }) From 997cfc21213c5aad10198aa19eb881a98d81ac7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 14 Apr 2022 08:49:04 +0200 Subject: [PATCH 15/49] Limit the search to the specified space in the reference --- search/pkg/search/index/index.go | 7 ++++--- search/pkg/search/index/index_test.go | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/search/pkg/search/index/index.go b/search/pkg/search/index/index.go index ed6d004e4..45a041b35 100644 --- a/search/pkg/search/index/index.go +++ b/search/pkg/search/index/index.go @@ -76,7 +76,8 @@ func (i *Index) Remove(ri *sprovider.ResourceInfo) error { func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) { query := bleve.NewConjunctionQuery( bleve.NewQueryStringQuery(req.Query), - bleve.NewQueryStringQuery("Path:"+req.Ref.Path+"*"), // Limit search to this directory in the space + bleve.NewQueryStringQuery("RootID:"+req.Ref.ResourceId.StorageId+"!"+req.Ref.ResourceId.OpaqueId), // Limit search to the space + bleve.NewQueryStringQuery("Path:"+req.Ref.Path+"*"), // Limit search to this directory in the space ) bleveReq := bleve.NewSearchRequest(query) bleveReq.Fields = []string{"*"} @@ -108,9 +109,9 @@ func BuildMapping() mapping.IndexMapping { func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocument { return &indexDocument{ - RootID: ref.ResourceId.GetStorageId() + "!" + ref.ResourceId.GetOpaqueId(), + RootID: idToBleveId(ref.ResourceId), Path: ref.Path, - ID: ri.Id.GetStorageId() + "!" + ri.Id.GetOpaqueId(), + ID: idToBleveId(ri.Id), Name: ri.Path, Size: ri.Size, } diff --git a/search/pkg/search/index/index_test.go b/search/pkg/search/index/index_test.go index 5e11cab2e..0945b1ae1 100644 --- a/search/pkg/search/index/index_test.go +++ b/search/pkg/search/index/index_test.go @@ -71,6 +71,21 @@ var _ = Describe("Index", func() { Expect(err).ToNot(HaveOccurred()) }) + It("scopes the search to the specified space", func() { + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: "differentstorageid", + OpaqueId: "differentopaqueid", + }, + }, + Query: "foo.pdf", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(0)) + }) + It("finds files by name, prefix or substring match", func() { queries := []string{"foo.pdf", "foo*", "*oo.p*"} for _, query := range queries { From 6573c2adbae4f6df4b3c401bf94c0362aa6017da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 14 Apr 2022 07:53:53 +0000 Subject: [PATCH 16/49] check in missing packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- search/cmd/search/main.go | 14 +++ webdav/pkg/errors/error.go | 170 ++++++++++++++++++++++++++++++ webdav/pkg/net/headers.go | 48 +++++++++ webdav/pkg/net/net.go | 12 +++ webdav/pkg/prop/prop.go | 125 ++++++++++++++++++++++ webdav/pkg/propfind/propfind.go | 180 ++++++++++++++++++++++++++++++++ 6 files changed, 549 insertions(+) create mode 100644 search/cmd/search/main.go create mode 100644 webdav/pkg/errors/error.go create mode 100644 webdav/pkg/net/headers.go create mode 100644 webdav/pkg/net/net.go create mode 100644 webdav/pkg/prop/prop.go create mode 100644 webdav/pkg/propfind/propfind.go diff --git a/search/cmd/search/main.go b/search/cmd/search/main.go new file mode 100644 index 000000000..71f142061 --- /dev/null +++ b/search/cmd/search/main.go @@ -0,0 +1,14 @@ +package main + +import ( + "os" + + "github.com/owncloud/ocis/search/pkg/command" + "github.com/owncloud/ocis/search/pkg/config/defaults" +) + +func main() { + if err := command.Execute(defaults.DefaultConfig()); err != nil { + os.Exit(1) + } +} diff --git a/webdav/pkg/errors/error.go b/webdav/pkg/errors/error.go new file mode 100644 index 000000000..ba7f9b908 --- /dev/null +++ b/webdav/pkg/errors/error.go @@ -0,0 +1,170 @@ +package errors + +import ( + "bytes" + "encoding/xml" + "net/http" + + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + "github.com/cs3org/reva/v2/pkg/rgrpc/status" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +var sabreException = map[int]string{ + + // the commented states have no corresponding exception in sabre/dav, + // see https://github.com/sabre-io/dav/tree/master/lib/DAV/Exception + + // http.StatusMultipleChoices: "Multiple Choices", + // http.StatusMovedPermanently: "Moved Permanently", + // http.StatusFound: "Found", + // http.StatusSeeOther: "See Other", + // http.StatusNotModified: "Not Modified", + // http.StatusUseProxy: "Use Proxy", + // http.StatusTemporaryRedirect: "Temporary Redirect", + // http.StatusPermanentRedirect: "Permanent Redirect", + + http.StatusBadRequest: "Sabre\\DAV\\Exception\\BadRequest", + http.StatusUnauthorized: "Sabre\\DAV\\Exception\\NotAuthenticated", + http.StatusPaymentRequired: "Sabre\\DAV\\Exception\\PaymentRequired", + http.StatusForbidden: "Sabre\\DAV\\Exception\\Forbidden", // InvalidResourceType, InvalidSyncToken, TooManyMatches + http.StatusNotFound: "Sabre\\DAV\\Exception\\NotFound", + http.StatusMethodNotAllowed: "Sabre\\DAV\\Exception\\MethodNotAllowed", + // http.StatusNotAcceptable: "Not Acceptable", + // http.StatusProxyAuthRequired: "Proxy Authentication Required", + // http.StatusRequestTimeout: "Request Timeout", + http.StatusConflict: "Sabre\\DAV\\Exception\\Conflict", // LockTokenMatchesRequestUri + // http.StatusGone: "Gone", + http.StatusLengthRequired: "Sabre\\DAV\\Exception\\LengthRequired", + http.StatusPreconditionFailed: "Sabre\\DAV\\Exception\\PreconditionFailed", + // http.StatusRequestEntityTooLarge: "Request Entity Too Large", + // http.StatusRequestURITooLong: "Request URI Too Long", + http.StatusUnsupportedMediaType: "Sabre\\DAV\\Exception\\UnsupportedMediaType", // ReportNotSupported + http.StatusRequestedRangeNotSatisfiable: "Sabre\\DAV\\Exception\\RequestedRangeNotSatisfiable", + // http.StatusExpectationFailed: "Expectation Failed", + // http.StatusTeapot: "I'm a teapot", + // http.StatusMisdirectedRequest: "Misdirected Request", + // http.StatusUnprocessableEntity: "Unprocessable Entity", + http.StatusLocked: "Sabre\\DAV\\Exception\\Locked", // ConflictingLock + // http.StatusFailedDependency: "Failed Dependency", + // http.StatusTooEarly: "Too Early", + // http.StatusUpgradeRequired: "Upgrade Required", + // http.StatusPreconditionRequired: "Precondition Required", + // http.StatusTooManyRequests: "Too Many Requests", + // http.StatusRequestHeaderFieldsTooLarge: "Request Header Fields Too Large", + // http.StatusUnavailableForLegalReasons: "Unavailable For Legal Reasons", + + // http.StatusInternalServerError: "Internal Server Error", + http.StatusNotImplemented: "Sabre\\DAV\\Exception\\NotImplemented", + // http.StatusBadGateway: "Bad Gateway", + http.StatusServiceUnavailable: "Sabre\\DAV\\Exception\\ServiceUnavailable", + // http.StatusGatewayTimeout: "Gateway Timeout", + // http.StatusHTTPVersionNotSupported: "HTTP Version Not Supported", + // http.StatusVariantAlsoNegotiates: "Variant Also Negotiates", + http.StatusInsufficientStorage: "Sabre\\DAV\\Exception\\InsufficientStorage", + // http.StatusLoopDetected: "Loop Detected", + // http.StatusNotExtended: "Not Extended", + // http.StatusNetworkAuthenticationRequired: "Network Authentication Required", +} + +// SabreException returns a sabre exception text for the HTTP status code. It returns the empty +// string if the code is unknown. +func SabreException(code int) string { + return sabreException[code] +} + +// Exception represents a ocdav exception +type Exception struct { + Code int + Message string + Header string +} + +// Marshal just calls the xml marshaller for a given exception. +func Marshal(code int, message string, header string) ([]byte, error) { + xmlstring, err := xml.Marshal(&ErrorXML{ + Xmlnsd: "DAV", + Xmlnss: "http://sabredav.org/ns", + Exception: sabreException[code], + Message: message, + Header: header, + }) + if err != nil { + return nil, err + } + var buf bytes.Buffer + buf.WriteString(xml.Header) + buf.Write(xmlstring) + return buf.Bytes(), err +} + +// ErrorXML holds the xml representation of an error +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_error +type ErrorXML struct { + XMLName xml.Name `xml:"d:error"` + Xmlnsd string `xml:"xmlns:d,attr"` + Xmlnss string `xml:"xmlns:s,attr"` + Exception string `xml:"s:exception"` + Message string `xml:"s:message"` + InnerXML []byte `xml:",innerxml"` + // Header is used to indicate the conflicting request header + Header string `xml:"s:header,omitempty"` +} + +var ( + // ErrInvalidDepth is an invalid depth header error + ErrInvalidDepth = errors.New("webdav: invalid depth") + // ErrInvalidPropfind is an invalid propfind error + ErrInvalidPropfind = errors.New("webdav: invalid propfind") + // ErrInvalidProppatch is an invalid proppatch error + ErrInvalidProppatch = errors.New("webdav: invalid proppatch") + // ErrInvalidLockInfo is an invalid lock error + ErrInvalidLockInfo = errors.New("webdav: invalid lock info") + // ErrUnsupportedLockInfo is an unsupported lock error + ErrUnsupportedLockInfo = errors.New("webdav: unsupported lock info") + // ErrInvalidTimeout is an invalid timeout error + ErrInvalidTimeout = errors.New("webdav: invalid timeout") + // ErrInvalidIfHeader is an invalid if header error + ErrInvalidIfHeader = errors.New("webdav: invalid If header") + // ErrUnsupportedMethod is an unsupported method error + ErrUnsupportedMethod = errors.New("webdav: unsupported method") + // ErrInvalidLockToken is an invalid lock token error + ErrInvalidLockToken = errors.New("webdav: invalid lock token") + // ErrConfirmationFailed is returned by a LockSystem's Confirm method. + ErrConfirmationFailed = errors.New("webdav: confirmation failed") + // ErrForbidden is returned by a LockSystem's Unlock method. + ErrForbidden = errors.New("webdav: forbidden") + // ErrLocked is returned by a LockSystem's Create, Refresh and Unlock methods. + ErrLocked = errors.New("webdav: locked") + // ErrNoSuchLock is returned by a LockSystem's Refresh and Unlock methods. + ErrNoSuchLock = errors.New("webdav: no such lock") + // ErrNotImplemented is returned when hitting not implemented code paths + ErrNotImplemented = errors.New("webdav: not implemented") +) + +// HandleErrorStatus checks the status code, logs a Debug or Error level message +// and writes an appropriate http status +func HandleErrorStatus(log *zerolog.Logger, w http.ResponseWriter, s *rpc.Status) { + hsc := status.HTTPStatusFromCode(s.Code) + if hsc == http.StatusInternalServerError { + log.Error().Interface("status", s).Int("code", hsc).Msg(http.StatusText(hsc)) + } else { + log.Debug().Interface("status", s).Int("code", hsc).Msg(http.StatusText(hsc)) + } + w.WriteHeader(hsc) +} + +// HandleWebdavError checks the status code, logs an error and creates a webdav response body +// if needed +func HandleWebdavError(log *zerolog.Logger, w http.ResponseWriter, b []byte, err error) { + if err != nil { + log.Error().Msgf("error marshaling xml response: %s", b) + w.WriteHeader(http.StatusInternalServerError) + return + } + _, err = w.Write(b) + if err != nil { + log.Err(err).Msg("error writing response") + } +} diff --git a/webdav/pkg/net/headers.go b/webdav/pkg/net/headers.go new file mode 100644 index 000000000..5c22d8509 --- /dev/null +++ b/webdav/pkg/net/headers.go @@ -0,0 +1,48 @@ +package net + +// Common HTTP headers. +const ( + HeaderAcceptRanges = "Accept-Ranges" + HeaderAccessControlAllowHeaders = "Access-Control-Allow-Headers" + HeaderAccessControlExposeHeaders = "Access-Control-Expose-Headers" + HeaderContentDisposistion = "Content-Disposition" + HeaderContentLength = "Content-Length" + HeaderContentRange = "Content-Range" + HeaderContentType = "Content-Type" + HeaderETag = "ETag" + HeaderLastModified = "Last-Modified" + HeaderLocation = "Location" + HeaderRange = "Range" + HeaderIfMatch = "If-Match" +) + +// webdav headers +const ( + HeaderDav = "DAV" // https://datatracker.ietf.org/doc/html/rfc4918#section-10.1 + HeaderDepth = "Depth" // https://datatracker.ietf.org/doc/html/rfc4918#section-10.2 + HeaderDestination = "Destination" // https://datatracker.ietf.org/doc/html/rfc4918#section-10.3 + HeaderIf = "If" // https://datatracker.ietf.org/doc/html/rfc4918#section-10.4 + HeaderLockToken = "Lock-Token" // https://datatracker.ietf.org/doc/html/rfc4918#section-10.5 + HeaderOverwrite = "Overwrite" // https://datatracker.ietf.org/doc/html/rfc4918#section-10.6 + HeaderTimeout = "Timeout" // https://datatracker.ietf.org/doc/html/rfc4918#section-10.7 +) + +// Non standard HTTP headers. +const ( + HeaderOCFileID = "OC-FileId" + HeaderOCETag = "OC-ETag" + HeaderOCChecksum = "OC-Checksum" + HeaderOCPermissions = "OC-Perm" + HeaderTusResumable = "Tus-Resumable" + HeaderTusVersion = "Tus-Version" + HeaderTusExtension = "Tus-Extension" + HeaderTusChecksumAlgorithm = "Tus-Checksum-Algorithm" + HeaderTusUploadExpires = "Upload-Expires" + HeaderUploadChecksum = "Upload-Checksum" + HeaderUploadLength = "Upload-Length" + HeaderUploadMetadata = "Upload-Metadata" + HeaderUploadOffset = "Upload-Offset" + HeaderOCMtime = "X-OC-Mtime" + HeaderExpectedEntityLength = "X-Expected-Entity-Length" + HeaderLitmus = "X-Litmus" +) diff --git a/webdav/pkg/net/net.go b/webdav/pkg/net/net.go new file mode 100644 index 000000000..6ac66b74a --- /dev/null +++ b/webdav/pkg/net/net.go @@ -0,0 +1,12 @@ +package net + +import ( + "net/url" +) + +// EncodePath encodes the path of a url. +// +// slashes (/) are treated as path-separators. +func EncodePath(path string) string { + return (&url.URL{Path: path}).EscapedPath() +} diff --git a/webdav/pkg/prop/prop.go b/webdav/pkg/prop/prop.go new file mode 100644 index 000000000..b34ce19fe --- /dev/null +++ b/webdav/pkg/prop/prop.go @@ -0,0 +1,125 @@ +package prop + +import ( + "bytes" + "encoding/xml" +) + +// PropertyXML represents a single DAV resource property as defined in RFC 4918. +// http://www.webdav.org/specs/rfc4918.html#data.model.for.resource.properties +type PropertyXML struct { + // XMLName is the fully qualified name that identifies this property. + XMLName xml.Name + + // Lang is an optional xml:lang attribute. + Lang string `xml:"xml:lang,attr,omitempty"` + + // InnerXML contains the XML representation of the property value. + // See http://www.webdav.org/specs/rfc4918.html#property_values + // + // Property values of complex type or mixed-content must have fully + // expanded XML namespaces or be self-contained with according + // XML namespace declarations. They must not rely on any XML + // namespace declarations within the scope of the XML document, + // even including the DAV: namespace. + InnerXML []byte `xml:",innerxml"` +} + +func xmlEscaped(val string) []byte { + buf := new(bytes.Buffer) + xml.Escape(buf, []byte(val)) + return buf.Bytes() +} + +// EscapedNS returns a new PropertyXML instance while xml-escaping the value +func EscapedNS(namespace string, local string, val string) PropertyXML { + return PropertyXML{ + XMLName: xml.Name{Space: namespace, Local: local}, + Lang: "", + InnerXML: xmlEscaped(val), + } +} + +// Escaped returns a new PropertyXML instance while xml-escaping the value +// TODO properly use the space +func Escaped(key, val string) PropertyXML { + return PropertyXML{ + XMLName: xml.Name{Space: "", Local: key}, + Lang: "", + InnerXML: xmlEscaped(val), + } +} + +// NotFound returns a new PropertyXML instance with an empty value +func NotFound(key string) PropertyXML { + return PropertyXML{ + XMLName: xml.Name{Space: "", Local: key}, + Lang: "", + } +} + +// NotFoundNS returns a new PropertyXML instance with the given namespace and an empty value +func NotFoundNS(namespace, key string) PropertyXML { + return PropertyXML{ + XMLName: xml.Name{Space: namespace, Local: key}, + Lang: "", + } +} + +// Raw returns a new PropertyXML instance for the given key/value pair +// TODO properly use the space +func Raw(key, val string) PropertyXML { + return PropertyXML{ + XMLName: xml.Name{Space: "", Local: key}, + Lang: "", + InnerXML: []byte(val), + } +} + +// Next returns the next token, if any, in the XML stream of d. +// RFC 4918 requires to ignore comments, processing instructions +// and directives. +// http://www.webdav.org/specs/rfc4918.html#property_values +// http://www.webdav.org/specs/rfc4918.html#xml-extensibility +func Next(d *xml.Decoder) (xml.Token, error) { + for { + t, err := d.Token() + if err != nil { + return t, err + } + switch t.(type) { + case xml.Comment, xml.Directive, xml.ProcInst: + continue + default: + return t, nil + } + } +} + +// ActiveLock holds active lock xml data +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_activelock +// +type ActiveLock struct { + XMLName xml.Name `xml:"activelock"` + Exclusive *struct{} `xml:"lockscope>exclusive,omitempty"` + Shared *struct{} `xml:"lockscope>shared,omitempty"` + Write *struct{} `xml:"locktype>write,omitempty"` + Depth string `xml:"depth"` + Owner Owner `xml:"owner,omitempty"` + Timeout string `xml:"timeout,omitempty"` + Locktoken string `xml:"locktoken>href"` + Lockroot string `xml:"lockroot>href,omitempty"` +} + +// Owner captures the inner UML of a lock owner element http://www.webdav.org/specs/rfc4918.html#ELEMENT_owner +type Owner struct { + InnerXML string `xml:",innerxml"` +} + +// Escape repaces ", &, ', < and > with their xml representation +func Escape(s string) string { + b := bytes.NewBuffer(nil) + _ = xml.EscapeText(b, []byte(s)) + return b.String() +} diff --git a/webdav/pkg/propfind/propfind.go b/webdav/pkg/propfind/propfind.go new file mode 100644 index 000000000..5a2f551cb --- /dev/null +++ b/webdav/pkg/propfind/propfind.go @@ -0,0 +1,180 @@ +package propfind + +import ( + "encoding/xml" + "fmt" + "io" + "net/http" + + "github.com/owncloud/ocis/webdav/pkg/errors" + "github.com/owncloud/ocis/webdav/pkg/prop" +) + +const ( + _spaceTypeProject = "project" +) + +type countingReader struct { + n int + r io.Reader +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += n + return n, err +} + +// Props represents properties related to a resource +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_prop (for propfind) +type Props []xml.Name + +// XML holds the xml representation of a propfind +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_propfind +type XML struct { + XMLName xml.Name `xml:"DAV: propfind"` + Allprop *struct{} `xml:"DAV: allprop"` + Propname *struct{} `xml:"DAV: propname"` + Prop Props `xml:"DAV: prop"` + Include Props `xml:"DAV: include"` +} + +// PropstatXML holds the xml representation of a propfind response +// http://www.webdav.org/specs/rfc4918.html#ELEMENT_propstat +type PropstatXML struct { + // Prop requires DAV: to be the default namespace in the enclosing + // XML. This is due to the standard encoding/xml package currently + // not honoring namespace declarations inside a xmltag with a + // parent element for anonymous slice elements. + // Use of multistatusWriter takes care of this. + Prop []prop.PropertyXML `xml:"d:prop>_ignored_"` + Status string `xml:"d:status"` + Error *errors.ErrorXML `xml:"d:error"` + ResponseDescription string `xml:"d:responsedescription,omitempty"` +} + +// ResponseXML holds the xml representation of a propfind response +type ResponseXML struct { + XMLName xml.Name `xml:"d:response"` + Href string `xml:"d:href"` + Propstat []PropstatXML `xml:"d:propstat"` + Status string `xml:"d:status,omitempty"` + Error *errors.ErrorXML `xml:"d:error"` + ResponseDescription string `xml:"d:responsedescription,omitempty"` +} + +// MultiStatusResponseXML holds the xml representation of a multistatus propfind response +type MultiStatusResponseXML struct { + XMLName xml.Name `xml:"d:multistatus"` + XmlnsS string `xml:"xmlns:s,attr,omitempty"` + XmlnsD string `xml:"xmlns:d,attr,omitempty"` + XmlnsOC string `xml:"xmlns:oc,attr,omitempty"` + + Responses []*ResponseXML `xml:"d:response"` +} + +// ResponseUnmarshalXML is a workaround for https://github.com/golang/go/issues/13400 +type ResponseUnmarshalXML struct { + XMLName xml.Name `xml:"response"` + Href string `xml:"href"` + Propstat []PropstatUnmarshalXML `xml:"propstat"` + Status string `xml:"status,omitempty"` + Error *errors.ErrorXML `xml:"d:error"` + ResponseDescription string `xml:"responsedescription,omitempty"` +} + +// MultiStatusResponseUnmarshalXML is a workaround for https://github.com/golang/go/issues/13400 +type MultiStatusResponseUnmarshalXML struct { + XMLName xml.Name `xml:"multistatus"` + XmlnsS string `xml:"xmlns:s,attr,omitempty"` + XmlnsD string `xml:"xmlns:d,attr,omitempty"` + XmlnsOC string `xml:"xmlns:oc,attr,omitempty"` + + Responses []*ResponseUnmarshalXML `xml:"response"` +} + +// PropstatUnmarshalXML is a workaround for https://github.com/golang/go/issues/13400 +type PropstatUnmarshalXML struct { + // Prop requires DAV: to be the default namespace in the enclosing + // XML. This is due to the standard encoding/xml package currently + // not honoring namespace declarations inside a xmltag with a + // parent element for anonymous slice elements. + // Use of multistatusWriter takes care of this. + Prop []*prop.PropertyXML `xml:"prop"` + Status string `xml:"status"` + Error *errors.ErrorXML `xml:"d:error"` + ResponseDescription string `xml:"responsedescription,omitempty"` +} + +// NewMultiStatusResponseXML returns a preconfigured instance of MultiStatusResponseXML +func NewMultiStatusResponseXML() *MultiStatusResponseXML { + return &MultiStatusResponseXML{ + XmlnsD: "DAV:", + XmlnsS: "http://sabredav.org/ns", + XmlnsOC: "http://owncloud.org/ns", + } +} + +// ReadPropfind extracts and parses the propfind XML information from a Reader +// from https://github.com/golang/net/blob/e514e69ffb8bc3c76a71ae40de0118d794855992/webdav/xml.go#L178-L205 +func ReadPropfind(r io.Reader) (pf XML, status int, err error) { + c := countingReader{r: r} + if err = xml.NewDecoder(&c).Decode(&pf); err != nil { + if err == io.EOF { + if c.n == 0 { + // An empty body means to propfind allprop. + // http://www.webdav.org/specs/rfc4918.html#METHOD_PROPFIND + return XML{Allprop: new(struct{})}, 0, nil + } + err = errors.ErrInvalidPropfind + } + return XML{}, http.StatusBadRequest, err + } + + if pf.Allprop == nil && pf.Include != nil { + return XML{}, http.StatusBadRequest, errors.ErrInvalidPropfind + } + if pf.Allprop != nil && (pf.Prop != nil || pf.Propname != nil) { + return XML{}, http.StatusBadRequest, errors.ErrInvalidPropfind + } + if pf.Prop != nil && pf.Propname != nil { + return XML{}, http.StatusBadRequest, errors.ErrInvalidPropfind + } + if pf.Propname == nil && pf.Allprop == nil && pf.Prop == nil { + // jfd: I think is perfectly valid ... treat it as allprop + return XML{Allprop: new(struct{})}, 0, nil + } + return pf, 0, nil +} + +// UnmarshalXML appends the property names enclosed within start to pn. +// +// It returns an error if start does not contain any properties or if +// properties contain values. Character data between properties is ignored. +func (pn *Props) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + for { + t, err := prop.Next(d) + if err != nil { + return err + } + switch e := t.(type) { + case xml.EndElement: + // jfd: I think is perfectly valid ... treat it as allprop + /* + if len(*pn) == 0 { + return fmt.Errorf("%s must not be empty", start.Name.Local) + } + */ + return nil + case xml.StartElement: + t, err = prop.Next(d) + if err != nil { + return err + } + if _, ok := t.(xml.EndElement); !ok { + return fmt.Errorf("unexpected token %T", t) + } + *pn = append(*pn, e.Name) + } + } +} From bba190f9803abaa959e9754bc01217e41ef2f35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 14 Apr 2022 08:19:51 +0000 Subject: [PATCH 17/49] add ocis search command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- ocis/pkg/command/search.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 ocis/pkg/command/search.go diff --git a/ocis/pkg/command/search.go b/ocis/pkg/command/search.go new file mode 100644 index 000000000..79a643b3f --- /dev/null +++ b/ocis/pkg/command/search.go @@ -0,0 +1,26 @@ +package command + +import ( + "github.com/owncloud/ocis/ocis-pkg/config" + "github.com/owncloud/ocis/ocis-pkg/config/parser" + "github.com/owncloud/ocis/ocis/pkg/register" + "github.com/owncloud/ocis/search/pkg/command" + "github.com/urfave/cli/v2" +) + +// SearchCommand is the entry point for the search command. +func SearchCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: cfg.Search.Service.Name, + Usage: subcommandDescription(cfg.Search.Service.Name), + Category: "extensions", + Before: func(ctx *cli.Context) error { + return parser.ParseConfig(cfg) + }, + Subcommands: command.GetCommands(cfg.Search), + } +} + +func init() { + register.AddCommand(SearchCommand) +} From a0d99464b2fdf0bbbe50561e1a10c5735e84a6b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 14 Apr 2022 10:51:21 +0200 Subject: [PATCH 18/49] Fix parsing the search pattern --- webdav/pkg/service/v0/search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webdav/pkg/service/v0/search.go b/webdav/pkg/service/v0/search.go index aa2a6fcb9..f10c4309b 100644 --- a/webdav/pkg/service/v0/search.go +++ b/webdav/pkg/service/v0/search.go @@ -130,7 +130,7 @@ type reportSearchFiles struct { Search reportSearchFilesSearch `xml:"search"` } type reportSearchFilesSearch struct { - Pattern string `xml:"search"` + Pattern string `xml:"pattern"` Limit int `xml:"limit"` Offset int `xml:"offset"` } From 2de90423b7b1c45e78447082898ffd25f82b8865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 14 Apr 2022 11:18:18 +0200 Subject: [PATCH 19/49] Actually call the search provider when searching --- go.mod | 4 +++- go.sum | 3 --- search/pkg/service/v0/service.go | 9 +++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 96bb7680b..b84ead247 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/asim/go-micro/plugins/wrapper/monitoring/prometheus/v4 v4.0.0-20220118152736-9e0be6c85d75 github.com/asim/go-micro/plugins/wrapper/trace/opencensus/v4 v4.0.0-20220118152736-9e0be6c85d75 github.com/blevesearch/bleve/v2 v2.3.2 + github.com/blevesearch/bleve_index_api v1.0.1 github.com/coreos/go-oidc/v3 v3.1.0 github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde github.com/cs3org/reva/v2 v2.2.0 @@ -104,7 +105,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/bits-and-blooms/bitset v1.2.1 // indirect - github.com/blevesearch/bleve_index_api v1.0.1 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.3 // indirect @@ -272,3 +272,5 @@ require ( // we need to use a fork to make the windows build pass replace github.com/pkg/xattr => github.com/micbar/xattr v0.4.6-0.20220215112335-88e74d648fb7 + +replace github.com/cs3org/reva/v2 => ../reva diff --git a/go.sum b/go.sum index 5347eda18..644f5fadd 100644 --- a/go.sum +++ b/go.sum @@ -335,11 +335,8 @@ github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3p github.com/crewjam/saml v0.4.6 h1:XCUFPkQSJLvzyl4cW9OvpWUbRf0gE7VUpU8ZnilbeM4= github.com/crewjam/saml v0.4.6/go.mod h1:ZBOXnNPFzB3CgOkRm7Nd6IVdkG+l/wF+0ZXLqD96t1A= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20220328105952-297bef33e13f/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde h1:WrD9O8ZaWvsm0eBzpzVBIuczDhqVq50Nmjc7PGHHA9Y= github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva/v2 v2.2.0 h1:OZRaJyz6/mLkR/B77XwYm9fwERRdk6LcU/7lLVP6tFI= -github.com/cs3org/reva/v2 v2.2.0/go.mod h1:l1dhodFXCp88/Lc0VhzSeaLSQUTn2AdwwkTFtaLCJhk= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= diff --git a/search/pkg/service/v0/service.go b/search/pkg/service/v0/service.go index 319d97112..79e1c03e3 100644 --- a/search/pkg/service/v0/service.go +++ b/search/pkg/service/v0/service.go @@ -53,5 +53,14 @@ type Service struct { } func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *searchsvc.SearchResponse) error { + res, err := s.provider.Search(ctx, &searchsvc.SearchRequest{ + Query: in.Query, + }) + if err != nil { + return nil + } + + out.Matches = res.Matches + out.NextPageToken = res.NextPageToken return nil } From dcb8e4f356ecc0067e523f925171050d0fbd8892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 19 Apr 2022 12:56:10 +0000 Subject: [PATCH 20/49] use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- extensions/proxy/pkg/config/defaults/defaultconfig.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/proxy/pkg/config/defaults/defaultconfig.go b/extensions/proxy/pkg/config/defaults/defaultconfig.go index 8dacb2608..f34da411d 100644 --- a/extensions/proxy/pkg/config/defaults/defaultconfig.go +++ b/extensions/proxy/pkg/config/defaults/defaultconfig.go @@ -100,8 +100,8 @@ func DefaultPolicies() []config.Policy { // TODO or we allow a REPORT on /dav/spaces to search all spaces and /dav/space/{spaceid} to search a specific space // send webdav REPORT requests to search service Method: "REPORT", - Endpoint: "/dav/", - Backend: "http://localhost:????", // TODO use registry? + Endpoint: "/remote.php/dav/", + Backend: "http://localhost:9115", // TODO use registry? }, { Endpoint: "/remote.php/", From c17717d7c9e39ea2a9936414ef891e9c5fd34af8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 22 Apr 2022 10:47:34 +0000 Subject: [PATCH 21/49] enable search in web ui MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- extensions/storage/pkg/command/frontend.go | 4 +++- extensions/web/pkg/config/defaults/defaultconfig.go | 3 +++ extensions/web/pkg/service/v0/service.go | 6 ------ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/extensions/storage/pkg/command/frontend.go b/extensions/storage/pkg/command/frontend.go index 26dba9add..69dd8b1a2 100644 --- a/extensions/storage/pkg/command/frontend.go +++ b/extensions/storage/pkg/command/frontend.go @@ -236,7 +236,9 @@ func frontendConfigFromStruct(c *cli.Context, cfg *config.Config, filesCfg map[s "preferred_upload_type": cfg.Reva.ChecksumPreferredUploadType, }, "files": filesCfg, - "dav": map[string]interface{}{}, + "dav": map[string]interface{}{ + "reports": []string{"search-files"}, + }, "files_sharing": map[string]interface{}{ "api_enabled": true, "resharing": false, diff --git a/extensions/web/pkg/config/defaults/defaultconfig.go b/extensions/web/pkg/config/defaults/defaultconfig.go index bbfad9bdf..cb5488b08 100644 --- a/extensions/web/pkg/config/defaults/defaultconfig.go +++ b/extensions/web/pkg/config/defaults/defaultconfig.go @@ -51,6 +51,9 @@ func DefaultConfig() *config.Config { Scope: "openid profile email", }, Apps: []string{"files", "search", "preview", "text-editor", "pdf-viewer", "external"}, + Options: map[string]interface{}{ + "hideSearchBar": false, + }, }, }, } diff --git a/extensions/web/pkg/service/v0/service.go b/extensions/web/pkg/service/v0/service.go index 3f762fded..dd6a9aa1a 100644 --- a/extensions/web/pkg/service/v0/service.go +++ b/extensions/web/pkg/service/v0/service.go @@ -66,12 +66,6 @@ func (p Web) getPayload() (payload []byte, err error) { if p.config.Web.Path == "" { // render dynamically using config - // provide default ocis-web options - if p.config.Web.Config.Options == nil { - p.config.Web.Config.Options = make(map[string]interface{}) - p.config.Web.Config.Options["hideSearchBar"] = true - } - // build theme url if themeServer, err := url.Parse(p.config.Web.ThemeServer); err == nil { p.config.Web.Config.Theme = themeServer.String() + p.config.Web.ThemePath From 5598fc04ec61afbdf566db3a5997d2f88b6d9f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 21 Apr 2022 12:01:25 +0200 Subject: [PATCH 22/49] Listen to nats and trigger the indexing of uploaded files --- extensions/search/pkg/config/config.go | 10 ++++ .../pkg/config/defaults/defaultconfig.go | 6 ++ .../search/pkg/search/index/index_test.go | 2 +- .../search/pkg/search/mocks/IndexClient.go | 15 +++++ .../pkg/search/provider/searchprovider.go | 55 +++++++++++++++++-- .../search/provider/searchprovider_test.go | 54 +++++++++++++++++- extensions/search/pkg/search/search.go | 2 + extensions/search/pkg/service/v0/service.go | 20 ++++++- 8 files changed, 154 insertions(+), 10 deletions(-) diff --git a/extensions/search/pkg/config/config.go b/extensions/search/pkg/config/config.go index 706a09cd9..788ceec85 100644 --- a/extensions/search/pkg/config/config.go +++ b/extensions/search/pkg/config/config.go @@ -20,6 +20,16 @@ type Config struct { Reva Reva `ocisConfig:"reva"` TokenManager TokenManager `ocisConfig:"token_manager"` + Events Events `yaml:"events"` + + MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;SEARCH_MACHINE_AUTH_API_KEY"` Context context.Context `ocisConfig:"-" yaml:"-"` } + +// Events combines the configuration options for the event bus. +type Events struct { + Endpoint string `yaml:"events_endpoint" env:"SEARCH_EVENTS_ENDPOINT" desc:"the address of the streaming service"` + Cluster string `yaml:"events_cluster" env:"SEARCH_EVENTS_CLUSTER" desc:"the clusterID of the streaming service. Mandatory when using nats"` + ConsumerGroup string `yaml:"events_group" env:"SEARCH_EVENTS_GROUP" desc:"the customergroup of the service. One group will only get one copy of an event"` +} diff --git a/extensions/search/pkg/config/defaults/defaultconfig.go b/extensions/search/pkg/config/defaults/defaultconfig.go index 006c84d6c..9f3368b0b 100644 --- a/extensions/search/pkg/config/defaults/defaultconfig.go +++ b/extensions/search/pkg/config/defaults/defaultconfig.go @@ -23,6 +23,12 @@ func DefaultConfig() *config.Config { TokenManager: config.TokenManager{ JWTSecret: "Pive-Fumkiu4", }, + Events: config.Events{ + Endpoint: "127.0.0.1:9233", + Cluster: "ocis-cluster", + ConsumerGroup: "search", + }, + MachineAuthAPIKey: "change-me-please", } } diff --git a/extensions/search/pkg/search/index/index_test.go b/extensions/search/pkg/search/index/index_test.go index 3fe6fb2b8..0d54f4275 100644 --- a/extensions/search/pkg/search/index/index_test.go +++ b/extensions/search/pkg/search/index/index_test.go @@ -177,7 +177,7 @@ var _ = Describe("Index", func() { }) }) - Describe("Index", func() { + Describe("Add", func() { It("adds a resourceInfo to the index", func() { err := i.Add(ref, ri) Expect(err).ToNot(HaveOccurred()) diff --git a/extensions/search/pkg/search/mocks/IndexClient.go b/extensions/search/pkg/search/mocks/IndexClient.go index e8a62ceca..c29b8aa9f 100644 --- a/extensions/search/pkg/search/mocks/IndexClient.go +++ b/extensions/search/pkg/search/mocks/IndexClient.go @@ -5,6 +5,7 @@ package mocks import ( context "context" + providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" mock "github.com/stretchr/testify/mock" v0 "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" @@ -15,6 +16,20 @@ type IndexClient struct { mock.Mock } +// Add provides a mock function with given fields: ref, ri +func (_m *IndexClient) Add(ref *providerv1beta1.Reference, ri *providerv1beta1.ResourceInfo) error { + ret := _m.Called(ref, ri) + + var r0 error + if rf, ok := ret.Get(0).(func(*providerv1beta1.Reference, *providerv1beta1.ResourceInfo) error); ok { + r0 = rf(ref, ri) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Search provides a mock function with given fields: ctx, req func (_m *IndexClient) Search(ctx context.Context, req *v0.SearchIndexRequest) (*v0.SearchIndexResponse, error) { ret := _m.Called(ctx, req) diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index 8735188d5..167f41418 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -5,26 +5,71 @@ import ( "strings" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/utils" "github.com/owncloud/ocis/extensions/search/pkg/search" + "google.golang.org/grpc/metadata" searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) type Provider struct { - gwClient gateway.GatewayAPIClient - indexClient search.IndexClient + gwClient gateway.GatewayAPIClient + indexClient search.IndexClient + machineAuthAPIKey string } -func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient) *Provider { +func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, machineAuthAPIKey string, eventsChan <-chan interface{}) *Provider { + go func() { + for { + ev := <-eventsChan + var ref *providerv1beta1.Reference + var owner *user.User + switch e := ev.(type) { + case events.FileUploaded: + ref = e.FileID + owner = &user.User{ + Id: e.Executant, + } + default: + // Not sure what to do here. Skip. + continue + } + + // Get auth + ownerCtx := ctxpkg.ContextSetUser(context.Background(), owner) + authRes, err := gwClient.Authenticate(ownerCtx, &gateway.AuthenticateRequest{ + Type: "machine", + ClientId: "userid:" + owner.Id.OpaqueId, + ClientSecret: machineAuthAPIKey, + }) + if err != nil || authRes.GetStatus().GetCode() != rpc.Code_CODE_OK { + // TODO: log error + } + ownerCtx = metadata.AppendToOutgoingContext(ownerCtx, ctxpkg.TokenHeader, authRes.Token) + + // Stat changed resource resource + statRes, err := gwClient.Stat(ownerCtx, &providerv1beta1.StatRequest{Ref: ref}) + if err != nil || statRes.Status.Code != rpc.Code_CODE_OK { + // TODO: log error + } + + indexClient.Add(ref, statRes.Info) + } + }() + return &Provider{ - gwClient: gwClient, - indexClient: indexClient, + gwClient: gwClient, + indexClient: indexClient, + machineAuthAPIKey: machineAuthAPIKey, } } diff --git a/extensions/search/pkg/search/provider/searchprovider_test.go b/extensions/search/pkg/search/provider/searchprovider_test.go index d883caf37..695dd143f 100644 --- a/extensions/search/pkg/search/provider/searchprovider_test.go +++ b/extensions/search/pkg/search/provider/searchprovider_test.go @@ -7,9 +7,11 @@ import ( . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/rgrpc/status" cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" "github.com/owncloud/ocis/extensions/search/pkg/search/mocks" @@ -24,7 +26,8 @@ var _ = Describe("Searchprovider", func() { gwClient *cs3mocks.GatewayAPIClient indexClient *mocks.IndexClient - ctx context.Context + ctx context.Context + eventsChan chan interface{} otherUser = &userv1beta1.User{ Id: &userv1beta1.UserId{ @@ -44,23 +47,68 @@ var _ = Describe("Searchprovider", func() { Root: &sprovider.ResourceId{OpaqueId: "personalspaceroot"}, Name: "personalspace", } + + ref = &sprovider.Reference{ + ResourceId: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "rootopaqueid", + }, + Path: "./foo.pdf", + } + ri = &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "opaqueid", + }, + Path: "foo.pdf", + Size: 12345, + } ) BeforeEach(func() { ctx = context.Background() + eventsChan = make(chan interface{}) gwClient = &cs3mocks.GatewayAPIClient{} indexClient = &mocks.IndexClient{} - p = provider.New(gwClient, indexClient) + p = provider.New(gwClient, indexClient, "", eventsChan) }) Describe("New", func() { It("returns a new instance", func() { - p := provider.New(gwClient, indexClient) + p := provider.New(gwClient, indexClient, "", eventsChan) Expect(p).ToNot(BeNil()) }) }) + Describe("events", func() { + BeforeEach(func() { + gwClient.On("Authenticate", mock.Anything, mock.Anything).Return(&gateway.AuthenticateResponse{ + Token: "authtoken", + }, nil) + gwClient.On("Stat", mock.Anything, mock.Anything).Return(&sprovider.StatResponse{ + Status: status.NewOK(context.Background()), + Info: ri, + }, nil) + }) + + It("trigger an index change", func() { + called := false + indexClient.On("Add", mock.Anything, mock.MatchedBy(func(riToIndex *sprovider.ResourceInfo) bool { + return riToIndex.Id.OpaqueId == ri.Id.OpaqueId + })).Return(nil).Run(func(args mock.Arguments) { + called = true + }) + eventsChan <- events.FileUploaded{ + FileID: ref, + } + + Eventually(func() bool { + return called + }).Should(BeTrue()) + }) + }) + Describe("Search", func() { It("fails when an empty query is given", func() { res, err := p.Search(ctx, &searchsvc.SearchRequest{ diff --git a/extensions/search/pkg/search/search.go b/extensions/search/pkg/search/search.go index b28563b77..4e23b568b 100644 --- a/extensions/search/pkg/search/search.go +++ b/extensions/search/pkg/search/search.go @@ -21,6 +21,7 @@ package search import ( "context" + providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) @@ -35,4 +36,5 @@ type ProviderClient interface { // IndexClient is the interface to the search index type IndexClient interface { Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) + Add(ref *providerv1beta1.Reference, ri *providerv1beta1.ResourceInfo) error } diff --git a/extensions/search/pkg/service/v0/service.go b/extensions/search/pkg/service/v0/service.go index e9ffd1782..5cedbf092 100644 --- a/extensions/search/pkg/service/v0/service.go +++ b/extensions/search/pkg/service/v0/service.go @@ -3,9 +3,13 @@ package service import ( "context" + "github.com/asim/go-micro/plugins/events/natsjs/v4" "github.com/blevesearch/bleve/v2" + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/events/server" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/owncloud/ocis/extensions/audit/pkg/types" "github.com/owncloud/ocis/extensions/search/pkg/config" "github.com/owncloud/ocis/extensions/search/pkg/search" "github.com/owncloud/ocis/extensions/search/pkg/search/index" @@ -20,6 +24,20 @@ func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, error) { logger := options.Logger cfg := options.Config + // Connect to nats to listen for changes that need to trigger an index update + evtsCfg := cfg.Events + client, err := server.NewNatsStream( + natsjs.Address(evtsCfg.Endpoint), + natsjs.ClusterID(evtsCfg.Cluster), + ) + if err != nil { + return nil, err + } + evts, err := events.Consume(client, evtsCfg.ConsumerGroup, types.RegisteredEvents()...) + if err != nil { + return nil, err + } + bleveIndex, err := bleve.NewMemOnly(index.BuildMapping()) if err != nil { return nil, err @@ -34,7 +52,7 @@ func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, error) { logger.Fatal().Err(err).Str("addr", cfg.Reva.Address).Msg("could not get reva client") } - provider := searchprovider.New(gwclient, index) + provider := searchprovider.New(gwclient, index, cfg.MachineAuthAPIKey, evts) return &Service{ id: cfg.GRPC.Namespace + "." + cfg.Service.Name, From 3ec212c0cbbdef27d177e7766afb8f99fce21808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 22 Apr 2022 12:57:42 +0200 Subject: [PATCH 23/49] Properly pass on the token in the context to the search provider --- .../pkg/search/provider/searchprovider_test.go | 2 +- extensions/search/pkg/service/v0/service.go | 14 +++++++++++++- extensions/webdav/pkg/service/v0/search.go | 11 ++++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/extensions/search/pkg/search/provider/searchprovider_test.go b/extensions/search/pkg/search/provider/searchprovider_test.go index 695dd143f..46a6d14d6 100644 --- a/extensions/search/pkg/search/provider/searchprovider_test.go +++ b/extensions/search/pkg/search/provider/searchprovider_test.go @@ -92,7 +92,7 @@ var _ = Describe("Searchprovider", func() { }, nil) }) - It("trigger an index change", func() { + It("trigger an index update when a file has been uploaded", func() { called := false indexClient.On("Add", mock.Anything, mock.MatchedBy(func(riToIndex *sprovider.ResourceInfo) bool { return riToIndex.Id.OpaqueId == ri.Id.OpaqueId diff --git a/extensions/search/pkg/service/v0/service.go b/extensions/search/pkg/service/v0/service.go index 5cedbf092..55a7c9754 100644 --- a/extensions/search/pkg/service/v0/service.go +++ b/extensions/search/pkg/service/v0/service.go @@ -2,12 +2,16 @@ package service import ( "context" + "errors" "github.com/asim/go-micro/plugins/events/natsjs/v4" "github.com/blevesearch/bleve/v2" + revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/events/server" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "go-micro.dev/v4/metadata" + grpcmetadata "google.golang.org/grpc/metadata" "github.com/owncloud/ocis/extensions/audit/pkg/types" "github.com/owncloud/ocis/extensions/search/pkg/config" @@ -71,11 +75,19 @@ type Service struct { } func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *searchsvc.SearchResponse) error { + // Get token from the context (go-micro) and make it known to the reva client too (grpc) + t, ok := metadata.Get(ctx, revactx.TokenHeader) + if !ok { + s.log.Error().Msg("Could not get token from context") + return errors.New("could not get token from context") + } + ctx = grpcmetadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) + res, err := s.provider.Search(ctx, &searchsvc.SearchRequest{ Query: in.Query, }) if err != nil { - return nil + return err } out.Matches = res.Matches diff --git a/extensions/webdav/pkg/service/v0/search.go b/extensions/webdav/pkg/service/v0/search.go index ce777a464..98e42d66f 100644 --- a/extensions/webdav/pkg/service/v0/search.go +++ b/extensions/webdav/pkg/service/v0/search.go @@ -5,14 +5,17 @@ import ( "encoding/xml" "io" "net/http" + "path" "strconv" + revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/owncloud/ocis/extensions/webdav/pkg/net" "github.com/owncloud/ocis/extensions/webdav/pkg/prop" "github.com/owncloud/ocis/extensions/webdav/pkg/propfind" searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" merrors "go-micro.dev/v4/errors" + "go-micro.dev/v4/metadata" ) const ( @@ -22,7 +25,6 @@ const ( // Search is the endpoint for retrieving search results for REPORT requests func (g Webdav) Search(w http.ResponseWriter, r *http.Request) { - rep, err := readReport(r.Body) if err != nil { renderError(w, r, errBadRequest(err.Error())) @@ -36,7 +38,10 @@ func (g Webdav) Search(w http.ResponseWriter, r *http.Request) { return } - rsp, err := g.searchClient.Search(r.Context(), &searchsvc.SearchRequest{ + t := r.Header.Get(TokenHeader) + ctx := revactx.ContextSetToken(r.Context(), t) + ctx = metadata.Set(ctx, revactx.TokenHeader, t) + rsp, err := g.searchClient.Search(ctx, &searchsvc.SearchRequest{ Query: rep.SearchFiles.Search.Pattern, }) if err != nil { @@ -93,7 +98,7 @@ func multistatusResponse(ctx context.Context, matches []*searchmsg.Match) ([]byt func matchToPropResponse(ctx context.Context, match *searchmsg.Match) (*propfind.ResponseXML, error) { response := propfind.ResponseXML{ - Href: net.EncodePath(match.Entity.Ref.Path), + Href: net.EncodePath(path.Join("/dav/spaces/", match.Entity.Ref.ResourceId.StorageId+"!"+match.Entity.Ref.ResourceId.OpaqueId, match.Entity.Ref.Path)), Propstat: []propfind.PropstatXML{}, } From ca29517527b2994cfb51d31b979f90b31039035f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 22 Apr 2022 14:33:24 +0200 Subject: [PATCH 24/49] Use proper natsjs plugin --- extensions/search/pkg/service/v0/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/search/pkg/service/v0/service.go b/extensions/search/pkg/service/v0/service.go index 55a7c9754..4bc989ad9 100644 --- a/extensions/search/pkg/service/v0/service.go +++ b/extensions/search/pkg/service/v0/service.go @@ -4,12 +4,12 @@ import ( "context" "errors" - "github.com/asim/go-micro/plugins/events/natsjs/v4" "github.com/blevesearch/bleve/v2" revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/events/server" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/go-micro/plugins/v4/events/natsjs" "go-micro.dev/v4/metadata" grpcmetadata "google.golang.org/grpc/metadata" From fe61ef58e6ae9ea5aa194b862cbea88af1d0061b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 22 Apr 2022 14:34:04 +0200 Subject: [PATCH 25/49] Fix searching files from the web ui --- extensions/webdav/pkg/service/v0/search.go | 2 +- extensions/webdav/pkg/service/v0/service.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/webdav/pkg/service/v0/search.go b/extensions/webdav/pkg/service/v0/search.go index 98e42d66f..0f802ce65 100644 --- a/extensions/webdav/pkg/service/v0/search.go +++ b/extensions/webdav/pkg/service/v0/search.go @@ -42,7 +42,7 @@ func (g Webdav) Search(w http.ResponseWriter, r *http.Request) { ctx := revactx.ContextSetToken(r.Context(), t) ctx = metadata.Set(ctx, revactx.TokenHeader, t) rsp, err := g.searchClient.Search(ctx, &searchsvc.SearchRequest{ - Query: rep.SearchFiles.Search.Pattern, + Query: "*" + rep.SearchFiles.Search.Pattern + "*", }) if err != nil { e := merrors.Parse(err.Error()) diff --git a/extensions/webdav/pkg/service/v0/service.go b/extensions/webdav/pkg/service/v0/service.go index 1e982f920..ad6faff6d 100644 --- a/extensions/webdav/pkg/service/v0/service.go +++ b/extensions/webdav/pkg/service/v0/service.go @@ -75,7 +75,7 @@ func NewService(opts ...Option) (Service, error) { r.Get("/remote.php/dav/public-files/{token}/*", svc.PublicThumbnail) r.Head("/remote.php/dav/public-files/{token}/*", svc.PublicThumbnailHead) - r.MethodFunc("REPORT", "/remote.php/dav/files/{id}", svc.Search) + r.MethodFunc("REPORT", "/remote.php/dav/files/{id}/*", svc.Search) }) return svc, nil From b14bded1d3df391a81297a7570b3ed410defc376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 22 Apr 2022 17:54:42 +0200 Subject: [PATCH 26/49] Limit the search to the relevant fields --- extensions/search/pkg/search/index/index.go | 2 +- extensions/search/pkg/search/index/index_test.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/extensions/search/pkg/search/index/index.go b/extensions/search/pkg/search/index/index.go index 45a041b35..865bb9b2b 100644 --- a/extensions/search/pkg/search/index/index.go +++ b/extensions/search/pkg/search/index/index.go @@ -75,7 +75,7 @@ func (i *Index) Remove(ri *sprovider.ResourceInfo) error { // Search searches the index according to the criteria specified in the given SearchIndexRequest func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) { query := bleve.NewConjunctionQuery( - bleve.NewQueryStringQuery(req.Query), + bleve.NewQueryStringQuery("Name:"+req.Query), bleve.NewQueryStringQuery("RootID:"+req.Ref.ResourceId.StorageId+"!"+req.Ref.ResourceId.OpaqueId), // Limit search to the space bleve.NewQueryStringQuery("Path:"+req.Ref.Path+"*"), // Limit search to this directory in the space ) diff --git a/extensions/search/pkg/search/index/index_test.go b/extensions/search/pkg/search/index/index_test.go index 0d54f4275..05c6d03dc 100644 --- a/extensions/search/pkg/search/index/index_test.go +++ b/extensions/search/pkg/search/index/index_test.go @@ -86,6 +86,21 @@ var _ = Describe("Index", func() { Expect(len(res.Matches)).To(Equal(0)) }) + It("limits the search to the relevant fields", func() { + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: ref.ResourceId.StorageId, + OpaqueId: ref.ResourceId.OpaqueId, + }, + }, + Query: "*" + ref.ResourceId.OpaqueId + "*", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(0)) + }) + It("finds files by name, prefix or substring match", func() { queries := []string{"foo.pdf", "foo*", "*oo.p*"} for _, query := range queries { From 962840ec55fc5b1a6574ffded43dcbe522480bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 22 Apr 2022 17:54:55 +0200 Subject: [PATCH 27/49] Fix file id field in the search response --- extensions/webdav/pkg/service/v0/search.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extensions/webdav/pkg/service/v0/search.go b/extensions/webdav/pkg/service/v0/search.go index 0f802ce65..0678765cc 100644 --- a/extensions/webdav/pkg/service/v0/search.go +++ b/extensions/webdav/pkg/service/v0/search.go @@ -96,7 +96,6 @@ func multistatusResponse(ctx context.Context, matches []*searchmsg.Match) ([]byt } func matchToPropResponse(ctx context.Context, match *searchmsg.Match) (*propfind.ResponseXML, error) { - response := propfind.ResponseXML{ Href: net.EncodePath(path.Join("/dav/spaces/", match.Entity.Ref.ResourceId.StorageId+"!"+match.Entity.Ref.ResourceId.OpaqueId, match.Entity.Ref.Path)), Propstat: []propfind.PropstatXML{}, @@ -107,7 +106,7 @@ func matchToPropResponse(ctx context.Context, match *searchmsg.Match) (*propfind Prop: []prop.PropertyXML{}, } - propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:id", match.Entity.Id.StorageId+"!"+match.Entity.Id.OpaqueId)) + propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:fileid", match.Entity.Id.StorageId+"!"+match.Entity.Id.OpaqueId)) size := strconv.FormatUint(match.Entity.Size, 10) propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:size", size)) From 51cf27d767c7547761d678760c1298a97fa69491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 25 Apr 2022 11:23:30 +0200 Subject: [PATCH 28/49] Add etag, mtime and mimetype to the search index and response --- extensions/search/pkg/search/index/index.go | 45 ++++-- .../search/pkg/search/index/index_test.go | 37 ++++- extensions/webdav/pkg/service/v0/search.go | 6 +- .../ocis/messages/accounts/v0/accounts.pb.go | 4 +- .../gen/ocis/messages/search/v0/search.pb.go | 139 ++++++++++++------ .../messages/search/v0/search.pb.micro.go | 1 + .../ocis/messages/settings/v0/settings.pb.go | 4 +- .../messages/thumbnails/v0/thumbnails.pb.go | 4 +- .../gen/ocis/services/search/v0/search.pb.go | 4 +- .../services/search/v0/search.swagger.json | 13 ++ .../ocis/services/settings/v0/settings.pb.go | 4 +- .../gen/ocis/services/store/v0/store.pb.go | 4 +- .../services/thumbnails/v0/thumbnails.pb.go | 4 +- .../ocis/messages/search/v0/search.proto | 8 +- 14 files changed, 196 insertions(+), 81 deletions(-) diff --git a/extensions/search/pkg/search/index/index.go b/extensions/search/pkg/search/index/index.go index 865bb9b2b..68c74cbfe 100644 --- a/extensions/search/pkg/search/index/index.go +++ b/extensions/search/pkg/search/index/index.go @@ -21,10 +21,12 @@ package index import ( "context" "strings" + "time" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/mapping" + "google.golang.org/protobuf/types/known/timestamppb" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" @@ -36,8 +38,11 @@ type indexDocument struct { Path string ID string - Name string - Size uint64 + Name string + Etag string + Size uint64 + Mtime string + MimeType string } // Index represents a bleve based search index @@ -108,20 +113,28 @@ func BuildMapping() mapping.IndexMapping { } func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocument { - return &indexDocument{ - RootID: idToBleveId(ref.ResourceId), - Path: ref.Path, - ID: idToBleveId(ri.Id), - Name: ri.Path, - Size: ri.Size, + doc := &indexDocument{ + RootID: idToBleveId(ref.ResourceId), + Path: ref.Path, + ID: idToBleveId(ri.Id), + Name: ri.Path, + Etag: ri.Etag, + Size: ri.Size, + MimeType: ri.MimeType, } + + if ri.Mtime != nil { + doc.Mtime = time.Unix(int64(ri.Mtime.Seconds), int64(ri.Mtime.Nanos)).UTC().Format(time.RFC3339) + } + + return doc } func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) { rootIDParts := strings.SplitN(fields["RootID"].(string), "!", 2) IDParts := strings.SplitN(fields["ID"].(string), "!", 2) - return &searchmsg.Match{ + match := &searchmsg.Match{ Entity: &searchmsg.Entity{ Ref: &searchmsg.Reference{ ResourceId: &searchmsg.ResourceID{ @@ -134,10 +147,18 @@ func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) { StorageId: IDParts[0], OpaqueId: IDParts[1], }, - Name: fields["Name"].(string), - Size: uint64(fields["Size"].(float64)), + Name: fields["Name"].(string), + Size: uint64(fields["Size"].(float64)), + Etag: fields["Etag"].(string), + MimeType: fields["MimeType"].(string), }, - }, nil + } + + if mtime, err := time.Parse(time.RFC3339, fields["Mtime"].(string)); err == nil { + match.Entity.LastModifiedTime = ×tamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())} + } + + return match, nil } func idToBleveId(id *sprovider.ResourceId) string { diff --git a/extensions/search/pkg/search/index/index_test.go b/extensions/search/pkg/search/index/index_test.go index 05c6d03dc..34c0957d0 100644 --- a/extensions/search/pkg/search/index/index_test.go +++ b/extensions/search/pkg/search/index/index_test.go @@ -5,6 +5,7 @@ import ( "github.com/blevesearch/bleve/v2" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/owncloud/ocis/extensions/search/pkg/search/index" searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" @@ -43,8 +44,11 @@ var _ = Describe("Index", func() { StorageId: "storageid", OpaqueId: "opaqueid", }, - Path: "foo.pdf", - Size: 12345, + Path: "foo.pdf", + Size: 12345, + Etag: "abcde", + MimeType: "application/pdf", + Mtime: &typesv1beta1.Timestamp{Seconds: 4000}, } }) @@ -101,6 +105,30 @@ var _ = Describe("Index", func() { Expect(len(res.Matches)).To(Equal(0)) }) + It("returns all desired fields", func() { + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: ref.ResourceId.StorageId, + OpaqueId: ref.ResourceId.OpaqueId, + }, + }, + Query: "foo.pdf", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + Expect(len(res.Matches)).To(Equal(1)) + match := res.Matches[0] + Expect(match.Entity.Ref.ResourceId.OpaqueId).To(Equal(ref.ResourceId.OpaqueId)) + Expect(match.Entity.Ref.Path).To(Equal(ref.Path)) + Expect(match.Entity.Id.OpaqueId).To(Equal(ri.Id.OpaqueId)) + Expect(match.Entity.Name).To(Equal(ri.Path)) + Expect(match.Entity.Size).To(Equal(ri.Size)) + Expect(match.Entity.Etag).To(Equal(ri.Etag)) + Expect(match.Entity.MimeType).To(Equal(ri.MimeType)) + Expect(uint64(match.Entity.LastModifiedTime.AsTime().Unix())).To(Equal(ri.Mtime.Seconds)) + }) + It("finds files by name, prefix or substring match", func() { queries := []string{"foo.pdf", "foo*", "*oo.p*"} for _, query := range queries { @@ -165,11 +193,6 @@ var _ = Describe("Index", func() { Expect(err).ToNot(HaveOccurred()) Expect(res).ToNot(BeNil()) Expect(len(res.Matches)).To(Equal(1), "query returned no result: "+query) - Expect(res.Matches[0].Entity.Ref.ResourceId.OpaqueId).To(Equal(nestedRef.ResourceId.OpaqueId)) - Expect(res.Matches[0].Entity.Ref.Path).To(Equal(nestedRef.Path)) - Expect(res.Matches[0].Entity.Id.OpaqueId).To(Equal(nestedRI.Id.OpaqueId)) - Expect(res.Matches[0].Entity.Name).To(Equal(nestedRI.Path)) - Expect(res.Matches[0].Entity.Size).To(Equal(nestedRI.Size)) } }) diff --git a/extensions/webdav/pkg/service/v0/search.go b/extensions/webdav/pkg/service/v0/search.go index 0678765cc..3bce68236 100644 --- a/extensions/webdav/pkg/service/v0/search.go +++ b/extensions/webdav/pkg/service/v0/search.go @@ -7,6 +7,7 @@ import ( "net/http" "path" "strconv" + "time" revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/owncloud/ocis/extensions/webdav/pkg/net" @@ -60,7 +61,6 @@ func (g Webdav) Search(w http.ResponseWriter, r *http.Request) { } func (g Webdav) sendSearchResponse(rsp *searchsvc.SearchResponse, w http.ResponseWriter, r *http.Request) { - responsesXML, err := multistatusResponse(r.Context(), rsp.Matches) if err != nil { g.log.Error().Err(err).Msg("error formatting propfind") @@ -107,9 +107,13 @@ func matchToPropResponse(ctx context.Context, match *searchmsg.Match) (*propfind } propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:fileid", match.Entity.Id.StorageId+"!"+match.Entity.Id.OpaqueId)) + propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("d:getetag", match.Entity.Etag)) + propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("d:getlastmodified", match.Entity.LastModifiedTime.AsTime().Format(time.RFC3339))) + propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("d:getcontenttype", match.Entity.MimeType)) size := strconv.FormatUint(match.Entity.Size, 10) propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:size", size)) + propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("d:getcontentlength", size)) // TODO find name for score property score := strconv.FormatFloat(float64(match.Score), 'f', -1, 64) diff --git a/protogen/gen/ocis/messages/accounts/v0/accounts.pb.go b/protogen/gen/ocis/messages/accounts/v0/accounts.pb.go index ced523b4a..3937c7aba 100644 --- a/protogen/gen/ocis/messages/accounts/v0/accounts.pb.go +++ b/protogen/gen/ocis/messages/accounts/v0/accounts.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc (unknown) // source: ocis/messages/accounts/v0/accounts.proto package v0 diff --git a/protogen/gen/ocis/messages/search/v0/search.pb.go b/protogen/gen/ocis/messages/search/v0/search.pb.go index 66f5d25dd..7085608ba 100644 --- a/protogen/gen/ocis/messages/search/v0/search.pb.go +++ b/protogen/gen/ocis/messages/search/v0/search.pb.go @@ -9,6 +9,7 @@ package v0 import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -135,10 +136,14 @@ type Entity struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Ref *Reference `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` - Id *ResourceID `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` - Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` - Size uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` + Ref *Reference `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` + Id *ResourceID `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + Etag string `protobuf:"bytes,4,opt,name=etag,proto3" json:"etag,omitempty"` + Size uint64 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"` + LastModifiedTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=last_modified_time,json=lastModifiedTime,proto3" json:"last_modified_time,omitempty"` + MimeType string `protobuf:"bytes,7,opt,name=mime_type,json=mimeType,proto3" json:"mime_type,omitempty"` + Permissions string `protobuf:"bytes,8,opt,name=permissions,proto3" json:"permissions,omitempty"` } func (x *Entity) Reset() { @@ -194,6 +199,13 @@ func (x *Entity) GetName() string { return "" } +func (x *Entity) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + func (x *Entity) GetSize() uint64 { if x != nil { return x.Size @@ -201,6 +213,27 @@ func (x *Entity) GetSize() uint64 { return 0 } +func (x *Entity) GetLastModifiedTime() *timestamppb.Timestamp { + if x != nil { + return x.LastModifiedTime + } + return nil +} + +func (x *Entity) GetMimeType() string { + if x != nil { + return x.MimeType + } + return "" +} + +func (x *Entity) GetPermissions() string { + if x != nil { + return x.Permissions + } + return "" +} + type Match struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -264,38 +297,50 @@ var file_ocis_messages_search_v0_search_proto_rawDesc = []byte{ 0x0a, 0x24, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x17, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x22, - 0x48, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x12, 0x1d, 0x0a, - 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, - 0x6f, 0x70, 0x61, 0x71, 0x75, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x6f, 0x70, 0x61, 0x71, 0x75, 0x65, 0x49, 0x64, 0x22, 0x65, 0x0a, 0x09, 0x52, 0x65, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x63, - 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, - 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, - 0x52, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, - 0x22, 0x9b, 0x01, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x03, 0x72, - 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, - 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, - 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x03, 0x72, 0x65, - 0x66, 0x12, 0x33, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x49, 0x44, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, - 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x56, - 0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x37, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, - 0x30, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, - 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, - 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, - 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x1a, + 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0x48, 0x0a, 0x0a, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x12, 0x1d, + 0x0a, 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, + 0x09, 0x6f, 0x70, 0x61, 0x71, 0x75, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x6f, 0x70, 0x61, 0x71, 0x75, 0x65, 0x49, 0x64, 0x22, 0x65, 0x0a, 0x09, 0x52, 0x65, + 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, + 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, + 0x44, 0x52, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, + 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, + 0x68, 0x22, 0xb8, 0x02, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x03, + 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x63, 0x69, 0x73, + 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x03, 0x72, + 0x65, 0x66, 0x12, 0x33, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, + 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x49, 0x44, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x65, + 0x74, 0x61, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x65, 0x74, 0x61, 0x67, 0x12, + 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, + 0x69, 0x7a, 0x65, 0x12, 0x48, 0x0a, 0x12, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, + 0x66, 0x69, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6c, 0x61, 0x73, + 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, + 0x09, 0x6d, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x6d, 0x69, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x65, + 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x56, 0x0a, 0x05, + 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x37, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, + 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, + 0x63, 0x6f, 0x72, 0x65, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63, + 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -312,21 +357,23 @@ func file_ocis_messages_search_v0_search_proto_rawDescGZIP() []byte { var file_ocis_messages_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_ocis_messages_search_v0_search_proto_goTypes = []interface{}{ - (*ResourceID)(nil), // 0: ocis.messages.search.v0.ResourceID - (*Reference)(nil), // 1: ocis.messages.search.v0.Reference - (*Entity)(nil), // 2: ocis.messages.search.v0.Entity - (*Match)(nil), // 3: ocis.messages.search.v0.Match + (*ResourceID)(nil), // 0: ocis.messages.search.v0.ResourceID + (*Reference)(nil), // 1: ocis.messages.search.v0.Reference + (*Entity)(nil), // 2: ocis.messages.search.v0.Entity + (*Match)(nil), // 3: ocis.messages.search.v0.Match + (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp } var file_ocis_messages_search_v0_search_proto_depIdxs = []int32{ 0, // 0: ocis.messages.search.v0.Reference.resource_id:type_name -> ocis.messages.search.v0.ResourceID 1, // 1: ocis.messages.search.v0.Entity.ref:type_name -> ocis.messages.search.v0.Reference 0, // 2: ocis.messages.search.v0.Entity.id:type_name -> ocis.messages.search.v0.ResourceID - 2, // 3: ocis.messages.search.v0.Match.entity:type_name -> ocis.messages.search.v0.Entity - 4, // [4:4] is the sub-list for method output_type - 4, // [4:4] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 4, // 3: ocis.messages.search.v0.Entity.last_modified_time:type_name -> google.protobuf.Timestamp + 2, // 4: ocis.messages.search.v0.Match.entity:type_name -> ocis.messages.search.v0.Entity + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_ocis_messages_search_v0_search_proto_init() } diff --git a/protogen/gen/ocis/messages/search/v0/search.pb.micro.go b/protogen/gen/ocis/messages/search/v0/search.pb.micro.go index 5f507f0ae..37fd9b52b 100644 --- a/protogen/gen/ocis/messages/search/v0/search.pb.micro.go +++ b/protogen/gen/ocis/messages/search/v0/search.pb.micro.go @@ -6,6 +6,7 @@ package v0 import ( fmt "fmt" proto "google.golang.org/protobuf/proto" + _ "google.golang.org/protobuf/types/known/timestamppb" math "math" ) diff --git a/protogen/gen/ocis/messages/settings/v0/settings.pb.go b/protogen/gen/ocis/messages/settings/v0/settings.pb.go index c4522665c..c03cacf63 100644 --- a/protogen/gen/ocis/messages/settings/v0/settings.pb.go +++ b/protogen/gen/ocis/messages/settings/v0/settings.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc (unknown) // source: ocis/messages/settings/v0/settings.proto package v0 diff --git a/protogen/gen/ocis/messages/thumbnails/v0/thumbnails.pb.go b/protogen/gen/ocis/messages/thumbnails/v0/thumbnails.pb.go index 25721838c..d6198d3d7 100644 --- a/protogen/gen/ocis/messages/thumbnails/v0/thumbnails.pb.go +++ b/protogen/gen/ocis/messages/thumbnails/v0/thumbnails.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc (unknown) // source: ocis/messages/thumbnails/v0/thumbnails.proto package v0 diff --git a/protogen/gen/ocis/services/search/v0/search.pb.go b/protogen/gen/ocis/services/search/v0/search.pb.go index 73d450a6b..7ccbc467b 100644 --- a/protogen/gen/ocis/services/search/v0/search.pb.go +++ b/protogen/gen/ocis/services/search/v0/search.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc (unknown) // source: ocis/services/search/v0/search.proto package v0 diff --git a/protogen/gen/ocis/services/search/v0/search.swagger.json b/protogen/gen/ocis/services/search/v0/search.swagger.json index dab8aa3ac..b06c03da8 100644 --- a/protogen/gen/ocis/services/search/v0/search.swagger.json +++ b/protogen/gen/ocis/services/search/v0/search.swagger.json @@ -137,9 +137,22 @@ "name": { "type": "string" }, + "etag": { + "type": "string" + }, "size": { "type": "string", "format": "uint64" + }, + "lastModifiedTime": { + "type": "string", + "format": "date-time" + }, + "mimeType": { + "type": "string" + }, + "permissions": { + "type": "string" } } }, diff --git a/protogen/gen/ocis/services/settings/v0/settings.pb.go b/protogen/gen/ocis/services/settings/v0/settings.pb.go index 5eeeb9566..52afe123c 100644 --- a/protogen/gen/ocis/services/settings/v0/settings.pb.go +++ b/protogen/gen/ocis/services/settings/v0/settings.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc (unknown) // source: ocis/services/settings/v0/settings.proto package v0 diff --git a/protogen/gen/ocis/services/store/v0/store.pb.go b/protogen/gen/ocis/services/store/v0/store.pb.go index 5dd2fc436..7d5b05547 100644 --- a/protogen/gen/ocis/services/store/v0/store.pb.go +++ b/protogen/gen/ocis/services/store/v0/store.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc (unknown) // source: ocis/services/store/v0/store.proto package v0 diff --git a/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go b/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go index cf949fb45..03daa47a7 100644 --- a/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go +++ b/protogen/gen/ocis/services/thumbnails/v0/thumbnails.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.17.3 +// protoc-gen-go v1.28.0 +// protoc (unknown) // source: ocis/services/thumbnails/v0/thumbnails.proto package v0 diff --git a/protogen/proto/ocis/messages/search/v0/search.proto b/protogen/proto/ocis/messages/search/v0/search.proto index 45b40df24..d80460af9 100644 --- a/protogen/proto/ocis/messages/search/v0/search.proto +++ b/protogen/proto/ocis/messages/search/v0/search.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package ocis.messages.search.v0; +import "google/protobuf/timestamp.proto"; + option go_package = "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0"; message ResourceID { @@ -18,7 +20,11 @@ message Entity { Reference ref = 1; ResourceID id = 2; string name = 3; - uint64 size = 4; + string etag = 4; + uint64 size = 5; + google.protobuf.Timestamp last_modified_time = 6; + string mime_type = 7; + string permissions = 8; } message Match { From 1a84afc401c5f6943a06b5154d7b9f8237052558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 25 Apr 2022 16:00:25 +0200 Subject: [PATCH 29/49] Persist the search index on disk --- extensions/search/pkg/config/config.go | 1 + extensions/search/pkg/config/defaults/defaultconfig.go | 4 ++++ extensions/search/pkg/service/v0/service.go | 9 +++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/extensions/search/pkg/config/config.go b/extensions/search/pkg/config/config.go index 788ceec85..ae90144ec 100644 --- a/extensions/search/pkg/config/config.go +++ b/extensions/search/pkg/config/config.go @@ -18,6 +18,7 @@ type Config struct { GRPC GRPC `ocisConfig:"grpc"` + Datapath string `yaml:"data_path" env:"SEARCH_DATA_PATH"` Reva Reva `ocisConfig:"reva"` TokenManager TokenManager `ocisConfig:"token_manager"` Events Events `yaml:"events"` diff --git a/extensions/search/pkg/config/defaults/defaultconfig.go b/extensions/search/pkg/config/defaults/defaultconfig.go index 9f3368b0b..8d18bf9e2 100644 --- a/extensions/search/pkg/config/defaults/defaultconfig.go +++ b/extensions/search/pkg/config/defaults/defaultconfig.go @@ -1,7 +1,10 @@ package defaults import ( + "path" + "github.com/owncloud/ocis/extensions/search/pkg/config" + "github.com/owncloud/ocis/ocis-pkg/config/defaults" ) func DefaultConfig() *config.Config { @@ -17,6 +20,7 @@ func DefaultConfig() *config.Config { Service: config.Service{ Name: "search", }, + Datapath: path.Join(defaults.BaseDataPath(), "search"), Reva: config.Reva{ Address: "127.0.0.1:9142", }, diff --git a/extensions/search/pkg/service/v0/service.go b/extensions/search/pkg/service/v0/service.go index 4bc989ad9..a214fa1bd 100644 --- a/extensions/search/pkg/service/v0/service.go +++ b/extensions/search/pkg/service/v0/service.go @@ -3,6 +3,7 @@ package service import ( "context" "errors" + "path/filepath" "github.com/blevesearch/bleve/v2" revactx "github.com/cs3org/reva/v2/pkg/ctx" @@ -42,9 +43,13 @@ func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, error) { return nil, err } - bleveIndex, err := bleve.NewMemOnly(index.BuildMapping()) + indexDir := filepath.Join(cfg.Datapath, "index.bleve") + bleveIndex, err := bleve.Open(indexDir) if err != nil { - return nil, err + bleveIndex, err = bleve.New(indexDir, index.BuildMapping()) + if err != nil { + return nil, err + } } index, err := index.New(bleveIndex) if err != nil { From d5084eea841f21b854a91a4045dba3dc8e63e3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 27 Apr 2022 09:30:08 +0200 Subject: [PATCH 30/49] Add command for (re-)indexing existing spaces --- extensions/search/pkg/command/index.go | 52 ++++++++++++++++++++++++++ extensions/search/pkg/command/root.go | 1 + 2 files changed, 53 insertions(+) create mode 100644 extensions/search/pkg/command/index.go diff --git a/extensions/search/pkg/command/index.go b/extensions/search/pkg/command/index.go new file mode 100644 index 000000000..3d57e9264 --- /dev/null +++ b/extensions/search/pkg/command/index.go @@ -0,0 +1,52 @@ +package command + +import ( + "context" + "fmt" + + "github.com/urfave/cli/v2" + + "github.com/owncloud/ocis/extensions/search/pkg/config" + "github.com/owncloud/ocis/extensions/search/pkg/config/parser" + "github.com/owncloud/ocis/ocis-pkg/service/grpc" + searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" +) + +// Index is the entrypoint for the server command. +func Index(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "index", + Usage: "index the files for one one more users", + Category: "index management", + Aliases: []string{"i"}, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "space", + Aliases: []string{"s"}, + Required: true, + Usage: "the id of the space to travers and index the files of", + }, + &cli.StringFlag{ + Name: "user", + Aliases: []string{"u"}, + Required: true, + Usage: "the username of the user tha shall be used to access the files", + }, + }, + Before: func(c *cli.Context) error { + return parser.ParseConfig(cfg) + }, + Action: func(c *cli.Context) error { + client := searchsvc.NewSearchProviderService("com.owncloud.api.search", grpc.DefaultClient) + _, err := client.IndexSpace(context.Background(), &searchsvc.IndexSpaceRequest{ + SpaceId: c.String("space"), + UserId: c.String("user"), + }) + if err != nil { + fmt.Println("failed to index space: " + err.Error()) + return err + } + return nil + }, + } +} diff --git a/extensions/search/pkg/command/root.go b/extensions/search/pkg/command/root.go index 2e282d6a6..a43261631 100644 --- a/extensions/search/pkg/command/root.go +++ b/extensions/search/pkg/command/root.go @@ -19,6 +19,7 @@ func GetCommands(cfg *config.Config) cli.Commands { Server(cfg), // interaction with this service + Index(cfg), // infos about this service Health(cfg), From 827885abad8c15d36f3f914c3f6552217676abbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 27 Apr 2022 09:48:26 +0200 Subject: [PATCH 31/49] Add a DocCount() method to the index client --- extensions/search/pkg/search/index/index.go | 5 +++ .../search/pkg/search/mocks/IndexClient.go | 35 +++++++++++++++++++ .../pkg/search/provider/searchprovider.go | 12 ++++--- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/extensions/search/pkg/search/index/index.go b/extensions/search/pkg/search/index/index.go index 68c74cbfe..a7752b383 100644 --- a/extensions/search/pkg/search/index/index.go +++ b/extensions/search/pkg/search/index/index.go @@ -66,6 +66,11 @@ func New(bleveIndex bleve.Index) (*Index, error) { }, nil } +// DocCount returns the number of elemenst in the index +func (i *Index) DocCount() (uint64, error) { + return i.bleveIndex.DocCount() +} + // Add adds a new entity to the Index func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error { entity := toEntity(ref, ri) diff --git a/extensions/search/pkg/search/mocks/IndexClient.go b/extensions/search/pkg/search/mocks/IndexClient.go index c29b8aa9f..fd8e3167b 100644 --- a/extensions/search/pkg/search/mocks/IndexClient.go +++ b/extensions/search/pkg/search/mocks/IndexClient.go @@ -30,6 +30,41 @@ func (_m *IndexClient) Add(ref *providerv1beta1.Reference, ri *providerv1beta1.R return r0 } +// DocCount provides a mock function with given fields: +func (_m *IndexClient) DocCount() (uint64, error) { + ret := _m.Called() + + var r0 uint64 + if rf, ok := ret.Get(0).(func() uint64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint64) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Remove provides a mock function with given fields: ri +func (_m *IndexClient) Remove(ri *providerv1beta1.ResourceInfo) error { + ret := _m.Called(ri) + + var r0 error + if rf, ok := ret.Get(0).(func(*providerv1beta1.ResourceInfo) error); ok { + r0 = rf(ri) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Search provides a mock function with given fields: ctx, req func (_m *IndexClient) Search(ctx context.Context, req *v0.SearchIndexRequest) (*v0.SearchIndexResponse, error) { ret := _m.Called(ctx, req) diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index 167f41418..4c8c6d423 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -66,11 +66,15 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach } }() - return &Provider{ - gwClient: gwClient, - indexClient: indexClient, - machineAuthAPIKey: machineAuthAPIKey, + return p +} + +func (p *Provider) logDocCount() { + c, err := p.indexClient.DocCount() + if err != nil { + p.logger.Error().Err(err).Msg("error getting document count from the index") } + p.logger.Debug().Interface("count", c).Msg("new document count") } func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) { From 656ef9d6aeb9e0a7d1b753fad7e59fb4706c1ca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 27 Apr 2022 09:50:53 +0200 Subject: [PATCH 32/49] Add a IndexSpace method to the search provider IndexSpace recursively traverses a given space and (re-)index all files it finds. --- extensions/search/pkg/search/index/index.go | 3 - .../search/pkg/search/index/index_test.go | 2 - .../search/pkg/search/mocks/ProviderClient.go | 23 ++ .../pkg/search/provider/searchprovider.go | 92 ++++++- .../search/provider/searchprovider_test.go | 53 +++- extensions/search/pkg/search/search.go | 3 + extensions/search/pkg/service/v0/service.go | 7 +- extensions/webdav/pkg/service/v0/search.go | 16 ++ .../gen/ocis/services/search/v0/search.pb.go | 242 ++++++++++++++---- .../services/search/v0/search.pb.micro.go | 31 +++ .../ocis/services/search/v0/search.pb.web.go | 95 +++++++ .../services/search/v0/search.swagger.json | 46 ++++ .../ocis/services/search/v0/search.proto | 16 +- 13 files changed, 545 insertions(+), 84 deletions(-) diff --git a/extensions/search/pkg/search/index/index.go b/extensions/search/pkg/search/index/index.go index a7752b383..ed4b88dfa 100644 --- a/extensions/search/pkg/search/index/index.go +++ b/extensions/search/pkg/search/index/index.go @@ -39,7 +39,6 @@ type indexDocument struct { ID string Name string - Etag string Size uint64 Mtime string MimeType string @@ -123,7 +122,6 @@ func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocume Path: ref.Path, ID: idToBleveId(ri.Id), Name: ri.Path, - Etag: ri.Etag, Size: ri.Size, MimeType: ri.MimeType, } @@ -154,7 +152,6 @@ func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) { }, Name: fields["Name"].(string), Size: uint64(fields["Size"].(float64)), - Etag: fields["Etag"].(string), MimeType: fields["MimeType"].(string), }, } diff --git a/extensions/search/pkg/search/index/index_test.go b/extensions/search/pkg/search/index/index_test.go index 34c0957d0..bde7dffef 100644 --- a/extensions/search/pkg/search/index/index_test.go +++ b/extensions/search/pkg/search/index/index_test.go @@ -46,7 +46,6 @@ var _ = Describe("Index", func() { }, Path: "foo.pdf", Size: 12345, - Etag: "abcde", MimeType: "application/pdf", Mtime: &typesv1beta1.Timestamp{Seconds: 4000}, } @@ -124,7 +123,6 @@ var _ = Describe("Index", func() { Expect(match.Entity.Id.OpaqueId).To(Equal(ri.Id.OpaqueId)) Expect(match.Entity.Name).To(Equal(ri.Path)) Expect(match.Entity.Size).To(Equal(ri.Size)) - Expect(match.Entity.Etag).To(Equal(ri.Etag)) Expect(match.Entity.MimeType).To(Equal(ri.MimeType)) Expect(uint64(match.Entity.LastModifiedTime.AsTime().Unix())).To(Equal(ri.Mtime.Seconds)) }) diff --git a/extensions/search/pkg/search/mocks/ProviderClient.go b/extensions/search/pkg/search/mocks/ProviderClient.go index be426b753..d1d551909 100644 --- a/extensions/search/pkg/search/mocks/ProviderClient.go +++ b/extensions/search/pkg/search/mocks/ProviderClient.go @@ -15,6 +15,29 @@ type ProviderClient struct { mock.Mock } +// IndexSpace provides a mock function with given fields: ctx, req +func (_m *ProviderClient) IndexSpace(ctx context.Context, req *v0.IndexSpaceRequest) (*v0.IndexSpaceResponse, error) { + ret := _m.Called(ctx, req) + + var r0 *v0.IndexSpaceResponse + if rf, ok := ret.Get(0).(func(context.Context, *v0.IndexSpaceRequest) *v0.IndexSpaceResponse); ok { + r0 = rf(ctx, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v0.IndexSpaceResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *v0.IndexSpaceRequest) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Search provides a mock function with given fields: ctx, req func (_m *ProviderClient) Search(ctx context.Context, req *v0.SearchRequest) (*v0.SearchResponse, error) { ret := _m.Called(ctx, req) diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index 4c8c6d423..5a3658f06 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -2,19 +2,23 @@ package provider import ( "context" + "fmt" + "path/filepath" "strings" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/storage/utils/walker" "github.com/cs3org/reva/v2/pkg/utils" "github.com/owncloud/ocis/extensions/search/pkg/search" + "github.com/owncloud/ocis/ocis-pkg/log" "google.golang.org/grpc/metadata" searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" @@ -22,16 +26,24 @@ import ( ) type Provider struct { + logger log.Logger gwClient gateway.GatewayAPIClient indexClient search.IndexClient machineAuthAPIKey string } -func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, machineAuthAPIKey string, eventsChan <-chan interface{}) *Provider { +func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, machineAuthAPIKey string, eventsChan <-chan interface{}, logger log.Logger) *Provider { + p := &Provider{ + gwClient: gwClient, + indexClient: indexClient, + machineAuthAPIKey: machineAuthAPIKey, + logger: logger, + } + go func() { for { ev := <-eventsChan - var ref *providerv1beta1.Reference + var ref *provider.Reference var owner *user.User switch e := ev.(type) { case events.FileUploaded: @@ -46,23 +58,29 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach // Get auth ownerCtx := ctxpkg.ContextSetUser(context.Background(), owner) - authRes, err := gwClient.Authenticate(ownerCtx, &gateway.AuthenticateRequest{ + authRes, err := p.gwClient.Authenticate(ownerCtx, &gateway.AuthenticateRequest{ Type: "machine", ClientId: "userid:" + owner.Id.OpaqueId, ClientSecret: machineAuthAPIKey, }) if err != nil || authRes.GetStatus().GetCode() != rpc.Code_CODE_OK { - // TODO: log error + p.logger.Error().Err(err).Interface("authRes", authRes).Msg("error using machine auth") } ownerCtx = metadata.AppendToOutgoingContext(ownerCtx, ctxpkg.TokenHeader, authRes.Token) // Stat changed resource resource - statRes, err := gwClient.Stat(ownerCtx, &providerv1beta1.StatRequest{Ref: ref}) + statRes, err := gwClient.Stat(ownerCtx, &provider.StatRequest{Ref: ref}) if err != nil || statRes.Status.Code != rpc.Code_CODE_OK { - // TODO: log error + p.logger.Error().Err(err).Interface("statRes", statRes).Msg("failed to stat the changed resource") + } + + err = p.indexClient.Add(ref, statRes.Info) + if err != nil { + p.logger.Error().Err(err).Msg("error adding resource to the index") + } else { + p.logDocCount() } - indexClient.Add(ref, statRes.Info) } }() @@ -82,7 +100,7 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s return nil, errtypes.PreconditionFailed("empty query provided") } - listSpacesRes, err := p.gwClient.ListStorageSpaces(ctx, &providerv1beta1.ListStorageSpacesRequest{ + listSpacesRes, err := p.gwClient.ListStorageSpaces(ctx, &provider.ListStorageSpacesRequest{ Opaque: &typesv1beta1.Opaque{Map: map[string]*typesv1beta1.OpaqueEntry{ "path": { Decoder: "plain", @@ -98,7 +116,7 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s for _, space := range listSpacesRes.StorageSpaces { pathPrefix := "" if space.SpaceType == "grant" { - gpRes, err := p.gwClient.GetPath(ctx, &providerv1beta1.GetPathRequest{ + gpRes, err := p.gwClient.GetPath(ctx, &provider.GetPathRequest{ ResourceId: space.Root, }) if err != nil { @@ -136,3 +154,57 @@ func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*s Matches: matches, }, nil } + +func (p *Provider) IndexSpace(ctx context.Context, req *searchsvc.IndexSpaceRequest) (*searchsvc.IndexSpaceResponse, error) { + // get user + res, err := p.gwClient.GetUserByClaim(context.Background(), &user.GetUserByClaimRequest{ + Claim: "username", + Value: req.UserId, + }) + if err != nil || res.Status.Code != rpc.Code_CODE_OK { + fmt.Println("error: Could not get user by userid") + return nil, err + } + + // Get auth context + ownerCtx := ctxpkg.ContextSetUser(context.Background(), res.User) + authRes, err := p.gwClient.Authenticate(ownerCtx, &gateway.AuthenticateRequest{ + Type: "machine", + ClientId: "userid:" + res.User.Id.OpaqueId, + ClientSecret: p.machineAuthAPIKey, + }) + if err != nil || authRes.GetStatus().GetCode() != rpc.Code_CODE_OK { + return nil, err + } + + if authRes.GetStatus().GetCode() != rpc.Code_CODE_OK { + return nil, fmt.Errorf("could not get authenticated context for user") + } + ownerCtx = metadata.AppendToOutgoingContext(ownerCtx, ctxpkg.TokenHeader, authRes.Token) + + // Walk the space and index all files + walker := walker.NewWalker(p.gwClient) + rootId := &provider.ResourceId{StorageId: req.SpaceId, OpaqueId: req.SpaceId} + err = walker.Walk(ownerCtx, rootId, func(wd string, info *provider.ResourceInfo, err error) error { + if err != nil { + p.logger.Error().Err(err).Msg("error walking the tree") + } + ref := &provider.Reference{ + Path: utils.MakeRelativePath(filepath.Join(wd, info.Path)), + ResourceId: rootId, + } + err = p.indexClient.Add(ref, info) + if err != nil { + p.logger.Error().Err(err).Msg("error adding resource to the index") + } else { + p.logger.Debug().Interface("ref", ref).Msg("added resource to index") + } + return nil + }) + if err != nil { + return nil, err + } + + p.logDocCount() + return &searchsvc.IndexSpaceResponse{}, nil +} diff --git a/extensions/search/pkg/search/provider/searchprovider_test.go b/extensions/search/pkg/search/provider/searchprovider_test.go index 46a6d14d6..30b3c13cf 100644 --- a/extensions/search/pkg/search/provider/searchprovider_test.go +++ b/extensions/search/pkg/search/provider/searchprovider_test.go @@ -16,6 +16,7 @@ import ( cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" "github.com/owncloud/ocis/extensions/search/pkg/search/mocks" provider "github.com/owncloud/ocis/extensions/search/pkg/search/provider" + "github.com/owncloud/ocis/ocis-pkg/log" searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) @@ -29,6 +30,12 @@ var _ = Describe("Searchprovider", func() { ctx context.Context eventsChan chan interface{} + logger = log.NewLogger() + user = &userv1beta1.User{ + Id: &userv1beta1.UserId{ + OpaqueId: "user", + }, + } otherUser = &userv1beta1.User{ Id: &userv1beta1.UserId{ OpaqueId: "otheruser", @@ -71,27 +78,27 @@ var _ = Describe("Searchprovider", func() { gwClient = &cs3mocks.GatewayAPIClient{} indexClient = &mocks.IndexClient{} - p = provider.New(gwClient, indexClient, "", eventsChan) + p = provider.New(gwClient, indexClient, "", eventsChan, logger) + + gwClient.On("Authenticate", mock.Anything, mock.Anything).Return(&gateway.AuthenticateResponse{ + Status: status.NewOK(ctx), + Token: "authtoken", + }, nil) + gwClient.On("Stat", mock.Anything, mock.Anything).Return(&sprovider.StatResponse{ + Status: status.NewOK(context.Background()), + Info: ri, + }, nil) + indexClient.On("DocCount").Return(uint64(1), nil) }) Describe("New", func() { It("returns a new instance", func() { - p := provider.New(gwClient, indexClient, "", eventsChan) + p := provider.New(gwClient, indexClient, "", eventsChan, logger) Expect(p).ToNot(BeNil()) }) }) Describe("events", func() { - BeforeEach(func() { - gwClient.On("Authenticate", mock.Anything, mock.Anything).Return(&gateway.AuthenticateResponse{ - Token: "authtoken", - }, nil) - gwClient.On("Stat", mock.Anything, mock.Anything).Return(&sprovider.StatResponse{ - Status: status.NewOK(context.Background()), - Info: ri, - }, nil) - }) - It("trigger an index update when a file has been uploaded", func() { called := false indexClient.On("Add", mock.Anything, mock.MatchedBy(func(riToIndex *sprovider.ResourceInfo) bool { @@ -100,7 +107,8 @@ var _ = Describe("Searchprovider", func() { called = true }) eventsChan <- events.FileUploaded{ - FileID: ref, + FileID: ref, + Executant: user.Id, } Eventually(func() bool { @@ -109,6 +117,25 @@ var _ = Describe("Searchprovider", func() { }) }) + Describe("IndexSpace", func() { + It("walks the space and indexes all files", func() { + gwClient.On("GetUserByClaim", mock.Anything, mock.Anything).Return(&userv1beta1.GetUserByClaimResponse{ + Status: status.NewOK(context.Background()), + User: user, + }, nil) + indexClient.On("Add", mock.Anything, mock.MatchedBy(func(riToIndex *sprovider.ResourceInfo) bool { + return riToIndex.Id.OpaqueId == ri.Id.OpaqueId + })).Return(nil) + + res, err := p.IndexSpace(ctx, &searchsvc.IndexSpaceRequest{ + SpaceId: "storageid", + UserId: "user", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res).ToNot(BeNil()) + }) + }) + Describe("Search", func() { It("fails when an empty query is given", func() { res, err := p.Search(ctx, &searchsvc.SearchRequest{ diff --git a/extensions/search/pkg/search/search.go b/extensions/search/pkg/search/search.go index 4e23b568b..8200a364a 100644 --- a/extensions/search/pkg/search/search.go +++ b/extensions/search/pkg/search/search.go @@ -31,10 +31,13 @@ import ( // ProviderClient is the interface to the search provider service type ProviderClient interface { Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) + IndexSpace(ctx context.Context, req *searchsvc.IndexSpaceRequest) (*searchsvc.IndexSpaceResponse, error) } // IndexClient is the interface to the search index type IndexClient interface { Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) Add(ref *providerv1beta1.Reference, ri *providerv1beta1.ResourceInfo) error + Remove(ri *providerv1beta1.ResourceInfo) error + DocCount() (uint64, error) } diff --git a/extensions/search/pkg/service/v0/service.go b/extensions/search/pkg/service/v0/service.go index a214fa1bd..90349b439 100644 --- a/extensions/search/pkg/service/v0/service.go +++ b/extensions/search/pkg/service/v0/service.go @@ -61,7 +61,7 @@ func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, error) { logger.Fatal().Err(err).Str("addr", cfg.Reva.Address).Msg("could not get reva client") } - provider := searchprovider.New(gwclient, index, cfg.MachineAuthAPIKey, evts) + provider := searchprovider.New(gwclient, index, cfg.MachineAuthAPIKey, evts, logger) return &Service{ id: cfg.GRPC.Namespace + "." + cfg.Service.Name, @@ -99,3 +99,8 @@ func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *s out.NextPageToken = res.NextPageToken return nil } + +func (s Service) IndexSpace(ctx context.Context, in *searchsvc.IndexSpaceRequest, out *searchsvc.IndexSpaceResponse) error { + _, err := s.provider.IndexSpace(ctx, in) + return err +} diff --git a/extensions/webdav/pkg/service/v0/search.go b/extensions/webdav/pkg/service/v0/search.go index 3bce68236..5a7d15f54 100644 --- a/extensions/webdav/pkg/service/v0/search.go +++ b/extensions/webdav/pkg/service/v0/search.go @@ -106,6 +106,22 @@ func matchToPropResponse(ctx context.Context, match *searchmsg.Match) (*propfind Prop: []prop.PropertyXML{}, } + // RDNVW + // 0 + // demo + // demo + // + // https://demo.owncloud.com/f/7 + // application/pdf + // + // + // done: + // 7 + // 6668668 + // 6668668 + // "0cdcdd1bb13a8fed3e54d3b2325dc97c" + // Mon, 25 Apr 2022 06:48:26 GMT + propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("oc:fileid", match.Entity.Id.StorageId+"!"+match.Entity.Id.OpaqueId)) propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("d:getetag", match.Entity.Etag)) propstatOK.Prop = append(propstatOK.Prop, prop.Escaped("d:getlastmodified", match.Entity.LastModifiedTime.AsTime().Format(time.RFC3339))) diff --git a/protogen/gen/ocis/services/search/v0/search.pb.go b/protogen/gen/ocis/services/search/v0/search.pb.go index 7ccbc467b..236bfe95a 100644 --- a/protogen/gen/ocis/services/search/v0/search.pb.go +++ b/protogen/gen/ocis/services/search/v0/search.pb.go @@ -278,6 +278,99 @@ func (x *SearchIndexResponse) GetNextPageToken() string { return "" } +type IndexSpaceRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + SpaceId string `protobuf:"bytes,1,opt,name=space_id,json=spaceId,proto3" json:"space_id,omitempty"` + UserId string `protobuf:"bytes,2,opt,name=user_id,json=userId,proto3" json:"user_id,omitempty"` +} + +func (x *IndexSpaceRequest) Reset() { + *x = IndexSpaceRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IndexSpaceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IndexSpaceRequest) ProtoMessage() {} + +func (x *IndexSpaceRequest) ProtoReflect() protoreflect.Message { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IndexSpaceRequest.ProtoReflect.Descriptor instead. +func (*IndexSpaceRequest) Descriptor() ([]byte, []int) { + return file_ocis_services_search_v0_search_proto_rawDescGZIP(), []int{4} +} + +func (x *IndexSpaceRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *IndexSpaceRequest) GetUserId() string { + if x != nil { + return x.UserId + } + return "" +} + +type IndexSpaceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *IndexSpaceResponse) Reset() { + *x = IndexSpaceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IndexSpaceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IndexSpaceResponse) ProtoMessage() {} + +func (x *IndexSpaceResponse) ProtoReflect() protoreflect.Message { + mi := &file_ocis_services_search_v0_search_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IndexSpaceResponse.ProtoReflect.Descriptor instead. +func (*IndexSpaceResponse) Descriptor() ([]byte, []int) { + return file_ocis_services_search_v0_search_proto_rawDescGZIP(), []int{5} +} + var File_ocis_services_search_v0_search_proto protoreflect.FileDescriptor var file_ocis_services_search_v0_search_proto_rawDesc = []byte{ @@ -329,48 +422,63 @@ var file_ocis_services_search_v0_search_proto_rawDesc = []byte{ 0x63, 0x68, 0x52, 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x32, 0x8d, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x50, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x7b, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x12, 0x26, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, - 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x20, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1a, 0x22, 0x15, 0x2f, 0x61, 0x70, 0x69, 0x2f, - 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x3a, 0x01, 0x2a, 0x32, 0x9d, 0x01, 0x0a, 0x0d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x50, 0x72, 0x6f, - 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x8b, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x12, 0x2b, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, + 0x6b, 0x65, 0x6e, 0x22, 0x47, 0x0a, 0x11, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x22, 0x14, 0x0a, 0x12, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x32, 0x9c, 0x02, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x50, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x7b, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, + 0x26, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, + 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, + 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x20, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x1a, 0x22, 0x15, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, + 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x3a, + 0x01, 0x2a, 0x12, 0x8c, 0x01, 0x0a, 0x0a, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, 0x63, + 0x65, 0x12, 0x2a, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, - 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x20, 0x22, 0x1b, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, - 0x72, 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x3a, 0x01, 0x2a, 0x42, 0xde, 0x02, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63, - 0x69, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x2f, 0x76, 0x30, 0x92, 0x41, 0x9c, 0x02, 0x12, 0xb4, 0x01, 0x0a, 0x1e, 0x6f, 0x77, 0x6e, - 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x65, 0x20, 0x53, - 0x63, 0x61, 0x6c, 0x65, 0x20, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x22, 0x47, 0x0a, 0x0d, 0x6f, - 0x77, 0x6e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, 0x48, 0x12, 0x20, 0x68, 0x74, - 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x1a, 0x14, - 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x2e, 0x63, 0x6f, 0x6d, 0x2a, 0x42, 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2d, 0x32, - 0x2e, 0x30, 0x12, 0x34, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, - 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, - 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, 0x2e, 0x30, 0x2e, 0x30, 0x2a, - 0x02, 0x01, 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3b, 0x0a, 0x10, 0x44, 0x65, 0x76, 0x65, 0x6c, - 0x6f, 0x70, 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x12, 0x27, 0x68, 0x74, 0x74, - 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x64, 0x65, - 0x76, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x73, 0x65, 0x61, - 0x72, 0x63, 0x68, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x70, 0x61, + 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x25, 0x82, 0xd3, 0xe4, 0x93, + 0x02, 0x1f, 0x22, 0x1a, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2d, 0x73, 0x70, 0x61, 0x63, 0x65, 0x3a, 0x01, + 0x2a, 0x32, 0x9d, 0x01, 0x0a, 0x0d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x76, 0x69, + 0x64, 0x65, 0x72, 0x12, 0x8b, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x2b, + 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6f, 0x63, + 0x69, 0x73, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x26, 0x82, 0xd3, 0xe4, 0x93, 0x02, + 0x20, 0x22, 0x1b, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x3a, 0x01, + 0x2a, 0x42, 0xde, 0x02, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63, 0x69, 0x73, + 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2f, + 0x76, 0x30, 0x92, 0x41, 0x9c, 0x02, 0x12, 0xb4, 0x01, 0x0a, 0x1e, 0x6f, 0x77, 0x6e, 0x43, 0x6c, + 0x6f, 0x75, 0x64, 0x20, 0x49, 0x6e, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x65, 0x20, 0x53, 0x63, 0x61, + 0x6c, 0x65, 0x20, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x22, 0x47, 0x0a, 0x0d, 0x6f, 0x77, 0x6e, + 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x20, 0x47, 0x6d, 0x62, 0x48, 0x12, 0x20, 0x68, 0x74, 0x74, 0x70, + 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, + 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x1a, 0x14, 0x73, 0x75, + 0x70, 0x70, 0x6f, 0x72, 0x74, 0x40, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x63, + 0x6f, 0x6d, 0x2a, 0x42, 0x0a, 0x0a, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2d, 0x32, 0x2e, 0x30, + 0x12, 0x34, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, + 0x69, 0x73, 0x2f, 0x62, 0x6c, 0x6f, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x4c, + 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x32, 0x05, 0x31, 0x2e, 0x30, 0x2e, 0x30, 0x2a, 0x02, 0x01, + 0x02, 0x32, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6a, + 0x73, 0x6f, 0x6e, 0x3a, 0x10, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x2f, 0x6a, 0x73, 0x6f, 0x6e, 0x72, 0x3b, 0x0a, 0x10, 0x44, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70, + 0x65, 0x72, 0x20, 0x4d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x12, 0x27, 0x68, 0x74, 0x74, 0x70, 0x73, + 0x3a, 0x2f, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x64, 0x65, 0x76, 0x2f, + 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -385,25 +493,29 @@ func file_ocis_services_search_v0_search_proto_rawDescGZIP() []byte { return file_ocis_services_search_v0_search_proto_rawDescData } -var file_ocis_services_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_ocis_services_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_ocis_services_search_v0_search_proto_goTypes = []interface{}{ (*SearchRequest)(nil), // 0: ocis.services.search.v0.SearchRequest (*SearchResponse)(nil), // 1: ocis.services.search.v0.SearchResponse (*SearchIndexRequest)(nil), // 2: ocis.services.search.v0.SearchIndexRequest (*SearchIndexResponse)(nil), // 3: ocis.services.search.v0.SearchIndexResponse - (*v0.Match)(nil), // 4: ocis.messages.search.v0.Match - (*v0.Reference)(nil), // 5: ocis.messages.search.v0.Reference + (*IndexSpaceRequest)(nil), // 4: ocis.services.search.v0.IndexSpaceRequest + (*IndexSpaceResponse)(nil), // 5: ocis.services.search.v0.IndexSpaceResponse + (*v0.Match)(nil), // 6: ocis.messages.search.v0.Match + (*v0.Reference)(nil), // 7: ocis.messages.search.v0.Reference } var file_ocis_services_search_v0_search_proto_depIdxs = []int32{ - 4, // 0: ocis.services.search.v0.SearchResponse.matches:type_name -> ocis.messages.search.v0.Match - 5, // 1: ocis.services.search.v0.SearchIndexRequest.ref:type_name -> ocis.messages.search.v0.Reference - 4, // 2: ocis.services.search.v0.SearchIndexResponse.matches:type_name -> ocis.messages.search.v0.Match + 6, // 0: ocis.services.search.v0.SearchResponse.matches:type_name -> ocis.messages.search.v0.Match + 7, // 1: ocis.services.search.v0.SearchIndexRequest.ref:type_name -> ocis.messages.search.v0.Reference + 6, // 2: ocis.services.search.v0.SearchIndexResponse.matches:type_name -> ocis.messages.search.v0.Match 0, // 3: ocis.services.search.v0.SearchProvider.Search:input_type -> ocis.services.search.v0.SearchRequest - 2, // 4: ocis.services.search.v0.IndexProvider.Search:input_type -> ocis.services.search.v0.SearchIndexRequest - 1, // 5: ocis.services.search.v0.SearchProvider.Search:output_type -> ocis.services.search.v0.SearchResponse - 3, // 6: ocis.services.search.v0.IndexProvider.Search:output_type -> ocis.services.search.v0.SearchIndexResponse - 5, // [5:7] is the sub-list for method output_type - 3, // [3:5] is the sub-list for method input_type + 4, // 4: ocis.services.search.v0.SearchProvider.IndexSpace:input_type -> ocis.services.search.v0.IndexSpaceRequest + 2, // 5: ocis.services.search.v0.IndexProvider.Search:input_type -> ocis.services.search.v0.SearchIndexRequest + 1, // 6: ocis.services.search.v0.SearchProvider.Search:output_type -> ocis.services.search.v0.SearchResponse + 5, // 7: ocis.services.search.v0.SearchProvider.IndexSpace:output_type -> ocis.services.search.v0.IndexSpaceResponse + 3, // 8: ocis.services.search.v0.IndexProvider.Search:output_type -> ocis.services.search.v0.SearchIndexResponse + 6, // [6:9] is the sub-list for method output_type + 3, // [3:6] is the sub-list for method input_type 3, // [3:3] is the sub-list for extension type_name 3, // [3:3] is the sub-list for extension extendee 0, // [0:3] is the sub-list for field type_name @@ -463,6 +575,30 @@ func file_ocis_services_search_v0_search_proto_init() { return nil } } + file_ocis_services_search_v0_search_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IndexSpaceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ocis_services_search_v0_search_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IndexSpaceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -470,7 +606,7 @@ func file_ocis_services_search_v0_search_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_ocis_services_search_v0_search_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 6, NumExtensions: 0, NumServices: 2, }, diff --git a/protogen/gen/ocis/services/search/v0/search.pb.micro.go b/protogen/gen/ocis/services/search/v0/search.pb.micro.go index 2c45e6a7e..06e5ea9ae 100644 --- a/protogen/gen/ocis/services/search/v0/search.pb.micro.go +++ b/protogen/gen/ocis/services/search/v0/search.pb.micro.go @@ -42,6 +42,13 @@ func NewSearchProviderEndpoints() []*api.Endpoint { Body: "*", Handler: "rpc", }, + { + Name: "SearchProvider.IndexSpace", + Path: []string{"/api/v0/search/index-space"}, + Method: []string{"POST"}, + Body: "*", + Handler: "rpc", + }, } } @@ -49,6 +56,7 @@ func NewSearchProviderEndpoints() []*api.Endpoint { type SearchProviderService interface { Search(ctx context.Context, in *SearchRequest, opts ...client.CallOption) (*SearchResponse, error) + IndexSpace(ctx context.Context, in *IndexSpaceRequest, opts ...client.CallOption) (*IndexSpaceResponse, error) } type searchProviderService struct { @@ -73,15 +81,27 @@ func (c *searchProviderService) Search(ctx context.Context, in *SearchRequest, o return out, nil } +func (c *searchProviderService) IndexSpace(ctx context.Context, in *IndexSpaceRequest, opts ...client.CallOption) (*IndexSpaceResponse, error) { + req := c.c.NewRequest(c.name, "SearchProvider.IndexSpace", in) + out := new(IndexSpaceResponse) + err := c.c.Call(ctx, req, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for SearchProvider service type SearchProviderHandler interface { Search(context.Context, *SearchRequest, *SearchResponse) error + IndexSpace(context.Context, *IndexSpaceRequest, *IndexSpaceResponse) error } func RegisterSearchProviderHandler(s server.Server, hdlr SearchProviderHandler, opts ...server.HandlerOption) error { type searchProvider interface { Search(ctx context.Context, in *SearchRequest, out *SearchResponse) error + IndexSpace(ctx context.Context, in *IndexSpaceRequest, out *IndexSpaceResponse) error } type SearchProvider struct { searchProvider @@ -94,6 +114,13 @@ func RegisterSearchProviderHandler(s server.Server, hdlr SearchProviderHandler, Body: "*", Handler: "rpc", })) + opts = append(opts, api.WithEndpoint(&api.Endpoint{ + Name: "SearchProvider.IndexSpace", + Path: []string{"/api/v0/search/index-space"}, + Method: []string{"POST"}, + Body: "*", + Handler: "rpc", + })) return s.Handle(s.NewHandler(&SearchProvider{h}, opts...)) } @@ -105,6 +132,10 @@ func (h *searchProviderHandler) Search(ctx context.Context, in *SearchRequest, o return h.SearchProviderHandler.Search(ctx, in, out) } +func (h *searchProviderHandler) IndexSpace(ctx context.Context, in *IndexSpaceRequest, out *IndexSpaceResponse) error { + return h.SearchProviderHandler.IndexSpace(ctx, in, out) +} + // Api Endpoints for IndexProvider service func NewIndexProviderEndpoints() []*api.Endpoint { diff --git a/protogen/gen/ocis/services/search/v0/search.pb.web.go b/protogen/gen/ocis/services/search/v0/search.pb.web.go index 46105d330..427033152 100644 --- a/protogen/gen/ocis/services/search/v0/search.pb.web.go +++ b/protogen/gen/ocis/services/search/v0/search.pb.web.go @@ -44,6 +44,28 @@ func (h *webSearchProviderHandler) Search(w http.ResponseWriter, r *http.Request render.JSON(w, r, resp) } +func (h *webSearchProviderHandler) IndexSpace(w http.ResponseWriter, r *http.Request) { + req := &IndexSpaceRequest{} + resp := &IndexSpaceResponse{} + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusPreconditionFailed) + return + } + + if err := h.h.IndexSpace( + r.Context(), + req, + resp, + ); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + render.Status(r, http.StatusCreated) + render.JSON(w, r, resp) +} + func RegisterSearchProviderWeb(r chi.Router, i SearchProviderHandler, middlewares ...func(http.Handler) http.Handler) { handler := &webSearchProviderHandler{ r: r, @@ -51,6 +73,7 @@ func RegisterSearchProviderWeb(r chi.Router, i SearchProviderHandler, middleware } r.MethodFunc("POST", "/api/v0/search/search", handler.Search) + r.MethodFunc("POST", "/api/v0/search/index-space", handler.IndexSpace) } type webIndexProviderHandler struct { @@ -236,3 +259,75 @@ func (m *SearchIndexResponse) UnmarshalJSON(b []byte) error { } var _ json.Unmarshaler = (*SearchIndexResponse)(nil) + +// IndexSpaceRequestJSONMarshaler describes the default jsonpb.Marshaler used by all +// instances of IndexSpaceRequest. This struct is safe to replace or modify but +// should not be done so concurrently. +var IndexSpaceRequestJSONMarshaler = new(jsonpb.Marshaler) + +// MarshalJSON satisfies the encoding/json Marshaler interface. This method +// uses the more correct jsonpb package to correctly marshal the message. +func (m *IndexSpaceRequest) MarshalJSON() ([]byte, error) { + if m == nil { + return json.Marshal(nil) + } + + buf := &bytes.Buffer{} + + if err := IndexSpaceRequestJSONMarshaler.Marshal(buf, m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +var _ json.Marshaler = (*IndexSpaceRequest)(nil) + +// IndexSpaceRequestJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all +// instances of IndexSpaceRequest. This struct is safe to replace or modify but +// should not be done so concurrently. +var IndexSpaceRequestJSONUnmarshaler = new(jsonpb.Unmarshaler) + +// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method +// uses the more correct jsonpb package to correctly unmarshal the message. +func (m *IndexSpaceRequest) UnmarshalJSON(b []byte) error { + return IndexSpaceRequestJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m) +} + +var _ json.Unmarshaler = (*IndexSpaceRequest)(nil) + +// IndexSpaceResponseJSONMarshaler describes the default jsonpb.Marshaler used by all +// instances of IndexSpaceResponse. This struct is safe to replace or modify but +// should not be done so concurrently. +var IndexSpaceResponseJSONMarshaler = new(jsonpb.Marshaler) + +// MarshalJSON satisfies the encoding/json Marshaler interface. This method +// uses the more correct jsonpb package to correctly marshal the message. +func (m *IndexSpaceResponse) MarshalJSON() ([]byte, error) { + if m == nil { + return json.Marshal(nil) + } + + buf := &bytes.Buffer{} + + if err := IndexSpaceResponseJSONMarshaler.Marshal(buf, m); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +var _ json.Marshaler = (*IndexSpaceResponse)(nil) + +// IndexSpaceResponseJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all +// instances of IndexSpaceResponse. This struct is safe to replace or modify but +// should not be done so concurrently. +var IndexSpaceResponseJSONUnmarshaler = new(jsonpb.Unmarshaler) + +// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method +// uses the more correct jsonpb package to correctly unmarshal the message. +func (m *IndexSpaceResponse) UnmarshalJSON(b []byte) error { + return IndexSpaceResponseJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m) +} + +var _ json.Unmarshaler = (*IndexSpaceResponse)(nil) diff --git a/protogen/gen/ocis/services/search/v0/search.swagger.json b/protogen/gen/ocis/services/search/v0/search.swagger.json index b06c03da8..26567aab8 100644 --- a/protogen/gen/ocis/services/search/v0/search.swagger.json +++ b/protogen/gen/ocis/services/search/v0/search.swagger.json @@ -32,6 +32,38 @@ "application/json" ], "paths": { + "/api/v0/search/index-space": { + "post": { + "operationId": "SearchProvider_IndexSpace", + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/v0IndexSpaceResponse" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/rpcStatus" + } + } + }, + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v0IndexSpaceRequest" + } + } + ], + "tags": [ + "SearchProvider" + ] + } + }, "/api/v0/search/index/search": { "post": { "operationId": "IndexProvider_Search", @@ -156,6 +188,20 @@ } } }, + "v0IndexSpaceRequest": { + "type": "object", + "properties": { + "spaceId": { + "type": "string" + }, + "userId": { + "type": "string" + } + } + }, + "v0IndexSpaceResponse": { + "type": "object" + }, "v0Match": { "type": "object", "properties": { diff --git a/protogen/proto/ocis/services/search/v0/search.proto b/protogen/proto/ocis/services/search/v0/search.proto index 8bc8fb392..7d78dae35 100644 --- a/protogen/proto/ocis/services/search/v0/search.proto +++ b/protogen/proto/ocis/services/search/v0/search.proto @@ -36,17 +36,21 @@ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { service SearchProvider { rpc Search(SearchRequest) returns (SearchResponse) { - // List method maps to HTTP GET option (google.api.http) = { post: "/api/v0/search/search", body: "*" }; }; + rpc IndexSpace(IndexSpaceRequest) returns (IndexSpaceResponse) { + option (google.api.http) = { + post: "/api/v0/search/index-space", + body: "*" + }; + } } service IndexProvider { rpc Search(SearchIndexRequest) returns (SearchIndexResponse) { - // List method maps to HTTP GET option (google.api.http) = { post: "/api/v0/search/index/search", body: "*" @@ -92,4 +96,12 @@ message SearchIndexResponse { // Token to retrieve the next page of results, or empty if there are no // more results in the list string next_page_token = 2; +} + +message IndexSpaceRequest { + string space_id = 1; + string user_id = 2; +} + +message IndexSpaceResponse { } \ No newline at end of file From 4dd89aa804071a145cd4c27fb3e55fdbf8d91894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 28 Apr 2022 08:39:12 +0200 Subject: [PATCH 33/49] Adapt to renamed field --- extensions/audit/pkg/types/conversion.go | 14 +++++++------- .../search/pkg/search/provider/searchprovider.go | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/extensions/audit/pkg/types/conversion.go b/extensions/audit/pkg/types/conversion.go index 5c3f3f1c8..fbb789ea5 100644 --- a/extensions/audit/pkg/types/conversion.go +++ b/extensions/audit/pkg/types/conversion.go @@ -234,7 +234,7 @@ func FilesAuditEvent(base AuditEvent, itemid, owner, path string) AuditEventFile // FileUploaded converts a FileUploaded event to an AuditEventFileCreated func FileUploaded(ev events.FileUploaded) AuditEventFileCreated { - iid, path, uid := extractFileDetails(ev.FileID, ev.Owner) + iid, path, uid := extractFileDetails(ev.Ref, ev.Owner) base := BasicAuditEvent(uid, "", MessageFileCreated(iid), ActionFileCreated) return AuditEventFileCreated{ AuditEventFiles: FilesAuditEvent(base, iid, uid, path), @@ -243,7 +243,7 @@ func FileUploaded(ev events.FileUploaded) AuditEventFileCreated { // FileDownloaded converts a FileDownloaded event to an AuditEventFileRead func FileDownloaded(ev events.FileDownloaded) AuditEventFileRead { - iid, path, uid := extractFileDetails(ev.FileID, ev.Owner) + iid, path, uid := extractFileDetails(ev.Ref, ev.Owner) base := BasicAuditEvent(uid, "", MessageFileRead(iid), ActionFileRead) return AuditEventFileRead{ AuditEventFiles: FilesAuditEvent(base, iid, uid, path), @@ -252,7 +252,7 @@ func FileDownloaded(ev events.FileDownloaded) AuditEventFileRead { // ItemMoved converts a ItemMoved event to an AuditEventFileRenamed func ItemMoved(ev events.ItemMoved) AuditEventFileRenamed { - iid, path, uid := extractFileDetails(ev.FileID, ev.Owner) + iid, path, uid := extractFileDetails(ev.Ref, ev.Owner) oldpath := "" if ev.OldReference != nil { @@ -268,7 +268,7 @@ func ItemMoved(ev events.ItemMoved) AuditEventFileRenamed { // ItemTrashed converts a ItemTrashed event to an AuditEventFileDeleted func ItemTrashed(ev events.ItemTrashed) AuditEventFileDeleted { - iid, path, uid := extractFileDetails(ev.FileID, ev.Owner) + iid, path, uid := extractFileDetails(ev.Ref, ev.Owner) base := BasicAuditEvent(uid, "", MessageFileTrashed(iid), ActionFileTrashed) return AuditEventFileDeleted{ AuditEventFiles: FilesAuditEvent(base, iid, uid, path), @@ -277,7 +277,7 @@ func ItemTrashed(ev events.ItemTrashed) AuditEventFileDeleted { // ItemPurged converts a ItemPurged event to an AuditEventFilePurged func ItemPurged(ev events.ItemPurged) AuditEventFilePurged { - iid, path, uid := extractFileDetails(ev.FileID, ev.Owner) + iid, path, uid := extractFileDetails(ev.Ref, ev.Owner) base := BasicAuditEvent(uid, "", MessageFilePurged(iid), ActionFilePurged) return AuditEventFilePurged{ AuditEventFiles: FilesAuditEvent(base, iid, uid, path), @@ -286,7 +286,7 @@ func ItemPurged(ev events.ItemPurged) AuditEventFilePurged { // ItemRestored converts a ItemRestored event to an AuditEventFileRestored func ItemRestored(ev events.ItemRestored) AuditEventFileRestored { - iid, path, uid := extractFileDetails(ev.FileID, ev.Owner) + iid, path, uid := extractFileDetails(ev.Ref, ev.Owner) oldpath := "" if ev.OldReference != nil { @@ -302,7 +302,7 @@ func ItemRestored(ev events.ItemRestored) AuditEventFileRestored { // FileVersionRestored converts a FileVersionRestored event to an AuditEventFileVersionRestored func FileVersionRestored(ev events.FileVersionRestored) AuditEventFileVersionRestored { - iid, path, uid := extractFileDetails(ev.FileID, ev.Owner) + iid, path, uid := extractFileDetails(ev.Ref, ev.Owner) base := BasicAuditEvent(uid, "", MessageFileVersionRestored(iid, ev.Key), ActionFileVersionRestored) return AuditEventFileVersionRestored{ AuditEventFiles: FilesAuditEvent(base, iid, uid, path), diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index 5a3658f06..cee5df910 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -47,7 +47,7 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach var owner *user.User switch e := ev.(type) { case events.FileUploaded: - ref = e.FileID + ref = e.Ref owner = &user.User{ Id: e.Executant, } From 0b0efd6799bef2d16cfe28a70ff18925fa3125e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 28 Apr 2022 08:39:21 +0200 Subject: [PATCH 34/49] Remove items from the index when they were deleted --- extensions/search/pkg/search/index/index.go | 4 ++-- .../search/pkg/search/index/index_test.go | 2 +- .../search/pkg/search/mocks/IndexClient.go | 4 ++-- .../pkg/search/provider/searchprovider.go | 20 ++++++++++++---- .../search/provider/searchprovider_test.go | 24 ++++++++++++++++++- extensions/search/pkg/search/search.go | 2 +- 6 files changed, 45 insertions(+), 11 deletions(-) diff --git a/extensions/search/pkg/search/index/index.go b/extensions/search/pkg/search/index/index.go index ed4b88dfa..f61c9e2a3 100644 --- a/extensions/search/pkg/search/index/index.go +++ b/extensions/search/pkg/search/index/index.go @@ -77,8 +77,8 @@ func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error } // Remove removes an entity from the index -func (i *Index) Remove(ri *sprovider.ResourceInfo) error { - return i.bleveIndex.Delete(idToBleveId(ri.Id)) +func (i *Index) Remove(id *sprovider.ResourceId) error { + return i.bleveIndex.Delete(idToBleveId(id)) } // Search searches the index according to the criteria specified in the given SearchIndexRequest diff --git a/extensions/search/pkg/search/index/index_test.go b/extensions/search/pkg/search/index/index_test.go index bde7dffef..98ae27dd1 100644 --- a/extensions/search/pkg/search/index/index_test.go +++ b/extensions/search/pkg/search/index/index_test.go @@ -248,7 +248,7 @@ var _ = Describe("Index", func() { count, _ := bleveIndex.DocCount() Expect(count).To(Equal(uint64(1))) - err = i.Remove(ri) + err = i.Remove(ri.Id) Expect(err).ToNot(HaveOccurred()) count, _ = bleveIndex.DocCount() Expect(count).To(Equal(uint64(0))) diff --git a/extensions/search/pkg/search/mocks/IndexClient.go b/extensions/search/pkg/search/mocks/IndexClient.go index fd8e3167b..dae95ec51 100644 --- a/extensions/search/pkg/search/mocks/IndexClient.go +++ b/extensions/search/pkg/search/mocks/IndexClient.go @@ -52,11 +52,11 @@ func (_m *IndexClient) DocCount() (uint64, error) { } // Remove provides a mock function with given fields: ri -func (_m *IndexClient) Remove(ri *providerv1beta1.ResourceInfo) error { +func (_m *IndexClient) Remove(ri *providerv1beta1.ResourceId) error { ret := _m.Called(ri) var r0 error - if rf, ok := ret.Get(0).(func(*providerv1beta1.ResourceInfo) error); ok { + if rf, ok := ret.Get(0).(func(*providerv1beta1.ResourceId) error); ok { r0 = rf(ri) } else { r0 = ret.Error(0) diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index cee5df910..471024a56 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -51,6 +51,12 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach owner = &user.User{ Id: e.Executant, } + case events.ItemTrashed: + err := p.indexClient.Remove(e.Id) + if err != nil { + p.logger.Error().Err(err).Interface("Id", e.Id).Msg("failed to remove item from index") + } + continue default: // Not sure what to do here. Skip. continue @@ -70,13 +76,19 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach // Stat changed resource resource statRes, err := gwClient.Stat(ownerCtx, &provider.StatRequest{Ref: ref}) - if err != nil || statRes.Status.Code != rpc.Code_CODE_OK { - p.logger.Error().Err(err).Interface("statRes", statRes).Msg("failed to stat the changed resource") + if err != nil { + p.logger.Error().Err(err).Msg("failed to stat the changed resource") + } + + switch statRes.Status.Code { + case rpc.Code_CODE_OK: + err = p.indexClient.Add(ref, statRes.Info) + default: + p.logger.Error().Interface("statRes", statRes).Msg("failed to stat the changed resource") } - err = p.indexClient.Add(ref, statRes.Info) if err != nil { - p.logger.Error().Err(err).Msg("error adding resource to the index") + p.logger.Error().Err(err).Msg("error adding updating the resource in the index") } else { p.logDocCount() } diff --git a/extensions/search/pkg/search/provider/searchprovider_test.go b/extensions/search/pkg/search/provider/searchprovider_test.go index 30b3c13cf..a622ea956 100644 --- a/extensions/search/pkg/search/provider/searchprovider_test.go +++ b/extensions/search/pkg/search/provider/searchprovider_test.go @@ -107,7 +107,29 @@ var _ = Describe("Searchprovider", func() { called = true }) eventsChan <- events.FileUploaded{ - FileID: ref, + Ref: ref, + Executant: user.Id, + } + + Eventually(func() bool { + return called + }).Should(BeTrue()) + }) + + It("removes an entry from the index when the file has been deleted", func() { + called := false + + gwClient.On("Stat", mock.Anything, mock.Anything).Return(&sprovider.StatResponse{ + Status: status.NewNotFound(context.Background(), ""), + }, nil) + indexClient.On("Remove", mock.MatchedBy(func(id *sprovider.ResourceId) bool { + return id.OpaqueId == ri.Id.OpaqueId + })).Return(nil).Run(func(args mock.Arguments) { + called = true + }) + eventsChan <- events.ItemTrashed{ + Ref: ref, + Id: ri.Id, Executant: user.Id, } diff --git a/extensions/search/pkg/search/search.go b/extensions/search/pkg/search/search.go index 8200a364a..b5e1ed11e 100644 --- a/extensions/search/pkg/search/search.go +++ b/extensions/search/pkg/search/search.go @@ -38,6 +38,6 @@ type ProviderClient interface { type IndexClient interface { Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) Add(ref *providerv1beta1.Reference, ri *providerv1beta1.ResourceInfo) error - Remove(ri *providerv1beta1.ResourceInfo) error + Remove(ri *providerv1beta1.ResourceId) error DocCount() (uint64, error) } From e5338dff55367bb78ff6b892df728e9ea56c7021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 28 Apr 2022 08:51:18 +0200 Subject: [PATCH 35/49] Trigger updates on all relevant events --- .../pkg/search/provider/searchprovider.go | 15 ++++++ .../search/provider/searchprovider_test.go | 51 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index 471024a56..223251c09 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -51,6 +51,21 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach owner = &user.User{ Id: e.Executant, } + case events.ItemMoved: + ref = e.Ref + owner = &user.User{ + Id: e.Executant, + } + case events.ItemRestored: + ref = e.Ref + owner = &user.User{ + Id: e.Executant, + } + case events.FileVersionRestored: + ref = e.Ref + owner = &user.User{ + Id: e.Executant, + } case events.ItemTrashed: err := p.indexClient.Remove(e.Id) if err != nil { diff --git a/extensions/search/pkg/search/provider/searchprovider_test.go b/extensions/search/pkg/search/provider/searchprovider_test.go index a622ea956..0ce1f4ab5 100644 --- a/extensions/search/pkg/search/provider/searchprovider_test.go +++ b/extensions/search/pkg/search/provider/searchprovider_test.go @@ -137,6 +137,57 @@ var _ = Describe("Searchprovider", func() { return called }).Should(BeTrue()) }) + + It("indexes items when they are being restored", func() { + called := false + indexClient.On("Add", mock.Anything, mock.MatchedBy(func(riToIndex *sprovider.ResourceInfo) bool { + return riToIndex.Id.OpaqueId == ri.Id.OpaqueId + })).Return(nil).Run(func(args mock.Arguments) { + called = true + }) + eventsChan <- events.ItemRestored{ + Ref: ref, + Executant: user.Id, + } + + Eventually(func() bool { + return called + }).Should(BeTrue()) + }) + + It("indexes items when a version has been restored", func() { + called := false + indexClient.On("Add", mock.Anything, mock.MatchedBy(func(riToIndex *sprovider.ResourceInfo) bool { + return riToIndex.Id.OpaqueId == ri.Id.OpaqueId + })).Return(nil).Run(func(args mock.Arguments) { + called = true + }) + eventsChan <- events.FileVersionRestored{ + Ref: ref, + Executant: user.Id, + } + + Eventually(func() bool { + return called + }).Should(BeTrue()) + }) + + It("indexes items when they are being moved", func() { + called := false + indexClient.On("Add", mock.Anything, mock.MatchedBy(func(riToIndex *sprovider.ResourceInfo) bool { + return riToIndex.Id.OpaqueId == ri.Id.OpaqueId + })).Return(nil).Run(func(args mock.Arguments) { + called = true + }) + eventsChan <- events.ItemMoved{ + Ref: ref, + Executant: user.Id, + } + + Eventually(func() bool { + return called + }).Should(BeTrue()) + }) }) Describe("IndexSpace", func() { From 68c5e2366ca1d3db7b8ddac0c85a27743dde5a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 28 Apr 2022 10:10:01 +0200 Subject: [PATCH 36/49] Store the parent id in the index to allow for handling directories --- extensions/search/pkg/search/index/index.go | 18 +++++++++++++++--- .../search/pkg/search/index/index_test.go | 5 +++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/extensions/search/pkg/search/index/index.go b/extensions/search/pkg/search/index/index.go index f61c9e2a3..ca550647f 100644 --- a/extensions/search/pkg/search/index/index.go +++ b/extensions/search/pkg/search/index/index.go @@ -34,9 +34,10 @@ import ( ) type indexDocument struct { - RootID string - Path string - ID string + RootID string + Path string + ID string + ParentID string Name string Size uint64 @@ -121,6 +122,7 @@ func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocume RootID: idToBleveId(ref.ResourceId), Path: ref.Path, ID: idToBleveId(ri.Id), + ParentID: idToBleveId(ri.ParentId), Name: ri.Path, Size: ri.Size, MimeType: ri.MimeType, @@ -159,10 +161,20 @@ func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) { if mtime, err := time.Parse(time.RFC3339, fields["Mtime"].(string)); err == nil { match.Entity.LastModifiedTime = ×tamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())} } + if fields["ParentID"] != "" { + parentIDParts := strings.SplitN(fields["ParentID"].(string), "!", 2) + match.Entity.ParentId = &searchmsg.ResourceID{ + StorageId: parentIDParts[0], + OpaqueId: parentIDParts[1], + } + } return match, nil } func idToBleveId(id *sprovider.ResourceId) string { + if id == nil { + return "" + } return id.StorageId + "!" + id.OpaqueId } diff --git a/extensions/search/pkg/search/index/index_test.go b/extensions/search/pkg/search/index/index_test.go index 98ae27dd1..333a94926 100644 --- a/extensions/search/pkg/search/index/index_test.go +++ b/extensions/search/pkg/search/index/index_test.go @@ -44,6 +44,10 @@ var _ = Describe("Index", func() { StorageId: "storageid", OpaqueId: "opaqueid", }, + ParentId: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "parentopaqueid", + }, Path: "foo.pdf", Size: 12345, MimeType: "application/pdf", @@ -124,6 +128,7 @@ var _ = Describe("Index", func() { Expect(match.Entity.Name).To(Equal(ri.Path)) Expect(match.Entity.Size).To(Equal(ri.Size)) Expect(match.Entity.MimeType).To(Equal(ri.MimeType)) + Expect(match.Entity.ParentId.OpaqueId).To(Equal(ri.ParentId.OpaqueId)) Expect(uint64(match.Entity.LastModifiedTime.AsTime().Unix())).To(Equal(ri.Mtime.Seconds)) }) From 6bd67c0f273723b640c46e2d847263d76bab4cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 29 Apr 2022 09:00:14 +0200 Subject: [PATCH 37/49] Mark items as deleted (recursively) but still keep them around --- extensions/search/pkg/search/index/index.go | 83 ++++++++-- .../search/pkg/search/index/index_test.go | 150 ++++++++++++++---- .../search/pkg/search/mocks/IndexClient.go | 18 ++- .../pkg/search/provider/searchprovider.go | 2 +- .../search/provider/searchprovider_test.go | 2 +- extensions/search/pkg/search/search.go | 3 +- .../gen/ocis/messages/search/v0/search.pb.go | 43 +++-- .../services/search/v0/search.swagger.json | 7 + .../ocis/messages/search/v0/search.proto | 2 + 9 files changed, 246 insertions(+), 64 deletions(-) diff --git a/extensions/search/pkg/search/index/index.go b/extensions/search/pkg/search/index/index.go index ca550647f..aa5fe0dd5 100644 --- a/extensions/search/pkg/search/index/index.go +++ b/extensions/search/pkg/search/index/index.go @@ -20,6 +20,7 @@ package index import ( "context" + "errors" "strings" "time" @@ -34,15 +35,17 @@ import ( ) type indexDocument struct { - RootID string - Path string - ID string - ParentID string + RootID string + Path string + ID string Name string Size uint64 Mtime string MimeType string + Type uint64 + + Deleted bool } // Index represents a bleve based search index @@ -77,15 +80,56 @@ func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error return i.bleveIndex.Index(idToBleveId(ri.Id), entity) } -// Remove removes an entity from the index -func (i *Index) Remove(id *sprovider.ResourceId) error { +// Delete marks an entity from the index as delete (still keeping it around) +func (i *Index) Delete(id *sprovider.ResourceId) error { + return i.markAsDeleted(idToBleveId(id)) +} + +func (i *Index) markAsDeleted(id string) error { + req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{id})) + req.Fields = []string{"*"} + res, err := i.bleveIndex.Search(req) + if err != nil { + return err + } + if res.Hits.Len() == 0 { + return errors.New("entity not found") + } + entity := fieldsToEntity(res.Hits[0].Fields) + entity.Deleted = true + + if entity.Type == uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) { + query := bleve.NewConjunctionQuery( + bleve.NewQueryStringQuery("RootID:"+entity.RootID), + bleve.NewQueryStringQuery("Path:"+entity.Path+"/*"), + ) + bleveReq := bleve.NewSearchRequest(query) + bleveReq.Fields = []string{"*"} + res, err := i.bleveIndex.Search(bleveReq) + if err != nil { + return err + } + + for _, h := range res.Hits { + i.markAsDeleted(h.ID) + } + } + + return i.bleveIndex.Index(entity.ID, entity) +} + +// Purge removes an entity from the index +func (i *Index) Purge(id *sprovider.ResourceId) error { return i.bleveIndex.Delete(idToBleveId(id)) } // Search searches the index according to the criteria specified in the given SearchIndexRequest func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) { + deletedQuery := bleve.NewBoolFieldQuery(false) + deletedQuery.SetField("Deleted") query := bleve.NewConjunctionQuery( bleve.NewQueryStringQuery("Name:"+req.Query), + deletedQuery, // Skip documents that have been marked as deleted bleve.NewQueryStringQuery("RootID:"+req.Ref.ResourceId.StorageId+"!"+req.Ref.ResourceId.OpaqueId), // Limit search to the space bleve.NewQueryStringQuery("Path:"+req.Ref.Path+"*"), // Limit search to this directory in the space ) @@ -122,10 +166,11 @@ func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocume RootID: idToBleveId(ref.ResourceId), Path: ref.Path, ID: idToBleveId(ri.Id), - ParentID: idToBleveId(ri.ParentId), Name: ri.Path, Size: ri.Size, MimeType: ri.MimeType, + Type: uint64(ri.Type), + Deleted: false, } if ri.Mtime != nil { @@ -135,6 +180,21 @@ func toEntity(ref *sprovider.Reference, ri *sprovider.ResourceInfo) *indexDocume return doc } +func fieldsToEntity(fields map[string]interface{}) *indexDocument { + doc := &indexDocument{ + RootID: fields["RootID"].(string), + Path: fields["Path"].(string), + ID: fields["ID"].(string), + Name: fields["Name"].(string), + Size: uint64(fields["Size"].(float64)), + Mtime: fields["Mtime"].(string), + MimeType: fields["MimeType"].(string), + Deleted: fields["Deleted"].(bool), + Type: uint64(fields["Type"].(float64)), + } + return doc +} + func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) { rootIDParts := strings.SplitN(fields["RootID"].(string), "!", 2) IDParts := strings.SplitN(fields["ID"].(string), "!", 2) @@ -154,20 +214,15 @@ func fromFields(fields map[string]interface{}) (*searchmsg.Match, error) { }, Name: fields["Name"].(string), Size: uint64(fields["Size"].(float64)), + Type: uint64(fields["Type"].(float64)), MimeType: fields["MimeType"].(string), + Deleted: fields["Deleted"].(bool), }, } if mtime, err := time.Parse(time.RFC3339, fields["Mtime"].(string)); err == nil { match.Entity.LastModifiedTime = ×tamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())} } - if fields["ParentID"] != "" { - parentIDParts := strings.SplitN(fields["ParentID"].(string), "!", 2) - match.Entity.ParentId = &searchmsg.ResourceID{ - StorageId: parentIDParts[0], - OpaqueId: parentIDParts[1], - } - } return match, nil } diff --git a/extensions/search/pkg/search/index/index_test.go b/extensions/search/pkg/search/index/index_test.go index 333a94926..7e0591810 100644 --- a/extensions/search/pkg/search/index/index_test.go +++ b/extensions/search/pkg/search/index/index_test.go @@ -18,10 +18,63 @@ var _ = Describe("Index", func() { var ( i *index.Index bleveIndex bleve.Index - ref *sprovider.Reference - ri *sprovider.ResourceInfo + ctx context.Context - ctx context.Context + rootId = &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "rootopaqueid", + } + ref = &sprovider.Reference{ + ResourceId: rootId, + Path: "./foo.pdf", + } + ri = &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "opaqueid", + }, + ParentId: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "someopaqueid", + }, + Path: "foo.pdf", + Size: 12345, + Type: sprovider.ResourceType_RESOURCE_TYPE_FILE, + MimeType: "application/pdf", + Mtime: &typesv1beta1.Timestamp{Seconds: 4000}, + } + parentRef = &sprovider.Reference{ + ResourceId: rootId, + Path: "./sudbir", + } + parentRi = &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "parentopaqueid", + }, + Path: "subdir", + Size: 12345, + Type: sprovider.ResourceType_RESOURCE_TYPE_CONTAINER, + Mtime: &typesv1beta1.Timestamp{Seconds: 4000}, + } + childRef = &sprovider.Reference{ + ResourceId: rootId, + Path: "./sudbir/child.pdf", + } + childRi = &sprovider.ResourceInfo{ + Id: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "childopaqueid", + }, + ParentId: &sprovider.ResourceId{ + StorageId: "storageid", + OpaqueId: "parentopaqueid", + }, + Path: "subdir", + Size: 12345, + Type: sprovider.ResourceType_RESOURCE_TYPE_FILE, + Mtime: &typesv1beta1.Timestamp{Seconds: 4000}, + } ) BeforeEach(func() { @@ -31,28 +84,6 @@ var _ = Describe("Index", func() { i, err = index.New(bleveIndex) Expect(err).ToNot(HaveOccurred()) - - ref = &sprovider.Reference{ - ResourceId: &sprovider.ResourceId{ - StorageId: "storageid", - OpaqueId: "rootopaqueid", - }, - Path: "./foo.pdf", - } - ri = &sprovider.ResourceInfo{ - Id: &sprovider.ResourceId{ - StorageId: "storageid", - OpaqueId: "opaqueid", - }, - ParentId: &sprovider.ResourceId{ - StorageId: "storageid", - OpaqueId: "parentopaqueid", - }, - Path: "foo.pdf", - Size: 12345, - MimeType: "application/pdf", - Mtime: &typesv1beta1.Timestamp{Seconds: 4000}, - } }) Describe("New", func() { @@ -127,8 +158,9 @@ var _ = Describe("Index", func() { Expect(match.Entity.Id.OpaqueId).To(Equal(ri.Id.OpaqueId)) Expect(match.Entity.Name).To(Equal(ri.Path)) Expect(match.Entity.Size).To(Equal(ri.Size)) + Expect(match.Entity.Type).To(Equal(uint64(ri.Type))) Expect(match.Entity.MimeType).To(Equal(ri.MimeType)) - Expect(match.Entity.ParentId.OpaqueId).To(Equal(ri.ParentId.OpaqueId)) + Expect(match.Entity.Deleted).To(BeFalse()) Expect(uint64(match.Entity.LastModifiedTime.AsTime().Unix())).To(Equal(ri.Mtime.Seconds)) }) @@ -247,16 +279,68 @@ var _ = Describe("Index", func() { }) Describe("Remove", func() { - It("removes a resource from the index", func() { - err := i.Add(ref, ri) + It("marks a resource as deleted", func() { + err := i.Add(parentRef, parentRi) Expect(err).ToNot(HaveOccurred()) - count, _ := bleveIndex.DocCount() - Expect(count).To(Equal(uint64(1))) + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Query: "subdir", + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rootId.StorageId, + OpaqueId: rootId.OpaqueId, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(len(res.Matches)).To(Equal(1)) - err = i.Remove(ri.Id) + err = i.Delete(parentRi.Id) Expect(err).ToNot(HaveOccurred()) - count, _ = bleveIndex.DocCount() - Expect(count).To(Equal(uint64(0))) + + res, err = i.Search(ctx, &searchsvc.SearchIndexRequest{ + Query: "subdir", + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rootId.StorageId, + OpaqueId: rootId.OpaqueId, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(len(res.Matches)).To(Equal(0)) + }) + + It("also marks child resources as deleted", func() { + err := i.Add(parentRef, parentRi) + Expect(err).ToNot(HaveOccurred()) + err = i.Add(childRef, childRi) + Expect(err).ToNot(HaveOccurred()) + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Query: "subdir", + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rootId.StorageId, + OpaqueId: rootId.OpaqueId, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(len(res.Matches)).To(Equal(2)) + + err = i.Delete(parentRi.Id) + Expect(err).ToNot(HaveOccurred()) + + res, err = i.Search(ctx, &searchsvc.SearchIndexRequest{ + Query: "subdir", + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rootId.StorageId, + OpaqueId: rootId.OpaqueId, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(len(res.Matches)).To(Equal(0)) }) }) }) diff --git a/extensions/search/pkg/search/mocks/IndexClient.go b/extensions/search/pkg/search/mocks/IndexClient.go index dae95ec51..51a07442c 100644 --- a/extensions/search/pkg/search/mocks/IndexClient.go +++ b/extensions/search/pkg/search/mocks/IndexClient.go @@ -30,6 +30,20 @@ func (_m *IndexClient) Add(ref *providerv1beta1.Reference, ri *providerv1beta1.R return r0 } +// Delete provides a mock function with given fields: ri +func (_m *IndexClient) Delete(ri *providerv1beta1.ResourceId) error { + ret := _m.Called(ri) + + var r0 error + if rf, ok := ret.Get(0).(func(*providerv1beta1.ResourceId) error); ok { + r0 = rf(ri) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // DocCount provides a mock function with given fields: func (_m *IndexClient) DocCount() (uint64, error) { ret := _m.Called() @@ -51,8 +65,8 @@ func (_m *IndexClient) DocCount() (uint64, error) { return r0, r1 } -// Remove provides a mock function with given fields: ri -func (_m *IndexClient) Remove(ri *providerv1beta1.ResourceId) error { +// Purge provides a mock function with given fields: ri +func (_m *IndexClient) Purge(ri *providerv1beta1.ResourceId) error { ret := _m.Called(ri) var r0 error diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index 223251c09..5d9ece27f 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -67,7 +67,7 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach Id: e.Executant, } case events.ItemTrashed: - err := p.indexClient.Remove(e.Id) + err := p.indexClient.Delete(e.Id) if err != nil { p.logger.Error().Err(err).Interface("Id", e.Id).Msg("failed to remove item from index") } diff --git a/extensions/search/pkg/search/provider/searchprovider_test.go b/extensions/search/pkg/search/provider/searchprovider_test.go index 0ce1f4ab5..4c878ffc1 100644 --- a/extensions/search/pkg/search/provider/searchprovider_test.go +++ b/extensions/search/pkg/search/provider/searchprovider_test.go @@ -122,7 +122,7 @@ var _ = Describe("Searchprovider", func() { gwClient.On("Stat", mock.Anything, mock.Anything).Return(&sprovider.StatResponse{ Status: status.NewNotFound(context.Background(), ""), }, nil) - indexClient.On("Remove", mock.MatchedBy(func(id *sprovider.ResourceId) bool { + indexClient.On("Delete", mock.MatchedBy(func(id *sprovider.ResourceId) bool { return id.OpaqueId == ri.Id.OpaqueId })).Return(nil).Run(func(args mock.Arguments) { called = true diff --git a/extensions/search/pkg/search/search.go b/extensions/search/pkg/search/search.go index b5e1ed11e..49779bc3e 100644 --- a/extensions/search/pkg/search/search.go +++ b/extensions/search/pkg/search/search.go @@ -38,6 +38,7 @@ type ProviderClient interface { type IndexClient interface { Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) Add(ref *providerv1beta1.Reference, ri *providerv1beta1.ResourceInfo) error - Remove(ri *providerv1beta1.ResourceId) error + Delete(ri *providerv1beta1.ResourceId) error + Purge(ri *providerv1beta1.ResourceId) error DocCount() (uint64, error) } diff --git a/protogen/gen/ocis/messages/search/v0/search.pb.go b/protogen/gen/ocis/messages/search/v0/search.pb.go index 7085608ba..1f3acae8c 100644 --- a/protogen/gen/ocis/messages/search/v0/search.pb.go +++ b/protogen/gen/ocis/messages/search/v0/search.pb.go @@ -144,6 +144,8 @@ type Entity struct { LastModifiedTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=last_modified_time,json=lastModifiedTime,proto3" json:"last_modified_time,omitempty"` MimeType string `protobuf:"bytes,7,opt,name=mime_type,json=mimeType,proto3" json:"mime_type,omitempty"` Permissions string `protobuf:"bytes,8,opt,name=permissions,proto3" json:"permissions,omitempty"` + Type uint64 `protobuf:"varint,9,opt,name=type,proto3" json:"type,omitempty"` + Deleted bool `protobuf:"varint,10,opt,name=deleted,proto3" json:"deleted,omitempty"` } func (x *Entity) Reset() { @@ -234,6 +236,20 @@ func (x *Entity) GetPermissions() string { return "" } +func (x *Entity) GetType() uint64 { + if x != nil { + return x.Type + } + return 0 +} + +func (x *Entity) GetDeleted() bool { + if x != nil { + return x.Deleted + } + return false +} + type Match struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -311,7 +327,7 @@ var file_ocis_messages_search_v0_search_proto_rawDesc = []byte{ 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, - 0x68, 0x22, 0xb8, 0x02, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x03, + 0x68, 0x22, 0xe6, 0x02, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x03, 0x72, @@ -330,17 +346,20 @@ var file_ocis_messages_search_v0_search_proto_rawDesc = []byte{ 0x09, 0x6d, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x56, 0x0a, 0x05, - 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x37, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, - 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, - 0x63, 0x6f, 0x72, 0x65, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63, - 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, - 0x63, 0x68, 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, + 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x56, 0x0a, 0x05, 0x4d, 0x61, + 0x74, 0x63, 0x68, 0x12, 0x37, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x63, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x45, 0x6e, + 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, 0x63, 0x6f, + 0x72, 0x65, 0x42, 0x3f, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6f, 0x77, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x6f, 0x63, 0x69, 0x73, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6f, 0x63, 0x69, 0x73, + 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x2f, 0x76, 0x30, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/protogen/gen/ocis/services/search/v0/search.swagger.json b/protogen/gen/ocis/services/search/v0/search.swagger.json index 26567aab8..55b025f12 100644 --- a/protogen/gen/ocis/services/search/v0/search.swagger.json +++ b/protogen/gen/ocis/services/search/v0/search.swagger.json @@ -185,6 +185,13 @@ }, "permissions": { "type": "string" + }, + "type": { + "type": "string", + "format": "uint64" + }, + "deleted": { + "type": "boolean" } } }, diff --git a/protogen/proto/ocis/messages/search/v0/search.proto b/protogen/proto/ocis/messages/search/v0/search.proto index d80460af9..b3729e47e 100644 --- a/protogen/proto/ocis/messages/search/v0/search.proto +++ b/protogen/proto/ocis/messages/search/v0/search.proto @@ -25,6 +25,8 @@ message Entity { google.protobuf.Timestamp last_modified_time = 6; string mime_type = 7; string permissions = 8; + uint64 type = 9; + bool deleted = 10; } message Match { From 15c60df064618ed3ca56468cbe13f79c0b46d334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 29 Apr 2022 09:16:28 +0200 Subject: [PATCH 38/49] Return up to 200 search results --- extensions/search/pkg/search/index/index.go | 2 +- go.mod | 18 +++--- go.sum | 66 +++++++++++++++++---- 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/extensions/search/pkg/search/index/index.go b/extensions/search/pkg/search/index/index.go index aa5fe0dd5..dc1401ede 100644 --- a/extensions/search/pkg/search/index/index.go +++ b/extensions/search/pkg/search/index/index.go @@ -134,6 +134,7 @@ func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) ( bleve.NewQueryStringQuery("Path:"+req.Ref.Path+"*"), // Limit search to this directory in the space ) bleveReq := bleve.NewSearchRequest(query) + bleveReq.Size = 200 bleveReq.Fields = []string{"*"} res, err := i.bleveIndex.Search(bleveReq) if err != nil { @@ -189,7 +190,6 @@ func fieldsToEntity(fields map[string]interface{}) *indexDocument { Size: uint64(fields["Size"].(float64)), Mtime: fields["Mtime"].(string), MimeType: fields["MimeType"].(string), - Deleted: fields["Deleted"].(bool), Type: uint64(fields["Type"].(float64)), } return doc diff --git a/go.mod b/go.mod index 21a50f1f0..042af7b18 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( ) require ( - contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect + contrib.go.opencensus.io/exporter/prometheus v0.4.1 // indirect github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e // indirect github.com/BurntSushi/toml v1.1.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -100,7 +100,7 @@ require ( github.com/armon/go-metrics v0.3.10 // indirect github.com/armon/go-radix v1.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/aws/aws-sdk-go v1.42.39 // indirect + github.com/aws/aws-sdk-go v1.43.28 // indirect github.com/beevik/etree v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect @@ -122,7 +122,7 @@ require ( github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f // indirect github.com/boombuler/barcode v1.0.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect - github.com/ceph/go-ceph v0.13.0 // indirect + github.com/ceph/go-ceph v0.15.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coreos/go-oidc v2.2.1+incompatible // indirect github.com/coreos/go-semver v0.3.0 // indirect @@ -130,6 +130,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/crewjam/httperr v0.2.0 // indirect github.com/crewjam/saml v0.4.6 // indirect + github.com/cs3org/reva v1.18.0 // indirect github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set v1.8.0 // indirect @@ -144,6 +145,7 @@ require ( github.com/gabriel-vasile/mimetype v1.4.0 // indirect github.com/gdexlab/go-render v1.0.1 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect + github.com/go-chi/chi v4.0.2+incompatible // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/go-git/go-git/v5 v5.4.2 // indirect @@ -166,7 +168,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/hashicorp/consul/api v1.11.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v1.1.0 // indirect + github.com/hashicorp/go-hclog v1.2.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-msgpack v1.1.5 // indirect github.com/hashicorp/go-plugin v1.4.3 // indirect @@ -198,7 +200,7 @@ require ( github.com/mileusna/useragent v1.0.2 // indirect github.com/minio/highwayhash v1.0.2 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.21 // indirect + github.com/minio/minio-go/v7 v7.0.24 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -216,11 +218,11 @@ require ( github.com/orcaman/concurrent-map v1.0.0 // indirect github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - github.com/pkg/xattr v0.4.4 // indirect + github.com/pkg/xattr v0.4.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect github.com/pquerna/otp v1.3.0 // indirect - github.com/prometheus/alertmanager v0.23.0 // indirect + github.com/prometheus/alertmanager v0.24.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect @@ -258,7 +260,7 @@ require ( golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect - golang.org/x/tools v0.1.8 // indirect + golang.org/x/tools v0.1.9 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/ini.v1 v1.66.2 // indirect diff --git a/go.sum b/go.sum index ba8f4693a..b0d518618 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,9 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.18.2/go.mod h1:AiIj7BWXyhO5gGVmYJ+S8tbkCx3yb0IMjua8Aw4naVM= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= -contrib.go.opencensus.io/exporter/prometheus v0.4.0 h1:0QfIkj9z/iVZgK31D9H9ohjjIDApI2GOPScCKwxedbs= contrib.go.opencensus.io/exporter/prometheus v0.4.0/go.mod h1:o7cosnyfuPVK0tB8q0QmaQNhGnptITnPQB+z1+qeFB0= +contrib.go.opencensus.io/exporter/prometheus v0.4.1 h1:oObVeKo2NxpdF/fIfrPsNj6K0Prg0R0mHM+uANlYMiM= +contrib.go.opencensus.io/exporter/prometheus v0.4.1/go.mod h1:t9wvfitlUjGXG2IXAZsuFq26mDGid/JwCEXp+gTG/9U= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v32.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -131,6 +132,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387 h1:loy0fjI90vF44BPW4ZYOkE3tDkGTy7yHURusOJimt+I= github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387/go.mod h1:GuR5j/NW7AU7tDAQUDGCtpiPxWIOy/c3kiRDnlwiCHc= github.com/aliyun/alibaba-cloud-sdk-go v1.61.976/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= @@ -169,8 +171,10 @@ github.com/aws/aws-sdk-go v1.37.27/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.41.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.42.39 h1:6Lso73VoCI8Zmv3zAMv4BNg2gHAKNOlbLv1s/ew90SI= github.com/aws/aws-sdk-go v1.42.39/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= +github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.43.28 h1:HrBUf2pYEMRB3GDkSa/bZ2lkZIe8gSUOz/IEupG1Te0= +github.com/aws/aws-sdk-go v1.43.28/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -236,10 +240,12 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/ceph/go-ceph v0.13.0 h1:69dgIPlNHD2OCz98T0benI4++vcnShGcpQK4RIALjw4= github.com/ceph/go-ceph v0.13.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= +github.com/ceph/go-ceph v0.15.0 h1:ILB3NaLWOtt4u/2d8I8HZTC4Ycm1PsOYVar3IFU1xlo= +github.com/ceph/go-ceph v0.15.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -306,8 +312,11 @@ github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3p github.com/crewjam/saml v0.4.6 h1:XCUFPkQSJLvzyl4cW9OvpWUbRf0gE7VUpU8ZnilbeM4= github.com/crewjam/saml v0.4.6/go.mod h1:ZBOXnNPFzB3CgOkRm7Nd6IVdkG+l/wF+0ZXLqD96t1A= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= +github.com/cs3org/go-cs3apis v0.0.0-20211214102128-4e8745ab1654/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde h1:WrD9O8ZaWvsm0eBzpzVBIuczDhqVq50Nmjc7PGHHA9Y= github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/reva v1.18.0 h1:MbPS5ZAa8RzKcTxAVeSDdISB3XXqLIxqB03BTN5ReBY= +github.com/cs3org/reva v1.18.0/go.mod h1:e5VDUDu4vVWIeVkZcW//n6UZzhGGMa+Tz/whCiX3N6o= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= @@ -362,6 +371,7 @@ github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59/go.mod h1:XYuK1S5 github.com/exoscale/egoscale v0.46.0/go.mod h1:mpEXBpROAa/2i5GC0r33rfxG+TxSEka11g1PIXt9+zc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= @@ -481,6 +491,7 @@ github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2 github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og= +github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= @@ -491,6 +502,7 @@ github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpX github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.20.0/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= @@ -501,6 +513,7 @@ github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3Hfo github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -511,6 +524,7 @@ github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hs github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4= github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o= +github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= @@ -518,6 +532,7 @@ github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2g github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= github.com/go-openapi/runtime v0.19.29/go.mod h1:BvrQtn6iVb2QmiVXRsFAm6ZCAZBpbVKFfN6QWCp582M= +github.com/go-openapi/runtime v0.23.1/go.mod h1:AKurw9fNre+h3ELZfk6ILsfvPN+bvvlaU/M9q/r9hpk= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= @@ -528,6 +543,7 @@ github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFu github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ= github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= @@ -539,6 +555,9 @@ github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLs github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= github.com/go-openapi/strfmt v0.20.1/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= github.com/go-openapi/strfmt v0.20.2/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk= +github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= +github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= +github.com/go-openapi/strfmt v0.21.2/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= @@ -549,6 +568,7 @@ github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5H github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= @@ -557,15 +577,21 @@ github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0 github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0= github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZIVCbJBpTUoY0= +github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -597,6 +623,7 @@ github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY9 github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= @@ -768,8 +795,9 @@ github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/S github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.1.0 h1:QsGcniKx5/LuX2eYoeL+Np3UKYPNaN7YKpTh29h8rbw= github.com/hashicorp/go-hclog v1.1.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= +github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -808,8 +836,9 @@ github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.2.4/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.0 h1:8+567mCcFDnS5ADl7lrpxPMWiFCElyUEeW0gtj34fMA= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM= +github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= @@ -881,6 +910,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4= github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -913,6 +943,7 @@ github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libregraph/idm v0.3.1-0.20220315094434-e9a5cff3dd05 h1:/I4f6c7ZGw16oTBAyhCD9Tf+arBHGvmxL9Drs/KRkRc= github.com/libregraph/idm v0.3.1-0.20220315094434-e9a5cff3dd05/go.mod h1:YQ21AOfZPcCZWX1uJYULZ8hNdrmxStg6egvXaS+ZvOM= @@ -935,6 +966,7 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= @@ -996,8 +1028,9 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.21 h1:xrc4BQr1Fa4s5RwY0xfMjPZFJ1bcYBCCHYlngBdWV+k= github.com/minio/minio-go/v7 v7.0.21/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do= +github.com/minio/minio-go/v7 v7.0.24 h1:HPlHiET6L5gIgrHRaw1xFo1OaN4bEP/082asWh3WJtI= +github.com/minio/minio-go/v7 v7.0.24/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= @@ -1058,6 +1091,7 @@ github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a h1:lem6QCvxR0Y28g github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= github.com/nats-io/nats-server/v2 v2.1.9/go.mod h1:9qVyoewoYXzG1ME9ox0HwkkzyYvnlBDugfR4Gg/8uHU= github.com/nats-io/nats-server/v2 v2.7.4/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= +github.com/nats-io/nats-server/v2 v2.8.0/go.mod h1:5vic7C58BFEVltiZhs7Kq81q2WcEPhJPsmNv1FOrdv0= github.com/nats-io/nats-server/v2 v2.8.1 h1:WZ9m/d8rklkWo6opo3X927vXnuaE00VEEl5zXcpL6qw= github.com/nats-io/nats-server/v2 v2.8.1/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4= github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE= @@ -1105,7 +1139,7 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.18.0/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= @@ -1153,8 +1187,9 @@ github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8 github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/pquerna/otp v1.3.0 h1:oJV/SkzR33anKXwQU3Of42rL4wbrffP4uvUf1SvS5Xs= github.com/pquerna/otp v1.3.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= -github.com/prometheus/alertmanager v0.23.0 h1:KIb9IChC3kg+1CC388qfr7bsT+tARpQqdsCMoatdObA= github.com/prometheus/alertmanager v0.23.0/go.mod h1:0MLTrjQI8EuVmvykEhcfr/7X0xmaDAZrqMgxIq3OXHk= +github.com/prometheus/alertmanager v0.24.0 h1:HBWR3lk4uy3ys+naDZthDdV7yEsxpaNeZuUS+hJgrOw= +github.com/prometheus/alertmanager v0.24.0/go.mod h1:r6fy/D7FRuZh5YbnX6J3MBY0eI4Pb5yPYS7/bPSXXqI= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= @@ -1164,6 +1199,7 @@ github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1189,6 +1225,7 @@ github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuI github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= github.com/prometheus/exporter-toolkit v0.6.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= +github.com/prometheus/exporter-toolkit v0.7.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= github.com/prometheus/procfs v0.0.0-20170703101242-e645f4e5aaa8/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1317,6 +1354,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df/go.mod h1:gCcfDlA1Y7GqOaeEKw5l9dOGx1VLdc/HuQSlQAaZ30s= github.com/studio-b12/gowebdav v0.0.0-20211109083228-3f8721cd4b6f/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -1418,6 +1456,9 @@ go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4S go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.mongodb.org/mongo-driver v1.7.2/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8= +go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -1609,6 +1650,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1747,6 +1789,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1769,12 +1812,14 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210317153231-de623e64d2a6/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210916214954-140adaaadfaf/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1875,8 +1920,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w= -golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2078,6 +2123,7 @@ gopkg.in/square/go-jose.v2 v2.4.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/telebot.v3 v3.0.0/go.mod h1:7rExV8/0mDDNu9epSrDm/8j22KLaActH1Tbee6YjzWg= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= From be15d63747d4758668f97ef2ea8e39a168fcc26b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 29 Apr 2022 10:31:02 +0200 Subject: [PATCH 39/49] Add items back to the index recursively when the parent is restored --- extensions/search/pkg/search/index/index.go | 15 ++-- .../search/pkg/search/index/index_test.go | 40 ++++++++++- .../search/pkg/search/mocks/IndexClient.go | 14 ++++ .../pkg/search/provider/searchprovider.go | 71 ++++++++++++------- .../search/provider/searchprovider_test.go | 6 +- extensions/search/pkg/search/search.go | 1 + 6 files changed, 113 insertions(+), 34 deletions(-) diff --git a/extensions/search/pkg/search/index/index.go b/extensions/search/pkg/search/index/index.go index dc1401ede..5a8984370 100644 --- a/extensions/search/pkg/search/index/index.go +++ b/extensions/search/pkg/search/index/index.go @@ -80,12 +80,17 @@ func (i *Index) Add(ref *sprovider.Reference, ri *sprovider.ResourceInfo) error return i.bleveIndex.Index(idToBleveId(ri.Id), entity) } -// Delete marks an entity from the index as delete (still keeping it around) +// Delete marks an entity from the index as deleten (still keeping it around) func (i *Index) Delete(id *sprovider.ResourceId) error { - return i.markAsDeleted(idToBleveId(id)) + return i.markAsDeleted(idToBleveId(id), true) } -func (i *Index) markAsDeleted(id string) error { +// Restore marks an entity from the index as not being deleted +func (i *Index) Restore(id *sprovider.ResourceId) error { + return i.markAsDeleted(idToBleveId(id), false) +} + +func (i *Index) markAsDeleted(id string, deleted bool) error { req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{id})) req.Fields = []string{"*"} res, err := i.bleveIndex.Search(req) @@ -96,7 +101,7 @@ func (i *Index) markAsDeleted(id string) error { return errors.New("entity not found") } entity := fieldsToEntity(res.Hits[0].Fields) - entity.Deleted = true + entity.Deleted = deleted if entity.Type == uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) { query := bleve.NewConjunctionQuery( @@ -111,7 +116,7 @@ func (i *Index) markAsDeleted(id string) error { } for _, h := range res.Hits { - i.markAsDeleted(h.ID) + i.markAsDeleted(h.ID, deleted) } } diff --git a/extensions/search/pkg/search/index/index_test.go b/extensions/search/pkg/search/index/index_test.go index 7e0591810..503ea0675 100644 --- a/extensions/search/pkg/search/index/index_test.go +++ b/extensions/search/pkg/search/index/index_test.go @@ -278,7 +278,45 @@ var _ = Describe("Index", func() { }) }) - Describe("Remove", func() { + Describe("Restore", func() { + It("also marks child resources as deleted", func() { + err := i.Add(parentRef, parentRi) + Expect(err).ToNot(HaveOccurred()) + err = i.Add(childRef, childRi) + Expect(err).ToNot(HaveOccurred()) + err = i.Delete(parentRi.Id) + Expect(err).ToNot(HaveOccurred()) + + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Query: "subdir", + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rootId.StorageId, + OpaqueId: rootId.OpaqueId, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(len(res.Matches)).To(Equal(0)) + + err = i.Restore(parentRi.Id) + Expect(err).ToNot(HaveOccurred()) + + res, err = i.Search(ctx, &searchsvc.SearchIndexRequest{ + Query: "subdir", + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rootId.StorageId, + OpaqueId: rootId.OpaqueId, + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(len(res.Matches)).To(Equal(2)) + }) + }) + + Describe("Delete", func() { It("marks a resource as deleted", func() { err := i.Add(parentRef, parentRi) Expect(err).ToNot(HaveOccurred()) diff --git a/extensions/search/pkg/search/mocks/IndexClient.go b/extensions/search/pkg/search/mocks/IndexClient.go index 51a07442c..e33db661e 100644 --- a/extensions/search/pkg/search/mocks/IndexClient.go +++ b/extensions/search/pkg/search/mocks/IndexClient.go @@ -79,6 +79,20 @@ func (_m *IndexClient) Purge(ri *providerv1beta1.ResourceId) error { return r0 } +// Restore provides a mock function with given fields: ri +func (_m *IndexClient) Restore(ri *providerv1beta1.ResourceId) error { + ret := _m.Called(ri) + + var r0 error + if rf, ok := ret.Get(0).(func(*providerv1beta1.ResourceId) error); ok { + r0 = rf(ri) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Search provides a mock function with given fields: ctx, req func (_m *IndexClient) Search(ctx context.Context, req *v0.SearchIndexRequest) (*v0.SearchIndexResponse, error) { ret := _m.Called(ctx, req) diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index 5d9ece27f..4bb74342f 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -46,6 +46,34 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach var ref *provider.Reference var owner *user.User switch e := ev.(type) { + case events.ItemTrashed: + err := p.indexClient.Delete(e.ID) + if err != nil { + p.logger.Error().Err(err).Interface("Id", e.ID).Msg("failed to remove item from index") + } + continue + case events.ItemRestored: + ref = e.Ref + owner = &user.User{ + Id: e.Executant, + } + + statRes, err := p.statResource(ref, owner) + if err != nil { + p.logger.Error().Err(err).Msg("failed to stat the changed resource") + } + + switch statRes.Status.Code { + case rpc.Code_CODE_OK: + err = p.indexClient.Restore(statRes.Info.Id) + if err != nil { + p.logger.Error().Err(err).Msg("failed to restore the changed resource in the index") + } + default: + p.logger.Error().Interface("statRes", statRes).Msg("failed to stat the changed resource") + } + + continue case events.FileUploaded: ref = e.Ref owner = &user.User{ @@ -56,41 +84,17 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach owner = &user.User{ Id: e.Executant, } - case events.ItemRestored: - ref = e.Ref - owner = &user.User{ - Id: e.Executant, - } case events.FileVersionRestored: ref = e.Ref owner = &user.User{ Id: e.Executant, } - case events.ItemTrashed: - err := p.indexClient.Delete(e.Id) - if err != nil { - p.logger.Error().Err(err).Interface("Id", e.Id).Msg("failed to remove item from index") - } - continue default: // Not sure what to do here. Skip. continue } - // Get auth - ownerCtx := ctxpkg.ContextSetUser(context.Background(), owner) - authRes, err := p.gwClient.Authenticate(ownerCtx, &gateway.AuthenticateRequest{ - Type: "machine", - ClientId: "userid:" + owner.Id.OpaqueId, - ClientSecret: machineAuthAPIKey, - }) - if err != nil || authRes.GetStatus().GetCode() != rpc.Code_CODE_OK { - p.logger.Error().Err(err).Interface("authRes", authRes).Msg("error using machine auth") - } - ownerCtx = metadata.AppendToOutgoingContext(ownerCtx, ctxpkg.TokenHeader, authRes.Token) - - // Stat changed resource resource - statRes, err := gwClient.Stat(ownerCtx, &provider.StatRequest{Ref: ref}) + statRes, err := p.statResource(ref, owner) if err != nil { p.logger.Error().Err(err).Msg("failed to stat the changed resource") } @@ -114,6 +118,23 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach return p } +func (p *Provider) statResource(ref *provider.Reference, owner *user.User) (*provider.StatResponse, error) { + // Get auth + ownerCtx := ctxpkg.ContextSetUser(context.Background(), owner) + authRes, err := p.gwClient.Authenticate(ownerCtx, &gateway.AuthenticateRequest{ + Type: "machine", + ClientId: "userid:" + owner.Id.OpaqueId, + ClientSecret: p.machineAuthAPIKey, + }) + if err != nil || authRes.GetStatus().GetCode() != rpc.Code_CODE_OK { + p.logger.Error().Err(err).Interface("authRes", authRes).Msg("error using machine auth") + } + ownerCtx = metadata.AppendToOutgoingContext(ownerCtx, ctxpkg.TokenHeader, authRes.Token) + + // Stat changed resource resource + return p.gwClient.Stat(ownerCtx, &provider.StatRequest{Ref: ref}) +} + func (p *Provider) logDocCount() { c, err := p.indexClient.DocCount() if err != nil { diff --git a/extensions/search/pkg/search/provider/searchprovider_test.go b/extensions/search/pkg/search/provider/searchprovider_test.go index 4c878ffc1..b989f1ce0 100644 --- a/extensions/search/pkg/search/provider/searchprovider_test.go +++ b/extensions/search/pkg/search/provider/searchprovider_test.go @@ -129,7 +129,7 @@ var _ = Describe("Searchprovider", func() { }) eventsChan <- events.ItemTrashed{ Ref: ref, - Id: ri.Id, + ID: ri.Id, Executant: user.Id, } @@ -140,8 +140,8 @@ var _ = Describe("Searchprovider", func() { It("indexes items when they are being restored", func() { called := false - indexClient.On("Add", mock.Anything, mock.MatchedBy(func(riToIndex *sprovider.ResourceInfo) bool { - return riToIndex.Id.OpaqueId == ri.Id.OpaqueId + indexClient.On("Restore", mock.MatchedBy(func(id *sprovider.ResourceId) bool { + return id.OpaqueId == ri.Id.OpaqueId })).Return(nil).Run(func(args mock.Arguments) { called = true }) diff --git a/extensions/search/pkg/search/search.go b/extensions/search/pkg/search/search.go index 49779bc3e..cb9bcbbb8 100644 --- a/extensions/search/pkg/search/search.go +++ b/extensions/search/pkg/search/search.go @@ -39,6 +39,7 @@ type IndexClient interface { Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) Add(ref *providerv1beta1.Reference, ri *providerv1beta1.ResourceInfo) error Delete(ri *providerv1beta1.ResourceId) error + Restore(ri *providerv1beta1.ResourceId) error Purge(ri *providerv1beta1.ResourceId) error DocCount() (uint64, error) } From 18d0fa24162aaa67e1663e3bc1ad0b771fe4e488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 29 Apr 2022 12:47:34 +0200 Subject: [PATCH 40/49] Move the children as well when a directory is moved --- extensions/search/pkg/search/index/index.go | 97 +++++++++++-- .../search/pkg/search/index/index_test.go | 137 ++++++++---------- .../search/pkg/search/mocks/IndexClient.go | 14 ++ .../pkg/search/provider/searchprovider.go | 21 ++- .../search/provider/searchprovider_test.go | 2 +- extensions/search/pkg/search/search.go | 1 + 6 files changed, 181 insertions(+), 91 deletions(-) diff --git a/extensions/search/pkg/search/index/index.go b/extensions/search/pkg/search/index/index.go index 5a8984370..8cba7f7a6 100644 --- a/extensions/search/pkg/search/index/index.go +++ b/extensions/search/pkg/search/index/index.go @@ -21,6 +21,7 @@ package index import ( "context" "errors" + "path" "strings" "time" @@ -30,6 +31,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/utils" searchmsg "github.com/owncloud/ocis/protogen/gen/ocis/messages/search/v0" searchsvc "github.com/owncloud/ocis/protogen/gen/ocis/services/search/v0" ) @@ -91,22 +93,17 @@ func (i *Index) Restore(id *sprovider.ResourceId) error { } func (i *Index) markAsDeleted(id string, deleted bool) error { - req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{id})) - req.Fields = []string{"*"} - res, err := i.bleveIndex.Search(req) + doc, err := i.updateEntity(id, func(doc *indexDocument) { + doc.Deleted = deleted + }) if err != nil { return err } - if res.Hits.Len() == 0 { - return errors.New("entity not found") - } - entity := fieldsToEntity(res.Hits[0].Fields) - entity.Deleted = deleted - if entity.Type == uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) { + if doc.Type == uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) { query := bleve.NewConjunctionQuery( - bleve.NewQueryStringQuery("RootID:"+entity.RootID), - bleve.NewQueryStringQuery("Path:"+entity.Path+"/*"), + bleve.NewQueryStringQuery("RootID:"+doc.RootID), + bleve.NewQueryStringQuery("Path:"+doc.Path+"/*"), ) bleveReq := bleve.NewSearchRequest(query) bleveReq.Fields = []string{"*"} @@ -116,11 +113,43 @@ func (i *Index) markAsDeleted(id string, deleted bool) error { } for _, h := range res.Hits { - i.markAsDeleted(h.ID, deleted) + _, err := i.updateEntity(h.ID, func(doc *indexDocument) { + doc.Deleted = deleted + }) + if err != nil { + return err + } } } - return i.bleveIndex.Index(entity.ID, entity) + return nil +} + +func (i *Index) updateEntity(id string, mutateFunc func(doc *indexDocument)) (*indexDocument, error) { + doc, err := i.getEntity(id) + if err != nil { + return nil, err + } + mutateFunc(doc) + err = i.bleveIndex.Index(doc.ID, doc) + if err != nil { + return nil, err + } + + return doc, nil +} + +func (i *Index) getEntity(id string) (*indexDocument, error) { + req := bleve.NewSearchRequest(bleve.NewDocIDQuery([]string{id})) + req.Fields = []string{"*"} + res, err := i.bleveIndex.Search(req) + if err != nil { + return nil, err + } + if res.Hits.Len() == 0 { + return nil, errors.New("entity not found") + } + return fieldsToEntity(res.Hits[0].Fields), nil } // Purge removes an entity from the index @@ -128,6 +157,48 @@ func (i *Index) Purge(id *sprovider.ResourceId) error { return i.bleveIndex.Delete(idToBleveId(id)) } +// Purge removes an entity from the index +func (i *Index) Move(ri *sprovider.ResourceInfo) error { + doc, err := i.getEntity(idToBleveId(ri.Id)) + if err != nil { + return err + } + oldName := doc.Path + newName := utils.MakeRelativePath(ri.Path) + + doc, err = i.updateEntity(idToBleveId(ri.Id), func(doc *indexDocument) { + doc.Path = newName + doc.Name = path.Base(newName) + }) + if err != nil { + return err + } + + if doc.Type == uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) { + query := bleve.NewConjunctionQuery( + bleve.NewQueryStringQuery("RootID:"+doc.RootID), + bleve.NewQueryStringQuery("Path:"+oldName+"/*"), + ) + bleveReq := bleve.NewSearchRequest(query) + bleveReq.Fields = []string{"*"} + res, err := i.bleveIndex.Search(bleveReq) + if err != nil { + return err + } + + for _, h := range res.Hits { + _, err := i.updateEntity(h.ID, func(doc *indexDocument) { + doc.Path = strings.Replace(doc.Path, oldName, newName, 1) + }) + if err != nil { + return err + } + } + } + + return nil +} + // Search searches the index according to the criteria specified in the given SearchIndexRequest func (i *Index) Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) { deletedQuery := bleve.NewBoolFieldQuery(false) diff --git a/extensions/search/pkg/search/index/index_test.go b/extensions/search/pkg/search/index/index_test.go index 503ea0675..8624e55a8 100644 --- a/extensions/search/pkg/search/index/index_test.go +++ b/extensions/search/pkg/search/index/index_test.go @@ -70,11 +70,25 @@ var _ = Describe("Index", func() { StorageId: "storageid", OpaqueId: "parentopaqueid", }, - Path: "subdir", + Path: "child.pdf", Size: 12345, Type: sprovider.ResourceType_RESOURCE_TYPE_FILE, Mtime: &typesv1beta1.Timestamp{Seconds: 4000}, } + + assertDocCount = func(rootId *sprovider.ResourceId, query string, expectedCount int) { + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ + Query: query, + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rootId.StorageId, + OpaqueId: rootId.OpaqueId, + }, + }, + }) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount)) + } ) BeforeEach(func() { @@ -278,50 +292,70 @@ var _ = Describe("Index", func() { }) }) - Describe("Restore", func() { + Describe("Delete", func() { + It("marks a resource as deleted", func() { + err := i.Add(parentRef, parentRi) + Expect(err).ToNot(HaveOccurred()) + assertDocCount(rootId, "subdir", 1) + + err = i.Delete(parentRi.Id) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootId, "subdir", 0) + }) + It("also marks child resources as deleted", func() { err := i.Add(parentRef, parentRi) Expect(err).ToNot(HaveOccurred()) err = i.Add(childRef, childRi) Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootId, "subdir", 1) + assertDocCount(rootId, "child.pdf", 1) + + err = i.Delete(parentRi.Id) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootId, "subdir", 0) + assertDocCount(rootId, "child.pdf", 0) + }) + }) + + Describe("Restore", func() { + It("also marks child resources as restored", func() { + err := i.Add(parentRef, parentRi) + Expect(err).ToNot(HaveOccurred()) + err = i.Add(childRef, childRi) + Expect(err).ToNot(HaveOccurred()) err = i.Delete(parentRi.Id) Expect(err).ToNot(HaveOccurred()) - res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ - Query: "subdir", - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: rootId.StorageId, - OpaqueId: rootId.OpaqueId, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(len(res.Matches)).To(Equal(0)) + assertDocCount(rootId, "subdir", 0) + assertDocCount(rootId, "child.pdf", 0) err = i.Restore(parentRi.Id) Expect(err).ToNot(HaveOccurred()) - res, err = i.Search(ctx, &searchsvc.SearchIndexRequest{ - Query: "subdir", - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: rootId.StorageId, - OpaqueId: rootId.OpaqueId, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(len(res.Matches)).To(Equal(2)) + assertDocCount(rootId, "subdir", 1) + assertDocCount(rootId, "child.pdf", 1) }) }) - Describe("Delete", func() { - It("marks a resource as deleted", func() { + Describe("Move", func() { + It("moves the parent and its child resources", func() { err := i.Add(parentRef, parentRi) Expect(err).ToNot(HaveOccurred()) + err = i.Add(childRef, childRi) + Expect(err).ToNot(HaveOccurred()) + + parentRi.Path = "newname" + err = i.Move(parentRi) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootId, "subdir", 0) + res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ - Query: "subdir", + Query: "child.pdf", Ref: &searchmsg.Reference{ ResourceId: &searchmsg.ResourceID{ StorageId: rootId.StorageId, @@ -331,54 +365,7 @@ var _ = Describe("Index", func() { }) Expect(err).ToNot(HaveOccurred()) Expect(len(res.Matches)).To(Equal(1)) - - err = i.Delete(parentRi.Id) - Expect(err).ToNot(HaveOccurred()) - - res, err = i.Search(ctx, &searchsvc.SearchIndexRequest{ - Query: "subdir", - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: rootId.StorageId, - OpaqueId: rootId.OpaqueId, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(len(res.Matches)).To(Equal(0)) - }) - - It("also marks child resources as deleted", func() { - err := i.Add(parentRef, parentRi) - Expect(err).ToNot(HaveOccurred()) - err = i.Add(childRef, childRi) - Expect(err).ToNot(HaveOccurred()) - res, err := i.Search(ctx, &searchsvc.SearchIndexRequest{ - Query: "subdir", - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: rootId.StorageId, - OpaqueId: rootId.OpaqueId, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(len(res.Matches)).To(Equal(2)) - - err = i.Delete(parentRi.Id) - Expect(err).ToNot(HaveOccurred()) - - res, err = i.Search(ctx, &searchsvc.SearchIndexRequest{ - Query: "subdir", - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: rootId.StorageId, - OpaqueId: rootId.OpaqueId, - }, - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(len(res.Matches)).To(Equal(0)) + Expect(res.Matches[0].Entity.Ref.Path).To(Equal("./newname/child.pdf")) }) }) }) diff --git a/extensions/search/pkg/search/mocks/IndexClient.go b/extensions/search/pkg/search/mocks/IndexClient.go index e33db661e..f87fd7264 100644 --- a/extensions/search/pkg/search/mocks/IndexClient.go +++ b/extensions/search/pkg/search/mocks/IndexClient.go @@ -65,6 +65,20 @@ func (_m *IndexClient) DocCount() (uint64, error) { return r0, r1 } +// Move provides a mock function with given fields: ri +func (_m *IndexClient) Move(ri *providerv1beta1.ResourceInfo) error { + ret := _m.Called(ri) + + var r0 error + if rf, ok := ret.Get(0).(func(*providerv1beta1.ResourceInfo) error); ok { + r0 = rf(ri) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Purge provides a mock function with given fields: ri func (_m *IndexClient) Purge(ri *providerv1beta1.ResourceId) error { ret := _m.Called(ri) diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index 4bb74342f..c7d05ef20 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -74,12 +74,29 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach } continue - case events.FileUploaded: + case events.ItemMoved: ref = e.Ref owner = &user.User{ Id: e.Executant, } - case events.ItemMoved: + + statRes, err := p.statResource(ref, owner) + if err != nil { + p.logger.Error().Err(err).Msg("failed to stat the changed resource") + } + + switch statRes.Status.Code { + case rpc.Code_CODE_OK: + err = p.indexClient.Move(statRes.Info) + if err != nil { + p.logger.Error().Err(err).Msg("failed to restore the changed resource in the index") + } + default: + p.logger.Error().Interface("statRes", statRes).Msg("failed to stat the changed resource") + } + + continue + case events.FileUploaded: ref = e.Ref owner = &user.User{ Id: e.Executant, diff --git a/extensions/search/pkg/search/provider/searchprovider_test.go b/extensions/search/pkg/search/provider/searchprovider_test.go index b989f1ce0..f6ea0de39 100644 --- a/extensions/search/pkg/search/provider/searchprovider_test.go +++ b/extensions/search/pkg/search/provider/searchprovider_test.go @@ -174,7 +174,7 @@ var _ = Describe("Searchprovider", func() { It("indexes items when they are being moved", func() { called := false - indexClient.On("Add", mock.Anything, mock.MatchedBy(func(riToIndex *sprovider.ResourceInfo) bool { + indexClient.On("Move", mock.Anything, mock.MatchedBy(func(riToIndex *sprovider.ResourceInfo) bool { return riToIndex.Id.OpaqueId == ri.Id.OpaqueId })).Return(nil).Run(func(args mock.Arguments) { called = true diff --git a/extensions/search/pkg/search/search.go b/extensions/search/pkg/search/search.go index cb9bcbbb8..14f0baf8c 100644 --- a/extensions/search/pkg/search/search.go +++ b/extensions/search/pkg/search/search.go @@ -38,6 +38,7 @@ type ProviderClient interface { type IndexClient interface { Search(ctx context.Context, req *searchsvc.SearchIndexRequest) (*searchsvc.SearchIndexResponse, error) Add(ref *providerv1beta1.Reference, ri *providerv1beta1.ResourceInfo) error + Move(ri *providerv1beta1.ResourceInfo) error Delete(ri *providerv1beta1.ResourceId) error Restore(ri *providerv1beta1.ResourceId) error Purge(ri *providerv1beta1.ResourceId) error From 0fdd23c2c61d04647e180fb00c9e7c3042adbd36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 29 Apr 2022 13:05:49 +0200 Subject: [PATCH 41/49] Fix search config after master merge --- extensions/search/pkg/config/defaults/defaultconfig.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/extensions/search/pkg/config/defaults/defaultconfig.go b/extensions/search/pkg/config/defaults/defaultconfig.go index 8d18bf9e2..abd40aa92 100644 --- a/extensions/search/pkg/config/defaults/defaultconfig.go +++ b/extensions/search/pkg/config/defaults/defaultconfig.go @@ -7,6 +7,14 @@ import ( "github.com/owncloud/ocis/ocis-pkg/config/defaults" ) +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + + EnsureDefaults(cfg) + + return cfg +} + func DefaultConfig() *config.Config { return &config.Config{ Debug: config.Debug{ From 851c64945f1f3b690a6fa1408a745f8bd4b6c5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 29 Apr 2022 13:59:09 +0200 Subject: [PATCH 42/49] Remove reva go mod replace --- go.mod | 2 -- go.sum | 2 -- 2 files changed, 4 deletions(-) diff --git a/go.mod b/go.mod index 7d806e3a7..291ea47d4 100644 --- a/go.mod +++ b/go.mod @@ -274,5 +274,3 @@ require ( // we need to use a fork to make the windows build pass replace github.com/pkg/xattr => github.com/micbar/xattr v0.4.6-0.20220215112335-88e74d648fb7 - -replace github.com/cs3org/reva/v2 => ../reva diff --git a/go.sum b/go.sum index 97e90625a..1843c2b33 100644 --- a/go.sum +++ b/go.sum @@ -318,8 +318,6 @@ github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde h1:WrD9O8ZaWvsm0 github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/reva v1.18.0 h1:MbPS5ZAa8RzKcTxAVeSDdISB3XXqLIxqB03BTN5ReBY= github.com/cs3org/reva v1.18.0/go.mod h1:e5VDUDu4vVWIeVkZcW//n6UZzhGGMa+Tz/whCiX3N6o= -github.com/cs3org/reva/v2 v2.0.0-20220427133111-618964eed515 h1:8pPCLxNXVz/q7PMM6Zq1lff3P8SFAu8/CXwB2eA21xc= -github.com/cs3org/reva/v2 v2.0.0-20220427133111-618964eed515/go.mod h1:2e/4HcIy54Mic3V7Ow0bz4n5dkZU0dHIZSWomFe5vng= github.com/cs3org/reva/v2 v2.0.0-20220427203355-0164880ac7d3 h1:6sKjGI0AUW5tBXWBduaBoc+9sNYZWQR894G0oFCbus0= github.com/cs3org/reva/v2 v2.0.0-20220427203355-0164880ac7d3/go.mod h1:2e/4HcIy54Mic3V7Ow0bz4n5dkZU0dHIZSWomFe5vng= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= From 498d078a06f4fad84df67a251a2b75b0853893b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 29 Apr 2022 15:23:27 +0200 Subject: [PATCH 43/49] Add changelog --- changelog/unreleased/search-extension.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/search-extension.md diff --git a/changelog/unreleased/search-extension.md b/changelog/unreleased/search-extension.md new file mode 100644 index 000000000..c4e7bffa4 --- /dev/null +++ b/changelog/unreleased/search-extension.md @@ -0,0 +1,5 @@ +Enhancement: Add initial version of the search extensions + +It is now possible to search for files and directories by their name using the web UI. Therefor new search extension indexes files in a persistent local index. + +https://github.com/owncloud/ocis/pull/3635 From cd285c9ad911d26d14f2c380e5413ca4e6d1fb2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 29 Apr 2022 18:42:33 +0200 Subject: [PATCH 44/49] Fix merge error --- extensions/web/pkg/config/defaults/defaultconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/web/pkg/config/defaults/defaultconfig.go b/extensions/web/pkg/config/defaults/defaultconfig.go index cb5488b08..084bdcfe8 100644 --- a/extensions/web/pkg/config/defaults/defaultconfig.go +++ b/extensions/web/pkg/config/defaults/defaultconfig.go @@ -50,7 +50,7 @@ func DefaultConfig() *config.Config { ResponseType: "code", Scope: "openid profile email", }, - Apps: []string{"files", "search", "preview", "text-editor", "pdf-viewer", "external"}, + Apps: []string{"files", "search", "preview", "text-editor", "pdf-viewer", "external", "user-management"}, Options: map[string]interface{}{ "hideSearchBar": false, }, From c720d18de8403a752ea6b87e1dd6c5a3aad2d5bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 2 May 2022 14:19:08 +0000 Subject: [PATCH 45/49] update config to init changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- extensions/search/pkg/command/server.go | 6 +++++- extensions/search/pkg/config/config.go | 7 +++---- extensions/search/pkg/config/defaults/defaultconfig.go | 7 ++++--- extensions/search/pkg/config/parser/parse.go | 10 +++++++++- extensions/search/pkg/config/reva.go | 5 ----- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/extensions/search/pkg/command/server.go b/extensions/search/pkg/command/server.go index 2c8ae18d5..2582213c4 100644 --- a/extensions/search/pkg/command/server.go +++ b/extensions/search/pkg/command/server.go @@ -23,7 +23,11 @@ func Server(cfg *config.Config) *cli.Command { Usage: fmt.Sprintf("start %s extension without runtime (unsupervised mode)", cfg.Service.Name), Category: "server", Before: func(c *cli.Context) error { - return parser.ParseConfig(cfg) + err := parser.ParseConfig(cfg) + if err != nil { + fmt.Printf("%v", err) + } + return err }, Action: func(c *cli.Context) error { logger := logging.Configure(cfg.Service.Name, cfg.Log) diff --git a/extensions/search/pkg/config/config.go b/extensions/search/pkg/config/config.go index ae90144ec..7c7ac33ff 100644 --- a/extensions/search/pkg/config/config.go +++ b/extensions/search/pkg/config/config.go @@ -18,10 +18,9 @@ type Config struct { GRPC GRPC `ocisConfig:"grpc"` - Datapath string `yaml:"data_path" env:"SEARCH_DATA_PATH"` - Reva Reva `ocisConfig:"reva"` - TokenManager TokenManager `ocisConfig:"token_manager"` - Events Events `yaml:"events"` + Datapath string `yaml:"data_path" env:"SEARCH_DATA_PATH"` + Reva Reva `ocisConfig:"reva"` + Events Events `yaml:"events"` MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;SEARCH_MACHINE_AUTH_API_KEY"` diff --git a/extensions/search/pkg/config/defaults/defaultconfig.go b/extensions/search/pkg/config/defaults/defaultconfig.go index abd40aa92..c3c40a7fe 100644 --- a/extensions/search/pkg/config/defaults/defaultconfig.go +++ b/extensions/search/pkg/config/defaults/defaultconfig.go @@ -32,9 +32,6 @@ func DefaultConfig() *config.Config { Reva: config.Reva{ Address: "127.0.0.1:9142", }, - TokenManager: config.TokenManager{ - JWTSecret: "Pive-Fumkiu4", - }, Events: config.Events{ Endpoint: "127.0.0.1:9233", Cluster: "ocis-cluster", @@ -67,6 +64,10 @@ func EnsureDefaults(cfg *config.Config) { } else if cfg.Tracing == nil { cfg.Tracing = &config.Tracing{} } + + if cfg.MachineAuthAPIKey == "" && cfg.Commons != nil && cfg.Commons.MachineAuthAPIKey != "" { + cfg.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey + } } func Sanitize(cfg *config.Config) { diff --git a/extensions/search/pkg/config/parser/parse.go b/extensions/search/pkg/config/parser/parse.go index b764382e9..9183f5be8 100644 --- a/extensions/search/pkg/config/parser/parse.go +++ b/extensions/search/pkg/config/parser/parse.go @@ -6,11 +6,12 @@ import ( "github.com/owncloud/ocis/extensions/search/pkg/config" "github.com/owncloud/ocis/extensions/search/pkg/config/defaults" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" + "github.com/owncloud/ocis/ocis-pkg/shared" "github.com/owncloud/ocis/ocis-pkg/config/envdecode" ) -// ParseConfig loads accounts configuration from known paths. +// ParseConfig loads configuration from known paths. func ParseConfig(cfg *config.Config) error { _, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg) if err != nil { @@ -29,5 +30,12 @@ func ParseConfig(cfg *config.Config) error { defaults.Sanitize(cfg) + return Validate(cfg) +} + +func Validate(cfg *config.Config) error { + if cfg.MachineAuthAPIKey == "" { + return shared.MissingMachineAuthApiKeyError(cfg.Service.Name) + } return nil } diff --git a/extensions/search/pkg/config/reva.go b/extensions/search/pkg/config/reva.go index 68bfd2f62..2b299a0f6 100644 --- a/extensions/search/pkg/config/reva.go +++ b/extensions/search/pkg/config/reva.go @@ -4,8 +4,3 @@ package config type Reva struct { Address string `ocisConfig:"address" env:"REVA_GATEWAY"` } - -// TokenManager is the config for using the reva token manager -type TokenManager struct { - JWTSecret string `ocisConfig:"jwt_secret" env:"OCIS_JWT_SECRET;SEARCH_JWT_SECRET"` -} From 4d888ccb7a1749eeb04a03710648f26240cecda9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 2 May 2022 14:49:30 +0000 Subject: [PATCH 46/49] fix proxy directorSelectionDirector test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- extensions/proxy/pkg/proxy/proxy_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/proxy/pkg/proxy/proxy_test.go b/extensions/proxy/pkg/proxy/proxy_test.go index 6beb637d9..2805ada18 100644 --- a/extensions/proxy/pkg/proxy/proxy_test.go +++ b/extensions/proxy/pkg/proxy/proxy_test.go @@ -127,9 +127,9 @@ func TestDirectorSelectionDirector(t *testing.T) { } for _, test := range table { - r := httptest.NewRequest(http.MethodGet, "/dav/files/demo/", nil) + r := httptest.NewRequest(test.method, "/dav/files/demo/", nil) p.directorSelectionDirector(r) - if r.Host != test.target { + if r.URL.Host != test.target { t.Errorf("TestDirectorSelectionDirector got host %s expected %s", r.Host, test.target) } From acd6b33eda8ad27f61accc08dd0a22d92dc7e847 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 2 May 2022 16:00:07 +0000 Subject: [PATCH 47/49] expect favorites to fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../expected-failures-graphAPI-on-OCIS-storage.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/acceptance/expected-failures-graphAPI-on-OCIS-storage.md b/tests/acceptance/expected-failures-graphAPI-on-OCIS-storage.md index f513a93f0..24c996481 100644 --- a/tests/acceptance/expected-failures-graphAPI-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-graphAPI-on-OCIS-storage.md @@ -1016,6 +1016,15 @@ _ocdav: api compatibility, return correct status code_ - [apiWebdavOperations/search.feature:265](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/search.feature#L265) - [apiWebdavOperations/search.feature:270](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/search.feature#L270) +#### [Support for favorites](https://github.com/owncloud/ocis/issues/1228) + +- [apiFavorites/favorites.feature:115](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L115) +- [apiFavorites/favorites.feature:116](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L116) +- [apiFavorites/favorites.feature:141](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L141) +- [apiFavorites/favorites.feature:142](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L142) +- [apiFavorites/favorites.feature:267](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L267) +- [apiFavorites/favorites.feature:268](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L268) + And other missing implementation of favorites - [apiFavorites/favorites.feature:162](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L162) From a1b34a62a725d140ab8a321f27a6a6a22cfda042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 2 May 2022 18:42:47 +0200 Subject: [PATCH 48/49] Also index new directories --- extensions/audit/pkg/types/events.go | 1 + extensions/search/pkg/search/provider/searchprovider.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/extensions/audit/pkg/types/events.go b/extensions/audit/pkg/types/events.go index 1862b1f80..2fe9d9aef 100644 --- a/extensions/audit/pkg/types/events.go +++ b/extensions/audit/pkg/types/events.go @@ -16,6 +16,7 @@ func RegisteredEvents() []events.Unmarshaller { events.ReceivedShareUpdated{}, events.LinkAccessed{}, events.LinkAccessFailed{}, + events.ContainerCreated{}, events.FileUploaded{}, events.FileDownloaded{}, events.ItemTrashed{}, diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index c7d05ef20..2f3b2993e 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -96,6 +96,11 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach } continue + case events.ContainerCreated: + ref = e.Ref + owner = &user.User{ + Id: e.Executant, + } case events.FileUploaded: ref = e.Ref owner = &user.User{ From fd4ec56b3e64c513137ce8a12869212b8140512d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 2 May 2022 18:43:03 +0200 Subject: [PATCH 49/49] Cleanup --- .../search/pkg/search/provider/searchprovider.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/extensions/search/pkg/search/provider/searchprovider.go b/extensions/search/pkg/search/provider/searchprovider.go index 2f3b2993e..3aa2f6cda 100644 --- a/extensions/search/pkg/search/provider/searchprovider.go +++ b/extensions/search/pkg/search/provider/searchprovider.go @@ -124,16 +124,14 @@ func New(gwClient gateway.GatewayAPIClient, indexClient search.IndexClient, mach switch statRes.Status.Code { case rpc.Code_CODE_OK: err = p.indexClient.Add(ref, statRes.Info) + if err != nil { + p.logger.Error().Err(err).Msg("error adding updating the resource in the index") + } else { + p.logDocCount() + } default: p.logger.Error().Interface("statRes", statRes).Msg("failed to stat the changed resource") } - - if err != nil { - p.logger.Error().Err(err).Msg("error adding updating the resource in the index") - } else { - p.logDocCount() - } - } }()