Compare commits

...
Author SHA1 Message Date
Jörn Friedrich Dreyer 1bafa4bd18 align the handler verifier, update docs
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-08 17:11:55 +02:00
Dominik Schmidt b18618ab4a Merge pull request #3484 from opencloud-eu/fix/tika-embedded-facet-leak
fix(search): extract facets from the main tika document only
2026-09-08 15:52:21 +02:00
Dominik Schmidt 65f19b5d71 fix(thumbnails): close rejected vips ref, map oversized convert error to forbidden 2026-09-08 15:28:37 +02:00
Dominik Schmidt 7f687ea288 fix(thumbnails): bound declared image dimensions before decoding
The imaging build decodes the full pixel buffer from the header-declared
dimensions before the existing MaxInputWidth/MaxInputHeight guard runs, so a
tiny crafted file whose header declares huge dimensions forces a multi-GB
allocation and can OOM the worker. Read the header with DecodeConfig and reject
oversized sources before the decode allocates, in both the imaging and vips
builds, and thread the limit through the audio cover-art and geogebra decoders
that decode a second attacker-controlled image.
2026-09-08 15:28:37 +02:00
Dominik Schmidt be83ed19e3 fix(search): take extraction facets from the main document only
The recursive tika response lists the file first, then its embedded
resources (cover art, thumbnails, the clip appended to a motion photo).
The loop applied getImage/getPhoto/getLocation/getAudio/getLivePhoto to
every part, so an mp3's embedded cover art leaked a 200x200 image facet
onto the track (and an embedded EXIF image would leak photo/location).
Read those facets from metas[0] only, like the video facet already does;
the loop now only concatenates title/content and detects the motion
photo clip.
2026-09-08 14:52:35 +02:00
Dominik Schmidt b5cb1cd83b test(search): re-search until the expected files are in the result (#3488)
Indexing of newly uploaded resources is asynchronous; the search step
slept a fixed 10s and searched once, so under load a just-uploaded file
was missing from the result and the following assertion failed. The
search step now remembers its query, and new eventually-contain steps
re-run the search until the expected entries are present (or the
WaitHelper timeout elapses), then assert as before. contentSearch
scenarios use them for their positive assertions; the exact-count check
runs after the expected entries have appeared.
2026-09-08 14:50:49 +02:00
zerox80 9797ee903f fix(config): correct pending version annotations
Use the supported %%NEXT%% marker for seven graph and policies settings so the environment annotation check passes.
2026-09-08 14:40:40 +02:00
Jörn Friedrich Dreyer 973dbbdf1a Merge pull request #3241 from opencloud-eu/activitylog-event-handler-split
Activitylog event handler split
2026-09-08 14:13:28 +02:00
Dominik Schmidt 1ce7f07523 Merge pull request #3486 from opencloud-eu/test/extractedprops-wait-for-values
test(search): wait for expected properties and documents
2026-09-08 14:05:12 +02:00
Dominik Schmidt 3b83221a87 test(search): retry the driveItem GET until the extracted properties match
Property extraction reaches the driveItem asynchronously, so the GET
step previously waited only until any extraction facet key appeared and
a separate step then validated the full schema, which could run on a
partial payload and fail intermittently. The GET and the status check
stay plain steps; a new explicit Then step re-fetches the item until
the response satisfies the expected schema (a partial never matches) or
the WaitHelper timeout elapses.
2026-09-08 13:39:00 +02:00
Jörn Friedrich Dreyer 391705b017 Merge pull request #3441 from opencloud-eu/dependabot/go_modules/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp-0.71.0
build(deps): bump go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp from 0.70.0 to 0.71.0
2026-09-08 10:47:59 +02:00
Alex Ababii 8875d062b0 upd tests for auth requests middelware 2026-09-08 09:37:51 +02:00
Alex Ababii d1d20edbd2 log jwt expired on debug level instead of error 2026-09-08 09:37:51 +02:00
dependabot[bot] b50c5e54d6 build(deps): bump go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp
Bumps [go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp](https://github.com/open-telemetry/opentelemetry-go-contrib) from 0.70.0 to 0.71.0.
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/zpages/v0.70.0...zpages/v0.71.0)

---
updated-dependencies:
- dependency-name: go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp
  dependency-version: 0.71.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-08 07:26:51 +00:00
opencloudeu c2a34e8c45 [tx] updated from transifex 2026-09-07 23:17:03 +00:00
Dominik Schmidt d6c6b6fd0d Merge pull request #3481 from opencloud-eu/fix/proxy-signed-url-methods
fix(proxy): restrict JWT signed urls to the allowed HTTP methods
2026-09-07 16:13:40 +02:00
Dominik Schmidt 7f7a938c3f fix(proxy): restrict JWT signed urls to the allowed HTTP methods
The legacy OC-Signature path already rejects methods outside
PRE_SIGNED_URL_ALLOWED_HTTP_METHODS, the JWT path did not. A leaked
signed download url could be used for PUT, DELETE, MOVE or PROPFIND
as the signing user for the lifetime of the signature.
2026-09-07 13:00:09 +02:00
Jörn Friedrich Dreyer cfefb83d26 revert unrelated changes 2026-09-07 12:52:33 +02:00
Jörn Friedrich Dreyer 9357124a91 add tests to extracted services
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:33 +02:00
Jörn Friedrich Dreyer c36af05d14 simplify ConnectNatsKV
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:33 +02:00
Jörn Friedrich Dreyer 7947c1c98a reduce packages
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:33 +02:00
Jörn Friedrich Dreyer d58c1e3dab use json encode
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:33 +02:00
Jörn Friedrich Dreyer 5a609c98dc handle error types
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:33 +02:00
Jörn Friedrich Dreyer 29913e1d7f revert to legacy autoack events
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:32 +02:00
Jörn Friedrich Dreyer 5028364e2c add tests to cover more ack scenarios
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:32 +02:00
Jörn Friedrich Dreyer b2b15d44f7 rename test files
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:32 +02:00
Jörn Friedrich Dreyer 6ea363b0b9 add debouncer tests
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:32 +02:00
Jörn Friedrich Dreyer 9594313243 add explicit ack
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:32 +02:00
Jörn Friedrich Dreyer 62d815abe9 split handlers, fix bugs
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:32 +02:00
Jörn Friedrich Dreyer 5923aa5e5e move event + http handler code to the events package
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2026-09-07 12:52:32 +02:00
Dominik Schmidt 81536bbd0e Merge pull request #3471 from opencloud-eu/feat/graph-expand-thumbnails
feat(graph): expand thumbnails on driveItems
2026-09-07 12:50:53 +02:00
Jörn Friedrich Dreyer be226ebb4c Merge pull request #3478 from opencloud-eu/reva-bump-main-20260907
[full-ci] chore: bump reva to latest main
2026-09-07 12:48:12 +02:00
Dominik Schmidt c2b37668e4 chore: bump reva to latest main 2026-09-07 09:33:59 +02:00
opencloudeu 4b7c6908f0 [tx] updated from transifex 2026-09-06 23:16:18 +00:00
Dominik Schmidt 6231a870d5 feat(graph): expand thumbnails on driveItems
$expand=thumbnails was only honored by sharedByMe and sharedWithMe. The
driveItem stat, the children listing and the root children listing now
honor it as well, so a client that lists a folder learns which items have
a preview instead of guessing from the mime type.

The thumbnails are set from the resource info the listing already has, so
a later preview check that needs more than the mime type has a single
place to sit. The two share listings carry driveItems only, they keep
matching on the mime type but share the url building.
2026-09-06 23:51:35 +02:00
opencloudeu 17003fe34b [tx] updated from transifex 2026-09-05 23:17:58 +00:00
Dominik Schmidt 2ce9faec5a Merge pull request #3444 from opencloud-eu/feat/driveitem-lock
graph: expose lockInfo on driveItems
2026-09-04 09:51:13 +02:00
opencloudeu f5696d40f8 [tx] updated from transifex 2026-09-03 23:17:09 +00:00
Dominik Schmidt 9a769b8019 build(deps): bump github.com/opencloud-eu/libre-graph-api-go to 45af3945a067 2026-09-03 16:02:43 +00:00
Dominik Schmidt bafbc88a40 graph: expose lockInfo on driveItems 2026-09-03 17:52:13 +02:00
Dominik Schmidt addb32812a Merge pull request #3438 from opencloud-eu/feat/driveitem-share-types
graph: expose @libre.graph.shareTypes on driveItems
2026-09-03 17:52:10 +02:00
Dominik Schmidt aa968ec898 graph: expose @libre.graph.shareTypes on driveItems 2026-09-03 16:59:29 +02:00
Dominik Schmidt 61037cc22f Merge pull request #3202 from opencloud-eu/feat/search-live-photo-facet
feat(search): live photo facet
2026-09-03 11:42:28 +02:00
Jörn Friedrich Dreyer 993265b64e Merge pull request #3445 from opencloud-eu/feat/graph-expand-children
feat: support $expand=children on the driveItem endpoint
2026-09-03 09:35:05 +02:00
Dominik Schmidt 5995938c34 chore(search): trim the live photo comments 2026-09-03 03:34:33 +02:00
Dominik Schmidt 0a4f3ed786 fix(search): keep livePhoto.contentId out of the search siblings 2026-09-03 03:34:33 +02:00
Dominik Schmidt a7214b3d73 fix(search): drop the live photo facet on an empty pairing id 2026-09-03 03:34:33 +02:00
Dominik Schmidt 81b0df1ad4 feat(search): live photo facet 2026-09-03 03:34:33 +02:00
Dominik Schmidt e6102c0405 Merge pull request #3200 from opencloud-eu/feat/search-motion-photo-facet
feat(search): motion photo facet
2026-09-03 03:34:29 +02:00
opencloudeu ab0b3c58f1 [tx] updated from transifex 2026-09-02 23:18:41 +00:00
Dominik Schmidt 3515594787 fix(search): keep the video facet on the file itself 2026-09-03 01:09:41 +02:00
Dominik Schmidt a8318fe03b refactor(search): take the motion photo video size from the video tika extracted 2026-09-03 00:32:07 +02:00
Dominik Schmidt 706aaf02f2 refactor(search): decide the motion photo from the file xmp and its extracted video 2026-09-03 00:32:06 +02:00
Dominik Schmidt 88f21d804b refactor(search): restore cs3 Retrieve, the download split had only one caller left 2026-09-03 00:31:40 +02:00
Dominik Schmidt 8aebf80124 refactor(search): confirm the motion photo video by type, not by name 2026-09-03 00:31:40 +02:00
Dominik Schmidt ae9fd595d9 chore(search): drop the dead legacy tika key 2026-09-03 00:31:40 +02:00
Dominik Schmidt 709962b616 feat(search): confirm the motion photo video via tika instead of reading bytes 2026-09-03 00:31:40 +02:00
Dominik Schmidt a2050d1fb9 feat(search): index and expose the motion photo facet 2026-09-03 00:31:40 +02:00
Dominik Schmidt 0f4b95b9ae feat(search): add RetrieveRange to the content retriever 2026-09-03 00:27:04 +02:00
Dominik Schmidt 42876ca61c refactor(search): let getFirstValue try multiple metadata keys 2026-09-03 00:27:04 +02:00
Jörn Friedrich Dreyer f5a064395a Merge pull request #3201 from opencloud-eu/feat/search-video-facet
feat(search): video facet
2026-09-03 00:20:02 +02:00
dependabot[bot] 6688a037f7 build(deps): bump go.opentelemetry.io/contrib/zpages
Bumps [go.opentelemetry.io/contrib/zpages](https://github.com/open-telemetry/opentelemetry-go-contrib) from 0.70.0 to 0.71.0.
- [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/zpages/v0.70.0...zpages/v0.71.0)

---
updated-dependencies:
- dependency-name: go.opentelemetry.io/contrib/zpages
  dependency-version: 0.71.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-02 16:48:58 +02:00
Jörn Friedrich Dreyer 20a579e6ea Merge pull request #3257 from opencloud-eu/fix/userlog_notifications_handling
fix: notification handling for share removal and space membership expiry
2026-09-02 10:25:07 +02:00
Florian Schade a52b55461e Merge pull request #3432 from pbleser-oc/chore/muse-dynamic-go-version
chore(mise): determine go version in mise.toml by parsing go.mod
2026-09-01 22:16:50 +02:00
Dominik Schmidt cc73c95a14 Merge pull request #3420 from fredrikblau/fix/3410-upload-file-date
fix(search): ignore invalid photo exposure times
2026-09-01 21:59:31 +02:00
Pascal Bleser 1ab6710e4c chore(mise): determine go version by parsing go.mod
Instead of statically defining the version of Go in mise.toml as well as
in its native files (go.mod), use a small shell script to parse go.mod
and determine the version dynamically from there.

That version is then available as an env variable in the mise.toml file
and can be referenced for go, in order to avoid having to keep those
versions in sync, especially since dependabot will update go in the
go.mod file, but not in mise.toml.
2026-09-01 21:21:59 +02:00
Dominik Schmidt 0750884597 perf: check the resource type before parsing the query 2026-09-01 20:04:22 +02:00
Dominik Schmidt aff81159a5 chore: drop the listDriveItemChildren comment 2026-09-01 20:01:54 +02:00
Dominik Schmidt 701d6fea17 chore: trim comments 2026-09-01 20:01:06 +02:00
Dominik Schmidt 74dd57d500 refactor: let formatDriveItems return values so children needs no copy 2026-09-01 19:59:51 +02:00
Dominik Schmidt b95e341f9c feat: support $expand=children on the driveItem endpoint 2026-09-01 19:55:14 +02:00
amir 67124a6c37 fix(search): ignore invalid exposure denominators 2026-09-01 20:24:44 +03:30
amir db4cee156b fix(search): ignore invalid photo exposure times 2026-09-01 20:24:19 +03:30
Dominik Schmidt 727c986de8 feat(search): index and expose the video facet 2026-09-01 16:26:01 +02:00
Alex Ababii 8d407417fe granular notifications enabled by default in config 2026-09-01 15:48:03 +02:00
Alex Ababii 4a9b3f4002 upd test cases after intorduced fixes 2026-09-01 15:48:01 +02:00
Alex Ababii 71eb0716dc fix in app space memebership expired notification 2026-09-01 15:42:03 +02:00
Alex Ababii 59fd05f0a9 SpaceMembershipExpired notification email fix 2026-09-01 15:42:03 +02:00
Alex Ababii 3bd17299e3 send email when a share is removed (backport https://github.com/owncloud/ocis/pull/10915) 2026-09-01 15:42:03 +02:00
Alex Ababii cdc806d21c bump reva version with backport and fixes for SpaceMembershipExpired 2026-09-01 15:42:02 +02:00
Alex Ababii 8997fbc326 removed env vars for granular notifications from docker and vscode config 2026-09-01 15:40:31 +02:00
Alex Ababii df16f6d07f fix userlog policy violation/antivirus event handling, enabled granular notifications 2026-09-01 15:40:31 +02:00
Dominik Schmidt f60e2e52e5 Merge pull request #3437 from opencloud-eu/feat/driveitem-pending-operations
graph: expose pendingOperations on driveItems
2026-09-01 15:35:24 +02:00
Pascal Bleser 249d105d27 Merge pull request #3293 from pbleser-oc/chore/#1312-graph-event-handler-split
chore(graph): disable HTTP or eventhandlers by configuration
2026-09-01 14:31:24 +02:00
Alex Ababii 5a7afe0bf6 Merge pull request #3348 from opencloud-eu/fix/posix_cli_commands
fix: posix cli commands
2026-09-01 14:07:17 +02:00
Dominik Schmidt 000c99c63b graph: expose pendingOperations on driveItems 2026-09-01 14:05:18 +02:00
Dominik Schmidt df8d0e8fd3 Merge pull request #3198 from opencloud-eu/feat/search-tika-facets
feat(search): extract more data from tika 4 (if available)
2026-09-01 11:45:11 +02:00
Dominik Schmidt f7c5d3adb4 Merge pull request #3113 from opencloud-eu/feat/driveitem-following-tags
graph: expose following state, tags and allowed actions on driveItems
2026-09-01 11:34:53 +02:00
Pascal Bleser e4365320de chore(graph): use metrics tools introduced in 8821053ff8
* introduce a fix in the metrics tools introduced with 8821053ff8 in
    order to deal with a struct that has fields that are not exported
    (as is the case here)

  * introduce a func in the toplevel metrics package that registers a
    single metric, as that needs to be done explicitly for unexported
    ones

  * in the graph service, use RegisterAll and BuildInfo from the
    toplevel metrics package, as introduced with 8821053ff8
2026-09-01 10:57:54 +02:00
Pascal Bleser 5b72318493 chore(graph): add metrics for HTTP API and LDAP
Introducing gowrap as a build-time tool to generate interface delegate
structs from templates:

 * added as a 'make go-generate' target in services/graph,
 * added as a build-time dependency in .bingo/

Introduce an LDAP client abstraction interface to be able to wrap the
go-ldap client API with metrics transparently (and possibly hooks and
such in the future), in order to use delegation patterns to measure the
time LDAP (client) operations take to finish, as well as to track their
results (success, failure, not-found).

Has two implementations that are generated using gowrap:

* a go-ldap adapter implementation that directly delegates to a go-ldap
  connection
* a time measuring and metrics collecting implementation that delegates
  to another LdapClient

The metrics collecting one is disabled by default, can be enabled with
GRAPH_LDAP_METRICS_DISABLE=false

It collects durations of outbound LDAP client operations into a histogram, as
well as the number of concurrent outbound LDAP operations in a gauge (via an
atomic int and a gauge function, as that performs best).

Add an HTTP middleware that measures how long Graph HTTP API requests
take, storing taken time into a histogram along with labels for

 * method,
 * path pattern (from the chi routes),
 * Graph API version prefix,
 * Graph API resource name,
 * and the resulting status code.

It also tracks the number of concurrent inbound Graph API HTTP requests
using a gauge (also using an atomic int and a gauge function).

Disabled by default, can be enabled with
GRAPH_HTTP_METRICS_DISABLE=false

Add Backend and EducationBackend delegate implementations that measure
execution time on the level of the higher API call operations there
(CreateUser, DeleteUser, ..., CreateSchool, ...), generated using
gowrap.

Disabled by default, can be enabled with
GRAPH_IDENTITY_BACKEND_METRICS_DISABLE=false

Also added a small k6 script to produce some read-only load on the Graph
API, for a casual test of the metrics, as well as k6 in mise.toml.

Make an internal changes to how singular LDAP entry searches work in the LDAP
identity backends:

 * check whether searches for a singular entry returns more than one
   result, in which case a new error TooManyResults is returned, instead
   of leaving that undetected, blindly taking the first result, and
   potentially risking data inconsistencies

Improve the loggers in identity backends by adding attributes for their
request targets (Reva gateway address or LDAP URI, respectively).

Also add a "backend" attribute for all Graph API logs (set to "ldap" or
"cs3"), to help debug potential issues, and remove them from all the logger
debug calls at the beginning of each LDAP-related function as those should
really be part of the logger and set beforehand.

The LDAP identity backend logger also has two new attributes to help
debugging with logs:

 * write (bool): whether write operations are enabled
 * refint (bool): whether refint is enabled or not

Also adds a dedicated counter metric for user password change operations.

Minor campfire improvements:

 * add a constructor func for the CS3 backend

 * add a constructor func for the LDAP backend

 * in the LDAP identity backend, in searchLDAPEntryByFilter (used by all
   search/get public functions), errors that occur when performing LDAP
   SEARCH operations were blindly mapped to a ItemNotFound error,
   instead of being analyzed as it could be caused by a technical error

 * in the requireadmin middleware, add debug logging to explain why a
   request is denied

 * when an LDAP password change fails because the user entry was not
   found in LDAP, we now have a log message that tracks that
2026-09-01 10:57:47 +02:00
Pascal Bleser e8cf677353 chore(graph): disable HTTP or eventhandlers by configuration
In the scope of the broader issue #1312, this PR deals with performing
those changes for the `graph` service, namely to add the ability to
disable the HTTP API or to disable the events API handler by
configuration.

It also adds metrics for the events processing, and tests for the events
processing.

The previous implementation was combining the HTTP server service and
the events consumption, which is why this PR refactors the composition
of those services:

 * the event consumption has been moved into its own service
 * the identity.Backend is created beforehand, and then injected as a
   collaborator in both the HTTP service as well as the event consumer
   service

It also adds metrics, mainly for the event processing.

To encourage re-use in latter implementations and changes, it also
introduces two top-level package changes:

 * internal/eventstest/events_test_helpers: contains a TestBus
   implementation to unit-test event consumers without NATS
 * internal/metricstest/metrics_test_helpers: contains assertion
   functions to test Prometheus metrics
2026-09-01 10:57:26 +02:00
Dominik Schmidt d2bcd3cc9e test(search): comment the root-id assumption, pin the OR fan-out 2026-09-01 10:57:26 +02:00
Dominik Schmidt 71b3104e7b fix(search): keep wildcard drive ids out of completion and pinning 2026-09-01 10:57:26 +02:00
Dominik Schmidt 5f03dcf768 test(search): pin the driveId scope in the parity suite
SCOPE-06..09 cover the completed root id, negation and composition; the cross-space fixture root follows the space-root convention (opaque id = space id).
2026-09-01 10:57:26 +02:00
Dominik Schmidt 25043babd5 perf(search): prune the space fan-out for drive-pinned queries
When top-level AND conjuncts pin the query to a single root
(driveId/RootID restrictions), only that space's index is asked; the
restriction itself stays in the query, so this is purely an
optimization. Conservative by design: any top-level OR, negated or
group-nested restriction leaves the fan-out untouched, searching a
space too many is wasted work while skipping one would be wrong.
Mountpoints are kept for result path mapping.

Costs one extra parse of the query in the service; parsing once and
handing the AST to the engines (which currently re-parse per space) is
a follow-up that changes the engine interface.
2026-09-01 10:57:26 +02:00
Dominik Schmidt 8bd130cc41 feat(search): driveId as a query field
scope: takes an opaque resource id, which is hostile to hand-written
queries. Accept driveId:"<storage$space>" as a regular KQL field
instead: it resolves to the indexed RootID, and a bare drive id is
completed to the root resource id (a space root's opaque id is its
space id). Full root ids pass through untouched.

Combined with path: this gives a readable location scope without any
token stripping: both are plain fields, so they compose with groups,
OR and NOT like everything else.
2026-09-01 10:57:26 +02:00
Dominik Schmidt 394b0fdc45 graph: expose @libre.graph.permissions.actions.allowedValues on driveItems
Opt-in via $select, on GetDriveItem and the root children listing.
2026-09-01 10:21:50 +02:00
Dominik Schmidt 11d8b8a382 graph: expose @libre.graph.tags on driveItems 2026-09-01 10:21:50 +02:00
Dominik Schmidt f0515bb28c graph: expose @libre.graph.me.following on driveItems 2026-09-01 10:21:50 +02:00
Alex Ababii d923bfe1f6 cli: upd check for removing empty dir 2026-09-01 10:03:34 +02:00
Alex Ababii 9822c1ffd4 upd ci config for cliCommands suite 2026-09-01 10:03:34 +02:00
Alex Ababii c742ff6798 fix for trash purge empty dirs for posix 2026-09-01 10:03:34 +02:00
Alex Ababii a2d5ee2bc6 fix revisions purge cli command for posix 2026-09-01 10:03:34 +02:00
Alex Ababii 7aa5b467f6 cli: fix for cleanup call 2026-09-01 10:03:34 +02:00
Alex Ababii 983c836653 cli: restore single item from trash bin, listing on empty thrash bin 2026-09-01 10:03:34 +02:00
Dominik Schmidt 7d4c6d6524 Merge pull request #3424 from opencloud-eu/feat/search-scope
feat(search): scope searches to a drive via the driveId field
2026-09-01 09:36:20 +02:00
Jörn Friedrich Dreyer a249fa5f74 Merge pull request #2662 from dschmidt/docs/adr-reflection-based-search-mapping
docs(adr): unified search index mapping
2026-09-01 09:11:09 +02:00
Dominik Schmidt 5e4f124e78 fix(search): find the language endpoint on tika 4, keep the legacy path 2026-09-01 09:09:55 +02:00
Dominik Schmidt a1e8aa783d fix(search): reject NaN coordinates and non-positive bitrates 2026-09-01 09:09:55 +02:00
Dominik Schmidt ae9ec4ce70 fix(search): harden facet value parsing 2026-09-01 09:09:55 +02:00
Dominik Schmidt c03b9c7229 fix(search): gate the audio facet inside getAudio, keep facets across meta entries 2026-09-01 09:09:55 +02:00
Dominik Schmidt 77bb974f9b test(search): run acceptance and CI against the released tika 4.0.0
Both tika pins (local compose and the CI service) point at apache/tika:4.0.0-full; the search README links tika version-less.
2026-09-01 09:09:55 +02:00
Dominik Schmidt 1be01ae50c fix(search): read the tika 4 content key, keep the legacy one
Tika 4.0.0 renamed the meta prefix from X-TIKA: to tk:.
2026-09-01 09:09:55 +02:00
Dominik Schmidt 2b4a6fd739 feat(search): populate audio and gps-altitude facets from tika, drop the audio-only gate 2026-09-01 09:09:55 +02:00
Dominik Schmidt 7e0f536b83 refactor(search): split the tika extractor and its tests into per-facet files 2026-09-01 09:09:55 +02:00
Jörn Friedrich Dreyer 49ebe072f1 Merge pull request #3426 from opencloud-eu/dependabot/go_modules/go.opentelemetry.io/otel-1.46.0
build(deps): bump go.opentelemetry.io/otel from 1.45.0 to 1.46.0
2026-09-01 08:25:05 +02:00
Jörn Friedrich Dreyer 8028e3e310 Merge pull request #3435 from fschade/test-search-cjk-parity
test(search): cover chinese search in the parity suite
2026-09-01 08:24:34 +02:00
Dominik Schmidt 7d75b98c69 Merge pull request #3423 from opencloud-eu/test/parity-readme-staleness
test(search): fail the parity suite when the committed matrix is stale
2026-09-01 08:24:15 +02:00
Florian Schade 9e8403818c test(search): cover chinese search in the parity suite 2026-09-01 07:46:03 +02:00
Dominik Schmidt ef41429838 test(search): rename the update flag to UPDATE_SEARCH_PARITY_MATRIX 2026-09-01 07:35:15 +02:00
Dominik Schmidt 7bf64e329c docs(adr): apply review wording suggestion 2026-09-01 07:26:32 +02:00
Dominik Schmidt 268a4497b4 Merge pull request #3197 from opencloud-eu/feat/search-schema-change-handling
feat(search): check the index schema on startup and refuse breaking changes
2026-09-01 07:17:29 +02:00
Florian Schade b4893baa81 Merge pull request #3379 from opencloud-eu/ci/search-opensearch-pipelines
ci: run search API and e2e suites against OpenSearch
2026-09-01 06:56:07 +02:00
opencloudeu ebf9566886 [tx] updated from transifex 2026-08-31 23:20:06 +00:00
Dominik Schmidt b149316349 docs(adr): keep the facet reachability gap out of the problem list 2026-08-31 21:22:22 +02:00
Dominik Schmidt c6ddc13055 docs(adr): date the context section 2026-08-31 21:20:54 +02:00
Dominik Schmidt 65ea519099 docs(adr): update to the implemented state 2026-08-31 21:16:43 +02:00
Dominik Schmidt 2041fdc259 test(search): comment the root-id assumption, pin the OR fan-out 2026-08-31 21:14:58 +02:00
Dominik Schmidt 8e68a34b8f fix(search): keep wildcard drive ids out of completion and pinning 2026-08-31 21:14:04 +02:00
Jörn Friedrich Dreyer 8f44f6eb42 Merge pull request #3429 from opencloud-eu/dependabot/go_modules/google.golang.org/grpc-1.83.2
build(deps): bump google.golang.org/grpc from 1.83.1 to 1.83.2
2026-08-31 18:10:47 +02:00
Jörn Friedrich Dreyer 0f9c6dc1ec Merge pull request #3425 from pbleser-oc/chore/mise-go-1.25.9
chore(mise): upgrade go in mise to 1.25.9 after 578861b473
2026-08-31 18:07:57 +02:00
dependabot[bot] 5bfbfd8ff7 build(deps): bump google.golang.org/grpc from 1.83.1 to 1.83.2
Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.83.1 to 1.83.2.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.83.1...v1.83.2)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-version: 1.83.2
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-08-31 14:45:06 +00:00
dependabot[bot] fe1e7632fa build(deps): bump go.opentelemetry.io/otel from 1.45.0 to 1.46.0
Bumps [go.opentelemetry.io/otel](https://github.com/open-telemetry/opentelemetry-go) from 1.45.0 to 1.46.0.
- [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases)
- [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.45.0...v1.46.0)

---
updated-dependencies:
- dependency-name: go.opentelemetry.io/otel
  dependency-version: 1.46.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-08-31 14:44:13 +00:00
Pascal Bleser 15b7fad504 chore(mise): upgrade go in mise to 1.25.9 after 578861b473
* upgrade the go version in mise.toml to 1.25.9 because dependabot
   upgraded go to 1.25.9 in go.mod

 * mise: upgrade dlv from 1.27.0 to 1.27.1:
   https://github.com/go-delve/delve/releases/tag/v1.27.1
2026-08-31 15:52:30 +02:00
v.scharf 5c97014cd2 ci: run search API and e2e suites against OpenSearch 2026-08-31 15:29:37 +02:00
Dominik Schmidt 774c732f52 test(search): pin the driveId scope in the parity suite
SCOPE-06..09 cover the completed root id, negation and composition; the cross-space fixture root follows the space-root convention (opaque id = space id).
2026-08-31 15:23:25 +02:00
Dominik Schmidt 95f175fe10 perf(search): prune the space fan-out for drive-pinned queries
When top-level AND conjuncts pin the query to a single root
(driveId/RootID restrictions), only that space's index is asked; the
restriction itself stays in the query, so this is purely an
optimization. Conservative by design: any top-level OR, negated or
group-nested restriction leaves the fan-out untouched, searching a
space too many is wasted work while skipping one would be wrong.
Mountpoints are kept for result path mapping.

Costs one extra parse of the query in the service; parsing once and
handing the AST to the engines (which currently re-parse per space) is
a follow-up that changes the engine interface.
2026-08-31 15:23:25 +02:00
Dominik Schmidt 094baab7be feat(search): driveId as a query field
scope: takes an opaque resource id, which is hostile to hand-written
queries. Accept driveId:"<storage$space>" as a regular KQL field
instead: it resolves to the indexed RootID, and a bare drive id is
completed to the root resource id (a space root's opaque id is its
space id). Full root ids pass through untouched.

Combined with path: this gives a readable location scope without any
token stripping: both are plain fields, so they compose with groups,
OR and NOT like everything else.
2026-08-31 15:23:25 +02:00
Dominik Schmidt feb13d426b test(search): name the update flag in the generated header 2026-08-31 15:16:15 +02:00
Dominik Schmidt b1533a3135 test(search): write the matrix only with UPDATE_MATRIX
A plain run compares and fails on a stale README without touching the working tree; UPDATE_MATRIX=true regenerates it.
2026-08-31 15:16:15 +02:00
Dominik Schmidt 874d8fdac2 test(search): fail the parity suite when the committed matrix is stale
The suite already regenerates README.md on every run; now a run whose content differs from the committed file also fails, so CI catches a stale matrix. The fresh content is on disk after the run, committing it is the fix.
2026-08-31 15:16:15 +02:00
Dominik Schmidt 0e2e2876f4 test(search): a golden diff says whether it needs a SchemaVersion bump
The failure runs the classifier on golden vs generated: additive means regenerate only, breaking means bump too.
2026-08-31 15:15:18 +02:00
Dominik Schmidt e11ce95d95 test(search): golden for the generated OpenSearch index definition
Pins the shipped schema as a reviewable diff; regenerate with UPDATE_GOLDEN=1.
2026-08-31 15:12:54 +02:00
Dominik Schmidt 7493f189a0 chore(search): adapt tests and golden to the v4 base
The refuse specs use registered analyzers (fulltext is gone), the golden regenerates via UPDATE_GOLDEN, MappingGetResp grew an accessor, and the parity suite passes the new NewBackend signature.
2026-08-31 15:12:54 +02:00
Dominik Schmidt 53fa66f120 chore(search): follow the rebased base
The opensearch-go bump renamed the mapping-get accessor, the schema check
takes a context and a logger now, and the golden bleve mapping carries the
word-broken Name and Title.
2026-08-31 15:12:54 +02:00
Dominik Schmidt 519dd112ea review: note SchemaVersion bump on bleve marshaling drift in golden test 2026-08-31 15:12:54 +02:00
Dominik Schmidt 414247d6c4 review: use any over interface{}, clone slice in AddBreaking 2026-08-31 15:12:54 +02:00
Dominik Schmidt 7ff2643d23 refactor(search): share SortedUnionKeys + Classification.AddBreaking
- export mapping.SortedUnionKeys and reuse it in bleve.compareKeysExcept
  instead of a copied union-of-keys block
- add Classification.AddBreaking to fold engine-specific breaking reasons and
  force the verdict, replacing the identical block in the bleve and opensearch
  Classify paths
2026-08-31 15:12:54 +02:00
Dominik Schmidt a0521f0260 review: trim verbose comments, cover Reconcile, table-driven Classify tests
- shorten the multi-line doc comments flagged as too verbose
- add reconcile_test.go: direct unit tests for Reconcile incl. the
  persisted-but-errored and classify-error branches (previously only
  reached indirectly through the engine integration tests)
- convert the 11 near-identical Classify It blocks to a DescribeTable
2026-08-31 15:12:54 +02:00
Dominik Schmidt 6b07a0d9dc chore(search): tighten the schema-reconcile comments 2026-08-31 15:12:54 +02:00
Dominik Schmidt dc863c64c6 refactor(search): single-source the new-index log message 2026-08-31 15:12:54 +02:00
Dominik Schmidt a8ffe91c79 feat(search): log a reindex hint when a fresh search index is created 2026-08-31 15:12:54 +02:00
Dominik Schmidt fec7536617 refactor(search): warn on a persisted additive change even when the reopen fails 2026-08-31 15:12:54 +02:00
Dominik Schmidt e8c93359b1 refactor(search): route schema verdict handling through a shared mapping.Reconcile 2026-08-31 15:12:54 +02:00
Dominik Schmidt 04c37d12d2 refactor(search): extract opensearch.NewClient out of server startup 2026-08-31 15:12:54 +02:00
Dominik Schmidt 9755728d43 refactor(search): harden the index-diff helpers against unset input 2026-08-31 15:12:54 +02:00
Dominik Schmidt 37eed7c321 refactor(search): only enforce analysis settings, tolerate operational drift 2026-08-31 15:12:54 +02:00
Dominik Schmidt a6bfb61935 refactor(search): make the breaking-schema error developer-facing 2026-08-31 15:12:54 +02:00
Dominik Schmidt 5abe2ccb26 fix(search): list the schema mismatch reasons on separate lines 2026-08-31 15:12:54 +02:00
Dominik Schmidt e9347998b7 fix(search): name the service to stop in the schema mismatch error 2026-08-31 15:12:54 +02:00
Dominik Schmidt 33299afbaa fix(search): address max-review findings
- the additive warnings advertise --all-spaces --force-rescan; a plain
  walk skips unchanged documents and never backfills the new fields
- Apply checks index existence first again, so a pre-provisioned index
  needs no create privilege and odd create-error shapes (string error
  bodies, cluster blocks) cannot fail a healthy startup; Create on 404
  keeps the typed already-exists swallow as the creation-race backstop
- number_of_replicas drift is not breaking, it is runtime-tunable and
  needs no rebuild
- bleve returns the classification alongside post-persist errors and
  the server warns before the error check, so the one-time additive
  warning is not lost when close or reopen fails
- a golden fixture pins the marshaled bleve mapping so a dependency
  bump that changes marshaling fails in CI instead of refusing every
  installation in the field
2026-08-31 15:12:54 +02:00
Dominik Schmidt 451e893d7d chore(search): tighten doc comments 2026-08-31 15:12:54 +02:00
Dominik Schmidt b67811ab3c chore(search): warn on additive opensearch changes and name the exact delete step
Addresses the two Copilot review comments on the PR: the additive
opensearch log now matches the bleve warning (level and re-index hint),
and the refuse message spells out how to delete the index per engine
(DELETE /<name> vs removing the bleve directory).
2026-08-31 15:12:54 +02:00
Dominik Schmidt 09ea7e5f15 chore(search): mention the impact of disabling search in the refuse message 2026-08-31 15:12:54 +02:00
Dominik Schmidt 1c130c448c chore(search): drop the changelog entry 2026-08-31 15:12:54 +02:00
Dominik Schmidt 03dfabddbb feat(search): check the index schema on startup and refuse breaking changes
Both engines now diff the stored/live index schema against the schema
generated from code when the service starts. A shared recursive
classifier in the mapping package is the single oracle:

- equal: start normally.
- additive (new fields without any indexed data): applied in place.
  OpenSearch gets a PUT _mapping with the full code properties, bleve
  persists the code mapping into the index (SetInternal + reopen) so
  the new fields are properly typed immediately and later startups
  classify equal. A startup warning lists the new fields because
  documents indexed before the upgrade lack them until re-indexed.
- breaking (changed definitions or analyzers, removed or renamed
  fields, or new fields that already contain data of unknown form):
  refuse to start with an error describing the rebuild procedure
  (delete the index, start, run "opencloud search index --all-spaces")
  and the OC_EXCLUDE_RUN_SERVICES=search escape hatch.

PUT _mapping is deliberately only the apply mechanism, never the
judge: its merge semantics cannot see removals or renames and it
accepts in-place updatable param changes with an ack. bleve
additionally checks idx.Fields() so previously dynamically indexed
data (which leaves no schema trace in bleve) is caught, matching by
exact name and by path prefix.

While at it: the OpenSearch startup check runs with a real,
minute-bounded context instead of context.TODO(), bleve indexes are
opened with a 5s bolt_timeout so a second process fails fast instead
of hanging on the file lock, and the reversed errors.Is arguments in
bleve.NewIndex were fixed.

https://github.com/opencloud-eu/opencloud/issues/3092
2026-08-31 15:12:54 +02:00
Dominik Schmidt d38fbc8e52 Merge pull request #3345 from opencloud-eu/refactor/search-mapping
refactor: reflection-based search mapping + location geopoint
2026-08-31 15:12:49 +02:00
Benedikt Kulmann 109e9555c8 chore(policies): disable gRPC or event handlers by configuration + metrics (#3287)
* add the ability to disable the gRPC API handler
   (POLICIES_GRPC_DISABLED)

 * add the ability to disable the Events API handler
   (POLICIES_EVENTS_DISABLED)

 * add metrics

 * add support for specifying rego files via the environment varirable
   POLICIES_ENGINE_FILES

 * for file paths specified in yaml or in POLICIES_ENGINE_FILES, support
   'config:' and 'data:' path prefixes

 * fix typos in the documentation, and try to make it more clear

 * add metrics to the documentation

 * add a section for testing in the documentation

 * introduces a new top-level package pkg/metrics/ with utilities for
   metrics that are backported from the groupware branch, with unit
   tests
2026-08-31 14:58:52 +02:00
Dominik Schmidt 1bf813068e docs(search): version-free index example in the README too 2026-08-31 14:17:29 +02:00
Dominik Schmidt aa2f7da029 docs(search): the unversioned index is the one up to 7.4 2026-08-31 14:15:17 +02:00
Dominik Schmidt 0188b18d2b docs(search): everything but the current index generation can go 2026-08-31 14:14:19 +02:00
Dominik Schmidt 73bfc1cf48 docs(search): just the reindex step 2026-08-31 14:12:52 +02:00
Dominik Schmidt 3a7d6ca54b docs(search): no index names in the migration steps, admins never type them 2026-08-31 14:12:38 +02:00
Dominik Schmidt 5d3c67b587 docs(search): the only migration path is a reindex
Copying the old index over misses the search sibling fields, copied documents would be unfindable.
2026-08-31 14:06:12 +02:00
Dominik Schmidt 2458d51ed1 feat(search): favorites are opaque user ids, no lowercase sibling
Same rule as the other ids; removing the sibling later would take another schema generation.
2026-08-31 13:50:36 +02:00
Dominik Schmidt d4f1ae910c fix(search): scope the OpenSearch purge to the resource's space
The delete-by-query matched the bare Path, so a purge could take same-path documents in other spaces with it; bleve was already RootID-scoped. Pinned as rootscope-04.
2026-08-31 13:50:36 +02:00
Dominik Schmidt 5abef94482 test(search): regenerate the parity matrix for the mediatype:file rows 2026-08-31 13:45:42 +02:00
Dominik Schmidt e14bef7ef3 chore(search): version-free schema examples in docs strings 2026-08-31 13:44:47 +02:00
Dominik Schmidt 98bdce156b chore: drop the duplicated vendor entry for analysis/char/regexp 2026-08-31 13:44:47 +02:00
Dominik Schmidt dd6b3548f9 fix(search): tighten the generated search siblings
The OpenSearch _lowercase siblings are search-only like bleve's (no doc_values, own map instance instead of aliasing the base), and the dead Path_words field is gone: _words exists for keyword fields only, as SearchSiblings declares.
2026-08-31 13:44:47 +02:00
Dominik Schmidt 9e8bb474b4 fix(search): compose the mediatype:file negation correctly
The file expansion is grouped again so its NOT stays atomic next to other terms (OpenSearch turned 'mediatype:file OR x' into '(NOT dir) AND x'), and the bleve compiler no longer re-keys resolved groups (a negated mediatype group targeted the raw 'mediatype' field and matched everything). Pinned as MEDIATYPE-07..10.
2026-08-31 13:43:19 +02:00
Dominik Schmidt 9503884472 docs(search): migration notes for schema v4
Also make the reindex copy safe to run after the service already indexed (op_type create, conflicts proceed).
2026-08-31 13:40:43 +02:00
Dominik Schmidt 2383bcddcc test(search): leave behavior to the parity suite
The engine suites keep what is engine-specific (index setup, health, purge-space batching); every behavior answer lives in the parity suite once. The shared test client learns IndicesCount, and FIELDS-15 pins the Size/Type gate for number queries.
2026-08-31 13:40:43 +02:00
Dominik Schmidt b92d671c02 chore(search): drop the orphaned hand-written v3 mapping leftovers 2026-08-31 13:40:43 +02:00
Dominik Schmidt 523d377a08 test(search): carry the parity suite over to the rebased semantics
Typed Mtime fixtures, VersionedIndexName, and audio.artist matches case-insensitively now (facets search case-insensitively by default).
2026-08-31 13:40:43 +02:00
Dominik Schmidt d5872aa851 fix(search): match fulltext phrases in order on bleve
Content:"reports monthly" must not match 'monthly reports'; the phrase runs on the analyzed field like on OpenSearch.
2026-08-31 13:40:43 +02:00
Dominik Schmidt 283de0d29a feat(search): adopt the parity-pinned query semantics on the sibling routing
From #3408: hidden takes bool words only, type categories map to the stored value in the shared pass, ? counts as a wildcard, a non-suffix wildcard on a word-broken field forgives the extension, = matches the whole value on the lowercased sibling, paths lose their trailing slash. Dead compiler helpers removed.
2026-08-31 13:40:42 +02:00
Dominik Schmidt ba210232ab feat(search): analyze Content by words on both engines, no stemming
Adopts the parity-pinned semantics from #3408: 'report' does not match 'reports'; the porter fulltext analyzer is gone.
2026-08-31 13:40:42 +02:00
Dominik Schmidt b5693d7058 chore(search): one schema generation, bump it to 4
The hand-written v3 of the interim mapping is taken; both engines derive index name and directory from search.SchemaVersion.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 903ef4521b feat(search): search facets by word too, whole values only by opt-out
SharePoint's default for a text property is word breaking, so ours is too:
every keyword field gets the _words sibling unless it opts out with
NoWordBreaker, which now carries SharePoint's polarity as well. Artist,
album, camera model and the other facets match by word like name and title;
tags and favorites stay one label, ids, paths and the mime type one value.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 96e728901e feat(search): search every keyword field case-insensitively by default
KQL searches case-insensitively, so every keyword field gets its lowercased
search sibling unless it opts out: ids are opaque, paths are POSIX, the mime
type is normalized already. That takes the facets along, artist or camera
model match regardless of case, while the case-preserved base still answers
and aggregates. Which siblings a field carries is decided once, from the
struct and the overrides, and the renderers, the document writer and the
query lowering all read it from there.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 6b07ea745c feat(search): split names and titles into words on both engines
A single word finds the names and titles that contain it, `report` finds
Report.txt, on bleve as well as on OpenSearch, which so far only did it by
accident of its dynamic mapping. Modelled like SharePoint's NoWordBreaker:
a keyword field is one whole value unless the override switches that off,
which adds a search-only _words sibling next to _lowercase, analyzed into
lowercased words (a dot is a word boundary, no stemming). The base stays the
whole value for returning and aggregating, wildcards and whole values keep
using _lowercase. Quotes do not change the meaning, a phrase is a phrase
either way, and there is no exact-match operator yet.
2026-08-31 13:40:42 +02:00
Dominik Schmidt e64692ce9e test(search): re-apply opensearch backend coverage under ginkgo 2026-08-31 13:40:42 +02:00
Dominik Schmidt 9c46713a65 refactor(search): make paths case-sensitive and scope searches at query level
Paths act as references (location scoping, deep links): /Foo and /foo
are distinct siblings, so path: matching must be exact. Case-insensitive
folder discovery is served by name: and its lowercase sibling. This also
matches bleve on main, where path queries have always been
case-sensitive.

Dropping the sibling removes its biggest maintenance cost: Path is the
one mutable sibling field, a move rewrites the paths of a whole subtree
and the OpenSearch move script had to rebuild Path_lowercase alongside
the base field.

With paths case-sensitive, the ref path scope moves into the query
itself: bleve as a term/prefix disjunction on the keyword Path,
OpenSearch as a term filter on its path_hierarchy tokens. The
post-filter that used to drop out-of-scope hits after the query ran is
gone; totals and paging now respect the scope instead of being computed
over the whole space, and a wrong-cased scope simply matches nothing.
2026-08-31 13:40:42 +02:00
Dominik Schmidt f6739f552d fix(search): bind a leading NOT to the term right after it, on both backends
A leading NOT next to an operator was miscompiled: the bleve compiler left the consumed term in `next`, so `NOT x AND y` dropped `y` and produced a self-contradicting clause; the OpenSearch transpiler checked nextOp==AND before prevOp==NOT, so the negated term landed in `must` instead of `must_not`. NOT is unary and binds to the node directly after it regardless of what follows. This also fixes `mediatype:file AND <term>` (the web Files filter) at the root, so the earlier mediatype:file group workaround is dropped.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 5d9a255b16 test(search): port fieldindex, mimetype and normalize tests to ginkgo 2026-08-31 13:40:42 +02:00
Dominik Schmidt 17144d4791 fix(search): keep OpenSearch path queries as unanalyzed term queries
A path value with spaces went through a match_phrase query, which analyzes
the query with the path_hierarchy analyzer; the resulting "." prefix token
matches every document in the space, breaking descendant matching and the
stale-path check after a move.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 0284224787 refactor(search): anchor the OpenSearch move prefix replace to the path start
The move script rewrote Path/Path_lowercase with painless String.replace, which replaces every occurrence of the old path, not just the leading prefix. OpenCloud paths are ./-prefixed so the full old path only occurs at the start and the result is byte-identical, but startsWith + substring makes the prefix-only intent explicit and robust to any path format. Not a live bug fix, a hardening.
2026-08-31 13:40:42 +02:00
Dominik Schmidt fc41995a47 fix(search): validate CaseInsensitive against the effective field type
The guard only rejected CaseInsensitive when a non-keyword/path Type was set explicitly. With no Type, isCasedType treated the field as cased, so CaseInsensitive on an inferred numeric/bool/datetime field passed validation but produced no _lowercase sibling, and the query would silently match nothing. Validate now falls back to the inferred Go type.
2026-08-31 13:40:42 +02:00
Dominik Schmidt c94059fb85 fix(search): keep mediatype:file atomic so it composes with other terms
mediatype:file expands to a NOT restriction. Spliced inline as `NOT MimeType:httpd/unix-directory`, the bleve compiler's NOT branch left a stale operand, so `mediatype:file AND name:x` dropped `name:x` and matched nothing (the web Files filter). It is now wrapped in a group so the negation stays atomic; verified fixing both bleve and OpenSearch.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 69c517a8b2 fix(search): nest json-tagged embedded structs instead of flattening them
resolveField marked every anonymous field embedded, so walkFields (mapping, field index, validate) and fillStruct (deserializer) flattened a json-tagged embedded struct, while conversions.To/encoding/json on the write path nests it under the tag, mapping and deserializing it at the wrong path. An anonymous field is now embedded only without a json tag name, matching encoding/json; fillStruct also recurses into a value nested struct. No current type has a tagged embedded struct, so runtime behavior is unchanged; this hardens the reflection walker.
2026-08-31 13:40:42 +02:00
Dominik Schmidt dad99e85b2 fix(search): make mediatype categories and MIME types case-insensitive
mediatype:Folder / mediatype:IMAGE resolved to a literal MimeType search and matched nothing because Expand switched on the raw value. The value is now lowercased in the lowering pass, so categories and literal MIME types match regardless of case, consistently on both backends.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 8cb973d695 test(search): cover mediatype and direct MimeType search on both backends
Adds bleve and OpenSearch coverage for category (image), literal MIME (image/svg+xml, with + and /), and raw MimeType: queries. Documents why MimeType skips the bleve escaper: it is not a bug, bleve treats / and + as literals mid-term, so a literal MIME still matches exactly while the category wildcard image/* keeps its *.
2026-08-31 13:40:42 +02:00
Dominik Schmidt c680a90e49 fix(search): reject CaseInsensitive on non-keyword/path fields
CaseInsensitive routes queries to a <field>_lowercase sibling that is only generated for keyword/path fields, so marking any other type CaseInsensitive would silently match nothing. Validate now rejects it up front.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 1bf15599f7 fix(search): write a consistent empty _lowercase sibling for empty arrays
The []any branch skipped the sibling for an empty array while the []string branch wrote an empty one; both now write it, matching the base field.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 727ca92afe fix(search): review fixes for path AND-term and content wildcard
bleve compiled a path restriction to a DisjunctionQuery, which mapBinary redistributes as an OR-chain, so `path:/Foo AND name:bar` matched the folder itself unconditionally. It is now a BooleanQuery (should: folder OR descendants), which mapBinary keeps atomic under an enclosing AND.

The OpenSearch full-text branch ran before the wildcard check, so `content:foo*` degraded to a phrase match and diverged from bleve; the wildcard check now comes first.

Adds the missing coverage the review flagged: path AND term, content wildcard, case-insensitive tags (the array sibling branch), and a spaced path with descendants on OpenSearch.
2026-08-31 13:40:42 +02:00
Dominik Schmidt e3c45dd1d0 fix(search): analyze OpenSearch full-text queries, stem Content like bleve
Single-term `content:` built an unanalyzed term query, so once this branch dropped the blanket query-value lowercasing, `content:Foo` missed on OpenSearch (bleve was unaffected, its query analyzes). Fielded full-text queries now use a match query. OpenSearch `Content` also gets a porter stemming analyzer (it used the default standard analyzer and never stemmed), so full-text search matches bleve on both case and stemming.
2026-08-31 13:40:42 +02:00
Dominik Schmidt ec58861e4e feat(search): per-field case-insensitive search via _lowercase siblings
Keyword and path fields always index their case-preserved base and, when CaseInsensitive is set, an additional <field>_lowercase sibling used only for matching. The KQL lowering marks a restriction case-insensitive; each backend searches the sibling and lowercases the query value the same way the sibling is precomputed at index time (Go strings.ToLower on both sides, so non-ASCII stays consistent).

Search always returns the case-preserved base, so the sibling never has to be read back. In bleve it is indexed but not stored, kept out of _all, and without doc values. In OpenSearch it deliberately stays in _source: excluding it would make every update-by-query script rebuild all siblings from the document via painless toLowerCase, which lowercases differently than Go and would drift from the query side. Keeping it in _source avoids that, and a lowercased copy of a name or path is negligible disk in a cluster.

The OpenSearch move script keeps the base and its sibling in sync by swapping the moved prefix in Path_lowercase and setting Name_lowercase from Go-lowercased params, so case-insensitive search still finds a file after it moves (previously the sibling went stale). bleve re-indexes the whole document on move/delete/restore, so its siblings stay fresh for free.

This also repairs OpenSearch path search (the query value was no longer folded to lowercase, so path:<Foo> returned nothing) and makes bleve path queries match a folder and its descendants like OpenSearch's path_hierarchy. The Path base stays case-preserved so the move/delete descendant update (an exact TermQuery on Path) matches mixed-case folders.
2026-08-31 13:40:42 +02:00
Dominik Schmidt fb22dd81a4 refactor(search): run the OpenSearch backend on the shared lowering pass
KQLToOpenSearchBoolQuery runs query.Normalize, then only value lowercasing stays
backend-specific; remapKey and unfoldValue are gone.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 1cba974c1d refactor(search): run the bleve backend on the shared lowering pass
The bleve Creator runs query.Normalize before compiling; the compiler consumes a
canonical AST with no field resolution or media-type special-casing.
2026-08-31 13:40:42 +02:00
Dominik Schmidt bf7d04f7a1 feat(search): add shared KQL lowering pass
query.Normalize resolves field names (query.ResolveField, from the derived
index + a small alias overlay) and expands media-type restrictions
(mimetype.Expand) once, between parse and backend compilation.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 45a8096617 feat(search): derive a case-insensitive field-name index from the resource struct
mapping.FieldNameIndex walks the struct and maps a lowercased field path to the
real field name, including nested facet sub-fields. Backend-neutral.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 65f615d927 refactor(kql): move parse/validation errors into pkg/kql
The KQL parser produced its own validation errors but imported them from the
search service's query package. Move them into pkg/kql and let the search
backend consume kql.IsValidationError, so the parser stops depending on a
service package.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 40a2f14600 fix(search): preserve query value case for case-sensitive fields on OpenSearch
OpenSearch lowercased every KQL query value, so exact-match queries on
case-preserved keyword fields (facet values, ids) never matched their stored
token. Fold the value only for fields with a lowercasing analyzer, mirroring the
bleve backend. The field set is derived once in search.LowercaseValueFields and
shared by both backends (bleve's local buildLowercaseFields is dropped).
2026-08-31 13:40:42 +02:00
Dominik Schmidt 7d55849900 feat(search): version the search index by schema version
Both backends carry a shared search.SchemaVersion in the index name
(OpenSearch <base>-vN) and data path (bleve-vN). A breaking schema change
bumps the version so the service builds a fresh index instead of colliding
with the incompatible previous one; the old index is left in place.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 450f380198 fix(search): type mtime as a date 2026-08-31 13:40:42 +02:00
Dominik Schmidt 7cc144a9c3 test(search): set Mtime on opensearch folder and root fixtures
The Mtime field is mapped as an OpenSearch `date`, which rejects an
empty value with `mapper_parsing_exception: cannot parse empty date`.
The folder and root fixtures had no Mtime, so serializing them to
`"Mtime": ""` made TestEngine_Purge/purge_resource_trees fail when the
document was indexed. Give both a valid RFC3339 Mtime, matching the
file fixture.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 6695f99c90 test(search): convert mapping package tests to ginkgo
New package, so use the repo's standard test framework.
2026-08-31 13:40:42 +02:00
Dominik Schmidt c6c36212f5 test(search): convert bleve geo/mtime tests to ginkgo
The package's engine suite is ginkgo; these new tests were plain.
2026-08-31 13:40:42 +02:00
Dominik Schmidt c384322d94 test(search): use RFC3339 Mtime in opensearch fixture
Mtime is now a date field; the fixture's Go-format string fails
OpenSearch date parsing.
2026-08-31 13:40:42 +02:00
Dominik Schmidt 29cfdb01a0 feat(search): index Location as a geopoint on both backends
Add a TypeGeopoint field type. The libregraph Location facet is kept as an
object (retrieval / numeric queries) and a sibling <name>_geopoint field
carries the {lat,lon} form for geo-distance / bbox / polygon queries,
uniform across bleve and OpenSearch via the shared mapping. PrepareForIndex
splices the sibling in at write time.
2026-08-31 13:40:42 +02:00
Dominik Schmidt e78cad1cbe refactor: reflection-based search mapping
Build the bleve and OpenSearch index mappings from the Go struct via
reflection (json tags + per-field overrides) instead of hand-rolled
mappings and hit deserializers. New mapping package: BleveBuildMapping,
OpenSearchBuildMapping, Deserialize[T], PrepareForIndex; field decoding is
fail-soft. Mtime is typed as a date so mtime ranges are chronological on
both backends. Route CS3 facet parsing through mapping.DeserializeStringMap.

The any-valued (bleve hit) and string-valued (CS3 metadata) deserializers
share one generic fillStruct walker with a per-value setLeaf callback.
2026-08-31 13:40:42 +02:00
Jörn Friedrich Dreyer 0becbb8212 Merge pull request #3419 from fredrikblau/fix/3413-oidc-nil-provider
fix(middleware): reject empty OIDC providers safely
2026-08-31 13:37:09 +02:00
Florian Schade a9da258dcc Merge pull request #3408 from fschade/fix-search-field-matching
fix(search): make openSearch and bleve behave the same
2026-08-31 13:24:49 +02:00
Florian Schade 1bde3d14bb fix(search): bring back rebase removals 2026-08-31 11:57:19 +02:00
Pascal Bleser 8821053ff8 chore(policies): disable gRPC or event handlers by configuration + metrics
* add the ability to disable the gRPC API handler
   (POLICIES_GRPC_DISABLED)

 * add the ability to disable the Events API handler
   (POLICIES_EVENTS_DISABLED)

 * add metrics

 * add support for specifying rego files via the environment varirable
   POLICIES_ENGINE_FILES

 * for file paths specified in yaml or in POLICIES_ENGINE_FILES, support
   'config:' and 'data:' path prefixes

 * fix typos in the documentation, and try to make it more clear

 * add metrics to the documentation

 * add a section for testing in the documentation

 * introduces a new top-level package pkg/metrics/ with utilities for
   metrics that are backported from the groupware branch, with unit
   tests
2026-08-31 11:54:32 +02:00
Florian Schade 5c153ff74a test(search): drop the engine overrides, both engines answer alike 2026-08-31 11:15:03 +02:00
Florian Schade 0106a700c2 test(search): follow the versioned index and pin the equals operator 2026-08-31 10:58:13 +02:00
dependabot[bot] 4639f8f7c6 build(deps): bump github.com/blevesearch/bleve/v2 from 2.6.0 to 2.6.1
Bumps [github.com/blevesearch/bleve/v2](https://github.com/blevesearch/bleve) from 2.6.0 to 2.6.1.
- [Release notes](https://github.com/blevesearch/bleve/releases)
- [Commits](https://github.com/blevesearch/bleve/compare/v2.6.0...v2.6.1)

---
updated-dependencies:
- dependency-name: github.com/blevesearch/bleve/v2
  dependency-version: 2.6.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-08-31 10:48:06 +02:00
Florian Schade bd4287762e fix(search): answer the same on both engines
both engines now agree on names, titles, tags, paths, types, sizes, dates,
hidden flags, facet values and wildcards. quotes only delimit phrases and
the equals operator matches the whole field value, following the kql spec.
the index name carries a generation so a changed mapping starts on a fresh
index, MIGRATION.md says how to fill it.
2026-08-31 10:44:03 +02:00
Florian Schade ffbfe171c3 Merge pull request #3418 from opencloud-eu/test/search-parity-suite
test(search): engine parity suite for bleve and opensearch
2026-08-31 10:22:25 +02:00
dependabot[bot] 1bd59d9210 build(deps): bump github.com/sirupsen/logrus from 1.10.0 to 1.10.1
Bumps [github.com/sirupsen/logrus](https://github.com/sirupsen/logrus) from 1.10.0 to 1.10.1.
- [Release notes](https://github.com/sirupsen/logrus/releases)
- [Changelog](https://github.com/sirupsen/logrus/blob/master/CHANGELOG.md)
- [Commits](https://github.com/sirupsen/logrus/compare/v1.10.0...v1.10.1)

---
updated-dependencies:
- dependency-name: github.com/sirupsen/logrus
  dependency-version: 1.10.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-08-31 10:00:26 +02:00
opencloudeu 32a35f8653 [tx] updated from transifex 2026-08-30 23:17:11 +00:00
amir 0a3378eef8 fix(middleware): reject empty OIDC providers safely 2026-08-29 19:29:18 +03:30
Dominik Schmidt fef2df8671 chore(codacy): exclude the generated search parity matrix
Markdown lint has nothing to say about a table the test suite writes.
2026-08-28 19:56:57 +02:00
Florian Schade 18759b0183 test(search): cover a deleted space on both engines 2026-08-28 19:38:09 +02:00
Dominik Schmidt 545aa114e8 test(search): let the parity suite replace the per-engine backend suites
Everything the bleve and OpenSearch backend suites checked now runs against
both engines in services/search/pkg/parity. The two cases without a twin
there join it (a facet value keeps its case, batches stay apart), the one
thing only OpenSearch can do, refuse an unhealthy cluster, stays in its
package.
2026-08-28 19:38:01 +02:00
Dominik Schmidt 889483d69c test(search): document where the engines still differ from the parity suite
The expectations are the desired state; where bleve or OpenSearch answer
differently on main today the case records that answer as an engine
override, so the suite is green and the README lists every divergence as
known.
2026-08-28 19:38:01 +02:00
Dominik Schmidt 1668e23f80 test(search): port the parity suite to ginkgo
Every case is one spec per engine, so one engine failing leaves the other's
answer in the matrix. Query groups share their engines through an Ordered
container, operations build their own. The README is assembled from report
entries in ReportAfterSuite, which also works across ginkgo -p processes, and
is only written once every planned row got an answer from every engine.

A case can carry, per engine, what that engine answers today where it still
differs from the expectation (engineOverrides): the spec asserts the override,
the README marks the row as known, and the override fails once the engine
answers as expected.
2026-08-28 19:37:45 +02:00
Dominik Schmidt c24bfb2e41 test(search): keep the OpenSearch testcontainer between runs on request
KEEP_TEST_CONTAINER=true skips the terminate; the next run picks the
container up by name instead of booting a fresh one.
2026-08-28 19:20:58 +02:00
Dominik Schmidt a03a294a12 test(search): make the OpenSearch testcontainer start reliably
Cold boots take well over the 5s startup timeout, and a full host disk
tripped the flood-stage create-index block mid-run; test indexes are tiny.
2026-08-28 19:15:51 +02:00
Dominik Schmidt a3e40a7471 test(search): run the parity suite against the current main
Move the OpenSearch test helpers to services/search/internal/opensearchtest so
the parity package can import them, and drop the index generation and clock
patch the suite was written against, neither exists on main yet.
2026-08-28 19:15:50 +02:00
Florian Schade 5601be1276 test(search): introduce engine parity suite 2026-08-28 18:41:37 +02:00
Dominik Schmidt 03cfbacafa docs(adr): add concrete mtime/name backend-drift examples 2026-08-18 17:57:27 +02:00
Dominik Schmidt 2223b8ed9d docs(adr): include location.* in the OpenSearch facet sub-strings list
The bullet enumerating which OpenSearch facet sub-fields change
from text+keyword to keyword-only listed audio, photo and image but
omitted location, which is in the same dynamic-template-handled
facet block per the context section. Add it for consistency.
2026-08-18 17:57:27 +02:00
Dominik Schmidt 8053275166 docs(adr): add 0005 unified search index mapping
Propose a single central Go-struct + overrides map as the source of
truth for the search index layout across bleve and OpenSearch — the
same definition drives the per-backend index mapping, the write-time
adapter, the hit-decoding path, and the KQL compiler's case-folding
rules, so the two backends cannot drift silently again.

Also records the end-to-end case-handling principle for facet data
(indexed as case-preserving keywords so aggregation buckets return
correct display values), the sibling-field pattern for geopoint on
Location, and the rationale for replacing the two backends' implicit
defaults with an explicit, backend-agnostic contract.

PR #2659 is a proof-of-concept implementation the proposal emerged
from; scope and APIs there will be revisited once this ADR lands.
2026-08-18 17:57:26 +02:00
643 changed files with 34707 additions and 21134 deletions

No files matched your search

+6
View File
@@ -65,6 +65,12 @@ $(GOVULNCHECK): $(BINGO_DIR)/govulncheck.mod
@echo "(re)installing $(GOBIN)/govulncheck-v1.1.4"
@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=govulncheck.mod -o=$(GOBIN)/govulncheck-v1.1.4 "golang.org/x/vuln/cmd/govulncheck"
GOWRAP := $(GOBIN)/gowrap-v1.4.3
$(GOWRAP): $(BINGO_DIR)/gowrap.mod
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
@echo "(re)installing $(GOBIN)/gowrap-v1.4.3"
@cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=gowrap.mod -o=$(GOBIN)/gowrap-v1.4.3 "github.com/hexdigest/gowrap/cmd/gowrap"
MOCKERY := $(GOBIN)/mockery-v3.4.0
$(MOCKERY): $(BINGO_DIR)/mockery.mod
@# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies.
+5
View File
@@ -0,0 +1,5 @@
module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT
go 1.25.8
require github.com/hexdigest/gowrap v1.4.3 // cmd/gowrap
+55
View File
@@ -0,0 +1,55 @@
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hexdigest/gowrap v1.4.3 h1:m+t8aj1pUiFQbEiE8QJg2xdYVH5DAMluLgZ9P/qEF0k=
github.com/hexdigest/gowrap v1.4.3/go.mod h1:XWL8oQW2H3fX5ll8oT3Fduh4mt2H3cUAGQHQLMUbmG4=
github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/copystructure v1.1.2 h1:Th2TIvG1+6ma3e/0/bopBKohOTY7s4dA8V2q4EUcBJ0=
github.com/mitchellh/copystructure v1.1.2/go.mod h1:EBArHfARyrSWO/+Wyr9zwEkc6XMFB9XyNgFNmRkZZU4=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE=
github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA=
github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+2
View File
@@ -24,6 +24,8 @@ GOLANGCI_LINT="${GOBIN}/golangci-lint-v1.64.6"
GOVULNCHECK="${GOBIN}/govulncheck-v1.1.4"
GOWRAP="${GOBIN}/gowrap-v1.4.3"
MOCKERY="${GOBIN}/mockery-v3.4.0"
MUTAGEN="${GOBIN}/mutagen-v0.18.1"
+2
View File
@@ -18,6 +18,8 @@ exclude_paths:
- 'deployments/**'
- "release-config.ts"
- 'tests/acceptance/expected-failures-*.md'
# written by the search engine parity suite, table rows are as wide as they are
- 'services/search/pkg/parity/README.md'
- 'tests/acceptance/bootstrap/**'
- 'tests/acceptance/TestHelpers/**'
- 'tests/acceptance/scripts/run.sh'
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
set -euo pipefail
# Purpose of this script is to parse the go.mod file to retrieve the version of
# Go that we are using, either from a "toolchain" directive (preferred), or from
# a "go" directive as a fallback.
#
# The script then outputs that version, where it will be picked up as an environment
# variable in mise.
F="./go.mod"
AWK=awk
# prefer 'gawk' over 'awk' to make sure we get GNU awk
command -v gawk &>/dev/null && AWK=gawk
[[ -e $F ]] || { echo "ERROR: failed to find $F" >&2; exit 1;}
GO_VERSION=$("$AWK" '/^toolchain/ {print $2; exit} /^go / {print $2}' "$F")
GO_VERSION=${GO_VERSION#go} # strip the potential 'go' prefix:
echo "${GO_VERSION}"
+45 -13
View File
@@ -8,7 +8,7 @@ docker_repo_slug = "opencloudeu/opencloud"
# images
ALPINE_GIT = "alpine/git:latest"
APACHE_TIKA = "apache/tika:3.2.3.0-full"
APACHE_TIKA = "apache/tika:4.0.0-full"
CHKO_DOCKER_PUSHRM = "chko/docker-pushrm:1"
CODACY_COVERAGE_REPORTER = "codacy/codacy-coverage-reporter:14.1.3"
COLLABORA_CODE = "collabora/code:24.04.5.1.1"
@@ -202,14 +202,14 @@ config = {
"apiSearch1",
],
"skip": False,
"nightlyOpenSearch": True,
"openSearch": True,
},
"search2": {
"suites": [
"apiSearch2",
],
"skip": False,
"nightlyOpenSearch": True,
"openSearch": True,
},
"sharingNg": {
"suites": [
@@ -273,7 +273,7 @@ config = {
],
"skip": False,
"tikaNeeded": True,
"nightlyOpenSearch": True,
"openSearch": True,
},
"ocm": {
"suites": [
@@ -331,6 +331,8 @@ config = {
"ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310",
"OC_ASYNC_UPLOADS": True,
"OC_ADD_RUN_SERVICES": "antivirus",
"OC_SERVICE_ACCOUNT_ID": "service-account-id",
"OC_SERVICE_ACCOUNT_SECRET": "service-account-secret",
},
},
"multiTenancy": {
@@ -406,6 +408,16 @@ config = {
"embed",
],
},
# Runs the search-related e2e-tests against an OpenSearch search-engine.
"search": {
"skip": False,
"suites": [
"search/",
"file-action/favorites.feature",
],
"tikaNeeded": True,
"openSearch": True,
},
},
"e2eMultiService": {
"testSuites": {
@@ -1253,10 +1265,8 @@ def build_api_test_workflow_matrix(ctx, storage, suite_cfg, default_cfg):
if matrix not in workflow_metrices and base in matrices:
workflow_metrices.append(matrix)
# Add an OpenSearch search-engine variant for nightly running search tests,
# or on demand when "opensearch" is specified in the PR title
run_open_search = ctx.build.event == "cron" or "opensearch" in ctx.build.title.lower()
if run_open_search and storage == "posix" and suite_cfg.get("nightlyOpenSearch", False):
# Add an OpenSearch search-engine variant for the search test suites.
if storage == "posix" and suite_cfg.get("openSearch", False):
os_matrix = {
"withRemotePhp": False,
"enableWatchFs": False,
@@ -1284,7 +1294,7 @@ def localApiTestPipeline(ctx):
"withRemotePhp": False,
"enableWatchFs": False,
"ldapNeeded": False,
"nightlyOpenSearch": False,
"openSearch": False,
}
if "localApiTests" in config:
@@ -1530,6 +1540,7 @@ def e2eTestPipeline(ctx):
"reportTracing": False,
"enableWatchFs": [False],
"storages": ["posix"],
"openSearch": False,
}
extra_server_environment = {
@@ -1573,14 +1584,30 @@ def e2eTestPipeline(ctx):
if "[decomposed]" in ctx.build.title.lower():
params["storages"] = ["decomposed"]
e2e_args = " ".join(params["suites"]) if params["suites"] else ""
if "with-tracing" in ctx.build.title.lower():
params["reportTracing"] = True
e2e_args = " ".join(params["suites"]) if params["suites"] else ""
# OpenSearch search-engine suites run posix-only without the watchfs matrix.
run_with_open_search = params["openSearch"]
if run_with_open_search:
params["storages"] = ["posix"]
params["enableWatchFs"] = [False]
for storage in params["storages"]:
for watch_fs_enabled in params["enableWatchFs"]:
pipeline_name = "test-e2e-%s-%s%s" % (name, storage, "-watchfs" if watch_fs_enabled else "")
pipeline_name = "test-e2e-%s-%s%s%s" % (
name,
storage,
"-watchfs" if watch_fs_enabled else "",
"-opensearch" if run_with_open_search else "",
)
server_environment = dict(extra_server_environment)
if run_with_open_search:
server_environment["SEARCH_ENGINE_TYPE"] = "open-search"
server_environment["SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES"] = "http://open-search:9200"
steps = \
evaluateWorkflowStep() + \
@@ -1589,9 +1616,10 @@ def e2eTestPipeline(ctx):
restoreWebPnpmCache() + \
restoreBrowsersCache() + \
(tikaService() if params["tikaNeeded"] else []) + \
(waitForOpenSearch() if run_with_open_search else []) + \
opencloudServer(
storage,
extra_server_environment = extra_server_environment,
extra_server_environment = server_environment,
tika_enabled = params["tikaNeeded"],
watch_fs_enabled = watch_fs_enabled,
) + \
@@ -1618,6 +1646,7 @@ def e2eTestPipeline(ctx):
pipeline = {
"name": pipeline_name,
"steps": steps,
"services": openSearchService() if run_with_open_search else [],
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)),
"when": e2e_trigger,
}
@@ -3415,6 +3444,9 @@ def tikaService():
return [{
"name": "tika",
"image": APACHE_TIKA,
# tika 4 discovers its plugins relative to the image working directory,
# the workspace default would leave the pipes fetchers empty
"directory": "/opt/tika-server",
"detach": True,
}, {
"name": "wait-for-tika-service",
@@ -0,0 +1,226 @@
---
title: "5. Unified Search Index Mapping"
---
* Status: accepted
* Deciders: @aduffeck, @butonic, @dschmidt, @fschade
* Date: 2026-04-23, accepted and updated to the implemented state 2026-08-31
Reference: implemented by https://github.com/opencloud-eu/opencloud/pull/3345 (reflection-based mapping, search siblings, shared query lowering) and https://github.com/opencloud-eu/opencloud/pull/3197 (schema versioning and startup checks). https://github.com/opencloud-eu/opencloud/pull/2659 was the original proof-of-concept.
## Context and Problem Statement
This section describes the state at decision time (April 2026); the implementation has since resolved the problems listed here.
The search service supports two backends, bleve (embedded) and
OpenSearch (external). Each backend currently carries its own,
independently maintained description of the index layout:
- The bleve backend hand-builds a document mapping that explicitly
declares only Name, Tags, Favorites and Content. Everything else,
including the entire facet block (audio, image, photo, location),
is left to bleve's dynamic mapping.
- The OpenSearch backend ships a static JSON template that covers a
similar but not identical subset, plus a few OpenSearch-specific
primitives (path_hierarchy analyzer, wildcard MimeType). It does
not list the facet sub-fields either; they are produced by
OpenSearch's dynamic templating at first write.
- The graph DriveItem assembly path keeps its own private copy of a
reflection-based walker to turn CS3 ArbitraryMetadata back into
typed libregraph facets, parallel to the search service's
reflection helpers but maintained separately.
- The bleve KQL compiler keeps a hand-maintained set of field names
whose query values need to be pre-lowercased, with a comment that
literally says "Keep in sync with index.go".
The current implementation has three concrete problems:
1. **The two backends do not behave the same.** Both rely on their
own implicit defaults for fields that are not explicitly
declared. The inferred shapes differ: bleve produces keyword-
analyzed text, OpenSearch produces a `text + keyword` multi-field
with auto-detected dates. Nobody has written down which behavior
is the intended one. Two concrete instances surfaced while building
#2659:
- **mtime** is stored as an RFC3339 string. OpenSearch's dynamic
mapping auto-detects it as `date`; bleve leaves it `keyword`. So
`mtime:>...` is a chronological range on OpenSearch but a
lexicographic string compare on bleve.
- **name/tags**: bleve indexes a single lowercase token (exact or
wildcard match only); OpenSearch word-tokenizes, so a bare
`name:report` matches "My Report.txt" on OpenSearch but not on
bleve.
2. **Drift risk.** The OpenSearch JSON template is a subset of what
actually gets indexed. Even where it overlaps with the bleve
mapping it diverges on analyzer choices. Because the facet
fields were not reachable from user queries at the time (no dot
syntax in the KQL compilers, no facet exposure on the hit and
REPORT paths), the divergence has been invisible, but it would
surface the moment the first working cross-backend facet query
landed.
3. **Per-facet cost.** Adding a new facet (motionPhoto, etc.)
requires coordinated edits across the proto message, both backend
mappings, the bleve hit converters, the OpenSearch convert
closures, the search service's metadata persistence, the graph
DriveItem assembly, and the KQL compiler's lowercasing set. Most
of those edits are boilerplate following a copy-paste pattern.
Adding a genuinely new index capability (geopoint, wildcard,
...) means wiring it in at every one of those sites, and there
is no single place to hook a type-specific adapter.
### A note on backwards compatibility
That the facet fields were unreachable at decision time has a
useful corollary for this ADR: **changing the indexed shape of the
facet fields cannot break any existing client of the search
service**, because no client could successfully read them. The behavior changes
discussed below are therefore additive in a literal sense; nothing
that works today stops working as a result.
## Decision Drivers
* **Predictable OpenCloud API behavior independent of backend.**
Consumers of the search service should be able to rely on the
documented behavior of the API, not on which backend happens to
be configured. Today the same query can give different results
depending on whether bleve or OpenSearch is wired in (bleve's
dynamic default is `keyword`, exact match; OpenSearch's dynamic
default is `text + keyword`, also matches sub-tokens of a
string). That is backend-implementation leakage, and trying to
keep the two implicit defaults synchronized has not worked.
* Single source of truth for the indexed schema, so the two backends
cannot drift silently again.
* Reduce the per-facet cost so future facets (motionPhoto and
whatever comes next) can be added with minimal boilerplate.
* Establish a single place to hook index-type-specific behavior, so
a new capability needs to be implemented at most once per backend
and then becomes available for any field uniformly.
* A one-time reindex is an acceptable upgrade path. Both bleve and
OpenSearch store their mapping alongside the data; existing
indexes keep serving queries against their stored shape without
any automatic reshaping. Benefiting from the new behavior is done
by creating a fresh index and re-ingesting, which is the normal
reindex flow, rather than by inventing migration tooling.
## Considered Options
### Option 1: Do nothing, keep relying on implicit backend defaults
Accept that bleve and OpenSearch each fall back to their own
dynamic-mapping defaults for whatever is not explicitly declared,
and treat the observable search behavior of OpenCloud as "whatever
the configured backend happens to do". Adding a facet stays a
copy-paste coordination across half a dozen sites; the existing
divergence between bleve (keyword) and OpenSearch (`text + keyword`
multi-field plus auto-date detection) stays silently in place
until a working query actually reaches the diverging field and
returns different answers on the two backends.
Low upfront work, but it makes the OpenCloud API behavior a
function of the backend rather than a contract, and it keeps the
per-facet boilerplate cost for every new field.
### Option 2: Generate one backend's mapping from the other
Treat one backend as canonical (likely bleve, because Go types) and
derive the other. Partial answer; it still does not help the reader
path or the graph walker, and still leaves per-facet boilerplate in
non-mapping code.
### Option 3: A struct-driven mapping (chosen)
Let the Go struct that represents an indexed document, together
with a small overrides map, be the single source of truth. A
reflection-based helper walks the struct via json tags and emits
each backend's index mapping. The same definition drives the
write-time path, the hit-decoding path, and the query compiler's
case-folding rules. Any future field follows one declaration in
one place and falls through the whole pipeline consistently.
## Decision Outcome
Adopt Option 3. The Go struct that represents an indexed document,
together with a small overrides map, becomes the single source of
truth for the search index. The bleve and OpenSearch index
mappings, the write-time conversion, the hit-decoding path, and
the query compiler's case-folding rules are all derived from that
same definition. Drift between backends is prevented by
construction, because there is no second place to edit.
The overrides surface stays small. Each entry declares one of a
handful of things per field: a semantic type for fields whose
intent cannot be inferred from the Go type (for example a path-
analyzed field, a fulltext field, a geopoint field), or search-
behavior flags (case-insensitivity, word breaking, inclusion in
the catch-all field). Any field that needs something beyond the
inferred defaults gets one line in the overrides map and that one
line flows through every derived piece. Overrides are validated at
startup so a typo fails loudly instead of silently disabling a
setting.
A practical consequence of having one place to hook things: when a
new capability is needed (a geopoint representation, a sibling
field for a different aggregation behavior, a different analyzer
for a class of fields, ...) it can be implemented once per backend
in the central pipeline. After that, turning the capability on for
a specific field is a single override entry, and both backends
adopt it the same way. This ADR does not decide which capabilities
to add, only that they will land in this uniform shape rather than
through coordinated per-site edits.
### Facet values are indexed as case-preserving keywords
All facet sub-fields, meaning any leaf inside `audio`, `photo`, `image`, `location` and the facets that followed (`video`, `motionPhoto`, `livePhoto`), keep a case-preserving keyword as their stored base field on both backends. The raw value the extractor saw, or the CS3 ArbitraryMetadata string, is what lands in the index, and it is what returning, sorting and aggregations read.
This is the single intended semantic for facets across bleve and
OpenSearch, and it is driven by what aggregations need.
Aggregation buckets ("group all files by `audio.artist`", "list
distinct `photo.cameraMake`") return bucket keys drawn from the
indexed terms. If the indexing analyzer lowercases (OpenSearch's
default `text + keyword` multi-field against the text leg, or a
`lowercaseKeyword`-style analyzer), the buckets come back lower-
cased: a distinct-artists query would answer `motörhead` and
`queen` instead of the original display casings, and two tag
writers using `Motörhead` versus `MOTÖRHEAD` would collapse into a
single bucket labelled `motörhead`. For a metadata display use
case (thumbnails, facet filters in the UI, distinct lists) that
behavior is not what we want.
Searching is layered on top as exactly the strict superset the proposal reserved for later, and it shipped with the implementation: every keyword field additionally gets search-only sibling fields derived from the same definition, a `_lowercase` keyword sibling (doc values disabled; serves wildcards and `=` whole-value matches) and a `_words` text sibling (`words` analyzer: dots to spaces, unicode tokenization, lowercasing, no stemming; serves token and phrase matches). Case-insensitive, word-broken search is the default for every keyword field including facets; fields opt out per override where that is wrong: opaque ids (`ID`, `RootID`, `ParentID`, `Favorites`, `livePhoto.contentId`), the POSIX `Path`, the normalized `MimeType`, and `Content`, which is a fulltext field of its own. Aggregation buckets keep their display casing because they read the base field, never the siblings.
The query side derives from the same source: the shared lowering pass resolves field names case-insensitively, folds values and routes each match to the right sibling (wildcards to `_lowercase`, tokens and phrases to `_words`, `=` as a whole-value term on `_lowercase`), and both backend compilers consume that one decision. The engine parity suite pins the resulting behavior against bleve and OpenSearch, so a divergence fails CI instead of surfacing in production. The case-sensitivity alignment started in #2633 is completed by deriving both sides from the same source.
### Schema versioning and upgrades
Index names carry a schema version derived from the single `search.SchemaVersion` constant (`opencloud-resource-v4`, `bleve-v4`). On startup the service classifies the stored mapping against the code: additive changes (new fields, unchanged analyzers) are reconciled in place without a version bump, breaking changes make the service refuse to start and name the reindex steps. The upgrade path is a plain reindex (`opencloud search index --all-spaces`) into the new versioned index; older indexes stay untouched and can be deleted afterwards (services/search/MIGRATION.md). Golden mapping tests on both backends pin the rendered mappings and reuse the same classifier to tell a contributor whether a change needs only a golden regeneration or a version bump.
### Known trade-off
The write-time pipeline produces the document as a generic map via
a json round-trip. The OpenSearch write path already does the
equivalent today via the same json-based conversion helper, so
that path is unchanged. The bleve write path, which previously
handed the struct directly to bleve's reflective indexer, now goes
through the same map-producing step and pays roughly the same
cost. On hot paths (initial indexing of a large space) this is
measurable but not significant; if it ever matters, a direct
reflection walker can replace the json round-trip without changing
any call site.
### Follow-ups out of scope for this ADR
- **WebDAV REPORT facet exposure.** The current webdav search
endpoint renders none of the facet fields back to the client.
This is a missing feature, not a regression of the proposal;
its natural resolution is to let the graph-search endpoint
(proposed in #3211) take over once graph search lands.
- **Graph search hit conversion.** Graph search (#3211) translates
proto hits back into libregraph DriveItems with the same
facet-copy helper the search service uses internally.
- **reva's PROPFIND facet listing** uses its own hand-maintained
per-facet key lists. reva deliberately does not depend on the
libregraph Go types, so unifying those key sets is a reva-side
decision tracked separately.
- **Write-path performance.** The json round-trip in the bleve
write path is an optional optimisation target with no call-site
impact when it lands.
+18 -19
View File
@@ -11,7 +11,7 @@ require (
github.com/Nerzal/gocloak/v13 v13.9.0
github.com/bbalet/stopwords v1.0.0
github.com/beevik/etree v1.7.1
github.com/blevesearch/bleve/v2 v2.6.0
github.com/blevesearch/bleve/v2 v2.6.1
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/coreos/go-oidc/v3 v3.20.0
github.com/cs3org/go-cs3apis v0.0.0-20260424072047-8d9ef7076ae9
@@ -22,7 +22,6 @@ require (
github.com/ggwhite/go-masker v1.1.0
github.com/go-chi/chi/v5 v5.3.2
github.com/go-chi/render v1.0.3
github.com/go-jose/go-jose/v3 v3.0.5
github.com/go-ldap/ldap/v3 v3.4.14
github.com/go-ldap/ldif v0.0.0-20200320164324-fd88d9b715b3
github.com/go-micro/plugins/v4/client/grpc v1.2.1
@@ -63,20 +62,21 @@ require (
github.com/onsi/gomega v1.42.1
github.com/open-policy-agent/opa v1.19.1
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260818063016-69f82a7dde55
github.com/opencloud-eu/reva/v2 v2.49.0
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067
github.com/opencloud-eu/reva/v2 v2.49.1-0.20260903122659-26f34ec05774
github.com/opensearch-project/opensearch-go/v4 v4.7.3
github.com/orcaman/concurrent-map v1.0.0
github.com/pkg/errors v0.9.1
github.com/pkg/xattr v0.4.12
github.com/prometheus/client_golang v1.24.1
github.com/prometheus/client_model v0.6.2
github.com/r3labs/sse/v2 v2.10.0
github.com/riandyrn/otelchi v0.12.3
github.com/rogpeppe/go-internal v1.16.0
github.com/rs/cors v1.11.1
github.com/rs/zerolog v1.35.1
github.com/shamaton/msgpack/v2 v2.4.1
github.com/sirupsen/logrus v1.10.0
github.com/sirupsen/logrus v1.10.1
github.com/spf13/afero v1.15.0
github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10
@@ -95,13 +95,13 @@ require (
go-micro.dev/v4 v4.11.0
go.etcd.io/bbolt v1.5.0
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
go.opentelemetry.io/contrib/zpages v0.70.0
go.opentelemetry.io/otel v1.45.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0
go.opentelemetry.io/contrib/zpages v0.71.0
go.opentelemetry.io/otel v1.46.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0
go.opentelemetry.io/otel/sdk v1.45.0
go.opentelemetry.io/otel/trace v1.45.0
go.opentelemetry.io/otel/sdk v1.46.0
go.opentelemetry.io/otel/trace v1.46.0
golang.org/x/crypto v0.55.0
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
golang.org/x/image v0.45.0
@@ -111,7 +111,7 @@ require (
golang.org/x/term v0.45.0
golang.org/x/text v0.41.0
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d
google.golang.org/grpc v1.83.1
google.golang.org/grpc v1.83.2
google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
@@ -141,13 +141,13 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/bitly/go-simplejson v0.5.0 // indirect
github.com/bits-and-blooms/bitset v1.24.2 // indirect
github.com/blevesearch/bleve_index_api v1.3.11 // indirect
github.com/blevesearch/geo v0.2.5 // indirect
github.com/blevesearch/go-faiss v1.1.0 // indirect
github.com/blevesearch/bleve_index_api v1.4.1 // indirect
github.com/blevesearch/geo v0.2.6 // indirect
github.com/blevesearch/go-faiss v1.1.5 // indirect
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
github.com/blevesearch/gtreap v0.1.1 // indirect
github.com/blevesearch/mmap-go v1.2.0 // indirect
github.com/blevesearch/scorch_segment_api/v2 v2.4.7 // indirect
github.com/blevesearch/scorch_segment_api/v2 v2.4.10 // indirect
github.com/blevesearch/segment v0.9.1 // indirect
github.com/blevesearch/snowballstem v0.9.0 // indirect
github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
@@ -158,7 +158,7 @@ require (
github.com/blevesearch/zapx/v14 v14.4.3 // indirect
github.com/blevesearch/zapx/v15 v15.4.3 // indirect
github.com/blevesearch/zapx/v16 v16.3.4 // indirect
github.com/blevesearch/zapx/v17 v17.1.2 // indirect
github.com/blevesearch/zapx/v17 v17.2.3 // indirect
github.com/bluele/gcache v0.0.2 // indirect
github.com/bombsimon/logrusr/v3 v3.1.0 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -207,6 +207,7 @@ require (
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.9.0 // indirect
github.com/go-git/go-git/v5 v5.19.2 // indirect
github.com/go-jose/go-jose/v3 v3.0.5 // indirect
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
@@ -323,8 +324,6 @@ require (
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/pquerna/cachecontrol v0.2.0 // indirect
github.com/prometheus/alertmanager v0.33.1 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/prometheus/statsd_exporter v0.22.8 // indirect
@@ -378,7 +377,7 @@ require (
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
go.opentelemetry.io/otel/metric v1.45.0 // indirect
go.opentelemetry.io/otel/metric v1.46.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
+34 -36
View File
@@ -146,22 +146,22 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE
github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0=
github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/blevesearch/bleve/v2 v2.6.0 h1:Cyd3dd4q5tCbOV8MnKUVRUDYMHOir9xn12NZzXVSEd4=
github.com/blevesearch/bleve/v2 v2.6.0/go.mod h1:gLmI8lWgHgrIYf7UpUX7JISI1CaqC6VScu46mHThuAY=
github.com/blevesearch/bleve_index_api v1.3.11 h1:x29vbV8OjWfLcrDVd7Lr1q+BkLNS0JWNEig0MCVnKH4=
github.com/blevesearch/bleve_index_api v1.3.11/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko=
github.com/blevesearch/geo v0.2.5 h1:yJg9FX1oRwLnjXSXF+ECHfXFTF4diF02Ca/qUGVjJhE=
github.com/blevesearch/geo v0.2.5/go.mod h1:Jhq7WE2K6mJTx1xS44M2pUO6Io+wjCSHh1+co3YOgH4=
github.com/blevesearch/go-faiss v1.1.0 h1:xM7Jc0ZUCv5lssG9Ohj3Jv0SdTpxcUABU1dDt9XVsc4=
github.com/blevesearch/go-faiss v1.1.0/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk=
github.com/blevesearch/bleve/v2 v2.6.1 h1:47vLskRTqxvQEtxVPYHjf5KpOgzD2msslXFjvUQCgWQ=
github.com/blevesearch/bleve/v2 v2.6.1/go.mod h1:Dvvx6ZoEBTOj6RSzfk0lEz0wce/qhe2yOUubXeuzd2c=
github.com/blevesearch/bleve_index_api v1.4.1 h1:CYIyecFlI+/RYjzUm+NmDjYbSvk870Bb7f+Vl4b12q8=
github.com/blevesearch/bleve_index_api v1.4.1/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko=
github.com/blevesearch/geo v0.2.6 h1:7K1oyQKYlauC+mJuo2AfNPyjN/4mihEoJMfyClVH1Mo=
github.com/blevesearch/geo v0.2.6/go.mod h1:6qzVUiB4BK47QkSZcRqiXEP2W3EeXuzM5XFTF8AdZ8A=
github.com/blevesearch/go-faiss v1.1.5 h1:/IU5lkOahH9Ghfk9n3F6N0XD7PYVXZJWmNDc9TtXuco=
github.com/blevesearch/go-faiss v1.1.5/go.mod h1:w3W9AiWsFRGVaMG+/cmJi7iHEAuGyC6blsgO1EzCK/M=
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
github.com/blevesearch/mmap-go v1.2.0 h1:l33nNKPFcBjJUMwem6sAYJPUzhUCABoK9FxZDGiFNBI=
github.com/blevesearch/mmap-go v1.2.0/go.mod h1:Vd6+20GBhEdwJnU1Xohgt88XCD/CTWcqbCNxkZpyBo0=
github.com/blevesearch/scorch_segment_api/v2 v2.4.7 h1:GlMzW08hcsM3DnLUxhyF/1PcDal1qtvvIuytuph5djw=
github.com/blevesearch/scorch_segment_api/v2 v2.4.7/go.mod h1://IJ7tG3QCf0cWW/aVSXqy77tc1AvLu3fcJLYEvOAFs=
github.com/blevesearch/scorch_segment_api/v2 v2.4.10 h1:C3873+iWZ0YJM2ijaSHhJJzSvD4x1k+5UaQdGygZVhM=
github.com/blevesearch/scorch_segment_api/v2 v2.4.10/go.mod h1:WUUkAocbkDlNK/kgAE13NvS9oxe+u618mYZ8sOvcCc4=
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
@@ -182,8 +182,8 @@ github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2l
github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw=
github.com/blevesearch/zapx/v16 v16.3.4 h1:hDAqA8qusZTNbPEL7//w5P65UZ2de6yhSeUaTbp0Po0=
github.com/blevesearch/zapx/v16 v16.3.4/go.mod h1:zqkPPqs9GS9FzVWzCO3Wf1X044yWAV17+4zb+FTiEHg=
github.com/blevesearch/zapx/v17 v17.1.2 h1:avbOk2igaASNoiy0BE/jPgcxAnRI2PGeydeP4hg7Ikk=
github.com/blevesearch/zapx/v17 v17.1.2/go.mod h1:WQObxKrqUX7cd0G1GMvDfc/bmZzQvoy7APOPimx7DiI=
github.com/blevesearch/zapx/v17 v17.2.3 h1:UYYJPAt5b2tVxldx5h0jmv23RMsg8/UZKFVya7v92po=
github.com/blevesearch/zapx/v17 v17.2.3/go.mod h1:r7mb4QWbDQSkbAnOjCb9iCfkcrzajB4yBdJpuBIo/fE=
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
@@ -940,10 +940,10 @@ github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-202505121527
github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-20250512152754-23325793059a/go.mod h1:pjcozWijkNPbEtX5SIQaxEW/h8VAVZYTLx+70bmB3LY=
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+lP5lUUIzjRGDg93WrQfZJZCaV1ZP3KeyXi8bzY=
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260818063016-69f82a7dde55 h1:dzYZ5iA6i0QoFUap9l6sZ4Ts2HXiXVfsOoiRYToYgm8=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260818063016-69f82a7dde55/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU=
github.com/opencloud-eu/reva/v2 v2.49.0 h1:AwECMDDth3NUaihZRf9bI9HNpWutvaRijOCvQyQfTT4=
github.com/opencloud-eu/reva/v2 v2.49.0/go.mod h1:Frg+UWnVcSy+412UB3l2LcD0KY8ZNu1samimwkNywbg=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067 h1:UkNMKauyJAzY6RE6mmthz9bQZLYkbvBuApm7ZDCparE=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU=
github.com/opencloud-eu/reva/v2 v2.49.1-0.20260903122659-26f34ec05774 h1:8N42Pwzyafdxq66zYP6kJ1CH/QBRpxovQXq/0Kt6bJY=
github.com/opencloud-eu/reva/v2 v2.49.1-0.20260903122659-26f34ec05774/go.mod h1:Frg+UWnVcSy+412UB3l2LcD0KY8ZNu1samimwkNywbg=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -1001,8 +1001,6 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:Om
github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k=
github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI=
github.com/pquerna/otp v1.3.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
github.com/prometheus/alertmanager v0.33.1 h1:PJHGvTdb8Q0ZEpJnWff120WxB2kxIScLa707AaIudTo=
github.com/prometheus/alertmanager v0.33.1/go.mod h1:V06Uc8EZ5X5wLOJRGhtXx+EE2LgrinFIADbKWMVm1RY=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
@@ -1115,8 +1113,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.10.0 h1:T8MxJJXVZkfcC5zSRMRAg2F8+lxjmUCGGWPzFxO+Msc=
github.com/sirupsen/logrus v1.10.0/go.mod h1:FXZFonkDAnFozmO+5hGAFvB0Yg9/j2SIhA/QuIkP180=
github.com/sirupsen/logrus v1.10.1 h1:xi4336Zh11WpU14fXR6I67V3yaTPQYwRx2WEtHbRg4Q=
github.com/sirupsen/logrus v1.10.1/go.mod h1:vsQHnG7xzNsxk3NrwboUiWPnIC3dmbjcGPykD7+tiHk=
github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8=
github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=
github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
@@ -1295,26 +1293,26 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
go.opentelemetry.io/contrib/zpages v0.70.0 h1:uBcclHekIrCRLO4KZfXiebDcWB52QQXaHGgvhUJ1K5I=
go.opentelemetry.io/contrib/zpages v0.70.0/go.mod h1:Nz9A1+68HVoJrk+cg92zJbcbHFAa+Jbl1uINwAlcK94=
go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU=
go.opentelemetry.io/contrib/zpages v0.71.0 h1:OrQSEmdlfknIusEyGj/t30uBR7MnbmGcCRBt5ov8/rA=
go.opentelemetry.io/contrib/zpages v0.71.0/go.mod h1:C3IIpqkQbjswcSREqxQLyEapF0SI9bqwJpxpjwtzFrE=
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0 h1:lsA/S1bxgdbyFGkTj+3meEdJ6ADVU7QoFstV6MXgE68=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.45.0/go.mod h1:L7u+MirGoB1bjeLH66+xDykF4RC8C3RN7lIFpBiewUo=
go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -1723,8 +1721,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y=
google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
google.golang.org/grpc/examples v0.0.0-20211102180624-670c133e568e h1:m7aQHHqd0q89mRwhwS9Bx2rjyl/hsFAeta+uGrHsQaU=
google.golang.org/grpc/examples v0.0.0-20211102180624-670c133e568e/go.mod h1:gID3PKrg7pWKntu9Ss6zTLJ0ttC0X9IHgREOCZwbCVU=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
@@ -0,0 +1,44 @@
package eventstest
import (
"encoding/json"
"reflect"
"github.com/google/uuid"
rev "github.com/opencloud-eu/reva/v2/pkg/events"
microevents "go-micro.dev/v4/events"
)
func NewTestBus() TestBus {
return TestBus(make(chan rev.Event))
}
type TestBus chan rev.Event
func (tb TestBus) Consume(_ string, _ ...microevents.ConsumeOption) (<-chan microevents.Event, error) {
ch := make(chan microevents.Event)
go func() {
for ev := range tb {
b, _ := json.Marshal(ev.Event)
ch <- microevents.Event{
Payload: b,
Metadata: map[string]string{
rev.MetadatakeyEventID: ev.ID,
rev.MetadatakeyEventType: ev.Type,
},
}
}
}()
return ch, nil
}
func (tb TestBus) Publish(e any) string {
ev := rev.Event{
ID: uuid.New().String(),
Type: reflect.TypeOf(e).String(),
Event: e,
}
tb <- ev
return ev.ID
}
@@ -0,0 +1,143 @@
package metricstest
import (
"fmt"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// copied and adapted from Prometheus testutil.ToFloat64(), since we don't import that package
func collect(c prometheus.Collector) []prometheus.Metric {
result := []prometheus.Metric{}
ch := make(chan prometheus.Metric)
done := make(chan struct{})
go func() {
for m := range ch {
result = append(result, m)
}
close(done)
}()
c.Collect(ch)
close(ch)
<-done
return result
}
func RequireIsNotSet(t require.TestingT, c prometheus.Collector, msgAndArgs ...any) {
if h, ok := t.(interface{ Helper() }); ok {
h.Helper()
}
if !IsNotSet(t, c, msgAndArgs) {
t.FailNow()
}
}
func IsNotSet(t assert.TestingT, c prometheus.Collector, msgAndArgs ...any) bool {
if h, ok := t.(interface{ Helper() }); ok {
h.Helper()
}
m := collect(c)
if len(m) > 0 {
return assert.Fail(t, "Metric exists while expected to not exist", msgAndArgs)
} else {
return true
}
}
func RequireEqual(t require.TestingT, expected float64, c prometheus.Collector, msgAndArgs ...any) {
if h, ok := t.(interface{ Helper() }); ok {
h.Helper()
}
if !Equal(t, expected, c, msgAndArgs) {
t.FailNow()
}
}
// copied and adapted from Prometheus testutil.ToFloat64(), since we don't import that package
func Equal(t assert.TestingT, expected float64, c prometheus.Collector, msgAndArgs ...any) bool {
if h, ok := t.(interface{ Helper() }); ok {
h.Helper()
}
m := collect(c)
if !assert.Len(t, m, 1, msgAndArgs...) {
return false
}
pb := &dto.Metric{}
err := m[0].Write(pb)
if !assert.NoError(t, err, msgAndArgs...) {
return false
}
if pb.Gauge != nil {
return assert.Equal(t, expected, pb.Gauge.GetValue(), msgAndArgs...)
} else if pb.Counter != nil {
return assert.Equal(t, expected, pb.Counter.GetValue(), msgAndArgs...)
} else if pb.Untyped != nil {
return assert.Equal(t, expected, pb.Untyped.GetValue(), msgAndArgs...)
} else {
return assert.Fail(t, fmt.Sprintf("collected a non-gauge/counter/untyped metric: %s", pb), msgAndArgs...)
}
}
func RequireEqualWithLabels(t require.TestingT, expectedValue float64, expectedLabels map[string]string, c prometheus.Collector, msgAndArgs ...any) {
if h, ok := t.(interface{ Helper() }); ok {
h.Helper()
}
if !EqualWithLabels(t, expectedValue, expectedLabels, c, msgAndArgs) {
t.FailNow()
}
}
func EqualWithLabels(t assert.TestingT, expectedValue float64, expectedLabels map[string]string, c prometheus.Collector, msgAndArgs ...any) bool {
if h, ok := t.(interface{ Helper() }); ok {
h.Helper()
}
m := collect(c)
if !assert.Len(t, m, 1, "collected %d metrics instead of exactly 1", len(m)) {
return false
}
pb := &dto.Metric{}
err := m[0].Write(pb)
if !assert.NoError(t, err) {
return false
}
if pb.Gauge != nil {
if !assert.Equal(t, expectedValue, pb.Gauge.GetValue()) {
return false
}
} else if pb.Counter != nil {
if !assert.Equal(t, expectedValue, pb.Counter.GetValue()) {
return false
}
} else if pb.Untyped != nil {
if !assert.Equal(t, expectedValue, pb.Untyped.GetValue()) {
return false
}
} else {
return assert.Fail(t, "collected a non-gauge/counter/untyped metric: %s", pb)
}
if !assert.NotNil(t, pb.Label) {
return false
}
actualLabels := map[string]string{}
for _, label := range pb.Label {
if !assert.NotNil(t, label) {
return false
}
if !assert.NotNil(t, label.Name) {
return false
}
if !assert.NotNil(t, label.Value) {
return false
}
actualLabels[*label.Name] = *label.Value
}
return assert.Equal(t, expectedLabels, actualLabels, msgAndArgs)
}
+7 -2
View File
@@ -1,9 +1,14 @@
[env]
GO_VERSION = "{{ exec(command='./.mise-go-version.sh') }}"
[tools]
go = "1.25"
go = "{{ env.GO_VERSION }}"
node = "24"
pnpm = "11.1.3"
"go:github.com/go-delve/delve/cmd/dlv" = "1.27.0"
"go:github.com/go-delve/delve/cmd/dlv" = "1.27.1"
"aqua:nats-io/natscli" = "0.4.0"
k6 = "2.2.0"
ginkgo = "latest"
[tasks.build]
description = "build"
+46 -17
View File
@@ -25,6 +25,8 @@ var (
_nodesGlobPattern = "spaces/*/*/nodes/"
)
const posixDriver = "posix"
// RevisionsCommand is the entrypoint for the revisions command.
func RevisionsCommand(cfg *config.Config) *cobra.Command {
revCmd := &cobra.Command{
@@ -86,23 +88,28 @@ func PurgeRevisionsCommand(cfg *config.Config) *cobra.Command {
mechanism = "glob"
}
var posix = cfg.StorageUsers.Driver == posixDriver
var ch <-chan string
switch mechanism {
default:
fallthrough
case "glob":
p := generatePath(basePath, rid)
p := generatePath(basePath, rid, posix)
if rid.GetOpaqueId() == "" {
p = filepath.Join(p, "*/*/*/*/*")
}
ch = revisions.Glob(p)
case "workers":
p := generatePath(basePath, rid)
p := generatePath(basePath, rid, posix)
ch = revisions.GlobWorkers(p, "/*", "/*/*/*/*")
case "list":
p := filepath.Join(basePath, "spaces")
p := basePath
if !posix {
p = filepath.Join(basePath, "spaces")
}
if rid != nil {
p = generatePath(basePath, rid)
p = generatePath(basePath, rid, posix)
}
ch = revisions.List(p, 10)
}
@@ -144,22 +151,44 @@ func printResults(countFiles, countBlobs, countRevisions int, dryRun bool) {
}
}
func generatePath(basePath string, rid *provider.ResourceId) string {
if rid == nil {
return filepath.Join(basePath, _nodesGlobPattern)
}
func generatePath(basePath string, rid *provider.ResourceId, posix bool) string {
// decomposedfs and posix store the revisions of a node at different
// locations on disk, so the path has to be built per driver:
// - decomposedfs: <basePath>/spaces/<pathified spaceID>/nodes/<pathified nodeID>.REV.<ts>
// - posix: <basePath>/<users|projects>/<spaceID>/.oc-nodes/<pathified nodeID>.REV.<ts>
if posix {
if rid == nil {
return filepath.Join(basePath, "*", "*", ".oc-nodes")
}
sid := lookup.Pathify(rid.GetSpaceId(), 1, 2)
if sid == "" {
return ""
}
nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2)
if nid != "" {
return filepath.Join(basePath, "*", "*", ".oc-nodes", nid+"*")
}
nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2)
if nid == "" {
return filepath.Join(basePath, "spaces", sid, "nodes")
}
if rid.GetSpaceId() == "" {
return ""
}
return filepath.Join(basePath, "*", rid.GetSpaceId(), ".oc-nodes")
} else {
// decomposedfs
if rid == nil {
return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*")
return filepath.Join(basePath, _nodesGlobPattern)
}
sid := lookup.Pathify(rid.GetSpaceId(), 1, 2)
if sid == "" {
return ""
}
nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2)
if nid == "" {
return filepath.Join(basePath, "spaces", sid, "nodes")
}
return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*")
}
}
func init() {
+22 -6
View File
@@ -1,7 +1,9 @@
package command
import (
"context"
"errors"
"time"
"github.com/spf13/viper"
@@ -21,6 +23,14 @@ import (
"github.com/spf13/cobra"
)
// need to be discussed, for now I will let it here
//
// Problem:
// in reva on CleanupStaleShares call there is migrations invokation, which at leat in tests takes some time,
// reva code was modified to wait until migrations are done, to prevent cases when migrations are stuck and the
// this executions is not returned this timeout is needed
const cleanupTimeout = 1 * time.Minute
// SharesCommand is the entrypoint for the groups command.
func SharesCommand(cfg *config.Config) *cobra.Command {
sharesCmd := &cobra.Command{
@@ -120,7 +130,11 @@ func cleanup(_ *cobra.Command, cfg *config.Config) error {
}
serviceUserCtx = l.WithContext(serviceUserCtx)
mgr.(*jsoncs3.Manager).CleanupStaleShares(serviceUserCtx)
cleanupCtx, cancel := context.WithTimeout(serviceUserCtx, cleanupTimeout)
defer cancel()
if err := mgr.(*jsoncs3.Manager).CleanupStaleShares(cleanupCtx); err != nil {
return configlog.ReturnError(err)
}
return nil
}
@@ -158,11 +172,13 @@ func revaShareConfig(cfg *sharing.Config) map[string]any {
"machine_auth_apikey": cfg.UserSharingDrivers.CS3.SystemUserAPIKey,
},
"jsoncs3": map[string]any{
"gateway_addr": cfg.Reva.Address,
"provider_addr": cfg.UserSharingDrivers.JSONCS3.ProviderAddr,
"service_user_id": cfg.UserSharingDrivers.JSONCS3.SystemUserID,
"service_user_idp": cfg.UserSharingDrivers.JSONCS3.SystemUserIDP,
"machine_auth_apikey": cfg.UserSharingDrivers.JSONCS3.SystemUserAPIKey,
"gateway_addr": cfg.Reva.Address,
"provider_addr": cfg.UserSharingDrivers.JSONCS3.ProviderAddr,
"system_user_id": cfg.UserSharingDrivers.JSONCS3.SystemUserID,
"system_user_idp": cfg.UserSharingDrivers.JSONCS3.SystemUserIDP,
"machine_auth_apikey": cfg.UserSharingDrivers.JSONCS3.SystemUserAPIKey,
"service_account_id": cfg.ServiceAccount.ServiceAccountID,
"service_account_secret": cfg.ServiceAccount.ServiceAccountSecret,
},
}
}
+2 -1
View File
@@ -36,7 +36,8 @@ func TrashPurgeEmptyDirsCommand(cfg *config.Config) *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) error {
basePath, _ := cmd.Flags().GetString("basepath")
dryRun, _ := cmd.Flags().GetBool("dry-run")
if err := trash.PurgeTrashEmptyPaths(basePath, dryRun); err != nil {
posix := cfg.StorageUsers.Driver == posixDriver
if err := trash.PurgeTrashEmptyPaths(basePath, dryRun, posix); err != nil {
fmt.Println(err)
return err
}
+45 -6
View File
@@ -10,12 +10,23 @@ import (
const (
// _trashGlobPattern is the glob pattern to find all trash items
_trashGlobPattern = "spaces/*/*/trash/*/*/*/*"
// _trashRootPattern is the glob pattern of the trash container root
_trashRootPattern = "spaces/*/*/trash"
// _posixTrashGlobPattern is the glob pattern to find all trash items on posix
_posixTrashGlobPattern = "*/*/.Trash/files/*"
// _posixTrashRootPattern is the glob pattern of the trash container root on posix
_posixTrashRootPattern = "*/*/.Trash/files"
)
// PurgeTrashEmptyPaths purges empty paths in the trash
func PurgeTrashEmptyPaths(p string, dryRun bool) error {
func PurgeTrashEmptyPaths(p string, dryRun bool, posix bool) error {
pattern := _trashGlobPattern
if posix {
pattern = _posixTrashGlobPattern
}
// we have all trash nodes in all spaces now
dirs, err := filepath.Glob(filepath.Join(p, _trashGlobPattern))
dirs, err := filepath.Glob(filepath.Join(p, pattern))
if err != nil {
return err
}
@@ -25,15 +36,26 @@ func PurgeTrashEmptyPaths(p string, dryRun bool) error {
}
for _, d := range dirs {
if err := removeEmptyFolder(d, dryRun); err != nil {
if err := removeEmptyFolder(d, dryRun, posix, p); err != nil {
return err
}
}
return nil
}
func removeEmptyFolder(path string, dryRun bool) error {
func removeEmptyFolder(path string, dryRun bool, posix bool, basePath string) error {
if dryRun {
if posix {
// on posix the ".trashitem" entries are the actual data and can be
// files, which are skipped, so we need to check here if the path
// is the actual dir, the same check for real removal part
fi, err := os.Stat(path)
if err != nil || !fi.IsDir() {
return nil
}
}
f, err := os.ReadDir(path)
if err != nil {
return err
@@ -43,6 +65,14 @@ func removeEmptyFolder(path string, dryRun bool) error {
}
return nil
}
if posix {
fi, err := os.Stat(path)
if err != nil || !fi.IsDir() {
return nil
}
}
if err := os.Remove(path); err != nil {
// we do not really care about the error here
// if the folder is not empty we will get an error,
@@ -50,8 +80,17 @@ func removeEmptyFolder(path string, dryRun bool) error {
return nil
}
nd := filepath.Dir(path)
if filepath.Base(nd) == "trash" {
if isTrashRoot(nd, basePath, posix) {
return nil
}
return removeEmptyFolder(nd, dryRun)
return removeEmptyFolder(nd, dryRun, posix, basePath)
}
func isTrashRoot(path, basePath string, posix bool) bool {
rootPattern := _trashRootPattern
if posix {
rootPattern = _posixTrashRootPattern
}
matched, _ := filepath.Match(filepath.Join(basePath, rootPattern), path)
return matched
}
+109
View File
@@ -0,0 +1,109 @@
package trash
import (
"os"
"testing"
"github.com/test-go/testify/require"
)
func TestIsTrashRootDecomposed(t *testing.T) {
storageRoot := "test_temp_" + t.Name()
defer os.RemoveAll(storageRoot)
require.True(t, isTrashRoot(storageRoot+"/spaces/id/id/trash", storageRoot, false))
require.False(t, isTrashRoot(storageRoot+"/spaces/id/id/trash/s1/s2/s3/node", storageRoot, false))
require.False(t, isTrashRoot(storageRoot+"/spaces/id/id/trash/s1/s2/trash/node", storageRoot, false))
}
func TestIsTrashRootPosix(t *testing.T) {
storageRoot := "test_temp_" + t.Name()
defer os.RemoveAll(storageRoot)
require.True(t, isTrashRoot(storageRoot+"/users/alice/.Trash/files", storageRoot, true))
require.False(t, isTrashRoot(storageRoot+"/users/alice/.Trash/files/item.trashitem", storageRoot, true))
require.False(t, isTrashRoot(storageRoot+"/users/alice/.Trash/files/item.trashitem/files", storageRoot, true))
}
func TestRemoveEmptyFolderPosix(t *testing.T) {
storageRoot := "test_temp_" + t.Name()
base := storageRoot + "/users/alice/.Trash/files"
defer os.RemoveAll(storageRoot)
emptyChain := base + "/empty.trashitem/sub/subsub"
require.NoError(t, os.MkdirAll(emptyChain, os.ModePerm))
nonEmpty := base + "/keep.trashitem"
require.NoError(t, os.MkdirAll(nonEmpty, os.ModePerm))
require.NoError(t, os.WriteFile(nonEmpty+"/file.txt", []byte("some text"), os.ModePerm))
require.NoError(t, removeEmptyFolder(emptyChain, false, true, storageRoot))
assertNoDirExists(t, emptyChain)
assertNoDirExists(t, base+"/empty.trashitem")
assertDirExists(t, base)
assertDirExists(t, nonEmpty)
}
func TestRemoveEmptyFolderPosixUserDirNamedFiles(t *testing.T) {
storageRoot := "test_temp_" + t.Name()
base := storageRoot + "/users/alice/.Trash/files"
defer os.RemoveAll(storageRoot)
nestedFilesDir := base + "/folder.trashitem/files/files"
require.NoError(t, os.MkdirAll(nestedFilesDir, os.ModePerm))
require.NoError(t, removeEmptyFolder(nestedFilesDir, false, true, storageRoot))
assertNoDirExists(t, nestedFilesDir)
assertNoDirExists(t, base+"/folder.trashitem/files")
assertNoDirExists(t, base+"/folder.trashitem")
assertDirExists(t, base)
}
func TestRemoveEmptyFolderDecomposed(t *testing.T) {
storageRoot := "test_temp_" + t.Name()
base := storageRoot + "/spaces/id/id/trash"
defer os.RemoveAll(storageRoot)
emptyChain := base + "/s1/s2/s3/node"
require.NoError(t, os.MkdirAll(emptyChain, os.ModePerm))
require.NoError(t, removeEmptyFolder(emptyChain, false, false, storageRoot))
assertNoDirExists(t, emptyChain)
assertNoDirExists(t, base+"/s1/s2/s3")
assertNoDirExists(t, base+"/s1/s2")
assertNoDirExists(t, base+"/s1")
assertDirExists(t, base)
}
func TestRemoveEmptyFolderDecomposedUserDirNamedTrash(t *testing.T) {
storageRoot := "test_temp_" + t.Name()
base := storageRoot + "/spaces/id/id/trash"
defer os.RemoveAll(storageRoot)
nestedTrashDir := base + "/s1/trash/s2/node"
require.NoError(t, os.MkdirAll(nestedTrashDir, os.ModePerm))
require.NoError(t, removeEmptyFolder(nestedTrashDir, false, false, storageRoot))
assertNoDirExists(t, nestedTrashDir)
assertNoDirExists(t, base+"/s1/trash/s2")
assertNoDirExists(t, base+"/s1/trash")
assertNoDirExists(t, base+"/s1")
assertDirExists(t, base)
}
func assertNoDirExists(t *testing.T, path string) {
t.Helper()
_, err := os.Stat(path)
require.True(t, os.IsNotExist(err))
}
func assertDirExists(t *testing.T, path string) {
t.Helper()
fi, err := os.Stat(path)
require.NoError(t, err)
require.True(t, fi.IsDir())
}
+12
View File
@@ -43,6 +43,10 @@ type StringNode struct {
*Base
Key string
Value string
Exact bool
// CaseInsensitive marks a case-insensitive restriction; set by the search
// lowering pass, a backend routes it to the field's lowercased form.
CaseInsensitive bool
}
// BooleanNode represents a bool value
@@ -60,6 +64,14 @@ type DateTimeNode struct {
Value time.Time
}
// NumberNode represents a numeric value
type NumberNode struct {
*Base
Key string
Operator *OperatorNode
Value float64
}
// OperatorNode represents an operator value like
// AND, OR, NOT, =, <= ... and so on
type OperatorNode struct {
+1
View File
@@ -21,6 +21,7 @@ func DiffAst(x, y any, opts ...cmp.Option) string {
cmpopts.IgnoreFields(ast.GroupNode{}, "Base"),
cmpopts.IgnoreFields(ast.BooleanNode{}, "Base"),
cmpopts.IgnoreFields(ast.DateTimeNode{}, "Base"),
cmpopts.IgnoreFields(ast.NumberNode{}, "Base"),
)...,
)
}
+12 -3
View File
@@ -2,11 +2,11 @@ package kql
import (
"fmt"
"strconv"
"time"
"github.com/jinzhu/now"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
func toNode[T ast.Node](in any) (T, error) {
@@ -84,7 +84,7 @@ func toTimeRange(in any) (*time.Time, *time.Time, error) {
value, err := toString(in)
if err != nil {
return &from, &to, &query.UnsupportedTimeRangeError{}
return &from, &to, &UnsupportedTimeRangeError{}
}
c := &now.Config{
@@ -131,8 +131,17 @@ func toTimeRange(in any) (*time.Time, *time.Time, error) {
}
if from.IsZero() || to.IsZero() {
return nil, nil, &query.UnsupportedTimeRangeError{}
return nil, nil, &UnsupportedTimeRangeError{}
}
return &from, &to, nil
}
func toFloat(v any) (float64, error) {
value, err := toString(v)
if err != nil {
return 0, err
}
return strconv.ParseFloat(value, 64)
}
+23 -4
View File
@@ -40,6 +40,7 @@ GroupNode <-
PropertyRestrictionNodes <-
YesNoPropertyRestrictionNode /
DateTimeRestrictionNode /
NumberRestrictionNode /
TextPropertyRestrictionNode
YesNoPropertyRestrictionNode <-
@@ -69,9 +70,22 @@ DateTimeRestrictionNode <-
return buildNaturalLanguageDateTimeNodes(k, v, c.text, c.pos)
}
NumberRestrictionNode <-
k:Key o:(
OperatorGreaterOrEqualNode /
OperatorLessOrEqualNode /
OperatorGreaterNode /
OperatorLessNode
) '"'? v:Number '"'? {
return buildNumberNode(k, o, v, c.text, c.pos)
}
TextPropertyRestrictionNode <-
k:Key (OperatorColonNode / OperatorEqualNode) v:(String / [^ ()]+){
return buildStringNode(k, v, c.text, c.pos)
k:Key OperatorEqualNode v:(String / [^ ()]+) {
return buildStringNode(k, v, true, c.text, c.pos)
} /
k:Key OperatorColonNode v:(String / [^ ()]+) {
return buildStringNode(k, v, false, c.text, c.pos)
}
////////////////////////////////////////////////////////
@@ -84,12 +98,12 @@ FreeTextKeywordNodes <-
PhraseNode <-
OperatorColonNode? _ v:String _ OperatorColonNode? {
return buildStringNode("", v, c.text, c.pos)
return buildStringNode("", v, false, c.text, c.pos)
}
WordNode <-
OperatorColonNode? _ v:[^ :()]+ _ OperatorColonNode? {
return buildStringNode("", v, c.text, c.pos)
return buildStringNode("", v, false, c.text, c.pos)
}
////////////////////////////////////////////////////////
@@ -229,6 +243,11 @@ String <-
return v, nil
}
Number <-
[0-9]+ ("." [0-9]+)? {
return string(c.text), nil
}
Digit <-
[0-9] {
return c.text, nil
+928 -483
View File
File diff suppressed because it is too large. Load diff
+80 -11
View File
@@ -9,7 +9,6 @@ import (
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/ast/test"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
tAssert "github.com/stretchr/testify/assert"
)
@@ -34,13 +33,13 @@ func TestParse_Spec(t *testing.T) {
},
{
name: `AND`,
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolAND},
},
},
{
name: `AND cat AND dog`,
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolAND},
},
},
@@ -80,13 +79,13 @@ func TestParse_Spec(t *testing.T) {
},
{
name: `OR`,
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolOR},
},
},
{
name: `OR cat AND dog`,
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolOR},
},
},
@@ -145,6 +144,22 @@ func TestParse_Spec(t *testing.T) {
},
},
},
{
name: `author="John Smith"`,
ast: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "author", Value: "John Smith", Exact: true},
},
},
},
{
name: `filename=budget.xlsx`,
ast: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "filename", Value: "budget.xlsx", Exact: true},
},
},
},
// 3.2.3 Implicit Operator for Property Restriction
{
name: `author:"John Smith" filetype:docx`,
@@ -423,6 +438,60 @@ func TestParse_Spec(t *testing.T) {
}
}
func TestParse_NumberRestrictionNode(t *testing.T) {
tests := []testCase{
{
name: "format",
query: join([]string{
`size>100`,
`size>"100"`,
`size>=15.5`,
`size<100`,
`size<=100`,
}),
ast: &ast.Ast{
Nodes: []ast.Node{
&ast.NumberNode{
Key: "size",
Operator: &ast.OperatorNode{Value: ">"},
Value: 100,
},
&ast.OperatorNode{Value: kql.BoolAND},
&ast.NumberNode{
Key: "size",
Operator: &ast.OperatorNode{Value: ">"},
Value: 100,
},
&ast.OperatorNode{Value: kql.BoolAND},
&ast.NumberNode{
Key: "size",
Operator: &ast.OperatorNode{Value: ">="},
Value: 15.5,
},
&ast.OperatorNode{Value: kql.BoolAND},
&ast.NumberNode{
Key: "size",
Operator: &ast.OperatorNode{Value: "<"},
Value: 100,
},
&ast.OperatorNode{Value: kql.BoolAND},
&ast.NumberNode{
Key: "size",
Operator: &ast.OperatorNode{Value: "<="},
Value: 100,
},
},
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
testKQL(t, tc)
})
}
}
func TestParse_DateTimeRestrictionNode(t *testing.T) {
tests := []testCase{
{
@@ -860,37 +929,37 @@ func TestParse_Errors(t *testing.T) {
tests := []testCase{
{
query: "animal:(mammal:cat mammal:dog reptile:turtle)",
error: query.NamedGroupInvalidNodesError{
error: kql.NamedGroupInvalidNodesError{
Node: &ast.StringNode{Key: "mammal", Value: "cat"},
},
},
{
query: "animal:(cat mammal:dog turtle)",
error: query.NamedGroupInvalidNodesError{
error: kql.NamedGroupInvalidNodesError{
Node: &ast.StringNode{Key: "mammal", Value: "dog"},
},
},
{
query: "animal:(AND cat)",
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolAND},
},
},
{
query: "animal:(OR cat)",
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolOR},
},
},
{
query: "(AND cat)",
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolAND},
},
},
{
query: "(OR cat)",
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolOR},
},
},
-11
View File
@@ -1,11 +0,0 @@
package kql
import (
"time"
)
// PatchTimeNow is here to patch the package time now func,
// which is used in the test suite
func PatchTimeNow(t func() time.Time) {
timeNow = t
}
@@ -1,4 +1,4 @@
package query
package kql
import (
"errors"
@@ -39,8 +39,9 @@ func (e UnsupportedTimeRangeError) Error() string {
return fmt.Sprintf("unable to convert '%v' to a time range", e.Value)
}
// IsValidationError says whether the query itself is at fault, which makes it a
// bad request and not an error of ours.
// IsValidationError reports whether err is one of the KQL parse/validation
// errors produced by this package, i.e. the query itself is at fault and the
// caller should treat it as a bad request.
func IsValidationError(err error) bool {
var (
startsWithBinaryOperator *StartsWithBinaryOperatorError
+32 -1
View File
@@ -50,7 +50,7 @@ func buildAST(n any, text []byte, pos position) (*ast.Ast, error) {
return a, nil
}
func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, error) {
func buildStringNode(k, v any, exact bool, text []byte, pos position) (*ast.StringNode, error) {
b, err := base(text, pos)
if err != nil {
return nil, err
@@ -70,6 +70,7 @@ func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, erro
Base: b,
Key: key,
Value: value,
Exact: exact,
}, nil
}
@@ -101,6 +102,36 @@ func buildDateTimeNode(k, o, v any, text []byte, pos position) (*ast.DateTimeNod
Value: value,
}, nil
}
func buildNumberNode(k, o, v any, text []byte, pos position) (*ast.NumberNode, error) {
b, err := base(text, pos)
if err != nil {
return nil, err
}
operator, err := toNode[*ast.OperatorNode](o)
if err != nil {
return nil, err
}
key, err := toString(k)
if err != nil {
return nil, err
}
value, err := toFloat(v)
if err != nil {
return nil, err
}
return &ast.NumberNode{
Base: b,
Key: key,
Operator: operator,
Value: value,
}, nil
}
func buildNaturalLanguageDateTimeNodes(k, v any, text []byte, pos position) ([]ast.Node, error) {
b, err := base(text, pos)
if err != nil {
+6
View File
@@ -47,3 +47,9 @@ func (b Builder) Build(q string) (*ast.Ast, error) {
// timeNow mirrors time.Now by default, the only reason why this exists
// is to monkey patch it from the tests. See PatchTimeNow
var timeNow = time.Now
// PatchTimeNow pins the clock the natural language dates resolve against,
// so a test can hold "today" still while it runs
func PatchTimeNow(t func() time.Time) {
timeNow = t
}
+1 -2
View File
@@ -5,7 +5,6 @@ import (
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
tAssert "github.com/stretchr/testify/assert"
)
@@ -22,7 +21,7 @@ func TestNewAST(t *testing.T) {
{
name: "error",
givenQuery: kql.BoolAND,
expectedError: query.StartsWithBinaryOperatorError{
expectedError: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolAND},
},
},
+3 -4
View File
@@ -2,7 +2,6 @@ package kql
import (
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
func validateAst(a *ast.Ast) error {
@@ -10,7 +9,7 @@ func validateAst(a *ast.Ast) error {
case *ast.OperatorNode:
switch node.Value {
case BoolAND, BoolOR:
return &query.StartsWithBinaryOperatorError{Node: node}
return &StartsWithBinaryOperatorError{Node: node}
}
}
return nil
@@ -21,14 +20,14 @@ func validateGroupNode(n *ast.GroupNode) error {
case *ast.OperatorNode:
switch node.Value {
case BoolAND, BoolOR:
return &query.StartsWithBinaryOperatorError{Node: node}
return &StartsWithBinaryOperatorError{Node: node}
}
}
if n.Key != "" {
for _, node := range n.Nodes {
if ast.NodeKey(node) != "" {
return &query.NamedGroupInvalidNodesError{Node: node}
return &NamedGroupInvalidNodesError{Node: node}
}
}
}
+189
View File
@@ -0,0 +1,189 @@
package metrics
import (
"fmt"
"reflect"
"strings"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/pkg/version"
"github.com/prometheus/client_golang/prometheus"
)
type BuildInfoMetric = *prometheus.GaugeVec
// Create a BuildInfo metric for the specified namespace and subsystem.
func BuildInfo(namespace, subsystem string) BuildInfoMetric {
return prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "build_info",
Help: "Build information",
}, []string{"version"})
}
// Determine the fully qualified name of a metric.
//
// Beware that this requires storing a value into the metric in order to make it
// visible in a temporary registry.
// If the metric is a MetricVec, it will be Reset().
func describe(metric prometheus.Collector, initialize func() error) (string, error) {
reg := prometheus.NewRegistry()
if err := reg.Register(metric); err != nil {
return "", err
}
if err := initialize(); err != nil {
return "", err
}
if resettable, ok := metric.(*prometheus.MetricVec); ok {
defer resettable.Reset()
}
fams, err := reg.Gather()
if err != nil {
return "", err
}
if len(fams) == 0 {
return "", fmt.Errorf("no metric families gathered")
}
return fams[0].GetName(), nil
}
// Take a struct that contains metrics as attributes and register all of them
// with the specified Registerer.
func RegisterAll(registerer prometheus.Registerer, m any, logger *log.Logger) error {
// we go over all of them, use this to keep track of succeesses and failures
total := 0
succeeded := []string{}
failed := map[string]error{}
// we need to use reflection here to iterate over the public metric attributes
// that are contained in it
r := reflect.ValueOf(m)
if r.Kind() == reflect.Pointer {
r = r.Elem()
}
for i := 0; i < r.NumField(); i++ {
t := r.Type().Field(i)
n := t.Name // the name of the attribute (not the name of the metric)
f := r.Field(i)
if !f.CanInterface() {
continue // we won't be able to process that one, most probably because it's not exported
}
v := f.Interface()
switch c := v.(type) {
case prometheus.Collector:
total++
if err := registerer.Register(c); err != nil {
switch err.(type) {
case prometheus.AlreadyRegisteredError:
// silently ignore this error, as this case can happen when the suture service decides to restart
err = nil
succeeded = append(succeeded, n)
default:
failed[n] = err
}
} else {
succeeded = append(succeeded, n)
// special post-treatment for the BuildInfo metric, as we have that one pretty much
// everywhere: set its value with the current version so we don't need to do that every time
switch buildInfo := c.(type) {
case BuildInfoMetric:
if name, err := describe(buildInfo, func() error { buildInfo.WithLabelValues("0").Set(0.0); return nil }); err != nil {
failed[n] = err
} else if strings.HasSuffix(name, "_build_info") {
buildInfo.Reset()
buildInfo.WithLabelValues(version.GetString()).Set(1)
}
}
}
case *prometheus.Desc,
prometheus.GaugeOpts,
prometheus.CounterOpts,
prometheus.HistogramOpts,
prometheus.SummaryOpts,
prometheus.UntypedOpts:
// skip these
default:
failed[n] = fmt.Errorf("unsupported metric '%s' of type %T", n, c)
}
}
if len(failed) > 0 {
failedMsgs := []string{}
for name, err := range failed {
failedMsgs = append(failedMsgs, fmt.Sprintf("'%s' (%v)", name, err))
}
msg := strings.Join(failedMsgs, ", ")
if logger != nil {
logger.Warn().Msgf("registered %d/%d metrics successfully (%d failed): %s", len(succeeded), total, len(failed), msg)
}
return fmt.Errorf("failed to register metrics: %s", msg)
} else {
if logger != nil {
logger.Debug().Msgf("registered %d/%d metrics successfully (%d failed)", len(succeeded), total, len(failed))
}
return nil
}
}
// Register all the metrics that are contained as public attributes in the struct,
// and log any errors that might occur while doing so.
func Register[M any](reg prometheus.Registerer, m M, logger *log.Logger) (M, error) {
lr := NewLoggingPrometheusRegisterer(reg, logger)
err := RegisterAll(lr, m, logger)
return m, err
}
// Register a single metric.
func RegisterMetric[M prometheus.Collector](reg prometheus.Registerer, m M, logger *log.Logger) error {
return NewLoggingPrometheusRegisterer(reg, logger).Register(m)
}
// Prometheus Registerer wrapper that logs every error that occurs when registering
// a metric, and delegates to an actual Registerer.
type LoggingPrometheusRegisterer struct {
delegate prometheus.Registerer
logger *log.Logger
}
// Instantiate a Prometheus Registerer wrapper that logs every error that occurs when registering
// a metric, and that delegates to an actual Registerer specified here.
func NewLoggingPrometheusRegisterer(delegate prometheus.Registerer, logger *log.Logger) *LoggingPrometheusRegisterer {
return &LoggingPrometheusRegisterer{
delegate: delegate,
logger: logger,
}
}
func (r *LoggingPrometheusRegisterer) Register(c prometheus.Collector) error {
err := r.delegate.Register(c)
if err != nil {
switch err.(type) {
case prometheus.AlreadyRegisteredError:
// silently ignore this error, as this case can happen when the suture service decides to restart
err = nil
default:
if r.logger != nil {
r.logger.Warn().Err(err).Msgf("failed to register metric")
}
}
}
return err
}
func (r *LoggingPrometheusRegisterer) MustRegister(collectors ...prometheus.Collector) {
for _, c := range collectors {
if err := r.Register(c); err != nil {
if r.logger != nil {
r.logger.Error().Err(err).Msg("failed to register metrics collector")
}
}
}
}
func (r *LoggingPrometheusRegisterer) Unregister(c prometheus.Collector) bool {
return r.delegate.Unregister(c)
}
var _ prometheus.Registerer = &LoggingPrometheusRegisterer{}
+153
View File
@@ -0,0 +1,153 @@
package metrics
import (
"fmt"
"math/rand/v2"
"testing"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/pkg/version"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
)
func randName() string {
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
n := 8 + rand.IntN(33)
b := make([]byte, n)
for i := range b {
b[i] = letterBytes[rand.IntN(len(letterBytes))]
}
return string(b)
}
func TestBuildInfo(t *testing.T) {
require := require.New(t)
namespace := "name-" + randName()
subsystem := "sub-" + randName()
expectedName := fmt.Sprintf("%s_%s_build_info", namespace, subsystem)
version := fmt.Sprintf("%d.%d.%d", rand.IntN(10), rand.IntN(10), rand.IntN(10))
g := BuildInfo(namespace, subsystem)
reg := prometheus.NewRegistry()
require.NoError(reg.Register(g))
{
mfs, err := reg.Gather()
require.NoError(err)
require.Len(mfs, 0)
}
g.WithLabelValues(version).Set(1)
{
mfs, err := reg.Gather()
require.NoError(err)
found := false
for _, mf := range mfs {
if mf.GetName() == expectedName {
found = true
ms := mf.GetMetric()
require.Len(ms, 1)
labels := ms[0].GetLabel()
require.Len(labels, 1)
require.NotNil(labels[0].Name)
require.Equal("version", *labels[0].Name)
require.NotNil(labels[0].Value)
require.Equal(version, *labels[0].Value)
require.Equal(1.0, ms[0].GetGauge().GetValue())
} else {
t.Fatalf("unexpected metric family %q", mf.GetName())
}
}
require.True(found, "failed to find metric %q", expectedName)
}
}
func TestRegisterAll(t *testing.T) {
require := require.New(t)
reg := prometheus.NewRegistry()
logger := log.NewLogger()
namespace := "name-" + randName()
subsystem := "sub-" + randName()
m := struct {
BuildInfo *prometheus.GaugeVec
Foo *prometheus.GaugeVec
Bar prometheus.Counter
}{
BuildInfo: BuildInfo(namespace, subsystem),
Foo: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "foo",
ConstLabels: prometheus.Labels{
"f": "oo",
"fo": "o",
},
}, []string{"oof"}),
Bar: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "bar",
}),
}
expectedNameForBuildInfo := namespace + "_" + subsystem + "_build_info"
expectedNameForFoo := namespace + "_" + subsystem + "_foo"
expectedNameForBar := namespace + "_" + subsystem + "_bar"
{
mfs, err := reg.Gather()
require.NoError(err)
require.Len(mfs, 0)
}
require.NoError(RegisterAll(reg, m, &logger))
{
mfs, err := reg.Gather()
require.NoError(err)
require.Len(mfs, 3)
found := 0
for _, mf := range mfs {
switch mf.GetName() {
case expectedNameForBuildInfo:
found++
ms := mf.GetMetric()
require.Len(ms, 1)
labels := ms[0].GetLabel()
require.Len(labels, 1)
require.NotNil(labels[0].Name)
require.Equal("version", *labels[0].Name)
require.NotNil(labels[0].Value)
require.Equal(version.GetString(), *labels[0].Value)
require.Equal(1.0, ms[0].GetGauge().GetValue())
case expectedNameForFoo:
found++
ms := mf.GetMetric()
require.Len(ms, 1)
labels := ms[0].GetLabel()
require.Len(labels, 3)
require.NotNil(labels[0].Name)
require.Equal("f", *labels[0].Name)
require.NotNil(labels[0].Value)
require.Equal("oo", *labels[0].Value)
require.NotNil(labels[1].Name)
require.Equal("fo", *labels[1].Name)
require.NotNil(labels[1].Value)
require.Equal("o", *labels[1].Value)
require.Equal(0.0, ms[0].GetGauge().GetValue())
case expectedNameForBar:
found++
ms := mf.GetMetric()
require.Len(ms, 1)
labels := ms[0].GetLabel()
require.Len(labels, 0)
require.Equal(0.0, ms[0].GetGauge().GetValue())
default:
t.Fatalf("unexpected metric family %q", mf.GetName())
}
}
require.Equal(3, found, "failed to find expected metrics")
}
}
+11 -1
View File
@@ -2,6 +2,7 @@ package middleware
import (
"context"
"errors"
"net/http"
"strings"
"sync"
@@ -61,7 +62,10 @@ func OidcAuth(opts ...Option) func(http.Handler) http.Handler {
provider, err = providerFunc()
}
initializeProviderLock.Unlock()
if err != nil {
if err != nil || provider == nil {
if err == nil {
err = errors.New("OIDC provider initialization returned nil")
}
opt.Logger.Error().Err(err).Msg("could not initialize OIDC provider")
w.WriteHeader(http.StatusInternalServerError)
return
@@ -82,6 +86,12 @@ func OidcAuth(opts ...Option) func(http.Handler) http.Handler {
w.WriteHeader(http.StatusUnauthorized)
return
}
if userInfo == nil {
opt.Logger.Error().Msg("OIDC provider returned empty user info")
w.Header().Add("WWW-Authenticate", `Bearer`)
w.WriteHeader(http.StatusUnauthorized)
return
}
claims := map[string]any{}
err = userInfo.Claims(&claims)
if err != nil {
@@ -535,6 +535,267 @@ func (x *Photo) GetTakenDateTime() *timestamppb.Timestamp {
return nil
}
type Video struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
AudioBitsPerSample *int32 `protobuf:"varint,1,opt,name=audioBitsPerSample,proto3,oneof" json:"audioBitsPerSample,omitempty"`
AudioChannels *int32 `protobuf:"varint,2,opt,name=audioChannels,proto3,oneof" json:"audioChannels,omitempty"`
AudioFormat *string `protobuf:"bytes,3,opt,name=audioFormat,proto3,oneof" json:"audioFormat,omitempty"`
AudioSamplesPerSecond *int32 `protobuf:"varint,4,opt,name=audioSamplesPerSecond,proto3,oneof" json:"audioSamplesPerSecond,omitempty"`
Bitrate *int32 `protobuf:"varint,5,opt,name=bitrate,proto3,oneof" json:"bitrate,omitempty"`
Duration *int64 `protobuf:"varint,6,opt,name=duration,proto3,oneof" json:"duration,omitempty"`
FourCC *string `protobuf:"bytes,7,opt,name=fourCC,proto3,oneof" json:"fourCC,omitempty"`
FrameRate *float64 `protobuf:"fixed64,8,opt,name=frameRate,proto3,oneof" json:"frameRate,omitempty"`
Height *int32 `protobuf:"varint,9,opt,name=height,proto3,oneof" json:"height,omitempty"`
Width *int32 `protobuf:"varint,10,opt,name=width,proto3,oneof" json:"width,omitempty"`
}
func (x *Video) Reset() {
*x = Video{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Video) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Video) ProtoMessage() {}
func (x *Video) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[6]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Video.ProtoReflect.Descriptor instead.
func (*Video) Descriptor() ([]byte, []int) {
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{6}
}
func (x *Video) GetAudioBitsPerSample() int32 {
if x != nil && x.AudioBitsPerSample != nil {
return *x.AudioBitsPerSample
}
return 0
}
func (x *Video) GetAudioChannels() int32 {
if x != nil && x.AudioChannels != nil {
return *x.AudioChannels
}
return 0
}
func (x *Video) GetAudioFormat() string {
if x != nil && x.AudioFormat != nil {
return *x.AudioFormat
}
return ""
}
func (x *Video) GetAudioSamplesPerSecond() int32 {
if x != nil && x.AudioSamplesPerSecond != nil {
return *x.AudioSamplesPerSecond
}
return 0
}
func (x *Video) GetBitrate() int32 {
if x != nil && x.Bitrate != nil {
return *x.Bitrate
}
return 0
}
func (x *Video) GetDuration() int64 {
if x != nil && x.Duration != nil {
return *x.Duration
}
return 0
}
func (x *Video) GetFourCC() string {
if x != nil && x.FourCC != nil {
return *x.FourCC
}
return ""
}
func (x *Video) GetFrameRate() float64 {
if x != nil && x.FrameRate != nil {
return *x.FrameRate
}
return 0
}
func (x *Video) GetHeight() int32 {
if x != nil && x.Height != nil {
return *x.Height
}
return 0
}
func (x *Video) GetWidth() int32 {
if x != nil && x.Width != nil {
return *x.Width
}
return 0
}
type MotionPhoto struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Version *int32 `protobuf:"varint,1,opt,name=version,proto3,oneof" json:"version,omitempty"`
PresentationTimestampUs *int64 `protobuf:"varint,2,opt,name=presentationTimestampUs,proto3,oneof" json:"presentationTimestampUs,omitempty"`
VideoSize *int64 `protobuf:"varint,3,opt,name=videoSize,proto3,oneof" json:"videoSize,omitempty"`
}
func (x *MotionPhoto) Reset() {
*x = MotionPhoto{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *MotionPhoto) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*MotionPhoto) ProtoMessage() {}
func (x *MotionPhoto) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[7]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use MotionPhoto.ProtoReflect.Descriptor instead.
func (*MotionPhoto) Descriptor() ([]byte, []int) {
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{7}
}
func (x *MotionPhoto) GetVersion() int32 {
if x != nil && x.Version != nil {
return *x.Version
}
return 0
}
func (x *MotionPhoto) GetPresentationTimestampUs() int64 {
if x != nil && x.PresentationTimestampUs != nil {
return *x.PresentationTimestampUs
}
return 0
}
func (x *MotionPhoto) GetVideoSize() int64 {
if x != nil && x.VideoSize != nil {
return *x.VideoSize
}
return 0
}
type LivePhoto struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
ContentId *string `protobuf:"bytes,1,opt,name=contentId,proto3,oneof" json:"contentId,omitempty"`
StillImageTimeUs *int64 `protobuf:"varint,2,opt,name=stillImageTimeUs,proto3,oneof" json:"stillImageTimeUs,omitempty"`
Auto *bool `protobuf:"varint,3,opt,name=auto,proto3,oneof" json:"auto,omitempty"`
VitalityScore *float64 `protobuf:"fixed64,4,opt,name=vitalityScore,proto3,oneof" json:"vitalityScore,omitempty"`
VitalityScoringVersion *int64 `protobuf:"varint,5,opt,name=vitalityScoringVersion,proto3,oneof" json:"vitalityScoringVersion,omitempty"`
}
func (x *LivePhoto) Reset() {
*x = LivePhoto{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *LivePhoto) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*LivePhoto) ProtoMessage() {}
func (x *LivePhoto) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[8]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use LivePhoto.ProtoReflect.Descriptor instead.
func (*LivePhoto) Descriptor() ([]byte, []int) {
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{8}
}
func (x *LivePhoto) GetContentId() string {
if x != nil && x.ContentId != nil {
return *x.ContentId
}
return ""
}
func (x *LivePhoto) GetStillImageTimeUs() int64 {
if x != nil && x.StillImageTimeUs != nil {
return *x.StillImageTimeUs
}
return 0
}
func (x *LivePhoto) GetAuto() bool {
if x != nil && x.Auto != nil {
return *x.Auto
}
return false
}
func (x *LivePhoto) GetVitalityScore() float64 {
if x != nil && x.VitalityScore != nil {
return *x.VitalityScore
}
return 0
}
func (x *LivePhoto) GetVitalityScoringVersion() int64 {
if x != nil && x.VitalityScoringVersion != nil {
return *x.VitalityScoringVersion
}
return 0
}
type Entity struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -560,12 +821,15 @@ type Entity struct {
Image *Image `protobuf:"bytes,18,opt,name=image,proto3" json:"image,omitempty"`
Photo *Photo `protobuf:"bytes,19,opt,name=photo,proto3" json:"photo,omitempty"`
Favorites []string `protobuf:"bytes,20,rep,name=favorites,proto3" json:"favorites,omitempty"`
MotionPhoto *MotionPhoto `protobuf:"bytes,21,opt,name=motionPhoto,proto3" json:"motionPhoto,omitempty"`
Video *Video `protobuf:"bytes,22,opt,name=video,proto3" json:"video,omitempty"`
LivePhoto *LivePhoto `protobuf:"bytes,23,opt,name=livePhoto,proto3" json:"livePhoto,omitempty"`
}
func (x *Entity) Reset() {
*x = Entity{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[6]
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -578,7 +842,7 @@ func (x *Entity) String() string {
func (*Entity) ProtoMessage() {}
func (x *Entity) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[6]
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[9]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -591,7 +855,7 @@ func (x *Entity) ProtoReflect() protoreflect.Message {
// Deprecated: Use Entity.ProtoReflect.Descriptor instead.
func (*Entity) Descriptor() ([]byte, []int) {
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{6}
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{9}
}
func (x *Entity) GetRef() *Reference {
@@ -734,6 +998,27 @@ func (x *Entity) GetFavorites() []string {
return nil
}
func (x *Entity) GetMotionPhoto() *MotionPhoto {
if x != nil {
return x.MotionPhoto
}
return nil
}
func (x *Entity) GetVideo() *Video {
if x != nil {
return x.Video
}
return nil
}
func (x *Entity) GetLivePhoto() *LivePhoto {
if x != nil {
return x.LivePhoto
}
return nil
}
type Match struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -748,7 +1033,7 @@ type Match struct {
func (x *Match) Reset() {
*x = Match{}
if protoimpl.UnsafeEnabled {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[7]
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -761,7 +1046,7 @@ func (x *Match) String() string {
func (*Match) ProtoMessage() {}
func (x *Match) ProtoReflect() protoreflect.Message {
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[7]
mi := &file_opencloud_messages_search_v0_search_proto_msgTypes[10]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -774,7 +1059,7 @@ func (x *Match) ProtoReflect() protoreflect.Message {
// Deprecated: Use Match.ProtoReflect.Descriptor instead.
func (*Match) Descriptor() ([]byte, []int) {
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{7}
return file_opencloud_messages_search_v0_search_proto_rawDescGZIP(), []int{10}
}
func (x *Match) GetEntity() *Entity {
@@ -908,62 +1193,141 @@ var file_opencloud_messages_search_v0_search_proto_rawDesc = []byte{
0x6c, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x69, 0x73, 0x6f, 0x42,
0x0e, 0x0a, 0x0c, 0x5f, 0x6f, 0x72, 0x69, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42,
0x10, 0x0a, 0x0e, 0x5f, 0x74, 0x61, 0x6b, 0x65, 0x6e, 0x44, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d,
0x65, 0x22, 0xfa, 0x06, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a, 0x03,
0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65, 0x6e,
0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73,
0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e,
0x63, 0x65, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x38, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20,
0x65, 0x22, 0x9b, 0x04, 0x0a, 0x05, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x33, 0x0a, 0x12, 0x61,
0x75, 0x64, 0x69, 0x6f, 0x42, 0x69, 0x74, 0x73, 0x50, 0x65, 0x72, 0x53, 0x61, 0x6d, 0x70, 0x6c,
0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x12, 0x61, 0x75, 0x64, 0x69, 0x6f,
0x42, 0x69, 0x74, 0x73, 0x50, 0x65, 0x72, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x88, 0x01, 0x01,
0x12, 0x29, 0x0a, 0x0d, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x48, 0x01, 0x52, 0x0d, 0x61, 0x75, 0x64, 0x69, 0x6f,
0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x88, 0x01, 0x01, 0x12, 0x25, 0x0a, 0x0b, 0x61,
0x75, 0x64, 0x69, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
0x48, 0x02, 0x52, 0x0b, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x88,
0x01, 0x01, 0x12, 0x39, 0x0a, 0x15, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x53, 0x61, 0x6d, 0x70, 0x6c,
0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28,
0x05, 0x48, 0x03, 0x52, 0x15, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65,
0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x88, 0x01, 0x01, 0x12, 0x1d, 0x0a,
0x07, 0x62, 0x69, 0x74, 0x72, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x48, 0x04,
0x52, 0x07, 0x62, 0x69, 0x74, 0x72, 0x61, 0x74, 0x65, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x08,
0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x48, 0x05,
0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a,
0x06, 0x66, 0x6f, 0x75, 0x72, 0x43, 0x43, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x48, 0x06, 0x52,
0x06, 0x66, 0x6f, 0x75, 0x72, 0x43, 0x43, 0x88, 0x01, 0x01, 0x12, 0x21, 0x0a, 0x09, 0x66, 0x72,
0x61, 0x6d, 0x65, 0x52, 0x61, 0x74, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x01, 0x48, 0x07, 0x52,
0x09, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x52, 0x61, 0x74, 0x65, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a,
0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x48, 0x08, 0x52,
0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x88, 0x01, 0x01, 0x12, 0x19, 0x0a, 0x05, 0x77, 0x69,
0x64, 0x74, 0x68, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x48, 0x09, 0x52, 0x05, 0x77, 0x69, 0x64,
0x74, 0x68, 0x88, 0x01, 0x01, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x42,
0x69, 0x74, 0x73, 0x50, 0x65, 0x72, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x42, 0x10, 0x0a, 0x0e,
0x5f, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x42, 0x0e,
0x0a, 0x0c, 0x5f, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x18,
0x0a, 0x16, 0x5f, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x50,
0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x62, 0x69, 0x74,
0x72, 0x61, 0x74, 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x66, 0x6f, 0x75, 0x72, 0x43, 0x43, 0x42, 0x0c, 0x0a, 0x0a,
0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x52, 0x61, 0x74, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x68,
0x65, 0x69, 0x67, 0x68, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22,
0xc4, 0x01, 0x0a, 0x0b, 0x4d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x12,
0x1d, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05,
0x48, 0x00, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x3d,
0x0a, 0x17, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x55, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x48,
0x01, 0x52, 0x17, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x55, 0x73, 0x88, 0x01, 0x01, 0x12, 0x21, 0x0a,
0x09, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03,
0x48, 0x02, 0x52, 0x09, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x53, 0x69, 0x7a, 0x65, 0x88, 0x01, 0x01,
0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x1a, 0x0a, 0x18,
0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d,
0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x55, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x76, 0x69, 0x64,
0x65, 0x6f, 0x53, 0x69, 0x7a, 0x65, 0x22, 0xb9, 0x02, 0x0a, 0x09, 0x4c, 0x69, 0x76, 0x65, 0x50,
0x68, 0x6f, 0x74, 0x6f, 0x12, 0x21, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x49,
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65,
0x6e, 0x74, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x2f, 0x0a, 0x10, 0x73, 0x74, 0x69, 0x6c, 0x6c,
0x49, 0x6d, 0x61, 0x67, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28,
0x03, 0x48, 0x01, 0x52, 0x10, 0x73, 0x74, 0x69, 0x6c, 0x6c, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x54,
0x69, 0x6d, 0x65, 0x55, 0x73, 0x88, 0x01, 0x01, 0x12, 0x17, 0x0a, 0x04, 0x61, 0x75, 0x74, 0x6f,
0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x48, 0x02, 0x52, 0x04, 0x61, 0x75, 0x74, 0x6f, 0x88, 0x01,
0x01, 0x12, 0x29, 0x0a, 0x0d, 0x76, 0x69, 0x74, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f,
0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x48, 0x03, 0x52, 0x0d, 0x76, 0x69, 0x74, 0x61,
0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x88, 0x01, 0x01, 0x12, 0x3b, 0x0a, 0x16,
0x76, 0x69, 0x74, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x56,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x48, 0x04, 0x52, 0x16,
0x76, 0x69, 0x74, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x56,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, 0x6f,
0x6e, 0x74, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x73, 0x74, 0x69, 0x6c,
0x6c, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x42, 0x07, 0x0a, 0x05,
0x5f, 0x61, 0x75, 0x74, 0x6f, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x76, 0x69, 0x74, 0x61, 0x6c, 0x69,
0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x19, 0x0a, 0x17, 0x5f, 0x76, 0x69, 0x74, 0x61,
0x6c, 0x69, 0x74, 0x79, 0x53, 0x63, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x65, 0x72, 0x73, 0x69,
0x6f, 0x6e, 0x22, 0xc9, 0x08, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a,
0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65,
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65,
0x6e, 0x63, 0x65, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x38, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x02,
0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x74, 0x61, 0x67, 0x18, 0x04,
0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x65, 0x74, 0x61, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69,
0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x48,
0x0a, 0x12, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f,
0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d,
0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69,
0x66, 0x69, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x69, 0x6d, 0x65,
0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d,
0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73,
0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x65, 0x72, 0x6d,
0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18,
0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64,
0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x64, 0x65,
0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x65, 0x52, 0x6f,
0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x68,
0x61, 0x72, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x45, 0x0a, 0x09, 0x70,
0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28,
0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61,
0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65,
0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74,
0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x09,
0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x68, 0x69, 0x67, 0x68, 0x6c, 0x69,
0x67, 0x68, 0x74, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x68, 0x69, 0x67, 0x68,
0x6c, 0x69, 0x67, 0x68, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x05, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x18,
0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75,
0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63,
0x68, 0x2e, 0x76, 0x30, 0x2e, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x52, 0x05, 0x61, 0x75, 0x64, 0x69,
0x6f, 0x12, 0x48, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x10, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e,
0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e,
0x76, 0x30, 0x2e, 0x47, 0x65, 0x6f, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x65,
0x73, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4e, 0x0a, 0x0e, 0x72,
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x11, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e,
0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e,
0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x02, 0x69,
0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x65, 0x74, 0x61, 0x67, 0x18, 0x04, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x65, 0x74, 0x61, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a,
0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x48, 0x0a,
0x12, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x74,
0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66,
0x69, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x69, 0x6d, 0x65, 0x5f,
0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6d, 0x65,
0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69,
0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x65, 0x72, 0x6d, 0x69,
0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x09,
0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65,
0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x64, 0x65, 0x6c,
0x65, 0x74, 0x65, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x68, 0x61, 0x72, 0x65, 0x52, 0x6f, 0x6f,
0x74, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x68, 0x61,
0x72, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x45, 0x0a, 0x09, 0x70, 0x61,
0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e,
0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67,
0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x52, 0x65, 0x73,
0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49,
0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x09, 0x52,
0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x68, 0x69, 0x67, 0x68, 0x6c, 0x69, 0x67,
0x68, 0x74, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x68, 0x69, 0x67, 0x68, 0x6c,
0x69, 0x67, 0x68, 0x74, 0x73, 0x12, 0x39, 0x0a, 0x05, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x18, 0x0f,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x76, 0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x0c, 0x72,
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x12, 0x39, 0x0a, 0x05, 0x69,
0x6d, 0x61, 0x67, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65,
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52,
0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x05, 0x70, 0x68, 0x6f, 0x74, 0x6f, 0x18,
0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75,
0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63,
0x68, 0x2e, 0x76, 0x30, 0x2e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x52, 0x05, 0x70, 0x68, 0x6f, 0x74,
0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74, 0x65, 0x73, 0x18, 0x14,
0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74, 0x65, 0x73, 0x12,
0x4b, 0x0a, 0x0b, 0x6d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x18, 0x15,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
0x2e, 0x76, 0x30, 0x2e, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x52, 0x05, 0x61, 0x75, 0x64, 0x69, 0x6f,
0x12, 0x48, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x10, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d,
0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76,
0x30, 0x2e, 0x47, 0x65, 0x6f, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x73,
0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4e, 0x0a, 0x0e, 0x72, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x69, 0x74, 0x65, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x11, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d,
0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76,
0x30, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x44, 0x52, 0x0c, 0x72, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x49, 0x64, 0x12, 0x39, 0x0a, 0x05, 0x69, 0x6d,
0x61, 0x67, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e,
0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73,
0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x05,
0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x05, 0x70, 0x68, 0x6f, 0x74, 0x6f, 0x18, 0x13,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64,
0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
0x2e, 0x76, 0x30, 0x2e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x52, 0x05, 0x70, 0x68, 0x6f, 0x74, 0x6f,
0x12, 0x1c, 0x0a, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74, 0x65, 0x73, 0x18, 0x14, 0x20,
0x03, 0x28, 0x09, 0x52, 0x09, 0x66, 0x61, 0x76, 0x6f, 0x72, 0x69, 0x74, 0x65, 0x73, 0x22, 0x5b,
0x2e, 0x76, 0x30, 0x2e, 0x4d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x52,
0x0b, 0x6d, 0x6f, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x12, 0x39, 0x0a, 0x05,
0x76, 0x69, 0x64, 0x65, 0x6f, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x70,
0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73,
0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x56, 0x69, 0x64, 0x65, 0x6f,
0x52, 0x05, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x45, 0x0a, 0x09, 0x6c, 0x69, 0x76, 0x65, 0x50,
0x68, 0x6f, 0x74, 0x6f, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x70, 0x65,
0x6e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e,
0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x76, 0x30, 0x2e, 0x4c, 0x69, 0x76, 0x65, 0x50, 0x68,
0x6f, 0x74, 0x6f, 0x52, 0x09, 0x6c, 0x69, 0x76, 0x65, 0x50, 0x68, 0x6f, 0x74, 0x6f, 0x22, 0x5b,
0x0a, 0x05, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x3c, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74,
0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x70, 0x65, 0x6e, 0x63, 0x6c,
0x6f, 0x75, 0x64, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x73, 0x65, 0x61,
@@ -990,7 +1354,7 @@ func file_opencloud_messages_search_v0_search_proto_rawDescGZIP() []byte {
return file_opencloud_messages_search_v0_search_proto_rawDescData
}
var file_opencloud_messages_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
var file_opencloud_messages_search_v0_search_proto_msgTypes = make([]protoimpl.MessageInfo, 11)
var file_opencloud_messages_search_v0_search_proto_goTypes = []interface{}{
(*ResourceID)(nil), // 0: opencloud.messages.search.v0.ResourceID
(*Reference)(nil), // 1: opencloud.messages.search.v0.Reference
@@ -998,28 +1362,34 @@ var file_opencloud_messages_search_v0_search_proto_goTypes = []interface{}{
(*Image)(nil), // 3: opencloud.messages.search.v0.Image
(*GeoCoordinates)(nil), // 4: opencloud.messages.search.v0.GeoCoordinates
(*Photo)(nil), // 5: opencloud.messages.search.v0.Photo
(*Entity)(nil), // 6: opencloud.messages.search.v0.Entity
(*Match)(nil), // 7: opencloud.messages.search.v0.Match
(*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp
(*Video)(nil), // 6: opencloud.messages.search.v0.Video
(*MotionPhoto)(nil), // 7: opencloud.messages.search.v0.MotionPhoto
(*LivePhoto)(nil), // 8: opencloud.messages.search.v0.LivePhoto
(*Entity)(nil), // 9: opencloud.messages.search.v0.Entity
(*Match)(nil), // 10: opencloud.messages.search.v0.Match
(*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp
}
var file_opencloud_messages_search_v0_search_proto_depIdxs = []int32{
0, // 0: opencloud.messages.search.v0.Reference.resource_id:type_name -> opencloud.messages.search.v0.ResourceID
8, // 1: opencloud.messages.search.v0.Photo.takenDateTime:type_name -> google.protobuf.Timestamp
11, // 1: opencloud.messages.search.v0.Photo.takenDateTime:type_name -> google.protobuf.Timestamp
1, // 2: opencloud.messages.search.v0.Entity.ref:type_name -> opencloud.messages.search.v0.Reference
0, // 3: opencloud.messages.search.v0.Entity.id:type_name -> opencloud.messages.search.v0.ResourceID
8, // 4: opencloud.messages.search.v0.Entity.last_modified_time:type_name -> google.protobuf.Timestamp
11, // 4: opencloud.messages.search.v0.Entity.last_modified_time:type_name -> google.protobuf.Timestamp
0, // 5: opencloud.messages.search.v0.Entity.parent_id:type_name -> opencloud.messages.search.v0.ResourceID
2, // 6: opencloud.messages.search.v0.Entity.audio:type_name -> opencloud.messages.search.v0.Audio
4, // 7: opencloud.messages.search.v0.Entity.location:type_name -> opencloud.messages.search.v0.GeoCoordinates
0, // 8: opencloud.messages.search.v0.Entity.remote_item_id:type_name -> opencloud.messages.search.v0.ResourceID
3, // 9: opencloud.messages.search.v0.Entity.image:type_name -> opencloud.messages.search.v0.Image
5, // 10: opencloud.messages.search.v0.Entity.photo:type_name -> opencloud.messages.search.v0.Photo
6, // 11: opencloud.messages.search.v0.Match.entity:type_name -> opencloud.messages.search.v0.Entity
12, // [12:12] is the sub-list for method output_type
12, // [12:12] is the sub-list for method input_type
12, // [12:12] is the sub-list for extension type_name
12, // [12:12] is the sub-list for extension extendee
0, // [0:12] is the sub-list for field type_name
7, // 11: opencloud.messages.search.v0.Entity.motionPhoto:type_name -> opencloud.messages.search.v0.MotionPhoto
6, // 12: opencloud.messages.search.v0.Entity.video:type_name -> opencloud.messages.search.v0.Video
8, // 13: opencloud.messages.search.v0.Entity.livePhoto:type_name -> opencloud.messages.search.v0.LivePhoto
9, // 14: opencloud.messages.search.v0.Match.entity:type_name -> opencloud.messages.search.v0.Entity
15, // [15:15] is the sub-list for method output_type
15, // [15:15] is the sub-list for method input_type
15, // [15:15] is the sub-list for extension type_name
15, // [15:15] is the sub-list for extension extendee
0, // [0:15] is the sub-list for field type_name
}
func init() { file_opencloud_messages_search_v0_search_proto_init() }
@@ -1101,7 +1471,7 @@ func file_opencloud_messages_search_v0_search_proto_init() {
}
}
file_opencloud_messages_search_v0_search_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Entity); i {
switch v := v.(*Video); i {
case 0:
return &v.state
case 1:
@@ -1113,6 +1483,42 @@ func file_opencloud_messages_search_v0_search_proto_init() {
}
}
file_opencloud_messages_search_v0_search_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*MotionPhoto); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_opencloud_messages_search_v0_search_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*LivePhoto); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_opencloud_messages_search_v0_search_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Entity); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_opencloud_messages_search_v0_search_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Match); i {
case 0:
return &v.state
@@ -1129,13 +1535,16 @@ func file_opencloud_messages_search_v0_search_proto_init() {
file_opencloud_messages_search_v0_search_proto_msgTypes[3].OneofWrappers = []interface{}{}
file_opencloud_messages_search_v0_search_proto_msgTypes[4].OneofWrappers = []interface{}{}
file_opencloud_messages_search_v0_search_proto_msgTypes[5].OneofWrappers = []interface{}{}
file_opencloud_messages_search_v0_search_proto_msgTypes[6].OneofWrappers = []interface{}{}
file_opencloud_messages_search_v0_search_proto_msgTypes[7].OneofWrappers = []interface{}{}
file_opencloud_messages_search_v0_search_proto_msgTypes[8].OneofWrappers = []interface{}{}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_opencloud_messages_search_v0_search_proto_rawDesc,
NumEnums: 0,
NumMessages: 8,
NumMessages: 11,
NumExtensions: 0,
NumServices: 0,
},
@@ -226,6 +226,42 @@ func (m *Photo) UnmarshalJSON(b []byte) error {
var _ json.Unmarshaler = (*Photo)(nil)
// VideoJSONMarshaler describes the default jsonpb.Marshaler used by all
// instances of Video. This struct is safe to replace or modify but
// should not be done so concurrently.
var VideoJSONMarshaler = new(jsonpb.Marshaler)
// MarshalJSON satisfies the encoding/json Marshaler interface. This method
// uses the more correct jsonpb package to correctly marshal the message.
func (m *Video) MarshalJSON() ([]byte, error) {
if m == nil {
return json.Marshal(nil)
}
buf := &bytes.Buffer{}
if err := VideoJSONMarshaler.Marshal(buf, m); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
var _ json.Marshaler = (*Video)(nil)
// VideoJSONUnmarshaler describes the default jsonpb.Unmarshaler used by all
// instances of Video. This struct is safe to replace or modify but
// should not be done so concurrently.
var VideoJSONUnmarshaler = new(jsonpb.Unmarshaler)
// UnmarshalJSON satisfies the encoding/json Unmarshaler interface. This method
// uses the more correct jsonpb package to correctly unmarshal the message.
func (m *Video) UnmarshalJSON(b []byte) error {
return VideoJSONUnmarshaler.Unmarshal(bytes.NewReader(b), m)
}
var _ json.Unmarshaler = (*Video)(nil)
// EntityJSONMarshaler describes the default jsonpb.Marshaler used by all
// instances of Entity. This struct is safe to replace or modify but
// should not be done so concurrently.
@@ -298,6 +298,15 @@
"items": {
"type": "string"
}
},
"motionPhoto": {
"$ref": "#/definitions/v0MotionPhoto"
},
"video": {
"$ref": "#/definitions/v0Video"
},
"livePhoto": {
"$ref": "#/definitions/v0LivePhoto"
}
}
},
@@ -376,6 +385,29 @@
}
}
},
"v0LivePhoto": {
"type": "object",
"properties": {
"contentId": {
"type": "string"
},
"stillImageTimeUs": {
"type": "string",
"format": "int64"
},
"auto": {
"type": "boolean"
},
"vitalityScore": {
"type": "number",
"format": "double"
},
"vitalityScoringVersion": {
"type": "string",
"format": "int64"
}
}
},
"v0Match": {
"type": "object",
"properties": {
@@ -390,6 +422,23 @@
}
}
},
"v0MotionPhoto": {
"type": "object",
"properties": {
"version": {
"type": "integer",
"format": "int32"
},
"presentationTimestampUs": {
"type": "string",
"format": "int64"
},
"videoSize": {
"type": "string",
"format": "int64"
}
}
},
"v0Photo": {
"type": "object",
"properties": {
@@ -531,6 +580,49 @@
"format": "int32"
}
}
},
"v0Video": {
"type": "object",
"properties": {
"audioBitsPerSample": {
"type": "integer",
"format": "int32"
},
"audioChannels": {
"type": "integer",
"format": "int32"
},
"audioFormat": {
"type": "string"
},
"audioSamplesPerSecond": {
"type": "integer",
"format": "int32"
},
"bitrate": {
"type": "integer",
"format": "int32"
},
"duration": {
"type": "string",
"format": "int64"
},
"fourCC": {
"type": "string"
},
"frameRate": {
"type": "number",
"format": "double"
},
"height": {
"type": "integer",
"format": "int32"
},
"width": {
"type": "integer",
"format": "int32"
}
}
}
},
"externalDocs": {
@@ -59,6 +59,33 @@ message Photo {
optional google.protobuf.Timestamp takenDateTime = 9;
}
message Video {
optional int32 audioBitsPerSample = 1;
optional int32 audioChannels = 2;
optional string audioFormat = 3;
optional int32 audioSamplesPerSecond = 4;
optional int32 bitrate = 5;
optional int64 duration = 6;
optional string fourCC = 7;
optional double frameRate = 8;
optional int32 height = 9;
optional int32 width = 10;
}
message MotionPhoto {
optional int32 version = 1;
optional int64 presentationTimestampUs = 2;
optional int64 videoSize = 3;
}
message LivePhoto {
optional string contentId = 1;
optional int64 stillImageTimeUs = 2;
optional bool auto = 3;
optional double vitalityScore = 4;
optional int64 vitalityScoringVersion = 5;
}
message Entity {
Reference ref = 1;
ResourceID id = 2;
@@ -80,6 +107,9 @@ message Entity {
Image image = 18;
Photo photo = 19;
repeated string favorites = 20;
MotionPhoto motionPhoto = 21;
Video video = 22;
LivePhoto livePhoto = 23;
}
message Match {
+132 -30
View File
@@ -3,17 +3,18 @@ package command
import (
"context"
"fmt"
"strings"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/pkg/runner"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/nats-io/nats.go"
"github.com/olekukonko/errors"
"github.com/spf13/cobra"
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
"github.com/opencloud-eu/opencloud/pkg/generators"
"github.com/opencloud-eu/opencloud/pkg/log"
natspkg "github.com/opencloud-eu/opencloud/pkg/nats"
"github.com/opencloud-eu/opencloud/pkg/registry"
"github.com/opencloud-eu/opencloud/pkg/runner"
ogrpc "github.com/opencloud-eu/opencloud/pkg/service/grpc"
"github.com/opencloud-eu/opencloud/pkg/tracing"
"github.com/opencloud-eu/opencloud/pkg/version"
@@ -24,6 +25,12 @@ import (
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/metrics"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/server/debug"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/server/http"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
svcEvents "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events"
svcHttp "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
var _registeredEvents = []events.Unmarshaller{
@@ -62,19 +69,11 @@ func Server(cfg *config.Config) *cobra.Command {
gr := runner.NewGroup()
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
mtrcs := metrics.New()
mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1)
defer cancel()
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
evStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig(cfg.Events))
if err != nil {
logger.Error().Err(err).Msg("Failed to initialize event stream")
return err
}
tm, err := pool.StringToTLSMode(cfg.GRPCClientTLS.Mode)
if err != nil {
logger.Error().Err(err).Msg("Failed to parse tls mode")
@@ -99,28 +98,101 @@ func Server(cfg *config.Config) *cobra.Command {
return err
}
hClient := ehsvc.NewEventHistoryService("eu.opencloud.api.eventhistory", grpcClient)
vClient := settingssvc.NewValueService("eu.opencloud.api.settings", grpcClient)
kv, err := ConnectNatsKV(cfg.Store)
if err != nil {
return err
}
activityLog, err := activitylog.New(kv,
activitylog.Logger(logger),
activitylog.MaxActivities(cfg.MaxActivities),
activitylog.WriteBufferDuration(cfg.WriteBufferDuration),
)
if err != nil {
logger.Error().Err(err).Msg("Failed to initialize activity log")
return err
}
{
svc, err := http.Server(
http.Logger(logger),
http.Config(cfg),
http.Context(ctx), // NOTE: not passing this "option" leads to a panic in go-micro
http.TraceProvider(tracerProvider),
http.Stream(evStream),
http.GatewaySelector(gatewaySelector),
http.HistoryClient(hClient),
http.ValueClient(vClient),
http.RegisteredEvents(_registeredEvents),
if !cfg.HTTP.Disabled {
hClient := ehsvc.NewEventHistoryService("eu.opencloud.api.eventhistory", grpcClient)
svc, err := svcHttp.New(
activityLog,
svcHttp.Logger(logger),
svcHttp.GatewaySelector(gatewaySelector),
svcHttp.RegisteredEvents(_registeredEvents),
//svcHttp.TraceProvider(tracerProvider),
svcHttp.HistoryClient(hClient),
)
if err != nil {
logger.Error().Err(err).Str("transport", "http").Msg("Failed to initialize server")
logger.Error().Err(err).Msg("handler init")
return err
}
// TODO svc = service.NewInstrument(svc, metrics)
// TODO svc = service.NewLogging(svc, logger) // this logs service specific data
// TODO svc = service.NewTracing(svc, traceProvider)
vClient := settingssvc.NewValueService("eu.opencloud.api.settings", grpcClient)
server, err := http.Server(
http.ValueClient(vClient),
http.Logger(logger),
http.Context(ctx),
http.Config(cfg),
http.Service(svc),
)
if err != nil {
logger.Info().
Err(err).
Str("transport", "http").
Msg("Failed to initialize server")
return err
}
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", svc))
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", server))
} else {
logger.Info().Msg("HTTP server disabled, not starting HTTP service")
}
if !cfg.Events.Disabled {
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
evStream, err := stream.NatsFromConfig(connName, false, stream.NatsConfig{
Endpoint: cfg.Events.Endpoint,
Cluster: cfg.Events.Cluster,
EnableTLS: cfg.Events.EnableTLS,
TLSInsecure: cfg.Events.TLSInsecure,
TLSRootCACertificate: cfg.Events.TLSRootCACertificate,
AuthUsername: cfg.Events.AuthUsername,
AuthPassword: cfg.Events.AuthPassword,
})
if err != nil {
logger.Error().Err(err).Msg("Failed to initialize event stream")
return err
}
eventSvc, err := svcEvents.New(
activityLog,
evStream,
svcEvents.Context(ctx),
svcEvents.Logger(logger),
svcEvents.ServiceAccount(cfg.ServiceAccount),
svcEvents.GatewaySelector(gatewaySelector),
svcEvents.RegisteredEvents(_registeredEvents),
svcEvents.NumConsumers(cfg.NumConsumers),
)
if err != nil {
logger.Error().Err(err).Str("transport", "event").Msg("Failed to initialize server")
return err
}
gr.Add(runner.New(cfg.Service.Name+".svc", func() error {
return eventSvc.Run()
}, func() {
eventSvc.Close()
}))
} else {
logger.Info().Msg("event listening disabled, not starting event service")
}
{
@@ -149,3 +221,33 @@ func Server(cfg *config.Config) *cobra.Command {
},
}
}
func ConnectNatsKV(cfg config.Store) (nats.KeyValue, error) {
// Connect to NATS servers
secureOption := natspkg.Secure(cfg.EnableTLS, cfg.TLSInsecure, cfg.TLSRootCACertificate)
conn, err := nats.Connect(strings.Join(cfg.Nodes, ","), secureOption, nats.UserInfo(cfg.AuthUsername, cfg.AuthPassword))
if err != nil {
return nil, err
}
js, err := conn.JetStream()
if err != nil {
return nil, err
}
kv, err := js.KeyValue(cfg.Database)
if err != nil {
if !errors.Is(err, nats.ErrBucketNotFound) {
return nil, errors.Wrapf(err, "Failed to get bucket (%s)", cfg.Database)
}
kv, err = js.CreateKeyValue(&nats.KeyValueConfig{
Bucket: cfg.Database,
})
if err != nil {
return nil, errors.Wrapf(err, "Failed to create bucket (%s)", cfg.Database)
}
}
return kv, nil
}
@@ -35,10 +35,12 @@ type Config struct {
WriteBufferDuration time.Duration `yaml:"write_buffer_duration" env:"ACTIVITYLOG_WRITE_BUFFER_DURATION" desc:"The duration to wait before flushing the write buffer. This is used to reduce the number of writes to the store." introductionVersion:"4.0.0"`
MaxActivities int `yaml:"max_activities" env:"ACTIVITYLOG_MAX_ACTIVITIES" desc:"The maximum number of activities to keep in the store per resource. If the number of activities exceeds this value, the oldest activities will be removed." introductionVersion:"4.0.0"`
NumConsumers int `yaml:"num_consumers" env:"ACTIVITYLOG_NUM_CONSUMERS" desc:"The amount of concurrent event consumers to start. Event consumers are used for updating the list of activities. Multiple consumers increase parallelisation, but will also increase CPU and memory demands." introductionVersion:"%NEXT%"`
}
// Events combines the configuration options for the event bus.
type Events struct {
Disabled bool `yaml:"disabled" env:"ACTIVITYLOG_EVENTS_DISABLED" desc:"Disables listening for events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%NEXT%"`
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system." introductionVersion:"1.0.0"`
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
@@ -77,6 +79,7 @@ type CORS struct {
// HTTP defines the available http configuration.
type HTTP struct {
Disabled bool `yaml:"disabled" env:"ACTIVITYLOG_HTTP_DISABLED" desc:"Disables the HTTP service. Set this to true if the service should only handle events." introductionVersion:"1.0.0"`
Addr string `yaml:"addr" env:"ACTIVITYLOG_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
Namespace string `yaml:"-"`
Root string `yaml:"root" env:"ACTIVITYLOG_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
@@ -52,7 +52,9 @@ func DefaultConfig() *config.Config {
},
},
WriteBufferDuration: 10 * time.Second,
MaxActivities: 6000,
// Nats runs into max payload exceeded errors at around 7k activities. Let's keep a buffer.
MaxActivities: 6000,
NumConsumers: 1,
}
}
@@ -4,6 +4,7 @@ import (
"errors"
occfg "github.com/opencloud-eu/opencloud/pkg/config"
"github.com/opencloud-eu/opencloud/pkg/shared"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config/defaults"
@@ -34,5 +35,8 @@ func ParseConfig(cfg *config.Config) error {
// Validate validates the config
func Validate(cfg *config.Config) error {
if cfg.Events.Disabled && cfg.HTTP.Disabled {
return shared.AllComponentsDisabledError(cfg.Service.Name)
}
return nil
}
+23 -71
View File
@@ -3,18 +3,12 @@ package http
import (
"context"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/log"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/metrics"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/spf13/pflag"
"go-micro.dev/v4/store"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/noop"
)
// Option defines a single option function.
@@ -22,19 +16,15 @@ type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Logger log.Logger
Context context.Context
Config *config.Config
Metrics *metrics.Metrics
Flags []pflag.Flag
Namespace string
Store store.Store
Stream events.Stream
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
TraceProvider trace.TracerProvider
HistoryClient ehsvc.EventHistoryService
ValueClient settingssvc.ValueService
RegisteredEvents []events.Unmarshaller
Name string
Namespace string
Logger log.Logger
Context context.Context
Config *config.Config
Flags []pflag.Flag
Service ActivityLogService
TraceProvider trace.TracerProvider
ValueClient settingssvc.ValueService
}
// newOptions initializes the available default options.
@@ -69,10 +59,10 @@ func Config(val *config.Config) Option {
}
}
// Metrics provides a function to set the metrics option.
func Metrics(val *metrics.Metrics) Option {
// Service provides a function to set the service option.
func Service(val ActivityLogService) Option {
return func(o *Options) {
o.Metrics = val
o.Service = val
}
}
@@ -83,58 +73,20 @@ func Flags(flags ...pflag.Flag) Option {
}
}
// Namespace provides a function to set the Namespace option.
func Namespace(val string) Option {
// TraceProvider provides a function to configure the trace provider
func TraceProvider(traceProvider trace.TracerProvider) Option {
return func(o *Options) {
o.Namespace = val
if traceProvider != nil {
o.TraceProvider = traceProvider
} else {
o.TraceProvider = noop.NewTracerProvider()
}
}
}
// Store provides a function to configure the store
func Store(store store.Store) Option {
// ValueClient adds a grpc client for the value service
func ValueClient(vs settingssvc.ValueService) Option {
return func(o *Options) {
o.Store = store
}
}
// Stream provides a function to configure the stream
func Stream(stream events.Stream) Option {
return func(o *Options) {
o.Stream = stream
}
}
// GatewaySelector provides a function to configure the gateway client selector
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
return func(o *Options) {
o.GatewaySelector = gatewaySelector
}
}
// HistoryClient provides a function to configure the event history client
func HistoryClient(h ehsvc.EventHistoryService) Option {
return func(o *Options) {
o.HistoryClient = h
}
}
// RegisteredEvents provides a function to register events
func RegisteredEvents(evs []events.Unmarshaller) Option {
return func(o *Options) {
o.RegisteredEvents = evs
}
}
// TraceProvider provides a function to set the TracerProvider option
func TraceProvider(val trace.TracerProvider) Option {
return func(o *Options) {
o.TraceProvider = val
}
}
// ValueClient provides a function to set the ValueClient options
func ValueClient(val settingssvc.ValueService) Option {
return func(o *Options) {
o.ValueClient = val
o.ValueClient = vs
}
}
+103 -46
View File
@@ -1,49 +1,63 @@
package http
import (
"fmt"
stdhttp "net/http"
"context"
"embed"
"encoding/json"
"errors"
"net/http"
"github.com/go-chi/chi/v5"
chimiddleware "github.com/go-chi/chi/v5/middleware"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/pkg/account"
"github.com/opencloud-eu/opencloud/pkg/cors"
"github.com/opencloud-eu/opencloud/pkg/l10n"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/pkg/middleware"
"github.com/opencloud-eu/opencloud/pkg/service/http"
"github.com/opencloud-eu/opencloud/pkg/tracing"
ohttp "github.com/opencloud-eu/opencloud/pkg/service/http"
"github.com/opencloud-eu/opencloud/pkg/version"
svc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"
"github.com/riandyrn/otelchi"
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
activityloghttp "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"go-micro.dev/v4"
"google.golang.org/grpc/metadata"
)
// Service is the service interface
type Service any
var (
//go:embed l10n/locale
_localeFS embed.FS
// subfolder where the translation files are stored
_localeSubPath = "l10n/locale"
// domain of the activitylog service (transifex)
_domain = "activitylog"
)
// Server initializes the http service and server.
func Server(opts ...Option) (http.Service, error) {
func Server(opts ...Option) (ohttp.Service, error) {
options := newOptions(opts...)
service := options.Service
service, err := http.NewService(
http.TLSConfig(options.Config.HTTP.TLS),
http.Logger(options.Logger),
http.Namespace(options.Config.HTTP.Namespace),
http.Name(options.Config.Service.Name),
http.Version(version.GetString()),
http.Address(options.Config.HTTP.Addr),
http.Context(options.Context),
http.Flags(options.Flags...),
http.TraceProvider(options.TraceProvider),
newService, err := ohttp.NewService(
ohttp.TLSConfig(options.Config.HTTP.TLS),
ohttp.Logger(options.Logger),
ohttp.Namespace(options.Config.HTTP.Namespace),
ohttp.Name(options.Config.Service.Name),
ohttp.Version(version.GetString()),
ohttp.Address(options.Config.HTTP.Addr),
ohttp.Context(options.Context),
ohttp.Flags(options.Flags...),
)
if err != nil {
options.Logger.Error().
Err(err).
Msg("Error initializing http service")
return http.Service{}, fmt.Errorf("could not initialize http service: %w", err)
return ohttp.Service{}, err
}
middlewares := []func(stdhttp.Handler) stdhttp.Handler{
middlewares := []func(http.Handler) http.Handler{
chimiddleware.RequestID,
middleware.Version(
options.Config.Service.Name,
@@ -52,6 +66,7 @@ func Server(opts ...Option) (http.Service, error) {
middleware.Logger(
options.Logger,
),
middleware.TraceContext,
middleware.ExtractAccountUUID(
account.Logger(options.Logger),
account.JWTSecret(options.Config.TokenManager.JWTSecret),
@@ -68,33 +83,75 @@ func Server(opts ...Option) (http.Service, error) {
mux := chi.NewMux()
mux.Use(middlewares...)
mux.Use(
otelchi.Middleware(
"actitivylog",
otelchi.WithChiRoutes(mux),
otelchi.WithTracerProvider(options.TraceProvider),
otelchi.WithPropagators(tracing.GetPropagator()),
),
)
t := l10n.NewTranslatorFromCommonConfig(options.Config.DefaultLanguage, _domain, options.Config.TranslationPath, _localeFS, _localeSubPath)
mux.Route(options.Config.HTTP.Root, func(r chi.Router) {
r.Get("/graph/v1beta1/extensions/org.libregraph/activities", GetItemActivitiesHandler(options.Logger, service, options.ValueClient, t))
})
handle, err := svc.New(
svc.Logger(options.Logger),
svc.Stream(options.Stream),
svc.Mux(mux),
svc.Config(options.Config),
svc.GatewaySelector(options.GatewaySelector),
svc.TraceProvider(options.TraceProvider),
svc.HistoryClient(options.HistoryClient),
svc.ValueClient(options.ValueClient),
svc.RegisteredEvents(options.RegisteredEvents),
)
err = micro.RegisterHandler(newService.Server(), mux)
if err != nil {
return http.Service{}, err
options.Logger.Fatal().Err(err).Msg("failed to register the handler")
}
if err := micro.RegisterHandler(service.Server(), handle); err != nil {
return http.Service{}, err
}
newService.Init()
return newService, nil
return service, nil
}
// Service defines the business logic implementations need to provide.
type ActivityLogService interface {
GetItemActivities(ctx context.Context, query, loc string, t l10n.Translator) ([]libregraph.Activity, error)
}
// GetActivitiesResponse is the response on GET activities requests
type GetActivitiesResponse struct {
Activities []libregraph.Activity `json:"value"`
}
func GetItemActivitiesHandler(log log.Logger, s ActivityLogService, vc settingssvc.ValueService, t l10n.Translator) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, r.Header.Get(revactx.TokenHeader))
activeUser, ok := revactx.ContextGetUser(ctx)
if !ok {
w.WriteHeader(http.StatusUnauthorized)
return
}
loc := l10n.MustGetUserLocale(ctx, activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), vc)
activities, err := s.GetItemActivities(ctx, r.URL.Query().Get("kql"), loc, t)
if err != nil {
switch {
case errors.Is(err, activityloghttp.ErrBadRequest):
log.Debug().Str("query", r.URL.Query().Get("kql")).Err(err).Msg("error getting activities")
w.WriteHeader(http.StatusBadRequest)
return
case errors.Is(err, activityloghttp.ErrForbidden):
log.Debug().Err(err).Msg("error getting activities")
w.WriteHeader(http.StatusForbidden)
return
default:
log.Error().Err(err).Msg("error getting activities")
w.WriteHeader(http.StatusInternalServerError)
return
}
}
res := GetActivitiesResponse{
Activities: activities,
}
w.Header().Set("Content-Type", "application/json; odata.metadata=minimal")
w.Header().Set("OData-Version", "4.0")
if reqID := chimiddleware.GetReqID(ctx); reqID != "" {
w.Header().Set("request-id", reqID)
}
w.Header().Set("Cache-Control", "no-cache")
if err := json.NewEncoder(w).Encode(res); err != nil {
log.Error().Err(err).Msg("error encoding activities")
w.WriteHeader(http.StatusInternalServerError)
return
}
}
}
@@ -0,0 +1,366 @@
package activitylog
import (
"context"
"encoding/base32"
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/jellydator/ttlcache/v2"
"github.com/nats-io/nats.go"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/vmihailenco/msgpack/v5"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
var tracer trace.Tracer
func init() {
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog")
}
var (
_maxActivitiesDefault = 6000
_writeBufferDuration = 10 * time.Second
)
// Activitylog stores and retrieves activities for resources and their parents from a nats kv
type ActivityLog struct {
log log.Logger
// FIXME the lock does not protect agains concurrent resource activities on multiple instances
// known since https://github.com/owncloud/ocis/pull/9361#pullrequestreview-2135350157
// current ocis discussion in https://github.com/owncloud/ocis/issues/12475
lock sync.RWMutex
debouncer *Debouncer
parentIdCache *ttlcache.Cache
natskv nats.KeyValue
maxActivities int
}
type batchInfo struct {
key string
count int
timestamp time.Time
}
// New creates a new ActivitylogService
func New(kv nats.KeyValue, opts ...Option) (*ActivityLog, error) {
o := &Options{
MaxActivities: _maxActivitiesDefault,
WriteBufferDuration: _writeBufferDuration,
Logger: log.NopLogger(),
}
for _, opt := range opts {
opt(o)
}
cache := ttlcache.NewCache()
err := cache.SetTTL(30 * time.Second)
if err != nil {
return nil, err
}
s := &ActivityLog{
log: o.Logger,
lock: sync.RWMutex{},
parentIdCache: cache,
maxActivities: o.MaxActivities,
natskv: kv,
}
s.debouncer = NewDebouncer(o.WriteBufferDuration, s.StoreActivity)
// run migrations
err = s.runMigrations(context.Background(), kv)
if err != nil {
return nil, err
}
return s, nil
}
// RemoveResource removes the resource from the store
func (a *ActivityLog) RemoveResource(rid *provider.ResourceId) error {
if rid == nil {
return fmt.Errorf("resource id is required")
}
a.lock.Lock()
defer a.lock.Unlock()
err := a.natskv.Delete(storagespace.FormatResourceID(rid))
if err != nil {
return fmt.Errorf("could not delete resource %s: %w", rid.OpaqueId, err)
}
return nil
}
// RawActivity represents an activity as it is stored in the activitylog store
type RawActivity struct {
EventID string `json:"event_id"`
Depth int `json:"depth"`
Timestamp time.Time `json:"timestamp"`
}
func (a *ActivityLog) AddActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time, getResource func(context.Context, *provider.Reference) (*provider.ResourceInfo, error)) error {
var (
err error
depth int
ref = initRef
)
ctx, span := tracer.Start(ctx, "AddActivity")
defer span.End()
for {
var info *provider.ResourceInfo
id := ref.GetResourceId()
if ref.Path != "" {
// Path based reference, we need to resolve the resource id
ctx, span = tracer.Start(ctx, "AddActivity.getResource")
info, err = getResource(ctx, ref)
span.End()
if err != nil {
// TODO If the resource was deleted should we still log an activity in the parent?
return fmt.Errorf("could not get resource info for reference %v: %w", ref, err)
}
id = info.GetId()
}
if id == nil {
return fmt.Errorf("resource id is required")
}
key := storagespace.FormatResourceID(id)
a.debouncer.Debounce(key, RawActivity{
EventID: eventID,
Depth: depth,
Timestamp: timestamp,
})
if id.OpaqueId == id.SpaceId {
// we are at the root of the space, no need to go further
break
}
// check if parent id is cached
// parent id is cached in the format <storageid>$<spaceid>!<resourceid>
// if it is not cached, get the resource info and cache it
if parentId == nil {
if v, err := a.parentIdCache.Get(key); err != nil {
if info == nil {
ctx, span := tracer.Start(ctx, "AddActivity.getResource parent")
info, err = getResource(ctx, ref)
span.End()
if err != nil || info.GetParentId() == nil || info.GetParentId().GetOpaqueId() == "" {
return fmt.Errorf("could not get parent id: %w", err)
}
}
parentId = info.GetParentId()
a.parentIdCache.Set(key, parentId)
} else {
parentId = v.(*provider.ResourceId)
}
} else {
a.log.Debug().Msg("parent id is cached")
}
depth++
ref = &provider.Reference{ResourceId: parentId}
parentId = nil // reset parent id so it's not reused in the next iteration
}
return nil
}
func (a *ActivityLog) StoreActivity(resourceID string, activities []RawActivity) error {
a.lock.Lock()
defer a.lock.Unlock()
ctx, span := tracer.Start(context.Background(), "storeActivity")
defer span.End()
_, subspan := tracer.Start(ctx, "storeActivity.Marshal")
b, err := msgpack.Marshal(activities)
if err != nil {
return err
}
subspan.End()
_, subspan = tracer.Start(ctx, "storeActivity.natskv.Put")
key := natsKey(resourceID, len(activities))
_, err = a.natskv.Put(key, b)
if err != nil {
return err
}
subspan.End()
ctx, subspan = tracer.Start(ctx, "storeActivity.enforceMaxActivities")
a.enforceMaxActivities(ctx, resourceID)
subspan.End()
return nil
}
func (a *ActivityLog) enforceMaxActivities(ctx context.Context, resourceID string) {
if a.maxActivities <= 0 {
return
}
key := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
_, subspan := tracer.Start(ctx, "enforceMaxActivities.watch")
watcher, err := a.natskv.Watch(key, nats.IgnoreDeletes())
if err != nil {
a.log.Error().Err(err).Str("resourceID", resourceID).Msg("could not watch")
return
}
defer watcher.Stop()
var keys []string
for update := range watcher.Updates() {
if update == nil {
break
}
var batchActivities []RawActivity
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
}
keys = append(keys, update.Key())
}
subspan.End()
_, subspan = tracer.Start(ctx, "enforceMaxActivities.compile")
// Parse keys into batches
batches := make([]batchInfo, 0)
var activitiesCount int
for _, k := range keys {
parts := strings.SplitN(k, ".", 3)
if len(parts) < 3 {
a.log.Warn().Str("key", k).Msg("skipping key, not enough parts")
continue
}
c, err := strconv.Atoi(parts[1])
if err != nil {
a.log.Warn().Str("key", k).Msg("skipping key, can not parse count")
continue
}
// parse timestamp
nano, err := strconv.ParseInt(parts[2], 10, 64)
if err != nil {
a.log.Warn().Str("key", k).Msg("skipping key, can not parse timestamp")
continue
}
batches = append(batches, batchInfo{
key: k,
count: c,
timestamp: time.Unix(0, nano),
})
activitiesCount += c
}
// sort batches by timestamp
sort.Slice(batches, func(i, j int) bool {
return batches[i].timestamp.Before(batches[j].timestamp)
})
subspan.End()
_, subspan = tracer.Start(ctx, "enforceMaxActivities.delete")
// remove oldest keys until we are at max activities
for _, b := range batches {
if activitiesCount-b.count < a.maxActivities {
break
}
activitiesCount -= b.count
err = a.natskv.Delete(b.key)
if err != nil {
a.log.Error().Err(err).Str("key", b.key).Msg("could not delete key")
break
}
}
subspan.End()
}
func (a *ActivityLog) InvalidateCachedParentID(purgeId *provider.ResourceId) {
// The parent id cache is populated lazily and its entries expire, so a
// missing key is the expected case rather than an error.
if err := a.parentIdCache.Remove(storagespace.FormatResourceID(purgeId)); err != nil {
a.log.Debug().Interface("event", purgeId).Err(err).Msg("could not delete parent id cache")
}
}
func natsKey(resourceID string, activitiesCount int) string {
return fmt.Sprintf("%s.%d.%d",
base32.StdEncoding.EncodeToString([]byte(resourceID)),
activitiesCount,
time.Now().UnixNano())
}
func (a *ActivityLog) Activities(rid *provider.ResourceId) ([]RawActivity, error) {
a.lock.RLock()
defer a.lock.RUnlock()
return a.activities(rid)
}
func (a *ActivityLog) activities(rid *provider.ResourceId) ([]RawActivity, error) {
resourceID := storagespace.FormatResourceID(rid)
glob := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
watcher, err := a.natskv.Watch(glob, nats.IgnoreDeletes())
if err != nil {
return nil, err
}
defer watcher.Stop()
var activities []RawActivity
for update := range watcher.Updates() {
if update == nil {
break
}
var batchActivities []RawActivity
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack")
}
activities = append(activities, batchActivities...)
}
return activities, nil
}
// RemoveActivities removes the activities from the given resource
func (a *ActivityLog) RemoveActivities(rid *provider.ResourceId, toDelete map[string]struct{}) error {
a.lock.Lock()
defer a.lock.Unlock()
curActivities, err := a.activities(rid)
if err != nil {
return err
}
var acts []RawActivity
for _, a := range curActivities {
if _, ok := toDelete[a.EventID]; !ok {
acts = append(acts, a)
}
}
b, err := json.Marshal(acts)
if err != nil {
return err
}
_, err = a.natskv.Put(storagespace.FormatResourceID(rid), b)
return err
}
@@ -0,0 +1,13 @@
package activitylog_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestActivitylog(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Activitylog Suite")
}
@@ -1,7 +1,6 @@
package service
package activitylog_test
import (
"bytes"
"context"
"net"
"os"
@@ -9,17 +8,13 @@ import (
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
nserver "github.com/nats-io/nats-server/v2/server"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/command"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
eventsmocks "github.com/opencloud-eu/reva/v2/pkg/events/mocks"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/test-go/testify/mock"
"go.opentelemetry.io/otel/trace/noop"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
)
var (
@@ -68,31 +63,25 @@ var _ = SynchronizedAfterSuite(func() {
var _ = Describe("ActivitylogService", func() {
var (
alog *ActivitylogService
alog *activitylog.ActivityLog
getResource func(_ context.Context, ref *provider.Reference) (*provider.ResourceInfo, error)
writebufferduration = 100 * time.Millisecond
)
JustBeforeEach(func() {
var err error
stream := &eventsmocks.Stream{}
stream.EXPECT().Consume(mock.Anything, mock.Anything).Return(nil, nil)
alog, err = New(
Config(&config.Config{
Service: config.Service{
Name: "activitylog-test",
},
Store: config.Store{
Store: "nats-js-kv",
Nodes: []string{server.Addr().String()},
Database: "activitylog-test-" + uuid.New().String(),
},
MaxActivities: 4,
WriteBufferDuration: writebufferduration,
}),
Stream(stream),
TraceProvider(noop.NewTracerProvider()),
Mux(chi.NewMux()),
db := "activitylog-test-" + uuid.New().String()
kv, err := command.ConnectNatsKV(config.Store{
Nodes: []string{server.Addr().String()},
Database: db,
})
Expect(err).ToNot(HaveOccurred())
alog, err = activitylog.New(
kv,
activitylog.MaxActivities(4),
activitylog.WriteBufferDuration(writebufferduration),
)
Expect(err).ToNot(HaveOccurred())
})
@@ -107,7 +96,7 @@ var _ = Describe("ActivitylogService", func() {
Name string
Tree map[string]*provider.ResourceInfo
Activities map[string]string
Expected map[string][]RawActivity
Expected map[string][]activitylog.RawActivity
}
testCases := []testCase{
@@ -121,7 +110,7 @@ var _ = Describe("ActivitylogService", func() {
Activities: map[string]string{
"activity": "base",
},
Expected: map[string][]RawActivity{
Expected: map[string][]activitylog.RawActivity{
"base": activitites("activity", 0),
"parent": activitites("activity", 1),
"spaceid": activitites("activity", 2),
@@ -138,7 +127,7 @@ var _ = Describe("ActivitylogService", func() {
"activity1": "base",
"activity2": "base",
},
Expected: map[string][]RawActivity{
Expected: map[string][]activitylog.RawActivity{
"base": activitites("activity1", 0, "activity2", 0),
"parent": activitites("activity1", 1, "activity2", 1),
"spaceid": activitites("activity1", 2, "activity2", 2),
@@ -155,7 +144,7 @@ var _ = Describe("ActivitylogService", func() {
}
for k, v := range tc.Activities {
err := alog.addActivity(context.Background(), reference(v), nil, k, time.Time{}, getResource)
err := alog.AddActivity(context.Background(), reference(v), nil, k, time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
}
})
@@ -194,9 +183,9 @@ var _ = Describe("ActivitylogService", func() {
It("debounces activities", func() {
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
@@ -207,7 +196,7 @@ var _ = Describe("ActivitylogService", func() {
})
It("adheres to the MaxActivities setting", func() {
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
activities, err := alog.Activities(resourceID("base"))
@@ -215,7 +204,7 @@ var _ = Describe("ActivitylogService", func() {
g.Expect(len(activities)).To(Equal(1))
}).Should(Succeed())
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
activities, err := alog.Activities(resourceID("base"))
@@ -223,11 +212,11 @@ var _ = Describe("ActivitylogService", func() {
g.Expect(len(activities)).To(Equal(2))
}).Should(Succeed())
err = alog.addActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
err = alog.addActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
err = alog.addActivity(context.Background(), reference("base"), nil, "activity5", time.Time{}, getResource)
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity5", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
@@ -244,9 +233,9 @@ var _ = Describe("ActivitylogService", func() {
return tree[ref.GetResourceId().GetOpaqueId()], nil
}
err := alog.addActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
err := alog.AddActivity(context.Background(), reference("base"), nil, "activity1", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
err = alog.addActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity2", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
@@ -255,9 +244,9 @@ var _ = Describe("ActivitylogService", func() {
g.Expect(activities).To(ConsistOf(activitites("activity1", 0, "activity2", 0)))
}).Should(Succeed())
err = alog.addActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity3", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
err = alog.addActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
err = alog.AddActivity(context.Background(), reference("base"), nil, "activity4", time.Time{}, getResource)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
@@ -268,47 +257,11 @@ var _ = Describe("ActivitylogService", func() {
})
})
})
Describe("removeCachedParentID", func() {
var logBuffer *bytes.Buffer
newLoggerAtLevel := func(level string) log.Logger {
logBuffer = &bytes.Buffer{}
return log.Logger{Logger: log.NewLogger(log.Level(level)).Output(logBuffer)}
}
It("does not log an error when the entry was never cached", func() {
alog.log = newLoggerAtLevel("error")
alog.removeCachedParentID(reference("never-cached"))
Expect(logBuffer.String()).To(BeEmpty())
})
It("logs a missing entry at debug level", func() {
alog.log = newLoggerAtLevel("debug")
alog.removeCachedParentID(reference("never-cached"))
Expect(logBuffer.String()).To(ContainSubstring("could not delete parent id cache"))
Expect(logBuffer.String()).To(ContainSubstring(`"level":"debug"`))
})
It("does not log at all when the entry was cached", func() {
alog.log = newLoggerAtLevel("debug")
ref := reference("cached")
Expect(alog.parentIdCache.Set(storagespace.FormatResourceID(ref.GetResourceId()), resourceID("parent"))).To(Succeed())
alog.removeCachedParentID(ref)
Expect(logBuffer.String()).To(BeEmpty())
})
})
})
func activitites(acts ...any) []RawActivity {
var activities []RawActivity
act := RawActivity{}
func activitites(acts ...any) []activitylog.RawActivity {
var activities []activitylog.RawActivity
act := activitylog.RawActivity{}
for _, a := range acts {
switch v := a.(type) {
case string:
@@ -0,0 +1,76 @@
package activitylog
import (
"sync"
"time"
)
// Debouncer is used to debounce writes to the activity log store.
type Debouncer struct {
after time.Duration
f func(id string, ra []RawActivity) error
pending sync.Map
inProgress sync.Map
mutex sync.Mutex
}
type queueItem struct {
activities []RawActivity
timer *time.Timer
}
// NewDebouncer returns a new Debouncer instance.
func NewDebouncer(d time.Duration, f func(id string, ra []RawActivity) error) *Debouncer {
return &Debouncer{
after: d,
f: f,
pending: sync.Map{},
inProgress: sync.Map{},
}
}
// Debounce restarts the debounce timer for the given space.
func (d *Debouncer) Debounce(id string, ra RawActivity) {
if d.after == 0 {
d.f(id, []RawActivity{ra})
return
}
d.mutex.Lock()
defer d.mutex.Unlock()
item := &queueItem{
activities: []RawActivity{ra},
}
if i, ok := d.pending.Load(id); ok {
// if the item is already in the queue, append the new activities
item, ok = i.(*queueItem)
if ok {
item.activities = append(item.activities, ra)
}
}
if item.timer == nil {
item.timer = time.AfterFunc(d.after, func() {
if _, ok := d.inProgress.Load(id); ok {
// Reschedule this run for when the previous run has finished
d.mutex.Lock()
if i, ok := d.pending.Load(id); ok {
i.(*queueItem).timer.Reset(d.after)
}
d.mutex.Unlock()
return
}
d.pending.Delete(id)
d.inProgress.Store(id, true)
defer d.inProgress.Delete(id)
d.f(id, item.activities)
})
}
d.pending.Store(id, item)
}
@@ -0,0 +1,121 @@
package activitylog_test
import (
"sync"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
)
var _ = Describe("Debouncer", func() {
var (
mu sync.Mutex
callbacks []activitylog.RawActivity
newCallback func(id string, ra []activitylog.RawActivity) error
)
BeforeEach(func() {
mu.Lock()
callbacks = nil
mu.Unlock()
newCallback = func(id string, ra []activitylog.RawActivity) error {
mu.Lock()
defer mu.Unlock()
callbacks = append(callbacks, ra...)
return nil
}
})
Context("with zero duration", func() {
It("calls the callback immediately", func() {
d := activitylog.NewDebouncer(0, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
Expect(callbacks).To(HaveLen(1))
Expect(callbacks[0].EventID).To(Equal("activity1"))
})
It("calls the callback immediately for each event", func() {
d := activitylog.NewDebouncer(0, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
d.Debounce("space2", activitylog.RawActivity{EventID: "activity2"})
Expect(callbacks).To(HaveLen(2))
})
})
Context("with non-zero duration", func() {
It("batches activities with the same id", func() {
d := activitylog.NewDebouncer(10*time.Millisecond, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
d.Debounce("space1", activitylog.RawActivity{EventID: "activity3"})
Eventually(func() int {
mu.Lock()
defer mu.Unlock()
return len(callbacks)
}).Should(Equal(3))
})
It("handles different ids independently", func() {
d := activitylog.NewDebouncer(10*time.Millisecond, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
d.Debounce("space2", activitylog.RawActivity{EventID: "activity2"})
Eventually(func() int {
mu.Lock()
defer mu.Unlock()
return len(callbacks)
}).Should(Equal(2))
})
It("batches activities that arrive within the debounce window", func() {
d := activitylog.NewDebouncer(100*time.Millisecond, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
time.Sleep(20 * time.Millisecond)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
time.Sleep(20 * time.Millisecond)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity3"})
Eventually(func() int {
mu.Lock()
defer mu.Unlock()
return len(callbacks)
}).Should(Equal(3))
})
It("processes new batch after previous completes", func() {
d := activitylog.NewDebouncer(5*time.Millisecond, newCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
time.Sleep(20 * time.Millisecond) // let first batch complete
d.Debounce("space1", activitylog.RawActivity{EventID: "activity2"})
Eventually(func() int {
mu.Lock()
defer mu.Unlock()
return len(callbacks)
}).Should(Equal(2))
})
It("skips duplicate write when timer fires during in-progress callback", func() {
slowCallback := func(id string, ra []activitylog.RawActivity) error {
time.Sleep(50 * time.Millisecond) // simulate slow write
mu.Lock()
defer mu.Unlock()
callbacks = append(callbacks, ra...)
return nil
}
d := activitylog.NewDebouncer(10*time.Millisecond, slowCallback)
d.Debounce("space1", activitylog.RawActivity{EventID: "activity1"})
time.Sleep(20 * time.Millisecond) // timer fires while callback is running
Eventually(func() int {
mu.Lock()
defer mu.Unlock()
return len(callbacks)
}).Should(Equal(1))
})
})
})
@@ -1,4 +1,4 @@
package service
package activitylog
import (
"context"
@@ -15,7 +15,7 @@ const currentMigrationVersion = "1"
// RunMigrations checks the activitylog data version and runs migrations if necessary.
// It should be called during service startup, after the NATS KeyValue store is initialized.
func (a *ActivitylogService) runMigrations(ctx context.Context, kv nats.KeyValue) error {
func (a *ActivityLog) runMigrations(ctx context.Context, kv nats.KeyValue) error {
entry, err := kv.Get(activitylogVersionKey)
if err == nats.ErrKeyNotFound {
a.log.Info().Msg("activitylog version key not found. Running migration to V1...")
@@ -40,7 +40,7 @@ func (a *ActivitylogService) runMigrations(ctx context.Context, kv nats.KeyValue
// For each such key, it creates a new key in the format "originalKey.count.timestamp"
// and stores the original list of strings (re-marshalled to messagepack) as its value.
// Finally, it sets the activitylog.version key to "1".
func (a *ActivitylogService) migrateToV1(_ context.Context, kv nats.KeyValue) error {
func (a *ActivityLog) migrateToV1(_ context.Context, kv nats.KeyValue) error {
lister, err := kv.ListKeys()
if err != nil {
return fmt.Errorf("migrateToV1: failed to list keys from NATS KV store: %w", err)
@@ -0,0 +1,35 @@
package activitylog
import (
"time"
"github.com/opencloud-eu/opencloud/pkg/log"
)
// Option for the activitylog service
type Option func(*Options)
// Options for the activitylog service
type Options struct {
Logger log.Logger
MaxActivities int
WriteBufferDuration time.Duration
}
// Logger configures a logger for the activitylog service
func Logger(log log.Logger) Option {
return func(o *Options) {
o.Logger = log
}
}
func MaxActivities(max int) Option {
return func(o *Options) {
o.MaxActivities = max
}
}
func WriteBufferDuration(d time.Duration) Option {
return func(o *Options) {
o.WriteBufferDuration = d
}
}
@@ -1,4 +1,4 @@
package service_test
package events_test
import (
"testing"
@@ -7,7 +7,7 @@ import (
. "github.com/onsi/gomega"
)
func TestService(t *testing.T) {
func TestEvents(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Service Suite")
RunSpecs(t, "Events Suite")
}
@@ -1,17 +1,14 @@
package service
package events
import (
"context"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/go-chi/chi/v5"
"github.com/opencloud-eu/opencloud/pkg/log"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"go.opentelemetry.io/otel/trace"
)
// Option for the activitylog service
@@ -19,17 +16,20 @@ type Option func(*Options)
// Options for the activitylog service
type Options struct {
Context context.Context
Logger log.Logger
Config *config.Config
TraceProvider trace.TracerProvider
ServiceAccount config.ServiceAccount
Stream events.Stream
RegisteredEvents []events.Unmarshaller
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
Mux *chi.Mux
HistoryClient ehsvc.EventHistoryService
ValueClient settingssvc.ValueService
WriteBufferDuration time.Duration
MaxActivities int
NumConsumers int
}
func Context(ctx context.Context) Option {
return func(o *Options) {
o.Context = ctx
}
}
// Logger configures a logger for the activitylog service
@@ -39,17 +39,10 @@ func Logger(log log.Logger) Option {
}
}
// Config adds the config for the activitylog service
func Config(c *config.Config) Option {
// ServiceAccount configures a service account for the activitylog service
func ServiceAccount(sa config.ServiceAccount) Option {
return func(o *Options) {
o.Config = c
}
}
// TraceProvider adds a tracer provider for the activitylog service
func TraceProvider(tp trace.TracerProvider) Option {
return func(o *Options) {
o.TraceProvider = tp
o.ServiceAccount = sa
}
}
@@ -74,23 +67,8 @@ func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient])
}
}
// Mux defines the muxer for the service
func Mux(m *chi.Mux) Option {
func NumConsumers(num int) Option {
return func(o *Options) {
o.Mux = m
}
}
// HistoryClient adds a grpc client for the eventhistory service
func HistoryClient(hc ehsvc.EventHistoryService) Option {
return func(o *Options) {
o.HistoryClient = hc
}
}
// ValueClient adds a grpc client for the value service
func ValueClient(vs settingssvc.ValueService) Option {
return func(o *Options) {
o.ValueClient = vs
o.NumConsumers = num
}
}
@@ -0,0 +1,302 @@
package events
import (
"context"
"fmt"
"path/filepath"
"sync"
"sync/atomic"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
var tracer trace.Tracer
func init() {
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events")
}
var (
_numConsumersDefault = 1
)
// ActivitylogService logs events per resource
type ActivitylogService struct {
ctx context.Context
sa config.ServiceAccount
log log.Logger
stream events.Stream
gws pool.Selectable[gateway.GatewayAPIClient]
al *activitylog.ActivityLog
numConsumers int
events []events.Unmarshaller
stopCh chan struct{}
stopped *atomic.Bool
}
// New creates a new ActivitylogService
func New(al *activitylog.ActivityLog, stream events.Stream, opts ...Option) (*ActivitylogService, error) {
o := &Options{
NumConsumers: _numConsumersDefault,
}
for _, opt := range opts {
opt(o)
}
s := &ActivitylogService{
ctx: o.Context,
log: o.Logger,
sa: o.ServiceAccount,
stream: stream,
gws: o.GatewaySelector,
events: o.RegisteredEvents,
numConsumers: o.NumConsumers,
al: al,
stopCh: make(chan struct{}, 1),
stopped: new(atomic.Bool),
}
return s, nil
}
// Run to fulfil Runner interface
func (s *ActivitylogService) Run() error {
ch, err := events.Consume(s.stream, "activitylog", s.events...)
if err != nil {
return err
}
var wg sync.WaitGroup
ctx, cancel := context.WithCancel(s.ctx)
defer cancel()
s.log.Debug().Int("worker.count", s.numConsumers).
Str("messaging.consumer.group.name", "activitylog").
Str("messaging.system", "nats").
Str("messaging.operation.name", "receive").
Msg("starting event processing workers")
// start workers
for i := 0; i < s.numConsumers; i++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
for {
select {
case <-ctx.Done():
return
case e, ok := <-ch:
if !ok {
return
}
if err := s.processEvent(e); err != nil {
s.log.Error().Err(err).
Int("worker", workerID).
Interface("event", e).
Msg("failed to process event")
}
}
}
}(i)
}
// wait for stop signal
<-s.stopCh
cancel() // signal workers to stop
wg.Wait()
return nil
}
// Close will make the service to stop processing, so the `Run`
// method can finish.
func (s *ActivitylogService) Close() {
if s.stopped.CompareAndSwap(false, true) {
close(s.stopCh)
}
}
func (s *ActivitylogService) processEvent(e events.Event) error {
ctx := e.GetTraceContext(s.ctx)
ctx, span := tracer.Start(ctx, "processEvent")
defer span.End()
s.log.Debug().Interface("event", e).Msg("updating activitylog")
switch ev := e.Event.(type) {
case events.UploadReady:
return s.AddActivity(ctx, ev.FileRef, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
case events.FileTouched:
return s.AddActivity(ctx, ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
// Disabled https://github.com/owncloud/ocis/issues/10293
//case events.FileDownloaded:
// we are only interested in public link downloads - so no need to store others.
//if ev.ImpersonatingUser.GetDisplayName() == "Public" {
// err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
//}
case events.ContainerCreated:
return s.AddActivity(ctx, ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemTrashed:
return s.AddActivityTrashed(ctx, ev.ID, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemPurged:
return s.al.RemoveResource(ev.ID)
case events.ItemMoved:
// remove the cached parent id for this resource
s.removeCachedParentID(ctx, ev.Ref)
return s.AddActivity(ctx, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.ShareCreated:
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
case events.ShareUpdated:
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
}
case events.ShareRemoved:
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, ev.Timestamp)
case events.LinkCreated:
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
case events.LinkUpdated:
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
}
case events.LinkRemoved:
return s.AddActivity(ctx, toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.SpaceShared:
return s.AddSpaceActivity(ctx, ev.ID, e.ID, ev.Timestamp)
case events.SpaceUnshared:
return s.AddSpaceActivity(ctx, ev.ID, e.ID, ev.Timestamp)
}
return nil
}
// AddActivity adds the activity to the given resource and all its parents
func (a *ActivitylogService) AddActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
ctx, span := tracer.Start(ctx, "AddActivity")
defer span.End()
gwc, err := a.gws.Next()
if err != nil {
return fmt.Errorf("cant get gateway client: %w", err)
}
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("cant get service user context: %w", err)
}
return a.al.AddActivity(ctx, initRef, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
return utils.GetResource(ctx, ref, gwc)
})
}
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
func (a *ActivitylogService) AddActivityTrashed(ctx context.Context, resourceID *provider.ResourceId, reference *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
ctx, span := tracer.Start(ctx, "AddActivityTrashed")
defer span.End()
gwc, err := a.gws.Next()
if err != nil {
return fmt.Errorf("cant get gateway client: %w", err)
}
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("cant get service user context: %w", err)
}
// store activity on trashed item
if err := a.al.StoreActivity(storagespace.FormatResourceID(resourceID), []activitylog.RawActivity{
{
EventID: eventID,
Depth: 0,
Timestamp: timestamp,
},
}); err != nil {
return fmt.Errorf("could not store activity: %w", err)
}
// get previous parent
ref := &provider.Reference{
ResourceId: reference.GetResourceId(),
Path: filepath.Dir(reference.GetPath()),
}
return a.al.AddActivity(ctx, ref, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
return utils.GetResource(ctx, ref, gwc)
})
}
// AddSpaceActivity adds the activity to the given spaceroot
func (a *ActivitylogService) AddSpaceActivity(ctx context.Context, spaceID *provider.StorageSpaceId, eventID string, timestamp time.Time) error {
_, span := tracer.Start(ctx, "AddSpaceActivity")
defer span.End()
// spaceID is in format <providerid>$<spaceid>
// activitylog service uses format <providerid>$<spaceid>!<resourceid>
// lets do some converting, shall we?
rid, err := storagespace.ParseID(spaceID.GetOpaqueId())
if err != nil {
return fmt.Errorf("could not parse space id: %w", err)
}
rid.OpaqueId = rid.GetSpaceId()
err = a.al.StoreActivity(storagespace.FormatResourceID(&rid), []activitylog.RawActivity{
{
EventID: eventID,
Depth: 0,
Timestamp: timestamp,
},
})
if err != nil {
return fmt.Errorf("could not store activity: %w", err)
}
return nil
}
func toRef(r *provider.ResourceId) *provider.Reference {
return &provider.Reference{
ResourceId: r,
}
}
func (a *ActivitylogService) removeCachedParentID(ctx context.Context, ref *provider.Reference) {
var span trace.Span
ctx, span = tracer.Start(ctx, "removeCachedParentID")
defer span.End()
purgeId := ref.GetResourceId()
if ref.GetPath() != "" {
gwc, err := a.gws.Next()
if err != nil {
a.log.Error().Err(err).Msg("could not get gateway client")
return
}
ctx, err = utils.GetServiceUserContextWithContext(ctx, gwc, a.sa.ServiceAccountID, a.sa.ServiceAccountSecret)
if err != nil {
a.log.Error().Err(err).Msg("could not get service user context")
return
}
info, err := utils.GetResource(ctx, ref, gwc)
if err != nil {
a.log.Error().Err(err).Msg("could not get resource info")
return
}
purgeId = info.GetId()
}
a.al.InvalidateCachedParentID(purgeId)
}
@@ -0,0 +1,123 @@
package events_test
import (
"context"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
eventssvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/events"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
var _ = Describe("ActivitylogService", func() {
Describe("New", func() {
var (
al *activitylog.ActivityLog
stream events.Stream
)
BeforeEach(func() {
al = &activitylog.ActivityLog{}
stream = nil
})
It("creates a service with minimal options", func() {
svc, err := eventssvc.New(al, stream)
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with context option", func() {
ctx := context.Background()
svc, err := eventssvc.New(al, stream, eventssvc.Context(ctx))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with logger option", func() {
logger := log.NopLogger()
svc, err := eventssvc.New(al, stream, eventssvc.Logger(logger))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with service account option", func() {
sa := config.ServiceAccount{
ServiceAccountID: "sa-id",
ServiceAccountSecret: "sa-secret",
}
svc, err := eventssvc.New(al, stream, eventssvc.ServiceAccount(sa))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with registered events", func() {
evts := []events.Unmarshaller{&events.UploadReady{}, &events.FileTouched{}}
svc, err := eventssvc.New(al, stream, eventssvc.RegisteredEvents(evts))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with gateway selector", func() {
var gs pool.Selectable[gateway.GatewayAPIClient]
svc, err := eventssvc.New(al, stream, eventssvc.GatewaySelector(gs))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with num consumers option", func() {
svc, err := eventssvc.New(al, stream, eventssvc.NumConsumers(5))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with all options", func() {
ctx := context.Background()
logger := log.NopLogger()
sa := config.ServiceAccount{
ServiceAccountID: "sa-id",
ServiceAccountSecret: "sa-secret",
}
evts := []events.Unmarshaller{&events.UploadReady{}, &events.ContainerCreated{}}
var gs pool.Selectable[gateway.GatewayAPIClient]
svc, err := eventssvc.New(
al,
stream,
eventssvc.Context(ctx),
eventssvc.Logger(logger),
eventssvc.ServiceAccount(sa),
eventssvc.RegisteredEvents(evts),
eventssvc.GatewaySelector(gs),
eventssvc.NumConsumers(3),
)
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
})
Describe("Close", func() {
It("can be called without panic on a new service", func() {
al := &activitylog.ActivityLog{}
svc, err := eventssvc.New(al, nil)
Expect(err).ToNot(HaveOccurred())
Expect(func() { svc.Close() }).ToNot(Panic())
})
It("can be called multiple times without panic", func() {
al := &activitylog.ActivityLog{}
svc, err := eventssvc.New(al, nil)
Expect(err).ToNot(HaveOccurred())
svc.Close()
svc.Close()
svc.Close()
})
})
})
@@ -0,0 +1,13 @@
package http_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestHTTP(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "HTTP Suite")
}
@@ -0,0 +1,59 @@
package http
import (
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/log"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
Logger log.Logger
RegisteredEvents []events.Unmarshaller
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
HistoryClient ehsvc.EventHistoryService
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
// Logger provides a function to set the logger option.
func Logger(val log.Logger) Option {
return func(o *Options) {
o.Logger = val
}
}
// RegisteredEvents registers the events the service should listen to
func RegisteredEvents(e []events.Unmarshaller) Option {
return func(o *Options) {
o.RegisteredEvents = e
}
}
// GatewaySelector adds a grpc client selector for the gateway service
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
return func(o *Options) {
o.GatewaySelector = gatewaySelector
}
}
// HistoryClient adds a grpc client for the eventhistory service
func HistoryClient(hc ehsvc.EventHistoryService) Option {
return func(o *Options) {
o.HistoryClient = hc
}
}
@@ -1,4 +1,4 @@
package service
package http
import (
"context"
@@ -44,11 +44,6 @@ var (
StrDescription = l10n.Template("description")
)
// GetActivitiesResponse is the response on GET activities requests
type GetActivitiesResponse struct {
Activities []libregraph.Activity `json:"value"`
}
// Resource represents an item such as a file or folder
type Resource struct {
ID string `json:"id"`
@@ -311,7 +306,7 @@ func NewActivity(message string, ts time.Time, eventID string, vars map[string]a
}
// GetVars calls other service to gather the required data for the activity variables
func (s *ActivitylogService) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]any, error) {
func (s *svc) GetVars(ctx context.Context, opts ...ActivityOption) (map[string]any, error) {
gwc, err := s.gws.Next()
if err != nil {
return nil, err
@@ -327,6 +322,12 @@ func (s *ActivitylogService) GetVars(ctx context.Context, opts ...ActivityOption
return vars, nil
}
func toSpace(r *provider.Reference) *provider.StorageSpaceId {
return &provider.StorageSpaceId{
OpaqueId: storagespace.FormatStorageID(r.GetResourceId().GetStorageId(), r.GetResourceId().GetSpaceId()),
}
}
func getFolderName(ctx context.Context, gwc gateway.GatewayAPIClient, ref *provider.Reference) string {
n := filepath.Base(filepath.Dir(ref.GetPath()))
if n == "." || n == "/" {
@@ -0,0 +1,150 @@
package http_test
import (
"context"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/l10n"
httpsvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
)
var _ = Describe("Response", func() {
Describe("NewActivity", func() {
It("creates an activity with the given parameters", func() {
ts := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
vars := map[string]any{
"user": "testuser",
"resource": "testfile.txt",
}
act := httpsvc.NewActivity("Test message", ts, "event-123", vars)
Expect(act.Id).To(Equal("event-123"))
Expect(act.Times.RecordedTime).To(Equal(ts))
Expect(act.Template.Message).To(Equal("Test message"))
Expect(act.Template.Variables).To(HaveKeyWithValue("user", "testuser"))
Expect(act.Template.Variables).To(HaveKeyWithValue("resource", "testfile.txt"))
})
It("handles empty variables map", func() {
act := httpsvc.NewActivity("", time.Time{}, "", map[string]any{})
Expect(act.Id).To(BeEmpty())
Expect(act.Times.RecordedTime).To(Equal(time.Time{}))
Expect(act.Template.Message).To(BeEmpty())
Expect(act.Template.Variables).To(BeEmpty())
})
})
Describe("WithOldResource", func() {
It("sets the oldResource variable from reference path", func() {
ref := &provider.Reference{
Path: "/old/path/oldname.txt",
}
vars := make(map[string]any)
opt := httpsvc.WithOldResource(ref)
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("oldResource"))
})
})
Describe("WithUser", func() {
It("returns error when no user is provided", func() {
opt := httpsvc.WithUser(nil, nil, nil)
err := opt(context.Background(), nil, make(map[string]any))
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("no user provided"))
})
It("uses impersonator when provided", func() {
impersonator := &user.User{
Id: &user.UserId{
OpaqueId: "imp-user-id",
},
DisplayName: "Impersonated User",
}
vars := make(map[string]any)
opt := httpsvc.WithUser(nil, nil, impersonator)
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("user"))
})
It("uses executing user when no impersonator", func() {
execUser := &user.User{
Id: &user.UserId{
OpaqueId: "exec-user-id",
},
DisplayName: "Executing User",
}
vars := make(map[string]any)
opt := httpsvc.WithUser(nil, execUser, nil)
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("user"))
})
})
Describe("WithVar", func() {
It("sets a simple key-value variable", func() {
vars := make(map[string]any)
opt := httpsvc.WithVar("token", "id123", "My Token")
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("token"))
})
})
Describe("WithTranslation", func() {
It("sets translated field variable", func() {
var t l10n.Translator
vars := make(map[string]any)
opt := httpsvc.WithTranslation(&t, "en", "field", []string{"permission"})
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("field"))
})
It("handles empty values slice", func() {
var t l10n.Translator
vars := make(map[string]any)
opt := httpsvc.WithTranslation(&t, "en", "field", []string{})
err := opt(context.Background(), nil, vars)
Expect(err).ToNot(HaveOccurred())
Expect(vars).To(HaveKey("field"))
})
})
Describe("ActivityOption type", func() {
It("allows composing multiple options", func() {
vars := make(map[string]any)
ctx := context.Background()
var gwc gateway.GatewayAPIClient
options := []httpsvc.ActivityOption{
httpsvc.WithVar("key1", "id1", "name1"),
httpsvc.WithVar("key2", "id2", "name2"),
}
for _, opt := range options {
err := opt(ctx, gwc, vars)
Expect(err).ToNot(HaveOccurred())
}
Expect(vars).To(HaveKey("key1"))
Expect(vars).To(HaveKey("key2"))
})
})
})
@@ -1,89 +1,97 @@
package service
package http
import (
"embed"
"encoding/json"
"errors"
"net/http"
"context"
"path/filepath"
"reflect"
"slices"
"strconv"
"strings"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"google.golang.org/grpc/metadata"
"github.com/olekukonko/errors"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/pkg/l10n"
"github.com/opencloud-eu/opencloud/pkg/log"
ehmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/eventhistory/v0"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
)
var (
//go:embed l10n/locale
_localeFS embed.FS
var tracer trace.Tracer
// subfolder where the translation files are stored
_localeSubPath = "l10n/locale"
// domain of the activitylog service (transifex)
_domain = "activitylog"
)
// ServeHTTP implements the http.Handler interface.
func (s *ActivitylogService) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.mux.ServeHTTP(w, r)
func init() {
tracer = otel.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http")
}
// HandleGetItemActivities handles the request to get the activities of an item.
func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, r.Header.Get(revactx.TokenHeader))
// New returns a new instance of Service
func New(al *activitylog.ActivityLog, opts ...Option) (*svc, error) {
o := newOptions(opts...)
activeUser, ok := revactx.ContextGetUser(ctx)
if !ok {
w.WriteHeader(http.StatusUnauthorized)
return
registeredEvents := make(map[string]events.Unmarshaller)
for _, e := range o.RegisteredEvents {
typ := reflect.TypeOf(e)
registeredEvents[typ.String()] = e
}
return &svc{
log: o.Logger,
evHistory: o.HistoryClient,
al: al,
registeredEvents: registeredEvents,
gws: o.GatewaySelector,
}, nil
}
type svc struct {
log log.Logger
evHistory ehsvc.EventHistoryService
gws pool.Selectable[gateway.GatewayAPIClient]
al *activitylog.ActivityLog
registeredEvents map[string]events.Unmarshaller
}
var (
ErrBadRequest = errors.New("bad request")
ErrForbidden = errors.New("forbidden")
)
func (s *svc) GetItemActivities(ctx context.Context, query, loc string, t l10n.Translator) ([]libregraph.Activity, error) {
gwc, err := s.gws.Next()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
return nil, err
}
rid, limit, rawActivityAccepted, activityAccepted, sort, err := s.getFilters(r.URL.Query().Get("kql"))
rid, limit, rawActivityAccepted, activityAccepted, sort, err := s.getFilters(query)
if err != nil {
s.log.Info().Str("query", r.URL.Query().Get("kql")).Err(err).Msg("error getting filters")
_, _ = w.Write([]byte(err.Error()))
w.WriteHeader(http.StatusBadRequest)
return
s.log.Info().Str("query", query).Err(err).Msg("error getting filters")
return nil, ErrBadRequest
}
info, err := utils.GetResourceByID(ctx, rid, gwc)
if err != nil {
w.WriteHeader(http.StatusForbidden)
return
return nil, ErrForbidden
}
// you need ListGrants to see activities
if !info.GetPermissionSet().GetListGrants() {
w.WriteHeader(http.StatusForbidden)
return
return nil, ErrForbidden
}
raw, err := s.Activities(rid)
raw, err := s.al.Activities(rid)
if err != nil {
s.log.Error().Err(err).Msg("error getting activities")
w.WriteHeader(http.StatusInternalServerError)
return
return nil, err
}
ids := make([]string, 0, len(raw))
@@ -96,21 +104,21 @@ func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *h
toDelete[a.EventID] = struct{}{}
}
evRes, err := s.evHistory.GetEvents(r.Context(), &ehsvc.GetEventsRequest{Ids: ids})
evRes, err := s.evHistory.GetEvents(ctx, &ehsvc.GetEventsRequest{Ids: ids})
if err != nil {
s.log.Error().Err(err).Msg("error getting events")
w.WriteHeader(http.StatusInternalServerError)
return
return nil, err
}
evs := evRes.GetEvents()
sort(evs)
resp := GetActivitiesResponse{Activities: make([]libregraph.Activity, 0, len(evRes.GetEvents()))}
// TODO cut the interface here?
activities := make([]libregraph.Activity, 0, len(evRes.GetEvents()))
for _, e := range evs {
delete(toDelete, e.GetId())
if limit > 0 && limit <= len(resp.Activities) {
if limit > 0 && limit <= len(activities) {
continue
}
@@ -124,9 +132,6 @@ func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *h
vars map[string]any
)
loc := l10n.MustGetUserLocale(r.Context(), activeUser.GetId().GetOpaqueId(), r.Header.Get(l10n.HeaderAcceptLanguage), s.valService)
t := l10n.NewTranslatorFromCommonConfig(s.cfg.DefaultLanguage, _domain, s.cfg.TranslationPath, _localeFS, _localeSubPath)
switch ev := s.unwrapEvent(e).(type) {
case nil:
// error already logged in unwrapEvent
@@ -224,35 +229,29 @@ func (s *ActivitylogService) HandleGetItemActivities(w http.ResponseWriter, r *h
continue
}
resp.Activities = append(resp.Activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars))
activities = append(activities, NewActivity(t.Translate(message, loc), ts, e.GetId(), vars))
}
// delete activities in separate go routine
if len(toDelete) > 0 {
go func() {
err := s.RemoveActivities(rid, toDelete)
err := s.al.RemoveActivities(rid, toDelete)
if err != nil {
s.log.Error().Err(err).Msg("error removing activities")
}
}()
}
return activities, nil
b, err := json.Marshal(resp)
if err != nil {
s.log.Error().Err(err).Msg("error marshalling activities")
w.WriteHeader(http.StatusInternalServerError)
return
}
if _, err := w.Write(b); err != nil {
s.log.Error().Err(err).Msg("error writing response")
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
}
func (s *ActivitylogService) unwrapEvent(e *ehmsg.Event) any {
func toRef(r *provider.ResourceId) *provider.Reference {
return &provider.Reference{
ResourceId: r,
}
}
func (s *svc) unwrapEvent(e *ehmsg.Event) any {
etype, ok := s.registeredEvents[e.GetType()]
if !ok {
s.log.Error().Str("eventid", e.GetId()).Str("eventtype", e.GetType()).Msg("event not registered")
@@ -268,13 +267,13 @@ func (s *ActivitylogService) unwrapEvent(e *ehmsg.Event) any {
return einterface
}
func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int, func(RawActivity) bool, func(*ehmsg.Event) bool, func([]*ehmsg.Event), error) {
func (s *svc) getFilters(query string) (*provider.ResourceId, int, func(activitylog.RawActivity) bool, func(*ehmsg.Event) bool, func([]*ehmsg.Event), error) {
qast, err := kql.Builder{}.Build(query)
if err != nil {
return nil, 0, nil, nil, nil, err
}
prefilters := make([]func(RawActivity) bool, 0)
prefilters := make([]func(activitylog.RawActivity) bool, 0)
postfilters := make([]func(*ehmsg.Event) bool, 0)
sortby := func(_ []*ehmsg.Event) {}
@@ -299,7 +298,7 @@ func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int
break
}
prefilters = append(prefilters, func(a RawActivity) bool {
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
return a.Depth <= depth
})
case "limit":
@@ -322,11 +321,11 @@ func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int
case *ast.DateTimeNode:
switch v.Operator.Value {
case "<", "<=":
prefilters = append(prefilters, func(a RawActivity) bool {
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
return a.Timestamp.Before(v.Value)
})
case ">", ">=":
prefilters = append(prefilters, func(a RawActivity) bool {
prefilters = append(prefilters, func(a activitylog.RawActivity) bool {
return a.Timestamp.After(v.Value)
})
}
@@ -345,7 +344,7 @@ func (s *ActivitylogService) getFilters(query string) (*provider.ResourceId, int
// space root requested - fix format
rid.OpaqueId = rid.GetSpaceId()
}
pref := func(a RawActivity) bool {
pref := func(a activitylog.RawActivity) bool {
for _, f := range prefilters {
if !f(a) {
return false
@@ -0,0 +1,76 @@
package http_test
import (
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/log"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/activitylog"
httpsvc "github.com/opencloud-eu/opencloud/services/activitylog/pkg/service/http"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
var _ = Describe("Service", func() {
Describe("New", func() {
var (
al *activitylog.ActivityLog
)
BeforeEach(func() {
al = &activitylog.ActivityLog{}
})
It("creates a service with default options", func() {
svc, err := httpsvc.New(al)
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with logger option", func() {
logger := log.NopLogger()
svc, err := httpsvc.New(al, httpsvc.Logger(logger))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with registered events", func() {
evts := []events.Unmarshaller{&events.UploadReady{}}
svc, err := httpsvc.New(al, httpsvc.RegisteredEvents(evts))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with gateway selector", func() {
var gs pool.Selectable[gateway.GatewayAPIClient]
svc, err := httpsvc.New(al, httpsvc.GatewaySelector(gs))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with history client", func() {
var hc ehsvc.EventHistoryService
svc, err := httpsvc.New(al, httpsvc.HistoryClient(hc))
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
It("creates a service with all options", func() {
logger := log.NopLogger()
evts := []events.Unmarshaller{&events.UploadReady{}, &events.FileTouched{}}
var gs pool.Selectable[gateway.GatewayAPIClient]
var hc ehsvc.EventHistoryService
svc, err := httpsvc.New(
al,
httpsvc.Logger(logger),
httpsvc.RegisteredEvents(evts),
httpsvc.GatewaySelector(gs),
httpsvc.HistoryClient(hc),
)
Expect(err).ToNot(HaveOccurred())
Expect(svc).ToNot(BeNil())
})
})
})
@@ -0,0 +1,102 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
# Translators:
# Roy-Magne Mo, 2026
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2026-09-06 23:15+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Roy-Magne Mo, 2026\n"
"Language-Team: Norwegian (https://app.transifex.com/opencloud-eu/teams/204053/no/)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Language: no\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
#: pkg/service/response.go:44
msgid "description"
msgstr "beskrivelse"
#: pkg/service/response.go:43
msgid "display name"
msgstr "visningsnavn"
#: pkg/service/response.go:42
msgid "expiration date"
msgstr "utløpsdato"
#: pkg/service/response.go:41
msgid "password"
msgstr "passord"
#: pkg/service/response.go:40
msgid "permission"
msgstr "tillatelse"
#: pkg/service/response.go:39
msgid "some field"
msgstr "et felt"
#: pkg/service/response.go:26
msgid "{resource} was downloaded via public link {token}"
msgstr "{resource} ble lastet ned via offentlig lenke {token}"
#: pkg/service/response.go:24
msgid "{user} added {resource} to {folder}"
msgstr "{user} la til {resource} i {folder}"
#: pkg/service/response.go:36
msgid "{user} added {sharee} as member of {space}"
msgstr "{user} la til {sharee} som medlem av {space}"
#: pkg/service/response.go:27
msgid "{user} deleted {resource} from {folder}"
msgstr "{user} slettet {resource} fra {folder}"
#: pkg/service/response.go:28
msgid "{user} moved {resource} to {folder}"
msgstr "{user} flyttet {resource} til {folder}"
#: pkg/service/response.go:35
msgid "{user} removed link to {resource}"
msgstr "{user} fjernet lenke til {resource}"
#: pkg/service/response.go:32
msgid "{user} removed {sharee} from {resource}"
msgstr "{user} fjernet {sharee} fra {resource}"
#: pkg/service/response.go:37
msgid "{user} removed {sharee} from {space}"
msgstr "{user} fjernet {sharee} fra {space}"
#: pkg/service/response.go:29
msgid "{user} renamed {oldResource} to {resource}"
msgstr "{user} ga nytt navn til {oldResource} som {resource}"
#: pkg/service/response.go:33
msgid "{user} shared {resource} via link"
msgstr "{user} delte {resource} via lenke"
#: pkg/service/response.go:30
msgid "{user} shared {resource} with {sharee}"
msgstr "{user} delte {resource} med {sharee}"
#: pkg/service/response.go:34
msgid "{user} updated {field} for a link {token} on {resource}"
msgstr "{user} oppdaterte {field} for en lenke {token} på {resource}"
#: pkg/service/response.go:31
msgid "{user} updated {field} for the {resource}"
msgstr "{user} oppdaterte {field} for {resource}"
#: pkg/service/response.go:25
msgid "{user} updated {resource} in {folder}"
msgstr "{user} oppdaterte {resource} i {folder}"
-676
View File
@@ -1,676 +0,0 @@
package service
import (
"context"
"crypto/tls"
"encoding/base32"
"encoding/json"
"fmt"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"sync"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/go-chi/chi/v5"
"github.com/jellydator/ttlcache/v2"
"github.com/nats-io/nats.go"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/pkg/errors"
"github.com/vmihailenco/msgpack/v5"
"go.opentelemetry.io/otel/trace"
"github.com/opencloud-eu/opencloud/pkg/log"
ehsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/eventhistory/v0"
settingssvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0"
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
)
// Nats runs into max payload exceeded errors at around 7k activities. Let's keep a buffer.
var _maxActivitiesDefault = 6000
// RawActivity represents an activity as it is stored in the activitylog store
type RawActivity struct {
EventID string `json:"event_id"`
Depth int `json:"depth"`
Timestamp time.Time `json:"timestamp"`
}
// ActivitylogService logs events per resource
type ActivitylogService struct {
cfg *config.Config
log log.Logger
events <-chan events.Event
gws pool.Selectable[gateway.GatewayAPIClient]
mux *chi.Mux
evHistory ehsvc.EventHistoryService
valService settingssvc.ValueService
lock sync.RWMutex
tp trace.TracerProvider
tracer trace.Tracer
debouncer *Debouncer
parentIdCache *ttlcache.Cache
natskv nats.KeyValue
maxActivities int
registeredEvents map[string]events.Unmarshaller
}
type Debouncer struct {
after time.Duration
f func(id string, ra []RawActivity) error
pending sync.Map
inProgress sync.Map
mutex sync.Mutex
}
type queueItem struct {
activities []RawActivity
timer *time.Timer
}
type batchInfo struct {
key string
count int
timestamp time.Time
}
// NewDebouncer returns a new Debouncer instance
func NewDebouncer(d time.Duration, f func(id string, ra []RawActivity) error) *Debouncer {
return &Debouncer{
after: d,
f: f,
pending: sync.Map{},
inProgress: sync.Map{},
}
}
// Debounce restarts the debounce timer for the given space
func (d *Debouncer) Debounce(id string, ra RawActivity) {
if d.after == 0 {
d.f(id, []RawActivity{ra})
return
}
d.mutex.Lock()
defer d.mutex.Unlock()
activities := []RawActivity{ra}
item := &queueItem{
activities: activities,
}
if i, ok := d.pending.Load(id); ok {
// if the item is already in the queue, append the new activities
item, ok = i.(*queueItem)
if ok {
item.activities = append(item.activities, ra)
}
}
if item.timer == nil {
item.timer = time.AfterFunc(d.after, func() {
if _, ok := d.inProgress.Load(id); ok {
// Reschedule this run for when the previous run has finished
d.mutex.Lock()
if i, ok := d.pending.Load(id); ok {
i.(*queueItem).timer.Reset(d.after)
}
d.mutex.Unlock()
return
}
d.pending.Delete(id)
d.inProgress.Store(id, true)
defer d.inProgress.Delete(id)
d.f(id, item.activities)
})
}
d.pending.Store(id, item)
}
// New creates a new ActivitylogService
func New(opts ...Option) (*ActivitylogService, error) {
o := &Options{
MaxActivities: _maxActivitiesDefault,
}
for _, opt := range opts {
opt(o)
}
if o.Stream == nil {
return nil, errors.New("stream is required")
}
ch, err := events.Consume(o.Stream, o.Config.Service.Name, o.RegisteredEvents...)
if err != nil {
return nil, err
}
cache := ttlcache.NewCache()
err = cache.SetTTL(30 * time.Second)
if err != nil {
return nil, err
}
// Connect to NATS servers
natsOptions := nats.Options{
Servers: o.Config.Store.Nodes,
}
if o.Config.Store.EnableTLS {
if o.Config.Store.TLSRootCACertificate != "" {
// when root ca is configured use it. an insecure flag is ignored.
nats.RootCAs(o.Config.Store.TLSRootCACertificate)(&natsOptions)
} else {
// enable tls and use insecure flag
nats.Secure(&tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: o.Config.Store.TLSInsecure})(&natsOptions)
}
}
if o.Config.Store.AuthUsername != "" && o.Config.Store.AuthPassword != "" {
nats.UserInfo(o.Config.Store.AuthUsername, o.Config.Store.AuthPassword)(&natsOptions)
}
conn, err := natsOptions.Connect()
if err != nil {
return nil, err
}
js, err := conn.JetStream()
if err != nil {
return nil, err
}
kv, err := js.KeyValue(o.Config.Store.Database)
if err != nil {
if !errors.Is(err, nats.ErrBucketNotFound) {
return nil, errors.Wrapf(err, "Failed to get bucket (%s)", o.Config.Store.Database)
}
kv, err = js.CreateKeyValue(&nats.KeyValueConfig{
Bucket: o.Config.Store.Database,
})
if err != nil {
return nil, errors.Wrapf(err, "Failed to create bucket (%s)", o.Config.Store.Database)
}
}
if err != nil {
return nil, err
}
s := &ActivitylogService{
log: o.Logger,
cfg: o.Config,
events: ch,
gws: o.GatewaySelector,
mux: o.Mux,
evHistory: o.HistoryClient,
valService: o.ValueClient,
lock: sync.RWMutex{},
registeredEvents: make(map[string]events.Unmarshaller),
tp: o.TraceProvider,
tracer: o.TraceProvider.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"),
parentIdCache: cache,
maxActivities: o.Config.MaxActivities,
natskv: kv,
}
s.debouncer = NewDebouncer(o.Config.WriteBufferDuration, s.storeActivity)
// run migrations
err = s.runMigrations(context.Background(), kv)
if err != nil {
return nil, err
}
s.mux.Get("/graph/v1beta1/extensions/org.libregraph/activities", s.HandleGetItemActivities)
for _, e := range o.RegisteredEvents {
typ := reflect.TypeOf(e)
s.registeredEvents[typ.String()] = e
}
go s.Run()
return s, nil
}
// Run runs the service
func (a *ActivitylogService) Run() {
for e := range a.events {
var err error
switch ev := e.Event.(type) {
case events.UploadReady:
err = a.AddActivity(ev.FileRef, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
case events.FileTouched:
err = a.AddActivity(ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
// Disabled https://github.com/owncloud/ocis/issues/10293
//case events.FileDownloaded:
// we are only interested in public link downloads - so no need to store others.
//if ev.ImpersonatingUser.GetDisplayName() == "Public" {
// err = a.AddActivity(ev.Ref, e.ID, utils.TSToTime(ev.Timestamp))
//}
case events.ContainerCreated:
err = a.AddActivity(ev.Ref, ev.ParentID, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemTrashed:
err = a.AddActivityTrashed(ev.ID, ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.ItemPurged:
err = a.RemoveResource(ev.ID)
case events.ItemMoved:
// remove the cached parent id for this resource
a.removeCachedParentID(ev.Ref)
err = a.AddActivity(ev.Ref, nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.ShareCreated:
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
case events.ShareUpdated:
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
}
case events.ShareRemoved:
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, ev.Timestamp)
case events.LinkCreated:
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.CTime))
case events.LinkUpdated:
if ev.Sharer != nil && ev.ItemID != nil && ev.Sharer.GetOpaqueId() != ev.ItemID.GetSpaceId() {
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.MTime))
}
case events.LinkRemoved:
err = a.AddActivity(toRef(ev.ItemID), nil, e.ID, utils.TSToTime(ev.Timestamp))
case events.SpaceShared:
err = a.AddSpaceActivity(ev.ID, e.ID, ev.Timestamp)
case events.SpaceUnshared:
err = a.AddSpaceActivity(ev.ID, e.ID, ev.Timestamp)
}
if err != nil {
a.log.Error().Err(err).Interface("event", e).Msg("could not process event")
}
}
}
// AddActivity adds the activity to the given resource and all its parents
func (a *ActivitylogService) AddActivity(initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
gwc, err := a.gws.Next()
if err != nil {
return fmt.Errorf("cant get gateway client: %w", err)
}
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("cant get service user context: %w", err)
}
var span trace.Span
ctx, span = a.tracer.Start(ctx, "AddActivity")
defer span.End()
return a.addActivity(ctx, initRef, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
return utils.GetResource(ctx, ref, gwc)
})
}
// AddActivityTrashed adds the activity to given trashed resource and all its former parents
func (a *ActivitylogService) AddActivityTrashed(resourceID *provider.ResourceId, reference *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time) error {
gwc, err := a.gws.Next()
if err != nil {
return fmt.Errorf("cant get gateway client: %w", err)
}
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
return fmt.Errorf("cant get service user context: %w", err)
}
// store activity on trashed item
if err := a.storeActivity(storagespace.FormatResourceID(resourceID), []RawActivity{
{
EventID: eventID,
Depth: 0,
Timestamp: timestamp,
},
}); err != nil {
return fmt.Errorf("could not store activity: %w", err)
}
// get previous parent
ref := &provider.Reference{
ResourceId: reference.GetResourceId(),
Path: filepath.Dir(reference.GetPath()),
}
var span trace.Span
ctx, span = a.tracer.Start(ctx, "AddActivityTrashed")
defer span.End()
return a.addActivity(ctx, ref, parentId, eventID, timestamp, func(ctx context.Context, ref *provider.Reference) (*provider.ResourceInfo, error) {
return utils.GetResource(ctx, ref, gwc)
})
}
// AddSpaceActivity adds the activity to the given spaceroot
func (a *ActivitylogService) AddSpaceActivity(spaceID *provider.StorageSpaceId, eventID string, timestamp time.Time) error {
// spaceID is in format <providerid>$<spaceid>
// activitylog service uses format <providerid>$<spaceid>!<resourceid>
// lets do some converting, shall we?
rid, err := storagespace.ParseID(spaceID.GetOpaqueId())
if err != nil {
return fmt.Errorf("could not parse space id: %w", err)
}
rid.OpaqueId = rid.GetSpaceId()
return a.storeActivity(storagespace.FormatResourceID(&rid), []RawActivity{
{
EventID: eventID,
Depth: 0,
Timestamp: timestamp,
},
})
}
// Activities returns the activities for the given resource
func (a *ActivitylogService) Activities(rid *provider.ResourceId) ([]RawActivity, error) {
a.lock.RLock()
defer a.lock.RUnlock()
return a.activities(rid)
}
// RemoveActivities removes the activities from the given resource
func (a *ActivitylogService) RemoveActivities(rid *provider.ResourceId, toDelete map[string]struct{}) error {
a.lock.Lock()
defer a.lock.Unlock()
curActivities, err := a.activities(rid)
if err != nil {
return err
}
var acts []RawActivity
for _, a := range curActivities {
if _, ok := toDelete[a.EventID]; !ok {
acts = append(acts, a)
}
}
b, err := json.Marshal(acts)
if err != nil {
return err
}
_, err = a.natskv.Put(storagespace.FormatResourceID(rid), b)
return err
}
// RemoveResource removes the resource from the store
func (a *ActivitylogService) RemoveResource(rid *provider.ResourceId) error {
if rid == nil {
return fmt.Errorf("resource id is required")
}
a.lock.Lock()
defer a.lock.Unlock()
return a.natskv.Delete(storagespace.FormatResourceID(rid))
}
func (a *ActivitylogService) activities(rid *provider.ResourceId) ([]RawActivity, error) {
resourceID := storagespace.FormatResourceID(rid)
glob := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
watcher, err := a.natskv.Watch(glob, nats.IgnoreDeletes())
if err != nil {
return nil, err
}
defer watcher.Stop()
var activities []RawActivity
for update := range watcher.Updates() {
if update == nil {
break
}
var batchActivities []RawActivity
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
}
activities = append(activities, batchActivities...)
}
return activities, nil
}
// note: getResource is abstracted to allow unit testing, in general this will just be utils.GetResource
func (a *ActivitylogService) addActivity(ctx context.Context, initRef *provider.Reference, parentId *provider.ResourceId, eventID string, timestamp time.Time, getResource func(context.Context, *provider.Reference) (*provider.ResourceInfo, error)) error {
var (
err error
depth int
ref = initRef
)
ctx, span := a.tracer.Start(ctx, "addActivity")
defer span.End()
for {
var info *provider.ResourceInfo
id := ref.GetResourceId()
if ref.Path != "" {
// Path based reference, we need to resolve the resource id
ctx, span = a.tracer.Start(ctx, "addActivity.getResource")
info, err = getResource(ctx, ref)
span.End()
if err != nil {
return fmt.Errorf("could not get resource info: %w", err)
}
id = info.GetId()
}
if id == nil {
return fmt.Errorf("resource id is required")
}
key := storagespace.FormatResourceID(id)
a.debouncer.Debounce(key, RawActivity{
EventID: eventID,
Depth: depth,
Timestamp: timestamp,
})
if id.OpaqueId == id.SpaceId {
// we are at the root of the space, no need to go further
break
}
// check if parent id is cached
// parent id is cached in the format <storageid>$<spaceid>!<resourceid>
// if it is not cached, get the resource info and cache it
if parentId == nil {
if v, err := a.parentIdCache.Get(key); err != nil {
if info == nil {
ctx, span := a.tracer.Start(ctx, "addActivity.getResource parent")
info, err = getResource(ctx, ref)
span.End()
if err != nil || info.GetParentId() == nil || info.GetParentId().GetOpaqueId() == "" {
return fmt.Errorf("could not get parent id: %w", err)
}
}
parentId = info.GetParentId()
a.parentIdCache.Set(key, parentId)
} else {
parentId = v.(*provider.ResourceId)
}
} else {
a.log.Debug().Msg("parent id is cached")
}
depth++
ref = &provider.Reference{ResourceId: parentId}
parentId = nil // reset parent id so it's not reused in the next iteration
}
return nil
}
func (a *ActivitylogService) storeActivity(resourceID string, activities []RawActivity) error {
a.lock.Lock()
defer a.lock.Unlock()
ctx, span := a.tracer.Start(context.Background(), "storeActivity")
defer span.End()
_, subspan := a.tracer.Start(ctx, "storeActivity.Marshal")
b, err := msgpack.Marshal(activities)
if err != nil {
return err
}
subspan.End()
_, subspan = a.tracer.Start(ctx, "storeActivity.natskv.Put")
key := natsKey(resourceID, len(activities))
_, err = a.natskv.Put(key, b)
if err != nil {
return err
}
subspan.End()
ctx, subspan = a.tracer.Start(ctx, "storeActivity.enforceMaxActivities")
a.enforceMaxActivities(ctx, resourceID)
subspan.End()
return nil
}
func (a *ActivitylogService) enforceMaxActivities(ctx context.Context, resourceID string) {
if a.maxActivities <= 0 {
return
}
key := fmt.Sprintf("%s.>", base32.StdEncoding.EncodeToString([]byte(resourceID)))
_, subspan := a.tracer.Start(ctx, "enforceMaxActivities.watch")
watcher, err := a.natskv.Watch(key, nats.IgnoreDeletes())
if err != nil {
a.log.Error().Err(err).Str("resourceID", resourceID).Msg("could not watch")
return
}
defer watcher.Stop()
var keys []string
for update := range watcher.Updates() {
if update == nil {
break
}
var batchActivities []RawActivity
if err := msgpack.Unmarshal(update.Value(), &batchActivities); err != nil {
a.log.Debug().Err(err).Str("resourceID", resourceID).Msg("could not unmarshal messagepack, trying json")
}
keys = append(keys, update.Key())
}
subspan.End()
_, subspan = a.tracer.Start(ctx, "enforceMaxActivities.compile")
// Parse keys into batches
batches := make([]batchInfo, 0)
var activitiesCount int
for _, k := range keys {
parts := strings.SplitN(k, ".", 3)
if len(parts) < 3 {
a.log.Warn().Str("key", k).Msg("skipping key, not enough parts")
continue
}
c, err := strconv.Atoi(parts[1])
if err != nil {
a.log.Warn().Str("key", k).Msg("skipping key, can not parse count")
continue
}
// parse timestamp
nano, err := strconv.ParseInt(parts[2], 10, 64)
if err != nil {
a.log.Warn().Str("key", k).Msg("skipping key, can not parse timestamp")
continue
}
batches = append(batches, batchInfo{
key: k,
count: c,
timestamp: time.Unix(0, nano),
})
activitiesCount += c
}
// sort batches by timestamp
sort.Slice(batches, func(i, j int) bool {
return batches[i].timestamp.Before(batches[j].timestamp)
})
subspan.End()
_, subspan = a.tracer.Start(ctx, "enforceMaxActivities.delete")
// remove oldest keys until we are at max activities
for _, b := range batches {
if activitiesCount-b.count < a.maxActivities {
break
}
activitiesCount -= b.count
err = a.natskv.Delete(b.key)
if err != nil {
a.log.Error().Err(err).Str("key", b.key).Msg("could not delete key")
break
}
}
subspan.End()
}
func toRef(r *provider.ResourceId) *provider.Reference {
return &provider.Reference{
ResourceId: r,
}
}
func toSpace(r *provider.Reference) *provider.StorageSpaceId {
return &provider.StorageSpaceId{
OpaqueId: storagespace.FormatStorageID(r.GetResourceId().GetStorageId(), r.GetResourceId().GetSpaceId()),
}
}
func (a *ActivitylogService) removeCachedParentID(ref *provider.Reference) {
purgeId := ref.GetResourceId()
if ref.GetPath() != "" {
gwc, err := a.gws.Next()
if err != nil {
a.log.Error().Err(err).Msg("could not get gateway client")
return
}
ctx, err := utils.GetServiceUserContext(a.cfg.ServiceAccount.ServiceAccountID, gwc, a.cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
a.log.Error().Err(err).Msg("could not get service user context")
return
}
info, err := utils.GetResource(ctx, ref, gwc)
if err != nil {
a.log.Error().Err(err).Msg("could not get resource info")
return
}
purgeId = info.GetId()
}
// The parent id cache is populated lazily and its entries expire, so a
// missing key is the expected case rather than an error.
if err := a.parentIdCache.Remove(storagespace.FormatResourceID(purgeId)); err != nil {
a.log.Debug().Interface("event", ref).Err(err).Msg("could not delete parent id cache")
}
}
func natsKey(resourceID string, activitiesCount int) string {
return fmt.Sprintf("%s.%d.%d",
base32.StdEncoding.EncodeToString([]byte(resourceID)),
activitiesCount,
time.Now().UnixNano())
}
-4
View File
@@ -63,10 +63,6 @@ Store specific notes:
- When using `nats-js-kv` it is recommended to set `OC_CACHE_STORE_NODES` to the same value as `OC_EVENTS_ENDPOINT`. That way the cache uses the same nats instance as the event bus.
- When using the `nats-js-kv` store, it is possible to set `OC_CACHE_DISABLE_PERSISTENCE` to instruct nats to not persist cache data on disc.
### Auto-Accept Shares
When setting the `SHARING_AUTO_ACCEPT_SHARES` to `true` (sharing service), all incoming shares will be accepted automatically. Users can overwrite this setting individually in their profile. The deprecated `FRONTEND_AUTO_ACCEPT_SHARES` is still supported for backwards compatibility.
## Passwords
### The Password Policy
@@ -148,6 +148,7 @@ func DefaultConfig() *config.Config {
Groupware: config.Groupware{
Enabled: false,
},
ConfigurableNotifications: true,
}
}
+2 -1
View File
@@ -13,7 +13,8 @@ include ../../.make/release.mk
include ../../.make/docs.mk
.PHONY: go-generate
go-generate: $(MOCKERY)
go-generate: $(MOCKERY) $(GOWRAP)
GOWRAP=$(GOWRAP) go generate ./...
$(MOCKERY)
.PHONY: l10n-pull
+93 -1
View File
@@ -168,7 +168,7 @@ The output of this command includes the following information for each role:
* `Condition`
* `Allowed resource actions`
**Example output (shortned)**
**Example output (shortened)**
```bash
+--------------------------------------+----------+--------------------------------+--------------------------------+------------------------------------------+
@@ -184,3 +184,95 @@ The output of this command includes the following information for each role:
+--------------------------------------+----------+--------------------------------+--------------------------------+------------------------------------------+
```
## API Handlers
To specialize `graph` service instances in order to scale them independently, it is possible to disable its API handlers:
* `GRAPH_HTTP_DISABLE`: when set to `true`, the service does not listen on HTTP and only consumes events (defaults to `false`)
* `GRAPH_EVENTS_DISABLE_CONSUMER`: when set to `true`, the service does not consome events and only listens on HTTP (defaults to `false`)
## Metrics
Metrics are disabled by default, and must be enabled using the following environment variables:
* `GRAPH_LDAP_METRICS_DISABLE`: set to `false` to enable metrics for the duration of outbound LDAP client operations (defaults to `true`)
* `GRAPH_HTTP_METRICS_DISABLE`: set to `false` to enable metrics for the duration of inbound Graph HTTP API requests (defaults to `true`)
* `GRAPH_IDENTITY_BACKEND_METRICS_DISABLE`: set to `false` to enable metrics for the duration of Graph identity backend operations (defaults to `true`)
The `graph` service provides the following metrics:
| Name | Description |
| ---- | ----------- |
| `opencloud_graph_build_info{version=...}` | Contains a label `version` that is set to the current version of the service, and always has a value of `1` |
| `opencloud_graph_events_enabled` | Is set to `1` if the Events API handler is enabled, or `0` if not |
| `opencloud_graph_http_enabled` | Is set to `1` if the HTTP API handler is enabled, or `0` if not |
| `opencloud_graph_events{event=...,result=...}` | Counts the number of events that have been consumed, with a `event` label that contains the name of the event, and a `result` label that is set to `success` or `failure` |
| `opencloud_graph_events_invalid` | Counts the number of invalid events that are malformed or are missing required data |
| `opencloud_graph_events_unsupported` | Counts the numbef of consumed events that cannot be processes by this service, should always be `0` |
| `opencloud_graph_user_password_changes{result=...,reason=...}` | Counts the number of user password change attempts, including the reason for failure when `result`=`failure` |
| `opencloud_graph_http_request_duration_seconds{method=...,path=...,version=...,resource=...,code=...,result=...}` | Histogram that measures the duration of Graph HTTP API requests, in buckets |
| `opencloud_graph_http_requests` | Gauge that counts the number of concurrent inbound HTTP requests to the Graph API |
| `opencloud_graph_ldap_client_operation_duration_seconds{uri=...,write=...,operation=...,result=...}` | Histogram that measures the duration of outbound LDAP operations |
| `opencloud_graph_ldap_client_operations{uri=...,write=...}` | Gauge that counts the number of concurrent outbound LDAP operations |
| `opencloud_graph_identity_backend_api_duration_seconds{type=...,operation=...,result=...}` | Histogram that measures the duration of requests to the Graph identity backend, in buckets |
To create some moderate load on a running `opencloud` instance, one can use the k6 script `load_test.js` as follows:
```bash
k6 run --vus=10 --duration=3m ./load_test.js
```
The following environment variables can be used to influence its behavior:
* `BASE_URL`: defaults to `https://localhost:9200`
* `USERNAME`: defaults to `alan`
* `PASSWORD`: defaults to `demo`
For example, to use a different user and a different URL:
```bash
k6 run --vus=10 --duration=3m -e USERNAME=lynn -e BASE_URL=https://localhost:9201 ./load_test.js
```
It is not meant to be a feature test suite, but merely a small k6 script to generate some read-only load in order to make Grafana displays.
### Graph User Password Change Counter Metric
For `opencloud_graph_user_password_changes`:
* `result` is either
* `success`: when the password was changed successfully
* `failure`: when the password could not be changed, the reason being tracked in the `reason` label
* `reason` is either
* empty when `result` is `success`
* `invalid`: when parameters were invalid, such as the new password being an empty password
* `error`: when an error prevented the password change, such as a network failure
* `wrong-password`: when the password change was refused because the current password is wrong
### Graph Inbound HTTP Request Duration Metrics
For `opencloud_graph_http_request_duration_seconds`:
* `method` is the HTTP method (`GET`, `PUT`, ...)
* `path` is the canonical request path with placeholders (e.g. `/v1beta1/drives/{driveID}/root/children`)
* `version` is the Graph API version (`v1beta` or `v1.0`)
* `resource` is the top-level resource after the version (`me`, `application`, `drives`, ...)
* `code` is the resulting HTTP status code (`200`, `404`, `500`, ...)
* `result` is one of `success`, `client-error`, `server-error`
### Graph Outbound LDAP Operation Duration Metrics
For `opencloud_graph_ldap_client_operation_duration_seconds`:
* `operation` is the name of the LDAP operation (`add`, `delete`, `modify`, `modify-dn`, ...)
* `result` is either `success`, `failure`, `read-only` (when attempting a write operation on a LDAP server that is configured as read-only in OpenCloud) or `not-found`
* `uri` contains the LDAP server URI the client is connected to
* `write` is set to `1` if the LDAP client is allowed to perform write operations, or to `0` if it is configured to be read-only
### Graph Identity Backend API Duration Metrics
* `type` is the type of the identity backend that is being used (`ldap` or `cs3`)
* `operation` is the name of the API operation (`create-user`, `get-users`, ...)
* `result` is `success`, `failure` or `not-found`
+80
View File
@@ -0,0 +1,80 @@
// Small k6 script to generate some load on read-only endpoints of
// the Graph API, for showcasing the metrics.
import http from 'k6/http';
import { check, sleep } from 'k6';
import encoding from 'k6/encoding';
// Configuration via environment variables with defaults
const BASE_URL = __ENV.BASE_URL || 'https://localhost:9200';
const USERNAME = __ENV.USERNAME || 'alan';
const PASSWORD = __ENV.PASSWORD || 'demo';
export const options = {
insecureSkipTLSVerify: true,
vus: 10,
thresholds: {
http_req_failed: ['rate<0.01'],
http_req_duration: ['p(95)<500'],
},
};
const credentials = `${USERNAME}:${PASSWORD}`;
const encodedCredentials = encoding.b64encode(credentials);
const params = {
headers: {
'Authorization': `Basic ${encodedCredentials}`,
'Accept': 'application/json',
},
};
export default function () {
// Fetch current user profile, including the list of groups the user is part of
let resMe = http.get(`${BASE_URL}/graph/v1.0/me?$expand=memberOf`, params);
const meOk = check(resMe, { 'GET /me status is 200': (r) => r.status === 200 });
sleep(0.1);
// extract the names of the groups the user is part of, because the user is allowed
// to retrieve information about those
let groupNames = [];
if (meOk && resMe.json() && resMe.json().memberOf) {
groupNames = (resMe.json().memberOf || []).map((group) => group.displayName);
}
// Fetch oneself using the users search API:
let resUsers = http.get(`${BASE_URL}/graph/v1.0/users?$search="${USERNAME}"`, params);
check(resUsers, { 'GET /users status is 200': (r) => r.status === 200 });
sleep(0.1);
// Fetch storage drives
let resDrives = http.get(`${BASE_URL}/graph/v1.0/drives`, params);
const drivesOk = check(resDrives, {
'GET /drives status is 200': (r) => r.status === 200,
});
sleep(0.1);
// For each of those drives, retrieve deeper information about each
if (drivesOk && resDrives.json() && resDrives.json().value) {
const drives = resDrives.json().value;
if (drives.length > 0) {
const driveId = drives[0].id;
let resDrive = http.get(`${BASE_URL}/graph/v1.0/drives/${driveId}`, params);
check(resDrive, {
'GET /drives/{id} status is 200': (r) => r.status === 200,
});
}
}
// For each of the groups the user is part of, retrieve information about each of them
// using the group searching endpoint
for (const group of groupNames) {
let resGroups = http.get(`${BASE_URL}/graph/v1.0/groups?$search="${group}"`, params);
const groupsOk = check(resGroups, {
'GET /groups status is 200': (r) => r.status === 200,
});
}
sleep(0.2);
}
+85 -5
View File
@@ -7,16 +7,22 @@ import (
"strings"
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
"github.com/opencloud-eu/opencloud/pkg/generators"
"github.com/opencloud-eu/opencloud/pkg/log"
natspkg "github.com/opencloud-eu/opencloud/pkg/nats"
"github.com/opencloud-eu/opencloud/pkg/runner"
"github.com/opencloud-eu/opencloud/pkg/tracing"
"github.com/opencloud-eu/opencloud/pkg/version"
"github.com/opencloud-eu/opencloud/services/graph/pkg/config"
"github.com/opencloud-eu/opencloud/services/graph/pkg/config/parser"
"github.com/opencloud-eu/opencloud/services/graph/pkg/identity"
"github.com/opencloud-eu/opencloud/services/graph/pkg/metrics"
"github.com/opencloud-eu/opencloud/services/graph/pkg/server/debug"
"github.com/opencloud-eu/opencloud/services/graph/pkg/server/http"
evc "github.com/opencloud-eu/opencloud/services/graph/pkg/service/events"
svc "github.com/opencloud-eu/opencloud/services/graph/pkg/service/v0"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
"github.com/prometheus/client_golang/prometheus"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -46,8 +52,17 @@ func Server(cfg *config.Config) *cobra.Command {
}
ctx := cfg.Context
mtrcs := metrics.New()
mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1)
prom := prometheus.DefaultRegisterer
// note that the function we pass here is tasked with decomposing Graph HTTP API
// request URL patterns into information that is then used for labels in metrics
// to track HTTP request processing durations, and it is located there to be close
// to the HTTP API route definitions, to improve chances of adapting it accordingly
// whenever those routes should change in the future
mtrcs, err := metrics.New(prom, &logger, svc.DecomposeGraphApiRequestPattern)
if err != nil {
return err
}
var kv jetstream.KeyValue
// Allow to run without a NATS store (e.g. for the standalone Education provisioning service)
@@ -78,9 +93,45 @@ func Server(cfg *config.Config) *cobra.Command {
}
}
identityBackendName := cfg.Identity.Backend // contains the name of the backend implementation to use
// since the identity backend in use is of prime importance to understand issues through logs, every
// log entry should contain a 'backend' entry with the name of the backend in use from here on:
logger = log.Logger{Logger: logger.With().Str("backend", identityBackendName).Logger()}
identityBackend, eduBackend, err := identity.CreateIdentityBackends(
identityBackendName,
cfg,
&logger,
prom,
traceProvider,
)
if err != nil {
logger.Error().Err(err).Msg("Error initializing the identity backend")
return fmt.Errorf("could not initialize identity backend: %w", err)
}
var eventsStream events.Stream
if cfg.Events.Endpoint != "" {
var err error
connName := generators.GenerateConnectionName(cfg.Service.Name, generators.NTypeBus)
eventsStream, err = stream.NatsFromConfig(connName, false, cfg.Events.ToNatsConfig())
if err != nil {
logger.Error().Err(err).Msg("Error initializing events publisher")
return fmt.Errorf("could not initialize events publisher: %w", err)
}
}
gr := runner.NewGroup()
{
if !cfg.HTTP.Disabled {
mtrcs.HttpEnabled.Set(1)
server, err := http.Server(
identityBackend,
eduBackend,
eventsStream,
http.Logger(logger),
http.Context(ctx),
http.Config(cfg),
@@ -92,8 +143,37 @@ func Server(cfg *config.Config) *cobra.Command {
logger.Error().Err(err).Str("transport", "http").Msg("Failed to initialize server")
return err
}
gr.Add(runner.NewGoMicroHttpServerRunner(cfg.Service.Name+".http", server))
} else {
mtrcs.HttpEnabled.Set(0)
logger.Info().Str("transport", "http").Msg("HTTP server is disabled")
}
if !cfg.Events.DisabledConsumer {
mtrcs.EventsEnabled.Set(1)
// even if events are enabled, we still need to differentiate between whether this process
// show be consuming events or not (and even when that is disabled, we still need to be
// able to produce events), which is why this is a separate setting;
// for context, see https://github.com/opencloud-eu/opencloud/issues/1312
logger := &log.Logger{Logger: logger.With().Str("transport", "events").Logger()}
eventConsumer, err := evc.NewService(cfg.Context, eventsStream, identityBackend, mtrcs, logger)
if err != nil {
return fmt.Errorf("could not initialize events consumer: %w", err)
}
gr.Add(runner.New(cfg.Service.Name+".svc", func() error {
return eventConsumer.Start()
}, func() {
err := eventConsumer.Close()
if err != nil {
logger.Error().Err(err).Msg("failed to stop event consumer")
}
}))
} else {
mtrcs.EventsEnabled.Set(0)
logger.Info().Str("transport", "events").Msg("event consumer is disabled")
}
{
+27 -2
View File
@@ -5,6 +5,7 @@ import (
"time"
"github.com/opencloud-eu/opencloud/pkg/shared"
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
)
// Config combines all available configuration parts.
@@ -59,6 +60,10 @@ type Spaces struct {
TranslationPath string `yaml:"translation_path" env:"OC_TRANSLATION_PATH;GRAPH_TRANSLATION_PATH" desc:"(optional) Set this to a path with custom translations to overwrite the builtin translations. Note that file and folder naming rules apply, see the documentation for more details." introductionVersion:"1.0.0"`
}
type LDAPMetrics struct {
Disabled bool `yaml:"disabled" env:"GRAPH_LDAP_METRICS_DISABLE" desc:"Disables the metrics for outbound LDAP operations." introductionVersion:"%%NEXT%%"`
}
type LDAP struct {
URI string `yaml:"uri" env:"OC_LDAP_URI;GRAPH_LDAP_URI" desc:"URI of the LDAP Server to connect to. Supported URI schemes are 'ldaps://' and 'ldap://'" introductionVersion:"1.0.0"`
CACert string `yaml:"cacert" env:"OC_LDAP_CACERT;GRAPH_LDAP_CACERT" desc:"Path/File name for the root CA certificate (in PEM format) used to validate TLS server certificates of the LDAP service. If not defined, the root directory derives from $OC_BASE_DATA_PATH/idm." introductionVersion:"1.0.0"`
@@ -96,6 +101,8 @@ type LDAP struct {
EducationResourcesEnabled bool `yaml:"education_resources_enabled" env:"GRAPH_LDAP_EDUCATION_RESOURCES_ENABLED" desc:"Enable LDAP support for managing education related resources." introductionVersion:"1.0.0"`
EducationConfig LDAPEducationConfig
Metrics LDAPMetrics `yaml:"metrics"`
}
// LDAPEducationConfig represents the LDAP configuration for education related resources
@@ -113,9 +120,14 @@ type LDAPEducationConfig struct {
SchoolTerminationGraceDays int `yaml:"school_termination_min_grace_days" env:"GRAPH_LDAP_SCHOOL_TERMINATION_MIN_GRACE_DAYS" desc:"When setting a 'terminationDate' for a school, require the date to be at least this number of days in the future." introductionVersion:"1.0.0"`
}
type IdentityMetrics struct {
Disabled bool `yaml:"disabled" env:"GRAPH_IDENTITY_BACKEND_METRICS_DISABLE" desc:"Disables the metrics for inbound identity backend operations." introductionVersion:"%%NEXT%%"`
}
type Identity struct {
Backend string `yaml:"backend" env:"GRAPH_IDENTITY_BACKEND" desc:"The user identity backend to use. Supported backend types are 'ldap' and 'cs3'." introductionVersion:"1.0.0"`
LDAP LDAP `yaml:"ldap"`
Backend string `yaml:"backend" env:"GRAPH_IDENTITY_BACKEND" desc:"The user identity backend to use. Supported backend types are 'ldap' and 'cs3'." introductionVersion:"1.0.0"`
LDAP LDAP `yaml:"ldap"`
Metrics IdentityMetrics `yaml:"metrics"`
}
// API represents API configuration parameters.
@@ -129,6 +141,7 @@ type API struct {
// Events combines the configuration options for the event bus.
type Events struct {
DisabledConsumer bool `yaml:"disabled_consumer" env:"GRAPH_EVENTS_DISABLE_CONSUMER" desc:"Disables consuming events. Set this to true if the service should only handle HTTP requests." introductionVersion:"%%NEXT%%"`
Endpoint string `yaml:"endpoint" env:"OC_EVENTS_ENDPOINT;GRAPH_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Set to a empty string to disable emitting events." introductionVersion:"1.0.0"`
Cluster string `yaml:"cluster" env:"OC_EVENTS_CLUSTER;GRAPH_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture." introductionVersion:"1.0.0"`
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_EVENTS_TLS_INSECURE;GRAPH_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"1.0.0"`
@@ -138,6 +151,18 @@ type Events struct {
AuthPassword string `yaml:"password" env:"OC_EVENTS_AUTH_PASSWORD;GRAPH_EVENTS_AUTH_PASSWORD" desc:"The password to authenticate with the events broker. The events broker is the OpenCloud service which receives and delivers events between the services." introductionVersion:"1.0.0"`
}
func (e Events) ToNatsConfig() stream.NatsConfig {
return stream.NatsConfig{
Endpoint: e.Endpoint,
Cluster: e.Cluster,
TLSInsecure: e.TLSInsecure,
TLSRootCACertificate: e.TLSRootCACertificate,
EnableTLS: e.EnableTLS,
AuthUsername: e.AuthUsername,
AuthPassword: e.AuthPassword,
}
}
// CORS defines the available cors configuration.
type CORS struct {
AllowedOrigins []string `yaml:"allow_origins" env:"OC_CORS_ALLOW_ORIGINS;GRAPH_CORS_ALLOW_ORIGINS" desc:"A list of allowed CORS origins. See following chapter for more details: *Access-Control-Allow-Origin* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
@@ -43,6 +43,7 @@ func DefaultConfig() *config.Config {
Token: "",
},
HTTP: config.HTTP{
Disabled: false,
Addr: "127.0.0.1:9120",
Namespace: "eu.opencloud.web",
Root: "/graph",
@@ -52,6 +53,11 @@ func DefaultConfig() *config.Config {
AllowedHeaders: []string{"Authorization", "Origin", "Content-Type", "Accept", "X-Requested-With", "X-Request-Id", "Purge", "Restore"},
AllowCredentials: true,
},
Metrics: config.HTTPMetrics{
// disabling inbound HTTP metrics collection by default for now, since the runtime performance impact is currently unclear;
// it is most likely to be negligible, but has not been measured yet to confirm
Disabled: true,
},
},
Service: config.Service{
Name: "graph",
@@ -80,6 +86,12 @@ func DefaultConfig() *config.Config {
},
Identity: config.Identity{
Backend: "ldap",
Metrics: config.IdentityMetrics{
// disabling identity backend opcall metrics collection by default for now, since
// the runtime performance impact is currently unclear;
// it is most likely to be negligible, but has not been measured yet to confirm
Disabled: true,
},
LDAP: config.LDAP{
URI: "ldap://localhost:9236",
Insecure: false,
@@ -109,6 +121,11 @@ func DefaultConfig() *config.Config {
GroupMemberAttribute: "member",
GroupIDAttribute: "openCloudUUID",
EducationResourcesEnabled: false,
Metrics: config.LDAPMetrics{
// disabling inbound HTTP metrics collection by default for now, since the runtime performance impact is currently unclear;
// it is most likely to be negligible, but has not been measured yet to confirm
Disabled: true,
},
},
},
Cache: &config.Cache{
@@ -118,9 +135,10 @@ func DefaultConfig() *config.Config {
TTL: time.Hour * 24,
},
Events: config.Events{
Endpoint: "127.0.0.1:9233",
Cluster: "opencloud-cluster",
EnableTLS: false,
DisabledConsumer: false,
Endpoint: "127.0.0.1:9233",
Cluster: "opencloud-cluster",
EnableTLS: false,
},
MaxConcurrency: 20,
UnifiedRoles: config.UnifiedRoles{
+6
View File
@@ -2,12 +2,18 @@ package config
import "github.com/opencloud-eu/opencloud/pkg/shared"
type HTTPMetrics struct {
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_METRICS_DISABLE" desc:"Disables the metrics for the HTTP service." introductionVersion:"%%NEXT%%"`
}
// HTTP defines the available http configuration.
type HTTP struct {
Disabled bool `yaml:"disabled" env:"GRAPH_HTTP_DISABLE" desc:"Disables the HTTP service. Set this to true if the service should only consume events." introductionVersion:"%%NEXT%%"`
Addr string `yaml:"addr" env:"GRAPH_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"1.0.0"`
Namespace string `yaml:"-"`
Root string `yaml:"root" env:"GRAPH_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"1.0.0"`
TLS shared.HTTPServiceTLS `yaml:"tls"`
APIToken string `yaml:"apitoken" env:"GRAPH_HTTP_API_TOKEN" desc:"An optional API bearer token" introductionVersion:"1.0.0"`
CORS CORS `yaml:"cors"`
Metrics HTTPMetrics `yaml:"metrics"`
}
@@ -39,6 +39,10 @@ func ParseConfig(cfg *config.Config) error {
}
func Validate(cfg *config.Config) error {
if cfg.HTTP.Disabled && cfg.Events.DisabledConsumer {
return shared.AllComponentsDisabledError(cfg.Service.Name)
}
if cfg.TokenManager.JWTSecret == "" {
return shared.MissingJWTTokenError(cfg.Service.Name)
}
+16
View File
@@ -50,6 +50,8 @@ const (
InvalidRequest
// ItemNotFound defines the error if the resource could not be found.
ItemNotFound
// TooManyResults defines the error if multiple results are found for a unique resource.
TooManyResults
// MalwareDetected defines the error if malware was detected in the requested resource.
MalwareDetected
// NameAlreadyExists defines the error if the specified item name already exists.
@@ -84,6 +86,7 @@ var errorCodes = [...]string{
"invalidRange",
"invalidRequest",
"itemNotFound",
"tooManyResults",
"malwareDetected",
"nameAlreadyExists",
"notAllowed",
@@ -206,3 +209,16 @@ func ToError(err error) (Error, bool) {
return Error{}, false
}
// Returns true if the error is of type Error and has an ErrorCode that matches
// the one specified as the second parameter, and false if not.
func IsErrorCode(err error, code ErrorCode) bool {
if err == nil {
return false
}
if e, ok := ToError(err); ok {
return e.errorCode == code
} else {
return false
}
}
+53
View File
@@ -1,5 +1,8 @@
package identity
//go:generate $GOWRAP gen -g -i Backend -t ./backend_prometheus.tmpl -o backend_prometheus.go
//go:generate $GOWRAP gen -g -i EducationBackend -t ./backend_prometheus.tmpl -o education_backend_prometheus.go
import (
"context"
"net/url"
@@ -18,6 +21,8 @@ var (
ErrReadOnly = errorcode.New(errorcode.NotAllowed, "server is configured read-only")
// ErrNotFound signals that the requested resource was not found.
ErrNotFound = errorcode.New(errorcode.ItemNotFound, "not found")
// ErrTooManyResults signals that multiple results were found when only one was expected
ErrTooManyResults = errorcode.New(errorcode.TooManyResults, "too many results")
// ErrUnsupportedFilter signals that the requested filter is not supported by the backend.
ErrUnsupportedFilter = godata.NotImplementedError("unsupported filter")
)
@@ -28,6 +33,24 @@ const (
UserTypeFederated = "Federated"
)
const (
MetricOpCreateUser = "create-user"
MetricOpDeleteUser = "delete-user"
MetricOpUpdateUser = "update-user"
MetricOpGetUser = "get-user"
MetricOpGetUsers = "get-users"
MetricOpFilterUsers = "filter-users"
MetricOpUpdateLastSignInDate = "update-last-signin-date"
MetricOpGetGroup = "get-group"
MetricOpGetGroups = "get-groups"
MetricOpCreateGroup = "create-group"
MetricOpDeleteGroup = "delete-group"
MetricOpUpdateGroupName = "update-group-name"
MetricOpAddMembersToGroup = "add-members-to-group"
MetricOpRemoveMemberFromGroup = "remove-member-from-group"
MetricOpGetGroupMembers = "get-group-members"
)
// Backend defines the Interface for an IdentityBackend implementation
type Backend interface {
// CreateUser creates a given user in the identity backend.
@@ -58,6 +81,36 @@ type Backend interface {
RemoveMemberFromGroup(ctx context.Context, groupID string, memberID string) error
}
const (
MetricOpCreateEducationSchool = "create-school"
MetricOpUpdateEducationSchool = "update-school"
MetricOpDeleteEducationSchool = "delete-school"
MetricOpGetEducationSchool = "get-school"
MetricOpGetEducationSchools = "get-schools"
MetricOpFilterEducationSchoolsByAttribute = "filter-schools-byattr"
MetricOpAddUsersToEducationSchool = "add-eduusers-to-school"
MetricOpRemoveUserFromEducationSchool = "remove-eduser-from-school"
MetricOpGetEducationSchoolClasses = "get-school-classes"
MetricOpAddClassesToEducationSchool = "add-classes-to-school"
MetricOpRemoveClassFromEducationSchool = "remove-class-from-school"
MetricOpAddTeacherToEducationClass = "add-teacher-to-class"
MetricOpCreateEducationUser = "create-eduser"
MetricOpDeleteEducationClass = "delete-class"
MetricOpDeleteEducationUser = "delete-eduser"
MetricOpFilterEducationUsersByAttribute = "filter-edusers"
MetricOpGetEducationClass = "get-class"
MetricOpGetEducationClassMembers = "get-class-members"
MetricOpGetEducationClassTeachers = "get-class-teachers"
MetricOpGetEducationClasses = "get-classes"
MetricOpGetEducationSchoolUsers = "get-school-edusers"
MetricOpGetEducationUser = "get-eduser"
MetricOpGetEducationUsers = "get-edusers"
MetricOpUpdateEducationUser = "update-eduser"
MetricOpRemoveTeacherFromEducationClass = "remove-teacher-from-class"
MetricOpUpdateEducationClass = "update-class"
MetricOpCreateEducationClass = "create-class"
)
// EducationBackend defines the Interface for an EducationBackend implementation
type EducationBackend interface {
// CreateEducationSchool creates the supplied school in the identity backend.
@@ -0,0 +1,500 @@
// Code generated by gowrap. DO NOT EDIT.
// template: backend_prometheus.tmpl
// gowrap: http://github.com/hexdigest/gowrap
package identity
import (
"context"
"net/url"
"time"
"github.com/CiscoM31/godata"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
"github.com/prometheus/client_golang/prometheus"
)
// PrometheusBackend implements Backend interface with all methods wrapped
// with Prometheus metrics
type PrometheusBackend struct {
delegate Backend
metric *prometheus.HistogramVec
}
var _ Backend = &PrometheusBackend{}
// returns an instance of the Backend decorated with prometheus metric
func NewPrometheusBackend(delegate Backend, metric *prometheus.HistogramVec) PrometheusBackend {
return PrometheusBackend{
delegate: delegate,
metric: metric,
}
}
// AddMembersToGroup implements Backend.AddMembersToGroup
func (_d PrometheusBackend) AddMembersToGroup(ctx context.Context, groupID string, memberID []string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpAddMembersToGroup, result).Observe(duration)
}()
return _d.delegate.AddMembersToGroup(ctx, groupID, memberID)
}
// CreateGroup implements Backend.CreateGroup
func (_d PrometheusBackend) CreateGroup(ctx context.Context, group libregraph.Group) (gp1 *libregraph.Group, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpCreateGroup, result).Observe(duration)
}()
return _d.delegate.CreateGroup(ctx, group)
}
// CreateUser implements Backend.CreateUser
func (_d PrometheusBackend) CreateUser(ctx context.Context, user libregraph.User) (up1 *libregraph.User, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpCreateUser, result).Observe(duration)
}()
return _d.delegate.CreateUser(ctx, user)
}
// DeleteGroup implements Backend.DeleteGroup
func (_d PrometheusBackend) DeleteGroup(ctx context.Context, id string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpDeleteGroup, result).Observe(duration)
}()
return _d.delegate.DeleteGroup(ctx, id)
}
// DeleteUser implements Backend.DeleteUser
func (_d PrometheusBackend) DeleteUser(ctx context.Context, nameOrID string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpDeleteUser, result).Observe(duration)
}()
return _d.delegate.DeleteUser(ctx, nameOrID)
}
// FilterUsers implements Backend.FilterUsers
func (_d PrometheusBackend) FilterUsers(ctx context.Context, oreq *godata.GoDataRequest, filter *godata.ParseNode) (upa1 []*libregraph.User, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpFilterUsers, result).Observe(duration)
}()
return _d.delegate.FilterUsers(ctx, oreq, filter)
}
// GetGroup implements Backend.GetGroup
func (_d PrometheusBackend) GetGroup(ctx context.Context, nameOrID string, queryParam url.Values) (gp1 *libregraph.Group, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
// it's a get operation that returns a pointer (and not an array): check whether that's nil or not
if gp1 == nil {
result = MetricResultNotFound
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetGroup, result).Observe(duration)
}()
return _d.delegate.GetGroup(ctx, nameOrID, queryParam)
}
// GetGroupMembers implements Backend.GetGroupMembers
func (_d PrometheusBackend) GetGroupMembers(ctx context.Context, id string, oreq *godata.GoDataRequest) (upa1 []*libregraph.User, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetGroupMembers, result).Observe(duration)
}()
return _d.delegate.GetGroupMembers(ctx, id, oreq)
}
// GetGroups implements Backend.GetGroups
func (_d PrometheusBackend) GetGroups(ctx context.Context, oreq *godata.GoDataRequest) (gpa1 []*libregraph.Group, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetGroups, result).Observe(duration)
}()
return _d.delegate.GetGroups(ctx, oreq)
}
// GetUser implements Backend.GetUser
func (_d PrometheusBackend) GetUser(ctx context.Context, nameOrID string, oreq *godata.GoDataRequest) (up1 *libregraph.User, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
// it's a get operation that returns a pointer (and not an array): check whether that's nil or not
if up1 == nil {
result = MetricResultNotFound
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetUser, result).Observe(duration)
}()
return _d.delegate.GetUser(ctx, nameOrID, oreq)
}
// GetUsers implements Backend.GetUsers
func (_d PrometheusBackend) GetUsers(ctx context.Context, oreq *godata.GoDataRequest) (upa1 []*libregraph.User, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetUsers, result).Observe(duration)
}()
return _d.delegate.GetUsers(ctx, oreq)
}
// RemoveMemberFromGroup implements Backend.RemoveMemberFromGroup
func (_d PrometheusBackend) RemoveMemberFromGroup(ctx context.Context, groupID string, memberID string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpRemoveMemberFromGroup, result).Observe(duration)
}()
return _d.delegate.RemoveMemberFromGroup(ctx, groupID, memberID)
}
// UpdateGroupName implements Backend.UpdateGroupName
func (_d PrometheusBackend) UpdateGroupName(ctx context.Context, groupID string, groupName string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpUpdateGroupName, result).Observe(duration)
}()
return _d.delegate.UpdateGroupName(ctx, groupID, groupName)
}
// UpdateLastSignInDate implements Backend.UpdateLastSignInDate
func (_d PrometheusBackend) UpdateLastSignInDate(ctx context.Context, userID string, timestamp time.Time) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpUpdateLastSignInDate, result).Observe(duration)
}()
return _d.delegate.UpdateLastSignInDate(ctx, userID, timestamp)
}
// UpdateUser implements Backend.UpdateUser
func (_d PrometheusBackend) UpdateUser(ctx context.Context, nameOrID string, user libregraph.UserUpdate) (up1 *libregraph.User, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpUpdateUser, result).Observe(duration)
}()
return _d.delegate.UpdateUser(ctx, nameOrID, user)
}
@@ -0,0 +1,73 @@
import (
"errors"
"time"
"github.com/go-ldap/ldap/v3"
"github.com/opencloud-eu/opencloud/services/graph/pkg/metrics"
"github.com/prometheus/client_golang/prometheus"
)
{{ $decorator := (or .Vars.DecoratorName (printf "Prometheus%s" .Interface.Name)) }}
// {{$decorator}} implements {{.Interface.Type}} interface with all methods wrapped
// with Prometheus metrics
type {{$decorator}} struct {
delegate {{.Interface.Type}}
metric *prometheus.HistogramVec
}
var _ {{.Interface.Type}} = &{{$decorator}}{}
// returns an instance of the {{.Interface.Type}} decorated with prometheus metric
func New{{$decorator}}(delegate {{.Interface.Type}}, metric *prometheus.HistogramVec) {{$decorator}} {
return {{$decorator}} {
delegate: delegate,
metric: metric,
}
}
{{range $method := .Interface.Methods}}
// {{$method.Name}} implements {{$.Interface.Type}}.{{$method.Name}}
func (_d {{$decorator}}) {{$method.Declaration}} {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
{{- if $method.ReturnsError}}
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
{{- if or (hasPrefix "Get" $method.Name) }}
{{- range $i, $result := $method.Results }}
{{- if and (hasPrefix "*" $result.Type) (not (hasPrefix "*[]" $result.Type)) }}
// it's a get operation that returns a pointer (and not an array): check whether that's nil or not
if {{$result.Name}} == nil {
result = MetricResultNotFound
}{{break}}
{{end}}
{{end}}
{{end}}
{{- range $i, $result := $method.Results }}
{{- if eq $result.Type "error" }}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
{{end}}
{{end}}
}
{{end}}
_d.metric.WithLabelValues(MetricOp{{upFirst $method.Name}}, result).Observe(duration)
}()
{{$method.Pass "_d.delegate."}}
}
{{end}}
+16
View File
@@ -28,6 +28,22 @@ type CS3 struct {
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
}
var _ Backend = &CS3{}
func NewCS3Backend(config *shared.Reva, gatewaySelector pool.Selectable[gateway.GatewayAPIClient], logger *log.Logger) (*CS3, error) {
logger = &log.Logger{Logger: logger.With().
// Str("backend", "cs3"). // already added upstream
Str("gateway", config.Address).
Logger(),
}
return &CS3{
Config: config,
GatewaySelector: gatewaySelector,
Logger: logger,
}, nil
}
// CreateUser implements the Backend Interface. It's currently not supported for the CS3 backend
func (i *CS3) CreateUser(ctx context.Context, user libregraph.User) (*libregraph.User, error) {
return nil, errNotImplemented
@@ -0,0 +1,871 @@
// Code generated by gowrap. DO NOT EDIT.
// template: backend_prometheus.tmpl
// gowrap: http://github.com/hexdigest/gowrap
package identity
import (
"context"
"time"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
"github.com/prometheus/client_golang/prometheus"
)
// PrometheusEducationBackend implements EducationBackend interface with all methods wrapped
// with Prometheus metrics
type PrometheusEducationBackend struct {
delegate EducationBackend
metric *prometheus.HistogramVec
}
var _ EducationBackend = &PrometheusEducationBackend{}
// returns an instance of the EducationBackend decorated with prometheus metric
func NewPrometheusEducationBackend(delegate EducationBackend, metric *prometheus.HistogramVec) PrometheusEducationBackend {
return PrometheusEducationBackend{
delegate: delegate,
metric: metric,
}
}
// AddClassesToEducationSchool implements EducationBackend.AddClassesToEducationSchool
func (_d PrometheusEducationBackend) AddClassesToEducationSchool(ctx context.Context, schoolNumberOrID string, memberIDs []string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpAddClassesToEducationSchool, result).Observe(duration)
}()
return _d.delegate.AddClassesToEducationSchool(ctx, schoolNumberOrID, memberIDs)
}
// AddTeacherToEducationClass implements EducationBackend.AddTeacherToEducationClass
func (_d PrometheusEducationBackend) AddTeacherToEducationClass(ctx context.Context, classID string, teacherID string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpAddTeacherToEducationClass, result).Observe(duration)
}()
return _d.delegate.AddTeacherToEducationClass(ctx, classID, teacherID)
}
// AddUsersToEducationSchool implements EducationBackend.AddUsersToEducationSchool
func (_d PrometheusEducationBackend) AddUsersToEducationSchool(ctx context.Context, schoolID string, memberID []string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpAddUsersToEducationSchool, result).Observe(duration)
}()
return _d.delegate.AddUsersToEducationSchool(ctx, schoolID, memberID)
}
// CreateEducationClass implements EducationBackend.CreateEducationClass
func (_d PrometheusEducationBackend) CreateEducationClass(ctx context.Context, class libregraph.EducationClass) (ep1 *libregraph.EducationClass, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpCreateEducationClass, result).Observe(duration)
}()
return _d.delegate.CreateEducationClass(ctx, class)
}
// CreateEducationSchool implements EducationBackend.CreateEducationSchool
func (_d PrometheusEducationBackend) CreateEducationSchool(ctx context.Context, group libregraph.EducationSchool) (ep1 *libregraph.EducationSchool, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpCreateEducationSchool, result).Observe(duration)
}()
return _d.delegate.CreateEducationSchool(ctx, group)
}
// CreateEducationUser implements EducationBackend.CreateEducationUser
func (_d PrometheusEducationBackend) CreateEducationUser(ctx context.Context, user libregraph.EducationUser) (ep1 *libregraph.EducationUser, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpCreateEducationUser, result).Observe(duration)
}()
return _d.delegate.CreateEducationUser(ctx, user)
}
// DeleteEducationClass implements EducationBackend.DeleteEducationClass
func (_d PrometheusEducationBackend) DeleteEducationClass(ctx context.Context, nameOrID string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpDeleteEducationClass, result).Observe(duration)
}()
return _d.delegate.DeleteEducationClass(ctx, nameOrID)
}
// DeleteEducationSchool implements EducationBackend.DeleteEducationSchool
func (_d PrometheusEducationBackend) DeleteEducationSchool(ctx context.Context, id string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpDeleteEducationSchool, result).Observe(duration)
}()
return _d.delegate.DeleteEducationSchool(ctx, id)
}
// DeleteEducationUser implements EducationBackend.DeleteEducationUser
func (_d PrometheusEducationBackend) DeleteEducationUser(ctx context.Context, nameOrID string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpDeleteEducationUser, result).Observe(duration)
}()
return _d.delegate.DeleteEducationUser(ctx, nameOrID)
}
// FilterEducationSchoolsByAttribute implements EducationBackend.FilterEducationSchoolsByAttribute
func (_d PrometheusEducationBackend) FilterEducationSchoolsByAttribute(ctx context.Context, attr string, value string) (epa1 []*libregraph.EducationSchool, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpFilterEducationSchoolsByAttribute, result).Observe(duration)
}()
return _d.delegate.FilterEducationSchoolsByAttribute(ctx, attr, value)
}
// FilterEducationUsersByAttribute implements EducationBackend.FilterEducationUsersByAttribute
func (_d PrometheusEducationBackend) FilterEducationUsersByAttribute(ctx context.Context, attr string, value string) (epa1 []*libregraph.EducationUser, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpFilterEducationUsersByAttribute, result).Observe(duration)
}()
return _d.delegate.FilterEducationUsersByAttribute(ctx, attr, value)
}
// GetEducationClass implements EducationBackend.GetEducationClass
func (_d PrometheusEducationBackend) GetEducationClass(ctx context.Context, namedOrID string) (ep1 *libregraph.EducationClass, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
// it's a get operation that returns a pointer (and not an array): check whether that's nil or not
if ep1 == nil {
result = MetricResultNotFound
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetEducationClass, result).Observe(duration)
}()
return _d.delegate.GetEducationClass(ctx, namedOrID)
}
// GetEducationClassMembers implements EducationBackend.GetEducationClassMembers
func (_d PrometheusEducationBackend) GetEducationClassMembers(ctx context.Context, nameOrID string) (epa1 []*libregraph.EducationUser, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetEducationClassMembers, result).Observe(duration)
}()
return _d.delegate.GetEducationClassMembers(ctx, nameOrID)
}
// GetEducationClassTeachers implements EducationBackend.GetEducationClassTeachers
func (_d PrometheusEducationBackend) GetEducationClassTeachers(ctx context.Context, classID string) (epa1 []*libregraph.EducationUser, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetEducationClassTeachers, result).Observe(duration)
}()
return _d.delegate.GetEducationClassTeachers(ctx, classID)
}
// GetEducationClasses implements EducationBackend.GetEducationClasses
func (_d PrometheusEducationBackend) GetEducationClasses(ctx context.Context) (epa1 []*libregraph.EducationClass, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetEducationClasses, result).Observe(duration)
}()
return _d.delegate.GetEducationClasses(ctx)
}
// GetEducationSchool implements EducationBackend.GetEducationSchool
func (_d PrometheusEducationBackend) GetEducationSchool(ctx context.Context, nameOrID string) (ep1 *libregraph.EducationSchool, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
// it's a get operation that returns a pointer (and not an array): check whether that's nil or not
if ep1 == nil {
result = MetricResultNotFound
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetEducationSchool, result).Observe(duration)
}()
return _d.delegate.GetEducationSchool(ctx, nameOrID)
}
// GetEducationSchoolClasses implements EducationBackend.GetEducationSchoolClasses
func (_d PrometheusEducationBackend) GetEducationSchoolClasses(ctx context.Context, schoolNumberOrID string) (epa1 []*libregraph.EducationClass, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetEducationSchoolClasses, result).Observe(duration)
}()
return _d.delegate.GetEducationSchoolClasses(ctx, schoolNumberOrID)
}
// GetEducationSchoolUsers implements EducationBackend.GetEducationSchoolUsers
func (_d PrometheusEducationBackend) GetEducationSchoolUsers(ctx context.Context, id string) (epa1 []*libregraph.EducationUser, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetEducationSchoolUsers, result).Observe(duration)
}()
return _d.delegate.GetEducationSchoolUsers(ctx, id)
}
// GetEducationSchools implements EducationBackend.GetEducationSchools
func (_d PrometheusEducationBackend) GetEducationSchools(ctx context.Context) (epa1 []*libregraph.EducationSchool, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetEducationSchools, result).Observe(duration)
}()
return _d.delegate.GetEducationSchools(ctx)
}
// GetEducationUser implements EducationBackend.GetEducationUser
func (_d PrometheusEducationBackend) GetEducationUser(ctx context.Context, nameOrID string) (ep1 *libregraph.EducationUser, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
// it's a get operation that returns a pointer (and not an array): check whether that's nil or not
if ep1 == nil {
result = MetricResultNotFound
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetEducationUser, result).Observe(duration)
}()
return _d.delegate.GetEducationUser(ctx, nameOrID)
}
// GetEducationUsers implements EducationBackend.GetEducationUsers
func (_d PrometheusEducationBackend) GetEducationUsers(ctx context.Context) (epa1 []*libregraph.EducationUser, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpGetEducationUsers, result).Observe(duration)
}()
return _d.delegate.GetEducationUsers(ctx)
}
// RemoveClassFromEducationSchool implements EducationBackend.RemoveClassFromEducationSchool
func (_d PrometheusEducationBackend) RemoveClassFromEducationSchool(ctx context.Context, schoolNumberOrID string, memberID string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpRemoveClassFromEducationSchool, result).Observe(duration)
}()
return _d.delegate.RemoveClassFromEducationSchool(ctx, schoolNumberOrID, memberID)
}
// RemoveTeacherFromEducationClass implements EducationBackend.RemoveTeacherFromEducationClass
func (_d PrometheusEducationBackend) RemoveTeacherFromEducationClass(ctx context.Context, classID string, teacherID string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpRemoveTeacherFromEducationClass, result).Observe(duration)
}()
return _d.delegate.RemoveTeacherFromEducationClass(ctx, classID, teacherID)
}
// RemoveUserFromEducationSchool implements EducationBackend.RemoveUserFromEducationSchool
func (_d PrometheusEducationBackend) RemoveUserFromEducationSchool(ctx context.Context, schoolID string, memberID string) (err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpRemoveUserFromEducationSchool, result).Observe(duration)
}()
return _d.delegate.RemoveUserFromEducationSchool(ctx, schoolID, memberID)
}
// UpdateEducationClass implements EducationBackend.UpdateEducationClass
func (_d PrometheusEducationBackend) UpdateEducationClass(ctx context.Context, id string, class libregraph.EducationClass) (ep1 *libregraph.EducationClass, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpUpdateEducationClass, result).Observe(duration)
}()
return _d.delegate.UpdateEducationClass(ctx, id, class)
}
// UpdateEducationSchool implements EducationBackend.UpdateEducationSchool
func (_d PrometheusEducationBackend) UpdateEducationSchool(ctx context.Context, numberOrID string, school libregraph.EducationSchool) (ep1 *libregraph.EducationSchool, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpUpdateEducationSchool, result).Observe(duration)
}()
return _d.delegate.UpdateEducationSchool(ctx, numberOrID, school)
}
// UpdateEducationUser implements EducationBackend.UpdateEducationUser
func (_d PrometheusEducationBackend) UpdateEducationUser(ctx context.Context, nameOrID string, user libregraph.EducationUser) (ep1 *libregraph.EducationUser, err error) {
_since := time.Now()
defer func() {
duration := time.Since(_since).Seconds()
result := MetricResultSuccess
if err != nil {
result = MetricResultFailure
if err == ErrReadOnly {
result = MetricResultReadOnly
}
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
} else {
if e, ok := errorcode.ToError(err); ok {
if e.GetCode() == errorcode.ItemNotFound {
result = MetricResultNotFound
}
}
}
_d.metric.WithLabelValues(MetricOpUpdateEducationUser, result).Observe(duration)
}()
return _d.delegate.UpdateEducationUser(ctx, nameOrID, user)
}
@@ -9,6 +9,8 @@ import (
// ErrEducationBackend is a dummy EducationBackend, doing nothing
type ErrEducationBackend struct{}
var _ EducationBackend = &ErrEducationBackend{}
// CreateEducationSchool creates the supplied school in the identity backend.
func (i *ErrEducationBackend) CreateEducationSchool(ctx context.Context, school libregraph.EducationSchool) (*libregraph.EducationSchool, error) {
return nil, errNotImplemented
+175
View File
@@ -0,0 +1,175 @@
package identity
import (
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"os"
"strings"
ldapv3 "github.com/go-ldap/ldap/v3"
ocldap "github.com/opencloud-eu/opencloud/pkg/ldap"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/pkg/registry"
"github.com/opencloud-eu/opencloud/services/graph/pkg/config"
"github.com/opencloud-eu/opencloud/services/graph/pkg/metrics"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
"github.com/opencloud-eu/reva/v2/pkg/utils/ldap"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/trace"
)
const (
cs3Backend = "cs3"
ldapBackend = "ldap"
)
var supportedBackends = []string{cs3Backend, ldapBackend}
func CreateIdentityBackends(name string, cfg *config.Config, logger *log.Logger, registrer prometheus.Registerer, traceProvider trace.TracerProvider) (Backend, EducationBackend, error) {
switch name {
case cs3Backend:
gatewaySelector, err := pool.GatewaySelector(
cfg.Reva.Address,
append(
cfg.Reva.GetRevaOptions(),
pool.WithRegistry(registry.GetRegistry()),
pool.WithTracerProvider(traceProvider),
)...,
)
if err != nil {
return nil, nil, err
}
if cs3, err := NewCS3Backend(cfg.Reva, gatewaySelector, logger); err != nil {
return nil, nil, err
} else {
return cs3, nil, nil
}
case ldapBackend:
var err error
var tlsConf *tls.Config
if cfg.Identity.LDAP.Insecure {
// When insecure is set to true then we don't need a certificate.
cfg.Identity.LDAP.CACert = ""
tlsConf = &tls.Config{
MinVersion: tls.VersionTLS12,
//nolint:gosec // We need the ability to run with "insecure" (dev/testing)
InsecureSkipVerify: cfg.Identity.LDAP.Insecure,
}
}
if cfg.Identity.LDAP.CACert != "" {
if err := ocldap.WaitForCA(*logger,
cfg.Identity.LDAP.Insecure,
cfg.Identity.LDAP.CACert); err != nil {
logger.Fatal().Err(err).Msg("The configured LDAP CA cert does not exist")
}
if tlsConf == nil {
tlsConf = &tls.Config{
MinVersion: tls.VersionTLS12,
}
}
certs := x509.NewCertPool()
pemData, err := os.ReadFile(cfg.Identity.LDAP.CACert)
if err != nil {
logger.Error().Err(err).Msg("Error initializing LDAP Backend")
return nil, nil, err
}
if !certs.AppendCertsFromPEM(pemData) {
logger.Error().Msg("Error initializing LDAP Backend. Adding CA cert failed")
return nil, nil, err
}
tlsConf.RootCAs = certs
}
ldapConfig := ldap.Config{
URI: cfg.Identity.LDAP.URI,
BindDN: cfg.Identity.LDAP.BindDN,
BindPassword: cfg.Identity.LDAP.BindPassword,
TLSConfig: tlsConf,
}
logger = &log.Logger{Logger: logger.With().
Str("ldap-uri", ldapConfig.URI).
Logger(),
}
conn := ldap.NewLDAPWithReconnect(ldapConfig)
conn.SetLogger(&logger.Logger)
lb, err := NewLDAPBackend(conn, cfg.Identity.LDAP, logger, metrics.Namespace, metrics.Subsystem, registrer)
if err != nil {
logger.Error().Err(err).Msg("Error initializing LDAP Backend")
return nil, nil, err
}
var identityBackend Backend = lb
var eduBackend EducationBackend = lb
if !cfg.Identity.Metrics.Disabled && registrer != nil {
backendApiOperationDuration := prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metrics.Namespace,
Subsystem: metrics.Subsystem,
Name: "identity_backend_api_duration_seconds",
Help: "Duration of API operations performed by the Graph service identity backend in seconds.",
Buckets: prometheus.DefBuckets,
ConstLabels: prometheus.Labels{
MetricLabelType: name,
},
}, []string{MetricLabelOperation, metrics.LabelResult})
if err := registrer.Register(backendApiOperationDuration); err != nil {
logger.Warn().Err(err).Msg("failed to register backend API operation duration metric")
}
identityBackend = NewPrometheusBackend(identityBackend, backendApiOperationDuration)
eduBackend = NewPrometheusEducationBackend(eduBackend, backendApiOperationDuration)
}
if !cfg.Identity.LDAP.EducationResourcesEnabled {
// in this case, simply bury the previous eduBackend, no need to wrap or anything: if we had
// a previous implementation in there that wrapped with metrics or such, we don't want to
// have any cross-cutting concerns running here, just use this implementation that returns
// errors on purpose and that's it:
eduBackend = &ErrEducationBackend{}
}
disableMechanismType, err := ParseDisableMechanismType(cfg.Identity.LDAP.DisableUserMechanism)
if err != nil {
logger.Error().Err(err).Msg("Error initializing LDAP Backend")
return nil, nil, err
}
if disableMechanismType == DisableMechanismGroup {
logger.Info().Msg("LocalUserDisable is true, will create group if not exists")
err := lb.CreateLDAPGroupByDN(cfg.Identity.LDAP.LdapDisabledUsersGroupDN)
if err != nil {
isAnError := false
var lerr *ldapv3.Error
if errors.As(err, &lerr) {
if lerr.ResultCode != ldapv3.LDAPResultEntryAlreadyExists {
isAnError = true
}
} else {
isAnError = true
}
if isAnError {
msg := "error adding group for disabling users"
logger.Error().Err(err).Str("local_user_disable", cfg.Identity.LDAP.LdapDisabledUsersGroupDN).Msg(msg)
return nil, nil, fmt.Errorf("%s: %w", msg, err)
}
}
}
return identityBackend, eduBackend, nil
default:
err := fmt.Errorf("unknown identity backend: %q, must be one of [%s]", name, strings.Join(supportedBackends, ", "))
logger.Error().Err(err).Msgf("failed to create identity backend %q", name)
return nil, nil, err
}
}
+180 -48
View File
@@ -8,6 +8,7 @@ import (
"slices"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/CiscoM31/godata"
@@ -15,10 +16,12 @@ import (
"github.com/google/uuid"
"github.com/libregraph/idm/pkg/ldapdn"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/graph/pkg/config"
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
"github.com/opencloud-eu/opencloud/services/graph/pkg/metrics"
"github.com/opencloud-eu/opencloud/services/graph/pkg/odata"
)
@@ -74,9 +77,12 @@ type LDAP struct {
educationConfig educationConfig
logger *log.Logger
conn ldap.Client
conn LdapClient
}
var _ Backend = &LDAP{}
var _ EducationBackend = &LDAP{}
type userAttributeMap struct {
displayName string
id string
@@ -107,11 +113,33 @@ func ParseDisableMechanismType(disableMechanism string) (DisableUserMechanismTyp
return t, nil
}
func NewLDAPBackend(lc ldap.Client, config config.LDAP, logger *log.Logger) (*LDAP, error) {
const (
MetricResultSuccess = "success"
MetricResultFailure = "failure"
MetricResultNotFound = "not-found"
MetricResultReadOnly = "read-only"
)
const (
MetricLabelOperation = "operation"
MetricLabelType = "type"
MetricLabelUri = "uri"
MetricLabelWrite = "write"
)
func NewLDAPBackend(lc ldap.Client, config config.LDAP, logger *log.Logger, namespace string, subsystem string, registry prometheus.Registerer) (*LDAP, error) {
if config.UserDisplayNameAttribute == "" || config.UserIDAttribute == "" ||
config.UserEmailAttribute == "" || config.UserNameAttribute == "" {
return nil, errors.New("invalid user attribute mappings")
}
logger = &log.Logger{Logger: logger.With().
// Str("backend", "ldap"). // already added upstream
Bool("write", config.WriteEnabled).
Bool("refint", config.RefintEnabled).
Logger(),
}
uam := userAttributeMap{
displayName: config.UserDisplayNameAttribute,
id: config.UserIDAttribute,
@@ -154,6 +182,65 @@ func NewLDAPBackend(lc ldap.Client, config config.LDAP, logger *log.Logger) (*LD
return nil, fmt.Errorf("error configuring disable user mechanism: %w", err)
}
var client LdapClient
client = NewGoLdapLdapClient(lc)
if !config.Metrics.Disabled && registry != nil {
// metrics are enabled for the LDAP identity backend
// use a 'write' label to indicate whether this instance is read-only
// (write=="0") or allowed to make changes (wrote=="1")
write := "0"
if config.WriteEnabled {
write = "1"
}
// a metric that tracks the duration of the LDAP operations we perform as an LDAP client,
// will be passed to a Prometheus wrapper around LdapClient below
ldapEgressDuration := prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "ldap_client_operation_duration_seconds",
Help: "Duration of LDAP operations performed by the Graph service in seconds.",
Buckets: prometheus.DefBuckets,
ConstLabels: prometheus.Labels{
MetricLabelUri: config.URI,
MetricLabelWrite: write,
},
}, []string{MetricLabelOperation, metrics.LabelResult})
if err := registry.Register(ldapEgressDuration); err != nil {
logger.Warn().Err(err).Msg("failed to register LDAP egress duration metric")
}
// a metric that tracks the number of ongoing concurrent LDAP operations, will also be
// passed to a Prometheus wrapper around LdapClient below;
// note that we use an atomic int as a gauge to count operations up and down using the
// wrapper, and then a gauge func that retrieves the current value of that atomic int
// whenever scraped by Prometheus, as that approach performs better than calling inc/dec
// on a Gauge object directly:
var inFlight atomic.Int64
ldapEgressInFlight := prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "ldap_client_operations",
Help: "Number of LDAP client operations in-flight in the Graph service.",
Unit: "operation",
ConstLabels: prometheus.Labels{
MetricLabelUri: config.URI,
MetricLabelWrite: write,
},
}, func() float64 {
// when scraped, we simply read the current value of the atomic int:
return float64(inFlight.Load())
})
if err := registry.Register(ldapEgressInFlight); err != nil {
logger.Warn().Err(err).Msg("failed to register LDAP egress in-flight metric")
}
// we sill use an LdapClient that wraps the "proper" LdapClient with recording the
// metrics referenced above:
client = NewPrometheusLdapClient(client, ldapEgressDuration, &inFlight)
}
return &LDAP{
useServerUUID: config.UseServerUUID,
usePwModifyExOp: config.UsePasswordModExOp,
@@ -174,7 +261,7 @@ func NewLDAPBackend(lc ldap.Client, config config.LDAP, logger *log.Logger) (*LD
disableUserMechanism: disableMechanismType,
localUserDisableGroupDN: config.LdapDisabledUsersGroupDN,
logger: logger,
conn: lc,
conn: client,
writeEnabled: config.WriteEnabled,
refintEnabled: config.RefintEnabled,
}, nil
@@ -185,7 +272,7 @@ func NewLDAPBackend(lc ldap.Client, config config.LDAP, logger *log.Logger) (*LD
// configured LDAP server
func (i *LDAP) CreateUser(ctx context.Context, user libregraph.User) (*libregraph.User, error) {
logger := i.logger.SubloggerWithRequestID(ctx)
logger.Debug().Str("backend", "ldap").Msg("CreateUser")
logger.Debug().Msg("CreateUser")
if !i.writeEnabled {
return nil, ErrReadOnly
}
@@ -222,14 +309,14 @@ func (i *LDAP) CreateUser(ctx context.Context, user libregraph.User) (*libregrap
if err != nil {
return nil, err
}
return i.createUserModelFromLDAP(e), nil
return i.createUserModelFromLDAP(e)
}
// DeleteUser implements the Backend Interface. It permanently deletes a User identified
// by name or id from the LDAP server
func (i *LDAP) DeleteUser(ctx context.Context, nameOrID string) error {
logger := i.logger.SubloggerWithRequestID(ctx)
logger.Debug().Str("backend", "ldap").Msg("DeleteUser")
logger.Debug().Msg("DeleteUser")
if !i.writeEnabled {
return ErrReadOnly
}
@@ -237,6 +324,7 @@ func (i *LDAP) DeleteUser(ctx context.Context, nameOrID string) error {
if err != nil {
return err
}
dr := ldap.DelRequest{DN: e.DN}
if err = i.conn.Del(&dr); err != nil {
msg := "error deleting user"
@@ -272,11 +360,11 @@ func (i *LDAP) DeleteUser(ctx context.Context, nameOrID string) error {
// UpdateUser implements the Backend Interface for the LDAP Backend
func (i *LDAP) UpdateUser(ctx context.Context, nameOrID string, user libregraph.UserUpdate) (*libregraph.User, error) {
logger := i.logger.SubloggerWithRequestID(ctx)
logger.Debug().Str("backend", "ldap").Msg("UpdateUser")
logger.Debug().Msg("UpdateUser")
if !i.writeEnabled {
// still allow to enable/disable user when using DisableMechanismGroup
if i.disableUserMechanism == DisableMechanismGroup && isUserEnabledUpdate(user) {
logger.Error().Str("backend", "ldap").Msg("Allowing accountEnabled Update on read-only backend")
logger.Error().Msg("Allowing accountEnabled Update on read-only backend")
} else {
return nil, ErrReadOnly
}
@@ -396,12 +484,16 @@ func (i *LDAP) UpdateUser(ctx context.Context, nameOrID string, user libregraph.
return nil, err
}
returnUser := i.createUserModelFromLDAP(e)
// To avoid a ldap lookup for group membership, set the enabled flag to same as input value
// since this would have been updated with group membership from the input anyway.
if user.AccountEnabled != nil && i.disableUserMechanism == DisableMechanismGroup {
returnUser.AccountEnabled = user.AccountEnabled
returnUser, err := i.createUserModelFromLDAP(e)
if err != nil {
return nil, err
}
if returnUser != nil {
// To avoid a ldap lookup for group membership, set the enabled flag to same as input value
// since this would have been updated with group membership from the input anyway.
if user.AccountEnabled != nil && i.disableUserMechanism == DisableMechanismGroup {
returnUser.AccountEnabled = user.AccountEnabled
}
}
return returnUser, nil
@@ -445,7 +537,7 @@ func (i *LDAP) getEntryByDN(dn string, attrs []string, filter string) (*ldap.Ent
nil,
)
i.logger.Debug().Str("backend", "ldap").
i.logger.Debug().
Str("base", searchRequest.BaseDN).
Str("filter", searchRequest.Filter).
Int("scope", searchRequest.Scope).
@@ -454,16 +546,34 @@ func (i *LDAP) getEntryByDN(dn string, attrs []string, filter string) (*ldap.Ent
Msg("getEntryByDN")
res, err := i.conn.Search(searchRequest)
if err != nil {
i.logger.Error().Err(err).Str("backend", "ldap").Str("dn", dn).Msg("Search ldap by DN failed")
return nil, errorcode.New(errorcode.ItemNotFound, "user lookup failed")
}
if len(res.Entries) == 0 {
return nil, ErrNotFound
i.logger.Error().Err(err).Str("dn", dn).Msg("Search ldap by DN failed")
msg := "user lookup failed"
errMap := ldapResultToErrMap{
ldap.LDAPResultNoSuchObject: errorcode.New(errorcode.ItemNotFound, msg),
ldap.LDAPResultUnwillingToPerform: errorcode.New(errorcode.NotAllowed, msg),
ldap.LDAPResultInsufficientAccessRights: errorcode.New(errorcode.NotAllowed, msg),
ldap.LDAPResultSizeLimitExceeded: errorcode.New(errorcode.TooManyResults, msg),
ldapGenericErr: errorcode.New(errorcode.GeneralException, msg),
}
return nil, i.mapLDAPError(err, errMap)
}
return res.Entries[0], nil
switch len(res.Entries) {
case 0:
return nil, ErrNotFound
case 1:
return res.Entries[0], nil
default:
return nil, ErrTooManyResults
}
}
// Retrieves a single entry from LDAP.
//
// It never returns nil for the *ldap.Entry:
// - if no object is found, it returns a ErrNotFound error
// - if more than one object is found, it returns a ErrTooManyResults error
// - if exactly one object is found, it returns that entry and no error
func (i *LDAP) searchLDAPEntryByFilter(basedn string, attrs []string, filter string) (*ldap.Entry, error) {
if filter == "" {
filter = "(objectclass=*)"
@@ -478,7 +588,7 @@ func (i *LDAP) searchLDAPEntryByFilter(basedn string, attrs []string, filter str
nil,
)
i.logger.Debug().Str("backend", "ldap").
i.logger.Debug().
Str("base", searchRequest.BaseDN).
Str("filter", searchRequest.Filter).
Int("scope", searchRequest.Scope).
@@ -487,14 +597,25 @@ func (i *LDAP) searchLDAPEntryByFilter(basedn string, attrs []string, filter str
Msg("getEntryByFilter")
res, err := i.conn.Search(searchRequest)
if err != nil {
i.logger.Error().Err(err).Str("backend", "ldap").Str("dn", basedn).Str("filter", filter).Msg("Search user by filter failed")
return nil, errorcode.New(errorcode.ItemNotFound, "user search failed")
i.logger.Error().Err(err).Str("dn", basedn).Str("filter", filter).Msg("Search user by filter failed")
msg := "user search failed"
errMap := ldapResultToErrMap{
ldap.LDAPResultNoSuchObject: errorcode.New(errorcode.ItemNotFound, msg),
ldap.LDAPResultUnwillingToPerform: errorcode.New(errorcode.NotAllowed, msg),
ldap.LDAPResultInsufficientAccessRights: errorcode.New(errorcode.NotAllowed, msg),
ldap.LDAPResultSizeLimitExceeded: errorcode.New(errorcode.TooManyResults, msg),
ldapGenericErr: errorcode.New(errorcode.GeneralException, msg),
}
return nil, i.mapLDAPError(err, errMap)
}
if len(res.Entries) == 0 {
switch len(res.Entries) {
case 0:
return nil, ErrNotFound
case 1:
return res.Entries[0], nil
default:
return nil, ErrTooManyResults
}
return res.Entries[0], nil
}
func filterEscapeAttribute(attribute string, binary bool, id string) (string, error) {
@@ -572,16 +693,16 @@ func (i *LDAP) getLDAPUserByFilter(filter string) (*ldap.Entry, error) {
// GetUser implements the Backend Interface.
func (i *LDAP) GetUser(ctx context.Context, nameOrID string, oreq *godata.GoDataRequest) (*libregraph.User, error) {
logger := i.logger.SubloggerWithRequestID(ctx)
logger.Debug().Str("backend", "ldap").Msg("GetUser")
logger.Debug().Msg("GetUser")
e, err := i.getLDAPUserByNameOrID(nameOrID)
if err != nil {
return nil, err
}
u := i.createUserModelFromLDAP(e)
if u == nil {
return nil, ErrNotFound
u, err := i.createUserModelFromLDAP(e)
if err != nil {
return nil, err
}
if i.disableUserMechanism != DisableMechanismNone {
@@ -601,7 +722,11 @@ func (i *LDAP) GetUser(ctx context.Context, nameOrID string, oreq *godata.GoData
if err != nil {
return nil, err
}
u.MemberOf = i.groupsFromLDAPEntries(userGroups)
if memberOf, err := i.groupsFromLDAPEntries(userGroups); err != nil {
// TODO: should we really just silently skip LDAP data model errors here, or rather return this as an error?
} else {
u.MemberOf = memberOf
}
}
return u, nil
}
@@ -614,7 +739,7 @@ func (i *LDAP) GetUsers(ctx context.Context, oreq *godata.GoDataRequest) ([]*lib
// FilterUsers implements the Backend Interface.
func (i *LDAP) FilterUsers(ctx context.Context, oreq *godata.GoDataRequest, filter *godata.ParseNode) ([]*libregraph.User, error) {
logger := i.logger.SubloggerWithRequestID(ctx)
logger.Debug().Str("backend", "ldap").Msg("GetUsers")
logger.Debug().Msg("GetUsers")
queryFilter, err := i.oDataFilterToLDAPFilter(filter)
if err != nil {
@@ -648,7 +773,7 @@ func (i *LDAP) FilterUsers(ctx context.Context, oreq *godata.GoDataRequest, filt
i.getUserAttrTypesForSearch(),
nil,
)
logger.Debug().Str("backend", "ldap").
logger.Debug().
Str("base", searchRequest.BaseDN).
Str("filter", searchRequest.Filter).
Int("scope", searchRequest.Scope).
@@ -676,9 +801,9 @@ func (i *LDAP) usersFromLDAPEntries(entries []*ldap.Entry, exp []string) ([]*lib
}
users := make([]*libregraph.User, 0, len(entries))
for _, e := range entries {
u := i.createUserModelFromLDAP(e)
// Skip invalid LDAP users
if u == nil {
u, err := i.createUserModelFromLDAP(e)
if u == nil || err != nil {
// Skip invalid LDAP users
continue
}
@@ -692,7 +817,11 @@ func (i *LDAP) usersFromLDAPEntries(entries []*ldap.Entry, exp []string) ([]*lib
if err != nil {
return nil, err
}
u.MemberOf = i.groupsFromLDAPEntries(userGroups)
if memberOf, err := i.groupsFromLDAPEntries(userGroups); err != nil {
// TODO: should we really just silently skip LDAP data model errors here, or rather return this as an error?
} else {
u.MemberOf = memberOf
}
}
users = append(users, u)
}
@@ -702,14 +831,15 @@ func (i *LDAP) usersFromLDAPEntries(entries []*ldap.Entry, exp []string) ([]*lib
// UpdateLastSignInDate implements the Backend Interface.
func (i *LDAP) UpdateLastSignInDate(ctx context.Context, userID string, timestamp time.Time) error {
if !i.writeEnabled {
i.logger.Debug().Str("backend", "ldap").Msg("The LDAP Server is readonly. Skipping update of last sign in date")
return nil
i.logger.Debug().Msg("The LDAP Server is readonly. Skipping update of last sign in date")
return nil // TODO: do we really want to just silently do nothing here, rather than returning an error?
}
e, err := i.getLDAPUserByID(userID)
switch {
case errors.Is(err, ErrNotFound):
i.logger.Warn().Err(err).Str("userID", userID).Msg("Failed to update last sign in date for user")
return nil
return nil // TODO questionable whether this should just fail silently because the user was not found
case err != nil:
return err
}
@@ -819,7 +949,7 @@ func (i *LDAP) renameMemberInGroup(ctx context.Context, group *ldap.Entry, oldMe
func (i *LDAP) updateUserPassword(ctx context.Context, dn, password string) error {
logger := i.logger.SubloggerWithRequestID(ctx)
logger.Debug().Str("backend", "ldap").Msg("updateUserPassword")
logger.Debug().Msg("updateUserPassword")
pwMod := ldap.PasswordModifyRequest{
UserIdentity: dn,
NewPassword: password,
@@ -860,9 +990,9 @@ func (i *LDAP) ldapUUIDtoString(e *ldap.Entry, attribute string, binary bool) (s
return e.GetEqualFoldAttributeValue(attribute), nil
}
func (i *LDAP) createUserModelFromLDAP(e *ldap.Entry) *libregraph.User {
func (i *LDAP) createUserModelFromLDAP(e *ldap.Entry) (*libregraph.User, error) {
if e == nil {
return nil
return nil, nil
}
opsan := e.GetEqualFoldAttributeValue(i.userAttributeMap.userName)
@@ -910,10 +1040,12 @@ func (i *LDAP) createUserModelFromLDAP(e *ldap.Entry) *libregraph.User {
case !errors.Is(err, errNotSet):
i.logger.Warn().Err(err).Str("dn", e.DN).Msg("Error getting last signin timestamp")
}
return user
return user, nil
}
err = errorcode.New(errorcode.GeneralException, "Invalid User. Missing username or id attribute")
i.logger.Warn().Str("dn", e.DN).Str("id", id).Str("username", opsan).Msg("Invalid User. Missing username or id attribute")
return nil
return nil, err
}
func (i *LDAP) userToLDAPAttrValues(user libregraph.User) (map[string][]string, error) {
@@ -1082,7 +1214,7 @@ func (i *LDAP) removeEntryByDNAndAttributeFromEntry(entry *ldap.Entry, dn string
}
}
if !found {
i.logger.Error().Str("backend", "ldap").Str("entry", entry.DN).Str("target", dn).
i.logger.Error().Str("entry", entry.DN).Str("target", dn).
Msg("The target value is not present in the attribute list")
return ErrNotFound
}
@@ -1126,7 +1258,7 @@ func (i *LDAP) removeEntryByDNAndAttributeFromEntry(entry *ldap.Entry, dn string
// expandLDAPAttributeEntries reads an attribute from a ldap entry and expands to users
func (i *LDAP) expandLDAPAttributeEntries(ctx context.Context, e *ldap.Entry, attribute, searchTerm string) ([]*ldap.Entry, error) {
logger := i.logger.SubloggerWithRequestID(ctx)
logger.Debug().Str("backend", "ldap").Msg("ExpandLDAPAttributeEntries")
logger.Debug().Msg("ExpandLDAPAttributeEntries")
result := []*ldap.Entry{}
for _, entryDN := range e.GetEqualFoldAttributeValues(attribute) {
Loaded 100 of 643 files, more files were not shown because too many files have changed in this diff. Show more