Compare commits

...

22 Commits

Author SHA1 Message Date
OpenCloud Devops
86dbae6412 🎉 Release 4.1.0 (#1960)
* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.0.1

* 🎉 Release 4.1.0

* 🎉 Release 4.1.0

* 🎉 Release 4.1.0

* 🎉 Release 4.1.0

* 🎉 Release 4.1.0

* 🎉 Release 4.1.0

* 🎉 Release 4.1.0
2025-12-15 19:51:20 +01:00
Benedikt Kulmann
87257623c6 chore: bump web to v4.3.0 (#2030) 2025-12-15 18:04:49 +01:00
Viktor Scharf
8aac5f6318 reva-bump-2.41.0 (#2032) 2025-12-15 17:03:58 +01:00
Anja Barz
4dcecbf5c0 fix typo (#2024) 2025-12-15 15:01:50 +01:00
Sawjan Gurung
cffeb4a690 [full-ci][tests-only] test: fix some test flakiness (#2003)
* test: check content after upload

Signed-off-by: Saw-jan <saw.jan.grg3e@gmail.com>

test: check content with retry

Signed-off-by: Saw-jan <saw.jan.grg3e@gmail.com>

* test: check empty body before json decoding

Signed-off-by: Saw-jan <saw.jan.grg3e@gmail.com>

* test: wait post-processing for webdav requests if applicable

Signed-off-by: Saw-jan <saw.jan.grg3e@gmail.com>

* test: check token before doing request

Signed-off-by: Saw-jan <saw.jan.grg3e@gmail.com>

test: check body before json decoding

Signed-off-by: Saw-jan <saw.jan.grg3e@gmail.com>

test: add wait step

Signed-off-by: Saw-jan <saw.jan.grg3e@gmail.com>

---------

Signed-off-by: Saw-jan <saw.jan.grg3e@gmail.com>
2025-12-15 14:45:12 +05:45
opencloudeu
826640c2c5 [tx] updated from transifex 2025-12-14 00:03:10 +00:00
opencloudeu
8a4785e0e7 [tx] updated from transifex 2025-12-13 00:03:05 +00:00
Viktor Scharf
e7d8f3f446 show edition in the opncloud version output (#2019) 2025-12-12 15:01:25 +01:00
Prashant Gurung
b1c9159bd1 skip test related pipelines for ready-release-go PRs (#2011)
Signed-off-by: prashant-gurung899 <prasantgrg777@gmail.com>
2025-12-12 14:30:26 +05:45
Sawjan Gurung
446ae35701 Merge pull request #2007 from opencloud-eu/ci/fix-translation-pipeline
ci: fix translation pipeline
2025-12-12 10:46:26 +05:45
Florian Schade
8f323c775a Merge pull request #2001 from fschade/release-channel-capability
enhancement: introduce release channel capability
2025-12-11 09:33:02 +01:00
Florian Schade
40d8aacea4 enhancement: introduce build time edition channels
be careful, the env:OC_EDITION, env:FRONTEND_EDITION, and conf:edition got removed as part of this commit, no deprecation because the flag is build time only!
2025-12-10 16:21:42 +01:00
Michael Flemming
4c5d5fb218 Merge pull request #1947 from opencloud-eu/production_to_rolling_image
create a rolling release on all tag events [CI SKIP]
2025-12-10 10:29:40 +01:00
Michael 'Flimmy' Flemming
ec30bcc030 ci: add logic to do multiple docker releases for non-patch production-releases 2025-12-09 16:45:09 +01:00
Michael 'Flimmy' Flemming
61a591bcba ci: move config getter from nested if to for-loop 2025-12-09 16:34:29 +01:00
Michael 'Flimmy' Flemming
fc9a62a2d8 add comments to nested ifs 2025-12-09 16:25:16 +01:00
Benedikt Kulmann
b595461ae7 Merge pull request #1995 from opencloud-eu/enforce-server-url-trailing-slash
fix: enforce trailing slash for server url
2025-12-09 16:16:06 +01:00
Sawjan Gurung
a3a1397e2d Merge pull request #1993 from opencloud-eu/test/add-mismatch-offset-test
[full-ci][tests-only] test: add test to check mismatch offset during TUS upload
2025-12-09 17:16:29 +05:45
Jörn Friedrich Dreyer
dbabedb90b Merge pull request #1996 from opencloud-eu/fix-policies-link
[docs] update policies link

pipeline failure unrelated
2025-12-09 12:21:42 +01:00
Jörn Friedrich Dreyer
dd4f2fe529 update policies link
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
2025-12-09 10:55:12 +01:00
Benedikt Kulmann
614d916978 fix: enforce trailing slash for server url 2025-12-09 10:14:08 +01:00
Saw-jan
ee16c0597c test: add test to check mismatch offset during TUS upload
Signed-off-by: Saw-jan <saw.jan.grg3e@gmail.com>
2025-12-09 12:51:35 +05:45
63 changed files with 880 additions and 142 deletions

View File

@@ -36,8 +36,18 @@ ifndef DATE
DATE := $(shell date -u '+%Y%m%d')
endif
LDFLAGS += -X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn -s -w -X "$(OC_REPO)/pkg/version.String=$(STRING)" -X "$(OC_REPO)/pkg/version.Tag=$(VERSION)" -X "$(OC_REPO)/pkg/version.Date=$(DATE)"
DEBUG_LDFLAGS += -X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn -X "$(OC_REPO)/pkg/version.String=$(STRING)" -X "$(OC_REPO)/pkg/version.Tag=$(VERSION)" -X "$(OC_REPO)/pkg/version.Date=$(DATE)"
LDFLAGS += -X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn -s -w \
-X "$(OC_REPO)/pkg/version.Edition=$(EDITION)" \
-X "$(OC_REPO)/pkg/version.String=$(STRING)" \
-X "$(OC_REPO)/pkg/version.Tag=$(VERSION)" \
-X "$(OC_REPO)/pkg/version.Date=$(DATE)"
DEBUG_LDFLAGS += -X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn \
-X "$(OC_REPO)/pkg/version.Edition=$(EDITION)" \
-X "$(OC_REPO)/pkg/version.String=$(STRING)" \
-X "$(OC_REPO)/pkg/version.Tag=$(VERSION)" \
-X "$(OC_REPO)/pkg/version.Date=$(DATE)"
DOCKER_LDFLAGS += -X "$(OC_REPO)/pkg/config/defaults.BaseDataPathType=path" -X "$(OC_REPO)/pkg/config/defaults.BaseDataPathValue=/var/lib/opencloud"
DOCKER_LDFLAGS += -X "$(OC_REPO)/pkg/config/defaults.BaseConfigPathType=path" -X "$(OC_REPO)/pkg/config/defaults.BaseConfigPathValue=/etc/opencloud"

View File

@@ -1,4 +1,4 @@
# The test runner source for UI tests
WEB_COMMITID=50e3fff6a518361d59cba864a927470f313b6f91
WEB_BRANCH=stable-4.2
WEB_COMMITID=3120ea384c7a9d1f1ea0c328965951fc06d66900
WEB_BRANCH=main

View File

@@ -388,6 +388,8 @@ config = {
"production": {
# NOTE: need to be updated if new production releases are determined
"tags": ["2.0", "4.0"],
# NOTE: need to be set to true if patch releases are made from stable-X-branches
"skip_rolling": "false",
"repo": docker_repo_slug,
"build_type": "production",
},
@@ -479,6 +481,10 @@ def main(ctx):
if ctx.build.event == "cron" and ctx.build.sender == "translation-sync":
return translation_sync(ctx)
is_release_pr = (ctx.build.event == "pull_request" and ctx.build.sender == "openclouders" and "🎉 release" in ctx.build.title.lower())
if is_release_pr:
return [licenseCheck(ctx)]
build_release_helpers = \
readyReleaseGo()
@@ -1608,31 +1614,40 @@ def uploadTracingResult(ctx):
def dockerReleases(ctx):
pipelines = []
docker_repos = []
docker_releases = []
build_type = ""
# only make realeases on tag events
if ctx.build.event == "tag":
tag = ctx.build.ref.replace("refs/tags/v", "").lower()
# iterate over production tags to see if this is a production release
is_production = False
skip_rolling = False
for prod_tag in config["dockerReleases"]["production"]["tags"]:
if tag.startswith(prod_tag):
is_production = True
skip_rolling = config["dockerReleases"]["production"]["skip_rolling"]
break
if is_production:
docker_repos.append(config["dockerReleases"]["production"]["repo"])
build_type = config["dockerReleases"]["production"]["build_type"]
docker_releases.append("production")
# a new production realease is also a rolling release
# unless skip_rolling is set in the config, i.e. for patch-releases on stable-branch
if not skip_rolling:
docker_releases.append("rolling")
else:
docker_repos.append(config["dockerReleases"]["rolling"]["repo"])
build_type = config["dockerReleases"]["rolling"]["build_type"]
docker_releases.append("rolling")
# on non tag events, do daily build
else:
docker_repos.append(config["dockerReleases"]["daily"]["repo"])
build_type = config["dockerReleases"]["daily"]["build_type"]
docker_releases.append("daily")
for repo in docker_repos:
for releaseConfigName in docker_releases:
repo = config["dockerReleases"][releaseConfigName]["repo"]
build_type = config["dockerReleases"][releaseConfigName]["build_type"]
repo_pipelines = []
repo_pipelines.append(dockerRelease(ctx, repo, build_type))
@@ -1652,6 +1667,7 @@ def dockerRelease(ctx, repo, build_type):
build_args = {
"REVISION": "%s" % ctx.build.commit,
"VERSION": "%s" % (ctx.build.ref.replace("refs/tags/", "") if ctx.build.event == "tag" else "daily"),
"EDITION": "stable" if build_type == "production" else "rolling",
}
# if no additional tag is given, the build-plugin adds latest
@@ -1815,6 +1831,7 @@ def binaryRelease(ctx, arch, depends_on = []):
"image": OC_CI_GOLANG,
"environment": {
"VERSION": (ctx.build.ref.replace("refs/tags/", "") if ctx.build.event == "tag" else "daily"),
"EDITION": "rolling",
"HTTP_PROXY": {
"from_secret": "ci_http_proxy",
},
@@ -2340,11 +2357,12 @@ def translation_sync(ctx):
"image": OC_CI_GOLANG,
"commands": [
"make l10n-read",
"mkdir tx && cd tx",
"curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash",
". ~/.profile",
"export PATH=$PATH:$(pwd) && cd ..",
"make l10n-push",
"make l10n-pull",
"rm tx",
"rm -rf tx",
"make l10n-clean",
],
"environment": {

View File

@@ -1,5 +1,41 @@
# Changelog
## [4.1.0](https://github.com/opencloud-eu/opencloud/releases/tag/v4.1.0) - 2025-12-15
### ❤️ Thanks to all contributors! ❤️
@JammingBen, @ScharfViktor, @Svanvith, @butonic, @flimmy, @fschade, @individual-it, @kulmann, @micbar, @prashant-gurung899, @saw-jan
### 📚 Documentation
- fix typo [[#2024](https://github.com/opencloud-eu/opencloud/pull/2024)]
- [docs] update policies link [[#1996](https://github.com/opencloud-eu/opencloud/pull/1996)]
- fix the link in quickstart script for itself [[#1956](https://github.com/opencloud-eu/opencloud/pull/1956)]
### ✅ Tests
- [full-ci][tests-only] test: fix some test flakiness [[#2003](https://github.com/opencloud-eu/opencloud/pull/2003)]
- [tests-only] Skip test related pipelines for ready-release-go PRs [[#2011](https://github.com/opencloud-eu/opencloud/pull/2011)]
- [full-ci][tests-only] test: add test to check mismatch offset during TUS upload [[#1993](https://github.com/opencloud-eu/opencloud/pull/1993)]
- [full-ci][tests-only] test: proper resource existence check [[#1990](https://github.com/opencloud-eu/opencloud/pull/1990)]
- check propfing after renaming data in file system [[#1809](https://github.com/opencloud-eu/opencloud/pull/1809)]
- fix-get-attribute-test [[#1974](https://github.com/opencloud-eu/opencloud/pull/1974)]
### 📈 Enhancement
- Show edition in opencloud version command [[#2019](https://github.com/opencloud-eu/opencloud/pull/2019)]
### 🐛 Bug Fixes
- fix: enforce trailing slash for server url [[#1995](https://github.com/opencloud-eu/opencloud/pull/1995)]
- fix: enhance resource creation with detailed process information [[#1978](https://github.com/opencloud-eu/opencloud/pull/1978)]
### 📦️ Dependencies
- chore: bump web to v4.3.0 [[#2030](https://github.com/opencloud-eu/opencloud/pull/2030)]
- reva-bump-2.41.0 [[#2032](https://github.com/opencloud-eu/opencloud/pull/2032)]
- build(deps): bump github.com/testcontainers/testcontainers-go from 0.39.0 to 0.40.0 [[#1931](https://github.com/opencloud-eu/opencloud/pull/1931)]
## [4.0.0](https://github.com/opencloud-eu/opencloud/releases/tag/v4.0.0) - 2025-12-01
### ❤️ Thanks to all contributors! ❤️

2
go.mod
View File

@@ -64,7 +64,7 @@ require (
github.com/open-policy-agent/opa v1.10.1
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76
github.com/opencloud-eu/reva/v2 v2.40.1
github.com/opencloud-eu/reva/v2 v2.41.0
github.com/opensearch-project/opensearch-go/v4 v4.5.0
github.com/orcaman/concurrent-map v1.0.0
github.com/pkg/errors v0.9.1

4
go.sum
View File

@@ -963,8 +963,8 @@ github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9 h1:dIft
github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9/go.mod h1:JWyDC6H+5oZRdUJUgKuaye+8Ph5hEs6HVzVoPKzWSGI=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76 h1:vD/EdfDUrv4omSFjrinT8Mvf+8D7f9g4vgQ2oiDrVUI=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q=
github.com/opencloud-eu/reva/v2 v2.40.1 h1:QwMkbGMhwDSwfk2WxbnTpIig2BugPBaVFjWcy2DSU3U=
github.com/opencloud-eu/reva/v2 v2.40.1/go.mod h1:DGH08n2mvtsQLkt8o15FV6m51FwSJJGhjR8Ty+iIJww=
github.com/opencloud-eu/reva/v2 v2.41.0 h1:oie8+sxcA+drREXRTqm0LmfUdy/mmaa6pA6wkdF6tF4=
github.com/opencloud-eu/reva/v2 v2.41.0/go.mod h1:DGH08n2mvtsQLkt8o15FV6m51FwSJJGhjR8Ty+iIJww=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=

View File

@@ -3,6 +3,7 @@ ARG TARGETOS
ARG TARGETARCH
ARG VERSION
ARG STRING
ARG EDITION
RUN apk add bash make git curl gcc musl-dev libc-dev binutils-gold inotify-tools vips-dev

View File

@@ -33,6 +33,7 @@ func VersionCommand(cfg *config.Config) *cli.Command {
Category: "info",
Action: func(c *cli.Context) error {
fmt.Println("Version: " + version.GetString())
fmt.Printf("Edition: %s\n", version.Edition)
fmt.Printf("Compiled: %s\n", version.Compiled())
if c.Bool(_skipServiceListingFlagName) {

View File

@@ -0,0 +1,4 @@
package version
// InitEdition exports the private edition initialization func for testing
var InitEdition = initEdition

View File

@@ -1,9 +1,27 @@
package version
import (
"fmt"
"slices"
"strings"
"time"
"github.com/Masterminds/semver"
"github.com/opencloud-eu/reva/v2/pkg/logger"
)
const (
// Dev is used as a placeholder.
Dev = "dev"
// EditionDev indicates the development build channel was used to build the binary.
EditionDev = Dev
// EditionRolling indicates the rolling release build channel was used to build the binary.
EditionRolling = "rolling"
// EditionStable indicates the stable release build channel was used to build the binary.
EditionStable = "stable"
// EditionLTS indicates the lts release build channel was used to build the binary.
EditionLTS = "lts"
)
var (
@@ -21,17 +39,56 @@ var (
// Date indicates the build date.
// This has been removed, it looks like you can only replace static strings with recent go versions
//Date = time.Now().Format("20060102")
Date = "dev"
Date = Dev
// Legacy defines the old long 4 number OpenCloud version needed for some clients
Legacy = "0.1.0.0"
// LegacyString defines the old OpenCloud version needed for some clients
LegacyString = "0.1.0"
// Edition describes the build channel (stable, rolling, nightly, daily, dev)
Edition = Dev // default for self-compiled builds
)
func init() { //nolint:gochecknoinits
if err := initEdition(); err != nil {
logger.New().Error().Err(err).Msg("falling back to dev")
}
}
func initEdition() error {
regularEditions := []string{EditionDev, EditionRolling, EditionStable}
versionedEditions := []string{EditionLTS}
if !slices.ContainsFunc(slices.Concat(regularEditions, versionedEditions), func(s string) bool {
isRegularEdition := slices.Contains(regularEditions, Edition)
if isRegularEdition && s == Edition {
return true
}
// handle editions with a version
editionParts := strings.Split(Edition, "-")
if len(editionParts) != 2 { // a versioned edition channel must consist of exactly 2 parts.
return false
}
isVersionedEdition := slices.Contains(versionedEditions, editionParts[0])
if !isVersionedEdition { // not all channels can contain version information
return false
}
_, err := semver.NewVersion(editionParts[1])
return err == nil
}) {
Edition = Dev
return fmt.Errorf(`unknown edition channel "%s"`, Edition)
}
return nil
}
// Compiled returns the compile time of this service.
func Compiled() time.Time {
if Date == "dev" {
if Date == Dev {
return time.Now()
}
t, _ := time.Parse("20060102", Date)

View File

@@ -0,0 +1,65 @@
package version_test
import (
"fmt"
"testing"
"github.com/opencloud-eu/opencloud/pkg/version"
)
func TestChannel(t *testing.T) {
tests := map[string]struct {
got string
valid bool
}{
"no channel, defaults to dev": {
got: "",
valid: false,
},
"dev channel": {
got: version.EditionDev,
valid: true,
},
"rolling channel": {
got: version.EditionRolling,
valid: true,
},
"stable channel": {
got: version.EditionStable,
valid: true,
},
"lts channel without version": {
got: version.EditionLTS,
valid: false,
},
"lts-1.0.0 channel": {
got: fmt.Sprintf("%s-1", version.EditionLTS),
valid: true,
},
"lts-one invalid version": {
got: fmt.Sprintf("%s-one", version.EditionLTS),
valid: false,
},
"known channel with version": {
got: fmt.Sprintf("%s-1", version.EditionStable),
valid: false,
},
"unknown channel": {
got: "foo",
valid: false,
},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
version.Edition = test.got
switch err := version.InitEdition(); {
case err != nil && !test.valid && version.Edition != version.Dev: // if a given edition is unknown, the value is always dev
fallthrough
case test.valid != (err == nil):
t.Fatalf("invalid edition: %s", version.Edition)
}
})
}
}

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-17 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Mário Machado, 2025\n"
"Language-Team: Portuguese (https://app.transifex.com/opencloud-eu/teams/204053/pt/)\n"

View File

@@ -33,7 +33,7 @@ type Config struct {
EnableFederatedSharingIncoming bool `yaml:"enable_federated_sharing_incoming" env:"OC_ENABLE_OCM;FRONTEND_ENABLE_FEDERATED_SHARING_INCOMING" desc:"Changing this value is NOT supported. Enables support for incoming federated sharing for clients. The backend behaviour is not changed." introductionVersion:"1.0.0"`
EnableFederatedSharingOutgoing bool `yaml:"enable_federated_sharing_outgoing" env:"OC_ENABLE_OCM;FRONTEND_ENABLE_FEDERATED_SHARING_OUTGOING" desc:"Changing this value is NOT supported. Enables support for outgoing federated sharing for clients. The backend behaviour is not changed." introductionVersion:"1.0.0"`
SearchMinLength int `yaml:"search_min_length" env:"FRONTEND_SEARCH_MIN_LENGTH" desc:"Minimum number of characters to enter before a client should start a search for Share receivers. This setting can be used to customize the user experience if e.g too many results are displayed." introductionVersion:"1.0.0"`
Edition string `yaml:"edition" env:"OC_EDITION;FRONTEND_EDITION" desc:"Edition of OpenCloud. Used for branding purposes." introductionVersion:"1.0.0"`
Edition string `desc:"Edition of OpenCloud. Used for branding purposes." introductionVersion:"1.0.0"`
DisableSSE bool `yaml:"disable_sse" env:"OC_DISABLE_SSE;FRONTEND_DISABLE_SSE" desc:"When set to true, clients are informed that the Server-Sent Events endpoint is not accessible." introductionVersion:"1.0.0"`
DisableRadicale bool `yaml:"disable_radicale" env:"FRONTEND_DISABLE_RADICALE" desc:"When set to true, clients are informed that the Radicale (CalDAV/CardDAV) is not accessible." introductionVersion:"4.0.0"`
DefaultLinkPermissions int `yaml:"default_link_permissions" env:"FRONTEND_DEFAULT_LINK_PERMISSIONS" desc:"Defines the default permissions a link is being created with. Possible values are 0 (= internal link, for instance members only) and 1 (= public link with viewer permissions). Defaults to 1." introductionVersion:"1.0.0"`

View File

@@ -5,6 +5,7 @@ import (
"github.com/opencloud-eu/opencloud/pkg/shared"
"github.com/opencloud-eu/opencloud/pkg/structs"
"github.com/opencloud-eu/opencloud/pkg/version"
"github.com/opencloud-eu/opencloud/services/frontend/pkg/config"
)
@@ -87,7 +88,7 @@ func DefaultConfig() *config.Config {
DefaultUploadProtocol: "tus",
DefaultLinkPermissions: 1,
SearchMinLength: 3,
Edition: "",
Edition: version.Edition,
CheckForUpdates: true,
Checksums: config.Checksums{
SupportedTypes: []string{"sha1", "md5", "adler32"},

View File

@@ -346,7 +346,7 @@ func FrontendConfigFromStruct(cfg *config.Config, logger log.Logger) (map[string
},
"version": map[string]interface{}{
"product": "OpenCloud",
"edition": "",
"edition": version.Edition,
"major": version.ParsedLegacy().Major(),
"minor": version.ParsedLegacy().Minor(),
"micro": version.ParsedLegacy().Patch(),

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-15 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Stephan Paternotte <stephan@paternottes.net>, 2025\n"
"Language-Team: Dutch (https://app.transifex.com/opencloud-eu/teams/204053/nl/)\n"

View File

@@ -14,7 +14,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-15 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Radoslaw Posim, 2025\n"
"Language-Team: Polish (https://app.transifex.com/opencloud-eu/teams/204053/pl/)\n"

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-17 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Mário Machado, 2025\n"
"Language-Team: Portuguese (https://app.transifex.com/opencloud-eu/teams/204053/pt/)\n"

View File

@@ -12,7 +12,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-21 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Lulufox, 2025\n"
"Language-Team: Russian (https://app.transifex.com/opencloud-eu/teams/204053/ru/)\n"

View File

@@ -12,7 +12,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-23 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: miguel tapias, 2025\n"
"Language-Team: Spanish (https://app.transifex.com/opencloud-eu/teams/204053/es/)\n"

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-24 00:02+0000\n"
"POT-Creation-Date: 2025-12-14 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Jiri Grönroos <jiri.gronroos@iki.fi>, 2025\n"
"Language-Team: Finnish (https://app.transifex.com/opencloud-eu/teams/204053/fi/)\n"

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-15 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Stephan Paternotte <stephan@paternottes.net>, 2025\n"
"Language-Team: Dutch (https://app.transifex.com/opencloud-eu/teams/204053/nl/)\n"

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-18 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Mário Machado, 2025\n"
"Language-Team: Portuguese (https://app.transifex.com/opencloud-eu/teams/204053/pt/)\n"

View File

@@ -12,7 +12,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-16 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Lulufox, 2025\n"
"Language-Team: Russian (https://app.transifex.com/opencloud-eu/teams/204053/ru/)\n"

View File

@@ -80,5 +80,5 @@ type Status struct {
Product string
ProductName string
ProductVersion string
Edition string `yaml:"edition" env:"OC_EDITION;OCDAV_EDITION" desc:"Edition of OpenCloud. Used for branding purposes." introductionVersion:"1.0.0"`
Edition string `desc:"Edition of OpenCloud. Used for branding purposes." introductionVersion:"1.0.0"`
}

View File

@@ -92,7 +92,7 @@ func DefaultConfig() *config.Config {
ProductVersion: version.GetString(),
Product: "OpenCloud",
ProductName: "OpenCloud",
Edition: "",
Edition: version.Edition,
},
}
}

View File

@@ -164,4 +164,4 @@ A good example of how such a file should be formatted can be found in the [Apach
## Example Policies
The policies service contains a set of preconfigured example policies. See the [deployment examples](https://github.com/opencloud-eu/opencloud/tree/main/deployments/examples) directory for details. The contained policies disallow OpenCloud to create certain file types, both via the proxy middleware and the events service via postprocessing.
The policies service contains a set of preconfigured example policies. See the [devtools policie](https://github.com/opencloud-eu/opencloud/tree/main/devtools/deployments/service_policies/policies/) directory for details. The contained policies disallow OpenCloud to create certain file types, both via the proxy middleware and the events service via postprocessing.

View File

@@ -7,5 +7,5 @@ type HTTP struct {
Namespace string `yaml:"-"`
TLSCert string `yaml:"tls_cert" env:"PROXY_TRANSPORT_TLS_CERT" desc:"Path/File name of the TLS server certificate (in PEM format) for the external http services. If not defined, the root directory derives from $OC_BASE_DATA_PATH/proxy." introductionVersion:"1.0.0"`
TLSKey string `yaml:"tls_key" env:"PROXY_TRANSPORT_TLS_KEY" desc:"Path/File name for the TLS certificate key (in PEM format) for the server certificate to use for the external http services. If not defined, the root directory derives from $OC_BASE_DATA_PATH/proxy." introductionVersion:"1.0.0"`
TLS bool `yaml:"tls" env:"PROXY_TLS" desc:"Enable/Disable HTTPS for external HTTP services. Must be set to 'true' if the built-in IDP service an no reverse proxy is used. See the text description for details." introductionVersion:"1.0.0"`
TLS bool `yaml:"tls" env:"PROXY_TLS" desc:"Enable/Disable HTTPS for external HTTP services. Must be set to 'true' if the built-in IDP service and no reverse proxy is used. See the text description for details." introductionVersion:"1.0.0"`
}

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-24 00:02+0000\n"
"POT-Creation-Date: 2025-12-14 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Jiri Grönroos <jiri.gronroos@iki.fi>, 2025\n"
"Language-Team: Finnish (https://app.transifex.com/opencloud-eu/teams/204053/fi/)\n"

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-18 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: idoet <idoet@protonmail.ch>, 2025\n"
"Language-Team: Indonesian (https://app.transifex.com/opencloud-eu/teams/204053/id/)\n"

View File

@@ -13,7 +13,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-15 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Radoslaw Posim, 2025\n"
"Language-Team: Polish (https://app.transifex.com/opencloud-eu/teams/204053/pl/)\n"

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-17 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Mário Machado, 2025\n"
"Language-Team: Portuguese (https://app.transifex.com/opencloud-eu/teams/204053/pt/)\n"

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-15 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Stephan Paternotte <stephan@paternottes.net>, 2025\n"
"Language-Team: Dutch (https://app.transifex.com/opencloud-eu/teams/204053/nl/)\n"

View File

@@ -13,7 +13,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-15 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Radoslaw Posim, 2025\n"
"Language-Team: Polish (https://app.transifex.com/opencloud-eu/teams/204053/pl/)\n"

View File

@@ -11,7 +11,7 @@ msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: EMAIL\n"
"POT-Creation-Date: 2025-11-17 00:02+0000\n"
"POT-Creation-Date: 2025-12-13 00:02+0000\n"
"PO-Revision-Date: 2025-01-27 10:17+0000\n"
"Last-Translator: Mário Machado, 2025\n"
"Language-Team: Portuguese (https://app.transifex.com/opencloud-eu/teams/204053/pt/)\n"

View File

@@ -1,6 +1,6 @@
SHELL := bash
NAME := web
WEB_ASSETS_VERSION = v4.2.1
WEB_ASSETS_VERSION = v4.3.0
WEB_ASSETS_BRANCH = main
ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI

View File

@@ -136,6 +136,9 @@ func (p Web) getPayload() (payload []byte, err error) {
p.config.Web.Config.Apps = make([]string, 0)
}
// ensure that the server url has a trailing slash
p.config.Web.Config.Server = strings.TrimRight(p.config.Web.Config.Server, "/") + "/"
return json.Marshal(p.config.Web.Config)
}

View File

@@ -33,6 +33,7 @@ use SimpleXMLElement;
use Sabre\Xml\LibXMLException;
use Sabre\Xml\Reader;
use GuzzleHttp\Pool;
use Symfony\Component\HttpFoundation\Response;
/**
* Helper for HTTP requests
@@ -74,7 +75,6 @@ class HttpRequestHelper {
* than download it all up-front.
* @param int|null $timeout
* @param Client|null $client
* @param string|null $bearerToken
*
* @return ResponseInterface
* @throws GuzzleException
@@ -92,8 +92,42 @@ class HttpRequestHelper {
bool $stream = false,
?int $timeout = 0,
?Client $client = null,
?string $bearerToken = null
): ResponseInterface {
$bearerToken = null;
if (TokenHelper::useBearerToken() && $user && $user !== 'public') {
$bearerToken = TokenHelper::getTokens($user, $password, $url)['access_token'];
// check token is still valid
$parsedUrl = parse_url($url);
$baseUrl = $parsedUrl['scheme'] . '://' . $parsedUrl['host'];
$baseUrl .= isset($parsedUrl['port']) ? ':' . $parsedUrl['port'] : '';
$testUrl = $baseUrl . "/graph/v1.0/use/$user";
if (OcHelper::isTestingOnReva()) {
$url = $baseUrl . "/ocs/v2.php/cloud/users/$user";
}
// check token validity with a GET request
$c = self::createClient(
$user,
$password,
$config,
$cookies,
$stream,
$timeout,
$bearerToken
);
$testReq = self::createRequest($testUrl, $xRequestId, 'GET');
try {
$testRes = $c->send($testReq);
} catch (RequestException $ex) {
$testRes = $ex->getResponse();
if ($testRes && $testRes->getStatusCode() === Response::HTTP_UNAUTHORIZED) {
// token is invalid or expired, get a new one
echo "[INFO] Bearer token expired or invalid, getting a new one...\n";
TokenHelper::clearAllTokens();
$bearerToken = TokenHelper::getTokens($user, $password, $url)['access_token'];
}
}
}
if ($client === null) {
$client = self::createClient(
$user,
@@ -160,6 +194,24 @@ class HttpRequestHelper {
}
HttpLogger::logResponse($response);
// wait for post-processing to finish if applicable
if (WebdavHelper::isDAVRequest($url)
&& \str_starts_with($url, OcHelper::getServerUrl())
&& \in_array($method, ["PUT", "MOVE", "COPY"])
&& \in_array($response->getStatusCode(), [Response::HTTP_CREATED, Response::HTTP_NO_CONTENT])
&& OcConfigHelper::getPostProcessingDelay() === 0
) {
if (\in_array($method, ["MOVE", "COPY"])) {
$url = $headers['Destination'];
}
WebDavHelper::waitForPostProcessingToFinish(
$url,
$user,
$password,
$headers,
);
}
return $response;
}
@@ -203,13 +255,6 @@ class HttpRequestHelper {
} else {
$debugResponses = false;
}
// use basic auth for 'public' user or no user
if ($user === 'public' || $user === null || $user === '') {
$bearerToken = null;
} else {
$useBearerToken = TokenHelper::useBearerToken();
$bearerToken = $useBearerToken ? TokenHelper::getTokens($user, $password, $url)['access_token'] : null;
}
$sendRetryLimit = self::numRetriesOnHttpTooEarly();
$sendCount = 0;
@@ -228,7 +273,6 @@ class HttpRequestHelper {
$stream,
$timeout,
$client,
$bearerToken,
);
if ($response->getStatusCode() >= 400
@@ -256,7 +300,8 @@ class HttpRequestHelper {
// we need to repeat the send request, because we got HTTP_TOO_EARLY or HTTP_CONFLICT
// wait 1 second before sending again, to give the server some time
// to finish whatever post-processing it might be doing.
self::debugResponse($response);
echo "[INFO] Received '" . $response->getStatusCode() .
"' status code, retrying request ($sendCount)...\n";
\sleep(1);
}
} while ($loopAgain);

View File

@@ -30,6 +30,26 @@ use Psr\Http\Message\ResponseInterface;
* A helper class for configuring OpenCloud server
*/
class OcConfigHelper {
public static $postProcessingDelay = 0;
/**
* @return int
*/
public static function getPostProcessingDelay(): int {
return self::$postProcessingDelay;
}
/**
* @param string $postProcessingDelay
*
* @return void
*/
public static function setPostProcessingDelay(string $postProcessingDelay): void {
// extract number from string
$delay = (int) filter_var($postProcessingDelay, FILTER_SANITIZE_NUMBER_INT);
self::$postProcessingDelay = $delay;
}
/**
* @param string $url
* @param string $method

View File

@@ -84,7 +84,9 @@ class TokenHelper {
$tokenData = [
'access_token' => $refreshedToken['access_token'],
'refresh_token' => $refreshedToken['refresh_token'],
'expires_at' => time() + 300 // 5 minutes
// set expiry to 240 (4 minutes) seconds to allow for some buffer
// token actually expires in 300 seconds (5 minutes)
'expires_at' => time() + 240
];
self::$tokenCache[$cacheKey] = $tokenData;
return $tokenData;
@@ -100,7 +102,9 @@ class TokenHelper {
$tokenData = [
'access_token' => $tokens['access_token'],
'refresh_token' => $tokens['refresh_token'],
'expires_at' => time() + 290 // set expiry to 290 seconds to allow for some buffer
// set expiry to 240 (4 minutes) seconds to allow for some buffer
// token actually expires in 300 seconds (5 minutes)
'expires_at' => time() + 240
];
// Save to cache

View File

@@ -923,4 +923,45 @@ class WebDavHelper {
$mtime = new DateTime($xmlPart[0]->__toString());
return $mtime->format('U');
}
/**
* wait until the reqeust doesn't return 425 anymore
*
* @param string $url
* @param ?string $user
* @param ?string $password
* @param ?array $headers
*
* @return void
*/
public static function waitForPostProcessingToFinish(
string $url,
?string $user = null,
?string $password = null,
?array $headers = [],
): void {
$retried = 0;
do {
$response = HttpRequestHelper::sendRequest(
$url,
'check-425-status',
'GET',
$user,
$password,
$headers,
);
$statusCode = $response->getStatusCode();
if ($statusCode !== 425) {
return;
}
$tryAgain = $retried < HttpRequestHelper::numRetriesOnHttpTooEarly();
if ($tryAgain) {
$retried += 1;
echo "[INFO] Waiting for post processing to finish, attempt ($retried)...\n";
// wait 1s and try again
\sleep(1);
}
} while ($tryAgain);
echo "[ERROR] 10 seconds timeout! Post processing did not finish in time.\n";
}
}

View File

@@ -2026,8 +2026,12 @@ class FeatureContext extends BehatVariablesContext {
if ($response === null) {
$response = $this->getResponse();
}
$body = (string)$response->getBody();
if (!$body) {
return [];
}
return \json_decode(
(string)$response->getBody(),
$body,
true
);
}

View File

@@ -68,6 +68,7 @@ class OcConfigContext implements Context {
$response->getStatusCode(),
"Failed to set async upload with delayed post processing"
);
OcConfigHelper::setPostProcessingDelay($delayTime);
}
/**
@@ -90,6 +91,9 @@ class OcConfigContext implements Context {
$response->getStatusCode(),
"Failed to set config $configVariable=$configValue"
);
if ($configVariable === "POSTPROCESSING_DELAY") {
OcConfigHelper::setPostProcessingDelay($configValue);
}
}
/**
@@ -184,6 +188,9 @@ class OcConfigContext implements Context {
$envs = [];
foreach ($table->getHash() as $row) {
$envs[$row['config']] = $row['value'];
if ($row['config'] === "POSTPROCESSING_DELAY") {
OcConfigHelper::setPostProcessingDelay($row['value']);
}
}
$response = OcConfigHelper::reConfigureOc($envs);
@@ -200,6 +207,7 @@ class OcConfigContext implements Context {
* @return void
*/
public function rollbackOc(): void {
OcConfigHelper::setPostProcessingDelay('0');
$response = OcConfigHelper::rollbackOc();
Assert::assertEquals(
200,

View File

@@ -607,7 +607,7 @@ trait Provisioning {
Assert::assertEquals(
201,
$response->getStatusCode(),
__METHOD__ . " cannot create user '$userName' using Graph API.\nResponse:" .
__METHOD__ . " cannot create user '$userName'.\nResponse:" .
json_encode($this->getJsonDecodedResponse($response))
);
@@ -1083,7 +1083,7 @@ trait Provisioning {
Assert::assertEquals(
201,
$response->getStatusCode(),
__METHOD__ . " cannot create user '$user' using Graph API.\nResponse:" .
__METHOD__ . " cannot create user '$user'.\nResponse:" .
json_encode($this->getJsonDecodedResponse($response))
);
$userId = $this->getJsonDecodedResponse($response)['id'];

View File

@@ -750,6 +750,9 @@ class SpacesContext implements Context {
} else {
$rawBody = $this->featureContext->getResponse()->getBody()->getContents();
}
if (!$rawBody) {
throw new Exception(__METHOD__ . " - Response body is empty");
}
$drives = json_decode($rawBody, true, 512, JSON_THROW_ON_ERROR);
if (isset($drives["value"])) {
$drives = $drives["value"];

View File

@@ -216,6 +216,44 @@ class TUSContext implements Context {
);
}
/**
* @When user :user sends a chunk to the last created TUS Location with offset :offset and data :data with retry on offset mismatch using the WebDAV API
*
* @param string $user
* @param string $offset
* @param string $data
*
* @return void
*
* @throws GuzzleException
* @throws JsonException
*/
public function userSendsAChunkToTUSLocationWithOffsetAndDataWithRetryOnOffsetMismatch(
string $user,
string $offset,
string $data,
): void {
$resourceLocation = $this->getLastTusResourceLocation();
$retried = 0;
do {
$tryAgain = false;
$response = $this->uploadChunkToTUSLocation($user, $resourceLocation, $offset, $data);
// retry on 409 Conflict (Offset mismatch during TUS upload)
if ($response->getStatusCode() === 409) {
$tryAgain = true;
}
$tryAgain = $tryAgain && $retried < HttpRequestHelper::numRetriesOnHttpTooEarly();
if ($tryAgain) {
$retried += 1;
echo "Offset mismatch during TUS upload, retrying ($retried)...\n";
// wait 1s and try again
\sleep(1);
}
} while ($tryAgain);
$this->featureContext->setResponse($response);
}
/**
* @When user :user sends a chunk to the last created TUS Location with offset :offset and data :data using the WebDAV API
*

View File

@@ -25,6 +25,7 @@ use GuzzleHttp\Exception\GuzzleException;
use PHPUnit\Framework\Assert;
use Psr\Http\Message\ResponseInterface;
use GuzzleHttp\Stream\StreamInterface;
use TestHelpers\OcConfigHelper;
use TestHelpers\OcHelper;
use TestHelpers\UploadHelper;
use TestHelpers\WebDavHelper;
@@ -743,6 +744,7 @@ trait WebDav {
/**
* @When the user waits for :time seconds for postprocessing to finish
* @When the user waits for :time seconds
*
* @param int $time
*
@@ -973,6 +975,61 @@ trait WebDav {
$this->checkDownloadedContentMatches($content, '', $response);
}
/**
* check file content with retry
*
* @param string $user
* @param string $fileName
* @param string $content
*
* @return void
* @throws Exception
*/
public function checkFileContentWithRetry(string $user, string $fileName, string $content): void {
$retried = 0;
do {
$tryAgain = false;
$response = $this->downloadFileAsUserUsingPassword($this->getActualUsername($user), $fileName);
$status = $response->getStatusCode();
$downloadedContent = $response->getBody()->getContents();
if ($status !== 200) {
$tryAgain = true;
$message = "Expected '200' but got '$status'";
} elseif ($downloadedContent !== $content) {
$tryAgain = true;
$message = "Expected content '$content' but got '$downloadedContent'";
}
$tryAgain = $tryAgain && $retried < HttpRequestHelper::numRetriesOnHttpTooEarly();
if ($tryAgain) {
$retried += 1;
echo "[INFO] File content mismatch. $message, checking content again ($retried)...\n";
// break the loop if status is 425 as the request will already be retried
if ($status === HttpRequestHelper::HTTP_TOO_EARLY) {
break;
}
// wait 1s and try again
\sleep(1);
}
} while ($tryAgain);
$this->theHTTPStatusCodeShouldBe(200, '', $response);
$this->checkDownloadedContentMatches($content, '', $response);
}
/**
* @Then as :user the final content of file :fileName should be :content
*
* @param string $user
* @param string $fileName
* @param string $content
*
* @return void
*/
public function asUserFinalContentOfFileShouldBe(string $user, string $fileName, string $content): void {
$this->checkFileContentWithRetry($user, $fileName, $content);
}
/**
* @Then /^the content of the following files for user "([^"]*)" should be "([^"]*)"$/
*
@@ -2272,6 +2329,11 @@ trait WebDav {
"HTTP status code was not 201 or 204 while trying to upload file '$destination' for user '$user'",
$response
);
// check uploaded content only if post-processing delay is not configured
if (OcConfigHelper::getPostProcessingDelay() === 0) {
$this->checkFileContentWithRetry($user, $destination, $content);
}
return $response->getHeader('oc-fileid');
}

View File

@@ -29,7 +29,7 @@ Feature: create a resources using collaborative posixfs
Scenario: create file
When the administrator creates the file "test.txt" with content "content" for user "Alice" on the POSIX filesystem
Then the command should be successful
And the content of file "/test.txt" for user "Alice" should be "content"
And as "Alice" the final content of file "test.txt" should be "content"
Scenario: create large file
@@ -41,21 +41,22 @@ Feature: create a resources using collaborative posixfs
Scenario: creates files sequentially in a folder
When the administrator creates 50 files sequentially in the directory "firstFolder" for user "Alice" on the POSIX filesystem
Then the command should be successful
And the content of file "/firstFolder/file_1.txt" for user "Alice" should be "file 1 content"
And the content of file "/firstFolder/file_50.txt" for user "Alice" should be "file 50 content"
And as "Alice" the final content of file "/firstFolder/file_1.txt" should be "file 1 content"
And as "Alice" the final content of file "/firstFolder/file_50.txt" should be "file 50 content"
Scenario: creates files in parallel in a folder
When the administrator creates 100 files in parallel in the directory "firstFolder" for user "Alice" on the POSIX filesystem
Then the command should be successful
And the content of file "/firstFolder/parallel_1.txt" for user "Alice" should be "parallel file 1 content"
And the content of file "/firstFolder/parallel_100.txt" for user "Alice" should be "parallel file 100 content"
And as "Alice" the final content of file "/firstFolder/parallel_1.txt" should be "parallel file 1 content"
And as "Alice" the final content of file "/firstFolder/parallel_100.txt" should be "parallel file 100 content"
Scenario: edit file
Given user "Alice" has uploaded file with content "content" to "test.txt"
When the administrator puts the content "new" into the file "test.txt" in the POSIX storage folder of user "Alice"
Then the content of file "/test.txt" for user "Alice" should be "contentnew"
Then the command should be successful
And as "Alice" the final content of file "test.txt" should be "contentnew"
Scenario: read file content
@@ -68,14 +69,14 @@ Feature: create a resources using collaborative posixfs
Given user "Alice" has uploaded file with content "content" to "test.txt"
When the administrator copies the file "test.txt" to the folder "firstFolder" for user "Alice" on the POSIX filesystem
Then the command should be successful
And the content of file "/firstFolder/test.txt" for user "Alice" should be "content"
And as "Alice" the final content of file "/firstFolder/test.txt" should be "content"
Scenario: rename file
Given user "Alice" has uploaded file with content "content" to "test.txt"
When the administrator renames the file "test.txt" to "new-name.txt" for user "Alice" on the POSIX filesystem
Then the command should be successful
And the content of file "/new-name.txt" for user "Alice" should be "content"
And as "Alice" the final content of file "/new-name.txt" should be "content"
Scenario: check propfind after rename file
@@ -97,14 +98,14 @@ Feature: create a resources using collaborative posixfs
Given the administrator has created the file "test.txt" with content "content" for user "Alice" on the POSIX filesystem
When the administrator renames the file "test.txt" to "test.md" for user "Alice" on the POSIX filesystem
Then the command should be successful
And the content of file "/test.md" for user "Alice" should be "content"
And as "Alice" the final content of file "/test.md" should be "content"
Scenario: move file to folder
Given user "Alice" has uploaded file with content "content" to "test.txt"
When the administrator moves the file "test.txt" to the folder "firstFolder" for user "Alice" on the POSIX filesystem
Then the command should be successful
And the content of file "/firstFolder/test.txt" for user "Alice" should be "content"
And as "Alice" the final content of file "/firstFolder/test.txt" should be "content"
And as "Alice" file "/test.txt" should not exist
@@ -202,4 +203,4 @@ Feature: create a resources using collaborative posixfs
And the administrator renames the file "test.txt" to "renamed.txt" for user "Alice" on the POSIX filesystem
And the administrator checks the attribute "user.oc.name" of file "renamed.txt" for user "Alice" on the POSIX filesystem
Then the command output should contain "renamed.txt"
And the content of file "/renamed.txt" for user "Alice" should be "content"
And as "Alice" the final content of file "/renamed.txt" should be "content"

View File

@@ -202,7 +202,7 @@ Feature: capabilities
"properties": {
"edition": {
"type": "string",
"enum": ["%edition%"]
"enum": ["dev"]
},
"product": {
"type": "string",
@@ -240,7 +240,7 @@ Feature: capabilities
},
"edition": {
"type": "string",
"enum": ["%edition%"]
"enum": ["dev"]
},
"product": {
"type": "string",

View File

@@ -58,7 +58,7 @@ Feature: default capabilities for normal user
"const": "%versionstring%"
},
"edition": {
"const": "%edition%"
"const": "dev"
},
"productname": {
"const": "%productname%"

View File

@@ -50,8 +50,7 @@ Feature: low level tests for upload of chunks
| Upload-Metadata | filename ZmlsZS50eHQ= |
When user "Alice" sends a chunk to the last created TUS Location with offset "0" and data "123" using the WebDAV API
And user "Alice" sends a chunk to the last created TUS Location with offset "3" and data "4567890" using the WebDAV API
And the user waits for "2" seconds for postprocessing to finish
And user "Alice" sends a chunk to the last created TUS Location with offset "3" and data "0000000" using the WebDAV API
And user "Alice" sends a chunk to the last created TUS Location with offset "3" and data "0000000" with retry on offset mismatch using the WebDAV API
Then the HTTP status code should be "404"
And the content of file "/file.txt" for user "Alice" should be "1234567890"
Examples:
@@ -61,6 +60,22 @@ Feature: low level tests for upload of chunks
| spaces |
Scenario Outline: send last chunk with mismatch offset
Given using <dav-path-version> DAV path
And user "Alice" has created a new TUS resource on the WebDAV API with these headers:
| Upload-Length | 10 |
# ZmlsZS50eHQ= is the base64 encode of file.txt
| Upload-Metadata | filename ZmlsZS50eHQ= |
When user "Alice" sends a chunk to the last created TUS Location with offset "0" and data "123" using the WebDAV API
And user "Alice" sends a chunk to the last created TUS Location with offset "2" and data "34567890" using the WebDAV API
Then the HTTP status code should be "409"
Examples:
| dav-path-version |
| old |
| new |
| spaces |
Scenario Outline: start with uploading not at the beginning of the file
Given using <dav-path-version> DAV path
And user "Alice" has created a new TUS resource on the WebDAV API with these headers:

View File

@@ -46,10 +46,27 @@ type Options struct {
WatchRoot string `mapstructure:"watch_root"` // base directory for the watch. events will be considered relative to this path
WatchNotificationBrokers string `mapstructure:"watch_notification_brokers"`
NatsWatcher NatsWatcherConfig `mapstructure:"natswatcher"`
// InotifyWatcher specific options
InotifyStatsFrequency time.Duration `mapstructure:"inotify_stats_frequency"`
}
// NatsWatcherConfig is the configuration needed for a NATS watcher event stream.
type NatsWatcherConfig struct {
Endpoint string `mapstructure:"address"`
Cluster string `mapstructure:"clusterID"`
Stream string `mapstructure:"stream"`
Durable string `mapstructure:"durable-name"`
TLSInsecure bool `mapstructure:"tls-insecure"`
TLSRootCACertificate string `mapstructure:"tls-root-ca-cert"`
EnableTLS bool `mapstructure:"enable-tls"`
AuthUsername string `mapstructure:"username"`
AuthPassword string `mapstructure:"password"`
MaxAckPending int `mapstructure:"max-ack-pending"`
AckWait time.Duration `mapstructure:"ack-wait"`
}
// New returns a new Options instance for the given configuration
func New(m map[string]interface{}) (*Options, error) {
// default to hybrid metadatabackend for posixfs

View File

@@ -21,6 +21,7 @@ package trashbin
import (
"context"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
@@ -589,6 +590,10 @@ func (tb *Trashbin) IsEmpty(ctx context.Context, spaceID string) bool {
}
dirItems, err := trash.ReadDir(1)
if err != nil {
if err == io.EOF {
// empty trash
return true
}
// if we cannot read the trash, we assume there are no trashed items
tb.log.Error().Err(err).Str("spaceID", spaceID).Msg("trashbin: error reading trash directory")
return true

View File

@@ -39,6 +39,7 @@ import (
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
@@ -54,16 +55,6 @@ type ScanDebouncer struct {
mutex sync.Mutex
}
type EventAction int
const (
ActionCreate EventAction = iota
ActionUpdate
ActionMove
ActionDelete
ActionMoveFrom
)
type queueItem struct {
item scanItem
timer *time.Timer
@@ -190,10 +181,10 @@ func (t *Tree) workScanQueue() {
}
// Scan scans the given path and updates the id chache
func (t *Tree) Scan(path string, action EventAction, isDir bool) error {
func (t *Tree) Scan(path string, action watcher.EventAction, isDir bool) error {
// cases:
switch action {
case ActionCreate:
case watcher.ActionCreate:
t.log.Debug().Str("path", path).Bool("isDir", isDir).Msg("scanning path (ActionCreate)")
if !isDir {
// 1. New file (could be emitted as part of a new directory)
@@ -225,7 +216,7 @@ func (t *Tree) Scan(path string, action EventAction, isDir bool) error {
})
}
case ActionUpdate:
case watcher.ActionUpdate:
t.log.Debug().Str("path", path).Bool("isDir", isDir).Msg("scanning path (ActionUpdate)")
// 3. Updated file
// -> update file unless parent directory is being rescanned
@@ -241,7 +232,7 @@ func (t *Tree) Scan(path string, action EventAction, isDir bool) error {
AssimilationCounter.WithLabelValues(_labelDir, _labelUpdated).Inc()
}
case ActionMove:
case watcher.ActionMove:
t.log.Debug().Str("path", path).Bool("isDir", isDir).Msg("scanning path (ActionMove)")
// 4. Moved file
// -> update file
@@ -258,7 +249,7 @@ func (t *Tree) Scan(path string, action EventAction, isDir bool) error {
AssimilationCounter.WithLabelValues(_labelDir, _labelMoved).Inc()
}
case ActionMoveFrom:
case watcher.ActionMoveFrom:
t.log.Debug().Str("path", path).Bool("isDir", isDir).Msg("scanning path (ActionMoveFrom)")
// 6. file/directory moved out of the watched directory
// -> remove from caches
@@ -279,7 +270,7 @@ func (t *Tree) Scan(path string, action EventAction, isDir bool) error {
// We do not do metrics here because this has been handled in `ActionMove`
case ActionDelete:
case watcher.ActionDelete:
t.log.Debug().Str("path", path).Bool("isDir", isDir).Msg("handling deleted item")
// 7. Deleted file or directory
@@ -426,6 +417,15 @@ func (t *Tree) assimilate(item scanItem) error {
}
}
fi, err := os.Lstat(item.Path)
if err != nil {
return err
}
if !fi.IsDir() && !fi.Mode().IsRegular() {
t.log.Trace().Str("path", item.Path).Msg("skipping non-regular file")
return nil
}
if id != "" {
// the file has an id set, we already know it from the past
@@ -451,20 +451,10 @@ func (t *Tree) assimilate(item scanItem) error {
// compare metadata mtime with actual mtime. if it matches AND the path hasn't changed (move operation)
// we can skip the assimilation because the file was handled by us
fi, err := os.Lstat(item.Path)
if err != nil {
return err
}
if previousPath == item.Path && mtime.Equal(fi.ModTime()) {
return nil
}
if !fi.IsDir() && !fi.Mode().IsRegular() {
t.log.Trace().Str("path", item.Path).Msg("skipping non-regular file")
return nil
}
// was it moved or copied/restored with a clashing id?
if ok && len(parentID) > 0 && previousPath != item.Path {
_, err := os.Stat(previousPath)
@@ -675,6 +665,7 @@ assimilate:
}
var n *node.Node
sizeDiff := int64(0)
if fi.IsDir() {
// The Space's name attribute might not match the directory name. Use the name as
// it was set before. Also the space root doesn't have a 'type' attribute
@@ -712,44 +703,46 @@ assimilate:
n.SpaceRoot = &node.Node{BaseNode: node.BaseNode{SpaceID: spaceID, ID: spaceID}}
prevBlobSize, err := previousAttribs.Int64(prefixes.BlobsizeAttr)
if err == nil && prevBlobSize != fi.Size() {
// file size changed, trigger propagation of tree size changes
err = t.Propagate(context.Background(), n, fi.Size()-prevBlobSize)
if err != nil {
t.log.Error().Err(err).Str("path", path).Msg("could not propagate tree size changes")
}
if err != nil || prevBlobSize < 0 {
prevBlobSize = 0
}
if prevBlobSize != fi.Size() {
sizeDiff = fi.Size() - prevBlobSize
}
}
attributes.SetTime(prefixes.MTimeAttr, fi.ModTime())
n.SpaceRoot = &node.Node{BaseNode: node.BaseNode{SpaceID: spaceID, ID: spaceID}}
if t.options.EnableFSRevisions {
if !fi.IsDir() && t.options.EnableFSRevisions {
go func() {
// Copy the previous current version to a revision
currentNode := node.NewBaseNode(n.SpaceID, n.ID+node.CurrentIDDelimiter, t.lookup)
currentPath := currentNode.InternalPath()
stat, err := os.Stat(currentPath)
if err != nil {
t.log.Error().Err(err).Str("path", path).Str("currentPath", currentPath).Msg("could not stat current path")
return
}
revisionPath := t.lookup.VersionPath(n.SpaceID, n.ID, stat.ModTime().UTC().Format(time.RFC3339Nano))
if err == nil {
revisionPath := t.lookup.VersionPath(n.SpaceID, n.ID, stat.ModTime().UTC().Format(time.RFC3339Nano))
err = os.Rename(currentPath, revisionPath)
if err != nil {
t.log.Error().Err(err).Str("path", path).Str("revisionPath", revisionPath).Msg("could not create revision")
return
err = os.Rename(currentPath, revisionPath)
if err != nil {
t.log.Error().Err(err).Str("path", path).Str("revisionPath", revisionPath).Msg("could not create revision")
return
}
}
// Copy the new version to the current version
if err := os.MkdirAll(filepath.Dir(currentPath), 0700); err != nil {
t.log.Error().Err(err).Str("path", path).Str("currentPath", currentPath).Msg("could not create base path for current file")
return
}
w, err := os.OpenFile(currentPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
t.log.Error().Err(err).Str("path", path).Str("currentPath", currentPath).Msg("could not open current path for writing")
return
}
defer w.Close()
r, err := os.OpenFile(n.InternalPath(), os.O_RDONLY, 0600)
r, err := os.OpenFile(path, os.O_RDONLY, 0600)
if err != nil {
t.log.Error().Err(err).Str("path", path).Msg("could not open file for reading")
return
@@ -775,7 +768,7 @@ assimilate:
}()
}
err = t.Propagate(context.Background(), n, 0)
err = t.Propagate(context.Background(), n, sizeDiff)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to propagate")
}

View File

@@ -8,6 +8,7 @@ import (
"encoding/json"
"path/filepath"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
kafka "github.com/segmentio/kafka-go"
@@ -97,17 +98,17 @@ func (w *CephFSWatcher) Watch(topic string) {
go func() {
switch {
case mask&CEPH_MDS_NOTIFY_DELETE > 0:
err = w.tree.Scan(path, ActionDelete, isDir)
err = w.tree.Scan(path, watcher.ActionDelete, isDir)
case mask&CEPH_MDS_NOTIFY_MOVED_TO > 0:
if ev.SrcMask > 0 {
// This is a move, clean up the old path
err = w.tree.Scan(filepath.Join(w.tree.options.WatchRoot, ev.SrcPath), ActionMoveFrom, isDir)
err = w.tree.Scan(filepath.Join(w.tree.options.WatchRoot, ev.SrcPath), watcher.ActionMoveFrom, isDir)
}
err = w.tree.Scan(path, ActionMove, isDir)
err = w.tree.Scan(path, watcher.ActionMove, isDir)
case mask&CEPH_MDS_NOTIFY_CREATE > 0:
err = w.tree.Scan(path, ActionCreate, isDir)
err = w.tree.Scan(path, watcher.ActionCreate, isDir)
case mask&CEPH_MDS_NOTIFY_CLOSE_WRITE > 0:
err = w.tree.Scan(path, ActionUpdate, isDir)
err = w.tree.Scan(path, watcher.ActionUpdate, isDir)
case mask&CEPH_MDS_NOTIFY_CLOSE > 0:
// ignore, already handled by CLOSE_WRITE
default:

View File

@@ -26,6 +26,7 @@ import (
"strconv"
"time"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher"
"github.com/rs/zerolog"
)
@@ -88,15 +89,15 @@ start:
go func() {
switch ev.Event {
case "CREATE":
err = w.tree.Scan(ev.Path, ActionCreate, false)
err = w.tree.Scan(ev.Path, watcher.ActionCreate, false)
case "CLOSE":
var bytesWritten int
bytesWritten, err = strconv.Atoi(ev.BytesWritten)
if err == nil && bytesWritten > 0 {
err = w.tree.Scan(ev.Path, ActionUpdate, false)
err = w.tree.Scan(ev.Path, watcher.ActionUpdate, false)
}
case "RENAME":
err = w.tree.Scan(ev.Path, ActionMove, false)
err = w.tree.Scan(ev.Path, watcher.ActionMove, false)
if warmupErr := w.tree.WarmupIDCache(ev.Path, false, false); warmupErr != nil {
w.log.Error().Err(warmupErr).Str("path", ev.Path).Msg("error warming up id cache")
}

View File

@@ -26,6 +26,7 @@ import (
"strconv"
"strings"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher"
"github.com/rs/zerolog"
kafka "github.com/segmentio/kafka-go"
)
@@ -77,21 +78,21 @@ func (w *GpfsWatchFolderWatcher) Watch(topic string) {
var err error
switch {
case strings.Contains(lwev.Event, "IN_DELETE"):
err = w.tree.Scan(path, ActionDelete, isDir)
err = w.tree.Scan(path, watcher.ActionDelete, isDir)
case strings.Contains(lwev.Event, "IN_MOVE_FROM"):
err = w.tree.Scan(path, ActionMoveFrom, isDir)
err = w.tree.Scan(path, watcher.ActionMoveFrom, isDir)
case strings.Contains(lwev.Event, "IN_CREATE"):
err = w.tree.Scan(path, ActionCreate, isDir)
err = w.tree.Scan(path, watcher.ActionCreate, isDir)
case strings.Contains(lwev.Event, "IN_CLOSE_WRITE"):
bytesWritten, convErr := strconv.Atoi(lwev.BytesWritten)
if convErr == nil && bytesWritten > 0 {
err = w.tree.Scan(path, ActionUpdate, isDir)
err = w.tree.Scan(path, watcher.ActionUpdate, isDir)
}
case strings.Contains(lwev.Event, "IN_MOVED_TO"):
err = w.tree.Scan(path, ActionMove, isDir)
err = w.tree.Scan(path, watcher.ActionMove, isDir)
}
if err != nil {
w.log.Error().Err(err).Str("path", path).Msg("error scanning path")

View File

@@ -30,6 +30,7 @@ import (
"time"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher"
"github.com/pablodz/inotifywaitgo/inotifywaitgo"
"github.com/rs/zerolog"
slogzerolog "github.com/samber/slog-zerolog/v2"
@@ -96,15 +97,15 @@ func (iw *InotifyWatcher) Watch(path string) {
var err error
switch e {
case inotifywaitgo.DELETE:
err = iw.tree.Scan(event.Filename, ActionDelete, event.IsDir)
err = iw.tree.Scan(event.Filename, watcher.ActionDelete, event.IsDir)
case inotifywaitgo.MOVED_FROM:
err = iw.tree.Scan(event.Filename, ActionMoveFrom, event.IsDir)
err = iw.tree.Scan(event.Filename, watcher.ActionMoveFrom, event.IsDir)
case inotifywaitgo.MOVED_TO:
err = iw.tree.Scan(event.Filename, ActionMove, event.IsDir)
err = iw.tree.Scan(event.Filename, watcher.ActionMove, event.IsDir)
case inotifywaitgo.CREATE:
err = iw.tree.Scan(event.Filename, ActionCreate, event.IsDir)
err = iw.tree.Scan(event.Filename, watcher.ActionCreate, event.IsDir)
case inotifywaitgo.CLOSE_WRITE:
err = iw.tree.Scan(event.Filename, ActionUpdate, event.IsDir)
err = iw.tree.Scan(event.Filename, watcher.ActionUpdate, event.IsDir)
case inotifywaitgo.CLOSE:
// ignore, already handled by CLOSE_WRITE
default:

View File

@@ -47,6 +47,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/trashbin"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher/natswatcher"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
@@ -147,6 +148,11 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin *
if err != nil {
return nil, err
}
case "natswatcher":
t.watcher, err = natswatcher.New(context.TODO(), t, o.NatsWatcher, o.WatchRoot, log)
if err != nil {
return nil, err
}
default:
t.watcher, err = NewInotifyWatcher(t, o, log)
if err != nil {
@@ -499,8 +505,18 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro
_, nodeID, err := t.lookup.IDsForPath(ctx, path)
if err != nil {
t.log.Error().Err(err).Str("path", path).Msg("failed to get ids for entry")
continue
// we don't know about this node yet for some reason, assimilate it on the fly
t.log.Info().Err(err).Str("path", path).Msg("encountered unknown entity while listing the directory. Assimilate.")
err = t.assimilate(scanItem{Path: path})
if err != nil {
t.log.Error().Err(err).Str("path", path).Msg("failed to assimilate node")
continue
}
_, nodeID, err = t.lookup.IDsForPath(ctx, path)
if err != nil || nodeID == "" {
t.log.Error().Err(err).Str("path", path).Msg("still could not resolve node after assimilation")
continue
}
}
child, err := node.ReadNode(ctx, t.lookup, n.SpaceID, nodeID, false, n.SpaceRoot, true)
@@ -708,9 +724,23 @@ func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) {
t.log.Error().Err(err).Str("spaceID", n.SpaceID).Str("id", n.ID).Str("path", path).Msg("could not cache id")
}
// Write mtime from filesystem to metadata to preven re-assimilation
d, err := os.Open(path)
if err != nil {
return err
}
fi, err := d.Stat()
if err != nil {
return err
}
mtime := fi.ModTime()
attributes := n.NodeMetadata(ctx)
attributes[prefixes.MTimeAttr] = []byte(mtime.UTC().Format(time.RFC3339Nano))
attributes[prefixes.IDAttr] = []byte(n.ID)
attributes[prefixes.TreesizeAttr] = []byte("0") // initialize as empty, TODO why bother? if it is not set we could treat it as 0?
if t.options.TreeTimeAccounting || t.options.TreeSizeAccounting {
attributes[prefixes.PropagationAttr] = []byte("1") // mark the node for propagation
}

View File

@@ -0,0 +1,11 @@
package watcher
type EventAction int
const (
ActionCreate EventAction = iota
ActionUpdate
ActionMove
ActionDelete
ActionMoveFrom
)

View File

@@ -0,0 +1,236 @@
package natswatcher
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"os"
"path/filepath"
"time"
"github.com/cenkalti/backoff"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher"
"github.com/rs/zerolog"
"github.com/vmihailenco/msgpack/v5"
)
// natsEvent represents the event encoded in MessagePack.
// we abbreviate the the properties to save some space
type natsEvent struct {
Event string `msgpack:"e"`
Path string `msgpack:"p,omitempty"`
ToPath string `msgpack:"t,omitempty"`
IsDir bool `msgpack:"d,omitempty"`
}
// NatsWatcher consumes filesystem-style events from NATS JetStream.
type NatsWatcher struct {
ctx context.Context
tree Scannable
log *zerolog.Logger
watchRoot string
config options.NatsWatcherConfig
}
type Scannable interface {
Scan(path string, action watcher.EventAction, isDir bool) error
}
// NewNatsWatcher creates a new NATS watcher.
func New(ctx context.Context, tree Scannable, cfg options.NatsWatcherConfig, watchRoot string, log *zerolog.Logger) (*NatsWatcher, error) {
return &NatsWatcher{
ctx: ctx,
tree: tree,
log: log,
watchRoot: watchRoot,
config: cfg,
}, nil
}
// Watch starts consuming events from a NATS JetStream subject
func (w *NatsWatcher) Watch(path string) {
w.log.Info().Str("stream", w.config.Stream).Msg("starting NATS watcher with auto-reconnect")
for {
select {
case <-w.ctx.Done():
w.log.Debug().Msg("context cancelled, stopping NATS watcher")
return
default:
}
// Try to connect with exponential backoff
nc, js, err := w.connectWithBackoff()
if err != nil {
w.log.Error().Err(err).Msg("failed to establish NATS connection after retries")
time.Sleep(5 * time.Second)
continue
}
if err := w.consume(js); err != nil {
w.log.Error().Err(err).Msg("NATS consumer exited with error, reconnecting")
}
_ = nc.Drain()
nc.Close()
time.Sleep(2 * time.Second)
}
}
// connectWithBackoff repeatedly attempts to connect to NATS JetStream with exponential backoff.
func (w *NatsWatcher) connectWithBackoff() (*nats.Conn, jetstream.JetStream, error) {
var nc *nats.Conn
var js jetstream.JetStream
b := backoff.NewExponentialBackOff()
b.InitialInterval = 1 * time.Second
b.MaxInterval = 30 * time.Second
b.MaxElapsedTime = 0 // never stop
connect := func() error {
select {
case <-w.ctx.Done():
return backoff.Permanent(w.ctx.Err())
default:
}
var err error
nc, err = w.connect()
if err != nil {
w.log.Warn().Err(err).Msg("failed to connect to NATS, retrying")
return err
}
js, err = jetstream.New(nc)
if err != nil {
nc.Close()
w.log.Warn().Err(err).Msg("failed to create jetstream context, retrying")
return err
}
w.log.Info().Str("endpoint", w.config.Endpoint).Msg("connected to NATS JetStream")
return nil
}
if err := backoff.Retry(connect, backoff.WithContext(b, w.ctx)); err != nil {
return nil, nil, err
}
return nc, js, nil
}
// consume subscribes to JetStream and handles messages.
func (w *NatsWatcher) consume(js jetstream.JetStream) error {
stream, err := js.Stream(w.ctx, w.config.Stream)
if err != nil {
return fmt.Errorf("failed to get stream: %w", err)
}
consumer, err := stream.CreateOrUpdateConsumer(w.ctx, jetstream.ConsumerConfig{
Durable: w.config.Durable,
AckPolicy: jetstream.AckExplicitPolicy,
MaxAckPending: w.config.MaxAckPending,
AckWait: w.config.AckWait,
})
if err != nil {
return fmt.Errorf("failed to create consumer: %w", err)
}
w.log.Info().
Str("stream", w.config.Stream).
Msg("started consuming from JetStream")
_, err = consumer.Consume(func(msg jetstream.Msg) {
defer func() {
if ackErr := msg.Ack(); ackErr != nil {
w.log.Warn().Err(ackErr).Msg("failed to ack message")
}
}()
var ev natsEvent
if err := msgpack.Unmarshal(msg.Data(), &ev); err != nil {
w.log.Error().Err(err).Msg("failed to decode MessagePack event")
return
}
w.handleEvent(ev)
})
if err != nil {
return fmt.Errorf("consumer error: %w", err)
}
<-w.ctx.Done()
return w.ctx.Err()
}
// connect establishes a single NATS connection with optional TLS and auth.
func (w *NatsWatcher) connect() (*nats.Conn, error) {
var tlsConf *tls.Config
if w.config.EnableTLS {
var rootCAPool *x509.CertPool
if w.config.TLSRootCACertificate != "" {
rootCrtFile, err := os.ReadFile(w.config.TLSRootCACertificate)
if err != nil {
return nil, fmt.Errorf("failed to read root CA: %w", err)
}
rootCAPool = x509.NewCertPool()
rootCAPool.AppendCertsFromPEM(rootCrtFile)
w.config.TLSInsecure = false
}
tlsConf = &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: w.config.TLSInsecure,
RootCAs: rootCAPool,
}
}
opts := []nats.Option{nats.Name("opencloud-posixfs-natswatcher")}
if tlsConf != nil {
opts = append(opts, nats.Secure(tlsConf))
}
if w.config.AuthUsername != "" && w.config.AuthPassword != "" {
opts = append(opts, nats.UserInfo(w.config.AuthUsername, w.config.AuthPassword))
}
return nats.Connect(w.config.Endpoint, opts...)
}
// handleEvent applies the event to the local tree.
func (w *NatsWatcher) handleEvent(ev natsEvent) {
var err error
// Determine the relevant path
path := filepath.Join(w.watchRoot, ev.Path)
switch ev.Event {
case "CREATE":
err = w.tree.Scan(path, watcher.ActionCreate, ev.IsDir)
case "MOVED_TO":
err = w.tree.Scan(path, watcher.ActionMove, ev.IsDir)
case "MOVE_FROM":
err = w.tree.Scan(path, watcher.ActionMoveFrom, ev.IsDir)
case "MOVE": // support event with source and target path
err = w.tree.Scan(path, watcher.ActionMoveFrom, ev.IsDir)
if err == nil {
w.log.Error().Err(err).Interface("event", ev).Msg("error processing event")
}
tgt := filepath.Join(w.watchRoot, ev.ToPath)
if tgt == "" {
w.log.Warn().Interface("event", ev).Msg("MOVE event missing target path")
} else {
err = w.tree.Scan(tgt, watcher.ActionMove, ev.IsDir)
}
case "CLOSE_WRITE":
err = w.tree.Scan(path, watcher.ActionUpdate, ev.IsDir)
case "DELETE":
err = w.tree.Scan(path, watcher.ActionDelete, ev.IsDir)
default:
w.log.Warn().Str("event", ev.Event).Msg("unhandled event type")
}
if err != nil {
w.log.Error().Err(err).Interface("event", ev).Msg("error processing event")
}
}

View File

@@ -2,6 +2,7 @@ package metadata
import (
"context"
"fmt"
"io"
"io/fs"
"os"
@@ -292,17 +293,19 @@ func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
}
}
xerrs := 0
total := 0
var xerr error
// error handling: Count if there are errors while setting the attribs.
// if there were any, return an error.
for key, val := range attribs {
total++
if xerr = xattr.Set(path, key, val); xerr != nil {
// log
xerrs++
}
}
if xerrs > 0 {
return errors.Wrap(xerr, "Failed to set all xattrs")
return fmt.Errorf("failed to set %d/%d xattrs: %w", xerrs, total, xerr)
}
attribs, err = b.getAll(ctx, n, true, false, false)

4
vendor/modules.txt vendored
View File

@@ -1368,7 +1368,7 @@ github.com/opencloud-eu/icap-client
# github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76
## explicit; go 1.18
github.com/opencloud-eu/libre-graph-api-go
# github.com/opencloud-eu/reva/v2 v2.40.1
# github.com/opencloud-eu/reva/v2 v2.41.0
## explicit; go 1.24.1
github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace
github.com/opencloud-eu/reva/v2/cmd/revad/runtime
@@ -1682,6 +1682,8 @@ github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/trashbin
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher/natswatcher
github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry
github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng
github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore