mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-08 11:53:07 -04:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1770793f26 | ||
|
|
7bb32387cb | ||
|
|
43166d138a | ||
|
|
502628c4d6 | ||
|
|
d6944d7b0b | ||
|
|
027a8cf9b6 | ||
|
|
0e6795492e | ||
|
|
0548d5db7e | ||
|
|
64be6d614f | ||
|
|
cbc6f8408a | ||
|
|
fa1e4e6104 | ||
|
|
cee105203c | ||
|
|
48f9874e48 | ||
|
|
8c40a33f6c | ||
|
|
aeff1f53b8 | ||
|
|
82294e3c71 | ||
|
|
be01619806 | ||
|
|
38440bd8fc | ||
|
|
066eabd0c9 | ||
|
|
3516dc7f2f | ||
|
|
954b8ccf41 | ||
|
|
6da9b1326a | ||
|
|
2711523aec | ||
|
|
5d8d022bd7 | ||
|
|
5218eeffdd | ||
|
|
a75f592d37 | ||
|
|
a17a70c8b7 | ||
|
|
df3a790a2f | ||
|
|
eb73d2af0e | ||
|
|
31f01c6d3d | ||
|
|
6597dad1f9 | ||
|
|
8b01d08ce3 | ||
|
|
c73ed01908 | ||
|
|
8c4f9d96c0 | ||
|
|
a83a3b7680 | ||
|
|
cee5657651 | ||
|
|
0a500d25bf | ||
|
|
be121d8a94 | ||
|
|
c067b65f3f | ||
|
|
c7ed57263c | ||
|
|
1b9cd062c7 | ||
|
|
ae53d5847e | ||
|
|
14b34903a7 | ||
|
|
a44539d5f6 | ||
|
|
69a1350921 | ||
|
|
d7375e9904 | ||
|
|
b4ed8c4aeb | ||
|
|
8fd9a82acf | ||
|
|
cd99a6fd44 | ||
|
|
c24b6f1c33 | ||
|
|
569e630fe0 | ||
|
|
7143dc9cba | ||
|
|
3f83cbc524 | ||
|
|
6ce5943e07 | ||
|
|
7eb08af8ae | ||
|
|
7306abaaf9 | ||
|
|
dd345bf48f | ||
|
|
f2e02e1f88 | ||
|
|
92a2980875 | ||
|
|
b25367a845 | ||
|
|
4d82968bd0 | ||
|
|
90cc015a9c | ||
|
|
6dcb7a1ff2 | ||
|
|
63664db2f6 | ||
|
|
f8e26d3198 | ||
|
|
a6ca2c538e | ||
|
|
56e03f12ce |
No files matched your search
+2
-2
@@ -1,4 +1,4 @@
|
||||
# The test runner source for UI tests
|
||||
WEB_COMMITID=2cee98bb27fab223fc4ce2455f5483ce7767ff1a
|
||||
WEB_BRANCH=main
|
||||
WEB_COMMITID=170677dc06b1dfab57bc4bea2dac8a5cd78b2784
|
||||
WEB_BRANCH=stable-7.1
|
||||
|
||||
+260
-143
@@ -75,11 +75,11 @@ OC_FED_DOMAIN = "%s:10200" % FED_OC_SERVER_NAME
|
||||
event = {
|
||||
"base": {
|
||||
"event": ["push", "manual"],
|
||||
"branch": "main",
|
||||
"branch": "stable-*",
|
||||
},
|
||||
"cron": {
|
||||
"event": "cron",
|
||||
"branch": "main",
|
||||
"branch": "stable-*",
|
||||
},
|
||||
"pull_request": {
|
||||
"event": "pull_request",
|
||||
@@ -89,6 +89,30 @@ event = {
|
||||
},
|
||||
}
|
||||
|
||||
OPENCLOUD_STORAGES = ["posix", "decomposed"]
|
||||
API_TEST_NIGHTLY_CI_MATRIX = {
|
||||
"posix": [
|
||||
{
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": True,
|
||||
},
|
||||
{
|
||||
"withRemotePhp": True,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
],
|
||||
"decomposed": [
|
||||
{
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
{
|
||||
"withRemotePhp": True,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# configuration
|
||||
config = {
|
||||
"cs3ApiTests": {
|
||||
@@ -121,7 +145,8 @@ config = {
|
||||
"apiSettings",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
"emailNeeded": True,
|
||||
"extraTestEnvironment": {
|
||||
"EMAIL_HOST": "email",
|
||||
@@ -144,14 +169,14 @@ config = {
|
||||
#"collaborativePosix",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
},
|
||||
"graphUserGroup": {
|
||||
"suites": [
|
||||
"apiGraphUserGroup",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
},
|
||||
"spaces": {
|
||||
"suites": [
|
||||
@@ -176,12 +201,14 @@ config = {
|
||||
"apiSearch1",
|
||||
],
|
||||
"skip": False,
|
||||
"nightlyOpenSearch": True,
|
||||
},
|
||||
"search2": {
|
||||
"suites": [
|
||||
"apiSearch2",
|
||||
],
|
||||
"skip": False,
|
||||
"nightlyOpenSearch": True,
|
||||
},
|
||||
"sharingNg": {
|
||||
"suites": [
|
||||
@@ -209,7 +236,7 @@ config = {
|
||||
"apiNotification",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"emailNeeded": True,
|
||||
"extraTestEnvironment": {
|
||||
"EMAIL_HOST": "email",
|
||||
@@ -245,13 +272,14 @@ config = {
|
||||
],
|
||||
"skip": False,
|
||||
"tikaNeeded": True,
|
||||
"nightlyOpenSearch": True,
|
||||
},
|
||||
"ocm": {
|
||||
"suites": [
|
||||
"apiOcm",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"federationServer": True,
|
||||
"emailNeeded": True,
|
||||
"extraTestEnvironment": {
|
||||
@@ -287,14 +315,15 @@ config = {
|
||||
"apiAuthApp",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
"cliCommands": {
|
||||
"suites": [
|
||||
"cliCommands",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"antivirusNeeded": True,
|
||||
"extraServerEnvironment": {
|
||||
"ANTIVIRUS_SCANNER_TYPE": "clamav",
|
||||
@@ -309,7 +338,7 @@ config = {
|
||||
"apiTenancy",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"ldapNeeded": True,
|
||||
"extraTestEnvironment": {
|
||||
"USE_PREPARED_LDAP_USERS": True,
|
||||
@@ -403,9 +432,10 @@ config = {
|
||||
"architectures": ["arm64", "amd64"],
|
||||
"production": {
|
||||
# NOTE: need to be updated if new production releases are determined
|
||||
"tags": ["2.0", "4.0"],
|
||||
"tags": ["2.0", "4.0", "7.2"],
|
||||
# NOTE: need to be set to true if patch releases are made from stable-X-branches
|
||||
"skip_rolling": "false",
|
||||
"skip_rolling": True,
|
||||
"skip_daily": True,
|
||||
"repo": docker_repo_slug,
|
||||
"build_type": "production",
|
||||
},
|
||||
@@ -789,30 +819,7 @@ def testOpencloud(ctx):
|
||||
],
|
||||
"environment": CI_HTTP_PROXY_ENV,
|
||||
},
|
||||
{
|
||||
"name": "open-search",
|
||||
"image": OPEN_SEARCH,
|
||||
"detach": True,
|
||||
"environment": {
|
||||
"discovery.type": "single-node",
|
||||
"DISABLE_INSTALL_DEMO_CONFIG": True,
|
||||
"DISABLE_SECURITY_PLUGIN": True,
|
||||
},
|
||||
"entrypoint": ["/usr/share/opensearch/opensearch-docker-entrypoint.sh", "opensearch"],
|
||||
},
|
||||
{
|
||||
"name": "wait-for-open-search",
|
||||
"image": OC_CI_ALPINE,
|
||||
"commands": [
|
||||
"bash -c '" +
|
||||
"until curl -sS \"http://open-search:9200/_cat/health?h=status\" | grep \"green\\|yellow\"; do\n" +
|
||||
" echo \"Waiting for http://open-search:9200 to be healthy...\"\n" +
|
||||
" sleep 5\n" +
|
||||
"done\n" +
|
||||
"echo \"http://open-search:9200 healthy...\"\n" +
|
||||
"'",
|
||||
],
|
||||
},
|
||||
] + waitForOpenSearch() + [
|
||||
{
|
||||
"name": "test",
|
||||
"image": OC_CI_GOLANG,
|
||||
@@ -844,6 +851,7 @@ def testOpencloud(ctx):
|
||||
pipeline = {
|
||||
"name": "test-lint-unit",
|
||||
"steps": steps,
|
||||
"services": openSearchService(),
|
||||
"when": [
|
||||
event["base"],
|
||||
event["cron"],
|
||||
@@ -1184,6 +1192,77 @@ def wopiValidatorTests(ctx, storage, wopiServerType):
|
||||
])
|
||||
return [pipeline]
|
||||
|
||||
def build_api_test_workflow_matrix(ctx, storage, suite_cfg, default_cfg):
|
||||
"""
|
||||
Generates a matrix of feature combinations to run API tests with.
|
||||
|
||||
Args:
|
||||
ctx: woodpecker context
|
||||
storage: opencloud storage type
|
||||
suite_cfg: suite config. E.g.: config["localApiTests"]["spaces"]
|
||||
default_cfg: default suite config values
|
||||
|
||||
Returns:
|
||||
A matrix to run API tests with. E.g.:
|
||||
[
|
||||
{
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
{
|
||||
"withRemotePhp": True,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
]
|
||||
"""
|
||||
|
||||
# default for PRs and commit push events
|
||||
matrices = [{
|
||||
"withRemotePhp": default_cfg["withRemotePhp"],
|
||||
"enableWatchFs": default_cfg["enableWatchFs"],
|
||||
}]
|
||||
if ctx.build.event == "cron":
|
||||
matrices = API_TEST_NIGHTLY_CI_MATRIX[storage]
|
||||
|
||||
override_with_remote_php = None
|
||||
override_enable_watch_fs = None
|
||||
if "withRemotePhp" in suite_cfg:
|
||||
override_with_remote_php = suite_cfg["withRemotePhp"]
|
||||
if "enableWatchFs" in suite_cfg:
|
||||
override_enable_watch_fs = suite_cfg["enableWatchFs"]
|
||||
|
||||
workflow_metrices = []
|
||||
for m in matrices:
|
||||
matrix = {
|
||||
"withRemotePhp": m["withRemotePhp"],
|
||||
"enableWatchFs": m["enableWatchFs"],
|
||||
"openSearch": False,
|
||||
}
|
||||
if override_with_remote_php != None:
|
||||
matrix["withRemotePhp"] = override_with_remote_php
|
||||
if override_enable_watch_fs != None:
|
||||
matrix["enableWatchFs"] = override_enable_watch_fs
|
||||
|
||||
base = {
|
||||
"withRemotePhp": matrix["withRemotePhp"],
|
||||
"enableWatchFs": matrix["enableWatchFs"],
|
||||
}
|
||||
if matrix not in workflow_metrices and base in matrices:
|
||||
workflow_metrices.append(matrix)
|
||||
|
||||
# Add an OpenSearch search-engine variant for nightly running search tests,
|
||||
# or on demand when "opensearch" is specified in the PR title
|
||||
run_open_search = ctx.build.event == "cron" or "opensearch" in ctx.build.title.lower()
|
||||
if run_open_search and storage == "posix" and suite_cfg.get("nightlyOpenSearch", False):
|
||||
os_matrix = {
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
"openSearch": True,
|
||||
}
|
||||
if os_matrix not in workflow_metrices:
|
||||
workflow_metrices.append(os_matrix)
|
||||
return workflow_metrices
|
||||
|
||||
def localApiTestPipeline(ctx):
|
||||
pipelines = []
|
||||
|
||||
@@ -1199,9 +1278,10 @@ def localApiTestPipeline(ctx):
|
||||
"federationServer": False,
|
||||
"collaborationServiceNeeded": False,
|
||||
"extraCollaborationEnvironment": {},
|
||||
"withRemotePhp": [False],
|
||||
"enableWatchFs": [False],
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
"ldapNeeded": False,
|
||||
"nightlyOpenSearch": False,
|
||||
}
|
||||
|
||||
if "localApiTests" in config:
|
||||
@@ -1211,68 +1291,80 @@ def localApiTestPipeline(ctx):
|
||||
for item in defaults:
|
||||
params[item] = matrix[item] if item in matrix else defaults[item]
|
||||
|
||||
if ctx.build.event == "cron":
|
||||
params["storages"] = OPENCLOUD_STORAGES
|
||||
|
||||
# skip CLI tests in nightly pipeline
|
||||
if name.startswith("cli"):
|
||||
continue
|
||||
|
||||
# use decomposed storage if specified in the PR title
|
||||
# run CLI tests only with decomposed storage
|
||||
if "[decomposed]" in ctx.build.title.lower() or name.startswith("cli"):
|
||||
params["storages"] = ["decomposed"]
|
||||
|
||||
if ctx.build.event == "cron":
|
||||
params["withRemotePhp"] = [True, False]
|
||||
params["enableWatchFs"] = [True, False]
|
||||
|
||||
# override withRemotePhp if specified in the suite config
|
||||
if "withRemotePhp" in matrix:
|
||||
params["withRemotePhp"] = matrix["withRemotePhp"]
|
||||
|
||||
for storage in params["storages"]:
|
||||
for run_with_remote_php in params["withRemotePhp"]:
|
||||
for run_with_watch_fs_enabled in params["enableWatchFs"]:
|
||||
pipeline_name = "test-API"
|
||||
if name.startswith("cli"):
|
||||
pipeline_name = "test-CLI"
|
||||
pipeline_name += "-%s" % name
|
||||
if not run_with_remote_php:
|
||||
pipeline_name += "-withoutRemotePhp"
|
||||
pipeline_name += "-%s" % storage
|
||||
if run_with_watch_fs_enabled:
|
||||
pipeline_name += "-watchfs"
|
||||
matrices = build_api_test_workflow_matrix(ctx, storage, matrix, defaults)
|
||||
for m in matrices:
|
||||
run_with_remote_php = m["withRemotePhp"]
|
||||
run_with_watch_fs = m["enableWatchFs"]
|
||||
run_with_open_search = m["openSearch"]
|
||||
|
||||
pipeline = {
|
||||
"name": pipeline_name,
|
||||
"steps": skipCheckStep(ctx, "acceptance-tests") + evaluateWorkflowStep() + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
||||
(tikaService() if params["tikaNeeded"] else []) +
|
||||
(waitForWebOffices(["https://collabora:9980", "https://onlyoffice", "http://fakeoffice:8080"]) if params["collaborationServiceNeeded"] else []) +
|
||||
(waitForClamavService() if params["antivirusNeeded"] else []) +
|
||||
(waitForEmailService() if params["emailNeeded"] else []) +
|
||||
(ldapService() if params["ldapNeeded"] else []) +
|
||||
(waitForLdapService() if params["ldapNeeded"] else []) +
|
||||
opencloudServer(
|
||||
storage,
|
||||
extra_server_environment = params["extraServerEnvironment"],
|
||||
with_wrapper = True,
|
||||
tika_enabled = params["tikaNeeded"],
|
||||
watch_fs_enabled = run_with_watch_fs_enabled,
|
||||
) +
|
||||
(opencloudServer(storage, deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"], watch_fs_enabled = run_with_watch_fs_enabled) if params["federationServer"] else []) +
|
||||
((wopiCollaborationService("fakeoffice") + wopiCollaborationService("collabora") + wopiCollaborationService("onlyoffice")) if params["collaborationServiceNeeded"] else []) +
|
||||
(openCloudHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) if params["collaborationServiceNeeded"] else []) +
|
||||
localApiTest(params["suites"], storage, params["extraTestEnvironment"], run_with_remote_php) +
|
||||
logRequests(),
|
||||
"services": (emailService() if params["emailNeeded"] else []) +
|
||||
(clamavService() if params["antivirusNeeded"] else []) +
|
||||
((fakeOffice() + collaboraService() + onlyofficeService()) if params["collaborationServiceNeeded"] else []),
|
||||
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
||||
"when": [
|
||||
event["base"],
|
||||
event["cron"],
|
||||
event["pull_request"],
|
||||
],
|
||||
}
|
||||
prefixStepCommands(pipeline, [
|
||||
". ./.woodpecker.env",
|
||||
'[ "$SKIP_WORKFLOW" = "true" ] && exit 0',
|
||||
])
|
||||
pipelines.append(pipeline)
|
||||
pipeline_name = "test-API"
|
||||
if name.startswith("cli"):
|
||||
pipeline_name = "test-CLI"
|
||||
pipeline_name += "-%s" % name
|
||||
pipeline_name += "-%s" % storage
|
||||
if run_with_remote_php:
|
||||
pipeline_name += "-withRemotePhp"
|
||||
if run_with_watch_fs:
|
||||
pipeline_name += "-watchfs"
|
||||
if run_with_open_search:
|
||||
pipeline_name += "-opensearch"
|
||||
|
||||
server_environment = dict(params["extraServerEnvironment"])
|
||||
if run_with_open_search:
|
||||
server_environment["SEARCH_ENGINE_TYPE"] = "open-search"
|
||||
server_environment["SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES"] = "http://open-search:9200"
|
||||
|
||||
pipeline = {
|
||||
"name": pipeline_name,
|
||||
"steps": skipCheckStep(ctx, "acceptance-tests") + evaluateWorkflowStep() + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
||||
(tikaService() if params["tikaNeeded"] else []) +
|
||||
(waitForOpenSearch() if run_with_open_search else []) +
|
||||
(waitForWebOffices(["https://collabora:9980", "https://onlyoffice", "http://fakeoffice:8080"]) if params["collaborationServiceNeeded"] else []) +
|
||||
(waitForClamavService() if params["antivirusNeeded"] else []) +
|
||||
(waitForEmailService() if params["emailNeeded"] else []) +
|
||||
(ldapService() if params["ldapNeeded"] else []) +
|
||||
(waitForLdapService() if params["ldapNeeded"] else []) +
|
||||
opencloudServer(
|
||||
storage,
|
||||
extra_server_environment = server_environment,
|
||||
with_wrapper = True,
|
||||
tika_enabled = params["tikaNeeded"],
|
||||
watch_fs_enabled = run_with_watch_fs,
|
||||
) +
|
||||
(opencloudServer(storage, deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"], watch_fs_enabled = run_with_watch_fs) if params["federationServer"] else []) +
|
||||
((wopiCollaborationService("fakeoffice") + wopiCollaborationService("collabora") + wopiCollaborationService("onlyoffice")) if params["collaborationServiceNeeded"] else []) +
|
||||
(openCloudHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) if params["collaborationServiceNeeded"] else []) +
|
||||
localApiTest(params["suites"], storage, params["extraTestEnvironment"], run_with_remote_php) +
|
||||
logRequests(),
|
||||
"services": (emailService() if params["emailNeeded"] else []) +
|
||||
(clamavService() if params["antivirusNeeded"] else []) +
|
||||
(openSearchService() if run_with_open_search else []) +
|
||||
((fakeOffice() + collaboraService() + onlyofficeService()) if params["collaborationServiceNeeded"] else []),
|
||||
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
||||
"when": [
|
||||
event["base"],
|
||||
event["cron"],
|
||||
event["pull_request"],
|
||||
],
|
||||
}
|
||||
prefixStepCommands(pipeline, [
|
||||
". ./.woodpecker.env",
|
||||
'[ "$SKIP_WORKFLOW" = "true" ] && exit 0',
|
||||
])
|
||||
pipelines.append(pipeline)
|
||||
return pipelines
|
||||
|
||||
def localApiTest(suites, storage = "decomposed", extra_environment = {}, with_remote_php = False):
|
||||
@@ -1316,8 +1408,8 @@ def localApiTest(suites, storage = "decomposed", extra_environment = {}, with_re
|
||||
|
||||
def coreApiTestPipeline(ctx):
|
||||
defaults = {
|
||||
"withRemotePhp": [False],
|
||||
"enableWatchFs": [False],
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
"storages": ["posix"],
|
||||
"numberOfParts": 7,
|
||||
"skipExceptParts": [],
|
||||
@@ -1334,63 +1426,60 @@ def coreApiTestPipeline(ctx):
|
||||
for item in defaults:
|
||||
params[item] = matrix[item] if item in matrix else defaults[item]
|
||||
|
||||
if ctx.build.event == "cron":
|
||||
params["storages"] = OPENCLOUD_STORAGES
|
||||
|
||||
# use decomposed storage if specified in the PR title
|
||||
if "[decomposed]" in ctx.build.title.lower():
|
||||
params["storages"] = ["decomposed"]
|
||||
|
||||
if ctx.build.event == "cron":
|
||||
params["withRemotePhp"] = [True, False]
|
||||
params["enableWatchFs"] = [True, False]
|
||||
|
||||
# override withRemotePhp if specified in the suite config
|
||||
if "withRemotePhp" in matrix:
|
||||
params["withRemotePhp"] = matrix["withRemotePhp"]
|
||||
|
||||
debugParts = params["skipExceptParts"]
|
||||
debugPartsEnabled = (len(debugParts) != 0)
|
||||
|
||||
for storage in params["storages"]:
|
||||
for runPart in range(1, params["numberOfParts"] + 1):
|
||||
for run_with_remote_php in params["withRemotePhp"]:
|
||||
for run_with_watch_fs_enabled in params["enableWatchFs"]:
|
||||
if not debugPartsEnabled or (debugPartsEnabled and runPart in debugParts):
|
||||
pipeline_name = "test-Core-API-%s" % runPart
|
||||
if not run_with_remote_php:
|
||||
pipeline_name += "-withoutRemotePhp"
|
||||
pipeline_name += "-%s" % storage
|
||||
if run_with_watch_fs_enabled:
|
||||
pipeline_name += "-watchfs"
|
||||
matrices = build_api_test_workflow_matrix(ctx, storage, matrix, defaults)
|
||||
for m in matrices:
|
||||
run_with_remote_php = m["withRemotePhp"]
|
||||
run_with_watch_fs = m["enableWatchFs"]
|
||||
if not debugPartsEnabled or (debugPartsEnabled and runPart in debugParts):
|
||||
pipeline_name = "test-Core-API-%s" % runPart
|
||||
pipeline_name += "-%s" % storage
|
||||
if run_with_remote_php:
|
||||
pipeline_name += "-withRemotePhp"
|
||||
if run_with_watch_fs:
|
||||
pipeline_name += "-watchfs"
|
||||
|
||||
pipeline = {
|
||||
"name": pipeline_name,
|
||||
"steps": skipCheckStep(ctx, "acceptance-tests") +
|
||||
evaluateWorkflowStep() +
|
||||
restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
||||
opencloudServer(
|
||||
storage,
|
||||
with_wrapper = True,
|
||||
watch_fs_enabled = run_with_watch_fs_enabled,
|
||||
) +
|
||||
coreApiTest(
|
||||
runPart,
|
||||
params["numberOfParts"],
|
||||
run_with_remote_php,
|
||||
storage,
|
||||
) +
|
||||
logRequests(),
|
||||
"services": redisForOCStorage(storage),
|
||||
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
||||
"when": [
|
||||
event["base"],
|
||||
event["cron"],
|
||||
event["pull_request"],
|
||||
],
|
||||
}
|
||||
prefixStepCommands(pipeline, [
|
||||
". ./.woodpecker.env",
|
||||
'[ "$SKIP_WORKFLOW" = "true" ] && exit 0',
|
||||
])
|
||||
pipelines.append(pipeline)
|
||||
pipeline = {
|
||||
"name": pipeline_name,
|
||||
"steps": skipCheckStep(ctx, "acceptance-tests") +
|
||||
evaluateWorkflowStep() +
|
||||
restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
||||
opencloudServer(
|
||||
storage,
|
||||
with_wrapper = True,
|
||||
watch_fs_enabled = run_with_watch_fs,
|
||||
) +
|
||||
coreApiTest(
|
||||
runPart,
|
||||
params["numberOfParts"],
|
||||
run_with_remote_php,
|
||||
storage,
|
||||
) +
|
||||
logRequests(),
|
||||
"services": redisForOCStorage(storage),
|
||||
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
||||
"when": [
|
||||
event["base"],
|
||||
event["cron"],
|
||||
event["pull_request"],
|
||||
],
|
||||
}
|
||||
prefixStepCommands(pipeline, [
|
||||
". ./.woodpecker.env",
|
||||
'[ "$SKIP_WORKFLOW" = "true" ] && exit 0',
|
||||
])
|
||||
pipelines.append(pipeline)
|
||||
return pipelines
|
||||
|
||||
def coreApiTest(part_number = 1, number_of_parts = 1, with_remote_php = False, storage = "posix"):
|
||||
@@ -1722,7 +1811,7 @@ def dockerReleases(ctx):
|
||||
docker_releases.append("rolling")
|
||||
|
||||
# on non tag events, do daily build
|
||||
else:
|
||||
elif not config["dockerReleases"]["production"]["skip_daily"]:
|
||||
docker_releases.append("daily")
|
||||
|
||||
for releaseConfigName in docker_releases:
|
||||
@@ -2084,6 +2173,7 @@ def readyReleaseGo():
|
||||
"image": READY_RELEASE_GO,
|
||||
"settings": {
|
||||
"git_email": "devops@opencloud.eu",
|
||||
"release_branch": "stable-7.2",
|
||||
"forge_type": "github",
|
||||
"forge_token": {
|
||||
"from_secret": "github_token",
|
||||
@@ -3251,6 +3341,33 @@ def tikaService():
|
||||
],
|
||||
}]
|
||||
|
||||
def openSearchService():
|
||||
return [{
|
||||
"name": "open-search",
|
||||
"image": OPEN_SEARCH,
|
||||
"environment": {
|
||||
"discovery.type": "single-node",
|
||||
"DISABLE_INSTALL_DEMO_CONFIG": True,
|
||||
"DISABLE_SECURITY_PLUGIN": True,
|
||||
},
|
||||
"entrypoint": ["/usr/share/opensearch/opensearch-docker-entrypoint.sh", "opensearch"],
|
||||
}]
|
||||
|
||||
def waitForOpenSearch():
|
||||
return [{
|
||||
"name": "wait-for-open-search",
|
||||
"image": OC_CI_ALPINE,
|
||||
"commands": [
|
||||
"bash -c '" +
|
||||
"until curl -sS \"http://open-search:9200/_cat/health?h=status\" | grep \"green\\|yellow\"; do\n" +
|
||||
" echo \"Waiting for http://open-search:9200 to be healthy...\"\n" +
|
||||
" sleep 5\n" +
|
||||
"done\n" +
|
||||
"echo \"http://open-search:9200 healthy...\"\n" +
|
||||
"'",
|
||||
],
|
||||
}]
|
||||
|
||||
def logRequests():
|
||||
return [{
|
||||
"name": "api-test-failure-logs",
|
||||
|
||||
@@ -1,5 +1,84 @@
|
||||
# Changelog
|
||||
|
||||
## [7.2.4](https://github.com/opencloud-eu/opencloud/releases/tag/v7.2.4) - 2026-08-20
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
@aduffeck, @rhafer, @v-scharf
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- [stable-7.2] fix(search): fix open-search kql field compilation [[#3375](https://github.com/opencloud-eu/opencloud/pull/3375)]
|
||||
- [stable-7.2] fix(search): ack every event the index service handles [[#3374](https://github.com/opencloud-eu/opencloud/pull/3374)]
|
||||
- [stable-7.2] fix(postprocessing): retry publishing events [[#3370](https://github.com/opencloud-eu/opencloud/pull/3370)]
|
||||
- [stable-7.2] Only log a debug message when an item is still in processing state [[#3369](https://github.com/opencloud-eu/opencloud/pull/3369)]
|
||||
- [stable-7.2] backport fixes for the opencloud posixfs command [[#3367](https://github.com/opencloud-eu/opencloud/pull/3367)]
|
||||
- [stable-7.2] Expose upload status and message [[#3365](https://github.com/opencloud-eu/opencloud/pull/3365)]
|
||||
- Fix opensearch space scope [stable-7.2] [[#3351](https://github.com/opencloud-eu/opencloud/pull/3351)]
|
||||
- [stable-7.2] fix(posixfs scan): Setup logger for scan command [[#3195](https://github.com/opencloud-eu/opencloud/pull/3195)]
|
||||
|
||||
### ✅ Tests
|
||||
|
||||
- [stable-7.2] api-tests against open-search in stable [[#3353](https://github.com/opencloud-eu/opencloud/pull/3353)]
|
||||
|
||||
## [7.2.3](https://github.com/opencloud-eu/opencloud/releases/tag/v7.2.3) - 2026-08-06
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
@aduffeck, @v-scharf
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix missing favorite flag on opensearch hits [[#3256](https://github.com/opencloud-eu/opencloud/pull/3256)]
|
||||
|
||||
## [7.2.2](https://github.com/opencloud-eu/opencloud/releases/tag/v7.2.2) - 2026-07-14
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
@kulmann, @v-scharf
|
||||
|
||||
### 📦️ Dependencies
|
||||
|
||||
- [full-ci] chore: bump web to v7.1.4 [[#3122](https://github.com/opencloud-eu/opencloud/pull/3122)]
|
||||
|
||||
## [7.2.1](https://github.com/opencloud-eu/opencloud/releases/tag/v7.2.1) - 2026-07-06
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
@JammingBen, @v-scharf
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix warming up the id cache for the user storage [[#3072](https://github.com/opencloud-eu/opencloud/pull/3072)]
|
||||
|
||||
### 📦️ Dependencies
|
||||
|
||||
- [full-ci] chore: bump web to v7.1.3 [[#3052](https://github.com/opencloud-eu/opencloud/pull/3052)]
|
||||
|
||||
## [7.2.0](https://github.com/opencloud-eu/opencloud/releases/tag/v7.2.0) - 2026-06-24
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
@Heiko-Pohl, @JammingBen, @ScharfViktor, @aduffeck, @butonic, @dragonchaser, @kulmann, @rhafer
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- [stable-7.2] Backport fixes from main [[#2999](https://github.com/opencloud-eu/opencloud/pull/2999)]
|
||||
- fix(idp): aarch64 build [[#2906](https://github.com/opencloud-eu/opencloud/pull/2906)]
|
||||
- use ldap instead of ldaps internally [[#2880](https://github.com/opencloud-eu/opencloud/pull/2880)]
|
||||
|
||||
### 📚 Documentation
|
||||
|
||||
- Rename role_name from "guest" to "user-light" [[#2912](https://github.com/opencloud-eu/opencloud/pull/2912)]
|
||||
|
||||
### 📦️ Dependencies
|
||||
|
||||
- [full-ci] chore: bump web to v7.1.2 [[#3012](https://github.com/opencloud-eu/opencloud/pull/3012)]
|
||||
- [full-ci] chore: bump web to v7.1.1 [[#2998](https://github.com/opencloud-eu/opencloud/pull/2998)]
|
||||
- bump reva to latest main [[#2922](https://github.com/opencloud-eu/opencloud/pull/2922)]
|
||||
- build(deps-dev): bump webpack-manifest-plugin from 5.0.0 to 6.0.1 in /services/idp [[#2884](https://github.com/opencloud-eu/opencloud/pull/2884)]
|
||||
- build(deps): bump axios from 1.16.0 to 1.16.1 in /services/idp [[#2883](https://github.com/opencloud-eu/opencloud/pull/2883)]
|
||||
|
||||
## [7.1.0](https://github.com/opencloud-eu/opencloud/releases/tag/v7.1.0) - 2026-06-02
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
+2
-3
@@ -21,15 +21,14 @@ COPY ./ /opencloud/
|
||||
WORKDIR /opencloud/opencloud
|
||||
RUN make node-generate-prod
|
||||
|
||||
FROM golang:1.24-alpine AS build
|
||||
RUN apk add bash make git curl gcc musl-dev libc-dev binutils-gold inotify-tools vips-dev
|
||||
FROM quay.io/opencloudeu/golang-ci:1.25 AS build
|
||||
|
||||
COPY --from=generate /opencloud /opencloud
|
||||
|
||||
WORKDIR /opencloud/opencloud
|
||||
RUN make go-generate build ENABLE_VIPS=true
|
||||
|
||||
FROM alpine:3.23
|
||||
FROM alpine:3.24
|
||||
|
||||
RUN apk add --no-cache attr ca-certificates curl mailcap tree vips && \
|
||||
echo 'hosts: files dns' >| /etc/nsswitch.conf
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
Bugfix: Retry publishing postprocessing events before giving up
|
||||
|
||||
A single transient failure while publishing an event to the event system took
|
||||
the whole server down. The postprocessing service treated every publish error
|
||||
as fatal and called log.Fatal, which exits the process and so also stopped all
|
||||
the other services running in the same binary. A burst of uploads was enough to
|
||||
run into one nats publish timeout and lose the server with it.
|
||||
|
||||
Publishing is now retried using the same exponential backoff that is already
|
||||
used for failed postprocessing steps. Between the attempts the source event is
|
||||
marked as in progress and the backoff is capped at half the ack wait, so the
|
||||
event should not be redelivered to another worker while we are still retrying.
|
||||
The number of retries is configurable via POSTPROCESSING_PUBLISH_MAX_RETRIES.
|
||||
|
||||
Should all attempts fail, the source event is no longer acknowledged. Before,
|
||||
the event was acknowledged even though its successor was never published, so
|
||||
the upload was left half processed in the store and did not recover on restart.
|
||||
|
||||
https://github.com/opencloud-eu/opencloud/issues/3271
|
||||
https://github.com/opencloud-eu/opencloud/issues/2232
|
||||
https://github.com/opencloud-eu/opencloud/issues/2422
|
||||
@@ -64,7 +64,7 @@ require (
|
||||
github.com/open-policy-agent/opa v1.15.2
|
||||
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d
|
||||
github.com/opencloud-eu/reva/v2 v2.46.3-0.20260610093751-a33d8108dd91
|
||||
github.com/opencloud-eu/reva/v2 v2.46.8
|
||||
github.com/opensearch-project/opensearch-go/v4 v4.6.0
|
||||
github.com/orcaman/concurrent-map v1.0.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
@@ -75,6 +75,7 @@ require (
|
||||
github.com/rogpeppe/go-internal v1.14.1
|
||||
github.com/rs/cors v1.11.1
|
||||
github.com/rs/zerolog v1.35.0
|
||||
github.com/shamaton/msgpack/v2 v2.4.1
|
||||
github.com/sirupsen/logrus v1.9.4
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
@@ -84,7 +85,6 @@ require (
|
||||
github.com/test-go/testify v1.1.4
|
||||
github.com/testcontainers/testcontainers-go v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/opensearch v0.42.0
|
||||
github.com/theckman/yacspin v0.13.12
|
||||
github.com/thejerf/suture/v4 v4.0.6
|
||||
github.com/tidwall/gjson v1.19.0
|
||||
github.com/tidwall/sjson v1.2.5
|
||||
@@ -345,7 +345,6 @@ require (
|
||||
github.com/sergi/go-diff v1.4.0 // indirect
|
||||
github.com/sethvargo/go-diceware v0.5.0 // indirect
|
||||
github.com/sethvargo/go-password v0.3.1 // indirect
|
||||
github.com/shamaton/msgpack/v2 v2.4.1 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
|
||||
github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
|
||||
github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 // indirect
|
||||
|
||||
@@ -948,8 +948,8 @@ github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+l
|
||||
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d h1:JcqGDiyrcaQwVyV861TUyQgO7uEmsjkhfm7aQd84dOw=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q=
|
||||
github.com/opencloud-eu/reva/v2 v2.46.3-0.20260610093751-a33d8108dd91 h1:A/a0d9UNclpNBWGp2NUDWF+qO+U/u38EBH4CIk2dqIE=
|
||||
github.com/opencloud-eu/reva/v2 v2.46.3-0.20260610093751-a33d8108dd91/go.mod h1:RoFQt+u7edxwzHr1IZ2Y6VaDinMiRPQupAvMBy3WVmE=
|
||||
github.com/opencloud-eu/reva/v2 v2.46.8 h1:stqn5VfGiIkR7+b/BIsFAqbLbrzSARu43I/t/6G9X3Q=
|
||||
github.com/opencloud-eu/reva/v2 v2.46.8/go.mod h1:RoFQt+u7edxwzHr1IZ2Y6VaDinMiRPQupAvMBy3WVmE=
|
||||
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
|
||||
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
@@ -1195,8 +1195,6 @@ github.com/testcontainers/testcontainers-go/modules/opensearch v0.42.0 h1:lXPr5N
|
||||
github.com/testcontainers/testcontainers-go/modules/opensearch v0.42.0/go.mod h1:KJ+m9onB785g9VDRDuhvUvsbkRjnFp3Uz+CHoHpPieA=
|
||||
github.com/thanhpk/randstr v1.0.6 h1:psAOktJFD4vV9NEVb3qkhRSMvYh4ORRaj1+w/hn4B+o=
|
||||
github.com/thanhpk/randstr v1.0.6/go.mod h1:M/H2P1eNLZzlDwAzpkkkUvoyNNMbzRGhESZuEQk3r0U=
|
||||
github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRzJr4=
|
||||
github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg=
|
||||
github.com/thejerf/suture/v4 v4.0.6 h1:QsuCEsCqb03xF9tPAsWAj8QOAJBgQI1c0VqJNaingg8=
|
||||
github.com/thejerf/suture/v4 v4.0.6/go.mod h1:gu9Y4dXNUWFrByqRt30Rm9/UZ0wzRSt9AJS6xu/ZGxU=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:alpine3.23 AS build
|
||||
FROM quay.io/opencloudeu/golang-ci:1.25 AS build
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
ARG VERSION
|
||||
@@ -6,8 +6,6 @@ ARG STRING
|
||||
ARG EDITION="dev"
|
||||
ARG SRCDIR
|
||||
|
||||
RUN apk add bash make git curl gcc musl-dev libc-dev binutils-gold inotify-tools vips-dev
|
||||
|
||||
WORKDIR /build
|
||||
RUN --mount=type=bind,target=/build,rw \
|
||||
--mount=type=cache,target=/go/pkg/mod \
|
||||
@@ -15,7 +13,7 @@ RUN --mount=type=bind,target=/build,rw \
|
||||
GOOS="${TARGETOS:-linux}" GOARCH="${TARGETARCH:-amd64}" ; \
|
||||
make -C ${SRCDIR:-.}/opencloud release-linux-docker-${TARGETARCH} ENABLE_VIPS=true DIST=/dist
|
||||
|
||||
FROM alpine:3.23
|
||||
FROM alpine:3.24
|
||||
ARG VERSION
|
||||
ARG REVISION
|
||||
ARG TARGETOS
|
||||
|
||||
+273
-429
@@ -1,17 +1,16 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/pkg/x/path/filepathx"
|
||||
storageUsersParser "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/services/storage-users/pkg/event"
|
||||
"github.com/opencloud-eu/opencloud/services/storage-users/pkg/revaconfig"
|
||||
@@ -19,26 +18,10 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/ignore"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/theckman/yacspin"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
)
|
||||
|
||||
// Define the names of the extended attributes we are working with.
|
||||
const (
|
||||
parentIDAttrName = "user.oc.parentid"
|
||||
idAttrName = "user.oc.id"
|
||||
spaceIDAttrName = "user.oc.space.id"
|
||||
ownerIDAttrName = "user.oc.owner.id"
|
||||
)
|
||||
|
||||
var (
|
||||
spinner *yacspin.Spinner
|
||||
restartRequired = false
|
||||
ignorer *ignore.Ignorer
|
||||
)
|
||||
|
||||
type IDCacher interface {
|
||||
@@ -72,11 +55,23 @@ func init() {
|
||||
|
||||
// scanCmd performs a posixfs id cache warmup scan
|
||||
func scanCmd(ocCfg *config.Config) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "scan",
|
||||
scanCmd := &cobra.Command{
|
||||
Use: "scan [path ...]",
|
||||
Short: "Perform a filesystem scan and update the ID and filemetadata cache",
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
Long: `Perform a filesystem scan and update the ID and filemetadata cache.
|
||||
|
||||
You can specify one or more paths to limit the scope of the scan.
|
||||
If no path is provided, the whole storage is checked, starting at the storage root directory.
|
||||
|
||||
The provided arguments determines the scope of the check:
|
||||
- a storage root: the whole storage (all personal and project spaces) is scanned
|
||||
- a space root: only that space is scanned
|
||||
- a file or directory: only that single resource is scanned (and its children, if it is a directory)
|
||||
|
||||
Any specified file or directory must be underneath the storage root directory and if that is not the case,
|
||||
the command is aborted with an error before performing any scanning.`,
|
||||
Args: cobra.ArbitraryArgs,
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
if err := parser.ParseConfig(ocCfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
@@ -93,432 +88,281 @@ func scanCmd(ocCfg *config.Config) *cobra.Command {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// We want to initialize the driver but disable scanfs on boot, so we can trigger it manually afterwards
|
||||
drivers := revaconfig.StorageProviderDrivers(cfg)
|
||||
drivers["posix"] = revaconfig.Posix(cfg, false, false)
|
||||
|
||||
var fsStream events.Stream
|
||||
var err error
|
||||
fsStream, err = event.NewStream(cfg)
|
||||
haltOnError, err := cmd.Flags().GetBool("halt-on-error")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create event stream for posix driver: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
f, ok := registry.NewFuncs["posix"]
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "posix driver not found in registry\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fs, err := f(drivers["posix"].(map[string]any), fsStream, nil)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s': %v\n", cfg.Driver, err)
|
||||
return err
|
||||
}
|
||||
|
||||
cacher, ok := fs.(IDCacher)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "The posix driver does not expose WarmupIDCache.\n")
|
||||
os.Exit(1)
|
||||
storageRoot := cfg.Drivers.Posix.Root
|
||||
paths := []string{storageRoot}
|
||||
if len(args) > 0 {
|
||||
paths = []string{}
|
||||
for _, v := range args {
|
||||
path := v
|
||||
if !filepath.IsAbs(path) {
|
||||
if v, err := filepath.Abs(path); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to make the specified path %q absolute: %v\n", v, err)
|
||||
os.Exit(1)
|
||||
} else {
|
||||
path = v
|
||||
}
|
||||
}
|
||||
// not ensuring whether the path is under the storage root here, will be done when iterating over them
|
||||
path = filepath.Clean(path)
|
||||
paths = append(paths, path)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("Starting posixfs scan...")
|
||||
err = cacher.WarmupIDCache(cfg.Drivers.Posix.Root, true, false)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Scan failed: %v\n", err)
|
||||
return err
|
||||
var scan func(path string) error = nil
|
||||
{
|
||||
// We want to initialize the driver but disable scanfs on boot, so we can trigger it manually afterwards
|
||||
drivers := revaconfig.StorageProviderDrivers(cfg)
|
||||
drivers["posix"] = revaconfig.Posix(cfg, false, false)
|
||||
|
||||
var fsStream events.Stream
|
||||
var err error
|
||||
fsStream, err = event.NewStream(cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create event stream for posix driver: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log := logger("posixfs")
|
||||
|
||||
f, ok := registry.NewFuncs["posix"]
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "posix driver not found in registry\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fs, err := f(drivers["posix"].(map[string]any), fsStream, &log)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s': %v\n", cfg.Driver, err)
|
||||
return err
|
||||
}
|
||||
|
||||
cacher, ok := fs.(IDCacher)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "The posix driver does not expose WarmupIDCache.\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
scan = func(path string) error {
|
||||
err := cacher.WarmupIDCache(path, true, false)
|
||||
if err != nil {
|
||||
logFailure("Error scanning path '%s': %v", path, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("Scan completed successfully.")
|
||||
return nil
|
||||
errors := processPosixFsResources(paths, !haltOnError,
|
||||
func(path string) error {
|
||||
fmt.Println("Scanning personal spaces...")
|
||||
return scan(path)
|
||||
},
|
||||
func(path string) error {
|
||||
fmt.Println("Scanning project spaces...")
|
||||
return scan(path)
|
||||
},
|
||||
func(path string) error {
|
||||
fmt.Printf("Scanning space '%s'...\n", path)
|
||||
return scan(path)
|
||||
},
|
||||
func(path string) error {
|
||||
fmt.Printf("Scanning '%s'...\n", path)
|
||||
return scan(path)
|
||||
},
|
||||
)
|
||||
|
||||
if len(errors) == 0 {
|
||||
fmt.Println("Scan completed successfully.")
|
||||
return nil
|
||||
} else {
|
||||
plural := "s"
|
||||
if len(errors) == 1 {
|
||||
plural = ""
|
||||
}
|
||||
verb := "completed"
|
||||
if haltOnError {
|
||||
verb = "aborted"
|
||||
}
|
||||
return fmt.Errorf("scan %s with %d error%s", verb, len(errors), plural)
|
||||
}
|
||||
},
|
||||
}
|
||||
return cmd
|
||||
scanCmd.Flags().BoolP("halt-on-error", "E", false, "Halt at once when an error occurs when processing one of the paths (default behaviour is to keep going and attempt to process all paths).")
|
||||
return scanCmd
|
||||
}
|
||||
|
||||
// consistencyCmd returns a command to check the consistency of the posixfs storage.
|
||||
func consistencyCmd(cfg *config.Config) *cobra.Command {
|
||||
func consistencyCmd(ocCfg *config.Config) *cobra.Command {
|
||||
consCmd := &cobra.Command{
|
||||
Use: "consistency",
|
||||
Short: "check the consistency of the posixfs storage",
|
||||
Use: "consistency [path ...]",
|
||||
Short: "Check the consistency of the posixfs storage",
|
||||
Long: `Check the consistency of the posixfs storage.
|
||||
|
||||
You can specify one or more paths to limit the scope of the check.
|
||||
If no path is provided, the whole storage is checked.
|
||||
|
||||
The provided arguments determines the scope of the check:
|
||||
- a storage root: the whole storage (all personal and project spaces) is checked
|
||||
- a space root: only that space is checked
|
||||
- a file or directory: only that single entity is checked (and its children, if it is a directory)`,
|
||||
Args: cobra.ArbitraryArgs,
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
|
||||
if err := parser.ParseConfig(ocCfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Parse storage users config
|
||||
ocCfg.StorageUsers.Commons = ocCfg.Commons
|
||||
|
||||
return configlog.ReturnFatal(storageUsersParser.ParseConfig(ocCfg.StorageUsers))
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return checkPosixfsConsistency(cmd, cfg)
|
||||
cfg := ocCfg.StorageUsers
|
||||
if len(args) == 0 {
|
||||
args = []string{cfg.Drivers.Posix.Root}
|
||||
}
|
||||
log := logger("posixfs")
|
||||
recalculateChecksums, err := cmd.Flags().GetBool("fix-checksums")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
drivers := revaconfig.StorageProviderDrivers(cfg)
|
||||
drivers["posix"] = revaconfig.Posix(cfg, false, false)
|
||||
opts, err := options.New(drivers["posix"].(map[string]any))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ignorer := ignore.NewIgnorer(opts, &log)
|
||||
|
||||
checker := &consistencyChecker{
|
||||
cfg: cfg,
|
||||
ignorer: ignorer,
|
||||
recalculateChecksums: recalculateChecksums,
|
||||
}
|
||||
|
||||
return checker.Check(args)
|
||||
},
|
||||
}
|
||||
consCmd.Flags().StringP("root", "r", "", "Path to the root directory of the posixfs storage")
|
||||
_ = consCmd.MarkFlagRequired("root")
|
||||
|
||||
consCmd.Flags().Bool("fix-checksums", false, "Recalculate and fix the file checksums. This reads every file and can be slow on large storages.")
|
||||
return consCmd
|
||||
}
|
||||
|
||||
// checkPosixfsConsistency checks the consistency of the posixfs storage.
|
||||
func checkPosixfsConsistency(cmd *cobra.Command, cfg *config.Config) error {
|
||||
rootPath, _ := cmd.Flags().GetString("root")
|
||||
indexesPath := filepath.Join(rootPath, "indexes")
|
||||
|
||||
opt, _ := options.New(map[string]interface{}{
|
||||
"root": rootPath,
|
||||
})
|
||||
log := zerolog.Nop()
|
||||
ignorer = ignore.NewIgnorer(opt, &log)
|
||||
|
||||
_, err := os.Stat(indexesPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("consistency check failed: '%s' is not a posixfs root", rootPath)
|
||||
}
|
||||
return fmt.Errorf("error accessing '%s': %w", indexesPath, err)
|
||||
}
|
||||
|
||||
spinnerCfg := yacspin.Config{
|
||||
Frequency: 100 * time.Millisecond,
|
||||
CharSet: yacspin.CharSets[11],
|
||||
StopCharacter: "✓",
|
||||
StopColors: []string{"fgGreen"},
|
||||
StopFailCharacter: "✗",
|
||||
StopFailColors: []string{"fgRed"},
|
||||
}
|
||||
|
||||
spinner, err = yacspin.New(spinnerCfg)
|
||||
err = spinner.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating spinner: %w", err)
|
||||
}
|
||||
|
||||
checkSpaces(filepath.Join(rootPath, "users"))
|
||||
spinner.Suffix(" Personal spaces check ")
|
||||
spinner.StopMessage("completed\n")
|
||||
spinner.Stop()
|
||||
|
||||
checkSpaces(filepath.Join(rootPath, "projects"))
|
||||
spinner.Suffix(" Project spaces check ")
|
||||
spinner.StopMessage("completed")
|
||||
spinner.Stop()
|
||||
|
||||
if restartRequired {
|
||||
fmt.Println("\n\n ⚠️ Please restart your openCloud instance to apply changes.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkSpaces(basePath string) {
|
||||
dirEntries, err := os.ReadDir(basePath)
|
||||
if err != nil {
|
||||
spinner.Message(fmt.Sprintf("Error reading spaces directory '%s'\n", basePath))
|
||||
spinner.StopFail()
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
if entry.IsDir() {
|
||||
fullPath := filepath.Join(basePath, entry.Name())
|
||||
checkSpace(fullPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkSpace(spacePath string) {
|
||||
spinner.Message("")
|
||||
spinner.Suffix(fmt.Sprintf(" Checking space '%s'", spacePath))
|
||||
|
||||
info, err := os.Stat(spacePath)
|
||||
if err != nil {
|
||||
logFailure("Error accessing path '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
if !info.IsDir() {
|
||||
logFailure("Error: The provided path '%s' is not a directory\n", spacePath)
|
||||
return
|
||||
}
|
||||
|
||||
spaceID, err := xattr.Get(spacePath, spaceIDAttrName)
|
||||
if err != nil || len(spaceID) == 0 {
|
||||
logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, spaceIDAttrName)
|
||||
return
|
||||
}
|
||||
|
||||
checkSpaceID(spacePath)
|
||||
checkNodeIDs(spacePath)
|
||||
}
|
||||
|
||||
func checkSpaceID(spacePath string) {
|
||||
spinner.Message(" - checking space ID uniqueness")
|
||||
|
||||
entries, uniqueIDs, oldestEntry, err := gatherAttributes(spacePath)
|
||||
if err != nil {
|
||||
logFailure("Failed to gather attributes: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if len(uniqueIDs) > 1 {
|
||||
spinner.Pause()
|
||||
fmt.Println("\n ⚠ Multiple space IDs found:")
|
||||
for id := range uniqueIDs {
|
||||
fmt.Printf(" - %s\n", id)
|
||||
}
|
||||
|
||||
fmt.Printf("\n ⏳ Oldest entry is '%s' (modified on %s).\n",
|
||||
filepath.Base(oldestEntry.Path), oldestEntry.ModTime.Format(time.RFC1123))
|
||||
|
||||
targetID := oldestEntry.ParentID
|
||||
fmt.Printf(" ✅ Proposed target Parent ID: %s\n", targetID)
|
||||
|
||||
fmt.Printf("\n Do you want to unify all parent IDs to '%s'? This will modify %d entries, the directory, and the user index. (y/N): ", targetID, len(entries))
|
||||
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
input, _ := reader.ReadString('\n')
|
||||
input = strings.TrimSpace(strings.ToLower(input))
|
||||
|
||||
if input != "y" {
|
||||
spinner.Unpause()
|
||||
logFailure("Operation cancelled by user.")
|
||||
return
|
||||
}
|
||||
restartRequired = true
|
||||
|
||||
obsoleteIDs := []string{}
|
||||
for id := range uniqueIDs {
|
||||
if id != targetID {
|
||||
obsoleteIDs = append(obsoleteIDs, id)
|
||||
}
|
||||
}
|
||||
fixSpaceID(spacePath, obsoleteIDs, targetID, entries)
|
||||
spinner.Unpause()
|
||||
}
|
||||
}
|
||||
|
||||
func walkParentIDs(dir string, parentID string) int {
|
||||
fixes := 0
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
logFailure("Error reading directory '%s': %v", dir, err)
|
||||
return 0
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
fullPath := filepath.Join(dir, entry.Name())
|
||||
|
||||
if ignorer.IsIgnored(fullPath) {
|
||||
continue
|
||||
}
|
||||
|
||||
actualParentID, err := xattr.Get(fullPath, parentIDAttrName)
|
||||
if err != nil || string(actualParentID) != parentID {
|
||||
err = xattr.Set(fullPath, parentIDAttrName, []byte(parentID))
|
||||
if err != nil {
|
||||
logFailure("Failed to fix parent ID for '%s': %v", fullPath, err)
|
||||
} else {
|
||||
spinner.Pause()
|
||||
fmt.Printf("\n + Fixed parent ID for '%s'\n", fullPath)
|
||||
spinner.Unpause()
|
||||
fixes++
|
||||
restartRequired = true
|
||||
}
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
nodeID, err := xattr.Get(fullPath, idAttrName)
|
||||
if err != nil || len(nodeID) == 0 {
|
||||
logFailure("Directory '%s' missing '%s', skipping its children", fullPath, idAttrName)
|
||||
continue
|
||||
}
|
||||
walkParentIDs(fullPath, string(nodeID))
|
||||
}
|
||||
}
|
||||
return fixes
|
||||
}
|
||||
|
||||
func checkNodeIDs(spacePath string) {
|
||||
spinner.Message(" - checking parent IDs")
|
||||
|
||||
rootID, err := xattr.Get(spacePath, idAttrName)
|
||||
if err != nil || len(rootID) == 0 {
|
||||
logFailure("Space root '%s' missing '%s' attribute", spacePath, idAttrName)
|
||||
return
|
||||
}
|
||||
|
||||
fixes := walkParentIDs(spacePath, string(rootID))
|
||||
|
||||
if fixes > 0 {
|
||||
spinner.Pause()
|
||||
fmt.Printf("\n ✓ Fixed %d incorrect parent IDs in %s\n", fixes, filepath.Base(spacePath))
|
||||
spinner.Unpause()
|
||||
}
|
||||
}
|
||||
|
||||
func fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) {
|
||||
// Set all parentid attributes to the proper space ID
|
||||
err := setAllParentIDAttributes(entries, targetID)
|
||||
if err != nil {
|
||||
logFailure("an error occurred during file attribute update: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update space ID itself
|
||||
fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), idAttrName, targetID)
|
||||
err = xattr.Set(spacePath, idAttrName, []byte(targetID))
|
||||
if err != nil {
|
||||
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
err = xattr.Set(spacePath, spaceIDAttrName, []byte(targetID))
|
||||
if err != nil {
|
||||
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
|
||||
// update the index
|
||||
err = updateOwnerIndexFile(spacePath, obsoleteIDs)
|
||||
if err != nil {
|
||||
logFailure("Could not update the owner index file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, error) {
|
||||
dirEntries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return nil, nil, EntryInfo{}, fmt.Errorf("failed to read directory: %w", err)
|
||||
}
|
||||
|
||||
var allEntries []EntryInfo
|
||||
uniqueIDs := make(map[string]struct{})
|
||||
var oldestEntry EntryInfo
|
||||
oldestTime := time.Now().Add(100 * 365 * 24 * time.Hour) // Set to a future date to find the oldest entry
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
fullPath := filepath.Join(path, entry.Name())
|
||||
if ignorer.IsIgnored(fullPath) {
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
fmt.Printf(" - Warning: could not stat %s: %v\n", entry.Name(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
parentID, err := xattr.Get(fullPath, parentIDAttrName)
|
||||
if err != nil {
|
||||
continue // Skip if attribute doesn't exist or can't be read
|
||||
}
|
||||
|
||||
entryInfo := EntryInfo{
|
||||
Path: fullPath,
|
||||
ModTime: info.ModTime(),
|
||||
ParentID: string(parentID),
|
||||
}
|
||||
|
||||
allEntries = append(allEntries, entryInfo)
|
||||
uniqueIDs[string(parentID)] = struct{}{}
|
||||
|
||||
if entryInfo.ModTime.Before(oldestTime) {
|
||||
oldestTime = entryInfo.ModTime
|
||||
oldestEntry = entryInfo
|
||||
}
|
||||
}
|
||||
|
||||
return allEntries, uniqueIDs, oldestEntry, nil
|
||||
}
|
||||
|
||||
func setAllParentIDAttributes(entries []EntryInfo, targetID string) error {
|
||||
fmt.Printf(" Setting all parent IDs to '%s':\n", targetID)
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.ParentID == targetID {
|
||||
fmt.Printf(" - Skipping '%s' (already has target ID).\n", filepath.Base(entry.Path))
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Printf(" - Removing all attributes from '%s'. It will be re-assimilated\n", filepath.Base(entry.Path))
|
||||
filepath.WalkDir(entry.Path, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking path '%s': %w", path, err)
|
||||
}
|
||||
|
||||
// Remove all attributes from the file.
|
||||
if err := removeAttributes(path); err != nil {
|
||||
fmt.Printf("failed to remove attributes from '%s': %v", path, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file.
|
||||
func updateOwnerIndexFile(basePath string, obsoleteIDs []string) error {
|
||||
fmt.Printf(" Rewriting index file '%s'\n", basePath)
|
||||
|
||||
ownerID, err := xattr.Get(basePath, ownerIDAttrName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err)
|
||||
}
|
||||
|
||||
indexPath := filepath.Join(basePath, "../../indexes/by-user-id", string(ownerID)+".mpk")
|
||||
indexPath = filepath.Clean(indexPath)
|
||||
|
||||
// Read the MessagePack file
|
||||
fileData, err := os.ReadFile(indexPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("index file does not exist, skipping update")
|
||||
}
|
||||
return fmt.Errorf("could not read index file: %w", err)
|
||||
}
|
||||
var indexMap map[string]string
|
||||
if err := msgpack.Unmarshal(fileData, &indexMap); err != nil {
|
||||
return fmt.Errorf("failed to parse MessagePack index file (is it corrupt?): %w", err)
|
||||
}
|
||||
|
||||
// Remove obsolete IDs from the map
|
||||
itemsRemoved := 0
|
||||
for _, id := range obsoleteIDs {
|
||||
if _, exists := indexMap[id]; exists {
|
||||
fmt.Printf(" - Removing obsolete ID '%s' from index.\n", id)
|
||||
delete(indexMap, id)
|
||||
itemsRemoved++
|
||||
} else {
|
||||
fmt.Printf(" - Obsolete ID '%s' not found in index\n", id)
|
||||
}
|
||||
}
|
||||
|
||||
if itemsRemoved == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write the data back to the file
|
||||
updatedData, err := msgpack.Marshal(&indexMap)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal updated index map: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(indexPath, updatedData, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write updated index file: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf(" ✓ Successfully removed %d item(s) and saved index file.\n", itemsRemoved)
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeAttributes(path string) error {
|
||||
attrNames, err := xattr.List(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list attributes for '%s': %w", path, err)
|
||||
}
|
||||
|
||||
for _, attrName := range attrNames {
|
||||
if err := xattr.Remove(path, attrName); err != nil {
|
||||
return fmt.Errorf("failed to remove attribute '%s' from '%s': %w", attrName, path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func logFailure(message string, args ...any) {
|
||||
spinner.StopFailMessage(fmt.Sprintf("\n"+message, args...))
|
||||
spinner.StopFail()
|
||||
spinner.Start()
|
||||
fmt.Fprintf(os.Stderr, message+"\n", args...)
|
||||
}
|
||||
|
||||
// findStorageRoot walks up the directory tree starting at path until it finds a
|
||||
// directory that contains an "indexes" subdirectory which marks the root of a
|
||||
// posixfs storage. A user directory inside a space might also be named "indexes",
|
||||
// so to disambiguate we require that the "indexes" directory is an internal
|
||||
// directory: the storage's own indexes directory is skipped during assimilation
|
||||
// and therefore never receives a node ID attribute, whereas a regular user
|
||||
// directory would have one.
|
||||
func findStorageRoot(path string) (string, error) {
|
||||
current := path
|
||||
for {
|
||||
indexesPath := filepath.Join(current, "indexes")
|
||||
if info, err := os.Stat(indexesPath); err == nil && info.IsDir() {
|
||||
if id, err := xattr.Get(indexesPath, prefixes.IDAttr); err != nil || len(id) == 0 {
|
||||
return current, nil
|
||||
}
|
||||
}
|
||||
|
||||
parent := filepath.Dir(current)
|
||||
if parent == current {
|
||||
return "", fmt.Errorf("'%s' does not appear to be inside a posixfs storage (no 'indexes' directory found)", path)
|
||||
}
|
||||
current = parent
|
||||
}
|
||||
}
|
||||
|
||||
// isSpaceRoot reports whether the given path is a space root, which is
|
||||
// identified by the presence of the space ID attribute.
|
||||
func isSpaceRoot(path string) bool {
|
||||
spaceID, err := xattr.Get(path, prefixes.SpaceIDAttr)
|
||||
return err == nil && len(spaceID) > 0
|
||||
}
|
||||
|
||||
// iterates over a list of paths and processes them all, using the appropriate function
|
||||
// depending on the type of resource
|
||||
//
|
||||
// note that whenever an error occurs, it collects that error and continues processing
|
||||
// subsequent paths, and then returns a slice of errors at the end (or an empty slice
|
||||
// if no errors occured)
|
||||
func processPosixFsResources(paths []string,
|
||||
keepGoing bool,
|
||||
personalSpaceDir func(string) error,
|
||||
projectSpaceDir func(string) error,
|
||||
spaceRoot func(string) error,
|
||||
entity func(string) error,
|
||||
) []error {
|
||||
// no need to guard this with a mutex for now, since the implementation is not parallelized
|
||||
errors := []error{}
|
||||
|
||||
for _, path := range paths {
|
||||
rootPath, err := findStorageRoot(path)
|
||||
if err != nil {
|
||||
errors = append(errors, err)
|
||||
logFailure("error: %s", err)
|
||||
if keepGoing {
|
||||
continue
|
||||
} else {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
path = filepath.Clean(path)
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
errors = append(errors, err)
|
||||
logFailure("error accessing '%s': %w", path, err)
|
||||
if keepGoing {
|
||||
continue
|
||||
} else {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
contained, _ := filepathx.IsSameOrContainedBy(rootPath, path)
|
||||
|
||||
switch {
|
||||
case path == rootPath:
|
||||
if err := personalSpaceDir(filepath.Join(path, "users")); err != nil {
|
||||
errors = append(errors, err)
|
||||
if !keepGoing {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
if err := projectSpaceDir(filepath.Join(path, "projects")); err != nil {
|
||||
errors = append(errors, err)
|
||||
if !keepGoing {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
case isSpaceRoot(path):
|
||||
if err := spaceRoot(path); err != nil {
|
||||
errors = append(errors, err)
|
||||
if !keepGoing {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
case contained:
|
||||
if err := entity(path); err != nil {
|
||||
errors = append(errors, err)
|
||||
if !keepGoing {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
default:
|
||||
err := fmt.Errorf("error: the provided path '%s' is neither a space root nor contained by the storage root '%s'", path, rootPath)
|
||||
errors = append(errors, err)
|
||||
logFailure(err.Error())
|
||||
if !keepGoing {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors
|
||||
}
|
||||
@@ -0,0 +1,489 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package command
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
storageUsersConfig "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/ignore"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/shamaton/msgpack/v2"
|
||||
)
|
||||
|
||||
type consistencyChecker struct {
|
||||
cfg *storageUsersConfig.Config
|
||||
ignorer *ignore.Ignorer
|
||||
recalculateChecksums bool
|
||||
|
||||
restartRequired bool
|
||||
}
|
||||
|
||||
// checkPosixfsConsistency checks the consistency of the posixfs storage. The
|
||||
// given path determines the scope of the check: the whole storage, a single
|
||||
// space or a single entity within a space.
|
||||
func (c *consistencyChecker) Check(paths []string) error {
|
||||
_ = processPosixFsResources(paths, true,
|
||||
func(path string) error {
|
||||
fmt.Println("Checking personal spaces...")
|
||||
c.checkSpaces(path)
|
||||
return nil
|
||||
},
|
||||
func(path string) error {
|
||||
fmt.Println("Checking project spaces...")
|
||||
c.checkSpaces(path)
|
||||
return nil
|
||||
},
|
||||
func(path string) error {
|
||||
fmt.Printf("Checking space '%s'...\n", path)
|
||||
c.checkSpace(path)
|
||||
return nil
|
||||
},
|
||||
func(path string) error {
|
||||
if c.ignorer.IsIgnored(path) {
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("Checking '%s'...\n", path)
|
||||
c.checkEntity(path)
|
||||
return nil
|
||||
},
|
||||
)
|
||||
if c.restartRequired {
|
||||
fmt.Println("\n\n ⚠️ Please restart your openCloud instance to apply changes.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) checkSpaces(basePath string) {
|
||||
dirEntries, err := os.ReadDir(basePath)
|
||||
if err != nil {
|
||||
logFailure("Error reading spaces directory '%s': %v", basePath, err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
if entry.IsDir() {
|
||||
fullPath := filepath.Join(basePath, entry.Name())
|
||||
c.checkSpace(fullPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) checkSpace(spacePath string) {
|
||||
info, err := os.Stat(spacePath)
|
||||
if err != nil {
|
||||
logFailure("Error accessing path '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
if !info.IsDir() {
|
||||
logFailure("Error: The provided path '%s' is not a directory\n", spacePath)
|
||||
return
|
||||
}
|
||||
|
||||
spaceID, err := xattr.Get(spacePath, prefixes.SpaceIDAttr)
|
||||
if err != nil || len(spaceID) == 0 {
|
||||
logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, prefixes.SpaceIDAttr)
|
||||
return
|
||||
}
|
||||
|
||||
c.checkSpaceID(spacePath)
|
||||
c.checkNodes(spacePath)
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) checkSpaceID(spacePath string) {
|
||||
entries, uniqueIDs, oldestEntry, err := c.gatherAttributes(spacePath)
|
||||
if err != nil {
|
||||
logFailure("Failed to gather attributes: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if len(uniqueIDs) > 1 {
|
||||
fmt.Println("\n ⚠ Multiple space IDs found:")
|
||||
for id := range uniqueIDs {
|
||||
fmt.Printf(" - %s\n", id)
|
||||
}
|
||||
|
||||
fmt.Printf("\n ⏳ Oldest entry is '%s' (modified on %s).\n",
|
||||
filepath.Base(oldestEntry.Path), oldestEntry.ModTime.Format(time.RFC1123))
|
||||
|
||||
targetID := oldestEntry.ParentID
|
||||
fmt.Printf(" ✅ Proposed target Parent ID: %s\n", targetID)
|
||||
|
||||
fmt.Printf("\n Do you want to unify all parent IDs to '%s'? This will modify %d entries, the directory, and the user index. (y/N): ", targetID, len(entries))
|
||||
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
input, _ := reader.ReadString('\n')
|
||||
input = strings.TrimSpace(strings.ToLower(input))
|
||||
|
||||
if input != "y" {
|
||||
logFailure("Operation cancelled by user.")
|
||||
return
|
||||
}
|
||||
c.restartRequired = true
|
||||
|
||||
obsoleteIDs := []string{}
|
||||
for id := range uniqueIDs {
|
||||
if id != targetID {
|
||||
obsoleteIDs = append(obsoleteIDs, id)
|
||||
}
|
||||
}
|
||||
c.fixSpaceID(spacePath, obsoleteIDs, targetID, entries)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) walkNodes(dir string, parentID string) int {
|
||||
fixes := 0
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
logFailure("Error reading directory '%s': %v", dir, err)
|
||||
return 0
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
fullPath := filepath.Join(dir, entry.Name())
|
||||
|
||||
if c.ignorer.IsIgnored(fullPath) {
|
||||
continue
|
||||
}
|
||||
|
||||
fixes += c.checkNodeAttributes(fullPath, entry.Name(), parentID, entry.IsDir())
|
||||
|
||||
if entry.IsDir() {
|
||||
nodeID, err := xattr.Get(fullPath, prefixes.IDAttr)
|
||||
if err != nil || len(nodeID) == 0 {
|
||||
logFailure("Directory '%s' missing '%s', skipping its children", fullPath, prefixes.IDAttr)
|
||||
continue
|
||||
}
|
||||
fixes += c.walkNodes(fullPath, string(nodeID))
|
||||
}
|
||||
}
|
||||
return fixes
|
||||
}
|
||||
|
||||
// checkNodeAttributes checks and fixes the parent ID and name attributes of a
|
||||
// single node. For files it additionally checks the blobsize and, when
|
||||
// requested, the checksums. It returns the number of fixes applied.
|
||||
func (c *consistencyChecker) checkNodeAttributes(path, name, parentID string, isDir bool) int {
|
||||
fixes := 0
|
||||
|
||||
// Check if the parent ID attribute matches the expected parent ID, if not, fix it.
|
||||
actualParentID, err := xattr.Get(path, prefixes.ParentidAttr)
|
||||
if err != nil || string(actualParentID) != parentID {
|
||||
if err := xattr.Set(path, prefixes.ParentidAttr, []byte(parentID)); err != nil {
|
||||
logFailure("Failed to fix parent ID for '%s': %v", path, err)
|
||||
} else {
|
||||
fmt.Printf(" + Fixed parent ID for '%s'\n", path)
|
||||
fixes++
|
||||
c.restartRequired = true
|
||||
}
|
||||
}
|
||||
|
||||
// Check that the name attribute matches the actual name of the file/directory, if not, fix it.
|
||||
nameAttr, err := xattr.Get(path, prefixes.NameAttr)
|
||||
if err != nil || string(nameAttr) != name {
|
||||
if err := xattr.Set(path, prefixes.NameAttr, []byte(name)); err != nil {
|
||||
logFailure("Failed to fix name attribute for '%s': %v", path, err)
|
||||
} else {
|
||||
fmt.Printf(" + Fixed name attribute for '%s'\n", path)
|
||||
fixes++
|
||||
c.restartRequired = true
|
||||
}
|
||||
}
|
||||
|
||||
if !isDir {
|
||||
fixes += c.checkBlobsize(path)
|
||||
if c.recalculateChecksums {
|
||||
fixes += c.fixChecksums(path)
|
||||
}
|
||||
}
|
||||
|
||||
return fixes
|
||||
}
|
||||
|
||||
// checkEntity checks a single file or directory within a space, including its own
|
||||
// parent ID, name and (for files) blobsize/checksums. If the entity is a directory
|
||||
// its children are checked recursively.
|
||||
func (c *consistencyChecker) checkEntity(path string) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
logFailure("Error accessing path '%s': %v", path, err)
|
||||
return
|
||||
}
|
||||
|
||||
// The expected parent ID is the ID attribute of the containing directory.
|
||||
parentDir := filepath.Dir(path)
|
||||
parentID, err := xattr.Get(parentDir, prefixes.IDAttr)
|
||||
if err != nil || len(parentID) == 0 {
|
||||
logFailure("Parent directory '%s' is missing the '%s' attribute", parentDir, prefixes.IDAttr)
|
||||
return
|
||||
}
|
||||
|
||||
fixes := c.checkNodeAttributes(path, info.Name(), string(parentID), info.IsDir())
|
||||
|
||||
if info.IsDir() {
|
||||
nodeID, err := xattr.Get(path, prefixes.IDAttr)
|
||||
if err != nil || len(nodeID) == 0 {
|
||||
logFailure("Directory '%s' missing '%s' attribute", path, prefixes.IDAttr)
|
||||
} else {
|
||||
fixes += c.walkNodes(path, string(nodeID))
|
||||
}
|
||||
}
|
||||
|
||||
if fixes > 0 {
|
||||
fmt.Printf(" ✓ Fixed %d incorrect node attributes for %s\n", fixes, filepath.Base(path))
|
||||
}
|
||||
}
|
||||
|
||||
// checkBlobsize verifies that the stored blobsize attribute matches the actual
|
||||
// file size and fixes it if it doesn't. It returns the number of fixes applied.
|
||||
func (c *consistencyChecker) checkBlobsize(path string) int {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
logFailure("Error accessing file '%s': %v", path, err)
|
||||
return 0
|
||||
}
|
||||
|
||||
expectedSize := strconv.FormatInt(info.Size(), 10)
|
||||
blobsize, err := xattr.Get(path, prefixes.BlobsizeAttr)
|
||||
if err == nil && string(blobsize) == expectedSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
if err := xattr.Set(path, prefixes.BlobsizeAttr, []byte(expectedSize)); err != nil {
|
||||
logFailure("Failed to fix blobsize for '%s': %v", path, err)
|
||||
return 0
|
||||
}
|
||||
|
||||
fmt.Printf(" + Fixed blobsize for '%s'\n", path)
|
||||
c.restartRequired = true
|
||||
return 1
|
||||
}
|
||||
|
||||
// fixChecksums recalculates the sha1, md5 and adler32 checksums of the file and
|
||||
// updates the stored attributes if they differ. It returns the number of fixes applied.
|
||||
func (c *consistencyChecker) fixChecksums(path string) int {
|
||||
sha1h, md5h, adler32h, err := node.CalculateChecksums(context.Background(), path)
|
||||
if err != nil {
|
||||
logFailure("Failed to calculate checksums for '%s': %v", path, err)
|
||||
return 0
|
||||
}
|
||||
|
||||
checksums := map[string][]byte{
|
||||
prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil),
|
||||
prefixes.ChecksumPrefix + "md5": md5h.Sum(nil),
|
||||
prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil),
|
||||
}
|
||||
|
||||
fixes := 0
|
||||
for attrName, sum := range checksums {
|
||||
current, err := xattr.Get(path, attrName)
|
||||
if err == nil && bytes.Equal(current, sum) {
|
||||
continue
|
||||
}
|
||||
if err := xattr.Set(path, attrName, sum); err != nil {
|
||||
logFailure("Failed to fix checksum '%s' for '%s': %v", attrName, path, err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" + Fixed checksum '%s' for '%s'\n", attrName, path)
|
||||
c.restartRequired = true
|
||||
fixes++
|
||||
}
|
||||
return fixes
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) checkNodes(spacePath string) {
|
||||
rootID, err := xattr.Get(spacePath, prefixes.IDAttr)
|
||||
if err != nil || len(rootID) == 0 {
|
||||
logFailure("Space root '%s' missing '%s' attribute", spacePath, prefixes.IDAttr)
|
||||
return
|
||||
}
|
||||
|
||||
fixes := c.walkNodes(spacePath, string(rootID))
|
||||
|
||||
if fixes > 0 {
|
||||
fmt.Printf(" ✓ Fixed %d incorrect node attributes in %s\n", fixes, filepath.Base(spacePath))
|
||||
}
|
||||
}
|
||||
|
||||
// fixSpaceID updates the parentid attributes of all entries in a space to a new target ID,
|
||||
// updates the space's own ID attributes, and removes obsolete IDs from the user index file.
|
||||
func (c *consistencyChecker) fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) {
|
||||
// Set all parentid attributes to the proper space ID
|
||||
err := setAllParentIDAttributes(entries, targetID)
|
||||
if err != nil {
|
||||
logFailure("an error occurred during file attribute update: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update space ID itself
|
||||
fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), prefixes.IDAttr, targetID)
|
||||
err = xattr.Set(spacePath, prefixes.IDAttr, []byte(targetID))
|
||||
if err != nil {
|
||||
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
err = xattr.Set(spacePath, prefixes.SpaceIDAttr, []byte(targetID))
|
||||
if err != nil {
|
||||
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
|
||||
// update the index
|
||||
err = c.updateOwnerIndexFile(spacePath, obsoleteIDs)
|
||||
if err != nil {
|
||||
logFailure("Could not update the owner index file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, error) {
|
||||
dirEntries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return nil, nil, EntryInfo{}, fmt.Errorf("failed to read directory: %w", err)
|
||||
}
|
||||
|
||||
var allEntries []EntryInfo
|
||||
uniqueIDs := make(map[string]struct{})
|
||||
var oldestEntry EntryInfo
|
||||
oldestTime := time.Now().Add(100 * 365 * 24 * time.Hour) // Set to a future date to find the oldest entry
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
fullPath := filepath.Join(path, entry.Name())
|
||||
if c.ignorer.IsIgnored(fullPath) {
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
fmt.Printf(" - Warning: could not stat %s: %v\n", entry.Name(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
parentID, err := xattr.Get(fullPath, prefixes.ParentidAttr)
|
||||
if err != nil {
|
||||
continue // Skip if attribute doesn't exist or can't be read
|
||||
}
|
||||
|
||||
entryInfo := EntryInfo{
|
||||
Path: fullPath,
|
||||
ModTime: info.ModTime(),
|
||||
ParentID: string(parentID),
|
||||
}
|
||||
|
||||
allEntries = append(allEntries, entryInfo)
|
||||
uniqueIDs[string(parentID)] = struct{}{}
|
||||
|
||||
if entryInfo.ModTime.Before(oldestTime) {
|
||||
oldestTime = entryInfo.ModTime
|
||||
oldestEntry = entryInfo
|
||||
}
|
||||
}
|
||||
|
||||
return allEntries, uniqueIDs, oldestEntry, nil
|
||||
}
|
||||
|
||||
func setAllParentIDAttributes(entries []EntryInfo, targetID string) error {
|
||||
fmt.Printf(" Setting all parent IDs to '%s':\n", targetID)
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.ParentID == targetID {
|
||||
fmt.Printf(" - Skipping '%s' (already has target ID).\n", filepath.Base(entry.Path))
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Printf(" - Removing all attributes from '%s'. It will be re-assimilated\n", filepath.Base(entry.Path))
|
||||
filepath.WalkDir(entry.Path, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking path '%s': %w", path, err)
|
||||
}
|
||||
|
||||
// Remove all attributes from the file.
|
||||
if err := removeAttributes(path); err != nil {
|
||||
fmt.Printf("failed to remove attributes from '%s': %v", path, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file.
|
||||
func (c *consistencyChecker) updateOwnerIndexFile(basePath string, obsoleteIDs []string) error {
|
||||
fmt.Printf(" Rewriting index file '%s'\n", basePath)
|
||||
|
||||
ownerID, err := xattr.Get(basePath, prefixes.OwnerIDAttr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err)
|
||||
}
|
||||
|
||||
indexPath := filepath.Join(c.cfg.Drivers.Posix.Root, "indexes", "by-user-id", string(ownerID)+".mpk")
|
||||
indexPath = filepath.Clean(indexPath)
|
||||
|
||||
// Read the MessagePack file
|
||||
fileData, err := os.ReadFile(indexPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("index file does not exist, skipping update")
|
||||
}
|
||||
return fmt.Errorf("could not read index file: %w", err)
|
||||
}
|
||||
var indexMap map[string]string
|
||||
if err := msgpack.Unmarshal(fileData, &indexMap); err != nil {
|
||||
return fmt.Errorf("failed to parse MessagePack index file (is it corrupt?): %w", err)
|
||||
}
|
||||
|
||||
// Remove obsolete IDs from the map
|
||||
itemsRemoved := 0
|
||||
for _, id := range obsoleteIDs {
|
||||
if _, exists := indexMap[id]; exists {
|
||||
fmt.Printf(" - Removing obsolete ID '%s' from index.\n", id)
|
||||
delete(indexMap, id)
|
||||
itemsRemoved++
|
||||
} else {
|
||||
fmt.Printf(" - Obsolete ID '%s' not found in index\n", id)
|
||||
}
|
||||
}
|
||||
|
||||
if itemsRemoved == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write the data back to the file
|
||||
updatedData, err := msgpack.Marshal(&indexMap)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal updated index map: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(indexPath, updatedData, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write updated index file: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf(" ✓ Successfully removed %d item(s) and saved index file.\n", itemsRemoved)
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeAttributes(path string) error {
|
||||
attrNames, err := xattr.List(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list attributes for '%s': %w", path, err)
|
||||
}
|
||||
|
||||
for _, attrName := range attrNames {
|
||||
if err := xattr.Remove(path, attrName); err != nil {
|
||||
return fmt.Errorf("failed to remove attribute '%s' from '%s': %w", attrName, path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -6,11 +6,13 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/opencloud-eu/opencloud/pkg/clihelper"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
oclog "github.com/opencloud-eu/opencloud/pkg/log"
|
||||
)
|
||||
|
||||
// Execute is the entry point for the opencloud command.
|
||||
@@ -38,3 +40,11 @@ func Execute() error {
|
||||
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP)
|
||||
return app.ExecuteContext(ctx)
|
||||
}
|
||||
|
||||
func logger(name string) zerolog.Logger {
|
||||
return oclog.NewLogger(
|
||||
oclog.Name(name),
|
||||
oclog.Level("info"),
|
||||
oclog.Pretty(true),
|
||||
oclog.Color(true)).Logger
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/parser"
|
||||
oclog "github.com/opencloud-eu/opencloud/pkg/log"
|
||||
mregistry "github.com/opencloud-eu/opencloud/pkg/registry"
|
||||
sharing "github.com/opencloud-eu/opencloud/services/sharing/pkg/config"
|
||||
sharingparser "github.com/opencloud-eu/opencloud/services/sharing/pkg/config/parser"
|
||||
@@ -85,7 +84,7 @@ func cleanup(_ *cobra.Command, cfg *config.Config) error {
|
||||
return configlog.ReturnError(errors.New("cleanup is only implemented for the jsoncs3 share manager"))
|
||||
}
|
||||
|
||||
l := logger()
|
||||
l := logger("migrate")
|
||||
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
|
||||
@@ -94,7 +93,7 @@ func cleanup(_ *cobra.Command, cfg *config.Config) error {
|
||||
if !ok {
|
||||
return configlog.ReturnError(errors.New("Unknown share manager type '" + driver + "'"))
|
||||
}
|
||||
mgr, err := f(rcfg[driver].(map[string]any), l)
|
||||
mgr, err := f(rcfg[driver].(map[string]any), &l)
|
||||
if err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
@@ -167,12 +166,3 @@ func revaShareConfig(cfg *sharing.Config) map[string]any {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func logger() *zerolog.Logger {
|
||||
log := oclog.NewLogger(
|
||||
oclog.Name("migrate"),
|
||||
oclog.Level("info"),
|
||||
oclog.Pretty(true),
|
||||
oclog.Color(true)).Logger
|
||||
return &log
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
func Secure(enableTLS, insecure bool, rootCA string) nats.Option {
|
||||
if enableTLS {
|
||||
if rootCA != "" {
|
||||
return nats.RootCAs(rootCA)
|
||||
}
|
||||
return nats.Secure(&tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: insecure,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -48,14 +48,17 @@ type HTTPServiceTLS struct {
|
||||
}
|
||||
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"OC_CACHE_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"OC_CACHE_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h." introductionVersion:"1.0.0"`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"auth_username" env:"OC_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"auth_password" env:"OC_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"OC_CACHE_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"OC_CACHE_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h." introductionVersion:"1.0.0"`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"auth_username" env:"OC_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"auth_password" env:"OC_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_CACHE_ENABLE_TLS" desc:"Enable TLS for the connection to file metadata cache." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_CACHE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_CACHE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided OC_CACHE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
|
||||
// Commons holds configuration that are common to all extensions. Each extension can then decide whether
|
||||
|
||||
@@ -34,7 +34,7 @@ var (
|
||||
// LatestTag is the latest released version plus the dev meta version.
|
||||
// Will be overwritten by the release pipeline
|
||||
// Needs a manual change for every tagged release
|
||||
LatestTag = "7.1.0+dev"
|
||||
LatestTag = "7.2.4+dev"
|
||||
|
||||
// Date indicates the build date.
|
||||
// This has been removed, it looks like you can only replace static strings with recent go versions
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package filepathx
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// JailJoin joins any number of path elements into a single path,
|
||||
@@ -10,3 +12,27 @@ import (
|
||||
func JailJoin(jail string, elem ...string) string {
|
||||
return filepath.Join(jail, filepath.Join(append([]string{"/"}, elem...)...))
|
||||
}
|
||||
|
||||
// Determines whether the file or directory 'child' is same as or underneath the directory 'parent'.
|
||||
//
|
||||
// Note that 'parent' is expected to be a directory.
|
||||
func IsSameOrContainedBy(parent string, child string) (bool, error) {
|
||||
absParent, err := filepath.Abs(parent)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to make parent directory absolute: %q: %w", parent, err)
|
||||
}
|
||||
|
||||
absChild, err := filepath.Abs(child)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to make child file/directory absolute: %q: %w", child, err)
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(absParent, absChild)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to determine the relative path between the parent directory %q and the child file/directory: %q: %w", absParent, absChild, err)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -1,9 +1,12 @@
|
||||
package filepathx_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/x/path/filepathx"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestJailJoin(t *testing.T) {
|
||||
@@ -61,3 +64,23 @@ func TestJailJoin(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSameOrContainedBy(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
parent string
|
||||
child string
|
||||
expected bool
|
||||
}{
|
||||
{"foo", "foo", true},
|
||||
{"/foo", "/foo", true},
|
||||
{"foo", "foo/bar", true},
|
||||
{"foo", "bar", false},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%s: %s vs %s", t.Name(), strings.ReplaceAll(tt.parent, "/", "."), strings.ReplaceAll(tt.child, "/", ".")), func(t *testing.T) {
|
||||
require := require.New(t)
|
||||
b, err := filepathx.IsSameOrContainedBy(tt.parent, tt.child)
|
||||
require.NoError(err)
|
||||
require.Equal(tt.expected, b)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -9,9 +9,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/stream"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/store"
|
||||
"github.com/spf13/cobra"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/pkg/generators"
|
||||
@@ -77,15 +75,6 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
return err
|
||||
}
|
||||
|
||||
evStore := store.Create(
|
||||
store.Store(cfg.Store.Store),
|
||||
store.TTL(cfg.Store.TTL),
|
||||
microstore.Nodes(cfg.Store.Nodes...),
|
||||
microstore.Database(cfg.Store.Database),
|
||||
microstore.Table(cfg.Store.Table),
|
||||
store.Authentication(cfg.Store.AuthUsername, cfg.Store.AuthPassword),
|
||||
)
|
||||
|
||||
tm, err := pool.StringToTLSMode(cfg.GRPCClientTLS.Mode)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to parse tls mode")
|
||||
@@ -120,7 +109,6 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
http.Context(ctx), // NOTE: not passing this "option" leads to a panic in go-micro
|
||||
http.TraceProvider(tracerProvider),
|
||||
http.Stream(evStream),
|
||||
http.Store(evStore),
|
||||
http.GatewaySelector(gatewaySelector),
|
||||
http.HistoryClient(hClient),
|
||||
http.ValueClient(vClient),
|
||||
|
||||
@@ -50,13 +50,15 @@ type Events struct {
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;ACTIVITYLOG_STORE" desc:"The type of the store. Supported values are: 'memory', 'nats-js-kv', 'redis-sentinel', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;ACTIVITYLOG_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"ACTIVITYLOG_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"ACTIVITYLOG_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;ACTIVITYLOG_STORE_TTL" desc:"Time to live for events in the store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;ACTIVITYLOG_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;ACTIVITYLOG_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;ACTIVITYLOG_STORE" desc:"The type of the store. Supported values are: 'memory', 'nats-js-kv', 'redis-sentinel', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;ACTIVITYLOG_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"ACTIVITYLOG_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;ACTIVITYLOG_STORE_TTL" desc:"Time to live for events in the store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;ACTIVITYLOG_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;ACTIVITYLOG_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_PERSISTENT_STORE_ENABLE_TLS;ACTIVITYLOG_STORE_ENABLE_TLS" desc:"Enable TLS for the connection to the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_PERSISTENT_STORE_TLS_INSECURE;ACTIVITYLOG_STORE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_PERSISTENT_STORE_TLS_ROOT_CA_CERTIFICATE;ACTIVITYLOG_STORE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided ACTIVITYLOG_STORE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
|
||||
// ServiceAccount is the configuration for the used service account
|
||||
|
||||
@@ -37,7 +37,6 @@ func DefaultConfig() *config.Config {
|
||||
Store: "nats-js-kv",
|
||||
Nodes: []string{"127.0.0.1:9233"},
|
||||
Database: "activitylog",
|
||||
Table: "",
|
||||
},
|
||||
RevaGateway: shared.DefaultRevaConfig().Address,
|
||||
DefaultLanguage: "en",
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -17,8 +18,13 @@ func Server(opts ...Option) (*http.Server, error) {
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("http reachability", checks.NewHTTPCheck(options.Config.HTTP.Addr))
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := healthHandlerConfiguration.
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint))
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption))
|
||||
|
||||
return debug.NewService(
|
||||
debug.Logger(options.Logger),
|
||||
|
||||
@@ -81,7 +81,6 @@ func Server(opts ...Option) (http.Service, error) {
|
||||
svc.Logger(options.Logger),
|
||||
svc.Stream(options.Stream),
|
||||
svc.Mux(mux),
|
||||
svc.Store(options.Store),
|
||||
svc.Config(options.Config),
|
||||
svc.GatewaySelector(options.GatewaySelector),
|
||||
svc.TraceProvider(options.TraceProvider),
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/services/activitylog/pkg/config"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
@@ -25,7 +24,6 @@ type Options struct {
|
||||
TraceProvider trace.TracerProvider
|
||||
Stream events.Stream
|
||||
RegisteredEvents []events.Unmarshaller
|
||||
Store microstore.Store
|
||||
GatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
Mux *chi.Mux
|
||||
HistoryClient ehsvc.EventHistoryService
|
||||
@@ -69,13 +67,6 @@ func RegisteredEvents(e []events.Unmarshaller) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// Store configures the store to use
|
||||
func Store(store microstore.Store) Option {
|
||||
return func(o *Options) {
|
||||
o.Store = store
|
||||
}
|
||||
}
|
||||
|
||||
// GatewaySelector adds a grpc client selector for the gateway service
|
||||
func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option {
|
||||
return func(o *Options) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base32"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -166,6 +167,18 @@ func New(opts ...Option) (*ActivitylogService, error) {
|
||||
natsOptions := nats.Options{
|
||||
Servers: o.Config.Store.Nodes,
|
||||
}
|
||||
if o.Config.Store.EnableTLS {
|
||||
if o.Config.Store.TLSRootCACertificate != "" {
|
||||
// when root ca is configured use it. an insecure flag is ignored.
|
||||
nats.RootCAs(o.Config.Store.TLSRootCACertificate)(&natsOptions)
|
||||
} else {
|
||||
// enable tls and use insecure flag
|
||||
nats.Secure(&tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: o.Config.Store.TLSInsecure})(&natsOptions)
|
||||
}
|
||||
}
|
||||
if o.Config.Store.AuthUsername != "" && o.Config.Store.AuthPassword != "" {
|
||||
nats.UserInfo(o.Config.Store.AuthUsername, o.Config.Store.AuthPassword)(&natsOptions)
|
||||
}
|
||||
conn, err := natsOptions.Connect()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -18,9 +19,14 @@ import (
|
||||
func Server(opts ...Option) (*http.Server, error) {
|
||||
options := newOptions(opts...)
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := handlers.NewCheckHandlerConfiguration().
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint)).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption)).
|
||||
WithCheck("antivirus reachability", func(ctx context.Context) error {
|
||||
cfg := options.Config
|
||||
switch cfg.Scanner.Type {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -13,9 +14,14 @@ import (
|
||||
func Server(opts ...Option) (*http.Server, error) {
|
||||
options := newOptions(opts...)
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := handlers.NewCheckHandlerConfiguration().
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint))
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption))
|
||||
|
||||
return debug.NewService(
|
||||
debug.Logger(options.Logger),
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -13,9 +14,14 @@ import (
|
||||
func Server(opts ...Option) (*http.Server, error) {
|
||||
options := newOptions(opts...)
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := handlers.NewCheckHandlerConfiguration().
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint))
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption))
|
||||
|
||||
return debug.NewService(
|
||||
debug.Logger(options.Logger),
|
||||
|
||||
@@ -83,7 +83,8 @@ EventLoop:
|
||||
if !ok {
|
||||
break EventLoop
|
||||
}
|
||||
cl.processEvent(event)
|
||||
|
||||
go cl.processEvent(event)
|
||||
|
||||
if cl.stopped.Load() {
|
||||
break EventLoop
|
||||
|
||||
@@ -101,6 +101,9 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
microstore.Database(cfg.Store.Database),
|
||||
microstore.Table(cfg.Store.Table),
|
||||
store.Authentication(cfg.Store.AuthUsername, cfg.Store.AuthPassword),
|
||||
store.TLSEnabled(cfg.Store.EnableTLS),
|
||||
store.TLSInsecure(cfg.Store.TLSInsecure),
|
||||
store.TLSRootCA(cfg.Store.TLSRootCACertificate),
|
||||
)
|
||||
|
||||
gr := runner.NewGroup()
|
||||
|
||||
@@ -4,11 +4,14 @@ import "time"
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;COLLABORATION_STORE" desc:"The type of the store. Supported values are: 'memory', 'nats-js-kv', 'redis-sentinel', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;COLLABORATION_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"COLLABORATION_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"COLLABORATION_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;COLLABORATION_STORE_TTL" desc:"Time to live for events in the store. Defaults to '30m' (30 minutes). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;COLLABORATION_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;COLLABORATION_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;COLLABORATION_STORE" desc:"The type of the store. Supported values are: 'memory', 'nats-js-kv', 'redis-sentinel', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;COLLABORATION_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"COLLABORATION_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"COLLABORATION_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;COLLABORATION_STORE_TTL" desc:"Time to live for events in the store. Defaults to '30m' (30 minutes). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;COLLABORATION_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;COLLABORATION_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_PERSISTENT_STORE_ENABLE_TLS;COLLABORATION_STORE_ENABLE_TLS" desc:"Enable TLS for the connection to the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_PERSISTENT_STORE_TLS_INSECURE;COLLABORATION_STORE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_PERSISTENT_STORE_TLS_ROOT_CA_CERTIFICATE;COLLABORATION_STORE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided COLLABORATION_STORE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
@@ -71,6 +71,9 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
microstore.Database(cfg.Store.Database),
|
||||
microstore.Table(cfg.Store.Table),
|
||||
store.Authentication(cfg.Store.AuthUsername, cfg.Store.AuthPassword),
|
||||
store.TLSEnabled(cfg.Store.EnableTLS),
|
||||
store.TLSInsecure(cfg.Store.TLSInsecure),
|
||||
store.TLSRootCA(cfg.Store.TLSRootCACertificate),
|
||||
)
|
||||
|
||||
service := grpc.NewService(
|
||||
|
||||
@@ -36,13 +36,16 @@ type GRPCConfig struct {
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;EVENTHISTORY_STORE" desc:"The type of the store. Supported values are: 'memory', 'nats-js-kv', 'redis-sentinel', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;EVENTHISTORY_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"EVENTHISTORY_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"EVENTHISTORY_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;EVENTHISTORY_STORE_TTL" desc:"Time to live for events in the store. Defaults to '336h' (2 weeks). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;EVENTHISTORY_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;EVENTHISTORY_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;EVENTHISTORY_STORE" desc:"The type of the store. Supported values are: 'memory', 'nats-js-kv', 'redis-sentinel', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;EVENTHISTORY_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"EVENTHISTORY_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"EVENTHISTORY_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;EVENTHISTORY_STORE_TTL" desc:"Time to live for events in the store. Defaults to '336h' (2 weeks). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;EVENTHISTORY_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;EVENTHISTORY_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_PERSISTENT_STORE_ENABLE_TLS;EVENTHISTORY_STORE_ENABLE_TLS" desc:"Enable TLS for the connection to the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_PERSISTENT_STORE_TLS_INSECURE;EVENTHISTORY_STORE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_PERSISTENT_STORE_TLS_ROOT_CA_CERTIFICATE;EVENTHISTORY_STORE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided EVENTHISTORY_STORE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -17,8 +18,13 @@ func Server(opts ...Option) (*http.Server, error) {
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("grpc reachability", checks.NewGRPCCheck(options.Config.GRPC.Addr))
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := healthHandlerConfiguration.
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint))
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption))
|
||||
|
||||
return debug.NewService(
|
||||
debug.Logger(options.Logger),
|
||||
|
||||
@@ -121,26 +121,29 @@ type DataGateway struct {
|
||||
}
|
||||
|
||||
type OCS struct {
|
||||
Prefix string `yaml:"prefix" env:"FRONTEND_OCS_PREFIX" desc:"URL path prefix for the OCS service. Note that the string must not start with '/'." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
SharePrefix string `yaml:"share_prefix" env:"FRONTEND_OCS_SHARE_PREFIX" desc:"Path prefix for shares as part of a CS3 resource. Note that the path must start with '/'." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
HomeNamespace string `yaml:"home_namespace" env:"FRONTEND_OCS_PERSONAL_NAMESPACE" desc:"Home namespace identifier." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
AdditionalInfoAttribute string `yaml:"additional_info_attribute" env:"FRONTEND_OCS_ADDITIONAL_INFO_ATTRIBUTE" desc:"Additional information attribute for the user like {{.Mail}}." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheType string `yaml:"stat_cache_type" env:"OC_CACHE_STORE;FRONTEND_OCS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_STORE, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OC_CACHE_STORE_NODES;FRONTEND_OCS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_STORE_NODES, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheDatabase string `yaml:"stat_cache_database" env:"OC_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
StatCacheTable string `yaml:"stat_cache_table" env:"FRONTEND_OCS_STAT_CACHE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OC_CACHE_TTL;FRONTEND_OCS_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_TTL, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheDisablePersistence bool `yaml:"stat_cache_disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;FRONTEND_OCS_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disable persistence of the cache. Only applies when using the 'nats-js-kv' store type. Defaults to false." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_DISABLE_PERSISTENCE, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheAuthUsername string `yaml:"stat_cache_auth_username" env:"OC_CACHE_AUTH_USERNAME;FRONTEND_OCS_STAT_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when using the 'nats-js-kv' store type." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_AUTH_USERNAME, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheAuthPassword string `yaml:"stat_cache_auth_password" env:"OC_CACHE_AUTH_PASSWORD;FRONTEND_OCS_STAT_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when using the 'nats-js-kv' store type." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_AUTH_PASSWORD, the OCS API is deprecated" deprecationReplacement:""`
|
||||
Prefix string `yaml:"prefix" env:"FRONTEND_OCS_PREFIX" desc:"URL path prefix for the OCS service. Note that the string must not start with '/'." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
SharePrefix string `yaml:"share_prefix" env:"FRONTEND_OCS_SHARE_PREFIX" desc:"Path prefix for shares as part of a CS3 resource. Note that the path must start with '/'." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
HomeNamespace string `yaml:"home_namespace" env:"FRONTEND_OCS_PERSONAL_NAMESPACE" desc:"Home namespace identifier." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
AdditionalInfoAttribute string `yaml:"additional_info_attribute" env:"FRONTEND_OCS_ADDITIONAL_INFO_ATTRIBUTE" desc:"Additional information attribute for the user like {{.Mail}}." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheType string `yaml:"stat_cache_type" env:"OC_CACHE_STORE;FRONTEND_OCS_STAT_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_STORE, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheNodes []string `yaml:"stat_cache_nodes" env:"OC_CACHE_STORE_NODES;FRONTEND_OCS_STAT_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_STORE_NODES, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheDatabase string `yaml:"stat_cache_database" env:"OC_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
StatCacheTable string `yaml:"stat_cache_table" env:"FRONTEND_OCS_STAT_CACHE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheTTL time.Duration `yaml:"stat_cache_ttl" env:"OC_CACHE_TTL;FRONTEND_OCS_STAT_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_TTL, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheDisablePersistence bool `yaml:"stat_cache_disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;FRONTEND_OCS_STAT_CACHE_DISABLE_PERSISTENCE" desc:"Disable persistence of the cache. Only applies when using the 'nats-js-kv' store type. Defaults to false." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_DISABLE_PERSISTENCE, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheAuthUsername string `yaml:"stat_cache_auth_username" env:"OC_CACHE_AUTH_USERNAME;FRONTEND_OCS_STAT_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when using the 'nats-js-kv' store type." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_AUTH_USERNAME, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheAuthPassword string `yaml:"stat_cache_auth_password" env:"OC_CACHE_AUTH_PASSWORD;FRONTEND_OCS_STAT_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when using the 'nats-js-kv' store type." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"FRONTEND_OCS_STAT_CACHE_AUTH_PASSWORD, the OCS API is deprecated" deprecationReplacement:""`
|
||||
StatCacheEnableTLS bool `yaml:"stat_cache_enable_tls" env:"OC_CACHE_ENABLE_TLS;FRONTEND_OCS_STAT_CACHE_ENABLE_TLS" desc:"Enable TLS for the connection to file metadata cache." introductionVersion:"7.2.0"`
|
||||
StatCacheTLSInsecure bool `yaml:"stat_cache_tls_insecure" env:"OC_INSECURE;OC_CACHE_TLS_INSECURE;FRONTEND_OCS_STAT_CACHE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
StatCacheTLSRootCACertificate string `yaml:"stat_cache_tls_root_ca_certificate" env:"OC_CACHE_TLS_ROOT_CA_CERTIFICATE;FRONTEND_OCS_STAT_CACHE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided FRONTEND_OCS_STAT_CACHE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
|
||||
CacheWarmupDriver string `yaml:"cache_warmup_driver,omitempty"` // not supported by the OpenCloud product, therefore not part of docs
|
||||
CacheWarmupDrivers CacheWarmupDrivers `yaml:"cache_warmup_drivers,omitempty"` // not supported by the OpenCloud product, therefore not part of docs
|
||||
EnableDenials bool `yaml:"enable_denials" env:"FRONTEND_OCS_ENABLE_DENIALS" desc:"EXPERIMENTAL: enable the feature to deny access on folders." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
ListOCMShares bool `yaml:"list_ocm_shares" env:"OC_ENABLE_OCM;FRONTEND_OCS_LIST_OCM_SHARES" desc:"Include OCM shares when listing shares. See the OCM service documentation for more details." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"FRONTEND_OCS_LIST_OCM_SHARES, the OCS API is deprecated" deprecationReplacement:""`
|
||||
IncludeOCMSharees bool `yaml:"include_ocm_sharees" env:"OC_ENABLE_OCM;FRONTEND_OCS_INCLUDE_OCM_SHAREES" desc:"Include OCM sharees when listing sharees." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"FRONTEND_OCS_INCLUDE_OCM_SHAREES, the OCS API is deprecated" deprecationReplacement:""`
|
||||
PublicShareMustHavePassword bool `yaml:"public_sharing_share_must_have_password" env:"OC_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD;FRONTEND_OCS_PUBLIC_SHARE_MUST_HAVE_PASSWORD" desc:"Set this to true if you want to enforce passwords on all public shares." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"FRONTEND_OCS_PUBLIC_SHARE_MUST_HAVE_PASSWORD, the OCS API is deprecated" deprecationReplacement:""`
|
||||
WriteablePublicShareMustHavePassword bool `yaml:"public_sharing_writeableshare_must_have_password" env:"OC_SHARING_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD;FRONTEND_OCS_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD" desc:"Set this to true if you want to enforce passwords for writable shares. Only effective if the setting for 'passwords on all public shares' is set to false." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"%%NEXT_PRODUCTION_VERSION%%" deprecationInfo:"FRONTEND_OCS_PUBLIC_WRITABLE_SHARE_MUST_HAVE_PASSWORD, the OCS API is deprecated" deprecationReplacement:""`
|
||||
EnableDenials bool `yaml:"enable_denials" env:"FRONTEND_OCS_ENABLE_DENIALS" desc:"EXPERIMENTAL: enable the feature to deny access on folders." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"The OCS API is deprecated" deprecationReplacement:""`
|
||||
ListOCMShares bool `yaml:"list_ocm_shares" env:"OC_ENABLE_OCM;FRONTEND_OCS_LIST_OCM_SHARES" desc:"Include OCM shares when listing shares. See the OCM service documentation for more details." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"FRONTEND_OCS_LIST_OCM_SHARES, the OCS API is deprecated" deprecationReplacement:""`
|
||||
IncludeOCMSharees bool `yaml:"include_ocm_sharees" env:"OC_ENABLE_OCM;FRONTEND_OCS_INCLUDE_OCM_SHAREES" desc:"Include OCM sharees when listing sharees." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"FRONTEND_OCS_INCLUDE_OCM_SHAREES, the OCS API is deprecated" deprecationReplacement:""`
|
||||
PublicShareMustHavePassword bool `yaml:"public_sharing_share_must_have_password" env:"OC_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD;FRONTEND_OCS_PUBLIC_SHARE_MUST_HAVE_PASSWORD" desc:"Set this to true if you want to enforce passwords on all public shares." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"FRONTEND_OCS_PUBLIC_SHARE_MUST_HAVE_PASSWORD, the OCS API is deprecated" deprecationReplacement:""`
|
||||
WriteablePublicShareMustHavePassword bool `yaml:"public_sharing_writeableshare_must_have_password" env:"OC_SHARING_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD;FRONTEND_OCS_PUBLIC_WRITEABLE_SHARE_MUST_HAVE_PASSWORD" desc:"Set this to true if you want to enforce passwords for writable shares. Only effective if the setting for 'passwords on all public shares' is set to false." introductionVersion:"1.0.0" deprecationVersion:"1.0.0" removalVersion:"7.2.0" deprecationInfo:"FRONTEND_OCS_PUBLIC_WRITABLE_SHARE_MUST_HAVE_PASSWORD, the OCS API is deprecated" deprecationReplacement:""`
|
||||
ShowUserEmailInResults bool `yaml:"show_email_in_results" env:"OC_SHOW_USER_EMAIL_IN_RESULTS" desc:"Include user email addresses in responses. If absent or set to false emails will be omitted from results. Please note that admin users can always see all email addresses." introductionVersion:"1.0.0"`
|
||||
}
|
||||
|
||||
|
||||
@@ -164,14 +164,17 @@ func FrontendConfigFromStruct(cfg *config.Config, logger log.Logger) (map[string
|
||||
"share_prefix": cfg.OCS.SharePrefix,
|
||||
"home_namespace": cfg.OCS.HomeNamespace,
|
||||
"stat_cache_config": map[string]any{
|
||||
"cache_store": cfg.OCS.StatCacheType,
|
||||
"cache_nodes": cfg.OCS.StatCacheNodes,
|
||||
"cache_database": cfg.OCS.StatCacheDatabase,
|
||||
"cache_table": cfg.OCS.StatCacheTable,
|
||||
"cache_ttl": cfg.OCS.StatCacheTTL,
|
||||
"cache_disable_persistence": cfg.OCS.StatCacheDisablePersistence,
|
||||
"cache_auth_username": cfg.OCS.StatCacheAuthUsername,
|
||||
"cache_auth_password": cfg.OCS.StatCacheAuthPassword,
|
||||
"cache_store": cfg.OCS.StatCacheType,
|
||||
"cache_nodes": cfg.OCS.StatCacheNodes,
|
||||
"cache_database": cfg.OCS.StatCacheDatabase,
|
||||
"cache_table": cfg.OCS.StatCacheTable,
|
||||
"cache_ttl": cfg.OCS.StatCacheTTL,
|
||||
"cache_disable_persistence": cfg.OCS.StatCacheDisablePersistence,
|
||||
"cache_auth_username": cfg.OCS.StatCacheAuthUsername,
|
||||
"cache_auth_password": cfg.OCS.StatCacheAuthPassword,
|
||||
"cache_tls_enabled": cfg.OCS.StatCacheEnableTLS,
|
||||
"cache_tls_insecure": cfg.OCS.StatCacheTLSInsecure,
|
||||
"cache_tls_root_ca_certificate": cfg.OCS.StatCacheTLSRootCACertificate,
|
||||
},
|
||||
"prefix": cfg.OCS.Prefix,
|
||||
"additional_info_attribute": cfg.OCS.AdditionalInfoAttribute,
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -17,8 +18,13 @@ func Server(opts ...Option) (*http.Server, error) {
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("web reachability", checks.NewHTTPCheck(options.Config.HTTP.Addr))
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := healthHandlerConfiguration.
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint))
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption))
|
||||
|
||||
return debug.NewService(
|
||||
debug.Logger(options.Logger),
|
||||
|
||||
@@ -84,11 +84,18 @@ type Cache struct {
|
||||
ProviderCacheDisablePersistence bool `yaml:"provider_cache_disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;GATEWAY_PROVIDER_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the provider cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
ProviderCacheAuthUsername string `yaml:"provider_cache_auth_username" env:"OC_CACHE_AUTH_USERNAME;GATEWAY_PROVIDER_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
ProviderCacheAuthPassword string `yaml:"provider_cache_auth_password" env:"OC_CACHE_AUTH_PASSWORD;GATEWAY_PROVIDER_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OC_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OC_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OC_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OC_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheDisablePersistence bool `yaml:"create_home_cache_disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;GATEWAY_CREATE_HOME_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the create home cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheAuthUsername string `yaml:"create_home_cache_auth_username" env:"OC_CACHE_AUTH_USERNAME;GATEWAY_CREATE_HOME_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheAuthPassword string `yaml:"create_home_cache_auth_password" env:"OC_CACHE_AUTH_PASSWORD;GATEWAY_CREATE_HOME_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
ProviderCacheEnableTLS bool `yaml:"provider_cache_enable_tls" env:"OC_CACHE_ENABLE_TLS;GATEWAY_PROVIDER_CACHE_ENABLE_TLS" desc:"Enable TLS for the connection to file metadata cache." introductionVersion:"7.2.0"`
|
||||
ProviderCacheTLSInsecure bool `yaml:"provider_cache_tls_insecure" env:"OC_INSECURE;OC_CACHE_TLS_INSECURE;GATEWAY_PROVIDER_CACHE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
ProviderCacheTLSRootCACertificate string `yaml:"provider_cache_tls_root_ca_certificate" env:"OC_CACHE_TLS_ROOT_CA_CERTIFICATE;GATEWAY_PROVIDER_CACHE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided GATEWAY_PROVIDER_CACHE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
|
||||
CreateHomeCacheStore string `yaml:"create_home_cache_store" env:"OC_CACHE_STORE;GATEWAY_CREATE_HOME_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheNodes []string `yaml:"create_home_cache_nodes" env:"OC_CACHE_STORE_NODES;GATEWAY_CREATE_HOME_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheDatabase string `yaml:"create_home_cache_database" env:"OC_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheTTL time.Duration `yaml:"create_home_cache_ttl" env:"OC_CACHE_TTL;GATEWAY_CREATE_HOME_CACHE_TTL" desc:"Default time to live for user info in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheDisablePersistence bool `yaml:"create_home_cache_disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;GATEWAY_CREATE_HOME_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the create home cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheAuthUsername string `yaml:"create_home_cache_auth_username" env:"OC_CACHE_AUTH_USERNAME;GATEWAY_CREATE_HOME_CACHE_AUTH_USERNAME" desc:"The username to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheAuthPassword string `yaml:"create_home_cache_auth_password" env:"OC_CACHE_AUTH_PASSWORD;GATEWAY_CREATE_HOME_CACHE_AUTH_PASSWORD" desc:"The password to use for authentication. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
CreateHomeCacheEnableTLS bool `yaml:"create_home_cache_enable_tls" env:"OC_CACHE_ENABLE_TLS;GATEWAY_CREATE_HOME_CACHE_ENABLE_TLS" desc:"Enable TLS for the connection to file metadata cache." introductionVersion:"7.2.0"`
|
||||
CreateHomeCacheTLSInsecure bool `yaml:"create_home_cache_tls_insecure" env:"OC_INSECURE;OC_CACHE_TLS_INSECURE;GATEWAY_CREATE_HOME_CACHE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
CreateHomeCacheTLSRootCACertificate string `yaml:"create_home_cache_tls_root_ca_certificate" env:"OC_CACHE_TLS_ROOT_CA_CERTIFICATE;GATEWAY_CREATE_HOME_CACHE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided GATEWAY_CREATE_HOME_CACHE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
@@ -45,15 +45,15 @@ func GatewayConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]a
|
||||
"groupprovidersvc": cfg.GroupsEndpoint,
|
||||
"permissionssvc": cfg.PermissionsEndpoint,
|
||||
// sharing is located on the sharing service
|
||||
"usershareprovidersvc": cfg.SharingEndpoint,
|
||||
"publicshareprovidersvc": cfg.SharingEndpoint,
|
||||
"ocmshareprovidersvc": cfg.OCMEndpoint,
|
||||
"ocminvitemanagersvc": cfg.OCMEndpoint,
|
||||
"ocmproviderauthorizersvc": cfg.OCMEndpoint,
|
||||
"ocmcoresvc": cfg.OCMEndpoint,
|
||||
"usershareprovidersvc": cfg.SharingEndpoint,
|
||||
"publicshareprovidersvc": cfg.SharingEndpoint,
|
||||
"ocmshareprovidersvc": cfg.OCMEndpoint,
|
||||
"ocminvitemanagersvc": cfg.OCMEndpoint,
|
||||
"ocmproviderauthorizersvc": cfg.OCMEndpoint,
|
||||
"ocmcoresvc": cfg.OCMEndpoint,
|
||||
"use_common_space_root_share_logic": true,
|
||||
"commit_share_to_storage_grant": cfg.CommitShareToStorageGrant,
|
||||
"share_folder": cfg.ShareFolder, // ShareFolder is the location where to create shares in the recipient's storage provider.
|
||||
"commit_share_to_storage_grant": cfg.CommitShareToStorageGrant,
|
||||
"share_folder": cfg.ShareFolder, // ShareFolder is the location where to create shares in the recipient's storage provider.
|
||||
// other
|
||||
"disable_home_creation_on_login": cfg.DisableHomeCreationOnLogin,
|
||||
"datagateway": strings.TrimRight(cfg.FrontendPublicURL, "/") + "/data",
|
||||
@@ -71,14 +71,17 @@ func GatewayConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]a
|
||||
"cache_auth_password": cfg.Cache.ProviderCacheAuthPassword,
|
||||
},
|
||||
"create_personal_space_cache_config": map[string]any{
|
||||
"cache_store": cfg.Cache.CreateHomeCacheStore,
|
||||
"cache_nodes": cfg.Cache.CreateHomeCacheNodes,
|
||||
"cache_database": cfg.Cache.CreateHomeCacheDatabase,
|
||||
"cache_table": "create_personal_space",
|
||||
"cache_ttl": cfg.Cache.CreateHomeCacheTTL,
|
||||
"cache_disable_persistence": cfg.Cache.CreateHomeCacheDisablePersistence,
|
||||
"cache_auth_username": cfg.Cache.CreateHomeCacheAuthUsername,
|
||||
"cache_auth_password": cfg.Cache.CreateHomeCacheAuthPassword,
|
||||
"cache_store": cfg.Cache.CreateHomeCacheStore,
|
||||
"cache_nodes": cfg.Cache.CreateHomeCacheNodes,
|
||||
"cache_database": cfg.Cache.CreateHomeCacheDatabase,
|
||||
"cache_table": "create_personal_space",
|
||||
"cache_ttl": cfg.Cache.CreateHomeCacheTTL,
|
||||
"cache_disable_persistence": cfg.Cache.CreateHomeCacheDisablePersistence,
|
||||
"cache_auth_username": cfg.Cache.CreateHomeCacheAuthUsername,
|
||||
"cache_auth_password": cfg.Cache.CreateHomeCacheAuthPassword,
|
||||
"cache_tls_enabled": cfg.Cache.CreateHomeCacheEnableTLS,
|
||||
"cache_tls_insecure": cfg.Cache.CreateHomeCacheTLSInsecure,
|
||||
"cache_tls_root_ca_certificate": cfg.Cache.CreateHomeCacheTLSRootCACertificate,
|
||||
},
|
||||
},
|
||||
"authregistry": map[string]any{
|
||||
|
||||
@@ -18,6 +18,7 @@ func Server(opts ...Option) (*http.Server, error) {
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("nats reachability", func(ctx context.Context) error {
|
||||
if options.Config.Cache.ProviderCacheStore == "nats-js-kv" && len(options.Config.Cache.ProviderCacheNodes) > 0 {
|
||||
// no secureOption because we cannot yet configure tls for the cache store
|
||||
return checks.NewNatsCheck(options.Config.Cache.ProviderCacheNodes[0])(ctx)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -4,9 +4,11 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/signal"
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
natspkg "github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/runner"
|
||||
"github.com/opencloud-eu/opencloud/pkg/tracing"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
@@ -50,13 +52,9 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
var kv jetstream.KeyValue
|
||||
// Allow to run without a NATS store (e.g. for the standalone Education provisioning service)
|
||||
if len(cfg.Store.Nodes) > 0 {
|
||||
//Connect to NATS servers
|
||||
natsOptions := nats.Options{
|
||||
Servers: cfg.Store.Nodes,
|
||||
User: cfg.Store.AuthUsername,
|
||||
Password: cfg.Store.AuthPassword,
|
||||
}
|
||||
conn, err := natsOptions.Connect()
|
||||
// Connect to NATS servers
|
||||
secureOption := natspkg.Secure(cfg.Store.EnableTLS, cfg.Store.TLSInsecure, cfg.Store.TLSRootCACertificate)
|
||||
conn, err := nats.Connect(strings.Join(cfg.Store.Nodes, ","), secureOption, nats.UserInfo(cfg.Store.AuthUsername, cfg.Store.AuthPassword))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -4,12 +4,15 @@ import "time"
|
||||
|
||||
// Cache defines the available configuration for a cache store
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE;GRAPH_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_CACHE_STORE_NODES;GRAPH_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"GRAPH_CACHE_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"GRAPH_CACHE_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL;GRAPH_CACHE_TTL" desc:"Time to live for cache records in the graph. Defaults to '336h' (2 weeks). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;GRAPH_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_CACHE_AUTH_USERNAME;GRAPH_CACHE_AUTH_USERNAME" desc:"The username to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_CACHE_AUTH_PASSWORD;GRAPH_CACHE_AUTH_PASSWORD" desc:"The password to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE;GRAPH_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_CACHE_STORE_NODES;GRAPH_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"GRAPH_CACHE_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"GRAPH_CACHE_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL;GRAPH_CACHE_TTL" desc:"Time to live for cache records in the graph. Defaults to '336h' (2 weeks). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;GRAPH_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_CACHE_AUTH_USERNAME;GRAPH_CACHE_AUTH_USERNAME" desc:"The username to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_CACHE_AUTH_PASSWORD;GRAPH_CACHE_AUTH_PASSWORD" desc:"The password to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_CACHE_ENABLE_TLS;GRAPH_CACHE_ENABLE_TLS" desc:"Enable TLS for the connection to file metadata cache." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_CACHE_TLS_INSECURE;GRAPH_CACHE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_CACHE_TLS_ROOT_CA_CERTIFICATE;GRAPH_CACHE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided GRAPH_CACHE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
@@ -172,8 +172,11 @@ type Metadata struct {
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;GRAPH_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"GRAPH_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;GRAPH_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;GRAPH_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;GRAPH_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"GRAPH_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;GRAPH_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;GRAPH_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_PERSISTENT_STORE_ENABLE_TLS;GRAPH_STORE_ENABLE_TLS" desc:"Enable TLS for the connection to the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_PERSISTENT_STORE_TLS_INSECURE;GRAPH_STORE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_PERSISTENT_STORE_TLS_ROOT_CA_CERTIFICATE;GRAPH_STORE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided GRAPH_STORE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -32,8 +33,13 @@ func Server(opts ...Option) (*http.Server, error) {
|
||||
|
||||
// only check nats if really needed
|
||||
if options.Config.Events.Endpoint != "" {
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration = readyHandlerConfiguration.
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint))
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption))
|
||||
}
|
||||
|
||||
return debug.NewService(
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/render"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/publicshare"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
@@ -30,16 +29,14 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
l10n_pkg "github.com/opencloud-eu/opencloud/services/graph/pkg/l10n"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/odata"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/identity"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/identity/cache"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/odata"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/unifiedrole"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/validate"
|
||||
)
|
||||
@@ -755,16 +752,8 @@ func (api DriveItemPermissionsApi) ListPermissions(w http.ResponseWriter, r *htt
|
||||
|
||||
loc := r.Header.Get(l10n.HeaderAcceptLanguage)
|
||||
w.Header().Add("Content-Language", loc)
|
||||
if loc != "" && loc != "en" {
|
||||
err := l10n_pkg.TranslateEntity(loc, "en", permissions,
|
||||
l10n.TranslateEach("LibreGraphPermissionsRolesAllowedValues",
|
||||
l10n.TranslateField("Description"),
|
||||
l10n.TranslateField("DisplayName"),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
api.logger.Error().Err(err).Msg("tranlation error")
|
||||
}
|
||||
for i := range permissions.LibreGraphPermissionsRolesAllowedValues {
|
||||
permissions.LibreGraphPermissionsRolesAllowedValues[i] = unifiedrole.LocalizeRole(&permissions.LibreGraphPermissionsRolesAllowedValues[i], loc)
|
||||
}
|
||||
|
||||
render.Status(r, http.StatusOK)
|
||||
@@ -806,16 +795,8 @@ func (api DriveItemPermissionsApi) ListSpaceRootPermissions(w http.ResponseWrite
|
||||
|
||||
loc := r.Header.Get(l10n.HeaderAcceptLanguage)
|
||||
w.Header().Add("Content-Language", loc)
|
||||
if loc != "" && loc != "en" {
|
||||
err := l10n_pkg.TranslateEntity(loc, "en", permissions,
|
||||
l10n.TranslateEach("LibreGraphPermissionsRolesAllowedValues",
|
||||
l10n.TranslateField("Description"),
|
||||
l10n.TranslateField("DisplayName"),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
api.logger.Error().Err(err).Msg("tranlation error")
|
||||
}
|
||||
for i := range permissions.LibreGraphPermissionsRolesAllowedValues {
|
||||
permissions.LibreGraphPermissionsRolesAllowedValues[i] = unifiedrole.LocalizeRole(&permissions.LibreGraphPermissionsRolesAllowedValues[i], loc)
|
||||
}
|
||||
|
||||
render.Status(r, http.StatusOK)
|
||||
|
||||
@@ -1119,7 +1119,7 @@ func (g Graph) DeleteDrive(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
case cs3rpc.Code_CODE_PERMISSION_DENIED:
|
||||
logger.Debug().Interface("id", rid).Msg("could not delete drive: permission denied")
|
||||
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, "drive not found")
|
||||
errorcode.AccessDenied.Render(w, r, http.StatusForbidden, "permission denied")
|
||||
return
|
||||
case cs3rpc.Code_CODE_NOT_FOUND:
|
||||
logger.Debug().Interface("id", rid).Msg("could not delete drive: drive not found")
|
||||
|
||||
@@ -7,14 +7,17 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/render"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/unifiedrole"
|
||||
)
|
||||
|
||||
// GetRoleDefinitions a list of permission roles than can be used when sharing with users or groups
|
||||
func (g Graph) GetRoleDefinitions(w http.ResponseWriter, r *http.Request) {
|
||||
loc := r.Header.Get(l10n.HeaderAcceptLanguage)
|
||||
w.Header().Add("Content-Language", loc)
|
||||
render.Status(r, http.StatusOK)
|
||||
render.JSON(w, r, g.availableRoles)
|
||||
render.JSON(w, r, unifiedrole.LocalizeRoles(g.availableRoles, loc))
|
||||
}
|
||||
|
||||
// GetRoleDefinition a permission role than can be used when sharing with users or groups
|
||||
@@ -32,6 +35,8 @@ func (g Graph) GetRoleDefinition(w http.ResponseWriter, r *http.Request) {
|
||||
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
loc := r.Header.Get(l10n.HeaderAcceptLanguage)
|
||||
w.Header().Add("Content-Language", loc)
|
||||
render.Status(r, http.StatusOK)
|
||||
render.JSON(w, r, role)
|
||||
render.JSON(w, r, unifiedrole.LocalizeRole(role, loc))
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package svc_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
cs3mocks "github.com/opencloud-eu/reva/v2/tests/cs3mocks/mocks"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/shared"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/mocks"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/config/defaults"
|
||||
service "github.com/opencloud-eu/opencloud/services/graph/pkg/service/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/unifiedrole"
|
||||
)
|
||||
|
||||
var _ = Describe("RoleManagement", func() {
|
||||
var (
|
||||
svc service.Service
|
||||
gatewayClient *cs3mocks.GatewayAPIClient
|
||||
gatewaySelector pool.Selectable[gateway.GatewayAPIClient]
|
||||
eventsPublisher mocks.Publisher
|
||||
permSvc mocks.Permissions
|
||||
cfg *config.Config
|
||||
rr *httptest.ResponseRecorder
|
||||
ctx context.Context
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
rr = httptest.NewRecorder()
|
||||
ctx = context.Background()
|
||||
|
||||
cfg = defaults.FullDefaultConfig()
|
||||
cfg.Identity.LDAP.CACert = ""
|
||||
cfg.TokenManager.JWTSecret = "loremipsum"
|
||||
cfg.Commons = &shared.Commons{}
|
||||
cfg.GRPCClientTLS = &shared.GRPCClientTLS{}
|
||||
|
||||
pool.RemoveSelector("GatewaySelector" + "eu.opencloud.api.gateway")
|
||||
gatewayClient = &cs3mocks.GatewayAPIClient{}
|
||||
gatewaySelector = pool.GetSelector[gateway.GatewayAPIClient](
|
||||
"GatewaySelector",
|
||||
"eu.opencloud.api.gateway",
|
||||
func(cc grpc.ClientConnInterface) gateway.GatewayAPIClient {
|
||||
return gatewayClient
|
||||
},
|
||||
)
|
||||
eventsPublisher = mocks.Publisher{}
|
||||
permSvc = mocks.Permissions{}
|
||||
|
||||
var err error
|
||||
svc, err = service.NewService(
|
||||
service.Config(cfg),
|
||||
service.WithGatewaySelector(gatewaySelector),
|
||||
service.EventsPublisher(&eventsPublisher),
|
||||
service.PermissionService(&permSvc),
|
||||
)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
Describe("GetRoleDefinitions", func() {
|
||||
It("returns all available roles in English when no Accept-Language is set", func() {
|
||||
r := httptest.NewRequest(http.MethodGet, "/graph/v1beta1/roleManagement/permissions/roleDefinitions", nil)
|
||||
r = r.WithContext(ctx)
|
||||
svc.ServeHTTP(rr, r)
|
||||
|
||||
Expect(rr.Code).To(Equal(http.StatusOK))
|
||||
|
||||
var roles []libregraph.UnifiedRoleDefinition
|
||||
Expect(json.Unmarshal(rr.Body.Bytes(), &roles)).To(Succeed())
|
||||
Expect(roles).NotTo(BeEmpty())
|
||||
|
||||
viewer := findRoleByID(roles, unifiedrole.UnifiedRoleViewerID)
|
||||
Expect(viewer).NotTo(BeNil())
|
||||
Expect(viewer.GetDisplayName()).To(Equal("Can view"))
|
||||
})
|
||||
|
||||
It("returns translated roles when Accept-Language is German", func() {
|
||||
r := httptest.NewRequest(http.MethodGet, "/graph/v1beta1/roleManagement/permissions/roleDefinitions", nil)
|
||||
r.Header.Set("Accept-Language", "de")
|
||||
r = r.WithContext(ctx)
|
||||
svc.ServeHTTP(rr, r)
|
||||
|
||||
Expect(rr.Code).To(Equal(http.StatusOK))
|
||||
Expect(rr.Header().Get("Content-Language")).To(Equal("de"))
|
||||
|
||||
var roles []libregraph.UnifiedRoleDefinition
|
||||
Expect(json.Unmarshal(rr.Body.Bytes(), &roles)).To(Succeed())
|
||||
|
||||
viewer := findRoleByID(roles, unifiedrole.UnifiedRoleViewerID)
|
||||
Expect(viewer).NotTo(BeNil())
|
||||
Expect(viewer.GetDisplayName()).To(Equal("Kann anzeigen"))
|
||||
Expect(viewer.GetDescription()).To(Equal("Ansehen und herunterladen."))
|
||||
})
|
||||
|
||||
It("does not mutate the global buildInRoles after a German request", func() {
|
||||
r := httptest.NewRequest(http.MethodGet, "/graph/v1beta1/roleManagement/permissions/roleDefinitions", nil)
|
||||
r.Header.Set("Accept-Language", "de")
|
||||
r = r.WithContext(ctx)
|
||||
svc.ServeHTTP(rr, r)
|
||||
Expect(rr.Code).To(Equal(http.StatusOK))
|
||||
|
||||
// A second request without a locale must still return English
|
||||
rr2 := httptest.NewRecorder()
|
||||
r2 := httptest.NewRequest(http.MethodGet, "/graph/v1beta1/roleManagement/permissions/roleDefinitions", nil)
|
||||
r2 = r2.WithContext(ctx)
|
||||
svc.ServeHTTP(rr2, r2)
|
||||
Expect(rr2.Code).To(Equal(http.StatusOK))
|
||||
|
||||
var roles []libregraph.UnifiedRoleDefinition
|
||||
Expect(json.Unmarshal(rr2.Body.Bytes(), &roles)).To(Succeed())
|
||||
viewer := findRoleByID(roles, unifiedrole.UnifiedRoleViewerID)
|
||||
Expect(viewer).NotTo(BeNil())
|
||||
Expect(viewer.GetDisplayName()).To(Equal("Can view"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetRoleDefinition", func() {
|
||||
It("returns a single role in English when no Accept-Language is set", func() {
|
||||
r := httptest.NewRequest(http.MethodGet, "/graph/v1beta1/roleManagement/permissions/roleDefinitions/"+unifiedrole.UnifiedRoleViewerID, nil)
|
||||
r = r.WithContext(ctx)
|
||||
svc.ServeHTTP(rr, r)
|
||||
|
||||
Expect(rr.Code).To(Equal(http.StatusOK))
|
||||
var role libregraph.UnifiedRoleDefinition
|
||||
Expect(json.Unmarshal(rr.Body.Bytes(), &role)).To(Succeed())
|
||||
Expect(role.GetDisplayName()).To(Equal("Can view"))
|
||||
})
|
||||
|
||||
It("returns a single role translated when Accept-Language is German", func() {
|
||||
r := httptest.NewRequest(http.MethodGet, "/graph/v1beta1/roleManagement/permissions/roleDefinitions/"+unifiedrole.UnifiedRoleViewerID, nil)
|
||||
r.Header.Set("Accept-Language", "de")
|
||||
r = r.WithContext(ctx)
|
||||
svc.ServeHTTP(rr, r)
|
||||
|
||||
Expect(rr.Code).To(Equal(http.StatusOK))
|
||||
Expect(rr.Header().Get("Content-Language")).To(Equal("de"))
|
||||
var role libregraph.UnifiedRoleDefinition
|
||||
Expect(json.Unmarshal(rr.Body.Bytes(), &role)).To(Succeed())
|
||||
Expect(role.GetDisplayName()).To(Equal("Kann anzeigen"))
|
||||
Expect(role.GetDescription()).To(Equal("Ansehen und herunterladen."))
|
||||
})
|
||||
|
||||
It("returns 404 for an unknown roleID", func() {
|
||||
r := httptest.NewRequest(http.MethodGet, "/graph/v1beta1/roleManagement/permissions/roleDefinitions/unknown-role-id", nil)
|
||||
r = r.WithContext(ctx)
|
||||
svc.ServeHTTP(rr, r)
|
||||
|
||||
Expect(rr.Code).To(Equal(http.StatusNotFound))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// findRoleByID returns the first role with the given ID from the slice, or nil.
|
||||
func findRoleByID(roles []libregraph.UnifiedRoleDefinition, id string) *libregraph.UnifiedRoleDefinition {
|
||||
for i := range roles {
|
||||
if roles[i].GetId() == id {
|
||||
return &roles[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -6,11 +6,11 @@ import (
|
||||
"strings"
|
||||
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/conversions"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/conversions"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/l10n"
|
||||
graphl10n "github.com/opencloud-eu/opencloud/services/graph/pkg/l10n"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -546,6 +546,46 @@ func weightRoles(roleSet []*libregraph.UnifiedRoleDefinition, constraints string
|
||||
return roleSet
|
||||
}
|
||||
|
||||
// cloneRole returns a shallow struct copy of r with independent allocations for
|
||||
// the Description and DisplayName pointer fields — the only fields mutated by
|
||||
// TranslateEntity. All other fields (Id, LibreGraphWeight, RolePermissions) are
|
||||
// either read-only or stripped before rendering, so sharing their values is safe.
|
||||
func cloneRole(r *libregraph.UnifiedRoleDefinition) libregraph.UnifiedRoleDefinition {
|
||||
c := *r
|
||||
if r.Description != nil {
|
||||
s := *r.Description
|
||||
c.Description = &s
|
||||
}
|
||||
if r.DisplayName != nil {
|
||||
s := *r.DisplayName
|
||||
c.DisplayName = &s
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// LocalizeRole returns a translated, independent copy of a single role definition.
|
||||
// The global buildInRoles singleton is never modified.
|
||||
func LocalizeRole(r *libregraph.UnifiedRoleDefinition, locale string) libregraph.UnifiedRoleDefinition {
|
||||
c := cloneRole(r)
|
||||
if locale != "" && locale != "en" {
|
||||
_ = graphl10n.TranslateEntity(locale, "en", &c,
|
||||
l10n.TranslateField("Description"),
|
||||
l10n.TranslateField("DisplayName"),
|
||||
)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// LocalizeRoles returns a translated, independent copy of each role definition.
|
||||
// The global buildInRoles singleton is never modified.
|
||||
func LocalizeRoles(roles []*libregraph.UnifiedRoleDefinition, locale string) []libregraph.UnifiedRoleDefinition {
|
||||
out := make([]libregraph.UnifiedRoleDefinition, len(roles))
|
||||
for i, r := range roles {
|
||||
out[i] = LocalizeRole(r, locale)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GetAllowedResourceActions returns the allowed resource actions for the provided role by condition
|
||||
func GetAllowedResourceActions(role *libregraph.UnifiedRoleDefinition, condition string) []string {
|
||||
if role == nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package unifiedrole_test
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -267,3 +268,84 @@ func TestGetAllowedResourceActions(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalizeRole_English(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
original := unifiedrole.RoleViewer
|
||||
|
||||
result := unifiedrole.LocalizeRole(original, "en")
|
||||
|
||||
// Strings are unchanged for English
|
||||
g.Expect(result.GetDisplayName()).To(Equal(original.GetDisplayName()))
|
||||
g.Expect(result.GetDescription()).To(Equal(original.GetDescription()))
|
||||
|
||||
// Result is an independent copy — mutating it must not touch the global
|
||||
translated := "mutated"
|
||||
result.DisplayName = &translated
|
||||
g.Expect(original.GetDisplayName()).NotTo(Equal("mutated"))
|
||||
}
|
||||
|
||||
func TestLocalizeRole_German(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
original := unifiedrole.RoleViewer
|
||||
|
||||
result := unifiedrole.LocalizeRole(original, "de")
|
||||
|
||||
g.Expect(result.GetDisplayName()).To(Equal("Kann anzeigen"))
|
||||
g.Expect(result.GetDescription()).To(Equal("Ansehen und herunterladen."))
|
||||
|
||||
// Global singleton must be untouched
|
||||
g.Expect(original.GetDisplayName()).NotTo(Equal("Kann anzeigen"))
|
||||
g.Expect(original.GetDescription()).NotTo(Equal("Ansehen und herunterladen."))
|
||||
}
|
||||
|
||||
func TestLocalizeRole_EmptyLocale(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
original := unifiedrole.RoleViewer
|
||||
|
||||
result := unifiedrole.LocalizeRole(original, "")
|
||||
|
||||
// Empty locale falls back to source strings
|
||||
g.Expect(result.GetDisplayName()).To(Equal(original.GetDisplayName()))
|
||||
g.Expect(result.GetDescription()).To(Equal(original.GetDescription()))
|
||||
}
|
||||
|
||||
func TestLocalizeRoles_German(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
roles := unifiedrole.BuildInRoles
|
||||
|
||||
results := unifiedrole.LocalizeRoles(roles, "de")
|
||||
|
||||
g.Expect(results).To(HaveLen(len(roles)))
|
||||
|
||||
// Every result is a value (not a pointer)
|
||||
for i, r := range results {
|
||||
// Id is preserved
|
||||
g.Expect(r.GetId()).To(Equal(roles[i].GetId()))
|
||||
// Global singleton is not mutated
|
||||
g.Expect(roles[i].GetDisplayName()).NotTo(Equal(r.GetDisplayName()),
|
||||
"global displayName for role %s was mutated", r.GetId())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalizeRole_ConcurrentCallsDoNotRace(t *testing.T) {
|
||||
// Run with -race to detect data races on the global buildInRoles strings.
|
||||
const goroutines = 20
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines)
|
||||
for i := 0; i < goroutines; i++ {
|
||||
locale := "de"
|
||||
if i%2 == 0 {
|
||||
locale = "fr"
|
||||
}
|
||||
go func(loc string) {
|
||||
defer wg.Done()
|
||||
_ = unifiedrole.LocalizeRoles(unifiedrole.BuildInRoles, loc)
|
||||
}(locale)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// After all concurrent translations the globals must still hold English strings
|
||||
g := NewWithT(t)
|
||||
g.Expect(unifiedrole.RoleViewer.GetDisplayName()).To(Equal("Can view"))
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -14,12 +15,17 @@ import (
|
||||
func Server(opts ...Option) (*http.Server, error) {
|
||||
options := newOptions(opts...)
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Nats.EnableTLS,
|
||||
options.Config.Nats.TLSSkipVerifyClientCert,
|
||||
options.Config.Nats.TLSCert,
|
||||
)
|
||||
// For nats readiness and liveness checks are identical
|
||||
// the nats server will neither be healthy nor ready when it can not reach the nats server/cluster
|
||||
checkHandler := handlers.NewCheckHandler(
|
||||
handlers.NewCheckHandlerConfiguration().
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Nats.Host+":"+strconv.Itoa(options.Config.Nats.Port))),
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Nats.Host+":"+strconv.Itoa(options.Config.Nats.Port), secureOption)),
|
||||
)
|
||||
|
||||
return debug.NewService(
|
||||
|
||||
@@ -130,6 +130,9 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
microstore.Database(cfg.Store.Database),
|
||||
microstore.Table(cfg.Store.Table),
|
||||
store.Authentication(cfg.Store.AuthUsername, cfg.Store.AuthPassword),
|
||||
store.TLSEnabled(cfg.Store.EnableTLS),
|
||||
store.TLSInsecure(cfg.Store.TLSInsecure),
|
||||
store.TLSRootCA(cfg.Store.TLSRootCACertificate),
|
||||
)
|
||||
|
||||
svc := service.NewEventsNotifier(evts, channel, logger, gatewaySelector, valueService,
|
||||
|
||||
@@ -70,11 +70,14 @@ type ServiceAccount struct {
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;NOTIFICATIONS_STORE" desc:"The type of the store. Supported values are: 'memory', 'nats-js-kv', 'redis-sentinel', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;NOTIFICATIONS_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"NOTIFICATIONS_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"NOTIFICATIONS_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;NOTIFICATIONS_STORE_TTL" desc:"Time to live for notifications in the store. Defaults to '336h' (2 weeks). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;NOTIFICATIONS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;NOTIFICATIONS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;NOTIFICATIONS_STORE" desc:"The type of the store. Supported values are: 'memory', 'nats-js-kv', 'redis-sentinel', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;NOTIFICATIONS_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"NOTIFICATIONS_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"NOTIFICATIONS_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;NOTIFICATIONS_STORE_TTL" desc:"Time to live for notifications in the store. Defaults to '336h' (2 weeks). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;NOTIFICATIONS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;NOTIFICATIONS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_PERSISTENT_STORE_ENABLE_TLS;NOTIFICATIONS_STORE_ENABLE_TLS" desc:"Enable TLS for the connection to the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_PERSISTENT_STORE_TLS_INSECURE;NOTIFICATIONS_STORE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_PERSISTENT_STORE_TLS_ROOT_CA_CERTIFICATE;NOTIFICATIONS_STORE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided NOTIFICATIONS_STORE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -14,9 +15,14 @@ import (
|
||||
func Server(opts ...Option) (*http.Server, error) {
|
||||
options := newOptions(opts...)
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Notifications.Events.EnableTLS,
|
||||
options.Config.Notifications.Events.TLSInsecure,
|
||||
options.Config.Notifications.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := handlers.NewCheckHandlerConfiguration().
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Notifications.Events.Endpoint)).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Notifications.Events.Endpoint, secureOption)).
|
||||
WithCheck("smtp-check", checks.NewTCPCheck(options.Config.Notifications.SMTP.Host+":"+strconv.Itoa(options.Config.Notifications.SMTP.Port)))
|
||||
|
||||
return debug.NewService(
|
||||
|
||||
@@ -81,6 +81,7 @@ func OCMConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]any {
|
||||
"events": map[string]any{
|
||||
"natsaddress": cfg.Events.Endpoint,
|
||||
"natsclusterid": cfg.Events.Cluster,
|
||||
"enabletls": cfg.Events.EnableTLS,
|
||||
"tlsinsecure": cfg.Events.TLSInsecure,
|
||||
"tlsrootcacertificate": cfg.Events.TLSRootCACertificate,
|
||||
"authusername": cfg.Events.AuthUsername,
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -17,8 +18,13 @@ func Server(opts ...Option) (*http.Server, error) {
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("web reachability", checks.NewHTTPCheck(options.Config.HTTP.Addr))
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := healthHandlerConfiguration.
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint)).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption)).
|
||||
WithCheck("grpc reachability", checks.NewGRPCCheck(options.Config.GRPC.Addr))
|
||||
|
||||
return debug.NewService(
|
||||
|
||||
@@ -31,9 +31,12 @@ type Config struct {
|
||||
|
||||
// SigningKeys is a store configuration.
|
||||
type SigningKeys struct {
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE" desc:"The type of the signing key store. Supported values are: 'redis-sentinel' and 'nats-js-kv'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"addresses" env:"OC_CACHE_STORE_NODES;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES" desc:"A list of nodes to access the configured store. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_TTL" desc:"Default time to live for signing keys. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_CACHE_AUTH_USERNAME;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_CACHE_AUTH_PASSWORD;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE" desc:"The type of the signing key store. Supported values are: 'redis-sentinel' and 'nats-js-kv'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"addresses" env:"OC_CACHE_STORE_NODES;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES" desc:"A list of nodes to access the configured store. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_TTL" desc:"Default time to live for signing keys. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_CACHE_AUTH_USERNAME;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_CACHE_AUTH_PASSWORD;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_CACHE_ENABLE_TLS;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_ENABLE_TLS" desc:"Enable TLS for the connection to file metadata cache." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_CACHE_TLS_INSECURE;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_CACHE_TLS_ROOT_CA_CERTIFICATE;OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided OCS_PRESIGNEDURL_SIGNING_KEYS_STORE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
@@ -21,6 +21,7 @@ func Server(opts ...Option) (*http.Server, error) {
|
||||
readyHandlerConfiguration := healthHandlerConfiguration.
|
||||
WithCheck("nats reachability", func(ctx context.Context) error {
|
||||
if len(options.Config.SigningKeys.Nodes) > 0 {
|
||||
// no secureOption because we cannot configure it
|
||||
return checks.NewNatsCheck(options.Config.SigningKeys.Nodes[0])(ctx)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -44,6 +44,9 @@ func Server(opts ...Option) (http.Service, error) {
|
||||
microstore.Database("proxy"),
|
||||
microstore.Table("signing-keys"),
|
||||
store.Authentication(options.Config.SigningKeys.AuthUsername, options.Config.SigningKeys.AuthPassword),
|
||||
store.TLSEnabled(options.Config.SigningKeys.EnableTLS),
|
||||
store.TLSInsecure(options.Config.SigningKeys.TLSInsecure),
|
||||
store.TLSRootCA(options.Config.SigningKeys.TLSRootCACertificate),
|
||||
)
|
||||
|
||||
handle := svc.NewService(
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -17,8 +18,13 @@ func Server(opts ...Option) (*http.Server, error) {
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("grpc reachability", checks.NewGRPCCheck(options.Config.GRPC.Addr))
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := healthHandlerConfiguration.
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint))
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption))
|
||||
|
||||
return debug.NewService(
|
||||
debug.Logger(options.Logger),
|
||||
|
||||
@@ -69,6 +69,8 @@ Once the service defined as custom step has finished its work, it should send an
|
||||
|
||||
The backoff behavior as mentioned in the `retry` outcome can be configured using the `POSTPROCESSING_RETRY_BACKOFF_DURATION` and `POSTPROCESSING_MAX_RETRIES` environment variables. The backoff duration is calculated using the following formula after each failure: `backoff_duration = POSTPROCESSING_RETRY_BACKOFF_DURATION * 2^(number of failures - 1)`. This means that the time between the next round grows exponentially limited by the number of retries. Steps that still don't succeed after the maximum number of retries will be automatically moved to the `abort` state.
|
||||
|
||||
The same backoff formula is used when publishing an event to the event system fails, which can happen when the event system is briefly unavailable or slow to acknowledge. The number of publish retries is configured with the `POSTPROCESSING_PUBLISH_MAX_RETRIES` environment variable, setting it to `0` disables retrying. A single wait is never longer than half the configured ack wait, so that the event being processed is not redelivered to another worker while the publish is still being retried. If publishing still fails after the last retry, the incoming event is not acknowledged so that it gets redelivered and postprocessing can pick up where it left off.
|
||||
|
||||
See the [cs3 org](https://github.com/cs3org/reva/blob/edge/pkg/events/postprocessing.go) for up-to-date information of reserved step names and event definitions.
|
||||
|
||||
## CLI Commands
|
||||
|
||||
@@ -56,6 +56,9 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
microstore.Database(cfg.Store.Database),
|
||||
microstore.Table(cfg.Store.Table),
|
||||
store.Authentication(cfg.Store.AuthUsername, cfg.Store.AuthPassword),
|
||||
store.TLSEnabled(cfg.Store.EnableTLS),
|
||||
store.TLSInsecure(cfg.Store.TLSInsecure),
|
||||
store.TLSRootCA(cfg.Store.TLSRootCACertificate),
|
||||
)
|
||||
|
||||
svc, err := service.NewPostprocessingService(ctx, logger, st, traceProvider, cfg)
|
||||
|
||||
@@ -32,6 +32,7 @@ type Postprocessing struct {
|
||||
|
||||
RetryBackoffDuration time.Duration `yaml:"retry_backoff_duration" env:"POSTPROCESSING_RETRY_BACKOFF_DURATION" desc:"The base for the exponential backoff duration before retrying a failed postprocessing step. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
MaxRetries int `yaml:"max_retries" env:"POSTPROCESSING_MAX_RETRIES" desc:"The maximum number of retries for a failed postprocessing step." introductionVersion:"1.0.0"`
|
||||
PublishMaxRetries int `yaml:"-"` // hardcoded default only, for the stable-7.2. Config setting will come with the next stable release
|
||||
}
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
@@ -59,11 +60,14 @@ type Debug struct {
|
||||
|
||||
// Store configures the store to use
|
||||
type Store struct {
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;POSTPROCESSING_STORE" desc:"The type of the store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;POSTPROCESSING_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"POSTPROCESSING_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"POSTPROCESSING_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;POSTPROCESSING_STORE_TTL" desc:"Time to live for events in the store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;POSTPROCESSING_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;POSTPROCESSING_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_PERSISTENT_STORE;POSTPROCESSING_STORE" desc:"The type of the store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"nodes" env:"OC_PERSISTENT_STORE_NODES;POSTPROCESSING_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"POSTPROCESSING_STORE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"POSTPROCESSING_STORE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_PERSISTENT_STORE_TTL;POSTPROCESSING_STORE_TTL" desc:"Time to live for events in the store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_PERSISTENT_STORE_AUTH_USERNAME;POSTPROCESSING_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_PERSISTENT_STORE_AUTH_PASSWORD;POSTPROCESSING_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_PERSISTENT_STORE_ENABLE_TLS;POSTPROCESSING_STORE_ENABLE_TLS" desc:"Enable TLS for the connection to the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_PERSISTENT_STORE_TLS_INSECURE;POSTPROCESSING_STORE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_PERSISTENT_STORE_TLS_ROOT_CA_CERTIFICATE;POSTPROCESSING_STORE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided POSTPROCESSING_STORE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
@@ -36,6 +36,7 @@ func DefaultConfig() *config.Config {
|
||||
Workers: 3,
|
||||
RetryBackoffDuration: 5 * time.Second,
|
||||
MaxRetries: 14,
|
||||
PublishMaxRetries: 5,
|
||||
},
|
||||
Store: config.Store{
|
||||
Store: "nats-js-kv",
|
||||
|
||||
@@ -86,7 +86,17 @@ func (pp *Postprocessing) Delay(f func(next any)) {
|
||||
|
||||
// BackoffDuration calculates the duration for exponential backoff based on the number of failures.
|
||||
func (pp *Postprocessing) BackoffDuration() time.Duration {
|
||||
return pp.config.RetryBackoffDuration * time.Duration(math.Pow(2, float64(pp.Failures-1)))
|
||||
return BackoffDuration(pp.config.RetryBackoffDuration, pp.Failures)
|
||||
}
|
||||
|
||||
// BackoffDuration calculates an exponential backoff duration from a base duration and the
|
||||
// number of failures that happened so far. The first failure waits the base duration, every
|
||||
// subsequent one waits twice as long as the previous one.
|
||||
func BackoffDuration(base time.Duration, failures int) time.Duration {
|
||||
if failures < 1 {
|
||||
return 0
|
||||
}
|
||||
return base * time.Duration(math.Pow(2, float64(failures-1)))
|
||||
}
|
||||
|
||||
func (pp *Postprocessing) next(current events.Postprocessingstep) any {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -13,9 +14,14 @@ import (
|
||||
func Server(opts ...Option) (*http.Server, error) {
|
||||
options := newOptions(opts...)
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Postprocessing.Events.EnableTLS,
|
||||
options.Config.Postprocessing.Events.TLSInsecure,
|
||||
options.Config.Postprocessing.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := handlers.NewCheckHandlerConfiguration().
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Postprocessing.Events.Endpoint))
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Postprocessing.Events.Endpoint, secureOption))
|
||||
|
||||
return debug.NewService(
|
||||
debug.Logger(options.Logger),
|
||||
|
||||
@@ -293,14 +293,66 @@ func (pps *PostprocessingService) processEvent(e raw.Event) error {
|
||||
}
|
||||
|
||||
if next != nil {
|
||||
if err := events.Publish(ctx, pps.pub, next); err != nil {
|
||||
if err := pps.publishWithRetry(ctx, &e, next); err != nil {
|
||||
// The successor event never made it onto the bus. Don't ack the source event so
|
||||
// jetstream redelivers it and the upload can continue instead of being stuck in
|
||||
// the store forever.
|
||||
ackEvent = false
|
||||
pps.log.Error().Err(err).Msg("unable to publish event")
|
||||
|
||||
if pps.stopped.Load() || ctx.Err() != nil {
|
||||
// we are shutting down anyway, no need to take the whole process down with us
|
||||
return fmt.Errorf("%w: unable to publish event", ErrEvent)
|
||||
}
|
||||
return fmt.Errorf("%w: unable to publish event", ErrFatal) // we can't publish -> we are screwed
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// publishWithRetry publishes an event, retrying transient failures of the event system with
|
||||
// the same exponential backoff that is used for failed postprocessing steps. Between the
|
||||
// attempts the source event is marked as in progress and the wait is capped to half the ack
|
||||
// wait, so jetstream should not hand the source event to a second worker while we are still
|
||||
// retrying. Note that marking the event as in progress is best effort: it is a fire and forget
|
||||
// publish, so it can be lost exactly when the event system is unhealthy. A redelivery during a
|
||||
// retry is therefore possible, it is just unlikely. If all attempts fail, the last error is
|
||||
// returned.
|
||||
func (pps *PostprocessingService) publishWithRetry(ctx context.Context, e *raw.Event, ev any) error {
|
||||
for attempt := 0; ; attempt++ {
|
||||
err := events.Publish(ctx, pps.pub, ev)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if attempt >= pps.c.PublishMaxRetries {
|
||||
return err
|
||||
}
|
||||
|
||||
backoff := postprocessing.BackoffDuration(pps.c.RetryBackoffDuration, attempt+1)
|
||||
// Never wait longer than half the ack wait. We only refresh the redelivery timer
|
||||
// between the attempts, so a longer wait would let jetstream hand the source event
|
||||
// to a second worker while we are still retrying here.
|
||||
if maxBackoff := pps.c.Events.AckWait / 2; maxBackoff > 0 && backoff > maxBackoff {
|
||||
backoff = maxBackoff
|
||||
}
|
||||
pps.log.Warn().Err(err).Int("attempt", attempt+1).Dur("backoff", backoff).Msg("unable to publish event, retrying")
|
||||
|
||||
// tell jetstream that we are still working on the source event
|
||||
if ipErr := e.InProgress(); ipErr != nil {
|
||||
pps.log.Debug().Err(ipErr).Msg("unable to mark event as in progress")
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return err
|
||||
case <-pps.stopCh:
|
||||
return err
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (pps *PostprocessingService) getPP(sto store.Store, uploadID string) (*postprocessing.Postprocessing, error) {
|
||||
recs, err := sto.Read(uploadID)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestPostprocessing(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Service Suite")
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/postprocessing/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/postprocessing/pkg/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/raw"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/store"
|
||||
microevents "go-micro.dev/v4/events"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
)
|
||||
|
||||
// errPublish is the error a nats publish returns when the ack of the message was not
|
||||
// received in time. It is transient: the very next publish usually succeeds.
|
||||
var errPublish = errors.New("nats: timeout")
|
||||
|
||||
// testPublisher is a publisher that fails the first `failures` publish attempts and
|
||||
// records everything it accepted afterwards.
|
||||
type testPublisher struct {
|
||||
mu sync.Mutex
|
||||
failures int
|
||||
attempts int
|
||||
accepted []any
|
||||
}
|
||||
|
||||
func (p *testPublisher) Publish(_ string, ev any, _ ...microevents.PublishOption) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.attempts++
|
||||
if p.attempts <= p.failures {
|
||||
return errPublish
|
||||
}
|
||||
p.accepted = append(p.accepted, ev)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *testPublisher) stats() (attempts int, accepted []any) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.attempts, p.accepted
|
||||
}
|
||||
|
||||
var _ = Describe("PostprocessingService", func() {
|
||||
var (
|
||||
cfg config.Postprocessing
|
||||
pub *testPublisher
|
||||
pps *PostprocessingService
|
||||
)
|
||||
|
||||
// newService builds a service that is wired to the fake publisher only. It deliberately
|
||||
// does not go through NewPostprocessingService, which would need a running nats.
|
||||
newService := func() *PostprocessingService {
|
||||
return &PostprocessingService{
|
||||
ctx: context.Background(),
|
||||
log: log.NopLogger(),
|
||||
pub: pub,
|
||||
steps: getSteps(cfg),
|
||||
store: store.Create(),
|
||||
c: cfg,
|
||||
tp: noop.NewTracerProvider(),
|
||||
metrics: metrics.New(),
|
||||
stopCh: make(chan struct{}, 1),
|
||||
}
|
||||
}
|
||||
|
||||
// bytesReceived is the event that starts a postprocessing chain. Handling it makes the
|
||||
// service publish the first StartPostprocessingStep, which is the publish that used to
|
||||
// take the whole process down when nats hiccuped.
|
||||
bytesReceived := func() raw.Event {
|
||||
ev := events.BytesReceived{
|
||||
UploadID: "upload-" + uuid.New().String(),
|
||||
Filename: "test.txt",
|
||||
Filesize: 1234,
|
||||
}
|
||||
return raw.Event{
|
||||
Event: events.Event{
|
||||
ID: uuid.New().String(),
|
||||
Type: reflect.TypeOf(ev).String(),
|
||||
Event: ev,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
cfg = config.Postprocessing{
|
||||
Steps: []string{"virusscan"},
|
||||
RetryBackoffDuration: 5 * time.Millisecond,
|
||||
MaxRetries: 14,
|
||||
PublishMaxRetries: 3,
|
||||
}
|
||||
pub = &testPublisher{}
|
||||
})
|
||||
|
||||
Describe("publishing the next event", func() {
|
||||
It("publishes once when the event system is healthy", func() {
|
||||
pps = newService()
|
||||
|
||||
Expect(pps.processEvent(bytesReceived())).To(Succeed())
|
||||
|
||||
attempts, accepted := pub.stats()
|
||||
Expect(attempts).To(Equal(1))
|
||||
Expect(accepted).To(HaveLen(1))
|
||||
Expect(accepted[0]).To(BeAssignableToTypeOf(events.StartPostprocessingStep{}))
|
||||
Expect(accepted[0].(events.StartPostprocessingStep).StepToStart).To(Equal(events.PPStepAntivirus))
|
||||
})
|
||||
|
||||
It("retries a transient publish failure and succeeds on a later attempt", func() {
|
||||
pub.failures = 2
|
||||
pps = newService()
|
||||
|
||||
Expect(pps.processEvent(bytesReceived())).To(Succeed())
|
||||
|
||||
attempts, accepted := pub.stats()
|
||||
Expect(attempts).To(Equal(3))
|
||||
Expect(accepted).To(HaveLen(1))
|
||||
Expect(accepted[0]).To(BeAssignableToTypeOf(events.StartPostprocessingStep{}))
|
||||
})
|
||||
|
||||
It("uses an exponential backoff between the attempts", func() {
|
||||
pub.failures = 3
|
||||
pps = newService()
|
||||
|
||||
start := time.Now()
|
||||
Expect(pps.processEvent(bytesReceived())).To(Succeed())
|
||||
elapsed := time.Since(start)
|
||||
|
||||
// 5ms + 10ms + 20ms, minus a margin so a coarse clock cannot make this flaky
|
||||
Expect(elapsed).To(BeNumerically(">=", 30*time.Millisecond))
|
||||
})
|
||||
|
||||
It("caps the backoff at half the ack wait", func() {
|
||||
// Without a cap the waits would grow to 10+20+40+80=150ms, far beyond the ack
|
||||
// wait, and jetstream would redeliver the source event to a second worker while
|
||||
// this one is still retrying. Capped at AckWait/2 they are 10+10+10+10=40ms.
|
||||
cfg.RetryBackoffDuration = 10 * time.Millisecond
|
||||
cfg.PublishMaxRetries = 4
|
||||
cfg.Events.AckWait = 20 * time.Millisecond
|
||||
pub.failures = 4
|
||||
pps = newService()
|
||||
|
||||
start := time.Now()
|
||||
Expect(pps.processEvent(bytesReceived())).To(Succeed())
|
||||
elapsed := time.Since(start)
|
||||
|
||||
Expect(elapsed).To(BeNumerically("<", 100*time.Millisecond))
|
||||
|
||||
attempts, _ := pub.stats()
|
||||
Expect(attempts).To(Equal(5))
|
||||
})
|
||||
|
||||
It("is fatal once the retries are genuinely exhausted", func() {
|
||||
pub.failures = 1000
|
||||
pps = newService()
|
||||
|
||||
err := pps.processEvent(bytesReceived())
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(errors.Is(err, ErrFatal)).To(BeTrue())
|
||||
|
||||
// the initial attempt plus PublishMaxRetries retries
|
||||
attempts, accepted := pub.stats()
|
||||
Expect(attempts).To(Equal(4))
|
||||
Expect(accepted).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("does not retry when retrying is disabled", func() {
|
||||
cfg.PublishMaxRetries = 0
|
||||
pub.failures = 1000
|
||||
pps = newService()
|
||||
|
||||
err := pps.processEvent(bytesReceived())
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(errors.Is(err, ErrFatal)).To(BeTrue())
|
||||
|
||||
attempts, _ := pub.stats()
|
||||
Expect(attempts).To(Equal(1))
|
||||
})
|
||||
|
||||
It("does not take the process down when the service is stopping", func() {
|
||||
pub.failures = 1000
|
||||
pps = newService()
|
||||
pps.Close()
|
||||
|
||||
err := pps.processEvent(bytesReceived())
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(errors.Is(err, ErrEvent)).To(BeTrue())
|
||||
Expect(errors.Is(err, ErrFatal)).To(BeFalse())
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -66,6 +66,9 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
microstore.Table(cfg.OIDC.UserinfoCache.Table),
|
||||
store.DisablePersistence(cfg.OIDC.UserinfoCache.DisablePersistence),
|
||||
store.Authentication(cfg.OIDC.UserinfoCache.AuthUsername, cfg.OIDC.UserinfoCache.AuthPassword),
|
||||
store.TLSEnabled(cfg.OIDC.UserinfoCache.EnableTLS),
|
||||
store.TLSInsecure(cfg.OIDC.UserinfoCache.TLSInsecure),
|
||||
store.TLSRootCA(cfg.OIDC.UserinfoCache.TLSRootCACertificate),
|
||||
)
|
||||
|
||||
signingKeyStore := store.Create(
|
||||
@@ -76,6 +79,9 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
microstore.Table("signing-keys"),
|
||||
store.DisablePersistence(cfg.PreSignedURL.SigningKeys.DisablePersistence),
|
||||
store.Authentication(cfg.PreSignedURL.SigningKeys.AuthUsername, cfg.PreSignedURL.SigningKeys.AuthPassword),
|
||||
store.TLSEnabled(cfg.PreSignedURL.SigningKeys.EnableTLS),
|
||||
store.TLSInsecure(cfg.PreSignedURL.SigningKeys.TLSInsecure),
|
||||
store.TLSRootCA(cfg.PreSignedURL.SigningKeys.TLSRootCACertificate),
|
||||
)
|
||||
|
||||
logger := log.Configure(cfg.Service.Name, cfg.Commons, cfg.LogLevel)
|
||||
|
||||
@@ -132,14 +132,17 @@ type JWKS struct {
|
||||
|
||||
// Cache is a TTL cache configuration.
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE;PROXY_OIDC_USERINFO_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"addresses" env:"OC_CACHE_STORE_NODES;PROXY_OIDC_USERINFO_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"OC_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"PROXY_OIDC_USERINFO_CACHE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL;PROXY_OIDC_USERINFO_CACHE_TTL" desc:"Default time to live for user info in the user info cache. This value is only applied when the token expiration cannot be extracted from the access tokens (e.g. when non-JWT access tokes are used). See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;PROXY_OIDC_USERINFO_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_CACHE_AUTH_USERNAME;PROXY_OIDC_USERINFO_CACHE_AUTH_USERNAME" desc:"The username to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_CACHE_AUTH_PASSWORD;PROXY_OIDC_USERINFO_CACHE_AUTH_PASSWORD" desc:"The password to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE;PROXY_OIDC_USERINFO_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"addresses" env:"OC_CACHE_STORE_NODES;PROXY_OIDC_USERINFO_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"OC_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
Table string `yaml:"table" env:"PROXY_OIDC_USERINFO_CACHE_TABLE" desc:"The database table the store should use." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL;PROXY_OIDC_USERINFO_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;PROXY_OIDC_USERINFO_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_CACHE_AUTH_USERNAME;PROXY_OIDC_USERINFO_CACHE_AUTH_USERNAME" desc:"The username to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_CACHE_AUTH_PASSWORD;PROXY_OIDC_USERINFO_CACHE_AUTH_PASSWORD" desc:"The password to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_CACHE_ENABLE_TLS;PROXY_OIDC_USERINFO_CACHE_ENABLE_TLS" desc:"Enable TLS for the connection to file metadata cache." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_CACHE_TLS_INSECURE;PROXY_OIDC_USERINFO_CACHE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_CACHE_TLS_ROOT_CA_CERTIFICATE;PROXY_OIDC_USERINFO_CACHE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided PROXY_OIDC_USERINFO_CACHE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
|
||||
// RoleAssignment contains the configuration for how to assign roles to users during login
|
||||
@@ -189,12 +192,15 @@ type PreSignedURL struct {
|
||||
|
||||
// SigningKeys is a store configuration.
|
||||
type SigningKeys struct {
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE" desc:"The type of the signing key store. Supported values are: 'redis-sentinel', 'nats-js-kv' and 'opencloudstoreservice' (deprecated). See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"addresses" env:"OC_CACHE_STORE_NODES;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES" desc:"A list of nodes to access the configured store. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_TTL" desc:"Default time to live for signing keys. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_DISABLE_PERSISTENCE" desc:"Disables persistence of the store. Only applies when store type 'nats-js-kv' is configured. Defaults to true." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_CACHE_AUTH_USERNAME;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_CACHE_AUTH_PASSWORD;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE" desc:"The type of the signing key store. Supported values are: 'redis-sentinel', 'nats-js-kv' and 'opencloudstoreservice' (deprecated). See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"addresses" env:"OC_CACHE_STORE_NODES;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_NODES" desc:"A list of nodes to access the configured store. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_TTL" desc:"Default time to live for signing keys. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_DISABLE_PERSISTENCE" desc:"Disables persistence of the store. Only applies when store type 'nats-js-kv' is configured. Defaults to true." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_CACHE_AUTH_USERNAME;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_USERNAME" desc:"The username to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_CACHE_AUTH_PASSWORD;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_AUTH_PASSWORD" desc:"The password to authenticate with the store. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_CACHE_ENABLE_TLS;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_ENABLE_TLS" desc:"Enable TLS for the connection to file metadata cache." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_CACHE_TLS_INSECURE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_CACHE_TLS_ROOT_CA_CERTIFICATE;PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided PROXY_PRESIGNEDURL_SIGNING_KEYS_STORE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
|
||||
// ClaimsSelectorConf is the config for the claims-selector
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
|
||||
"github.com/opencloud-eu/opencloud/services/proxy/pkg/router"
|
||||
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/status"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
@@ -48,9 +49,13 @@ func (m createHome) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
ctx, span := m.tracer.Start(req.Context(), fmt.Sprintf("%s %s", req.Method, req.URL.Path), trace.WithSpanKind(trace.SpanKindServer))
|
||||
req = req.WithContext(ctx)
|
||||
defer span.End()
|
||||
if !m.shouldServe(req) {
|
||||
next := func() {
|
||||
span.End()
|
||||
m.next.ServeHTTP(w, req)
|
||||
}
|
||||
|
||||
if !m.shouldServe(req) {
|
||||
next()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -63,6 +68,10 @@ func (m createHome) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
createHomeReq := &provider.CreateHomeRequest{}
|
||||
u, ok := revactx.ContextGetUser(ctx)
|
||||
if ok {
|
||||
if u.GetId().GetType() == userv1beta1.UserType_USER_TYPE_LIGHTWEIGHT || u.GetId().GetType() == userv1beta1.UserType_USER_TYPE_SERVICE {
|
||||
next()
|
||||
return
|
||||
}
|
||||
roleIDs, err := m.getUserRoles(u)
|
||||
if err != nil {
|
||||
m.logger.Error().Err(err).Str("userid", u.Id.OpaqueId).Msg("failed to get roles for user")
|
||||
@@ -88,12 +97,12 @@ func (m createHome) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
}
|
||||
}
|
||||
span.End()
|
||||
m.next.ServeHTTP(w, req)
|
||||
next()
|
||||
}
|
||||
|
||||
func (m createHome) shouldServe(req *http.Request) bool {
|
||||
return req.Header.Get(revactx.TokenHeader) != ""
|
||||
ri := router.ContextRoutingInfo(req.Context())
|
||||
return req.Header.Get(revactx.TokenHeader) != "" && !ri.IsRouteUnprotected()
|
||||
}
|
||||
|
||||
func (m createHome) getUserRoles(user *userv1beta1.User) ([]string, error) {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
"github.com/opencloud-eu/opencloud/services/proxy/pkg/config"
|
||||
@@ -21,8 +22,13 @@ func Server(opts ...Option) (*http.Server, error) {
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("web reachability", checks.NewHTTPCheck(options.Config.HTTP.Addr))
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := healthHandlerConfiguration.
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint))
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption))
|
||||
|
||||
var configDumpFunc http.HandlerFunc = configDump(options.Config)
|
||||
return debug.NewService(
|
||||
|
||||
@@ -173,13 +173,13 @@ func (b *Backend) Upsert(id string, r search.Resource) error {
|
||||
return batch.Push()
|
||||
}
|
||||
|
||||
func (b *Backend) Move(rootID, parentID, location string) error {
|
||||
func (b *Backend) Move(id, parentID, targetPath string) error {
|
||||
batch, err := b.NewBatch(defaultBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := batch.Move(rootID, parentID, location); err != nil {
|
||||
if err := batch.Move(id, parentID, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -225,6 +225,31 @@ func (b *Backend) Purge(id string, onlyDeleted bool) error {
|
||||
return batch.Push()
|
||||
}
|
||||
|
||||
func (b *Backend) PurgeSpace(rootID string) error {
|
||||
for {
|
||||
req := bleve.NewSearchRequest(&query.TermQuery{FieldVal: "RootID", Term: rootID})
|
||||
req.Size = defaultBatchSize
|
||||
|
||||
res, err := b.index.Search(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if res.Hits.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
batch := b.index.NewBatch()
|
||||
for _, hit := range res.Hits {
|
||||
batch.Delete(hit.ID)
|
||||
}
|
||||
|
||||
if err := b.index.Batch(batch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Backend) NewBatch(size int) (search.BatchOperator, error) {
|
||||
return NewBatch(b.index, size)
|
||||
}
|
||||
@@ -103,6 +103,52 @@ var _ = Describe("Bleve", func() {
|
||||
}
|
||||
})
|
||||
|
||||
Describe("PurgeSpace", func() {
|
||||
It("takes every record of that space out of the index", func() {
|
||||
otherSpace := search.Resource{
|
||||
ID: "1$9!9",
|
||||
RootID: "1$9!9",
|
||||
Path: ".",
|
||||
Document: content.Document{Name: "other"},
|
||||
}
|
||||
for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} {
|
||||
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
|
||||
}
|
||||
|
||||
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)), "only the records of that space are gone")
|
||||
})
|
||||
|
||||
It("takes a space out that holds more records than one round", func() {
|
||||
otherSpace := search.Resource{
|
||||
ID: "1$9!9",
|
||||
RootID: "1$9!9",
|
||||
Path: ".",
|
||||
Document: content.Document{Name: "other"},
|
||||
}
|
||||
Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed())
|
||||
|
||||
for i := range 120 {
|
||||
resource := search.Resource{
|
||||
ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i),
|
||||
RootID: rootResource.RootID,
|
||||
Path: fmt.Sprintf("./file-%d", i),
|
||||
Document: content.Document{Name: fmt.Sprintf("file-%d", i)},
|
||||
}
|
||||
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
|
||||
}
|
||||
|
||||
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)), "only the record of the other space is left")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("New", func() {
|
||||
It("returns a new index instance", func() {
|
||||
b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
|
||||
|
||||
@@ -40,14 +40,14 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Batch) Move(id, parentID, location string) error {
|
||||
func (b *Batch) Move(id string, parentID string, targetPath string) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
rootResource, err := searchResourceByID(id, b.index)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
currentPath := rootResource.Path
|
||||
nextPath := utils.MakeRelativePath(location)
|
||||
nextPath := utils.MakeRelativePath(targetPath)
|
||||
|
||||
rootResource.Path = nextPath
|
||||
rootResource.Name = path.Base(nextPath)
|
||||
|
||||
@@ -102,9 +102,9 @@ func searchResourceByID(id string, index bleve.Index) (*search.Resource, error)
|
||||
return matchToResource(res.Hits[0]), nil
|
||||
}
|
||||
|
||||
func searchResourcesByPath(rootId, lookupPath string, index bleve.Index) ([]*search.Resource, error) {
|
||||
func searchResourcesByPath(rootID string, lookupPath string, index bleve.Index) ([]*search.Resource, error) {
|
||||
q := bleve.NewConjunctionQuery(
|
||||
bleve.NewQueryStringQuery("RootID:"+rootId),
|
||||
bleve.NewQueryStringQuery("RootID:"+rootID),
|
||||
bleve.NewQueryStringQuery("Path:"+escapeQuery(lookupPath+"/*")),
|
||||
)
|
||||
bleveReq := bleve.NewSearchRequest(q)
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
|
||||
searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
@@ -68,7 +70,10 @@ func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error)
|
||||
|
||||
func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
|
||||
boolQuery, err := convert.KQLToOpenSearchBoolQuery(sir.Query)
|
||||
if err != nil {
|
||||
switch {
|
||||
case searchQuery.IsValidationError(err):
|
||||
return nil, errtypes.BadRequest(err.Error())
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to convert KQL query to OpenSearch bool query: %w", err)
|
||||
}
|
||||
|
||||
@@ -194,13 +199,13 @@ func (b *Backend) Upsert(id string, r search.Resource) error {
|
||||
return batch.Push()
|
||||
}
|
||||
|
||||
func (b *Backend) Move(id string, parentID string, target string) error {
|
||||
func (b *Backend) Move(id string, parentID string, targetPath string) error {
|
||||
batch, err := b.NewBatch(defaultBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := batch.Move(id, parentID, target); err != nil {
|
||||
if err := batch.Move(id, parentID, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -246,6 +251,32 @@ func (b *Backend) Purge(id string, onlyDeleted bool) error {
|
||||
return batch.Push()
|
||||
}
|
||||
|
||||
func (b *Backend) PurgeSpace(rootID string) error {
|
||||
req, err := osu.BuildDocumentDeleteByQueryReq(
|
||||
opensearchgoAPI.DocumentDeleteByQueryReq{
|
||||
Indices: []string{b.index},
|
||||
Params: opensearchgoAPI.DocumentDeleteByQueryParams{
|
||||
WaitForCompletion: conversions.ToPointer(true),
|
||||
Refresh: conversions.ToPointer(true),
|
||||
},
|
||||
},
|
||||
osu.NewBoolQuery().Must(osu.NewTermQuery[string]("RootID").Value(rootID)),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to build the space purge request %s: %w", rootID, err)
|
||||
}
|
||||
|
||||
resp, err := b.client.Document.DeleteByQuery(context.TODO(), req)
|
||||
switch {
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to purge space %s: %w", rootID, err)
|
||||
case len(resp.Failures) != 0:
|
||||
return fmt.Errorf("failed to purge space %s: %v", rootID, resp.Failures)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) NewBatch(size int) (search.BatchOperator, error) {
|
||||
return NewBatch(b.client, b.index, size)
|
||||
}
|
||||
@@ -1,283 +1,613 @@
|
||||
package opensearch_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
opensearchgo "github.com/opensearch-project/opensearch-go/v4"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
func TestNewBackend(t *testing.T) {
|
||||
t.Run("fails to create if the cluster is not healthy", func(t *testing.T) {
|
||||
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
|
||||
Client: opensearchgo.Config{
|
||||
Addresses: []string{"http://localhost:1025"},
|
||||
func TestOpenSearchBackend(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "OpenSearch Backend Suite")
|
||||
}
|
||||
|
||||
func deleteIndexOnCleanup(tc *opensearchtest.TestClient, indexName string) {
|
||||
DeferCleanup(func() {
|
||||
Expect(tc.IndicesDelete(context.Background(), []string{indexName})).To(Succeed())
|
||||
})
|
||||
}
|
||||
|
||||
func resourceByID(tc *opensearchtest.TestClient, index, id string) search.Resource {
|
||||
GinkgoHelper()
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"ids": map[string]any{
|
||||
"values": []string{id},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "failed to create OpenSearch client")
|
||||
|
||||
backend, err := opensearch.NewBackend("test-engine-new-engine", client)
|
||||
require.Nil(t, backend)
|
||||
require.ErrorIs(t, err, opensearch.ErrUnhealthyCluster)
|
||||
},
|
||||
})
|
||||
|
||||
resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(index, strings.NewReader(body)).Hits)
|
||||
Expect(resources).To(HaveLen(1))
|
||||
return resources[0]
|
||||
}
|
||||
|
||||
func TestEngine_Search(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-search"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
// otherRoot returns a copy of the given resource that lives in a different root (space)
|
||||
// while keeping the same path, so it can be used to assert that cross-root updates do
|
||||
// not affect identically-named resources in other roots.
|
||||
func otherRoot(r search.Resource) search.Resource {
|
||||
r.ID = "2$2!3"
|
||||
r.RootID = "2$2!1"
|
||||
r.ParentID = "2$2!2"
|
||||
return r
|
||||
}
|
||||
|
||||
func newBackend(indexName string, resources ...search.Resource) (*opensearch.Backend, *opensearchtest.TestClient) {
|
||||
GinkgoHelper()
|
||||
|
||||
tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
for _, r := range resources {
|
||||
tc.Require.DocumentCreate(indexName, r.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), r)))
|
||||
}
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, len(resources))
|
||||
|
||||
t.Run("most simple search", func(t *testing.T) {
|
||||
resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, resp.Matches, 1)
|
||||
require.Equal(t, int32(1), resp.TotalMatches)
|
||||
require.Equal(t, document.ID, fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId))
|
||||
})
|
||||
|
||||
t.Run("ignores files that are marked as deleted", func(t *testing.T) {
|
||||
deletedDocument := opensearchtest.Testdata.Resources.File
|
||||
deletedDocument.ID = "1$2!4"
|
||||
deletedDocument.Deleted = true
|
||||
|
||||
tc.Require.DocumentCreate(indexName, deletedDocument.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, deletedDocument)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, resp.Matches, 1)
|
||||
require.Equal(t, int32(1), resp.TotalMatches)
|
||||
require.Equal(t, document.ID, fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId))
|
||||
})
|
||||
return backend, tc
|
||||
}
|
||||
|
||||
func TestEngine_Upsert(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-upsert"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("upsert with full document", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
require.NoError(t, backend.Upsert(document.ID, document))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngine_Move(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-move"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("moves the document to a new path", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(t, map[string]any{
|
||||
"query": map[string]any{
|
||||
"ids": map[string]any{
|
||||
"values": []string{document.ID},
|
||||
var _ = Describe("Backend", func() {
|
||||
Describe("NewBackend", func() {
|
||||
It("fails to create if the cluster is not healthy", func() {
|
||||
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
|
||||
Client: opensearchgo.Config{
|
||||
Addresses: []string{"http://localhost:1025"},
|
||||
},
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred(), "failed to create OpenSearch client")
|
||||
|
||||
backend, err := opensearch.NewBackend("test-engine-new-engine", client)
|
||||
Expect(backend).To(BeNil())
|
||||
Expect(err).To(MatchError(opensearch.ErrUnhealthyCluster))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Search", func() {
|
||||
const indexName = "opencloud-test-engine-search"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
document search.Resource
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
document = opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
|
||||
resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](t, tc.Require.Search(indexName, strings.NewReader(body)).Hits)
|
||||
require.Len(t, resources, 1)
|
||||
require.Equal(t, document.Path, resources[0].Path)
|
||||
It("performs the most simple search", func() {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(1))
|
||||
Expect(resp.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID))
|
||||
})
|
||||
|
||||
document.Path = "./new/path/to/resource"
|
||||
require.NoError(t, backend.Move(document.ID, document.ParentID, document.Path))
|
||||
It("ignores files that are marked as deleted", func() {
|
||||
deletedDocument := opensearchtest.Testdata.Resources.File
|
||||
deletedDocument.ID = "1$2!4"
|
||||
deletedDocument.Deleted = true
|
||||
|
||||
resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](t, tc.Require.Search(indexName, strings.NewReader(body)).Hits)
|
||||
require.Len(t, resources, 1)
|
||||
require.Equal(t, document.Path, resources[0].Path)
|
||||
tc.Require.DocumentCreate(indexName, deletedDocument.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), deletedDocument)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(1))
|
||||
Expect(resp.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngine_Delete(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-delete"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
Describe("Upsert", func() {
|
||||
const indexName = "opencloud-test-engine-upsert"
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
t.Run("mark document as deleted", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(t, map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"Deleted": map[string]any{
|
||||
"value": true,
|
||||
It("upserts a full document", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
Expect(backend.Upsert(document.ID, document)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Move", func() {
|
||||
const indexName = "opencloud-test-engine-move"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("moves the document to a new path", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"ids": map[string]any{
|
||||
"values": []string{document.ID},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits)
|
||||
Expect(resources).To(HaveLen(1))
|
||||
Expect(resources[0].Path).To(Equal(document.Path))
|
||||
|
||||
document.Path = "./new/path/to/resource"
|
||||
Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed())
|
||||
|
||||
resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits)
|
||||
Expect(resources).To(HaveLen(1))
|
||||
Expect(resources[0].Path).To(Equal(document.Path))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete", func() {
|
||||
const indexName = "opencloud-test-engine-delete"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
|
||||
It("marks the document as deleted", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
require.NoError(t, backend.Delete(document.ID))
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngine_Restore(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-restore"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("mark document as not deleted", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
document.Deleted = true
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(t, map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"Deleted": map[string]any{
|
||||
"value": true,
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"Deleted": map[string]any{
|
||||
"value": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
|
||||
|
||||
Expect(backend.Delete(document.ID)).To(Succeed())
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Restore", func() {
|
||||
const indexName = "opencloud-test-engine-restore"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
|
||||
It("marks the document as not deleted", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
document.Deleted = true
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
require.NoError(t, backend.Restore(document.ID))
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
|
||||
})
|
||||
}
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"Deleted": map[string]any{
|
||||
"value": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
func TestEngine_Purge(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-purge"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("purge with full document", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
require.NoError(t, backend.Purge(document.ID, false))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
Expect(backend.Restore(document.ID)).To(Succeed())
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("purge resource trees", func(t *testing.T) {
|
||||
resourceFolder := opensearchtest.Testdata.Resources.Folder
|
||||
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFolder)))
|
||||
Describe("Purge", func() {
|
||||
const indexName = "opencloud-test-engine-purge"
|
||||
|
||||
resourceFile := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFile)))
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
require.NoError(t, backend.Purge(resourceFolder.ID, false))
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
It("purges a full document", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
Expect(backend.Purge(document.ID, false)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
})
|
||||
|
||||
It("purges resource trees", func() {
|
||||
resourceFolder := opensearchtest.Testdata.Resources.Folder
|
||||
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder)))
|
||||
|
||||
resourceFile := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile)))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
Expect(backend.Purge(resourceFolder.ID, false)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
})
|
||||
|
||||
It("purges resource trees and ignores undeleted resources", func() {
|
||||
resourceFolder := opensearchtest.Testdata.Resources.Folder
|
||||
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder)))
|
||||
|
||||
resourceFile := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile)))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
Expect(backend.Delete(resourceFile.ID)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
Expect(backend.Purge(resourceFolder.ID, true)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("purge resource trees and ignores undeleted resources", func(t *testing.T) {
|
||||
resourceFolder := opensearchtest.Testdata.Resources.Folder
|
||||
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFolder)))
|
||||
Describe("PurgeSpace", func() {
|
||||
const indexName = "opencloud-test-engine-purge-space"
|
||||
|
||||
resourceFile := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFile)))
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
require.NoError(t, backend.Delete(resourceFile.ID))
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
require.NoError(t, backend.Purge(resourceFolder.ID, true))
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
It("takes every record of that space out of the index", func() {
|
||||
gone := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, gone.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), gone)))
|
||||
|
||||
stays := opensearchtest.Testdata.Resources.File
|
||||
stays.ID = "1$2!3"
|
||||
stays.RootID = "1$2!2"
|
||||
tc.Require.DocumentCreate(indexName, stays.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), stays)))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
Expect(backend.PurgeSpace(gone.RootID)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
left := opensearchtest.SearchHitsMustBeConverted[search.Resource](
|
||||
GinkgoTB(),
|
||||
tc.Require.Search(indexName, strings.NewReader(`{"query":{"match_all":{}}}`)).Hits,
|
||||
)
|
||||
Expect(left).To(HaveLen(1), "only the records of that space are gone")
|
||||
Expect(left[0].ID).To(Equal(stays.ID))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngine_DocCount(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-doc-count"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
Describe("DocCount", func() {
|
||||
const indexName = "opencloud-test-engine-doc-count"
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
t.Run("ignore deleted documents", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
count, err := backend.DocCount()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(1), count)
|
||||
It("ignores deleted documents", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, map[string]any{
|
||||
"doc": map[string]any{
|
||||
"Deleted": true,
|
||||
count, err := backend.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"doc": map[string]any{
|
||||
"Deleted": true,
|
||||
},
|
||||
})))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
count, err = backend.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(0)))
|
||||
})
|
||||
})
|
||||
|
||||
// The following specs ensure that updates which affect a resource and its descendants
|
||||
// (Delete, Restore, Move) are scoped to the root (space) of the target resource. Two
|
||||
// resources living in different roots may share the exact same path, so matching by
|
||||
// path alone would incorrectly update the wrong resource.
|
||||
Describe("updateSelfAndDescendants root scope", func() {
|
||||
It("deletes only the resource in the target root", func() {
|
||||
const indexName = "opencloud-test-engine-root-scope-delete"
|
||||
|
||||
target := opensearchtest.Testdata.Resources.File
|
||||
other := otherRoot(target)
|
||||
|
||||
backend, tc := newBackend(indexName, target, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
Expect(backend.Delete(target.ID)).To(Succeed())
|
||||
|
||||
Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeTrue(), "target resource should be marked as deleted")
|
||||
Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeFalse(), "resource in a different root must not be affected")
|
||||
})
|
||||
|
||||
It("restores only the resource in the target root", func() {
|
||||
const indexName = "opencloud-test-engine-root-scope-restore"
|
||||
|
||||
target := opensearchtest.Testdata.Resources.File
|
||||
target.Deleted = true
|
||||
other := otherRoot(target)
|
||||
|
||||
backend, tc := newBackend(indexName, target, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
Expect(backend.Restore(target.ID)).To(Succeed())
|
||||
|
||||
Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeFalse(), "target resource should be restored")
|
||||
Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeTrue(), "resource in a different root must not be affected")
|
||||
})
|
||||
|
||||
It("moves only the resource in the target root", func() {
|
||||
const indexName = "opencloud-test-engine-root-scope-move"
|
||||
|
||||
target := opensearchtest.Testdata.Resources.File
|
||||
other := otherRoot(target)
|
||||
|
||||
backend, tc := newBackend(indexName, target, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
Expect(backend.Move(target.ID, target.ParentID, "./new/path/to/resource")).To(Succeed())
|
||||
|
||||
Expect(resourceByID(tc, indexName, target.ID).Path).To(Equal("./new/path/to/resource"), "target resource should be moved")
|
||||
Expect(resourceByID(tc, indexName, other.ID).Path).To(Equal(other.Path), "resource in a different root must not be moved")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("SearchInAnalyzedFields", func() {
|
||||
const indexName = "opencloud-test-engine-search-analyzed-fields"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
dashed := opensearchtest.Testdata.Resources.Folder
|
||||
dashed.ID = "1$1!10"
|
||||
dashed.Name = "new-folder"
|
||||
dashed.Path = "./new-folder"
|
||||
dashed.Title = "quarterly report"
|
||||
|
||||
plain := opensearchtest.Testdata.Resources.Folder
|
||||
plain.ID = "1$1!11"
|
||||
plain.Name = "documents"
|
||||
plain.Path = "./documents"
|
||||
plain.Title = "notes"
|
||||
|
||||
spaced := opensearchtest.Testdata.Resources.Folder
|
||||
spaced.ID = "1$1!12"
|
||||
spaced.Name = "foo bar"
|
||||
spaced.Path = "./foo bar"
|
||||
spaced.Title = "spaced out"
|
||||
|
||||
backend, tc = newBackend(indexName, dashed, plain, spaced)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
})
|
||||
|
||||
DescribeTable("finds what the analyzer made of the value",
|
||||
func(query string, want []string) {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
names := make([]string, 0, len(resp.Matches))
|
||||
for _, match := range resp.Matches {
|
||||
names = append(names, match.Entity.Name)
|
||||
}
|
||||
Expect(names).To(ConsistOf(want))
|
||||
},
|
||||
})))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
count, err = backend.DocCount()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(0), count)
|
||||
Entry("the full name with the dash", "new-folder", []string{"new-folder"}),
|
||||
Entry("one token of it", "new", []string{"new-folder"}),
|
||||
Entry("a name without a dash", "documents", []string{"documents"}),
|
||||
Entry("a wildcard", "*folder*", []string{"new-folder"}),
|
||||
// the shape the web client sends for every name search
|
||||
Entry("a wildcard around the whole dashed name", `name:"*new-folder*"`, []string{"new-folder"}),
|
||||
Entry("a wildcard spanning the dash", `name:"*w-fol*"`, []string{"new-folder"}),
|
||||
Entry("a wildcard in a different case", `name:"*NEW-FOLDER*"`, []string{"new-folder"}),
|
||||
Entry("a wildcard spanning a space", `name:"*oo ba*"`, []string{"foo bar"}),
|
||||
Entry("a wildcard around a name with a space", `name:"*foo bar*"`, []string{"foo bar"}),
|
||||
Entry("a name with a space", `name:"foo bar"`, []string{"foo bar"}),
|
||||
Entry("a title of two words", `Title:"quarterly report"`, []string{"new-folder"}),
|
||||
Entry("one token of a title", "Title:quarterly", []string{"new-folder"}),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
Describe("SearchByTag", func() {
|
||||
const indexName = "opencloud-test-engine-search-by-tag"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tagged := opensearchtest.Testdata.Resources.Folder
|
||||
tagged.ID = "1$1!20"
|
||||
tagged.Name = "tagged"
|
||||
tagged.Path = "./tagged"
|
||||
tagged.Tags = []string{"foo-bar"}
|
||||
|
||||
other := opensearchtest.Testdata.Resources.Folder
|
||||
other.ID = "1$1!21"
|
||||
other.Name = "other"
|
||||
other.Path = "./other"
|
||||
other.Tags = []string{"foo"}
|
||||
|
||||
backend, tc = newBackend(indexName, tagged, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
})
|
||||
|
||||
// a tag is one label, not prose, so it matches as a whole or not at all
|
||||
DescribeTable("matches a tag as a whole",
|
||||
func(query string, want []string) {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
names := make([]string, 0, len(resp.Matches))
|
||||
for _, match := range resp.Matches {
|
||||
names = append(names, match.Entity.Name)
|
||||
}
|
||||
Expect(names).To(ConsistOf(want))
|
||||
},
|
||||
Entry("the whole tag", `tag:("foo-bar")`, []string{"tagged"}),
|
||||
Entry("a token of a tag does not match it", `tag:("foo")`, []string{"other"}),
|
||||
Entry("a tag in a different case", `tag:("FOO-BAR")`, []string{"tagged"}),
|
||||
Entry("a wildcard reaches both", `tag:("*foo*")`, []string{"tagged", "other"}),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("SearchWithAnInvalidQuery", func() {
|
||||
const indexName = "opencloud-test-engine-search-invalid-query"
|
||||
|
||||
It("answers with a bad request", func() {
|
||||
backend, tc := newBackend(indexName)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
_, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "AND mediatype:document"})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest("")))
|
||||
Expect(err.Error()).To(Equal(`error: bad request: the expression can't begin from a binary operator: 'AND'`))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -63,7 +63,7 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Batch) Move(id, parentID, location string) error {
|
||||
func (b *Batch) Move(id string, parentID string, targetPath string) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
op := func() error {
|
||||
return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(rootResource search.Resource) *osu.BodyParamScript {
|
||||
@@ -77,8 +77,8 @@ func (b *Batch) Move(id, parentID, location string) error {
|
||||
"id": id,
|
||||
"parentID": parentID,
|
||||
"oldPath": rootResource.Path,
|
||||
"newPath": utils.MakeRelativePath(location),
|
||||
"newName": path.Base(utils.MakeRelativePath(location)),
|
||||
"newPath": utils.MakeRelativePath(targetPath),
|
||||
"newName": path.Base(utils.MakeRelativePath(targetPath)),
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package convert_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestConvert(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "OpenSearch Convert Suite")
|
||||
}
|
||||
@@ -14,7 +14,7 @@ var (
|
||||
func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
|
||||
kqlAst, err := kql.Builder{}.Build(kqlQuery)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build query: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
kqlNodes, err := ExpandKQL(kqlAst.Nodes)
|
||||
|
||||
@@ -3,6 +3,7 @@ package convert
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -93,47 +94,13 @@ func (t kqlOpensearchTranspiler) getOperatorValueAt(nodes []ast.Node, i int) str
|
||||
}
|
||||
|
||||
func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
|
||||
var builder osu.Builder
|
||||
|
||||
switch node := node.(type) {
|
||||
case *ast.BooleanNode:
|
||||
return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
|
||||
case *ast.StringNode:
|
||||
isWildcard := strings.Contains(node.Value, "*")
|
||||
if isWildcard {
|
||||
return osu.NewWildcardQuery(node.Key).Value(node.Value), nil
|
||||
}
|
||||
|
||||
totalTerms := strings.Split(node.Value, " ")
|
||||
isSingleTerm := len(totalTerms) == 1
|
||||
isMultiTerm := len(totalTerms) >= 1
|
||||
switch {
|
||||
case isSingleTerm:
|
||||
return osu.NewTermQuery[string](node.Key).Value(node.Value), nil
|
||||
case isMultiTerm:
|
||||
return osu.NewMatchPhraseQuery(node.Key).Query(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported string node value: %s", node.Value)
|
||||
return stringNodeQuery(node), nil
|
||||
case *ast.DateTimeNode:
|
||||
if node.Operator == nil {
|
||||
return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
query := osu.NewRangeQuery[time.Time](node.Key)
|
||||
|
||||
switch node.Operator.Value {
|
||||
case ">":
|
||||
return query.Gt(node.Value), nil
|
||||
case ">=":
|
||||
return query.Gte(node.Value), nil
|
||||
case "<":
|
||||
return query.Lt(node.Value), nil
|
||||
case "<=":
|
||||
return query.Lte(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
|
||||
return dateTimeNodeQuery(node)
|
||||
case *ast.GroupNode:
|
||||
group, err := t.transpile(node.Nodes)
|
||||
if err != nil {
|
||||
@@ -145,3 +112,56 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
|
||||
|
||||
return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node)
|
||||
}
|
||||
|
||||
// stringNodeQuery picks the query a string node turns into.
|
||||
func stringNodeQuery(node *ast.StringNode) osu.Builder {
|
||||
isWildcard := strings.Contains(node.Value, "*")
|
||||
|
||||
switch {
|
||||
// Name: "*oo-bar", "*oo ba*", "*OO*"
|
||||
// Title: "*rterly rep*"
|
||||
// Tags: "*spaced tag*"
|
||||
case isWildcard && slices.Contains([]string{"Name", "Title", "Tags"}, node.Key):
|
||||
return osu.NewWildcardQuery(node.Key + ".keyword").
|
||||
Value(node.Value).
|
||||
Params(&osu.WildcardQueryParams{CaseInsensitive: true})
|
||||
// Path: "./foo*", MimeType: "*plain"
|
||||
case isWildcard:
|
||||
return osu.NewWildcardQuery(node.Key).Value(node.Value)
|
||||
// Tags: "foo-bar", "spaced tag", "FOO-BAR"
|
||||
case node.Key == "Tags":
|
||||
return osu.NewTermQuery[string](node.Key + ".keyword").
|
||||
Value(node.Value).
|
||||
Params(&osu.TermQueryParams{CaseInsensitive: true})
|
||||
// Name: "foo-bar", "foo bar"
|
||||
// Title: "quarterly report"
|
||||
// Content: "foo bar"
|
||||
case slices.Contains([]string{"Name", "Title", "Content"}, node.Key):
|
||||
return osu.NewMatchPhraseQuery(node.Key).Query(node.Value)
|
||||
// Path: "./foo bar", MimeType: "text/plain"
|
||||
default:
|
||||
return osu.NewTermQuery[string](node.Key).Value(node.Value)
|
||||
}
|
||||
}
|
||||
|
||||
// dateTimeNodeQuery turns a date time node into a range query.
|
||||
func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) {
|
||||
if node.Operator == nil {
|
||||
return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
query := osu.NewRangeQuery[time.Time](node.Key)
|
||||
|
||||
switch node.Operator.Value {
|
||||
case ">":
|
||||
return query.Gt(node.Value), nil
|
||||
case ">=":
|
||||
return query.Gte(node.Value), nil
|
||||
case "<":
|
||||
return query.Lt(node.Value), nil
|
||||
case "<=":
|
||||
return query.Lte(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
|
||||
}
|
||||
@@ -16,13 +16,13 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
|
||||
// kql to os dsl - type tests
|
||||
{
|
||||
Name: "term query - string node",
|
||||
Name: "match phrase query - string node on an analyzed field",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
},
|
||||
{
|
||||
Name: "term query - boolean node - true",
|
||||
@@ -58,7 +58,18 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
&ast.StringNode{Key: "Name", Value: "open*"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewWildcardQuery("Name").Value("open*"),
|
||||
Want: osu.NewWildcardQuery("Name.keyword").
|
||||
Value("open*").
|
||||
Params(&osu.WildcardQueryParams{CaseInsensitive: true}),
|
||||
},
|
||||
{
|
||||
Name: "wildcard query - string node without a keyword sub field",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Content", Value: "open*"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewWildcardQuery("Content").Value("open*"),
|
||||
},
|
||||
{
|
||||
Name: "bool query",
|
||||
@@ -71,8 +82,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().Must(
|
||||
osu.NewTermQuery[string]("Name").Value("a"),
|
||||
osu.NewTermQuery[string]("Name").Value("b"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("a"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("b"),
|
||||
),
|
||||
},
|
||||
{
|
||||
@@ -84,7 +95,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
}},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("any"),
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query("any"),
|
||||
},
|
||||
{
|
||||
Name: "range query >",
|
||||
@@ -146,7 +157,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
},
|
||||
{
|
||||
Name: "[* *]",
|
||||
@@ -158,7 +169,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -173,7 +184,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -189,7 +200,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -217,7 +228,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
).
|
||||
MustNot(
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
@@ -237,7 +248,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
osu.NewTermQuery[string]("age").Value("44"),
|
||||
),
|
||||
|
||||
@@ -55,11 +55,12 @@ func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match,
|
||||
SpaceId: resourceParentID.GetSpaceId(),
|
||||
OpaqueId: resourceParentID.GetOpaqueId(),
|
||||
},
|
||||
Size: resource.Size,
|
||||
Type: resource.Type,
|
||||
MimeType: resource.MimeType,
|
||||
Deleted: resource.Deleted,
|
||||
Tags: resource.Tags,
|
||||
Size: resource.Size,
|
||||
Type: resource.Type,
|
||||
MimeType: resource.MimeType,
|
||||
Deleted: resource.Deleted,
|
||||
Tags: resource.Tags,
|
||||
Favorites: resource.Favorites,
|
||||
Highlights: func() string {
|
||||
contentHighlights, ok := hit.Highlight["Content"]
|
||||
if !ok {
|
||||
|
||||
@@ -2,37 +2,114 @@ package convert_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
func TestOpenSearchHitToMatch(t *testing.T) {
|
||||
resource := opensearchtest.Testdata.Resources.File
|
||||
resource.MimeType = "audio/anything"
|
||||
|
||||
hit := opensearchgoAPI.SearchHit{
|
||||
Score: 1.1,
|
||||
Source: json.RawMessage(opensearchtest.JSONMustMarshal(t, resource)),
|
||||
}
|
||||
match, err := convert.OpenSearchHitToMatch(hit)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, hit.Score, match.Score)
|
||||
assert.Equal(t, resource.Name, match.Entity.Name)
|
||||
t.Parallel()
|
||||
t.Run("converts the audio field to the expected type", func(t *testing.T) {
|
||||
// searchMessage.Audio contains int64, int32 ... values that are converted to strings by the JSON marshaler,
|
||||
// so we need to convert the resource.Audio to align the expectations for the JSON comparison.
|
||||
audio, err := conversions.To[*searchMessage.Audio](resource.Audio)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, resource.Audio.Bitrate, match.Entity.Audio.Bitrate)
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, audio), opensearchtest.JSONMustMarshal(t, match.Entity.Audio))
|
||||
})
|
||||
// jsonMarshal marshals data to a JSON string, failing the running spec on error.
|
||||
func jsonMarshal(data any) string {
|
||||
GinkgoHelper()
|
||||
b, err := json.Marshal(data)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return string(b)
|
||||
}
|
||||
|
||||
var _ = Describe("OpenSearchHitToMatch", func() {
|
||||
var (
|
||||
resource search.Resource
|
||||
hit opensearchgoAPI.SearchHit
|
||||
mtime time.Time
|
||||
match *searchMessage.Match
|
||||
err error
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
resource = opensearchtest.Testdata.Resources.File
|
||||
resource.MimeType = "audio/mpeg"
|
||||
mtime = time.Date(2025, 7, 24, 15, 15, 1, 0, time.UTC)
|
||||
resource.Mtime = mtime.Format(time.RFC3339)
|
||||
resource.Favorites = []string{"cbf24bce-3e6e-4d9e-a2a2-cbf24bce3e6e"}
|
||||
|
||||
hit = opensearchgoAPI.SearchHit{
|
||||
Score: 1.1,
|
||||
Source: json.RawMessage(jsonMarshal(resource)),
|
||||
Highlight: map[string][]string{
|
||||
"Content": {"first match", "second match"},
|
||||
},
|
||||
}
|
||||
|
||||
match, err = convert.OpenSearchHitToMatch(hit)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("maps the score", func() {
|
||||
Expect(match.Score).To(Equal(hit.Score))
|
||||
})
|
||||
|
||||
It("maps all resource fields to the entity", func() {
|
||||
entity := match.Entity
|
||||
Expect(entity).ToNot(BeNil())
|
||||
|
||||
// reference (derived from RootID) and path
|
||||
Expect(entity.Ref.ResourceId.StorageId).To(Equal("1"))
|
||||
Expect(entity.Ref.ResourceId.SpaceId).To(Equal("1"))
|
||||
Expect(entity.Ref.ResourceId.OpaqueId).To(Equal("1"))
|
||||
Expect(entity.Ref.Path).To(Equal(resource.Path))
|
||||
|
||||
// resource id
|
||||
Expect(entity.Id.StorageId).To(Equal("1"))
|
||||
Expect(entity.Id.SpaceId).To(Equal("1"))
|
||||
Expect(entity.Id.OpaqueId).To(Equal("3"))
|
||||
|
||||
// parent id
|
||||
Expect(entity.ParentId.StorageId).To(Equal("1"))
|
||||
Expect(entity.ParentId.SpaceId).To(Equal("1"))
|
||||
Expect(entity.ParentId.OpaqueId).To(Equal("2"))
|
||||
|
||||
// scalar fields
|
||||
Expect(entity.Name).To(Equal(resource.Name))
|
||||
Expect(entity.Size).To(Equal(resource.Size))
|
||||
Expect(entity.Type).To(Equal(resource.Type))
|
||||
Expect(entity.MimeType).To(Equal(resource.MimeType))
|
||||
Expect(entity.Deleted).To(Equal(resource.Deleted))
|
||||
Expect(entity.Tags).To(Equal(resource.Tags))
|
||||
Expect(entity.Favorites).To(Equal(resource.Favorites))
|
||||
|
||||
// highlights are joined together
|
||||
Expect(entity.Highlights).To(Equal("first match; second match"))
|
||||
|
||||
// last modified time is parsed from the Mtime
|
||||
Expect(entity.LastModifiedTime).ToNot(BeNil())
|
||||
Expect(entity.LastModifiedTime.Seconds).To(Equal(mtime.Unix()))
|
||||
})
|
||||
|
||||
It("converts the media metadata to the expected types", func() {
|
||||
// searchMessage.Audio contains int64, int32 ... values that are converted to strings by the JSON marshaler,
|
||||
// so we need to convert the resource fields to align the expectations for the JSON comparison.
|
||||
expectedAudio, err := conversions.To[*searchMessage.Audio](resource.Audio)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(match.Entity.Audio.Bitrate).To(Equal(resource.Audio.Bitrate))
|
||||
Expect(jsonMarshal(match.Entity.Audio)).To(MatchJSON(jsonMarshal(expectedAudio)))
|
||||
|
||||
expectedImage, err := conversions.To[*searchMessage.Image](resource.Image)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(jsonMarshal(match.Entity.Image)).To(MatchJSON(jsonMarshal(expectedImage)))
|
||||
|
||||
expectedLocation, err := conversions.To[*searchMessage.GeoCoordinates](resource.Location)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(jsonMarshal(match.Entity.Location)).To(MatchJSON(jsonMarshal(expectedLocation)))
|
||||
|
||||
expectedPhoto, err := conversions.To[*searchMessage.Photo](resource.Photo)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(jsonMarshal(match.Entity.Photo)).To(MatchJSON(jsonMarshal(expectedPhoto)))
|
||||
})
|
||||
})
|
||||
@@ -11,20 +11,20 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
)
|
||||
|
||||
var TimeMustParse = func(t *testing.T, ts string) time.Time {
|
||||
var TimeMustParse = func(t testing.TB, ts string) time.Time {
|
||||
tp, err := time.Parse(time.RFC3339Nano, ts)
|
||||
require.NoError(t, err, "failed to parse time %s", ts)
|
||||
|
||||
return tp
|
||||
}
|
||||
|
||||
func JSONMustMarshal(t *testing.T, data any) string {
|
||||
func JSONMustMarshal(t testing.TB, data any) string {
|
||||
jsonData, err := json.Marshal(data)
|
||||
require.NoError(t, err, "failed to marshal data to JSON")
|
||||
return string(jsonData)
|
||||
}
|
||||
|
||||
func SearchHitsMustBeConverted[T any](t *testing.T, hits []opensearchgoAPI.SearchHit) []T {
|
||||
func SearchHitsMustBeConverted[T any](t testing.TB, hits []opensearchgoAPI.SearchHit) []T {
|
||||
ts := make([]T, len(hits))
|
||||
for i, hit := range hits {
|
||||
resource, err := conversions.To[T](hit.Source)
|
||||
|
||||
@@ -20,7 +20,7 @@ type TestClient struct {
|
||||
Require *testRequireClient
|
||||
}
|
||||
|
||||
func NewDefaultTestClient(t *testing.T, cfg config.EngineOpenSearchClient) *TestClient {
|
||||
func NewDefaultTestClient(t testing.TB, cfg config.EngineOpenSearchClient) *TestClient {
|
||||
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
|
||||
Client: opensearchgo.Config{
|
||||
Addresses: cfg.Addresses,
|
||||
@@ -33,7 +33,7 @@ func NewDefaultTestClient(t *testing.T, cfg config.EngineOpenSearchClient) *Test
|
||||
return NewTestClient(t, client)
|
||||
}
|
||||
|
||||
func NewTestClient(t *testing.T, client *opensearchgoAPI.Client) *TestClient {
|
||||
func NewTestClient(t testing.TB, client *opensearchgoAPI.Client) *TestClient {
|
||||
tc := &TestClient{c: client}
|
||||
trc := &testRequireClient{tc: tc, t: t}
|
||||
tc.Require = trc
|
||||
@@ -210,7 +210,7 @@ func (tc *TestClient) Search(ctx context.Context, index string, body io.Reader)
|
||||
|
||||
type testRequireClient struct {
|
||||
tc *TestClient
|
||||
t *testing.T
|
||||
t testing.TB
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) IndicesReset(indices []string) {
|
||||
|
||||
@@ -56,7 +56,9 @@ func updateSelfAndDescendants(ctx context.Context, client *opensearchgoAPI.Clien
|
||||
WaitForCompletion: conversions.ToPointer(true),
|
||||
},
|
||||
},
|
||||
osu.NewTermQuery[string]("Path").Value(resource.Path),
|
||||
osu.NewBoolQuery().
|
||||
Must(osu.NewTermQuery[string]("RootID").Value(resource.RootID)).
|
||||
Must(osu.NewTermQuery[string]("Path").Value(resource.Path)),
|
||||
osu.UpdateByQueryBodyParams{
|
||||
Script: scriptProvider(resource),
|
||||
},
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
@@ -38,10 +39,16 @@ func (e UnsupportedTimeRangeError) Error() string {
|
||||
return fmt.Sprintf("unable to convert '%v' to a time range", e.Value)
|
||||
}
|
||||
|
||||
// IsValidationError says whether the query itself is at fault, which makes it a
|
||||
// bad request and not an error of ours.
|
||||
func IsValidationError(err error) bool {
|
||||
switch err.(type) {
|
||||
case *StartsWithBinaryOperatorError, *NamedGroupInvalidNodesError, *UnsupportedTimeRangeError:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
var (
|
||||
startsWithBinaryOperator *StartsWithBinaryOperatorError
|
||||
namedGroupInvalidNodes *NamedGroupInvalidNodesError
|
||||
unsupportedTimeRange *UnsupportedTimeRangeError
|
||||
)
|
||||
|
||||
return errors.As(err, &startsWithBinaryOperator) ||
|
||||
errors.As(err, &namedGroupInvalidNodes) ||
|
||||
errors.As(err, &unsupportedTimeRange)
|
||||
}
|
||||
@@ -88,8 +88,8 @@ func (_c *BatchOperator_Delete_Call) RunAndReturn(run func(id string) error) *Ba
|
||||
}
|
||||
|
||||
// Move provides a mock function for the type BatchOperator
|
||||
func (_mock *BatchOperator) Move(rootID string, parentID string, location string) error {
|
||||
ret := _mock.Called(rootID, parentID, location)
|
||||
func (_mock *BatchOperator) Move(id string, parentID string, targetPath string) error {
|
||||
ret := _mock.Called(id, parentID, targetPath)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Move")
|
||||
@@ -97,7 +97,7 @@ func (_mock *BatchOperator) Move(rootID string, parentID string, location string
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(string, string, string) error); ok {
|
||||
r0 = returnFunc(rootID, parentID, location)
|
||||
r0 = returnFunc(id, parentID, targetPath)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
@@ -110,14 +110,14 @@ type BatchOperator_Move_Call struct {
|
||||
}
|
||||
|
||||
// Move is a helper method to define mock.On call
|
||||
// - rootID string
|
||||
// - id string
|
||||
// - parentID string
|
||||
// - location string
|
||||
func (_e *BatchOperator_Expecter) Move(rootID interface{}, parentID interface{}, location interface{}) *BatchOperator_Move_Call {
|
||||
return &BatchOperator_Move_Call{Call: _e.mock.On("Move", rootID, parentID, location)}
|
||||
// - targetPath string
|
||||
func (_e *BatchOperator_Expecter) Move(id interface{}, parentID interface{}, targetPath interface{}) *BatchOperator_Move_Call {
|
||||
return &BatchOperator_Move_Call{Call: _e.mock.On("Move", id, parentID, targetPath)}
|
||||
}
|
||||
|
||||
func (_c *BatchOperator_Move_Call) Run(run func(rootID string, parentID string, location string)) *BatchOperator_Move_Call {
|
||||
func (_c *BatchOperator_Move_Call) Run(run func(id string, parentID string, targetPath string)) *BatchOperator_Move_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 string
|
||||
if args[0] != nil {
|
||||
@@ -145,7 +145,7 @@ func (_c *BatchOperator_Move_Call) Return(err error) *BatchOperator_Move_Call {
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *BatchOperator_Move_Call) RunAndReturn(run func(rootID string, parentID string, location string) error) *BatchOperator_Move_Call {
|
||||
func (_c *BatchOperator_Move_Call) RunAndReturn(run func(id string, parentID string, targetPath string) error) *BatchOperator_Move_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
@@ -144,8 +144,8 @@ func (_c *Engine_DocCount_Call) RunAndReturn(run func() (uint64, error)) *Engine
|
||||
}
|
||||
|
||||
// Move provides a mock function for the type Engine
|
||||
func (_mock *Engine) Move(id string, parentid string, target string) error {
|
||||
ret := _mock.Called(id, parentid, target)
|
||||
func (_mock *Engine) Move(id string, parentID string, targetPath string) error {
|
||||
ret := _mock.Called(id, parentID, targetPath)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Move")
|
||||
@@ -153,7 +153,7 @@ func (_mock *Engine) Move(id string, parentid string, target string) error {
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(string, string, string) error); ok {
|
||||
r0 = returnFunc(id, parentid, target)
|
||||
r0 = returnFunc(id, parentID, targetPath)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
@@ -167,13 +167,13 @@ type Engine_Move_Call struct {
|
||||
|
||||
// Move is a helper method to define mock.On call
|
||||
// - id string
|
||||
// - parentid string
|
||||
// - target string
|
||||
func (_e *Engine_Expecter) Move(id interface{}, parentid interface{}, target interface{}) *Engine_Move_Call {
|
||||
return &Engine_Move_Call{Call: _e.mock.On("Move", id, parentid, target)}
|
||||
// - parentID string
|
||||
// - targetPath string
|
||||
func (_e *Engine_Expecter) Move(id interface{}, parentID interface{}, targetPath interface{}) *Engine_Move_Call {
|
||||
return &Engine_Move_Call{Call: _e.mock.On("Move", id, parentID, targetPath)}
|
||||
}
|
||||
|
||||
func (_c *Engine_Move_Call) Run(run func(id string, parentid string, target string)) *Engine_Move_Call {
|
||||
func (_c *Engine_Move_Call) Run(run func(id string, parentID string, targetPath string)) *Engine_Move_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 string
|
||||
if args[0] != nil {
|
||||
@@ -201,7 +201,7 @@ func (_c *Engine_Move_Call) Return(err error) *Engine_Move_Call {
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Engine_Move_Call) RunAndReturn(run func(id string, parentid string, target string) error) *Engine_Move_Call {
|
||||
func (_c *Engine_Move_Call) RunAndReturn(run func(id string, parentID string, targetPath string) error) *Engine_Move_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
@@ -325,6 +325,57 @@ func (_c *Engine_Purge_Call) RunAndReturn(run func(id string, onlyDeleted bool)
|
||||
return _c
|
||||
}
|
||||
|
||||
// PurgeSpace provides a mock function for the type Engine
|
||||
func (_mock *Engine) PurgeSpace(rootID string) error {
|
||||
ret := _mock.Called(rootID)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for PurgeSpace")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(string) error); ok {
|
||||
r0 = returnFunc(rootID)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Engine_PurgeSpace_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PurgeSpace'
|
||||
type Engine_PurgeSpace_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// PurgeSpace is a helper method to define mock.On call
|
||||
// - rootID string
|
||||
func (_e *Engine_Expecter) PurgeSpace(rootID interface{}) *Engine_PurgeSpace_Call {
|
||||
return &Engine_PurgeSpace_Call{Call: _e.mock.On("PurgeSpace", rootID)}
|
||||
}
|
||||
|
||||
func (_c *Engine_PurgeSpace_Call) Run(run func(rootID string)) *Engine_PurgeSpace_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 string
|
||||
if args[0] != nil {
|
||||
arg0 = args[0].(string)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Engine_PurgeSpace_Call) Return(err error) *Engine_PurgeSpace_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Engine_PurgeSpace_Call) RunAndReturn(run func(rootID string) error) *Engine_PurgeSpace_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Restore provides a mock function for the type Engine
|
||||
func (_mock *Engine) Restore(id string) error {
|
||||
ret := _mock.Called(id)
|
||||
|
||||
@@ -40,8 +40,8 @@ func (_m *Searcher) EXPECT() *Searcher_Expecter {
|
||||
}
|
||||
|
||||
// IndexSpace provides a mock function for the type Searcher
|
||||
func (_mock *Searcher) IndexSpace(rID *providerv1beta1.StorageSpaceId, forceRescan bool) error {
|
||||
ret := _mock.Called(rID, forceRescan)
|
||||
func (_mock *Searcher) IndexSpace(spaceID *providerv1beta1.StorageSpaceId, forceRescan bool) error {
|
||||
ret := _mock.Called(spaceID, forceRescan)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for IndexSpace")
|
||||
@@ -49,7 +49,7 @@ func (_mock *Searcher) IndexSpace(rID *providerv1beta1.StorageSpaceId, forceResc
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(*providerv1beta1.StorageSpaceId, bool) error); ok {
|
||||
r0 = returnFunc(rID, forceRescan)
|
||||
r0 = returnFunc(spaceID, forceRescan)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
@@ -62,13 +62,13 @@ type Searcher_IndexSpace_Call struct {
|
||||
}
|
||||
|
||||
// IndexSpace is a helper method to define mock.On call
|
||||
// - rID *providerv1beta1.StorageSpaceId
|
||||
// - spaceID *providerv1beta1.StorageSpaceId
|
||||
// - forceRescan bool
|
||||
func (_e *Searcher_Expecter) IndexSpace(rID interface{}, forceRescan interface{}) *Searcher_IndexSpace_Call {
|
||||
return &Searcher_IndexSpace_Call{Call: _e.mock.On("IndexSpace", rID, forceRescan)}
|
||||
func (_e *Searcher_Expecter) IndexSpace(spaceID interface{}, forceRescan interface{}) *Searcher_IndexSpace_Call {
|
||||
return &Searcher_IndexSpace_Call{Call: _e.mock.On("IndexSpace", spaceID, forceRescan)}
|
||||
}
|
||||
|
||||
func (_c *Searcher_IndexSpace_Call) Run(run func(rID *providerv1beta1.StorageSpaceId, forceRescan bool)) *Searcher_IndexSpace_Call {
|
||||
func (_c *Searcher_IndexSpace_Call) Run(run func(spaceID *providerv1beta1.StorageSpaceId, forceRescan bool)) *Searcher_IndexSpace_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 *providerv1beta1.StorageSpaceId
|
||||
if args[0] != nil {
|
||||
@@ -91,7 +91,7 @@ func (_c *Searcher_IndexSpace_Call) Return(err error) *Searcher_IndexSpace_Call
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Searcher_IndexSpace_Call) RunAndReturn(run func(rID *providerv1beta1.StorageSpaceId, forceRescan bool) error) *Searcher_IndexSpace_Call {
|
||||
func (_c *Searcher_IndexSpace_Call) RunAndReturn(run func(spaceID *providerv1beta1.StorageSpaceId, forceRescan bool) error) *Searcher_IndexSpace_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
@@ -188,8 +188,8 @@ func (_c *Searcher_PurgeDeleted_Call) RunAndReturn(run func(spaceID *providerv1b
|
||||
}
|
||||
|
||||
// PurgeItem provides a mock function for the type Searcher
|
||||
func (_mock *Searcher) PurgeItem(rID *providerv1beta1.Reference) {
|
||||
_mock.Called(rID)
|
||||
func (_mock *Searcher) PurgeItem(ref *providerv1beta1.Reference) {
|
||||
_mock.Called(ref)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -199,12 +199,12 @@ type Searcher_PurgeItem_Call struct {
|
||||
}
|
||||
|
||||
// PurgeItem is a helper method to define mock.On call
|
||||
// - rID *providerv1beta1.Reference
|
||||
func (_e *Searcher_Expecter) PurgeItem(rID interface{}) *Searcher_PurgeItem_Call {
|
||||
return &Searcher_PurgeItem_Call{Call: _e.mock.On("PurgeItem", rID)}
|
||||
// - ref *providerv1beta1.Reference
|
||||
func (_e *Searcher_Expecter) PurgeItem(ref interface{}) *Searcher_PurgeItem_Call {
|
||||
return &Searcher_PurgeItem_Call{Call: _e.mock.On("PurgeItem", ref)}
|
||||
}
|
||||
|
||||
func (_c *Searcher_PurgeItem_Call) Run(run func(rID *providerv1beta1.Reference)) *Searcher_PurgeItem_Call {
|
||||
func (_c *Searcher_PurgeItem_Call) Run(run func(ref *providerv1beta1.Reference)) *Searcher_PurgeItem_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 *providerv1beta1.Reference
|
||||
if args[0] != nil {
|
||||
@@ -222,11 +222,62 @@ func (_c *Searcher_PurgeItem_Call) Return() *Searcher_PurgeItem_Call {
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Searcher_PurgeItem_Call) RunAndReturn(run func(rID *providerv1beta1.Reference)) *Searcher_PurgeItem_Call {
|
||||
func (_c *Searcher_PurgeItem_Call) RunAndReturn(run func(ref *providerv1beta1.Reference)) *Searcher_PurgeItem_Call {
|
||||
_c.Run(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// PurgeSpace provides a mock function for the type Searcher
|
||||
func (_mock *Searcher) PurgeSpace(spaceID *providerv1beta1.StorageSpaceId) error {
|
||||
ret := _mock.Called(spaceID)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for PurgeSpace")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(*providerv1beta1.StorageSpaceId) error); ok {
|
||||
r0 = returnFunc(spaceID)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Searcher_PurgeSpace_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PurgeSpace'
|
||||
type Searcher_PurgeSpace_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// PurgeSpace is a helper method to define mock.On call
|
||||
// - spaceID *providerv1beta1.StorageSpaceId
|
||||
func (_e *Searcher_Expecter) PurgeSpace(spaceID interface{}) *Searcher_PurgeSpace_Call {
|
||||
return &Searcher_PurgeSpace_Call{Call: _e.mock.On("PurgeSpace", spaceID)}
|
||||
}
|
||||
|
||||
func (_c *Searcher_PurgeSpace_Call) Run(run func(spaceID *providerv1beta1.StorageSpaceId)) *Searcher_PurgeSpace_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 *providerv1beta1.StorageSpaceId
|
||||
if args[0] != nil {
|
||||
arg0 = args[0].(*providerv1beta1.StorageSpaceId)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Searcher_PurgeSpace_Call) Return(err error) *Searcher_PurgeSpace_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Searcher_PurgeSpace_Call) RunAndReturn(run func(spaceID *providerv1beta1.StorageSpaceId) error) *Searcher_PurgeSpace_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// RestoreItem provides a mock function for the type Searcher
|
||||
func (_mock *Searcher) RestoreItem(ref *providerv1beta1.Reference) {
|
||||
_mock.Called(ref)
|
||||
@@ -336,8 +387,8 @@ func (_c *Searcher_Search_Call) RunAndReturn(run func(ctx context.Context, req *
|
||||
}
|
||||
|
||||
// TrashItem provides a mock function for the type Searcher
|
||||
func (_mock *Searcher) TrashItem(rID *providerv1beta1.ResourceId) {
|
||||
_mock.Called(rID)
|
||||
func (_mock *Searcher) TrashItem(resourceID *providerv1beta1.ResourceId) {
|
||||
_mock.Called(resourceID)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -347,12 +398,12 @@ type Searcher_TrashItem_Call struct {
|
||||
}
|
||||
|
||||
// TrashItem is a helper method to define mock.On call
|
||||
// - rID *providerv1beta1.ResourceId
|
||||
func (_e *Searcher_Expecter) TrashItem(rID interface{}) *Searcher_TrashItem_Call {
|
||||
return &Searcher_TrashItem_Call{Call: _e.mock.On("TrashItem", rID)}
|
||||
// - resourceID *providerv1beta1.ResourceId
|
||||
func (_e *Searcher_Expecter) TrashItem(resourceID interface{}) *Searcher_TrashItem_Call {
|
||||
return &Searcher_TrashItem_Call{Call: _e.mock.On("TrashItem", resourceID)}
|
||||
}
|
||||
|
||||
func (_c *Searcher_TrashItem_Call) Run(run func(rID *providerv1beta1.ResourceId)) *Searcher_TrashItem_Call {
|
||||
func (_c *Searcher_TrashItem_Call) Run(run func(resourceID *providerv1beta1.ResourceId)) *Searcher_TrashItem_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 *providerv1beta1.ResourceId
|
||||
if args[0] != nil {
|
||||
@@ -370,7 +421,7 @@ func (_c *Searcher_TrashItem_Call) Return() *Searcher_TrashItem_Call {
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Searcher_TrashItem_Call) RunAndReturn(run func(rID *providerv1beta1.ResourceId)) *Searcher_TrashItem_Call {
|
||||
func (_c *Searcher_TrashItem_Call) RunAndReturn(run func(resourceID *providerv1beta1.ResourceId)) *Searcher_TrashItem_Call {
|
||||
_c.Run(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
@@ -29,17 +29,18 @@ type Engine interface {
|
||||
DocCount() (uint64, error)
|
||||
|
||||
Upsert(id string, r Resource) error
|
||||
Move(id string, parentid string, target string) error
|
||||
Move(id string, parentID string, targetPath string) error
|
||||
Delete(id string) error
|
||||
Restore(id string) error
|
||||
Purge(id string, onlyDeleted bool) error
|
||||
PurgeSpace(rootID string) error
|
||||
|
||||
NewBatch(batchSize int) (BatchOperator, error)
|
||||
}
|
||||
|
||||
type BatchOperator interface {
|
||||
Upsert(id string, r Resource) error
|
||||
Move(rootID, parentID, location string) error
|
||||
Move(id string, parentID string, targetPath string) error
|
||||
Delete(id string) error
|
||||
Restore(id string) error
|
||||
Purge(id string, onlyDeleted bool) error
|
||||
|
||||
@@ -46,11 +46,12 @@ const (
|
||||
type Searcher interface {
|
||||
Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error)
|
||||
|
||||
IndexSpace(rID *provider.StorageSpaceId, forceRescan bool) error
|
||||
IndexSpace(spaceID *provider.StorageSpaceId, forceRescan bool) error
|
||||
PurgeSpace(spaceID *provider.StorageSpaceId) error
|
||||
PurgeDeleted(spaceID *provider.StorageSpaceId) error
|
||||
|
||||
TrashItem(rID *provider.ResourceId)
|
||||
PurgeItem(rID *provider.Reference)
|
||||
TrashItem(resourceID *provider.ResourceId)
|
||||
PurgeItem(ref *provider.Reference)
|
||||
UpsertItem(ref *provider.Reference)
|
||||
RestoreItem(ref *provider.Reference)
|
||||
MoveItem(ref *provider.Reference)
|
||||
@@ -533,9 +534,9 @@ func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId, forceRescan bool)
|
||||
}
|
||||
|
||||
// TrashItem marks the item as deleted.
|
||||
func (s *Service) TrashItem(rID *provider.ResourceId) {
|
||||
if err := s.engine.Delete(storagespace.FormatResourceID(rID)); err != nil {
|
||||
s.logger.Error().Err(err).Interface("Id", rID).Msg("failed to remove item from index")
|
||||
func (s *Service) TrashItem(resourceID *provider.ResourceId) {
|
||||
if err := s.engine.Delete(storagespace.FormatResourceID(resourceID)); err != nil {
|
||||
s.logger.Info().Err(err).Interface("Id", resourceID).Msg("failed to remove item from index")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -554,6 +555,31 @@ func (s *Service) PurgeItem(ref *provider.Reference) {
|
||||
logDocCount(s.engine, s.logger)
|
||||
}
|
||||
|
||||
func (s *Service) PurgeSpace(spaceID *provider.StorageSpaceId) error {
|
||||
if spaceID == nil {
|
||||
return fmt.Errorf("spaceID must not be nil")
|
||||
}
|
||||
|
||||
rootID, err := storagespace.ParseID(spaceID.GetOpaqueId())
|
||||
if err != nil {
|
||||
s.logger.Error().Err(err).Str("space_id", spaceID.GetOpaqueId()).Msg("invalid space id")
|
||||
return err
|
||||
}
|
||||
if rootID.StorageId == "" || rootID.SpaceId == "" {
|
||||
return fmt.Errorf("invalid space id %s", spaceID.GetOpaqueId())
|
||||
}
|
||||
rootID.OpaqueId = rootID.SpaceId
|
||||
|
||||
if err := s.engine.PurgeSpace(storagespace.FormatResourceID(&rootID)); err != nil {
|
||||
s.logger.Error().Err(err).Str("space_id", spaceID.GetOpaqueId()).Msg("failed to purge the space from the index")
|
||||
return err
|
||||
}
|
||||
|
||||
logDocCount(s.engine, s.logger)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) PurgeDeleted(spaceID *provider.StorageSpaceId) error {
|
||||
if spaceID == nil {
|
||||
return fmt.Errorf("spaceID must not be nil")
|
||||
@@ -592,6 +618,11 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) {
|
||||
return
|
||||
}
|
||||
|
||||
if utils.IsProcessing(stat.Info) {
|
||||
s.logger.Debug().Str("path", path).Msg("resource is still being processed. Skipping.")
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := s.extractor.Extract(ctx, stat.Info)
|
||||
if err != nil {
|
||||
s.logger.Error().Err(err).Msg("failed to extract resource content")
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/checks"
|
||||
"github.com/opencloud-eu/opencloud/pkg/handlers"
|
||||
"github.com/opencloud-eu/opencloud/pkg/nats"
|
||||
"github.com/opencloud-eu/opencloud/pkg/service/debug"
|
||||
"github.com/opencloud-eu/opencloud/pkg/version"
|
||||
)
|
||||
@@ -19,8 +20,13 @@ func Server(opts ...Option) (*http.Server, error) {
|
||||
WithLogger(options.Logger).
|
||||
WithCheck("grpc reachability", checks.NewGRPCCheck(options.Config.GRPC.Addr))
|
||||
|
||||
secureOption := nats.Secure(
|
||||
options.Config.Events.EnableTLS,
|
||||
options.Config.Events.TLSInsecure,
|
||||
options.Config.Events.TLSRootCACertificate,
|
||||
)
|
||||
readyHandlerConfiguration := healthHandlerConfiguration.
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint)).
|
||||
WithCheck("nats reachability", checks.NewNatsCheck(options.Config.Events.Endpoint, secureOption)).
|
||||
WithCheck("tika-check", func(ctx context.Context) error {
|
||||
if options.Config.Extractor.Type == "tika" {
|
||||
u, err := url.Parse(options.Config.Extractor.Tika.TikaURL)
|
||||
|
||||
@@ -2,19 +2,21 @@ package event
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/metrics"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/raw"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/metrics"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
@@ -61,6 +63,7 @@ func New(ctx context.Context, stream raw.Stream, logger log.Logger, tp trace.Tra
|
||||
events.TagsAdded{},
|
||||
events.TagsRemoved{},
|
||||
events.SpaceRenamed{},
|
||||
events.SpaceDeleted{},
|
||||
events.LabelAdded{},
|
||||
events.LabelRemoved{},
|
||||
},
|
||||
@@ -163,49 +166,64 @@ func (s Service) processEvent(e raw.Event) error {
|
||||
_, span := tracer.Start(ctx, "processEvent")
|
||||
defer span.End()
|
||||
|
||||
e.InProgress() // let nats know that we are processing this event
|
||||
if err := e.InProgress(); err != nil {
|
||||
s.log.Debug().Err(err).Interface("event", e).Msg("failed to set event progress")
|
||||
}
|
||||
|
||||
s.log.Debug().Interface("event", e).Msg("updating index")
|
||||
|
||||
ack := e.Ack
|
||||
|
||||
debounce := func(id *provider.StorageSpaceId) {
|
||||
ack = func() error { return nil }
|
||||
s.indexSpaceDebouncer.Debounce(id, e.Ack)
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
switch ev := e.Event.Event.(type) {
|
||||
case events.ItemTrashed:
|
||||
s.index.TrashItem(ev.ID)
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.ItemPurged:
|
||||
s.index.PurgeItem(ev.Ref)
|
||||
e.Ack()
|
||||
case events.TrashbinPurged:
|
||||
s.index.PurgeDeleted(getSpaceID(ev.Ref))
|
||||
e.Ack()
|
||||
err = s.index.PurgeDeleted(getSpaceID(ev.Ref))
|
||||
case events.ItemMoved:
|
||||
s.index.MoveItem(ev.Ref)
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.ItemRestored:
|
||||
s.index.RestoreItem(ev.Ref)
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.ContainerCreated:
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.FileTouched:
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.FileVersionRestored:
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.TagsAdded:
|
||||
s.index.UpsertItem(ev.Ref)
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.TagsRemoved:
|
||||
s.index.UpsertItem(ev.Ref)
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.FileUploaded:
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.UploadReady:
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.FileRef), e.Ack)
|
||||
debounce(getSpaceID(ev.FileRef))
|
||||
case events.SpaceRenamed:
|
||||
s.indexSpaceDebouncer.Debounce(ev.ID, e.Ack)
|
||||
debounce(ev.ID)
|
||||
case events.SpaceDeleted:
|
||||
err = s.index.PurgeSpace(ev.ID)
|
||||
case events.LabelAdded:
|
||||
s.index.UpsertItem(ev.Ref)
|
||||
case events.LabelRemoved:
|
||||
s.index.UpsertItem(ev.Ref)
|
||||
default:
|
||||
s.log.Error().Interface("event", e).Msg("unhandled event type, acknowledged without indexing")
|
||||
}
|
||||
return nil
|
||||
|
||||
return errors.Join(err, ack())
|
||||
}
|
||||
|
||||
func monitorMetrics(ctx context.Context, stream raw.Stream, name string, m *metrics.Metrics, logger log.Logger) {
|
||||
|
||||
@@ -50,6 +50,11 @@ var _ = DescribeTable("event",
|
||||
return int(calls.Load())
|
||||
}, "2s").Should(Equal(len(mcks)))
|
||||
},
|
||||
Entry("SpaceDeleted", []string{"PurgeSpace"}, events.SpaceDeleted{}, false),
|
||||
Entry("ItemPurged", []string{"PurgeItem"}, events.ItemPurged{}, false),
|
||||
Entry("TrashbinPurged", []string{"PurgeDeleted"}, events.TrashbinPurged{}, false),
|
||||
Entry("LabelAdded", []string{"UpsertItem"}, events.LabelAdded{}, false),
|
||||
Entry("LabelRemoved", []string{"UpsertItem"}, events.LabelRemoved{}, false),
|
||||
Entry("ItemTrashed", []string{"TrashItem", "IndexSpace"}, events.ItemTrashed{}, false),
|
||||
Entry("ItemMoved", []string{"MoveItem", "IndexSpace"}, events.ItemMoved{}, false),
|
||||
Entry("ItemRestored", []string{"RestoreItem", "IndexSpace"}, events.ItemRestored{}, false),
|
||||
|
||||
@@ -55,13 +55,16 @@ type Metadata struct {
|
||||
|
||||
// Cache configures the cache of the Metadata store
|
||||
type Cache struct {
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE;SETTINGS_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"addresses" env:"OC_CACHE_STORE_NODES;SETTINGS_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"OC_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
FileTable string `yaml:"files_table" env:"SETTINGS_FILE_CACHE_TABLE" desc:"The database table the store should use for the file cache." introductionVersion:"1.0.0"`
|
||||
DirectoryTable string `yaml:"directories_table" env:"SETTINGS_DIRECTORY_CACHE_TABLE" desc:"The database table the store should use for the directory cache." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL;SETTINGS_CACHE_TTL" desc:"Default time to live for entries in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;SETTINGS_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_CACHE_AUTH_USERNAME;SETTINGS_CACHE_AUTH_USERNAME" desc:"The username to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_CACHE_AUTH_PASSWORD;SETTINGS_CACHE_AUTH_PASSWORD" desc:"The password to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
Store string `yaml:"store" env:"OC_CACHE_STORE;SETTINGS_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'redis-sentinel', 'nats-js-kv', 'noop'. See the text description for details." introductionVersion:"1.0.0"`
|
||||
Nodes []string `yaml:"addresses" env:"OC_CACHE_STORE_NODES;SETTINGS_CACHE_STORE_NODES" desc:"A list of nodes to access the configured store. This has no effect when 'memory' store is configured. Note that the behaviour how nodes are used is dependent on the library of the configured store. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
Database string `yaml:"database" env:"OC_CACHE_DATABASE" desc:"The database name the configured store should use." introductionVersion:"1.0.0"`
|
||||
FileTable string `yaml:"files_table" env:"SETTINGS_FILE_CACHE_TABLE" desc:"The database table the store should use for the file cache." introductionVersion:"1.0.0"`
|
||||
DirectoryTable string `yaml:"directories_table" env:"SETTINGS_DIRECTORY_CACHE_TABLE" desc:"The database table the store should use for the directory cache." introductionVersion:"1.0.0"`
|
||||
TTL time.Duration `yaml:"ttl" env:"OC_CACHE_TTL;SETTINGS_CACHE_TTL" desc:"Default time to live for entries in the cache. Only applied when access tokens has no expiration. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
DisablePersistence bool `yaml:"disable_persistence" env:"OC_CACHE_DISABLE_PERSISTENCE;SETTINGS_CACHE_DISABLE_PERSISTENCE" desc:"Disables persistence of the cache. Only applies when store type 'nats-js-kv' is configured. Defaults to false." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"username" env:"OC_CACHE_AUTH_USERNAME;SETTINGS_CACHE_AUTH_USERNAME" desc:"The username to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"password" env:"OC_CACHE_AUTH_PASSWORD;SETTINGS_CACHE_AUTH_PASSWORD" desc:"The password to authenticate with the cache. Only applies when store type 'nats-js-kv' is configured." introductionVersion:"1.0.0"`
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_CACHE_ENABLE_TLS;SETTINGS_CACHE_ENABLE_TLS" desc:"Enable TLS for the connection to file metadata cache." introductionVersion:"7.2.0"`
|
||||
TLSInsecure bool `yaml:"tls_insecure" env:"OC_INSECURE;OC_CACHE_TLS_INSECURE;SETTINGS_CACHE_TLS_INSECURE" desc:"Whether to verify the server TLS certificates." introductionVersion:"7.2.0"`
|
||||
TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_CACHE_TLS_ROOT_CA_CERTIFICATE;SETTINGS_CACHE_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided SETTINGS_CACHE_TLS_INSECURE will be seen as false." introductionVersion:"7.2.0"`
|
||||
}
|
||||
@@ -140,6 +140,9 @@ func (c *CachedMDC) Init(ctx context.Context, id string) error {
|
||||
microstore.Table(c.cfg.Metadata.Cache.DirectoryTable),
|
||||
store.DisablePersistence(c.cfg.Metadata.Cache.DisablePersistence),
|
||||
store.Authentication(c.cfg.Metadata.Cache.AuthUsername, c.cfg.Metadata.Cache.AuthPassword),
|
||||
store.TLSEnabled(c.cfg.Metadata.Cache.EnableTLS),
|
||||
store.TLSInsecure(c.cfg.Metadata.Cache.TLSInsecure),
|
||||
store.TLSRootCA(c.cfg.Metadata.Cache.TLSRootCACertificate),
|
||||
)
|
||||
c.filesCache = store.Create(
|
||||
store.Store(c.cfg.Metadata.Cache.Store),
|
||||
@@ -149,6 +152,9 @@ func (c *CachedMDC) Init(ctx context.Context, id string) error {
|
||||
microstore.Table(c.cfg.Metadata.Cache.FileTable),
|
||||
store.DisablePersistence(c.cfg.Metadata.Cache.DisablePersistence),
|
||||
store.Authentication(c.cfg.Metadata.Cache.AuthUsername, c.cfg.Metadata.Cache.AuthPassword),
|
||||
store.TLSEnabled(c.cfg.Metadata.Cache.EnableTLS),
|
||||
store.TLSInsecure(c.cfg.Metadata.Cache.TLSInsecure),
|
||||
store.TLSRootCA(c.cfg.Metadata.Cache.TLSRootCACertificate),
|
||||
)
|
||||
return c.next.Init(ctx, id)
|
||||
}
|
||||
|
||||
@@ -155,6 +155,7 @@ type Events struct {
|
||||
EnableTLS bool `yaml:"enable_tls" env:"OC_EVENTS_ENABLE_TLS;SHARING_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the OpenCloud service which receives and delivers events between the services." introductionVersion:"1.0.0"`
|
||||
AuthUsername string `yaml:"auth_username" env:"OC_EVENTS_AUTH_USERNAME;SHARING_EVENTS_AUTH_USERNAME" desc:"Username for the events broker." introductionVersion:"1.0.0"`
|
||||
AuthPassword string `yaml:"auth_password" env:"OC_EVENTS_AUTH_PASSWORD;SHARING_EVENTS_AUTH_PASSWORD" desc:"Password for the events broker." introductionVersion:"1.0.0"`
|
||||
// TODO use TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OC_EVENTS_TLS_ROOT_CA_CERTIFICATE;SHARING_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided SHARING_EVENTS_TLS_INSECURE will be seen as false." introductionVersion:"1.0.0"`
|
||||
}
|
||||
|
||||
// PasswordPolicy configures reva password policy
|
||||
|
||||
Loaded 100 of 173 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user