mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-08 11:53:07 -04:00
Compare commits
38
Commits
v7.2.0
..
stable-7.2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1770793f26 | ||
|
|
7bb32387cb | ||
|
|
43166d138a | ||
|
|
502628c4d6 | ||
|
|
d6944d7b0b | ||
|
|
027a8cf9b6 | ||
|
|
0e6795492e | ||
|
|
0548d5db7e | ||
|
|
64be6d614f | ||
|
|
cbc6f8408a | ||
|
|
fa1e4e6104 | ||
|
|
cee105203c | ||
|
|
48f9874e48 | ||
|
|
8c40a33f6c | ||
|
|
aeff1f53b8 | ||
|
|
82294e3c71 | ||
|
|
be01619806 | ||
|
|
38440bd8fc | ||
|
|
066eabd0c9 | ||
|
|
3516dc7f2f | ||
|
|
954b8ccf41 | ||
|
|
6da9b1326a | ||
|
|
2711523aec | ||
|
|
5d8d022bd7 | ||
|
|
5218eeffdd | ||
|
|
a75f592d37 | ||
|
|
a17a70c8b7 | ||
|
|
df3a790a2f | ||
|
|
eb73d2af0e | ||
|
|
31f01c6d3d | ||
|
|
6597dad1f9 | ||
|
|
8b01d08ce3 | ||
|
|
c73ed01908 | ||
|
|
8c4f9d96c0 | ||
|
|
a83a3b7680 | ||
|
|
cee5657651 | ||
|
|
0a500d25bf | ||
|
|
be121d8a94 |
No files matched your search
+1
-1
@@ -1,4 +1,4 @@
|
||||
# The test runner source for UI tests
|
||||
WEB_COMMITID=096ba683b62ab19ee7ff5dabe1225506ef50713c
|
||||
WEB_COMMITID=170677dc06b1dfab57bc4bea2dac8a5cd78b2784
|
||||
WEB_BRANCH=stable-7.1
|
||||
|
||||
+254
-139
@@ -89,6 +89,30 @@ event = {
|
||||
},
|
||||
}
|
||||
|
||||
OPENCLOUD_STORAGES = ["posix", "decomposed"]
|
||||
API_TEST_NIGHTLY_CI_MATRIX = {
|
||||
"posix": [
|
||||
{
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": True,
|
||||
},
|
||||
{
|
||||
"withRemotePhp": True,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
],
|
||||
"decomposed": [
|
||||
{
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
{
|
||||
"withRemotePhp": True,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# configuration
|
||||
config = {
|
||||
"cs3ApiTests": {
|
||||
@@ -121,7 +145,8 @@ config = {
|
||||
"apiSettings",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
"emailNeeded": True,
|
||||
"extraTestEnvironment": {
|
||||
"EMAIL_HOST": "email",
|
||||
@@ -144,14 +169,14 @@ config = {
|
||||
#"collaborativePosix",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
},
|
||||
"graphUserGroup": {
|
||||
"suites": [
|
||||
"apiGraphUserGroup",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
},
|
||||
"spaces": {
|
||||
"suites": [
|
||||
@@ -176,12 +201,14 @@ config = {
|
||||
"apiSearch1",
|
||||
],
|
||||
"skip": False,
|
||||
"nightlyOpenSearch": True,
|
||||
},
|
||||
"search2": {
|
||||
"suites": [
|
||||
"apiSearch2",
|
||||
],
|
||||
"skip": False,
|
||||
"nightlyOpenSearch": True,
|
||||
},
|
||||
"sharingNg": {
|
||||
"suites": [
|
||||
@@ -209,7 +236,7 @@ config = {
|
||||
"apiNotification",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"emailNeeded": True,
|
||||
"extraTestEnvironment": {
|
||||
"EMAIL_HOST": "email",
|
||||
@@ -245,13 +272,14 @@ config = {
|
||||
],
|
||||
"skip": False,
|
||||
"tikaNeeded": True,
|
||||
"nightlyOpenSearch": True,
|
||||
},
|
||||
"ocm": {
|
||||
"suites": [
|
||||
"apiOcm",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"federationServer": True,
|
||||
"emailNeeded": True,
|
||||
"extraTestEnvironment": {
|
||||
@@ -287,14 +315,15 @@ config = {
|
||||
"apiAuthApp",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
"cliCommands": {
|
||||
"suites": [
|
||||
"cliCommands",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"antivirusNeeded": True,
|
||||
"extraServerEnvironment": {
|
||||
"ANTIVIRUS_SCANNER_TYPE": "clamav",
|
||||
@@ -309,7 +338,7 @@ config = {
|
||||
"apiTenancy",
|
||||
],
|
||||
"skip": False,
|
||||
"withRemotePhp": [False],
|
||||
"withRemotePhp": False,
|
||||
"ldapNeeded": True,
|
||||
"extraTestEnvironment": {
|
||||
"USE_PREPARED_LDAP_USERS": True,
|
||||
@@ -405,7 +434,7 @@ config = {
|
||||
# NOTE: need to be updated if new production releases are determined
|
||||
"tags": ["2.0", "4.0", "7.2"],
|
||||
# NOTE: need to be set to true if patch releases are made from stable-X-branches
|
||||
"skip_rolling": False,
|
||||
"skip_rolling": True,
|
||||
"skip_daily": True,
|
||||
"repo": docker_repo_slug,
|
||||
"build_type": "production",
|
||||
@@ -790,30 +819,7 @@ def testOpencloud(ctx):
|
||||
],
|
||||
"environment": CI_HTTP_PROXY_ENV,
|
||||
},
|
||||
{
|
||||
"name": "open-search",
|
||||
"image": OPEN_SEARCH,
|
||||
"detach": True,
|
||||
"environment": {
|
||||
"discovery.type": "single-node",
|
||||
"DISABLE_INSTALL_DEMO_CONFIG": True,
|
||||
"DISABLE_SECURITY_PLUGIN": True,
|
||||
},
|
||||
"entrypoint": ["/usr/share/opensearch/opensearch-docker-entrypoint.sh", "opensearch"],
|
||||
},
|
||||
{
|
||||
"name": "wait-for-open-search",
|
||||
"image": OC_CI_ALPINE,
|
||||
"commands": [
|
||||
"bash -c '" +
|
||||
"until curl -sS \"http://open-search:9200/_cat/health?h=status\" | grep \"green\\|yellow\"; do\n" +
|
||||
" echo \"Waiting for http://open-search:9200 to be healthy...\"\n" +
|
||||
" sleep 5\n" +
|
||||
"done\n" +
|
||||
"echo \"http://open-search:9200 healthy...\"\n" +
|
||||
"'",
|
||||
],
|
||||
},
|
||||
] + waitForOpenSearch() + [
|
||||
{
|
||||
"name": "test",
|
||||
"image": OC_CI_GOLANG,
|
||||
@@ -845,6 +851,7 @@ def testOpencloud(ctx):
|
||||
pipeline = {
|
||||
"name": "test-lint-unit",
|
||||
"steps": steps,
|
||||
"services": openSearchService(),
|
||||
"when": [
|
||||
event["base"],
|
||||
event["cron"],
|
||||
@@ -1185,6 +1192,77 @@ def wopiValidatorTests(ctx, storage, wopiServerType):
|
||||
])
|
||||
return [pipeline]
|
||||
|
||||
def build_api_test_workflow_matrix(ctx, storage, suite_cfg, default_cfg):
|
||||
"""
|
||||
Generates a matrix of feature combinations to run API tests with.
|
||||
|
||||
Args:
|
||||
ctx: woodpecker context
|
||||
storage: opencloud storage type
|
||||
suite_cfg: suite config. E.g.: config["localApiTests"]["spaces"]
|
||||
default_cfg: default suite config values
|
||||
|
||||
Returns:
|
||||
A matrix to run API tests with. E.g.:
|
||||
[
|
||||
{
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
{
|
||||
"withRemotePhp": True,
|
||||
"enableWatchFs": False,
|
||||
},
|
||||
]
|
||||
"""
|
||||
|
||||
# default for PRs and commit push events
|
||||
matrices = [{
|
||||
"withRemotePhp": default_cfg["withRemotePhp"],
|
||||
"enableWatchFs": default_cfg["enableWatchFs"],
|
||||
}]
|
||||
if ctx.build.event == "cron":
|
||||
matrices = API_TEST_NIGHTLY_CI_MATRIX[storage]
|
||||
|
||||
override_with_remote_php = None
|
||||
override_enable_watch_fs = None
|
||||
if "withRemotePhp" in suite_cfg:
|
||||
override_with_remote_php = suite_cfg["withRemotePhp"]
|
||||
if "enableWatchFs" in suite_cfg:
|
||||
override_enable_watch_fs = suite_cfg["enableWatchFs"]
|
||||
|
||||
workflow_metrices = []
|
||||
for m in matrices:
|
||||
matrix = {
|
||||
"withRemotePhp": m["withRemotePhp"],
|
||||
"enableWatchFs": m["enableWatchFs"],
|
||||
"openSearch": False,
|
||||
}
|
||||
if override_with_remote_php != None:
|
||||
matrix["withRemotePhp"] = override_with_remote_php
|
||||
if override_enable_watch_fs != None:
|
||||
matrix["enableWatchFs"] = override_enable_watch_fs
|
||||
|
||||
base = {
|
||||
"withRemotePhp": matrix["withRemotePhp"],
|
||||
"enableWatchFs": matrix["enableWatchFs"],
|
||||
}
|
||||
if matrix not in workflow_metrices and base in matrices:
|
||||
workflow_metrices.append(matrix)
|
||||
|
||||
# Add an OpenSearch search-engine variant for nightly running search tests,
|
||||
# or on demand when "opensearch" is specified in the PR title
|
||||
run_open_search = ctx.build.event == "cron" or "opensearch" in ctx.build.title.lower()
|
||||
if run_open_search and storage == "posix" and suite_cfg.get("nightlyOpenSearch", False):
|
||||
os_matrix = {
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
"openSearch": True,
|
||||
}
|
||||
if os_matrix not in workflow_metrices:
|
||||
workflow_metrices.append(os_matrix)
|
||||
return workflow_metrices
|
||||
|
||||
def localApiTestPipeline(ctx):
|
||||
pipelines = []
|
||||
|
||||
@@ -1200,9 +1278,10 @@ def localApiTestPipeline(ctx):
|
||||
"federationServer": False,
|
||||
"collaborationServiceNeeded": False,
|
||||
"extraCollaborationEnvironment": {},
|
||||
"withRemotePhp": [False],
|
||||
"enableWatchFs": [False],
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
"ldapNeeded": False,
|
||||
"nightlyOpenSearch": False,
|
||||
}
|
||||
|
||||
if "localApiTests" in config:
|
||||
@@ -1212,68 +1291,80 @@ def localApiTestPipeline(ctx):
|
||||
for item in defaults:
|
||||
params[item] = matrix[item] if item in matrix else defaults[item]
|
||||
|
||||
if ctx.build.event == "cron":
|
||||
params["storages"] = OPENCLOUD_STORAGES
|
||||
|
||||
# skip CLI tests in nightly pipeline
|
||||
if name.startswith("cli"):
|
||||
continue
|
||||
|
||||
# use decomposed storage if specified in the PR title
|
||||
# run CLI tests only with decomposed storage
|
||||
if "[decomposed]" in ctx.build.title.lower() or name.startswith("cli"):
|
||||
params["storages"] = ["decomposed"]
|
||||
|
||||
if ctx.build.event == "cron":
|
||||
params["withRemotePhp"] = [True, False]
|
||||
params["enableWatchFs"] = [True, False]
|
||||
|
||||
# override withRemotePhp if specified in the suite config
|
||||
if "withRemotePhp" in matrix:
|
||||
params["withRemotePhp"] = matrix["withRemotePhp"]
|
||||
|
||||
for storage in params["storages"]:
|
||||
for run_with_remote_php in params["withRemotePhp"]:
|
||||
for run_with_watch_fs_enabled in params["enableWatchFs"]:
|
||||
pipeline_name = "test-API"
|
||||
if name.startswith("cli"):
|
||||
pipeline_name = "test-CLI"
|
||||
pipeline_name += "-%s" % name
|
||||
if not run_with_remote_php:
|
||||
pipeline_name += "-withoutRemotePhp"
|
||||
pipeline_name += "-%s" % storage
|
||||
if run_with_watch_fs_enabled:
|
||||
pipeline_name += "-watchfs"
|
||||
matrices = build_api_test_workflow_matrix(ctx, storage, matrix, defaults)
|
||||
for m in matrices:
|
||||
run_with_remote_php = m["withRemotePhp"]
|
||||
run_with_watch_fs = m["enableWatchFs"]
|
||||
run_with_open_search = m["openSearch"]
|
||||
|
||||
pipeline = {
|
||||
"name": pipeline_name,
|
||||
"steps": skipCheckStep(ctx, "acceptance-tests") + evaluateWorkflowStep() + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
||||
(tikaService() if params["tikaNeeded"] else []) +
|
||||
(waitForWebOffices(["https://collabora:9980", "https://onlyoffice", "http://fakeoffice:8080"]) if params["collaborationServiceNeeded"] else []) +
|
||||
(waitForClamavService() if params["antivirusNeeded"] else []) +
|
||||
(waitForEmailService() if params["emailNeeded"] else []) +
|
||||
(ldapService() if params["ldapNeeded"] else []) +
|
||||
(waitForLdapService() if params["ldapNeeded"] else []) +
|
||||
opencloudServer(
|
||||
storage,
|
||||
extra_server_environment = params["extraServerEnvironment"],
|
||||
with_wrapper = True,
|
||||
tika_enabled = params["tikaNeeded"],
|
||||
watch_fs_enabled = run_with_watch_fs_enabled,
|
||||
) +
|
||||
(opencloudServer(storage, deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"], watch_fs_enabled = run_with_watch_fs_enabled) if params["federationServer"] else []) +
|
||||
((wopiCollaborationService("fakeoffice") + wopiCollaborationService("collabora") + wopiCollaborationService("onlyoffice")) if params["collaborationServiceNeeded"] else []) +
|
||||
(openCloudHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) if params["collaborationServiceNeeded"] else []) +
|
||||
localApiTest(params["suites"], storage, params["extraTestEnvironment"], run_with_remote_php) +
|
||||
logRequests(),
|
||||
"services": (emailService() if params["emailNeeded"] else []) +
|
||||
(clamavService() if params["antivirusNeeded"] else []) +
|
||||
((fakeOffice() + collaboraService() + onlyofficeService()) if params["collaborationServiceNeeded"] else []),
|
||||
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
||||
"when": [
|
||||
event["base"],
|
||||
event["cron"],
|
||||
event["pull_request"],
|
||||
],
|
||||
}
|
||||
prefixStepCommands(pipeline, [
|
||||
". ./.woodpecker.env",
|
||||
'[ "$SKIP_WORKFLOW" = "true" ] && exit 0',
|
||||
])
|
||||
pipelines.append(pipeline)
|
||||
pipeline_name = "test-API"
|
||||
if name.startswith("cli"):
|
||||
pipeline_name = "test-CLI"
|
||||
pipeline_name += "-%s" % name
|
||||
pipeline_name += "-%s" % storage
|
||||
if run_with_remote_php:
|
||||
pipeline_name += "-withRemotePhp"
|
||||
if run_with_watch_fs:
|
||||
pipeline_name += "-watchfs"
|
||||
if run_with_open_search:
|
||||
pipeline_name += "-opensearch"
|
||||
|
||||
server_environment = dict(params["extraServerEnvironment"])
|
||||
if run_with_open_search:
|
||||
server_environment["SEARCH_ENGINE_TYPE"] = "open-search"
|
||||
server_environment["SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES"] = "http://open-search:9200"
|
||||
|
||||
pipeline = {
|
||||
"name": pipeline_name,
|
||||
"steps": skipCheckStep(ctx, "acceptance-tests") + evaluateWorkflowStep() + restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
||||
(tikaService() if params["tikaNeeded"] else []) +
|
||||
(waitForOpenSearch() if run_with_open_search else []) +
|
||||
(waitForWebOffices(["https://collabora:9980", "https://onlyoffice", "http://fakeoffice:8080"]) if params["collaborationServiceNeeded"] else []) +
|
||||
(waitForClamavService() if params["antivirusNeeded"] else []) +
|
||||
(waitForEmailService() if params["emailNeeded"] else []) +
|
||||
(ldapService() if params["ldapNeeded"] else []) +
|
||||
(waitForLdapService() if params["ldapNeeded"] else []) +
|
||||
opencloudServer(
|
||||
storage,
|
||||
extra_server_environment = server_environment,
|
||||
with_wrapper = True,
|
||||
tika_enabled = params["tikaNeeded"],
|
||||
watch_fs_enabled = run_with_watch_fs,
|
||||
) +
|
||||
(opencloudServer(storage, deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"], watch_fs_enabled = run_with_watch_fs) if params["federationServer"] else []) +
|
||||
((wopiCollaborationService("fakeoffice") + wopiCollaborationService("collabora") + wopiCollaborationService("onlyoffice")) if params["collaborationServiceNeeded"] else []) +
|
||||
(openCloudHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) if params["collaborationServiceNeeded"] else []) +
|
||||
localApiTest(params["suites"], storage, params["extraTestEnvironment"], run_with_remote_php) +
|
||||
logRequests(),
|
||||
"services": (emailService() if params["emailNeeded"] else []) +
|
||||
(clamavService() if params["antivirusNeeded"] else []) +
|
||||
(openSearchService() if run_with_open_search else []) +
|
||||
((fakeOffice() + collaboraService() + onlyofficeService()) if params["collaborationServiceNeeded"] else []),
|
||||
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
||||
"when": [
|
||||
event["base"],
|
||||
event["cron"],
|
||||
event["pull_request"],
|
||||
],
|
||||
}
|
||||
prefixStepCommands(pipeline, [
|
||||
". ./.woodpecker.env",
|
||||
'[ "$SKIP_WORKFLOW" = "true" ] && exit 0',
|
||||
])
|
||||
pipelines.append(pipeline)
|
||||
return pipelines
|
||||
|
||||
def localApiTest(suites, storage = "decomposed", extra_environment = {}, with_remote_php = False):
|
||||
@@ -1317,8 +1408,8 @@ def localApiTest(suites, storage = "decomposed", extra_environment = {}, with_re
|
||||
|
||||
def coreApiTestPipeline(ctx):
|
||||
defaults = {
|
||||
"withRemotePhp": [False],
|
||||
"enableWatchFs": [False],
|
||||
"withRemotePhp": False,
|
||||
"enableWatchFs": False,
|
||||
"storages": ["posix"],
|
||||
"numberOfParts": 7,
|
||||
"skipExceptParts": [],
|
||||
@@ -1335,63 +1426,60 @@ def coreApiTestPipeline(ctx):
|
||||
for item in defaults:
|
||||
params[item] = matrix[item] if item in matrix else defaults[item]
|
||||
|
||||
if ctx.build.event == "cron":
|
||||
params["storages"] = OPENCLOUD_STORAGES
|
||||
|
||||
# use decomposed storage if specified in the PR title
|
||||
if "[decomposed]" in ctx.build.title.lower():
|
||||
params["storages"] = ["decomposed"]
|
||||
|
||||
if ctx.build.event == "cron":
|
||||
params["withRemotePhp"] = [True, False]
|
||||
params["enableWatchFs"] = [True, False]
|
||||
|
||||
# override withRemotePhp if specified in the suite config
|
||||
if "withRemotePhp" in matrix:
|
||||
params["withRemotePhp"] = matrix["withRemotePhp"]
|
||||
|
||||
debugParts = params["skipExceptParts"]
|
||||
debugPartsEnabled = (len(debugParts) != 0)
|
||||
|
||||
for storage in params["storages"]:
|
||||
for runPart in range(1, params["numberOfParts"] + 1):
|
||||
for run_with_remote_php in params["withRemotePhp"]:
|
||||
for run_with_watch_fs_enabled in params["enableWatchFs"]:
|
||||
if not debugPartsEnabled or (debugPartsEnabled and runPart in debugParts):
|
||||
pipeline_name = "test-Core-API-%s" % runPart
|
||||
if not run_with_remote_php:
|
||||
pipeline_name += "-withoutRemotePhp"
|
||||
pipeline_name += "-%s" % storage
|
||||
if run_with_watch_fs_enabled:
|
||||
pipeline_name += "-watchfs"
|
||||
matrices = build_api_test_workflow_matrix(ctx, storage, matrix, defaults)
|
||||
for m in matrices:
|
||||
run_with_remote_php = m["withRemotePhp"]
|
||||
run_with_watch_fs = m["enableWatchFs"]
|
||||
if not debugPartsEnabled or (debugPartsEnabled and runPart in debugParts):
|
||||
pipeline_name = "test-Core-API-%s" % runPart
|
||||
pipeline_name += "-%s" % storage
|
||||
if run_with_remote_php:
|
||||
pipeline_name += "-withRemotePhp"
|
||||
if run_with_watch_fs:
|
||||
pipeline_name += "-watchfs"
|
||||
|
||||
pipeline = {
|
||||
"name": pipeline_name,
|
||||
"steps": skipCheckStep(ctx, "acceptance-tests") +
|
||||
evaluateWorkflowStep() +
|
||||
restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
||||
opencloudServer(
|
||||
storage,
|
||||
with_wrapper = True,
|
||||
watch_fs_enabled = run_with_watch_fs_enabled,
|
||||
) +
|
||||
coreApiTest(
|
||||
runPart,
|
||||
params["numberOfParts"],
|
||||
run_with_remote_php,
|
||||
storage,
|
||||
) +
|
||||
logRequests(),
|
||||
"services": redisForOCStorage(storage),
|
||||
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
||||
"when": [
|
||||
event["base"],
|
||||
event["cron"],
|
||||
event["pull_request"],
|
||||
],
|
||||
}
|
||||
prefixStepCommands(pipeline, [
|
||||
". ./.woodpecker.env",
|
||||
'[ "$SKIP_WORKFLOW" = "true" ] && exit 0',
|
||||
])
|
||||
pipelines.append(pipeline)
|
||||
pipeline = {
|
||||
"name": pipeline_name,
|
||||
"steps": skipCheckStep(ctx, "acceptance-tests") +
|
||||
evaluateWorkflowStep() +
|
||||
restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
||||
opencloudServer(
|
||||
storage,
|
||||
with_wrapper = True,
|
||||
watch_fs_enabled = run_with_watch_fs,
|
||||
) +
|
||||
coreApiTest(
|
||||
runPart,
|
||||
params["numberOfParts"],
|
||||
run_with_remote_php,
|
||||
storage,
|
||||
) +
|
||||
logRequests(),
|
||||
"services": redisForOCStorage(storage),
|
||||
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
||||
"when": [
|
||||
event["base"],
|
||||
event["cron"],
|
||||
event["pull_request"],
|
||||
],
|
||||
}
|
||||
prefixStepCommands(pipeline, [
|
||||
". ./.woodpecker.env",
|
||||
'[ "$SKIP_WORKFLOW" = "true" ] && exit 0',
|
||||
])
|
||||
pipelines.append(pipeline)
|
||||
return pipelines
|
||||
|
||||
def coreApiTest(part_number = 1, number_of_parts = 1, with_remote_php = False, storage = "posix"):
|
||||
@@ -3253,6 +3341,33 @@ def tikaService():
|
||||
],
|
||||
}]
|
||||
|
||||
def openSearchService():
|
||||
return [{
|
||||
"name": "open-search",
|
||||
"image": OPEN_SEARCH,
|
||||
"environment": {
|
||||
"discovery.type": "single-node",
|
||||
"DISABLE_INSTALL_DEMO_CONFIG": True,
|
||||
"DISABLE_SECURITY_PLUGIN": True,
|
||||
},
|
||||
"entrypoint": ["/usr/share/opensearch/opensearch-docker-entrypoint.sh", "opensearch"],
|
||||
}]
|
||||
|
||||
def waitForOpenSearch():
|
||||
return [{
|
||||
"name": "wait-for-open-search",
|
||||
"image": OC_CI_ALPINE,
|
||||
"commands": [
|
||||
"bash -c '" +
|
||||
"until curl -sS \"http://open-search:9200/_cat/health?h=status\" | grep \"green\\|yellow\"; do\n" +
|
||||
" echo \"Waiting for http://open-search:9200 to be healthy...\"\n" +
|
||||
" sleep 5\n" +
|
||||
"done\n" +
|
||||
"echo \"http://open-search:9200 healthy...\"\n" +
|
||||
"'",
|
||||
],
|
||||
}]
|
||||
|
||||
def logRequests():
|
||||
return [{
|
||||
"name": "api-test-failure-logs",
|
||||
|
||||
@@ -1,5 +1,60 @@
|
||||
# Changelog
|
||||
|
||||
## [7.2.4](https://github.com/opencloud-eu/opencloud/releases/tag/v7.2.4) - 2026-08-20
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
@aduffeck, @rhafer, @v-scharf
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- [stable-7.2] fix(search): fix open-search kql field compilation [[#3375](https://github.com/opencloud-eu/opencloud/pull/3375)]
|
||||
- [stable-7.2] fix(search): ack every event the index service handles [[#3374](https://github.com/opencloud-eu/opencloud/pull/3374)]
|
||||
- [stable-7.2] fix(postprocessing): retry publishing events [[#3370](https://github.com/opencloud-eu/opencloud/pull/3370)]
|
||||
- [stable-7.2] Only log a debug message when an item is still in processing state [[#3369](https://github.com/opencloud-eu/opencloud/pull/3369)]
|
||||
- [stable-7.2] backport fixes for the opencloud posixfs command [[#3367](https://github.com/opencloud-eu/opencloud/pull/3367)]
|
||||
- [stable-7.2] Expose upload status and message [[#3365](https://github.com/opencloud-eu/opencloud/pull/3365)]
|
||||
- Fix opensearch space scope [stable-7.2] [[#3351](https://github.com/opencloud-eu/opencloud/pull/3351)]
|
||||
- [stable-7.2] fix(posixfs scan): Setup logger for scan command [[#3195](https://github.com/opencloud-eu/opencloud/pull/3195)]
|
||||
|
||||
### ✅ Tests
|
||||
|
||||
- [stable-7.2] api-tests against open-search in stable [[#3353](https://github.com/opencloud-eu/opencloud/pull/3353)]
|
||||
|
||||
## [7.2.3](https://github.com/opencloud-eu/opencloud/releases/tag/v7.2.3) - 2026-08-06
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
@aduffeck, @v-scharf
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix missing favorite flag on opensearch hits [[#3256](https://github.com/opencloud-eu/opencloud/pull/3256)]
|
||||
|
||||
## [7.2.2](https://github.com/opencloud-eu/opencloud/releases/tag/v7.2.2) - 2026-07-14
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
@kulmann, @v-scharf
|
||||
|
||||
### 📦️ Dependencies
|
||||
|
||||
- [full-ci] chore: bump web to v7.1.4 [[#3122](https://github.com/opencloud-eu/opencloud/pull/3122)]
|
||||
|
||||
## [7.2.1](https://github.com/opencloud-eu/opencloud/releases/tag/v7.2.1) - 2026-07-06
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
@JammingBen, @v-scharf
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix warming up the id cache for the user storage [[#3072](https://github.com/opencloud-eu/opencloud/pull/3072)]
|
||||
|
||||
### 📦️ Dependencies
|
||||
|
||||
- [full-ci] chore: bump web to v7.1.3 [[#3052](https://github.com/opencloud-eu/opencloud/pull/3052)]
|
||||
|
||||
## [7.2.0](https://github.com/opencloud-eu/opencloud/releases/tag/v7.2.0) - 2026-06-24
|
||||
|
||||
### ❤️ Thanks to all contributors! ❤️
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
Bugfix: Retry publishing postprocessing events before giving up
|
||||
|
||||
A single transient failure while publishing an event to the event system took
|
||||
the whole server down. The postprocessing service treated every publish error
|
||||
as fatal and called log.Fatal, which exits the process and so also stopped all
|
||||
the other services running in the same binary. A burst of uploads was enough to
|
||||
run into one nats publish timeout and lose the server with it.
|
||||
|
||||
Publishing is now retried using the same exponential backoff that is already
|
||||
used for failed postprocessing steps. Between the attempts the source event is
|
||||
marked as in progress and the backoff is capped at half the ack wait, so the
|
||||
event should not be redelivered to another worker while we are still retrying.
|
||||
The number of retries is configurable via POSTPROCESSING_PUBLISH_MAX_RETRIES.
|
||||
|
||||
Should all attempts fail, the source event is no longer acknowledged. Before,
|
||||
the event was acknowledged even though its successor was never published, so
|
||||
the upload was left half processed in the store and did not recover on restart.
|
||||
|
||||
https://github.com/opencloud-eu/opencloud/issues/3271
|
||||
https://github.com/opencloud-eu/opencloud/issues/2232
|
||||
https://github.com/opencloud-eu/opencloud/issues/2422
|
||||
@@ -64,7 +64,7 @@ require (
|
||||
github.com/open-policy-agent/opa v1.15.2
|
||||
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d
|
||||
github.com/opencloud-eu/reva/v2 v2.46.6
|
||||
github.com/opencloud-eu/reva/v2 v2.46.8
|
||||
github.com/opensearch-project/opensearch-go/v4 v4.6.0
|
||||
github.com/orcaman/concurrent-map v1.0.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
@@ -75,6 +75,7 @@ require (
|
||||
github.com/rogpeppe/go-internal v1.14.1
|
||||
github.com/rs/cors v1.11.1
|
||||
github.com/rs/zerolog v1.35.0
|
||||
github.com/shamaton/msgpack/v2 v2.4.1
|
||||
github.com/sirupsen/logrus v1.9.4
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
@@ -84,7 +85,6 @@ require (
|
||||
github.com/test-go/testify v1.1.4
|
||||
github.com/testcontainers/testcontainers-go v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/opensearch v0.42.0
|
||||
github.com/theckman/yacspin v0.13.12
|
||||
github.com/thejerf/suture/v4 v4.0.6
|
||||
github.com/tidwall/gjson v1.19.0
|
||||
github.com/tidwall/sjson v1.2.5
|
||||
@@ -345,7 +345,6 @@ require (
|
||||
github.com/sergi/go-diff v1.4.0 // indirect
|
||||
github.com/sethvargo/go-diceware v0.5.0 // indirect
|
||||
github.com/sethvargo/go-password v0.3.1 // indirect
|
||||
github.com/shamaton/msgpack/v2 v2.4.1 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
|
||||
github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
|
||||
github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 // indirect
|
||||
|
||||
@@ -948,8 +948,8 @@ github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+l
|
||||
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d h1:JcqGDiyrcaQwVyV861TUyQgO7uEmsjkhfm7aQd84dOw=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q=
|
||||
github.com/opencloud-eu/reva/v2 v2.46.6 h1:Q20wE4A2OpWVyDnRaC4KW1+nzo+DQaK+RK5y3RFnWIg=
|
||||
github.com/opencloud-eu/reva/v2 v2.46.6/go.mod h1:RoFQt+u7edxwzHr1IZ2Y6VaDinMiRPQupAvMBy3WVmE=
|
||||
github.com/opencloud-eu/reva/v2 v2.46.8 h1:stqn5VfGiIkR7+b/BIsFAqbLbrzSARu43I/t/6G9X3Q=
|
||||
github.com/opencloud-eu/reva/v2 v2.46.8/go.mod h1:RoFQt+u7edxwzHr1IZ2Y6VaDinMiRPQupAvMBy3WVmE=
|
||||
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
|
||||
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
@@ -1195,8 +1195,6 @@ github.com/testcontainers/testcontainers-go/modules/opensearch v0.42.0 h1:lXPr5N
|
||||
github.com/testcontainers/testcontainers-go/modules/opensearch v0.42.0/go.mod h1:KJ+m9onB785g9VDRDuhvUvsbkRjnFp3Uz+CHoHpPieA=
|
||||
github.com/thanhpk/randstr v1.0.6 h1:psAOktJFD4vV9NEVb3qkhRSMvYh4ORRaj1+w/hn4B+o=
|
||||
github.com/thanhpk/randstr v1.0.6/go.mod h1:M/H2P1eNLZzlDwAzpkkkUvoyNNMbzRGhESZuEQk3r0U=
|
||||
github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRzJr4=
|
||||
github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg=
|
||||
github.com/thejerf/suture/v4 v4.0.6 h1:QsuCEsCqb03xF9tPAsWAj8QOAJBgQI1c0VqJNaingg8=
|
||||
github.com/thejerf/suture/v4 v4.0.6/go.mod h1:gu9Y4dXNUWFrByqRt30Rm9/UZ0wzRSt9AJS6xu/ZGxU=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
|
||||
+273
-446
@@ -1,17 +1,16 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/pkg/x/path/filepathx"
|
||||
storageUsersParser "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/services/storage-users/pkg/event"
|
||||
"github.com/opencloud-eu/opencloud/services/storage-users/pkg/revaconfig"
|
||||
@@ -19,27 +18,10 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/ignore"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/theckman/yacspin"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
)
|
||||
|
||||
// Define the names of the extended attributes we are working with.
|
||||
const (
|
||||
parentIDAttrName = "user.oc.parentid"
|
||||
idAttrName = "user.oc.id"
|
||||
nameAttrName = "user.oc.name"
|
||||
spaceIDAttrName = "user.oc.space.id"
|
||||
ownerIDAttrName = "user.oc.owner.id"
|
||||
)
|
||||
|
||||
var (
|
||||
spinner *yacspin.Spinner
|
||||
restartRequired = false
|
||||
ignorer *ignore.Ignorer
|
||||
)
|
||||
|
||||
type IDCacher interface {
|
||||
@@ -73,11 +55,23 @@ func init() {
|
||||
|
||||
// scanCmd performs a posixfs id cache warmup scan
|
||||
func scanCmd(ocCfg *config.Config) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "scan",
|
||||
scanCmd := &cobra.Command{
|
||||
Use: "scan [path ...]",
|
||||
Short: "Perform a filesystem scan and update the ID and filemetadata cache",
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
Long: `Perform a filesystem scan and update the ID and filemetadata cache.
|
||||
|
||||
You can specify one or more paths to limit the scope of the scan.
|
||||
If no path is provided, the whole storage is checked, starting at the storage root directory.
|
||||
|
||||
The provided arguments determines the scope of the check:
|
||||
- a storage root: the whole storage (all personal and project spaces) is scanned
|
||||
- a space root: only that space is scanned
|
||||
- a file or directory: only that single resource is scanned (and its children, if it is a directory)
|
||||
|
||||
Any specified file or directory must be underneath the storage root directory and if that is not the case,
|
||||
the command is aborted with an error before performing any scanning.`,
|
||||
Args: cobra.ArbitraryArgs,
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
if err := parser.ParseConfig(ocCfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
@@ -94,448 +88,281 @@ func scanCmd(ocCfg *config.Config) *cobra.Command {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// We want to initialize the driver but disable scanfs on boot, so we can trigger it manually afterwards
|
||||
drivers := revaconfig.StorageProviderDrivers(cfg)
|
||||
drivers["posix"] = revaconfig.Posix(cfg, false, false)
|
||||
|
||||
var fsStream events.Stream
|
||||
var err error
|
||||
fsStream, err = event.NewStream(cfg)
|
||||
haltOnError, err := cmd.Flags().GetBool("halt-on-error")
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create event stream for posix driver: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
f, ok := registry.NewFuncs["posix"]
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "posix driver not found in registry\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fs, err := f(drivers["posix"].(map[string]any), fsStream, nil)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s': %v\n", cfg.Driver, err)
|
||||
return err
|
||||
}
|
||||
|
||||
cacher, ok := fs.(IDCacher)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "The posix driver does not expose WarmupIDCache.\n")
|
||||
os.Exit(1)
|
||||
storageRoot := cfg.Drivers.Posix.Root
|
||||
paths := []string{storageRoot}
|
||||
if len(args) > 0 {
|
||||
paths = []string{}
|
||||
for _, v := range args {
|
||||
path := v
|
||||
if !filepath.IsAbs(path) {
|
||||
if v, err := filepath.Abs(path); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to make the specified path %q absolute: %v\n", v, err)
|
||||
os.Exit(1)
|
||||
} else {
|
||||
path = v
|
||||
}
|
||||
}
|
||||
// not ensuring whether the path is under the storage root here, will be done when iterating over them
|
||||
path = filepath.Clean(path)
|
||||
paths = append(paths, path)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("Starting posixfs scan...")
|
||||
err = cacher.WarmupIDCache(cfg.Drivers.Posix.Root, true, false)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Scan failed: %v\n", err)
|
||||
return err
|
||||
var scan func(path string) error = nil
|
||||
{
|
||||
// We want to initialize the driver but disable scanfs on boot, so we can trigger it manually afterwards
|
||||
drivers := revaconfig.StorageProviderDrivers(cfg)
|
||||
drivers["posix"] = revaconfig.Posix(cfg, false, false)
|
||||
|
||||
var fsStream events.Stream
|
||||
var err error
|
||||
fsStream, err = event.NewStream(cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to create event stream for posix driver: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log := logger("posixfs")
|
||||
|
||||
f, ok := registry.NewFuncs["posix"]
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "posix driver not found in registry\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fs, err := f(drivers["posix"].(map[string]any), fsStream, &log)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s': %v\n", cfg.Driver, err)
|
||||
return err
|
||||
}
|
||||
|
||||
cacher, ok := fs.(IDCacher)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "The posix driver does not expose WarmupIDCache.\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
scan = func(path string) error {
|
||||
err := cacher.WarmupIDCache(path, true, false)
|
||||
if err != nil {
|
||||
logFailure("Error scanning path '%s': %v", path, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("Scan completed successfully.")
|
||||
return nil
|
||||
errors := processPosixFsResources(paths, !haltOnError,
|
||||
func(path string) error {
|
||||
fmt.Println("Scanning personal spaces...")
|
||||
return scan(path)
|
||||
},
|
||||
func(path string) error {
|
||||
fmt.Println("Scanning project spaces...")
|
||||
return scan(path)
|
||||
},
|
||||
func(path string) error {
|
||||
fmt.Printf("Scanning space '%s'...\n", path)
|
||||
return scan(path)
|
||||
},
|
||||
func(path string) error {
|
||||
fmt.Printf("Scanning '%s'...\n", path)
|
||||
return scan(path)
|
||||
},
|
||||
)
|
||||
|
||||
if len(errors) == 0 {
|
||||
fmt.Println("Scan completed successfully.")
|
||||
return nil
|
||||
} else {
|
||||
plural := "s"
|
||||
if len(errors) == 1 {
|
||||
plural = ""
|
||||
}
|
||||
verb := "completed"
|
||||
if haltOnError {
|
||||
verb = "aborted"
|
||||
}
|
||||
return fmt.Errorf("scan %s with %d error%s", verb, len(errors), plural)
|
||||
}
|
||||
},
|
||||
}
|
||||
return cmd
|
||||
scanCmd.Flags().BoolP("halt-on-error", "E", false, "Halt at once when an error occurs when processing one of the paths (default behaviour is to keep going and attempt to process all paths).")
|
||||
return scanCmd
|
||||
}
|
||||
|
||||
// consistencyCmd returns a command to check the consistency of the posixfs storage.
|
||||
func consistencyCmd(cfg *config.Config) *cobra.Command {
|
||||
func consistencyCmd(ocCfg *config.Config) *cobra.Command {
|
||||
consCmd := &cobra.Command{
|
||||
Use: "consistency",
|
||||
Short: "check the consistency of the posixfs storage",
|
||||
Use: "consistency [path ...]",
|
||||
Short: "Check the consistency of the posixfs storage",
|
||||
Long: `Check the consistency of the posixfs storage.
|
||||
|
||||
You can specify one or more paths to limit the scope of the check.
|
||||
If no path is provided, the whole storage is checked.
|
||||
|
||||
The provided arguments determines the scope of the check:
|
||||
- a storage root: the whole storage (all personal and project spaces) is checked
|
||||
- a space root: only that space is checked
|
||||
- a file or directory: only that single entity is checked (and its children, if it is a directory)`,
|
||||
Args: cobra.ArbitraryArgs,
|
||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
|
||||
if err := parser.ParseConfig(ocCfg, true); err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
|
||||
// Parse storage users config
|
||||
ocCfg.StorageUsers.Commons = ocCfg.Commons
|
||||
|
||||
return configlog.ReturnFatal(storageUsersParser.ParseConfig(ocCfg.StorageUsers))
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return checkPosixfsConsistency(cmd, cfg)
|
||||
cfg := ocCfg.StorageUsers
|
||||
if len(args) == 0 {
|
||||
args = []string{cfg.Drivers.Posix.Root}
|
||||
}
|
||||
log := logger("posixfs")
|
||||
recalculateChecksums, err := cmd.Flags().GetBool("fix-checksums")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
drivers := revaconfig.StorageProviderDrivers(cfg)
|
||||
drivers["posix"] = revaconfig.Posix(cfg, false, false)
|
||||
opts, err := options.New(drivers["posix"].(map[string]any))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ignorer := ignore.NewIgnorer(opts, &log)
|
||||
|
||||
checker := &consistencyChecker{
|
||||
cfg: cfg,
|
||||
ignorer: ignorer,
|
||||
recalculateChecksums: recalculateChecksums,
|
||||
}
|
||||
|
||||
return checker.Check(args)
|
||||
},
|
||||
}
|
||||
consCmd.Flags().StringP("root", "r", "", "Path to the root directory of the posixfs storage")
|
||||
_ = consCmd.MarkFlagRequired("root")
|
||||
|
||||
consCmd.Flags().Bool("fix-checksums", false, "Recalculate and fix the file checksums. This reads every file and can be slow on large storages.")
|
||||
return consCmd
|
||||
}
|
||||
|
||||
// checkPosixfsConsistency checks the consistency of the posixfs storage.
|
||||
func checkPosixfsConsistency(cmd *cobra.Command, cfg *config.Config) error {
|
||||
rootPath, _ := cmd.Flags().GetString("root")
|
||||
indexesPath := filepath.Join(rootPath, "indexes")
|
||||
|
||||
opt, _ := options.New(map[string]interface{}{
|
||||
"root": rootPath,
|
||||
})
|
||||
log := zerolog.Nop()
|
||||
ignorer = ignore.NewIgnorer(opt, &log)
|
||||
|
||||
_, err := os.Stat(indexesPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("consistency check failed: '%s' is not a posixfs root", rootPath)
|
||||
}
|
||||
return fmt.Errorf("error accessing '%s': %w", indexesPath, err)
|
||||
}
|
||||
|
||||
spinnerCfg := yacspin.Config{
|
||||
Frequency: 100 * time.Millisecond,
|
||||
CharSet: yacspin.CharSets[11],
|
||||
StopCharacter: "✓",
|
||||
StopColors: []string{"fgGreen"},
|
||||
StopFailCharacter: "✗",
|
||||
StopFailColors: []string{"fgRed"},
|
||||
}
|
||||
|
||||
spinner, err = yacspin.New(spinnerCfg)
|
||||
err = spinner.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating spinner: %w", err)
|
||||
}
|
||||
|
||||
checkSpaces(filepath.Join(rootPath, "users"))
|
||||
spinner.Suffix(" Personal spaces check ")
|
||||
spinner.StopMessage("completed\n")
|
||||
spinner.Stop()
|
||||
|
||||
checkSpaces(filepath.Join(rootPath, "projects"))
|
||||
spinner.Suffix(" Project spaces check ")
|
||||
spinner.StopMessage("completed")
|
||||
spinner.Stop()
|
||||
|
||||
if restartRequired {
|
||||
fmt.Println("\n\n ⚠️ Please restart your openCloud instance to apply changes.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkSpaces(basePath string) {
|
||||
dirEntries, err := os.ReadDir(basePath)
|
||||
if err != nil {
|
||||
spinner.Message(fmt.Sprintf("Error reading spaces directory '%s'\n", basePath))
|
||||
spinner.StopFail()
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
if entry.IsDir() {
|
||||
fullPath := filepath.Join(basePath, entry.Name())
|
||||
checkSpace(fullPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkSpace(spacePath string) {
|
||||
spinner.Message("")
|
||||
spinner.Suffix(fmt.Sprintf(" Checking space '%s'", spacePath))
|
||||
|
||||
info, err := os.Stat(spacePath)
|
||||
if err != nil {
|
||||
logFailure("Error accessing path '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
if !info.IsDir() {
|
||||
logFailure("Error: The provided path '%s' is not a directory\n", spacePath)
|
||||
return
|
||||
}
|
||||
|
||||
spaceID, err := xattr.Get(spacePath, spaceIDAttrName)
|
||||
if err != nil || len(spaceID) == 0 {
|
||||
logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, spaceIDAttrName)
|
||||
return
|
||||
}
|
||||
|
||||
checkSpaceID(spacePath)
|
||||
checkNodeIDs(spacePath)
|
||||
}
|
||||
|
||||
func checkSpaceID(spacePath string) {
|
||||
spinner.Message(" - checking space ID uniqueness")
|
||||
|
||||
entries, uniqueIDs, oldestEntry, err := gatherAttributes(spacePath)
|
||||
if err != nil {
|
||||
logFailure("Failed to gather attributes: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if len(uniqueIDs) > 1 {
|
||||
spinner.Pause()
|
||||
fmt.Println("\n ⚠ Multiple space IDs found:")
|
||||
for id := range uniqueIDs {
|
||||
fmt.Printf(" - %s\n", id)
|
||||
}
|
||||
|
||||
fmt.Printf("\n ⏳ Oldest entry is '%s' (modified on %s).\n",
|
||||
filepath.Base(oldestEntry.Path), oldestEntry.ModTime.Format(time.RFC1123))
|
||||
|
||||
targetID := oldestEntry.ParentID
|
||||
fmt.Printf(" ✅ Proposed target Parent ID: %s\n", targetID)
|
||||
|
||||
fmt.Printf("\n Do you want to unify all parent IDs to '%s'? This will modify %d entries, the directory, and the user index. (y/N): ", targetID, len(entries))
|
||||
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
input, _ := reader.ReadString('\n')
|
||||
input = strings.TrimSpace(strings.ToLower(input))
|
||||
|
||||
if input != "y" {
|
||||
spinner.Unpause()
|
||||
logFailure("Operation cancelled by user.")
|
||||
return
|
||||
}
|
||||
restartRequired = true
|
||||
|
||||
obsoleteIDs := []string{}
|
||||
for id := range uniqueIDs {
|
||||
if id != targetID {
|
||||
obsoleteIDs = append(obsoleteIDs, id)
|
||||
}
|
||||
}
|
||||
fixSpaceID(spacePath, obsoleteIDs, targetID, entries)
|
||||
spinner.Unpause()
|
||||
}
|
||||
}
|
||||
|
||||
func walkNodes(dir string, parentID string) int {
|
||||
fixes := 0
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
logFailure("Error reading directory '%s': %v", dir, err)
|
||||
return 0
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
fullPath := filepath.Join(dir, entry.Name())
|
||||
|
||||
if ignorer.IsIgnored(fullPath) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if the parent ID attribute matches the expected parent ID, if not, fix it.
|
||||
actualParentID, err := xattr.Get(fullPath, parentIDAttrName)
|
||||
if err != nil || string(actualParentID) != parentID {
|
||||
err = xattr.Set(fullPath, parentIDAttrName, []byte(parentID))
|
||||
if err != nil {
|
||||
logFailure("Failed to fix parent ID for '%s': %v", fullPath, err)
|
||||
} else {
|
||||
spinner.Pause()
|
||||
fmt.Printf(" + Fixed parent ID for '%s'", fullPath)
|
||||
spinner.Unpause()
|
||||
fixes++
|
||||
restartRequired = true
|
||||
}
|
||||
}
|
||||
|
||||
// Check that the name attribute matches the actual name of the file/directory, if not, fix it.
|
||||
nameAttr, err := xattr.Get(fullPath, nameAttrName)
|
||||
if err != nil || string(nameAttr) != entry.Name() {
|
||||
err = xattr.Set(fullPath, nameAttrName, []byte(entry.Name()))
|
||||
if err != nil {
|
||||
logFailure("Failed to fix name attribute for '%s': %v", fullPath, err)
|
||||
} else {
|
||||
spinner.Pause()
|
||||
fmt.Printf(" + Fixed name attribute for '%s'", fullPath)
|
||||
spinner.Unpause()
|
||||
fixes++
|
||||
restartRequired = true
|
||||
}
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
nodeID, err := xattr.Get(fullPath, idAttrName)
|
||||
if err != nil || len(nodeID) == 0 {
|
||||
logFailure("Directory '%s' missing '%s', skipping its children", fullPath, idAttrName)
|
||||
continue
|
||||
}
|
||||
walkNodes(fullPath, string(nodeID))
|
||||
}
|
||||
}
|
||||
return fixes
|
||||
}
|
||||
|
||||
func checkNodeIDs(spacePath string) {
|
||||
spinner.Message(" - checking nodes")
|
||||
|
||||
rootID, err := xattr.Get(spacePath, idAttrName)
|
||||
if err != nil || len(rootID) == 0 {
|
||||
logFailure("Space root '%s' missing '%s' attribute", spacePath, idAttrName)
|
||||
return
|
||||
}
|
||||
|
||||
fixes := walkNodes(spacePath, string(rootID))
|
||||
|
||||
if fixes > 0 {
|
||||
spinner.Pause()
|
||||
fmt.Printf("\n ✓ Fixed %d incorrect node attributes in %s\n", fixes, filepath.Base(spacePath))
|
||||
spinner.Unpause()
|
||||
}
|
||||
}
|
||||
|
||||
func fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) {
|
||||
// Set all parentid attributes to the proper space ID
|
||||
err := setAllParentIDAttributes(entries, targetID)
|
||||
if err != nil {
|
||||
logFailure("an error occurred during file attribute update: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update space ID itself
|
||||
fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), idAttrName, targetID)
|
||||
err = xattr.Set(spacePath, idAttrName, []byte(targetID))
|
||||
if err != nil {
|
||||
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
err = xattr.Set(spacePath, spaceIDAttrName, []byte(targetID))
|
||||
if err != nil {
|
||||
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
|
||||
// update the index
|
||||
err = updateOwnerIndexFile(spacePath, obsoleteIDs)
|
||||
if err != nil {
|
||||
logFailure("Could not update the owner index file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, error) {
|
||||
dirEntries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return nil, nil, EntryInfo{}, fmt.Errorf("failed to read directory: %w", err)
|
||||
}
|
||||
|
||||
var allEntries []EntryInfo
|
||||
uniqueIDs := make(map[string]struct{})
|
||||
var oldestEntry EntryInfo
|
||||
oldestTime := time.Now().Add(100 * 365 * 24 * time.Hour) // Set to a future date to find the oldest entry
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
fullPath := filepath.Join(path, entry.Name())
|
||||
if ignorer.IsIgnored(fullPath) {
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
fmt.Printf(" - Warning: could not stat %s: %v\n", entry.Name(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
parentID, err := xattr.Get(fullPath, parentIDAttrName)
|
||||
if err != nil {
|
||||
continue // Skip if attribute doesn't exist or can't be read
|
||||
}
|
||||
|
||||
entryInfo := EntryInfo{
|
||||
Path: fullPath,
|
||||
ModTime: info.ModTime(),
|
||||
ParentID: string(parentID),
|
||||
}
|
||||
|
||||
allEntries = append(allEntries, entryInfo)
|
||||
uniqueIDs[string(parentID)] = struct{}{}
|
||||
|
||||
if entryInfo.ModTime.Before(oldestTime) {
|
||||
oldestTime = entryInfo.ModTime
|
||||
oldestEntry = entryInfo
|
||||
}
|
||||
}
|
||||
|
||||
return allEntries, uniqueIDs, oldestEntry, nil
|
||||
}
|
||||
|
||||
func setAllParentIDAttributes(entries []EntryInfo, targetID string) error {
|
||||
fmt.Printf(" Setting all parent IDs to '%s':\n", targetID)
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.ParentID == targetID {
|
||||
fmt.Printf(" - Skipping '%s' (already has target ID).\n", filepath.Base(entry.Path))
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Printf(" - Removing all attributes from '%s'. It will be re-assimilated\n", filepath.Base(entry.Path))
|
||||
filepath.WalkDir(entry.Path, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking path '%s': %w", path, err)
|
||||
}
|
||||
|
||||
// Remove all attributes from the file.
|
||||
if err := removeAttributes(path); err != nil {
|
||||
fmt.Printf("failed to remove attributes from '%s': %v", path, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file.
|
||||
func updateOwnerIndexFile(basePath string, obsoleteIDs []string) error {
|
||||
fmt.Printf(" Rewriting index file '%s'\n", basePath)
|
||||
|
||||
ownerID, err := xattr.Get(basePath, ownerIDAttrName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err)
|
||||
}
|
||||
|
||||
indexPath := filepath.Join(basePath, "../../indexes/by-user-id", string(ownerID)+".mpk")
|
||||
indexPath = filepath.Clean(indexPath)
|
||||
|
||||
// Read the MessagePack file
|
||||
fileData, err := os.ReadFile(indexPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("index file does not exist, skipping update")
|
||||
}
|
||||
return fmt.Errorf("could not read index file: %w", err)
|
||||
}
|
||||
var indexMap map[string]string
|
||||
if err := msgpack.Unmarshal(fileData, &indexMap); err != nil {
|
||||
return fmt.Errorf("failed to parse MessagePack index file (is it corrupt?): %w", err)
|
||||
}
|
||||
|
||||
// Remove obsolete IDs from the map
|
||||
itemsRemoved := 0
|
||||
for _, id := range obsoleteIDs {
|
||||
if _, exists := indexMap[id]; exists {
|
||||
fmt.Printf(" - Removing obsolete ID '%s' from index.\n", id)
|
||||
delete(indexMap, id)
|
||||
itemsRemoved++
|
||||
} else {
|
||||
fmt.Printf(" - Obsolete ID '%s' not found in index\n", id)
|
||||
}
|
||||
}
|
||||
|
||||
if itemsRemoved == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write the data back to the file
|
||||
updatedData, err := msgpack.Marshal(&indexMap)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal updated index map: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(indexPath, updatedData, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write updated index file: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf(" ✓ Successfully removed %d item(s) and saved index file.\n", itemsRemoved)
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeAttributes(path string) error {
|
||||
attrNames, err := xattr.List(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list attributes for '%s': %w", path, err)
|
||||
}
|
||||
|
||||
for _, attrName := range attrNames {
|
||||
if err := xattr.Remove(path, attrName); err != nil {
|
||||
return fmt.Errorf("failed to remove attribute '%s' from '%s': %w", attrName, path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func logFailure(message string, args ...any) {
|
||||
spinner.StopFailMessage(fmt.Sprintf("\n"+message, args...))
|
||||
spinner.StopFail()
|
||||
spinner.Start()
|
||||
fmt.Fprintf(os.Stderr, message+"\n", args...)
|
||||
}
|
||||
|
||||
// findStorageRoot walks up the directory tree starting at path until it finds a
|
||||
// directory that contains an "indexes" subdirectory which marks the root of a
|
||||
// posixfs storage. A user directory inside a space might also be named "indexes",
|
||||
// so to disambiguate we require that the "indexes" directory is an internal
|
||||
// directory: the storage's own indexes directory is skipped during assimilation
|
||||
// and therefore never receives a node ID attribute, whereas a regular user
|
||||
// directory would have one.
|
||||
func findStorageRoot(path string) (string, error) {
|
||||
current := path
|
||||
for {
|
||||
indexesPath := filepath.Join(current, "indexes")
|
||||
if info, err := os.Stat(indexesPath); err == nil && info.IsDir() {
|
||||
if id, err := xattr.Get(indexesPath, prefixes.IDAttr); err != nil || len(id) == 0 {
|
||||
return current, nil
|
||||
}
|
||||
}
|
||||
|
||||
parent := filepath.Dir(current)
|
||||
if parent == current {
|
||||
return "", fmt.Errorf("'%s' does not appear to be inside a posixfs storage (no 'indexes' directory found)", path)
|
||||
}
|
||||
current = parent
|
||||
}
|
||||
}
|
||||
|
||||
// isSpaceRoot reports whether the given path is a space root, which is
|
||||
// identified by the presence of the space ID attribute.
|
||||
func isSpaceRoot(path string) bool {
|
||||
spaceID, err := xattr.Get(path, prefixes.SpaceIDAttr)
|
||||
return err == nil && len(spaceID) > 0
|
||||
}
|
||||
|
||||
// iterates over a list of paths and processes them all, using the appropriate function
|
||||
// depending on the type of resource
|
||||
//
|
||||
// note that whenever an error occurs, it collects that error and continues processing
|
||||
// subsequent paths, and then returns a slice of errors at the end (or an empty slice
|
||||
// if no errors occured)
|
||||
func processPosixFsResources(paths []string,
|
||||
keepGoing bool,
|
||||
personalSpaceDir func(string) error,
|
||||
projectSpaceDir func(string) error,
|
||||
spaceRoot func(string) error,
|
||||
entity func(string) error,
|
||||
) []error {
|
||||
// no need to guard this with a mutex for now, since the implementation is not parallelized
|
||||
errors := []error{}
|
||||
|
||||
for _, path := range paths {
|
||||
rootPath, err := findStorageRoot(path)
|
||||
if err != nil {
|
||||
errors = append(errors, err)
|
||||
logFailure("error: %s", err)
|
||||
if keepGoing {
|
||||
continue
|
||||
} else {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
path = filepath.Clean(path)
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
errors = append(errors, err)
|
||||
logFailure("error accessing '%s': %w", path, err)
|
||||
if keepGoing {
|
||||
continue
|
||||
} else {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
contained, _ := filepathx.IsSameOrContainedBy(rootPath, path)
|
||||
|
||||
switch {
|
||||
case path == rootPath:
|
||||
if err := personalSpaceDir(filepath.Join(path, "users")); err != nil {
|
||||
errors = append(errors, err)
|
||||
if !keepGoing {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
if err := projectSpaceDir(filepath.Join(path, "projects")); err != nil {
|
||||
errors = append(errors, err)
|
||||
if !keepGoing {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
case isSpaceRoot(path):
|
||||
if err := spaceRoot(path); err != nil {
|
||||
errors = append(errors, err)
|
||||
if !keepGoing {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
case contained:
|
||||
if err := entity(path); err != nil {
|
||||
errors = append(errors, err)
|
||||
if !keepGoing {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
default:
|
||||
err := fmt.Errorf("error: the provided path '%s' is neither a space root nor contained by the storage root '%s'", path, rootPath)
|
||||
errors = append(errors, err)
|
||||
logFailure(err.Error())
|
||||
if !keepGoing {
|
||||
return errors
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors
|
||||
}
|
||||
@@ -0,0 +1,489 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package command
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
storageUsersConfig "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/ignore"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/shamaton/msgpack/v2"
|
||||
)
|
||||
|
||||
type consistencyChecker struct {
|
||||
cfg *storageUsersConfig.Config
|
||||
ignorer *ignore.Ignorer
|
||||
recalculateChecksums bool
|
||||
|
||||
restartRequired bool
|
||||
}
|
||||
|
||||
// checkPosixfsConsistency checks the consistency of the posixfs storage. The
|
||||
// given path determines the scope of the check: the whole storage, a single
|
||||
// space or a single entity within a space.
|
||||
func (c *consistencyChecker) Check(paths []string) error {
|
||||
_ = processPosixFsResources(paths, true,
|
||||
func(path string) error {
|
||||
fmt.Println("Checking personal spaces...")
|
||||
c.checkSpaces(path)
|
||||
return nil
|
||||
},
|
||||
func(path string) error {
|
||||
fmt.Println("Checking project spaces...")
|
||||
c.checkSpaces(path)
|
||||
return nil
|
||||
},
|
||||
func(path string) error {
|
||||
fmt.Printf("Checking space '%s'...\n", path)
|
||||
c.checkSpace(path)
|
||||
return nil
|
||||
},
|
||||
func(path string) error {
|
||||
if c.ignorer.IsIgnored(path) {
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("Checking '%s'...\n", path)
|
||||
c.checkEntity(path)
|
||||
return nil
|
||||
},
|
||||
)
|
||||
if c.restartRequired {
|
||||
fmt.Println("\n\n ⚠️ Please restart your openCloud instance to apply changes.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) checkSpaces(basePath string) {
|
||||
dirEntries, err := os.ReadDir(basePath)
|
||||
if err != nil {
|
||||
logFailure("Error reading spaces directory '%s': %v", basePath, err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
if entry.IsDir() {
|
||||
fullPath := filepath.Join(basePath, entry.Name())
|
||||
c.checkSpace(fullPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) checkSpace(spacePath string) {
|
||||
info, err := os.Stat(spacePath)
|
||||
if err != nil {
|
||||
logFailure("Error accessing path '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
if !info.IsDir() {
|
||||
logFailure("Error: The provided path '%s' is not a directory\n", spacePath)
|
||||
return
|
||||
}
|
||||
|
||||
spaceID, err := xattr.Get(spacePath, prefixes.SpaceIDAttr)
|
||||
if err != nil || len(spaceID) == 0 {
|
||||
logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, prefixes.SpaceIDAttr)
|
||||
return
|
||||
}
|
||||
|
||||
c.checkSpaceID(spacePath)
|
||||
c.checkNodes(spacePath)
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) checkSpaceID(spacePath string) {
|
||||
entries, uniqueIDs, oldestEntry, err := c.gatherAttributes(spacePath)
|
||||
if err != nil {
|
||||
logFailure("Failed to gather attributes: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if len(uniqueIDs) > 1 {
|
||||
fmt.Println("\n ⚠ Multiple space IDs found:")
|
||||
for id := range uniqueIDs {
|
||||
fmt.Printf(" - %s\n", id)
|
||||
}
|
||||
|
||||
fmt.Printf("\n ⏳ Oldest entry is '%s' (modified on %s).\n",
|
||||
filepath.Base(oldestEntry.Path), oldestEntry.ModTime.Format(time.RFC1123))
|
||||
|
||||
targetID := oldestEntry.ParentID
|
||||
fmt.Printf(" ✅ Proposed target Parent ID: %s\n", targetID)
|
||||
|
||||
fmt.Printf("\n Do you want to unify all parent IDs to '%s'? This will modify %d entries, the directory, and the user index. (y/N): ", targetID, len(entries))
|
||||
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
input, _ := reader.ReadString('\n')
|
||||
input = strings.TrimSpace(strings.ToLower(input))
|
||||
|
||||
if input != "y" {
|
||||
logFailure("Operation cancelled by user.")
|
||||
return
|
||||
}
|
||||
c.restartRequired = true
|
||||
|
||||
obsoleteIDs := []string{}
|
||||
for id := range uniqueIDs {
|
||||
if id != targetID {
|
||||
obsoleteIDs = append(obsoleteIDs, id)
|
||||
}
|
||||
}
|
||||
c.fixSpaceID(spacePath, obsoleteIDs, targetID, entries)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) walkNodes(dir string, parentID string) int {
|
||||
fixes := 0
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
logFailure("Error reading directory '%s': %v", dir, err)
|
||||
return 0
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
fullPath := filepath.Join(dir, entry.Name())
|
||||
|
||||
if c.ignorer.IsIgnored(fullPath) {
|
||||
continue
|
||||
}
|
||||
|
||||
fixes += c.checkNodeAttributes(fullPath, entry.Name(), parentID, entry.IsDir())
|
||||
|
||||
if entry.IsDir() {
|
||||
nodeID, err := xattr.Get(fullPath, prefixes.IDAttr)
|
||||
if err != nil || len(nodeID) == 0 {
|
||||
logFailure("Directory '%s' missing '%s', skipping its children", fullPath, prefixes.IDAttr)
|
||||
continue
|
||||
}
|
||||
fixes += c.walkNodes(fullPath, string(nodeID))
|
||||
}
|
||||
}
|
||||
return fixes
|
||||
}
|
||||
|
||||
// checkNodeAttributes checks and fixes the parent ID and name attributes of a
|
||||
// single node. For files it additionally checks the blobsize and, when
|
||||
// requested, the checksums. It returns the number of fixes applied.
|
||||
func (c *consistencyChecker) checkNodeAttributes(path, name, parentID string, isDir bool) int {
|
||||
fixes := 0
|
||||
|
||||
// Check if the parent ID attribute matches the expected parent ID, if not, fix it.
|
||||
actualParentID, err := xattr.Get(path, prefixes.ParentidAttr)
|
||||
if err != nil || string(actualParentID) != parentID {
|
||||
if err := xattr.Set(path, prefixes.ParentidAttr, []byte(parentID)); err != nil {
|
||||
logFailure("Failed to fix parent ID for '%s': %v", path, err)
|
||||
} else {
|
||||
fmt.Printf(" + Fixed parent ID for '%s'\n", path)
|
||||
fixes++
|
||||
c.restartRequired = true
|
||||
}
|
||||
}
|
||||
|
||||
// Check that the name attribute matches the actual name of the file/directory, if not, fix it.
|
||||
nameAttr, err := xattr.Get(path, prefixes.NameAttr)
|
||||
if err != nil || string(nameAttr) != name {
|
||||
if err := xattr.Set(path, prefixes.NameAttr, []byte(name)); err != nil {
|
||||
logFailure("Failed to fix name attribute for '%s': %v", path, err)
|
||||
} else {
|
||||
fmt.Printf(" + Fixed name attribute for '%s'\n", path)
|
||||
fixes++
|
||||
c.restartRequired = true
|
||||
}
|
||||
}
|
||||
|
||||
if !isDir {
|
||||
fixes += c.checkBlobsize(path)
|
||||
if c.recalculateChecksums {
|
||||
fixes += c.fixChecksums(path)
|
||||
}
|
||||
}
|
||||
|
||||
return fixes
|
||||
}
|
||||
|
||||
// checkEntity checks a single file or directory within a space, including its own
|
||||
// parent ID, name and (for files) blobsize/checksums. If the entity is a directory
|
||||
// its children are checked recursively.
|
||||
func (c *consistencyChecker) checkEntity(path string) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
logFailure("Error accessing path '%s': %v", path, err)
|
||||
return
|
||||
}
|
||||
|
||||
// The expected parent ID is the ID attribute of the containing directory.
|
||||
parentDir := filepath.Dir(path)
|
||||
parentID, err := xattr.Get(parentDir, prefixes.IDAttr)
|
||||
if err != nil || len(parentID) == 0 {
|
||||
logFailure("Parent directory '%s' is missing the '%s' attribute", parentDir, prefixes.IDAttr)
|
||||
return
|
||||
}
|
||||
|
||||
fixes := c.checkNodeAttributes(path, info.Name(), string(parentID), info.IsDir())
|
||||
|
||||
if info.IsDir() {
|
||||
nodeID, err := xattr.Get(path, prefixes.IDAttr)
|
||||
if err != nil || len(nodeID) == 0 {
|
||||
logFailure("Directory '%s' missing '%s' attribute", path, prefixes.IDAttr)
|
||||
} else {
|
||||
fixes += c.walkNodes(path, string(nodeID))
|
||||
}
|
||||
}
|
||||
|
||||
if fixes > 0 {
|
||||
fmt.Printf(" ✓ Fixed %d incorrect node attributes for %s\n", fixes, filepath.Base(path))
|
||||
}
|
||||
}
|
||||
|
||||
// checkBlobsize verifies that the stored blobsize attribute matches the actual
|
||||
// file size and fixes it if it doesn't. It returns the number of fixes applied.
|
||||
func (c *consistencyChecker) checkBlobsize(path string) int {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
logFailure("Error accessing file '%s': %v", path, err)
|
||||
return 0
|
||||
}
|
||||
|
||||
expectedSize := strconv.FormatInt(info.Size(), 10)
|
||||
blobsize, err := xattr.Get(path, prefixes.BlobsizeAttr)
|
||||
if err == nil && string(blobsize) == expectedSize {
|
||||
return 0
|
||||
}
|
||||
|
||||
if err := xattr.Set(path, prefixes.BlobsizeAttr, []byte(expectedSize)); err != nil {
|
||||
logFailure("Failed to fix blobsize for '%s': %v", path, err)
|
||||
return 0
|
||||
}
|
||||
|
||||
fmt.Printf(" + Fixed blobsize for '%s'\n", path)
|
||||
c.restartRequired = true
|
||||
return 1
|
||||
}
|
||||
|
||||
// fixChecksums recalculates the sha1, md5 and adler32 checksums of the file and
|
||||
// updates the stored attributes if they differ. It returns the number of fixes applied.
|
||||
func (c *consistencyChecker) fixChecksums(path string) int {
|
||||
sha1h, md5h, adler32h, err := node.CalculateChecksums(context.Background(), path)
|
||||
if err != nil {
|
||||
logFailure("Failed to calculate checksums for '%s': %v", path, err)
|
||||
return 0
|
||||
}
|
||||
|
||||
checksums := map[string][]byte{
|
||||
prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil),
|
||||
prefixes.ChecksumPrefix + "md5": md5h.Sum(nil),
|
||||
prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil),
|
||||
}
|
||||
|
||||
fixes := 0
|
||||
for attrName, sum := range checksums {
|
||||
current, err := xattr.Get(path, attrName)
|
||||
if err == nil && bytes.Equal(current, sum) {
|
||||
continue
|
||||
}
|
||||
if err := xattr.Set(path, attrName, sum); err != nil {
|
||||
logFailure("Failed to fix checksum '%s' for '%s': %v", attrName, path, err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf(" + Fixed checksum '%s' for '%s'\n", attrName, path)
|
||||
c.restartRequired = true
|
||||
fixes++
|
||||
}
|
||||
return fixes
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) checkNodes(spacePath string) {
|
||||
rootID, err := xattr.Get(spacePath, prefixes.IDAttr)
|
||||
if err != nil || len(rootID) == 0 {
|
||||
logFailure("Space root '%s' missing '%s' attribute", spacePath, prefixes.IDAttr)
|
||||
return
|
||||
}
|
||||
|
||||
fixes := c.walkNodes(spacePath, string(rootID))
|
||||
|
||||
if fixes > 0 {
|
||||
fmt.Printf(" ✓ Fixed %d incorrect node attributes in %s\n", fixes, filepath.Base(spacePath))
|
||||
}
|
||||
}
|
||||
|
||||
// fixSpaceID updates the parentid attributes of all entries in a space to a new target ID,
|
||||
// updates the space's own ID attributes, and removes obsolete IDs from the user index file.
|
||||
func (c *consistencyChecker) fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) {
|
||||
// Set all parentid attributes to the proper space ID
|
||||
err := setAllParentIDAttributes(entries, targetID)
|
||||
if err != nil {
|
||||
logFailure("an error occurred during file attribute update: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update space ID itself
|
||||
fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), prefixes.IDAttr, targetID)
|
||||
err = xattr.Set(spacePath, prefixes.IDAttr, []byte(targetID))
|
||||
if err != nil {
|
||||
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
err = xattr.Set(spacePath, prefixes.SpaceIDAttr, []byte(targetID))
|
||||
if err != nil {
|
||||
logFailure("Failed to set attribute on directory '%s': %v", spacePath, err)
|
||||
return
|
||||
}
|
||||
|
||||
// update the index
|
||||
err = c.updateOwnerIndexFile(spacePath, obsoleteIDs)
|
||||
if err != nil {
|
||||
logFailure("Could not update the owner index file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *consistencyChecker) gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, error) {
|
||||
dirEntries, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return nil, nil, EntryInfo{}, fmt.Errorf("failed to read directory: %w", err)
|
||||
}
|
||||
|
||||
var allEntries []EntryInfo
|
||||
uniqueIDs := make(map[string]struct{})
|
||||
var oldestEntry EntryInfo
|
||||
oldestTime := time.Now().Add(100 * 365 * 24 * time.Hour) // Set to a future date to find the oldest entry
|
||||
|
||||
for _, entry := range dirEntries {
|
||||
fullPath := filepath.Join(path, entry.Name())
|
||||
if c.ignorer.IsIgnored(fullPath) {
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
fmt.Printf(" - Warning: could not stat %s: %v\n", entry.Name(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
parentID, err := xattr.Get(fullPath, prefixes.ParentidAttr)
|
||||
if err != nil {
|
||||
continue // Skip if attribute doesn't exist or can't be read
|
||||
}
|
||||
|
||||
entryInfo := EntryInfo{
|
||||
Path: fullPath,
|
||||
ModTime: info.ModTime(),
|
||||
ParentID: string(parentID),
|
||||
}
|
||||
|
||||
allEntries = append(allEntries, entryInfo)
|
||||
uniqueIDs[string(parentID)] = struct{}{}
|
||||
|
||||
if entryInfo.ModTime.Before(oldestTime) {
|
||||
oldestTime = entryInfo.ModTime
|
||||
oldestEntry = entryInfo
|
||||
}
|
||||
}
|
||||
|
||||
return allEntries, uniqueIDs, oldestEntry, nil
|
||||
}
|
||||
|
||||
func setAllParentIDAttributes(entries []EntryInfo, targetID string) error {
|
||||
fmt.Printf(" Setting all parent IDs to '%s':\n", targetID)
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.ParentID == targetID {
|
||||
fmt.Printf(" - Skipping '%s' (already has target ID).\n", filepath.Base(entry.Path))
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Printf(" - Removing all attributes from '%s'. It will be re-assimilated\n", filepath.Base(entry.Path))
|
||||
filepath.WalkDir(entry.Path, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking path '%s': %w", path, err)
|
||||
}
|
||||
|
||||
// Remove all attributes from the file.
|
||||
if err := removeAttributes(path); err != nil {
|
||||
fmt.Printf("failed to remove attributes from '%s': %v", path, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file.
|
||||
func (c *consistencyChecker) updateOwnerIndexFile(basePath string, obsoleteIDs []string) error {
|
||||
fmt.Printf(" Rewriting index file '%s'\n", basePath)
|
||||
|
||||
ownerID, err := xattr.Get(basePath, prefixes.OwnerIDAttr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err)
|
||||
}
|
||||
|
||||
indexPath := filepath.Join(c.cfg.Drivers.Posix.Root, "indexes", "by-user-id", string(ownerID)+".mpk")
|
||||
indexPath = filepath.Clean(indexPath)
|
||||
|
||||
// Read the MessagePack file
|
||||
fileData, err := os.ReadFile(indexPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("index file does not exist, skipping update")
|
||||
}
|
||||
return fmt.Errorf("could not read index file: %w", err)
|
||||
}
|
||||
var indexMap map[string]string
|
||||
if err := msgpack.Unmarshal(fileData, &indexMap); err != nil {
|
||||
return fmt.Errorf("failed to parse MessagePack index file (is it corrupt?): %w", err)
|
||||
}
|
||||
|
||||
// Remove obsolete IDs from the map
|
||||
itemsRemoved := 0
|
||||
for _, id := range obsoleteIDs {
|
||||
if _, exists := indexMap[id]; exists {
|
||||
fmt.Printf(" - Removing obsolete ID '%s' from index.\n", id)
|
||||
delete(indexMap, id)
|
||||
itemsRemoved++
|
||||
} else {
|
||||
fmt.Printf(" - Obsolete ID '%s' not found in index\n", id)
|
||||
}
|
||||
}
|
||||
|
||||
if itemsRemoved == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write the data back to the file
|
||||
updatedData, err := msgpack.Marshal(&indexMap)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal updated index map: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(indexPath, updatedData, 0644); err != nil {
|
||||
return fmt.Errorf("failed to write updated index file: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf(" ✓ Successfully removed %d item(s) and saved index file.\n", itemsRemoved)
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeAttributes(path string) error {
|
||||
attrNames, err := xattr.List(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list attributes for '%s': %w", path, err)
|
||||
}
|
||||
|
||||
for _, attrName := range attrNames {
|
||||
if err := xattr.Remove(path, attrName); err != nil {
|
||||
return fmt.Errorf("failed to remove attribute '%s' from '%s': %w", attrName, path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -6,11 +6,13 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/opencloud/pkg/register"
|
||||
"github.com/opencloud-eu/opencloud/pkg/clihelper"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
oclog "github.com/opencloud-eu/opencloud/pkg/log"
|
||||
)
|
||||
|
||||
// Execute is the entry point for the opencloud command.
|
||||
@@ -38,3 +40,11 @@ func Execute() error {
|
||||
ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP)
|
||||
return app.ExecuteContext(ctx)
|
||||
}
|
||||
|
||||
func logger(name string) zerolog.Logger {
|
||||
return oclog.NewLogger(
|
||||
oclog.Name(name),
|
||||
oclog.Level("info"),
|
||||
oclog.Pretty(true),
|
||||
oclog.Color(true)).Logger
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/parser"
|
||||
oclog "github.com/opencloud-eu/opencloud/pkg/log"
|
||||
mregistry "github.com/opencloud-eu/opencloud/pkg/registry"
|
||||
sharing "github.com/opencloud-eu/opencloud/services/sharing/pkg/config"
|
||||
sharingparser "github.com/opencloud-eu/opencloud/services/sharing/pkg/config/parser"
|
||||
@@ -85,7 +84,7 @@ func cleanup(_ *cobra.Command, cfg *config.Config) error {
|
||||
return configlog.ReturnError(errors.New("cleanup is only implemented for the jsoncs3 share manager"))
|
||||
}
|
||||
|
||||
l := logger()
|
||||
l := logger("migrate")
|
||||
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
|
||||
@@ -94,7 +93,7 @@ func cleanup(_ *cobra.Command, cfg *config.Config) error {
|
||||
if !ok {
|
||||
return configlog.ReturnError(errors.New("Unknown share manager type '" + driver + "'"))
|
||||
}
|
||||
mgr, err := f(rcfg[driver].(map[string]any), l)
|
||||
mgr, err := f(rcfg[driver].(map[string]any), &l)
|
||||
if err != nil {
|
||||
return configlog.ReturnError(err)
|
||||
}
|
||||
@@ -167,12 +166,3 @@ func revaShareConfig(cfg *sharing.Config) map[string]any {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func logger() *zerolog.Logger {
|
||||
log := oclog.NewLogger(
|
||||
oclog.Name("migrate"),
|
||||
oclog.Level("info"),
|
||||
oclog.Pretty(true),
|
||||
oclog.Color(true)).Logger
|
||||
return &log
|
||||
}
|
||||
@@ -34,7 +34,7 @@ var (
|
||||
// LatestTag is the latest released version plus the dev meta version.
|
||||
// Will be overwritten by the release pipeline
|
||||
// Needs a manual change for every tagged release
|
||||
LatestTag = "7.2.0-rc.1+dev"
|
||||
LatestTag = "7.2.4+dev"
|
||||
|
||||
// Date indicates the build date.
|
||||
// This has been removed, it looks like you can only replace static strings with recent go versions
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package filepathx
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// JailJoin joins any number of path elements into a single path,
|
||||
@@ -10,3 +12,27 @@ import (
|
||||
func JailJoin(jail string, elem ...string) string {
|
||||
return filepath.Join(jail, filepath.Join(append([]string{"/"}, elem...)...))
|
||||
}
|
||||
|
||||
// Determines whether the file or directory 'child' is same as or underneath the directory 'parent'.
|
||||
//
|
||||
// Note that 'parent' is expected to be a directory.
|
||||
func IsSameOrContainedBy(parent string, child string) (bool, error) {
|
||||
absParent, err := filepath.Abs(parent)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to make parent directory absolute: %q: %w", parent, err)
|
||||
}
|
||||
|
||||
absChild, err := filepath.Abs(child)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to make child file/directory absolute: %q: %w", child, err)
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(absParent, absChild)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to determine the relative path between the parent directory %q and the child file/directory: %q: %w", absParent, absChild, err)
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -1,9 +1,12 @@
|
||||
package filepathx_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/x/path/filepathx"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestJailJoin(t *testing.T) {
|
||||
@@ -61,3 +64,23 @@ func TestJailJoin(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSameOrContainedBy(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
parent string
|
||||
child string
|
||||
expected bool
|
||||
}{
|
||||
{"foo", "foo", true},
|
||||
{"/foo", "/foo", true},
|
||||
{"foo", "foo/bar", true},
|
||||
{"foo", "bar", false},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%s: %s vs %s", t.Name(), strings.ReplaceAll(tt.parent, "/", "."), strings.ReplaceAll(tt.child, "/", ".")), func(t *testing.T) {
|
||||
require := require.New(t)
|
||||
b, err := filepathx.IsSameOrContainedBy(tt.parent, tt.child)
|
||||
require.NoError(err)
|
||||
require.Equal(tt.expected, b)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -69,6 +69,8 @@ Once the service defined as custom step has finished its work, it should send an
|
||||
|
||||
The backoff behavior as mentioned in the `retry` outcome can be configured using the `POSTPROCESSING_RETRY_BACKOFF_DURATION` and `POSTPROCESSING_MAX_RETRIES` environment variables. The backoff duration is calculated using the following formula after each failure: `backoff_duration = POSTPROCESSING_RETRY_BACKOFF_DURATION * 2^(number of failures - 1)`. This means that the time between the next round grows exponentially limited by the number of retries. Steps that still don't succeed after the maximum number of retries will be automatically moved to the `abort` state.
|
||||
|
||||
The same backoff formula is used when publishing an event to the event system fails, which can happen when the event system is briefly unavailable or slow to acknowledge. The number of publish retries is configured with the `POSTPROCESSING_PUBLISH_MAX_RETRIES` environment variable, setting it to `0` disables retrying. A single wait is never longer than half the configured ack wait, so that the event being processed is not redelivered to another worker while the publish is still being retried. If publishing still fails after the last retry, the incoming event is not acknowledged so that it gets redelivered and postprocessing can pick up where it left off.
|
||||
|
||||
See the [cs3 org](https://github.com/cs3org/reva/blob/edge/pkg/events/postprocessing.go) for up-to-date information of reserved step names and event definitions.
|
||||
|
||||
## CLI Commands
|
||||
|
||||
@@ -32,6 +32,7 @@ type Postprocessing struct {
|
||||
|
||||
RetryBackoffDuration time.Duration `yaml:"retry_backoff_duration" env:"POSTPROCESSING_RETRY_BACKOFF_DURATION" desc:"The base for the exponential backoff duration before retrying a failed postprocessing step. See the Environment Variable Types description for more details." introductionVersion:"1.0.0"`
|
||||
MaxRetries int `yaml:"max_retries" env:"POSTPROCESSING_MAX_RETRIES" desc:"The maximum number of retries for a failed postprocessing step." introductionVersion:"1.0.0"`
|
||||
PublishMaxRetries int `yaml:"-"` // hardcoded default only, for the stable-7.2. Config setting will come with the next stable release
|
||||
}
|
||||
|
||||
// Events combines the configuration options for the event bus.
|
||||
|
||||
@@ -36,6 +36,7 @@ func DefaultConfig() *config.Config {
|
||||
Workers: 3,
|
||||
RetryBackoffDuration: 5 * time.Second,
|
||||
MaxRetries: 14,
|
||||
PublishMaxRetries: 5,
|
||||
},
|
||||
Store: config.Store{
|
||||
Store: "nats-js-kv",
|
||||
|
||||
@@ -86,7 +86,17 @@ func (pp *Postprocessing) Delay(f func(next any)) {
|
||||
|
||||
// BackoffDuration calculates the duration for exponential backoff based on the number of failures.
|
||||
func (pp *Postprocessing) BackoffDuration() time.Duration {
|
||||
return pp.config.RetryBackoffDuration * time.Duration(math.Pow(2, float64(pp.Failures-1)))
|
||||
return BackoffDuration(pp.config.RetryBackoffDuration, pp.Failures)
|
||||
}
|
||||
|
||||
// BackoffDuration calculates an exponential backoff duration from a base duration and the
|
||||
// number of failures that happened so far. The first failure waits the base duration, every
|
||||
// subsequent one waits twice as long as the previous one.
|
||||
func BackoffDuration(base time.Duration, failures int) time.Duration {
|
||||
if failures < 1 {
|
||||
return 0
|
||||
}
|
||||
return base * time.Duration(math.Pow(2, float64(failures-1)))
|
||||
}
|
||||
|
||||
func (pp *Postprocessing) next(current events.Postprocessingstep) any {
|
||||
|
||||
@@ -293,14 +293,66 @@ func (pps *PostprocessingService) processEvent(e raw.Event) error {
|
||||
}
|
||||
|
||||
if next != nil {
|
||||
if err := events.Publish(ctx, pps.pub, next); err != nil {
|
||||
if err := pps.publishWithRetry(ctx, &e, next); err != nil {
|
||||
// The successor event never made it onto the bus. Don't ack the source event so
|
||||
// jetstream redelivers it and the upload can continue instead of being stuck in
|
||||
// the store forever.
|
||||
ackEvent = false
|
||||
pps.log.Error().Err(err).Msg("unable to publish event")
|
||||
|
||||
if pps.stopped.Load() || ctx.Err() != nil {
|
||||
// we are shutting down anyway, no need to take the whole process down with us
|
||||
return fmt.Errorf("%w: unable to publish event", ErrEvent)
|
||||
}
|
||||
return fmt.Errorf("%w: unable to publish event", ErrFatal) // we can't publish -> we are screwed
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// publishWithRetry publishes an event, retrying transient failures of the event system with
|
||||
// the same exponential backoff that is used for failed postprocessing steps. Between the
|
||||
// attempts the source event is marked as in progress and the wait is capped to half the ack
|
||||
// wait, so jetstream should not hand the source event to a second worker while we are still
|
||||
// retrying. Note that marking the event as in progress is best effort: it is a fire and forget
|
||||
// publish, so it can be lost exactly when the event system is unhealthy. A redelivery during a
|
||||
// retry is therefore possible, it is just unlikely. If all attempts fail, the last error is
|
||||
// returned.
|
||||
func (pps *PostprocessingService) publishWithRetry(ctx context.Context, e *raw.Event, ev any) error {
|
||||
for attempt := 0; ; attempt++ {
|
||||
err := events.Publish(ctx, pps.pub, ev)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if attempt >= pps.c.PublishMaxRetries {
|
||||
return err
|
||||
}
|
||||
|
||||
backoff := postprocessing.BackoffDuration(pps.c.RetryBackoffDuration, attempt+1)
|
||||
// Never wait longer than half the ack wait. We only refresh the redelivery timer
|
||||
// between the attempts, so a longer wait would let jetstream hand the source event
|
||||
// to a second worker while we are still retrying here.
|
||||
if maxBackoff := pps.c.Events.AckWait / 2; maxBackoff > 0 && backoff > maxBackoff {
|
||||
backoff = maxBackoff
|
||||
}
|
||||
pps.log.Warn().Err(err).Int("attempt", attempt+1).Dur("backoff", backoff).Msg("unable to publish event, retrying")
|
||||
|
||||
// tell jetstream that we are still working on the source event
|
||||
if ipErr := e.InProgress(); ipErr != nil {
|
||||
pps.log.Debug().Err(ipErr).Msg("unable to mark event as in progress")
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return err
|
||||
case <-pps.stopCh:
|
||||
return err
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (pps *PostprocessingService) getPP(sto store.Store, uploadID string) (*postprocessing.Postprocessing, error) {
|
||||
recs, err := sto.Read(uploadID)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestPostprocessing(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Service Suite")
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/postprocessing/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/postprocessing/pkg/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/raw"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/store"
|
||||
microevents "go-micro.dev/v4/events"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
)
|
||||
|
||||
// errPublish is the error a nats publish returns when the ack of the message was not
|
||||
// received in time. It is transient: the very next publish usually succeeds.
|
||||
var errPublish = errors.New("nats: timeout")
|
||||
|
||||
// testPublisher is a publisher that fails the first `failures` publish attempts and
|
||||
// records everything it accepted afterwards.
|
||||
type testPublisher struct {
|
||||
mu sync.Mutex
|
||||
failures int
|
||||
attempts int
|
||||
accepted []any
|
||||
}
|
||||
|
||||
func (p *testPublisher) Publish(_ string, ev any, _ ...microevents.PublishOption) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.attempts++
|
||||
if p.attempts <= p.failures {
|
||||
return errPublish
|
||||
}
|
||||
p.accepted = append(p.accepted, ev)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *testPublisher) stats() (attempts int, accepted []any) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.attempts, p.accepted
|
||||
}
|
||||
|
||||
var _ = Describe("PostprocessingService", func() {
|
||||
var (
|
||||
cfg config.Postprocessing
|
||||
pub *testPublisher
|
||||
pps *PostprocessingService
|
||||
)
|
||||
|
||||
// newService builds a service that is wired to the fake publisher only. It deliberately
|
||||
// does not go through NewPostprocessingService, which would need a running nats.
|
||||
newService := func() *PostprocessingService {
|
||||
return &PostprocessingService{
|
||||
ctx: context.Background(),
|
||||
log: log.NopLogger(),
|
||||
pub: pub,
|
||||
steps: getSteps(cfg),
|
||||
store: store.Create(),
|
||||
c: cfg,
|
||||
tp: noop.NewTracerProvider(),
|
||||
metrics: metrics.New(),
|
||||
stopCh: make(chan struct{}, 1),
|
||||
}
|
||||
}
|
||||
|
||||
// bytesReceived is the event that starts a postprocessing chain. Handling it makes the
|
||||
// service publish the first StartPostprocessingStep, which is the publish that used to
|
||||
// take the whole process down when nats hiccuped.
|
||||
bytesReceived := func() raw.Event {
|
||||
ev := events.BytesReceived{
|
||||
UploadID: "upload-" + uuid.New().String(),
|
||||
Filename: "test.txt",
|
||||
Filesize: 1234,
|
||||
}
|
||||
return raw.Event{
|
||||
Event: events.Event{
|
||||
ID: uuid.New().String(),
|
||||
Type: reflect.TypeOf(ev).String(),
|
||||
Event: ev,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
cfg = config.Postprocessing{
|
||||
Steps: []string{"virusscan"},
|
||||
RetryBackoffDuration: 5 * time.Millisecond,
|
||||
MaxRetries: 14,
|
||||
PublishMaxRetries: 3,
|
||||
}
|
||||
pub = &testPublisher{}
|
||||
})
|
||||
|
||||
Describe("publishing the next event", func() {
|
||||
It("publishes once when the event system is healthy", func() {
|
||||
pps = newService()
|
||||
|
||||
Expect(pps.processEvent(bytesReceived())).To(Succeed())
|
||||
|
||||
attempts, accepted := pub.stats()
|
||||
Expect(attempts).To(Equal(1))
|
||||
Expect(accepted).To(HaveLen(1))
|
||||
Expect(accepted[0]).To(BeAssignableToTypeOf(events.StartPostprocessingStep{}))
|
||||
Expect(accepted[0].(events.StartPostprocessingStep).StepToStart).To(Equal(events.PPStepAntivirus))
|
||||
})
|
||||
|
||||
It("retries a transient publish failure and succeeds on a later attempt", func() {
|
||||
pub.failures = 2
|
||||
pps = newService()
|
||||
|
||||
Expect(pps.processEvent(bytesReceived())).To(Succeed())
|
||||
|
||||
attempts, accepted := pub.stats()
|
||||
Expect(attempts).To(Equal(3))
|
||||
Expect(accepted).To(HaveLen(1))
|
||||
Expect(accepted[0]).To(BeAssignableToTypeOf(events.StartPostprocessingStep{}))
|
||||
})
|
||||
|
||||
It("uses an exponential backoff between the attempts", func() {
|
||||
pub.failures = 3
|
||||
pps = newService()
|
||||
|
||||
start := time.Now()
|
||||
Expect(pps.processEvent(bytesReceived())).To(Succeed())
|
||||
elapsed := time.Since(start)
|
||||
|
||||
// 5ms + 10ms + 20ms, minus a margin so a coarse clock cannot make this flaky
|
||||
Expect(elapsed).To(BeNumerically(">=", 30*time.Millisecond))
|
||||
})
|
||||
|
||||
It("caps the backoff at half the ack wait", func() {
|
||||
// Without a cap the waits would grow to 10+20+40+80=150ms, far beyond the ack
|
||||
// wait, and jetstream would redeliver the source event to a second worker while
|
||||
// this one is still retrying. Capped at AckWait/2 they are 10+10+10+10=40ms.
|
||||
cfg.RetryBackoffDuration = 10 * time.Millisecond
|
||||
cfg.PublishMaxRetries = 4
|
||||
cfg.Events.AckWait = 20 * time.Millisecond
|
||||
pub.failures = 4
|
||||
pps = newService()
|
||||
|
||||
start := time.Now()
|
||||
Expect(pps.processEvent(bytesReceived())).To(Succeed())
|
||||
elapsed := time.Since(start)
|
||||
|
||||
Expect(elapsed).To(BeNumerically("<", 100*time.Millisecond))
|
||||
|
||||
attempts, _ := pub.stats()
|
||||
Expect(attempts).To(Equal(5))
|
||||
})
|
||||
|
||||
It("is fatal once the retries are genuinely exhausted", func() {
|
||||
pub.failures = 1000
|
||||
pps = newService()
|
||||
|
||||
err := pps.processEvent(bytesReceived())
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(errors.Is(err, ErrFatal)).To(BeTrue())
|
||||
|
||||
// the initial attempt plus PublishMaxRetries retries
|
||||
attempts, accepted := pub.stats()
|
||||
Expect(attempts).To(Equal(4))
|
||||
Expect(accepted).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("does not retry when retrying is disabled", func() {
|
||||
cfg.PublishMaxRetries = 0
|
||||
pub.failures = 1000
|
||||
pps = newService()
|
||||
|
||||
err := pps.processEvent(bytesReceived())
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(errors.Is(err, ErrFatal)).To(BeTrue())
|
||||
|
||||
attempts, _ := pub.stats()
|
||||
Expect(attempts).To(Equal(1))
|
||||
})
|
||||
|
||||
It("does not take the process down when the service is stopping", func() {
|
||||
pub.failures = 1000
|
||||
pps = newService()
|
||||
pps.Close()
|
||||
|
||||
err := pps.processEvent(bytesReceived())
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(errors.Is(err, ErrEvent)).To(BeTrue())
|
||||
Expect(errors.Is(err, ErrFatal)).To(BeFalse())
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -173,13 +173,13 @@ func (b *Backend) Upsert(id string, r search.Resource) error {
|
||||
return batch.Push()
|
||||
}
|
||||
|
||||
func (b *Backend) Move(rootID, parentID, location string) error {
|
||||
func (b *Backend) Move(id, parentID, targetPath string) error {
|
||||
batch, err := b.NewBatch(defaultBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := batch.Move(rootID, parentID, location); err != nil {
|
||||
if err := batch.Move(id, parentID, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -225,6 +225,31 @@ func (b *Backend) Purge(id string, onlyDeleted bool) error {
|
||||
return batch.Push()
|
||||
}
|
||||
|
||||
func (b *Backend) PurgeSpace(rootID string) error {
|
||||
for {
|
||||
req := bleve.NewSearchRequest(&query.TermQuery{FieldVal: "RootID", Term: rootID})
|
||||
req.Size = defaultBatchSize
|
||||
|
||||
res, err := b.index.Search(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if res.Hits.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
batch := b.index.NewBatch()
|
||||
for _, hit := range res.Hits {
|
||||
batch.Delete(hit.ID)
|
||||
}
|
||||
|
||||
if err := b.index.Batch(batch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Backend) NewBatch(size int) (search.BatchOperator, error) {
|
||||
return NewBatch(b.index, size)
|
||||
}
|
||||
@@ -103,6 +103,52 @@ var _ = Describe("Bleve", func() {
|
||||
}
|
||||
})
|
||||
|
||||
Describe("PurgeSpace", func() {
|
||||
It("takes every record of that space out of the index", func() {
|
||||
otherSpace := search.Resource{
|
||||
ID: "1$9!9",
|
||||
RootID: "1$9!9",
|
||||
Path: ".",
|
||||
Document: content.Document{Name: "other"},
|
||||
}
|
||||
for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} {
|
||||
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
|
||||
}
|
||||
|
||||
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)), "only the records of that space are gone")
|
||||
})
|
||||
|
||||
It("takes a space out that holds more records than one round", func() {
|
||||
otherSpace := search.Resource{
|
||||
ID: "1$9!9",
|
||||
RootID: "1$9!9",
|
||||
Path: ".",
|
||||
Document: content.Document{Name: "other"},
|
||||
}
|
||||
Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed())
|
||||
|
||||
for i := range 120 {
|
||||
resource := search.Resource{
|
||||
ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i),
|
||||
RootID: rootResource.RootID,
|
||||
Path: fmt.Sprintf("./file-%d", i),
|
||||
Document: content.Document{Name: fmt.Sprintf("file-%d", i)},
|
||||
}
|
||||
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
|
||||
}
|
||||
|
||||
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)), "only the record of the other space is left")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("New", func() {
|
||||
It("returns a new index instance", func() {
|
||||
b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
|
||||
|
||||
@@ -40,14 +40,14 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Batch) Move(id, parentID, location string) error {
|
||||
func (b *Batch) Move(id string, parentID string, targetPath string) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
rootResource, err := searchResourceByID(id, b.index)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
currentPath := rootResource.Path
|
||||
nextPath := utils.MakeRelativePath(location)
|
||||
nextPath := utils.MakeRelativePath(targetPath)
|
||||
|
||||
rootResource.Path = nextPath
|
||||
rootResource.Name = path.Base(nextPath)
|
||||
|
||||
@@ -102,9 +102,9 @@ func searchResourceByID(id string, index bleve.Index) (*search.Resource, error)
|
||||
return matchToResource(res.Hits[0]), nil
|
||||
}
|
||||
|
||||
func searchResourcesByPath(rootId, lookupPath string, index bleve.Index) ([]*search.Resource, error) {
|
||||
func searchResourcesByPath(rootID string, lookupPath string, index bleve.Index) ([]*search.Resource, error) {
|
||||
q := bleve.NewConjunctionQuery(
|
||||
bleve.NewQueryStringQuery("RootID:"+rootId),
|
||||
bleve.NewQueryStringQuery("RootID:"+rootID),
|
||||
bleve.NewQueryStringQuery("Path:"+escapeQuery(lookupPath+"/*")),
|
||||
)
|
||||
bleveReq := bleve.NewSearchRequest(q)
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
|
||||
searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
@@ -68,7 +70,10 @@ func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error)
|
||||
|
||||
func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
|
||||
boolQuery, err := convert.KQLToOpenSearchBoolQuery(sir.Query)
|
||||
if err != nil {
|
||||
switch {
|
||||
case searchQuery.IsValidationError(err):
|
||||
return nil, errtypes.BadRequest(err.Error())
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to convert KQL query to OpenSearch bool query: %w", err)
|
||||
}
|
||||
|
||||
@@ -194,13 +199,13 @@ func (b *Backend) Upsert(id string, r search.Resource) error {
|
||||
return batch.Push()
|
||||
}
|
||||
|
||||
func (b *Backend) Move(id string, parentID string, target string) error {
|
||||
func (b *Backend) Move(id string, parentID string, targetPath string) error {
|
||||
batch, err := b.NewBatch(defaultBatchSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := batch.Move(id, parentID, target); err != nil {
|
||||
if err := batch.Move(id, parentID, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -246,6 +251,32 @@ func (b *Backend) Purge(id string, onlyDeleted bool) error {
|
||||
return batch.Push()
|
||||
}
|
||||
|
||||
func (b *Backend) PurgeSpace(rootID string) error {
|
||||
req, err := osu.BuildDocumentDeleteByQueryReq(
|
||||
opensearchgoAPI.DocumentDeleteByQueryReq{
|
||||
Indices: []string{b.index},
|
||||
Params: opensearchgoAPI.DocumentDeleteByQueryParams{
|
||||
WaitForCompletion: conversions.ToPointer(true),
|
||||
Refresh: conversions.ToPointer(true),
|
||||
},
|
||||
},
|
||||
osu.NewBoolQuery().Must(osu.NewTermQuery[string]("RootID").Value(rootID)),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to build the space purge request %s: %w", rootID, err)
|
||||
}
|
||||
|
||||
resp, err := b.client.Document.DeleteByQuery(context.TODO(), req)
|
||||
switch {
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to purge space %s: %w", rootID, err)
|
||||
case len(resp.Failures) != 0:
|
||||
return fmt.Errorf("failed to purge space %s: %v", rootID, resp.Failures)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Backend) NewBatch(size int) (search.BatchOperator, error) {
|
||||
return NewBatch(b.client, b.index, size)
|
||||
}
|
||||
@@ -1,283 +1,613 @@
|
||||
package opensearch_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
opensearchgo "github.com/opensearch-project/opensearch-go/v4"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
func TestNewBackend(t *testing.T) {
|
||||
t.Run("fails to create if the cluster is not healthy", func(t *testing.T) {
|
||||
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
|
||||
Client: opensearchgo.Config{
|
||||
Addresses: []string{"http://localhost:1025"},
|
||||
func TestOpenSearchBackend(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "OpenSearch Backend Suite")
|
||||
}
|
||||
|
||||
func deleteIndexOnCleanup(tc *opensearchtest.TestClient, indexName string) {
|
||||
DeferCleanup(func() {
|
||||
Expect(tc.IndicesDelete(context.Background(), []string{indexName})).To(Succeed())
|
||||
})
|
||||
}
|
||||
|
||||
func resourceByID(tc *opensearchtest.TestClient, index, id string) search.Resource {
|
||||
GinkgoHelper()
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"ids": map[string]any{
|
||||
"values": []string{id},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "failed to create OpenSearch client")
|
||||
|
||||
backend, err := opensearch.NewBackend("test-engine-new-engine", client)
|
||||
require.Nil(t, backend)
|
||||
require.ErrorIs(t, err, opensearch.ErrUnhealthyCluster)
|
||||
},
|
||||
})
|
||||
|
||||
resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(index, strings.NewReader(body)).Hits)
|
||||
Expect(resources).To(HaveLen(1))
|
||||
return resources[0]
|
||||
}
|
||||
|
||||
func TestEngine_Search(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-search"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
// otherRoot returns a copy of the given resource that lives in a different root (space)
|
||||
// while keeping the same path, so it can be used to assert that cross-root updates do
|
||||
// not affect identically-named resources in other roots.
|
||||
func otherRoot(r search.Resource) search.Resource {
|
||||
r.ID = "2$2!3"
|
||||
r.RootID = "2$2!1"
|
||||
r.ParentID = "2$2!2"
|
||||
return r
|
||||
}
|
||||
|
||||
func newBackend(indexName string, resources ...search.Resource) (*opensearch.Backend, *opensearchtest.TestClient) {
|
||||
GinkgoHelper()
|
||||
|
||||
tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
for _, r := range resources {
|
||||
tc.Require.DocumentCreate(indexName, r.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), r)))
|
||||
}
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, len(resources))
|
||||
|
||||
t.Run("most simple search", func(t *testing.T) {
|
||||
resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, resp.Matches, 1)
|
||||
require.Equal(t, int32(1), resp.TotalMatches)
|
||||
require.Equal(t, document.ID, fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId))
|
||||
})
|
||||
|
||||
t.Run("ignores files that are marked as deleted", func(t *testing.T) {
|
||||
deletedDocument := opensearchtest.Testdata.Resources.File
|
||||
deletedDocument.ID = "1$2!4"
|
||||
deletedDocument.Deleted = true
|
||||
|
||||
tc.Require.DocumentCreate(indexName, deletedDocument.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, deletedDocument)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, resp.Matches, 1)
|
||||
require.Equal(t, int32(1), resp.TotalMatches)
|
||||
require.Equal(t, document.ID, fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId))
|
||||
})
|
||||
return backend, tc
|
||||
}
|
||||
|
||||
func TestEngine_Upsert(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-upsert"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("upsert with full document", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
require.NoError(t, backend.Upsert(document.ID, document))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngine_Move(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-move"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("moves the document to a new path", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(t, map[string]any{
|
||||
"query": map[string]any{
|
||||
"ids": map[string]any{
|
||||
"values": []string{document.ID},
|
||||
var _ = Describe("Backend", func() {
|
||||
Describe("NewBackend", func() {
|
||||
It("fails to create if the cluster is not healthy", func() {
|
||||
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
|
||||
Client: opensearchgo.Config{
|
||||
Addresses: []string{"http://localhost:1025"},
|
||||
},
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred(), "failed to create OpenSearch client")
|
||||
|
||||
backend, err := opensearch.NewBackend("test-engine-new-engine", client)
|
||||
Expect(backend).To(BeNil())
|
||||
Expect(err).To(MatchError(opensearch.ErrUnhealthyCluster))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Search", func() {
|
||||
const indexName = "opencloud-test-engine-search"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
document search.Resource
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
document = opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
|
||||
resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](t, tc.Require.Search(indexName, strings.NewReader(body)).Hits)
|
||||
require.Len(t, resources, 1)
|
||||
require.Equal(t, document.Path, resources[0].Path)
|
||||
It("performs the most simple search", func() {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(1))
|
||||
Expect(resp.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID))
|
||||
})
|
||||
|
||||
document.Path = "./new/path/to/resource"
|
||||
require.NoError(t, backend.Move(document.ID, document.ParentID, document.Path))
|
||||
It("ignores files that are marked as deleted", func() {
|
||||
deletedDocument := opensearchtest.Testdata.Resources.File
|
||||
deletedDocument.ID = "1$2!4"
|
||||
deletedDocument.Deleted = true
|
||||
|
||||
resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](t, tc.Require.Search(indexName, strings.NewReader(body)).Hits)
|
||||
require.Len(t, resources, 1)
|
||||
require.Equal(t, document.Path, resources[0].Path)
|
||||
tc.Require.DocumentCreate(indexName, deletedDocument.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), deletedDocument)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(1))
|
||||
Expect(resp.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngine_Delete(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-delete"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
Describe("Upsert", func() {
|
||||
const indexName = "opencloud-test-engine-upsert"
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
t.Run("mark document as deleted", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(t, map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"Deleted": map[string]any{
|
||||
"value": true,
|
||||
It("upserts a full document", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
Expect(backend.Upsert(document.ID, document)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Move", func() {
|
||||
const indexName = "opencloud-test-engine-move"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("moves the document to a new path", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"ids": map[string]any{
|
||||
"values": []string{document.ID},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits)
|
||||
Expect(resources).To(HaveLen(1))
|
||||
Expect(resources[0].Path).To(Equal(document.Path))
|
||||
|
||||
document.Path = "./new/path/to/resource"
|
||||
Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed())
|
||||
|
||||
resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits)
|
||||
Expect(resources).To(HaveLen(1))
|
||||
Expect(resources[0].Path).To(Equal(document.Path))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete", func() {
|
||||
const indexName = "opencloud-test-engine-delete"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
|
||||
It("marks the document as deleted", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
require.NoError(t, backend.Delete(document.ID))
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngine_Restore(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-restore"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("mark document as not deleted", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
document.Deleted = true
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(t, map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"Deleted": map[string]any{
|
||||
"value": true,
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"Deleted": map[string]any{
|
||||
"value": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
|
||||
|
||||
Expect(backend.Delete(document.ID)).To(Succeed())
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Restore", func() {
|
||||
const indexName = "opencloud-test-engine-restore"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
|
||||
It("marks the document as not deleted", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
document.Deleted = true
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
require.NoError(t, backend.Restore(document.ID))
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
|
||||
})
|
||||
}
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"Deleted": map[string]any{
|
||||
"value": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
func TestEngine_Purge(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-purge"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("purge with full document", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
require.NoError(t, backend.Purge(document.ID, false))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
Expect(backend.Restore(document.ID)).To(Succeed())
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("purge resource trees", func(t *testing.T) {
|
||||
resourceFolder := opensearchtest.Testdata.Resources.Folder
|
||||
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFolder)))
|
||||
Describe("Purge", func() {
|
||||
const indexName = "opencloud-test-engine-purge"
|
||||
|
||||
resourceFile := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFile)))
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
require.NoError(t, backend.Purge(resourceFolder.ID, false))
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
It("purges a full document", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
Expect(backend.Purge(document.ID, false)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
})
|
||||
|
||||
It("purges resource trees", func() {
|
||||
resourceFolder := opensearchtest.Testdata.Resources.Folder
|
||||
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder)))
|
||||
|
||||
resourceFile := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile)))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
Expect(backend.Purge(resourceFolder.ID, false)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
})
|
||||
|
||||
It("purges resource trees and ignores undeleted resources", func() {
|
||||
resourceFolder := opensearchtest.Testdata.Resources.Folder
|
||||
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder)))
|
||||
|
||||
resourceFile := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile)))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
Expect(backend.Delete(resourceFile.ID)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
Expect(backend.Purge(resourceFolder.ID, true)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("purge resource trees and ignores undeleted resources", func(t *testing.T) {
|
||||
resourceFolder := opensearchtest.Testdata.Resources.Folder
|
||||
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFolder)))
|
||||
Describe("PurgeSpace", func() {
|
||||
const indexName = "opencloud-test-engine-purge-space"
|
||||
|
||||
resourceFile := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, resourceFile)))
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
require.NoError(t, backend.Delete(resourceFile.ID))
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
require.NoError(t, backend.Purge(resourceFolder.ID, true))
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
It("takes every record of that space out of the index", func() {
|
||||
gone := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, gone.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), gone)))
|
||||
|
||||
stays := opensearchtest.Testdata.Resources.File
|
||||
stays.ID = "1$2!3"
|
||||
stays.RootID = "1$2!2"
|
||||
tc.Require.DocumentCreate(indexName, stays.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), stays)))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
Expect(backend.PurgeSpace(gone.RootID)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
left := opensearchtest.SearchHitsMustBeConverted[search.Resource](
|
||||
GinkgoTB(),
|
||||
tc.Require.Search(indexName, strings.NewReader(`{"query":{"match_all":{}}}`)).Hits,
|
||||
)
|
||||
Expect(left).To(HaveLen(1), "only the records of that space are gone")
|
||||
Expect(left[0].ID).To(Equal(stays.ID))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngine_DocCount(t *testing.T) {
|
||||
indexName := "opencloud-test-engine-doc-count"
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
Describe("DocCount", func() {
|
||||
const indexName = "opencloud-test-engine-doc-count"
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
require.NoError(t, err)
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
t.Run("ignore deleted documents", func(t *testing.T) {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
count, err := backend.DocCount()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(1), count)
|
||||
It("ignores deleted documents", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(t, map[string]any{
|
||||
"doc": map[string]any{
|
||||
"Deleted": true,
|
||||
count, err := backend.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"doc": map[string]any{
|
||||
"Deleted": true,
|
||||
},
|
||||
})))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
count, err = backend.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(0)))
|
||||
})
|
||||
})
|
||||
|
||||
// The following specs ensure that updates which affect a resource and its descendants
|
||||
// (Delete, Restore, Move) are scoped to the root (space) of the target resource. Two
|
||||
// resources living in different roots may share the exact same path, so matching by
|
||||
// path alone would incorrectly update the wrong resource.
|
||||
Describe("updateSelfAndDescendants root scope", func() {
|
||||
It("deletes only the resource in the target root", func() {
|
||||
const indexName = "opencloud-test-engine-root-scope-delete"
|
||||
|
||||
target := opensearchtest.Testdata.Resources.File
|
||||
other := otherRoot(target)
|
||||
|
||||
backend, tc := newBackend(indexName, target, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
Expect(backend.Delete(target.ID)).To(Succeed())
|
||||
|
||||
Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeTrue(), "target resource should be marked as deleted")
|
||||
Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeFalse(), "resource in a different root must not be affected")
|
||||
})
|
||||
|
||||
It("restores only the resource in the target root", func() {
|
||||
const indexName = "opencloud-test-engine-root-scope-restore"
|
||||
|
||||
target := opensearchtest.Testdata.Resources.File
|
||||
target.Deleted = true
|
||||
other := otherRoot(target)
|
||||
|
||||
backend, tc := newBackend(indexName, target, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
Expect(backend.Restore(target.ID)).To(Succeed())
|
||||
|
||||
Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeFalse(), "target resource should be restored")
|
||||
Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeTrue(), "resource in a different root must not be affected")
|
||||
})
|
||||
|
||||
It("moves only the resource in the target root", func() {
|
||||
const indexName = "opencloud-test-engine-root-scope-move"
|
||||
|
||||
target := opensearchtest.Testdata.Resources.File
|
||||
other := otherRoot(target)
|
||||
|
||||
backend, tc := newBackend(indexName, target, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
Expect(backend.Move(target.ID, target.ParentID, "./new/path/to/resource")).To(Succeed())
|
||||
|
||||
Expect(resourceByID(tc, indexName, target.ID).Path).To(Equal("./new/path/to/resource"), "target resource should be moved")
|
||||
Expect(resourceByID(tc, indexName, other.ID).Path).To(Equal(other.Path), "resource in a different root must not be moved")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("SearchInAnalyzedFields", func() {
|
||||
const indexName = "opencloud-test-engine-search-analyzed-fields"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
dashed := opensearchtest.Testdata.Resources.Folder
|
||||
dashed.ID = "1$1!10"
|
||||
dashed.Name = "new-folder"
|
||||
dashed.Path = "./new-folder"
|
||||
dashed.Title = "quarterly report"
|
||||
|
||||
plain := opensearchtest.Testdata.Resources.Folder
|
||||
plain.ID = "1$1!11"
|
||||
plain.Name = "documents"
|
||||
plain.Path = "./documents"
|
||||
plain.Title = "notes"
|
||||
|
||||
spaced := opensearchtest.Testdata.Resources.Folder
|
||||
spaced.ID = "1$1!12"
|
||||
spaced.Name = "foo bar"
|
||||
spaced.Path = "./foo bar"
|
||||
spaced.Title = "spaced out"
|
||||
|
||||
backend, tc = newBackend(indexName, dashed, plain, spaced)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
})
|
||||
|
||||
DescribeTable("finds what the analyzer made of the value",
|
||||
func(query string, want []string) {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
names := make([]string, 0, len(resp.Matches))
|
||||
for _, match := range resp.Matches {
|
||||
names = append(names, match.Entity.Name)
|
||||
}
|
||||
Expect(names).To(ConsistOf(want))
|
||||
},
|
||||
})))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
count, err = backend.DocCount()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(0), count)
|
||||
Entry("the full name with the dash", "new-folder", []string{"new-folder"}),
|
||||
Entry("one token of it", "new", []string{"new-folder"}),
|
||||
Entry("a name without a dash", "documents", []string{"documents"}),
|
||||
Entry("a wildcard", "*folder*", []string{"new-folder"}),
|
||||
// the shape the web client sends for every name search
|
||||
Entry("a wildcard around the whole dashed name", `name:"*new-folder*"`, []string{"new-folder"}),
|
||||
Entry("a wildcard spanning the dash", `name:"*w-fol*"`, []string{"new-folder"}),
|
||||
Entry("a wildcard in a different case", `name:"*NEW-FOLDER*"`, []string{"new-folder"}),
|
||||
Entry("a wildcard spanning a space", `name:"*oo ba*"`, []string{"foo bar"}),
|
||||
Entry("a wildcard around a name with a space", `name:"*foo bar*"`, []string{"foo bar"}),
|
||||
Entry("a name with a space", `name:"foo bar"`, []string{"foo bar"}),
|
||||
Entry("a title of two words", `Title:"quarterly report"`, []string{"new-folder"}),
|
||||
Entry("one token of a title", "Title:quarterly", []string{"new-folder"}),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
Describe("SearchByTag", func() {
|
||||
const indexName = "opencloud-test-engine-search-by-tag"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tagged := opensearchtest.Testdata.Resources.Folder
|
||||
tagged.ID = "1$1!20"
|
||||
tagged.Name = "tagged"
|
||||
tagged.Path = "./tagged"
|
||||
tagged.Tags = []string{"foo-bar"}
|
||||
|
||||
other := opensearchtest.Testdata.Resources.Folder
|
||||
other.ID = "1$1!21"
|
||||
other.Name = "other"
|
||||
other.Path = "./other"
|
||||
other.Tags = []string{"foo"}
|
||||
|
||||
backend, tc = newBackend(indexName, tagged, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
})
|
||||
|
||||
// a tag is one label, not prose, so it matches as a whole or not at all
|
||||
DescribeTable("matches a tag as a whole",
|
||||
func(query string, want []string) {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
names := make([]string, 0, len(resp.Matches))
|
||||
for _, match := range resp.Matches {
|
||||
names = append(names, match.Entity.Name)
|
||||
}
|
||||
Expect(names).To(ConsistOf(want))
|
||||
},
|
||||
Entry("the whole tag", `tag:("foo-bar")`, []string{"tagged"}),
|
||||
Entry("a token of a tag does not match it", `tag:("foo")`, []string{"other"}),
|
||||
Entry("a tag in a different case", `tag:("FOO-BAR")`, []string{"tagged"}),
|
||||
Entry("a wildcard reaches both", `tag:("*foo*")`, []string{"tagged", "other"}),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("SearchWithAnInvalidQuery", func() {
|
||||
const indexName = "opencloud-test-engine-search-invalid-query"
|
||||
|
||||
It("answers with a bad request", func() {
|
||||
backend, tc := newBackend(indexName)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
_, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "AND mediatype:document"})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest("")))
|
||||
Expect(err.Error()).To(Equal(`error: bad request: the expression can't begin from a binary operator: 'AND'`))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -63,7 +63,7 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Batch) Move(id, parentID, location string) error {
|
||||
func (b *Batch) Move(id string, parentID string, targetPath string) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
op := func() error {
|
||||
return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(rootResource search.Resource) *osu.BodyParamScript {
|
||||
@@ -77,8 +77,8 @@ func (b *Batch) Move(id, parentID, location string) error {
|
||||
"id": id,
|
||||
"parentID": parentID,
|
||||
"oldPath": rootResource.Path,
|
||||
"newPath": utils.MakeRelativePath(location),
|
||||
"newName": path.Base(utils.MakeRelativePath(location)),
|
||||
"newPath": utils.MakeRelativePath(targetPath),
|
||||
"newName": path.Base(utils.MakeRelativePath(targetPath)),
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package convert_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestConvert(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "OpenSearch Convert Suite")
|
||||
}
|
||||
@@ -14,7 +14,7 @@ var (
|
||||
func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
|
||||
kqlAst, err := kql.Builder{}.Build(kqlQuery)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build query: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
kqlNodes, err := ExpandKQL(kqlAst.Nodes)
|
||||
|
||||
@@ -3,6 +3,7 @@ package convert
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -93,47 +94,13 @@ func (t kqlOpensearchTranspiler) getOperatorValueAt(nodes []ast.Node, i int) str
|
||||
}
|
||||
|
||||
func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
|
||||
var builder osu.Builder
|
||||
|
||||
switch node := node.(type) {
|
||||
case *ast.BooleanNode:
|
||||
return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
|
||||
case *ast.StringNode:
|
||||
isWildcard := strings.Contains(node.Value, "*")
|
||||
if isWildcard {
|
||||
return osu.NewWildcardQuery(node.Key).Value(node.Value), nil
|
||||
}
|
||||
|
||||
totalTerms := strings.Split(node.Value, " ")
|
||||
isSingleTerm := len(totalTerms) == 1
|
||||
isMultiTerm := len(totalTerms) >= 1
|
||||
switch {
|
||||
case isSingleTerm:
|
||||
return osu.NewTermQuery[string](node.Key).Value(node.Value), nil
|
||||
case isMultiTerm:
|
||||
return osu.NewMatchPhraseQuery(node.Key).Query(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported string node value: %s", node.Value)
|
||||
return stringNodeQuery(node), nil
|
||||
case *ast.DateTimeNode:
|
||||
if node.Operator == nil {
|
||||
return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
query := osu.NewRangeQuery[time.Time](node.Key)
|
||||
|
||||
switch node.Operator.Value {
|
||||
case ">":
|
||||
return query.Gt(node.Value), nil
|
||||
case ">=":
|
||||
return query.Gte(node.Value), nil
|
||||
case "<":
|
||||
return query.Lt(node.Value), nil
|
||||
case "<=":
|
||||
return query.Lte(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
|
||||
return dateTimeNodeQuery(node)
|
||||
case *ast.GroupNode:
|
||||
group, err := t.transpile(node.Nodes)
|
||||
if err != nil {
|
||||
@@ -145,3 +112,56 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
|
||||
|
||||
return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node)
|
||||
}
|
||||
|
||||
// stringNodeQuery picks the query a string node turns into.
|
||||
func stringNodeQuery(node *ast.StringNode) osu.Builder {
|
||||
isWildcard := strings.Contains(node.Value, "*")
|
||||
|
||||
switch {
|
||||
// Name: "*oo-bar", "*oo ba*", "*OO*"
|
||||
// Title: "*rterly rep*"
|
||||
// Tags: "*spaced tag*"
|
||||
case isWildcard && slices.Contains([]string{"Name", "Title", "Tags"}, node.Key):
|
||||
return osu.NewWildcardQuery(node.Key + ".keyword").
|
||||
Value(node.Value).
|
||||
Params(&osu.WildcardQueryParams{CaseInsensitive: true})
|
||||
// Path: "./foo*", MimeType: "*plain"
|
||||
case isWildcard:
|
||||
return osu.NewWildcardQuery(node.Key).Value(node.Value)
|
||||
// Tags: "foo-bar", "spaced tag", "FOO-BAR"
|
||||
case node.Key == "Tags":
|
||||
return osu.NewTermQuery[string](node.Key + ".keyword").
|
||||
Value(node.Value).
|
||||
Params(&osu.TermQueryParams{CaseInsensitive: true})
|
||||
// Name: "foo-bar", "foo bar"
|
||||
// Title: "quarterly report"
|
||||
// Content: "foo bar"
|
||||
case slices.Contains([]string{"Name", "Title", "Content"}, node.Key):
|
||||
return osu.NewMatchPhraseQuery(node.Key).Query(node.Value)
|
||||
// Path: "./foo bar", MimeType: "text/plain"
|
||||
default:
|
||||
return osu.NewTermQuery[string](node.Key).Value(node.Value)
|
||||
}
|
||||
}
|
||||
|
||||
// dateTimeNodeQuery turns a date time node into a range query.
|
||||
func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) {
|
||||
if node.Operator == nil {
|
||||
return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
query := osu.NewRangeQuery[time.Time](node.Key)
|
||||
|
||||
switch node.Operator.Value {
|
||||
case ">":
|
||||
return query.Gt(node.Value), nil
|
||||
case ">=":
|
||||
return query.Gte(node.Value), nil
|
||||
case "<":
|
||||
return query.Lt(node.Value), nil
|
||||
case "<=":
|
||||
return query.Lte(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
|
||||
}
|
||||
@@ -16,13 +16,13 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
|
||||
// kql to os dsl - type tests
|
||||
{
|
||||
Name: "term query - string node",
|
||||
Name: "match phrase query - string node on an analyzed field",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
},
|
||||
{
|
||||
Name: "term query - boolean node - true",
|
||||
@@ -58,7 +58,18 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
&ast.StringNode{Key: "Name", Value: "open*"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewWildcardQuery("Name").Value("open*"),
|
||||
Want: osu.NewWildcardQuery("Name.keyword").
|
||||
Value("open*").
|
||||
Params(&osu.WildcardQueryParams{CaseInsensitive: true}),
|
||||
},
|
||||
{
|
||||
Name: "wildcard query - string node without a keyword sub field",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Content", Value: "open*"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewWildcardQuery("Content").Value("open*"),
|
||||
},
|
||||
{
|
||||
Name: "bool query",
|
||||
@@ -71,8 +82,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().Must(
|
||||
osu.NewTermQuery[string]("Name").Value("a"),
|
||||
osu.NewTermQuery[string]("Name").Value("b"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("a"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("b"),
|
||||
),
|
||||
},
|
||||
{
|
||||
@@ -84,7 +95,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
}},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("any"),
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query("any"),
|
||||
},
|
||||
{
|
||||
Name: "range query >",
|
||||
@@ -146,7 +157,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
},
|
||||
{
|
||||
Name: "[* *]",
|
||||
@@ -158,7 +169,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -173,7 +184,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -189,7 +200,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -217,7 +228,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
).
|
||||
MustNot(
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
@@ -237,7 +248,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
osu.NewTermQuery[string]("age").Value("44"),
|
||||
),
|
||||
|
||||
@@ -55,11 +55,12 @@ func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match,
|
||||
SpaceId: resourceParentID.GetSpaceId(),
|
||||
OpaqueId: resourceParentID.GetOpaqueId(),
|
||||
},
|
||||
Size: resource.Size,
|
||||
Type: resource.Type,
|
||||
MimeType: resource.MimeType,
|
||||
Deleted: resource.Deleted,
|
||||
Tags: resource.Tags,
|
||||
Size: resource.Size,
|
||||
Type: resource.Type,
|
||||
MimeType: resource.MimeType,
|
||||
Deleted: resource.Deleted,
|
||||
Tags: resource.Tags,
|
||||
Favorites: resource.Favorites,
|
||||
Highlights: func() string {
|
||||
contentHighlights, ok := hit.Highlight["Content"]
|
||||
if !ok {
|
||||
|
||||
@@ -2,37 +2,114 @@ package convert_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
func TestOpenSearchHitToMatch(t *testing.T) {
|
||||
resource := opensearchtest.Testdata.Resources.File
|
||||
resource.MimeType = "audio/anything"
|
||||
|
||||
hit := opensearchgoAPI.SearchHit{
|
||||
Score: 1.1,
|
||||
Source: json.RawMessage(opensearchtest.JSONMustMarshal(t, resource)),
|
||||
}
|
||||
match, err := convert.OpenSearchHitToMatch(hit)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, hit.Score, match.Score)
|
||||
assert.Equal(t, resource.Name, match.Entity.Name)
|
||||
t.Parallel()
|
||||
t.Run("converts the audio field to the expected type", func(t *testing.T) {
|
||||
// searchMessage.Audio contains int64, int32 ... values that are converted to strings by the JSON marshaler,
|
||||
// so we need to convert the resource.Audio to align the expectations for the JSON comparison.
|
||||
audio, err := conversions.To[*searchMessage.Audio](resource.Audio)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, resource.Audio.Bitrate, match.Entity.Audio.Bitrate)
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, audio), opensearchtest.JSONMustMarshal(t, match.Entity.Audio))
|
||||
})
|
||||
// jsonMarshal marshals data to a JSON string, failing the running spec on error.
|
||||
func jsonMarshal(data any) string {
|
||||
GinkgoHelper()
|
||||
b, err := json.Marshal(data)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return string(b)
|
||||
}
|
||||
|
||||
var _ = Describe("OpenSearchHitToMatch", func() {
|
||||
var (
|
||||
resource search.Resource
|
||||
hit opensearchgoAPI.SearchHit
|
||||
mtime time.Time
|
||||
match *searchMessage.Match
|
||||
err error
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
resource = opensearchtest.Testdata.Resources.File
|
||||
resource.MimeType = "audio/mpeg"
|
||||
mtime = time.Date(2025, 7, 24, 15, 15, 1, 0, time.UTC)
|
||||
resource.Mtime = mtime.Format(time.RFC3339)
|
||||
resource.Favorites = []string{"cbf24bce-3e6e-4d9e-a2a2-cbf24bce3e6e"}
|
||||
|
||||
hit = opensearchgoAPI.SearchHit{
|
||||
Score: 1.1,
|
||||
Source: json.RawMessage(jsonMarshal(resource)),
|
||||
Highlight: map[string][]string{
|
||||
"Content": {"first match", "second match"},
|
||||
},
|
||||
}
|
||||
|
||||
match, err = convert.OpenSearchHitToMatch(hit)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("maps the score", func() {
|
||||
Expect(match.Score).To(Equal(hit.Score))
|
||||
})
|
||||
|
||||
It("maps all resource fields to the entity", func() {
|
||||
entity := match.Entity
|
||||
Expect(entity).ToNot(BeNil())
|
||||
|
||||
// reference (derived from RootID) and path
|
||||
Expect(entity.Ref.ResourceId.StorageId).To(Equal("1"))
|
||||
Expect(entity.Ref.ResourceId.SpaceId).To(Equal("1"))
|
||||
Expect(entity.Ref.ResourceId.OpaqueId).To(Equal("1"))
|
||||
Expect(entity.Ref.Path).To(Equal(resource.Path))
|
||||
|
||||
// resource id
|
||||
Expect(entity.Id.StorageId).To(Equal("1"))
|
||||
Expect(entity.Id.SpaceId).To(Equal("1"))
|
||||
Expect(entity.Id.OpaqueId).To(Equal("3"))
|
||||
|
||||
// parent id
|
||||
Expect(entity.ParentId.StorageId).To(Equal("1"))
|
||||
Expect(entity.ParentId.SpaceId).To(Equal("1"))
|
||||
Expect(entity.ParentId.OpaqueId).To(Equal("2"))
|
||||
|
||||
// scalar fields
|
||||
Expect(entity.Name).To(Equal(resource.Name))
|
||||
Expect(entity.Size).To(Equal(resource.Size))
|
||||
Expect(entity.Type).To(Equal(resource.Type))
|
||||
Expect(entity.MimeType).To(Equal(resource.MimeType))
|
||||
Expect(entity.Deleted).To(Equal(resource.Deleted))
|
||||
Expect(entity.Tags).To(Equal(resource.Tags))
|
||||
Expect(entity.Favorites).To(Equal(resource.Favorites))
|
||||
|
||||
// highlights are joined together
|
||||
Expect(entity.Highlights).To(Equal("first match; second match"))
|
||||
|
||||
// last modified time is parsed from the Mtime
|
||||
Expect(entity.LastModifiedTime).ToNot(BeNil())
|
||||
Expect(entity.LastModifiedTime.Seconds).To(Equal(mtime.Unix()))
|
||||
})
|
||||
|
||||
It("converts the media metadata to the expected types", func() {
|
||||
// searchMessage.Audio contains int64, int32 ... values that are converted to strings by the JSON marshaler,
|
||||
// so we need to convert the resource fields to align the expectations for the JSON comparison.
|
||||
expectedAudio, err := conversions.To[*searchMessage.Audio](resource.Audio)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(match.Entity.Audio.Bitrate).To(Equal(resource.Audio.Bitrate))
|
||||
Expect(jsonMarshal(match.Entity.Audio)).To(MatchJSON(jsonMarshal(expectedAudio)))
|
||||
|
||||
expectedImage, err := conversions.To[*searchMessage.Image](resource.Image)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(jsonMarshal(match.Entity.Image)).To(MatchJSON(jsonMarshal(expectedImage)))
|
||||
|
||||
expectedLocation, err := conversions.To[*searchMessage.GeoCoordinates](resource.Location)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(jsonMarshal(match.Entity.Location)).To(MatchJSON(jsonMarshal(expectedLocation)))
|
||||
|
||||
expectedPhoto, err := conversions.To[*searchMessage.Photo](resource.Photo)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(jsonMarshal(match.Entity.Photo)).To(MatchJSON(jsonMarshal(expectedPhoto)))
|
||||
})
|
||||
})
|
||||
@@ -11,20 +11,20 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
)
|
||||
|
||||
var TimeMustParse = func(t *testing.T, ts string) time.Time {
|
||||
var TimeMustParse = func(t testing.TB, ts string) time.Time {
|
||||
tp, err := time.Parse(time.RFC3339Nano, ts)
|
||||
require.NoError(t, err, "failed to parse time %s", ts)
|
||||
|
||||
return tp
|
||||
}
|
||||
|
||||
func JSONMustMarshal(t *testing.T, data any) string {
|
||||
func JSONMustMarshal(t testing.TB, data any) string {
|
||||
jsonData, err := json.Marshal(data)
|
||||
require.NoError(t, err, "failed to marshal data to JSON")
|
||||
return string(jsonData)
|
||||
}
|
||||
|
||||
func SearchHitsMustBeConverted[T any](t *testing.T, hits []opensearchgoAPI.SearchHit) []T {
|
||||
func SearchHitsMustBeConverted[T any](t testing.TB, hits []opensearchgoAPI.SearchHit) []T {
|
||||
ts := make([]T, len(hits))
|
||||
for i, hit := range hits {
|
||||
resource, err := conversions.To[T](hit.Source)
|
||||
|
||||
@@ -20,7 +20,7 @@ type TestClient struct {
|
||||
Require *testRequireClient
|
||||
}
|
||||
|
||||
func NewDefaultTestClient(t *testing.T, cfg config.EngineOpenSearchClient) *TestClient {
|
||||
func NewDefaultTestClient(t testing.TB, cfg config.EngineOpenSearchClient) *TestClient {
|
||||
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
|
||||
Client: opensearchgo.Config{
|
||||
Addresses: cfg.Addresses,
|
||||
@@ -33,7 +33,7 @@ func NewDefaultTestClient(t *testing.T, cfg config.EngineOpenSearchClient) *Test
|
||||
return NewTestClient(t, client)
|
||||
}
|
||||
|
||||
func NewTestClient(t *testing.T, client *opensearchgoAPI.Client) *TestClient {
|
||||
func NewTestClient(t testing.TB, client *opensearchgoAPI.Client) *TestClient {
|
||||
tc := &TestClient{c: client}
|
||||
trc := &testRequireClient{tc: tc, t: t}
|
||||
tc.Require = trc
|
||||
@@ -210,7 +210,7 @@ func (tc *TestClient) Search(ctx context.Context, index string, body io.Reader)
|
||||
|
||||
type testRequireClient struct {
|
||||
tc *TestClient
|
||||
t *testing.T
|
||||
t testing.TB
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) IndicesReset(indices []string) {
|
||||
|
||||
@@ -56,7 +56,9 @@ func updateSelfAndDescendants(ctx context.Context, client *opensearchgoAPI.Clien
|
||||
WaitForCompletion: conversions.ToPointer(true),
|
||||
},
|
||||
},
|
||||
osu.NewTermQuery[string]("Path").Value(resource.Path),
|
||||
osu.NewBoolQuery().
|
||||
Must(osu.NewTermQuery[string]("RootID").Value(resource.RootID)).
|
||||
Must(osu.NewTermQuery[string]("Path").Value(resource.Path)),
|
||||
osu.UpdateByQueryBodyParams{
|
||||
Script: scriptProvider(resource),
|
||||
},
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
@@ -38,10 +39,16 @@ func (e UnsupportedTimeRangeError) Error() string {
|
||||
return fmt.Sprintf("unable to convert '%v' to a time range", e.Value)
|
||||
}
|
||||
|
||||
// IsValidationError says whether the query itself is at fault, which makes it a
|
||||
// bad request and not an error of ours.
|
||||
func IsValidationError(err error) bool {
|
||||
switch err.(type) {
|
||||
case *StartsWithBinaryOperatorError, *NamedGroupInvalidNodesError, *UnsupportedTimeRangeError:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
var (
|
||||
startsWithBinaryOperator *StartsWithBinaryOperatorError
|
||||
namedGroupInvalidNodes *NamedGroupInvalidNodesError
|
||||
unsupportedTimeRange *UnsupportedTimeRangeError
|
||||
)
|
||||
|
||||
return errors.As(err, &startsWithBinaryOperator) ||
|
||||
errors.As(err, &namedGroupInvalidNodes) ||
|
||||
errors.As(err, &unsupportedTimeRange)
|
||||
}
|
||||
@@ -88,8 +88,8 @@ func (_c *BatchOperator_Delete_Call) RunAndReturn(run func(id string) error) *Ba
|
||||
}
|
||||
|
||||
// Move provides a mock function for the type BatchOperator
|
||||
func (_mock *BatchOperator) Move(rootID string, parentID string, location string) error {
|
||||
ret := _mock.Called(rootID, parentID, location)
|
||||
func (_mock *BatchOperator) Move(id string, parentID string, targetPath string) error {
|
||||
ret := _mock.Called(id, parentID, targetPath)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Move")
|
||||
@@ -97,7 +97,7 @@ func (_mock *BatchOperator) Move(rootID string, parentID string, location string
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(string, string, string) error); ok {
|
||||
r0 = returnFunc(rootID, parentID, location)
|
||||
r0 = returnFunc(id, parentID, targetPath)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
@@ -110,14 +110,14 @@ type BatchOperator_Move_Call struct {
|
||||
}
|
||||
|
||||
// Move is a helper method to define mock.On call
|
||||
// - rootID string
|
||||
// - id string
|
||||
// - parentID string
|
||||
// - location string
|
||||
func (_e *BatchOperator_Expecter) Move(rootID interface{}, parentID interface{}, location interface{}) *BatchOperator_Move_Call {
|
||||
return &BatchOperator_Move_Call{Call: _e.mock.On("Move", rootID, parentID, location)}
|
||||
// - targetPath string
|
||||
func (_e *BatchOperator_Expecter) Move(id interface{}, parentID interface{}, targetPath interface{}) *BatchOperator_Move_Call {
|
||||
return &BatchOperator_Move_Call{Call: _e.mock.On("Move", id, parentID, targetPath)}
|
||||
}
|
||||
|
||||
func (_c *BatchOperator_Move_Call) Run(run func(rootID string, parentID string, location string)) *BatchOperator_Move_Call {
|
||||
func (_c *BatchOperator_Move_Call) Run(run func(id string, parentID string, targetPath string)) *BatchOperator_Move_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 string
|
||||
if args[0] != nil {
|
||||
@@ -145,7 +145,7 @@ func (_c *BatchOperator_Move_Call) Return(err error) *BatchOperator_Move_Call {
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *BatchOperator_Move_Call) RunAndReturn(run func(rootID string, parentID string, location string) error) *BatchOperator_Move_Call {
|
||||
func (_c *BatchOperator_Move_Call) RunAndReturn(run func(id string, parentID string, targetPath string) error) *BatchOperator_Move_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
@@ -144,8 +144,8 @@ func (_c *Engine_DocCount_Call) RunAndReturn(run func() (uint64, error)) *Engine
|
||||
}
|
||||
|
||||
// Move provides a mock function for the type Engine
|
||||
func (_mock *Engine) Move(id string, parentid string, target string) error {
|
||||
ret := _mock.Called(id, parentid, target)
|
||||
func (_mock *Engine) Move(id string, parentID string, targetPath string) error {
|
||||
ret := _mock.Called(id, parentID, targetPath)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for Move")
|
||||
@@ -153,7 +153,7 @@ func (_mock *Engine) Move(id string, parentid string, target string) error {
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(string, string, string) error); ok {
|
||||
r0 = returnFunc(id, parentid, target)
|
||||
r0 = returnFunc(id, parentID, targetPath)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
@@ -167,13 +167,13 @@ type Engine_Move_Call struct {
|
||||
|
||||
// Move is a helper method to define mock.On call
|
||||
// - id string
|
||||
// - parentid string
|
||||
// - target string
|
||||
func (_e *Engine_Expecter) Move(id interface{}, parentid interface{}, target interface{}) *Engine_Move_Call {
|
||||
return &Engine_Move_Call{Call: _e.mock.On("Move", id, parentid, target)}
|
||||
// - parentID string
|
||||
// - targetPath string
|
||||
func (_e *Engine_Expecter) Move(id interface{}, parentID interface{}, targetPath interface{}) *Engine_Move_Call {
|
||||
return &Engine_Move_Call{Call: _e.mock.On("Move", id, parentID, targetPath)}
|
||||
}
|
||||
|
||||
func (_c *Engine_Move_Call) Run(run func(id string, parentid string, target string)) *Engine_Move_Call {
|
||||
func (_c *Engine_Move_Call) Run(run func(id string, parentID string, targetPath string)) *Engine_Move_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 string
|
||||
if args[0] != nil {
|
||||
@@ -201,7 +201,7 @@ func (_c *Engine_Move_Call) Return(err error) *Engine_Move_Call {
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Engine_Move_Call) RunAndReturn(run func(id string, parentid string, target string) error) *Engine_Move_Call {
|
||||
func (_c *Engine_Move_Call) RunAndReturn(run func(id string, parentID string, targetPath string) error) *Engine_Move_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
@@ -325,6 +325,57 @@ func (_c *Engine_Purge_Call) RunAndReturn(run func(id string, onlyDeleted bool)
|
||||
return _c
|
||||
}
|
||||
|
||||
// PurgeSpace provides a mock function for the type Engine
|
||||
func (_mock *Engine) PurgeSpace(rootID string) error {
|
||||
ret := _mock.Called(rootID)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for PurgeSpace")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(string) error); ok {
|
||||
r0 = returnFunc(rootID)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Engine_PurgeSpace_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PurgeSpace'
|
||||
type Engine_PurgeSpace_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// PurgeSpace is a helper method to define mock.On call
|
||||
// - rootID string
|
||||
func (_e *Engine_Expecter) PurgeSpace(rootID interface{}) *Engine_PurgeSpace_Call {
|
||||
return &Engine_PurgeSpace_Call{Call: _e.mock.On("PurgeSpace", rootID)}
|
||||
}
|
||||
|
||||
func (_c *Engine_PurgeSpace_Call) Run(run func(rootID string)) *Engine_PurgeSpace_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 string
|
||||
if args[0] != nil {
|
||||
arg0 = args[0].(string)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Engine_PurgeSpace_Call) Return(err error) *Engine_PurgeSpace_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Engine_PurgeSpace_Call) RunAndReturn(run func(rootID string) error) *Engine_PurgeSpace_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Restore provides a mock function for the type Engine
|
||||
func (_mock *Engine) Restore(id string) error {
|
||||
ret := _mock.Called(id)
|
||||
|
||||
@@ -40,8 +40,8 @@ func (_m *Searcher) EXPECT() *Searcher_Expecter {
|
||||
}
|
||||
|
||||
// IndexSpace provides a mock function for the type Searcher
|
||||
func (_mock *Searcher) IndexSpace(rID *providerv1beta1.StorageSpaceId, forceRescan bool) error {
|
||||
ret := _mock.Called(rID, forceRescan)
|
||||
func (_mock *Searcher) IndexSpace(spaceID *providerv1beta1.StorageSpaceId, forceRescan bool) error {
|
||||
ret := _mock.Called(spaceID, forceRescan)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for IndexSpace")
|
||||
@@ -49,7 +49,7 @@ func (_mock *Searcher) IndexSpace(rID *providerv1beta1.StorageSpaceId, forceResc
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(*providerv1beta1.StorageSpaceId, bool) error); ok {
|
||||
r0 = returnFunc(rID, forceRescan)
|
||||
r0 = returnFunc(spaceID, forceRescan)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
@@ -62,13 +62,13 @@ type Searcher_IndexSpace_Call struct {
|
||||
}
|
||||
|
||||
// IndexSpace is a helper method to define mock.On call
|
||||
// - rID *providerv1beta1.StorageSpaceId
|
||||
// - spaceID *providerv1beta1.StorageSpaceId
|
||||
// - forceRescan bool
|
||||
func (_e *Searcher_Expecter) IndexSpace(rID interface{}, forceRescan interface{}) *Searcher_IndexSpace_Call {
|
||||
return &Searcher_IndexSpace_Call{Call: _e.mock.On("IndexSpace", rID, forceRescan)}
|
||||
func (_e *Searcher_Expecter) IndexSpace(spaceID interface{}, forceRescan interface{}) *Searcher_IndexSpace_Call {
|
||||
return &Searcher_IndexSpace_Call{Call: _e.mock.On("IndexSpace", spaceID, forceRescan)}
|
||||
}
|
||||
|
||||
func (_c *Searcher_IndexSpace_Call) Run(run func(rID *providerv1beta1.StorageSpaceId, forceRescan bool)) *Searcher_IndexSpace_Call {
|
||||
func (_c *Searcher_IndexSpace_Call) Run(run func(spaceID *providerv1beta1.StorageSpaceId, forceRescan bool)) *Searcher_IndexSpace_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 *providerv1beta1.StorageSpaceId
|
||||
if args[0] != nil {
|
||||
@@ -91,7 +91,7 @@ func (_c *Searcher_IndexSpace_Call) Return(err error) *Searcher_IndexSpace_Call
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Searcher_IndexSpace_Call) RunAndReturn(run func(rID *providerv1beta1.StorageSpaceId, forceRescan bool) error) *Searcher_IndexSpace_Call {
|
||||
func (_c *Searcher_IndexSpace_Call) RunAndReturn(run func(spaceID *providerv1beta1.StorageSpaceId, forceRescan bool) error) *Searcher_IndexSpace_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
@@ -188,8 +188,8 @@ func (_c *Searcher_PurgeDeleted_Call) RunAndReturn(run func(spaceID *providerv1b
|
||||
}
|
||||
|
||||
// PurgeItem provides a mock function for the type Searcher
|
||||
func (_mock *Searcher) PurgeItem(rID *providerv1beta1.Reference) {
|
||||
_mock.Called(rID)
|
||||
func (_mock *Searcher) PurgeItem(ref *providerv1beta1.Reference) {
|
||||
_mock.Called(ref)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -199,12 +199,12 @@ type Searcher_PurgeItem_Call struct {
|
||||
}
|
||||
|
||||
// PurgeItem is a helper method to define mock.On call
|
||||
// - rID *providerv1beta1.Reference
|
||||
func (_e *Searcher_Expecter) PurgeItem(rID interface{}) *Searcher_PurgeItem_Call {
|
||||
return &Searcher_PurgeItem_Call{Call: _e.mock.On("PurgeItem", rID)}
|
||||
// - ref *providerv1beta1.Reference
|
||||
func (_e *Searcher_Expecter) PurgeItem(ref interface{}) *Searcher_PurgeItem_Call {
|
||||
return &Searcher_PurgeItem_Call{Call: _e.mock.On("PurgeItem", ref)}
|
||||
}
|
||||
|
||||
func (_c *Searcher_PurgeItem_Call) Run(run func(rID *providerv1beta1.Reference)) *Searcher_PurgeItem_Call {
|
||||
func (_c *Searcher_PurgeItem_Call) Run(run func(ref *providerv1beta1.Reference)) *Searcher_PurgeItem_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 *providerv1beta1.Reference
|
||||
if args[0] != nil {
|
||||
@@ -222,11 +222,62 @@ func (_c *Searcher_PurgeItem_Call) Return() *Searcher_PurgeItem_Call {
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Searcher_PurgeItem_Call) RunAndReturn(run func(rID *providerv1beta1.Reference)) *Searcher_PurgeItem_Call {
|
||||
func (_c *Searcher_PurgeItem_Call) RunAndReturn(run func(ref *providerv1beta1.Reference)) *Searcher_PurgeItem_Call {
|
||||
_c.Run(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// PurgeSpace provides a mock function for the type Searcher
|
||||
func (_mock *Searcher) PurgeSpace(spaceID *providerv1beta1.StorageSpaceId) error {
|
||||
ret := _mock.Called(spaceID)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for PurgeSpace")
|
||||
}
|
||||
|
||||
var r0 error
|
||||
if returnFunc, ok := ret.Get(0).(func(*providerv1beta1.StorageSpaceId) error); ok {
|
||||
r0 = returnFunc(spaceID)
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
return r0
|
||||
}
|
||||
|
||||
// Searcher_PurgeSpace_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PurgeSpace'
|
||||
type Searcher_PurgeSpace_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// PurgeSpace is a helper method to define mock.On call
|
||||
// - spaceID *providerv1beta1.StorageSpaceId
|
||||
func (_e *Searcher_Expecter) PurgeSpace(spaceID interface{}) *Searcher_PurgeSpace_Call {
|
||||
return &Searcher_PurgeSpace_Call{Call: _e.mock.On("PurgeSpace", spaceID)}
|
||||
}
|
||||
|
||||
func (_c *Searcher_PurgeSpace_Call) Run(run func(spaceID *providerv1beta1.StorageSpaceId)) *Searcher_PurgeSpace_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 *providerv1beta1.StorageSpaceId
|
||||
if args[0] != nil {
|
||||
arg0 = args[0].(*providerv1beta1.StorageSpaceId)
|
||||
}
|
||||
run(
|
||||
arg0,
|
||||
)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Searcher_PurgeSpace_Call) Return(err error) *Searcher_PurgeSpace_Call {
|
||||
_c.Call.Return(err)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Searcher_PurgeSpace_Call) RunAndReturn(run func(spaceID *providerv1beta1.StorageSpaceId) error) *Searcher_PurgeSpace_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// RestoreItem provides a mock function for the type Searcher
|
||||
func (_mock *Searcher) RestoreItem(ref *providerv1beta1.Reference) {
|
||||
_mock.Called(ref)
|
||||
@@ -336,8 +387,8 @@ func (_c *Searcher_Search_Call) RunAndReturn(run func(ctx context.Context, req *
|
||||
}
|
||||
|
||||
// TrashItem provides a mock function for the type Searcher
|
||||
func (_mock *Searcher) TrashItem(rID *providerv1beta1.ResourceId) {
|
||||
_mock.Called(rID)
|
||||
func (_mock *Searcher) TrashItem(resourceID *providerv1beta1.ResourceId) {
|
||||
_mock.Called(resourceID)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -347,12 +398,12 @@ type Searcher_TrashItem_Call struct {
|
||||
}
|
||||
|
||||
// TrashItem is a helper method to define mock.On call
|
||||
// - rID *providerv1beta1.ResourceId
|
||||
func (_e *Searcher_Expecter) TrashItem(rID interface{}) *Searcher_TrashItem_Call {
|
||||
return &Searcher_TrashItem_Call{Call: _e.mock.On("TrashItem", rID)}
|
||||
// - resourceID *providerv1beta1.ResourceId
|
||||
func (_e *Searcher_Expecter) TrashItem(resourceID interface{}) *Searcher_TrashItem_Call {
|
||||
return &Searcher_TrashItem_Call{Call: _e.mock.On("TrashItem", resourceID)}
|
||||
}
|
||||
|
||||
func (_c *Searcher_TrashItem_Call) Run(run func(rID *providerv1beta1.ResourceId)) *Searcher_TrashItem_Call {
|
||||
func (_c *Searcher_TrashItem_Call) Run(run func(resourceID *providerv1beta1.ResourceId)) *Searcher_TrashItem_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
var arg0 *providerv1beta1.ResourceId
|
||||
if args[0] != nil {
|
||||
@@ -370,7 +421,7 @@ func (_c *Searcher_TrashItem_Call) Return() *Searcher_TrashItem_Call {
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Searcher_TrashItem_Call) RunAndReturn(run func(rID *providerv1beta1.ResourceId)) *Searcher_TrashItem_Call {
|
||||
func (_c *Searcher_TrashItem_Call) RunAndReturn(run func(resourceID *providerv1beta1.ResourceId)) *Searcher_TrashItem_Call {
|
||||
_c.Run(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
@@ -29,17 +29,18 @@ type Engine interface {
|
||||
DocCount() (uint64, error)
|
||||
|
||||
Upsert(id string, r Resource) error
|
||||
Move(id string, parentid string, target string) error
|
||||
Move(id string, parentID string, targetPath string) error
|
||||
Delete(id string) error
|
||||
Restore(id string) error
|
||||
Purge(id string, onlyDeleted bool) error
|
||||
PurgeSpace(rootID string) error
|
||||
|
||||
NewBatch(batchSize int) (BatchOperator, error)
|
||||
}
|
||||
|
||||
type BatchOperator interface {
|
||||
Upsert(id string, r Resource) error
|
||||
Move(rootID, parentID, location string) error
|
||||
Move(id string, parentID string, targetPath string) error
|
||||
Delete(id string) error
|
||||
Restore(id string) error
|
||||
Purge(id string, onlyDeleted bool) error
|
||||
|
||||
@@ -46,11 +46,12 @@ const (
|
||||
type Searcher interface {
|
||||
Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error)
|
||||
|
||||
IndexSpace(rID *provider.StorageSpaceId, forceRescan bool) error
|
||||
IndexSpace(spaceID *provider.StorageSpaceId, forceRescan bool) error
|
||||
PurgeSpace(spaceID *provider.StorageSpaceId) error
|
||||
PurgeDeleted(spaceID *provider.StorageSpaceId) error
|
||||
|
||||
TrashItem(rID *provider.ResourceId)
|
||||
PurgeItem(rID *provider.Reference)
|
||||
TrashItem(resourceID *provider.ResourceId)
|
||||
PurgeItem(ref *provider.Reference)
|
||||
UpsertItem(ref *provider.Reference)
|
||||
RestoreItem(ref *provider.Reference)
|
||||
MoveItem(ref *provider.Reference)
|
||||
@@ -533,9 +534,9 @@ func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId, forceRescan bool)
|
||||
}
|
||||
|
||||
// TrashItem marks the item as deleted.
|
||||
func (s *Service) TrashItem(rID *provider.ResourceId) {
|
||||
if err := s.engine.Delete(storagespace.FormatResourceID(rID)); err != nil {
|
||||
s.logger.Info().Err(err).Interface("Id", rID).Msg("failed to remove item from index")
|
||||
func (s *Service) TrashItem(resourceID *provider.ResourceId) {
|
||||
if err := s.engine.Delete(storagespace.FormatResourceID(resourceID)); err != nil {
|
||||
s.logger.Info().Err(err).Interface("Id", resourceID).Msg("failed to remove item from index")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -554,6 +555,31 @@ func (s *Service) PurgeItem(ref *provider.Reference) {
|
||||
logDocCount(s.engine, s.logger)
|
||||
}
|
||||
|
||||
func (s *Service) PurgeSpace(spaceID *provider.StorageSpaceId) error {
|
||||
if spaceID == nil {
|
||||
return fmt.Errorf("spaceID must not be nil")
|
||||
}
|
||||
|
||||
rootID, err := storagespace.ParseID(spaceID.GetOpaqueId())
|
||||
if err != nil {
|
||||
s.logger.Error().Err(err).Str("space_id", spaceID.GetOpaqueId()).Msg("invalid space id")
|
||||
return err
|
||||
}
|
||||
if rootID.StorageId == "" || rootID.SpaceId == "" {
|
||||
return fmt.Errorf("invalid space id %s", spaceID.GetOpaqueId())
|
||||
}
|
||||
rootID.OpaqueId = rootID.SpaceId
|
||||
|
||||
if err := s.engine.PurgeSpace(storagespace.FormatResourceID(&rootID)); err != nil {
|
||||
s.logger.Error().Err(err).Str("space_id", spaceID.GetOpaqueId()).Msg("failed to purge the space from the index")
|
||||
return err
|
||||
}
|
||||
|
||||
logDocCount(s.engine, s.logger)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) PurgeDeleted(spaceID *provider.StorageSpaceId) error {
|
||||
if spaceID == nil {
|
||||
return fmt.Errorf("spaceID must not be nil")
|
||||
@@ -592,6 +618,11 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) {
|
||||
return
|
||||
}
|
||||
|
||||
if utils.IsProcessing(stat.Info) {
|
||||
s.logger.Debug().Str("path", path).Msg("resource is still being processed. Skipping.")
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := s.extractor.Extract(ctx, stat.Info)
|
||||
if err != nil {
|
||||
s.logger.Error().Err(err).Msg("failed to extract resource content")
|
||||
|
||||
@@ -2,19 +2,21 @@ package event
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/metrics"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/raw"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/metrics"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
@@ -61,6 +63,7 @@ func New(ctx context.Context, stream raw.Stream, logger log.Logger, tp trace.Tra
|
||||
events.TagsAdded{},
|
||||
events.TagsRemoved{},
|
||||
events.SpaceRenamed{},
|
||||
events.SpaceDeleted{},
|
||||
events.LabelAdded{},
|
||||
events.LabelRemoved{},
|
||||
},
|
||||
@@ -163,49 +166,64 @@ func (s Service) processEvent(e raw.Event) error {
|
||||
_, span := tracer.Start(ctx, "processEvent")
|
||||
defer span.End()
|
||||
|
||||
e.InProgress() // let nats know that we are processing this event
|
||||
if err := e.InProgress(); err != nil {
|
||||
s.log.Debug().Err(err).Interface("event", e).Msg("failed to set event progress")
|
||||
}
|
||||
|
||||
s.log.Debug().Interface("event", e).Msg("updating index")
|
||||
|
||||
ack := e.Ack
|
||||
|
||||
debounce := func(id *provider.StorageSpaceId) {
|
||||
ack = func() error { return nil }
|
||||
s.indexSpaceDebouncer.Debounce(id, e.Ack)
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
switch ev := e.Event.Event.(type) {
|
||||
case events.ItemTrashed:
|
||||
s.index.TrashItem(ev.ID)
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.ItemPurged:
|
||||
s.index.PurgeItem(ev.Ref)
|
||||
e.Ack()
|
||||
case events.TrashbinPurged:
|
||||
s.index.PurgeDeleted(getSpaceID(ev.Ref))
|
||||
e.Ack()
|
||||
err = s.index.PurgeDeleted(getSpaceID(ev.Ref))
|
||||
case events.ItemMoved:
|
||||
s.index.MoveItem(ev.Ref)
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.ItemRestored:
|
||||
s.index.RestoreItem(ev.Ref)
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.ContainerCreated:
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.FileTouched:
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.FileVersionRestored:
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.TagsAdded:
|
||||
s.index.UpsertItem(ev.Ref)
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.TagsRemoved:
|
||||
s.index.UpsertItem(ev.Ref)
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.FileUploaded:
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), e.Ack)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.UploadReady:
|
||||
s.indexSpaceDebouncer.Debounce(getSpaceID(ev.FileRef), e.Ack)
|
||||
debounce(getSpaceID(ev.FileRef))
|
||||
case events.SpaceRenamed:
|
||||
s.indexSpaceDebouncer.Debounce(ev.ID, e.Ack)
|
||||
debounce(ev.ID)
|
||||
case events.SpaceDeleted:
|
||||
err = s.index.PurgeSpace(ev.ID)
|
||||
case events.LabelAdded:
|
||||
s.index.UpsertItem(ev.Ref)
|
||||
case events.LabelRemoved:
|
||||
s.index.UpsertItem(ev.Ref)
|
||||
default:
|
||||
s.log.Error().Interface("event", e).Msg("unhandled event type, acknowledged without indexing")
|
||||
}
|
||||
return nil
|
||||
|
||||
return errors.Join(err, ack())
|
||||
}
|
||||
|
||||
func monitorMetrics(ctx context.Context, stream raw.Stream, name string, m *metrics.Metrics, logger log.Logger) {
|
||||
|
||||
@@ -50,6 +50,11 @@ var _ = DescribeTable("event",
|
||||
return int(calls.Load())
|
||||
}, "2s").Should(Equal(len(mcks)))
|
||||
},
|
||||
Entry("SpaceDeleted", []string{"PurgeSpace"}, events.SpaceDeleted{}, false),
|
||||
Entry("ItemPurged", []string{"PurgeItem"}, events.ItemPurged{}, false),
|
||||
Entry("TrashbinPurged", []string{"PurgeDeleted"}, events.TrashbinPurged{}, false),
|
||||
Entry("LabelAdded", []string{"UpsertItem"}, events.LabelAdded{}, false),
|
||||
Entry("LabelRemoved", []string{"UpsertItem"}, events.LabelRemoved{}, false),
|
||||
Entry("ItemTrashed", []string{"TrashItem", "IndexSpace"}, events.ItemTrashed{}, false),
|
||||
Entry("ItemMoved", []string{"MoveItem", "IndexSpace"}, events.ItemMoved{}, false),
|
||||
Entry("ItemRestored", []string{"RestoreItem", "IndexSpace"}, events.ItemRestored{}, false),
|
||||
|
||||
@@ -27,17 +27,19 @@ import (
|
||||
|
||||
// Session contains the information of an upload session
|
||||
type Session struct {
|
||||
ID string `json:"id"`
|
||||
Space string `json:"space"`
|
||||
Filename string `json:"filename"`
|
||||
Offset int64 `json:"offset"`
|
||||
Size int64 `json:"size"`
|
||||
Executant userpb.UserId `json:"executant"`
|
||||
SpaceOwner *userpb.UserId `json:"spaceowner,omitempty"`
|
||||
Expires time.Time `json:"expires"`
|
||||
Processing bool `json:"processing"`
|
||||
ScanDate time.Time `json:"virus_scan_date"`
|
||||
ScanResult string `json:"virus_scan_result"`
|
||||
ID string `json:"id"`
|
||||
Space string `json:"space"`
|
||||
Filename string `json:"filename"`
|
||||
Offset int64 `json:"offset"`
|
||||
Size int64 `json:"size"`
|
||||
Executant userpb.UserId `json:"executant"`
|
||||
SpaceOwner *userpb.UserId `json:"spaceowner,omitempty"`
|
||||
Expires time.Time `json:"expires"`
|
||||
Processing bool `json:"processing"`
|
||||
ScanDate time.Time `json:"virus_scan_date"`
|
||||
ScanResult string `json:"virus_scan_result"`
|
||||
Status string `json:"status"`
|
||||
StatusMessage string `json:"status_message"`
|
||||
}
|
||||
|
||||
// Uploads is the entry point for the uploads command
|
||||
@@ -123,7 +125,7 @@ func ListUploadSessions(cfg *config.Config) *cobra.Command {
|
||||
fmt.Println(buildInfo(filter))
|
||||
|
||||
table = tablewriter.NewTable(os.Stdout, tablewriter.WithHeaderAutoFormat(tw.Off))
|
||||
table.Header([]string{"Space", "Upload Id", "Name", "Offset", "Size", "Executant", "Owner", "Expires", "Processing", "Scan Date", "Scan Result"})
|
||||
table.Header([]string{"Space", "Upload Id", "Name", "Status", "Message", "Offset", "Size", "Executant", "Owner", "Expires", "Scan Date", "Scan Result"})
|
||||
}
|
||||
|
||||
for _, u := range uploads {
|
||||
@@ -131,17 +133,18 @@ func ListUploadSessions(cfg *config.Config) *cobra.Command {
|
||||
sr, sd := u.ScanData()
|
||||
|
||||
session := Session{
|
||||
Space: ref.GetResourceId().GetSpaceId(),
|
||||
ID: u.ID(),
|
||||
Filename: u.Filename(),
|
||||
Offset: u.Offset(),
|
||||
Size: u.Size(),
|
||||
Executant: u.Executant(),
|
||||
SpaceOwner: u.SpaceOwner(),
|
||||
Expires: u.Expires(),
|
||||
Processing: u.IsProcessing(),
|
||||
ScanDate: sd,
|
||||
ScanResult: sr,
|
||||
Space: ref.GetResourceId().GetSpaceId(),
|
||||
ID: u.ID(),
|
||||
Filename: u.Filename(),
|
||||
Offset: u.Offset(),
|
||||
Size: u.Size(),
|
||||
Executant: u.Executant(),
|
||||
SpaceOwner: u.SpaceOwner(),
|
||||
Expires: u.Expires(),
|
||||
ScanDate: sd,
|
||||
ScanResult: sr,
|
||||
Status: string(u.Status()),
|
||||
StatusMessage: u.StatusMessage(),
|
||||
}
|
||||
|
||||
if renderJson {
|
||||
@@ -151,12 +154,13 @@ func ListUploadSessions(cfg *config.Config) *cobra.Command {
|
||||
session.Space,
|
||||
session.ID,
|
||||
session.Filename,
|
||||
session.Status,
|
||||
session.StatusMessage,
|
||||
strconv.FormatInt(session.Offset, 10),
|
||||
strconv.FormatInt(session.Size, 10),
|
||||
session.Executant.OpaqueId,
|
||||
session.SpaceOwner.GetOpaqueId(),
|
||||
session.Expires.Format(time.RFC3339),
|
||||
strconv.FormatBool(session.Processing),
|
||||
session.ScanDate.Format(time.RFC3339),
|
||||
session.ScanResult,
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
SHELL := bash
|
||||
NAME := web
|
||||
WEB_ASSETS_VERSION = v7.1.2
|
||||
WEB_ASSETS_VERSION = v7.1.5
|
||||
WEB_ASSETS_BRANCH = stable-7.1
|
||||
|
||||
ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI
|
||||
|
||||
@@ -114,7 +114,7 @@ Feature: search for favorites
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result should contain "0" entries
|
||||
|
||||
|
||||
@issue-3309
|
||||
Scenario: search for favorite files after deleting a file
|
||||
Given user "Alice" has uploaded file "filesForUpload/textfile.txt" to "textfile.txt"
|
||||
And user "Alice" has marked file "textfile.txt" as favorite from space "Personal"
|
||||
|
||||
@@ -465,3 +465,39 @@ Feature: Search
|
||||
| old |
|
||||
| new |
|
||||
| spaces |
|
||||
|
||||
@issue-internal-326
|
||||
Scenario Outline: deleting a resource in one space does not remove identically-named paths from other spaces from the search index
|
||||
Given user "Alice" has created a folder "foo" in space "Personal"
|
||||
And user "Alice" has uploaded a file inside space "Personal" with content "alice content" to "foo/alice-file.txt"
|
||||
And user "Brian" has created a folder "foo" in space "Personal"
|
||||
And user "Brian" has uploaded a file inside space "Personal" with content "brian content" to "foo/brian-file.txt"
|
||||
And user "Brian" has removed the folder "foo" from space "Personal"
|
||||
And using <dav-path-version> DAV path
|
||||
When user "Alice" searches for "*file.txt*" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result should contain "1" entries
|
||||
And the search result of user "Alice" should contain these entries:
|
||||
| alice-file.txt |
|
||||
Examples:
|
||||
| dav-path-version |
|
||||
| old |
|
||||
| new |
|
||||
| spaces |
|
||||
|
||||
@issue-internal-326
|
||||
Scenario Outline: deleting a resource in one project space does not remove identically-named paths from other project spaces from the search index
|
||||
Given user "Alice" has created a space "project-1" with the default quota using the Graph API
|
||||
And user "Alice" has created a folder "folderMain/folder" in space "project-1"
|
||||
And user "Alice" has removed the folder "folderMain" from space "project-1"
|
||||
And using <dav-path-version> DAV path
|
||||
When user "Alice" searches for "*insideTheFolder*" using the WebDAV API
|
||||
Then the HTTP status code should be "207"
|
||||
And the search result should contain "1" entries
|
||||
And the search result of user "Alice" should contain these entries:
|
||||
| insideTheFolder.txt |
|
||||
Examples:
|
||||
| dav-path-version |
|
||||
| old |
|
||||
| new |
|
||||
| spaces |
|
||||
@@ -163,7 +163,7 @@ Feature: Share spaces
|
||||
| sharee | Brian |
|
||||
| shareType | user |
|
||||
| permissionsRole | <space-role> |
|
||||
When user "Alice" shares a space "share space" with settings:
|
||||
When user "Alice" updates the space "share space" with settings:
|
||||
| shareWith | Brian |
|
||||
| role | <share-space-role> |
|
||||
Then the HTTP status code should be "200"
|
||||
|
||||
+3
-3
@@ -27,15 +27,15 @@ func NewNatsKeyValue(c Config, log *zerolog.Logger) (jetstream.KeyValue, error)
|
||||
}
|
||||
opts := []nats.Option{
|
||||
nats.DisconnectErrHandler(func(nc *nats.Conn, err error) {
|
||||
log.Error().Err(err).Msg("Disconnected from NATS. Trying to reconnect...")
|
||||
log.Error().Err(err).Str("database", c.Database).Msg("Disconnected from NATS. Trying to reconnect...")
|
||||
}),
|
||||
nats.ReconnectHandler(func(nc *nats.Conn) {
|
||||
log.Error().Msgf("Successfully reconnected to NATS at: %s", nc.ConnectedUrl())
|
||||
log.Error().Str("database", c.Database).Msgf("Successfully reconnected to NATS at: %s", nc.ConnectedUrl())
|
||||
}),
|
||||
nats.ClosedHandler(func(nc *nats.Conn) {
|
||||
// Alright, it's time to give up. Send ourselves a SIGTERM to trigger a graceful
|
||||
// shutdown of the service.
|
||||
log.Error().Msg("NATS connection closed permanently. Shutting down...")
|
||||
log.Error().Str("database", c.Database).Msg("NATS connection closed permanently. Shutting down...")
|
||||
|
||||
pid := os.Getpid()
|
||||
process, err := os.FindProcess(pid)
|
||||
|
||||
Generated
Vendored
+64
-25
@@ -26,28 +26,34 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/pkg/xattr"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
)
|
||||
|
||||
const (
|
||||
TMPDir = ".oc-tmp"
|
||||
TMPDir = ".oc-tmp"
|
||||
fsyncEvery = 16 << 20 // 16 MiB
|
||||
)
|
||||
|
||||
// Blobstore provides an interface to an filesystem based blobstore
|
||||
type Blobstore struct {
|
||||
root string
|
||||
|
||||
canUseRenameForUpload bool
|
||||
}
|
||||
|
||||
// New returns a new Blobstore
|
||||
func New(root string) (*Blobstore, error) {
|
||||
return &Blobstore{
|
||||
root: root,
|
||||
root: root,
|
||||
canUseRenameForUpload: true, // let's assume the upload area is on the same device as the blobstore root by default
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -61,29 +67,40 @@ func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
sourceFile, err := os.Open(source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open source file '%s': %v", source, err)
|
||||
}
|
||||
defer func() {
|
||||
_ = sourceFile.Close()
|
||||
}()
|
||||
|
||||
tempFile, err := os.OpenFile(tempName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create temp file '%s': %v", tempName, err)
|
||||
if bs.canUseRenameForUpload {
|
||||
err := os.Rename(source, tempName)
|
||||
switch {
|
||||
case err == nil:
|
||||
// continue
|
||||
case errors.Is(err, syscall.EXDEV):
|
||||
// the upload and target file are on different devices, we need to copy the file instead of renaming it
|
||||
bs.canUseRenameForUpload = false
|
||||
default:
|
||||
return fmt.Errorf("failed to move source file '%s' to temp file '%s' - %v", source, tempName, err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := tempFile.ReadFrom(sourceFile); err != nil {
|
||||
return fmt.Errorf("failed to write data from source file '%s' to temp file '%s' - %v", source, tempName, err)
|
||||
}
|
||||
if !bs.canUseRenameForUpload {
|
||||
sourceFile, err := os.Open(source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open source file '%s': %v", source, err)
|
||||
}
|
||||
defer func() {
|
||||
_ = sourceFile.Close()
|
||||
}()
|
||||
|
||||
if err := tempFile.Sync(); err != nil {
|
||||
return fmt.Errorf("failed to sync temp file '%s' - %v", tempName, err)
|
||||
}
|
||||
tempFile, err := os.OpenFile(tempName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create temp file '%s': %v", tempName, err)
|
||||
}
|
||||
|
||||
if err := tempFile.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close temp file '%s' - %v", tempName, err)
|
||||
if err := copyWithPeriodicSync(tempFile, sourceFile); err != nil {
|
||||
return fmt.Errorf("failed to copy source file '%s' to temp file '%s' - %v", source, tempName, err)
|
||||
}
|
||||
|
||||
if err := tempFile.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close temp file '%s' - %v", tempName, err)
|
||||
}
|
||||
}
|
||||
|
||||
nodeAttributes, err := n.Xattrs(context.Background())
|
||||
@@ -137,11 +154,15 @@ func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error {
|
||||
}
|
||||
|
||||
// also "upload" the file to a local path, e.g., for keeping the "current" version of the file
|
||||
if err := os.MkdirAll(filepath.Dir(copyTarget), 0700); err != nil {
|
||||
return err
|
||||
sourceFile, err := os.Open(source)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "could not open source file '%s' for reading", source)
|
||||
}
|
||||
defer func() {
|
||||
_ = sourceFile.Close()
|
||||
}()
|
||||
|
||||
if _, err := sourceFile.Seek(0, 0); err != nil {
|
||||
if err := os.MkdirAll(filepath.Dir(copyTarget), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -153,7 +174,7 @@ func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error {
|
||||
_ = copyFile.Close()
|
||||
}()
|
||||
|
||||
if _, err := copyFile.ReadFrom(sourceFile); err != nil {
|
||||
if err := copyWithPeriodicSync(copyFile, sourceFile); err != nil {
|
||||
return errors.Wrapf(err, "could not write blob copy of '%s' to '%s'", n.InternalPath(), copyTarget)
|
||||
}
|
||||
|
||||
@@ -173,3 +194,21 @@ func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) {
|
||||
func (bs *Blobstore) Delete(node *node.Node) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyWithPeriodicSync(dst, src *os.File) error {
|
||||
for {
|
||||
n, err := io.CopyN(dst, src, fsyncEvery)
|
||||
if n > 0 {
|
||||
if serr := disk.Fdatasync(dst); serr != nil {
|
||||
return serr
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
_ = dst.Sync()
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
-42
@@ -41,7 +41,6 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"github.com/rs/zerolog"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
@@ -398,46 +397,14 @@ func refFromCS3(b []byte) (*provider.Reference, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CopyMetadata copies all extended attributes from source to target.
|
||||
// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix
|
||||
// For the source file, a shared lock is acquired.
|
||||
// NOTE: target resource will be write locked!
|
||||
func (lu *Lookup) CopyMetadata(ctx context.Context, src, target metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error) {
|
||||
// Acquire a read log on the source node
|
||||
// write lock existing node before reading treesize or tree time
|
||||
lock, err := lockedfile.OpenFile(lu.MetadataBackend().LockfilePath(src), os.O_RDONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "xattrs: Unable to lock source to read")
|
||||
}
|
||||
defer func() {
|
||||
rerr := lock.Close()
|
||||
|
||||
// if err is non nil we do not overwrite that
|
||||
if err == nil {
|
||||
err = rerr
|
||||
}
|
||||
}()
|
||||
|
||||
return lu.CopyMetadataWithSourceLock(ctx, src, target, filter, lock, acquireTargetLock)
|
||||
}
|
||||
|
||||
// CopyMetadataWithSourceLock copies all extended attributes from source to target.
|
||||
// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix
|
||||
// For the source file, a matching lockedfile is required.
|
||||
// NOTE: target resource will be write locked!
|
||||
func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, src, target metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), lockedSource *lockedfile.File, acquireTargetLock bool) (err error) {
|
||||
switch {
|
||||
case lockedSource == nil:
|
||||
return errors.New("no lock provided")
|
||||
case lockedSource.Name() != lu.MetadataBackend().LockfilePath(src):
|
||||
return errors.New("lockpath does not match filepath")
|
||||
}
|
||||
|
||||
attrs, err := lu.metadataBackend.AllWithLockedSource(ctx, src, lockedSource)
|
||||
// CopyMetadata copies all extended attributes from source to target. The optional filter
|
||||
// function can be used to filter by attribute name, e.g. by checking a prefix.
|
||||
//
|
||||
// Locking is derived from the nodes: the source's metadata lock is acquired for the duration
|
||||
// of the read unless the caller already holds it (src.LockHeld()), and the target is write
|
||||
// locked while its attributes are written unless the caller already holds its lock
|
||||
func (lu *Lookup) CopyMetadata(ctx context.Context, src, target metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool)) (err error) {
|
||||
attrs, err := lu.metadataBackend.All(ctx, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -453,7 +420,7 @@ func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, src, target me
|
||||
newAttrs[attrName] = val
|
||||
}
|
||||
|
||||
return lu.MetadataBackend().SetMultiple(ctx, target, newAttrs, acquireTargetLock)
|
||||
return lu.MetadataBackend().SetMultiple(ctx, target, newAttrs)
|
||||
}
|
||||
|
||||
// GenerateSpaceID generates a space id for the given space type and owner
|
||||
|
||||
Generated
Vendored
+2
-2
@@ -68,7 +68,7 @@ func (m *Manager) TMTime(ctx context.Context, n *node.Node) (time.Time, error) {
|
||||
// If tmtime is nil, the tmtime attribute is removed.
|
||||
func (m *Manager) SetTMTime(ctx context.Context, n *node.Node, tmtime *time.Time) error {
|
||||
if tmtime == nil {
|
||||
return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr, true)
|
||||
return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr)
|
||||
}
|
||||
return n.SetXattrString(ctx, prefixes.TreeMTimeAttr, tmtime.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
@@ -113,7 +113,7 @@ func (m *Manager) DTime(ctx context.Context, n *node.Node) (tmTime time.Time, er
|
||||
// If t is nil, the dtime attribute is removed.
|
||||
func (m *Manager) SetDTime(ctx context.Context, n *node.Node, t *time.Time) (err error) {
|
||||
if t == nil {
|
||||
return n.RemoveXattr(ctx, prefixes.DTimeAttr, true)
|
||||
return n.RemoveXattr(ctx, prefixes.DTimeAttr)
|
||||
}
|
||||
return n.SetXattrString(ctx, prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
+15
-1
@@ -40,6 +40,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/internal/goroutinelock"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
@@ -66,6 +67,7 @@ type trashNode struct {
|
||||
spaceID string
|
||||
id string
|
||||
path string
|
||||
lock goroutinelock.Lock
|
||||
}
|
||||
|
||||
func (tn *trashNode) GetSpaceID() string {
|
||||
@@ -80,6 +82,18 @@ func (tn *trashNode) InternalPath() string {
|
||||
return tn.path
|
||||
}
|
||||
|
||||
func (tn *trashNode) LockHeld() bool {
|
||||
return tn.lock.Held()
|
||||
}
|
||||
|
||||
func (tn *trashNode) SetLockHeld(held bool) {
|
||||
if held {
|
||||
tn.lock.Hold()
|
||||
return
|
||||
}
|
||||
tn.lock.Release()
|
||||
}
|
||||
|
||||
const (
|
||||
trashHeader = `[Trash Info]`
|
||||
timeFormat = "2006-01-02T15:04:05"
|
||||
@@ -345,7 +359,7 @@ func (tb *Trashbin) RestoreRecycleItem(ctx context.Context, spaceID string, key,
|
||||
if err = tb.lu.MetadataBackend().SetMultiple(ctx, trashedNode, map[string][]byte{
|
||||
prefixes.NameAttr: []byte(filepath.Base(restorePath)),
|
||||
prefixes.ParentidAttr: []byte(parentID),
|
||||
}, true); err != nil {
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("posixfs: failed to update trashed node metadata: %w", err)
|
||||
}
|
||||
|
||||
|
||||
+36
-14
@@ -42,6 +42,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/internal/goroutinelock"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
@@ -72,20 +73,33 @@ type assimilationNode struct {
|
||||
path string
|
||||
nodeId string
|
||||
spaceID string
|
||||
lock goroutinelock.Lock
|
||||
}
|
||||
|
||||
func (d assimilationNode) GetID() string {
|
||||
func (d *assimilationNode) GetID() string {
|
||||
return d.nodeId
|
||||
}
|
||||
|
||||
func (d assimilationNode) GetSpaceID() string {
|
||||
func (d *assimilationNode) GetSpaceID() string {
|
||||
return d.spaceID
|
||||
}
|
||||
|
||||
func (d assimilationNode) InternalPath() string {
|
||||
func (d *assimilationNode) InternalPath() string {
|
||||
return d.path
|
||||
}
|
||||
|
||||
func (d *assimilationNode) LockHeld() bool {
|
||||
return d.lock.Held()
|
||||
}
|
||||
|
||||
func (d *assimilationNode) SetLockHeld(held bool) {
|
||||
if held {
|
||||
d.lock.Hold()
|
||||
return
|
||||
}
|
||||
d.lock.Release()
|
||||
}
|
||||
|
||||
// NewScanDebouncer returns a new SpaceDebouncer instance
|
||||
func NewScanDebouncer(d time.Duration, f func(item scanItem)) *ScanDebouncer {
|
||||
return &ScanDebouncer{
|
||||
@@ -673,7 +687,8 @@ func (t *Tree) assimilate(item scanItem) error {
|
||||
func (t *Tree) updateFile(path, id, spaceID string, fi fs.FileInfo) (fs.FileInfo, node.Attributes, error) {
|
||||
retries := 1
|
||||
parentID := ""
|
||||
bn := assimilationNode{spaceID: spaceID, nodeId: id, path: path}
|
||||
bn := &assimilationNode{spaceID: spaceID, nodeId: id, path: path}
|
||||
bn.SetLockHeld(true)
|
||||
assimilate:
|
||||
if id != spaceID {
|
||||
// read parent
|
||||
@@ -714,7 +729,7 @@ assimilate:
|
||||
}
|
||||
}
|
||||
|
||||
attrs, err := t.lookup.MetadataBackend().AllWithLockedSource(context.Background(), bn, nil)
|
||||
attrs, err := t.lookup.MetadataBackend().All(context.Background(), bn)
|
||||
if err != nil && !metadata.IsAttrUnset(err) {
|
||||
return nil, nil, errors.Wrap(err, "failed to get item attribs")
|
||||
}
|
||||
@@ -788,6 +803,8 @@ assimilate:
|
||||
go func() {
|
||||
// Copy the previous current version to a revision
|
||||
currentNode := node.NewBaseNode(n.SpaceID, n.ID+node.CurrentIDDelimiter, t.lookup)
|
||||
currentNode.SetLockHeld(true)
|
||||
|
||||
currentPath := currentNode.InternalPath()
|
||||
stat, err := os.Stat(currentPath)
|
||||
if err == nil {
|
||||
@@ -834,7 +851,7 @@ assimilate:
|
||||
attributeName == prefixes.TypeAttr ||
|
||||
attributeName == prefixes.BlobIDAttr ||
|
||||
attributeName == prefixes.BlobsizeAttr
|
||||
}, false)
|
||||
})
|
||||
if err != nil {
|
||||
t.log.Error().Err(err).Str("currentPath", currentPath).Str("path", path).Msg("failed to copy xattrs to 'current' file")
|
||||
return
|
||||
@@ -848,7 +865,7 @@ assimilate:
|
||||
}
|
||||
|
||||
t.log.Debug().Str("path", path).Interface("attributes", attributes).Msg("setting attributes")
|
||||
err = t.lookup.MetadataBackend().SetMultiple(context.Background(), bn, attributes, false)
|
||||
err = t.lookup.MetadataBackend().SetMultiple(context.Background(), bn, attributes)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "failed to set attributes")
|
||||
}
|
||||
@@ -856,7 +873,7 @@ assimilate:
|
||||
// clear the status attribute if it was set before, if there was any upload to this file in progress
|
||||
// it needs notice that this file was changes meanwhile.
|
||||
if _, ok := previousAttribs[prefixes.StatusPrefix]; ok {
|
||||
err = t.lookup.MetadataBackend().Remove(context.Background(), bn, prefixes.StatusPrefix, false)
|
||||
err = t.lookup.MetadataBackend().Remove(context.Background(), bn, prefixes.StatusPrefix)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "failed to clear status attribute")
|
||||
}
|
||||
@@ -897,14 +914,19 @@ func (t *Tree) WarmupIDCache(root string, assimilate, onlyDirty bool) error {
|
||||
}
|
||||
|
||||
// skip irrelevant files
|
||||
if !t.Ignorer.IsSpaceRoot(path) && t.Ignorer.IsIgnored(path) {
|
||||
if info.IsDir() {
|
||||
if t.Ignorer.IsIgnored(path) {
|
||||
switch {
|
||||
case t.Ignorer.IsSpaceRoot(path):
|
||||
// ignore the space root itself, but do not skip the whole tree
|
||||
return nil
|
||||
case t.Ignorer.IsRootPath(path):
|
||||
// ignor the root path itself, but do not skip the whole tree
|
||||
return nil
|
||||
case info.IsDir():
|
||||
return filepath.SkipDir
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if t.Ignorer.IsRootPath(path) {
|
||||
return nil // ignore the root paths
|
||||
}
|
||||
|
||||
if !info.IsDir() && !info.Mode().IsRegular() {
|
||||
|
||||
+6
-8
@@ -30,7 +30,6 @@ import (
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
@@ -46,7 +45,7 @@ import (
|
||||
// can be kept in the same location.
|
||||
|
||||
// CreateRevision creates a new version of the node
|
||||
func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string, f *lockedfile.File) (string, error) {
|
||||
func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string) (string, error) {
|
||||
revNode := node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+version, tp.lookup)
|
||||
versionPath := revNode.InternalPath()
|
||||
|
||||
@@ -119,13 +118,13 @@ func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string
|
||||
}
|
||||
|
||||
// copy blob metadata to version node
|
||||
if err := tp.lookup.CopyMetadataWithSourceLock(ctx, n, revNode, func(attributeName string, value []byte) (newValue []byte, copy bool) {
|
||||
if err := tp.lookup.CopyMetadata(ctx, n, revNode, func(attributeName string, value []byte) (newValue []byte, copy bool) {
|
||||
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
|
||||
attributeName == prefixes.TypeAttr ||
|
||||
attributeName == prefixes.BlobIDAttr ||
|
||||
attributeName == prefixes.BlobsizeAttr ||
|
||||
attributeName == prefixes.MTimeAttr
|
||||
}, f, true); err != nil {
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -300,7 +299,7 @@ func (tp *Tree) RestoreRevision(ctx context.Context, srcNode, targetNode metadat
|
||||
attributeName == prefixes.TypeAttr ||
|
||||
attributeName == prefixes.BlobIDAttr ||
|
||||
attributeName == prefixes.BlobsizeAttr
|
||||
}, false)
|
||||
})
|
||||
if err != nil {
|
||||
return errtypes.InternalError("failed to copy blob xattrs to old revision to node: " + err.Error())
|
||||
}
|
||||
@@ -317,8 +316,7 @@ func (tp *Tree) RestoreRevision(ctx context.Context, srcNode, targetNode metadat
|
||||
err = tp.lookup.MetadataBackend().SetMultiple(ctx, targetNode,
|
||||
map[string][]byte{
|
||||
prefixes.MTimeAttr: []byte(mtime.UTC().Format(time.RFC3339Nano)),
|
||||
},
|
||||
false)
|
||||
})
|
||||
if err != nil {
|
||||
return errtypes.InternalError("failed to set mtime attribute on node: " + err.Error())
|
||||
}
|
||||
@@ -350,7 +348,7 @@ func (tp *Tree) RestoreRevision(ctx context.Context, srcNode, targetNode metadat
|
||||
attributeName == prefixes.TypeAttr ||
|
||||
attributeName == prefixes.BlobIDAttr ||
|
||||
attributeName == prefixes.BlobsizeAttr
|
||||
}, false)
|
||||
})
|
||||
if err != nil {
|
||||
return errtypes.InternalError("failed to copy xattrs to 'current' file: " + err.Error())
|
||||
}
|
||||
|
||||
+5
-5
@@ -394,7 +394,7 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool,
|
||||
attributes[prefixes.MTimeAttr] = []byte(mtime.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
|
||||
err = n.SetXattrsWithContext(ctx, attributes, false)
|
||||
err = n.SetXattrsWithContext(ctx, attributes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -477,7 +477,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node)
|
||||
attribs := node.Attributes{}
|
||||
attribs.SetString(prefixes.ParentidAttr, newNode.ParentID)
|
||||
attribs.SetString(prefixes.NameAttr, newNode.Name)
|
||||
if err := newNode.SetXattrsWithContext(ctx, attribs, true); err != nil {
|
||||
if err := newNode.SetXattrsWithContext(ctx, attribs); err != nil {
|
||||
return errors.Wrap(err, "posixfs: could not update node attributes")
|
||||
}
|
||||
|
||||
@@ -726,7 +726,7 @@ func (t *Tree) WriteBlob(n *node.Node, source string) error {
|
||||
attrs.SetString(prefixes.BlobIDAttr, n.BlobID)
|
||||
attrs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize)
|
||||
|
||||
err := t.lookup.MetadataBackend().SetMultiple(context.Background(), node.NewBaseNode(n.SpaceID, n.ID+node.CurrentIDDelimiter, t.lookup), attrs, true)
|
||||
err := t.lookup.MetadataBackend().SetMultiple(context.Background(), node.NewBaseNode(n.SpaceID, n.ID+node.CurrentIDDelimiter, t.lookup), attrs)
|
||||
if err != nil {
|
||||
t.log.Error().Err(err).Str("spaceID", n.SpaceID).Str("id", n.ID).Msg("could not copy metadata to current revision")
|
||||
}
|
||||
@@ -796,7 +796,7 @@ func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (met
|
||||
mtime := fi.ModTime()
|
||||
err = n.SetXattrsWithContext(ctx, map[string][]byte{
|
||||
prefixes.MTimeAttr: []byte(mtime.UTC().Format(time.RFC3339Nano)),
|
||||
}, false)
|
||||
})
|
||||
if err != nil {
|
||||
t.log.Error().Err(err).Str("path", n.InternalPath()).Msg("could not set mtime attribute on new node")
|
||||
}
|
||||
@@ -868,5 +868,5 @@ func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) {
|
||||
if t.options.TreeTimeAccounting || t.options.TreeSizeAccounting {
|
||||
attributes[prefixes.PropagationAttr] = []byte("1") // mark the node for propagation
|
||||
}
|
||||
return n.SetXattrsWithContext(ctx, attributes, false)
|
||||
return n.SetXattrsWithContext(ctx, attributes)
|
||||
}
|
||||
Generated
Vendored
+62
@@ -0,0 +1,62 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package goroutinelock
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// Lock tracks which goroutine currently owns a metadata lock.
|
||||
//
|
||||
// Held() only returns true in the owning goroutine, which prevents lock-held
|
||||
// state from leaking across goroutine boundaries when helper nodes are shared.
|
||||
type Lock struct {
|
||||
gid atomic.Uint64
|
||||
}
|
||||
|
||||
var goidLogOnce sync.Once
|
||||
|
||||
func (o *Lock) Hold() {
|
||||
o.gid.Store(goid())
|
||||
}
|
||||
|
||||
func (o *Lock) Release() {
|
||||
o.gid.Store(0)
|
||||
}
|
||||
|
||||
func (o *Lock) Held() bool {
|
||||
return o.gid.Load() == goid()
|
||||
}
|
||||
|
||||
func goid() uint64 {
|
||||
var buf [64]byte
|
||||
n := runtime.Stack(buf[:], false)
|
||||
|
||||
// The header has the form "goroutine 1234 [running]:".
|
||||
fields := bytes.Fields(buf[:n])
|
||||
if len(fields) < 2 {
|
||||
goidLogOnce.Do(func() {
|
||||
log.Error().
|
||||
Str("stack_header", string(buf[:n])).
|
||||
Msg("goroutinelock: could not determine goroutine id from runtime stack header")
|
||||
})
|
||||
return 0
|
||||
}
|
||||
id, err := strconv.ParseUint(string(fields[1]), 10, 64)
|
||||
if err != nil {
|
||||
goidLogOnce.Do(func() {
|
||||
log.Error().Err(err).
|
||||
Str("goroutine_id_field", string(fields[1])).
|
||||
Msg("goroutinelock: could not parse goroutine id")
|
||||
})
|
||||
return 0
|
||||
}
|
||||
return id
|
||||
}
|
||||
Generated
Vendored
+25
@@ -82,6 +82,7 @@ var (
|
||||
events.PostprocessingFinished{},
|
||||
events.PostprocessingStepFinished{},
|
||||
events.RestartPostprocessing{},
|
||||
events.StartPostprocessingStep{},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -440,6 +441,22 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) {
|
||||
}); err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to publish BytesReceived event")
|
||||
}
|
||||
case events.StartPostprocessingStep:
|
||||
sublog := log.With().Str("event", "StartPostprocessingStep").Str("uploadid", ev.UploadID).Logger()
|
||||
if ev.UploadID == "" {
|
||||
sublog.Error().Msg("UploadID is empty, cannot start postprocessing step")
|
||||
continue
|
||||
}
|
||||
session, err := fs.sessionStore.Get(ctx, ev.UploadID)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to get upload")
|
||||
continue
|
||||
}
|
||||
session.SetStatus(upload.SessionStatusProcessing, "started postprocessing step: "+string(ev.StepToStart))
|
||||
err = session.Persist(ctx)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to persist upload session after starting postprocessing step")
|
||||
}
|
||||
case events.PostprocessingStepFinished:
|
||||
sublog := log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger()
|
||||
if ev.FinishedStep != events.PPStepAntivirus {
|
||||
@@ -551,6 +568,14 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) {
|
||||
sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger()
|
||||
|
||||
session.SetScanData(res.Description, res.Scandate)
|
||||
var status string
|
||||
switch ev.Outcome {
|
||||
case events.PPOutcomeAbort, events.PPOutcomeDelete:
|
||||
status = upload.SessionStatusFailed
|
||||
default:
|
||||
status = upload.SessionStatusProcessing
|
||||
}
|
||||
session.SetStatus(status, "finished postprocessing step: "+string(ev.FinishedStep))
|
||||
if err := session.Persist(ctx); err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to persist scan results")
|
||||
}
|
||||
|
||||
Generated
Vendored
+12
@@ -0,0 +1,12 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//go:build !linux && !freebsd && !darwin
|
||||
|
||||
package disk
|
||||
|
||||
import "os"
|
||||
|
||||
func Fdatasync(f *os.File) error {
|
||||
return nil
|
||||
}
|
||||
Generated
Vendored
+15
@@ -0,0 +1,15 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//go:build linux
|
||||
|
||||
package disk
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func Fdatasync(f *os.File) error {
|
||||
return syscall.Fdatasync(int(f.Fd()))
|
||||
}
|
||||
Generated
Vendored
+12
@@ -0,0 +1,12 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//go:build freebsd || darwin
|
||||
|
||||
package disk
|
||||
|
||||
import "os"
|
||||
|
||||
func Fdatasync(file *os.File) error {
|
||||
return file.Sync()
|
||||
}
|
||||
+2
-2
@@ -223,7 +223,7 @@ func (fs *Decomposedfs) RemoveGrant(ctx context.Context, ref *provider.Reference
|
||||
}
|
||||
}
|
||||
|
||||
if err := grantNode.DeleteGrant(ctx, g, false); err != nil {
|
||||
if err := grantNode.DeleteGrant(ctx, g); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -349,7 +349,7 @@ func (fs *Decomposedfs) storeGrant(ctx context.Context, n *node.Node, g *provide
|
||||
attribs := node.Attributes{
|
||||
prefixes.GrantPrefix + principal: value,
|
||||
}
|
||||
if err := n.SetXattrsWithContext(ctx, attribs, false); err != nil {
|
||||
if err := n.SetXattrsWithContext(ctx, attribs); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).
|
||||
Str("principal", principal).Msg("Could not set grant for principal")
|
||||
return err
|
||||
|
||||
Generated
Vendored
+8
-41
@@ -36,7 +36,6 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
@@ -314,45 +313,13 @@ func refFromCS3(b []byte) (*provider.Reference, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CopyMetadata copies all extended attributes from source to target.
|
||||
// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix
|
||||
// For the source file, a shared lock is acquired.
|
||||
// NOTE: target resource will be write locked!
|
||||
func (lu *Lookup) CopyMetadata(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error) {
|
||||
// Acquire a read log on the source node
|
||||
// write lock existing node before reading treesize or tree time
|
||||
lock, err := lockedfile.OpenFile(lu.MetadataBackend().LockfilePath(sourceNode), os.O_RDONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "xattrs: Unable to lock source to read")
|
||||
}
|
||||
defer func() {
|
||||
rerr := lock.Close()
|
||||
|
||||
// if err is non nil we do not overwrite that
|
||||
if err == nil {
|
||||
err = rerr
|
||||
}
|
||||
}()
|
||||
|
||||
return lu.CopyMetadataWithSourceLock(ctx, sourceNode, targetNode, filter, lock, acquireTargetLock)
|
||||
}
|
||||
|
||||
// CopyMetadataWithSourceLock copies all extended attributes from source to target.
|
||||
// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix
|
||||
// For the source file, a matching lockedfile is required.
|
||||
// NOTE: target resource will be write locked!
|
||||
func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), lockedSource *lockedfile.File, acquireTargetLock bool) (err error) {
|
||||
switch {
|
||||
case lockedSource == nil:
|
||||
return errors.New("no lock provided")
|
||||
case lockedSource.Name() != lu.MetadataBackend().LockfilePath(sourceNode):
|
||||
return errors.New("lockpath does not match filepath")
|
||||
}
|
||||
|
||||
// CopyMetadata copies all extended attributes from source to target. The optional filter
|
||||
// function can be used to filter by attribute name, e.g. by checking a prefix.
|
||||
//
|
||||
// Locking is derived from the nodes: the source's metadata lock is acquired for the duration
|
||||
// of the read unless the caller already holds it (sourceNode.LockHeld()), and the target is
|
||||
// write locked while its attributes are written unless the caller already holds its lock
|
||||
func (lu *Lookup) CopyMetadata(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool)) (err error) {
|
||||
attrs, err := lu.metadataBackend.All(ctx, sourceNode)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -369,7 +336,7 @@ func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, sourceNode, ta
|
||||
newAttrs[attrName] = val
|
||||
}
|
||||
|
||||
return lu.MetadataBackend().SetMultiple(ctx, targetNode, newAttrs, acquireTargetLock)
|
||||
return lu.MetadataBackend().SetMultiple(ctx, targetNode, newAttrs)
|
||||
}
|
||||
|
||||
func (lu *Lookup) PurgeNode(n *node.Node) error {
|
||||
|
||||
+1
-1
@@ -142,7 +142,7 @@ func (fs *Decomposedfs) UnsetArbitraryMetadata(ctx context.Context, ref *provide
|
||||
|
||||
errs := []error{}
|
||||
for _, k := range keys {
|
||||
if err = n.RemoveXattr(ctx, prefixes.MetadataPrefix+k, true); err != nil {
|
||||
if err = n.RemoveXattr(ctx, prefixes.MetadataPrefix+k); err != nil {
|
||||
if metadata.IsAttrUnset(err) {
|
||||
continue // already gone, ignore
|
||||
}
|
||||
|
||||
Generated
Vendored
+29
-40
@@ -3,7 +3,6 @@ package metadata
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -117,8 +116,8 @@ func (b HybridBackend) list(ctx context.Context, n MetadataNode) (attribs []stri
|
||||
return xattr.List(filePath)
|
||||
}
|
||||
|
||||
// All reads all extended attributes for a node, protected by a
|
||||
// shared file lock
|
||||
// All reads all extended attributes for a node. It acquires a shared file lock
|
||||
// unless the caller already holds the node's metadata lock (n.LockHeld()).
|
||||
func (b HybridBackend) All(ctx context.Context, n MetadataNode) (map[string][]byte, error) {
|
||||
attribs := map[string][]byte{}
|
||||
|
||||
@@ -127,29 +126,18 @@ func (b HybridBackend) All(ctx context.Context, n MetadataNode) (map[string][]by
|
||||
return attribs, err
|
||||
}
|
||||
|
||||
unlock, err := b.Lock(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = unlock() }()
|
||||
|
||||
return b.getAll(ctx, n, false, false)
|
||||
}
|
||||
|
||||
// AllWithLockedSource reads all extended attributes from the given reader.
|
||||
// The path argument is used for storing the data in the cache
|
||||
func (b HybridBackend) AllWithLockedSource(ctx context.Context, n MetadataNode, _ io.Reader) (map[string][]byte, error) {
|
||||
attribs := map[string][]byte{}
|
||||
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs)
|
||||
if err == nil {
|
||||
return attribs, err
|
||||
if !n.LockHeld() {
|
||||
unlock, err := b.Lock(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = unlock() }()
|
||||
}
|
||||
|
||||
return b.getAll(ctx, n, false, false)
|
||||
return b.getAll(ctx, n, false)
|
||||
}
|
||||
|
||||
func (b HybridBackend) getAll(ctx context.Context, n MetadataNode, skipCache, skipOffloaded bool) (map[string][]byte, error) {
|
||||
func (b HybridBackend) getAll(ctx context.Context, n MetadataNode, skipOffloaded bool) (map[string][]byte, error) {
|
||||
attribs := map[string][]byte{}
|
||||
attrNames, err := b.list(ctx, n)
|
||||
if err != nil {
|
||||
@@ -215,13 +203,13 @@ func (b HybridBackend) getAll(ctx context.Context, n MetadataNode, skipCache, sk
|
||||
|
||||
// Set sets one attribute for the given path
|
||||
func (b HybridBackend) Set(ctx context.Context, n MetadataNode, key string, val []byte) (err error) {
|
||||
return b.SetMultiple(ctx, n, map[string][]byte{key: val}, true)
|
||||
return b.SetMultiple(ctx, n, map[string][]byte{key: val})
|
||||
}
|
||||
|
||||
// SetMultiple sets a set of attribute for the given path
|
||||
func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte, acquireLock bool) (err error) {
|
||||
func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte) (err error) {
|
||||
path := n.InternalPath()
|
||||
if acquireLock {
|
||||
if !n.LockHeld() {
|
||||
unlock, err := b.Lock(n)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -247,7 +235,7 @@ func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
|
||||
mdSize += len(attribs[key]) + len(key)
|
||||
}
|
||||
}
|
||||
existingAttribs, err := b.getAll(ctx, n, true, true)
|
||||
existingAttribs, err := b.getAll(ctx, n, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -320,7 +308,7 @@ func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
|
||||
}
|
||||
|
||||
// Update the cache with the new values
|
||||
_, err = b.getAll(ctx, n, true, false)
|
||||
_, err = b.getAll(ctx, n, false)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -331,7 +319,7 @@ func (b HybridBackend) offloadMetadata(ctx context.Context, n MetadataNode) erro
|
||||
var xerr error
|
||||
|
||||
// collect attributes to move
|
||||
existingAttribs, err := b.getAll(ctx, n, true, true)
|
||||
existingAttribs, err := b.getAll(ctx, n, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -377,9 +365,9 @@ func (b HybridBackend) offloadMetadata(ctx context.Context, n MetadataNode) erro
|
||||
}
|
||||
|
||||
// Remove an extended attribute key
|
||||
func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string, acquireLock bool) error {
|
||||
func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string) error {
|
||||
path := n.InternalPath()
|
||||
if acquireLock {
|
||||
if !n.LockHeld() {
|
||||
unlock, err := b.Lock(n)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -444,7 +432,7 @@ func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string, a
|
||||
}
|
||||
|
||||
// Update the cache with the new values
|
||||
_, err := b.getAll(ctx, n, true, false)
|
||||
_, err := b.getAll(ctx, n, false)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -462,7 +450,7 @@ func (b HybridBackend) Purge(ctx context.Context, n MetadataNode) error {
|
||||
}
|
||||
defer func() { _ = unlock() }()
|
||||
|
||||
attribs, err := b.getAll(ctx, n, true, false)
|
||||
attribs, err := b.getAll(ctx, n, false)
|
||||
if err == nil {
|
||||
for attr := range attribs {
|
||||
if strings.HasPrefix(attr, prefixes.OcPrefix) {
|
||||
@@ -504,11 +492,10 @@ func (b HybridBackend) LockfilePath(n MetadataNode) string {
|
||||
|
||||
// Lock locks the metadata for the given path
|
||||
func (b HybridBackend) Lock(n MetadataNode) (UnlockFunc, error) {
|
||||
f, _, err := b.LockAndRead(n)
|
||||
return f, err
|
||||
}
|
||||
if n.LockHeld() {
|
||||
return func() error { return nil }, nil
|
||||
}
|
||||
|
||||
func (b HybridBackend) LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error) {
|
||||
metaLockPath := b.LockfilePath(n)
|
||||
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
@@ -516,20 +503,22 @@ func (b HybridBackend) LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error
|
||||
// create the parent directory
|
||||
err = os.MkdirAll(filepath.Dir(metaLockPath), 0700)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
mlock, err = lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
n.SetLockHeld(true)
|
||||
return func() error {
|
||||
n.SetLockHeld(false)
|
||||
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
|
||||
return mlock.Close()
|
||||
}, mlock, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b HybridBackend) cacheKey(n MetadataNode) string {
|
||||
|
||||
Generated
Vendored
+38
-60
@@ -86,12 +86,12 @@ func (b MessagePackBackend) IdentifyPath(_ context.Context, path string) (string
|
||||
|
||||
// All reads all extended attributes for a node
|
||||
func (b MessagePackBackend) All(ctx context.Context, n MetadataNode) (map[string][]byte, error) {
|
||||
return b.loadAttributes(ctx, n, nil)
|
||||
return b.loadAttributes(ctx, n)
|
||||
}
|
||||
|
||||
// Get an extended attribute value for the given key
|
||||
func (b MessagePackBackend) Get(ctx context.Context, n MetadataNode, key string) ([]byte, error) {
|
||||
attribs, err := b.loadAttributes(ctx, n, nil)
|
||||
attribs, err := b.loadAttributes(ctx, n)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
@@ -104,7 +104,7 @@ func (b MessagePackBackend) Get(ctx context.Context, n MetadataNode, key string)
|
||||
|
||||
// GetInt64 reads a string as int64 from the xattrs
|
||||
func (b MessagePackBackend) GetInt64(ctx context.Context, n MetadataNode, key string) (int64, error) {
|
||||
attribs, err := b.loadAttributes(ctx, n, nil)
|
||||
attribs, err := b.loadAttributes(ctx, n)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -121,26 +121,20 @@ func (b MessagePackBackend) GetInt64(ctx context.Context, n MetadataNode, key st
|
||||
|
||||
// Set sets one attribute for the given path
|
||||
func (b MessagePackBackend) Set(ctx context.Context, n MetadataNode, key string, val []byte) error {
|
||||
return b.SetMultiple(ctx, n, map[string][]byte{key: val}, true)
|
||||
return b.SetMultiple(ctx, n, map[string][]byte{key: val})
|
||||
}
|
||||
|
||||
// SetMultiple sets a set of attribute for the given path
|
||||
func (b MessagePackBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte, acquireLock bool) error {
|
||||
return b.saveAttributes(ctx, n, attribs, nil, acquireLock)
|
||||
func (b MessagePackBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte) error {
|
||||
return b.saveAttributes(ctx, n, attribs, nil)
|
||||
}
|
||||
|
||||
// Remove an extended attribute key
|
||||
func (b MessagePackBackend) Remove(ctx context.Context, n MetadataNode, key string, acquireLock bool) error {
|
||||
return b.saveAttributes(ctx, n, nil, []string{key}, acquireLock)
|
||||
func (b MessagePackBackend) Remove(ctx context.Context, n MetadataNode, key string) error {
|
||||
return b.saveAttributes(ctx, n, nil, []string{key})
|
||||
}
|
||||
|
||||
// AllWithLockedSource reads all extended attributes from the given reader (if possible).
|
||||
// The path argument is used for storing the data in the cache
|
||||
func (b MessagePackBackend) AllWithLockedSource(ctx context.Context, n MetadataNode, source io.Reader) (map[string][]byte, error) {
|
||||
return b.loadAttributes(ctx, n, source)
|
||||
}
|
||||
|
||||
func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode, setAttribs map[string][]byte, deleteAttribs []string, acquireLock bool) error {
|
||||
func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode, setAttribs map[string][]byte, deleteAttribs []string) error {
|
||||
var (
|
||||
err error
|
||||
)
|
||||
@@ -155,7 +149,7 @@ func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode,
|
||||
}()
|
||||
|
||||
metaPath := b.MetadataPath(n)
|
||||
if acquireLock {
|
||||
if !n.LockHeld() {
|
||||
unlock, err := b.Lock(n)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -211,7 +205,7 @@ func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode,
|
||||
return err
|
||||
}
|
||||
|
||||
func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode, source io.Reader) (map[string][]byte, error) {
|
||||
func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode) (map[string][]byte, error) {
|
||||
ctx, span := tracer.Start(ctx, "loadAttributes")
|
||||
defer span.End()
|
||||
attribs := map[string][]byte{}
|
||||
@@ -221,39 +215,31 @@ func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode,
|
||||
}
|
||||
|
||||
metaPath := b.MetadataPath(n)
|
||||
var msgBytes []byte
|
||||
|
||||
if source == nil {
|
||||
// // No cached entry found. Read from storage and store in cache
|
||||
_, subspan := tracer.Start(ctx, "os.OpenFile")
|
||||
// source, err = lockedfile.Open(metaPath)
|
||||
source, err = os.Open(metaPath)
|
||||
subspan.End()
|
||||
// // No cached entry found. Read from storage and store in cache
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// some of the caller rely on ENOTEXISTS to be returned when the
|
||||
// actual file (not the metafile) does not exist in order to
|
||||
// determine whether a node exists or not -> stat the actual node
|
||||
_, subspan := tracer.Start(ctx, "os.Stat")
|
||||
_, err := os.Stat(n.InternalPath())
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return attribs, nil // no attributes set yet
|
||||
// No cached entry found. Read from storage and store in cache
|
||||
_, subspan := tracer.Start(ctx, "os.Open")
|
||||
source, err := os.Open(metaPath)
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// some of the callers rely on ENOTEXISTS to be returned when the
|
||||
// actual file (not the metafile) does not exist in order to
|
||||
// determine whether a node exists or not -> stat the actual node
|
||||
_, subspan := tracer.Start(ctx, "os.Stat")
|
||||
_, err := os.Stat(n.InternalPath())
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return attribs, nil // no attributes set yet
|
||||
}
|
||||
_, subspan = tracer.Start(ctx, "io.ReadAll")
|
||||
msgBytes, err = io.ReadAll(source)
|
||||
source.(*os.File).Close()
|
||||
subspan.End()
|
||||
} else {
|
||||
_, subspan := tracer.Start(ctx, "io.ReadAll")
|
||||
msgBytes, err = io.ReadAll(source)
|
||||
subspan.End()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, subspan = tracer.Start(ctx, "io.ReadAll")
|
||||
msgBytes, err := io.ReadAll(source)
|
||||
_ = source.Close()
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -264,7 +250,7 @@ func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode,
|
||||
}
|
||||
}
|
||||
|
||||
_, subspan := tracer.Start(ctx, "metaCache.PushToCache")
|
||||
_, subspan = tracer.Start(ctx, "metaCache.PushToCache")
|
||||
err = b.metaCache.PushToCache(b.cacheKey(n), attribs)
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
@@ -314,31 +300,23 @@ func (MessagePackBackend) LockfilePath(n MetadataNode) string { return n.Interna
|
||||
|
||||
// Lock locks the metadata for the given path
|
||||
func (b MessagePackBackend) Lock(n MetadataNode) (UnlockFunc, error) {
|
||||
if n.LockHeld() {
|
||||
return func() error { return nil }, nil
|
||||
}
|
||||
|
||||
metaLockPath := b.LockfilePath(n)
|
||||
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n.SetLockHeld(true)
|
||||
return func() error {
|
||||
n.SetLockHeld(false)
|
||||
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
|
||||
return mlock.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b MessagePackBackend) LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error) {
|
||||
unlock, err := b.Lock(n)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
f, err := os.Open(b.MetadataPath(n))
|
||||
if err != nil {
|
||||
_ = unlock()
|
||||
return nil, nil, err
|
||||
}
|
||||
return unlock, f, nil
|
||||
}
|
||||
|
||||
func (b MessagePackBackend) cacheKey(n MetadataNode) string {
|
||||
return n.GetSpaceID() + "/" + n.GetID()
|
||||
}
|
||||
Generated
Vendored
+15
-18
@@ -21,7 +21,6 @@ package metadata
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
@@ -42,6 +41,11 @@ type MetadataNode interface {
|
||||
GetSpaceID() string
|
||||
GetID() string
|
||||
InternalPath() string
|
||||
// LockHeld reports whether the caller already holds this node's metadata lock.
|
||||
// Backends use it to decide whether they must (re-)acquire the lock when reading
|
||||
// metadata; re-acquiring a held, non-reentrant advisory file lock self-deadlocks.
|
||||
LockHeld() bool
|
||||
SetLockHeld(held bool)
|
||||
}
|
||||
|
||||
// Backend defines the interface for file attribute backends
|
||||
@@ -49,17 +53,21 @@ type Backend interface {
|
||||
Name() string
|
||||
IdentifyPath(ctx context.Context, path string) (string, string, string, time.Time, error)
|
||||
|
||||
// All reads all extended attributes for a node. It acquires the node's metadata
|
||||
// lock unless n.LockHeld() reports that the caller already holds it.
|
||||
All(ctx context.Context, n MetadataNode) (map[string][]byte, error)
|
||||
AllWithLockedSource(ctx context.Context, n MetadataNode, source io.Reader) (map[string][]byte, error)
|
||||
|
||||
Get(ctx context.Context, n MetadataNode, key string) ([]byte, error)
|
||||
GetInt64(ctx context.Context, n MetadataNode, key string) (int64, error)
|
||||
Set(ctx context.Context, n MetadataNode, key string, val []byte) error
|
||||
SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte, acquireLock bool) error
|
||||
Remove(ctx context.Context, n MetadataNode, key string, acquireLock bool) error
|
||||
// SetMultiple sets multiple extended attributes for a node. It acquires the node's
|
||||
// metadata lock unless n.LockHeld() reports that the caller already holds it.
|
||||
SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte) error
|
||||
// Remove removes an extended attribute key. It acquires the node's metadata lock
|
||||
// unless n.LockHeld() reports that the caller already holds it.
|
||||
Remove(ctx context.Context, n MetadataNode, key string) error
|
||||
|
||||
Lock(n MetadataNode) (UnlockFunc, error)
|
||||
LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error)
|
||||
Purge(ctx context.Context, n MetadataNode) error
|
||||
Rename(oldNode, newNode MetadataNode) error
|
||||
MetadataPath(n MetadataNode) string
|
||||
@@ -100,12 +108,12 @@ func (NullBackend) Set(ctx context.Context, n MetadataNode, key string, val []by
|
||||
}
|
||||
|
||||
// SetMultiple sets a set of attribute for the given path
|
||||
func (NullBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte, acquireLock bool) error {
|
||||
func (NullBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte) error {
|
||||
return errUnconfiguredError
|
||||
}
|
||||
|
||||
// Remove removes an extended attribute key
|
||||
func (NullBackend) Remove(ctx context.Context, n MetadataNode, key string, acquireLock bool) error {
|
||||
func (NullBackend) Remove(ctx context.Context, n MetadataNode, key string) error {
|
||||
return errUnconfiguredError
|
||||
}
|
||||
|
||||
@@ -114,11 +122,6 @@ func (NullBackend) Lock(n MetadataNode) (UnlockFunc, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// LockAndRead locks the metadata for reading
|
||||
func (NullBackend) LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// IsMetaFile returns whether the given path represents a meta file
|
||||
func (NullBackend) IsMetaFile(path string) bool { return false }
|
||||
|
||||
@@ -133,9 +136,3 @@ func (NullBackend) MetadataPath(n MetadataNode) string { return "" }
|
||||
|
||||
// LockfilePath returns the path of the lock file
|
||||
func (NullBackend) LockfilePath(n MetadataNode) string { return "" }
|
||||
|
||||
// AllWithLockedSource reads all extended attributes from the given reader
|
||||
// The path argument is used for storing the data in the cache
|
||||
func (NullBackend) AllWithLockedSource(ctx context.Context, n MetadataNode, source io.Reader) (map[string][]byte, error) {
|
||||
return nil, errUnconfiguredError
|
||||
}
|
||||
Generated
Vendored
+29
-33
@@ -20,7 +20,6 @@ package metadata
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -100,17 +99,21 @@ func (b XattrsBackend) list(ctx context.Context, n MetadataNode, acquireLock boo
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n.SetLockHeld(true)
|
||||
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
|
||||
defer f.Close()
|
||||
defer func() {
|
||||
n.SetLockHeld(false)
|
||||
_ = f.Close()
|
||||
}()
|
||||
|
||||
}
|
||||
return xattr.List(filePath)
|
||||
}
|
||||
|
||||
// All reads all extended attributes for a node, protected by a
|
||||
// shared file lock
|
||||
// All reads all extended attributes for a node. It acquires a shared file lock
|
||||
// unless the caller already holds the node's metadata lock (n.LockHeld()).
|
||||
func (b XattrsBackend) All(ctx context.Context, n MetadataNode) (map[string][]byte, error) {
|
||||
return b.getAll(ctx, n, false, true)
|
||||
return b.getAll(ctx, n, false, !n.LockHeld())
|
||||
}
|
||||
|
||||
func (b XattrsBackend) getAll(ctx context.Context, n MetadataNode, skipCache, acquireLock bool) (map[string][]byte, error) {
|
||||
@@ -163,13 +166,13 @@ func (b XattrsBackend) getAll(ctx context.Context, n MetadataNode, skipCache, ac
|
||||
|
||||
// Set sets one attribute for the given path
|
||||
func (b XattrsBackend) Set(ctx context.Context, n MetadataNode, key string, val []byte) (err error) {
|
||||
return b.SetMultiple(ctx, n, map[string][]byte{key: val}, true)
|
||||
return b.SetMultiple(ctx, n, map[string][]byte{key: val})
|
||||
}
|
||||
|
||||
// SetMultiple sets a set of attribute for the given path
|
||||
func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte, acquireLock bool) (err error) {
|
||||
func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs map[string][]byte) (err error) {
|
||||
path := n.InternalPath()
|
||||
if acquireLock {
|
||||
if !n.LockHeld() {
|
||||
err := os.MkdirAll(filepath.Dir(path), 0700)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -178,8 +181,12 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n.SetLockHeld(true)
|
||||
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
|
||||
defer lockedFile.Close()
|
||||
defer func() {
|
||||
n.SetLockHeld(false)
|
||||
_ = lockedFile.Close()
|
||||
}()
|
||||
}
|
||||
|
||||
// error handling: Count if there are errors while setting the attribs.
|
||||
@@ -206,15 +213,19 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
|
||||
}
|
||||
|
||||
// Remove an extended attribute key
|
||||
func (b XattrsBackend) Remove(ctx context.Context, n MetadataNode, key string, acquireLock bool) error {
|
||||
func (b XattrsBackend) Remove(ctx context.Context, n MetadataNode, key string) error {
|
||||
path := n.InternalPath()
|
||||
if acquireLock {
|
||||
if !n.LockHeld() {
|
||||
lockedFile, err := lockedfile.OpenFile(path+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n.SetLockHeld(true)
|
||||
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
|
||||
defer lockedFile.Close()
|
||||
defer func() {
|
||||
n.SetLockHeld(false)
|
||||
_ = lockedFile.Close()
|
||||
}()
|
||||
}
|
||||
|
||||
err := xattr.Remove(path, key)
|
||||
@@ -275,38 +286,23 @@ func (XattrsBackend) LockfilePath(n MetadataNode) string { return n.InternalPath
|
||||
|
||||
// Lock locks the metadata for the given path
|
||||
func (b XattrsBackend) Lock(n MetadataNode) (UnlockFunc, error) {
|
||||
if n.LockHeld() {
|
||||
return func() error { return nil }, nil
|
||||
}
|
||||
|
||||
metaLockPath := b.LockfilePath(n)
|
||||
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n.SetLockHeld(true)
|
||||
return func() error {
|
||||
n.SetLockHeld(false)
|
||||
// Warning: do not remove the lockfile or we may lock the same file more than once, https://github.com/opencloud-eu/opencloud/issues/1793
|
||||
return mlock.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// LockAndRead locks the metadata for reading
|
||||
func (b XattrsBackend) LockAndRead(n MetadataNode) (UnlockFunc, io.Reader, error) {
|
||||
unlock, err := b.Lock(n)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
f, err := os.Open(b.MetadataPath(n))
|
||||
if err != nil {
|
||||
_ = unlock()
|
||||
return nil, nil, err
|
||||
}
|
||||
return unlock, f, nil
|
||||
}
|
||||
|
||||
// AllWithLockedSource reads all extended attributes from the given reader.
|
||||
// The path argument is used for storing the data in the cache
|
||||
func (b XattrsBackend) AllWithLockedSource(ctx context.Context, n MetadataNode, _ io.Reader) (map[string][]byte, error) {
|
||||
return b.All(ctx, n)
|
||||
}
|
||||
|
||||
func (b XattrsBackend) cacheKey(n MetadataNode) string {
|
||||
// rootPath is guaranteed to have no trailing slash
|
||||
// the cache key shouldn't begin with a slash as some stores drop it which can cause
|
||||
|
||||
+55
-32
@@ -44,13 +44,13 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/mime"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/internal/goroutinelock"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/ace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/grants"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
@@ -139,7 +139,7 @@ type Tree interface {
|
||||
BuildSpaceIDIndexEntry(spaceID string) string
|
||||
ResolveSpaceIDIndexEntry(spaceID string) (string, error)
|
||||
|
||||
CreateRevision(ctx context.Context, n *Node, version string, f *lockedfile.File) (string, error)
|
||||
CreateRevision(ctx context.Context, n *Node, version string) (string, error)
|
||||
ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error)
|
||||
DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string, openReaderFunc func(md *provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error)
|
||||
|
||||
@@ -166,8 +166,7 @@ type PathLookup interface {
|
||||
MetadataBackend() metadata.Backend
|
||||
TimeManager() TimeManager
|
||||
ReadBlobIDAndSizeAttr(ctx context.Context, n metadata.MetadataNode, attrs Attributes) (string, int64, error)
|
||||
CopyMetadataWithSourceLock(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), lockedSource *lockedfile.File, acquireTargetLock bool) (err error)
|
||||
CopyMetadata(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error)
|
||||
CopyMetadata(ctx context.Context, sourceNode, targetNode metadata.MetadataNode, filter func(attributeName string, value []byte) (newValue []byte, copy bool)) (err error)
|
||||
|
||||
PurgeNode(n *Node) error
|
||||
}
|
||||
@@ -181,6 +180,12 @@ type BaseNode struct {
|
||||
SpaceID string
|
||||
ID string
|
||||
|
||||
// lock records which goroutine (if any) holds this node's metadata lock, so the
|
||||
// metadata backend must not (re-)acquire it when reading attributes. Ownership is
|
||||
// scoped to the acquiring goroutine: a node leaked into another goroutine reports
|
||||
// LockHeld() == false there and takes its own lock rather than riding a stale flag.
|
||||
lock goroutinelock.Lock
|
||||
|
||||
lu PathLookup
|
||||
internalPathID string
|
||||
internalPath string
|
||||
@@ -197,6 +202,19 @@ func NewBaseNode(spaceID, nodeID string, lu PathLookup) *BaseNode {
|
||||
func (n *BaseNode) GetSpaceID() string { return n.SpaceID }
|
||||
func (n *BaseNode) GetID() string { return n.ID }
|
||||
|
||||
// LockHeld reports whether the calling goroutine holds this node's metadata lock.
|
||||
func (n *BaseNode) LockHeld() bool { return n.lock.Held() }
|
||||
|
||||
// SetLockHeld records whether the calling goroutine holds this node's metadata
|
||||
// lock
|
||||
func (n *BaseNode) SetLockHeld(held bool) {
|
||||
if held {
|
||||
n.lock.Hold()
|
||||
return
|
||||
}
|
||||
n.lock.Release()
|
||||
}
|
||||
|
||||
// InternalPath returns the internal path of the Node
|
||||
func (n *BaseNode) InternalPath() string {
|
||||
if len(n.internalPath) > 0 && n.ID == n.internalPathID {
|
||||
@@ -355,15 +373,15 @@ func LockAndReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID, intern
|
||||
ctx, span := tracer.Start(ctx, "LockAndReadNode")
|
||||
defer span.End()
|
||||
|
||||
_, subspan := tracer.Start(ctx, "lockedfile.OpenFile")
|
||||
_, subspan := tracer.Start(ctx, "MetadataBackend.Lock")
|
||||
bn := NewBaseNode(spaceID, nodeID, lu)
|
||||
unlock, r, err := lu.MetadataBackend().LockAndRead(bn)
|
||||
unlock, err := lu.MetadataBackend().Lock(bn)
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
n, err := readNode(ctx, lu, spaceID, nodeID, internalPath, canListDisabledSpace, spaceRoot, skipParentCheck, r)
|
||||
n, err := readNode(ctx, lu, spaceID, nodeID, internalPath, canListDisabledSpace, spaceRoot, skipParentCheck, true)
|
||||
if err != nil {
|
||||
_ = unlock()
|
||||
return nil, nil, err
|
||||
@@ -373,17 +391,22 @@ func LockAndReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID, intern
|
||||
return n, nil, errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
|
||||
}
|
||||
|
||||
return n, unlock, nil
|
||||
n.SetLockHeld(true)
|
||||
return n, func() error {
|
||||
n.SetLockHeld(false)
|
||||
return unlock()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ReadNode creates a new instance from an id and checks if it exists
|
||||
func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath string, canListDisabledSpace bool, spaceRoot *Node, skipParentCheck bool) (*Node, error) {
|
||||
return readNode(ctx, lu, spaceID, nodeID, internalPath, canListDisabledSpace, spaceRoot, skipParentCheck, nil)
|
||||
return readNode(ctx, lu, spaceID, nodeID, internalPath, canListDisabledSpace, spaceRoot, skipParentCheck, false)
|
||||
}
|
||||
|
||||
// readNode reads a node by its id. If a reader is provided, it will be passed to the metadata backend to read the metadata.
|
||||
// This is useful when the caller already holds a lock to prevent deadlocks when reading the metadata.
|
||||
func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath string, canListDisabledSpace bool, spaceRoot *Node, skipParentCheck bool, r io.Reader) (*Node, error) {
|
||||
// readNode reads a node by its id. When alreadyLocked is true the caller already
|
||||
// holds the node's metadata lock, so the attributes are read without re-acquiring it
|
||||
// (which would dead-lock e.g. the hybrid backend).
|
||||
func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath string, canListDisabledSpace bool, spaceRoot *Node, skipParentCheck bool, alreadyLocked bool) (*Node, error) {
|
||||
ctx, span := tracer.Start(ctx, "ReadNode")
|
||||
defer span.End()
|
||||
var err error
|
||||
@@ -398,12 +421,15 @@ func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath
|
||||
},
|
||||
}
|
||||
spaceRoot.SpaceRoot = spaceRoot
|
||||
if alreadyLocked && nodeID == spaceID {
|
||||
spaceRoot.SetLockHeld(true)
|
||||
}
|
||||
|
||||
// If we hold the lock on the space root itself, prime its attribute cache
|
||||
// through the no-lock path so the owner/name/disabled reads below do not try
|
||||
// to re-acquire the already-held lock and self-deadlock.
|
||||
if r != nil && nodeID == spaceID {
|
||||
_, err = spaceRoot.XattrsWithReader(ctx, r)
|
||||
if alreadyLocked && nodeID == spaceID {
|
||||
_, err = spaceRoot.Xattrs(ctx)
|
||||
switch {
|
||||
case metadata.IsNotExist(err):
|
||||
return spaceRoot, nil // swallow not found, the node defaults to exists = false
|
||||
@@ -464,6 +490,9 @@ func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath
|
||||
},
|
||||
SpaceRoot: spaceRoot,
|
||||
}
|
||||
if alreadyLocked {
|
||||
n.SetLockHeld(true)
|
||||
}
|
||||
if internalPath != "" {
|
||||
n.internalPath = internalPath
|
||||
}
|
||||
@@ -477,7 +506,7 @@ func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath
|
||||
}()
|
||||
|
||||
var attrs Attributes
|
||||
attrs, err = n.XattrsWithReader(ctx, r)
|
||||
attrs, err = n.Xattrs(ctx)
|
||||
switch {
|
||||
case metadata.IsNotExist(err):
|
||||
return n, nil // swallow not found, the node defaults to exists = false
|
||||
@@ -570,9 +599,9 @@ func (n *Node) Child(ctx context.Context, name string) (*Node, error) {
|
||||
return readNode, nil
|
||||
}
|
||||
|
||||
// ParentWithReader returns the parent node
|
||||
func (n *Node) ParentWithReader(ctx context.Context, r io.Reader) (*Node, error) {
|
||||
_, span := tracer.Start(ctx, "ParentWithReader")
|
||||
// Parent returns the parent node
|
||||
func (n *Node) Parent(ctx context.Context) (*Node, error) {
|
||||
_, span := tracer.Start(ctx, "Parent")
|
||||
defer span.End()
|
||||
if n.ParentID == "" {
|
||||
return nil, fmt.Errorf("decomposedfs: root has no parent")
|
||||
@@ -586,8 +615,7 @@ func (n *Node) ParentWithReader(ctx context.Context, r io.Reader) (*Node, error)
|
||||
SpaceRoot: n.SpaceRoot,
|
||||
}
|
||||
|
||||
// fill metadata cache using the reader
|
||||
attrs, err := p.XattrsWithReader(ctx, r)
|
||||
attrs, err := p.Xattrs(ctx)
|
||||
switch {
|
||||
case metadata.IsNotExist(err):
|
||||
return p, nil // swallow not found, the node defaults to exists = false
|
||||
@@ -602,11 +630,6 @@ func (n *Node) ParentWithReader(ctx context.Context, r io.Reader) (*Node, error)
|
||||
return p, err
|
||||
}
|
||||
|
||||
// Parent returns the parent node
|
||||
func (n *Node) Parent(ctx context.Context) (p *Node, err error) {
|
||||
return n.ParentWithReader(ctx, nil)
|
||||
}
|
||||
|
||||
// Owner returns the space owner
|
||||
func (n *Node) Owner() *userpb.UserId {
|
||||
return n.SpaceRoot.owner
|
||||
@@ -740,7 +763,7 @@ func (n *Node) SetMtimeString(ctx context.Context, mtime string) error {
|
||||
// SetMTime writes the UTC mtime to the extended attributes or removes the attribute if nil is passed
|
||||
func (n *Node) SetMtime(ctx context.Context, t *time.Time) (err error) {
|
||||
if t == nil {
|
||||
return n.RemoveXattr(ctx, prefixes.MTimeAttr, true)
|
||||
return n.RemoveXattr(ctx, prefixes.MTimeAttr)
|
||||
}
|
||||
return n.SetXattrString(ctx, prefixes.MTimeAttr, t.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
@@ -780,7 +803,7 @@ func (n *Node) SetFavorite(ctx context.Context, uid *userpb.UserId) error {
|
||||
func (n *Node) UnsetFavorite(ctx context.Context, uid *userpb.UserId) error {
|
||||
// the favorite flag is specific to the user, so we need to incorporate the userid
|
||||
fa := prefixes.FavoriteKey(uid)
|
||||
return n.RemoveXattr(ctx, fa, true)
|
||||
return n.RemoveXattr(ctx, fa)
|
||||
}
|
||||
|
||||
// IsDir returns true if the node is a directory
|
||||
@@ -1117,7 +1140,7 @@ func (n *Node) SetChecksum(ctx context.Context, csType string, h hash.Hash) (err
|
||||
|
||||
// UnsetTempEtag removes the temporary etag attribute
|
||||
func (n *Node) UnsetTempEtag(ctx context.Context) (err error) {
|
||||
return n.RemoveXattr(ctx, prefixes.TmpEtagAttr, true)
|
||||
return n.RemoveXattr(ctx, prefixes.TmpEtagAttr)
|
||||
}
|
||||
|
||||
func isGrantExpired(g *provider.Grant) bool {
|
||||
@@ -1278,7 +1301,7 @@ func (n *Node) ReadGrant(ctx context.Context, grantee string) (g *provider.Grant
|
||||
}
|
||||
|
||||
// ReadGrant reads a CS3 grant
|
||||
func (n *Node) DeleteGrant(ctx context.Context, g *provider.Grant, acquireLock bool) (err error) {
|
||||
func (n *Node) DeleteGrant(ctx context.Context, g *provider.Grant) (err error) {
|
||||
|
||||
var attr string
|
||||
if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP {
|
||||
@@ -1287,7 +1310,7 @@ func (n *Node) DeleteGrant(ctx context.Context, g *provider.Grant, acquireLock b
|
||||
attr = prefixes.GrantUserAcePrefix + g.Grantee.GetUserId().OpaqueId
|
||||
}
|
||||
|
||||
if err = n.RemoveXattr(ctx, attr, acquireLock); err != nil {
|
||||
if err = n.RemoveXattr(ctx, attr); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1379,7 +1402,7 @@ func (n *Node) UnmarkProcessing(ctx context.Context, uploadID string) error {
|
||||
// file started another postprocessing later - do not remove
|
||||
return nil
|
||||
}
|
||||
return n.RemoveXattr(ctx, prefixes.StatusPrefix, true)
|
||||
return n.RemoveXattr(ctx, prefixes.StatusPrefix)
|
||||
}
|
||||
|
||||
// IsProcessing returns true if the node is currently being processed
|
||||
@@ -1404,7 +1427,7 @@ func (n *Node) SetScanData(ctx context.Context, info string, date time.Time) err
|
||||
attribs := Attributes{}
|
||||
attribs.SetString(prefixes.ScanStatusPrefix, info)
|
||||
attribs.SetString(prefixes.ScanDatePrefix, date.Format(time.RFC3339Nano))
|
||||
return n.SetXattrsWithContext(ctx, attribs, true)
|
||||
return n.SetXattrsWithContext(ctx, attribs)
|
||||
}
|
||||
|
||||
// ScanData returns scanning information of the node
|
||||
|
||||
Generated
Vendored
+25
-36
@@ -20,7 +20,6 @@ package node
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/fs"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -72,7 +71,7 @@ func (md Attributes) SetTime(key string, t time.Time) {
|
||||
}
|
||||
|
||||
// SetXattrs sets multiple extended attributes on the write-through cache/node
|
||||
func (n *Node) SetXattrsWithContext(ctx context.Context, attribs map[string][]byte, acquireLock bool) (err error) {
|
||||
func (n *Node) SetXattrsWithContext(ctx context.Context, attribs map[string][]byte) (err error) {
|
||||
_, span := tracer.Start(ctx, "SetXattrsWithContext")
|
||||
defer span.End()
|
||||
if n.xattrsCache != nil {
|
||||
@@ -81,12 +80,12 @@ func (n *Node) SetXattrsWithContext(ctx context.Context, attribs map[string][]by
|
||||
}
|
||||
}
|
||||
|
||||
return n.lu.MetadataBackend().SetMultiple(ctx, n, attribs, acquireLock)
|
||||
return n.lu.MetadataBackend().SetMultiple(ctx, n, attribs)
|
||||
}
|
||||
|
||||
// SetXattrs sets multiple extended attributes on the write-through cache/node
|
||||
func (n *Node) SetXattrs(attribs map[string][]byte, acquireLock bool) (err error) {
|
||||
return n.SetXattrsWithContext(context.Background(), attribs, acquireLock)
|
||||
func (n *Node) SetXattrs(attribs map[string][]byte) (err error) {
|
||||
return n.SetXattrsWithContext(context.Background(), attribs)
|
||||
}
|
||||
|
||||
// SetXattr sets an extended attribute on the write-through cache/node
|
||||
@@ -108,33 +107,28 @@ func (n *Node) SetXattrString(ctx context.Context, key, val string) (err error)
|
||||
}
|
||||
|
||||
// RemoveXattr removes an extended attribute from the write-through cache/node
|
||||
func (n *Node) RemoveXattr(ctx context.Context, key string, acquireLock bool) error {
|
||||
func (n *Node) RemoveXattr(ctx context.Context, key string) error {
|
||||
if n.xattrsCache != nil {
|
||||
delete(n.xattrsCache, key)
|
||||
}
|
||||
return n.lu.MetadataBackend().Remove(ctx, n, key, acquireLock)
|
||||
return n.lu.MetadataBackend().Remove(ctx, n, key)
|
||||
}
|
||||
|
||||
// XattrsWithReader returns the extended attributes of the node. If the attributes have already
|
||||
// been cached they are not read from disk again.
|
||||
func (n *Node) XattrsWithReader(ctx context.Context, r io.Reader) (Attributes, error) {
|
||||
// loadXattrs reads and caches the node's extended attributes using the given load
|
||||
// function. It centralizes the empty-node guard and the write-through caching that
|
||||
// is shared by all the Xattrs accessors below.
|
||||
func (n *Node) loadXattrs(load func() (Attributes, error)) (Attributes, error) {
|
||||
if n.ID == "" {
|
||||
// Do not try to read the attribute of an empty node. The InternalPath points to the
|
||||
// base nodes directory in this case.
|
||||
return Attributes{}, &xattr.Error{Op: "node.XattrsWithReader", Path: n.InternalPath(), Err: xattr.ENOATTR}
|
||||
return Attributes{}, &xattr.Error{Op: "node.Xattrs", Path: n.InternalPath(), Err: xattr.ENOATTR}
|
||||
}
|
||||
|
||||
if n.xattrsCache != nil {
|
||||
return n.xattrsCache, nil
|
||||
}
|
||||
|
||||
var attrs Attributes
|
||||
var err error
|
||||
if r != nil {
|
||||
attrs, err = n.lu.MetadataBackend().AllWithLockedSource(ctx, n, r)
|
||||
} else {
|
||||
attrs, err = n.lu.MetadataBackend().All(ctx, n)
|
||||
}
|
||||
attrs, err := load()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -143,37 +137,32 @@ func (n *Node) XattrsWithReader(ctx context.Context, r io.Reader) (Attributes, e
|
||||
return n.xattrsCache, nil
|
||||
}
|
||||
|
||||
// Xattrs returns the extended attributes of the node. If the attributes have already
|
||||
// been cached they are not read from disk again.
|
||||
// Xattrs returns the extended attributes of the node, acquiring the metadata lock as
|
||||
// needed. If the attributes have already been cached they are not read from disk
|
||||
// again.
|
||||
func (n *Node) Xattrs(ctx context.Context) (Attributes, error) {
|
||||
return n.XattrsWithReader(ctx, nil)
|
||||
return n.loadXattrs(func() (Attributes, error) {
|
||||
return n.lu.MetadataBackend().All(ctx, n)
|
||||
})
|
||||
}
|
||||
|
||||
// Xattr returns an extended attribute of the node. If the attributes have already
|
||||
// been cached it is not read from disk again.
|
||||
func (n *Node) Xattr(ctx context.Context, key string) ([]byte, error) {
|
||||
path := n.InternalPath()
|
||||
|
||||
if path == "" {
|
||||
// Do not try to read the attribute of an non-existing node
|
||||
// Do not try to read the attribute of a non-existing node
|
||||
return []byte{}, fs.ErrNotExist
|
||||
}
|
||||
|
||||
if n.ID == "" {
|
||||
// Do not try to read the attribute of an empty node. The InternalPath points to the
|
||||
// base nodes directory in this case.
|
||||
return []byte{}, &xattr.Error{Op: "node.Xattr", Path: path, Name: key, Err: xattr.ENOATTR}
|
||||
attrs, err := n.loadXattrs(func() (Attributes, error) {
|
||||
return n.lu.MetadataBackend().All(ctx, n)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if n.xattrsCache == nil {
|
||||
attrs, err := n.lu.MetadataBackend().All(ctx, n)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
n.xattrsCache = attrs
|
||||
}
|
||||
|
||||
if val, ok := n.xattrsCache[key]; ok {
|
||||
if val, ok := attrs[key]; ok {
|
||||
return val, nil
|
||||
}
|
||||
// wrap the error as xattr does
|
||||
|
||||
Generated
Vendored
+1
-1
@@ -127,7 +127,7 @@ func New(m map[string]interface{}) (*Options, error) {
|
||||
o.GatewayAddr = sharedconf.GetGatewaySVC(o.GatewayAddr)
|
||||
|
||||
if o.MetadataBackend == "" {
|
||||
o.MetadataBackend = "xattrs"
|
||||
o.MetadataBackend = "messagepack"
|
||||
}
|
||||
|
||||
// ensure user layout has no starting or trailing /
|
||||
|
||||
+3
-4
@@ -27,7 +27,6 @@ import (
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
@@ -99,12 +98,12 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer
|
||||
}
|
||||
|
||||
// write lock node before copying metadata
|
||||
f, err := lockedfile.OpenFile(fs.lu.MetadataBackend().LockfilePath(n), os.O_RDWR|os.O_CREATE, 0600)
|
||||
unlock, err := fs.lu.MetadataBackend().Lock(n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = f.Close()
|
||||
_ = unlock()
|
||||
_ = os.Remove(fs.lu.MetadataBackend().LockfilePath(n))
|
||||
}()
|
||||
|
||||
@@ -116,7 +115,7 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer
|
||||
}
|
||||
|
||||
// create a revision of the current node
|
||||
if _, err := fs.tp.CreateRevision(ctx, n, mtime.UTC().Format(time.RFC3339Nano), f); err != nil {
|
||||
if _, err := fs.tp.CreateRevision(ctx, n, mtime.UTC().Format(time.RFC3339Nano)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
+3
-3
@@ -196,7 +196,7 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr
|
||||
metadata.SetString(prefixes.SpaceAliasAttr, alias)
|
||||
}
|
||||
|
||||
if err := root.SetXattrsWithContext(ctx, metadata, true); err != nil {
|
||||
if err := root.SetXattrsWithContext(ctx, metadata); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -697,7 +697,7 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up
|
||||
}
|
||||
metadata[prefixes.TreeMTimeAttr] = []byte(time.Now().UTC().Format(time.RFC3339Nano))
|
||||
|
||||
err = spaceNode.SetXattrsWithContext(ctx, metadata, true)
|
||||
err = spaceNode.SetXattrsWithContext(ctx, metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -935,7 +935,7 @@ func (fs *Decomposedfs) StorageSpaceFromNode(ctx context.Context, n *node.Node,
|
||||
// This way we don't have to have a cron job checking the grants in regular intervals.
|
||||
// The tradeof obviously is that this code is here.
|
||||
if isGrantExpired(g) {
|
||||
if err := n.DeleteGrant(ctx, g, true); err != nil {
|
||||
if err := n.DeleteGrant(ctx, g); err != nil {
|
||||
sublog.Error().Err(err).Str("grantee", id).
|
||||
Msg("failed to delete expired space grant")
|
||||
}
|
||||
|
||||
Generated
Vendored
+3
-3
@@ -54,7 +54,7 @@ func (dtm *Manager) MTime(ctx context.Context, n *node.Node) (time.Time, error)
|
||||
// If the time is nil, the attribute is removed.
|
||||
func (dtm *Manager) SetMTime(ctx context.Context, n *node.Node, mtime *time.Time) error {
|
||||
if mtime == nil {
|
||||
return n.RemoveXattr(ctx, prefixes.MTimeAttr, true)
|
||||
return n.RemoveXattr(ctx, prefixes.MTimeAttr)
|
||||
}
|
||||
return n.SetXattrString(ctx, prefixes.MTimeAttr, mtime.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
@@ -75,7 +75,7 @@ func (dtm *Manager) TMTime(ctx context.Context, n *node.Node) (time.Time, error)
|
||||
// If the time is nil, the attribute is removed.
|
||||
func (dtm *Manager) SetTMTime(ctx context.Context, n *node.Node, tmtime *time.Time) error {
|
||||
if tmtime == nil {
|
||||
return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr, true)
|
||||
return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr)
|
||||
}
|
||||
return n.SetXattrString(ctx, prefixes.TreeMTimeAttr, tmtime.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
@@ -121,7 +121,7 @@ func (dtm *Manager) DTime(ctx context.Context, n *node.Node) (tmTime time.Time,
|
||||
// If the time is nil, the attribute is removed.
|
||||
func (dtm *Manager) SetDTime(ctx context.Context, n *node.Node, t *time.Time) (err error) {
|
||||
if t == nil {
|
||||
return n.RemoveXattr(ctx, prefixes.DTimeAttr, true)
|
||||
return n.RemoveXattr(ctx, prefixes.DTimeAttr)
|
||||
}
|
||||
return n.SetXattrString(ctx, prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
Generated
Vendored
+18
-19
@@ -43,12 +43,6 @@ const (
|
||||
|
||||
var _propagationGracePeriod = 3 * time.Minute
|
||||
|
||||
type PropagationNode interface {
|
||||
GetSpaceID() string
|
||||
GetID() string
|
||||
InternalPath() string
|
||||
}
|
||||
|
||||
// AsyncPropagator implements asynchronous treetime & treesize propagation
|
||||
type AsyncPropagator struct {
|
||||
treeSizeAccounting bool
|
||||
@@ -130,8 +124,8 @@ func NewAsyncPropagator(treeSizeAccounting, treeTimeAccounting bool, o options.A
|
||||
now := time.Now()
|
||||
_ = os.Chtimes(changesDirPath, now, now)
|
||||
|
||||
n := node.NewBaseNode(parts[0], strings.TrimSuffix(parts[1], ".processing"), lookup)
|
||||
p.propagate(context.Background(), n, true, *log)
|
||||
nodeID := strings.TrimSuffix(parts[1], ".processing")
|
||||
p.propagate(context.Background(), parts[0], nodeID, true, *log)
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -164,14 +158,17 @@ func (p AsyncPropagator) Propagate(ctx context.Context, n *node.Node, sizeDiff i
|
||||
SyncTime: time.Now().UTC(),
|
||||
SizeDiff: sizeDiff,
|
||||
}
|
||||
go p.queuePropagation(ctx, n, c, log)
|
||||
|
||||
// Hand only the node's logical identity to the goroutine, never the live
|
||||
// *node.Node: sharing the instance would leak and race its metadata-lock state.
|
||||
go p.queuePropagation(ctx, n.SpaceID, n.ID, c, log)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p AsyncPropagator) queuePropagation(ctx context.Context, n *node.Node, change Change, log zerolog.Logger) {
|
||||
func (p AsyncPropagator) queuePropagation(ctx context.Context, spaceID, nodeID string, change Change, log zerolog.Logger) {
|
||||
// add a change to the parent node
|
||||
changePath := p.changesPath(n.SpaceID, n.ID, uuid.New().String()+".mpk")
|
||||
changePath := p.changesPath(spaceID, nodeID, uuid.New().String()+".mpk")
|
||||
|
||||
data, err := msgpack.Marshal(change)
|
||||
if err != nil {
|
||||
@@ -212,7 +209,7 @@ func (p AsyncPropagator) queuePropagation(ctx context.Context, n *node.Node, cha
|
||||
|
||||
log.Debug().Msg("propagating")
|
||||
// add a change to the parent node
|
||||
changeDirPath := p.changesPath(n.SpaceID, n.ID, "")
|
||||
changeDirPath := p.changesPath(spaceID, nodeID, "")
|
||||
|
||||
// first rename the existing node dir
|
||||
err = os.Rename(changeDirPath, changeDirPath+".processing")
|
||||
@@ -224,11 +221,11 @@ func (p AsyncPropagator) queuePropagation(ctx context.Context, n *node.Node, cha
|
||||
// -> ignore, the previous propagation will pick the new changes up
|
||||
return
|
||||
}
|
||||
p.propagate(ctx, n, false, log)
|
||||
p.propagate(ctx, spaceID, nodeID, false, log)
|
||||
}
|
||||
|
||||
func (p AsyncPropagator) propagate(ctx context.Context, pn PropagationNode, recalculateTreeSize bool, log zerolog.Logger) {
|
||||
changeDirPath := p.changesPath(pn.GetSpaceID(), pn.GetID(), "")
|
||||
func (p AsyncPropagator) propagate(ctx context.Context, spaceID, nodeID string, recalculateTreeSize bool, log zerolog.Logger) {
|
||||
changeDirPath := p.changesPath(spaceID, nodeID, "")
|
||||
processingPath := changeDirPath + ".processing"
|
||||
|
||||
cleanup := func() {
|
||||
@@ -285,8 +282,10 @@ func (p AsyncPropagator) propagate(ctx context.Context, pn PropagationNode, reca
|
||||
|
||||
attrs := node.Attributes{}
|
||||
|
||||
// lock parent before reading treesize or tree time
|
||||
// we deliberately allow reading disabled spaces so that the metadata is always consistent (see https://github.com/opencloud-eu/reva/issues/747)
|
||||
_, subspan = tracer.Start(ctx, "node.LockAndReadNode")
|
||||
n, unlock, err := node.LockAndReadNode(ctx, p.lookup, pn.GetSpaceID(), pn.GetID(), "", false, nil, false)
|
||||
n, unlock, err := node.LockAndReadNode(ctx, p.lookup, spaceID, nodeID, "", true, nil, false)
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
if n != nil && !n.Exists {
|
||||
@@ -385,7 +384,7 @@ func (p AsyncPropagator) propagate(ctx context.Context, pn PropagationNode, reca
|
||||
log.Debug().Uint64("newSize", newSize).Msg("updated treesize of node")
|
||||
}
|
||||
|
||||
if err = n.SetXattrsWithContext(ctx, attrs, false); err != nil {
|
||||
if err = n.SetXattrsWithContext(ctx, attrs); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to update extend attributes of node")
|
||||
cleanup()
|
||||
return
|
||||
@@ -400,7 +399,7 @@ func (p AsyncPropagator) propagate(ctx context.Context, pn PropagationNode, reca
|
||||
cleanup()
|
||||
|
||||
if !n.IsSpaceRoot(ctx) {
|
||||
p.queuePropagation(ctx, n, pc, log)
|
||||
p.queuePropagation(ctx, n.SpaceID, n.ID, pc, log)
|
||||
}
|
||||
|
||||
// Check for a changes dir that might have been added meanwhile and pick it up
|
||||
@@ -416,7 +415,7 @@ func (p AsyncPropagator) propagate(ctx context.Context, pn PropagationNode, reca
|
||||
// -> ignore, the previous propagation will pick the new changes up
|
||||
return
|
||||
}
|
||||
p.propagate(ctx, n, false, log)
|
||||
p.propagate(ctx, spaceID, nodeID, false, log)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Generated
Vendored
+3
-2
@@ -92,7 +92,8 @@ func (p SyncPropagator) propagateItem(ctx context.Context, n *node.Node, sTime t
|
||||
attrs := node.Attributes{}
|
||||
|
||||
// lock parent before reading treesize or tree time
|
||||
n, unlock, err := node.LockAndReadNode(ctx, p.lookup, n.SpaceID, n.ParentID, "", false, n.SpaceRoot, false)
|
||||
// we deliberately allow reading disabled spaces so that the metadata is always consistent (see https://github.com/opencloud-eu/reva/issues/747)
|
||||
n, unlock, err := node.LockAndReadNode(ctx, p.lookup, n.SpaceID, n.ParentID, "", true, n.SpaceRoot, false)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
@@ -190,7 +191,7 @@ func (p SyncPropagator) propagateItem(ctx context.Context, n *node.Node, sTime t
|
||||
log.Debug().Uint64("newSize", newSize).Msg("updated treesize of parent node")
|
||||
}
|
||||
|
||||
if err = n.SetXattrsWithContext(ctx, attrs, false); err != nil {
|
||||
if err = n.SetXattrsWithContext(ctx, attrs); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to update extend attributes of parent node")
|
||||
return n, true, err
|
||||
}
|
||||
|
||||
Generated
Vendored
+8
-8
@@ -30,7 +30,6 @@ import (
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
@@ -49,8 +48,10 @@ import (
|
||||
// We can add a background process to move old revisions to a slower storage
|
||||
// and replace the revision file with a symbolic link in the future, if necessary.
|
||||
|
||||
// CreateVersion creates a new version of the node
|
||||
func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string, f *lockedfile.File) (string, error) {
|
||||
// CreateRevision creates a new version of the node. The caller MUST already hold the
|
||||
// metadata lock of n, as the node's metadata is read (without re-locking) to copy the
|
||||
// blob metadata onto the new revision.
|
||||
func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string) (string, error) {
|
||||
versionNode := node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+version, tp.lookup)
|
||||
versionPath := versionNode.InternalPath()
|
||||
|
||||
@@ -116,13 +117,13 @@ func (tp *Tree) CreateRevision(ctx context.Context, n *node.Node, version string
|
||||
defer vf.Close()
|
||||
|
||||
// copy blob metadata to version node
|
||||
if err := tp.lookup.CopyMetadataWithSourceLock(ctx, n, versionNode, func(attributeName string, value []byte) (newValue []byte, copy bool) {
|
||||
if err := tp.lookup.CopyMetadata(ctx, n, versionNode, func(attributeName string, value []byte) (newValue []byte, copy bool) {
|
||||
return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) ||
|
||||
attributeName == prefixes.TypeAttr ||
|
||||
attributeName == prefixes.BlobIDAttr ||
|
||||
attributeName == prefixes.BlobsizeAttr ||
|
||||
attributeName == prefixes.MTimeAttr
|
||||
}, f, true); err != nil {
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -336,7 +337,7 @@ func (tp *Tree) RestoreRevision(ctx context.Context, sourceNode, targetNode meta
|
||||
attributeName == prefixes.TypeAttr ||
|
||||
attributeName == prefixes.BlobIDAttr ||
|
||||
attributeName == prefixes.BlobsizeAttr
|
||||
}, false)
|
||||
})
|
||||
if err != nil {
|
||||
return errtypes.InternalError("failed to copy blob xattrs to old revision to node: " + err.Error())
|
||||
}
|
||||
@@ -347,8 +348,7 @@ func (tp *Tree) RestoreRevision(ctx context.Context, sourceNode, targetNode meta
|
||||
err = tp.lookup.MetadataBackend().SetMultiple(ctx, targetNode,
|
||||
map[string][]byte{
|
||||
prefixes.MTimeAttr: []byte(mtime.UTC().Format(time.RFC3339Nano)),
|
||||
},
|
||||
false)
|
||||
})
|
||||
if err != nil {
|
||||
return errtypes.InternalError("failed to set mtime attribute on node: " + err.Error())
|
||||
}
|
||||
|
||||
+9
-9
@@ -156,7 +156,7 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool,
|
||||
return errors.Wrap(err, "Decomposedfs: could not set mtime")
|
||||
}
|
||||
}
|
||||
err = n.SetXattrsWithContext(ctx, attributes, true)
|
||||
err = n.SetXattrsWithContext(ctx, attributes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -288,7 +288,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node)
|
||||
attribs := node.Attributes{}
|
||||
attribs.SetString(prefixes.ParentidAttr, newNode.ParentID)
|
||||
attribs.SetString(prefixes.NameAttr, newNode.Name)
|
||||
if err := oldNode.SetXattrsWithContext(ctx, attribs, true); err != nil {
|
||||
if err := oldNode.SetXattrsWithContext(ctx, attribs); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: could not update old node attributes")
|
||||
}
|
||||
|
||||
@@ -458,7 +458,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) {
|
||||
trashLink := filepath.Join(t.options.Root, "spaces", lookup.Pathify(n.SpaceRoot.ID, 1, 2), "trash", lookup.Pathify(n.ID, 4, 2))
|
||||
if err := os.MkdirAll(filepath.Dir(trashLink), 0700); err != nil {
|
||||
// Roll back changes
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true)
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -471,7 +471,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) {
|
||||
err = os.Symlink("../../../../../nodes/"+lookup.Pathify(n.ID, 4, 2)+node.TrashIDDelimiter+deletionTime, trashLink)
|
||||
if err != nil {
|
||||
// Roll back changes
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true)
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -485,12 +485,12 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) {
|
||||
// To roll back changes
|
||||
// TODO remove symlink
|
||||
// Roll back changes
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true)
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
|
||||
return
|
||||
}
|
||||
err = t.lookup.MetadataBackend().Rename(n, trashNode)
|
||||
if err != nil {
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true)
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
|
||||
_ = os.Rename(trashPath, nodePath)
|
||||
return
|
||||
}
|
||||
@@ -507,7 +507,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) {
|
||||
// TODO revert the rename
|
||||
// TODO remove symlink
|
||||
// Roll back changes
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true)
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -580,7 +580,7 @@ func (t *Tree) RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPa
|
||||
// set ParentidAttr to restorePath's node parent id
|
||||
attrs.SetString(prefixes.ParentidAttr, targetNode.ParentID)
|
||||
|
||||
if err = t.lookup.MetadataBackend().SetMultiple(ctx, restoreNode, map[string][]byte(attrs), true); err != nil {
|
||||
if err = t.lookup.MetadataBackend().SetMultiple(ctx, restoreNode, map[string][]byte(attrs)); err != nil {
|
||||
return nil, errors.Wrap(err, "Decomposedfs: could not update recycle node")
|
||||
}
|
||||
|
||||
@@ -850,7 +850,7 @@ func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) {
|
||||
if t.options.TreeTimeAccounting || t.options.TreeSizeAccounting {
|
||||
attributes[prefixes.PropagationAttr] = []byte("1") // mark the node for propagation
|
||||
}
|
||||
return n.SetXattrsWithContext(ctx, attributes, true)
|
||||
return n.SetXattrsWithContext(ctx, attributes)
|
||||
}
|
||||
|
||||
var nodeIDRegep = regexp.MustCompile(`.*/nodes/([^.]*).*`)
|
||||
|
||||
Generated
Vendored
+24
-3
@@ -46,6 +46,12 @@ type DecomposedFsSession struct {
|
||||
info tusd.FileInfo
|
||||
}
|
||||
|
||||
const (
|
||||
SessionStatusUploading = "uploading"
|
||||
SessionStatusProcessing = "processing"
|
||||
SessionStatusFailed = "failed"
|
||||
)
|
||||
|
||||
// Context returns a context with the user, logger and lockid used when initiating the upload session
|
||||
func (session *DecomposedFsSession) Context(ctx context.Context) context.Context { // restore logger from file info
|
||||
sub := session.store.log.With().Int("pid", os.Getpid()).Logger()
|
||||
@@ -307,10 +313,9 @@ func (session *DecomposedFsSession) MTime() time.Time {
|
||||
return t
|
||||
}
|
||||
|
||||
// IsProcessing returns true if all bytes have been received. The session then has entered postprocessing state.
|
||||
// IsProcessing returns true if the upload is in the processing state, meaning that the upload has finished and postprocessing is still running.
|
||||
func (session *DecomposedFsSession) IsProcessing() bool {
|
||||
// We might need a more sophisticated way to determine processing status soon
|
||||
return session.info.Size == session.info.Offset && session.info.MetaData["scanResult"] == ""
|
||||
return session.info.MetaData["status"] == SessionStatusProcessing
|
||||
}
|
||||
|
||||
// binPath returns the path to the file storing the binary data.
|
||||
@@ -339,6 +344,22 @@ func (session *DecomposedFsSession) ScanData() (string, time.Time) {
|
||||
return session.info.MetaData["scanResult"], d
|
||||
}
|
||||
|
||||
// SetStatus sets the status of the upload session
|
||||
func (session *DecomposedFsSession) SetStatus(status, msg string) {
|
||||
session.info.MetaData["status"] = status
|
||||
session.info.MetaData["statusMessage"] = msg
|
||||
}
|
||||
|
||||
// Status returns the status of the upload session
|
||||
func (session *DecomposedFsSession) Status() string {
|
||||
return session.info.MetaData["status"]
|
||||
}
|
||||
|
||||
// StatusMessage returns the status message of the upload session
|
||||
func (session *DecomposedFsSession) StatusMessage() string {
|
||||
return session.info.MetaData["statusMessage"]
|
||||
}
|
||||
|
||||
// sessionPath returns the path to the .info file storing the file's info.
|
||||
func sessionPath(root, id string) string {
|
||||
return filepath.Join(root, "uploads", id+".info")
|
||||
|
||||
Generated
Vendored
+7
-11
@@ -44,7 +44,6 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"github.com/rs/zerolog"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
)
|
||||
@@ -95,7 +94,9 @@ func (store DecomposedFsStore) New(ctx context.Context) *DecomposedFsSession {
|
||||
Storage: map[string]string{
|
||||
"Type": "DecomposedFsStore",
|
||||
},
|
||||
MetaData: tusd.MetaData{},
|
||||
MetaData: tusd.MetaData{
|
||||
"status": string(SessionStatusUploading),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -286,7 +287,7 @@ func (store DecomposedFsStore) CreateNodeForUpload(ctx context.Context, session
|
||||
}
|
||||
|
||||
// update node metadata with new blobid etc
|
||||
err = n.SetXattrsWithContext(ctx, initAttrs, false)
|
||||
err = n.SetXattrsWithContext(ctx, initAttrs)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Decomposedfs: could not write metadata")
|
||||
}
|
||||
@@ -306,16 +307,11 @@ func (store DecomposedFsStore) updateExistingNode(ctx context.Context, session *
|
||||
defer span.End()
|
||||
|
||||
// write lock existing node before reading any metadata
|
||||
f, err := lockedfile.OpenFile(store.lu.MetadataBackend().LockfilePath(n), os.O_RDWR|os.O_CREATE, 0600)
|
||||
unlock, err := store.lu.MetadataBackend().Lock(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
unlock := func() error {
|
||||
// NOTE: to prevent stale NFS file handles do not remove lock file!
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
old, _ := node.ReadNode(ctx, store.lu, spaceID, n.ID, "", false, nil, false)
|
||||
if _, err := node.CheckQuota(ctx, n.SpaceRoot, true, uint64(old.Blobsize), fsize); err != nil {
|
||||
return unlock, err
|
||||
@@ -366,7 +362,7 @@ func (store DecomposedFsStore) updateExistingNode(ctx context.Context, session *
|
||||
span.AddEvent("CreateVersion")
|
||||
timestamp := oldNodeMtime.UTC().Format(time.RFC3339Nano)
|
||||
versionID := n.ID + node.RevisionIDDelimiter + timestamp
|
||||
versionPath, err := session.store.tp.CreateRevision(ctx, n, timestamp, f)
|
||||
versionPath, err := session.store.tp.CreateRevision(ctx, n, timestamp)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrExist) {
|
||||
return unlock, err
|
||||
@@ -389,7 +385,7 @@ func (store DecomposedFsStore) updateExistingNode(ctx context.Context, session *
|
||||
}
|
||||
|
||||
// clean revision file
|
||||
if versionPath, err = session.store.tp.CreateRevision(ctx, n, timestamp, f); err != nil {
|
||||
if versionPath, err = session.store.tp.CreateRevision(ctx, n, timestamp); err != nil {
|
||||
return unlock, err
|
||||
}
|
||||
}
|
||||
|
||||
Generated
Vendored
+39
-27
@@ -35,7 +35,6 @@ import (
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
@@ -46,6 +45,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/utils/download"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
@@ -71,6 +71,10 @@ func (session *DecomposedFsSession) WriteChunk(ctx context.Context, _ int64, src
|
||||
return 0, err
|
||||
}
|
||||
defer func() {
|
||||
// sync the written chunk to disk. This ensures that the upload can be resumed,
|
||||
// and helps to prevent issues with filesystem/journal freezes at the end of the upload
|
||||
// when committing a large fsync operation on slow disks.
|
||||
_ = disk.Fdatasync(file)
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
@@ -190,8 +194,18 @@ func (session *DecomposedFsSession) FinishUploadDecomposed(ctx context.Context)
|
||||
|
||||
n, err := session.store.CreateNodeForUpload(ctx, session, attrs)
|
||||
if err != nil {
|
||||
session.SetStatus(SessionStatusFailed, err.Error())
|
||||
if perr := session.Persist(ctx); perr != nil {
|
||||
log.Error().Err(perr).Msg("failed to persist upload session after setting status to failed")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
session.SetStatus(SessionStatusProcessing, "")
|
||||
if err = session.Persist(ctx); err != nil {
|
||||
log.Error().Err(err).Msg("failed to persist upload session after setting status to processing")
|
||||
}
|
||||
|
||||
// increase the processing counter for every started processing
|
||||
// will be decreased in Cleanup()
|
||||
metrics.UploadProcessing.Inc()
|
||||
@@ -291,7 +305,7 @@ func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) {
|
||||
ctx, span := tracer.Start(session.Context(ctx), "Finalize")
|
||||
defer span.End()
|
||||
|
||||
revisionNode := node.New(session.SpaceID(), session.NodeID(), "", "", session.Size(), session.ID(),
|
||||
n := node.New(session.SpaceID(), session.NodeID(), "", "", session.Size(), session.ID(),
|
||||
provider.ResourceType_RESOURCE_TYPE_FILE, session.SpaceOwner(), session.store.lu)
|
||||
|
||||
var (
|
||||
@@ -305,19 +319,20 @@ func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) {
|
||||
case spaceRoot.InternalPath() == "":
|
||||
return fmt.Errorf("space root for space id %s has no valid internal path", session.SpaceID())
|
||||
default:
|
||||
revisionNode.SpaceRoot = spaceRoot
|
||||
n.SpaceRoot = spaceRoot
|
||||
}
|
||||
|
||||
// lock the node before writing the blob
|
||||
lockedNode, err := lockedfile.OpenFile(session.store.lu.MetadataBackend().LockfilePath(revisionNode), os.O_RDWR|os.O_CREATE, 0600)
|
||||
// lock the node before reading its metadata and writing the blob
|
||||
unlock, err := session.store.lu.MetadataBackend().Lock(n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = lockedNode.Close()
|
||||
_ = unlock()
|
||||
}()
|
||||
|
||||
attribs, err := revisionNode.XattrsWithReader(ctx, lockedNode)
|
||||
// Read the node attributes while holding the lock acquired above.
|
||||
attribs, err := n.Xattrs(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -327,32 +342,27 @@ func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) {
|
||||
|
||||
// another upload on this node is in progress or has finished since we started
|
||||
if !isProcessing || processingID != session.ID() {
|
||||
versionID := revisionNode.ID + node.RevisionIDDelimiter + session.MTime().UTC().Format(time.RFC3339Nano)
|
||||
versionID := n.ID + node.RevisionIDDelimiter + session.MTime().UTC().Format(time.RFC3339Nano)
|
||||
// There should be a revision node (created by the other upload that finished before us), read it and upload our blob there.
|
||||
existingRevisionNode, err := node.ReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, spaceRoot, false)
|
||||
existingRevisionNode, revisionNodeUnlock, err := node.LockAndReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, spaceRoot, false)
|
||||
if err != nil || !existingRevisionNode.Exists {
|
||||
// The revision node has not been created. Likely because the file on disk was modified externally and re-assilimated (watchfs == true)
|
||||
// Let's create the revision node now and upload the blob to it.
|
||||
revisionNode, err = session.createRevisionNodeForUpload(ctx, revisionNode, session.MTime().UTC().Format(time.RFC3339Nano), lockedNode)
|
||||
n, revisionNodeUnlock, err = session.createRevisionNodeForUpload(ctx, n, session.MTime().UTC().Format(time.RFC3339Nano))
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Debug().Err(err).Str("versionID", session.MTime().UTC().Format(time.RFC3339Nano)).Msg("failed to create revision node for upload finalization")
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
revisionNode = existingRevisionNode
|
||||
n = existingRevisionNode
|
||||
}
|
||||
// lock this node as well, before writing the blob
|
||||
revisionNodeUnlock, err := session.store.lu.MetadataBackend().Lock(revisionNode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
appctx.GetLogger(ctx).Debug().Str("new nodepath", revisionNode.InternalPath()).Msg("uploading to revision node, that was created for us by another upload")
|
||||
appctx.GetLogger(ctx).Debug().Str("new nodepath", n.InternalPath()).Msg("uploading to revision node, that was created for us by another upload")
|
||||
defer func() { _ = revisionNodeUnlock() }()
|
||||
}
|
||||
|
||||
// upload the data to the blobstore
|
||||
_, subspan := tracer.Start(ctx, "WriteBlob")
|
||||
err = session.store.tp.WriteBlob(revisionNode, session.binPath())
|
||||
err = session.store.tp.WriteBlob(n, session.binPath())
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to upload file to blobstore")
|
||||
@@ -361,19 +371,19 @@ func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (session *DecomposedFsSession) createRevisionNodeForUpload(ctx context.Context, baseNode *node.Node, rev string, lockedNode *lockedfile.File) (*node.Node, error) {
|
||||
func (session *DecomposedFsSession) createRevisionNodeForUpload(ctx context.Context, baseNode *node.Node, rev string) (*node.Node, func() error, error) {
|
||||
versionID := baseNode.ID + node.RevisionIDDelimiter + rev
|
||||
log := appctx.GetLogger(ctx)
|
||||
_, err := session.store.tp.CreateRevision(ctx, baseNode, rev, lockedNode)
|
||||
_, err := session.store.tp.CreateRevision(ctx, baseNode, rev)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("versionID", versionID).Msg("failed to create revision node for upload")
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// FIXME: We already calculated the checksums in FinishUpload, we should maybe pass them via the session instead of recalculating them here
|
||||
sha1h, md5h, adler32h, err := node.CalculateChecksums(ctx, session.binPath())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// update checksums
|
||||
@@ -382,7 +392,7 @@ func (session *DecomposedFsSession) createRevisionNodeForUpload(ctx context.Cont
|
||||
prefixes.ChecksumPrefix + "md5": md5h.Sum(nil),
|
||||
prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil),
|
||||
}
|
||||
revisionNode, err := node.ReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, baseNode.SpaceRoot, false)
|
||||
revisionNode, unlock, err := node.LockAndReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, baseNode.SpaceRoot, false)
|
||||
if err == nil {
|
||||
mtime := session.MTime()
|
||||
attrs.SetString(prefixes.BlobIDAttr, session.ID())
|
||||
@@ -392,14 +402,16 @@ func (session *DecomposedFsSession) createRevisionNodeForUpload(ctx context.Cont
|
||||
|
||||
err = session.store.lu.TimeManager().OverrideMtime(ctx, revisionNode, &attrs, mtime)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime")
|
||||
_ = unlock()
|
||||
return nil, nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime")
|
||||
}
|
||||
err = revisionNode.SetXattrsWithContext(ctx, attrs, false)
|
||||
err = revisionNode.SetXattrsWithContext(ctx, attrs)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Decomposedfs: failed to set node attributes")
|
||||
_ = unlock()
|
||||
return nil, nil, errors.Wrap(err, "Decomposedfs: failed to set node attributes")
|
||||
}
|
||||
}
|
||||
return revisionNode, err
|
||||
return revisionNode, unlock, err
|
||||
}
|
||||
|
||||
func checkHash(expected string, h hash.Hash) error {
|
||||
|
||||
+5
@@ -80,6 +80,11 @@ type UploadSession interface {
|
||||
|
||||
// ScanData returns the scan data for the UploadSession
|
||||
ScanData() (string, time.Time)
|
||||
|
||||
// Status returns the status for the UploadSession
|
||||
Status() string
|
||||
// StatusMessage returns the status message for the UploadSession
|
||||
StatusMessage() string
|
||||
}
|
||||
|
||||
// UploadSessionFilter can be used to filter upload sessions
|
||||
|
||||
Generated
Vendored
+16
@@ -338,6 +338,22 @@ func (s *OcisSession) ScanData() (string, time.Time) {
|
||||
return s.info.MetaData["scanResult"], d
|
||||
}
|
||||
|
||||
func (s *OcisSession) SetStatus(status string) {
|
||||
s.info.MetaData["status"] = status
|
||||
}
|
||||
|
||||
func (s *OcisSession) Status() string {
|
||||
return s.info.MetaData["status"]
|
||||
}
|
||||
|
||||
func (s *OcisSession) SetStatusMessage(message string) {
|
||||
s.info.MetaData["status_message"] = message
|
||||
}
|
||||
|
||||
func (s *OcisSession) StatusMessage() string {
|
||||
return s.info.MetaData["status_message"]
|
||||
}
|
||||
|
||||
// sessionPath returns the path to the .info file storing the file's info.
|
||||
func sessionPath(root, id string) string {
|
||||
return filepath.Join(root, "uploads", id+".info")
|
||||
|
||||
+6
@@ -252,6 +252,12 @@ func IsSpaceRoot(ri *storageprovider.ResourceInfo) bool {
|
||||
return f.GetOpaqueId() == s.GetOpaqueId() && f.GetSpaceId() == s.GetSpaceId()
|
||||
}
|
||||
|
||||
// IsProcessing checks if the given resource info is still being processed
|
||||
// (e.g. postprocessing after an upload has not finished yet).
|
||||
func IsProcessing(ri *storageprovider.ResourceInfo) bool {
|
||||
return ReadPlainFromOpaque(ri.GetOpaque(), "status") == "processing"
|
||||
}
|
||||
|
||||
func checkStatusCode(reason, message string, code rpc.Code) error {
|
||||
if code == rpc.Code_CODE_OK {
|
||||
return nil
|
||||
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool
|
||||
*.out
|
||||
coverage.txt
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# README GIF generation files
|
||||
*.mov
|
||||
*.mp4
|
||||
*.gif
|
||||
*.prproj
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
-71
@@ -1,71 +0,0 @@
|
||||
run:
|
||||
tests: true
|
||||
|
||||
# all available settings of specific linters
|
||||
linters-settings:
|
||||
govet:
|
||||
# report about shadowed variables
|
||||
check-shadowing: true
|
||||
gofmt:
|
||||
# simplify code: gofmt with `-s` option, true by default
|
||||
simplify: true
|
||||
dupl:
|
||||
# tokens count to trigger issue, 150 by default
|
||||
threshold: 100
|
||||
goconst:
|
||||
# minimal length of string constant, 3 by default
|
||||
min-len: 3
|
||||
# minimal occurrences count to trigger, 3 by default
|
||||
min-occurrences: 3
|
||||
misspell:
|
||||
# Correct spellings using locale preferences for US or UK.
|
||||
# Default is to use a neutral variety of English.
|
||||
# Setting locale to US will correct the British spelling of 'colour' to 'color'.
|
||||
locale: US
|
||||
staticcheck:
|
||||
checks: [ "all" ]
|
||||
revive:
|
||||
confidence: 0.8
|
||||
ignore-generated-header: true
|
||||
rules:
|
||||
- name: context-keys-type
|
||||
- name: time-naming
|
||||
- name: var-declaration
|
||||
- name: unexported-return
|
||||
- name: errorf
|
||||
- name: blank-imports
|
||||
- name: context-as-argument
|
||||
- name: dot-imports
|
||||
- name: error-return
|
||||
- name: error-strings
|
||||
- name: error-naming
|
||||
- name: exported
|
||||
- name: increment-decrement
|
||||
- name: var-naming
|
||||
- name: package-comments
|
||||
- name: range
|
||||
- name: receiver-naming
|
||||
- name: indent-error-flow
|
||||
- name: superfluous-else
|
||||
- name: struct-tag
|
||||
- name: modifies-value-receiver
|
||||
- name: range-val-in-closure
|
||||
- name: range-val-address
|
||||
- name: atomic
|
||||
- name: empty-lines
|
||||
- name: early-return
|
||||
- name: useless-break
|
||||
|
||||
linters:
|
||||
enable:
|
||||
- revive
|
||||
- govet
|
||||
- gosec
|
||||
- staticcheck
|
||||
- typecheck
|
||||
fast: false
|
||||
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
exclude:
|
||||
- G104
|
||||
-201
@@ -1,201 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-274
@@ -1,274 +0,0 @@
|
||||
# Yet Another CLi Spinner (for Go)
|
||||
[](https://github.com/theckman/yacspin/blob/master/LICENSE)
|
||||
[](https://godoc.org/github.com/theckman/yacspin)
|
||||
[](https://github.com/theckman/yacspin/releases)
|
||||
[](https://github.com/theckman/yacspin/actions/workflows/tests.yaml)
|
||||
[](https://goreportcard.com/report/github.com/theckman/yacspin)
|
||||
[](https://codecov.io/gh/theckman/yacspin)
|
||||
|
||||
Package `yacspin` provides yet another CLi spinner for Go, taking inspiration
|
||||
(and some utility code) from the https://github.com/briandowns/spinner project.
|
||||
Specifically `yacspin` borrows the default character sets, and color mappings to
|
||||
github.com/fatih/color colors, from that project.
|
||||
|
||||
## License
|
||||
Because this package adopts the spinner character sets from https://github.com/briandowns/spinner,
|
||||
this package is released under the Apache 2.0 License.
|
||||
|
||||
## Yet Another CLi Spinner?
|
||||
This project was created after it was realized that the most popular spinner
|
||||
library for Go had some limitations, that couldn't be fixed without a massive
|
||||
overhaul of the API.
|
||||
|
||||
The other spinner ties the ability to show updated messages to the spinner's
|
||||
animation, meaning you can't always show all the information you want to the end
|
||||
user without changing the animation speed. This means you need to trade off
|
||||
animation aesthetics to show "realtime" information. It was a goal to avoid this
|
||||
problem.
|
||||
|
||||
In addition, there were also some API design choices that have made it unsafe
|
||||
for concurrent use, which presents challenges when trying to update the text in
|
||||
the spinner while it's animating. This could result in undefined behavior due to
|
||||
data races.
|
||||
|
||||
There were also some variable-width spinners in that other project that did
|
||||
not render correctly. Because the width of the spinner animation would change,
|
||||
so would the position of the message on the screen. `yacspin` uses a dynamic
|
||||
width when animating, so your message should appear static relative to the
|
||||
animating spinner.
|
||||
|
||||
Finally, there was an interest in the spinner being able to represent a task, and to
|
||||
indicate whether it failed or was successful. This would have further compounded
|
||||
the API changes needed above to support in an intuitive way.
|
||||
|
||||
This project takes inspiration from that other project, and takes a new approach
|
||||
to address the challenges above.
|
||||
|
||||
## Features
|
||||
#### Provided Spinners
|
||||
There are over 90 spinners available in the `CharSets` package variable. They
|
||||
were borrowed from [github.com/briandowns/spinner](https://github.com/briandowns/spinner).
|
||||
There is a table with most of the spinners [at the bottom of this README](#Spinners).
|
||||
|
||||
#### Dynamic Width of Animation
|
||||
Because of how some spinners are animated, they may have different widths are
|
||||
different times in the animation. `yacspin` calculates the maximum width, and
|
||||
pads the animation to ensure the text's position on the screen doesn't change.
|
||||
This results in a smoother looking animation.
|
||||
|
||||
##### yacspin
|
||||

|
||||
|
||||
##### other spinners
|
||||

|
||||
|
||||
#### Success and Failure Results
|
||||
The spinner has both a `Stop()` and `StopFail()` method, which allows the
|
||||
spinner to result in a success message or a failure message. The messages,
|
||||
colors, and even the character used to denote success or failure are
|
||||
customizable in either the initial config or via the spinner's methods.
|
||||
|
||||
By doing this you can use a single `yacspin` spinner to display the status of a
|
||||
list of tasks being executed serially.
|
||||
|
||||
##### Stop
|
||||

|
||||
|
||||
##### StopFail
|
||||

|
||||
|
||||
#### Animation At End of Line
|
||||
The `SpinnerAtEnd` field of the `Config` struct allows you to specify whether
|
||||
the spinner is rendered at the end of the line instead of the beginning. The
|
||||
default value (`false`) results in the spinner being rendered at the beginning
|
||||
of the line.
|
||||
|
||||
#### Concurrency
|
||||
The spinner is safe for concurrent use, so you can update any of its settings
|
||||
via methods whether the spinner is stopped or is currently animating.
|
||||
|
||||
#### Live Updates
|
||||
Most spinners tie the ability to show new messages with the animation of the
|
||||
spinner. So if the spinner animates every 200ms, you can only show updated
|
||||
information every 200ms. If you wanted more frequent updates, you'd need to
|
||||
tradeoff the asthetics of the animation to display more data.
|
||||
|
||||
This spinner updates the printed information of the spinner immediately on
|
||||
change, without the animation updating. This allows you to use an animation
|
||||
speed that looks astheticaly pleasing, while also knowing the data presented to
|
||||
the user will be updated live.
|
||||
|
||||
You can see this in action in the following gif, where the filenames being
|
||||
uploaded are rendered independent of the spinner being animated:
|
||||
|
||||

|
||||
|
||||
#### Pausing for Updates
|
||||
Sometimes you want to change a few settings, and don't want the spinner to
|
||||
render your partially applied configuration. If your spinner is running, and you
|
||||
want to change a few configuration items via method calls, you can `Pause()` the
|
||||
spinner first. After making the changes you can call `Unpause()`, and it will
|
||||
continue rendering like normal with the newly applied configuration.
|
||||
|
||||
#### Supporting Non-Interactive (TTY) Output Targets
|
||||
`yacspin` also has native support for non-interactive (TTY) output targets. By
|
||||
default this is detected in the constructor, or can be overriden via the
|
||||
`TerminalMode` `Config` struct field. When detecting the application is not
|
||||
running withn a TTY session, the behavior of the spinner is different.
|
||||
|
||||
Specifically, when this is automatically detected the spinner no longer uses
|
||||
colors, disables the automatic spinner animation, and instead only animates the
|
||||
spinner when updating the message. In addition, each animation is rendered on a
|
||||
new line instead of overwriting the current line.
|
||||
|
||||
This should result in human-readable output without any changes needed by
|
||||
consumers, even when the system is writing to a non-TTY destination.
|
||||
|
||||
#### Manually Stepping Animation
|
||||
If you'd like to manually animate the spinner, you can do so by setting the
|
||||
`TerminalMode` to `ForceNoTTYMode | ForceSmartTerminalMode`. In this mode the
|
||||
spinner will still use colors and other text stylings, but the animation only
|
||||
happens when data is updated and on individual lines. You can accomplish this by
|
||||
calling the `Message()` method with the same used previously.
|
||||
|
||||
## Usage
|
||||
```
|
||||
go get github.com/theckman/yacspin
|
||||
```
|
||||
|
||||
Within the `yacspin` package there are some default spinners stored in the
|
||||
`yacspin.CharSets` variable, and you can also provide your own. There is also a
|
||||
list of known colors in the `yacspin.ValidColors` variable.
|
||||
|
||||
### Example
|
||||
|
||||
There are runnable examples in the [examples/](https://github.com/theckman/yacspin/tree/master/examples)
|
||||
directory, with one simple example and one more advanced one. Here is a quick
|
||||
snippet showing usage from a very high level, with error handling omitted:
|
||||
|
||||
```Go
|
||||
cfg := yacspin.Config{
|
||||
Frequency: 100 * time.Millisecond,
|
||||
CharSet: yacspin.CharSets[59],
|
||||
Suffix: " backing up database to S3",
|
||||
SuffixAutoColon: true,
|
||||
Message: "exporting data",
|
||||
StopCharacter: "✓",
|
||||
StopColors: []string{"fgGreen"},
|
||||
}
|
||||
|
||||
spinner, err := yacspin.New(cfg)
|
||||
// handle the error
|
||||
|
||||
err = spinner.Start()
|
||||
|
||||
// doing some work
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
spinner.Message("uploading data")
|
||||
|
||||
// upload...
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
err = spinner.Stop()
|
||||
```
|
||||
|
||||
## Spinners
|
||||
|
||||
The spinner animations below are recorded at a refresh frequency of 200ms. Some
|
||||
animations may look better at a different speed, so play around with the
|
||||
frequency until you find a value you find aesthetically pleasing.
|
||||
|
||||
yacspin.CharSets index | sample gif (Frequency: 200ms)
|
||||
-----------------------|------------------------------
|
||||
0 | 
|
||||
1 | 
|
||||
2 | 
|
||||
3 | 
|
||||
4 | 
|
||||
5 | 
|
||||
6 | 
|
||||
7 | 
|
||||
8 | 
|
||||
9 | 
|
||||
10 | 
|
||||
11 | 
|
||||
12 | 
|
||||
13 | 
|
||||
14 | 
|
||||
15 | 
|
||||
16 | 
|
||||
17 | 
|
||||
18 | 
|
||||
19 | 
|
||||
20 | 
|
||||
21 | 
|
||||
22 | 
|
||||
23 | 
|
||||
24 | 
|
||||
25 | 
|
||||
26 | 
|
||||
27 | 
|
||||
28 | 
|
||||
29 | 
|
||||
30 | 
|
||||
31 | 
|
||||
32 | 
|
||||
33 | 
|
||||
34 | 
|
||||
35 | 
|
||||
36 | 
|
||||
37 | 
|
||||
38 | 
|
||||
39 | 
|
||||
40 | 
|
||||
41 | 
|
||||
42 | 
|
||||
43 | 
|
||||
44 | 
|
||||
45 | 
|
||||
46 | 
|
||||
47 | 
|
||||
48 | 
|
||||
49 | 
|
||||
50 | 
|
||||
51 | 
|
||||
52 | 
|
||||
53 | 
|
||||
54 | 
|
||||
55 | 
|
||||
56 | 
|
||||
57 | 
|
||||
58 | 
|
||||
59 | 
|
||||
60 | 
|
||||
61 | 
|
||||
62 | 
|
||||
63 | 
|
||||
64 | 
|
||||
65 | 
|
||||
66 | 
|
||||
67 | 
|
||||
68 | 
|
||||
69 | 
|
||||
70 | 
|
||||
71 | 
|
||||
72 | 
|
||||
73 | 
|
||||
74 | 
|
||||
75 | 
|
||||
76 | 
|
||||
77 | 
|
||||
78 | 
|
||||
79 | 
|
||||
80 | 
|
||||
81 | 
|
||||
82 | 
|
||||
83 | 
|
||||
84 | 
|
||||
85 | 
|
||||
86 | 
|
||||
87 | 
|
||||
88 | 
|
||||
89 | 
|
||||
90 | 
|
||||
-121
@@ -1,121 +0,0 @@
|
||||
// Copyright (c) 2021 Brian J. Downs
|
||||
// Copyright (c) 2019-2021 Tim Heckman
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Please see the LICENSE file for the copy of the Apache 2.0 License.
|
||||
//
|
||||
// This file was copied from: https://github.com/briandowns/spinner
|
||||
//
|
||||
// Modifications:
|
||||
//
|
||||
// - removed runtime generation of CharSets 37 and 38; made them literals
|
||||
// - fixed pipe spinner (32) animation, by adding missing frame
|
||||
|
||||
package yacspin
|
||||
|
||||
// CharSets contains the default character sets from
|
||||
// https://github.com/briandowns/spinner.
|
||||
var CharSets = map[int][]string{
|
||||
0: {"←", "↖", "↑", "↗", "→", "↘", "↓", "↙"},
|
||||
1: {"▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃", "▁"},
|
||||
2: {"▖", "▘", "▝", "▗"},
|
||||
3: {"┤", "┘", "┴", "└", "├", "┌", "┬", "┐"},
|
||||
4: {"◢", "◣", "◤", "◥"},
|
||||
5: {"◰", "◳", "◲", "◱"},
|
||||
6: {"◴", "◷", "◶", "◵"},
|
||||
7: {"◐", "◓", "◑", "◒"},
|
||||
8: {".", "o", "O", "@", "*"},
|
||||
9: {"|", "/", "-", "\\"},
|
||||
10: {"◡◡", "⊙⊙", "◠◠"},
|
||||
11: {"⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"},
|
||||
12: {">))'>", " >))'>", " >))'>", " >))'>", " >))'>", " <'((<", " <'((<", " <'((<"},
|
||||
13: {"⠁", "⠂", "⠄", "⡀", "⢀", "⠠", "⠐", "⠈"},
|
||||
14: {"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"},
|
||||
15: {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"},
|
||||
16: {"▉", "▊", "▋", "▌", "▍", "▎", "▏", "▎", "▍", "▌", "▋", "▊", "▉"},
|
||||
17: {"■", "□", "▪", "▫"},
|
||||
18: {"←", "↑", "→", "↓"},
|
||||
19: {"╫", "╪"},
|
||||
20: {"⇐", "⇖", "⇑", "⇗", "⇒", "⇘", "⇓", "⇙"},
|
||||
21: {"⠁", "⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈", "⠈"},
|
||||
22: {"⠈", "⠉", "⠋", "⠓", "⠒", "⠐", "⠐", "⠒", "⠖", "⠦", "⠤", "⠠", "⠠", "⠤", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋", "⠉", "⠈"},
|
||||
23: {"⠁", "⠉", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠤", "⠄", "⠄", "⠤", "⠴", "⠲", "⠒", "⠂", "⠂", "⠒", "⠚", "⠙", "⠉", "⠁"},
|
||||
24: {"⠋", "⠙", "⠚", "⠒", "⠂", "⠂", "⠒", "⠲", "⠴", "⠦", "⠖", "⠒", "⠐", "⠐", "⠒", "⠓", "⠋"},
|
||||
25: {"ヲ", "ァ", "ィ", "ゥ", "ェ", "ォ", "ャ", "ュ", "ョ", "ッ", "ア", "イ", "ウ", "エ", "オ", "カ", "キ", "ク", "ケ", "コ", "サ", "シ", "ス", "セ", "ソ", "タ", "チ", "ツ", "テ", "ト", "ナ", "ニ", "ヌ", "ネ", "ノ", "ハ", "ヒ", "フ", "ヘ", "ホ", "マ", "ミ", "ム", "メ", "モ", "ヤ", "ユ", "ヨ", "ラ", "リ", "ル", "レ", "ロ", "ワ", "ン"},
|
||||
26: {".", "..", "..."},
|
||||
27: {"▁", "▂", "▃", "▄", "▅", "▆", "▇", "█", "▉", "▊", "▋", "▌", "▍", "▎", "▏", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█", "▇", "▆", "▅", "▄", "▃", "▂", "▁"},
|
||||
28: {".", "o", "O", "°", "O", "o", "."},
|
||||
29: {"+", "x"},
|
||||
30: {"v", "<", "^", ">"},
|
||||
31: {">>--->", " >>--->", " >>--->", " >>--->", " >>--->", " <---<<", " <---<<", " <---<<", " <---<<", "<---<<"},
|
||||
32: {"|", "||", "|||", "||||", "|||||", "||||||", "|||||||", "||||||||", "|||||||", "||||||", "|||||", "||||", "|||", "||", "|"},
|
||||
33: {"[ ]", "[= ]", "[== ]", "[=== ]", "[==== ]", "[===== ]", "[====== ]", "[======= ]", "[======== ]", "[========= ]", "[==========]"},
|
||||
34: {"(*---------)", "(-*--------)", "(--*-------)", "(---*------)", "(----*-----)", "(-----*----)", "(------*---)", "(-------*--)", "(--------*-)", "(---------*)"},
|
||||
35: {"█▒▒▒▒▒▒▒▒▒", "███▒▒▒▒▒▒▒", "█████▒▒▒▒▒", "███████▒▒▒", "██████████"},
|
||||
36: {"[ ]", "[=> ]", "[===> ]", "[=====> ]", "[======> ]", "[========> ]", "[==========> ]", "[============> ]", "[==============> ]", "[================> ]", "[==================> ]", "[===================>]"},
|
||||
37: {"🕐", "🕑", "🕒", "🕓", "🕔", "🕕", "🕖", "🕗", "🕘", "🕙", "🕚", "🕛"}, // clock emoji: one per hour for hours 1~12
|
||||
38: {"🕐", "🕜", "🕑", "🕝", "🕒", "🕞", "🕓", "🕟", "🕔", "🕠", "🕕", "🕡", "🕖", "🕢", "🕗", "🕣", "🕘", "🕤", "🕙", "🕥", "🕚", "🕦", "🕛", "🕧"}, // clock emoji: one per half hour for hours 1~12
|
||||
39: {"🌍", "🌎", "🌏"},
|
||||
40: {"◜", "◝", "◞", "◟"},
|
||||
41: {"⬒", "⬔", "⬓", "⬕"},
|
||||
42: {"⬖", "⬘", "⬗", "⬙"},
|
||||
43: {"[>>> >]", "[]>>>> []", "[] >>>> []", "[] >>>> []", "[] >>>> []", "[] >>>>[]", "[>> >>]"},
|
||||
44: {"♠", "♣", "♥", "♦"},
|
||||
45: {"➞", "➟", "➠", "➡", "➠", "➟"},
|
||||
46: {" | ", ` \ `, "_ ", ` \ `, " | ", " / ", " _", " / "},
|
||||
47: {" . . . .", ". . . .", ". . . .", ". . . .", ". . . . ", ". . . . ."},
|
||||
48: {" | ", " / ", " _ ", ` \ `, " | ", ` \ `, " _ ", " / "},
|
||||
49: {"⎺", "⎻", "⎼", "⎽", "⎼", "⎻"},
|
||||
50: {"▹▹▹▹▹", "▸▹▹▹▹", "▹▸▹▹▹", "▹▹▸▹▹", "▹▹▹▸▹", "▹▹▹▹▸"},
|
||||
51: {"[ ]", "[ =]", "[ ==]", "[ ===]", "[====]", "[=== ]", "[== ]", "[= ]"},
|
||||
52: {"( ● )", "( ● )", "( ● )", "( ● )", "( ●)", "( ● )", "( ● )", "( ● )", "( ● )"},
|
||||
53: {"✶", "✸", "✹", "✺", "✹", "✷"},
|
||||
54: {"▐|\\____________▌", "▐_|\\___________▌", "▐__|\\__________▌", "▐___|\\_________▌", "▐____|\\________▌", "▐_____|\\_______▌", "▐______|\\______▌", "▐_______|\\_____▌", "▐________|\\____▌", "▐_________|\\___▌", "▐__________|\\__▌", "▐___________|\\_▌", "▐____________|\\▌", "▐____________/|▌", "▐___________/|_▌", "▐__________/|__▌", "▐_________/|___▌", "▐________/|____▌", "▐_______/|_____▌", "▐______/|______▌", "▐_____/|_______▌", "▐____/|________▌", "▐___/|_________▌", "▐__/|__________▌", "▐_/|___________▌", "▐/|____________▌"},
|
||||
55: {"▐⠂ ▌", "▐⠈ ▌", "▐ ⠂ ▌", "▐ ⠠ ▌", "▐ ⡀ ▌", "▐ ⠠ ▌", "▐ ⠂ ▌", "▐ ⠈ ▌", "▐ ⠂ ▌", "▐ ⠠ ▌", "▐ ⡀ ▌", "▐ ⠠ ▌", "▐ ⠂ ▌", "▐ ⠈ ▌", "▐ ⠂▌", "▐ ⠠▌", "▐ ⡀▌", "▐ ⠠ ▌", "▐ ⠂ ▌", "▐ ⠈ ▌", "▐ ⠂ ▌", "▐ ⠠ ▌", "▐ ⡀ ▌", "▐ ⠠ ▌", "▐ ⠂ ▌", "▐ ⠈ ▌", "▐ ⠂ ▌", "▐ ⠠ ▌", "▐ ⡀ ▌", "▐⠠ ▌"},
|
||||
56: {"¿", "?"},
|
||||
57: {"⢹", "⢺", "⢼", "⣸", "⣇", "⡧", "⡗", "⡏"},
|
||||
58: {"⢄", "⢂", "⢁", "⡁", "⡈", "⡐", "⡠"},
|
||||
59: {". ", ".. ", "...", " ..", " .", " "},
|
||||
60: {".", "o", "O", "°", "O", "o", "."},
|
||||
61: {"▓", "▒", "░"},
|
||||
62: {"▌", "▀", "▐", "▄"},
|
||||
63: {"⊶", "⊷"},
|
||||
64: {"▪", "▫"},
|
||||
65: {"□", "■"},
|
||||
66: {"▮", "▯"},
|
||||
67: {"-", "=", "≡"},
|
||||
68: {"d", "q", "p", "b"},
|
||||
69: {"∙∙∙", "●∙∙", "∙●∙", "∙∙●", "∙∙∙"},
|
||||
70: {"🌑 ", "🌒 ", "🌓 ", "🌔 ", "🌕 ", "🌖 ", "🌗 ", "🌘 "},
|
||||
71: {"☗", "☖"},
|
||||
72: {"⧇", "⧆"},
|
||||
73: {"◉", "◎"},
|
||||
74: {"㊂", "㊀", "㊁"},
|
||||
75: {"⦾", "⦿"},
|
||||
76: {"ဝ", "၀"},
|
||||
77: {"▌", "▀", "▐▄"},
|
||||
78: {"⠈⠁", "⠈⠑", "⠈⠱", "⠈⡱", "⢀⡱", "⢄⡱", "⢄⡱", "⢆⡱", "⢎⡱", "⢎⡰", "⢎⡠", "⢎⡀", "⢎⠁", "⠎⠁", "⠊⠁"},
|
||||
79: {"________", "-_______", "_-______", "__-_____", "___-____", "____-___", "_____-__", "______-_", "_______-", "________", "_______-", "______-_", "_____-__", "____-___", "___-____", "__-_____", "_-______", "-_______", "________"},
|
||||
80: {"|_______", "_/______", "__-_____", "___\\____", "____|___", "_____/__", "______-_", "_______\\", "_______|", "______\\_", "_____-__", "____/___", "___|____", "__\\_____", "_-______"},
|
||||
81: {"□", "◱", "◧", "▣", "■"},
|
||||
82: {"□", "◱", "▨", "▩", "■"},
|
||||
83: {"░", "▒", "▓", "█"},
|
||||
84: {"░", "█"},
|
||||
85: {"⚪", "⚫"},
|
||||
86: {"◯", "⬤"},
|
||||
87: {"▱", "▰"},
|
||||
88: {"➊", "➋", "➌", "➍", "➎", "➏", "➐", "➑", "➒", "➓"},
|
||||
89: {"½", "⅓", "⅔", "¼", "¾", "⅛", "⅜", "⅝", "⅞"},
|
||||
90: {"↞", "↟", "↠", "↡"},
|
||||
}
|
||||
-174
@@ -1,174 +0,0 @@
|
||||
// This file is available under the Apache 2.0 License
|
||||
// This file was copied from: https://github.com/briandowns/spinner
|
||||
//
|
||||
// Please see the LICENSE file for the copy of the Apache 2.0 License.
|
||||
//
|
||||
// Modifications:
|
||||
//
|
||||
// - made validColors set map more idiomatic with an empty struct value
|
||||
// - added a function for creating color functions from color list
|
||||
|
||||
package yacspin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/fatih/color"
|
||||
)
|
||||
|
||||
// ValidColors holds the list of the strings that are mapped to
|
||||
// github.com/fatih/color color attributes. Any of these colors / attributes can
|
||||
// be used with the *Spinner type, and it should be reflected in the output.
|
||||
var ValidColors = map[string]struct{}{
|
||||
// default colors for backwards compatibility
|
||||
"black": {},
|
||||
"red": {},
|
||||
"green": {},
|
||||
"yellow": {},
|
||||
"blue": {},
|
||||
"magenta": {},
|
||||
"cyan": {},
|
||||
"white": {},
|
||||
|
||||
// attributes
|
||||
"reset": {},
|
||||
"bold": {},
|
||||
"faint": {},
|
||||
"italic": {},
|
||||
"underline": {},
|
||||
"blinkslow": {},
|
||||
"blinkrapid": {},
|
||||
"reversevideo": {},
|
||||
"concealed": {},
|
||||
"crossedout": {},
|
||||
|
||||
// foreground text
|
||||
"fgBlack": {},
|
||||
"fgRed": {},
|
||||
"fgGreen": {},
|
||||
"fgYellow": {},
|
||||
"fgBlue": {},
|
||||
"fgMagenta": {},
|
||||
"fgCyan": {},
|
||||
"fgWhite": {},
|
||||
|
||||
// foreground Hi-Intensity text
|
||||
"fgHiBlack": {},
|
||||
"fgHiRed": {},
|
||||
"fgHiGreen": {},
|
||||
"fgHiYellow": {},
|
||||
"fgHiBlue": {},
|
||||
"fgHiMagenta": {},
|
||||
"fgHiCyan": {},
|
||||
"fgHiWhite": {},
|
||||
|
||||
// background text
|
||||
"bgBlack": {},
|
||||
"bgRed": {},
|
||||
"bgGreen": {},
|
||||
"bgYellow": {},
|
||||
"bgBlue": {},
|
||||
"bgMagenta": {},
|
||||
"bgCyan": {},
|
||||
"bgWhite": {},
|
||||
|
||||
// background Hi-Intensity text
|
||||
"bgHiBlack": {},
|
||||
"bgHiRed": {},
|
||||
"bgHiGreen": {},
|
||||
"bgHiYellow": {},
|
||||
"bgHiBlue": {},
|
||||
"bgHiMagenta": {},
|
||||
"bgHiCyan": {},
|
||||
"bgHiWhite": {},
|
||||
}
|
||||
|
||||
// returns a valid color's foreground text color attribute
|
||||
var colorAttributeMap = map[string]color.Attribute{
|
||||
// default colors for backwards compatibility
|
||||
"black": color.FgBlack,
|
||||
"red": color.FgRed,
|
||||
"green": color.FgGreen,
|
||||
"yellow": color.FgYellow,
|
||||
"blue": color.FgBlue,
|
||||
"magenta": color.FgMagenta,
|
||||
"cyan": color.FgCyan,
|
||||
"white": color.FgWhite,
|
||||
|
||||
// attributes
|
||||
"reset": color.Reset,
|
||||
"bold": color.Bold,
|
||||
"faint": color.Faint,
|
||||
"italic": color.Italic,
|
||||
"underline": color.Underline,
|
||||
"blinkslow": color.BlinkSlow,
|
||||
"blinkrapid": color.BlinkRapid,
|
||||
"reversevideo": color.ReverseVideo,
|
||||
"concealed": color.Concealed,
|
||||
"crossedout": color.CrossedOut,
|
||||
|
||||
// foreground text colors
|
||||
"fgBlack": color.FgBlack,
|
||||
"fgRed": color.FgRed,
|
||||
"fgGreen": color.FgGreen,
|
||||
"fgYellow": color.FgYellow,
|
||||
"fgBlue": color.FgBlue,
|
||||
"fgMagenta": color.FgMagenta,
|
||||
"fgCyan": color.FgCyan,
|
||||
"fgWhite": color.FgWhite,
|
||||
|
||||
// foreground Hi-Intensity text colors
|
||||
"fgHiBlack": color.FgHiBlack,
|
||||
"fgHiRed": color.FgHiRed,
|
||||
"fgHiGreen": color.FgHiGreen,
|
||||
"fgHiYellow": color.FgHiYellow,
|
||||
"fgHiBlue": color.FgHiBlue,
|
||||
"fgHiMagenta": color.FgHiMagenta,
|
||||
"fgHiCyan": color.FgHiCyan,
|
||||
"fgHiWhite": color.FgHiWhite,
|
||||
|
||||
// background text colors
|
||||
"bgBlack": color.BgBlack,
|
||||
"bgRed": color.BgRed,
|
||||
"bgGreen": color.BgGreen,
|
||||
"bgYellow": color.BgYellow,
|
||||
"bgBlue": color.BgBlue,
|
||||
"bgMagenta": color.BgMagenta,
|
||||
"bgCyan": color.BgCyan,
|
||||
"bgWhite": color.BgWhite,
|
||||
|
||||
// background Hi-Intensity text colors
|
||||
"bgHiBlack": color.BgHiBlack,
|
||||
"bgHiRed": color.BgHiRed,
|
||||
"bgHiGreen": color.BgHiGreen,
|
||||
"bgHiYellow": color.BgHiYellow,
|
||||
"bgHiBlue": color.BgHiBlue,
|
||||
"bgHiMagenta": color.BgHiMagenta,
|
||||
"bgHiCyan": color.BgHiCyan,
|
||||
"bgHiWhite": color.BgHiWhite,
|
||||
}
|
||||
|
||||
// validColor will make sure the given color is actually allowed
|
||||
func validColor(c string) bool {
|
||||
_, ok := ValidColors[c]
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
func colorFunc(colors ...string) (func(format string, a ...interface{}) string, error) {
|
||||
if len(colors) == 0 {
|
||||
return fmt.Sprintf, nil
|
||||
}
|
||||
|
||||
attrib := make([]color.Attribute, len(colors))
|
||||
|
||||
for i, color := range colors {
|
||||
if !validColor(color) {
|
||||
return nil, fmt.Errorf("%s is not a valid color", color)
|
||||
}
|
||||
|
||||
attrib[i] = colorAttributeMap[color]
|
||||
}
|
||||
|
||||
return color.New(attrib...).SprintfFunc(), nil
|
||||
}
|
||||
-1191
File diff suppressed because it is too large.
Load diff
Vendored
+3
-4
@@ -1360,7 +1360,7 @@ github.com/opencloud-eu/icap-client
|
||||
# github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d
|
||||
## explicit; go 1.18
|
||||
github.com/opencloud-eu/libre-graph-api-go
|
||||
# github.com/opencloud-eu/reva/v2 v2.46.6
|
||||
# github.com/opencloud-eu/reva/v2 v2.46.8
|
||||
## explicit; go 1.25.0
|
||||
github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace
|
||||
github.com/opencloud-eu/reva/v2/cmd/revad/runtime
|
||||
@@ -1660,8 +1660,10 @@ github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher/natswatcher
|
||||
github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry
|
||||
github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng
|
||||
github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore
|
||||
github.com/opencloud-eu/reva/v2/pkg/storage/internal/goroutinelock
|
||||
github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs
|
||||
github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects
|
||||
github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk
|
||||
github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup
|
||||
github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata
|
||||
github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes
|
||||
@@ -2079,9 +2081,6 @@ github.com/testcontainers/testcontainers-go/wait
|
||||
# github.com/testcontainers/testcontainers-go/modules/opensearch v0.42.0
|
||||
## explicit; go 1.25.0
|
||||
github.com/testcontainers/testcontainers-go/modules/opensearch
|
||||
# github.com/theckman/yacspin v0.13.12
|
||||
## explicit; go 1.17
|
||||
github.com/theckman/yacspin
|
||||
# github.com/thejerf/suture/v4 v4.0.6
|
||||
## explicit; go 1.9
|
||||
github.com/thejerf/suture/v4
|
||||
|
||||
Reference in new issue
Block a user