mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-13 06:09:21 -04:00
api-test: add posixfs scan and consistency CLI tests (#3263)
* api-test: add posixfs scan and consistency CLI tests * fix after review
This commit is contained in:
Viktor Scharf authored and GitHub committed
1 parent
3a3cba602c
commit
97908ac6c1
3 files changed
+424
-5
No files matched your search
@@ -38,18 +38,27 @@ class CliContext implements Context {
|
||||
private SpacesContext $spacesContext;
|
||||
|
||||
/**
|
||||
* opencloud users storage path
|
||||
* opencloud storage root path (with $HOME/~ expanded for CI)
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public static function getUsersStoragePath(): string {
|
||||
public static function getStorageRootPath(): string {
|
||||
$path = getenv('OC_STORAGE_PATH') ?: '/var/lib/opencloud/storage/users';
|
||||
// need for CI
|
||||
$home = getenv('HOME');
|
||||
$path = preg_replace('#^~/#', $home . '/', $path);
|
||||
$path = str_replace('$HOME', $home, $path);
|
||||
|
||||
return rtrim($path, '/') . '/users';
|
||||
return rtrim($path, '/');
|
||||
}
|
||||
|
||||
/**
|
||||
* opencloud users storage path
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public static function getUsersStoragePath(): string {
|
||||
return self::getStorageRootPath() . '/users';
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -58,8 +67,7 @@ class CliContext implements Context {
|
||||
* @return string
|
||||
*/
|
||||
public static function getProjectsStoragePath(): string {
|
||||
$path = getenv('OC_STORAGE_PATH') ?: '/var/lib/opencloud/storage/users';
|
||||
return $path . '/projects';
|
||||
return self::getStorageRootPath() . '/projects';
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1039,4 +1047,261 @@ class CliContext implements Context {
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
* on-disk path of a project space root
|
||||
*
|
||||
* @param string $space
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
private function getProjectSpaceStoragePath(string $space): string {
|
||||
$spaceId = $this->spacesContext->getSpaceIdByName($this->featureContext->getAdminUsername(), $space);
|
||||
$spaceId = explode('$', $spaceId)[1];
|
||||
return $this->getProjectsStoragePath() . "/$spaceId";
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator scans the whole storage using the CLI')]
|
||||
public function theAdministratorScansTheWholeStorageUsingTheCli(): void {
|
||||
$body = [
|
||||
"command" => "posixfs scan"
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param string $folder
|
||||
* @param string $user
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator scans the folder :folder of user :user using the CLI')]
|
||||
public function theAdministratorScansTheFolderOfUserUsingTheCli(string $folder, string $user): void {
|
||||
$userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id');
|
||||
$storagePath = $this->getUsersStoragePath();
|
||||
$body = [
|
||||
"command" => "posixfs scan $storagePath/$userUuid/$folder"
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param string $space
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator scans the space :space using the CLI')]
|
||||
public function theAdministratorScansTheSpaceUsingTheCli(string $space): void {
|
||||
$body = [
|
||||
"command" => "posixfs scan " . $this->getProjectSpaceStoragePath($space)
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator checks the posixfs consistency using the CLI')]
|
||||
public function theAdministratorChecksThePosixfsConsistencyUsingTheCli(): void {
|
||||
$body = [
|
||||
"command" => "posixfs consistency"
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param string $flag
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator checks the posixfs consistency using the CLI with flag :flag')]
|
||||
public function theAdministratorChecksThePosixfsConsistencyUsingTheCliWithFlag(string $flag): void {
|
||||
$body = [
|
||||
"command" => "posixfs consistency $flag"
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param string $file
|
||||
* @param string $content
|
||||
* @param string $space
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator creates the file :file with content :content in the space :space on the POSIX filesystem')]
|
||||
public function theAdministratorCreatesFileInSpaceOnPosix(string $file, string $content, string $space): void {
|
||||
$fullPath = $this->getProjectSpaceStoragePath($space) . "/$file";
|
||||
$safeContent = escapeshellarg($content);
|
||||
$body = [
|
||||
"command" => "echo -n $safeContent > $fullPath",
|
||||
"raw" => true
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
$this->waitForPath($fullPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* reads xattrs from disk with getfattr, bypassing the driver's on-the-fly assimilation
|
||||
*
|
||||
* @param string $file
|
||||
* @param string $user
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator gets the extended attributes of file :file of user :user on the POSIX filesystem')]
|
||||
public function theAdministratorGetsExtendedAttributesOfFileOfUser(string $file, string $user): void {
|
||||
$userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id');
|
||||
$storagePath = $this->getUsersStoragePath();
|
||||
$body = [
|
||||
"command" => "getfattr -d $storagePath/$userUuid/$file",
|
||||
"raw" => true
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param string $file
|
||||
* @param string $space
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator gets the extended attributes of file :file in the space :space on the POSIX filesystem')]
|
||||
public function theAdministratorGetsExtendedAttributesOfFileInSpace(string $file, string $space): void {
|
||||
$body = [
|
||||
"command" => "getfattr -d " . $this->getProjectSpaceStoragePath($space) . "/$file",
|
||||
"raw" => true
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param string $attribute
|
||||
* @param string $file
|
||||
* @param string $user
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator gets the extended attribute :attribute of file :file of user :user on the POSIX filesystem')]
|
||||
public function theAdministratorGetsExtendedAttributeOfFileOfUser(string $attribute, string $file, string $user): void {
|
||||
$userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id');
|
||||
$storagePath = $this->getUsersStoragePath();
|
||||
$body = [
|
||||
"command" => "getfattr -n " . escapeshellarg($attribute) . " --only-values $storagePath/$userUuid/$file",
|
||||
"raw" => true
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param string $attribute
|
||||
* @param string $file
|
||||
* @param string $user
|
||||
* @param string $value
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator sets the extended attribute :attribute of file :file of user :user to :value on the POSIX filesystem')]
|
||||
public function theAdministratorSetsExtendedAttributeOfFileOfUser(
|
||||
string $attribute,
|
||||
string $file,
|
||||
string $user,
|
||||
string $value
|
||||
): void {
|
||||
$userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id');
|
||||
$storagePath = $this->getUsersStoragePath();
|
||||
$body = [
|
||||
"command" => "setfattr -n " . escapeshellarg($attribute)
|
||||
. " -v " . escapeshellarg($value) . " $storagePath/$userUuid/$file",
|
||||
"raw" => true
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param string $file
|
||||
* @param string $user
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator scans the file :file of user :user using the CLI')]
|
||||
public function theAdministratorScansTheFileOfUserUsingTheCli(string $file, string $user): void {
|
||||
$userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id');
|
||||
$storagePath = $this->getUsersStoragePath();
|
||||
$body = [
|
||||
"command" => "posixfs scan $storagePath/$userUuid/$file"
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param string $path
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator scans path :path using the CLI')]
|
||||
public function theAdministratorScansPathUsingTheCli(string $path): void {
|
||||
$body = [
|
||||
"command" => "posixfs scan $path"
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
* non-existing path first, so with "-E" the scan aborts before the valid folder
|
||||
*
|
||||
* @param string $folder
|
||||
* @param string $user
|
||||
* @param string $flag
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[When('the administrator scans a non-existing path and the folder :folder of user :user using the CLI with flag :flag')]
|
||||
public function theAdministratorScansNonExistingPathAndFolderUsingTheCli(
|
||||
string $folder,
|
||||
string $user,
|
||||
string $flag
|
||||
): void {
|
||||
$userUuid = $this->featureContext->getAttributeOfCreatedUser($user, 'id');
|
||||
$storagePath = $this->getUsersStoragePath();
|
||||
$nonExistingPath = "$storagePath/$userUuid/nonExistingPath";
|
||||
$validPath = "$storagePath/$userUuid/$folder";
|
||||
$flagPart = $flag !== "" ? "$flag " : "";
|
||||
$body = [
|
||||
"command" => "posixfs scan {$flagPart}$nonExistingPath $validPath"
|
||||
];
|
||||
$this->featureContext->setResponse(CliHelper::runCommand($body));
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
#[Then('the command should not be successful')]
|
||||
public function theCommandShouldNotBeSuccessful(): void {
|
||||
$response = $this->featureContext->getResponse();
|
||||
$this->featureContext->theHTTPStatusCodeShouldBe(200, '', $response);
|
||||
|
||||
$jsonResponse = $this->featureContext->getJsonDecodedResponse($response);
|
||||
|
||||
Assert::assertNotSame(
|
||||
0,
|
||||
$jsonResponse["exitCode"],
|
||||
"Expected command to fail with a non-zero exit code, but got 0. Message: " . $jsonResponse["message"]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
@env-config @skipOnOpencloud-decomposed-Storage @skipOnOpencloud-decomposeds3-Storage
|
||||
Feature: posixfs consistency CLI command
|
||||
As an administrator
|
||||
I want to check and repair the consistency of the posix filesystem metadata
|
||||
So that files with broken metadata (name, blobsize, parent id, ...) get fixed
|
||||
|
||||
# WATCH_FS=false so the watcher does not re-assimilate the corrupted metadata before the command runs
|
||||
Background:
|
||||
Given the config "STORAGE_USERS_POSIX_WATCH_FS" has been set to "false"
|
||||
And user "Alice" has been created with default attributes
|
||||
|
||||
|
||||
Scenario: consistency fixes a corrupted name attribute
|
||||
Given user "Alice" has uploaded file with content "content" to "textfile.txt"
|
||||
When the administrator sets the extended attribute "user.oc.name" of file "textfile.txt" of user "Alice" to "corrupted.txt" on the POSIX filesystem
|
||||
Then the command should be successful
|
||||
When the administrator gets the extended attribute "user.oc.name" of file "textfile.txt" of user "Alice" on the POSIX filesystem
|
||||
Then the command output should contain "corrupted.txt"
|
||||
When the administrator checks the posixfs consistency using the CLI
|
||||
Then the command should be successful
|
||||
And the command output should contain "Fixed name attribute"
|
||||
When the administrator gets the extended attribute "user.oc.name" of file "textfile.txt" of user "Alice" on the POSIX filesystem
|
||||
Then the command output should contain "textfile.txt"
|
||||
And the command output should not contain "corrupted.txt"
|
||||
|
||||
|
||||
Scenario: consistency fixes a corrupted blobsize attribute
|
||||
Given user "Alice" has uploaded file with content "content" to "textfile.txt"
|
||||
When the administrator sets the extended attribute "user.oc.blobsize" of file "textfile.txt" of user "Alice" to "0" on the POSIX filesystem
|
||||
Then the command should be successful
|
||||
When the administrator checks the posixfs consistency using the CLI
|
||||
Then the command should be successful
|
||||
And the command output should contain "Fixed blobsize"
|
||||
When the administrator gets the extended attribute "user.oc.blobsize" of file "textfile.txt" of user "Alice" on the POSIX filesystem
|
||||
Then the command output should contain "7"
|
||||
|
||||
|
||||
Scenario: consistency fixes corrupted checksums only with the --fix-checksums flag
|
||||
Given user "Alice" has uploaded file with content "content" to "textfile.txt"
|
||||
When the administrator sets the extended attribute "user.oc.cs.sha1" of file "textfile.txt" of user "Alice" to "corrupted" on the POSIX filesystem
|
||||
Then the command should be successful
|
||||
When the administrator checks the posixfs consistency using the CLI
|
||||
Then the command should be successful
|
||||
And the command output should not contain "Fixed checksum"
|
||||
When the administrator checks the posixfs consistency using the CLI with flag "--fix-checksums"
|
||||
Then the command should be successful
|
||||
And the command output should contain "Fixed checksum"
|
||||
When the administrator checks the posixfs consistency using the CLI with flag "--fix-checksums"
|
||||
Then the command should be successful
|
||||
And the command output should not contain "Fixed checksum"
|
||||
@@ -0,0 +1,104 @@
|
||||
@env-config @skipOnOpencloud-decomposed-Storage @skipOnOpencloud-decomposeds3-Storage
|
||||
Feature: posixfs scan CLI command
|
||||
As an administrator
|
||||
I want to scan the posix filesystem for files that were added outside of OpenCloud
|
||||
So that their metadata (ID cache and file metadata) gets assimilated and they become usable
|
||||
|
||||
# WATCH_FS=false so only the scan command assimilates the files; proof read via getfattr from disk
|
||||
Background:
|
||||
Given the config "STORAGE_USERS_POSIX_WATCH_FS" has been set to "false"
|
||||
And user "Alice" has been created with default attributes
|
||||
|
||||
|
||||
Scenario: scan the whole storage assimilates files added directly to the personal and project spaces
|
||||
Given the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API
|
||||
And user "Alice" has created a space "projectSpace1" with the default quota using the Graph API
|
||||
And user "Alice" has created a space "projectSpace2" with the default quota using the Graph API
|
||||
And the administrator has created the file "scanned.txt" with content "personal" for user "Alice" on the POSIX filesystem
|
||||
And the administrator creates the file "inProject1.txt" with content "project one" in the space "projectSpace1" on the POSIX filesystem
|
||||
And the administrator creates the file "inProject2.txt" with content "project two" in the space "projectSpace2" on the POSIX filesystem
|
||||
When the administrator gets the extended attributes of file "scanned.txt" of user "Alice" on the POSIX filesystem
|
||||
Then the command output should not contain "user.oc.id"
|
||||
And the administrator gets the extended attributes of file "inProject1.txt" in the space "projectSpace1" on the POSIX filesystem
|
||||
And the command output should not contain "user.oc.id"
|
||||
And the administrator gets the extended attributes of file "inProject2.txt" in the space "projectSpace2" on the POSIX filesystem
|
||||
And the command output should not contain "user.oc.id"
|
||||
When the administrator scans the whole storage using the CLI
|
||||
Then the command should be successful
|
||||
And the command output should contain "Scan completed successfully."
|
||||
When the administrator gets the extended attributes of file "scanned.txt" of user "Alice" on the POSIX filesystem
|
||||
Then the command output should contain "user.oc.id"
|
||||
When the administrator gets the extended attributes of file "inProject1.txt" in the space "projectSpace1" on the POSIX filesystem
|
||||
Then the command output should contain "user.oc.id"
|
||||
When the administrator gets the extended attributes of file "inProject2.txt" in the space "projectSpace2" on the POSIX filesystem
|
||||
Then the command output should contain "user.oc.id"
|
||||
And as "Alice" the final content of file "scanned.txt" should be "personal"
|
||||
And using spaces DAV path
|
||||
And for user "Alice" the space "projectSpace1" should contain these entries:
|
||||
| inProject1.txt |
|
||||
And for user "Alice" the space "projectSpace2" should contain these entries:
|
||||
| inProject2.txt |
|
||||
|
||||
|
||||
Scenario: scan a single folder assimilates the files inside it
|
||||
Given the administrator creates the folder "scanFolder" for user "Alice" on the POSIX filesystem
|
||||
And the administrator has created the file "scanFolder/inside.txt" with content "inside content" for user "Alice" on the POSIX filesystem
|
||||
When the administrator gets the extended attributes of file "scanFolder/inside.txt" of user "Alice" on the POSIX filesystem
|
||||
Then the command output should not contain "user.oc.id"
|
||||
And the administrator scans the folder "scanFolder" of user "Alice" using the CLI
|
||||
And the command should be successful
|
||||
And the command output should contain "Scan completed successfully."
|
||||
When the administrator gets the extended attributes of file "scanFolder/inside.txt" of user "Alice" on the POSIX filesystem
|
||||
Then the command output should contain "user.oc.id"
|
||||
And as "Alice" the final content of file "scanFolder/inside.txt" should be "inside content"
|
||||
|
||||
|
||||
Scenario: scan a single project space assimilates only that space
|
||||
Given the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API
|
||||
And user "Alice" has created a space "projectSpace1" with the default quota using the Graph API
|
||||
And user "Alice" has created a space "projectSpace2" with the default quota using the Graph API
|
||||
And the administrator creates the file "inProject1.txt" with content "project one" in the space "projectSpace1" on the POSIX filesystem
|
||||
And the administrator creates the file "inProject2.txt" with content "project two" in the space "projectSpace2" on the POSIX filesystem
|
||||
When the administrator scans the space "projectSpace1" using the CLI
|
||||
Then the command should be successful
|
||||
And the command output should contain "Scan completed successfully."
|
||||
And the administrator gets the extended attributes of file "inProject1.txt" in the space "projectSpace1" on the POSIX filesystem
|
||||
And the command output should contain "user.oc.id"
|
||||
And the administrator gets the extended attributes of file "inProject2.txt" in the space "projectSpace2" on the POSIX filesystem
|
||||
And the command output should not contain "user.oc.id"
|
||||
And using spaces DAV path
|
||||
And for user "Alice" the space "projectSpace1" should contain these entries:
|
||||
| inProject1.txt |
|
||||
|
||||
|
||||
Scenario: scan a single regular file assimilates only that file
|
||||
Given the administrator has created the file "singleFile.txt" with content "single" for user "Alice" on the POSIX filesystem
|
||||
And the administrator has created the file "otherFile.txt" with content "other" for user "Alice" on the POSIX filesystem
|
||||
When the administrator scans the file "singleFile.txt" of user "Alice" using the CLI
|
||||
Then the command should be successful
|
||||
And the command output should contain "Scan completed successfully."
|
||||
And the administrator gets the extended attributes of file "singleFile.txt" of user "Alice" on the POSIX filesystem
|
||||
And the command output should contain "user.oc.id"
|
||||
And the administrator gets the extended attributes of file "otherFile.txt" of user "Alice" on the POSIX filesystem
|
||||
And the command output should not contain "user.oc.id"
|
||||
And as "Alice" the final content of file "singleFile.txt" should be "single"
|
||||
|
||||
|
||||
Scenario: scanning a path outside the storage root fails
|
||||
When the administrator scans path "/tmp" using the CLI
|
||||
Then the command should not be successful
|
||||
And the command output should contain "does not appear to be inside a posixfs storage"
|
||||
|
||||
|
||||
Scenario Outline: the halt-on-error flag controls whether the scan continues after an error
|
||||
Given the administrator creates the folder "validFolder" for user "Alice" on the POSIX filesystem
|
||||
And the administrator has created the file "validFolder/inside.txt" with content "keep going" for user "Alice" on the POSIX filesystem
|
||||
When the administrator scans a non-existing path and the folder "validFolder" of user "Alice" using the CLI with flag "<flag>"
|
||||
Then the command should not be successful
|
||||
And the command output should contain "<message>"
|
||||
And the administrator gets the extended attributes of file "validFolder/inside.txt" of user "Alice" on the POSIX filesystem
|
||||
And the command output <shouldOrNot> contain "user.oc.id"
|
||||
Examples:
|
||||
| flag | message | shouldOrNot |
|
||||
| | scan completed with 1 error | should |
|
||||
| -E | scan aborted with 1 error | should not |
|
||||
Reference in new issue
Block a user