From eadf736e98f31bfb3a2765acd178748b321b34a1 Mon Sep 17 00:00:00 2001 From: Sebastian Stenzel Date: Wed, 16 Dec 2015 18:37:08 +0100 Subject: [PATCH] added new "shortening layer" responsible for shortening long file names the crypto layer is no longer resposible for the postprocessing of long names, as this is an unrelated task without any security implications --- main/crypto-layer/pom.xml | 4 + .../crypto/engine/impl/CryptorImpl.java | 15 +-- .../engine/impl/FilenameCryptorImpl.java | 13 +- .../org/cryptomator/crypto/fs/CryptoFile.java | 3 +- .../crypto/fs/CryptoFileSystem.java | 7 -- .../cryptomator/crypto/fs/CryptoFolder.java | 15 +-- .../org/cryptomator/crypto/fs/CryptoNode.java | 8 -- .../crypto/engine/impl/CryptorImplTest.java | 44 +++++++ .../crypto/fs/CryptoFileSystemTest.java | 2 +- .../fs/EncryptAndShortenIntegrationTest.java | 62 ++++++++++ main/pom.xml | 6 + main/shortening-layer/.gitignore | 1 + main/shortening-layer/pom.xml | 58 +++++++++ .../shortening/FilenameShortener.java | 111 +++++++++++++++++ .../shortening/ShorteningFile.java | 38 ++++++ .../shortening/ShorteningFileSystem.java | 29 +++++ .../shortening/ShorteningFolder.java | 115 ++++++++++++++++++ .../shortening/ShorteningNode.java | 68 +++++++++++ .../cryptomator/shortening/package-info.java | 6 + .../shortening/ShorteningFileSystemTest.java | 115 ++++++++++++++++++ 20 files changed, 675 insertions(+), 45 deletions(-) create mode 100644 main/crypto-layer/src/test/java/org/cryptomator/crypto/fs/EncryptAndShortenIntegrationTest.java create mode 100644 main/shortening-layer/.gitignore create mode 100644 main/shortening-layer/pom.xml create mode 100644 main/shortening-layer/src/main/java/org/cryptomator/shortening/FilenameShortener.java create mode 100644 main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFile.java create mode 100644 main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFileSystem.java create mode 100644 main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFolder.java create mode 100644 main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningNode.java create mode 100644 main/shortening-layer/src/main/java/org/cryptomator/shortening/package-info.java create mode 100644 main/shortening-layer/src/test/java/org/cryptomator/shortening/ShorteningFileSystemTest.java diff --git a/main/crypto-layer/pom.xml b/main/crypto-layer/pom.xml index f0514bb3b..ca0a54e35 100644 --- a/main/crypto-layer/pom.xml +++ b/main/crypto-layer/pom.xml @@ -27,6 +27,10 @@ org.cryptomator filesystem-api + + org.cryptomator + shortening-layer + diff --git a/main/crypto-layer/src/main/java/org/cryptomator/crypto/engine/impl/CryptorImpl.java b/main/crypto-layer/src/main/java/org/cryptomator/crypto/engine/impl/CryptorImpl.java index fb62c6cf2..330d5ce39 100644 --- a/main/crypto-layer/src/main/java/org/cryptomator/crypto/engine/impl/CryptorImpl.java +++ b/main/crypto-layer/src/main/java/org/cryptomator/crypto/engine/impl/CryptorImpl.java @@ -46,17 +46,18 @@ public class CryptorImpl implements Cryptor { @Override public FilenameCryptor getFilenameCryptor() { // lazy initialization pattern as proposed here http://stackoverflow.com/a/30247202/4014509 - FilenameCryptor cryptor = filenameCryptor.get(); - if (cryptor == null) { - cryptor = new FilenameCryptorImpl(encryptionKey, macKey); - if (filenameCryptor.compareAndSet(null, cryptor)) { - return cryptor; + final FilenameCryptor existingCryptor = filenameCryptor.get(); + if (existingCryptor != null) { + return existingCryptor; + } else { + final FilenameCryptorImpl newCryptor = new FilenameCryptorImpl(encryptionKey, macKey); + if (filenameCryptor.compareAndSet(null, newCryptor)) { + return newCryptor; } else { // CAS failed: other thread set an object + newCryptor.destroy(); return filenameCryptor.get(); } - } else { - return cryptor; } } diff --git a/main/crypto-layer/src/main/java/org/cryptomator/crypto/engine/impl/FilenameCryptorImpl.java b/main/crypto-layer/src/main/java/org/cryptomator/crypto/engine/impl/FilenameCryptorImpl.java index ea1dfe96d..508f39f27 100644 --- a/main/crypto-layer/src/main/java/org/cryptomator/crypto/engine/impl/FilenameCryptorImpl.java +++ b/main/crypto-layer/src/main/java/org/cryptomator/crypto/engine/impl/FilenameCryptorImpl.java @@ -15,7 +15,6 @@ import java.security.NoSuchAlgorithmException; import javax.crypto.AEADBadTagException; import javax.crypto.SecretKey; -import javax.security.auth.DestroyFailedException; import org.apache.commons.codec.binary.Base32; import org.apache.commons.codec.binary.BaseNCodec; @@ -26,7 +25,7 @@ import org.cryptomator.siv.SivMode; class FilenameCryptorImpl implements FilenameCryptor { private static final BaseNCodec BASE32 = new Base32(); - private static final ThreadLocal SHA256 = new ThreadLocalSha256(); + private static final ThreadLocal SHA1 = new ThreadLocalSha1(); private static final SivMode AES_SIV = new SivMode(); private final SecretKey encryptionKey; @@ -44,7 +43,7 @@ class FilenameCryptorImpl implements FilenameCryptor { public String hashDirectoryId(String cleartextDirectoryId) { final byte[] cleartextBytes = cleartextDirectoryId.getBytes(StandardCharsets.UTF_8); byte[] encryptedBytes = AES_SIV.encrypt(encryptionKey, macKey, cleartextBytes); - final byte[] hashedBytes = SHA256.get().digest(encryptedBytes); + final byte[] hashedBytes = SHA1.get().digest(encryptedBytes); return BASE32.encodeAsString(hashedBytes); } @@ -66,14 +65,14 @@ class FilenameCryptorImpl implements FilenameCryptor { } } - private static class ThreadLocalSha256 extends ThreadLocal { + private static class ThreadLocalSha1 extends ThreadLocal { @Override protected MessageDigest initialValue() { try { - return MessageDigest.getInstance("SHA-256"); + return MessageDigest.getInstance("SHA-1"); } catch (NoSuchAlgorithmException e) { - throw new AssertionError("SHA-256 exists in every JVM"); + throw new AssertionError("SHA-1 exists in every JVM"); } } @@ -88,7 +87,7 @@ class FilenameCryptorImpl implements FilenameCryptor { /* ======================= destruction ======================= */ @Override - public void destroy() throws DestroyFailedException { + public void destroy() { TheDestroyer.destroyQuietly(encryptionKey); TheDestroyer.destroyQuietly(macKey); } diff --git a/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFile.java b/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFile.java index b79691c13..37d0267ba 100644 --- a/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFile.java +++ b/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFile.java @@ -26,9 +26,8 @@ public class CryptoFile extends CryptoNode implements File { super(parent, name, cryptor); } - @Override String encryptedName() { - return name() + FILE_EXT; + return cryptor.getFilenameCryptor().encryptFilename(name()) + FILE_EXT; } @Override diff --git a/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFileSystem.java b/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFileSystem.java index bf44affde..e2471c91e 100644 --- a/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFileSystem.java +++ b/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFileSystem.java @@ -29,7 +29,6 @@ public class CryptoFileSystem extends CryptoFolder implements FileSystem { private static final Logger LOG = LoggerFactory.getLogger(CryptoFileSystem.class); private static final String DATA_ROOT_DIR = "d"; - private static final String METADATA_ROOT_DIR = "m"; private static final String ROOT_DIR_FILE = "root"; private static final String MASTERKEY_FILENAME = "masterkey.cryptomator"; private static final String MASTERKEY_BACKUP_FILENAME = "masterkey.cryptomator.bkup"; @@ -96,11 +95,6 @@ public class CryptoFileSystem extends CryptoFolder implements FileSystem { return physicalRoot.folder(DATA_ROOT_DIR); } - @Override - Folder physicalMetadataRoot() { - return physicalRoot.folder(METADATA_ROOT_DIR); - } - @Override public Optional parent() { return Optional.empty(); @@ -119,7 +113,6 @@ public class CryptoFileSystem extends CryptoFolder implements FileSystem { @Override public void create(FolderCreateMode mode) { physicalDataRoot().create(mode); - physicalMetadataRoot().create(mode); final File dirFile = physicalFile(); final String directoryId = getDirectoryId(); try (WritableFile writable = dirFile.openWritable(1, TimeUnit.SECONDS)) { diff --git a/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFolder.java b/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFolder.java index c86c89885..13b8ae220 100644 --- a/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFolder.java +++ b/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoFolder.java @@ -12,8 +12,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -21,7 +19,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; -import org.apache.commons.codec.binary.Base32; import org.apache.commons.lang3.StringUtils; import org.cryptomator.crypto.engine.Cryptor; import org.cryptomator.filesystem.File; @@ -41,9 +38,8 @@ class CryptoFolder extends CryptoNode implements Folder { super(parent, name, cryptor); } - @Override String encryptedName() { - return name() + FILE_EXT; + return cryptor.getFilenameCryptor().encryptFilename(name()) + FILE_EXT; } protected String getDirectoryId() { @@ -72,14 +68,7 @@ class CryptoFolder extends CryptoNode implements Folder { } Folder physicalFolder() { - final String encryptedThenHashedDirId; - try { - final byte[] hash = MessageDigest.getInstance("SHA-1").digest(getDirectoryId().getBytes()); - encryptedThenHashedDirId = new Base32().encodeAsString(hash); - } catch (NoSuchAlgorithmException e) { - throw new AssertionError("SHA-1 exists in every JVM"); - } - // TODO actual encryption + final String encryptedThenHashedDirId = cryptor.getFilenameCryptor().hashDirectoryId(getDirectoryId()); return physicalDataRoot().folder(encryptedThenHashedDirId.substring(0, 2)).folder(encryptedThenHashedDirId.substring(2)); } diff --git a/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoNode.java b/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoNode.java index de7ec1c65..10feba12a 100644 --- a/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoNode.java +++ b/main/crypto-layer/src/main/java/org/cryptomator/crypto/fs/CryptoNode.java @@ -30,10 +30,6 @@ abstract class CryptoNode implements Node { return parent.physicalDataRoot(); } - Folder physicalMetadataRoot() { - return parent.physicalMetadataRoot(); - } - @Override public Optional parent() { return Optional.of(parent); @@ -44,10 +40,6 @@ abstract class CryptoNode implements Node { return name; } - String encryptedName() { - return name(); - } - @Override public boolean exists() { return parent.children().anyMatch(node -> node.equals(this)); diff --git a/main/crypto-layer/src/test/java/org/cryptomator/crypto/engine/impl/CryptorImplTest.java b/main/crypto-layer/src/test/java/org/cryptomator/crypto/engine/impl/CryptorImplTest.java index c6cee4b48..f4127a45f 100644 --- a/main/crypto-layer/src/test/java/org/cryptomator/crypto/engine/impl/CryptorImplTest.java +++ b/main/crypto-layer/src/test/java/org/cryptomator/crypto/engine/impl/CryptorImplTest.java @@ -11,8 +11,10 @@ package org.cryptomator.crypto.engine.impl; import java.io.IOException; import java.security.SecureRandom; import java.util.Arrays; +import java.util.concurrent.atomic.AtomicReference; import org.cryptomator.crypto.engine.Cryptor; +import org.cryptomator.crypto.engine.FilenameCryptor; import org.junit.Assert; import org.junit.Test; @@ -50,4 +52,46 @@ public class CryptorImplTest { Assert.assertArrayEquals(expectedMasterKey.getBytes(), masterkeyFile); } + @Test + public void testGetFilenameCryptorAfterUnlocking() { + final String testMasterKey = "{\"version\":3,\"scryptSalt\":\"AAAAAAAAAAA=\",\"scryptCostParam\":2,\"scryptBlockSize\":8," // + + "\"primaryMasterKey\":\"mM+qoQ+o0qvPTiDAZYt+flaC3WbpNAx1sTXaUzxwpy0M9Ctj6Tih/Q==\"," // + + "\"hmacMasterKey\":\"mM+qoQ+o0qvPTiDAZYt+flaC3WbpNAx1sTXaUzxwpy0M9Ctj6Tih/Q==\"}"; + final Cryptor cryptor = new CryptorImpl(RANDOM_MOCK); + cryptor.readKeysFromMasterkeyFile(testMasterKey.getBytes(), "asd"); + Assert.assertNotNull(cryptor.getFilenameCryptor()); + } + + @Test(expected = RuntimeException.class) + public void testGetFilenameCryptorBeforeUnlocking() { + final Cryptor cryptor = new CryptorImpl(RANDOM_MOCK); + cryptor.getFilenameCryptor(); + } + + @Test + public void testConcurrentGetFilenameCryptor() throws InterruptedException { + final String testMasterKey = "{\"version\":3,\"scryptSalt\":\"AAAAAAAAAAA=\",\"scryptCostParam\":2,\"scryptBlockSize\":8," // + + "\"primaryMasterKey\":\"mM+qoQ+o0qvPTiDAZYt+flaC3WbpNAx1sTXaUzxwpy0M9Ctj6Tih/Q==\"," // + + "\"hmacMasterKey\":\"mM+qoQ+o0qvPTiDAZYt+flaC3WbpNAx1sTXaUzxwpy0M9Ctj6Tih/Q==\"}"; + final Cryptor cryptor = new CryptorImpl(RANDOM_MOCK); + cryptor.readKeysFromMasterkeyFile(testMasterKey.getBytes(), "asd"); + + final AtomicReference receivedByT1 = new AtomicReference<>(); + final Thread t1 = new Thread(() -> { + receivedByT1.set(cryptor.getFilenameCryptor()); + }); + + final AtomicReference receivedByT2 = new AtomicReference<>(); + final Thread t2 = new Thread(() -> { + receivedByT2.set(cryptor.getFilenameCryptor()); + }); + t1.start(); + t2.start(); + t1.join(); + t2.join(); + // It is not guaranteed, both threads will enter getFilenameCryptor() exactly simultaneously. (But logging shows it is very likely) + // In any case both threads should receive the same FilenameCryptor + Assert.assertSame(receivedByT1.get(), receivedByT2.get()); + } + } diff --git a/main/crypto-layer/src/test/java/org/cryptomator/crypto/fs/CryptoFileSystemTest.java b/main/crypto-layer/src/test/java/org/cryptomator/crypto/fs/CryptoFileSystemTest.java index 20b8bfa29..276f5c171 100644 --- a/main/crypto-layer/src/test/java/org/cryptomator/crypto/fs/CryptoFileSystemTest.java +++ b/main/crypto-layer/src/test/java/org/cryptomator/crypto/fs/CryptoFileSystemTest.java @@ -49,7 +49,7 @@ public class CryptoFileSystemTest { Assert.assertTrue(masterkeyBkupFile.exists()); fs.create(FolderCreateMode.INCLUDING_PARENTS); Assert.assertTrue(physicalDataRoot.exists()); - Assert.assertEquals(4, physicalFs.children().count()); // d + m + masterkey.cryptomator + masterkey.cryptomator.bkup + Assert.assertEquals(3, physicalFs.children().count()); // d + masterkey.cryptomator + masterkey.cryptomator.bkup Assert.assertEquals(1, physicalDataRoot.files().count()); // ROOT file Assert.assertEquals(1, physicalDataRoot.folders().count()); // ROOT directory diff --git a/main/crypto-layer/src/test/java/org/cryptomator/crypto/fs/EncryptAndShortenIntegrationTest.java b/main/crypto-layer/src/test/java/org/cryptomator/crypto/fs/EncryptAndShortenIntegrationTest.java new file mode 100644 index 000000000..ee05654e3 --- /dev/null +++ b/main/crypto-layer/src/test/java/org/cryptomator/crypto/fs/EncryptAndShortenIntegrationTest.java @@ -0,0 +1,62 @@ +package org.cryptomator.crypto.fs; + +import java.security.SecureRandom; +import java.util.Arrays; + +import org.cryptomator.crypto.engine.Cryptor; +import org.cryptomator.crypto.engine.impl.CryptorImpl; +import org.cryptomator.filesystem.FileSystem; +import org.cryptomator.filesystem.Folder; +import org.cryptomator.filesystem.FolderCreateMode; +import org.cryptomator.filesystem.Node; +import org.cryptomator.filesystem.inmem.InMemoryFileSystem; +import org.cryptomator.shortening.ShorteningFileSystem; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EncryptAndShortenIntegrationTest { + + private static final Logger LOG = LoggerFactory.getLogger(EncryptAndShortenIntegrationTest.class); + + private static final SecureRandom RANDOM_MOCK = new SecureRandom() { + + private static final long serialVersionUID = 1505563778398085504L; + + @Override + public void nextBytes(byte[] bytes) { + Arrays.fill(bytes, (byte) 0x00); + } + + }; + + @Test + public void testEncryptionOfLongFolderNames() { + final FileSystem physicalFs = new InMemoryFileSystem(); + final FileSystem shorteningFs = new ShorteningFileSystem(physicalFs, physicalFs.folder("m"), 70); + final Cryptor cryptor = new CryptorImpl(RANDOM_MOCK); + cryptor.randomizeMasterkey(); + final FileSystem fs = new CryptoFileSystem(shorteningFs, cryptor, "foo"); + fs.create(FolderCreateMode.FAIL_IF_PARENT_IS_MISSING); + final Folder shortFolder = fs.folder("normal folder name"); + shortFolder.create(FolderCreateMode.FAIL_IF_PARENT_IS_MISSING); + final Folder longFolder = fs.folder("this will be a long filename after encryption"); + longFolder.create(FolderCreateMode.FAIL_IF_PARENT_IS_MISSING); + + // the long name will produce a metadata file on the physical layer: + LOG.debug("Physical file system:\n" + DirectoryPrinter.print(physicalFs)); + Assert.assertEquals(1, physicalFs.folder("m").folders().count()); + + // on the second layer all .lng files are resolved to their actual names: + LOG.debug("Unlimited filename length:\n" + DirectoryPrinter.print(shorteningFs)); + DirectoryWalker.walk(shorteningFs, node -> { + Assert.assertFalse(node.name().endsWith(".lng")); + }); + + // on the third (cleartext layer) we have cleartext names on the root level: + LOG.debug("Cleartext files:\n" + DirectoryPrinter.print(fs)); + Assert.assertArrayEquals(new String[] {"normal folder name", "this will be a long filename after encryption"}, fs.folders().map(Node::name).sorted().toArray()); + } + +} diff --git a/main/pom.xml b/main/pom.xml index a8ac6baba..d759f2fd8 100644 --- a/main/pom.xml +++ b/main/pom.xml @@ -64,6 +64,11 @@ filesystem-inmemory ${project.version} + + org.cryptomator + shortening-layer + ${project.version} + org.cryptomator crypto-layer @@ -215,6 +220,7 @@ crypto-aes core ui + shortening-layer diff --git a/main/shortening-layer/.gitignore b/main/shortening-layer/.gitignore new file mode 100644 index 000000000..b83d22266 --- /dev/null +++ b/main/shortening-layer/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/main/shortening-layer/pom.xml b/main/shortening-layer/pom.xml new file mode 100644 index 000000000..01cfd1366 --- /dev/null +++ b/main/shortening-layer/pom.xml @@ -0,0 +1,58 @@ + + + + 4.0.0 + + org.cryptomator + main + 0.11.0-SNAPSHOT + + shortening-layer + Cryptomator name shortening filesystem layer + + + + org.cryptomator + filesystem-api + + + + + org.apache.commons + commons-lang3 + + + commons-codec + commons-codec + + + + + com.fasterxml.jackson.core + jackson-databind + + + + + org.cryptomator + filesystem-inmemory + test + + + + + + + org.jacoco + jacoco-maven-plugin + + + + \ No newline at end of file diff --git a/main/shortening-layer/src/main/java/org/cryptomator/shortening/FilenameShortener.java b/main/shortening-layer/src/main/java/org/cryptomator/shortening/FilenameShortener.java new file mode 100644 index 000000000..10286b8b1 --- /dev/null +++ b/main/shortening-layer/src/main/java/org/cryptomator/shortening/FilenameShortener.java @@ -0,0 +1,111 @@ +package org.cryptomator.shortening; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.commons.codec.binary.Base32; +import org.apache.commons.codec.binary.BaseNCodec; +import org.cryptomator.filesystem.File; +import org.cryptomator.filesystem.Folder; +import org.cryptomator.filesystem.FolderCreateMode; +import org.cryptomator.filesystem.ReadableFile; +import org.cryptomator.filesystem.WritableFile; + +class FilenameShortener { + + private static final String LONG_NAME_FILE_EXT = ".lng"; + private static final ThreadLocal SHA1 = new ThreadLocalSha1(); + private static final BaseNCodec BASE32 = new Base32(); + private final Folder metadataRoot; + private final int threshold; + + public FilenameShortener(Folder metadataRoot, int threshold) { + this.metadataRoot = metadataRoot; + this.threshold = threshold; + } + + public String inflate(String shortName) { + if (shortName.endsWith(LONG_NAME_FILE_EXT)) { + return loadMapping(shortName); + } else { + return shortName; + } + } + + public String deflate(String longName) { + if (longName.length() < threshold) { + return longName; + } else { + final byte[] hashBytes = SHA1.get().digest(longName.getBytes()); + final String hash = BASE32.encodeAsString(hashBytes); + return hash + LONG_NAME_FILE_EXT; + } + } + + public boolean isShortened(String name) { + return name.endsWith(LONG_NAME_FILE_EXT); + } + + public void saveMapping(String longName, String shortName) { + final File mappingFile = mappingFile(shortName); + if (!mappingFile.exists()) { + mappingFile.parent().get().create(FolderCreateMode.INCLUDING_PARENTS); + try (WritableFile writable = mappingFile.openWritable(1, TimeUnit.SECONDS)) { + writable.write(ByteBuffer.wrap(longName.getBytes(StandardCharsets.UTF_8))); + } catch (TimeoutException e) { + throw new UncheckedIOException(new IOException("Failed to lock mapping file in time. " + mappingFile, e)); + } + } + } + + private File mappingFile(String deflated) { + final Folder folder = metadataRoot.folder(deflated.substring(0, 2)).folder(deflated.substring(2, 4)); + return folder.file(deflated); + } + + private String loadMapping(String shortName) { + final File mappingFile = mappingFile(shortName); + if (!mappingFile.exists()) { + throw new UncheckedIOException(new FileNotFoundException("Mapping file not found " + mappingFile)); + } else { + try (ReadableFile readable = mappingFile.openReadable(1, TimeUnit.SECONDS)) { + // TODO buffer might be to small + final ByteBuffer buf = ByteBuffer.allocate(1024); + readable.read(buf); + buf.flip(); + final byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } catch (TimeoutException e) { + throw new UncheckedIOException(new IOException("Failed to lock mapping file in time. " + mappingFile, e)); + } + } + } + + private static class ThreadLocalSha1 extends ThreadLocal { + + @Override + protected MessageDigest initialValue() { + try { + return MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw new AssertionError("SHA-1 exists in every JVM"); + } + } + + @Override + public MessageDigest get() { + final MessageDigest messageDigest = super.get(); + messageDigest.reset(); + return messageDigest; + } + } + +} diff --git a/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFile.java b/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFile.java new file mode 100644 index 000000000..75c4680e5 --- /dev/null +++ b/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFile.java @@ -0,0 +1,38 @@ +package org.cryptomator.shortening; + +import java.io.UncheckedIOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.cryptomator.filesystem.File; +import org.cryptomator.filesystem.ReadableFile; +import org.cryptomator.filesystem.WritableFile; + +class ShorteningFile extends ShorteningNodeimplements File { + + private final FilenameShortener shortener; + + public ShorteningFile(ShorteningFolder parent, File delegate, String longName, FilenameShortener shortener) { + super(parent, delegate, longName); + this.shortener = shortener; + } + + @Override + public ReadableFile openReadable(long timeout, TimeUnit unit) throws UncheckedIOException, TimeoutException { + return delegate.openReadable(timeout, unit); + } + + @Override + public WritableFile openWritable(long timeout, TimeUnit unit) throws UncheckedIOException, TimeoutException { + if (shortener.isShortened(shortName())) { + shortener.saveMapping(name(), shortName()); + } + return delegate.openWritable(timeout, unit); + } + + @Override + public String toString() { + return name(); + } + +} diff --git a/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFileSystem.java b/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFileSystem.java new file mode 100644 index 000000000..3923d1837 --- /dev/null +++ b/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFileSystem.java @@ -0,0 +1,29 @@ +package org.cryptomator.shortening; + +import java.util.Optional; + +import org.cryptomator.filesystem.FileSystem; +import org.cryptomator.filesystem.Folder; + +public class ShorteningFileSystem extends ShorteningFolder implements FileSystem { + + public ShorteningFileSystem(Folder root, Folder metadataRoot, int threshold) { + super(null, root, "", metadataRoot, new FilenameShortener(metadataRoot, threshold)); + } + + @Override + public Optional parent() { + return Optional.empty(); + } + + @Override + public boolean exists() { + return true; + } + + @Override + public void delete() { + // no-op. + } + +} diff --git a/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFolder.java b/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFolder.java new file mode 100644 index 000000000..df0295c33 --- /dev/null +++ b/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningFolder.java @@ -0,0 +1,115 @@ +package org.cryptomator.shortening; + +import java.io.FileNotFoundException; +import java.io.UncheckedIOException; +import java.nio.file.FileAlreadyExistsException; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import org.cryptomator.filesystem.File; +import org.cryptomator.filesystem.Folder; +import org.cryptomator.filesystem.FolderCreateMode; +import org.cryptomator.filesystem.Node; + +class ShorteningFolder extends ShorteningNodeimplements Folder { + + private final Folder metadataRoot; + private final FilenameShortener shortener; + + public ShorteningFolder(ShorteningFolder parent, Folder delegate, String longName, Folder metadataRoot, FilenameShortener shortener) { + super(parent, delegate, longName); + this.metadataRoot = metadataRoot; + this.shortener = shortener; + } + + @Override + public Stream children() { + return Stream.concat(this.files(), this.folders()); + } + + private ShorteningFile existingFile(File original) { + final String longName = shortener.inflate(original.name()); + return new ShorteningFile(this, original, longName, shortener); + } + + @Override + public File file(String name) { + final File original = delegate.file(shortener.deflate(name)); + if (metadataRoot.equals(original)) { // comparing apples and oranges, but we don't know if the underlying fs distinguishes files and folders... + throw new UncheckedIOException("'" + name + "' is a reserved name.", new FileAlreadyExistsException(name)); + } + return new ShorteningFile(this, original, name, shortener); + } + + @Override + public Stream files() throws UncheckedIOException { + return delegate.files().map(this::existingFile); + } + + private ShorteningFolder existingFolder(Folder original) { + final String longName = shortener.inflate(original.name()); + return new ShorteningFolder(this, original, longName, metadataRoot, shortener); + } + + @Override + public Folder folder(String name) { + final Folder original = delegate.folder(shortener.deflate(name)); + if (metadataRoot.equals(original)) { + throw new UncheckedIOException("'" + name + "' is a reserved name.", new FileAlreadyExistsException(name)); + } + return new ShorteningFolder(this, original, name, metadataRoot, shortener); + } + + @Override + public Stream folders() { + // if metadataRoot is inside our filesystem, we must filter it out: + final Predicate equalsMetadataRoot = (Node node) -> metadataRoot.equals(node); + return delegate.folders().filter(equalsMetadataRoot.negate()).map(this::existingFolder); + } + + @Override + public void create(FolderCreateMode mode) { + if (!parent().get().exists() && FolderCreateMode.FAIL_IF_PARENT_IS_MISSING.equals(mode)) { + throw new UncheckedIOException(new FileNotFoundException(parent().get().name())); + } else if (!parent().get().exists() && FolderCreateMode.INCLUDING_PARENTS.equals(mode)) { + parent().get().create(mode); + } + assert parent().get().exists(); + if (shortener.isShortened(shortName())) { + shortener.saveMapping(name(), shortName()); + } + delegate.create(mode); + } + + @Override + public void delete() { + delegate.delete(); + } + + @Override + public void moveTo(Folder target) { + if (target instanceof ShorteningFolder) { + moveToInternal((ShorteningFolder) target); + } else { + throw new UnsupportedOperationException("Can not move ShorteningFolder to conventional folder."); + } + } + + private void moveToInternal(ShorteningFolder target) { + if (this.isAncestorOf(target) || target.isAncestorOf(this)) { + throw new IllegalArgumentException("Can not move directories containing one another (src: " + this + ", dst: " + target + ")"); + } + + if (!target.exists()) { + target.create(FolderCreateMode.INCLUDING_PARENTS); + } + + delegate.moveTo(target.delegate); + } + + @Override + public String toString() { + return name() + "/"; + } + +} diff --git a/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningNode.java b/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningNode.java new file mode 100644 index 000000000..d3a62983c --- /dev/null +++ b/main/shortening-layer/src/main/java/org/cryptomator/shortening/ShorteningNode.java @@ -0,0 +1,68 @@ +package org.cryptomator.shortening; + +import java.time.Instant; +import java.util.Optional; + +import org.cryptomator.filesystem.Folder; +import org.cryptomator.filesystem.Node; + +class ShorteningNode implements Node { + + protected final E delegate; + private final ShorteningFolder parent; + private final String longName; + private final String shortName; + + public ShorteningNode(ShorteningFolder parent, E delegate, String longName) { + this.delegate = delegate; + this.parent = parent; + this.shortName = delegate.name(); + this.longName = longName; + } + + @Override + public String name() { + return longName; + } + + protected String shortName() { + return shortName; + } + + @Override + public Optional parent() { + return Optional.ofNullable(parent); + } + + @Override + public boolean exists() { + return delegate.exists(); + } + + @Override + public Instant lastModified() { + return delegate.lastModified(); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((longName == null) ? 0 : longName.hashCode()); + result = prime * result + ((parent == null) ? 0 : parent.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof ShorteningNode) { + ShorteningNode other = (ShorteningNode) obj; + return this.getClass() == other.getClass() // + && (this.parent == null && other.parent == null || this.parent.equals(other.parent)) // + && (this.longName == null && other.longName == null || this.longName.equals(other.longName)); + } else { + return false; + } + } + +} diff --git a/main/shortening-layer/src/main/java/org/cryptomator/shortening/package-info.java b/main/shortening-layer/src/main/java/org/cryptomator/shortening/package-info.java new file mode 100644 index 000000000..26f68c001 --- /dev/null +++ b/main/shortening-layer/src/main/java/org/cryptomator/shortening/package-info.java @@ -0,0 +1,6 @@ +/** + * Provides a decoration layer for the {@link org.cryptomator.filesystem Filesystem API}. + * {@link org.cryptomator.filesystem.File File} and {@link org.cryptomator.filesystem.Folder Folder} names exceeding a certain length limit will be mapped to shorter equivalents. + * The mapping itself is stored in metadata files inside the m/ directory on root level. + */ +package org.cryptomator.shortening; \ No newline at end of file diff --git a/main/shortening-layer/src/test/java/org/cryptomator/shortening/ShorteningFileSystemTest.java b/main/shortening-layer/src/test/java/org/cryptomator/shortening/ShorteningFileSystemTest.java new file mode 100644 index 000000000..0917fb2d6 --- /dev/null +++ b/main/shortening-layer/src/test/java/org/cryptomator/shortening/ShorteningFileSystemTest.java @@ -0,0 +1,115 @@ +package org.cryptomator.shortening; + +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.cryptomator.filesystem.File; +import org.cryptomator.filesystem.FileSystem; +import org.cryptomator.filesystem.Folder; +import org.cryptomator.filesystem.FolderCreateMode; +import org.cryptomator.filesystem.ReadableFile; +import org.cryptomator.filesystem.WritableFile; +import org.cryptomator.filesystem.inmem.InMemoryFileSystem; +import org.junit.Assert; +import org.junit.Test; + +public class ShorteningFileSystemTest { + + @Test + public void testCreationOfInvisibleMetadataFolder() { + final FileSystem underlyingFs = new InMemoryFileSystem(); + final Folder metadataRoot = underlyingFs.folder("m"); + final FileSystem fs = new ShorteningFileSystem(underlyingFs, metadataRoot, 10); + fs.folder("morethantenchars").create(FolderCreateMode.FAIL_IF_PARENT_IS_MISSING); + Assert.assertTrue(metadataRoot.exists()); + Assert.assertEquals(1, fs.folders().count()); + } + + @Test(expected = UncheckedIOException.class) + public void testPreventCreationOfMetadataFolder() { + final FileSystem underlyingFs = new InMemoryFileSystem(); + final Folder metadataRoot = underlyingFs.folder("m"); + final FileSystem fs = new ShorteningFileSystem(underlyingFs, metadataRoot, 10); + fs.folder("m"); + } + + @Test + public void testDeflate() { + final FileSystem underlyingFs = new InMemoryFileSystem(); + final Folder metadataRoot = underlyingFs.folder("m"); + final FileSystem fs = new ShorteningFileSystem(underlyingFs, metadataRoot, 10); + final Folder longNamedFolder = fs.folder("morethantenchars"); // base32(sha1(morethantenchars)) = QMJL5GQUETRX2YRV6XDTJQ6NNM7IEUHP + final File correspondingMetadataFile = metadataRoot.folder("QM").folder("JL").file("QMJL5GQUETRX2YRV6XDTJQ6NNM7IEUHP.lng"); + longNamedFolder.create(FolderCreateMode.FAIL_IF_PARENT_IS_MISSING); + Assert.assertTrue(longNamedFolder.exists()); + Assert.assertTrue(correspondingMetadataFile.exists()); + } + + @Test + public void testDeflateAndInflateFolder() { + final FileSystem underlyingFs = new InMemoryFileSystem(); + final Folder metadataRoot = underlyingFs.folder("m"); + final FileSystem fs1 = new ShorteningFileSystem(underlyingFs, metadataRoot, 10); + final Folder longNamedFolder1 = fs1.folder("morethantenchars"); + longNamedFolder1.create(FolderCreateMode.FAIL_IF_PARENT_IS_MISSING); + + final FileSystem fs2 = new ShorteningFileSystem(underlyingFs, metadataRoot, 10); + final Folder longNamedFolder2 = fs2.folder("morethantenchars"); + Assert.assertTrue(longNamedFolder2.exists()); + } + + @Test + public void testDeflateAndInflateFolderAndFile() throws UncheckedIOException, TimeoutException { + final FileSystem underlyingFs = new InMemoryFileSystem(); + final Folder metadataRoot = underlyingFs.folder("m"); + + // write: + final FileSystem fs1 = new ShorteningFileSystem(underlyingFs, metadataRoot, 10); + fs1.folder("morethantenchars").create(FolderCreateMode.INCLUDING_PARENTS); + try (WritableFile file = fs1.folder("morethantenchars").file("morethanelevenchars.txt").openWritable(1, TimeUnit.MILLISECONDS)) { + file.write(ByteBuffer.wrap("hello world".getBytes())); + } + + // read + final FileSystem fs2 = new ShorteningFileSystem(underlyingFs, metadataRoot, 10); + try (ReadableFile file = fs2.folder("morethantenchars").file("morethanelevenchars.txt").openReadable(1, TimeUnit.MILLISECONDS)) { + ByteBuffer buf = ByteBuffer.allocate(11); + file.read(buf); + Assert.assertEquals("hello world", new String(buf.array())); + } + } + + @Test + public void testPassthroughShortNamedFiles() throws UncheckedIOException, TimeoutException { + final FileSystem underlyingFs = new InMemoryFileSystem(); + final Folder metadataRoot = underlyingFs.folder("m"); + final FileSystem fs = new ShorteningFileSystem(underlyingFs, metadataRoot, 10); + + // of folders: + underlyingFs.folder("foo").folder("bar").create(FolderCreateMode.INCLUDING_PARENTS); + Assert.assertTrue(fs.folder("foo").folder("bar").exists()); + + // from underlying: + try (WritableFile file = underlyingFs.folder("foo").file("test1.txt").openWritable(1, TimeUnit.MILLISECONDS)) { + file.write(ByteBuffer.wrap("hello world".getBytes())); + } + try (ReadableFile file = fs.folder("foo").file("test1.txt").openReadable(1, TimeUnit.MILLISECONDS)) { + ByteBuffer buf = ByteBuffer.allocate(11); + file.read(buf); + Assert.assertEquals("hello world", new String(buf.array())); + } + + // to underlying: + try (WritableFile file = fs.folder("foo").file("test2.txt").openWritable(1, TimeUnit.MILLISECONDS)) { + file.write(ByteBuffer.wrap("hello world".getBytes())); + } + try (ReadableFile file = underlyingFs.folder("foo").file("test2.txt").openReadable(1, TimeUnit.MILLISECONDS)) { + ByteBuffer buf = ByteBuffer.allocate(11); + file.read(buf); + Assert.assertEquals("hello world", new String(buf.array())); + } + } + +}