mirror of
https://github.com/kopia/kopia.git
synced 2026-10-26 16:18:53 -04:00
API tweaks and godoc
This commit is contained in:
1 parent
b4758bdc79
commit
91bfa7cba3
15 files changed
+133
-155
No files matched your search
@@ -17,7 +17,7 @@ var (
|
||||
createCommandRepository = createCommand.Flag("repository", "Repository path.").Default("colocated").String()
|
||||
createObjectFormat = createCommand.Flag("repo-format", "Format of repository objects.").PlaceHolder("FORMAT").Default(repo.DefaultObjectFormat).Enum(supportedObjectFormats()...)
|
||||
|
||||
createMaxBlobSize = createCommand.Flag("max-blob-size", "Maximum size of a data chunk.").PlaceHolder("KB").Default("20480").Int()
|
||||
createMaxBlockSize = createCommand.Flag("max-blob-size", "Maximum size of a data chunk.").PlaceHolder("KB").Default("20480").Int()
|
||||
createInlineBlobSize = createCommand.Flag("inline-blob-size", "Maximum size of an inline data chunk.").PlaceHolder("KB").Default("32").Int()
|
||||
createVaultEncryptionFormat = createCommand.Flag("vault-encryption", "Vault encryption.").PlaceHolder("FORMAT").Default("aes-256").Enum(supportedVaultEncryptionFormats()...)
|
||||
createOverwrite = createCommand.Flag("overwrite", "Overwrite existing data (DANGEROUS).").Bool()
|
||||
@@ -44,11 +44,11 @@ func vaultFormat() (*vault.Format, error) {
|
||||
|
||||
func repositoryFormat() (*repo.Format, error) {
|
||||
f := &repo.Format{
|
||||
Version: 1,
|
||||
Secret: make([]byte, 32),
|
||||
MaxBlobSize: int32(*createMaxBlobSize * 1024),
|
||||
MaxInlineBlobSize: int32(*createInlineBlobSize * 1024),
|
||||
ObjectFormat: *createObjectFormat,
|
||||
Version: 1,
|
||||
Secret: make([]byte, 32),
|
||||
MaxBlockSize: int32(*createMaxBlockSize * 1024),
|
||||
MaxInlineContentLength: int32(*createInlineBlobSize * 1024),
|
||||
ObjectFormat: *createObjectFormat,
|
||||
}
|
||||
|
||||
_, err := io.ReadFull(rand.Reader, f.Secret)
|
||||
@@ -102,7 +102,7 @@ func runCreateCommand(context *kingpin.ParseContext) error {
|
||||
fmt.Printf(
|
||||
"Initializing repository in with format %v and maximum object size %v.\n",
|
||||
repoFormat.ObjectFormat,
|
||||
repoFormat.MaxBlobSize)
|
||||
repoFormat.MaxBlockSize)
|
||||
|
||||
vf, err := vaultFormat()
|
||||
if err != nil {
|
||||
@@ -115,7 +115,7 @@ func runCreateCommand(context *kingpin.ParseContext) error {
|
||||
}
|
||||
|
||||
// Make repository to make sure the format is supported.
|
||||
_, err = repo.NewRepository(repositoryStorage, repoFormat)
|
||||
_, err = repo.New(repositoryStorage, repoFormat)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to initialize repository: %v", err)
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ func runLSCommand(context *kingpin.ParseContext) error {
|
||||
}
|
||||
}
|
||||
|
||||
dir := fs.NewRepositoryDirectory(mgr, oid)
|
||||
dir := fs.NewDirectory(mgr, oid)
|
||||
entries, err := dir.Readdir()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -47,7 +47,7 @@ func runMountCommand(context *kingpin.ParseContext) error {
|
||||
return err
|
||||
}
|
||||
|
||||
rootNode := kopiafuse.NewDirectoryNode(fs.NewRepositoryDirectory(r, oid))
|
||||
rootNode := kopiafuse.NewDirectoryNode(fs.NewDirectory(r, oid))
|
||||
|
||||
fusefs.Serve(fuseConnection, &root{rootNode})
|
||||
|
||||
|
||||
@@ -41,8 +41,8 @@ func runRepositoryInfoCommand(context *kingpin.ParseContext) error {
|
||||
fmt.Println(" Secret: ", len(f.Secret), "bytes")
|
||||
}
|
||||
fmt.Println(" ID Format: ", f.ObjectFormat)
|
||||
fmt.Println(" Blob Size: ", f.MaxBlobSize/1024, "KB")
|
||||
fmt.Println(" Inline Blob Size:", f.MaxInlineBlobSize/1024, "KB")
|
||||
fmt.Println(" Blob Size: ", f.MaxBlockSize/1024, "KB")
|
||||
fmt.Println(" Inline Blob Size:", f.MaxInlineContentLength/1024, "KB")
|
||||
r, err := v.OpenRepository()
|
||||
if err == nil {
|
||||
fmt.Println(" Storage: ", r.Storage())
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ func parseObjectID(id string, vlt *vault.Vault) (repo.ObjectID, error) {
|
||||
return repo.NullObjectID, fmt.Errorf("cannot open repository: %v", err)
|
||||
}
|
||||
|
||||
dir := fs.NewRepositoryDirectory(r, oid)
|
||||
dir := fs.NewDirectory(r, oid)
|
||||
if err != nil {
|
||||
return repo.NullObjectID, err
|
||||
}
|
||||
|
||||
+2
-2
@@ -90,8 +90,8 @@ func withMetadata(rc io.ReadCloser, md *EntryMetadata) EntryMetadataReadCloser {
|
||||
}
|
||||
}
|
||||
|
||||
// NewRepositoryDirectory returns Directory based on repository object with the specified ID.
|
||||
func NewRepositoryDirectory(r repo.Repository, objectID repo.ObjectID) Directory {
|
||||
// NewDirectory returns Directory based on repository object with the specified ID.
|
||||
func NewDirectory(r repo.Repository, objectID repo.ObjectID) Directory {
|
||||
d := newRepoEntry(r, &EntryMetadata{
|
||||
Name: "/",
|
||||
ObjectID: objectID,
|
||||
|
||||
+5
-5
@@ -42,13 +42,13 @@ func newUploadTestHarness() *uploadTestHarness {
|
||||
}
|
||||
|
||||
format := repo.Format{
|
||||
Version: 1,
|
||||
ObjectFormat: "TESTONLY_MD5",
|
||||
MaxBlobSize: 1000,
|
||||
MaxInlineBlobSize: 0,
|
||||
Version: 1,
|
||||
ObjectFormat: "TESTONLY_MD5",
|
||||
MaxBlockSize: 1000,
|
||||
MaxInlineContentLength: 0,
|
||||
}
|
||||
|
||||
repo, err := repo.NewRepository(storage, &format)
|
||||
repo, err := repo.New(storage, &format)
|
||||
if err != nil {
|
||||
panic("unable to create repository: " + err.Error())
|
||||
}
|
||||
|
||||
+19
-13
@@ -7,22 +7,27 @@ import (
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"hash"
|
||||
)
|
||||
|
||||
// Format describes the format of object data.
|
||||
// Format describes the format of objects in a repository.
|
||||
type Format struct {
|
||||
Version int32 `json:"version,omitempty"`
|
||||
ObjectFormat string `json:"objectFormat,omitempty"`
|
||||
Secret []byte `json:"secret,omitempty"`
|
||||
MaxInlineBlobSize int32 `json:"maxInlineBlobSize,omitempty"`
|
||||
MaxBlobSize int32 `json:"maxBlobSize,omitempty"`
|
||||
Version int32 `json:"version,omitempty"` // version number, must be "1"
|
||||
ObjectFormat string `json:"objectFormat,omitempty"` // identifier of object format
|
||||
Secret []byte `json:"secret,omitempty"` // HMAC secret used to generate encryption keys
|
||||
MaxInlineContentLength int32 `json:"maxInlineContentLength,omitempty"` // maximum size of object to be considered for inline storage within ObjectID
|
||||
MaxBlockSize int32 `json:"maxBlockSize,omitempty"` // maximum size of storage block
|
||||
}
|
||||
|
||||
// ObjectFormatter performs data hashing and encryption of a data block when storing object in a repository,
|
||||
// ObjectFormatter performs data block ID computation and encryption of a block of data when storing object in a repository,
|
||||
type ObjectFormatter interface {
|
||||
HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte)
|
||||
// ComputeBlockIDAndKey computes ID of the storage block and encryption key for the specified block of data.
|
||||
// The secret should be used for HMAC.
|
||||
ComputeBlockIDAndKey(data []byte, secret []byte) (blockID string, cryptoKey []byte)
|
||||
|
||||
// CreateCipher returns a new instance of a block cipher with the specified key.
|
||||
CreateCipher(key []byte) (cipher.Block, error)
|
||||
}
|
||||
|
||||
@@ -31,11 +36,12 @@ type unencryptedFormat struct {
|
||||
fold int
|
||||
}
|
||||
|
||||
func (fi *unencryptedFormat) HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte) {
|
||||
blockID = hashContent(fi.hashCtor, data, secret)
|
||||
func (fi *unencryptedFormat) ComputeBlockIDAndKey(data []byte, secret []byte) (blockID string, cryptoKey []byte) {
|
||||
h := hashContent(fi.hashCtor, data, secret)
|
||||
if fi.fold > 0 {
|
||||
blockID = fold(blockID, fi.fold)
|
||||
h = fold(h, fi.fold)
|
||||
}
|
||||
blockID = hex.EncodeToString(h)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -50,10 +56,10 @@ type encryptedFormat struct {
|
||||
keyBytes int
|
||||
}
|
||||
|
||||
func (fi *encryptedFormat) HashBuffer(data []byte, secret []byte) (blockID []byte, cryptoKey []byte) {
|
||||
func (fi *encryptedFormat) ComputeBlockIDAndKey(data []byte, secret []byte) (blockID string, cryptoKey []byte) {
|
||||
h := hashContent(fi.hashCtor, data, secret)
|
||||
p := len(h) - fi.keyBytes
|
||||
blockID = h[0:p]
|
||||
blockID = hex.EncodeToString(h[0:p])
|
||||
cryptoKey = h[p:]
|
||||
return
|
||||
}
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ func TestFormat(t *testing.T) {
|
||||
|
||||
for k, v := range SupportedFormats {
|
||||
t.Logf("testing %v", k)
|
||||
blk, key := v.HashBuffer(data, secret)
|
||||
blk, key := v.ComputeBlockIDAndKey(data, secret)
|
||||
if key != nil {
|
||||
cipher, err := v.CreateCipher(key)
|
||||
if err != nil || cipher == nil {
|
||||
|
||||
@@ -182,7 +182,7 @@ func (w *objectWriter) Result(forceStored bool) (ObjectID, error) {
|
||||
return NullObjectID, nil
|
||||
}
|
||||
|
||||
if w.buffer.Len() < int(w.repo.format.MaxInlineBlobSize) {
|
||||
if w.buffer.Len() < int(w.repo.format.MaxInlineContentLength) {
|
||||
return InlineObjectID(w.buffer.Bytes()), nil
|
||||
}
|
||||
}
|
||||
|
||||
+13
-12
@@ -63,28 +63,29 @@ var (
|
||||
)
|
||||
|
||||
// UIString returns string representation of ObjectID that is suitable for displaying in the UI.
|
||||
// Note that the object ID name may contain its encryption key, which is sensitive.
|
||||
func (m *ObjectID) UIString() string {
|
||||
if m.StorageBlock != "" {
|
||||
//
|
||||
// Note that the object ID name often contains its encryption key, which is sensitive and can be quite long (~100 characters long).
|
||||
func (oid *ObjectID) UIString() string {
|
||||
if oid.StorageBlock != "" {
|
||||
var encryptionSuffix string
|
||||
|
||||
if m.EncryptionKey != nil {
|
||||
encryptionSuffix = "." + hex.EncodeToString(m.EncryptionKey)
|
||||
if oid.EncryptionKey != nil {
|
||||
encryptionSuffix = "." + hex.EncodeToString(oid.EncryptionKey)
|
||||
}
|
||||
|
||||
if m.Indirect > 0 {
|
||||
return fmt.Sprintf("L%v,%v%v", m.Indirect, m.StorageBlock, encryptionSuffix)
|
||||
if oid.Indirect > 0 {
|
||||
return fmt.Sprintf("L%v,%v%v", oid.Indirect, oid.StorageBlock, encryptionSuffix)
|
||||
}
|
||||
|
||||
return "D" + m.StorageBlock + encryptionSuffix
|
||||
return "D" + oid.StorageBlock + encryptionSuffix
|
||||
}
|
||||
|
||||
if m.Content != nil {
|
||||
return "B" + inlineContentEncoding.EncodeToString(m.Content)
|
||||
if oid.Content != nil {
|
||||
return "B" + inlineContentEncoding.EncodeToString(oid.Content)
|
||||
}
|
||||
|
||||
if m.Section != nil {
|
||||
return fmt.Sprintf("S%v,%v,%v", m.Section.Start, m.Section.Length, m.Section.Base.UIString())
|
||||
if oid.Section != nil {
|
||||
return fmt.Sprintf("S%v,%v,%v", oid.Section.Start, oid.Section.Length, oid.Section.Base.UIString())
|
||||
}
|
||||
|
||||
return "B"
|
||||
|
||||
+64
-93
@@ -5,12 +5,10 @@ import (
|
||||
"bytes"
|
||||
"crypto/cipher"
|
||||
"crypto/hmac"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -24,7 +22,7 @@ const (
|
||||
minLocatorSizeBytes = 16
|
||||
)
|
||||
|
||||
// ObjectReader allows reading, seeking and closing of a repository object.
|
||||
// ObjectReader allows reading, seeking, getting the length of and closing of a repository object.
|
||||
type ObjectReader interface {
|
||||
io.Reader
|
||||
io.Seeker
|
||||
@@ -41,14 +39,18 @@ type Repository interface {
|
||||
// NewWriter opens an ObjectWriter for writing new content to the storage.
|
||||
NewWriter(options ...WriterOption) ObjectWriter
|
||||
|
||||
// Open creates an io.ReadSeeker for reading object with a specified ID.
|
||||
// Open creates an ObjectReader for reading object with a specified ID.
|
||||
Open(objectID ObjectID) (ObjectReader, error)
|
||||
|
||||
// Flush ensures that all pending writes have completed.
|
||||
Flush() error
|
||||
Storage() storage.Storage
|
||||
Close()
|
||||
|
||||
ResetStats()
|
||||
// Storage returns the underlying Storage.
|
||||
Storage() storage.Storage
|
||||
|
||||
// Close
|
||||
Close() error
|
||||
|
||||
Stats() RepositoryStats
|
||||
}
|
||||
|
||||
@@ -81,34 +83,33 @@ type repository struct {
|
||||
formatter ObjectFormatter
|
||||
}
|
||||
|
||||
func (repo *repository) Close() {
|
||||
repo.Flush()
|
||||
repo.bufferManager.close()
|
||||
func (r *repository) Close() error {
|
||||
if err := r.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
r.bufferManager.close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (repo *repository) Flush() error {
|
||||
if f, ok := repo.storage.(storage.Flusher); ok {
|
||||
func (r *repository) Flush() error {
|
||||
if f, ok := r.storage.(storage.Flusher); ok {
|
||||
return f.Flush()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (repo *repository) ResetStats() {
|
||||
repo.stats = &RepositoryStats{}
|
||||
func (r *repository) Stats() RepositoryStats {
|
||||
return *r.stats
|
||||
}
|
||||
|
||||
func (repo *repository) Stats() RepositoryStats {
|
||||
return *repo.stats
|
||||
func (r *repository) Storage() storage.Storage {
|
||||
return r.storage
|
||||
}
|
||||
|
||||
func (repo *repository) Storage() storage.Storage {
|
||||
return repo.storage
|
||||
}
|
||||
|
||||
func (repo *repository) NewWriter(options ...WriterOption) ObjectWriter {
|
||||
func (r *repository) NewWriter(options ...WriterOption) ObjectWriter {
|
||||
result := &objectWriter{
|
||||
repo: repo,
|
||||
repo: r,
|
||||
blockTracker: &blockTracker{},
|
||||
}
|
||||
|
||||
@@ -119,12 +120,12 @@ func (repo *repository) NewWriter(options ...WriterOption) ObjectWriter {
|
||||
return result
|
||||
}
|
||||
|
||||
func (repo *repository) Open(objectID ObjectID) (ObjectReader, error) {
|
||||
func (r *repository) Open(objectID ObjectID) (ObjectReader, error) {
|
||||
// log.Printf("Repository::Open %v", objectID.String())
|
||||
// defer log.Printf("finished Repository::Open() %v", objectID.String())
|
||||
|
||||
if objectID.Section != nil {
|
||||
baseReader, err := repo.Open(objectID.Section.Base)
|
||||
baseReader, err := r.Open(objectID.Section.Base)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create base reader: %v", err)
|
||||
}
|
||||
@@ -133,13 +134,13 @@ func (repo *repository) Open(objectID ObjectID) (ObjectReader, error) {
|
||||
}
|
||||
|
||||
if objectID.Indirect > 0 {
|
||||
r, err := repo.Open(removeIndirection(objectID))
|
||||
rd, err := r.Open(removeIndirection(objectID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
seekTable, err := repo.flattenListChunk(r)
|
||||
seekTable, err := r.flattenListChunk(rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -147,32 +148,13 @@ func (repo *repository) Open(objectID ObjectID) (ObjectReader, error) {
|
||||
totalLength := seekTable[len(seekTable)-1].endOffset()
|
||||
|
||||
return &objectReader{
|
||||
repo: repo,
|
||||
repo: r,
|
||||
seekTable: seekTable,
|
||||
totalLength: totalLength,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return repo.newRawReader(objectID)
|
||||
}
|
||||
|
||||
func dumpObject(repo *repository, oid ObjectID) {
|
||||
oid.Indirect = 0
|
||||
log.Printf(" dumping %v", oid.UIString())
|
||||
defer log.Printf(" end of %v", oid.UIString())
|
||||
r, err := repo.Open(oid)
|
||||
if err != nil {
|
||||
log.Printf("failed to open %v: %v", oid, err)
|
||||
return
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
b, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
log.Printf("failed to read %v: %v", oid, err)
|
||||
return
|
||||
}
|
||||
log.Printf("%x", b)
|
||||
return r.newRawReader(objectID)
|
||||
}
|
||||
|
||||
// RepositoryOption controls the behavior of Repository.
|
||||
@@ -201,14 +183,10 @@ func hmacFunc(key []byte, hf func() hash.Hash) func() hash.Hash {
|
||||
}
|
||||
}
|
||||
|
||||
// NewRepository creates new Repository with the specified storage, options, and key provider.
|
||||
func NewRepository(
|
||||
r storage.Storage,
|
||||
f *Format,
|
||||
options ...RepositoryOption,
|
||||
) (Repository, error) {
|
||||
if f.MaxBlobSize < 100 {
|
||||
return nil, fmt.Errorf("MaxBlobSize is not set")
|
||||
// New creates a Repository with the specified storage, format and options.
|
||||
func New(s storage.Storage, f *Format, options ...RepositoryOption) (Repository, error) {
|
||||
if f.MaxBlockSize < 100 {
|
||||
return nil, fmt.Errorf("MaxBlockSize is not set")
|
||||
}
|
||||
|
||||
sf := SupportedFormats[f.ObjectFormat]
|
||||
@@ -216,31 +194,27 @@ func NewRepository(
|
||||
return nil, fmt.Errorf("unknown object format: %v", f.ObjectFormat)
|
||||
}
|
||||
|
||||
repo := &repository{
|
||||
storage: r,
|
||||
r := &repository{
|
||||
storage: s,
|
||||
format: *f,
|
||||
stats: &RepositoryStats{},
|
||||
}
|
||||
|
||||
repo.formatter = sf
|
||||
r.formatter = sf
|
||||
|
||||
for _, o := range options {
|
||||
if err := o(repo); err != nil {
|
||||
repo.Close()
|
||||
if err := o(r); err != nil {
|
||||
r.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
repo.bufferManager = newBufferManager(int(repo.format.MaxBlobSize))
|
||||
r.bufferManager = newBufferManager(int(r.format.MaxBlockSize))
|
||||
|
||||
return repo, nil
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (repo *repository) hashBuffer(data []byte) ([]byte, []byte) {
|
||||
return repo.formatter.HashBuffer(data, repo.format.Secret)
|
||||
}
|
||||
|
||||
func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, storage.ReaderWithLength, error) {
|
||||
func (r *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, storage.ReaderWithLength, error) {
|
||||
var data []byte
|
||||
if buffer != nil {
|
||||
data = buffer.Bytes()
|
||||
@@ -248,32 +222,32 @@ func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string
|
||||
|
||||
var blockCipher cipher.Block
|
||||
|
||||
contentHash, encryptionKey := repo.hashBuffer(data)
|
||||
blockID, encryptionKey := r.formatter.ComputeBlockIDAndKey(data, r.format.Secret)
|
||||
if encryptionKey != nil {
|
||||
var err error
|
||||
blockCipher, err = repo.formatter.CreateCipher(encryptionKey)
|
||||
blockCipher, err = r.formatter.CreateCipher(encryptionKey)
|
||||
if err != nil {
|
||||
return NullObjectID, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
objectID := ObjectID{
|
||||
StorageBlock: prefix + hex.EncodeToString(contentHash),
|
||||
StorageBlock: prefix + blockID,
|
||||
}
|
||||
|
||||
if len(encryptionKey) > 0 {
|
||||
objectID.EncryptionKey = encryptionKey
|
||||
}
|
||||
|
||||
atomic.AddInt32(&repo.stats.HashedBlocks, 1)
|
||||
atomic.AddInt64(&repo.stats.HashedBytes, int64(len(data)))
|
||||
atomic.AddInt32(&r.stats.HashedBlocks, 1)
|
||||
atomic.AddInt64(&r.stats.HashedBytes, int64(len(data)))
|
||||
|
||||
if buffer == nil {
|
||||
return objectID, storage.NewReader(bytes.NewBuffer(nil)), nil
|
||||
}
|
||||
|
||||
blockReader := repo.bufferManager.returnBufferOnClose(buffer)
|
||||
blockReader = newCountingReader(blockReader, &repo.stats.BytesWrittenToStorage)
|
||||
blockReader := r.bufferManager.returnBufferOnClose(buffer)
|
||||
blockReader = newCountingReader(blockReader, &r.stats.BytesWrittenToStorage)
|
||||
|
||||
if len(encryptionKey) > 0 {
|
||||
// Since we're not sharing the key, all-zero IV is ok.
|
||||
@@ -282,16 +256,14 @@ func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string
|
||||
|
||||
blockReader = newCountingReader(
|
||||
newEncryptingReader(blockReader, ctr),
|
||||
&repo.stats.EncryptedBytes)
|
||||
&r.stats.EncryptedBytes)
|
||||
}
|
||||
|
||||
return objectID, blockReader, nil
|
||||
}
|
||||
|
||||
func (repo *repository) flattenListChunk(rawReader io.Reader) ([]indirectObjectEntry, error) {
|
||||
|
||||
r := bufio.NewReader(rawReader)
|
||||
pr, err := jsonstream.NewReader(r, indirectStreamType)
|
||||
func (r *repository) flattenListChunk(rawReader io.Reader) ([]indirectObjectEntry, error) {
|
||||
pr, err := jsonstream.NewReader(bufio.NewReader(rawReader), indirectStreamType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -324,28 +296,28 @@ func removeIndirection(o ObjectID) ObjectID {
|
||||
return o
|
||||
}
|
||||
|
||||
func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) {
|
||||
func (r *repository) newRawReader(objectID ObjectID) (ObjectReader, error) {
|
||||
if objectID.Content != nil {
|
||||
return newObjectReaderWithData(objectID.Content), nil
|
||||
}
|
||||
|
||||
blockID := objectID.StorageBlock
|
||||
payload, err := repo.storage.GetBlock(blockID)
|
||||
payload, err := r.storage.GetBlock(blockID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
atomic.AddInt32(&repo.stats.BlocksReadFromStorage, 1)
|
||||
atomic.AddInt64(&repo.stats.BytesReadFromStorage, int64(len(payload)))
|
||||
atomic.AddInt32(&r.stats.BlocksReadFromStorage, 1)
|
||||
atomic.AddInt64(&r.stats.BytesReadFromStorage, int64(len(payload)))
|
||||
|
||||
if objectID.EncryptionKey == nil {
|
||||
if err := repo.verifyChecksum(payload, objectID.StorageBlock); err != nil {
|
||||
if err := r.verifyChecksum(payload, objectID.StorageBlock); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newObjectReaderWithData(payload), nil
|
||||
}
|
||||
|
||||
blockCipher, err := repo.formatter.CreateCipher(objectID.EncryptionKey)
|
||||
blockCipher, err := r.formatter.CreateCipher(objectID.EncryptionKey)
|
||||
if err != nil {
|
||||
return nil, errors.New("cannot create cipher")
|
||||
}
|
||||
@@ -356,24 +328,23 @@ func (repo *repository) newRawReader(objectID ObjectID) (ObjectReader, error) {
|
||||
|
||||
// Since the encryption key is a function of data, we must be able to generate exactly the same key
|
||||
// after decrypting the content. This serves as a checksum.
|
||||
atomic.AddInt64(&repo.stats.DecryptedBytes, int64(len(payload)))
|
||||
atomic.AddInt64(&r.stats.DecryptedBytes, int64(len(payload)))
|
||||
|
||||
if err := repo.verifyChecksum(payload, objectID.StorageBlock); err != nil {
|
||||
if err := r.verifyChecksum(payload, objectID.StorageBlock); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return newObjectReaderWithData(payload), nil
|
||||
}
|
||||
|
||||
func (repo *repository) verifyChecksum(data []byte, blockID string) error {
|
||||
payloadHash, _ := repo.hashBuffer(data)
|
||||
checksum := hex.EncodeToString(payloadHash)
|
||||
if !strings.HasSuffix(string(blockID), checksum) {
|
||||
atomic.AddInt32(&repo.stats.InvalidBlocks, 1)
|
||||
func (r *repository) verifyChecksum(data []byte, blockID string) error {
|
||||
expectedBlockID, _ := r.formatter.ComputeBlockIDAndKey(data, r.format.Secret)
|
||||
if !strings.HasSuffix(string(blockID), expectedBlockID) {
|
||||
atomic.AddInt32(&r.stats.InvalidBlocks, 1)
|
||||
return fmt.Errorf("invalid checksum for blob: '%v'", blockID)
|
||||
}
|
||||
|
||||
atomic.AddInt32(&repo.stats.ValidBlocks, 1)
|
||||
atomic.AddInt32(&r.stats.ValidBlocks, 1)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+12
-12
@@ -19,10 +19,10 @@ import (
|
||||
|
||||
func testFormat() *Format {
|
||||
return &Format{
|
||||
Version: 1,
|
||||
MaxBlobSize: 200,
|
||||
MaxInlineBlobSize: 20,
|
||||
ObjectFormat: "TESTONLY_MD5",
|
||||
Version: 1,
|
||||
MaxBlockSize: 200,
|
||||
MaxInlineContentLength: 20,
|
||||
ObjectFormat: "TESTONLY_MD5",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ func setupTest(t *testing.T) (data map[string][]byte, repo Repository) {
|
||||
data = map[string][]byte{}
|
||||
st := storagetesting.NewMapStorage(data)
|
||||
|
||||
repo, err := NewRepository(st, testFormat())
|
||||
repo, err := New(st, testFormat())
|
||||
if err != nil {
|
||||
t.Errorf("cannot create manager: %v", err)
|
||||
}
|
||||
@@ -194,7 +194,7 @@ func TestHMAC(t *testing.T) {
|
||||
s.ObjectFormat = "TESTONLY_MD5"
|
||||
s.Secret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25}
|
||||
|
||||
repo, err := NewRepository(storagetesting.NewMapStorage(data), s)
|
||||
repo, err := New(storagetesting.NewMapStorage(data), s)
|
||||
if err != nil {
|
||||
t.Errorf("cannot create manager: %v", err)
|
||||
}
|
||||
@@ -360,7 +360,7 @@ func TestFormats(t *testing.T) {
|
||||
Version: 1,
|
||||
ObjectFormat: objectFormat,
|
||||
Secret: []byte("key"),
|
||||
MaxBlobSize: 10000,
|
||||
MaxBlockSize: 10000,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -379,7 +379,7 @@ func TestFormats(t *testing.T) {
|
||||
format: &Format{
|
||||
Version: 1,
|
||||
ObjectFormat: "TESTONLY_MD5",
|
||||
MaxBlobSize: 10000,
|
||||
MaxBlockSize: 10000,
|
||||
Secret: []byte{}, // HMAC with zero-byte secret
|
||||
},
|
||||
oids: map[string]ObjectID{
|
||||
@@ -391,7 +391,7 @@ func TestFormats(t *testing.T) {
|
||||
format: &Format{
|
||||
Version: 1,
|
||||
ObjectFormat: "TESTONLY_MD5",
|
||||
MaxBlobSize: 10000,
|
||||
MaxBlockSize: 10000,
|
||||
Secret: nil, // non-HMAC version
|
||||
},
|
||||
oids: map[string]ObjectID{
|
||||
@@ -450,7 +450,7 @@ func TestFormats(t *testing.T) {
|
||||
st := storagetesting.NewMapStorage(data)
|
||||
|
||||
t.Logf("verifying %v", c.format)
|
||||
repo, err := NewRepository(st, c.format)
|
||||
repo, err := New(st, c.format)
|
||||
if err != nil {
|
||||
t.Errorf("cannot create manager: %v", err)
|
||||
continue
|
||||
@@ -491,10 +491,10 @@ func TestInvalidEncryptionKey(t *testing.T) {
|
||||
Version: 1,
|
||||
ObjectFormat: "ENCRYPTED_HMAC_SHA512_384_AES256",
|
||||
Secret: []byte("key"),
|
||||
MaxBlobSize: 1000,
|
||||
MaxBlockSize: 1000,
|
||||
}
|
||||
|
||||
repo, err := NewRepository(st, &format)
|
||||
repo, err := New(st, &format)
|
||||
if err != nil {
|
||||
t.Errorf("cannot create manager: %v", err)
|
||||
return
|
||||
|
||||
+1
-1
@@ -209,7 +209,7 @@ func (v *Vault) OpenRepository() (repo.Repository, error) {
|
||||
st = v.storage
|
||||
}
|
||||
|
||||
return repo.NewRepository(st, v.repoConfig.Format)
|
||||
return repo.New(st, v.repoConfig.Format)
|
||||
}
|
||||
|
||||
// Get returns the contents of a specified vault item.
|
||||
|
||||
+2
-2
@@ -63,12 +63,12 @@ func verifyVault(t *testing.T, vaultPath string, repoPath string) {
|
||||
|
||||
repoFormat := &repo.Format{
|
||||
Version: 1,
|
||||
MaxBlobSize: 1000000,
|
||||
MaxBlockSize: 1000000,
|
||||
ObjectFormat: "UNENCRYPTED_HMAC_SHA256",
|
||||
Secret: []byte{1, 2, 3},
|
||||
}
|
||||
|
||||
_, err = repo.NewRepository(repoStorage, repoFormat)
|
||||
_, err = repo.New(repoStorage, repoFormat)
|
||||
if err != nil {
|
||||
t.Errorf("can't create repository: %v", err)
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user