diff --git a/backend/hasher/hasher_internal_test.go b/backend/hasher/hasher_internal_test.go index e289d8f0d..6c6746a6c 100644 --- a/backend/hasher/hasher_internal_test.go +++ b/backend/hasher/hasher_internal_test.go @@ -69,12 +69,66 @@ func (f *Fs) testUploadFromCrypt(t *testing.T) { _ = operations.Purge(ctx, f, dirName) } +func (f *Fs) testUpdateStoresHash(t *testing.T) { + // make a temporary local remote + tempRoot, err := fstest.LocalRemote() + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tempRoot) + }() + + // make a temporary crypt remote as source + ctx := context.Background() + pass := obscure.MustObscure("crypt") + remote := fmt.Sprintf(`:crypt,remote="%s",password="%s":`, tempRoot, pass) + cryptFs, err := fs.NewFs(ctx, remote) + require.NoError(t, err) + + const dirName = "update_hash_1" + const fileName = dirName + "/file_update_1" + const longTime = fs.ModTimeNotSupported + hashType := f.keepHashes.GetOne() + + // upload initial file to hasher via Put + src1 := putFile(ctx, t, cryptFs, fileName, "initial content") + in1, err := src1.Open(ctx) + require.NoError(t, err) + dst, err := f.Put(ctx, in1, src1) + require.NoError(t, err) + require.NotNil(t, dst) + + // verify hash was stored after Put + var hash1 string + if f.opt.MaxAge > 0 { + hash1, err = f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime) + assert.NoError(t, err) + assert.NotEmpty(t, hash1) + } + + // update the file with new content via Update (this is what sync does for replacements) + src2 := putFile(ctx, t, cryptFs, fileName, "updated content") + in2, err := src2.Open(ctx) + require.NoError(t, err) + err = dst.Update(ctx, in2, src2) + require.NoError(t, err) + + // verify hash was stored after Update and is different from the original + if f.opt.MaxAge > 0 { + hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime) + assert.NoError(t, err) + assert.NotEmpty(t, hash2, "hash should be stored after Update") + assert.NotEqual(t, hash1, hash2, "hash should change after Update with different content") + } + _ = operations.Purge(ctx, f, dirName) +} + // InternalTest dispatches all internal tests func (f *Fs) InternalTest(t *testing.T) { if !kv.Supported() { t.Skip("hasher is not supported on this OS") } t.Run("UploadFromCrypt", f.testUploadFromCrypt) + t.Run("UpdateStoresHash", f.testUpdateStoresHash) } var _ fstests.InternalTester = (*Fs)(nil) diff --git a/backend/hasher/object.go b/backend/hasher/object.go index 1003807c8..4e33a53d0 100644 --- a/backend/hasher/object.go +++ b/backend/hasher/object.go @@ -131,10 +131,62 @@ func (o *Object) updateHashes(ctx context.Context) error { return nil } +// wrapInHashingReader wraps in with a hashing reader if needed and +// returns the (possibly wrapped) reader. After the transfer completes +// successfully, call the returned storeHashes function to store the +// computed hashes. +func (f *Fs) wrapInHashingReader(ctx context.Context, in io.Reader, src fs.ObjectInfo) (wrapIn io.Reader, storeHashes func(o *Object)) { + var ( + common hash.Set + rehash bool + hashes hashMap + ) + if fsrc := src.Fs(); fsrc != nil { + common = fsrc.Hashes().Overlap(f.keepHashes) + // Rehash if source does not have all required hashes or hashing is slow + rehash = fsrc.Features().SlowHash || common != f.keepHashes + } + + wrapIn = in + if rehash { + r, err := f.newHashingReader(ctx, in, func(sums hashMap) { + hashes = sums + }) + fs.Debugf(src, "Rehash in-fly due to incomplete or slow source set %v (err: %v)", common, err) + if err == nil { + wrapIn = r + } else { + rehash = false + } + } + + storeHashes = func(o *Object) { + if !rehash { + hashes = hashMap{} + for _, ht := range common.Array() { + if h, e := src.Hash(ctx, ht); e == nil && h != "" { + hashes[ht] = h + } + } + } + if len(hashes) > 0 { + err := o.putHashes(ctx, hashes) + fs.Debugf(o, "Applied %d source hashes, err: %v", len(hashes), err) + } + } + return wrapIn, storeHashes +} + // Update the object with the given data, time and size. func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + wrapIn, storeHashes := o.f.wrapInHashingReader(ctx, in, src) _ = o.f.pruneHash(src.Remote()) - return o.Object.Update(ctx, in, src, options...) + err := o.Object.Update(ctx, wrapIn, src, options...) + if err != nil { + return err + } + storeHashes(o) + return nil } // Remove an object. @@ -189,51 +241,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (r io.ReadC // Put data into the remote path with given modTime and size func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { - var ( - o fs.Object - common hash.Set - rehash bool - hashes hashMap - ) - if fsrc := src.Fs(); fsrc != nil { - common = fsrc.Hashes().Overlap(f.keepHashes) - // Rehash if source does not have all required hashes or hashing is slow - rehash = fsrc.Features().SlowHash || common != f.keepHashes - } - - wrapIn := in - if rehash { - r, err := f.newHashingReader(ctx, in, func(sums hashMap) { - hashes = sums - }) - fs.Debugf(src, "Rehash in-fly due to incomplete or slow source set %v (err: %v)", common, err) - if err == nil { - wrapIn = r - } else { - rehash = false - } - } - + wrapIn, storeHashes := f.wrapInHashingReader(ctx, in, src) _ = f.pruneHash(src.Remote()) oResult, err := f.Fs.Put(ctx, wrapIn, src, options...) - o, err = f.wrapObject(oResult, err) + o, err := f.wrapObject(oResult, err) if err != nil { return nil, err } - - if !rehash { - hashes = hashMap{} - for _, ht := range common.Array() { - if h, e := src.Hash(ctx, ht); e == nil && h != "" { - hashes[ht] = h - } - } - } - if len(hashes) > 0 { - err := o.(*Object).putHashes(ctx, hashes) - fs.Debugf(o, "Applied %d source hashes, err: %v", len(hashes), err) - } - return o, err + storeHashes(o.(*Object)) + return o, nil } type hashingReader struct {