mirror of
https://github.com/kopia/kopia.git
synced 2026-01-25 23:08:01 -05:00
361 lines
9.1 KiB
Go
361 lines
9.1 KiB
Go
package object
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/md5"
|
|
cryptorand "crypto/rand"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"reflect"
|
|
"runtime/debug"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/kopia/kopia/block"
|
|
|
|
"github.com/kopia/kopia/internal/config"
|
|
"github.com/kopia/kopia/internal/jsonstream"
|
|
"github.com/kopia/kopia/storage"
|
|
)
|
|
|
|
type fakeBlockManager struct {
|
|
mu sync.Mutex
|
|
data map[string][]byte
|
|
}
|
|
|
|
func (f *fakeBlockManager) GetBlock(ctx context.Context, blockID string) ([]byte, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
|
|
if d, ok := f.data[blockID]; ok {
|
|
return append([]byte(nil), d...), nil
|
|
}
|
|
|
|
return nil, storage.ErrBlockNotFound
|
|
}
|
|
|
|
func (f *fakeBlockManager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error) {
|
|
h := md5.New()
|
|
h.Write(data)
|
|
blockID := prefix + string(hex.EncodeToString(h.Sum(nil)))
|
|
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
|
|
f.data[blockID] = append([]byte(nil), data...)
|
|
return blockID, nil
|
|
}
|
|
|
|
func (f *fakeBlockManager) BlockInfo(ctx context.Context, blockID string) (block.Info, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
|
|
if d, ok := f.data[blockID]; ok {
|
|
return block.Info{BlockID: blockID, Length: uint32(len(d))}, nil
|
|
}
|
|
|
|
return block.Info{}, storage.ErrBlockNotFound
|
|
}
|
|
|
|
func (f *fakeBlockManager) Flush(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
func setupTest(t *testing.T) (map[string][]byte, *Manager) {
|
|
return setupTestWithData(t, map[string][]byte{}, ManagerOptions{})
|
|
}
|
|
|
|
func setupTestWithData(t *testing.T, data map[string][]byte, opts ManagerOptions) (map[string][]byte, *Manager) {
|
|
r, err := NewObjectManager(context.Background(), &fakeBlockManager{data: data}, config.RepositoryObjectFormat{
|
|
FormattingOptions: block.FormattingOptions{
|
|
Version: 1,
|
|
},
|
|
MaxBlockSize: 200,
|
|
Splitter: "FIXED",
|
|
}, opts)
|
|
if err != nil {
|
|
t.Fatalf("can't create object manager: %v", err)
|
|
}
|
|
|
|
return data, r
|
|
}
|
|
|
|
func TestWriters(t *testing.T) {
|
|
ctx := context.Background()
|
|
cases := []struct {
|
|
data []byte
|
|
objectID ID
|
|
}{
|
|
{
|
|
[]byte("the quick brown fox jumps over the lazy dog"),
|
|
"77add1d5f41223d5582fca736a5cb335",
|
|
},
|
|
{make([]byte, 100), "6d0bb00954ceb7fbee436bb55a8397a9"}, // 100 zero bytes
|
|
}
|
|
|
|
for _, c := range cases {
|
|
data, om := setupTest(t)
|
|
|
|
writer := om.NewWriter(ctx, WriterOptions{})
|
|
|
|
writer.Write(c.data)
|
|
|
|
result, err := writer.Result()
|
|
if err != nil {
|
|
t.Errorf("error getting writer results for %v, expected: %v", c.data, c.objectID.String())
|
|
continue
|
|
}
|
|
|
|
om.writeBackWG.Wait()
|
|
|
|
if !objectIDsEqual(result, c.objectID) {
|
|
t.Errorf("incorrect result for %v, expected: %v got: %v", c.data, c.objectID.String(), result.String())
|
|
}
|
|
|
|
if _, ok := c.objectID.BlockID(); !ok {
|
|
if len(data) != 0 {
|
|
t.Errorf("unexpected data written to the storage: %v", data)
|
|
}
|
|
} else {
|
|
if len(data) != 1 {
|
|
// 1 data block
|
|
t.Errorf("unexpected data written to the storage: %v", data)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func objectIDsEqual(o1 ID, o2 ID) bool {
|
|
return reflect.DeepEqual(o1, o2)
|
|
}
|
|
|
|
func TestWriterCompleteChunkInTwoWrites(t *testing.T) {
|
|
ctx := context.Background()
|
|
_, om := setupTest(t)
|
|
|
|
bytes := make([]byte, 100)
|
|
writer := om.NewWriter(ctx, WriterOptions{})
|
|
writer.Write(bytes[0:50])
|
|
writer.Write(bytes[0:50])
|
|
result, err := writer.Result()
|
|
if !objectIDsEqual(result, "6d0bb00954ceb7fbee436bb55a8397a9") {
|
|
t.Errorf("unexpected result: %v err: %v", result, err)
|
|
}
|
|
}
|
|
|
|
func verifyIndirectBlock(ctx context.Context, t *testing.T, r *Manager, oid ID) {
|
|
for indexBlockID, isIndirect := oid.IndexObjectID(); isIndirect; indexBlockID, isIndirect = indexBlockID.IndexObjectID() {
|
|
rd, err := r.Open(ctx, indexBlockID)
|
|
if err != nil {
|
|
t.Errorf("unable to open %v: %v", oid.String(), err)
|
|
return
|
|
}
|
|
defer rd.Close()
|
|
|
|
pr, err := jsonstream.NewReader(rd, indirectStreamType, nil)
|
|
if err != nil {
|
|
t.Errorf("cannot open indirect stream: %v", err)
|
|
return
|
|
}
|
|
for {
|
|
v := indirectObjectEntry{}
|
|
if err := pr.Read(&v); err != nil {
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
t.Errorf("err: %v", err)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIndirection(t *testing.T) {
|
|
ctx := context.Background()
|
|
cases := []struct {
|
|
dataLength int
|
|
expectedBlockCount int
|
|
expectedIndirection int
|
|
}{
|
|
{dataLength: 200, expectedBlockCount: 1, expectedIndirection: 0},
|
|
{dataLength: 250, expectedBlockCount: 3, expectedIndirection: 1},
|
|
{dataLength: 1400, expectedBlockCount: 7, expectedIndirection: 3},
|
|
{dataLength: 2000, expectedBlockCount: 8, expectedIndirection: 3},
|
|
{dataLength: 3000, expectedBlockCount: 9, expectedIndirection: 3},
|
|
{dataLength: 4000, expectedBlockCount: 14, expectedIndirection: 4},
|
|
{dataLength: 10000, expectedBlockCount: 24, expectedIndirection: 4},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
data, om := setupTest(t)
|
|
|
|
contentBytes := make([]byte, c.dataLength)
|
|
|
|
writer := om.NewWriter(ctx, WriterOptions{})
|
|
writer.Write(contentBytes)
|
|
result, err := writer.Result()
|
|
if err != nil {
|
|
t.Errorf("error getting writer results: %v", err)
|
|
}
|
|
|
|
if indirectionLevel(result) != c.expectedIndirection {
|
|
t.Errorf("incorrect indirection level for size: %v: %v, expected %v", c.dataLength, indirectionLevel(result), c.expectedIndirection)
|
|
}
|
|
|
|
if got, want := len(data), c.expectedBlockCount; got != want {
|
|
t.Errorf("unexpected block count for %v: %v, expected %v", c.dataLength, got, want)
|
|
}
|
|
|
|
om.Flush(ctx)
|
|
|
|
l, b, err := om.VerifyObject(ctx, result)
|
|
if err != nil {
|
|
t.Errorf("error verifying %q: %v", result, err)
|
|
}
|
|
|
|
if got, want := int(l), len(contentBytes); got != want {
|
|
t.Errorf("got invalid byte count for %q: %v, wanted %v", result, got, want)
|
|
}
|
|
|
|
if got, want := len(b), c.expectedBlockCount; got != want {
|
|
t.Errorf("invalid block count for %v, got %v, wanted %v", result, got, want)
|
|
}
|
|
|
|
verifyIndirectBlock(ctx, t, om, result)
|
|
}
|
|
}
|
|
|
|
func indirectionLevel(oid ID) int {
|
|
indexObjectID, ok := oid.IndexObjectID()
|
|
if !ok {
|
|
return 0
|
|
}
|
|
|
|
return 1 + indirectionLevel(indexObjectID)
|
|
}
|
|
|
|
func TestHMAC(t *testing.T) {
|
|
ctx := context.Background()
|
|
content := bytes.Repeat([]byte{0xcd}, 50)
|
|
|
|
_, om := setupTest(t)
|
|
|
|
w := om.NewWriter(ctx, WriterOptions{})
|
|
w.Write(content)
|
|
result, err := w.Result()
|
|
if result.String() != "999732b72ceff665b3f7608411db66a4" {
|
|
t.Errorf("unexpected result: %v err: %v", result.String(), err)
|
|
}
|
|
}
|
|
|
|
func TestReader(t *testing.T) {
|
|
ctx := context.Background()
|
|
data, om := setupTest(t)
|
|
|
|
storedPayload := []byte("foo\nbar")
|
|
data["a76999788386641a3ec798554f1fe7e6"] = storedPayload
|
|
|
|
cases := []struct {
|
|
text string
|
|
payload []byte
|
|
}{
|
|
{"a76999788386641a3ec798554f1fe7e6", storedPayload},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
objectID, err := ParseID(c.text)
|
|
if err != nil {
|
|
t.Errorf("cannot parse object ID: %v", err)
|
|
continue
|
|
}
|
|
|
|
reader, err := om.Open(ctx, objectID)
|
|
if err != nil {
|
|
t.Errorf("cannot create reader for %v: %v", objectID, err)
|
|
continue
|
|
}
|
|
|
|
d, err := ioutil.ReadAll(reader)
|
|
if err != nil {
|
|
t.Errorf("cannot read all data for %v: %v", objectID, err)
|
|
continue
|
|
}
|
|
if !bytes.Equal(d, c.payload) {
|
|
t.Errorf("incorrect payload for %v: expected: %v got: %v", objectID, c.payload, d)
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReaderStoredBlockNotFound(t *testing.T) {
|
|
ctx := context.Background()
|
|
_, om := setupTest(t)
|
|
|
|
objectID, err := ParseID("deadbeef")
|
|
if err != nil {
|
|
t.Errorf("cannot parse object ID: %v", err)
|
|
}
|
|
reader, err := om.Open(ctx, objectID)
|
|
if err != storage.ErrBlockNotFound || reader != nil {
|
|
t.Errorf("unexpected result: reader: %v err: %v", reader, err)
|
|
}
|
|
}
|
|
|
|
func TestEndToEndReadAndSeek(t *testing.T) {
|
|
ctx := context.Background()
|
|
_, om := setupTest(t)
|
|
|
|
for _, size := range []int{1, 199, 200, 201, 9999, 512434} {
|
|
// Create some random data sample of the specified size.
|
|
randomData := make([]byte, size)
|
|
cryptorand.Read(randomData)
|
|
|
|
writer := om.NewWriter(ctx, WriterOptions{})
|
|
writer.Write(randomData)
|
|
objectID, err := writer.Result()
|
|
writer.Close()
|
|
if err != nil {
|
|
t.Errorf("cannot get writer result for %v: %v", size, err)
|
|
continue
|
|
}
|
|
|
|
verify(ctx, t, om, objectID, randomData, fmt.Sprintf("%v %v", objectID, size))
|
|
}
|
|
}
|
|
|
|
func verify(ctx context.Context, t *testing.T, om *Manager, objectID ID, expectedData []byte, testCaseID string) {
|
|
t.Helper()
|
|
reader, err := om.Open(ctx, objectID)
|
|
if err != nil {
|
|
t.Errorf("cannot get reader for %v (%v): %v %v", testCaseID, objectID, err, string(debug.Stack()))
|
|
return
|
|
}
|
|
|
|
for i := 0; i < 20; i++ {
|
|
sampleSize := int(rand.Int31n(300))
|
|
seekOffset := int(rand.Int31n(int32(len(expectedData))))
|
|
if seekOffset+sampleSize > len(expectedData) {
|
|
sampleSize = len(expectedData) - seekOffset
|
|
}
|
|
if sampleSize > 0 {
|
|
got := make([]byte, sampleSize)
|
|
if offset, err := reader.Seek(int64(seekOffset), 0); err != nil || offset != int64(seekOffset) {
|
|
t.Errorf("seek error: %v offset=%v expected:%v", err, offset, seekOffset)
|
|
}
|
|
if n, err := reader.Read(got); err != nil || n != sampleSize {
|
|
t.Errorf("invalid data: n=%v, expected=%v, err:%v", n, sampleSize, err)
|
|
}
|
|
|
|
expected := expectedData[seekOffset : seekOffset+sampleSize]
|
|
|
|
if !bytes.Equal(expected, got) {
|
|
t.Errorf("incorrect data read for %v: expected: %x, got: %x", testCaseID, expected, got)
|
|
}
|
|
}
|
|
}
|
|
}
|