Files
kopia/internal/blobtesting/map.go
Jarek Kowalski 60977812f0 Support for gather writes (#373)
, where blob.Storage.PutBlob gets a list of slices and writes them sequentially 
* performance: added gather.Bytes and gather.WriteBuffer

They are similar to bytes.Buffer but instead of managing a single
byte slice, they maintain a list of slices that and when they run out of
space they allocate new fixed-size slice from a free list.

This helps keep memory allocations completely under control regardless
of the size of data written.

* switch from byte slices and bytes.Buffer to gather.Bytes.

This is mostly mechanical, the only cases where it's not involve blob
storage providers, where we leverage the fact that we don't need to
ever concatenate the slices into one and instead we can do gather
writes.

* PR feedback
2020-03-24 15:05:52 -07:00

156 lines
2.8 KiB
Go

package blobtesting
import (
"bytes"
"context"
"errors"
"sort"
"strings"
"sync"
"time"
"github.com/kopia/kopia/repo/blob"
)
// DataMap is a map of blob ID to their contents.
type DataMap map[blob.ID][]byte
type mapStorage struct {
data DataMap
keyTime map[blob.ID]time.Time
timeNow func() time.Time
mutex sync.RWMutex
}
func (s *mapStorage) GetBlob(ctx context.Context, id blob.ID, offset, length int64) ([]byte, error) {
s.mutex.RLock()
defer s.mutex.RUnlock()
data, ok := s.data[id]
if ok {
data = append([]byte(nil), data...)
if length < 0 {
return data, nil
}
if int(offset) > len(data) || offset < 0 {
return nil, errors.New("invalid offset")
}
data = data[offset:]
if int(length) > len(data) {
return nil, errors.New("invalid length")
}
return data[0:length], nil
}
return nil, blob.ErrBlobNotFound
}
func (s *mapStorage) PutBlob(ctx context.Context, id blob.ID, data blob.Bytes) error {
s.mutex.Lock()
defer s.mutex.Unlock()
if _, ok := s.data[id]; ok {
return nil
}
s.keyTime[id] = s.timeNow()
var b bytes.Buffer
data.WriteTo(&b) //nolint:errcheck
s.data[id] = b.Bytes()
return nil
}
func (s *mapStorage) DeleteBlob(ctx context.Context, id blob.ID) error {
s.mutex.Lock()
defer s.mutex.Unlock()
delete(s.data, id)
delete(s.keyTime, id)
return nil
}
func (s *mapStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback func(blob.Metadata) error) error {
s.mutex.RLock()
keys := []blob.ID{}
for k := range s.data {
if strings.HasPrefix(string(k), string(prefix)) {
keys = append(keys, k)
}
}
s.mutex.RUnlock()
sort.Slice(keys, func(i, j int) bool {
return keys[i] < keys[j]
})
for _, k := range keys {
s.mutex.RLock()
v, ok := s.data[k]
ts := s.keyTime[k]
s.mutex.RUnlock()
if !ok {
continue
}
if err := callback(blob.Metadata{
BlobID: k,
Length: int64(len(v)),
Timestamp: ts,
}); err != nil {
return err
}
}
return nil
}
func (s *mapStorage) Close(ctx context.Context) error {
return nil
}
func (s *mapStorage) TouchBlob(ctx context.Context, blobID blob.ID, threshold time.Duration) error {
s.mutex.Lock()
defer s.mutex.Unlock()
if v, ok := s.keyTime[blobID]; ok {
n := s.timeNow()
if n.Sub(v) >= threshold {
s.keyTime[blobID] = n
}
}
return nil
}
func (s *mapStorage) ConnectionInfo() blob.ConnectionInfo {
// unsupported
return blob.ConnectionInfo{}
}
// NewMapStorage returns an implementation of Storage backed by the contents of given map.
// Used primarily for testing.
func NewMapStorage(data DataMap, keyTime map[blob.ID]time.Time, timeNow func() time.Time) blob.Storage {
if keyTime == nil {
keyTime = make(map[blob.ID]time.Time)
}
if timeNow == nil {
timeNow = time.Now
}
return &mapStorage{data: data, keyTime: keyTime, timeNow: timeNow}
}