mirror of
https://github.com/kopia/kopia.git
synced 2026-03-13 11:46:55 -04:00
, where blob.Storage.PutBlob gets a list of slices and writes them sequentially * performance: added gather.Bytes and gather.WriteBuffer They are similar to bytes.Buffer but instead of managing a single byte slice, they maintain a list of slices that and when they run out of space they allocate new fixed-size slice from a free list. This helps keep memory allocations completely under control regardless of the size of data written. * switch from byte slices and bytes.Buffer to gather.Bytes. This is mostly mechanical, the only cases where it's not involve blob storage providers, where we leverage the fact that we don't need to ever concatenate the slices into one and instead we can do gather writes. * PR feedback
156 lines
2.8 KiB
Go
156 lines
2.8 KiB
Go
package blobtesting
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/kopia/kopia/repo/blob"
|
|
)
|
|
|
|
// DataMap is a map of blob ID to their contents.
|
|
type DataMap map[blob.ID][]byte
|
|
|
|
type mapStorage struct {
|
|
data DataMap
|
|
keyTime map[blob.ID]time.Time
|
|
timeNow func() time.Time
|
|
mutex sync.RWMutex
|
|
}
|
|
|
|
func (s *mapStorage) GetBlob(ctx context.Context, id blob.ID, offset, length int64) ([]byte, error) {
|
|
s.mutex.RLock()
|
|
defer s.mutex.RUnlock()
|
|
|
|
data, ok := s.data[id]
|
|
if ok {
|
|
data = append([]byte(nil), data...)
|
|
|
|
if length < 0 {
|
|
return data, nil
|
|
}
|
|
|
|
if int(offset) > len(data) || offset < 0 {
|
|
return nil, errors.New("invalid offset")
|
|
}
|
|
|
|
data = data[offset:]
|
|
if int(length) > len(data) {
|
|
return nil, errors.New("invalid length")
|
|
}
|
|
|
|
return data[0:length], nil
|
|
}
|
|
|
|
return nil, blob.ErrBlobNotFound
|
|
}
|
|
|
|
func (s *mapStorage) PutBlob(ctx context.Context, id blob.ID, data blob.Bytes) error {
|
|
s.mutex.Lock()
|
|
defer s.mutex.Unlock()
|
|
|
|
if _, ok := s.data[id]; ok {
|
|
return nil
|
|
}
|
|
|
|
s.keyTime[id] = s.timeNow()
|
|
|
|
var b bytes.Buffer
|
|
|
|
data.WriteTo(&b) //nolint:errcheck
|
|
|
|
s.data[id] = b.Bytes()
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *mapStorage) DeleteBlob(ctx context.Context, id blob.ID) error {
|
|
s.mutex.Lock()
|
|
defer s.mutex.Unlock()
|
|
|
|
delete(s.data, id)
|
|
delete(s.keyTime, id)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *mapStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback func(blob.Metadata) error) error {
|
|
s.mutex.RLock()
|
|
|
|
keys := []blob.ID{}
|
|
|
|
for k := range s.data {
|
|
if strings.HasPrefix(string(k), string(prefix)) {
|
|
keys = append(keys, k)
|
|
}
|
|
}
|
|
|
|
s.mutex.RUnlock()
|
|
|
|
sort.Slice(keys, func(i, j int) bool {
|
|
return keys[i] < keys[j]
|
|
})
|
|
|
|
for _, k := range keys {
|
|
s.mutex.RLock()
|
|
v, ok := s.data[k]
|
|
ts := s.keyTime[k]
|
|
s.mutex.RUnlock()
|
|
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
if err := callback(blob.Metadata{
|
|
BlobID: k,
|
|
Length: int64(len(v)),
|
|
Timestamp: ts,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *mapStorage) Close(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
func (s *mapStorage) TouchBlob(ctx context.Context, blobID blob.ID, threshold time.Duration) error {
|
|
s.mutex.Lock()
|
|
defer s.mutex.Unlock()
|
|
|
|
if v, ok := s.keyTime[blobID]; ok {
|
|
n := s.timeNow()
|
|
if n.Sub(v) >= threshold {
|
|
s.keyTime[blobID] = n
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *mapStorage) ConnectionInfo() blob.ConnectionInfo {
|
|
// unsupported
|
|
return blob.ConnectionInfo{}
|
|
}
|
|
|
|
// NewMapStorage returns an implementation of Storage backed by the contents of given map.
|
|
// Used primarily for testing.
|
|
func NewMapStorage(data DataMap, keyTime map[blob.ID]time.Time, timeNow func() time.Time) blob.Storage {
|
|
if keyTime == nil {
|
|
keyTime = make(map[blob.ID]time.Time)
|
|
}
|
|
|
|
if timeNow == nil {
|
|
timeNow = time.Now
|
|
}
|
|
|
|
return &mapStorage{data: data, keyTime: keyTime, timeNow: timeNow}
|
|
}
|