mirror of
https://github.com/kopia/kopia.git
synced 2026-01-05 04:57:53 -05:00
78edd92692
This may be a breaking change for users who rely on particular kopia metrics (unlikely):
- introduced blob-level metrics:
* `kopia_blob_download_full_blob_bytes_total`
* `kopia_blob_download_partial_blob_bytes_total`
* `kopia_blob_upload_bytes_total`
* `kopia_blob_storage_latency_ms` - per-method latency distribution
* `kopia_blob_errors_total` - per-method error counter
- updated cache metrics to indicate particular cache
* `kopia_cache_hit_bytes_total{cache="CACHE_TYPE"}`
* `kopia_cache_hit_total{cache="CACHE_TYPE"}`
* `kopia_cache_malformed_total{cache="CACHE_TYPE"}`
* `kopia_cache_miss_total{cache="CACHE_TYPE"}`
* `kopia_cache_miss_errors_total{cache="CACHE_TYPE"}`
* `kopia_cache_miss_bytes_total{cache="CACHE_TYPE"}`
* `kopia_cache_store_errors_total{cache="CACHE_TYPE"}`
where `CACHE_TYPE` is one of `contents`, `metadata` or `index-blobs`
- reorganized and unified content-level metrics:
* `kopia_content_write_bytes_total`
* `kopia_content_write_duration_nanos_total`
* `kopia_content_compression_attempted_bytes_total`
* `kopia_content_compression_attempted_duration_nanos_total`
* `kopia_content_compression_savings_bytes_total`
* `kopia_content_compressible_bytes_total`
* `kopia_content_non_compressible_bytes_total`
* `kopia_content_after_compression_bytes_total`
* `kopia_content_decompressed_bytes_total`
* `kopia_content_decompressed_duration_nanos_total`
* `kopia_content_encrypted_bytes_total`
* `kopia_content_encrypted_duration_nanos_total`
* `kopia_content_hashed_bytes_total`
* `kopia_content_hashed_duration_nanos_total`
* `kopia_content_deduplicated_bytes_total`
* `kopia_content_read_bytes_total`
* `kopia_content_read_duration_nanos_total`
* `kopia_content_decrypted_bytes_total`
* `kopia_content_decrypted_duration_nanos_total`
* `kopia_content_uploaded_bytes_total`
Also introduced `internal/metrics` framework which constructs Prometheus metrics in a uniform way and will allow us to include some of these metrics in telemetry report in future PRs.
94 lines
2.9 KiB
Go
94 lines
2.9 KiB
Go
package cache_test
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/kopia/kopia/internal/blobtesting"
|
|
"github.com/kopia/kopia/internal/cache"
|
|
"github.com/kopia/kopia/internal/gather"
|
|
"github.com/kopia/kopia/internal/testlogging"
|
|
"github.com/kopia/kopia/internal/testutil"
|
|
"github.com/kopia/kopia/repo/blob"
|
|
)
|
|
|
|
func TestContentCacheForMetadata(t *testing.T) {
|
|
ctx := testlogging.Context(t)
|
|
|
|
underlyingData := blobtesting.DataMap{}
|
|
underlying := blobtesting.NewMapStorage(underlyingData, nil, nil)
|
|
|
|
td := testutil.TempDirectory(t)
|
|
|
|
metadataCache, err := cache.NewContentCache(ctx, underlying, cache.Options{
|
|
BaseCacheDirectory: td,
|
|
CacheSubDir: "subdir",
|
|
HMACSecret: []byte{1, 2, 3},
|
|
FetchFullBlobs: true,
|
|
Sweep: cache.SweepSettings{
|
|
MaxSizeBytes: 100,
|
|
},
|
|
}, nil)
|
|
require.NoError(t, err)
|
|
|
|
cacheStorage := metadataCache.CacheStorage()
|
|
|
|
var tmp gather.WriteBuffer
|
|
defer tmp.Close()
|
|
|
|
// get something we don't have in the underlying storage
|
|
require.ErrorIs(t, metadataCache.GetContent(ctx, "key1", "blob1", 0, 3, &tmp), blob.ErrBlobNotFound)
|
|
|
|
require.NoError(t, underlying.PutBlob(ctx, "blob1", gather.FromSlice([]byte{1, 2, 3, 4, 5, 6}), blob.PutOptions{}))
|
|
|
|
require.NoError(t, metadataCache.GetContent(ctx, "key1", "blob1", 0, -1, &tmp))
|
|
require.Equal(t, []byte{1, 2, 3, 4, 5, 6}, tmp.ToByteSlice())
|
|
|
|
require.NoError(t, metadataCache.GetContent(ctx, "key1", "blob1", 0, 3, &tmp))
|
|
require.Equal(t, []byte{1, 2, 3}, tmp.ToByteSlice())
|
|
|
|
cacheEntries, err := blob.ListAllBlobs(ctx, cacheStorage, "")
|
|
require.NoError(t, err)
|
|
// cache has the entire blob
|
|
require.Len(t, cacheEntries, 1)
|
|
|
|
require.NoError(t, metadataCache.GetContent(ctx, "key1", "blob1", 3, 3, &tmp))
|
|
require.Equal(t, []byte{4, 5, 6}, tmp.ToByteSlice())
|
|
|
|
cacheEntries, err = blob.ListAllBlobs(ctx, cacheStorage, "")
|
|
require.NoError(t, err)
|
|
// cache has the entire blob
|
|
require.Len(t, cacheEntries, 1)
|
|
|
|
metadataCache.Close(ctx)
|
|
|
|
// get slice with cache miss
|
|
require.NoError(t, underlying.PutBlob(ctx, "blob2", gather.FromSlice([]byte{1, 2, 3, 4, 5, 6}), blob.PutOptions{}))
|
|
require.NoError(t, metadataCache.GetContent(ctx, "key1", "blob2", 3, 3, &tmp))
|
|
require.Equal(t, []byte{4, 5, 6}, tmp.ToByteSlice())
|
|
}
|
|
|
|
func TestContentCacheForMetadata_Passthrough(t *testing.T) {
|
|
underlyingData := blobtesting.DataMap{}
|
|
underlying := blobtesting.NewMapStorage(underlyingData, nil, nil)
|
|
|
|
ctx := testlogging.Context(t)
|
|
|
|
metadataCache, err := cache.NewContentCache(ctx, underlying, cache.Options{
|
|
BaseCacheDirectory: "",
|
|
Sweep: cache.SweepSettings{
|
|
MaxSizeBytes: 100,
|
|
},
|
|
}, nil)
|
|
|
|
require.NoError(t, err)
|
|
require.NoError(t, underlying.PutBlob(ctx, "blob1", gather.FromSlice([]byte{1, 2, 3, 4, 5, 6}), blob.PutOptions{}))
|
|
|
|
var tmp gather.WriteBuffer
|
|
defer tmp.Close()
|
|
|
|
require.NoError(t, metadataCache.GetContent(ctx, "key1", "blob1", 0, -1, &tmp))
|
|
require.Equal(t, []byte{1, 2, 3, 4, 5, 6}, tmp.ToByteSlice())
|
|
}
|