diff --git a/cli/command_cache_clear.go b/cli/command_cache_clear.go index c482323af..6ddba9f37 100644 --- a/cli/command_cache_clear.go +++ b/cli/command_cache_clear.go @@ -16,8 +16,13 @@ cacheClearCommandPartial = cacheClearCommand.Flag("partial", "Specifies the cache to clear").Enum("contents", "indexes", "metadata", "own-writes", "blob-list") ) -func runCacheClearCommand(ctx context.Context, rep repo.DirectRepository) error { - d := rep.CachingOptions().CacheDirectory +func runCacheClearCommand(ctx context.Context, rep repo.Repository) error { + opts, err := repo.GetCachingOptions(ctx, repositoryConfigFileName()) + if err != nil { + return errors.Wrap(err, "error getting caching options") + } + + d := opts.CacheDirectory if d == "" { return errors.New("caching not enabled") } @@ -52,5 +57,5 @@ func clearCacheDirectory(ctx context.Context, d string) error { } func init() { - cacheClearCommand.Action(directRepositoryReadAction(runCacheClearCommand)) + cacheClearCommand.Action(repositoryReaderAction(runCacheClearCommand)) } diff --git a/cli/command_cache_info.go b/cli/command_cache_info.go index f082e2782..f720c43e4 100644 --- a/cli/command_cache_info.go +++ b/cli/command_cache_info.go @@ -17,20 +17,26 @@ cacheInfoPathOnly = cacheInfoCommand.Flag("path", "Only display cache path").Bool() ) -func runCacheInfoCommand(ctx context.Context, rep repo.DirectRepository) error { +func runCacheInfoCommand(ctx context.Context, rep repo.Repository) error { + opts, err := repo.GetCachingOptions(ctx, repositoryConfigFileName()) + if err != nil { + return errors.Wrap(err, "error getting cache options") + } + if *cacheInfoPathOnly { - fmt.Println(rep.CachingOptions().CacheDirectory) + fmt.Println(opts.CacheDirectory) return nil } - entries, err := ioutil.ReadDir(rep.CachingOptions().CacheDirectory) + entries, err := ioutil.ReadDir(opts.CacheDirectory) if err != nil { return errors.Wrap(err, "unable to scan cache directory") } path2Limit := map[string]int64{ - "contents": rep.CachingOptions().MaxCacheSizeBytes, - "metadata": rep.CachingOptions().MaxMetadataCacheSizeBytes, + "contents": opts.MaxCacheSizeBytes, + "metadata": opts.MaxMetadataCacheSizeBytes, + "server-contents": opts.MaxCacheSizeBytes, } for _, ent := range entries { @@ -38,7 +44,7 @@ func runCacheInfoCommand(ctx context.Context, rep repo.DirectRepository) error { continue } - subdir := filepath.Join(rep.CachingOptions().CacheDirectory, ent.Name()) + subdir := filepath.Join(opts.CacheDirectory, ent.Name()) fileCount, totalFileSize, err := scanCacheDir(subdir) if err != nil { @@ -51,7 +57,7 @@ func runCacheInfoCommand(ctx context.Context, rep repo.DirectRepository) error { } if ent.Name() == "blob-list" { - maybeLimit = fmt.Sprintf(" (duration %vs)", rep.CachingOptions().MaxListCacheDurationSec) + maybeLimit = fmt.Sprintf(" (duration %vs)", opts.MaxListCacheDurationSec) } fmt.Printf("%v: %v files %v%v\n", subdir, fileCount, units.BytesStringBase10(totalFileSize), maybeLimit) @@ -64,5 +70,5 @@ func runCacheInfoCommand(ctx context.Context, rep repo.DirectRepository) error { } func init() { - cacheInfoCommand.Action(directRepositoryReadAction(runCacheInfoCommand)) + cacheInfoCommand.Action(repositoryReaderAction(runCacheInfoCommand)) } diff --git a/cli/command_cache_set.go b/cli/command_cache_set.go index 9278bc221..db060a397 100644 --- a/cli/command_cache_set.go +++ b/cli/command_cache_set.go @@ -18,8 +18,11 @@ cacheSetMaxListCacheDuration = cacheSetParamsCommand.Flag("max-list-cache-duration", "Duration of index cache").Default("-1ns").Duration() ) -func runCacheSetCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { - opts := rep.CachingOptions().CloneOrDefault() +func runCacheSetCommand(ctx context.Context, rep repo.RepositoryWriter) error { + opts, err := repo.GetCachingOptions(ctx, repositoryConfigFileName()) + if err != nil { + return errors.Wrap(err, "error getting caching options") + } changed := 0 @@ -53,9 +56,9 @@ func runCacheSetCommand(ctx context.Context, rep repo.DirectRepositoryWriter) er return errors.Errorf("no changes") } - return rep.SetCachingOptions(ctx, opts) + return repo.SetCachingOptions(ctx, repositoryConfigFileName(), opts) } func init() { - cacheSetParamsCommand.Action(directRepositoryWriteAction(runCacheSetCommand)) + cacheSetParamsCommand.Action(repositoryWriterAction(runCacheSetCommand)) } diff --git a/cli/command_repository_connect.go b/cli/command_repository_connect.go index fda6ec294..0d6d6a1cf 100644 --- a/cli/command_repository_connect.go +++ b/cli/command_repository_connect.go @@ -31,7 +31,7 @@ func setupConnectOptions(cmd *kingpin.CmdClause) { // Set up flags shared between 'create' and 'connect'. Note that because those flags are used by both command // we must use *Var() methods, otherwise one of the commands would always get default flag values. cmd.Flag("persist-credentials", "Persist credentials").Default("true").BoolVar(&connectPersistCredentials) - cmd.Flag("cache-directory", "Cache directory").PlaceHolder("PATH").StringVar(&connectCacheDirectory) + cmd.Flag("cache-directory", "Cache directory").PlaceHolder("PATH").Envar("KOPIA_CACHE_DIRECTORY").StringVar(&connectCacheDirectory) cmd.Flag("content-cache-size-mb", "Size of local content cache").PlaceHolder("MB").Default("5000").Int64Var(&connectMaxCacheSizeMB) cmd.Flag("metadata-cache-size-mb", "Size of local metadata cache").PlaceHolder("MB").Default("5000").Int64Var(&connectMaxMetadataCacheSizeMB) cmd.Flag("max-list-cache-duration", "Duration of index cache").Default("30s").Hidden().DurationVar(&connectMaxListCacheDuration) diff --git a/cli/command_repository_connect_from_config.go b/cli/command_repository_connect_from_config.go index 7475ba295..d7c56a5e6 100644 --- a/cli/command_repository_connect_from_config.go +++ b/cli/command_repository_connect_from_config.go @@ -2,7 +2,6 @@ import ( "context" - "os" "github.com/alecthomas/kingpin" "github.com/pkg/errors" @@ -33,17 +32,10 @@ func connectToStorageFromConfig(ctx context.Context, isNew bool) (blob.Storage, } func connectToStorageFromConfigFile(ctx context.Context) (blob.Storage, error) { - var cfg repo.LocalConfig - - f, err := os.Open(connectFromConfigFile) //nolint:gosec + cfg, err := repo.LoadConfigFromFile(connectFromConfigFile) if err != nil { return nil, errors.Wrap(err, "unable to open config") } - defer f.Close() //nolint:errcheck,gosec - - if err := cfg.Load(f); err != nil { - return nil, errors.Wrap(err, "unable to load config") - } return blob.NewStorage(ctx, *cfg.Storage) } diff --git a/htmlui/src/RepoStatus.js b/htmlui/src/RepoStatus.js index bdc9b1325..73c30a740 100644 --- a/htmlui/src/RepoStatus.js +++ b/htmlui/src/RepoStatus.js @@ -141,10 +141,6 @@ export class RepoStatus extends Component { Config File - - Cache Directory - - diff --git a/internal/server/api_repo.go b/internal/server/api_repo.go index 66526ba28..8a2d57e9b 100644 --- a/internal/server/api_repo.go +++ b/internal/server/api_repo.go @@ -49,7 +49,6 @@ func (s *Server) handleRepoStatus(ctx context.Context, r *http.Request, body []b return &serverapi.StatusResponse{ Connected: true, ConfigFile: dr.ConfigFilename(), - CacheDir: dr.CachingOptions().CacheDirectory, Hash: dr.ContentReader().ContentFormat().Hash, Encryption: dr.ContentReader().ContentFormat().Encryption, MaxPackSize: dr.ContentReader().ContentFormat().MaxPackSize, diff --git a/internal/server/grpc_session.go b/internal/server/grpc_session.go index acafe0468..e7380dcf9 100644 --- a/internal/server/grpc_session.go +++ b/internal/server/grpc_session.go @@ -59,9 +59,11 @@ func (s *Server) authenticateGRPCSession(ctx context.Context) (string, error) { if s.authenticator(ctx, s.rep, username, password) { return username, nil } + + return "", status.Errorf(codes.PermissionDenied, "access denied for %v", username) } - return "", status.Errorf(codes.PermissionDenied, "access denied") + return "", status.Errorf(codes.PermissionDenied, "missing credentials") } // Session handles GRPC session from a repository client. diff --git a/internal/serverapi/serverapi.go b/internal/serverapi/serverapi.go index 7222dcfff..db57fd5f3 100644 --- a/internal/serverapi/serverapi.go +++ b/internal/serverapi/serverapi.go @@ -20,7 +20,6 @@ type StatusResponse struct { Connected bool `json:"connected"` ConfigFile string `json:"configFile,omitempty"` - CacheDir string `json:"cacheDir,omitempty"` Hash string `json:"hash,omitempty"` Encryption string `json:"encryption,omitempty"` Splitter string `json:"splitter,omitempty"` diff --git a/internal/tlsutil/tlsutil.go b/internal/tlsutil/tlsutil.go index b238dc3b7..23305aa25 100644 --- a/internal/tlsutil/tlsutil.go +++ b/internal/tlsutil/tlsutil.go @@ -136,13 +136,19 @@ func verifyPeerCertificate(sha256Fingerprint string) func(rawCerts [][]byte, ver sha256Fingerprint = strings.ToLower(sha256Fingerprint) return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + var serverCerts []string + for _, c := range rawCerts { h := sha256.Sum256(c) - if hex.EncodeToString(h[:]) == sha256Fingerprint { + serverCert := hex.EncodeToString(h[:]) + + if serverCert == sha256Fingerprint { return nil } + + serverCerts = append(serverCerts, serverCert) } - return errors.Errorf("can't find certificate matching SHA256 fingerprint %q", sha256Fingerprint) + return errors.Errorf("can't find certificate matching SHA256 fingerprint %q (server had %v)", sha256Fingerprint, serverCerts) } } diff --git a/repo/api_server_repository.go b/repo/api_server_repository.go index a885078ee..db10f46a2 100644 --- a/repo/api_server_repository.go +++ b/repo/api_server_repository.go @@ -5,10 +5,7 @@ "encoding/hex" "encoding/json" "fmt" - "io/ioutil" "net/url" - "os" - "path/filepath" "time" "github.com/pkg/errors" @@ -40,7 +37,8 @@ type apiServerRepository struct { omgr *object.Manager wso WriteSessionOptions - contentCache *cache.PersistentCache + isSharedReadOnlySession bool + contentCache *cache.PersistentCache } func (r *apiServerRepository) APIServerURL() string { @@ -148,6 +146,7 @@ func (r *apiServerRepository) NewWriter(ctx context.Context, opt WriteSessionOpt w.omgr = omgr w.wso = opt + w.isSharedReadOnlySession = false return w, nil } @@ -195,6 +194,11 @@ func (r *apiServerRepository) WriteContent(ctx context.Context, data []byte, pre return "", errors.Wrapf(err, "error writing content %v", contentID) } + if prefix != "" { + // add all prefixed contents to the cache. + r.contentCache.Put(ctx, string(contentID), data) + } + return contentID, nil } @@ -204,8 +208,17 @@ func (r *apiServerRepository) UpdateDescription(d string) { } func (r *apiServerRepository) Close(ctx context.Context) error { - if err := r.omgr.Close(); err != nil { - return errors.Wrap(err, "error closing object manager") + if r.omgr != nil { + if err := r.omgr.Close(); err != nil { + return errors.Wrap(err, "error closing object manager") + } + + r.omgr = nil + } + + if r.isSharedReadOnlySession && r.contentCache != nil { + r.contentCache.Close(ctx) + r.contentCache = nil } return nil @@ -233,6 +246,7 @@ func openRestAPIRepository(ctx context.Context, si *APIServerInfo, cliOpts Clien wso: WriteSessionOptions{ OnUpload: func(i int64) {}, }, + isSharedReadOnlySession: true, } var p remoterepoapi.Parameters @@ -265,19 +279,13 @@ func ConnectAPIServer(ctx context.Context, configFile string, si *APIServerInfo, lc := LocalConfig{ APIServer: si, ClientOptions: opt.ClientOptions.ApplyDefaults(ctx, "API Server: "+si.BaseURL), - Caching: opt.CachingOptions.CloneOrDefault(), } - d, err := json.MarshalIndent(&lc, "", " ") - if err != nil { - return errors.Wrap(err, "unable to marshal config JSON") + if err := setupCachingOptionsWithDefaults(ctx, configFile, &lc, &opt.CachingOptions, []byte(si.BaseURL)); err != nil { + return errors.Wrap(err, "unable to set up caching") } - if err = os.MkdirAll(filepath.Dir(configFile), 0o700); err != nil { - return errors.Wrap(err, "unable to create config directory") - } - - if err = ioutil.WriteFile(configFile, d, 0o600); err != nil { + if err := lc.writeToFile(configFile); err != nil { return errors.Wrap(err, "unable to write config file") } diff --git a/repo/caching.go b/repo/caching.go new file mode 100644 index 000000000..5a27b91ea --- /dev/null +++ b/repo/caching.go @@ -0,0 +1,77 @@ +package repo + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/repo/content" +) + +// GetCachingOptions reads caching configuration for a given repository. +func GetCachingOptions(ctx context.Context, configFile string) (*content.CachingOptions, error) { + lc, err := LoadConfigFromFile(configFile) + if err != nil { + return nil, err + } + + return lc.Caching.CloneOrDefault(), nil +} + +// SetCachingOptions changes caching configuration for a given repository. +func SetCachingOptions(ctx context.Context, configFile string, opt *content.CachingOptions) error { + lc, err := LoadConfigFromFile(configFile) + if err != nil { + return err + } + + if err = setupCachingOptionsWithDefaults(ctx, configFile, lc, opt, nil); err != nil { + return errors.Wrap(err, "unable to set up caching") + } + + return lc.writeToFile(configFile) +} + +func setupCachingOptionsWithDefaults(ctx context.Context, configPath string, lc *LocalConfig, opt *content.CachingOptions, uniqueID []byte) error { + opt = opt.CloneOrDefault() + + if opt.MaxCacheSizeBytes == 0 { + lc.Caching = &content.CachingOptions{} + return nil + } + + if lc.Caching == nil { + lc.Caching = &content.CachingOptions{} + } + + if opt.CacheDirectory == "" { + cacheDir, err := os.UserCacheDir() + if err != nil { + return errors.Wrap(err, "unable to determine cache directory") + } + + h := sha256.New() + h.Write(uniqueID) //nolint:errcheck + h.Write([]byte(configPath)) //nolint:errcheck + lc.Caching.CacheDirectory = filepath.Join(cacheDir, "kopia", hex.EncodeToString(h.Sum(nil))[0:16]) + } else { + d, err := filepath.Abs(opt.CacheDirectory) + if err != nil { + return errors.Wrap(err, "unable to determine absolute cache path") + } + + lc.Caching.CacheDirectory = d + } + + lc.Caching.MaxCacheSizeBytes = opt.MaxCacheSizeBytes + lc.Caching.MaxMetadataCacheSizeBytes = opt.MaxMetadataCacheSizeBytes + lc.Caching.MaxListCacheDurationSec = opt.MaxListCacheDurationSec + + log(ctx).Debugf("Creating cache directory '%v' with max size %v", lc.Caching.CacheDirectory, lc.Caching.MaxCacheSizeBytes) + + return nil +} diff --git a/repo/connect.go b/repo/connect.go index 274b408c8..cd4c340b3 100644 --- a/repo/connect.go +++ b/repo/connect.go @@ -2,10 +2,6 @@ import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "io/ioutil" "os" "path/filepath" @@ -54,20 +50,11 @@ func Connect(ctx context.Context, configFile string, st blob.Storage, password s lc.Storage = &ci lc.ClientOptions = opt.ClientOptions.ApplyDefaults(ctx, "Repository in "+st.DisplayName()) - if err = setupCaching(ctx, configFile, &lc, &opt.CachingOptions, f.UniqueID); err != nil { + if err = setupCachingOptionsWithDefaults(ctx, configFile, &lc, &opt.CachingOptions, f.UniqueID); err != nil { return errors.Wrap(err, "unable to set up caching") } - d, err := json.MarshalIndent(&lc, "", " ") - if err != nil { - return errors.Wrap(err, "unable to serialize JSON") - } - - if err = os.MkdirAll(filepath.Dir(configFile), 0o700); err != nil { - return errors.Wrap(err, "unable to create config directory") - } - - if err = ioutil.WriteFile(configFile, d, 0o600); err != nil { + if err := lc.writeToFile(configFile); err != nil { return errors.Wrap(err, "unable to write config file") } @@ -98,56 +85,9 @@ func verifyConnect(ctx context.Context, configFile, password string, persist boo return r.Close(ctx) } -func setupCaching(ctx context.Context, configPath string, lc *LocalConfig, opt *content.CachingOptions, uniqueID []byte) error { - opt = opt.CloneOrDefault() - - if opt.MaxCacheSizeBytes == 0 { - lc.Caching = &content.CachingOptions{} - return nil - } - - if lc.Caching == nil { - lc.Caching = &content.CachingOptions{} - } - - if opt.CacheDirectory == "" { - cacheDir, err := os.UserCacheDir() - if err != nil { - return errors.Wrap(err, "unable to determine cache directory") - } - - h := sha256.New() - h.Write(uniqueID) //nolint:errcheck - h.Write([]byte(configPath)) //nolint:errcheck - opt.CacheDirectory = filepath.Join(cacheDir, "kopia", hex.EncodeToString(h.Sum(nil))[0:16]) - } - - var err error - - // try computing relative pathname from config dir to the cache dir. - lc.Caching.CacheDirectory, err = filepath.Rel(filepath.Dir(configPath), opt.CacheDirectory) - - if err != nil { - // fall back to storing absolute path - lc.Caching.CacheDirectory, err = filepath.Abs(opt.CacheDirectory) - } - - if err != nil { - return errors.Wrap(err, "error computing cache directory") - } - - lc.Caching.MaxCacheSizeBytes = opt.MaxCacheSizeBytes - lc.Caching.MaxMetadataCacheSizeBytes = opt.MaxMetadataCacheSizeBytes - lc.Caching.MaxListCacheDurationSec = opt.MaxListCacheDurationSec - - log(ctx).Debugf("Creating cache directory '%v' with max size %v", lc.Caching.CacheDirectory, lc.Caching.MaxCacheSizeBytes) - - return nil -} - // Disconnect removes the specified configuration file and any local cache directories. func Disconnect(ctx context.Context, configFile string) error { - cfg, err := loadConfigFromFile(configFile) + cfg, err := LoadConfigFromFile(configFile) if err != nil { return err } @@ -155,6 +95,10 @@ func Disconnect(ctx context.Context, configFile string) error { deletePassword(ctx, configFile) if cfg.Caching != nil && cfg.Caching.CacheDirectory != "" { + if !filepath.IsAbs(cfg.Caching.CacheDirectory) { + return errors.Errorf("cache directory was not absolute, refusing to delete") + } + if err = os.RemoveAll(cfg.Caching.CacheDirectory); err != nil { log(ctx).Warningf("unable to remove cache directory: %v", err) } @@ -170,21 +114,12 @@ func Disconnect(ctx context.Context, configFile string) error { // SetClientOptions updates client options stored in the provided configuration file. func SetClientOptions(ctx context.Context, configFile string, cliOpt ClientOptions) error { - lc, err := loadConfigFromFile(configFile) + lc, err := LoadConfigFromFile(configFile) if err != nil { return err } lc.ClientOptions = cliOpt - d, err := json.MarshalIndent(lc, "", " ") - if err != nil { - return errors.Wrap(err, "error marshaling config JSON") - } - - if err = ioutil.WriteFile(configFile, d, 0o600); err != nil { - return errors.Wrap(err, "unable to write config file") - } - - return nil + return lc.writeToFile(configFile) } diff --git a/repo/grpc_repository_client.go b/repo/grpc_repository_client.go index 90c19f406..fe7909cba 100644 --- a/repo/grpc_repository_client.go +++ b/repo/grpc_repository_client.go @@ -481,7 +481,7 @@ func errorFromSessionResponse(rr *apipb.ErrorResponse) error { case apipb.ErrorResponse_CONTENT_NOT_FOUND: return content.ErrContentNotFound case apipb.ErrorResponse_STREAM_BROKEN: - return io.EOF + return errors.Wrap(io.EOF, rr.Message) default: return errors.New(rr.Message) } @@ -552,6 +552,11 @@ func (r *grpcRepositoryClient) WriteContent(ctx context.Context, data []byte, pr return "", err } + if prefix != "" { + // add all prefixed contents to the cache. + r.contentCache.Put(ctx, string(contentID), data) + } + return v.(content.ID), nil } @@ -586,10 +591,17 @@ func (r *grpcRepositoryClient) UpdateDescription(d string) { } func (r *grpcRepositoryClient) Close(ctx context.Context) error { + if r.omgr == nil { + // already closed + return nil + } + if err := r.omgr.Close(); err != nil { return errors.Wrap(err, "error closing object manager") } + r.omgr = nil + if atomic.AddInt32(r.connRefCount, -1) == 0 { log(ctx).Debugf("closing GPRC connection to %v", r.conn.Target()) diff --git a/repo/local_config.go b/repo/local_config.go index 8a4df85f6..3da2c5b6c 100644 --- a/repo/local_config.go +++ b/repo/local_config.go @@ -1,13 +1,15 @@ package repo import ( + "bytes" "context" "encoding/json" - "io" "os" + "path/filepath" "github.com/pkg/errors" + "github.com/kopia/kopia/internal/atomicfile" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/object" @@ -88,26 +90,34 @@ type repositoryObjectFormat struct { object.Format } -// Load reads local configuration from the specified reader. -func (lc *LocalConfig) Load(r io.Reader) error { - *lc = LocalConfig{} - return json.NewDecoder(r).Decode(lc) -} +// writeToFile writes the config to a given file. +func (lc *LocalConfig) writeToFile(filename string) error { + lc2 := *lc -// Save writes the configuration to the specified writer. -func (lc *LocalConfig) Save(w io.Writer) error { - b, err := json.MarshalIndent(lc, "", " ") - if err != nil { - return nil + if lc.Caching != nil { + lc2.Caching = lc.Caching.CloneOrDefault() + + // try computing relative pathname from config dir to the cache dir. + d, err := filepath.Rel(filepath.Dir(filename), lc.Caching.CacheDirectory) + if err == nil { + lc2.Caching.CacheDirectory = d + } } - _, err = w.Write(b) + b, err := json.MarshalIndent(lc2, "", " ") + if err != nil { + return errors.Wrap(err, "error creating config file contents") + } - return errors.Wrap(err, "error saving local config") + if err = os.MkdirAll(filepath.Dir(filename), 0o700); err != nil { + return errors.Wrap(err, "unable to create config directory") + } + + return errors.Wrap(atomicfile.Write(filename, bytes.NewReader(b)), "error writing file") } -// loadConfigFromFile reads the local configuration from the specified file. -func loadConfigFromFile(fileName string) (*LocalConfig, error) { +// LoadConfigFromFile reads the local configuration from the specified file. +func LoadConfigFromFile(fileName string) (*LocalConfig, error) { f, err := os.Open(fileName) //nolint:gosec if err != nil { return nil, errors.Wrap(err, "error loading config file") @@ -116,8 +126,20 @@ func loadConfigFromFile(fileName string) (*LocalConfig, error) { var lc LocalConfig - if err := lc.Load(f); err != nil { - return nil, err + if err := json.NewDecoder(f).Decode(&lc); err != nil { + return nil, errors.Wrap(err, "error decoding config json") + } + + // cache directory is stored as relative to config file name, resolve it to absolute. + if lc.Caching != nil { + if lc.Caching.CacheDirectory != "" && !filepath.IsAbs(lc.Caching.CacheDirectory) { + lc.Caching.CacheDirectory = filepath.Join(filepath.Dir(fileName), lc.Caching.CacheDirectory) + } + + // override cache directory from the environment variable. + if cd := os.Getenv("KOPIA_CACHE_DIRECTORY"); cd != "" && filepath.IsAbs(cd) { + lc.Caching.CacheDirectory = cd + } } return &lc, nil diff --git a/repo/local_config_test.go b/repo/local_config_test.go new file mode 100644 index 000000000..63564dfb4 --- /dev/null +++ b/repo/local_config_test.go @@ -0,0 +1,84 @@ +package repo + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/repo/content" +) + +func TestLocalConfig_withCaching(t *testing.T) { + td := testutil.TempDirectory(t) + + originalLC := &LocalConfig{ + Caching: &content.CachingOptions{ + CacheDirectory: filepath.Join(td, "cache-dir"), + }, + } + + cfgFile := filepath.Join(td, "repository.config") + must(t, originalLC.writeToFile(cfgFile)) + + rawLC := LocalConfig{} + mustParseJSONFile(t, cfgFile, &rawLC) + + loadedLC, err := LoadConfigFromFile(cfgFile) + must(t, err) + + if filepath.IsAbs(rawLC.Caching.CacheDirectory) { + t.Fatalf("cache directory must be stored relative, was %v", rawLC.Caching.CacheDirectory) + } + + if got, want := loadedLC.Caching.CacheDirectory, originalLC.Caching.CacheDirectory; got != want { + t.Fatalf("cache directory did not round trip: %v, want %v", got, want) + } +} + +func TestLocalConfig_noCaching(t *testing.T) { + td := testutil.TempDirectory(t) + + originalLC := &LocalConfig{} + + cfgFile := filepath.Join(td, "repository.config") + must(t, originalLC.writeToFile(cfgFile)) + + rawLC := LocalConfig{} + mustParseJSONFile(t, cfgFile, &rawLC) + + loadedLC, err := LoadConfigFromFile(cfgFile) + must(t, err) + + if got, want := loadedLC.Caching, originalLC.Caching; got != want { + t.Fatalf("cacheing did not round trip: %v, want %v", got, want) + } +} + +func TestLocalConfig_notFound(t *testing.T) { + if _, err := LoadConfigFromFile("nosuchfile.json"); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unexpected error %v: wanted ErrNotExist", err) + } +} + +func must(t *testing.T, err error) { + t.Helper() + + if err != nil { + t.Fatal(err) + } +} + +func mustParseJSONFile(t *testing.T, fname string, o interface{}) { + t.Helper() + + f, err := os.Open(fname) + must(t, err) + + defer f.Close() + + must(t, json.NewDecoder(f).Decode(o)) +} diff --git a/repo/open.go b/repo/open.go index 74ecf7778..227d928f7 100644 --- a/repo/open.go +++ b/repo/open.go @@ -3,7 +3,6 @@ import ( "bytes" "context" - "encoding/json" "io/ioutil" "os" "path/filepath" @@ -72,18 +71,11 @@ func Open(ctx context.Context, configFile, password string, options *Options) (r return nil, errors.Wrap(err, "error resolving config file path") } - lc, err := loadConfigFromFile(configFile) + lc, err := LoadConfigFromFile(configFile) if err != nil { return nil, err } - // cache directory is stored as relative to config file name, resolve it to absolute. - if lc.Caching != nil { - if lc.Caching.CacheDirectory != "" && !filepath.IsAbs(lc.Caching.CacheDirectory) { - lc.Caching.CacheDirectory = filepath.Join(filepath.Dir(configFile), lc.Caching.CacheDirectory) - } - } - if lc.APIServer != nil { return OpenAPIServer(ctx, lc.APIServer, lc.ClientOptions, lc.Caching, password) } @@ -279,29 +271,6 @@ func writeCacheMarker(cacheDir string) error { return f.Close() } -// SetCachingOptions changes caching configuration for a given repository. -func (r *directRepository) SetCachingOptions(ctx context.Context, opt *content.CachingOptions) error { - lc, err := loadConfigFromFile(r.configFile) - if err != nil { - return err - } - - if err = setupCaching(ctx, r.configFile, lc, opt, r.uniqueID); err != nil { - return errors.Wrap(err, "unable to set up caching") - } - - d, err := json.MarshalIndent(&lc, "", " ") - if err != nil { - return errors.Wrap(err, "error marshaling JSON") - } - - if err := ioutil.WriteFile(r.configFile, d, 0o600); err != nil { - return nil - } - - return nil -} - func readAndCacheFormatBlobBytes(ctx context.Context, st blob.Storage, cacheDirectory string) ([]byte, error) { cachedFile := filepath.Join(cacheDirectory, "kopia.repository") diff --git a/repo/repository.go b/repo/repository.go index b1e379506..1fbbf8eef 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -59,9 +59,6 @@ type DirectRepository interface { ConfigFilename() string DeriveKey(purpose []byte, keyLength int) []byte Token(password string) (string, error) - - CachingOptions() *content.CachingOptions - SetCachingOptions(ctx context.Context, opt *content.CachingOptions) error } // DirectRepositoryWriter provides low-level write access to the repository. @@ -238,11 +235,6 @@ func (r *directRepository) Flush(ctx context.Context) error { return r.cmgr.Flush(ctx) } -// CachingOptions returns caching options. -func (r *directRepository) CachingOptions() *content.CachingOptions { - return r.cachingOptions.CloneOrDefault() -} - // ObjectFormat returns the object format. func (r *directRepository) ObjectFormat() object.Format { return r.omgr.Format