From 1f1465f4ba30ef45eff3d61793e7a37bcadcd684 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sun, 7 Mar 2021 11:25:21 -0800 Subject: [PATCH] Improvements and cleanups for connecting to kopia server (#870) * repo: refactored connect code set up cache for server repositories - improved logic to close the cache on last connection - preemptively add all contents with a prefix to the cache - refactored how config is loaded and saved Now cache dir will be stored as relative and resolved to absolute as part of loading and saving the file, in all other places cache dir is expected to be absolute. * server: removed cache directory from the API and UI This won't be easily available and does not seem useful to expose anyway. * cli: enabled cache commands for server repositories * cli: added KOPIA_CACHE_DIRECTORY environment variable This is used on two occassions - when setting up connection (it gets persisted in the config) and later when opening (to override the cache location from config). It makes setting up docker container with mounted cache somewhat easier with one environment variable. * cli: show cache size for the server cache * tls: present more helpful error message that includes SHA256 fingerprint of the TLS server on mismatch * server: return the name of user who attempted to login when authentication fails --- cli/command_cache_clear.go | 11 ++- cli/command_cache_info.go | 22 +++-- cli/command_cache_set.go | 11 ++- cli/command_repository_connect.go | 2 +- cli/command_repository_connect_from_config.go | 10 +-- htmlui/src/RepoStatus.js | 4 - internal/server/api_repo.go | 1 - internal/server/grpc_session.go | 4 +- internal/serverapi/serverapi.go | 1 - internal/tlsutil/tlsutil.go | 10 ++- repo/api_server_repository.go | 38 +++++---- repo/caching.go | 77 +++++++++++++++++ repo/connect.go | 83 ++---------------- repo/grpc_repository_client.go | 14 +++- repo/local_config.go | 56 +++++++++---- repo/local_config_test.go | 84 +++++++++++++++++++ repo/open.go | 33 +------- repo/repository.go | 8 -- 18 files changed, 288 insertions(+), 181 deletions(-) create mode 100644 repo/caching.go create mode 100644 repo/local_config_test.go diff --git a/cli/command_cache_clear.go b/cli/command_cache_clear.go index c482323af..6ddba9f37 100644 --- a/cli/command_cache_clear.go +++ b/cli/command_cache_clear.go @@ -16,8 +16,13 @@ cacheClearCommandPartial = cacheClearCommand.Flag("partial", "Specifies the cache to clear").Enum("contents", "indexes", "metadata", "own-writes", "blob-list") ) -func runCacheClearCommand(ctx context.Context, rep repo.DirectRepository) error { - d := rep.CachingOptions().CacheDirectory +func runCacheClearCommand(ctx context.Context, rep repo.Repository) error { + opts, err := repo.GetCachingOptions(ctx, repositoryConfigFileName()) + if err != nil { + return errors.Wrap(err, "error getting caching options") + } + + d := opts.CacheDirectory if d == "" { return errors.New("caching not enabled") } @@ -52,5 +57,5 @@ func clearCacheDirectory(ctx context.Context, d string) error { } func init() { - cacheClearCommand.Action(directRepositoryReadAction(runCacheClearCommand)) + cacheClearCommand.Action(repositoryReaderAction(runCacheClearCommand)) } diff --git a/cli/command_cache_info.go b/cli/command_cache_info.go index f082e2782..f720c43e4 100644 --- a/cli/command_cache_info.go +++ b/cli/command_cache_info.go @@ -17,20 +17,26 @@ cacheInfoPathOnly = cacheInfoCommand.Flag("path", "Only display cache path").Bool() ) -func runCacheInfoCommand(ctx context.Context, rep repo.DirectRepository) error { +func runCacheInfoCommand(ctx context.Context, rep repo.Repository) error { + opts, err := repo.GetCachingOptions(ctx, repositoryConfigFileName()) + if err != nil { + return errors.Wrap(err, "error getting cache options") + } + if *cacheInfoPathOnly { - fmt.Println(rep.CachingOptions().CacheDirectory) + fmt.Println(opts.CacheDirectory) return nil } - entries, err := ioutil.ReadDir(rep.CachingOptions().CacheDirectory) + entries, err := ioutil.ReadDir(opts.CacheDirectory) if err != nil { return errors.Wrap(err, "unable to scan cache directory") } path2Limit := map[string]int64{ - "contents": rep.CachingOptions().MaxCacheSizeBytes, - "metadata": rep.CachingOptions().MaxMetadataCacheSizeBytes, + "contents": opts.MaxCacheSizeBytes, + "metadata": opts.MaxMetadataCacheSizeBytes, + "server-contents": opts.MaxCacheSizeBytes, } for _, ent := range entries { @@ -38,7 +44,7 @@ func runCacheInfoCommand(ctx context.Context, rep repo.DirectRepository) error { continue } - subdir := filepath.Join(rep.CachingOptions().CacheDirectory, ent.Name()) + subdir := filepath.Join(opts.CacheDirectory, ent.Name()) fileCount, totalFileSize, err := scanCacheDir(subdir) if err != nil { @@ -51,7 +57,7 @@ func runCacheInfoCommand(ctx context.Context, rep repo.DirectRepository) error { } if ent.Name() == "blob-list" { - maybeLimit = fmt.Sprintf(" (duration %vs)", rep.CachingOptions().MaxListCacheDurationSec) + maybeLimit = fmt.Sprintf(" (duration %vs)", opts.MaxListCacheDurationSec) } fmt.Printf("%v: %v files %v%v\n", subdir, fileCount, units.BytesStringBase10(totalFileSize), maybeLimit) @@ -64,5 +70,5 @@ func runCacheInfoCommand(ctx context.Context, rep repo.DirectRepository) error { } func init() { - cacheInfoCommand.Action(directRepositoryReadAction(runCacheInfoCommand)) + cacheInfoCommand.Action(repositoryReaderAction(runCacheInfoCommand)) } diff --git a/cli/command_cache_set.go b/cli/command_cache_set.go index 9278bc221..db060a397 100644 --- a/cli/command_cache_set.go +++ b/cli/command_cache_set.go @@ -18,8 +18,11 @@ cacheSetMaxListCacheDuration = cacheSetParamsCommand.Flag("max-list-cache-duration", "Duration of index cache").Default("-1ns").Duration() ) -func runCacheSetCommand(ctx context.Context, rep repo.DirectRepositoryWriter) error { - opts := rep.CachingOptions().CloneOrDefault() +func runCacheSetCommand(ctx context.Context, rep repo.RepositoryWriter) error { + opts, err := repo.GetCachingOptions(ctx, repositoryConfigFileName()) + if err != nil { + return errors.Wrap(err, "error getting caching options") + } changed := 0 @@ -53,9 +56,9 @@ func runCacheSetCommand(ctx context.Context, rep repo.DirectRepositoryWriter) er return errors.Errorf("no changes") } - return rep.SetCachingOptions(ctx, opts) + return repo.SetCachingOptions(ctx, repositoryConfigFileName(), opts) } func init() { - cacheSetParamsCommand.Action(directRepositoryWriteAction(runCacheSetCommand)) + cacheSetParamsCommand.Action(repositoryWriterAction(runCacheSetCommand)) } diff --git a/cli/command_repository_connect.go b/cli/command_repository_connect.go index fda6ec294..0d6d6a1cf 100644 --- a/cli/command_repository_connect.go +++ b/cli/command_repository_connect.go @@ -31,7 +31,7 @@ func setupConnectOptions(cmd *kingpin.CmdClause) { // Set up flags shared between 'create' and 'connect'. Note that because those flags are used by both command // we must use *Var() methods, otherwise one of the commands would always get default flag values. cmd.Flag("persist-credentials", "Persist credentials").Default("true").BoolVar(&connectPersistCredentials) - cmd.Flag("cache-directory", "Cache directory").PlaceHolder("PATH").StringVar(&connectCacheDirectory) + cmd.Flag("cache-directory", "Cache directory").PlaceHolder("PATH").Envar("KOPIA_CACHE_DIRECTORY").StringVar(&connectCacheDirectory) cmd.Flag("content-cache-size-mb", "Size of local content cache").PlaceHolder("MB").Default("5000").Int64Var(&connectMaxCacheSizeMB) cmd.Flag("metadata-cache-size-mb", "Size of local metadata cache").PlaceHolder("MB").Default("5000").Int64Var(&connectMaxMetadataCacheSizeMB) cmd.Flag("max-list-cache-duration", "Duration of index cache").Default("30s").Hidden().DurationVar(&connectMaxListCacheDuration) diff --git a/cli/command_repository_connect_from_config.go b/cli/command_repository_connect_from_config.go index 7475ba295..d7c56a5e6 100644 --- a/cli/command_repository_connect_from_config.go +++ b/cli/command_repository_connect_from_config.go @@ -2,7 +2,6 @@ import ( "context" - "os" "github.com/alecthomas/kingpin" "github.com/pkg/errors" @@ -33,17 +32,10 @@ func connectToStorageFromConfig(ctx context.Context, isNew bool) (blob.Storage, } func connectToStorageFromConfigFile(ctx context.Context) (blob.Storage, error) { - var cfg repo.LocalConfig - - f, err := os.Open(connectFromConfigFile) //nolint:gosec + cfg, err := repo.LoadConfigFromFile(connectFromConfigFile) if err != nil { return nil, errors.Wrap(err, "unable to open config") } - defer f.Close() //nolint:errcheck,gosec - - if err := cfg.Load(f); err != nil { - return nil, errors.Wrap(err, "unable to load config") - } return blob.NewStorage(ctx, *cfg.Storage) } diff --git a/htmlui/src/RepoStatus.js b/htmlui/src/RepoStatus.js index bdc9b1325..73c30a740 100644 --- a/htmlui/src/RepoStatus.js +++ b/htmlui/src/RepoStatus.js @@ -141,10 +141,6 @@ export class RepoStatus extends Component { Config File - - Cache Directory - - diff --git a/internal/server/api_repo.go b/internal/server/api_repo.go index 66526ba28..8a2d57e9b 100644 --- a/internal/server/api_repo.go +++ b/internal/server/api_repo.go @@ -49,7 +49,6 @@ func (s *Server) handleRepoStatus(ctx context.Context, r *http.Request, body []b return &serverapi.StatusResponse{ Connected: true, ConfigFile: dr.ConfigFilename(), - CacheDir: dr.CachingOptions().CacheDirectory, Hash: dr.ContentReader().ContentFormat().Hash, Encryption: dr.ContentReader().ContentFormat().Encryption, MaxPackSize: dr.ContentReader().ContentFormat().MaxPackSize, diff --git a/internal/server/grpc_session.go b/internal/server/grpc_session.go index acafe0468..e7380dcf9 100644 --- a/internal/server/grpc_session.go +++ b/internal/server/grpc_session.go @@ -59,9 +59,11 @@ func (s *Server) authenticateGRPCSession(ctx context.Context) (string, error) { if s.authenticator(ctx, s.rep, username, password) { return username, nil } + + return "", status.Errorf(codes.PermissionDenied, "access denied for %v", username) } - return "", status.Errorf(codes.PermissionDenied, "access denied") + return "", status.Errorf(codes.PermissionDenied, "missing credentials") } // Session handles GRPC session from a repository client. diff --git a/internal/serverapi/serverapi.go b/internal/serverapi/serverapi.go index 7222dcfff..db57fd5f3 100644 --- a/internal/serverapi/serverapi.go +++ b/internal/serverapi/serverapi.go @@ -20,7 +20,6 @@ type StatusResponse struct { Connected bool `json:"connected"` ConfigFile string `json:"configFile,omitempty"` - CacheDir string `json:"cacheDir,omitempty"` Hash string `json:"hash,omitempty"` Encryption string `json:"encryption,omitempty"` Splitter string `json:"splitter,omitempty"` diff --git a/internal/tlsutil/tlsutil.go b/internal/tlsutil/tlsutil.go index b238dc3b7..23305aa25 100644 --- a/internal/tlsutil/tlsutil.go +++ b/internal/tlsutil/tlsutil.go @@ -136,13 +136,19 @@ func verifyPeerCertificate(sha256Fingerprint string) func(rawCerts [][]byte, ver sha256Fingerprint = strings.ToLower(sha256Fingerprint) return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + var serverCerts []string + for _, c := range rawCerts { h := sha256.Sum256(c) - if hex.EncodeToString(h[:]) == sha256Fingerprint { + serverCert := hex.EncodeToString(h[:]) + + if serverCert == sha256Fingerprint { return nil } + + serverCerts = append(serverCerts, serverCert) } - return errors.Errorf("can't find certificate matching SHA256 fingerprint %q", sha256Fingerprint) + return errors.Errorf("can't find certificate matching SHA256 fingerprint %q (server had %v)", sha256Fingerprint, serverCerts) } } diff --git a/repo/api_server_repository.go b/repo/api_server_repository.go index a885078ee..db10f46a2 100644 --- a/repo/api_server_repository.go +++ b/repo/api_server_repository.go @@ -5,10 +5,7 @@ "encoding/hex" "encoding/json" "fmt" - "io/ioutil" "net/url" - "os" - "path/filepath" "time" "github.com/pkg/errors" @@ -40,7 +37,8 @@ type apiServerRepository struct { omgr *object.Manager wso WriteSessionOptions - contentCache *cache.PersistentCache + isSharedReadOnlySession bool + contentCache *cache.PersistentCache } func (r *apiServerRepository) APIServerURL() string { @@ -148,6 +146,7 @@ func (r *apiServerRepository) NewWriter(ctx context.Context, opt WriteSessionOpt w.omgr = omgr w.wso = opt + w.isSharedReadOnlySession = false return w, nil } @@ -195,6 +194,11 @@ func (r *apiServerRepository) WriteContent(ctx context.Context, data []byte, pre return "", errors.Wrapf(err, "error writing content %v", contentID) } + if prefix != "" { + // add all prefixed contents to the cache. + r.contentCache.Put(ctx, string(contentID), data) + } + return contentID, nil } @@ -204,8 +208,17 @@ func (r *apiServerRepository) UpdateDescription(d string) { } func (r *apiServerRepository) Close(ctx context.Context) error { - if err := r.omgr.Close(); err != nil { - return errors.Wrap(err, "error closing object manager") + if r.omgr != nil { + if err := r.omgr.Close(); err != nil { + return errors.Wrap(err, "error closing object manager") + } + + r.omgr = nil + } + + if r.isSharedReadOnlySession && r.contentCache != nil { + r.contentCache.Close(ctx) + r.contentCache = nil } return nil @@ -233,6 +246,7 @@ func openRestAPIRepository(ctx context.Context, si *APIServerInfo, cliOpts Clien wso: WriteSessionOptions{ OnUpload: func(i int64) {}, }, + isSharedReadOnlySession: true, } var p remoterepoapi.Parameters @@ -265,19 +279,13 @@ func ConnectAPIServer(ctx context.Context, configFile string, si *APIServerInfo, lc := LocalConfig{ APIServer: si, ClientOptions: opt.ClientOptions.ApplyDefaults(ctx, "API Server: "+si.BaseURL), - Caching: opt.CachingOptions.CloneOrDefault(), } - d, err := json.MarshalIndent(&lc, "", " ") - if err != nil { - return errors.Wrap(err, "unable to marshal config JSON") + if err := setupCachingOptionsWithDefaults(ctx, configFile, &lc, &opt.CachingOptions, []byte(si.BaseURL)); err != nil { + return errors.Wrap(err, "unable to set up caching") } - if err = os.MkdirAll(filepath.Dir(configFile), 0o700); err != nil { - return errors.Wrap(err, "unable to create config directory") - } - - if err = ioutil.WriteFile(configFile, d, 0o600); err != nil { + if err := lc.writeToFile(configFile); err != nil { return errors.Wrap(err, "unable to write config file") } diff --git a/repo/caching.go b/repo/caching.go new file mode 100644 index 000000000..5a27b91ea --- /dev/null +++ b/repo/caching.go @@ -0,0 +1,77 @@ +package repo + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/repo/content" +) + +// GetCachingOptions reads caching configuration for a given repository. +func GetCachingOptions(ctx context.Context, configFile string) (*content.CachingOptions, error) { + lc, err := LoadConfigFromFile(configFile) + if err != nil { + return nil, err + } + + return lc.Caching.CloneOrDefault(), nil +} + +// SetCachingOptions changes caching configuration for a given repository. +func SetCachingOptions(ctx context.Context, configFile string, opt *content.CachingOptions) error { + lc, err := LoadConfigFromFile(configFile) + if err != nil { + return err + } + + if err = setupCachingOptionsWithDefaults(ctx, configFile, lc, opt, nil); err != nil { + return errors.Wrap(err, "unable to set up caching") + } + + return lc.writeToFile(configFile) +} + +func setupCachingOptionsWithDefaults(ctx context.Context, configPath string, lc *LocalConfig, opt *content.CachingOptions, uniqueID []byte) error { + opt = opt.CloneOrDefault() + + if opt.MaxCacheSizeBytes == 0 { + lc.Caching = &content.CachingOptions{} + return nil + } + + if lc.Caching == nil { + lc.Caching = &content.CachingOptions{} + } + + if opt.CacheDirectory == "" { + cacheDir, err := os.UserCacheDir() + if err != nil { + return errors.Wrap(err, "unable to determine cache directory") + } + + h := sha256.New() + h.Write(uniqueID) //nolint:errcheck + h.Write([]byte(configPath)) //nolint:errcheck + lc.Caching.CacheDirectory = filepath.Join(cacheDir, "kopia", hex.EncodeToString(h.Sum(nil))[0:16]) + } else { + d, err := filepath.Abs(opt.CacheDirectory) + if err != nil { + return errors.Wrap(err, "unable to determine absolute cache path") + } + + lc.Caching.CacheDirectory = d + } + + lc.Caching.MaxCacheSizeBytes = opt.MaxCacheSizeBytes + lc.Caching.MaxMetadataCacheSizeBytes = opt.MaxMetadataCacheSizeBytes + lc.Caching.MaxListCacheDurationSec = opt.MaxListCacheDurationSec + + log(ctx).Debugf("Creating cache directory '%v' with max size %v", lc.Caching.CacheDirectory, lc.Caching.MaxCacheSizeBytes) + + return nil +} diff --git a/repo/connect.go b/repo/connect.go index 274b408c8..cd4c340b3 100644 --- a/repo/connect.go +++ b/repo/connect.go @@ -2,10 +2,6 @@ import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "io/ioutil" "os" "path/filepath" @@ -54,20 +50,11 @@ func Connect(ctx context.Context, configFile string, st blob.Storage, password s lc.Storage = &ci lc.ClientOptions = opt.ClientOptions.ApplyDefaults(ctx, "Repository in "+st.DisplayName()) - if err = setupCaching(ctx, configFile, &lc, &opt.CachingOptions, f.UniqueID); err != nil { + if err = setupCachingOptionsWithDefaults(ctx, configFile, &lc, &opt.CachingOptions, f.UniqueID); err != nil { return errors.Wrap(err, "unable to set up caching") } - d, err := json.MarshalIndent(&lc, "", " ") - if err != nil { - return errors.Wrap(err, "unable to serialize JSON") - } - - if err = os.MkdirAll(filepath.Dir(configFile), 0o700); err != nil { - return errors.Wrap(err, "unable to create config directory") - } - - if err = ioutil.WriteFile(configFile, d, 0o600); err != nil { + if err := lc.writeToFile(configFile); err != nil { return errors.Wrap(err, "unable to write config file") } @@ -98,56 +85,9 @@ func verifyConnect(ctx context.Context, configFile, password string, persist boo return r.Close(ctx) } -func setupCaching(ctx context.Context, configPath string, lc *LocalConfig, opt *content.CachingOptions, uniqueID []byte) error { - opt = opt.CloneOrDefault() - - if opt.MaxCacheSizeBytes == 0 { - lc.Caching = &content.CachingOptions{} - return nil - } - - if lc.Caching == nil { - lc.Caching = &content.CachingOptions{} - } - - if opt.CacheDirectory == "" { - cacheDir, err := os.UserCacheDir() - if err != nil { - return errors.Wrap(err, "unable to determine cache directory") - } - - h := sha256.New() - h.Write(uniqueID) //nolint:errcheck - h.Write([]byte(configPath)) //nolint:errcheck - opt.CacheDirectory = filepath.Join(cacheDir, "kopia", hex.EncodeToString(h.Sum(nil))[0:16]) - } - - var err error - - // try computing relative pathname from config dir to the cache dir. - lc.Caching.CacheDirectory, err = filepath.Rel(filepath.Dir(configPath), opt.CacheDirectory) - - if err != nil { - // fall back to storing absolute path - lc.Caching.CacheDirectory, err = filepath.Abs(opt.CacheDirectory) - } - - if err != nil { - return errors.Wrap(err, "error computing cache directory") - } - - lc.Caching.MaxCacheSizeBytes = opt.MaxCacheSizeBytes - lc.Caching.MaxMetadataCacheSizeBytes = opt.MaxMetadataCacheSizeBytes - lc.Caching.MaxListCacheDurationSec = opt.MaxListCacheDurationSec - - log(ctx).Debugf("Creating cache directory '%v' with max size %v", lc.Caching.CacheDirectory, lc.Caching.MaxCacheSizeBytes) - - return nil -} - // Disconnect removes the specified configuration file and any local cache directories. func Disconnect(ctx context.Context, configFile string) error { - cfg, err := loadConfigFromFile(configFile) + cfg, err := LoadConfigFromFile(configFile) if err != nil { return err } @@ -155,6 +95,10 @@ func Disconnect(ctx context.Context, configFile string) error { deletePassword(ctx, configFile) if cfg.Caching != nil && cfg.Caching.CacheDirectory != "" { + if !filepath.IsAbs(cfg.Caching.CacheDirectory) { + return errors.Errorf("cache directory was not absolute, refusing to delete") + } + if err = os.RemoveAll(cfg.Caching.CacheDirectory); err != nil { log(ctx).Warningf("unable to remove cache directory: %v", err) } @@ -170,21 +114,12 @@ func Disconnect(ctx context.Context, configFile string) error { // SetClientOptions updates client options stored in the provided configuration file. func SetClientOptions(ctx context.Context, configFile string, cliOpt ClientOptions) error { - lc, err := loadConfigFromFile(configFile) + lc, err := LoadConfigFromFile(configFile) if err != nil { return err } lc.ClientOptions = cliOpt - d, err := json.MarshalIndent(lc, "", " ") - if err != nil { - return errors.Wrap(err, "error marshaling config JSON") - } - - if err = ioutil.WriteFile(configFile, d, 0o600); err != nil { - return errors.Wrap(err, "unable to write config file") - } - - return nil + return lc.writeToFile(configFile) } diff --git a/repo/grpc_repository_client.go b/repo/grpc_repository_client.go index 90c19f406..fe7909cba 100644 --- a/repo/grpc_repository_client.go +++ b/repo/grpc_repository_client.go @@ -481,7 +481,7 @@ func errorFromSessionResponse(rr *apipb.ErrorResponse) error { case apipb.ErrorResponse_CONTENT_NOT_FOUND: return content.ErrContentNotFound case apipb.ErrorResponse_STREAM_BROKEN: - return io.EOF + return errors.Wrap(io.EOF, rr.Message) default: return errors.New(rr.Message) } @@ -552,6 +552,11 @@ func (r *grpcRepositoryClient) WriteContent(ctx context.Context, data []byte, pr return "", err } + if prefix != "" { + // add all prefixed contents to the cache. + r.contentCache.Put(ctx, string(contentID), data) + } + return v.(content.ID), nil } @@ -586,10 +591,17 @@ func (r *grpcRepositoryClient) UpdateDescription(d string) { } func (r *grpcRepositoryClient) Close(ctx context.Context) error { + if r.omgr == nil { + // already closed + return nil + } + if err := r.omgr.Close(); err != nil { return errors.Wrap(err, "error closing object manager") } + r.omgr = nil + if atomic.AddInt32(r.connRefCount, -1) == 0 { log(ctx).Debugf("closing GPRC connection to %v", r.conn.Target()) diff --git a/repo/local_config.go b/repo/local_config.go index 8a4df85f6..3da2c5b6c 100644 --- a/repo/local_config.go +++ b/repo/local_config.go @@ -1,13 +1,15 @@ package repo import ( + "bytes" "context" "encoding/json" - "io" "os" + "path/filepath" "github.com/pkg/errors" + "github.com/kopia/kopia/internal/atomicfile" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/content" "github.com/kopia/kopia/repo/object" @@ -88,26 +90,34 @@ type repositoryObjectFormat struct { object.Format } -// Load reads local configuration from the specified reader. -func (lc *LocalConfig) Load(r io.Reader) error { - *lc = LocalConfig{} - return json.NewDecoder(r).Decode(lc) -} +// writeToFile writes the config to a given file. +func (lc *LocalConfig) writeToFile(filename string) error { + lc2 := *lc -// Save writes the configuration to the specified writer. -func (lc *LocalConfig) Save(w io.Writer) error { - b, err := json.MarshalIndent(lc, "", " ") - if err != nil { - return nil + if lc.Caching != nil { + lc2.Caching = lc.Caching.CloneOrDefault() + + // try computing relative pathname from config dir to the cache dir. + d, err := filepath.Rel(filepath.Dir(filename), lc.Caching.CacheDirectory) + if err == nil { + lc2.Caching.CacheDirectory = d + } } - _, err = w.Write(b) + b, err := json.MarshalIndent(lc2, "", " ") + if err != nil { + return errors.Wrap(err, "error creating config file contents") + } - return errors.Wrap(err, "error saving local config") + if err = os.MkdirAll(filepath.Dir(filename), 0o700); err != nil { + return errors.Wrap(err, "unable to create config directory") + } + + return errors.Wrap(atomicfile.Write(filename, bytes.NewReader(b)), "error writing file") } -// loadConfigFromFile reads the local configuration from the specified file. -func loadConfigFromFile(fileName string) (*LocalConfig, error) { +// LoadConfigFromFile reads the local configuration from the specified file. +func LoadConfigFromFile(fileName string) (*LocalConfig, error) { f, err := os.Open(fileName) //nolint:gosec if err != nil { return nil, errors.Wrap(err, "error loading config file") @@ -116,8 +126,20 @@ func loadConfigFromFile(fileName string) (*LocalConfig, error) { var lc LocalConfig - if err := lc.Load(f); err != nil { - return nil, err + if err := json.NewDecoder(f).Decode(&lc); err != nil { + return nil, errors.Wrap(err, "error decoding config json") + } + + // cache directory is stored as relative to config file name, resolve it to absolute. + if lc.Caching != nil { + if lc.Caching.CacheDirectory != "" && !filepath.IsAbs(lc.Caching.CacheDirectory) { + lc.Caching.CacheDirectory = filepath.Join(filepath.Dir(fileName), lc.Caching.CacheDirectory) + } + + // override cache directory from the environment variable. + if cd := os.Getenv("KOPIA_CACHE_DIRECTORY"); cd != "" && filepath.IsAbs(cd) { + lc.Caching.CacheDirectory = cd + } } return &lc, nil diff --git a/repo/local_config_test.go b/repo/local_config_test.go new file mode 100644 index 000000000..63564dfb4 --- /dev/null +++ b/repo/local_config_test.go @@ -0,0 +1,84 @@ +package repo + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/repo/content" +) + +func TestLocalConfig_withCaching(t *testing.T) { + td := testutil.TempDirectory(t) + + originalLC := &LocalConfig{ + Caching: &content.CachingOptions{ + CacheDirectory: filepath.Join(td, "cache-dir"), + }, + } + + cfgFile := filepath.Join(td, "repository.config") + must(t, originalLC.writeToFile(cfgFile)) + + rawLC := LocalConfig{} + mustParseJSONFile(t, cfgFile, &rawLC) + + loadedLC, err := LoadConfigFromFile(cfgFile) + must(t, err) + + if filepath.IsAbs(rawLC.Caching.CacheDirectory) { + t.Fatalf("cache directory must be stored relative, was %v", rawLC.Caching.CacheDirectory) + } + + if got, want := loadedLC.Caching.CacheDirectory, originalLC.Caching.CacheDirectory; got != want { + t.Fatalf("cache directory did not round trip: %v, want %v", got, want) + } +} + +func TestLocalConfig_noCaching(t *testing.T) { + td := testutil.TempDirectory(t) + + originalLC := &LocalConfig{} + + cfgFile := filepath.Join(td, "repository.config") + must(t, originalLC.writeToFile(cfgFile)) + + rawLC := LocalConfig{} + mustParseJSONFile(t, cfgFile, &rawLC) + + loadedLC, err := LoadConfigFromFile(cfgFile) + must(t, err) + + if got, want := loadedLC.Caching, originalLC.Caching; got != want { + t.Fatalf("cacheing did not round trip: %v, want %v", got, want) + } +} + +func TestLocalConfig_notFound(t *testing.T) { + if _, err := LoadConfigFromFile("nosuchfile.json"); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unexpected error %v: wanted ErrNotExist", err) + } +} + +func must(t *testing.T, err error) { + t.Helper() + + if err != nil { + t.Fatal(err) + } +} + +func mustParseJSONFile(t *testing.T, fname string, o interface{}) { + t.Helper() + + f, err := os.Open(fname) + must(t, err) + + defer f.Close() + + must(t, json.NewDecoder(f).Decode(o)) +} diff --git a/repo/open.go b/repo/open.go index 74ecf7778..227d928f7 100644 --- a/repo/open.go +++ b/repo/open.go @@ -3,7 +3,6 @@ import ( "bytes" "context" - "encoding/json" "io/ioutil" "os" "path/filepath" @@ -72,18 +71,11 @@ func Open(ctx context.Context, configFile, password string, options *Options) (r return nil, errors.Wrap(err, "error resolving config file path") } - lc, err := loadConfigFromFile(configFile) + lc, err := LoadConfigFromFile(configFile) if err != nil { return nil, err } - // cache directory is stored as relative to config file name, resolve it to absolute. - if lc.Caching != nil { - if lc.Caching.CacheDirectory != "" && !filepath.IsAbs(lc.Caching.CacheDirectory) { - lc.Caching.CacheDirectory = filepath.Join(filepath.Dir(configFile), lc.Caching.CacheDirectory) - } - } - if lc.APIServer != nil { return OpenAPIServer(ctx, lc.APIServer, lc.ClientOptions, lc.Caching, password) } @@ -279,29 +271,6 @@ func writeCacheMarker(cacheDir string) error { return f.Close() } -// SetCachingOptions changes caching configuration for a given repository. -func (r *directRepository) SetCachingOptions(ctx context.Context, opt *content.CachingOptions) error { - lc, err := loadConfigFromFile(r.configFile) - if err != nil { - return err - } - - if err = setupCaching(ctx, r.configFile, lc, opt, r.uniqueID); err != nil { - return errors.Wrap(err, "unable to set up caching") - } - - d, err := json.MarshalIndent(&lc, "", " ") - if err != nil { - return errors.Wrap(err, "error marshaling JSON") - } - - if err := ioutil.WriteFile(r.configFile, d, 0o600); err != nil { - return nil - } - - return nil -} - func readAndCacheFormatBlobBytes(ctx context.Context, st blob.Storage, cacheDirectory string) ([]byte, error) { cachedFile := filepath.Join(cacheDirectory, "kopia.repository") diff --git a/repo/repository.go b/repo/repository.go index b1e379506..1fbbf8eef 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -59,9 +59,6 @@ type DirectRepository interface { ConfigFilename() string DeriveKey(purpose []byte, keyLength int) []byte Token(password string) (string, error) - - CachingOptions() *content.CachingOptions - SetCachingOptions(ctx context.Context, opt *content.CachingOptions) error } // DirectRepositoryWriter provides low-level write access to the repository. @@ -238,11 +235,6 @@ func (r *directRepository) Flush(ctx context.Context) error { return r.cmgr.Flush(ctx) } -// CachingOptions returns caching options. -func (r *directRepository) CachingOptions() *content.CachingOptions { - return r.cachingOptions.CloneOrDefault() -} - // ObjectFormat returns the object format. func (r *directRepository) ObjectFormat() object.Format { return r.omgr.Format