diff --git a/cli/command_repository_set_parameters.go b/cli/command_repository_set_parameters.go index 00e160129..4470f4575 100644 --- a/cli/command_repository_set_parameters.go +++ b/cli/command_repository_set_parameters.go @@ -121,9 +121,29 @@ func (c *commandRepositorySetParameters) setRetentionModeParameter(ctx context.C log(ctx).Infof(" - setting %v to %s.\n", desc, v) } -func (c *commandRepositorySetParameters) run(ctx context.Context, rep repo.DirectRepositoryWriter) error { - var anyChange bool +func (c *commandRepositorySetParameters) updateEpochParameters(mp *format.MutableParameters, anyChange, upgradeToEpochManager *bool) { + *anyChange = true + if !mp.EpochParameters.Enabled { + mp.EpochParameters = epoch.DefaultParameters() + mp.IndexVersion = 2 + *upgradeToEpochManager = true + } + + if mp.Version < format.FormatVersion2 { + mp.Version = format.FormatVersion2 + } +} + +func (c *commandRepositorySetParameters) disableBlobRetention(ctx context.Context, blobcfg *format.BlobStorageConfiguration, anyChange *bool) { + log(ctx).Infof("disabling blob retention") + + blobcfg.RetentionMode = "" + blobcfg.RetentionPeriod = 0 + *anyChange = true +} + +func (c *commandRepositorySetParameters) run(ctx context.Context, rep repo.DirectRepositoryWriter) error { mp, err := rep.FormatManager().GetMutableParameters() if err != nil { return errors.Wrap(err, "mutable parameters") @@ -139,38 +159,28 @@ func (c *commandRepositorySetParameters) run(ctx context.Context, rep repo.Direc return errors.Wrap(err, "unable to get required features") } + anyChange := false upgradeToEpochManager := false if c.upgradeRepositoryFormat { - anyChange = true - - if !mp.EpochParameters.Enabled { - mp.EpochParameters = epoch.DefaultParameters() - mp.IndexVersion = 2 - upgradeToEpochManager = true - } - - if mp.Version < format.FormatVersion2 { - mp.Version = format.FormatVersion2 - } + c.updateEpochParameters(&mp, &anyChange, &upgradeToEpochManager) } c.setSizeMBParameter(ctx, c.maxPackSizeMB, "maximum pack size", &mp.MaxPackSize, &anyChange) // prevent downgrade of index format - if c.indexFormatVersion > mp.IndexVersion { - c.setIntParameter(ctx, c.indexFormatVersion, "index format version", &mp.IndexVersion, &anyChange) - } else if c.indexFormatVersion > 0 && c.indexFormatVersion != mp.IndexVersion { - return errors.Errorf("index format version can only be upgraded") + if c.indexFormatVersion != 0 { + if c.indexFormatVersion > mp.IndexVersion { + c.setIntParameter(ctx, c.indexFormatVersion, "index format version", &mp.IndexVersion, &anyChange) + } else { + return errors.Errorf("index format version can only be upgraded") + } } if c.retentionMode == "none" { if blobcfg.IsRetentionEnabled() { - log(ctx).Infof("disabling blob retention") - - blobcfg.RetentionMode = "" - blobcfg.RetentionPeriod = 0 - anyChange = true + // disable blob retention if already enabled + c.disableBlobRetention(ctx, &blobcfg, &anyChange) } } else { c.setRetentionModeParameter(ctx, blob.RetentionMode(c.retentionMode), "storage backend blob retention mode", &blobcfg.RetentionMode, &anyChange) diff --git a/cli/command_repository_upgrade.go b/cli/command_repository_upgrade.go index 9d3aeccec..f54c8a494 100644 --- a/cli/command_repository_upgrade.go +++ b/cli/command_repository_upgrade.go @@ -42,9 +42,8 @@ type commandRepositoryUpgrade struct { ) const ( - commitModeCommitOnValidationSuccess string = "" - commitModeAlwaysCommit = "always" - commitModeNeverCommit = "never" + commitModeAlwaysCommit = "always" + commitModeNeverCommit = "never" ) func (c *commandRepositoryUpgrade) setup(svc advancedAppServices, parent commandParent) { @@ -91,22 +90,21 @@ func (c *commandRepositoryUpgrade) setup(svc advancedAppServices, parent command c.svc = svc } -// assign store the info struct in a map that can be used to compare indexes +// assign store the info struct in a map that can be used to compare indexes. func assign(iif content.Info, i int, m map[content.ID][2]index.Info) { v := m[iif.GetContentID()] v[i] = iif m[iif.GetContentID()] = v } -// loadIndexBlobs load index blobs into indexEntries map +// loadIndexBlobs load index blobs into indexEntries map. func loadIndexBlobs(ctx context.Context, indexEntries map[content.ID][2]index.Info, sm *content.SharedManager, which int, indexBlobInfos []content.IndexBlobInfo) error { d := gather.WriteBuffer{} - - for _, indexBlobInfo := range indexBlobInfos { + for _, indexBlobInfo := range indexBlobInfos { blobID := indexBlobInfo.BlobID - indexInfos, err := sm.LoadIndexBlob(ctx, blobID, d) + indexInfos, err := sm.LoadIndexBlob(ctx, blobID, &d) if err != nil { return errors.Wrapf(err, "failed to load index blob with BlobID %s", blobID) } @@ -114,15 +112,14 @@ func loadIndexBlobs(ctx context.Context, indexEntries map[content.ID][2]index.In for _, indexInfo := range indexInfos { assign(indexInfo, which, indexEntries) } - } + return nil } // validateAction returns an error of the new V1 index blob content does not match the source V0 index blob content. // This is used to check that the upgraded index (V1 index) reflects the content of the old V0 index. func (c *commandRepositoryUpgrade) validateAction(ctx context.Context, rep repo.DirectRepositoryWriter) error { - indexEntries := map[content.ID][2]index.Info{} sm := rep.ContentManager().SharedManager @@ -158,10 +155,12 @@ func (c *commandRepositoryUpgrade) validateAction(ctx context.Context, rep repo. } else { err = checkIndexInfo(indexEntryPairs[0], indexEntryPairs[1]) } + if err != nil { break } } + if err == nil { log(ctx).Infof("index validation succeeded") return nil @@ -172,12 +171,14 @@ func (c *commandRepositoryUpgrade) validateAction(ctx context.Context, rep repo. log(ctx).Errorf("%v", err) return nil } + return err } // checkIndexInfo compare two index infos. If a mismatch exists, return an error with diagnostic information. func checkIndexInfo(i0, i1 index.Info) error { var err error + switch { case i0.GetFormatVersion() != i1.GetFormatVersion(): err = errors.Errorf("mismatched FormatVersions: %v %v", i0.GetFormatVersion(), i1.GetFormatVersion()) @@ -196,11 +197,13 @@ func checkIndexInfo(i0, i1 index.Info) error { case i0.GetTimestampSeconds() != i1.GetTimestampSeconds(): err = errors.Errorf("mismatched TimestampSeconds: %v %v", i0.GetTimestampSeconds(), i1.GetTimestampSeconds()) } + if err != nil { return errors.Wrapf(err, "index blobs do not match: %v, %v", string(i0.GetPackBlobID()), string(i1.GetPackBlobID())) } + return nil } @@ -429,6 +432,7 @@ func (c *commandRepositoryUpgrade) commitUpgrade(ctx context.Context, rep repo.D log(ctx).Infof("Commit mode is set to 'never'. Skipping commit.") return nil } + if err := rep.FormatManager().CommitUpgrade(ctx); err != nil { return errors.Wrap(err, "error finalizing upgrade") } diff --git a/cli/command_repository_upgrade_test.go b/cli/command_repository_upgrade_test.go index aaeaa4538..a1c18ab99 100644 --- a/cli/command_repository_upgrade_test.go +++ b/cli/command_repository_upgrade_test.go @@ -95,7 +95,6 @@ func (s *formatSpecificTestSuite) TestRepositoryCorruptedUpgrade(t *testing.T) { "--status-poll-interval", "1s", "--max-permitted-clock-drift", "1s") } - } func (s *formatSpecificTestSuite) TestRepositoryUpgradeCommitNever(t *testing.T) { diff --git a/repo/content/committed_read_manager.go b/repo/content/committed_read_manager.go index 2a03b94d1..ff05020cf 100644 --- a/repo/content/committed_read_manager.go +++ b/repo/content/committed_read_manager.go @@ -119,8 +119,8 @@ type SharedManager struct { } // LoadIndexBlob return index information loaded from the specified blob. -func (sm *SharedManager) LoadIndexBlob(ctx context.Context, ibid blob.ID, d gather.WriteBuffer) ([]Info, error) { - err := sm.st.GetBlob(ctx, ibid, 0, -1, &d) +func (sm *SharedManager) LoadIndexBlob(ctx context.Context, ibid blob.ID, d *gather.WriteBuffer) ([]Info, error) { + err := sm.st.GetBlob(ctx, ibid, 0, -1, d) if err != nil { return nil, errors.Wrapf(err, "could not find index blob %q", ibid) } diff --git a/repo/content/index_blob_manager_v0.go b/repo/content/index_blob_manager_v0.go index c6e8ea487..46fe5e203 100644 --- a/repo/content/index_blob_manager_v0.go +++ b/repo/content/index_blob_manager_v0.go @@ -67,7 +67,6 @@ type indexBlobManagerV0 struct { // ListIndexBlobInfos list active blob info structs. Also returns time of latest content deletion commit. func (m *indexBlobManagerV0) ListIndexBlobInfos(ctx context.Context) ([]IndexBlobInfo, time.Time, error) { - activeIndexBlobs, t0, err := m.listActiveIndexBlobs(ctx) if err != nil { return nil, time.Time{}, err @@ -80,7 +79,7 @@ func (m *indexBlobManagerV0) ListIndexBlobInfos(ctx context.Context) ([]IndexBlo if activeIndexBlob.BlobID == format.LegacyIndexPoisonBlobID { continue } - + q = append(q, activeIndexBlob) } diff --git a/tests/testenv/cli_test_env.go b/tests/testenv/cli_test_env.go index 27091fa62..45f1dd31c 100644 --- a/tests/testenv/cli_test_env.go +++ b/tests/testenv/cli_test_env.go @@ -109,7 +109,7 @@ func (e *CLITest) RunAndExpectSuccess(t *testing.T, args ...string) []string { return stdout } -// TweakFile writes a 0x00 byte at a random point in a file. Used to simulate file corruption +// TweakFile writes a xor-ed byte at a random point in a file. Used to simulate file corruption. func (e *CLITest) TweakFile(t *testing.T, dirn, fglob string) { t.Helper() @@ -118,13 +118,16 @@ func (e *CLITest) TweakFile(t *testing.T, dirn, fglob string) { mch, err := fs.Glob(os.DirFS(dirn), fglob) require.NoError(t, err) require.Greater(t, len(mch), 0) + // grab a random file in the directory dirn fn := mch[rand.Intn(len(mch))] f, err := os.OpenFile(path.Join(dirn, fn), os.O_RDWR, os.FileMode(RwUserGroupOther)) require.NoError(t, err) + // find the length of the file, then seek to a random location l, err := f.Seek(0, io.SeekEnd) require.NoError(t, err) + i := rand.Int63n(l) bs := [1]byte{}