diff --git a/block/block_formatting_options.go b/block/block_formatting_options.go index 857e390cf..674c454e3 100644 --- a/block/block_formatting_options.go +++ b/block/block_formatting_options.go @@ -2,10 +2,9 @@ package block // FormattingOptions describes the rules for formatting blocks in repository. type FormattingOptions struct { - Version int `json:"version,omitempty"` // version number, must be "1" - BlockFormat string `json:"objectFormat,omitempty"` // identifier of the block format - HMACSecret []byte `json:"secret,omitempty"` // HMAC secret used to generate encryption keys - MasterKey []byte `json:"masterKey,omitempty"` // master encryption key (SIV-mode encryption only) - MaxPackedContentLength int `json:"maxPackedContentLength,omitempty"` // maximum size of object to be considered for storage in a pack - MaxPackSize int `json:"maxPackSize,omitempty"` // maximum size of a pack object + Version int `json:"version,omitempty"` // version number, must be "1" + BlockFormat string `json:"objectFormat,omitempty"` // identifier of the block format + HMACSecret []byte `json:"secret,omitempty"` // HMAC secret used to generate encryption keys + MasterKey []byte `json:"masterKey,omitempty"` // master encryption key (SIV-mode encryption only) + MaxPackSize int `json:"maxPackSize,omitempty"` // maximum size of a pack object } diff --git a/block/block_manager.go b/block/block_manager.go index 96c70178e..40368a680 100644 --- a/block/block_manager.go +++ b/block/block_manager.go @@ -29,6 +29,7 @@ const ( defaultMinPreambleLength = 32 defaultMaxPreambleLength = 32 defaultPaddingUnit = 4096 + maxInlineContentLength = 100000 // amount of block data to store in the index block itself ) var zeroTime time.Time @@ -66,7 +67,7 @@ type Manager struct { currentPackData []byte currentPackIndex *blockmgrpb.Index - maxPackedContentLength int + maxInlineContentLength int maxPackSize int formatter Formatter @@ -91,6 +92,7 @@ func (bm *Manager) DeleteBlock(blockID string) error { delete(bm.blockIDToIndex, blockID) delete(bm.currentPackIndex.Items, blockID) + delete(bm.currentPackIndex.InlineItems, blockID) bm.currentPackIndex.DeletedItems = append(bm.currentPackIndex.DeletedItems, blockID) return nil } @@ -184,6 +186,7 @@ func (bm *Manager) close() error { func (bm *Manager) startPackIndexLocked() { bm.currentPackIndex = &blockmgrpb.Index{ Items: make(map[string]uint64), + InlineItems: make(map[string][]byte), CreateTimeNanos: uint64(bm.timeNow().UnixNano()), } bm.currentPackData = []byte{} @@ -234,7 +237,17 @@ func (bm *Manager) finishAllOpenPacksLocked() error { } func (bm *Manager) finishPackLocked() error { - if len(bm.currentPackIndex.Items)+len(bm.currentPackIndex.DeletedItems) > 0 { + if len(bm.currentPackIndex.Items)+len(bm.currentPackIndex.DeletedItems)+len(bm.currentPackIndex.InlineItems) > 0 { + if len(bm.currentPackData) < bm.maxInlineContentLength { + for k, os := range bm.currentPackIndex.Items { + offset, size := unpackOffsetAndSize(os) + dat := bm.currentPackData[offset : offset+size] + bm.currentPackIndex.InlineItems[k] = dat + } + + bm.currentPackIndex.Items = nil + bm.currentPackData = nil + } if bm.currentPackData != nil { if bm.paddingUnit > 0 { if missing := bm.paddingUnit - (len(bm.currentPackData) % bm.paddingUnit); missing > 0 { @@ -372,6 +385,10 @@ func (bm *Manager) loadMergedPackIndexLocked() ([]*blockmgrpb.Index, []string, t return } + // for i, v := range pi { + // log.Printf("pi[%v]: %v", i, proto.MarshalTextString(v)) + // } + mu.Lock() blockIDs = append(blockIDs, b) indexes = append(indexes, pi) @@ -430,6 +447,16 @@ func dedupeBlockIDsAndIndex(ndx []*blockmgrpb.Index) (blockToIndex, packToIndex if o := blockToIndex[blockID]; o != nil { // this pack is same or newer. delete(o.Items, blockID) + delete(o.InlineItems, blockID) + } + + blockToIndex[blockID] = pck + } + for blockID := range pck.InlineItems { + if o := blockToIndex[blockID]; o != nil { + // this pack is same or newer. + delete(o.Items, blockID) + delete(o.InlineItems, blockID) } blockToIndex[blockID] = pck @@ -446,7 +473,7 @@ func dedupeBlockIDsAndIndex(ndx []*blockmgrpb.Index) (blockToIndex, packToIndex func removeEmptyIndexes(ndx []*blockmgrpb.Index) []*blockmgrpb.Index { var res []*blockmgrpb.Index for _, n := range ndx { - if len(n.Items) > 0 { + if len(n.Items)+len(n.InlineItems) > 0 { res = append(res, n) } } @@ -626,6 +653,16 @@ func (bm *Manager) Repackage(maxLength uint64) error { return fmt.Errorf("unable to re-package %q: %v", blockID, err) } } + + for blockID, blockData := range m.InlineItems { + if done[blockID] { + continue + } + done[blockID] = true + if err := bm.addToPackLocked(blockID, blockData, true); err != nil { + return fmt.Errorf("unable to re-package %q: %v", blockID, err) + } + } } return nil @@ -749,6 +786,15 @@ func (bm *Manager) blockInfoLocked(blockID string) (Info, error) { return Info{}, storage.ErrBlockNotFound } + if d, ok := ndx.InlineItems[blockID]; ok { + return Info{ + BlockID: blockID, + Timestamp: time.Unix(0, int64(ndx.CreateTimeNanos)), + PackOffset: 0, + Length: int64(len(d)), + }, nil + } + offset, size := unpackOffsetAndSize(ndx.Items[blockID]) return Info{ @@ -763,6 +809,12 @@ func (bm *Manager) blockInfoLocked(blockID string) (Info, error) { func (bm *Manager) getBlockInternalLocked(blockID string) ([]byte, error) { bm.assertLocked() + if ndx, ok := bm.blockIDToIndex[blockID]; ok { + if ii, ok := ndx.InlineItems[blockID]; ok { + return ii, nil + } + } + s, err := bm.blockInfoLocked(blockID) if err != nil { return nil, err @@ -897,7 +949,6 @@ func newManagerWithTime(st storage.Storage, f FormattingOptions, caching Caching timeNow: timeNow, flushPackIndexesAfter: timeNow().Add(flushPackIndexTimeout), pendingPackIndexes: nil, - maxPackedContentLength: f.MaxPackedContentLength, maxPackSize: f.MaxPackSize, formatter: formatter, blockIDToIndex: make(map[string]*blockmgrpb.Index), @@ -905,7 +956,8 @@ func newManagerWithTime(st storage.Storage, f FormattingOptions, caching Caching minPreambleLength: defaultMinPreambleLength, maxPreambleLength: defaultMaxPreambleLength, paddingUnit: defaultPaddingUnit, - cache: newBlockCache(st, caching), + maxInlineContentLength: maxInlineContentLength, + cache: newBlockCache(st, caching), } m.startPackIndexLocked() diff --git a/block/block_manager_test.go b/block/block_manager_test.go index aa21f13b5..80a56d8d4 100644 --- a/block/block_manager_test.go +++ b/block/block_manager_test.go @@ -23,8 +23,7 @@ import ( ) const ( - maxPackedContentLength = 1000 - maxPackSize = 2000 + maxPackSize = 2000 ) var fakeTime = time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) @@ -93,7 +92,7 @@ func TestBlockManagerDedupesPendingBlocks(t *testing.T) { bm := newTestBlockManager(data, keyTime, nil) for i := 0; i < 100; i++ { - writeBlockAndVerify(t, bm, seededRandomData(0, maxPackedContentLength-1)) + writeBlockAndVerify(t, bm, seededRandomData(0, 999)) } if got, want := len(data), 0; got != want { t.Errorf("unexpected number of blocks: %v, wanted %v", got, want) @@ -475,10 +474,11 @@ func newTestBlockManager(data map[string][]byte, keyTime map[string]time.Time, t timeFunc = fakeTimeNowWithAutoAdvance(fakeTime, 1) } bm, err := newManagerWithTime(st, FormattingOptions{ - BlockFormat: "TESTONLY_MD5", - MaxPackedContentLength: maxPackedContentLength, - MaxPackSize: maxPackSize, + BlockFormat: "TESTONLY_MD5", + MaxPackSize: maxPackSize, }, CachingOptions{}, timeFunc) + + bm.maxInlineContentLength = 0 if err != nil { panic("can't create block manager: " + err.Error()) } diff --git a/cli/command_block_index_show.go b/cli/command_block_index_show.go index acf4cdb16..197770dce 100644 --- a/cli/command_block_index_show.go +++ b/cli/command_block_index_show.go @@ -22,7 +22,7 @@ func runShowBlockIndexesAction(context *kingpin.ParseContext) error { defer rep.Close() blockIDs := *blockIndexShowIDs - if len(blockIDs) == 1 || blockIDs[0] == "active" { + if len(blockIDs) == 1 && blockIDs[0] == "active" { b, err := rep.Blocks.ActiveIndexBlocks() if err != nil { return err @@ -55,11 +55,15 @@ func runShowBlockIndexesAction(context *kingpin.ParseContext) error { blockID string offset uint32 size uint32 + inline bool } var lines []unpacked for blk, os := range ndx.Items { - lines = append(lines, unpacked{blk, uint32(os >> 32), uint32(os)}) + lines = append(lines, unpacked{blk, uint32(os >> 32), uint32(os), false}) + } + for blk, d := range ndx.InlineItems { + lines = append(lines, unpacked{blk, 0, uint32(len(d)), true}) } switch *blockIndexShowSort { case "offset": @@ -76,7 +80,11 @@ func runShowBlockIndexesAction(context *kingpin.ParseContext) error { }) } for _, l := range lines { - fmt.Printf(" added %-40v offset:%-10v size:%v\n", l.blockID, l.offset, l.size) + if l.inline { + fmt.Printf(" added %-40v size:%v (inline)\n", l.blockID, l.size) + } else { + fmt.Printf(" added %-40v offset:%-10v size:%v\n", l.blockID, l.offset, l.size) + } } for _, del := range ndx.DeletedItems { fmt.Printf(" deleted %v\n", del) diff --git a/cli/command_repository_status.go b/cli/command_repository_status.go index 7e9cf5af9..da44bd93e 100644 --- a/cli/command_repository_status.go +++ b/cli/command_repository_status.go @@ -62,7 +62,6 @@ func runStatusCommand(context *kingpin.ParseContext) error { fmt.Println() fmt.Printf("Object manager: v%v\n", rep.Objects.Format.Version) fmt.Printf("Block format: %v\n", rep.Blocks.Format.BlockFormat) - fmt.Printf("Max packed len: %v\n", units.BytesStringBase2(int64(rep.Blocks.Format.MaxPackedContentLength))) fmt.Printf("Max pack length: %v\n", units.BytesStringBase2(int64(rep.Blocks.Format.MaxPackSize))) fmt.Printf("Splitter: %v%v\n", rep.Objects.Format.Splitter, splitterExtraInfo) diff --git a/internal/blockmgrpb/block_index.pb.go b/internal/blockmgrpb/block_index.pb.go index cd870f444..8c0bfc1a0 100644 --- a/internal/blockmgrpb/block_index.pb.go +++ b/internal/blockmgrpb/block_index.pb.go @@ -36,6 +36,7 @@ type Index struct { CreateTimeNanos uint64 `protobuf:"varint,3,opt,name=create_time_nanos,json=createTimeNanos,proto3" json:"create_time_nanos,omitempty"` Items map[string]uint64 `protobuf:"bytes,4,rep,name=items" json:"items,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` DeletedItems []string `protobuf:"bytes,5,rep,name=deleted_items,json=deletedItems" json:"deleted_items,omitempty"` + InlineItems map[string][]byte `protobuf:"bytes,6,rep,name=inline_items,json=inlineItems" json:"inline_items,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *Index) Reset() { *m = Index{} } @@ -78,6 +79,13 @@ func (m *Index) GetDeletedItems() []string { return nil } +func (m *Index) GetInlineItems() map[string][]byte { + if m != nil { + return m.InlineItems + } + return nil +} + type Indexes struct { Indexes []*Index `protobuf:"bytes,1,rep,name=indexes" json:"indexes,omitempty"` } @@ -160,6 +168,29 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], s) } } + if len(m.InlineItems) > 0 { + for k, _ := range m.InlineItems { + dAtA[i] = 0x32 + i++ + v := m.InlineItems[k] + byteSize := 0 + if len(v) > 0 { + byteSize = 1 + len(v) + sovBlockIndex(uint64(len(v))) + } + mapSize := 1 + len(k) + sovBlockIndex(uint64(len(k))) + byteSize + i = encodeVarintBlockIndex(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintBlockIndex(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + if len(v) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintBlockIndex(dAtA, i, uint64(len(v))) + i += copy(dAtA[i:], v) + } + } + } return i, nil } @@ -229,6 +260,18 @@ func (m *Index) Size() (n int) { n += 1 + l + sovBlockIndex(uint64(l)) } } + if len(m.InlineItems) > 0 { + for k, v := range m.InlineItems { + _ = k + _ = v + l = 0 + if len(v) > 0 { + l = 1 + len(v) + sovBlockIndex(uint64(len(v))) + } + mapEntrySize := 1 + len(k) + sovBlockIndex(uint64(len(k))) + l + n += mapEntrySize + 1 + sovBlockIndex(uint64(mapEntrySize)) + } + } return n } @@ -489,6 +532,125 @@ func (m *Index) Unmarshal(dAtA []byte) error { } m.DeletedItems = append(m.DeletedItems, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field InlineItems", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlockIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBlockIndex + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.InlineItems == nil { + m.InlineItems = make(map[string][]byte) + } + var mapkey string + mapvalue := []byte{} + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlockIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlockIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthBlockIndex + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapbyteLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBlockIndex + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapbyteLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intMapbyteLen := int(mapbyteLen) + if intMapbyteLen < 0 { + return ErrInvalidLengthBlockIndex + } + postbytesIndex := iNdEx + intMapbyteLen + if postbytesIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = make([]byte, mapbyteLen) + copy(mapvalue, dAtA[iNdEx:postbytesIndex]) + iNdEx = postbytesIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipBlockIndex(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBlockIndex + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.InlineItems[mapkey] = mapvalue + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipBlockIndex(dAtA[iNdEx:]) @@ -699,24 +861,26 @@ var ( func init() { proto.RegisterFile("internal/blockmgrpb/block_index.proto", fileDescriptorBlockIndex) } var fileDescriptorBlockIndex = []byte{ - // 292 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x90, 0xcd, 0x4a, 0xc3, 0x40, - 0x14, 0x85, 0x9d, 0xa6, 0x69, 0xe9, 0xad, 0x45, 0x3b, 0xb8, 0x08, 0x22, 0x31, 0x44, 0xc4, 0xa0, - 0x10, 0x41, 0x37, 0xc5, 0x65, 0xc1, 0x45, 0x40, 0x5c, 0x04, 0x57, 0x6e, 0x86, 0xfc, 0x5c, 0xea, - 0x90, 0x64, 0x12, 0x92, 0x51, 0xec, 0xce, 0xc7, 0xf0, 0x91, 0x5c, 0xfa, 0x08, 0x12, 0x5f, 0x44, - 0x66, 0x26, 0xd2, 0xdd, 0x39, 0xdf, 0xbd, 0x87, 0x39, 0x73, 0xe1, 0x9c, 0x0b, 0x89, 0xad, 0x48, - 0xca, 0xeb, 0xb4, 0xac, 0xb3, 0xa2, 0xda, 0xb4, 0x4d, 0x6a, 0x24, 0xe3, 0x22, 0xc7, 0xf7, 0xb0, - 0x69, 0x6b, 0x59, 0xfb, 0x1f, 0x23, 0xb0, 0x23, 0xe5, 0xa9, 0x0f, 0x8b, 0x26, 0xc9, 0x0a, 0x36, - 0xec, 0xe4, 0x0e, 0xf1, 0x48, 0x30, 0x8b, 0xe7, 0x0a, 0xae, 0x15, 0x8b, 0x72, 0x7a, 0x0a, 0xda, - 0xb2, 0x12, 0xc5, 0x46, 0xbe, 0x38, 0x23, 0x8f, 0x04, 0xe3, 0x18, 0x14, 0x7a, 0xd0, 0x84, 0x5e, - 0xc2, 0x32, 0x6b, 0x31, 0x91, 0xc8, 0x24, 0xaf, 0x90, 0x89, 0x44, 0xd4, 0x9d, 0x63, 0xe9, 0xb5, - 0x03, 0x33, 0x78, 0xe2, 0x15, 0x3e, 0x2a, 0x4c, 0x2f, 0xc0, 0xe6, 0x12, 0xab, 0xce, 0x19, 0x7b, - 0x56, 0x30, 0xbf, 0x59, 0x86, 0xba, 0x47, 0x18, 0x29, 0x76, 0x2f, 0x64, 0xbb, 0x8d, 0xcd, 0x9c, - 0x9e, 0xc1, 0x22, 0xc7, 0x12, 0x25, 0xe6, 0xcc, 0x04, 0x6c, 0xcf, 0x0a, 0x66, 0xf1, 0xfe, 0x00, - 0x75, 0xe0, 0x78, 0x05, 0xb0, 0x4b, 0xd2, 0x43, 0xb0, 0x0a, 0xdc, 0x0e, 0x5f, 0x50, 0x92, 0x1e, - 0x81, 0xfd, 0x96, 0x94, 0xaf, 0x38, 0x94, 0x36, 0xe6, 0x6e, 0xb4, 0x22, 0xfe, 0x15, 0x4c, 0xf5, - 0xcb, 0xd8, 0x51, 0x0f, 0xa6, 0xdc, 0x48, 0x87, 0xe8, 0x52, 0x13, 0x53, 0x2a, 0xfe, 0xc7, 0xeb, - 0x93, 0xaf, 0xde, 0x25, 0xdf, 0xbd, 0x4b, 0x7e, 0x7a, 0x97, 0x7c, 0xfe, 0xba, 0x7b, 0xcf, 0xb0, - 0xbb, 0x6f, 0x3a, 0xd1, 0x47, 0xbd, 0xfd, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x26, 0xd1, 0xd9, 0xf1, - 0x7d, 0x01, 0x00, 0x00, + // 330 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x91, 0xc1, 0x4a, 0xf3, 0x40, + 0x14, 0x85, 0xff, 0x69, 0x9a, 0x96, 0xde, 0xb6, 0xfc, 0xed, 0x20, 0x18, 0x8a, 0xc4, 0x50, 0x11, + 0x83, 0x42, 0x05, 0xdd, 0x94, 0x2e, 0x5c, 0x14, 0x5c, 0x14, 0xc4, 0x45, 0x70, 0xe5, 0x66, 0x48, + 0x9b, 0x4b, 0x1d, 0x3a, 0x99, 0x94, 0x74, 0x14, 0xfb, 0x26, 0x3e, 0x92, 0x4b, 0x1f, 0x41, 0xe2, + 0x6b, 0xb8, 0x90, 0x99, 0x49, 0x49, 0x71, 0xe3, 0xee, 0xde, 0xef, 0xdc, 0x73, 0x38, 0xc3, 0xc0, + 0x29, 0x97, 0x0a, 0x73, 0x19, 0x8b, 0xcb, 0xb9, 0xc8, 0x16, 0xab, 0x74, 0x99, 0xaf, 0xe7, 0x76, + 0x64, 0x5c, 0x26, 0xf8, 0x3a, 0x5a, 0xe7, 0x99, 0xca, 0x86, 0xdf, 0x35, 0x70, 0x67, 0x7a, 0xa7, + 0x43, 0xe8, 0xae, 0xe3, 0xc5, 0x8a, 0x95, 0x37, 0x89, 0x47, 0x02, 0x12, 0xb6, 0xa2, 0xb6, 0x86, + 0x53, 0xcd, 0x66, 0x09, 0x3d, 0x06, 0xb3, 0x32, 0x81, 0x72, 0xa9, 0x9e, 0xbc, 0x5a, 0x40, 0xc2, + 0x7a, 0x04, 0x1a, 0xdd, 0x19, 0x42, 0xcf, 0xa1, 0xbf, 0xc8, 0x31, 0x56, 0xc8, 0x14, 0x4f, 0x91, + 0xc9, 0x58, 0x66, 0x1b, 0xcf, 0x31, 0x67, 0xff, 0xad, 0xf0, 0xc0, 0x53, 0xbc, 0xd7, 0x98, 0x9e, + 0x81, 0xcb, 0x15, 0xa6, 0x1b, 0xaf, 0x1e, 0x38, 0x61, 0xfb, 0xaa, 0x3f, 0x32, 0x3d, 0x46, 0x33, + 0xcd, 0x6e, 0xa5, 0xca, 0xb7, 0x91, 0xd5, 0xe9, 0x09, 0x74, 0x13, 0x14, 0xa8, 0x30, 0x61, 0xd6, + 0xe0, 0x06, 0x4e, 0xd8, 0x8a, 0x3a, 0x25, 0x34, 0x06, 0x3a, 0x81, 0x0e, 0x97, 0x82, 0x4b, 0x2c, + 0x6f, 0x1a, 0x26, 0xf4, 0x70, 0x17, 0x6a, 0xa4, 0xbd, 0xe8, 0x36, 0xaf, 0xc8, 0x60, 0x0c, 0x50, + 0x49, 0xb4, 0x07, 0xce, 0x0a, 0xb7, 0xe5, 0xf3, 0xf5, 0x48, 0x0f, 0xc0, 0x7d, 0x89, 0xc5, 0x33, + 0x96, 0x0f, 0xb6, 0xcb, 0xa4, 0x36, 0x26, 0x83, 0x1b, 0xe8, 0xfd, 0x8e, 0xfe, 0xcb, 0xdf, 0xd9, + 0xf3, 0x0f, 0x2f, 0xa0, 0x69, 0x0a, 0xe2, 0x86, 0x06, 0xd0, 0xe4, 0x76, 0xf4, 0x88, 0xe9, 0xde, + 0xb0, 0xdd, 0xa3, 0x1d, 0x9e, 0x1e, 0xbd, 0x17, 0x3e, 0xf9, 0x28, 0x7c, 0xf2, 0x59, 0xf8, 0xe4, + 0xed, 0xcb, 0xff, 0xf7, 0x08, 0xd5, 0xdf, 0xce, 0x1b, 0xe6, 0x43, 0xaf, 0x7f, 0x02, 0x00, 0x00, + 0xff, 0xff, 0x4a, 0x30, 0xc8, 0xed, 0xf9, 0x01, 0x00, 0x00, } diff --git a/internal/blockmgrpb/block_index.proto b/internal/blockmgrpb/block_index.proto index a1eeba71d..d0824fc45 100644 --- a/internal/blockmgrpb/block_index.proto +++ b/internal/blockmgrpb/block_index.proto @@ -8,6 +8,7 @@ message Index { uint64 create_time_nanos = 3; map items = 4; repeated string deleted_items = 5; + map inline_items = 6; } message Indexes { diff --git a/manifest/manifest_manager.go b/manifest/manifest_manager.go index 4adb0c58c..f551b397e 100644 --- a/manifest/manifest_manager.go +++ b/manifest/manifest_manager.go @@ -214,7 +214,7 @@ func (m *Manager) load() error { return fmt.Errorf("unable to list manifest blocks: %v", err) } - log.Printf("loaded %v blocks", len(blocks)) + log.Printf("found %v manifest blocks", len(blocks)) return m.loadManifestBlocks(blocks) } @@ -252,7 +252,7 @@ func (m *Manager) loadManifestBlocks(blocks []block.Info) error { wg.Wait() close(errors) close(manifests) - log.Debug().Dur("duration_ms", time.Since(t0)).Msgf("finished loading blocks.") + log.Debug().Dur("duration_ms", time.Since(t0)).Msgf("finished loading manifest blocks.") // if there was any error, forward it if err := <-errors; err != nil { return err diff --git a/manifest/manifest_manager_test.go b/manifest/manifest_manager_test.go index db1aa43b4..61a32066a 100644 --- a/manifest/manifest_manager_test.go +++ b/manifest/manifest_manager_test.go @@ -174,9 +174,8 @@ func newManagerForTesting(t *testing.T, data map[string][]byte, keyTime map[stri st := storagetesting.NewMapStorage(data, keyTime) bm, err := block.NewManager(st, block.FormattingOptions{ - BlockFormat: "TESTONLY_MD5", - MaxPackedContentLength: 10000, - MaxPackSize: 100000, + BlockFormat: "TESTONLY_MD5", + MaxPackSize: 100000, }, block.CachingOptions{}) if err != nil { return nil, fmt.Errorf("can't create block manager: %v", err) diff --git a/repo/repository_test.go b/repo/repository_test.go index 106ce42de..6e910ed1a 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -89,9 +89,9 @@ func TestWriters(t *testing.T) { repo.Blocks.Flush() - if got, want := len(data), 3; got != want { - // 1 format block + 1 data block + 1 pack index block - t.Errorf("unexpected data written to the storage (%v), wanted %v: %v", len(data), 3, data) + if got, want := len(data), 2; got != want { + // 1 format block + 1 pack index block (including inline data blocks) + t.Errorf("unexpected data written to the storage (%v), wanted %v: %v", got, want, data) dumpBlockManagerData(data) } } @@ -169,8 +169,8 @@ func TestPackingSimple(t *testing.T) { t.Errorf("oid3a(%q) != oid3b(%q)", got, want) } - // format + index + pack - if got, want := len(data), 3; got != want { + // format + index + if got, want := len(data), 2; got != want { t.Errorf("got unexpected repository contents %v items, wanted %v", got, want) } repo.Close()