changed how small files are stored to avoid creating very small data files by embedding them directly in the index packs

block/block_formatting_options.go

@@ -2,10 +2,9 @@ package block
 
 // FormattingOptions describes the rules for formatting blocks in repository.
 type FormattingOptions struct {
-	Version                int    `json:"version,omitempty"`                // version number, must be "1"
-	BlockFormat            string `json:"objectFormat,omitempty"`           // identifier of the block format
-	HMACSecret             []byte `json:"secret,omitempty"`                 // HMAC secret used to generate encryption keys
-	MasterKey              []byte `json:"masterKey,omitempty"`              // master encryption key (SIV-mode encryption only)
-	MaxPackedContentLength int    `json:"maxPackedContentLength,omitempty"` // maximum size of object to be considered for storage in a pack
-	MaxPackSize            int    `json:"maxPackSize,omitempty"`            // maximum size of a pack object
+	Version     int    `json:"version,omitempty"`      // version number, must be "1"
+	BlockFormat string `json:"objectFormat,omitempty"` // identifier of the block format
+	HMACSecret  []byte `json:"secret,omitempty"`       // HMAC secret used to generate encryption keys
+	MasterKey   []byte `json:"masterKey,omitempty"`    // master encryption key (SIV-mode encryption only)
+	MaxPackSize int    `json:"maxPackSize,omitempty"`  // maximum size of a pack object
 }

block/block_manager.go

@@ -29,6 +29,7 @@ const (
 	defaultMinPreambleLength = 32
 	defaultMaxPreambleLength = 32
 	defaultPaddingUnit       = 4096
+	maxInlineContentLength   = 100000 // amount of block data to store in the index block itself
 )
 
 var zeroTime time.Time
@@ -66,7 +67,7 @@ type Manager struct {
 	currentPackData  []byte
 	currentPackIndex *blockmgrpb.Index
 
-	maxPackedContentLength int
+	maxInlineContentLength int
 	maxPackSize            int
 	formatter              Formatter
 
@@ -91,6 +92,7 @@ func (bm *Manager) DeleteBlock(blockID string) error {
 
 	delete(bm.blockIDToIndex, blockID)
 	delete(bm.currentPackIndex.Items, blockID)
+	delete(bm.currentPackIndex.InlineItems, blockID)
 	bm.currentPackIndex.DeletedItems = append(bm.currentPackIndex.DeletedItems, blockID)
 	return nil
 }
@@ -184,6 +186,7 @@ func (bm *Manager) close() error {
 func (bm *Manager) startPackIndexLocked() {
 	bm.currentPackIndex = &blockmgrpb.Index{
 		Items:           make(map[string]uint64),
+		InlineItems:     make(map[string][]byte),
 		CreateTimeNanos: uint64(bm.timeNow().UnixNano()),
 	}
 	bm.currentPackData = []byte{}
@@ -234,7 +237,17 @@ func (bm *Manager) finishAllOpenPacksLocked() error {
 }
 
 func (bm *Manager) finishPackLocked() error {
-	if len(bm.currentPackIndex.Items)+len(bm.currentPackIndex.DeletedItems) > 0 {
+	if len(bm.currentPackIndex.Items)+len(bm.currentPackIndex.DeletedItems)+len(bm.currentPackIndex.InlineItems) > 0 {
+		if len(bm.currentPackData) < bm.maxInlineContentLength {
+			for k, os := range bm.currentPackIndex.Items {
+				offset, size := unpackOffsetAndSize(os)
+				dat := bm.currentPackData[offset : offset+size]
+				bm.currentPackIndex.InlineItems[k] = dat
+			}
+
+			bm.currentPackIndex.Items = nil
+			bm.currentPackData = nil
+		}
 		if bm.currentPackData != nil {
 			if bm.paddingUnit > 0 {
 				if missing := bm.paddingUnit - (len(bm.currentPackData) % bm.paddingUnit); missing > 0 {
@@ -372,6 +385,10 @@ func (bm *Manager) loadMergedPackIndexLocked() ([]*blockmgrpb.Index, []string, t
 					return
 				}
 
+				// for i, v := range pi {
+				// 	log.Printf("pi[%v]: %v", i, proto.MarshalTextString(v))
+				// }
+
 				mu.Lock()
 				blockIDs = append(blockIDs, b)
 				indexes = append(indexes, pi)
@@ -430,6 +447,16 @@ func dedupeBlockIDsAndIndex(ndx []*blockmgrpb.Index) (blockToIndex, packToIndex
 			if o := blockToIndex[blockID]; o != nil {
 				// this pack is same or newer.
 				delete(o.Items, blockID)
+				delete(o.InlineItems, blockID)
+			}
+
+			blockToIndex[blockID] = pck
+		}
+		for blockID := range pck.InlineItems {
+			if o := blockToIndex[blockID]; o != nil {
+				// this pack is same or newer.
+				delete(o.Items, blockID)
+				delete(o.InlineItems, blockID)
 			}
 
 			blockToIndex[blockID] = pck
@@ -446,7 +473,7 @@ func dedupeBlockIDsAndIndex(ndx []*blockmgrpb.Index) (blockToIndex, packToIndex
 func removeEmptyIndexes(ndx []*blockmgrpb.Index) []*blockmgrpb.Index {
 	var res []*blockmgrpb.Index
 	for _, n := range ndx {
-		if len(n.Items) > 0 {
+		if len(n.Items)+len(n.InlineItems) > 0 {
 			res = append(res, n)
 		}
 	}
@@ -626,6 +653,16 @@ func (bm *Manager) Repackage(maxLength uint64) error {
 				return fmt.Errorf("unable to re-package %q: %v", blockID, err)
 			}
 		}
+
+		for blockID, blockData := range m.InlineItems {
+			if done[blockID] {
+				continue
+			}
+			done[blockID] = true
+			if err := bm.addToPackLocked(blockID, blockData, true); err != nil {
+				return fmt.Errorf("unable to re-package %q: %v", blockID, err)
+			}
+		}
 	}
 
 	return nil
@@ -749,6 +786,15 @@ func (bm *Manager) blockInfoLocked(blockID string) (Info, error) {
 		return Info{}, storage.ErrBlockNotFound
 	}
 
+	if d, ok := ndx.InlineItems[blockID]; ok {
+		return Info{
+			BlockID:    blockID,
+			Timestamp:  time.Unix(0, int64(ndx.CreateTimeNanos)),
+			PackOffset: 0,
+			Length:     int64(len(d)),
+		}, nil
+	}
+
 	offset, size := unpackOffsetAndSize(ndx.Items[blockID])
 
 	return Info{
@@ -763,6 +809,12 @@ func (bm *Manager) blockInfoLocked(blockID string) (Info, error) {
 func (bm *Manager) getBlockInternalLocked(blockID string) ([]byte, error) {
 	bm.assertLocked()
 
+	if ndx, ok := bm.blockIDToIndex[blockID]; ok {
+		if ii, ok := ndx.InlineItems[blockID]; ok {
+			return ii, nil
+		}
+	}
+
 	s, err := bm.blockInfoLocked(blockID)
 	if err != nil {
 		return nil, err
@@ -897,7 +949,6 @@ func newManagerWithTime(st storage.Storage, f FormattingOptions, caching Caching
 		timeNow:                timeNow,
 		flushPackIndexesAfter:  timeNow().Add(flushPackIndexTimeout),
 		pendingPackIndexes:     nil,
-		maxPackedContentLength: f.MaxPackedContentLength,
 		maxPackSize:            f.MaxPackSize,
 		formatter:              formatter,
 		blockIDToIndex:         make(map[string]*blockmgrpb.Index),
@@ -905,7 +956,8 @@ func newManagerWithTime(st storage.Storage, f FormattingOptions, caching Caching
 		minPreambleLength:      defaultMinPreambleLength,
 		maxPreambleLength:      defaultMaxPreambleLength,
 		paddingUnit:            defaultPaddingUnit,
-		cache:                  newBlockCache(st, caching),
+		maxInlineContentLength: maxInlineContentLength,
+		cache: newBlockCache(st, caching),
 	}
 
 	m.startPackIndexLocked()

block/block_manager_test.go

@@ -23,8 +23,7 @@ import (
 )
 
 const (
-	maxPackedContentLength = 1000
-	maxPackSize            = 2000
+	maxPackSize = 2000
 )
 
 var fakeTime = time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)
@@ -93,7 +92,7 @@ func TestBlockManagerDedupesPendingBlocks(t *testing.T) {
 	bm := newTestBlockManager(data, keyTime, nil)
 
 	for i := 0; i < 100; i++ {
-		writeBlockAndVerify(t, bm, seededRandomData(0, maxPackedContentLength-1))
+		writeBlockAndVerify(t, bm, seededRandomData(0, 999))
 	}
 	if got, want := len(data), 0; got != want {
 		t.Errorf("unexpected number of blocks: %v, wanted %v", got, want)
@@ -475,10 +474,11 @@ func newTestBlockManager(data map[string][]byte, keyTime map[string]time.Time, t
 		timeFunc = fakeTimeNowWithAutoAdvance(fakeTime, 1)
 	}
 	bm, err := newManagerWithTime(st, FormattingOptions{
-		BlockFormat:            "TESTONLY_MD5",
-		MaxPackedContentLength: maxPackedContentLength,
-		MaxPackSize:            maxPackSize,
+		BlockFormat: "TESTONLY_MD5",
+		MaxPackSize: maxPackSize,
 	}, CachingOptions{}, timeFunc)
+
+	bm.maxInlineContentLength = 0
 	if err != nil {
 		panic("can't create block manager: " + err.Error())
 	}

cli/command_block_index_show.go

@@ -22,7 +22,7 @@ func runShowBlockIndexesAction(context *kingpin.ParseContext) error {
 	defer rep.Close()
 
 	blockIDs := *blockIndexShowIDs
-	if len(blockIDs) == 1 || blockIDs[0] == "active" {
+	if len(blockIDs) == 1 && blockIDs[0] == "active" {
 		b, err := rep.Blocks.ActiveIndexBlocks()
 		if err != nil {
 			return err
@@ -55,11 +55,15 @@ func runShowBlockIndexesAction(context *kingpin.ParseContext) error {
 				blockID string
 				offset  uint32
 				size    uint32
+				inline  bool
 			}
 			var lines []unpacked
 
 			for blk, os := range ndx.Items {
-				lines = append(lines, unpacked{blk, uint32(os >> 32), uint32(os)})
+				lines = append(lines, unpacked{blk, uint32(os >> 32), uint32(os), false})
+			}
+			for blk, d := range ndx.InlineItems {
+				lines = append(lines, unpacked{blk, 0, uint32(len(d)), true})
 			}
 			switch *blockIndexShowSort {
 			case "offset":
@@ -76,7 +80,11 @@ func runShowBlockIndexesAction(context *kingpin.ParseContext) error {
 				})
 			}
 			for _, l := range lines {
-				fmt.Printf("  added %-40v offset:%-10v size:%v\n", l.blockID, l.offset, l.size)
+				if l.inline {
+					fmt.Printf("  added %-40v size:%v (inline)\n", l.blockID, l.size)
+				} else {
+					fmt.Printf("  added %-40v offset:%-10v size:%v\n", l.blockID, l.offset, l.size)
+				}
 			}
 			for _, del := range ndx.DeletedItems {
 				fmt.Printf("  deleted %v\n", del)

cli/command_repository_status.go

@@ -62,7 +62,6 @@ func runStatusCommand(context *kingpin.ParseContext) error {
 	fmt.Println()
 	fmt.Printf("Object manager:      v%v\n", rep.Objects.Format.Version)
 	fmt.Printf("Block format:        %v\n", rep.Blocks.Format.BlockFormat)
-	fmt.Printf("Max packed len:      %v\n", units.BytesStringBase2(int64(rep.Blocks.Format.MaxPackedContentLength)))
 	fmt.Printf("Max pack length:     %v\n", units.BytesStringBase2(int64(rep.Blocks.Format.MaxPackSize)))
 	fmt.Printf("Splitter:            %v%v\n", rep.Objects.Format.Splitter, splitterExtraInfo)
 

internal/blockmgrpb/block_index.pb.go

@@ -36,6 +36,7 @@ type Index struct {
 	CreateTimeNanos uint64            `protobuf:"varint,3,opt,name=create_time_nanos,json=createTimeNanos,proto3" json:"create_time_nanos,omitempty"`
 	Items           map[string]uint64 `protobuf:"bytes,4,rep,name=items" json:"items,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
 	DeletedItems    []string          `protobuf:"bytes,5,rep,name=deleted_items,json=deletedItems" json:"deleted_items,omitempty"`
+	InlineItems     map[string][]byte `protobuf:"bytes,6,rep,name=inline_items,json=inlineItems" json:"inline_items,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
 func (m *Index) Reset()                    { *m = Index{} }
@@ -78,6 +79,13 @@ func (m *Index) GetDeletedItems() []string {
 	return nil
 }
 
+func (m *Index) GetInlineItems() map[string][]byte {
+	if m != nil {
+		return m.InlineItems
+	}
+	return nil
+}
+
 type Indexes struct {
 	Indexes []*Index `protobuf:"bytes,1,rep,name=indexes" json:"indexes,omitempty"`
 }
@@ -160,6 +168,29 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) {
 			i += copy(dAtA[i:], s)
 		}
 	}
+	if len(m.InlineItems) > 0 {
+		for k, _ := range m.InlineItems {
+			dAtA[i] = 0x32
+			i++
+			v := m.InlineItems[k]
+			byteSize := 0
+			if len(v) > 0 {
+				byteSize = 1 + len(v) + sovBlockIndex(uint64(len(v)))
+			}
+			mapSize := 1 + len(k) + sovBlockIndex(uint64(len(k))) + byteSize
+			i = encodeVarintBlockIndex(dAtA, i, uint64(mapSize))
+			dAtA[i] = 0xa
+			i++
+			i = encodeVarintBlockIndex(dAtA, i, uint64(len(k)))
+			i += copy(dAtA[i:], k)
+			if len(v) > 0 {
+				dAtA[i] = 0x12
+				i++
+				i = encodeVarintBlockIndex(dAtA, i, uint64(len(v)))
+				i += copy(dAtA[i:], v)
+			}
+		}
+	}
 	return i, nil
 }
 
@@ -229,6 +260,18 @@ func (m *Index) Size() (n int) {
 			n += 1 + l + sovBlockIndex(uint64(l))
 		}
 	}
+	if len(m.InlineItems) > 0 {
+		for k, v := range m.InlineItems {
+			_ = k
+			_ = v
+			l = 0
+			if len(v) > 0 {
+				l = 1 + len(v) + sovBlockIndex(uint64(len(v)))
+			}
+			mapEntrySize := 1 + len(k) + sovBlockIndex(uint64(len(k))) + l
+			n += mapEntrySize + 1 + sovBlockIndex(uint64(mapEntrySize))
+		}
+	}
 	return n
 }
 
@@ -489,6 +532,125 @@ func (m *Index) Unmarshal(dAtA []byte) error {
 			}
 			m.DeletedItems = append(m.DeletedItems, string(dAtA[iNdEx:postIndex]))
 			iNdEx = postIndex
+		case 6:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field InlineItems", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowBlockIndex
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthBlockIndex
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if m.InlineItems == nil {
+				m.InlineItems = make(map[string][]byte)
+			}
+			var mapkey string
+			mapvalue := []byte{}
+			for iNdEx < postIndex {
+				entryPreIndex := iNdEx
+				var wire uint64
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return ErrIntOverflowBlockIndex
+					}
+					if iNdEx >= l {
+						return io.ErrUnexpectedEOF
+					}
+					b := dAtA[iNdEx]
+					iNdEx++
+					wire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				fieldNum := int32(wire >> 3)
+				if fieldNum == 1 {
+					var stringLenmapkey uint64
+					for shift := uint(0); ; shift += 7 {
+						if shift >= 64 {
+							return ErrIntOverflowBlockIndex
+						}
+						if iNdEx >= l {
+							return io.ErrUnexpectedEOF
+						}
+						b := dAtA[iNdEx]
+						iNdEx++
+						stringLenmapkey |= (uint64(b) & 0x7F) << shift
+						if b < 0x80 {
+							break
+						}
+					}
+					intStringLenmapkey := int(stringLenmapkey)
+					if intStringLenmapkey < 0 {
+						return ErrInvalidLengthBlockIndex
+					}
+					postStringIndexmapkey := iNdEx + intStringLenmapkey
+					if postStringIndexmapkey > l {
+						return io.ErrUnexpectedEOF
+					}
+					mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
+					iNdEx = postStringIndexmapkey
+				} else if fieldNum == 2 {
+					var mapbyteLen uint64
+					for shift := uint(0); ; shift += 7 {
+						if shift >= 64 {
+							return ErrIntOverflowBlockIndex
+						}
+						if iNdEx >= l {
+							return io.ErrUnexpectedEOF
+						}
+						b := dAtA[iNdEx]
+						iNdEx++
+						mapbyteLen |= (uint64(b) & 0x7F) << shift
+						if b < 0x80 {
+							break
+						}
+					}
+					intMapbyteLen := int(mapbyteLen)
+					if intMapbyteLen < 0 {
+						return ErrInvalidLengthBlockIndex
+					}
+					postbytesIndex := iNdEx + intMapbyteLen
+					if postbytesIndex > l {
+						return io.ErrUnexpectedEOF
+					}
+					mapvalue = make([]byte, mapbyteLen)
+					copy(mapvalue, dAtA[iNdEx:postbytesIndex])
+					iNdEx = postbytesIndex
+				} else {
+					iNdEx = entryPreIndex
+					skippy, err := skipBlockIndex(dAtA[iNdEx:])
+					if err != nil {
+						return err
+					}
+					if skippy < 0 {
+						return ErrInvalidLengthBlockIndex
+					}
+					if (iNdEx + skippy) > postIndex {
+						return io.ErrUnexpectedEOF
+					}
+					iNdEx += skippy
+				}
+			}
+			m.InlineItems[mapkey] = mapvalue
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipBlockIndex(dAtA[iNdEx:])
@@ -699,24 +861,26 @@ var (
 func init() { proto.RegisterFile("internal/blockmgrpb/block_index.proto", fileDescriptorBlockIndex) }
 
 var fileDescriptorBlockIndex = []byte{
-	// 292 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x90, 0xcd, 0x4a, 0xc3, 0x40,
-	0x14, 0x85, 0x9d, 0xa6, 0x69, 0xe9, 0xad, 0x45, 0x3b, 0xb8, 0x08, 0x22, 0x31, 0x44, 0xc4, 0xa0,
-	0x10, 0x41, 0x37, 0xc5, 0x65, 0xc1, 0x45, 0x40, 0x5c, 0x04, 0x57, 0x6e, 0x86, 0xfc, 0x5c, 0xea,
-	0x90, 0x64, 0x12, 0x92, 0x51, 0xec, 0xce, 0xc7, 0xf0, 0x91, 0x5c, 0xfa, 0x08, 0x12, 0x5f, 0x44,
-	0x66, 0x26, 0xd2, 0xdd, 0x39, 0xdf, 0xbd, 0x87, 0x39, 0x73, 0xe1, 0x9c, 0x0b, 0x89, 0xad, 0x48,
-	0xca, 0xeb, 0xb4, 0xac, 0xb3, 0xa2, 0xda, 0xb4, 0x4d, 0x6a, 0x24, 0xe3, 0x22, 0xc7, 0xf7, 0xb0,
-	0x69, 0x6b, 0x59, 0xfb, 0x1f, 0x23, 0xb0, 0x23, 0xe5, 0xa9, 0x0f, 0x8b, 0x26, 0xc9, 0x0a, 0x36,
-	0xec, 0xe4, 0x0e, 0xf1, 0x48, 0x30, 0x8b, 0xe7, 0x0a, 0xae, 0x15, 0x8b, 0x72, 0x7a, 0x0a, 0xda,
-	0xb2, 0x12, 0xc5, 0x46, 0xbe, 0x38, 0x23, 0x8f, 0x04, 0xe3, 0x18, 0x14, 0x7a, 0xd0, 0x84, 0x5e,
-	0xc2, 0x32, 0x6b, 0x31, 0x91, 0xc8, 0x24, 0xaf, 0x90, 0x89, 0x44, 0xd4, 0x9d, 0x63, 0xe9, 0xb5,
-	0x03, 0x33, 0x78, 0xe2, 0x15, 0x3e, 0x2a, 0x4c, 0x2f, 0xc0, 0xe6, 0x12, 0xab, 0xce, 0x19, 0x7b,
-	0x56, 0x30, 0xbf, 0x59, 0x86, 0xba, 0x47, 0x18, 0x29, 0x76, 0x2f, 0x64, 0xbb, 0x8d, 0xcd, 0x9c,
-	0x9e, 0xc1, 0x22, 0xc7, 0x12, 0x25, 0xe6, 0xcc, 0x04, 0x6c, 0xcf, 0x0a, 0x66, 0xf1, 0xfe, 0x00,
-	0x75, 0xe0, 0x78, 0x05, 0xb0, 0x4b, 0xd2, 0x43, 0xb0, 0x0a, 0xdc, 0x0e, 0x5f, 0x50, 0x92, 0x1e,
-	0x81, 0xfd, 0x96, 0x94, 0xaf, 0x38, 0x94, 0x36, 0xe6, 0x6e, 0xb4, 0x22, 0xfe, 0x15, 0x4c, 0xf5,
-	0xcb, 0xd8, 0x51, 0x0f, 0xa6, 0xdc, 0x48, 0x87, 0xe8, 0x52, 0x13, 0x53, 0x2a, 0xfe, 0xc7, 0xeb,
-	0x93, 0xaf, 0xde, 0x25, 0xdf, 0xbd, 0x4b, 0x7e, 0x7a, 0x97, 0x7c, 0xfe, 0xba, 0x7b, 0xcf, 0xb0,
-	0xbb, 0x6f, 0x3a, 0xd1, 0x47, 0xbd, 0xfd, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x26, 0xd1, 0xd9, 0xf1,
-	0x7d, 0x01, 0x00, 0x00,
+	// 330 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x91, 0xc1, 0x4a, 0xf3, 0x40,
+	0x14, 0x85, 0xff, 0x69, 0x9a, 0x96, 0xde, 0xb6, 0xfc, 0xed, 0x20, 0x18, 0x8a, 0xc4, 0x50, 0x11,
+	0x83, 0x42, 0x05, 0xdd, 0x94, 0x2e, 0x5c, 0x14, 0x5c, 0x14, 0xc4, 0x45, 0x70, 0xe5, 0x66, 0x48,
+	0x9b, 0x4b, 0x1d, 0x3a, 0x99, 0x94, 0x74, 0x14, 0xfb, 0x26, 0x3e, 0x92, 0x4b, 0x1f, 0x41, 0xe2,
+	0x6b, 0xb8, 0x90, 0x99, 0x49, 0x49, 0x71, 0xe3, 0xee, 0xde, 0xef, 0xdc, 0x73, 0x38, 0xc3, 0xc0,
+	0x29, 0x97, 0x0a, 0x73, 0x19, 0x8b, 0xcb, 0xb9, 0xc8, 0x16, 0xab, 0x74, 0x99, 0xaf, 0xe7, 0x76,
+	0x64, 0x5c, 0x26, 0xf8, 0x3a, 0x5a, 0xe7, 0x99, 0xca, 0x86, 0xdf, 0x35, 0x70, 0x67, 0x7a, 0xa7,
+	0x43, 0xe8, 0xae, 0xe3, 0xc5, 0x8a, 0x95, 0x37, 0x89, 0x47, 0x02, 0x12, 0xb6, 0xa2, 0xb6, 0x86,
+	0x53, 0xcd, 0x66, 0x09, 0x3d, 0x06, 0xb3, 0x32, 0x81, 0x72, 0xa9, 0x9e, 0xbc, 0x5a, 0x40, 0xc2,
+	0x7a, 0x04, 0x1a, 0xdd, 0x19, 0x42, 0xcf, 0xa1, 0xbf, 0xc8, 0x31, 0x56, 0xc8, 0x14, 0x4f, 0x91,
+	0xc9, 0x58, 0x66, 0x1b, 0xcf, 0x31, 0x67, 0xff, 0xad, 0xf0, 0xc0, 0x53, 0xbc, 0xd7, 0x98, 0x9e,
+	0x81, 0xcb, 0x15, 0xa6, 0x1b, 0xaf, 0x1e, 0x38, 0x61, 0xfb, 0xaa, 0x3f, 0x32, 0x3d, 0x46, 0x33,
+	0xcd, 0x6e, 0xa5, 0xca, 0xb7, 0x91, 0xd5, 0xe9, 0x09, 0x74, 0x13, 0x14, 0xa8, 0x30, 0x61, 0xd6,
+	0xe0, 0x06, 0x4e, 0xd8, 0x8a, 0x3a, 0x25, 0x34, 0x06, 0x3a, 0x81, 0x0e, 0x97, 0x82, 0x4b, 0x2c,
+	0x6f, 0x1a, 0x26, 0xf4, 0x70, 0x17, 0x6a, 0xa4, 0xbd, 0xe8, 0x36, 0xaf, 0xc8, 0x60, 0x0c, 0x50,
+	0x49, 0xb4, 0x07, 0xce, 0x0a, 0xb7, 0xe5, 0xf3, 0xf5, 0x48, 0x0f, 0xc0, 0x7d, 0x89, 0xc5, 0x33,
+	0x96, 0x0f, 0xb6, 0xcb, 0xa4, 0x36, 0x26, 0x83, 0x1b, 0xe8, 0xfd, 0x8e, 0xfe, 0xcb, 0xdf, 0xd9,
+	0xf3, 0x0f, 0x2f, 0xa0, 0x69, 0x0a, 0xe2, 0x86, 0x06, 0xd0, 0xe4, 0x76, 0xf4, 0x88, 0xe9, 0xde,
+	0xb0, 0xdd, 0xa3, 0x1d, 0x9e, 0x1e, 0xbd, 0x17, 0x3e, 0xf9, 0x28, 0x7c, 0xf2, 0x59, 0xf8, 0xe4,
+	0xed, 0xcb, 0xff, 0xf7, 0x08, 0xd5, 0xdf, 0xce, 0x1b, 0xe6, 0x43, 0xaf, 0x7f, 0x02, 0x00, 0x00,
+	0xff, 0xff, 0x4a, 0x30, 0xc8, 0xed, 0xf9, 0x01, 0x00, 0x00,
 }

internal/blockmgrpb/block_index.proto

@@ -8,6 +8,7 @@ message Index {
   uint64 create_time_nanos = 3;
   map<string, uint64> items = 4;
   repeated string deleted_items = 5;
+  map<string, bytes> inline_items = 6;
 }
 
 message Indexes {

manifest/manifest_manager.go

@@ -214,7 +214,7 @@ func (m *Manager) load() error {
 		return fmt.Errorf("unable to list manifest blocks: %v", err)
 	}
 
-	log.Printf("loaded %v blocks", len(blocks))
+	log.Printf("found %v manifest blocks", len(blocks))
 	return m.loadManifestBlocks(blocks)
 }
 
@@ -252,7 +252,7 @@ func (m *Manager) loadManifestBlocks(blocks []block.Info) error {
 	wg.Wait()
 	close(errors)
 	close(manifests)
-	log.Debug().Dur("duration_ms", time.Since(t0)).Msgf("finished loading blocks.")
+	log.Debug().Dur("duration_ms", time.Since(t0)).Msgf("finished loading manifest blocks.")
 	// if there was any error, forward it
 	if err := <-errors; err != nil {
 		return err

manifest/manifest_manager_test.go

@@ -174,9 +174,8 @@ func newManagerForTesting(t *testing.T, data map[string][]byte, keyTime map[stri
 	st := storagetesting.NewMapStorage(data, keyTime)
 
 	bm, err := block.NewManager(st, block.FormattingOptions{
-		BlockFormat:            "TESTONLY_MD5",
-		MaxPackedContentLength: 10000,
-		MaxPackSize:            100000,
+		BlockFormat: "TESTONLY_MD5",
+		MaxPackSize: 100000,
 	}, block.CachingOptions{})
 	if err != nil {
 		return nil, fmt.Errorf("can't create block manager: %v", err)

repo/repository_test.go

@@ -89,9 +89,9 @@ func TestWriters(t *testing.T) {
 
 		repo.Blocks.Flush()
 
-		if got, want := len(data), 3; got != want {
-			// 1 format block + 1 data block + 1 pack index block
-			t.Errorf("unexpected data written to the storage (%v), wanted %v: %v", len(data), 3, data)
+		if got, want := len(data), 2; got != want {
+			// 1 format block + 1 pack index block (including inline data blocks)
+			t.Errorf("unexpected data written to the storage (%v), wanted %v: %v", got, want, data)
 			dumpBlockManagerData(data)
 		}
 	}
@@ -169,8 +169,8 @@ func TestPackingSimple(t *testing.T) {
 		t.Errorf("oid3a(%q) != oid3b(%q)", got, want)
 	}
 
-	// format + index + pack
-	if got, want := len(data), 3; got != want {
+	// format + index
+	if got, want := len(data), 2; got != want {
 		t.Errorf("got unexpected repository contents %v items, wanted %v", got, want)
 	}
 	repo.Close()