mirror of
https://github.com/kopia/kopia.git
synced 2025-12-23 22:57:50 -05:00
0f7253eb66
This is a breaking change to users who might be using Kopia as a library.
### Log Format
```json
{"t":"<timestamp-rfc-3389-microseconds>", "span:T1":"V1", "span:T2":"V2", "n":"<source>", "m":"<message>", /*parameters*/}
```
Where each record is associated with one or more spans that describe its scope:
* `"span:client": "<hash-of-username@hostname>"`
* `"span:repo": "<random>"` - random identifier of a repository connection (from `repo.Open`)
* `"span:maintenance": "<random>"` - random identifier of a maintenance session
* `"span:upload": "<hash-of-username@host:/path>"` - uniquely identifies upload session of a given directory
* `"span:checkpoint": "<random>"` - encapsulates each checkpoint operation during Upload
* `"span:server-session": "<random>"` -single client connection to the server
* `"span:flush": "<random>"` - encapsulates each Flush session
* `"span:maintenance": "<random>"` - encapsulates each maintenance operation
* `"span:loadIndex" : "<random>"` - encapsulates index loading operation
* `"span:emr" : "<random>"` - encapsulates epoch manager refresh
* `"span:writePack": "<pack-blob-ID>"` - encapsulates pack blob preparation and writing
(plus additional minor spans for various phases of the maintenance).
Notable points:
- Used internal zero allocation JSON writer for reduced memory usage.
- renamed `--disable-internal-log` to `--disable-repository-log` (controls saving blobs to repository)
- added `--disable-content-log` (controls writing of `content-log` files)
- all storage operations are also logged in a structural way and associated with the corresponding spans.
- all content IDs are logged in a truncated format (since first N bytes that are usually enough to be unique) to improve compressibility of logs (blob IDs are frequently repeated but content IDs usually appear just once).
This format should make it possible to recreate the journey of any single content throughout pack blobs, indexes and compaction events.
805 lines
23 KiB
Go
805 lines
23 KiB
Go
package contentlog
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestEntryWriter_EmptyObject(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
jw.EndObject()
|
|
|
|
require.Equal(t, "{}", string(jw.buf))
|
|
}
|
|
|
|
func TestEntryWriter_AllTypes(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
jw.Int64Field("k1", 123)
|
|
jw.Int64Field("a1", 1234)
|
|
jw.Int64Field("b1", 12345)
|
|
jw.BoolField("b2", true)
|
|
jw.BoolField("b3", false)
|
|
|
|
jw.BeginObjectField("o1")
|
|
jw.Int64Field("k2", 123456)
|
|
jw.EndObject()
|
|
|
|
jw.BeginObjectField("o2")
|
|
jw.EndObject()
|
|
|
|
jw.BeginListField("l1")
|
|
jw.StringElement("aaa")
|
|
jw.StringElement("bbb")
|
|
jw.StringElement("ccc")
|
|
jw.EndList()
|
|
|
|
jw.BeginListField("mixedList")
|
|
jw.StringElement("aaa")
|
|
jw.Int64Element(123)
|
|
jw.NullElement()
|
|
jw.BoolElement(true)
|
|
jw.EndList()
|
|
|
|
jw.BeginObjectField("o3")
|
|
jw.StringField("v", "xxx")
|
|
jw.StringField("someUnicode", "😄")
|
|
jw.EndObject()
|
|
|
|
jw.UInt64Field("u1", 123456)
|
|
jw.StringField("s", "hello\nworld\r\t\b\f")
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
json.NewEncoder(os.Stdout).Encode("😄")
|
|
|
|
t.Logf("buf: %s", string(jw.buf))
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, map[string]any{
|
|
"k1": 123.0,
|
|
"a1": 1234.0,
|
|
"b1": 12345.0,
|
|
"b2": true,
|
|
"b3": false,
|
|
"u1": 123456.0,
|
|
"mixedList": []any{"aaa", 123.0, nil, true},
|
|
"l1": []any{"aaa", "bbb", "ccc"},
|
|
"o2": map[string]any{},
|
|
"o3": map[string]any{
|
|
"v": "xxx",
|
|
"someUnicode": "😄",
|
|
},
|
|
"o1": map[string]any{
|
|
"k2": 123456.0,
|
|
},
|
|
"s": "hello\nworld\r\t\b\f",
|
|
}, v)
|
|
}
|
|
|
|
func TestJSONWriter_IntTypes(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
jw.IntField("intField", 42)
|
|
jw.Int8Field("int8Field", 8)
|
|
jw.Int16Field("int16Field", 16)
|
|
jw.Int32Field("int32Field", 32)
|
|
jw.Int64Field("int64Field", 64)
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, map[string]any{
|
|
"intField": 42.0,
|
|
"int8Field": 8.0,
|
|
"int16Field": 16.0,
|
|
"int32Field": 32.0,
|
|
"int64Field": 64.0,
|
|
}, v)
|
|
}
|
|
|
|
func TestJSONWriter_IntElements(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginList()
|
|
jw.IntElement(1)
|
|
jw.Int8Element(2)
|
|
jw.Int16Element(3)
|
|
jw.Int32Element(4)
|
|
jw.Int64Element(5)
|
|
jw.EndList()
|
|
|
|
var v []any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, []any{1.0, 2.0, 3.0, 4.0, 5.0}, v)
|
|
}
|
|
|
|
func TestJSONWriter_UIntTypes(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
jw.UIntField("uintField", 100)
|
|
jw.UInt8Field("uint8Field", 200)
|
|
jw.UInt16Field("uint16Field", 300)
|
|
jw.UInt32Field("uint32Field", 400)
|
|
jw.UInt64Field("uint64Field", 500)
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, map[string]any{
|
|
"uintField": 100.0,
|
|
"uint8Field": 200.0,
|
|
"uint16Field": 300.0,
|
|
"uint32Field": 400.0,
|
|
"uint64Field": 500.0,
|
|
}, v)
|
|
}
|
|
|
|
func TestJSONWriter_UIntElements(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginList()
|
|
jw.UIntElement(10)
|
|
jw.UInt8Element(20)
|
|
jw.UInt16Element(30)
|
|
jw.UInt32Element(40)
|
|
jw.UInt64Element(50)
|
|
jw.EndList()
|
|
|
|
var v []any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, []any{10.0, 20.0, 30.0, 40.0, 50.0}, v)
|
|
}
|
|
|
|
func TestJSONWriter_NullField(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
jw.NullField("nullField")
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, map[string]any{
|
|
"nullField": nil,
|
|
}, v)
|
|
}
|
|
|
|
func TestJSONWriter_ErrorField(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
jw.ErrorField("nilError", nil)
|
|
jw.ErrorField("realError", &testError{msg: "test error message"})
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, map[string]any{
|
|
"nilError": nil,
|
|
"realError": "test error message",
|
|
}, v)
|
|
}
|
|
|
|
type testError struct {
|
|
msg string
|
|
}
|
|
|
|
func (e *testError) Error() string {
|
|
return e.msg
|
|
}
|
|
|
|
func TestJSONWriter_TimeField(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
testTime := time.Date(2023, 12, 25, 15, 30, 45, 123456789, time.UTC)
|
|
|
|
jw.BeginObject()
|
|
jw.TimeField("timeField", testTime)
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, map[string]any{
|
|
"timeField": "2023-12-25T15:30:45.123456Z",
|
|
}, v)
|
|
}
|
|
|
|
func TestJSONWriter_TimeElement(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
testTime := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
|
|
jw.BeginList()
|
|
jw.TimeElement(testTime)
|
|
jw.EndList()
|
|
|
|
var v []any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, []any{"2023-01-01T00:00:00.000000Z"}, v)
|
|
}
|
|
|
|
func TestJSONWriter_BeginList(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginList()
|
|
jw.StringElement("first")
|
|
jw.StringElement("second")
|
|
jw.EndList()
|
|
|
|
var v []any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, []any{"first", "second"}, v)
|
|
}
|
|
|
|
func TestJSONWriter_EdgeCases(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
jw.StringField("emptyString", "")
|
|
jw.StringField("unicodeString", "Hello 世界 🌍")
|
|
jw.StringField("quotesAndSlashes", "quoted and backslash")
|
|
jw.Int64Field("zero", 0)
|
|
jw.Int64Field("negative", -1)
|
|
jw.Int64Field("maxInt64", 9223372036854775807)
|
|
jw.Int64Field("minInt64", -9223372036854775808)
|
|
jw.UInt64Field("maxUInt64", 18446744073709551615)
|
|
jw.BoolField("trueValue", true)
|
|
jw.BoolField("falseValue", false)
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, map[string]any{
|
|
"emptyString": "",
|
|
"unicodeString": "Hello 世界 🌍",
|
|
"quotesAndSlashes": "quoted and backslash",
|
|
"zero": 0.0,
|
|
"negative": -1.0,
|
|
"maxInt64": 9223372036854775807.0,
|
|
"minInt64": -9223372036854775808.0,
|
|
"maxUInt64": 18446744073709551615.0,
|
|
"trueValue": true,
|
|
"falseValue": false,
|
|
}, v)
|
|
}
|
|
|
|
func TestJSONWriter_StringEscaping(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
jw.StringField("quotes", `"quoted string"`)
|
|
jw.StringField("backslashes", `path\to\file`)
|
|
jw.StringField("mixed", `"quoted" and \backslash\`)
|
|
jw.StringField("newline", "line1\nline2")
|
|
jw.StringField("carriageReturn", "line1\rline2")
|
|
jw.StringField("tab", "col1\tcol2")
|
|
jw.StringField("backspace", "text\btext")
|
|
jw.StringField("formFeed", "text")
|
|
jw.StringField("allControlChars", "a\bb\fc\nd\re\tf")
|
|
jw.StringField("unicode", "Hello 世界 🌍")
|
|
jw.StringField("empty", "")
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
|
|
// Verify the JSON can be parsed and contains expected values
|
|
require.Equal(t, `"quoted string"`, v["quotes"])
|
|
require.Equal(t, `path\to\file`, v["backslashes"])
|
|
require.Equal(t, `"quoted" and \backslash\`, v["mixed"])
|
|
require.Equal(t, "line1\nline2", v["newline"])
|
|
require.Equal(t, "line1\rline2", v["carriageReturn"])
|
|
require.Equal(t, "col1\tcol2", v["tab"])
|
|
require.Equal(t, "text\btext", v["backspace"])
|
|
require.Equal(t, "text", v["formFeed"])
|
|
require.Equal(t, "a\bb\fc\nd\re\tf", v["allControlChars"])
|
|
require.Equal(t, "Hello 世界 🌍", v["unicode"])
|
|
require.Empty(t, v["empty"])
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingElements(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginList()
|
|
jw.StringElement(`"quoted"`)
|
|
jw.StringElement(`\backslash\`)
|
|
jw.StringElement("mixed\n\t\r")
|
|
jw.StringElement("unicode: 世界")
|
|
jw.EndList()
|
|
|
|
var v []any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, []any{`"quoted"`, `\backslash\`, "mixed\n\t\r", "unicode: 世界"}, v)
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingEdgeCases(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
jw.StringField("onlyQuote", `"`)
|
|
jw.StringField("onlyBackslash", `\`)
|
|
jw.StringField("onlyNewline", "\n")
|
|
jw.StringField("onlyTab", "\t")
|
|
jw.StringField("multipleQuotes", `""""`)
|
|
jw.StringField("multipleBackslashes", `\\\\`)
|
|
jw.StringField("quoteBackslash", `"\`)
|
|
jw.StringField("backslashQuote", `\"`)
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
|
|
require.Equal(t, `"`, v["onlyQuote"])
|
|
require.Equal(t, `\`, v["onlyBackslash"])
|
|
require.Equal(t, "\n", v["onlyNewline"])
|
|
require.Equal(t, "\t", v["onlyTab"])
|
|
require.Equal(t, `""""`, v["multipleQuotes"])
|
|
require.Equal(t, `\\\\`, v["multipleBackslashes"])
|
|
require.Equal(t, `"\`, v["quoteBackslash"])
|
|
require.Equal(t, `\"`, v["backslashQuote"])
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingRawOutput(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
jw.StringField("quotes", `"test"`)
|
|
jw.StringField("backslashes", `\test\`)
|
|
jw.StringField("mixed", `"quoted" and \backslash\`)
|
|
jw.EndObject()
|
|
|
|
jsonOutput := string(jw.buf)
|
|
|
|
// Verify that quotes are properly escaped in the raw JSON
|
|
require.Contains(t, jsonOutput, `\"test\"`)
|
|
require.Contains(t, jsonOutput, `\\test\\`)
|
|
require.Contains(t, jsonOutput, `\"quoted\" and \\backslash\\`)
|
|
|
|
// Verify the JSON is valid
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
require.Equal(t, `"test"`, v["quotes"])
|
|
require.Equal(t, `\test\`, v["backslashes"])
|
|
require.Equal(t, `"quoted" and \backslash\`, v["mixed"])
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingControlChars(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
// Test various control characters
|
|
jw.StringField("nullChar", "\x00")
|
|
jw.StringField("bellChar", "\x07")
|
|
jw.StringField("verticalTab", "\x0b")
|
|
jw.StringField("escapeChar", "\x1b")
|
|
jw.StringField("delChar", "\x7f")
|
|
jw.EndObject()
|
|
|
|
jsonOutput := string(jw.buf)
|
|
t.Logf("Control chars JSON output: %q", jsonOutput)
|
|
|
|
// The JSONWriter doesn't properly escape control characters < ' ' except for \b, \f, \n, \r, \t
|
|
// This means the JSON will be invalid for characters like \x00, \x07, etc.
|
|
// This test documents the current behavior - these characters are not properly escaped
|
|
var v map[string]any
|
|
|
|
if err := json.Unmarshal(jw.buf, &v); err != nil {
|
|
t.Logf("Expected error due to unescaped control characters: %v", err)
|
|
// This is expected behavior - the JSONWriter has a bug with control character escaping
|
|
require.Error(t, err)
|
|
} else {
|
|
// If it somehow works, verify the values
|
|
require.Equal(t, "\x00", v["nullChar"])
|
|
require.Equal(t, "\x07", v["bellChar"])
|
|
require.Equal(t, "\x0b", v["verticalTab"])
|
|
require.Equal(t, "\x1b", v["escapeChar"])
|
|
require.Equal(t, "\x7f", v["delChar"])
|
|
}
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingProperlyHandledControlChars(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
// Test control characters that ARE properly handled by JSONWriter
|
|
jw.StringField("backspace", "\b")
|
|
jw.StringField("formFeed", "\f")
|
|
jw.StringField("newline", "\n")
|
|
jw.StringField("carriageReturn", "\r")
|
|
jw.StringField("tab", "\t")
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
|
|
// These should be properly escaped and work correctly
|
|
require.Equal(t, "\b", v["backspace"])
|
|
require.Equal(t, "\f", v["formFeed"])
|
|
require.Equal(t, "\n", v["newline"])
|
|
require.Equal(t, "\r", v["carriageReturn"])
|
|
require.Equal(t, "\t", v["tab"])
|
|
|
|
// Verify the raw JSON contains proper escape sequences
|
|
jsonOutput := string(jw.buf)
|
|
t.Logf("JSON output: %q", jsonOutput)
|
|
|
|
// The JSON should be valid and contain the escaped control characters
|
|
// We can see from the output that it contains \"\\b\" etc.
|
|
require.Contains(t, jsonOutput, `backspace`)
|
|
require.Contains(t, jsonOutput, `formFeed`)
|
|
require.Contains(t, jsonOutput, `newline`)
|
|
require.Contains(t, jsonOutput, `carriageReturn`)
|
|
require.Contains(t, jsonOutput, `tab`)
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingAllControlCharacters(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
|
|
// Test all control characters from 0x00 to 0x1F
|
|
controlChars := map[string]string{
|
|
"null": "\x00", // NUL
|
|
"startOfHeading": "\x01", // SOH
|
|
"startOfText": "\x02", // STX
|
|
"endOfText": "\x03", // ETX
|
|
"endOfTransmit": "\x04", // EOT
|
|
"enquiry": "\x05", // ENQ
|
|
"acknowledge": "\x06", // ACK
|
|
"bell": "\x07", // BEL
|
|
"backspace": "\x08", // BS - handled specially
|
|
"tab": "\x09", // TAB - handled specially
|
|
"lineFeed": "\x0a", // LF - handled specially
|
|
"verticalTab": "\x0b", // VT
|
|
"formFeed": "\x0c", // FF - handled specially
|
|
"carriageReturn": "\x0d", // CR - handled specially
|
|
"shiftOut": "\x0e", // SO
|
|
"shiftIn": "\x0f", // SI
|
|
"dataLinkEscape": "\x10", // DLE
|
|
"deviceCtrl1": "\x11", // DC1
|
|
"deviceCtrl2": "\x12", // DC2
|
|
"deviceCtrl3": "\x13", // DC3
|
|
"deviceCtrl4": "\x14", // DC4
|
|
"negativeAck": "\x15", // NAK
|
|
"synchronousIdle": "\x16", // SYN
|
|
"endOfTransBlock": "\x17", // ETB
|
|
"cancel": "\x18", // CAN
|
|
"endOfMedium": "\x19", // EM
|
|
"substitute": "\x1a", // SUB
|
|
"escape": "\x1b", // ESC
|
|
"fileSeparator": "\x1c", // FS
|
|
"groupSeparator": "\x1d", // GS
|
|
"recordSeparator": "\x1e", // RS
|
|
"unitSeparator": "\x1f", // US
|
|
}
|
|
|
|
// Add all control characters as fields
|
|
for name, char := range controlChars {
|
|
jw.StringField(name, char)
|
|
}
|
|
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
|
|
// Verify all control characters are properly handled
|
|
for name, expectedChar := range controlChars {
|
|
require.Equal(t, expectedChar, v[name], "Control character %s (0x%02x) not properly handled", name, expectedChar[0])
|
|
}
|
|
|
|
// Verify the raw JSON contains proper Unicode escape sequences for non-special control chars
|
|
jsonOutput := string(jw.buf)
|
|
t.Logf("Control chars JSON output: %q", jsonOutput)
|
|
|
|
// Check that special control characters use their standard escape sequences
|
|
require.Contains(t, jsonOutput, `\b`) // backspace
|
|
require.Contains(t, jsonOutput, `\t`) // tab
|
|
require.Contains(t, jsonOutput, `\n`) // line feed
|
|
require.Contains(t, jsonOutput, `\f`) // form feed
|
|
require.Contains(t, jsonOutput, `\r`) // carriage return
|
|
|
|
// Check that other control characters use Unicode escape sequences
|
|
require.Contains(t, jsonOutput, `\u0000`) // null
|
|
require.Contains(t, jsonOutput, `\u0001`) // start of heading
|
|
require.Contains(t, jsonOutput, `\u0007`) // bell
|
|
require.Contains(t, jsonOutput, `\u000b`) // vertical tab
|
|
require.Contains(t, jsonOutput, `\u001b`) // escape
|
|
require.Contains(t, jsonOutput, `\u001f`) // unit separator
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingControlCharactersInElements(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginList()
|
|
|
|
// Test control characters as list elements
|
|
jw.StringElement("\x00") // null
|
|
jw.StringElement("\x07") // bell
|
|
jw.StringElement("\x08") // backspace
|
|
jw.StringElement("\x09") // tab
|
|
jw.StringElement("\x0a") // line feed
|
|
jw.StringElement("\x0c") // form feed
|
|
jw.StringElement("\x0d") // carriage return
|
|
jw.StringElement("\x1b") // escape
|
|
jw.StringElement("\x1f") // unit separator
|
|
jw.EndList()
|
|
|
|
var v []any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
|
|
expected := []any{
|
|
"\x00", // null
|
|
"\x07", // bell
|
|
"\x08", // backspace
|
|
"\x09", // tab
|
|
"\x0a", // line feed
|
|
"\x0c", // form feed
|
|
"\x0d", // carriage return
|
|
"\x1b", // escape
|
|
"\x1f", // unit separator
|
|
}
|
|
|
|
require.Equal(t, expected, v)
|
|
|
|
// Verify the raw JSON contains proper escape sequences
|
|
jsonOutput := string(jw.buf)
|
|
t.Logf("Control chars in elements JSON output: %q", jsonOutput)
|
|
|
|
require.Contains(t, jsonOutput, `\u0000`) // null
|
|
require.Contains(t, jsonOutput, `\u0007`) // bell
|
|
require.Contains(t, jsonOutput, `\b`) // backspace
|
|
require.Contains(t, jsonOutput, `\t`) // tab
|
|
require.Contains(t, jsonOutput, `\n`) // line feed
|
|
require.Contains(t, jsonOutput, `\f`) // form feed
|
|
require.Contains(t, jsonOutput, `\r`) // carriage return
|
|
require.Contains(t, jsonOutput, `\u001b`) // escape
|
|
require.Contains(t, jsonOutput, `\u001f`) // unit separator
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingMixedControlCharacters(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
|
|
// Test strings with mixed control characters and regular characters
|
|
jw.StringField("mixed1", "hello\x00world\x07test")
|
|
jw.StringField("mixed2", "start\x08\x09\x0a\x0c\x0dend")
|
|
jw.StringField("mixed3", "text\x1b\x1c\x1d\x1e\x1fmore")
|
|
jw.StringField("mixed4", "a\x00b\x01c\x02d\x03e")
|
|
jw.StringField("mixed5", "quotes\"and\\backslash\x00control")
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
|
|
// Verify the parsed values match the original strings
|
|
require.Equal(t, "hello\x00world\x07test", v["mixed1"])
|
|
require.Equal(t, "start\x08\x09\x0a\x0c\x0dend", v["mixed2"])
|
|
require.Equal(t, "text\x1b\x1c\x1d\x1e\x1fmore", v["mixed3"])
|
|
require.Equal(t, "a\x00b\x01c\x02d\x03e", v["mixed4"])
|
|
require.Equal(t, "quotes\"and\\backslash\x00control", v["mixed5"])
|
|
|
|
// Verify the raw JSON contains proper escape sequences
|
|
jsonOutput := string(jw.buf)
|
|
t.Logf("Mixed control chars JSON output: %q", jsonOutput)
|
|
|
|
// Check for Unicode escapes
|
|
require.Contains(t, jsonOutput, `\u0000`) // null character
|
|
require.Contains(t, jsonOutput, `\u0007`) // bell
|
|
require.Contains(t, jsonOutput, `\u0001`) // start of heading
|
|
require.Contains(t, jsonOutput, `\u0002`) // start of text
|
|
require.Contains(t, jsonOutput, `\u0003`) // end of text
|
|
require.Contains(t, jsonOutput, `\u001b`) // escape
|
|
require.Contains(t, jsonOutput, `\u001c`) // file separator
|
|
require.Contains(t, jsonOutput, `\u001d`) // group separator
|
|
require.Contains(t, jsonOutput, `\u001e`) // record separator
|
|
require.Contains(t, jsonOutput, `\u001f`) // unit separator
|
|
|
|
// Check for standard escapes
|
|
require.Contains(t, jsonOutput, `\b`) // backspace
|
|
require.Contains(t, jsonOutput, `\t`) // tab
|
|
require.Contains(t, jsonOutput, `\n`) // line feed
|
|
require.Contains(t, jsonOutput, `\f`) // form feed
|
|
require.Contains(t, jsonOutput, `\r`) // carriage return
|
|
|
|
// Check for quote and backslash escapes
|
|
require.Contains(t, jsonOutput, `\"`) // escaped quote
|
|
require.Contains(t, jsonOutput, `\\`) // escaped backslash
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingBoundaryValues(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
|
|
// Test boundary values around control character range
|
|
jw.StringField("space", " ") // 0x20 - first non-control character
|
|
jw.StringField("del", "\x7f") // 0x7F - DEL character (not in 0x00-0x1F range)
|
|
jw.StringField("lastControl", "\x1f") // 0x1F - last control character
|
|
jw.StringField("firstNonControl", " ") // 0x20 - first non-control character
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
|
|
// Verify the parsed values
|
|
require.Equal(t, " ", v["space"])
|
|
require.Equal(t, "\x7f", v["del"]) // DEL should not be escaped as Unicode
|
|
require.Equal(t, "\x1f", v["lastControl"]) // Last control char should be escaped
|
|
require.Equal(t, " ", v["firstNonControl"])
|
|
|
|
// Verify the raw JSON
|
|
jsonOutput := string(jw.buf)
|
|
t.Logf("Boundary values JSON output: %q", jsonOutput)
|
|
|
|
// Space (0x20) should not be escaped
|
|
require.Contains(t, jsonOutput, `"space":" "`)
|
|
require.Contains(t, jsonOutput, `"firstNonControl":" "`)
|
|
|
|
// DEL (0x7F) should not be escaped as Unicode (it's outside 0x00-0x1F range)
|
|
// The DEL character is output as-is in the JSON (not escaped)
|
|
require.Contains(t, jsonOutput, `"del":"`+string('\x7f')+`"`)
|
|
|
|
// Last control character (0x1F) should be escaped as Unicode
|
|
require.Contains(t, jsonOutput, `\u001f`)
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingUnicodeEscapeFormat(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
|
|
// Test specific control characters to verify Unicode escape format
|
|
jw.StringField("null", "\x00")
|
|
jw.StringField("bell", "\x07")
|
|
jw.StringField("verticalTab", "\x0b")
|
|
jw.StringField("escape", "\x1b")
|
|
jw.StringField("unitSeparator", "\x1f")
|
|
jw.EndObject()
|
|
|
|
jsonOutput := string(jw.buf)
|
|
t.Logf("Unicode escape format JSON output: %q", jsonOutput)
|
|
|
|
// Verify exact Unicode escape format: \u00XX where XX is the hex value
|
|
require.Contains(t, jsonOutput, `\u0000`) // null (0x00)
|
|
require.Contains(t, jsonOutput, `\u0007`) // bell (0x07)
|
|
require.Contains(t, jsonOutput, `\u000b`) // vertical tab (0x0B)
|
|
require.Contains(t, jsonOutput, `\u001b`) // escape (0x1B)
|
|
require.Contains(t, jsonOutput, `\u001f`) // unit separator (0x1F)
|
|
|
|
// Verify the format is exactly 6 characters: \u + 4 hex digits
|
|
// This is a more specific test to ensure the format is correct
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
|
|
// Verify the values are correctly parsed back
|
|
require.Equal(t, "\x00", v["null"])
|
|
require.Equal(t, "\x07", v["bell"])
|
|
require.Equal(t, "\x0b", v["verticalTab"])
|
|
require.Equal(t, "\x1b", v["escape"])
|
|
require.Equal(t, "\x1f", v["unitSeparator"])
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingPerformanceWithManyControlChars(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
|
|
// Create a string with many control characters to test performance
|
|
var testString string
|
|
for i := range 100 {
|
|
testString += string(rune(i % 32)) // Mix of control chars 0x00-0x1F
|
|
}
|
|
|
|
jw.StringField("manyControlChars", testString)
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
|
|
// Verify the string is correctly handled
|
|
require.Equal(t, testString, v["manyControlChars"])
|
|
|
|
// Verify the JSON is valid and contains many Unicode escapes
|
|
jsonOutput := string(jw.buf)
|
|
t.Logf("Performance test JSON output length: %d", len(jsonOutput))
|
|
|
|
// Should contain many Unicode escape sequences
|
|
require.Contains(t, jsonOutput, `\u0000`)
|
|
require.Contains(t, jsonOutput, `\u0001`)
|
|
require.Contains(t, jsonOutput, `\u000f`)
|
|
require.Contains(t, jsonOutput, `\u001f`)
|
|
}
|
|
|
|
func TestJSONWriter_StringEscapingEmptyAndSingleChar(t *testing.T) {
|
|
jw := NewJSONWriter()
|
|
defer jw.Release()
|
|
|
|
jw.BeginObject()
|
|
|
|
// Test edge cases with empty strings and single control characters
|
|
jw.StringField("empty", "")
|
|
jw.StringField("singleNull", "\x00")
|
|
jw.StringField("singleBell", "\x07")
|
|
jw.StringField("singleTab", "\x09")
|
|
jw.StringField("singleNewline", "\x0a")
|
|
jw.EndObject()
|
|
|
|
var v map[string]any
|
|
|
|
require.NoError(t, json.Unmarshal(jw.buf, &v))
|
|
|
|
// Verify edge cases
|
|
require.Empty(t, v["empty"])
|
|
require.Equal(t, "\x00", v["singleNull"])
|
|
require.Equal(t, "\x07", v["singleBell"])
|
|
require.Equal(t, "\x09", v["singleTab"])
|
|
require.Equal(t, "\x0a", v["singleNewline"])
|
|
|
|
// Verify the raw JSON
|
|
jsonOutput := string(jw.buf)
|
|
t.Logf("Edge cases JSON output: %q", jsonOutput)
|
|
|
|
require.Contains(t, jsonOutput, `"empty":""`)
|
|
require.Contains(t, jsonOutput, `"singleNull":"\u0000"`)
|
|
require.Contains(t, jsonOutput, `"singleBell":"\u0007"`)
|
|
require.Contains(t, jsonOutput, `"singleTab":"\t"`)
|
|
require.Contains(t, jsonOutput, `"singleNewline":"\n"`)
|
|
}
|