",
config: newTestConfig(100, 10, 5),
expectedChunks: 1, // Should still parse leniently
},
{
name: "Deeply Nested HTML",
html: "
",
config: newTestConfig(100, 10, 5),
expectedChunks: 1,
},
{
name: "HTML Entities and Special Characters",
html: "
This is & some text with <entities> and unicode © characters.
",
config: newTestConfig(100, 10, 5),
expectedChunks: 1,
},
{
name: "Excessive Whitespace",
html: "
\n\t leading and trailing spaces \n\n
",
config: newTestConfig(100, 10, 5),
expectedChunks: 1,
validate: func(t *testing.T, chunks []string) {
assert.NotContains(t, chunks[0], " ", "Whitespace should be normalized")
},
},
{
name: "Giant Token Test - Single Massive Block",
html: "
" + strings.Repeat("word ", 1000) + "
", // 1000 tokens in one block
config: newTestConfig(50, 20, 10),
validate: func(t *testing.T, chunks []string) {
assert.Greater(t, len(chunks), 1, "An oversized block must be split across chunks, not truncated")
kept := strings.Count(strings.Join(chunks, " "), "word")
assert.GreaterOrEqual(t, kept, 950, "Nearly every word must survive; truncation used to drop the tail")
},
expectedChunks: -1, // Validated via validate func
},
{
name: "Oversized Wrapper Div Splits Into Children",
html: "
" + strings.Repeat("
"+strings.Repeat("word ", 100)+"
", 10) + "
",
config: newTestConfig(150, 20, 10),
validate: func(t *testing.T, chunks []string) {
assert.Greater(t, len(chunks), 1, "Wrapper div content should be split, not truncated to one chunk")
full := strings.Join(chunks, " ")
assert.Greater(t, len(full), 4000, "Most of the wrapped content should survive chunking")
},
expectedChunks: -1, // Validated via validate func
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
chunks, err := ChunkHTMLContent("Edge Case", tc.html, tc.config)
if tc.expectedError != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), tc.expectedError)
} else {
require.NoError(t, err)
if tc.expectedChunks >= 0 {
assert.Len(t, chunks, tc.expectedChunks)
}
if tc.validate != nil {
tc.validate(t, chunks)
}
}
})
}
}
func TestChunkHTMLContent_PlainText(t *testing.T) {
testCases := []struct {
name string
content string
}{
{"short plain text", "25 MB"},
{"plain sentence no tags", "The binary size of libredesk is about 25 MB."},
{"text with inline tag only", "Size is
25 MB"},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
chunks, err := ChunkHTMLContent("Whats binary size of libredesk", tc.content)
require.NoError(t, err)
require.Len(t, chunks, 1, "plain text must still produce one chunk to embed")
assert.Contains(t, chunks[0], "25 MB")
})
}
}
// TestChunkHTMLContent_NoSilentContentLoss covers the ways content used to vanish from the index
// with no error: text outside a block element was never collected, and an atomic block over
// MaxTokens was truncated instead of split.
func TestChunkHTMLContent_NoSilentContentLoss(t *testing.T) {
t.Run("Loose Text Around A Block Element", func(t *testing.T) {
chunks, err := ChunkHTMLContent("Refunds",
"Refunds are issued within thirty days.\n"+
"
\n"+
"Email support to start a refund.")
require.NoError(t, err)
joined := strings.Join(chunks, "\n")
assert.Contains(t, joined, "Refunds are issued within thirty days", "Prose before the block must be indexed")
assert.Contains(t, joined, "Email support to start a refund", "Prose after the block must be indexed")
assert.Contains(t, joined, "Plan", "The block itself must still be indexed")
})
t.Run("Oversized Loose Text Splits", func(t *testing.T) {
body := "HEAD_MARKER. " + strings.Repeat("The refund window is thirty days from purchase. ", 400) + "TAIL_MARKER."
chunks, err := ChunkHTMLContent("Refunds", body, newTestConfig(100, 30, 10))
require.NoError(t, err)
joined := strings.Join(chunks, "\n")
assert.Greater(t, len(chunks), 1, "Oversized loose text must split across chunks")
assert.Contains(t, joined, "HEAD_MARKER")
assert.Contains(t, joined, "TAIL_MARKER", "The tail must survive; truncation used to drop it")
})
t.Run("Oversized Split Keeps Sentence Terminators", func(t *testing.T) {
body := strings.Repeat("Can I get a refund? Yes you can! Contact support today. ", 40)
chunks, err := ChunkHTMLContent("Refunds", body, newTestConfig(30, 10, 5))
require.NoError(t, err)
joined := strings.Join(chunks, "\n")
assert.Contains(t, joined, "Can I get a refund?", "Question marks must survive sentence splitting")
assert.Contains(t, joined, "Yes you can!", "Exclamation marks must survive sentence splitting")
})
t.Run("Markup-Like Overlap Text Must Not Swallow The Next Chunk", func(t *testing.T) {
chunks, err := ChunkHTMLContent("Widget",
"
Install the widget first. Then add the <script> tag to your page.
"+
"
REAL_MARKER content that must survive chunking in the second chunk.
",
newTestConfig(20, 10, 8))
require.NoError(t, err)
assert.Contains(t, strings.Join(chunks, "\n"), "REAL_MARKER",
"An unescaped
Reset your password from settings.
trailing note