cd3a6a227b
* doc: split blog index into Hindsight and Hindsight Cloud sections - Tag the document upload post with `hindsight-cloud` - BlogListPage renders two sections, capping Cloud at 3 posts with a "View all →" link - Swizzle BlogTagsPostsPage so /blog/tags/hindsight-cloud uses the custom grid layout * doc: attribute blog posts to Nicolò Boschi with GitHub profile image Replace the generic "Hindsight Team" author with the real author entry (nicoloboschi) across all 15 blog posts. GitHub profile image is loaded from https://github.com/nicoloboschi.png. * doc: add Hindsight Team title to nicoloboschi author * doc: assign blog posts to correct authors based on git blame - Add benfrank241 (Ben Bartholomew) and chrislatimer (Chris Latimer) to authors.yml - Assign 7 posts to Ben, 1 post to Chris, remainder stay with Nicolò * fix: strip null bytes from parsed file content before retain * test: add tests for sanitize_llm_output * fix: retry retain DB transaction on deadlock during parallel document processing
37 lines
1.2 KiB
Python
37 lines
1.2 KiB
Python
import pytest
|
|
|
|
from hindsight_api.engine.llm_wrapper import sanitize_llm_output
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"input_text, expected",
|
|
[
|
|
# Null bytes stripped
|
|
("hello\x00world", "helloworld"),
|
|
("FIRST\u0000PAGE", "FIRSTPAGE"),
|
|
# Multiple null bytes
|
|
("\x00\x00text\x00", "text"),
|
|
# Other control characters stripped (non-whitespace)
|
|
("text\x01\x02\x03end", "textend"),
|
|
("text\x08end", "textend"), # backspace
|
|
("text\x0cend", "textend"), # form feed
|
|
("text\x0bend", "textend"), # vertical tab
|
|
("text\x1fend", "textend"), # unit separator
|
|
("text\x7fend", "textend"), # DEL
|
|
# Whitespace preserved
|
|
("hello\tworld", "hello\tworld"),
|
|
("hello\nworld", "hello\nworld"),
|
|
("hello\r\nworld", "hello\r\nworld"),
|
|
# Unicode surrogates stripped
|
|
("text\ud800end", "textend"),
|
|
("text\udfffend", "textend"),
|
|
# Clean text unchanged
|
|
("normal text", "normal text"),
|
|
("unicode: café naïve", "unicode: café naïve"),
|
|
# Edge cases
|
|
("", ""),
|
|
(None, None),
|
|
],
|
|
)
|
|
def test_sanitize_llm_output(input_text, expected):
|
|
assert sanitize_llm_output(input_text) == expected
|