fix: strip null bytes from parsed file content before retain (#535)

* doc: split blog index into Hindsight and Hindsight Cloud sections

- Tag the document upload post with `hindsight-cloud`
- BlogListPage renders two sections, capping Cloud at 3 posts with a "View all →" link
- Swizzle BlogTagsPostsPage so /blog/tags/hindsight-cloud uses the custom grid layout

* doc: attribute blog posts to Nicolò Boschi with GitHub profile image

Replace the generic "Hindsight Team" author with the real author entry
(nicoloboschi) across all 15 blog posts. GitHub profile image is loaded
from https://github.com/nicoloboschi.png.

* doc: add Hindsight Team title to nicoloboschi author

* doc: assign blog posts to correct authors based on git blame

- Add benfrank241 (Ben Bartholomew) and chrislatimer (Chris Latimer) to authors.yml
- Assign 7 posts to Ben, 1 post to Chris, remainder stay with Nicolò

* fix: strip null bytes from parsed file content before retain

* test: add tests for sanitize_llm_output

* fix: retry retain DB transaction on deadlock during parallel document processing

hindsight-api/hindsight_api/engine/db_utils.py

@@ -58,6 +58,12 @@ async def retry_with_backoff(
             last_exception = e
             if attempt < max_retries:
                 delay = min(base_delay * (2**attempt), max_delay)
+                if isinstance(e, asyncpg.exceptions.DeadlockDetectedError):
+                    logger.warning(
+                        f"Deadlock detected during parallel document processing — this is expected and will resolve automatically "
+                        f"(attempt {attempt + 1}/{max_retries + 1}, retrying in {delay:.1f}s)"
+                    )
+                else:
                     logger.warning(
                         f"Database operation failed (attempt {attempt + 1}/{max_retries + 1}): {e}. "
                         f"Retrying in {delay:.1f}s..."

hindsight-api/hindsight_api/engine/memory_engine.py

@@ -168,7 +168,7 @@ from enum import Enum
 from ..metrics import get_metrics_collector
 from ..pg0 import EmbeddedPostgres, parse_pg0_url
 from .entity_resolver import EntityResolver
-from .llm_wrapper import LLMConfig, requires_api_key
+from .llm_wrapper import LLMConfig, requires_api_key, sanitize_llm_output
 from .query_analyzer import QueryAnalyzer
 from .reflect import run_reflect_agent
 from .reflect.tools import tool_expand, tool_recall, tool_search_mental_models, tool_search_observations
@@ -664,7 +664,7 @@ class MemoryEngine(MemoryEngineInterface):
                 filename=filename,
                 content_type=task_dict.get("content_type"),
             )
-            markdown_content = convert_result.content
+            markdown_content = sanitize_llm_output(convert_result.content) or ""
             winning_parser = convert_result.parser_name
         except Exception as e:
             # Re-raise with filename context for better error reporting

hindsight-api/hindsight_api/engine/retain/orchestrator.py

@@ -11,7 +11,7 @@ from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime
 from typing import Any
 
-from ..db_utils import acquire_with_retry
+from ..db_utils import acquire_with_retry, retry_with_backoff
 from . import bank_utils
 
 
@@ -269,9 +269,6 @@ async def retain_batch(
         for extracted_fact, embedding in zip(extracted_facts, embeddings)
     ]
 
-    # Track document IDs for logging
-    document_ids_added = []
-
     # Group contents by document_id for document tracking and chunk storage
     from collections import defaultdict
 
@@ -280,7 +277,21 @@ async def retain_batch(
         doc_id = content_dict.get("document_id")
         contents_by_doc[doc_id].append((idx, content_dict))
 
-    # Step 4: Database transaction
+    # Step 4: Database transaction (retried on deadlock)
+    result_unit_ids: list[list[str]] = []
+
+    log_buffer_pre_db = len(log_buffer)
+
+    async def _run_db_work() -> None:
+        nonlocal result_unit_ids
+
+        # Reset per-fact mutations and log buffer so each retry attempt starts clean
+        del log_buffer[log_buffer_pre_db:]
+        document_ids_added: list[str] = []
+        for pf in processed_facts:
+            pf.document_id = None
+            pf.chunk_id = None
+
         async with acquire_with_retry(pool) as conn:
             async with conn.transaction():
                 # Handle document tracking for all documents
@@ -507,6 +518,7 @@ async def retain_batch(
 
             logger.info("\n" + "\n".join(log_buffer) + "\n")
 
+    await retry_with_backoff(_run_db_work)
     return result_unit_ids, usage
 
 

hindsight-api/tests/test_llm_wrapper.py

@@ -0,0 +1,37 @@
+import pytest
+
+from hindsight_api.engine.llm_wrapper import sanitize_llm_output
+
+
+@pytest.mark.parametrize(
+    "input_text, expected",
+    [
+        # Null bytes stripped
+        ("hello\x00world", "helloworld"),
+        ("FIRST\u0000PAGE", "FIRSTPAGE"),
+        # Multiple null bytes
+        ("\x00\x00text\x00", "text"),
+        # Other control characters stripped (non-whitespace)
+        ("text\x01\x02\x03end", "textend"),
+        ("text\x08end", "textend"),  # backspace
+        ("text\x0cend", "textend"),  # form feed
+        ("text\x0bend", "textend"),  # vertical tab
+        ("text\x1fend", "textend"),  # unit separator
+        ("text\x7fend", "textend"),  # DEL
+        # Whitespace preserved
+        ("hello\tworld", "hello\tworld"),
+        ("hello\nworld", "hello\nworld"),
+        ("hello\r\nworld", "hello\r\nworld"),
+        # Unicode surrogates stripped
+        ("text\ud800end", "textend"),
+        ("text\udfffend", "textend"),
+        # Clean text unchanged
+        ("normal text", "normal text"),
+        ("unicode: café naïve", "unicode: café naïve"),
+        # Edge cases
+        ("", ""),
+        (None, None),
+    ],
+)
+def test_sanitize_llm_output(input_text, expected):
+    assert sanitize_llm_output(input_text) == expected