Fix entity_id null constraint for non-ASCII entity names (#612)

* Fix entity_id null constraint for non-ASCII entity names (Turkish İ etc.)

Python's str.lower() and PostgreSQL's LOWER() produce different results for
some Unicode characters. The most common case is Turkish İ (U+0130):
  Python:     'İstanbul'.lower() == 'i\u0307stanbul' (i + combining dot, 2 chars)
  PostgreSQL: LOWER('İstanbul') == 'istanbul' (plain i, 1 char)

In _resolve_from_candidates, the fallback SELECT for conflicted entity names
passed Python-lowercased strings to LOWER(canonical_name) = ANY($names), so
PostgreSQL couldn't match them. entity_ids[idx] stayed None, which then
caused a NOT NULL violation on unit_entities.entity_id, failing the entire
retain.

Fix: pass original mixed-case names to the fallback SELECT and use
LOWER(canonical_name) = ANY(SELECT LOWER(n) FROM unnest($2) AS n) so
PostgreSQL lowercases both sides identically. The query also returns the
original input_name so we can add a Python-lowercased key to id_by_name
for the assignment loop that uses Python-lowercased keys.

* Add regression test for Unicode entity conflict

hindsight-api-slim/hindsight_api/engine/entity_resolver.py

@@ -477,19 +477,42 @@ class EntityResolver:
             id_by_name: dict[str, str] = {row["name_lower"]: row["id"] for row in inserted_rows}
 
             # Fallback SELECT for names that conflicted (another worker won the race).
-            missing = [n for n, _ in sorted_groups if n not in id_by_name]
-            if missing:
+            #
+            # IMPORTANT: we must let PostgreSQL do the lowercasing on BOTH sides of the
+            # comparison.  Python's str.lower() and PostgreSQL's LOWER() differ for some
+            # Unicode characters — most notably Turkish İ (U+0130):
+            #   Python:     'İstanbul'.lower()  == 'i\u0307stanbul'  (i + combining dot, 2 chars)
+            #   PostgreSQL: LOWER('İstanbul')   == 'istanbul'         (plain i, 1 char)
+            # Passing a Python-lowercased name to "LOWER(canonical_name) = ANY($2::text[])"
+            # would fail to match the stored entity, leaving entity_id as None and causing
+            # a NOT NULL constraint violation on unit_entities.entity_id.
+            #
+            # Fix: pass the original (mixed-case) input names and use
+            # "LOWER(canonical_name) = ANY(SELECT LOWER(n) FROM unnest($2) AS n)" so
+            # PostgreSQL lowercases both sides identically.  The query also returns the
+            # original input_name so we can index id_by_name by Python's lower() of that
+            # name, which is what the assignment loop below uses as its lookup key.
+            missing_original = [g.name for name_lower, g in sorted_groups if name_lower not in id_by_name]
+            if missing_original:
                 existing_rows = await conn.fetch(
                     f"""
-                    SELECT id, LOWER(canonical_name) AS name_lower
-                    FROM {fq_table("entities")}
-                    WHERE bank_id = $1 AND LOWER(canonical_name) = ANY($2::text[])
+                    SELECT e.id, LOWER(e.canonical_name) AS name_lower, inputs.input_name
+                    FROM {fq_table("entities")} e
+                    JOIN (
+                        SELECT LOWER(n) AS input_name_lower, n AS input_name
+                        FROM unnest($2::text[]) AS n
+                    ) AS inputs ON LOWER(e.canonical_name) = inputs.input_name_lower
+                    WHERE e.bank_id = $1
                     """,
                     bank_id,
-                    missing,
+                    missing_original,
                 )
                 for row in existing_rows:
                     id_by_name[row["name_lower"]] = row["id"]
+                    # Also index by Python's lower() of the original input name so the
+                    # assignment loop (which uses Python-lowercased keys) finds it even
+                    # when Python and PostgreSQL produce different lowercase strings.
+                    id_by_name[row["input_name"].lower()] = row["id"]
 
             # Assign entity IDs back and queue one stat per original mention so that
             # flush_pending_stats() increments mention_count by the true mention count,

hindsight-api-slim/tests/test_entity_resolver.py

@@ -0,0 +1,70 @@
+"""
+Tests for EntityResolver edge cases.
+"""
+
+import uuid
+from datetime import datetime, timezone
+
+import asyncpg
+import pytest
+
+from hindsight_api.engine.entity_resolver import EntityResolver
+from hindsight_api.pg0 import resolve_database_url
+
+
+@pytest.mark.asyncio
+async def test_resolve_entities_batch_handles_unicode_lower_conflicts(pg0_db_url):
+    """
+    Existing entities with PostgreSQL/Python lowercase mismatches should resolve
+    to the conflicted row instead of leaving a missing entity_id.
+    """
+    resolved_url = await resolve_database_url(pg0_db_url)
+    pool = await asyncpg.create_pool(resolved_url, min_size=1, max_size=2, command_timeout=30)
+    bank_id = f"test-entity-resolver-{uuid.uuid4().hex[:8]}"
+    event_date = datetime(2024, 1, 15, tzinfo=timezone.utc)
+    resolver = EntityResolver(pool=pool, entity_lookup="full")
+
+    try:
+        async with pool.acquire() as conn:
+            existing_entity_id = await conn.fetchval(
+                """
+                INSERT INTO entities (bank_id, canonical_name, first_seen, last_seen, mention_count)
+                VALUES ($1, $2, $3, $3, 1)
+                RETURNING id
+                """,
+                bank_id,
+                "İstanbul",
+                event_date,
+            )
+
+            resolved_ids = await resolver.resolve_entities_batch(
+                bank_id=bank_id,
+                entities_data=[
+                    {
+                        "text": "istanbul",
+                        "nearby_entities": [],
+                        "event_date": event_date,
+                    }
+                ],
+                context="unicode case mismatch",
+                unit_event_date=event_date,
+                conn=conn,
+            )
+
+            entity_rows = await conn.fetch(
+                """
+                SELECT id, canonical_name
+                FROM entities
+                WHERE bank_id = $1
+                ORDER BY canonical_name
+                """,
+                bank_id,
+            )
+
+        assert resolved_ids == [existing_entity_id]
+        assert len(entity_rows) == 1
+        assert entity_rows[0]["id"] == existing_entity_id
+        assert entity_rows[0]["canonical_name"] == "İstanbul"
+    finally:
+        await pool.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
+        await pool.close()