diff --git a/hindsight-api-slim/hindsight_api/engine/entity_resolver.py b/hindsight-api-slim/hindsight_api/engine/entity_resolver.py index 5bb1421f..6fc8f290 100644 --- a/hindsight-api-slim/hindsight_api/engine/entity_resolver.py +++ b/hindsight-api-slim/hindsight_api/engine/entity_resolver.py @@ -477,19 +477,42 @@ class EntityResolver: id_by_name: dict[str, str] = {row["name_lower"]: row["id"] for row in inserted_rows} # Fallback SELECT for names that conflicted (another worker won the race). - missing = [n for n, _ in sorted_groups if n not in id_by_name] - if missing: + # + # IMPORTANT: we must let PostgreSQL do the lowercasing on BOTH sides of the + # comparison. Python's str.lower() and PostgreSQL's LOWER() differ for some + # Unicode characters — most notably Turkish İ (U+0130): + # Python: 'İstanbul'.lower() == 'i\u0307stanbul' (i + combining dot, 2 chars) + # PostgreSQL: LOWER('İstanbul') == 'istanbul' (plain i, 1 char) + # Passing a Python-lowercased name to "LOWER(canonical_name) = ANY($2::text[])" + # would fail to match the stored entity, leaving entity_id as None and causing + # a NOT NULL constraint violation on unit_entities.entity_id. + # + # Fix: pass the original (mixed-case) input names and use + # "LOWER(canonical_name) = ANY(SELECT LOWER(n) FROM unnest($2) AS n)" so + # PostgreSQL lowercases both sides identically. The query also returns the + # original input_name so we can index id_by_name by Python's lower() of that + # name, which is what the assignment loop below uses as its lookup key. + missing_original = [g.name for name_lower, g in sorted_groups if name_lower not in id_by_name] + if missing_original: existing_rows = await conn.fetch( f""" - SELECT id, LOWER(canonical_name) AS name_lower - FROM {fq_table("entities")} - WHERE bank_id = $1 AND LOWER(canonical_name) = ANY($2::text[]) + SELECT e.id, LOWER(e.canonical_name) AS name_lower, inputs.input_name + FROM {fq_table("entities")} e + JOIN ( + SELECT LOWER(n) AS input_name_lower, n AS input_name + FROM unnest($2::text[]) AS n + ) AS inputs ON LOWER(e.canonical_name) = inputs.input_name_lower + WHERE e.bank_id = $1 """, bank_id, - missing, + missing_original, ) for row in existing_rows: id_by_name[row["name_lower"]] = row["id"] + # Also index by Python's lower() of the original input name so the + # assignment loop (which uses Python-lowercased keys) finds it even + # when Python and PostgreSQL produce different lowercase strings. + id_by_name[row["input_name"].lower()] = row["id"] # Assign entity IDs back and queue one stat per original mention so that # flush_pending_stats() increments mention_count by the true mention count, diff --git a/hindsight-api-slim/tests/test_entity_resolver.py b/hindsight-api-slim/tests/test_entity_resolver.py new file mode 100644 index 00000000..988abaa5 --- /dev/null +++ b/hindsight-api-slim/tests/test_entity_resolver.py @@ -0,0 +1,70 @@ +""" +Tests for EntityResolver edge cases. +""" + +import uuid +from datetime import datetime, timezone + +import asyncpg +import pytest + +from hindsight_api.engine.entity_resolver import EntityResolver +from hindsight_api.pg0 import resolve_database_url + + +@pytest.mark.asyncio +async def test_resolve_entities_batch_handles_unicode_lower_conflicts(pg0_db_url): + """ + Existing entities with PostgreSQL/Python lowercase mismatches should resolve + to the conflicted row instead of leaving a missing entity_id. + """ + resolved_url = await resolve_database_url(pg0_db_url) + pool = await asyncpg.create_pool(resolved_url, min_size=1, max_size=2, command_timeout=30) + bank_id = f"test-entity-resolver-{uuid.uuid4().hex[:8]}" + event_date = datetime(2024, 1, 15, tzinfo=timezone.utc) + resolver = EntityResolver(pool=pool, entity_lookup="full") + + try: + async with pool.acquire() as conn: + existing_entity_id = await conn.fetchval( + """ + INSERT INTO entities (bank_id, canonical_name, first_seen, last_seen, mention_count) + VALUES ($1, $2, $3, $3, 1) + RETURNING id + """, + bank_id, + "İstanbul", + event_date, + ) + + resolved_ids = await resolver.resolve_entities_batch( + bank_id=bank_id, + entities_data=[ + { + "text": "istanbul", + "nearby_entities": [], + "event_date": event_date, + } + ], + context="unicode case mismatch", + unit_event_date=event_date, + conn=conn, + ) + + entity_rows = await conn.fetch( + """ + SELECT id, canonical_name + FROM entities + WHERE bank_id = $1 + ORDER BY canonical_name + """, + bank_id, + ) + + assert resolved_ids == [existing_entity_id] + assert len(entity_rows) == 1 + assert entity_rows[0]["id"] == existing_entity_id + assert entity_rows[0]["canonical_name"] == "İstanbul" + finally: + await pool.execute("DELETE FROM entities WHERE bank_id = $1", bank_id) + await pool.close()