438ce98b40
* Fix entity_id null constraint for non-ASCII entity names (Turkish İ etc.)
Python's str.lower() and PostgreSQL's LOWER() produce different results for
some Unicode characters. The most common case is Turkish İ (U+0130):
Python: 'İstanbul'.lower() == 'i\u0307stanbul' (i + combining dot, 2 chars)
PostgreSQL: LOWER('İstanbul') == 'istanbul' (plain i, 1 char)
In _resolve_from_candidates, the fallback SELECT for conflicted entity names
passed Python-lowercased strings to LOWER(canonical_name) = ANY($names), so
PostgreSQL couldn't match them. entity_ids[idx] stayed None, which then
caused a NOT NULL violation on unit_entities.entity_id, failing the entire
retain.
Fix: pass original mixed-case names to the fallback SELECT and use
LOWER(canonical_name) = ANY(SELECT LOWER(n) FROM unnest($2) AS n) so
PostgreSQL lowercases both sides identically. The query also returns the
original input_name so we can add a Python-lowercased key to id_by_name
for the assignment loop that uses Python-lowercased keys.
* Add regression test for Unicode entity conflict
70 lines
2.3 KiB
Python
70 lines
2.3 KiB
Python
"""
|
|
Tests for EntityResolver edge cases.
|
|
"""
|
|
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
import asyncpg
|
|
import pytest
|
|
|
|
from hindsight_api.engine.entity_resolver import EntityResolver
|
|
from hindsight_api.pg0 import resolve_database_url
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_entities_batch_handles_unicode_lower_conflicts(pg0_db_url):
|
|
"""
|
|
Existing entities with PostgreSQL/Python lowercase mismatches should resolve
|
|
to the conflicted row instead of leaving a missing entity_id.
|
|
"""
|
|
resolved_url = await resolve_database_url(pg0_db_url)
|
|
pool = await asyncpg.create_pool(resolved_url, min_size=1, max_size=2, command_timeout=30)
|
|
bank_id = f"test-entity-resolver-{uuid.uuid4().hex[:8]}"
|
|
event_date = datetime(2024, 1, 15, tzinfo=timezone.utc)
|
|
resolver = EntityResolver(pool=pool, entity_lookup="full")
|
|
|
|
try:
|
|
async with pool.acquire() as conn:
|
|
existing_entity_id = await conn.fetchval(
|
|
"""
|
|
INSERT INTO entities (bank_id, canonical_name, first_seen, last_seen, mention_count)
|
|
VALUES ($1, $2, $3, $3, 1)
|
|
RETURNING id
|
|
""",
|
|
bank_id,
|
|
"İstanbul",
|
|
event_date,
|
|
)
|
|
|
|
resolved_ids = await resolver.resolve_entities_batch(
|
|
bank_id=bank_id,
|
|
entities_data=[
|
|
{
|
|
"text": "istanbul",
|
|
"nearby_entities": [],
|
|
"event_date": event_date,
|
|
}
|
|
],
|
|
context="unicode case mismatch",
|
|
unit_event_date=event_date,
|
|
conn=conn,
|
|
)
|
|
|
|
entity_rows = await conn.fetch(
|
|
"""
|
|
SELECT id, canonical_name
|
|
FROM entities
|
|
WHERE bank_id = $1
|
|
ORDER BY canonical_name
|
|
""",
|
|
bank_id,
|
|
)
|
|
|
|
assert resolved_ids == [existing_entity_id]
|
|
assert len(entity_rows) == 1
|
|
assert entity_rows[0]["id"] == existing_entity_id
|
|
assert entity_rows[0]["canonical_name"] == "İstanbul"
|
|
finally:
|
|
await pool.execute("DELETE FROM entities WHERE bank_id = $1", bank_id)
|
|
await pool.close()
|