correlator: delete stale scenes on re-extraction instead of leaving them to rot (#96/#114)

Review of #132 found a blocker: PATCH /calls/{id}/transcript wipes tags/severity/location/units/embedding before re-extraction, but not the new scenes map, and doc_set(merge=True) can only add/overwrite nested map keys, never remove one. A call corrected from 3 scenes to 1 kept scenes.1/scenes.2 with pre-correction transcripts and incident_ids forever -- corrupting the exact per-scene tally #96 exists to make trustworthy, and able to re-feed stale text into #114's summarizer fix if a stale scene's incident_id still names a real incident.

Fix: fstore.doc_update(...,{"scenes": fstore.DELETE_FIELD}) -- a real delete, not a merge over an empty map. Added fstore.DELETE_FIELD (re-exports the real firebase_admin sentinel) and stubbed it in the sandboxed test conftest, which didn't have it. Also softened an overclaiming docstring: the Firestore nested-merge behavior is verified against the doc_set wrapper's pass-through, not against live Firestore.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Claude-Session: https://claude.ai/code/session_01Tbknwttzou4s46PAykmtix

drb-c2-core/tests/conftest.py

@@ -34,6 +34,10 @@ except ModuleNotFoundError:
     # into dicts that tests compare against, and a MagicMock compares unequal
     # to itself across attribute accesses.
     _fs.SERVER_TIMESTAMP = "__SERVER_TIMESTAMP__"
+    # Same reasoning as SERVER_TIMESTAMP above: a distinct sentinel, not a
+    # MagicMock, so `fstore.DELETE_FIELD is fs.DELETE_FIELD` and dict/`is`
+    # comparisons against it in tests (server-26#96/#114, PR #132) behave.
+    _fs.DELETE_FIELD = "__DELETE_FIELD__"
 
     _auth = ModuleType("firebase_admin.auth")
     _auth.verify_id_token = MagicMock()

drb-c2-core/tests/test_reprocess_clears_stale_scenes.py

@@ -0,0 +1,89 @@
+"""
+server-26#96/#114 review (PR #132): `PATCH /calls/{id}/transcript` clears
+stale intelligence fields before re-extraction runs, but `doc_set(...,
+merge=True)` can only add/overwrite keys in a nested map, never remove one.
+A call corrected from 3 scenes down to 1 would keep `scenes.1`/`scenes.2`
+with pre-correction transcripts and incident_ids forever -- corrupting the
+per-scene tally #96 exists to make trustworthy, and re-feeding stale text
+into #114's summarizer fix if a stale scene's incident_id still names a real
+incident. The fix deletes the field with `fstore.DELETE_FIELD` instead of
+merging over it with an empty map (which is a no-op).
+"""
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from fastapi import BackgroundTasks
+
+from app.internal import firestore as fstore
+from app.routers.calls import TranscriptUpdate, patch_transcript
+
+
+@pytest.mark.asyncio
+async def test_transcript_correction_deletes_the_scenes_field_not_merges_over_it():
+    call = {
+        "call_id": "call-1",
+        "system_id": "sys-1",
+        "node_id": "node-1",
+        "transcript": "old raw text",
+        # Simulates a prior 3-scene call, per #96's schema.
+        "scenes": {
+            "0": {"transcript": "scene zero", "incident_id": "inc-a", "corr_debug": {}},
+            "1": {"transcript": "scene one", "incident_id": "inc-b", "corr_debug": {}},
+        },
+    }
+
+    doc_set_calls: list[tuple] = []
+    doc_update_calls: list[tuple] = []
+
+    async def fake_doc_get(collection, doc_id):
+        if collection == "calls" and doc_id == "call-1":
+            return call
+        return None
+
+    async def fake_doc_set(collection, doc_id, data, merge=True):
+        doc_set_calls.append((collection, doc_id, data))
+
+    async def fake_doc_update(collection, doc_id, data):
+        doc_update_calls.append((collection, doc_id, data))
+
+    fake_flags = (None, lambda name: name == "correlation_enabled")
+
+    with patch("app.routers.calls.fstore.doc_get", new=fake_doc_get), \
+         patch("app.routers.calls.fstore.doc_set", new=fake_doc_set), \
+         patch("app.routers.calls.fstore.doc_update", new=fake_doc_update), \
+         patch("app.internal.feature_flags.resolve_flags", new=AsyncMock(return_value=fake_flags)):
+        result = await patch_transcript(
+            call_id="call-1",
+            body=TranscriptUpdate(transcript="corrected text"),
+            background_tasks=BackgroundTasks(),
+            _={},
+        )
+
+    assert result == {"ok": True, "call_id": "call-1"}
+
+    # The stale scenes map must be DELETED, not merged over with {} (a no-op
+    # under Firestore's set(merge=True) semantics) and not left untouched by
+    # a doc_set call that never mentions it.
+    scenes_deletions = [
+        (coll, doc_id, data) for (coll, doc_id, data) in doc_update_calls
+        if coll == "calls" and doc_id == "call-1" and "scenes" in data
+    ]
+    assert len(scenes_deletions) == 1, (
+        f"expected exactly one doc_update clearing 'scenes', got {doc_update_calls}"
+    )
+    assert scenes_deletions[0][2]["scenes"] is fstore.DELETE_FIELD
+
+    # And no doc_set call should paper over the same field with an empty map
+    # instead -- that would silently do nothing and leave stale scenes intact.
+    for (coll, doc_id, data) in doc_set_calls:
+        if coll == "calls" and doc_id == "call-1":
+            assert "scenes" not in data, (
+                "a doc_set (merge=True) write must never carry 'scenes' -- "
+                "merging {} over an existing map is a no-op, not a delete"
+            )
+
+
+def test_delete_field_is_the_real_firestore_sentinel():
+    """Catches an import-path typo turning this into a silent no-op sentinel."""
+    from firebase_admin import firestore as fs
+    assert fstore.DELETE_FIELD is fs.DELETE_FIELD