Discord leases: free releasing tokens on gone nodes; stamp before building desired list

- reconcile_once only walked desired=on streams, so a token left "releasing"
  by /leave while its node was offline (or a deleted node) never returned to
  the pool until that node came back (server-26#204 leak shape). Sweep leased
  tokens whose node is offline/deleted past the lease grace.
- publish_desired stamped release_seq AFTER building the list, so a stop
  landing between the two got a seq whose list still held the stream; an ack
  for it could free a token the node was still told to use. Stamp first.

Three regression tests; each fails on the previous commit.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Logan Cusano
2026-10-04 18:16:31 -04:00
co-authored by Claude Opus 5.5
parent a31a8df17c
commit 03ba7eacd9
2 changed files with 82 additions and 1 deletions
+54
View File
@@ -358,6 +358,60 @@ async def test_persistent_stream_moves_after_grace_adhoc_stops(fs, monkeypatch):
assert all((_lease(fs, t) or {}).get("node_id") != "n1" for t in ("t1", "t2"))
async def test_leave_while_node_offline_frees_token_after_grace(fs, monkeypatch):
"""A stream stopped while its node was down leaves no desired=on stream
behind; the token must still return to the pool once the grace passes."""
monkeypatch.setattr(streams.settings, "discord_lease_offline_release_seconds", 600)
_system(fs); _node(fs, "n1"); _token(fs, "t1")
await _start()
_node(fs, "n1", status="offline", age=120)
await streams.stop_stream("g1:c1")
await streams.reconcile_once()
assert _lease(fs, "t1")["releasing"] is True # inside the grace: held
_node(fs, "n1", status="offline", age=900)
await streams.reconcile_once()
assert _lease(fs, "t1") is None
async def test_deleted_node_lease_freed_after_grace_from_last_report(fs, monkeypatch):
monkeypatch.setattr(streams.settings, "discord_lease_offline_release_seconds", 600)
_system(fs); _node(fs, "n1"); _token(fs, "t1")
await _start()
await streams.stop_stream("g1:c1")
fs.put("discord_nodes", "n1", {**(fs.get("discord_nodes", "n1") or {}),
"actual_at": datetime.now(timezone.utc).isoformat()})
await fs.doc_delete("nodes", "n1")
await streams.reconcile_once()
assert _lease(fs, "t1") is not None # heard from recently: held
old = (datetime.now(timezone.utc) - timedelta(seconds=900)).isoformat()
fs.put("discord_nodes", "n1", {**fs.get("discord_nodes", "n1"), "actual_at": old})
await streams.reconcile_once()
assert _lease(fs, "t1") is None
async def test_release_seq_never_names_a_list_that_still_holds_the_stream(fs, monkeypatch):
"""A stop landing between building the desired list and stamping must not
be stamped with that list's seq (the list still contains the stream)."""
_system(fs); _node(fs, "n1"); _token(fs, "t1")
await _start()
real = streams.desired_for_node
fired = []
async def _racy(node_id):
out = await real(node_id)
if not fired:
fired.append(1)
await leases.detach_stream("g1:c1", {"desired": "off"})
return out
monkeypatch.setattr(streams, "desired_for_node", _racy)
await streams.publish_desired("n1")
racy_seq = fs.published[-1][1]["seq"]
assert any(d["stream_id"] == "g1:c1" for d in fs.published[-1][1]["streams"])
rseq = _lease(fs, "t1")["release_seq"]
assert rseq is None or rseq > racy_seq
assert await leases.settle_releases("n1", set(), racy_seq) == []
async def test_retuned_node_gives_up_a_persistent_stream(fs):
_system(fs); _system(fs, "sys-2"); _node(fs, "n1"); _token(fs, "t1")
await _start(mode="persistent")