Discord leases: free releasing tokens on gone nodes; stamp before building desired list
- reconcile_once only walked desired=on streams, so a token left "releasing" by /leave while its node was offline (or a deleted node) never returned to the pool until that node came back (server-26#204 leak shape). Sweep leased tokens whose node is offline/deleted past the lease grace. - publish_desired stamped release_seq AFTER building the list, so a stop landing between the two got a seq whose list still held the stream; an ack for it could free a token the node was still told to use. Stamp first. Three regression tests; each fails on the previous commit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a31a8df17c
commit
03ba7eacd9
@@ -358,6 +358,60 @@ async def test_persistent_stream_moves_after_grace_adhoc_stops(fs, monkeypatch):
|
||||
assert all((_lease(fs, t) or {}).get("node_id") != "n1" for t in ("t1", "t2"))
|
||||
|
||||
|
||||
async def test_leave_while_node_offline_frees_token_after_grace(fs, monkeypatch):
|
||||
"""A stream stopped while its node was down leaves no desired=on stream
|
||||
behind; the token must still return to the pool once the grace passes."""
|
||||
monkeypatch.setattr(streams.settings, "discord_lease_offline_release_seconds", 600)
|
||||
_system(fs); _node(fs, "n1"); _token(fs, "t1")
|
||||
await _start()
|
||||
_node(fs, "n1", status="offline", age=120)
|
||||
await streams.stop_stream("g1:c1")
|
||||
await streams.reconcile_once()
|
||||
assert _lease(fs, "t1")["releasing"] is True # inside the grace: held
|
||||
_node(fs, "n1", status="offline", age=900)
|
||||
await streams.reconcile_once()
|
||||
assert _lease(fs, "t1") is None
|
||||
|
||||
|
||||
async def test_deleted_node_lease_freed_after_grace_from_last_report(fs, monkeypatch):
|
||||
monkeypatch.setattr(streams.settings, "discord_lease_offline_release_seconds", 600)
|
||||
_system(fs); _node(fs, "n1"); _token(fs, "t1")
|
||||
await _start()
|
||||
await streams.stop_stream("g1:c1")
|
||||
fs.put("discord_nodes", "n1", {**(fs.get("discord_nodes", "n1") or {}),
|
||||
"actual_at": datetime.now(timezone.utc).isoformat()})
|
||||
await fs.doc_delete("nodes", "n1")
|
||||
await streams.reconcile_once()
|
||||
assert _lease(fs, "t1") is not None # heard from recently: held
|
||||
old = (datetime.now(timezone.utc) - timedelta(seconds=900)).isoformat()
|
||||
fs.put("discord_nodes", "n1", {**fs.get("discord_nodes", "n1"), "actual_at": old})
|
||||
await streams.reconcile_once()
|
||||
assert _lease(fs, "t1") is None
|
||||
|
||||
|
||||
async def test_release_seq_never_names_a_list_that_still_holds_the_stream(fs, monkeypatch):
|
||||
"""A stop landing between building the desired list and stamping must not
|
||||
be stamped with that list's seq (the list still contains the stream)."""
|
||||
_system(fs); _node(fs, "n1"); _token(fs, "t1")
|
||||
await _start()
|
||||
real = streams.desired_for_node
|
||||
fired = []
|
||||
|
||||
async def _racy(node_id):
|
||||
out = await real(node_id)
|
||||
if not fired:
|
||||
fired.append(1)
|
||||
await leases.detach_stream("g1:c1", {"desired": "off"})
|
||||
return out
|
||||
monkeypatch.setattr(streams, "desired_for_node", _racy)
|
||||
await streams.publish_desired("n1")
|
||||
racy_seq = fs.published[-1][1]["seq"]
|
||||
assert any(d["stream_id"] == "g1:c1" for d in fs.published[-1][1]["streams"])
|
||||
rseq = _lease(fs, "t1")["release_seq"]
|
||||
assert rseq is None or rseq > racy_seq
|
||||
assert await leases.settle_releases("n1", set(), racy_seq) == []
|
||||
|
||||
|
||||
async def test_retuned_node_gives_up_a_persistent_stream(fs):
|
||||
_system(fs); _system(fs, "sys-2"); _node(fs, "n1"); _token(fs, "t1")
|
||||
await _start(mode="persistent")
|
||||
|
||||
Reference in New Issue
Block a user