Discord radio channels: wire leases + desired-state streams (server-26#203, #204)
c2-core
- discord_streams/discord_leases wired in: POST /discord/streams,
POST /discord/guilds/{id}/stop, GET /discord/guilds/{id}/overview,
GET /discord/streams/{id}, PUT /discord/guilds/{id}/link (admin),
GET /discord/node/{node}/tokens/{id} (node key, lease holder only).
- Retained nodes/{id}/discord/desired (token ids, never token strings),
nodes/+/discord/actual subscribed and handled; per-node publish lock keeps
the retained list the newest; failed publishes retried by the reconciler.
- Tokens freed only after the node acks logout (or offline past
discord_lease_offline_release_seconds) - sweeper and C2 startup no longer
release leases.
- /nodes/{id}/command discord_join/leave map onto streams; CommandPayload
keeps preferred_token_id. /tokens: admin-only list, validate on add against
Discord, refresh, flush via transactional force release.
- Guild -> org via discord_guilds, unlinked guilds fall back to
DISCORD_UNLINKED_GUILD_ORG_ID (default "founding").
- tests/fake_firestore.py: optimistic-transaction fake; 27 lease/race tests.
discord bot
- /join, /leave, /status, autocomplete scoped to the calling guild; /leave
never touches another server; Manage Channels default permission;
/join edits its reply with the node's real result; /help copy fixed.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
0a724a064f
commit
a31a8df17c
@@ -210,6 +210,24 @@ class Settings(BaseSettings):
|
||||
# deployment will set this up, and skipping it must be silent.
|
||||
ai_alert_webhook_url: str = ""
|
||||
|
||||
# --- Discord radio channels (server-26#203/#204, node-26#25) -----------
|
||||
# Voice connections one node may carry at once (≈64 kbps upstream each —
|
||||
# this is what protects an LTE node).
|
||||
discord_node_max_streams: int = 4
|
||||
# Radio channels one Discord server may have on at once.
|
||||
discord_guild_max_streams: int = 4
|
||||
# How often the reconciler re-places persistent streams / retries publishes.
|
||||
discord_reconcile_interval_seconds: int = 30
|
||||
# A node offline this long has its radio-bot tokens returned to the pool.
|
||||
# MUST stay above the edge's DISCORD_MQTT_LOSS_STOP_SECONDS (default 300):
|
||||
# by then the edge has logged every radio bot out by itself, so the same
|
||||
# token can never end up logged in on two nodes.
|
||||
discord_lease_offline_release_seconds: int = 600
|
||||
# Org a Discord server that was never linked to an org (discord_guilds/{id})
|
||||
# acts for. Default keeps today's single-org deployment working unchanged;
|
||||
# set it to "" to require every server to be linked first.
|
||||
discord_unlinked_guild_org_id: str = "founding"
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
|
||||
|
||||
@@ -318,7 +318,11 @@ async def release_node_leases(node_id: str) -> list[str]:
|
||||
|
||||
|
||||
async def force_release_token(token_id: str) -> Optional[dict]:
|
||||
"""Admin override (/tokens/flush, Force release). Returns the old lease."""
|
||||
"""Admin override (/tokens/flush, delete). Frees the token and detaches
|
||||
every stream that named it, in one transaction, so no stream doc is left
|
||||
pointing at a token whose lease no longer lists it. Returns the old lease
|
||||
(``None`` if the token was not leased or does not exist); the caller
|
||||
republishes desired state for ``lease["node_id"]`` so the edge logs out."""
|
||||
db = fstore.db
|
||||
|
||||
def _txn(txn):
|
||||
@@ -327,7 +331,83 @@ async def force_release_token(token_id: str) -> Optional[dict]:
|
||||
if not snap.exists:
|
||||
return None
|
||||
lease = snap.to_dict().get("lease")
|
||||
stream_refs = []
|
||||
for sid in (lease or {}).get("streams") or []:
|
||||
sref = db.collection(STREAMS).document(sid)
|
||||
ssnap = sref.get(transaction=txn)
|
||||
if ssnap.exists and (ssnap.to_dict() or {}).get("token_id") == token_id:
|
||||
stream_refs.append(sref)
|
||||
txn.update(ref, _legacy_mirror(None))
|
||||
for sref in stream_refs:
|
||||
txn.set(sref, {"token_id": None, "lease_id": None}, merge=True)
|
||||
return lease
|
||||
|
||||
return await fstore.run_transaction(_txn)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Token validation / guild membership (server-26#204 item 5)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DISCORD_API = "https://discord.com/api/v10"
|
||||
|
||||
|
||||
class TokenRejected(Exception):
|
||||
"""Discord answered 401: the string is not a working bot token."""
|
||||
|
||||
|
||||
async def inspect_token(token: str) -> dict:
|
||||
"""Ask Discord who this token is and which servers it is in.
|
||||
|
||||
Returns ``{bot_user_id, bot_name, application_id, guild_ids, valid,
|
||||
validated_at, check_error}``. Raises TokenRejected on a 401. Any other
|
||||
failure (Discord down, rate limit) is reported in ``check_error`` with
|
||||
``valid``/``guild_ids`` = None — unknown, not bad: a token is never
|
||||
rejected because Discord had a bad minute.
|
||||
"""
|
||||
import httpx
|
||||
headers = {"Authorization": f"Bot {token}"}
|
||||
out: dict = {"valid": None, "guild_ids": None, "check_error": None,
|
||||
"validated_at": _now().isoformat()}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10, base_url=DISCORD_API, headers=headers) as client:
|
||||
me = await client.get("/users/@me")
|
||||
if me.status_code == 401:
|
||||
raise TokenRejected("Discord rejected this bot token (401).")
|
||||
me.raise_for_status()
|
||||
user = me.json()
|
||||
out["bot_user_id"] = str(user.get("id"))
|
||||
out["bot_name"] = user.get("username")
|
||||
app = await client.get("/oauth2/applications/@me")
|
||||
if app.status_code == 200:
|
||||
out["application_id"] = str(app.json().get("id"))
|
||||
guilds = await client.get("/users/@me/guilds", params={"limit": 200})
|
||||
guilds.raise_for_status()
|
||||
out["guild_ids"] = [str(g["id"]) for g in guilds.json()]
|
||||
out["valid"] = True
|
||||
except TokenRejected:
|
||||
raise
|
||||
except Exception as e: # network / 5xx / 429 — unknown, not invalid
|
||||
out["check_error"] = str(e)[:200]
|
||||
return out
|
||||
|
||||
|
||||
async def refresh_membership(token_ids: list[str]) -> int:
|
||||
"""Re-read guild membership for these tokens. Returns how many changed."""
|
||||
changed = 0
|
||||
for tok_id in token_ids:
|
||||
tok = await fstore.doc_get(TOKENS, tok_id)
|
||||
if not tok or not tok.get("token"):
|
||||
continue
|
||||
try:
|
||||
info = await inspect_token(tok["token"])
|
||||
except TokenRejected as e:
|
||||
await fstore.doc_update(TOKENS, tok_id, {"valid": False, "check_error": str(e),
|
||||
"validated_at": _now().isoformat()})
|
||||
changed += 1
|
||||
continue
|
||||
updates = {k: v for k, v in info.items() if v is not None or k == "check_error"}
|
||||
if updates.get("guild_ids") != tok.get("guild_ids") or updates.get("valid") != tok.get("valid"):
|
||||
changed += 1
|
||||
await fstore.doc_update(TOKENS, tok_id, updates)
|
||||
return changed
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
Discord radio channels as desired state (server-26#203 MVP).
|
||||
Discord radio channels as desired state (server-26#203, #204; node-26#25).
|
||||
|
||||
One ``discord_streams/{guild_id}:{channel_id}`` doc per Discord voice channel
|
||||
that should carry radio. Fields:
|
||||
@@ -7,14 +7,15 @@ that should carry radio. Fields:
|
||||
org_id, guild_id, channel_id, channel_name, guild_name,
|
||||
system_id, system_name,
|
||||
mode: "persistent" | "adhoc"
|
||||
persistent — stays connected and reconnects by itself (node reboot,
|
||||
Discord voice drop, C2 restart) until someone stops it.
|
||||
adhoc — the Discord bot sets desired=off after ``idle_minutes``
|
||||
with no humans in the voice channel.
|
||||
persistent — stays connected and is re-placed by the reconciler (node
|
||||
reboot, node lost, C2 restart) until someone stops it.
|
||||
adhoc — stopped instead of re-placed when its node goes away.
|
||||
(Idle auto-dismiss is not built yet — see the PR.)
|
||||
desired: "on" | "off"
|
||||
node_id, token_id, lease_id — placement (written with the lease, atomically)
|
||||
actual: {state, error, since, at} — reported by the node, never guessed here
|
||||
state: pending | connecting | connected | error | stopped | node_offline
|
||||
state: pending | connecting | connected | error | stopping |
|
||||
node_offline | paused_locally
|
||||
tg_filter: None — phase-2 seam (per-channel talkgroup gate); always None now
|
||||
now_playing_message_id, created_by_discord_id, created_at, updated_at
|
||||
|
||||
@@ -23,12 +24,13 @@ retains ``nodes/{node}/discord/desired`` (full list for that node, with a
|
||||
monotonically increasing ``seq``) -> the edge reconciles and publishes
|
||||
``nodes/{node}/discord/actual`` -> ``handle_actual`` updates ``actual`` and
|
||||
settles token releases. ``reconcile_loop`` re-places persistent streams whose
|
||||
node went away.
|
||||
node went away and retries desired-state publishes that never reached the
|
||||
broker.
|
||||
|
||||
The desired payload carries token **ids**, never token strings: the edge
|
||||
fetches the string over HTTPS (GET /discord/node/tokens/{id}) with its own
|
||||
node API key, and C2 hands it out only to the node holding the lease. A
|
||||
retained MQTT message therefore never stores a bot credential on the broker.
|
||||
fetches the string over HTTPS (GET /discord/node/{node_id}/tokens/{id}) with
|
||||
its own node API key, and C2 hands it out only to the node holding the lease.
|
||||
A retained MQTT message therefore never stores a bot credential on the broker.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -42,7 +44,8 @@ from app.internal import discord_leases as leases
|
||||
from app.internal.logger import logger
|
||||
|
||||
STREAMS = leases.STREAMS
|
||||
NODE_STATE = "discord_nodes" # discord_nodes/{node_id}: {seq, applied_seq, actual_at}
|
||||
GUILDS = "discord_guilds" # discord_guilds/{guild_id}: {org_id, linked_by, linked_at}
|
||||
NODE_STATE = "discord_nodes" # discord_nodes/{node_id}: {seq, published, applied_seq, actual_at}
|
||||
ONLINE = ("online", "recording")
|
||||
|
||||
|
||||
@@ -56,33 +59,56 @@ def _effective_system(node: dict) -> Optional[str]:
|
||||
return node.get("assigned_system_id")
|
||||
|
||||
|
||||
async def _node_stream_counts(org_id: Optional[str]) -> dict[str, int]:
|
||||
def _parse_ts(value) -> Optional[datetime]:
|
||||
try:
|
||||
ts = datetime.fromisoformat(value) if isinstance(value, str) else value
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if isinstance(ts, datetime) and ts.tzinfo is None:
|
||||
ts = ts.replace(tzinfo=timezone.utc)
|
||||
return ts if isinstance(ts, datetime) else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Guild -> org
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def resolve_guild_org(guild_id: str) -> Optional[str]:
|
||||
"""The org a Discord server acts for: its ``discord_guilds`` link, else
|
||||
``settings.discord_unlinked_guild_org_id`` (None when that is empty)."""
|
||||
link = await fstore.doc_get(GUILDS, str(guild_id))
|
||||
if link and link.get("org_id"):
|
||||
return link["org_id"]
|
||||
return settings.discord_unlinked_guild_org_id or None
|
||||
|
||||
|
||||
async def _node_stream_counts(exclude_stream_id: Optional[str] = None) -> dict[str, int]:
|
||||
streams = await fstore.collection_list(STREAMS, desired="on")
|
||||
counts: dict[str, int] = {}
|
||||
for s in streams:
|
||||
if s.get("node_id"):
|
||||
if s.get("node_id") and s.get("stream_id") != exclude_stream_id:
|
||||
counts[s["node_id"]] = counts.get(s["node_id"], 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
async def candidate_nodes(org_id: str, system_id: str, exclude: Optional[str] = None) -> list[dict]:
|
||||
def _node_usable(n: dict, org_id: str, system_id: str) -> bool:
|
||||
return (
|
||||
n.get("org_id") == org_id
|
||||
and n.get("status") in ONLINE
|
||||
and n.get("approval_status") in (None, "approved")
|
||||
and _effective_system(n) == system_id
|
||||
)
|
||||
|
||||
|
||||
async def candidate_nodes(org_id: str, system_id: str, exclude_stream_id: Optional[str] = None) -> list[dict]:
|
||||
"""Online, approved nodes of this org currently listening to ``system_id``,
|
||||
best first: nodes already carrying streams (one gateway session can serve
|
||||
several guilds) but under the per-node cap, then the rest."""
|
||||
nodes = await fstore.collection_list("nodes", org_id=org_id)
|
||||
counts = await _node_stream_counts(org_id)
|
||||
counts = await _node_stream_counts(exclude_stream_id)
|
||||
cap = settings.discord_node_max_streams
|
||||
out = []
|
||||
for n in nodes:
|
||||
if n.get("node_id") == exclude:
|
||||
continue
|
||||
if n.get("status") not in ONLINE or n.get("approval_status") not in (None, "approved"):
|
||||
continue
|
||||
if _effective_system(n) != system_id:
|
||||
continue
|
||||
if counts.get(n["node_id"], 0) >= cap:
|
||||
continue
|
||||
out.append(n)
|
||||
out = [n for n in nodes
|
||||
if n.get("node_id") and _node_usable(n, org_id, system_id) and counts.get(n["node_id"], 0) < cap]
|
||||
out.sort(key=lambda n: (-counts.get(n["node_id"], 0), n.get("name") or n["node_id"]))
|
||||
return out
|
||||
|
||||
@@ -91,13 +117,27 @@ async def candidate_nodes(org_id: str, system_id: str, exclude: Optional[str] =
|
||||
# Desired state -> node
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# One c2-core process publishes desired state. Serialising bump->build->publish
|
||||
# per node is what keeps the RETAINED message the newest one: two unserialised
|
||||
# publishes could land seq 7 then seq 6, and a reconnecting edge would then be
|
||||
# handed the stale list.
|
||||
_publish_locks: dict[str, asyncio.Lock] = {}
|
||||
|
||||
|
||||
def _lock_for(node_id: str) -> asyncio.Lock:
|
||||
lock = _publish_locks.get(node_id)
|
||||
if lock is None:
|
||||
lock = _publish_locks[node_id] = asyncio.Lock()
|
||||
return lock
|
||||
|
||||
|
||||
async def _bump_seq(node_id: str) -> int:
|
||||
db = fstore.db
|
||||
|
||||
def _txn(txn):
|
||||
ref = db.collection(NODE_STATE).document(node_id)
|
||||
snap = ref.get(transaction=txn)
|
||||
seq = int((snap.to_dict() or {}).get("seq") or 0) + 1 if snap.exists else 1
|
||||
seq = int(((snap.to_dict() or {}) if snap.exists else {}).get("seq") or 0) + 1
|
||||
txn.set(ref, {"node_id": node_id, "seq": seq}, merge=True)
|
||||
return seq
|
||||
|
||||
@@ -125,14 +165,23 @@ async def desired_for_node(node_id: str) -> list[dict]:
|
||||
return out
|
||||
|
||||
|
||||
async def publish_desired(node_id: str) -> int:
|
||||
"""Retain the node's full stream list; stamp releasing tokens with the seq."""
|
||||
async def publish_desired(node_id: str) -> bool:
|
||||
"""Retain the node's full stream list under a fresh ``seq``.
|
||||
|
||||
Releasing tokens are stamped with that seq BEFORE the publish, so the
|
||||
earliest ack that can free them is one for a list that already lacks them.
|
||||
Returns whether the broker took the message; a miss is recorded and the
|
||||
reconciler retries it (nothing is lost — the streams stay desired)."""
|
||||
from app.internal.mqtt_handler import mqtt_handler
|
||||
seq = await _bump_seq(node_id)
|
||||
payload = {"seq": seq, "streams": await desired_for_node(node_id), "issued_at": _now_iso()}
|
||||
mqtt_handler.publish_discord_desired(node_id, payload)
|
||||
await leases.stamp_release_seq(node_id, seq)
|
||||
return seq
|
||||
async with _lock_for(node_id):
|
||||
seq = await _bump_seq(node_id)
|
||||
payload = {"seq": seq, "streams": await desired_for_node(node_id), "issued_at": _now_iso()}
|
||||
await leases.stamp_release_seq(node_id, seq)
|
||||
ok = mqtt_handler.publish_discord_desired(node_id, payload)
|
||||
await fstore.doc_set(NODE_STATE, node_id, {"published": bool(ok), "published_at": _now_iso()})
|
||||
if not ok:
|
||||
logger.warning(f"Discord desired state for {node_id} (seq {seq}) not delivered — reconciler will retry")
|
||||
return bool(ok)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -149,11 +198,12 @@ class StreamError(Exception):
|
||||
|
||||
|
||||
ERROR_TEXT = {
|
||||
"no_org": "This Discord server is not linked to a DRB organization yet.",
|
||||
"no_node": "No online node in your organization is listening to that system right now.",
|
||||
"pool_empty": "No radio bots have been added to DRB yet. A platform admin adds them under Radio bots.",
|
||||
"no_bot_in_guild": "No free radio bot is a member of this server. Invite one with the link below, then try again.",
|
||||
"pool_exhausted": "Every radio bot that can join this server is already busy. Stop another radio channel or add a bot.",
|
||||
"limit": "This server already has its maximum number of radio channels. Stop one with /radio stop first.",
|
||||
"limit": "This server already has its maximum number of radio channels. Stop one with /leave first.",
|
||||
}
|
||||
|
||||
|
||||
@@ -175,6 +225,9 @@ async def start_stream(
|
||||
it for a channel that is already on reuses the node and the lease."""
|
||||
if mode not in ("persistent", "adhoc"):
|
||||
raise StreamError("bad_mode", "mode must be persistent or adhoc", status=400)
|
||||
if not org_id:
|
||||
raise StreamError("no_org", ERROR_TEXT["no_org"], status=403)
|
||||
guild_id, channel_id = str(guild_id), str(channel_id)
|
||||
stream_id = leases.stream_id_for(guild_id, channel_id)
|
||||
existing = await fstore.doc_get(STREAMS, stream_id) or {}
|
||||
|
||||
@@ -186,9 +239,14 @@ async def start_stream(
|
||||
system = await fstore.doc_get_cached("systems", system_id) or {}
|
||||
|
||||
if node_id:
|
||||
candidates = [await fstore.doc_get("nodes", node_id) or {"node_id": node_id}]
|
||||
node = await fstore.doc_get("nodes", node_id)
|
||||
# A pinned node still has to be this org's and approved; it may be
|
||||
# listening to anything (the web "Join Discord" button pins a node).
|
||||
if not node or node.get("org_id") != org_id or node.get("approval_status") not in (None, "approved"):
|
||||
raise StreamError("no_node", ERROR_TEXT["no_node"])
|
||||
candidates = [node]
|
||||
else:
|
||||
candidates = await candidate_nodes(org_id, system_id)
|
||||
candidates = await candidate_nodes(org_id, system_id, exclude_stream_id=stream_id)
|
||||
# Keep the current node when it is still a valid choice (reuse the lease).
|
||||
cur = existing.get("node_id")
|
||||
if cur and existing.get("system_id") == system_id:
|
||||
@@ -200,10 +258,12 @@ async def start_stream(
|
||||
fields = {
|
||||
"stream_id": stream_id, "org_id": org_id, "guild_id": guild_id, "channel_id": channel_id,
|
||||
"system_id": system_id, "system_name": system.get("name") or system_id,
|
||||
"mode": mode, "desired": "on", "tg_filter": None,
|
||||
"actual": {"state": "pending", "error": None, "since": now, "at": now},
|
||||
"mode": mode, "desired": "on", "tg_filter": None, "stop_reason": None,
|
||||
"updated_at": now,
|
||||
}
|
||||
prev_actual = existing.get("actual") or {}
|
||||
if existing.get("desired") != "on" or prev_actual.get("state") not in ("connected", "connecting"):
|
||||
fields["actual"] = {"state": "pending", "error": None, "since": now, "at": now}
|
||||
if channel_name:
|
||||
fields["channel_name"] = channel_name
|
||||
if guild_name:
|
||||
@@ -215,19 +275,31 @@ async def start_stream(
|
||||
|
||||
last: Optional[leases.LeaseResult] = None
|
||||
prev_node = existing.get("node_id")
|
||||
refreshed = False
|
||||
for node in candidates:
|
||||
res = await leases.lease_for_stream(stream_id, node_id=node["node_id"], org_id=org_id,
|
||||
preferred_token_id=preferred_token_id, stream_fields=fields)
|
||||
if not res.ok and res.error == "no_bot_in_guild" and not refreshed:
|
||||
# Membership is cached on the token doc; someone may have just
|
||||
# invited a bot. Re-read it from Discord once, then retry.
|
||||
refreshed = True
|
||||
tokens = await fstore.collection_list(leases.TOKENS)
|
||||
stale = [t["token_id"] for t in tokens
|
||||
if t.get("token_id") and t.get("guild_ids") is not None and guild_id not in t["guild_ids"]]
|
||||
if await leases.refresh_membership(stale):
|
||||
res = await leases.lease_for_stream(stream_id, node_id=node["node_id"], org_id=org_id,
|
||||
preferred_token_id=preferred_token_id, stream_fields=fields)
|
||||
if res.ok:
|
||||
await publish_desired(node["node_id"])
|
||||
delivered = await publish_desired(node["node_id"])
|
||||
if prev_node and prev_node != node["node_id"]:
|
||||
await publish_desired(prev_node)
|
||||
logger.info(f"Discord stream {stream_id} -> node {node['node_id']} token {res.token_id}"
|
||||
f" ({'reused' if res.reused else 'new'} lease)")
|
||||
return {**existing, **fields, "node_id": node["node_id"], "node_name": node.get("name"),
|
||||
"token_id": res.token_id, "lease_id": res.lease_id, "reused": res.reused}
|
||||
"token_id": res.token_id, "lease_id": res.lease_id, "reused": res.reused,
|
||||
"delivered": delivered}
|
||||
last = res
|
||||
code = last.error if last else "no_node"
|
||||
code = last.error if last and last.error else "no_node"
|
||||
raise StreamError(code, ERROR_TEXT.get(code, code), invite=last.invite if last else [])
|
||||
|
||||
|
||||
@@ -246,22 +318,27 @@ async def stop_stream(stream_id: str, reason: str = "stopped") -> bool:
|
||||
return True
|
||||
|
||||
|
||||
async def stop_guild(guild_id: str, reason: str) -> int:
|
||||
n = 0
|
||||
for s in await fstore.collection_list(STREAMS, guild_id=guild_id):
|
||||
if s.get("desired") == "on":
|
||||
await stop_stream(s["stream_id"], reason)
|
||||
n += 1
|
||||
return n
|
||||
async def stop_guild(guild_id: str, reason: str, channel_id: Optional[str] = None) -> list[str]:
|
||||
"""Stop every live stream in ONE Discord server (optionally one channel).
|
||||
The /leave fix for server-26#203: never touches another server's streams."""
|
||||
stopped = []
|
||||
for s in await fstore.collection_list(STREAMS, guild_id=str(guild_id)):
|
||||
if s.get("desired") != "on":
|
||||
continue
|
||||
if channel_id and s.get("channel_id") != str(channel_id):
|
||||
continue
|
||||
await stop_stream(s["stream_id"], reason)
|
||||
stopped.append(s["stream_id"])
|
||||
return stopped
|
||||
|
||||
|
||||
async def stop_node(node_id: str, reason: str) -> int:
|
||||
n = 0
|
||||
async def stop_node(node_id: str, reason: str) -> list[str]:
|
||||
stopped = []
|
||||
for s in await fstore.collection_list(STREAMS, node_id=node_id):
|
||||
if s.get("desired") == "on":
|
||||
await stop_stream(s["stream_id"], reason)
|
||||
n += 1
|
||||
return n
|
||||
stopped.append(s["stream_id"])
|
||||
return stopped
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -275,8 +352,11 @@ async def handle_actual(node_id: str, payload: dict) -> None:
|
||||
"""``nodes/{node_id}/discord/actual``. A node may only report on streams
|
||||
currently placed on it — anything else is ignored, so a compromised node
|
||||
cannot rewrite another node's (or org's) stream state."""
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
reported = payload.get("streams") or {}
|
||||
now = _now_iso()
|
||||
recheck: set = set()
|
||||
if isinstance(reported, dict):
|
||||
for stream_id, st in list(reported.items())[:50]:
|
||||
if not isinstance(st, dict):
|
||||
@@ -287,16 +367,23 @@ async def handle_actual(node_id: str, payload: dict) -> None:
|
||||
state = st.get("state") if st.get("state") in _VALID_STATES else "error"
|
||||
error = st.get("error")
|
||||
error = str(error)[:300] if error else None
|
||||
if st.get("code") == "not_in_guild" and doc.get("token_id"):
|
||||
recheck.add(doc["token_id"])
|
||||
prev = doc.get("actual") or {}
|
||||
since = prev.get("since") if prev.get("state") == state else now
|
||||
await fstore.doc_update(STREAMS, stream_id, {
|
||||
if prev.get("state") == state and prev.get("error") == error:
|
||||
since = prev.get("since") or now
|
||||
else:
|
||||
since = now
|
||||
await fstore.doc_update(STREAMS, str(stream_id), {
|
||||
"actual": {"state": state, "error": error, "since": since, "at": now},
|
||||
})
|
||||
if recheck:
|
||||
await leases.refresh_membership(sorted(recheck))
|
||||
|
||||
if "sessions" in payload:
|
||||
sessions = {str(t) for t in (payload.get("sessions") or [])}
|
||||
applied = payload.get("applied_seq")
|
||||
applied = int(applied) if isinstance(applied, (int, float)) else None
|
||||
applied = int(applied) if isinstance(applied, (int, float)) and not isinstance(applied, bool) else None
|
||||
freed = await leases.settle_releases(node_id, sessions, applied)
|
||||
for tok in freed:
|
||||
logger.info(f"Radio bot token {tok} released — node {node_id} confirmed it logged out")
|
||||
@@ -309,7 +396,7 @@ async def handle_actual(node_id: str, payload: dict) -> None:
|
||||
|
||||
async def release_offline_node(node_id: str) -> None:
|
||||
"""Node offline past the lease grace: free its tokens (the edge already
|
||||
logged them out — see release_node_leases) and re-place its streams."""
|
||||
logged them out — see release_node_leases) and unplace its streams."""
|
||||
orphaned = await leases.release_node_leases(node_id)
|
||||
if not orphaned:
|
||||
return
|
||||
@@ -324,38 +411,57 @@ async def release_offline_node(node_id: str) -> None:
|
||||
logger.warning(f"Node {node_id} offline past lease grace — released {len(orphaned)} radio stream(s)")
|
||||
|
||||
|
||||
async def _unplace(s: dict, why: str) -> None:
|
||||
"""Take a stream off its node (token detached, node republished) but leave
|
||||
it desired, so the reconciler can place it somewhere valid."""
|
||||
now = _now_iso()
|
||||
old = await leases.detach_stream(s["stream_id"], {
|
||||
"node_id": None,
|
||||
"actual": {"state": "error", "error": why, "since": now, "at": now},
|
||||
})
|
||||
if old:
|
||||
await publish_desired(old)
|
||||
|
||||
|
||||
async def reconcile_once() -> None:
|
||||
nodes = {n["node_id"]: n for n in await fstore.collection_list("nodes") if n.get("node_id")}
|
||||
grace = timedelta(seconds=settings.discord_lease_offline_release_seconds)
|
||||
now = datetime.now(timezone.utc)
|
||||
released: set = set()
|
||||
for s in await fstore.collection_list(STREAMS, desired="on"):
|
||||
node = nodes.get(s.get("node_id") or "")
|
||||
online = bool(node) and node.get("status") in ONLINE and _effective_system(node) == s.get("system_id")
|
||||
if online and s.get("token_id"):
|
||||
continue
|
||||
if node and s.get("token_id"):
|
||||
# Node offline (or retuned): hold the lease until the grace passes.
|
||||
last_seen = node.get("last_seen")
|
||||
try:
|
||||
ls = datetime.fromisoformat(last_seen) if isinstance(last_seen, str) else last_seen
|
||||
if ls and ls.tzinfo is None:
|
||||
ls = ls.replace(tzinfo=timezone.utc)
|
||||
except Exception:
|
||||
ls = None
|
||||
if node.get("status") in ONLINE or (ls and now - ls < grace):
|
||||
if node.get("status") not in ONLINE:
|
||||
if node.get("status") in ONLINE:
|
||||
if _effective_system(node) == s.get("system_id"):
|
||||
continue # healthy placement
|
||||
# Node retuned to another system under a live stream.
|
||||
if s.get("mode") != "persistent":
|
||||
await stop_stream(s["stream_id"], "node_retuned")
|
||||
continue
|
||||
await _unplace(s, "The node carrying this channel switched systems; finding another.")
|
||||
else:
|
||||
ls = _parse_ts(node.get("last_seen"))
|
||||
if ls and now - ls < grace:
|
||||
await _mark(s, "node_offline", "The node carrying this channel is offline; waiting for it.")
|
||||
continue
|
||||
await release_offline_node(node["node_id"])
|
||||
continue # hold the lease through the grace window
|
||||
if node["node_id"] not in released:
|
||||
await release_offline_node(node["node_id"])
|
||||
released.add(node["node_id"])
|
||||
if s.get("mode") != "persistent":
|
||||
await stop_stream(s["stream_id"], "node_lost")
|
||||
continue
|
||||
org_id = s.get("org_id")
|
||||
try:
|
||||
await start_stream(org_id=s["org_id"], guild_id=s["guild_id"], channel_id=s["channel_id"],
|
||||
await start_stream(org_id=org_id, guild_id=s["guild_id"], channel_id=s["channel_id"],
|
||||
system_id=s["system_id"], mode="persistent")
|
||||
except StreamError as e:
|
||||
await _mark(s, "error", e.message)
|
||||
|
||||
# Desired-state publishes the broker never took (MQTT was down).
|
||||
for st in await fstore.collection_list(NODE_STATE, published=False):
|
||||
if st.get("node_id"):
|
||||
await publish_desired(st["node_id"])
|
||||
|
||||
|
||||
async def _mark(s: dict, state: str, error: str) -> None:
|
||||
prev = s.get("actual") or {}
|
||||
|
||||
@@ -35,6 +35,8 @@ class MQTTHandler:
|
||||
client.subscribe("nodes/+/checkin", qos=1)
|
||||
client.subscribe("nodes/+/status", qos=1)
|
||||
client.subscribe("nodes/+/metadata", qos=1)
|
||||
# Edge -> C2 actual state of its Discord radio streams (node-26#25).
|
||||
client.subscribe("nodes/+/discord/actual", qos=1)
|
||||
# TODO(mqtt-cutover): drop this subscribe once the enrollment/HTTP
|
||||
# credentials flow (routers/enrollment.py) is stable in prod and
|
||||
# node-26 (the one live node) has been migrated. See
|
||||
@@ -61,6 +63,14 @@ class MQTTHandler:
|
||||
|
||||
async def _dispatch(self, topic: str, payload: dict):
|
||||
parts = topic.split("/")
|
||||
# nodes/{node_id}/discord/actual — the one four-level node topic.
|
||||
if len(parts) == 4 and parts[0] == "nodes" and parts[2:] == ["discord", "actual"]:
|
||||
from app.internal import discord_streams
|
||||
try:
|
||||
await discord_streams.handle_actual(parts[1], payload)
|
||||
except Exception as e:
|
||||
logger.error(f"MQTT dispatch error [discord/actual] from {parts[1]}: {e}")
|
||||
return
|
||||
# Expected: nodes/{node_id}/{type}
|
||||
if len(parts) != 3 or parts[0] != "nodes":
|
||||
return
|
||||
@@ -179,11 +189,11 @@ class MQTTHandler:
|
||||
|
||||
await fstore.doc_update("nodes", node_id, updates)
|
||||
|
||||
# NOTE: discord_connected in checkins is informational only — do NOT release the
|
||||
# token here. The bot watchdog reconnects on transient Discord drops, so a single
|
||||
# checkin with discord_connected=False during a brief reconnect window would
|
||||
# incorrectly free the token while the bot is still active. Token release is
|
||||
# handled exclusively by the discord_leave command and the node offline sweeper.
|
||||
# NOTE: discord_connected in checkins is informational only — never release a
|
||||
# token from it. Radio-bot tokens are freed only when the node acks (over
|
||||
# nodes/{id}/discord/actual) that it logged the bot out, or by the
|
||||
# reconciler once the node has been offline past the lease grace — see
|
||||
# app/internal/discord_leases.py.
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Status update
|
||||
@@ -335,6 +345,19 @@ class MQTTHandler:
|
||||
else:
|
||||
logger.warning(f"MQTT not connected — could not push config to {node_id}")
|
||||
|
||||
def publish_discord_desired(self, node_id: str, payload: dict) -> bool:
|
||||
"""Retained full list of the Discord streams this node should carry
|
||||
(app/internal/discord_streams.py). Carries token IDs, never token
|
||||
strings — the broker never stores a bot credential."""
|
||||
topic = f"nodes/{node_id}/discord/desired"
|
||||
if self._client and self._connected:
|
||||
info = self._client.publish(topic, json.dumps(payload), qos=1, retain=True)
|
||||
if getattr(info, "rc", 0) == mqtt.MQTT_ERR_SUCCESS:
|
||||
logger.info(f"Discord desired state seq {payload.get('seq')} -> {node_id}")
|
||||
return True
|
||||
logger.warning(f"MQTT not connected — Discord desired state for {node_id} not published")
|
||||
return False
|
||||
|
||||
def publish_node_key(self, node_id: str, api_key: str):
|
||||
"""Publish the provisioned API key to the node (retained so it survives reconnects).
|
||||
TODO(mqtt-cutover): dead once nodes.py's callers switch to the HTTP
|
||||
|
||||
@@ -53,8 +53,11 @@ async def _sweep():
|
||||
node_id = node.get("node_id")
|
||||
await fstore.doc_update("nodes", node_id, {"status": "offline"})
|
||||
logger.info(f"Node {node_id} marked offline (last seen: {last_seen.isoformat()})")
|
||||
from app.routers.tokens import release_token
|
||||
await release_token(node_id)
|
||||
# Radio-bot tokens are NOT freed here any more (node-26#25): the
|
||||
# node may still be logged in to Discord on them. The Discord
|
||||
# reconciler (internal/discord_streams.py) frees them once the node
|
||||
# has been offline past discord_lease_offline_release_seconds — by
|
||||
# which point the edge has logged its bots out by itself.
|
||||
continue
|
||||
|
||||
# Check for expired system overrides (only for fixed nodes with timeout enforced)
|
||||
|
||||
+25
-8
@@ -10,6 +10,7 @@ from app.internal.summarizer import summarizer_loop
|
||||
from app.internal.vocabulary_learner import vocabulary_induction_loop
|
||||
from app.internal.recorrelation_sweep import recorrelation_loop
|
||||
from app.internal import ai_health
|
||||
from app.internal import discord_streams
|
||||
from app.config import settings
|
||||
from app.internal.auth import (
|
||||
require_firebase_token,
|
||||
@@ -18,31 +19,42 @@ from app.internal.auth import (
|
||||
)
|
||||
from app.routers import nodes, systems, calls, upload, tokens, incidents, alerts, admin, trips, places, links, users
|
||||
from app.routers import enrollment, media, org, waitlist, telemetry, replay, traffic
|
||||
from app.routers import discord as discord_router
|
||||
from app.internal import dynsec
|
||||
from app.internal import firestore as fstore
|
||||
|
||||
|
||||
async def _release_orphaned_tokens():
|
||||
"""Release all in-use tokens on startup — voice connections don't survive server restarts."""
|
||||
async def _clear_legacy_token_flags():
|
||||
"""Tokens written by the pre-lease code carry ``in_use`` with no ``lease``.
|
||||
|
||||
Nothing reads ``in_use`` for decisions any more (discord_leases.py decides
|
||||
on ``lease``), but the /tokens page shows it, so a legacy "in use" with no
|
||||
lease behind it would be a permanent false label. A token that HAS a lease
|
||||
is left alone: voice connections now belong to the edge and survive a C2
|
||||
restart, so releasing leases at startup (what this used to do) would hand
|
||||
a still-logged-in token to a second node.
|
||||
"""
|
||||
def _find():
|
||||
from app.internal.firestore import db
|
||||
return [d for d in db.collection("bot_tokens").where("in_use", "==", True).stream()]
|
||||
|
||||
results = await asyncio.to_thread(_find)
|
||||
cleared = 0
|
||||
for doc in results:
|
||||
if (doc.to_dict() or {}).get("lease"):
|
||||
continue
|
||||
await fstore.doc_update("bot_tokens", doc.id, {
|
||||
"in_use": False,
|
||||
"assigned_node_id": None,
|
||||
"assigned_at": None,
|
||||
"lease": None, "in_use": False, "assigned_node_id": None, "assigned_at": None,
|
||||
})
|
||||
if results:
|
||||
logger.info(f"Released {len(results)} orphaned token(s) on startup.")
|
||||
cleared += 1
|
||||
if cleared:
|
||||
logger.info(f"Cleared legacy in_use flag on {cleared} token(s) with no lease.")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
logger.info("DRB C2 Core starting.")
|
||||
await _release_orphaned_tokens()
|
||||
await _clear_legacy_token_flags()
|
||||
|
||||
# dynsec bootstrap + reconcile — must happen before mqtt_handler.connect()
|
||||
# so that by the time the app is serving requests, c2-core's own dynsec
|
||||
@@ -65,6 +77,7 @@ async def lifespan(app: FastAPI):
|
||||
summarizer_task = asyncio.create_task(summarizer_loop())
|
||||
induction_task = asyncio.create_task(vocabulary_induction_loop())
|
||||
recorrelation_task = asyncio.create_task(recorrelation_loop())
|
||||
discord_task = asyncio.create_task(discord_streams.reconcile_loop())
|
||||
|
||||
yield # --- app running ---
|
||||
|
||||
@@ -73,6 +86,7 @@ async def lifespan(app: FastAPI):
|
||||
summarizer_task.cancel()
|
||||
induction_task.cancel()
|
||||
recorrelation_task.cancel()
|
||||
discord_task.cancel()
|
||||
await mqtt_handler.disconnect()
|
||||
|
||||
|
||||
@@ -139,6 +153,9 @@ app.include_router(waitlist.router) # public — no auth, source-IP rate limi
|
||||
# public by necessity — an <audio src> can't send a bearer token, so the
|
||||
# short-lived HMAC in the URL is the credential. Checked inline in media.py.
|
||||
app.include_router(media.router)
|
||||
# auth is per-endpoint: bot routes take the service key (or a platform admin);
|
||||
# the token-fetch route takes the calling node's own API key.
|
||||
app.include_router(discord_router.router)
|
||||
# NOTE: there used to be an app.routers.mqtt_auth router here (an HTTP
|
||||
# backend for the mosquitto-go-auth plugin). That plugin's upstream project
|
||||
# is archived (no CVE patches) and was rejected for an internet-facing
|
||||
|
||||
@@ -112,6 +112,9 @@ class CommandPayload(BaseModel):
|
||||
action: str # discord_join / discord_leave / op25_restart
|
||||
guild_id: Optional[str] = None
|
||||
channel_id: Optional[str] = None
|
||||
# discord_join only: try this pool bot first (server-26#204 item 1 — it
|
||||
# used to be dropped by pydantic before the router ever read it).
|
||||
preferred_token_id: Optional[str] = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
"""
|
||||
Discord radio channels — the HTTP face of internal/discord_streams.py.
|
||||
|
||||
Every bot-facing route is scoped to ONE Discord server (``guild_id`` in the
|
||||
path or body) and acts for the org that server is linked to
|
||||
(server-26#203): /leave in server A can no longer touch server B, and the
|
||||
system/node/bot lists a server sees are its own org's.
|
||||
|
||||
Auth:
|
||||
* bot routes — the Discord bot's service key, or a platform admin.
|
||||
* PUT /discord/guilds/{id}/link — platform admin only.
|
||||
* GET /discord/node/{node_id}/tokens/{token_id} — that node's own API key;
|
||||
the token string is handed only to the node currently holding its lease.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Security
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.config import settings
|
||||
from app.internal import discord_leases as leases
|
||||
from app.internal import discord_streams as streams
|
||||
from app.internal import firestore as fstore
|
||||
from app.internal.auth import require_admin_token, require_service_key_or_admin
|
||||
from app.internal.logger import logger
|
||||
|
||||
router = APIRouter(prefix="/discord", tags=["discord"])
|
||||
_bearer = HTTPBearer(auto_error=False)
|
||||
|
||||
|
||||
def _error(e: streams.StreamError) -> JSONResponse:
|
||||
return JSONResponse(status_code=e.status,
|
||||
content={"detail": e.message, "code": e.code, "invite": e.invite})
|
||||
|
||||
|
||||
async def _org_or_403(guild_id: str) -> str:
|
||||
org_id = await streams.resolve_guild_org(guild_id)
|
||||
if not org_id:
|
||||
raise HTTPException(403, streams.ERROR_TEXT["no_org"])
|
||||
return org_id
|
||||
|
||||
|
||||
def _public_stream(s: dict) -> dict:
|
||||
keep = ("stream_id", "guild_id", "channel_id", "channel_name", "system_id", "system_name", "mode",
|
||||
"desired", "node_id", "token_id", "actual", "stop_reason", "updated_at", "delivered", "reused")
|
||||
return {k: s.get(k) for k in keep if k in s}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bot routes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/guilds/{guild_id}/overview")
|
||||
async def guild_overview(guild_id: str, _: dict = Depends(require_service_key_or_admin)):
|
||||
"""Everything one Discord server may see: its org's systems and nodes, its
|
||||
own streams, and which radio bots could serve it. No token strings."""
|
||||
org_id = await streams.resolve_guild_org(guild_id)
|
||||
if not org_id:
|
||||
return {"guild_id": guild_id, "org_id": None, "linked": False,
|
||||
"systems": [], "nodes": [], "streams": [], "bots": []}
|
||||
link = await fstore.doc_get(streams.GUILDS, guild_id)
|
||||
systems = await fstore.collection_list("systems", org_id=org_id)
|
||||
nodes = await fstore.collection_list("nodes", org_id=org_id)
|
||||
gstreams = [s for s in await fstore.collection_list(streams.STREAMS, guild_id=guild_id)
|
||||
if s.get("desired") == "on"]
|
||||
bots = []
|
||||
for t in await fstore.collection_list(leases.TOKENS):
|
||||
if t.get("org_id") not in (None, org_id) or t.get("valid") is False:
|
||||
continue
|
||||
lease = t.get("lease") or {}
|
||||
member = t.get("guild_ids")
|
||||
bots.append({
|
||||
"token_id": t.get("token_id"),
|
||||
"name": t.get("name"),
|
||||
"busy_here": any(leases.guild_of(s) == guild_id for s in lease.get("streams") or []),
|
||||
"member": None if member is None else guild_id in member,
|
||||
"invite_url": leases.invite_url(t.get("application_id"), guild_id),
|
||||
})
|
||||
return {
|
||||
"guild_id": guild_id,
|
||||
"org_id": org_id,
|
||||
"linked": bool(link and link.get("org_id")),
|
||||
"systems": [{"system_id": s.get("system_id"), "name": s.get("name")} for s in systems],
|
||||
"nodes": [{"node_id": n.get("node_id"), "name": n.get("name"), "status": n.get("status"),
|
||||
"system_id": streams._effective_system(n)} for n in nodes],
|
||||
"streams": [_public_stream(s) for s in gstreams],
|
||||
"bots": bots,
|
||||
}
|
||||
|
||||
|
||||
class StreamStart(BaseModel):
|
||||
guild_id: str
|
||||
channel_id: str
|
||||
system_id: str
|
||||
mode: str = "adhoc"
|
||||
preferred_token_id: Optional[str] = None
|
||||
channel_name: Optional[str] = None
|
||||
guild_name: Optional[str] = None
|
||||
created_by: Optional[str] = None
|
||||
|
||||
|
||||
@router.post("/streams")
|
||||
async def start_stream(body: StreamStart, _: dict = Depends(require_service_key_or_admin)):
|
||||
org_id = await _org_or_403(body.guild_id)
|
||||
system = await fstore.doc_get_cached("systems", body.system_id)
|
||||
if not system or system.get("org_id") not in (None, org_id):
|
||||
raise HTTPException(404, "System not found.")
|
||||
try:
|
||||
doc = await streams.start_stream(
|
||||
org_id=org_id, guild_id=body.guild_id, channel_id=body.channel_id, system_id=body.system_id,
|
||||
mode=body.mode, channel_name=body.channel_name, guild_name=body.guild_name,
|
||||
created_by=body.created_by, preferred_token_id=body.preferred_token_id,
|
||||
max_streams=settings.discord_guild_max_streams,
|
||||
)
|
||||
except streams.StreamError as e:
|
||||
return _error(e)
|
||||
return _public_stream(doc)
|
||||
|
||||
|
||||
@router.get("/streams/{stream_id}")
|
||||
async def get_stream(stream_id: str, _: dict = Depends(require_service_key_or_admin)):
|
||||
doc = await fstore.doc_get(streams.STREAMS, stream_id)
|
||||
if not doc:
|
||||
raise HTTPException(404, "Stream not found.")
|
||||
return _public_stream(doc)
|
||||
|
||||
|
||||
class GuildStop(BaseModel):
|
||||
channel_id: Optional[str] = None
|
||||
reason: str = "stopped"
|
||||
|
||||
|
||||
@router.post("/guilds/{guild_id}/stop")
|
||||
async def stop_guild(guild_id: str, body: GuildStop, _: dict = Depends(require_service_key_or_admin)):
|
||||
stopped = await streams.stop_guild(guild_id, body.reason[:60], channel_id=body.channel_id)
|
||||
return {"stopped": stopped}
|
||||
|
||||
|
||||
class GuildLink(BaseModel):
|
||||
org_id: str
|
||||
|
||||
|
||||
@router.put("/guilds/{guild_id}/link")
|
||||
async def link_guild(guild_id: str, body: GuildLink, decoded: dict = Depends(require_admin_token)):
|
||||
org = await fstore.doc_get("organizations", body.org_id)
|
||||
if not org:
|
||||
raise HTTPException(404, "Organization not found.")
|
||||
await fstore.doc_set(streams.GUILDS, guild_id, {
|
||||
"guild_id": guild_id, "org_id": body.org_id,
|
||||
"linked_by": decoded.get("uid"), "linked_at": datetime.now(timezone.utc).isoformat(),
|
||||
})
|
||||
return {"ok": True, "guild_id": guild_id, "org_id": body.org_id}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Node route — token pickup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/node/{node_id}/tokens/{token_id}")
|
||||
async def node_token(
|
||||
node_id: str,
|
||||
token_id: str,
|
||||
credentials: Optional[HTTPAuthorizationCredentials] = Security(_bearer),
|
||||
):
|
||||
"""The bot token string, for the node that holds its lease and nobody else."""
|
||||
if not credentials:
|
||||
raise HTTPException(401, "Missing authorization")
|
||||
key_doc = await fstore.doc_get("node_keys", node_id)
|
||||
stored = (key_doc or {}).get("api_key") or ""
|
||||
if not stored or not secrets.compare_digest(stored, credentials.credentials):
|
||||
raise HTTPException(401, "Invalid node API key")
|
||||
tok = await fstore.doc_get(leases.TOKENS, token_id)
|
||||
lease = (tok or {}).get("lease") or {}
|
||||
if not tok or lease.get("node_id") != node_id or lease.get("releasing"):
|
||||
logger.warning(f"Node {node_id} asked for token {token_id} it does not hold a lease on")
|
||||
raise HTTPException(403, "This node does not hold a lease on that token.")
|
||||
return {"token_id": token_id, "token": tok["token"], "lease_id": lease.get("lease_id"),
|
||||
"name": tok.get("name")}
|
||||
@@ -14,7 +14,7 @@ from app.internal.auth import (
|
||||
require_service_or_firebase_token,
|
||||
resolve_caller_org_id,
|
||||
)
|
||||
from app.routers.tokens import assign_token, release_token
|
||||
from app.internal import discord_streams
|
||||
|
||||
router = APIRouter(prefix="/nodes", tags=["nodes"])
|
||||
|
||||
@@ -138,27 +138,28 @@ async def send_command(
|
||||
|
||||
payload = cmd.model_dump(exclude_none=True)
|
||||
|
||||
# Discord join/leave are desired state now (server-26#203/#204, node-26#25):
|
||||
# they never go to the node as commands, and never carry a token string.
|
||||
if cmd.action == "discord_join":
|
||||
# Resolve system doc once — used for preferred token and presence name.
|
||||
system_doc = None
|
||||
system_id = node.get("assigned_system_id")
|
||||
if system_id:
|
||||
system_doc = await fstore.doc_get_cached("systems", system_id)
|
||||
if not cmd.guild_id or not cmd.channel_id:
|
||||
raise HTTPException(400, "guild_id and channel_id are required.")
|
||||
system_id = discord_streams._effective_system(node)
|
||||
if not system_id:
|
||||
raise HTTPException(409, "This node is not listening to a system yet.")
|
||||
try:
|
||||
doc = await discord_streams.start_stream(
|
||||
org_id=node.get("org_id"), guild_id=cmd.guild_id, channel_id=cmd.channel_id,
|
||||
system_id=system_id, node_id=node_id, preferred_token_id=cmd.preferred_token_id,
|
||||
mode="adhoc",
|
||||
)
|
||||
except discord_streams.StreamError as e:
|
||||
raise HTTPException(e.status, e.message)
|
||||
return {"ok": True, "stream_id": doc["stream_id"], "token_id": doc.get("token_id"),
|
||||
"delivered": doc.get("delivered")}
|
||||
|
||||
# Explicit preferred_token_id in the request beats the system-level preference.
|
||||
preferred = payload.pop("preferred_token_id", None) or (system_doc or {}).get("preferred_token_id")
|
||||
token = await assign_token(node_id, preferred_token_id=preferred)
|
||||
if not token:
|
||||
raise HTTPException(503, "No Discord bot tokens available in the pool.")
|
||||
payload["token"] = token
|
||||
|
||||
# Pass system name so the bot can set its Discord presence on join.
|
||||
system_name = (system_doc or {}).get("name")
|
||||
if system_name:
|
||||
payload["system_name"] = system_name
|
||||
|
||||
elif cmd.action == "discord_leave":
|
||||
await release_token(node_id)
|
||||
if cmd.action == "discord_leave":
|
||||
stopped = await discord_streams.stop_node(node_id, "stopped_from_web")
|
||||
return {"ok": True, "stopped": stopped}
|
||||
|
||||
if not mqtt_handler.send_command(node_id, payload):
|
||||
raise HTTPException(503, "MQTT broker unavailable — command not delivered.")
|
||||
|
||||
@@ -2,9 +2,10 @@ import uuid
|
||||
from fastapi import APIRouter, HTTPException, Depends
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
from datetime import datetime, timezone
|
||||
from app.internal import firestore as fstore
|
||||
from app.internal import discord_leases as leases
|
||||
from app.internal.auth import require_admin_token
|
||||
from app.internal.logger import logger
|
||||
|
||||
router = APIRouter(prefix="/tokens", tags=["tokens"])
|
||||
|
||||
@@ -12,53 +13,84 @@ router = APIRouter(prefix="/tokens", tags=["tokens"])
|
||||
class TokenCreate(BaseModel):
|
||||
name: str # friendly label e.g. "DRB Bot 1"
|
||||
token: str # the actual Discord bot token
|
||||
# None = platform pool. An org id = that org's bring-your-own bot, never
|
||||
# leased to anyone else (discord_leases.pick_token).
|
||||
org_id: Optional[str] = None
|
||||
|
||||
|
||||
def _masked(t: dict) -> dict:
|
||||
out = {**t, "token": "•••" + (t.get("token") or "")[-4:]}
|
||||
out["invite_url"] = leases.invite_url(t.get("application_id"))
|
||||
return out
|
||||
|
||||
|
||||
async def _republish_after_release(lease: Optional[dict]) -> None:
|
||||
if lease and lease.get("node_id"):
|
||||
from app.internal.discord_streams import publish_desired
|
||||
await publish_desired(lease["node_id"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CRUD
|
||||
# CRUD — platform admin only (server-26#204 item 6: the list used to be open
|
||||
# to any signed-in user of any org)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("")
|
||||
async def list_tokens():
|
||||
async def list_tokens(_: dict = Depends(require_admin_token)):
|
||||
"""List all tokens. The actual token string is masked for safety."""
|
||||
tokens = await fstore.collection_list("bot_tokens")
|
||||
return [
|
||||
{**t, "token": "•••" + t["token"][-4:]}
|
||||
for t in tokens
|
||||
]
|
||||
return [_masked(t) for t in await fstore.collection_list("bot_tokens")]
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def add_token(body: TokenCreate, _: dict = Depends(require_admin_token)):
|
||||
"""Add a radio bot. The token is checked against Discord first
|
||||
(server-26#204 item 5): a string Discord answers 401 for is refused; if
|
||||
Discord is merely unreachable the token is stored with membership unknown."""
|
||||
try:
|
||||
info = await leases.inspect_token(body.token.strip())
|
||||
except leases.TokenRejected as e:
|
||||
raise HTTPException(400, str(e))
|
||||
token_id = str(uuid.uuid4())
|
||||
doc = {
|
||||
"token_id": token_id,
|
||||
"name": body.name,
|
||||
"token": body.token,
|
||||
"token": body.token.strip(),
|
||||
"org_id": body.org_id,
|
||||
"lease": None,
|
||||
"in_use": False,
|
||||
"assigned_node_id": None,
|
||||
"assigned_at": None,
|
||||
**info,
|
||||
}
|
||||
await fstore.doc_set("bot_tokens", token_id, doc, merge=False)
|
||||
return {"token_id": token_id, "name": body.name}
|
||||
return {"token_id": token_id, "name": body.name, "valid": info.get("valid"),
|
||||
"guild_ids": info.get("guild_ids"), "check_error": info.get("check_error"),
|
||||
"invite_url": leases.invite_url(info.get("application_id"))}
|
||||
|
||||
|
||||
@router.post("/{token_id}/refresh", status_code=200)
|
||||
async def refresh_token(token_id: str, _: dict = Depends(require_admin_token)):
|
||||
"""Re-check a bot with Discord (valid? which servers is it in?)."""
|
||||
if not await fstore.doc_get("bot_tokens", token_id):
|
||||
raise HTTPException(404, "Token not found.")
|
||||
await leases.refresh_membership([token_id])
|
||||
return _masked(await fstore.doc_get("bot_tokens", token_id) or {})
|
||||
|
||||
|
||||
@router.post("/flush", status_code=200)
|
||||
async def flush_tokens(_: dict = Depends(require_admin_token)):
|
||||
"""Force-release all in-use tokens (admin utility — use when tokens get orphaned)."""
|
||||
def _find():
|
||||
from app.internal.firestore import db
|
||||
return [d for d in db.collection("bot_tokens").where("in_use", "==", True).stream()]
|
||||
|
||||
import asyncio
|
||||
results = await asyncio.to_thread(_find)
|
||||
for doc in results:
|
||||
await fstore.doc_update("bot_tokens", doc.id, {
|
||||
"in_use": False,
|
||||
"assigned_node_id": None,
|
||||
"assigned_at": None,
|
||||
})
|
||||
return {"released": len(results)}
|
||||
"""Force-release every leased token (admin last resort). Each holding node
|
||||
is sent a desired-state list without the token, so it logs the bot out;
|
||||
persistent streams that lost their token are re-placed by the reconciler."""
|
||||
released = 0
|
||||
for t in await fstore.collection_list("bot_tokens"):
|
||||
if not (t.get("lease") or t.get("in_use")):
|
||||
continue
|
||||
lease = await leases.force_release_token(t["token_id"])
|
||||
released += 1
|
||||
await _republish_after_release(lease)
|
||||
logger.warning(f"Admin flushed {released} radio-bot token lease(s)")
|
||||
return {"released": released}
|
||||
|
||||
|
||||
@router.put("/{token_id}/prefer/{system_id}", status_code=200)
|
||||
@@ -98,62 +130,6 @@ async def delete_token(token_id: str, _: dict = Depends(require_admin_token)):
|
||||
existing = await fstore.doc_get("bot_tokens", token_id)
|
||||
if not existing:
|
||||
raise HTTPException(404, "Token not found.")
|
||||
if existing.get("in_use"):
|
||||
raise HTTPException(409, "Token is currently in use by a node.")
|
||||
if existing.get("lease"):
|
||||
raise HTTPException(409, "Token is currently leased to a node. Stop its radio channels (or flush) first.")
|
||||
await fstore.doc_delete("bot_tokens", token_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers — used by the nodes router, not exposed via HTTP
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def assign_token(node_id: str, preferred_token_id: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
Find a free token, mark it as in-use, return the token string.
|
||||
If preferred_token_id is given, try that token first (only if it's free).
|
||||
Returns None if no tokens are available.
|
||||
"""
|
||||
def _find_free(preferred: Optional[str]):
|
||||
from app.internal.firestore import db
|
||||
if preferred:
|
||||
snap = db.collection("bot_tokens").document(preferred).get()
|
||||
if snap.exists and not snap.to_dict().get("in_use"):
|
||||
return [snap]
|
||||
docs = db.collection("bot_tokens").where("in_use", "==", False).limit(1).stream()
|
||||
return [d for d in docs]
|
||||
|
||||
import asyncio
|
||||
results = await asyncio.to_thread(_find_free, preferred_token_id)
|
||||
if not results:
|
||||
return None
|
||||
|
||||
doc = results[0]
|
||||
token_id = doc.id
|
||||
token_value = doc.to_dict()["token"]
|
||||
|
||||
await fstore.doc_update("bot_tokens", token_id, {
|
||||
"in_use": True,
|
||||
"assigned_node_id": node_id,
|
||||
"assigned_at": datetime.now(timezone.utc),
|
||||
})
|
||||
return token_value
|
||||
|
||||
|
||||
async def release_token(node_id: str) -> None:
|
||||
"""Free whichever token is currently assigned to this node."""
|
||||
def _find_assigned():
|
||||
from app.internal.firestore import db
|
||||
return [
|
||||
d for d in db.collection("bot_tokens")
|
||||
.where("assigned_node_id", "==", node_id)
|
||||
.stream()
|
||||
]
|
||||
|
||||
import asyncio
|
||||
results = await asyncio.to_thread(_find_assigned)
|
||||
for doc in results:
|
||||
await fstore.doc_update("bot_tokens", doc.id, {
|
||||
"in_use": False,
|
||||
"assigned_node_id": None,
|
||||
"assigned_at": None,
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user