Initial recording scraper

- Update the wrapper called when a feed encounters an error
    - Will now use a more robus backoff system
    - Waits in increments of 30 seconds
    - Keeps track of ignored attempts and 'count'

- Updated wrapper to remove source from backoff list
    - Now removes the object after the first attempt irrespective of deletion station

Client/.env.example

@@ -1,18 +1,10 @@
 DEBUG="client:*"
-# Bot Config
-# Discord Bot Token
-TOKEN=""
-# Discord Bot Application ID
-APPLICATION_ID=""
-# Default Guild ID
-GUILD_ID=""
-
 # Audio Config
 AUDIO_DEVICE_ID=""
 AUDIO_DEVICE_NAME=""
 
 # Client Config
-CLIENT_ID=
+CLIENT_ID=0
 CLIENT_NAME=""
 CLIENT_IP=""
 CLIENT_PORT=3010

Client/controllers/botController.js

@@ -24,7 +24,7 @@ exports.getStatus = (req, res) => {
  * Start the bot and join the server and preset specified
  */
 exports.joinServer = async (req, res) => {
-    if (!req.body.clientId || !req.body.channelId) return res.send("500").json({"message": "You must include the client ID (discord token), channel ID (The discord ID of the channel to connect to)"});
+    if (!req.body.clientId || !req.body.channelId) return res.status(500).json({"message": "You must include the client ID (discord token), channel ID (The discord ID of the channel to connect to)"});
     const deviceId = process.env.AUDIO_DEVICE_ID;
     const channelId = req.body.channelId;
     const clientId = req.body.clientId;

Client/setup.sh

@@ -13,6 +13,9 @@ ls -ld $SCRIPT_DIR | awk '{print $3}' >> ./config/installerName
 useradd -M RadioNode
 usermod -s -L RadioNode
 
+# Create the .env file from the example
+cp $SCRIPT_DIR/.env.example $SCRIPT_DIR/.env
+
 # Change the ownership of the directory to the service user
 chown RadioNode -R $SCRIPT_DIR
 
@@ -27,7 +30,7 @@ apt-get update
 apt-get upgrade -y
 
 # Install the necessary packages
-apt-get install -y nodejs npm libopus-dev gcc make portaudio19-dev libportaudio2 libpulse-dev pulseaudio apulse python3 pip
+apt-get install -y nodejs portaudio19-dev libportaudio2 libpulse-dev pulseaudio apulse python3 python3-pip
 
 # Ensure pulse audio is running
 pulseaudio

Server/TODO.md

@@ -1,10 +0,0 @@
-## TODOs 
-- ~~Create balance command for user to view their balance~~
-- ~~Create a command that explains the pricing~~
-- ~~Update welcome and insufficient replies ~~
-- ~~add a section for the help menu to show items that need tokens~~
-- add a limiter to the rss feeds to slow down the sends when multiple updates are occurring 
-- add a blank .env file to the git
-- clean up logging
-- ensure documentation for functions
-- merge with Discord CnC

Server/clientIds.json.EXAMPLE

@@ -0,0 +1,6 @@
+{
+    "[ID from Discord]": {
+        "name": "[Nickname of the Bot]",
+        "id": "[Client ID from Discord Dev Portal]"
+    }
+}

Server/commands/join.js

@@ -0,0 +1,120 @@
+// Modules
+const { customSlashCommandBuilder } = require('../utilities/customSlashCommandBuilder');
+const { DebugBuilder } = require("../utilities/debugBuilder");
+const { getMembersInRole, getAllClientIds } = require("../utilities/utils");
+const { requestOptions, sendHttpRequest } = require("../utilities/httpRequests");
+const { getOnlineNodes, updateNodeInfo, addNodeConnection, getConnectionByNodeId } = require("../utilities/mysqlHandler");
+
+// Global Vars
+const log = new DebugBuilder("server", "join");
+
+/**
+ * * This wrapper will check if there is an available node with the requested preset and if so checks for an available client ID to join with
+ * 
+ * @param {*} presetName The preset name to listen to on the client
+ * @param {*} channelId The channel ID to join the bot to 
+ * @param {*} clientIdsUsed EITHER A collection of clients that are currently connected OR a single discord client ID (NOT dev portal ID) that should be used to join the server with
+ * @returns 
+ */
+async function joinServerWrapper(presetName, channelId, clientIdsUsed) {
+	// Get nodes online
+	var onlineNodes = await new Promise((recordResolve, recordReject) => {
+		getOnlineNodes((nodeRows) => {
+			recordResolve(nodeRows);
+		});
+	});	
+
+	// Check which nodes have the selected preset
+	onlineNodes = onlineNodes.filter(node => node.nearbySystems.includes(presetName));
+	log.DEBUG("Filtered Online Nodes: ", onlineNodes);
+
+	// Check if any nodes with this preset are available
+	var nodesCurrentlyAvailable = [];
+	for (const node of onlineNodes) {
+		const currentConnection = await getConnectionByNodeId(node.id);
+		log.DEBUG("Checking to see if there is a connection for Node: ", node, currentConnection);
+		if(!currentConnection) nodesCurrentlyAvailable.push(node);		
+	}
+	log.DEBUG("Nodes Currently Available: ", nodesCurrentlyAvailable);
+
+	// If not, let the user know	
+	if (!nodesCurrentlyAvailable.length > 0) return Error("All nodes with this channel are unavailable, consider swapping one of the currently joined bots.");
+
+	// If so, join with the first node
+	var availableClientIds = await getAllClientIds();
+	log.DEBUG("All clients: ", Object.keys(availableClientIds));
+
+	var selectedClientId;
+	if (typeof clientIdsUsed === 'string') {
+		for (const availableClientId of availableClientIds) {
+			if (availableClientId.discordId != clientIdsUsed ) selectedClientId = availableClientId;
+		}		
+	}
+	else {
+		log.DEBUG("Client IDs Used: ", clientIdsUsed.keys());
+		for (const usedClientId of clientIdsUsed.keys()) {
+			log.DEBUG("Used Client ID: ", usedClientId);
+			availableClientIds = availableClientIds.filter(cid => cid.discordId != usedClientId);			
+		}
+
+		log.DEBUG("Available Client IDs: ", availableClientIds);
+
+		if (!Object.keys(availableClientIds).length > 0) return log.ERROR("All client ID have been used, consider swapping one of the curretly joined bots or adding more Client IDs to the pool.")	
+		selectedClientId = availableClientIds[0];
+	}
+
+	const selectedNode = nodesCurrentlyAvailable[0];
+	
+	const reqOptions = new requestOptions("/bot/join", "POST", selectedNode.ip, selectedNode.port);	
+	const postObject = {
+		"channelId": channelId,
+		"clientId": selectedClientId.clientId,
+		"presetName": presetName
+	};
+	log.INFO("Post Object: ", postObject);
+	sendHttpRequest(reqOptions, JSON.stringify(postObject), async (responseObj) => {
+		log.VERBOSE("Response Object from node ", selectedNode, responseObj);
+		if (!responseObj || !responseObj.statusCode == 200) return false;		
+		// Node has connected to discord
+		// Updating node Object in DB
+		const updatedNode = await updateNodeInfo(selectedNode);
+		log.DEBUG("Updated Node: ", updatedNode);
+
+		// Adding a new node connection
+		const nodeConnection = await addNodeConnection(selectedNode, selectedClientId);
+		log.DEBUG("Node Connection: ", nodeConnection);
+	});	
+}
+exports.joinServerWrapper = joinServerWrapper;
+
+module.exports = {
+	data: new customSlashCommandBuilder()
+		.setName('join')
+		.setDescription('Join the channel you are in with the preset you choose')
+		.addAllSystemPresetOptions(),
+	example: "join",
+	isPrivileged: false,
+	requiresTokens: false,
+	defaultTokenUsage: 0,
+	deferInitialReply: true,
+	async execute(interaction) {
+		try{
+			const guildId = interaction.guild.id;
+			const presetName = interaction.options.getString('preset');
+			if (!interaction.member.voice.channel.id) return interaction.editReply(`You need to be in a voice channel, ${interaction.user}`)
+			const channelId = interaction.member.voice.channel.id;
+			log.DEBUG(`Join requested by: ${interaction.user.username}, to: '${presetName}', in channel: ${channelId} / ${guildId}`);			
+
+			const onlineBots = await getMembersInRole(interaction);
+
+			log.DEBUG("Online Bots: ", onlineBots);
+
+			await joinServerWrapper(presetName, channelId, onlineBots.online);
+			await interaction.editReply('**Pong.**');
+			//await interaction.channel.send('**Pong.**'); // This will send a message to the channel of the interaction outside of the initial reply
+		}catch(err){
+			log.ERROR(err)
+            //await interaction.reply(err.toString());
+		}
+	}
+}

Server/commands/leave.js

@@ -0,0 +1,78 @@
+// Modules
+const { customSlashCommandBuilder } = require('../utilities/customSlashCommandBuilder');
+const { DebugBuilder } = require("../utilities/debugBuilder");
+const { getAllClientIds, getKeyByArrayValue } = require("../utilities/utils");
+const { requestOptions, sendHttpRequest } = require("../utilities/httpRequests");
+const { checkNodeConnectionByClientId, removeNodeConnectionByNodeId, updateNodeInfo, getConnectedNodes, getAllConnections } = require('../utilities/mysqlHandler');
+
+// Global Vars
+const log = new DebugBuilder("server", "leave");
+const logAC = new DebugBuilder("server", "leave_autocorrect");
+
+async function leaveServerWrapper(clientIdObject) {
+	if (!clientIdObject.clientId || !clientIdObject.name) return log.ERROR("Tried to leave server without client ID and/or Name");
+
+	const node = await checkNodeConnectionByClientId(clientIdObject);
+	
+	reqOptions = new requestOptions("/bot/leave", "POST", node.ip, node.port);
+
+	const responseObj = await new Promise((recordResolve, recordReject) => {
+		sendHttpRequest(reqOptions, JSON.stringify({}), async (responseObj) => {
+			recordResolve(responseObj);
+		});
+	});	
+
+	log.VERBOSE("Response Object from node ", node, responseObj);
+	if (!responseObj || !responseObj.statusCode == 202) return false;		
+	// Node has disconnected from discor
+	// Removing the node connection from the DB
+	const removedConnection = removeNodeConnectionByNodeId(node.id);
+	log.DEBUG("Removed Node Connection: ", removedConnection);
+
+	return;
+}
+exports.leaveServerWrapper = leaveServerWrapper;
+
+module.exports = {
+	data: new customSlashCommandBuilder()
+		.setName('leave')
+		.setDescription('Disconnect a bot from the server')
+		.addStringOption(option => 
+			option.setName("bot")
+			.setDescription("The bot to disconnect from the server")
+			.setAutocomplete(true)
+			.setRequired(true)),
+	example: "leave",
+	isPrivileged: false,
+	requiresTokens: false,
+	defaultTokenUsage: 0,
+	deferInitialReply: true,
+	async autocomplete(interaction) {
+		const focusedValue = interaction.options.getFocused();
+		const connections = await getAllConnections();
+		const filtered = connections.filter(conn => String(conn.clientObject.name).startsWith(focusedValue)).map(conn => conn.clientObject.name);
+		logAC.DEBUG("Focused Value: ", focusedValue, connections, filtered);
+		await interaction.respond(
+			filtered.map(option => ({ name: option, value: option })),
+		);		
+	},
+	async execute(interaction) {
+		try{
+			const guildId = interaction.guild.id;
+			const botName = interaction.options.getString('bot');
+			log.DEBUG("Bot Name: ", botName)			
+			const clinetIds = await getAllClientIds();
+			log.DEBUG("Client names: ", clinetIds);
+			const clientDiscordId = getKeyByArrayValue(clinetIds, {'name': botName});
+			log.DEBUG("Selected bot: ", clinetIds[clientDiscordId]);
+			// Need to create a table in DB to keep track of what bots have what IDs or an endpoint on the clients to return what ID they are running with
+			await leaveServerWrapper(clinetIds[clientDiscordId]);
+			
+			await interaction.editReply(`**${clinetIds[clientDiscordId].name}** has been disconnected`);  // This will reply to the initial interaction
+			//await interaction.channel.send('**word.**'); // This will send a message to the channel of the interaction outside of the initial reply
+		}catch(err){
+			log.ERROR(err)
+            //await interaction.reply(err.toString());
+		}
+	}
+}

Server/commands/ping.js

@@ -18,6 +18,9 @@ module.exports = {
 	requiresTokens: false,
 	defaultTokenUsage: 0,
 	deferInitialReply: false,
+	/*async autocomplete(interaction) {
+		const focusedValue = interaction.options.getFocused();		
+	},*/
 	async execute(interaction) {
 		try{
 			await interaction.channel.send('**Pong.**'); // TODO - Add insults as the response to this command

Server/controllers/nodesController.js

@@ -6,6 +6,7 @@ const {getAllNodes, addNewNode, updateNodeInfo, getNodeInfoFromId, getOnlineNode
 const utils = require("../utilities/utils");
 const { sendHttpRequest, requestOptions } = require("../utilities/httpRequests.js");
 const { nodeObject } = require("../utilities/recordHelper.js");
+const { joinServerWrapper } = require("../commands/join");
 
 const refreshInterval = process.env.NODE_MONITOR_REFRESH_INTERVAL ?? 1200000;
 
@@ -90,6 +91,21 @@ exports.nodeCheckIn = async  (req, res) => {
 
 }
 
+/**
+ * Request the node to join the specified server/channel and listen to the specified resource
+ * 
+ * @param req.body.clientId The client ID to join discord with (NOT dev portal ID, right click user -> Copy ID)
+ * @param req.body.channelId The Channel ID to join in Discord
+ * @param req.body.presetName The preset name to listen to in Discord
+ */
+exports.requestNodeJoinServer = async (req, res) => {
+    if (!req.body.clientId || !req.body.channelId || !req.body.presetName) return res.status(400).json("Missing information in request, requires clientId, channelId, presetName");
+    await joinServerWrapper(req.body.presetName, req.body.channelId, req.body.clientId)
+}
+
+/**
+ * The node monitor service, this will periodically check in on the online nodes to make sure they are still online
+ */
 exports.nodeMonitorService = class nodeMonitorService {
     constructor() {
     }

Server/events/interactionCreate.js

@@ -8,45 +8,53 @@ const log = new DebugBuilder("server", "interactionCreate");
 module.exports = {
 	name: Events.InteractionCreate,
 	async execute(interaction) {
+		const command = interaction.client.commands.get(interaction.commandName);
+		log.VERBOSE("Interaction for command: ", command);
+
+		// Execute autocomplete if the user is checking autocomplete
+		if (interaction.isAutocomplete()) {
+			log.DEBUG("Running autocomplete for command: ", command.data.name);
+			return await command.autocomplete(interaction);
+		}
+
+		// Check if the interaction is a command
 		if (!interaction.isChatInputCommand()) return;
 
-	const command = interaction.client.commands.get(interaction.commandName);	
+		if (!command) {
+			log.ERROR(`No command matching ${interaction.commandName} was found.`);
+			return;
+		}
 
-	if (!command) {
+		log.DEBUG(`${interaction.member.user} is running '${interaction.commandName}'`);
-		log.ERROR(`No command matching ${interaction.commandName} was found.`);
-		return;
-	}
 
-	log.DEBUG(`${interaction.member.user} is running '${interaction.commandName}'`);
+		await authorizeCommand(interaction, command, async () => {
-
+			await authorizeTokenUsage(interaction, command, undefined, async () => {
-	await authorizeCommand(interaction, command, async () => {
+				try {
-		await authorizeTokenUsage(interaction, command, undefined, async () => {
+					if (command.deferInitialReply) {
-			try {
+						try {
-				if (command.deferInitialReply) {
+							if (interaction.options.getBool('public') && interaction.options.getBool('public') == false) await interaction.deferReply({ ephemeral: true });
-					try {
+							else await interaction.deferReply({ ephemeral: false });
-						if (interaction.options.getBool('public') && interaction.options.getBool('public') == false) await interaction.deferReply({ ephemeral: true });
-						else await interaction.deferReply({ ephemeral: false });
-					}
-					catch (err) {
-						if (err instanceof TypeError) {
-							// The public option doesn't exist in this command
-							await interaction.deferReply({ ephemeral: false });
-						} else {
-							throw err;
 						}
+						catch (err) {
+							if (err instanceof TypeError) {
+								// The public option doesn't exist in this command
+								await interaction.deferReply({ ephemeral: false });
+							} else {
+								throw err;
+							}
 
+						}
+					} 
+					command.execute(interaction);
+				} catch (error) {
+					log.ERROR(error);
+					if (interaction.replied || interaction.deferred) {
+						interaction.followUp({ content: 'There was an error while executing this command!', ephemeral: true });
+					} else {
+						interaction.reply({ content: 'There was an error while executing this command!', ephemeral: true });
 					}
-				} 
-				command.execute(interaction);
-			} catch (error) {
-				log.ERROR(error);
-				if (interaction.replied || interaction.deferred) {
-					interaction.followUp({ content: 'There was an error while executing this command!', ephemeral: true });
-				} else {
-					interaction.reply({ content: 'There was an error while executing this command!', ephemeral: true });
 				}
-			}
+			})		
-		})		
+		});
-	});
 	},
 };

Server/events/messageCreate.js

@@ -9,6 +9,6 @@ const log = new DebugBuilder("server", "messageCreate");
 module.exports = {
 	name: Events.MessageCreate,
 	async execute(interaction) {
-		await linkCop(interaction);
+		//await linkCop(interaction);
 	},
 };

Server/libCore.js

@@ -32,17 +32,38 @@ var runningPostsToRemove = [{
 }]
 */
 var runningPostsToRemove = {};
-const sourceFailureLimit = process.env.SOURCE_FAILURE_LIMIT ?? 3;
+const sourceFailureLimit = process.env.SOURCE_FAILURE_LIMIT ?? 15;
 
 /**
+ * Wrapper for feeds that cause errors. By default it will wait over a day for the source to come back online before deleting it.
  * 
- * @param {*} sourceURL
+ * @param {string} sourceURL The URL of the feed source causing issues
  */
 exports.removeSource = function removeSource(sourceURL) {
   log.INFO("Removing source URL: ", sourceURL);
-  if (!sourceURL in runningPostsToRemove) {runningPostsToRemove[sourceURL] = 1; return;}
+  // Check to see if this is the first time this source has been attempted
+  if (!Object.keys(runningPostsToRemove).includes(sourceURL)) {
+    runningPostsToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 }; 
+    return;
+  }  
+
+  const backoffDateTimeDifference = (Date.now() - new Date(runningPostsToRemove[sourceURL].timestamp));
+  const backoffWaitTime = (runningPostsToRemove[sourceURL].count * 30000);
+
+  log.DEBUG("Datetime", runningPostsToRemove[sourceURL], backoffDateTimeDifference, backoffWaitTime);
+
+  // Check to see if the last error occurred within the backoff period or if we should try again
+  if (backoffDateTimeDifference <= backoffWaitTime) {
+    runningPostsToRemove[sourceURL].ignoredAttempts +=1; 
+    return;
+  }
   
-  if (runningPostsToRemove[sourceURL] < sourceFailureLimit) {runningPostsToRemove[sourceURL] += 1; return;}
+  // Increase the retry counter 
+  if (runningPostsToRemove[sourceURL].count < sourceFailureLimit) {
+    runningPostsToRemove[sourceURL].count += 1; 
+    runningPostsToRemove[sourceURL].timestamp = Date.now();
+    return;
+  }
 
   feedStorage.getRecordBy('link', sourceURL, (err, record) => {
     if (err) log.ERROR("Error getting record from feedStorage", err);
@@ -62,13 +83,14 @@ exports.removeSource = function removeSource(sourceURL) {
 /**
  * Unset a source URL from deletion if the source has not already been deleted
  * @param {*} sourceURL The source URL to be unset from deletion
- * @returns {*}
  */
 exports.unsetRemoveSource = function unsetRemoveSource(sourceURL) {
   log.INFO("Unsetting source URL from deletion (if not already deleted): ", sourceURL);
-  if (!sourceURL in runningPostsToRemove) return;
+  if (!Object.keys(runningPostsToRemove).includes(sourceURL)) return;
   
-  if (runningPostsToRemove[sourceURL] > sourceFailureLimit) return delete runningPostsToRemove[sourceURL];
+  delete runningPostsToRemove[sourceURL];
+
+  return
 }
 
 /**

Server/modules/radioRecordingScraper/recordings_spider.py

@@ -0,0 +1,40 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+class RecordingSpider(scrapy.Spider):
+    name = "recording-scraper"
+    start_urls = [
+        'https://radio.vpn.cusano.net/sdr/transmissions',
+    ]
+
+    def parse(self, response):
+        print("ASDASDD")
+        print(response)
+        for row in response.css("tr"):            
+            if row.css('td.py-1'):
+                links = row.css('a')
+                rows = row.css('td.py-1')
+                print(row)
+                yield {
+                    'device': rows[0],
+                    'date': rows[1],
+                    'duration': rows[2],
+                    "frequency": rows[3],
+                    "link": links[0].attrib["href"],
+                }
+            
+        next_page_url = response.css("a.page-link > a::attr(href)").extract_first()
+        if next_page_url is not None:
+            yield scrapy.Request(response.urljoin(next_page_url))
+
+
+process = CrawlerProcess(
+    settings={
+        "FEEDS": {
+            "items.json": {"format": "json"},
+        },
+    }
+)
+
+process.crawl(RecordingSpider)
+process.start()  # the script will block here until the crawling is finished

Server/modules/radioRecordingScraper/requirements.txt

@@ -0,0 +1,3 @@
+scrapy
+fake-useragent
+beautifulsoup4

Server/package.json

@@ -4,30 +4,30 @@
   "description": "Discord RSS News Bot",
   "main": "index.js",
   "dependencies": {
-    "@discordjs/builders": "~1.4.0",
+    "@discordjs/builders": "^1.6.3",
-    "@discordjs/rest": "~1.5.0",
+    "@discordjs/rest": "^1.7.1",
-    "axios": "~1.3.4",
+    "axios": "^1.4.0",
-    "chatgpt": "~4.7.2",
+    "chatgpt": "^5.2.4",
-    "cookie-parser": "~1.4.4",
+    "cookie-parser": "^1.4.6",
-    "debug": "~2.6.9",
+    "debug": "^4.3.4",
-    "discord-api-types": "~0.37.35",
+    "discord-api-types": "^0.37.42",
-    "discord.js": "~14.7.1",
+    "discord.js": "^14.11.0",
-    "dotenv": "~16.0.3",
+    "dotenv": "^16.0.3",
-    "ejs": "~2.6.1",
+    "ejs": "^3.1.9",
-    "express": "~4.18.2",
+    "express": "^4.18.2",
-    "fs": "~0.0.1-security",
+    "fs": "^0.0.1-security",
-    "gpt-3-encoder": "~1.1.4",
+    "gpt-3-encoder": "^1.1.4",
-    "http-errors": "~1.6.3",
+    "http-errors": "*",
-    "jsdoc": "~3.6.7",
+    "jsdoc": "^4.0.2",
-    "jsonfile": "~6.1.0",
+    "jsonfile": "^6.1.0",
-    "morgan": "~1.9.1",
+    "morgan": "^1.10.0",
-    "mysql": "~2.18.1",
+    "mysql": "^2.18.1",
-    "node-html-markdown": "~1.3.0",
+    "node-html-markdown": "^1.3.0",
-    "node-html-parser": "~6.1.5",
+    "node-html-parser": "^6.1.5",
-    "openai": "~3.1.0",
+    "openai": "^3.2.1",
-    "parse-files": "~0.1.1",
+    "parse-files": "^0.1.1",
-    "rss-parser": "~3.12.0",
+    "rss-parser": "^3.13.0",
-    "user-agents": "~1.0.1303"
+    "user-agents": "^1.0.1393"
   },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",

Server/routes/nodes.js

@@ -28,4 +28,8 @@ router.get('/nodeInfo', nodesController.getNodeInfo);
 // Client checkin with the server to update information
 router.post('/nodeCheckIn', nodesController.nodeCheckIn);
 
+// TODO Need to authenticate this request
+// Request a particular client to join a particular channel listening to a particular preset
+router.post('/joinServer', nodesController.requestNodeJoinServer);
+
 module.exports = router;

Server/utilities/customSlashCommandBuilder.js

@@ -0,0 +1,46 @@
+const { SlashCommandBuilder, SlashCommandStringOption } = require('discord.js');
+const { DebugBuilder } = require("../utilities/debugBuilder");
+const { BufferToJson } = require("../utilities/utils");
+const log = new DebugBuilder("server", "customSlashCommandBuilder");
+
+const { getAllNodes, getAllNodesSync } = require("../utilities/mysqlHandler");
+
+exports.customSlashCommandBuilder = class customSlashCommandBuilder extends SlashCommandBuilder {
+    constructor() {
+        super();
+    }
+
+    async addAllSystemPresetOptions() {
+        const nodeObjects = await new Promise((recordResolve, recordReject) => {
+            getAllNodes((nodeRows) => {
+                recordResolve(nodeRows);
+            });
+        });
+        log.DEBUG("Node objects: ", nodeObjects);
+        var presetsAvailable = [];
+        for (const nodeObject of nodeObjects) {            
+            log.DEBUG("Node object: ", nodeObject);
+            for (const presetName in nodeObject.nearbySystems) presetsAvailable.push(nodeObject.nearbySystems[presetName]);
+        }
+
+        log.DEBUG("All Presets available: ", presetsAvailable);
+
+        // Remove duplicates
+        presetsAvailable = [...new Set(presetsAvailable)];
+        log.DEBUG("DeDuped Presets available: ", presetsAvailable);
+        
+        this.addStringOption(option => option.setName("preset").setRequired(true).setDescription("The channels"));
+        for (const preset of presetsAvailable){
+            log.DEBUG("Preset: ", preset);
+            this.options[0].addChoices({
+                'name': String(preset),
+                'value': String(preset)
+            });
+        }
+        log.DEBUG("Preset Options: ", this);
+
+        return this;
+    }
+        
+}
+

Server/utilities/deployCommands.js

@@ -22,6 +22,7 @@ exports.deploy = (clientId, guildIDs) => {
 	// Grab the SlashCommandBuilder#toJSON() output of each command's data for deployment
 	for (const file of commandFiles) {
 		const command = require(`${path.resolve(commandsPath, file)}`);
+		log.VERBOSE('Deploying Command: ', command);		
 		commands.push(command.data.toJSON());
 	}
 

Server/utilities/httpRequests.js

@@ -3,6 +3,7 @@ const { DebugBuilder } = require("../utilities/debugBuilder.js");
 const log = new DebugBuilder("server", "httpRequests");
 // Modules
 const http = require("http");
+const { isJsonString } = require("./utils.js");
 
 exports.requestOptions = class requestOptions {
     /**
@@ -39,7 +40,7 @@ exports.sendHttpRequest = function sendHttpRequest(requestOptions, data, callbac
         res.on('data', (data) => {
             const responseObject = {
                 "statusCode": res.statusCode,
-                "body": (requestOptions.method === "POST") ? JSON.parse(data) : data.toString()
+                "body": (isJsonString(data.toString)) ? JSON.parse(data) : data.toString()
             };
             log.DEBUG("Response Object: ", responseObject);
             callback(responseObject);

Server/utilities/mysqlHandler.js

@@ -1,6 +1,11 @@
 require('dotenv').config();
 const mysql = require('mysql');
 const utils = require('./utils');
+const { nodeObject, clientObject, connectionObject } = require("./recordHelper");
+const { DebugBuilder } = require("../utilities/debugBuilder");
+const { BufferToJson, getClientObjectByClientID } = require("../utilities/utils");
+
+const log = new DebugBuilder("server", "mysSQLHandler");
  
 const connection = mysql.createPool({
     host: process.env.NODE_DB_HOST,
@@ -10,24 +15,93 @@ const connection = mysql.createPool({
 });
 
 const nodesTable = `${process.env.NODE_DB_NAME}.nodes`;
+const nodeConnectionsTable = `${process.env.NODE_DB_NAME}.node_connections`;
+
+/**
+ * Return a node object from a single SQL row
+ * 
+ * @param {object} row The row to convert to a node object
+ * @returns {nodeObject} The converted node object to be used downstream
+ */
+function returnNodeObjectFromRow(row) {
+    return new nodeObject({
+        _id: row.id,
+        _name: row.name,
+        _ip: row.ip,
+        _port: row.port,
+        _location: row.location,
+        _nearbySystems: BufferToJson(row.nearbySystems),
+        _online: (row.online === 1) ? true : false,
+        });
+}
+
+/**
+ * Wrapper to convert an array of rows to an array of nodeObjects
+ * 
+ * @param {array} rows The array of SQL results to be converted into node objects
+ * @returns {array} An array of node objects
+ */
+function returnNodeObjectFromRows(rows) {
+    var i = 0;
+    for (var row of rows){
+        log.DEBUG("Row: ", row);
+        rows[i] = returnNodeObjectFromRow(row);
+        i += 1;
+    }
+
+    log.DEBUG("Converted Objects from Rows: ", rows);
+    return rows;
+}
+
+/**
+ * Returns a connection object from an SQL row
+ * 
+ * @param {*} row The SQL row to convert to a connection object
+ * @returns {connectionObject}
+ */
+async function returnConnectionObjectFromRow(row) {
+    if (Array.isArray(row)) row = row[0]
+    log.DEBUG("Connection row: ", row);
+    return new connectionObject({
+        _connection_id: row.connection_id,
+        _node: await getNodeInfoFromId(row.id),
+        _client_object: await getClientObjectByClientID(row.discord_client_id)
+    });
+}
 
 /** Get all nodes the server knows about regardless of status
  * @param {*} callback Callback function
  */
 exports.getAllNodes = (callback) => {
     const sqlQuery = `SELECT * FROM ${nodesTable}`
-    runSQL(sqlQuery, (rows) => {
+    runSQL(sqlQuery, (rows) => {        
-        return callback(rows);
+        if(!rows || rows.length == 0) callback(undefined);
+                
+        return callback(returnNodeObjectFromRows(rows));
     })
 }
 
+/**
+ * Get all Nodes synchronously **May not be working**
+ * 
+ * @returns 
+ */
+exports.getAllNodesSync = async () => {
+    const sqlQuery = `SELECT * FROM ${nodesTable}`    
+    const rows = await runSQL(sqlQuery);
+
+    console.log("Rows: ", rows);
+
+    return returnNodeObjectFromRows(rows);
+}
+
 /** Get all nodes that have the online status set true (are online)
  * @param callback Callback function
  */
 exports.getOnlineNodes = (callback) => {
     const sqlQuery = `SELECT * FROM ${nodesTable} WHERE online = 1;`
     runSQL(sqlQuery, (rows) => {
-        return callback(rows);
+        return callback(returnNodeObjectFromRows(rows));
     })
 }
 
@@ -35,45 +109,60 @@ exports.getOnlineNodes = (callback) => {
  * @param nodeId The ID of the node 
  * @param callback Callback function
  */ 
-exports.getNodeInfoFromId = (nodeId, callback) => {
+ async function getNodeInfoFromId(nodeId, callback = undefined) {
+    if (!nodeId) throw new Error("No node ID given when trying to fetch node");
+    log.DEBUG("Getting node from ID: ", nodeId);
     const sqlQuery = `SELECT * FROM ${nodesTable} WHERE id = ${nodeId}`
-    runSQL(sqlQuery, (rows) => {
+
-        // Call back the first (and theoretically only) row
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
-        // Specify 0 so downstream functions don't have to worry about it
+		runSQL(sqlQuery, (rows) => {
-        return callback(rows[0]);
+            recordResolve(rows);
-    })
+        })
+	});
+
+    // Call back the first (and theoretically only) row
+    // Specify 0 so downstream functions don't have to worry about it
+    return (callback) ? callback(returnNodeObjectFromRow(sqlResponse[0])) : returnNodeObjectFromRow(sqlResponse[0]);
 }
+exports.getNodeInfoFromId = getNodeInfoFromId
 
 /** Add a new node to the DB
  * @param nodeObject Node information object
  * @param callback Callback function
  */
-exports.addNewNode = (nodeObject, callback) => {
+exports.addNewNode = async (nodeObject, callback) => {
     if (!nodeObject.name) throw new Error("No name provided");
     const name = nodeObject.name,
         ip = nodeObject.ip,
         port = nodeObject.port,
         location = nodeObject.location,
         nearbySystems = utils.JsonToBuffer(nodeObject.nearbySystems),
-        online = nodeObject.online;
+        online = nodeObject.online,
-    const sqlQuery  = `INSERT INTO ${nodesTable} (name, ip, port, location, nearbySystems, online) VALUES ('${name}', '${ip}', ${port}, '${location}', '${nearbySystems}', ${online})`;
+        connected = 0;
+    const sqlQuery  = `INSERT INTO ${nodesTable} (name, ip, port, location, nearbySystems, online, connected) VALUES ('${name}', '${ip}', ${port}, '${location}', '${nearbySystems}', ${online}, ${connected})`;
 
-    runSQL(sqlQuery, (rows) => {
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
-        return callback(rows);
+		runSQL(sqlQuery, (rows) => {
-    })
+            recordResolve(rows);
+        })
+	});
+
+    // Call back the first (and theoretically only) row
+    // Specify 0 so downstream functions don't have to worry about it
+    return (callback) ? callback(returnNodeObjectFromRow(sqlResponse)) : returnNodeObjectFromRow(sqlResponse);
 }
 
 /** Update the known info on a node
  * @param nodeObject Node information object
  * @param callback Callback function
  */
-exports.updateNodeInfo = (nodeObject, callback) => {
+exports.updateNodeInfo = async (nodeObject, callback = undefined) => {
     if(!nodeObject.id) throw new Error("Attempted to updated node without providing ID", nodeObject);
     const name = nodeObject.name,
         ip = nodeObject.ip,
         port = nodeObject.port,
         location = nodeObject.location,
-        online = nodeObject.online;
+        online = nodeObject.online
     let queryParams = [],
         nearbySystems = nodeObject.nearbySystems;
 
@@ -91,7 +180,7 @@ exports.updateNodeInfo = (nodeObject, callback) => {
     }
 
     let sqlQuery = `UPDATE ${nodesTable} SET`
-    if (!queryParams || queryParams.length === 0) return callback(undefined);
+    if (!queryParams || queryParams.length === 0) return (callback) ? callback(undefined) : undefined;
     if (queryParams.length === 1) {
         sqlQuery = `${sqlQuery} ${queryParams[0]}`
     } else {
@@ -110,21 +199,167 @@ exports.updateNodeInfo = (nodeObject, callback) => {
 
     sqlQuery = `${sqlQuery} WHERE id = ${nodeObject.id};`
 
-    runSQL(sqlQuery, (rows) => {
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
-        if (rows.affectedRows === 1) return callback(true);
+		runSQL(sqlQuery, (rows) => {
-        else return callback(rows);
+            recordResolve(rows);
-    })
+        })
+	});
+
+    if (sqlResponse.affectedRows === 1) return (callback) ? callback(true) : true;
+    else return (callback) ? callback(returnNodeObjectFromRows(sqlResponse)) : returnNodeObjectFromRows(sqlResponse);
+}
+
+/**
+ * Add a new connection to the DB when a bot has been connected to the server
+ * 
+ * @param {*} nodeObject The node object that is being used for this connection
+ * @param {*} clientId The client ID Object being used for this connection
+ * @param {*} callback [OPTIONAL] The callback function to be called with the results, will return otherwise
+ */
+exports.addNodeConnection = (nodeObject, clientObject, callback = undefined) => {
+    if (!nodeObject.id || !clientObject.clientId) throw new Error("Tried to add a connection without a client and/or node ID");
+    const sqlQuery  = `INSERT INTO ${nodeConnectionsTable} (id, discord_client_id) VALUES (${nodeObject.id}, '${clientObject.clientId}')`;
+
+    const sqlResponse = new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    
+    if (!sqlResponse) throw new Error("No result from added connection"); 
+    return (callback) ? callback(true) : true;
+}
+
+/**
+ * Check what node is connected with a given client ID object
+ * 
+ * @param {*} clientId The client ID object used to search for a connected node
+ * @param {*} callback [OPTIONAL] The callback function to be called with the results, return will be used otherwise
+ */
+exports.checkNodeConnectionByClientId = async (clientId, callback = undefined) => {
+    if (!clientId.clientId) throw new Error("Tried to check a connection without a client ID");
+    const sqlQuery  = `SELECT * FROM ${nodeConnectionsTable} WHERE discord_client_id = '${clientId.clientId}'`;
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    
+    log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
+    
+    if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
+    const newNodeObject = await getNodeInfoFromId(sqlResponse[0].id);
+    log.DEBUG("Node Object from SQL Response: ", newNodeObject);
+    return (callback) ? callback(newNodeObject) : newNodeObject;
+}
+
+/**
+ * Get a connection by node ID
+ * 
+ * @param {*} nodeId The ID to search for a connection with
+ * @param {*} callback [OPTIONAL] The callback function to be called with the results, return will be used otherwise
+ * @returns {connectionObject}
+ */
+exports.getConnectionByNodeId = async (nodeId, callback = undefined) => {
+    const sqlQuery  = `SELECT * FROM ${nodeConnectionsTable} WHERE id = '${nodeId}'`;
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    
+    log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
+    
+    if (!sqlResponse | sqlResponse.length == 0) return (callback) ? callback(undefined) : undefined;
+    const newConnectionObject = await returnConnectionObjectFromRow(sqlResponse)
+    log.DEBUG("Connection Object from SQL Response: ", newConnectionObject);
+    return (callback) ? callback(newConnectionObject) : newConnectionObject;
+}
+
+/**
+ * Remove a node connection by the node
+ * 
+ * @param {*} nodeId The node ID of the node to remove connections of
+ * @param {*} callback [OPTIONAL] The callback function to callback with the results, return will be used otherwise
+ * @returns 
+ */
+exports.removeNodeConnectionByNodeId = async (nodeId, callback = undefined) => {    
+    const sqlQuery  = `DELETE FROM ${nodeConnectionsTable} WHERE id = '${nodeId}'`;
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    log.VERBOSE("SQL Response from removing connection: ", sqlResponse);
+    
+    if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
+    return (callback) ? callback(sqlResponse) : sqlResponse;
+}
+
+/**
+ * Gets all connected nodes
+ * 
+ * @param {*} callback [OPTIONAL] The callback function to callback with the results, return will be used otherwise
+ * @returns {nodeObject}
+ */
+exports.getConnectedNodes = async (callback = undefined) =>  {
+    const sqlQuery  = `SELECT * FROM ${nodeConnectionsTable}`;
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    
+    log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
+    
+    if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
+    var nodeObjects = []
+    for (const row of sqlResponse) {
+        const newNodeObject = await getNodeInfoFromId(row.id);
+        log.DEBUG("Node Object from SQL Response: ", newNodeObject);
+        nodeObjects.push(newNodeObject);
+    }    
+    return (callback) ? callback(nodeObjects) : nodeObjects;
+}
+
+/**
+ * Returns all connections
+ * 
+ * @param {*} callback [OPTIONAL] The callback function to callback with the results, return will be used otherwise
+ * @returns {connectionObject}
+ */
+exports.getAllConnections = async (callback = undefined) => {
+    const sqlQuery  = `SELECT * FROM ${nodeConnectionsTable}`;
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    
+    log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
+    
+    if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
+    var connectionObjects = []
+    for (const row of sqlResponse) {
+        connectionObjects.push(await returnConnectionObjectFromRow(row));
+    }    
+    return (callback) ? callback(connectionObjects) : connectionObjects;
 }
 
 // Function to run and handle SQL errors
-function runSQL(sqlQuery, callback, error = (err) => {
+function runSQL(sqlQuery, callback = undefined, error = (err) => {
     console.log(err);
     throw err;
 }) {
     connection.query(sqlQuery, (err, rows) => {
         if (err) return error(err);
-        //console.log('The rows are:', rows);
+        //console.log('The rows are:', rows);        
-        return callback(rows);
+        return (callback) ? callback(rows) : rows
     })
 }
 

Server/utilities/recordHelper.js

@@ -109,8 +109,10 @@ class nodeObject {
      * @param {*} param0._ip The IP that the master can contact the node at
      * @param {*} param0._port The port that the client is listening on
      * @param {*} param0._location The physical location of the node
-     * @param {*} param0._online An integer representation of the online status of the bot, ie 0=off, 1=on
+     * @param {*} param0._online True/False if the node is online or offline 
-     * @param {*} param0._nearbySystems An object array of nearby systems
+     * @param {*} param0._connected True/False if the bot is connected to discord or not
+     * @param {*} param0._connection The connection Object associated with the node, null if not checked, undefined if none exists 
+     * @param {*} param0._nearbySystems An object array of nearby systems     
      */
     constructor({ _id = null, _name = null, _ip = null, _port = null, _location = null, _nearbySystems = null, _online = null }) {
         this.id = _id;
@@ -123,4 +125,42 @@ class nodeObject {
     }
 }
 
-exports.nodeObject = nodeObject;
+exports.nodeObject = nodeObject;
+
+/**
+ * This object represents a discord bot's client information
+ */
+class clientObject {
+    /**
+     * 
+     * @param {*} param0._discord_id The discord id from the node, as seen when right clicking -> copy ID
+     * @param {*} param0._name The name of the bot associated with the IDs
+     * @param {*} param0._client_id The client ID of the bot needed to connect to Discord
+     */
+    constructor({_discord_id = null, _name = null, _client_id = null,}) {
+        this.discordId = _discord_id;
+        this.name = _name;
+        this.clientId = _client_id;
+    }
+}
+
+exports.clientObject = clientObject;
+
+/**
+ * This object represents a discord node connection
+ */
+class connectionObject {
+    /**
+     * 
+     * @param {*} param0._connection_id The connection ID associated with the connection in the database
+     * @param {*} param0._node The node associated with the connection
+     * @param {*} param0._client_object The client object associated with the connection
+     */
+    constructor({_connection_id = null, _node = null, _client_object}) {
+        this.connectionId = _connection_id;
+        this.node = _node;
+        this.clientObject = _client_object;
+    }
+}
+
+exports.connectionObject = connectionObject;

Server/utilities/utils.js

@@ -1,3 +1,10 @@
+// Debug
+const { DebugBuilder } = require("../utilities/debugBuilder");
+const { clientObject } = require("./recordHelper");
+const { readFileSync } = require('fs');
+const log = new DebugBuilder("server", "utils");
+const path = require('path');
+
 // Convert a JSON object to a buffer for the DB
 exports.JsonToBuffer = (jsonObject) => {
     return Buffer.from(JSON.stringify(jsonObject))
@@ -8,6 +15,46 @@ exports.BufferToJson = (buffer) => {
     return JSON.parse(buffer.toString());
 }
 
+/**
+ * **DISUSED**
+ * 
+ * @param {string} presetName The present name to sanitize
+ * @returns {string} The sanitized preset name to be used elsewhere 
+ */
+exports.SanitizePresetName = (presetName) => {
+    return String(presetName).toLowerCase().replace(/[\W_]+/g,"-")
+}
+
+/**
+ * Get online, offline and total members in a guild. Optionally a group can be specified to get members' statuses.
+ * 
+ * @param interaction Discord interaction object
+ * @param param0.roleName {OPTIONAL} The role name to check the members in; Defaults to 'Bots'
+ */
+exports.getMembersInRole = async (interaction, roleName = "Bots" ) => {
+    log.DEBUG("Fetching all members");
+    var guild = await interaction.client.guilds.fetch({ guild: interaction.guild.id, cache: false }); //cache all members in the server
+    await guild.members.fetch({cache: false});
+    await guild.roles.fetch({cache: false});
+    log.VERBOSE("Guild: ", guild);
+    const role = await guild.roles.cache.find(role => role.name === roleName); //the role to check
+    log.DEBUG("Role to check members from: ", role);
+    log.DEBUG("Members of role: ", role.members);
+    
+    // This is not working, can't get the status of the users, rest of join is untested
+    const onlineMembers = await role.members.filter(member => member.voice.channel !== null);
+    const offlineMembers = await role.members.filter(member => member.voice.channel === null);
+    const allMembers = await role.members;
+
+    log.VERBOSE("All members: ", allMembers, onlineMembers, offlineMembers)
+
+    return {
+        'online': onlineMembers, 
+        'offline': offlineMembers, 
+        'all': allMembers
+    }
+}
+
 /** Find a key in an object by its value
  * 
  * @param {*} object The object to search
@@ -15,5 +62,58 @@ exports.BufferToJson = (buffer) => {
  * @returns The key of the object that contains the value
  */
 exports.getKeyByArrayValue = (object, value) => {
-    return Object.keys(object).find(key => object[key].includes(value));
+    if (typeof value == "string") return Object.keys(object).find(key => object[key].includes(value));
-  }
+    const valueKey = Object.keys(value)[0];    
+    return Object.keys(object).find(key => (object[key][valueKey] == value[valueKey]));
+  }
+
+/**
+ * Check to see if the input is a valid JSON string
+ * 
+ * @param {*} str The string to check for valud JSON
+ * @returns {true|false}
+ */
+exports.isJsonString = (str) => {
+    try {
+        JSON.parse(str);
+    } catch (e) {
+        return false;
+    }
+    return true;
+}
+
+/**
+ * Get all client IDs from the saved JSON file
+ * 
+ * @returns Object of Client IDs
+ */
+exports.getAllClientIds = () => {
+    const jsonClientIds = JSON.parse(readFileSync(path.resolve(__dirname, '../clientIds.json')));
+    var clientObjects = [];
+    for (const jsonClientId of Object.keys(jsonClientIds)){
+        clientObjects.push(new clientObject({
+            _discord_id: jsonClientId,
+            _name: jsonClientIds[jsonClientId].name,
+            _client_id: jsonClientIds[jsonClientId].id
+        }))
+    }
+    return clientObjects;
+}
+
+/**
+ * Gets a client object froma discord client ID
+ * 
+ * @param {*} clientId The discord client ID to get the client object of
+ * @returns {clientObject|undefined}
+ */
+exports.getClientObjectByClientID = (clientId) => {
+    const clientObjects = this.getAllClientIds();
+    log.DEBUG("All client IDs: ", clientObjects);
+    for (const clientObject of clientObjects){
+        if (clientObject.clientId == clientId) {
+            log.DEBUG("Found client ID from given ID: ", clientObject);
+            return clientObject
+        }
+    }
+    return undefined
+}