Initial recording scraper

Server/commands/giveRole.js

@@ -1,52 +0,0 @@
-const { SlashCommandBuilder } = require('discord.js');
-const { DebugBuilder } = require("../utilities/debugBuilder");
-const log = new DebugBuilder("server", "give-role");
-
-module.exports = {
-	data: new SlashCommandBuilder()
-	.setName('give-role')
-	.setDescription('Use this command to give a role you have to another member.')
-	.addUserOption(option =>
-		option.setName('user')
-			.setDescription('The user you wish to give the role to ')
-			.setRequired(true))
-	.addRoleOption(option =>
-		option.setName('role')
-			.setDescription('The role you wish to give the selected user')
-			.setRequired(true)),
-	example: "give-role",
-	isPrivileged: false,
-	requiresTokens: false,
-	defaultTokenUsage: 0,
-	deferInitialReply: true,
-	/*async autocomplete(interaction) {
-		const focusedValue = interaction.options.getFocused();		
-	},*/
-	async execute(interaction) {
-		try{
-			// The role to give to the user
-			const selectedRole = interaction.options.getRole('role');
-
-			// The user who should be given the role
-			var selectedUser = interaction.options.getUser("user");
-			selectedUser = interaction.guild.members.cache.get(selectedUser.id);
-
-
-			// The user who initiated the command
-			const initUser = interaction.member;
-
-			log.DEBUG("Give Role DEBUG: ", initUser, selectedRole, selectedUser);
-
-			// Check if the user has the role selected
-			if (!initUser.roles.cache.find(role => role.name === selectedRole.name)) return await interaction.editReply(`Sorry ${initUser}, you don't have the group ${selectedRole} and thus you cannot give it to ${selectedUser}`);
-
-			// Give the selected user the role and let both the user and the initiator know
-			await selectedUser.roles.add(selectedRole);
-
-			return await interaction.editReply(`Ok ${initUser}, ${selectedUser} has been given the ${selectedRole} role!`)
-		}catch(err){
-			log.ERROR(err)        
-            //await interaction.reply(err.toString());
-		}
-	}
-};

Server/commands/join.js

@@ -1,9 +1,9 @@
 // Modules
-const { SlashCommandBuilder } = require('discord.js');
+const { customSlashCommandBuilder } = require('../utilities/customSlashCommandBuilder');
 const { DebugBuilder } = require("../utilities/debugBuilder");
-const { getMembersInRole, getAllClientIds, filterAutocompleteValues } = require("../utilities/utils");
+const { getMembersInRole, getAllClientIds } = require("../utilities/utils");
 const { requestOptions, sendHttpRequest } = require("../utilities/httpRequests");
-const { getOnlineNodes, updateNodeInfo, addNodeConnection, getConnectionByNodeId, getAllConnections } = require("../utilities/mysqlHandler");
+const { getOnlineNodes, updateNodeInfo, addNodeConnection, getConnectionByNodeId } = require("../utilities/mysqlHandler");
 
 // Global Vars
 const log = new DebugBuilder("server", "join");
@@ -13,10 +13,10 @@ const log = new DebugBuilder("server", "join");
  * 
  * @param {*} presetName The preset name to listen to on the client
  * @param {*} channelId The channel ID to join the bot to 
- * @param {*} connections EITHER A collection of clients that are currently connected OR a single discord client ID (NOT dev portal ID) that should be used to join the server with
+ * @param {*} clientIdsUsed EITHER A collection of clients that are currently connected OR a single discord client ID (NOT dev portal ID) that should be used to join the server with
  * @returns 
  */
-async function joinServerWrapper(presetName, channelId, connections) {
+async function joinServerWrapper(presetName, channelId, clientIdsUsed) {
 	// Get nodes online
 	var onlineNodes = await new Promise((recordResolve, recordReject) => {
 		getOnlineNodes((nodeRows) => {
@@ -45,16 +45,16 @@ async function joinServerWrapper(presetName, channelId, connections) {
 	log.DEBUG("All clients: ", Object.keys(availableClientIds));
 
 	var selectedClientId;
-	if (typeof connections === 'string') {
+	if (typeof clientIdsUsed === 'string') {
 		for (const availableClientId of availableClientIds) {
-			if (availableClientId.discordId != connections ) selectedClientId = availableClientId;
+			if (availableClientId.discordId != clientIdsUsed ) selectedClientId = availableClientId;
 		}		
 	}
 	else {
-		log.DEBUG("Open connections: ", connections);
+		log.DEBUG("Client IDs Used: ", clientIdsUsed.keys());
-		for (const connection of connections) {
+		for (const usedClientId of clientIdsUsed.keys()) {
-			log.DEBUG("Used Client ID: ", connection);
+			log.DEBUG("Used Client ID: ", usedClientId);
-			availableClientIds = availableClientIds.filter(cid => cid.discordId != connection.clientObject.discordId);			
+			availableClientIds = availableClientIds.filter(cid => cid.discordId != usedClientId);			
 		}
 
 		log.DEBUG("Available Client IDs: ", availableClientIds);
@@ -84,48 +84,19 @@ async function joinServerWrapper(presetName, channelId, connections) {
 		const nodeConnection = await addNodeConnection(selectedNode, selectedClientId);
 		log.DEBUG("Node Connection: ", nodeConnection);
 	});	
-
-	return selectedClientId;
 }
 exports.joinServerWrapper = joinServerWrapper;
 
 module.exports = {
-	data: new SlashCommandBuilder()
+	data: new customSlashCommandBuilder()
 		.setName('join')
 		.setDescription('Join the channel you are in with the preset you choose')
-		.addStringOption(option => 
+		.addAllSystemPresetOptions(),
-			option.setName("preset")
-			.setDescription("The preset you would like to listen to")
-			.setAutocomplete(true)
-			.setRequired(true)),
 	example: "join",
 	isPrivileged: false,
 	requiresTokens: false,
 	defaultTokenUsage: 0,
 	deferInitialReply: true,
-	async autocomplete(interaction) {
-		const nodeObjects = await new Promise((recordResolve, recordReject) => {
-            getOnlineNodes((nodeRows) => {
-                recordResolve(nodeRows);
-            });
-        });
-        log.DEBUG("Node objects: ", nodeObjects);
-        var presetsAvailable = [];
-        for (const nodeObject of nodeObjects) {            
-            log.DEBUG("Node object: ", nodeObject);
-            for (const presetName in nodeObject.nearbySystems) presetsAvailable.push(nodeObject.nearbySystems[presetName]);
-        }
-
-        log.DEBUG("All Presets available: ", presetsAvailable);
-
-        // Remove duplicates
-        options = [...new Set(presetsAvailable)];
-        log.DEBUG("DeDuped Presets available: ", options);
-
-		// Filter the results to what the user is entering
-		filterAutocompleteValues(interaction, options);
-				
-	},
 	async execute(interaction) {
 		try{
 			const guildId = interaction.guild.id;
@@ -134,13 +105,12 @@ module.exports = {
 			const channelId = interaction.member.voice.channel.id;
 			log.DEBUG(`Join requested by: ${interaction.user.username}, to: '${presetName}', in channel: ${channelId} / ${guildId}`);			
 
-			const connections = await getAllConnections();
+			const onlineBots = await getMembersInRole(interaction);
 
-			log.DEBUG("Current Connections: ", connections);
+			log.DEBUG("Online Bots: ", onlineBots);
 
-			const selectedClientId = await joinServerWrapper(presetName, channelId, connections);
+			await joinServerWrapper(presetName, channelId, onlineBots.online);
-
+			await interaction.editReply('**Pong.**');
-			await interaction.editReply(`Ok, ${interaction.member}. **${selectedClientId.name}** is joining your channel.`);
 			//await interaction.channel.send('**Pong.**'); // This will send a message to the channel of the interaction outside of the initial reply
 		}catch(err){
 			log.ERROR(err)

Server/commands/leave.js

@@ -1,12 +1,13 @@
 // Modules
-const { SlashCommandBuilder } = require('discord.js');
+const { customSlashCommandBuilder } = require('../utilities/customSlashCommandBuilder');
 const { DebugBuilder } = require("../utilities/debugBuilder");
-const { getAllClientIds, getKeyByArrayValue, filterAutocompleteValues } = require("../utilities/utils");
+const { getAllClientIds, getKeyByArrayValue } = require("../utilities/utils");
 const { requestOptions, sendHttpRequest } = require("../utilities/httpRequests");
-const { checkNodeConnectionByClientId, removeNodeConnectionByNodeId, getAllConnections } = require('../utilities/mysqlHandler');
+const { checkNodeConnectionByClientId, removeNodeConnectionByNodeId, updateNodeInfo, getConnectedNodes, getAllConnections } = require('../utilities/mysqlHandler');
 
 // Global Vars
 const log = new DebugBuilder("server", "leave");
+const logAC = new DebugBuilder("server", "leave_autocorrect");
 
 async function leaveServerWrapper(clientIdObject) {
 	if (!clientIdObject.clientId || !clientIdObject.name) return log.ERROR("Tried to leave server without client ID and/or Name");
@@ -33,7 +34,7 @@ async function leaveServerWrapper(clientIdObject) {
 exports.leaveServerWrapper = leaveServerWrapper;
 
 module.exports = {
-	data: new SlashCommandBuilder()
+	data: new customSlashCommandBuilder()
 		.setName('leave')
 		.setDescription('Disconnect a bot from the server')
 		.addStringOption(option => 
@@ -47,12 +48,17 @@ module.exports = {
 	defaultTokenUsage: 0,
 	deferInitialReply: true,
 	async autocomplete(interaction) {
+		const focusedValue = interaction.options.getFocused();
 		const connections = await getAllConnections();
-		const options = connections.map(conn => conn.clientObject.name);
+		const filtered = connections.filter(conn => String(conn.clientObject.name).startsWith(focusedValue)).map(conn => conn.clientObject.name);
-		await filterAutocompleteValues(interaction, options);
+		logAC.DEBUG("Focused Value: ", focusedValue, connections, filtered);
+		await interaction.respond(
+			filtered.map(option => ({ name: option, value: option })),
+		);		
 	},
 	async execute(interaction) {
 		try{
+			const guildId = interaction.guild.id;
 			const botName = interaction.options.getString('bot');
 			log.DEBUG("Bot Name: ", botName)			
 			const clinetIds = await getAllClientIds();

Server/commands/remove.js

@@ -7,7 +7,7 @@ const log = new DebugBuilder("server", "remove");
 module.exports = {
 	data: new SlashCommandBuilder()
 	.setName('remove')
-	.setDescription('Remove an RSS source by it\'s title')
+	.setDescription('Remove an RSS source by it\' title')
 	.addStringOption(option =>
 		option.setName('title')
 			.setDescription('The title of the source to remove')

Server/libCore.js

@@ -5,7 +5,7 @@ const { FeedStorage, PostStorage } = require("./libStorage");
 const libUtils = require("./libUtils");
 const { DebugBuilder } = require("./utilities/debugBuilder");
 const log = new DebugBuilder("server", "libCore");
-const mysql = require("mysql2");
+const mysql = require("mysql");
 
 const UserAgent = require("user-agents");
 process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();

Server/libStorage.js

@@ -8,7 +8,7 @@ const { RSSSourceRecord, RSSPostRecord } = require("./utilities/recordHelper");
 
 // Storage Specific Modules
 // MySQL
-const mysql = require("mysql2");
+const mysql = require("mysql");
 
 const rssFeedsTable = process.env.DB_RSS_FEEDS_TABLE;
 const rssPostsTable = process.env.DB_RSS_POSTS_TABLE;

Server/modules/radioRecordingScraper/recordings_spider.py

@@ -0,0 +1,40 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+class RecordingSpider(scrapy.Spider):
+    name = "recording-scraper"
+    start_urls = [
+        'https://radio.vpn.cusano.net/sdr/transmissions',
+    ]
+
+    def parse(self, response):
+        print("ASDASDD")
+        print(response)
+        for row in response.css("tr"):            
+            if row.css('td.py-1'):
+                links = row.css('a')
+                rows = row.css('td.py-1')
+                print(row)
+                yield {
+                    'device': rows[0],
+                    'date': rows[1],
+                    'duration': rows[2],
+                    "frequency": rows[3],
+                    "link": links[0].attrib["href"],
+                }
+            
+        next_page_url = response.css("a.page-link > a::attr(href)").extract_first()
+        if next_page_url is not None:
+            yield scrapy.Request(response.urljoin(next_page_url))
+
+
+process = CrawlerProcess(
+    settings={
+        "FEEDS": {
+            "items.json": {"format": "json"},
+        },
+    }
+)
+
+process.crawl(RecordingSpider)
+process.start()  # the script will block here until the crawling is finished

Server/modules/radioRecordingScraper/requirements.txt

@@ -0,0 +1,3 @@
+scrapy
+fake-useragent
+beautifulsoup4

Server/package.json

@@ -21,7 +21,7 @@
     "jsdoc": "^4.0.2",
     "jsonfile": "^6.1.0",
     "morgan": "^1.10.0",
-    "mysql2": "^3.3.5",
+    "mysql": "^2.18.1",
     "node-html-markdown": "^1.3.0",
     "node-html-parser": "^6.1.5",
     "openai": "^3.2.1",

Server/utilities/customSlashCommandBuilder.js

@@ -0,0 +1,46 @@
+const { SlashCommandBuilder, SlashCommandStringOption } = require('discord.js');
+const { DebugBuilder } = require("../utilities/debugBuilder");
+const { BufferToJson } = require("../utilities/utils");
+const log = new DebugBuilder("server", "customSlashCommandBuilder");
+
+const { getAllNodes, getAllNodesSync } = require("../utilities/mysqlHandler");
+
+exports.customSlashCommandBuilder = class customSlashCommandBuilder extends SlashCommandBuilder {
+    constructor() {
+        super();
+    }
+
+    async addAllSystemPresetOptions() {
+        const nodeObjects = await new Promise((recordResolve, recordReject) => {
+            getAllNodes((nodeRows) => {
+                recordResolve(nodeRows);
+            });
+        });
+        log.DEBUG("Node objects: ", nodeObjects);
+        var presetsAvailable = [];
+        for (const nodeObject of nodeObjects) {            
+            log.DEBUG("Node object: ", nodeObject);
+            for (const presetName in nodeObject.nearbySystems) presetsAvailable.push(nodeObject.nearbySystems[presetName]);
+        }
+
+        log.DEBUG("All Presets available: ", presetsAvailable);
+
+        // Remove duplicates
+        presetsAvailable = [...new Set(presetsAvailable)];
+        log.DEBUG("DeDuped Presets available: ", presetsAvailable);
+        
+        this.addStringOption(option => option.setName("preset").setRequired(true).setDescription("The channels"));
+        for (const preset of presetsAvailable){
+            log.DEBUG("Preset: ", preset);
+            this.options[0].addChoices({
+                'name': String(preset),
+                'value': String(preset)
+            });
+        }
+        log.DEBUG("Preset Options: ", this);
+
+        return this;
+    }
+        
+}
+

Server/utilities/deployCommands.js

@@ -11,14 +11,14 @@ const path = require('node:path');
 const { DebugBuilder } = require("./debugBuilder");
 const log = new DebugBuilder("server", "deployCommands");
 
-var commands = [];
+const commands = [];
 // Grab all the command files from the commands directory you created earlier
 const commandsPath = path.resolve(__dirname, '../commands');
 const commandFiles = fs.readdirSync(commandsPath).filter(file => file.endsWith('.js'));
 
 exports.deploy = (clientId, guildIDs) => {
 	log.DEBUG("Deploying commands for: ", guildIDs);
-	if (!Array.isArray(guildIDs)) guildIDs = [guildIDs];
+	if (Array.isArray(guildIDs)) guildIDs = [guildIDs];
 	// Grab the SlashCommandBuilder#toJSON() output of each command's data for deployment
 	for (const file of commandFiles) {
 		const command = require(`${path.resolve(commandsPath, file)}`);
@@ -48,35 +48,3 @@ exports.deploy = (clientId, guildIDs) => {
 		})()
 	}
 };
-
-/**
- * Remove all commands for a given bot in a given guild
- * 
- * @param {*} clientId The client ID of the bot to remove commands from 
- * @param {*} guildId The ID of the guild to remove the bot commands from
- */
-exports.removeAll = (clientId, guildId) => {	
-	if (!Array.isArray(guildId)) guildIDs = [guildId];
-	log.DEBUG("Removing commands for: ", clientId, guildIDs);
-
-	commands = [];
-
-	const rest = new REST({ version: '10' }).setToken(token);
-	for (const guildId of guildIDs){		
-		(async () => {
-			try {
-				log.DEBUG(`Started refreshing ${commands.length} application (/) commands for guild ID: ${guildId}.`);			
-				// The put method is used to fully refresh all commands in the guild with the current set
-				const data = await rest.put(
-					Routes.applicationGuildCommands(clientId, guildId),
-					{ body: commands },
-				);
-
-				log.DEBUG(`Successfully reloaded ${data.length} application (/) commands for guild ID: ${guildId}.`);
-			} catch (error) {
-				// And of course, make sure you catch and log any errors!
-				log.ERROR("ERROR Deploying commands: ", error, "Body from error: ", commands);
-			}
-		})()
-	}
-}

Server/utilities/mysqlHandler.js

@@ -1,5 +1,5 @@
 require('dotenv').config();
-const mysql = require('mysql2');
+const mysql = require('mysql');
 const utils = require('./utils');
 const { nodeObject, clientObject, connectionObject } = require("./recordHelper");
 const { DebugBuilder } = require("../utilities/debugBuilder");

Server/utilities/utils.js

@@ -3,7 +3,6 @@ const { DebugBuilder } = require("../utilities/debugBuilder");
 const { clientObject } = require("./recordHelper");
 const { readFileSync } = require('fs');
 const log = new DebugBuilder("server", "utils");
-const logAC = new DebugBuilder("server", "command-autocorrect");
 const path = require('path');
 
 // Convert a JSON object to a buffer for the DB
@@ -118,19 +117,3 @@ exports.getClientObjectByClientID = (clientId) => {
     }
     return undefined
 }
-
-
-exports.filterAutocompleteValues = async (interaction, options) => {
-    // Get the command used
-    const command = interaction.command;
-
-    // Find values that start with what the user is entering
-    const focusedValue = interaction.options.getFocused();
-    const filtered = options.filter(preset => preset.startsWith(focusedValue));
-
-    // Give the query response to the user
-    logAC.DEBUG("Focused Value: ", command, focusedValue, options, filtered);
-    await interaction.respond(
-        filtered.map(option => ({ name: option, value: option })),
-    );
-}