Initial recording scraper

Server/commands/join.js

@@ -1,7 +1,7 @@
 // Modules
-const { SlashCommandBuilder } = require('discord.js');
+const { customSlashCommandBuilder } = require('../utilities/customSlashCommandBuilder');
 const { DebugBuilder } = require("../utilities/debugBuilder");
-const { getMembersInRole, getAllClientIds, filterAutocompleteValues } = require("../utilities/utils");
+const { getMembersInRole, getAllClientIds } = require("../utilities/utils");
 const { requestOptions, sendHttpRequest } = require("../utilities/httpRequests");
 const { getOnlineNodes, updateNodeInfo, addNodeConnection, getConnectionByNodeId } = require("../utilities/mysqlHandler");
 
@@ -84,48 +84,19 @@ async function joinServerWrapper(presetName, channelId, clientIdsUsed) {
 		const nodeConnection = await addNodeConnection(selectedNode, selectedClientId);
 		log.DEBUG("Node Connection: ", nodeConnection);
 	});	
-
-	return selectedClientId;
 }
 exports.joinServerWrapper = joinServerWrapper;
 
 module.exports = {
-	data: new SlashCommandBuilder()
+	data: new customSlashCommandBuilder()
 		.setName('join')
 		.setDescription('Join the channel you are in with the preset you choose')
-		.addStringOption(option => 
+		.addAllSystemPresetOptions(),
-			option.setName("preset")
-			.setDescription("The preset you would like to listen to")
-			.setAutocomplete(true)
-			.setRequired(true)),
 	example: "join",
 	isPrivileged: false,
 	requiresTokens: false,
 	defaultTokenUsage: 0,
 	deferInitialReply: true,
-	async autocomplete(interaction) {
-		const nodeObjects = await new Promise((recordResolve, recordReject) => {
-            getOnlineNodes((nodeRows) => {
-                recordResolve(nodeRows);
-            });
-        });
-        log.DEBUG("Node objects: ", nodeObjects);
-        var presetsAvailable = [];
-        for (const nodeObject of nodeObjects) {            
-            log.DEBUG("Node object: ", nodeObject);
-            for (const presetName in nodeObject.nearbySystems) presetsAvailable.push(nodeObject.nearbySystems[presetName]);
-        }
-
-        log.DEBUG("All Presets available: ", presetsAvailable);
-
-        // Remove duplicates
-        options = [...new Set(presetsAvailable)];
-        log.DEBUG("DeDuped Presets available: ", options);
-
-		// Filter the results to what the user is entering
-		filterAutocompleteValues(interaction, options);
-				
-	},
 	async execute(interaction) {
 		try{
 			const guildId = interaction.guild.id;
@@ -138,9 +109,8 @@ module.exports = {
 
 			log.DEBUG("Online Bots: ", onlineBots);
 
-			const selectedClientId = await joinServerWrapper(presetName, channelId, onlineBots.online);
+			await joinServerWrapper(presetName, channelId, onlineBots.online);
-
+			await interaction.editReply('**Pong.**');
-			await interaction.editReply(`Ok, ${interaction.member}. **${selectedClientId.name}** is joining your channel.`);
 			//await interaction.channel.send('**Pong.**'); // This will send a message to the channel of the interaction outside of the initial reply
 		}catch(err){
 			log.ERROR(err)

Server/commands/leave.js

@@ -1,12 +1,13 @@
 // Modules
-const { SlashCommandBuilder } = require('discord.js');
+const { customSlashCommandBuilder } = require('../utilities/customSlashCommandBuilder');
 const { DebugBuilder } = require("../utilities/debugBuilder");
-const { getAllClientIds, getKeyByArrayValue, filterAutocompleteValues } = require("../utilities/utils");
+const { getAllClientIds, getKeyByArrayValue } = require("../utilities/utils");
 const { requestOptions, sendHttpRequest } = require("../utilities/httpRequests");
-const { checkNodeConnectionByClientId, removeNodeConnectionByNodeId, getAllConnections } = require('../utilities/mysqlHandler');
+const { checkNodeConnectionByClientId, removeNodeConnectionByNodeId, updateNodeInfo, getConnectedNodes, getAllConnections } = require('../utilities/mysqlHandler');
 
 // Global Vars
 const log = new DebugBuilder("server", "leave");
+const logAC = new DebugBuilder("server", "leave_autocorrect");
 
 async function leaveServerWrapper(clientIdObject) {
 	if (!clientIdObject.clientId || !clientIdObject.name) return log.ERROR("Tried to leave server without client ID and/or Name");
@@ -33,7 +34,7 @@ async function leaveServerWrapper(clientIdObject) {
 exports.leaveServerWrapper = leaveServerWrapper;
 
 module.exports = {
-	data: new SlashCommandBuilder()
+	data: new customSlashCommandBuilder()
 		.setName('leave')
 		.setDescription('Disconnect a bot from the server')
 		.addStringOption(option => 
@@ -46,13 +47,18 @@ module.exports = {
 	requiresTokens: false,
 	defaultTokenUsage: 0,
 	deferInitialReply: true,
-	async autocomplete(interaction) {		
+	async autocomplete(interaction) {
+		const focusedValue = interaction.options.getFocused();
 		const connections = await getAllConnections();
-		const options = connections.map(conn => conn.clientObject.name);
+		const filtered = connections.filter(conn => String(conn.clientObject.name).startsWith(focusedValue)).map(conn => conn.clientObject.name);
-		await filterAutocompleteValues(interaction, options);
+		logAC.DEBUG("Focused Value: ", focusedValue, connections, filtered);
+		await interaction.respond(
+			filtered.map(option => ({ name: option, value: option })),
+		);		
 	},
 	async execute(interaction) {
-		try{			
+		try{
+			const guildId = interaction.guild.id;
 			const botName = interaction.options.getString('bot');
 			log.DEBUG("Bot Name: ", botName)			
 			const clinetIds = await getAllClientIds();

Server/libCore.js

@@ -5,7 +5,7 @@ const { FeedStorage, PostStorage } = require("./libStorage");
 const libUtils = require("./libUtils");
 const { DebugBuilder } = require("./utilities/debugBuilder");
 const log = new DebugBuilder("server", "libCore");
-const mysql = require("mysql2");
+const mysql = require("mysql");
 
 const UserAgent = require("user-agents");
 process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();

Server/libStorage.js

@@ -8,7 +8,7 @@ const { RSSSourceRecord, RSSPostRecord } = require("./utilities/recordHelper");
 
 // Storage Specific Modules
 // MySQL
-const mysql = require("mysql2");
+const mysql = require("mysql");
 
 const rssFeedsTable = process.env.DB_RSS_FEEDS_TABLE;
 const rssPostsTable = process.env.DB_RSS_POSTS_TABLE;

Server/modules/radioRecordingScraper/recordings_spider.py

@@ -0,0 +1,40 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+class RecordingSpider(scrapy.Spider):
+    name = "recording-scraper"
+    start_urls = [
+        'https://radio.vpn.cusano.net/sdr/transmissions',
+    ]
+
+    def parse(self, response):
+        print("ASDASDD")
+        print(response)
+        for row in response.css("tr"):            
+            if row.css('td.py-1'):
+                links = row.css('a')
+                rows = row.css('td.py-1')
+                print(row)
+                yield {
+                    'device': rows[0],
+                    'date': rows[1],
+                    'duration': rows[2],
+                    "frequency": rows[3],
+                    "link": links[0].attrib["href"],
+                }
+            
+        next_page_url = response.css("a.page-link > a::attr(href)").extract_first()
+        if next_page_url is not None:
+            yield scrapy.Request(response.urljoin(next_page_url))
+
+
+process = CrawlerProcess(
+    settings={
+        "FEEDS": {
+            "items.json": {"format": "json"},
+        },
+    }
+)
+
+process.crawl(RecordingSpider)
+process.start()  # the script will block here until the crawling is finished

Server/modules/radioRecordingScraper/requirements.txt

@@ -0,0 +1,3 @@
+scrapy
+fake-useragent
+beautifulsoup4

Server/package.json

@@ -21,7 +21,7 @@
     "jsdoc": "^4.0.2",
     "jsonfile": "^6.1.0",
     "morgan": "^1.10.0",
-    "mysql2": "^3.3.5",
+    "mysql": "^2.18.1",
     "node-html-markdown": "^1.3.0",
     "node-html-parser": "^6.1.5",
     "openai": "^3.2.1",

Server/utilities/customSlashCommandBuilder.js

@@ -0,0 +1,46 @@
+const { SlashCommandBuilder, SlashCommandStringOption } = require('discord.js');
+const { DebugBuilder } = require("../utilities/debugBuilder");
+const { BufferToJson } = require("../utilities/utils");
+const log = new DebugBuilder("server", "customSlashCommandBuilder");
+
+const { getAllNodes, getAllNodesSync } = require("../utilities/mysqlHandler");
+
+exports.customSlashCommandBuilder = class customSlashCommandBuilder extends SlashCommandBuilder {
+    constructor() {
+        super();
+    }
+
+    async addAllSystemPresetOptions() {
+        const nodeObjects = await new Promise((recordResolve, recordReject) => {
+            getAllNodes((nodeRows) => {
+                recordResolve(nodeRows);
+            });
+        });
+        log.DEBUG("Node objects: ", nodeObjects);
+        var presetsAvailable = [];
+        for (const nodeObject of nodeObjects) {            
+            log.DEBUG("Node object: ", nodeObject);
+            for (const presetName in nodeObject.nearbySystems) presetsAvailable.push(nodeObject.nearbySystems[presetName]);
+        }
+
+        log.DEBUG("All Presets available: ", presetsAvailable);
+
+        // Remove duplicates
+        presetsAvailable = [...new Set(presetsAvailable)];
+        log.DEBUG("DeDuped Presets available: ", presetsAvailable);
+        
+        this.addStringOption(option => option.setName("preset").setRequired(true).setDescription("The channels"));
+        for (const preset of presetsAvailable){
+            log.DEBUG("Preset: ", preset);
+            this.options[0].addChoices({
+                'name': String(preset),
+                'value': String(preset)
+            });
+        }
+        log.DEBUG("Preset Options: ", this);
+
+        return this;
+    }
+        
+}
+

Server/utilities/mysqlHandler.js

@@ -1,5 +1,5 @@
 require('dotenv').config();
-const mysql = require('mysql2');
+const mysql = require('mysql');
 const utils = require('./utils');
 const { nodeObject, clientObject, connectionObject } = require("./recordHelper");
 const { DebugBuilder } = require("../utilities/debugBuilder");

Server/utilities/utils.js

@@ -3,7 +3,6 @@ const { DebugBuilder } = require("../utilities/debugBuilder");
 const { clientObject } = require("./recordHelper");
 const { readFileSync } = require('fs');
 const log = new DebugBuilder("server", "utils");
-const logAC = new DebugBuilder("server", "command-autocorrect");
 const path = require('path');
 
 // Convert a JSON object to a buffer for the DB
@@ -117,20 +116,4 @@ exports.getClientObjectByClientID = (clientId) => {
         }
     }
     return undefined
-}
-
-
-exports.filterAutocompleteValues = async (interaction, options) => {
-    // Get the command used
-    const command = interaction.command;
-
-    // Find values that start with what the user is entering
-    const focusedValue = interaction.options.getFocused();
-    const filtered = options.filter(preset => preset.startsWith(focusedValue));
-
-    // Give the query response to the user
-    logAC.DEBUG("Focused Value: ", command, focusedValue, options, filtered);
-    await interaction.respond(
-        filtered.map(option => ({ name: option, value: option })),
-    );
 }