24 Commits

Author SHA1 Message Date
Logan Cusano
77deb3ba2b Initial recording scraper 2023-06-17 17:33:24 -04:00
Logan Cusano
f4475dc9d7 #19
- Update the wrapper called when a feed encounters an error
    - Will now use a more robus backoff system
    - Waits in increments of 30 seconds
    - Keeps track of ignored attempts and 'count'

- Updated wrapper to remove source from backoff list
    - Now removes the object after the first attempt irrespective of deletion station
2023-06-16 23:26:38 -04:00
Logan Cusano
c4650a9e99 Make the bot option in the leave command required 2023-06-16 22:02:54 -04:00
Logan Cusano
f5e119d845 Bugfixes and functional #7 & #9
- #7 needs to error check more
- both need to be cleaned up
2023-06-11 04:40:40 -04:00
Logan Cusano
e8d68b2da7 Initial #7 & #9
- Working commands
- Keeps track of open connections
2023-06-10 22:16:39 -04:00
Logan Cusano
041e0d485d Fix error status in client join 2023-06-10 20:46:43 -04:00
Logan Cusano
fc11324714 Add function to get all client IDs from JSON file #7 2023-06-04 00:24:50 -04:00
Logan Cusano
c6c048c919 Update default command with autocomplete 2023-06-03 23:35:07 -04:00
Logan Cusano
8ab611836b Allow commands to use autocomplete 2023-06-03 23:31:27 -04:00
7d8ad68e27 Merge pull request 'Add join command to server #7' (#15) from Add-join-command-to-server-#7 into master
Reviewed-on: #15
2023-06-03 23:05:39 -04:00
200ca9c926 Merge branch 'master' into Add-join-command-to-server-#7 2023-06-03 23:05:12 -04:00
Logan Cusano
fa2f28207e wrapping up join command
- API untested
2023-06-03 23:00:50 -04:00
Logan Cusano
5c8414b4d8 Moved to issues in git 2023-06-03 22:58:31 -04:00
Logan Cusano
edaeb261f7 Add additional info on connection status to nodeObject 2023-06-03 19:00:16 -04:00
Logan Cusano
c31ccff5ca Added JSDoc to Join wrapper and updated wrapper to also take just a client ID string 2023-06-03 16:03:07 -04:00
Logan Cusano
d2186e9471 Added a join command #7
- Added a JSON example for Known Client IDs
- Implemented a custom slash command builder to add the available presets as options in the discord command
2023-06-03 15:47:07 -04:00
Logan Cusano
07743cf8a3 Updated requirements and versions 2023-06-03 15:43:15 -04:00
Logan Cusano
18afa7c058 Added extra logging when deploying commands 2023-06-03 15:42:40 -04:00
Logan Cusano
a5cff9ec7e Check if the returned data from HTTP is valid JSON and parse if so, return the string if not 2023-06-03 15:42:23 -04:00
Logan Cusano
9450b78bd4 Updated all functions to return nodeObjects instead of SQL response 2023-06-03 15:41:29 -04:00
Logan Cusano
5757c51fa3 Added new utils
- isJsonString
    - This can be used to check if a string is valid json before parsing it
- getMembersInRole
    - This can be used to check online/offline/all members in a role
- (unused) SanitizePresetName
2023-06-03 15:40:48 -04:00
Logan Cusano
fa91cbc887 Used env var for the listening port 2023-06-03 15:39:16 -04:00
Logan Cusano
7fbaf31335 Updated server intents 2023-06-03 15:38:40 -04:00
Logan Cusano
0280cb5384 Update gitignore 2023-06-03 15:38:24 -04:00
24 changed files with 1321 additions and 1347 deletions

View File

@@ -24,7 +24,7 @@ exports.getStatus = (req, res) => {
* Start the bot and join the server and preset specified
*/
exports.joinServer = async (req, res) => {
if (!req.body.clientId || !req.body.channelId) return res.send("500").json({"message": "You must include the client ID (discord token), channel ID (The discord ID of the channel to connect to)"});
if (!req.body.clientId || !req.body.channelId) return res.status(500).json({"message": "You must include the client ID (discord token), channel ID (The discord ID of the channel to connect to)"});
const deviceId = process.env.AUDIO_DEVICE_ID;
const channelId = req.body.channelId;
const clientId = req.body.clientId;

1
Server/.gitignore vendored
View File

@@ -4,3 +4,4 @@ package-lock.json
*.bak
*.log
*._.*
clientIds.json

View File

@@ -1,10 +0,0 @@
## TODOs
- ~~Create balance command for user to view their balance~~
- ~~Create a command that explains the pricing~~
- ~~Update welcome and insufficient replies ~~
- ~~add a section for the help menu to show items that need tokens~~
- add a limiter to the rss feeds to slow down the sends when multiple updates are occurring
- add a blank .env file to the git
- clean up logging
- ensure documentation for functions
- merge with Discord CnC

View File

@@ -0,0 +1,6 @@
{
"[ID from Discord]": {
"name": "[Nickname of the Bot]",
"id": "[Client ID from Discord Dev Portal]"
}
}

120
Server/commands/join.js Normal file
View File

@@ -0,0 +1,120 @@
// Modules
const { customSlashCommandBuilder } = require('../utilities/customSlashCommandBuilder');
const { DebugBuilder } = require("../utilities/debugBuilder");
const { getMembersInRole, getAllClientIds } = require("../utilities/utils");
const { requestOptions, sendHttpRequest } = require("../utilities/httpRequests");
const { getOnlineNodes, updateNodeInfo, addNodeConnection, getConnectionByNodeId } = require("../utilities/mysqlHandler");
// Global Vars
const log = new DebugBuilder("server", "join");
/**
* * This wrapper will check if there is an available node with the requested preset and if so checks for an available client ID to join with
*
* @param {*} presetName The preset name to listen to on the client
* @param {*} channelId The channel ID to join the bot to
* @param {*} clientIdsUsed EITHER A collection of clients that are currently connected OR a single discord client ID (NOT dev portal ID) that should be used to join the server with
* @returns
*/
async function joinServerWrapper(presetName, channelId, clientIdsUsed) {
// Get nodes online
var onlineNodes = await new Promise((recordResolve, recordReject) => {
getOnlineNodes((nodeRows) => {
recordResolve(nodeRows);
});
});
// Check which nodes have the selected preset
onlineNodes = onlineNodes.filter(node => node.nearbySystems.includes(presetName));
log.DEBUG("Filtered Online Nodes: ", onlineNodes);
// Check if any nodes with this preset are available
var nodesCurrentlyAvailable = [];
for (const node of onlineNodes) {
const currentConnection = await getConnectionByNodeId(node.id);
log.DEBUG("Checking to see if there is a connection for Node: ", node, currentConnection);
if(!currentConnection) nodesCurrentlyAvailable.push(node);
}
log.DEBUG("Nodes Currently Available: ", nodesCurrentlyAvailable);
// If not, let the user know
if (!nodesCurrentlyAvailable.length > 0) return Error("All nodes with this channel are unavailable, consider swapping one of the currently joined bots.");
// If so, join with the first node
var availableClientIds = await getAllClientIds();
log.DEBUG("All clients: ", Object.keys(availableClientIds));
var selectedClientId;
if (typeof clientIdsUsed === 'string') {
for (const availableClientId of availableClientIds) {
if (availableClientId.discordId != clientIdsUsed ) selectedClientId = availableClientId;
}
}
else {
log.DEBUG("Client IDs Used: ", clientIdsUsed.keys());
for (const usedClientId of clientIdsUsed.keys()) {
log.DEBUG("Used Client ID: ", usedClientId);
availableClientIds = availableClientIds.filter(cid => cid.discordId != usedClientId);
}
log.DEBUG("Available Client IDs: ", availableClientIds);
if (!Object.keys(availableClientIds).length > 0) return log.ERROR("All client ID have been used, consider swapping one of the curretly joined bots or adding more Client IDs to the pool.")
selectedClientId = availableClientIds[0];
}
const selectedNode = nodesCurrentlyAvailable[0];
const reqOptions = new requestOptions("/bot/join", "POST", selectedNode.ip, selectedNode.port);
const postObject = {
"channelId": channelId,
"clientId": selectedClientId.clientId,
"presetName": presetName
};
log.INFO("Post Object: ", postObject);
sendHttpRequest(reqOptions, JSON.stringify(postObject), async (responseObj) => {
log.VERBOSE("Response Object from node ", selectedNode, responseObj);
if (!responseObj || !responseObj.statusCode == 200) return false;
// Node has connected to discord
// Updating node Object in DB
const updatedNode = await updateNodeInfo(selectedNode);
log.DEBUG("Updated Node: ", updatedNode);
// Adding a new node connection
const nodeConnection = await addNodeConnection(selectedNode, selectedClientId);
log.DEBUG("Node Connection: ", nodeConnection);
});
}
exports.joinServerWrapper = joinServerWrapper;
module.exports = {
data: new customSlashCommandBuilder()
.setName('join')
.setDescription('Join the channel you are in with the preset you choose')
.addAllSystemPresetOptions(),
example: "join",
isPrivileged: false,
requiresTokens: false,
defaultTokenUsage: 0,
deferInitialReply: true,
async execute(interaction) {
try{
const guildId = interaction.guild.id;
const presetName = interaction.options.getString('preset');
if (!interaction.member.voice.channel.id) return interaction.editReply(`You need to be in a voice channel, ${interaction.user}`)
const channelId = interaction.member.voice.channel.id;
log.DEBUG(`Join requested by: ${interaction.user.username}, to: '${presetName}', in channel: ${channelId} / ${guildId}`);
const onlineBots = await getMembersInRole(interaction);
log.DEBUG("Online Bots: ", onlineBots);
await joinServerWrapper(presetName, channelId, onlineBots.online);
await interaction.editReply('**Pong.**');
//await interaction.channel.send('**Pong.**'); // This will send a message to the channel of the interaction outside of the initial reply
}catch(err){
log.ERROR(err)
//await interaction.reply(err.toString());
}
}
}

78
Server/commands/leave.js Normal file
View File

@@ -0,0 +1,78 @@
// Modules
const { customSlashCommandBuilder } = require('../utilities/customSlashCommandBuilder');
const { DebugBuilder } = require("../utilities/debugBuilder");
const { getAllClientIds, getKeyByArrayValue } = require("../utilities/utils");
const { requestOptions, sendHttpRequest } = require("../utilities/httpRequests");
const { checkNodeConnectionByClientId, removeNodeConnectionByNodeId, updateNodeInfo, getConnectedNodes, getAllConnections } = require('../utilities/mysqlHandler');
// Global Vars
const log = new DebugBuilder("server", "leave");
const logAC = new DebugBuilder("server", "leave_autocorrect");
async function leaveServerWrapper(clientIdObject) {
if (!clientIdObject.clientId || !clientIdObject.name) return log.ERROR("Tried to leave server without client ID and/or Name");
const node = await checkNodeConnectionByClientId(clientIdObject);
reqOptions = new requestOptions("/bot/leave", "POST", node.ip, node.port);
const responseObj = await new Promise((recordResolve, recordReject) => {
sendHttpRequest(reqOptions, JSON.stringify({}), async (responseObj) => {
recordResolve(responseObj);
});
});
log.VERBOSE("Response Object from node ", node, responseObj);
if (!responseObj || !responseObj.statusCode == 202) return false;
// Node has disconnected from discor
// Removing the node connection from the DB
const removedConnection = removeNodeConnectionByNodeId(node.id);
log.DEBUG("Removed Node Connection: ", removedConnection);
return;
}
exports.leaveServerWrapper = leaveServerWrapper;
module.exports = {
data: new customSlashCommandBuilder()
.setName('leave')
.setDescription('Disconnect a bot from the server')
.addStringOption(option =>
option.setName("bot")
.setDescription("The bot to disconnect from the server")
.setAutocomplete(true)
.setRequired(true)),
example: "leave",
isPrivileged: false,
requiresTokens: false,
defaultTokenUsage: 0,
deferInitialReply: true,
async autocomplete(interaction) {
const focusedValue = interaction.options.getFocused();
const connections = await getAllConnections();
const filtered = connections.filter(conn => String(conn.clientObject.name).startsWith(focusedValue)).map(conn => conn.clientObject.name);
logAC.DEBUG("Focused Value: ", focusedValue, connections, filtered);
await interaction.respond(
filtered.map(option => ({ name: option, value: option })),
);
},
async execute(interaction) {
try{
const guildId = interaction.guild.id;
const botName = interaction.options.getString('bot');
log.DEBUG("Bot Name: ", botName)
const clinetIds = await getAllClientIds();
log.DEBUG("Client names: ", clinetIds);
const clientDiscordId = getKeyByArrayValue(clinetIds, {'name': botName});
log.DEBUG("Selected bot: ", clinetIds[clientDiscordId]);
// Need to create a table in DB to keep track of what bots have what IDs or an endpoint on the clients to return what ID they are running with
await leaveServerWrapper(clinetIds[clientDiscordId]);
await interaction.editReply(`**${clinetIds[clientDiscordId].name}** has been disconnected`); // This will reply to the initial interaction
//await interaction.channel.send('**word.**'); // This will send a message to the channel of the interaction outside of the initial reply
}catch(err){
log.ERROR(err)
//await interaction.reply(err.toString());
}
}
}

View File

@@ -18,6 +18,9 @@ module.exports = {
requiresTokens: false,
defaultTokenUsage: 0,
deferInitialReply: false,
/*async autocomplete(interaction) {
const focusedValue = interaction.options.getFocused();
},*/
async execute(interaction) {
try{
await interaction.channel.send('**Pong.**'); // TODO - Add insults as the response to this command

View File

@@ -6,6 +6,7 @@ const {getAllNodes, addNewNode, updateNodeInfo, getNodeInfoFromId, getOnlineNode
const utils = require("../utilities/utils");
const { sendHttpRequest, requestOptions } = require("../utilities/httpRequests.js");
const { nodeObject } = require("../utilities/recordHelper.js");
const { joinServerWrapper } = require("../commands/join");
const refreshInterval = process.env.NODE_MONITOR_REFRESH_INTERVAL ?? 1200000;
@@ -90,6 +91,21 @@ exports.nodeCheckIn = async (req, res) => {
}
/**
* Request the node to join the specified server/channel and listen to the specified resource
*
* @param req.body.clientId The client ID to join discord with (NOT dev portal ID, right click user -> Copy ID)
* @param req.body.channelId The Channel ID to join in Discord
* @param req.body.presetName The preset name to listen to in Discord
*/
exports.requestNodeJoinServer = async (req, res) => {
if (!req.body.clientId || !req.body.channelId || !req.body.presetName) return res.status(400).json("Missing information in request, requires clientId, channelId, presetName");
await joinServerWrapper(req.body.presetName, req.body.channelId, req.body.clientId)
}
/**
* The node monitor service, this will periodically check in on the online nodes to make sure they are still online
*/
exports.nodeMonitorService = class nodeMonitorService {
constructor() {
}

View File

@@ -8,45 +8,53 @@ const log = new DebugBuilder("server", "interactionCreate");
module.exports = {
name: Events.InteractionCreate,
async execute(interaction) {
const command = interaction.client.commands.get(interaction.commandName);
log.VERBOSE("Interaction for command: ", command);
// Execute autocomplete if the user is checking autocomplete
if (interaction.isAutocomplete()) {
log.DEBUG("Running autocomplete for command: ", command.data.name);
return await command.autocomplete(interaction);
}
// Check if the interaction is a command
if (!interaction.isChatInputCommand()) return;
const command = interaction.client.commands.get(interaction.commandName);
if (!command) {
log.ERROR(`No command matching ${interaction.commandName} was found.`);
return;
}
if (!command) {
log.ERROR(`No command matching ${interaction.commandName} was found.`);
return;
}
log.DEBUG(`${interaction.member.user} is running '${interaction.commandName}'`);
log.DEBUG(`${interaction.member.user} is running '${interaction.commandName}'`);
await authorizeCommand(interaction, command, async () => {
await authorizeTokenUsage(interaction, command, undefined, async () => {
try {
if (command.deferInitialReply) {
try {
if (interaction.options.getBool('public') && interaction.options.getBool('public') == false) await interaction.deferReply({ ephemeral: true });
else await interaction.deferReply({ ephemeral: false });
}
catch (err) {
if (err instanceof TypeError) {
// The public option doesn't exist in this command
await interaction.deferReply({ ephemeral: false });
} else {
throw err;
await authorizeCommand(interaction, command, async () => {
await authorizeTokenUsage(interaction, command, undefined, async () => {
try {
if (command.deferInitialReply) {
try {
if (interaction.options.getBool('public') && interaction.options.getBool('public') == false) await interaction.deferReply({ ephemeral: true });
else await interaction.deferReply({ ephemeral: false });
}
catch (err) {
if (err instanceof TypeError) {
// The public option doesn't exist in this command
await interaction.deferReply({ ephemeral: false });
} else {
throw err;
}
}
}
command.execute(interaction);
} catch (error) {
log.ERROR(error);
if (interaction.replied || interaction.deferred) {
interaction.followUp({ content: 'There was an error while executing this command!', ephemeral: true });
} else {
interaction.reply({ content: 'There was an error while executing this command!', ephemeral: true });
}
}
command.execute(interaction);
} catch (error) {
log.ERROR(error);
if (interaction.replied || interaction.deferred) {
interaction.followUp({ content: 'There was an error while executing this command!', ephemeral: true });
} else {
interaction.reply({ content: 'There was an error while executing this command!', ephemeral: true });
}
}
})
});
})
});
},
};

View File

@@ -9,6 +9,6 @@ const log = new DebugBuilder("server", "messageCreate");
module.exports = {
name: Events.MessageCreate,
async execute(interaction) {
await linkCop(interaction);
//await linkCop(interaction);
},
};

View File

@@ -26,7 +26,7 @@ const {
} = require('discord.js');
const client = new Client({
intents: [GatewayIntentBits.GuildMessages, GatewayIntentBits.Guilds]
intents: [GatewayIntentBits.GuildMessages, GatewayIntentBits.Guilds, GatewayIntentBits.GuildPresences, GatewayIntentBits.GuildMembers]
});
prefix = process.env.PREFIX
@@ -118,10 +118,21 @@ const commandFiles = fs.readdirSync(commandsPath).filter(file => file.endsWith('
for (const file of commandFiles) {
const filePath = path.join(commandsPath, file);
const command = require(filePath);
log.DEBUG("Importing command: ", command.data.name);
// Set a new item in the Collection
// With the key as the command name and the value as the exported module
client.commands.set(command.data.name, command);
if (command.data instanceof Promise) {
command.data.then(async (builder) => {
command.data = builder;
log.DEBUG("Importing command: ", command.data.name, command);
// Set a new item in the Collection
// With the key as the command name and the value as the exported module
client.commands.set(command.data.name, command);
});
}
else {
log.DEBUG("Importing command: ", command.data.name, command);
// Set a new item in the Collection
// With the key as the command name and the value as the exported module
client.commands.set(command.data.name, command);
}
}
// Run when the bot is ready
@@ -137,7 +148,7 @@ client.on('ready', () => {
runHTTPServer();
log.DEBUG("Starting Node Monitoring Service");
runNodeMonitorService();
//runNodeMonitorService();
log.DEBUG("Starting RSS watcher");
runRssService();

View File

@@ -32,17 +32,38 @@ var runningPostsToRemove = [{
}]
*/
var runningPostsToRemove = {};
const sourceFailureLimit = process.env.SOURCE_FAILURE_LIMIT ?? 3;
const sourceFailureLimit = process.env.SOURCE_FAILURE_LIMIT ?? 15;
/**
* Wrapper for feeds that cause errors. By default it will wait over a day for the source to come back online before deleting it.
*
* @param {*} sourceURL
* @param {string} sourceURL The URL of the feed source causing issues
*/
exports.removeSource = function removeSource(sourceURL) {
log.INFO("Removing source URL: ", sourceURL);
if (!sourceURL in runningPostsToRemove) {runningPostsToRemove[sourceURL] = 1; return;}
// Check to see if this is the first time this source has been attempted
if (!Object.keys(runningPostsToRemove).includes(sourceURL)) {
runningPostsToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 };
return;
}
if (runningPostsToRemove[sourceURL] < sourceFailureLimit) {runningPostsToRemove[sourceURL] += 1; return;}
const backoffDateTimeDifference = (Date.now() - new Date(runningPostsToRemove[sourceURL].timestamp));
const backoffWaitTime = (runningPostsToRemove[sourceURL].count * 30000);
log.DEBUG("Datetime", runningPostsToRemove[sourceURL], backoffDateTimeDifference, backoffWaitTime);
// Check to see if the last error occurred within the backoff period or if we should try again
if (backoffDateTimeDifference <= backoffWaitTime) {
runningPostsToRemove[sourceURL].ignoredAttempts +=1;
return;
}
// Increase the retry counter
if (runningPostsToRemove[sourceURL].count < sourceFailureLimit) {
runningPostsToRemove[sourceURL].count += 1;
runningPostsToRemove[sourceURL].timestamp = Date.now();
return;
}
feedStorage.getRecordBy('link', sourceURL, (err, record) => {
if (err) log.ERROR("Error getting record from feedStorage", err);
@@ -62,13 +83,14 @@ exports.removeSource = function removeSource(sourceURL) {
/**
* Unset a source URL from deletion if the source has not already been deleted
* @param {*} sourceURL The source URL to be unset from deletion
* @returns {*}
*/
exports.unsetRemoveSource = function unsetRemoveSource(sourceURL) {
log.INFO("Unsetting source URL from deletion (if not already deleted): ", sourceURL);
if (!sourceURL in runningPostsToRemove) return;
if (!Object.keys(runningPostsToRemove).includes(sourceURL)) return;
if (runningPostsToRemove[sourceURL] > sourceFailureLimit) return delete runningPostsToRemove[sourceURL];
delete runningPostsToRemove[sourceURL];
return
}
/**

View File

@@ -4,6 +4,7 @@ const log = new DebugBuilder("server", "libUtils");
const { NodeHtmlMarkdown } = require('node-html-markdown');
const { parse } = require("node-html-parser");
const crypto = require("crypto");
require('dotenv').config();
const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g;
const youtubeVideoRegex = /((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)/g
@@ -55,6 +56,7 @@ exports.onError = (error) => {
throw error;
}
var port = process.env.HTTP_PORT;
var bind = typeof port === 'string'
? 'Pipe ' + port
: 'Port ' + port;

View File

@@ -0,0 +1,40 @@
import scrapy
from scrapy.crawler import CrawlerProcess
class RecordingSpider(scrapy.Spider):
name = "recording-scraper"
start_urls = [
'https://radio.vpn.cusano.net/sdr/transmissions',
]
def parse(self, response):
print("ASDASDD")
print(response)
for row in response.css("tr"):
if row.css('td.py-1'):
links = row.css('a')
rows = row.css('td.py-1')
print(row)
yield {
'device': rows[0],
'date': rows[1],
'duration': rows[2],
"frequency": rows[3],
"link": links[0].attrib["href"],
}
next_page_url = response.css("a.page-link > a::attr(href)").extract_first()
if next_page_url is not None:
yield scrapy.Request(response.urljoin(next_page_url))
process = CrawlerProcess(
settings={
"FEEDS": {
"items.json": {"format": "json"},
},
}
)
process.crawl(RecordingSpider)
process.start() # the script will block here until the crawling is finished

View File

@@ -0,0 +1,3 @@
scrapy
fake-useragent
beautifulsoup4

1713
Server/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -4,30 +4,30 @@
"description": "Discord RSS News Bot",
"main": "index.js",
"dependencies": {
"@discordjs/builders": "~1.4.0",
"@discordjs/rest": "~1.5.0",
"axios": "~1.3.4",
"chatgpt": "~4.7.2",
"cookie-parser": "~1.4.4",
"debug": "~2.6.9",
"discord-api-types": "~0.37.35",
"discord.js": "~14.7.1",
"dotenv": "~16.0.3",
"ejs": "~2.6.1",
"express": "~4.18.2",
"fs": "~0.0.1-security",
"gpt-3-encoder": "~1.1.4",
"http-errors": "~1.6.3",
"jsdoc": "~3.6.7",
"jsonfile": "~6.1.0",
"morgan": "~1.9.1",
"mysql": "~2.18.1",
"node-html-markdown": "~1.3.0",
"node-html-parser": "~6.1.5",
"openai": "~3.1.0",
"parse-files": "~0.1.1",
"rss-parser": "~3.12.0",
"user-agents": "~1.0.1303"
"@discordjs/builders": "^1.6.3",
"@discordjs/rest": "^1.7.1",
"axios": "^1.4.0",
"chatgpt": "^5.2.4",
"cookie-parser": "^1.4.6",
"debug": "^4.3.4",
"discord-api-types": "^0.37.42",
"discord.js": "^14.11.0",
"dotenv": "^16.0.3",
"ejs": "^3.1.9",
"express": "^4.18.2",
"fs": "^0.0.1-security",
"gpt-3-encoder": "^1.1.4",
"http-errors": "*",
"jsdoc": "^4.0.2",
"jsonfile": "^6.1.0",
"morgan": "^1.10.0",
"mysql": "^2.18.1",
"node-html-markdown": "^1.3.0",
"node-html-parser": "^6.1.5",
"openai": "^3.2.1",
"parse-files": "^0.1.1",
"rss-parser": "^3.13.0",
"user-agents": "^1.0.1393"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",

View File

@@ -28,4 +28,8 @@ router.get('/nodeInfo', nodesController.getNodeInfo);
// Client checkin with the server to update information
router.post('/nodeCheckIn', nodesController.nodeCheckIn);
// TODO Need to authenticate this request
// Request a particular client to join a particular channel listening to a particular preset
router.post('/joinServer', nodesController.requestNodeJoinServer);
module.exports = router;

View File

@@ -0,0 +1,46 @@
const { SlashCommandBuilder, SlashCommandStringOption } = require('discord.js');
const { DebugBuilder } = require("../utilities/debugBuilder");
const { BufferToJson } = require("../utilities/utils");
const log = new DebugBuilder("server", "customSlashCommandBuilder");
const { getAllNodes, getAllNodesSync } = require("../utilities/mysqlHandler");
exports.customSlashCommandBuilder = class customSlashCommandBuilder extends SlashCommandBuilder {
constructor() {
super();
}
async addAllSystemPresetOptions() {
const nodeObjects = await new Promise((recordResolve, recordReject) => {
getAllNodes((nodeRows) => {
recordResolve(nodeRows);
});
});
log.DEBUG("Node objects: ", nodeObjects);
var presetsAvailable = [];
for (const nodeObject of nodeObjects) {
log.DEBUG("Node object: ", nodeObject);
for (const presetName in nodeObject.nearbySystems) presetsAvailable.push(nodeObject.nearbySystems[presetName]);
}
log.DEBUG("All Presets available: ", presetsAvailable);
// Remove duplicates
presetsAvailable = [...new Set(presetsAvailable)];
log.DEBUG("DeDuped Presets available: ", presetsAvailable);
this.addStringOption(option => option.setName("preset").setRequired(true).setDescription("The channels"));
for (const preset of presetsAvailable){
log.DEBUG("Preset: ", preset);
this.options[0].addChoices({
'name': String(preset),
'value': String(preset)
});
}
log.DEBUG("Preset Options: ", this);
return this;
}
}

View File

@@ -22,6 +22,7 @@ exports.deploy = (clientId, guildIDs) => {
// Grab the SlashCommandBuilder#toJSON() output of each command's data for deployment
for (const file of commandFiles) {
const command = require(`${path.resolve(commandsPath, file)}`);
log.VERBOSE('Deploying Command: ', command);
commands.push(command.data.toJSON());
}

View File

@@ -3,6 +3,7 @@ const { DebugBuilder } = require("../utilities/debugBuilder.js");
const log = new DebugBuilder("server", "httpRequests");
// Modules
const http = require("http");
const { isJsonString } = require("./utils.js");
exports.requestOptions = class requestOptions {
/**
@@ -39,7 +40,7 @@ exports.sendHttpRequest = function sendHttpRequest(requestOptions, data, callbac
res.on('data', (data) => {
const responseObject = {
"statusCode": res.statusCode,
"body": (requestOptions.method === "POST") ? JSON.parse(data) : data.toString()
"body": (isJsonString(data.toString)) ? JSON.parse(data) : data.toString()
};
log.DEBUG("Response Object: ", responseObject);
callback(responseObject);

View File

@@ -1,6 +1,11 @@
require('dotenv').config();
const mysql = require('mysql');
const utils = require('./utils');
const { nodeObject, clientObject, connectionObject } = require("./recordHelper");
const { DebugBuilder } = require("../utilities/debugBuilder");
const { BufferToJson, getClientObjectByClientID } = require("../utilities/utils");
const log = new DebugBuilder("server", "mysSQLHandler");
const connection = mysql.createPool({
host: process.env.NODE_DB_HOST,
@@ -10,6 +15,59 @@ const connection = mysql.createPool({
});
const nodesTable = `${process.env.NODE_DB_NAME}.nodes`;
const nodeConnectionsTable = `${process.env.NODE_DB_NAME}.node_connections`;
/**
* Return a node object from a single SQL row
*
* @param {object} row The row to convert to a node object
* @returns {nodeObject} The converted node object to be used downstream
*/
function returnNodeObjectFromRow(row) {
return new nodeObject({
_id: row.id,
_name: row.name,
_ip: row.ip,
_port: row.port,
_location: row.location,
_nearbySystems: BufferToJson(row.nearbySystems),
_online: (row.online === 1) ? true : false,
});
}
/**
* Wrapper to convert an array of rows to an array of nodeObjects
*
* @param {array} rows The array of SQL results to be converted into node objects
* @returns {array} An array of node objects
*/
function returnNodeObjectFromRows(rows) {
var i = 0;
for (var row of rows){
log.DEBUG("Row: ", row);
rows[i] = returnNodeObjectFromRow(row);
i += 1;
}
log.DEBUG("Converted Objects from Rows: ", rows);
return rows;
}
/**
* Returns a connection object from an SQL row
*
* @param {*} row The SQL row to convert to a connection object
* @returns {connectionObject}
*/
async function returnConnectionObjectFromRow(row) {
if (Array.isArray(row)) row = row[0]
log.DEBUG("Connection row: ", row);
return new connectionObject({
_connection_id: row.connection_id,
_node: await getNodeInfoFromId(row.id),
_client_object: await getClientObjectByClientID(row.discord_client_id)
});
}
/** Get all nodes the server knows about regardless of status
* @param {*} callback Callback function
@@ -17,17 +75,33 @@ const nodesTable = `${process.env.NODE_DB_NAME}.nodes`;
exports.getAllNodes = (callback) => {
const sqlQuery = `SELECT * FROM ${nodesTable}`
runSQL(sqlQuery, (rows) => {
return callback(rows);
if(!rows || rows.length == 0) callback(undefined);
return callback(returnNodeObjectFromRows(rows));
})
}
/**
* Get all Nodes synchronously **May not be working**
*
* @returns
*/
exports.getAllNodesSync = async () => {
const sqlQuery = `SELECT * FROM ${nodesTable}`
const rows = await runSQL(sqlQuery);
console.log("Rows: ", rows);
return returnNodeObjectFromRows(rows);
}
/** Get all nodes that have the online status set true (are online)
* @param callback Callback function
*/
exports.getOnlineNodes = (callback) => {
const sqlQuery = `SELECT * FROM ${nodesTable} WHERE online = 1;`
runSQL(sqlQuery, (rows) => {
return callback(rows);
return callback(returnNodeObjectFromRows(rows));
})
}
@@ -35,45 +109,60 @@ exports.getOnlineNodes = (callback) => {
* @param nodeId The ID of the node
* @param callback Callback function
*/
exports.getNodeInfoFromId = (nodeId, callback) => {
async function getNodeInfoFromId(nodeId, callback = undefined) {
if (!nodeId) throw new Error("No node ID given when trying to fetch node");
log.DEBUG("Getting node from ID: ", nodeId);
const sqlQuery = `SELECT * FROM ${nodesTable} WHERE id = ${nodeId}`
runSQL(sqlQuery, (rows) => {
// Call back the first (and theoretically only) row
// Specify 0 so downstream functions don't have to worry about it
return callback(rows[0]);
})
const sqlResponse = await new Promise((recordResolve, recordReject) => {
runSQL(sqlQuery, (rows) => {
recordResolve(rows);
})
});
// Call back the first (and theoretically only) row
// Specify 0 so downstream functions don't have to worry about it
return (callback) ? callback(returnNodeObjectFromRow(sqlResponse[0])) : returnNodeObjectFromRow(sqlResponse[0]);
}
exports.getNodeInfoFromId = getNodeInfoFromId
/** Add a new node to the DB
* @param nodeObject Node information object
* @param callback Callback function
*/
exports.addNewNode = (nodeObject, callback) => {
exports.addNewNode = async (nodeObject, callback) => {
if (!nodeObject.name) throw new Error("No name provided");
const name = nodeObject.name,
ip = nodeObject.ip,
port = nodeObject.port,
location = nodeObject.location,
nearbySystems = utils.JsonToBuffer(nodeObject.nearbySystems),
online = nodeObject.online;
const sqlQuery = `INSERT INTO ${nodesTable} (name, ip, port, location, nearbySystems, online) VALUES ('${name}', '${ip}', ${port}, '${location}', '${nearbySystems}', ${online})`;
online = nodeObject.online,
connected = 0;
const sqlQuery = `INSERT INTO ${nodesTable} (name, ip, port, location, nearbySystems, online, connected) VALUES ('${name}', '${ip}', ${port}, '${location}', '${nearbySystems}', ${online}, ${connected})`;
runSQL(sqlQuery, (rows) => {
return callback(rows);
})
const sqlResponse = await new Promise((recordResolve, recordReject) => {
runSQL(sqlQuery, (rows) => {
recordResolve(rows);
})
});
// Call back the first (and theoretically only) row
// Specify 0 so downstream functions don't have to worry about it
return (callback) ? callback(returnNodeObjectFromRow(sqlResponse)) : returnNodeObjectFromRow(sqlResponse);
}
/** Update the known info on a node
* @param nodeObject Node information object
* @param callback Callback function
*/
exports.updateNodeInfo = (nodeObject, callback) => {
exports.updateNodeInfo = async (nodeObject, callback = undefined) => {
if(!nodeObject.id) throw new Error("Attempted to updated node without providing ID", nodeObject);
const name = nodeObject.name,
ip = nodeObject.ip,
port = nodeObject.port,
location = nodeObject.location,
online = nodeObject.online;
online = nodeObject.online
let queryParams = [],
nearbySystems = nodeObject.nearbySystems;
@@ -91,7 +180,7 @@ exports.updateNodeInfo = (nodeObject, callback) => {
}
let sqlQuery = `UPDATE ${nodesTable} SET`
if (!queryParams || queryParams.length === 0) return callback(undefined);
if (!queryParams || queryParams.length === 0) return (callback) ? callback(undefined) : undefined;
if (queryParams.length === 1) {
sqlQuery = `${sqlQuery} ${queryParams[0]}`
} else {
@@ -110,21 +199,167 @@ exports.updateNodeInfo = (nodeObject, callback) => {
sqlQuery = `${sqlQuery} WHERE id = ${nodeObject.id};`
runSQL(sqlQuery, (rows) => {
if (rows.affectedRows === 1) return callback(true);
else return callback(rows);
})
const sqlResponse = await new Promise((recordResolve, recordReject) => {
runSQL(sqlQuery, (rows) => {
recordResolve(rows);
})
});
if (sqlResponse.affectedRows === 1) return (callback) ? callback(true) : true;
else return (callback) ? callback(returnNodeObjectFromRows(sqlResponse)) : returnNodeObjectFromRows(sqlResponse);
}
/**
* Add a new connection to the DB when a bot has been connected to the server
*
* @param {*} nodeObject The node object that is being used for this connection
* @param {*} clientId The client ID Object being used for this connection
* @param {*} callback [OPTIONAL] The callback function to be called with the results, will return otherwise
*/
exports.addNodeConnection = (nodeObject, clientObject, callback = undefined) => {
if (!nodeObject.id || !clientObject.clientId) throw new Error("Tried to add a connection without a client and/or node ID");
const sqlQuery = `INSERT INTO ${nodeConnectionsTable} (id, discord_client_id) VALUES (${nodeObject.id}, '${clientObject.clientId}')`;
const sqlResponse = new Promise((recordResolve, recordReject) => {
runSQL(sqlQuery, (rows) => {
recordResolve(rows);
})
});
if (!sqlResponse) throw new Error("No result from added connection");
return (callback) ? callback(true) : true;
}
/**
* Check what node is connected with a given client ID object
*
* @param {*} clientId The client ID object used to search for a connected node
* @param {*} callback [OPTIONAL] The callback function to be called with the results, return will be used otherwise
*/
exports.checkNodeConnectionByClientId = async (clientId, callback = undefined) => {
if (!clientId.clientId) throw new Error("Tried to check a connection without a client ID");
const sqlQuery = `SELECT * FROM ${nodeConnectionsTable} WHERE discord_client_id = '${clientId.clientId}'`;
const sqlResponse = await new Promise((recordResolve, recordReject) => {
runSQL(sqlQuery, (rows) => {
recordResolve(rows);
})
});
log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
const newNodeObject = await getNodeInfoFromId(sqlResponse[0].id);
log.DEBUG("Node Object from SQL Response: ", newNodeObject);
return (callback) ? callback(newNodeObject) : newNodeObject;
}
/**
* Get a connection by node ID
*
* @param {*} nodeId The ID to search for a connection with
* @param {*} callback [OPTIONAL] The callback function to be called with the results, return will be used otherwise
* @returns {connectionObject}
*/
exports.getConnectionByNodeId = async (nodeId, callback = undefined) => {
const sqlQuery = `SELECT * FROM ${nodeConnectionsTable} WHERE id = '${nodeId}'`;
const sqlResponse = await new Promise((recordResolve, recordReject) => {
runSQL(sqlQuery, (rows) => {
recordResolve(rows);
})
});
log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
if (!sqlResponse | sqlResponse.length == 0) return (callback) ? callback(undefined) : undefined;
const newConnectionObject = await returnConnectionObjectFromRow(sqlResponse)
log.DEBUG("Connection Object from SQL Response: ", newConnectionObject);
return (callback) ? callback(newConnectionObject) : newConnectionObject;
}
/**
* Remove a node connection by the node
*
* @param {*} nodeId The node ID of the node to remove connections of
* @param {*} callback [OPTIONAL] The callback function to callback with the results, return will be used otherwise
* @returns
*/
exports.removeNodeConnectionByNodeId = async (nodeId, callback = undefined) => {
const sqlQuery = `DELETE FROM ${nodeConnectionsTable} WHERE id = '${nodeId}'`;
const sqlResponse = await new Promise((recordResolve, recordReject) => {
runSQL(sqlQuery, (rows) => {
recordResolve(rows);
})
});
log.VERBOSE("SQL Response from removing connection: ", sqlResponse);
if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
return (callback) ? callback(sqlResponse) : sqlResponse;
}
/**
* Gets all connected nodes
*
* @param {*} callback [OPTIONAL] The callback function to callback with the results, return will be used otherwise
* @returns {nodeObject}
*/
exports.getConnectedNodes = async (callback = undefined) => {
const sqlQuery = `SELECT * FROM ${nodeConnectionsTable}`;
const sqlResponse = await new Promise((recordResolve, recordReject) => {
runSQL(sqlQuery, (rows) => {
recordResolve(rows);
})
});
log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
var nodeObjects = []
for (const row of sqlResponse) {
const newNodeObject = await getNodeInfoFromId(row.id);
log.DEBUG("Node Object from SQL Response: ", newNodeObject);
nodeObjects.push(newNodeObject);
}
return (callback) ? callback(nodeObjects) : nodeObjects;
}
/**
* Returns all connections
*
* @param {*} callback [OPTIONAL] The callback function to callback with the results, return will be used otherwise
* @returns {connectionObject}
*/
exports.getAllConnections = async (callback = undefined) => {
const sqlQuery = `SELECT * FROM ${nodeConnectionsTable}`;
const sqlResponse = await new Promise((recordResolve, recordReject) => {
runSQL(sqlQuery, (rows) => {
recordResolve(rows);
})
});
log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
var connectionObjects = []
for (const row of sqlResponse) {
connectionObjects.push(await returnConnectionObjectFromRow(row));
}
return (callback) ? callback(connectionObjects) : connectionObjects;
}
// Function to run and handle SQL errors
function runSQL(sqlQuery, callback, error = (err) => {
function runSQL(sqlQuery, callback = undefined, error = (err) => {
console.log(err);
throw err;
}) {
connection.query(sqlQuery, (err, rows) => {
if (err) return error(err);
//console.log('The rows are:', rows);
return callback(rows);
return (callback) ? callback(rows) : rows
})
}

View File

@@ -109,7 +109,9 @@ class nodeObject {
* @param {*} param0._ip The IP that the master can contact the node at
* @param {*} param0._port The port that the client is listening on
* @param {*} param0._location The physical location of the node
* @param {*} param0._online An integer representation of the online status of the bot, ie 0=off, 1=on
* @param {*} param0._online True/False if the node is online or offline
* @param {*} param0._connected True/False if the bot is connected to discord or not
* @param {*} param0._connection The connection Object associated with the node, null if not checked, undefined if none exists
* @param {*} param0._nearbySystems An object array of nearby systems
*/
constructor({ _id = null, _name = null, _ip = null, _port = null, _location = null, _nearbySystems = null, _online = null }) {
@@ -124,3 +126,41 @@ class nodeObject {
}
exports.nodeObject = nodeObject;
/**
* This object represents a discord bot's client information
*/
class clientObject {
/**
*
* @param {*} param0._discord_id The discord id from the node, as seen when right clicking -> copy ID
* @param {*} param0._name The name of the bot associated with the IDs
* @param {*} param0._client_id The client ID of the bot needed to connect to Discord
*/
constructor({_discord_id = null, _name = null, _client_id = null,}) {
this.discordId = _discord_id;
this.name = _name;
this.clientId = _client_id;
}
}
exports.clientObject = clientObject;
/**
* This object represents a discord node connection
*/
class connectionObject {
/**
*
* @param {*} param0._connection_id The connection ID associated with the connection in the database
* @param {*} param0._node The node associated with the connection
* @param {*} param0._client_object The client object associated with the connection
*/
constructor({_connection_id = null, _node = null, _client_object}) {
this.connectionId = _connection_id;
this.node = _node;
this.clientObject = _client_object;
}
}
exports.connectionObject = connectionObject;

View File

@@ -1,3 +1,10 @@
// Debug
const { DebugBuilder } = require("../utilities/debugBuilder");
const { clientObject } = require("./recordHelper");
const { readFileSync } = require('fs');
const log = new DebugBuilder("server", "utils");
const path = require('path');
// Convert a JSON object to a buffer for the DB
exports.JsonToBuffer = (jsonObject) => {
return Buffer.from(JSON.stringify(jsonObject))
@@ -8,6 +15,46 @@ exports.BufferToJson = (buffer) => {
return JSON.parse(buffer.toString());
}
/**
* **DISUSED**
*
* @param {string} presetName The present name to sanitize
* @returns {string} The sanitized preset name to be used elsewhere
*/
exports.SanitizePresetName = (presetName) => {
return String(presetName).toLowerCase().replace(/[\W_]+/g,"-")
}
/**
* Get online, offline and total members in a guild. Optionally a group can be specified to get members' statuses.
*
* @param interaction Discord interaction object
* @param param0.roleName {OPTIONAL} The role name to check the members in; Defaults to 'Bots'
*/
exports.getMembersInRole = async (interaction, roleName = "Bots" ) => {
log.DEBUG("Fetching all members");
var guild = await interaction.client.guilds.fetch({ guild: interaction.guild.id, cache: false }); //cache all members in the server
await guild.members.fetch({cache: false});
await guild.roles.fetch({cache: false});
log.VERBOSE("Guild: ", guild);
const role = await guild.roles.cache.find(role => role.name === roleName); //the role to check
log.DEBUG("Role to check members from: ", role);
log.DEBUG("Members of role: ", role.members);
// This is not working, can't get the status of the users, rest of join is untested
const onlineMembers = await role.members.filter(member => member.voice.channel !== null);
const offlineMembers = await role.members.filter(member => member.voice.channel === null);
const allMembers = await role.members;
log.VERBOSE("All members: ", allMembers, onlineMembers, offlineMembers)
return {
'online': onlineMembers,
'offline': offlineMembers,
'all': allMembers
}
}
/** Find a key in an object by its value
*
* @param {*} object The object to search
@@ -15,5 +62,58 @@ exports.BufferToJson = (buffer) => {
* @returns The key of the object that contains the value
*/
exports.getKeyByArrayValue = (object, value) => {
return Object.keys(object).find(key => object[key].includes(value));
if (typeof value == "string") return Object.keys(object).find(key => object[key].includes(value));
const valueKey = Object.keys(value)[0];
return Object.keys(object).find(key => (object[key][valueKey] == value[valueKey]));
}
/**
* Check to see if the input is a valid JSON string
*
* @param {*} str The string to check for valud JSON
* @returns {true|false}
*/
exports.isJsonString = (str) => {
try {
JSON.parse(str);
} catch (e) {
return false;
}
return true;
}
/**
* Get all client IDs from the saved JSON file
*
* @returns Object of Client IDs
*/
exports.getAllClientIds = () => {
const jsonClientIds = JSON.parse(readFileSync(path.resolve(__dirname, '../clientIds.json')));
var clientObjects = [];
for (const jsonClientId of Object.keys(jsonClientIds)){
clientObjects.push(new clientObject({
_discord_id: jsonClientId,
_name: jsonClientIds[jsonClientId].name,
_client_id: jsonClientIds[jsonClientId].id
}))
}
return clientObjects;
}
/**
* Gets a client object froma discord client ID
*
* @param {*} clientId The discord client ID to get the client object of
* @returns {clientObject|undefined}
*/
exports.getClientObjectByClientID = (clientId) => {
const clientObjects = this.getAllClientIds();
log.DEBUG("All client IDs: ", clientObjects);
for (const clientObject of clientObjects){
if (clientObject.clientId == clientId) {
log.DEBUG("Found client ID from given ID: ", clientObject);
return clientObject
}
}
return undefined
}