From fac5274715de95834089dd8ee7087e9901a0470d Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Wed, 22 May 2024 00:17:06 -0400 Subject: [PATCH 1/9] #5 added debugger --- modules/debugger.mjs | 67 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 modules/debugger.mjs diff --git a/modules/debugger.mjs b/modules/debugger.mjs new file mode 100644 index 0000000..8a3f524 --- /dev/null +++ b/modules/debugger.mjs @@ -0,0 +1,67 @@ +// Import necessary modules +import debug from 'debug'; +import { config } from 'dotenv'; +import { writeFile } from 'fs'; +import { inspect } from 'util'; + +// Load environment variables +config(); + +const logLocation = process.env.LOG_LOCATION; + +const writeToLog = async (logMessage, appName) => { + logMessage = `${String(logMessage)}\n`; + + writeFile( + logLocation ?? `./${appName}.log`, + logMessage, + { encoding: "utf-8", flag: 'a+' }, + (err) => { + if (err) console.error(err); + } + ); +}; + +/** + * Create the different logging methods for a function + * Namespace template = ("[app]:[fileName]:['INFO', 'WARNING', 'DEBUG', 'ERROR']") + * @param {string} appName The name of the app to be used in the 'app' portion of the namespace + * @param {string} fileName The name of the file calling the builder to be used in the 'fileName' portion of the namespace + */ +export class DebugBuilder { + constructor(appName, fileName) { + this.INFO = (...messageParts) => { + const _info = debug(`${appName}:${fileName}:INFO`); + _info(messageParts); + writeToLog(`${new Date().toLocaleString('en-US', { timeZone: 'America/New_York' })} - ${appName}:${fileName}:INFO\t-\t${messageParts.map(messagePart => inspect(messagePart))}`, appName); + }; + + this.DEBUG = (...messageParts) => { + const _debug = debug(`${appName}:${fileName}:DEBUG`); + _debug(messageParts); + writeToLog(`${new Date().toLocaleString('en-US', { timeZone: 'America/New_York' })} - ${appName}:${fileName}:DEBUG\t-\t${messageParts.map(messagePart => inspect(messagePart))}`, appName); + }; + + this.VERBOSE = (...messageParts) => { + const _verbose = debug(`${appName}:${fileName}:VERBOSE`); + _verbose(messageParts); + writeToLog(`${new Date().toLocaleString('en-US', { timeZone: 'America/New_York' })} - ${appName}:${fileName}:VERBOSE\t-\t${messageParts.map(messagePart => inspect(messagePart))}`, appName); + }; + + this.WARN = (...messageParts) => { + const _warn = debug(`${appName}:${fileName}:WARNING`); + _warn(messageParts); + writeToLog(`${new Date().toLocaleString('en-US', { timeZone: 'America/New_York' })} - ${appName}:${fileName}:WARNING\t-\t${messageParts.map(messagePart => inspect(messagePart))}`, appName); + }; + + this.ERROR = (...messageParts) => { + const _error = debug(`${appName}:${fileName}:ERROR`); + _error(messageParts); + writeToLog(`${new Date().toLocaleString('en-US', { timeZone: 'America/New_York' })} - ${appName}:${fileName}:ERROR\t-\t${messageParts.map(messagePart => inspect(messagePart))}`, appName); + if (process.env.EXIT_ON_ERROR && process.env.EXIT_ON_ERROR > 0) { + writeToLog("!--- EXITING ---!", appName); + setTimeout(() => process.exit(), process.env.EXIT_ON_ERROR_DELAY ?? 0); + } + }; + } +} From 4e71c7b167616b9d6601807f3341bc06b4e8446c Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Wed, 22 May 2024 00:17:36 -0400 Subject: [PATCH 2/9] Initial RSS implementation - Added debug command to trigger RSS feed update from discord --- discordBot/commands/add.mjs | 71 +++++++ discordBot/commands/triggerRss.mjs | 45 +++++ discordBot/modules/rssWrappers.mjs | 107 +++++++++++ modules/mongo-wrappers/mongoFeedsWrappers.mjs | 88 +++++++++ package-lock.json | 174 +++++++++++++++++- package.json | 5 +- rss-manager/feedHandler.mjs | 156 ++++++++++++++++ rss-manager/rssController.mjs | 37 ++++ 8 files changed, 680 insertions(+), 3 deletions(-) create mode 100644 discordBot/commands/add.mjs create mode 100644 discordBot/commands/triggerRss.mjs create mode 100644 discordBot/modules/rssWrappers.mjs create mode 100644 modules/mongo-wrappers/mongoFeedsWrappers.mjs create mode 100644 rss-manager/feedHandler.mjs create mode 100644 rss-manager/rssController.mjs diff --git a/discordBot/commands/add.mjs b/discordBot/commands/add.mjs new file mode 100644 index 0000000..0259c85 --- /dev/null +++ b/discordBot/commands/add.mjs @@ -0,0 +1,71 @@ + +import { SlashCommandBuilder } from 'discord.js'; +import { DebugBuilder } from "../../modules/debugger.mjs"; +import { addSource } from '../../rss-manager/feedHandler.mjs' +const log = new DebugBuilder("server", "add"); + +// Exporting data property that contains the command structure for discord including any params +export const data = new SlashCommandBuilder() + .setName('add') + .setDescription('Add RSS Source') + .addStringOption(option => + option.setName('title') + .setDescription('The title of the RSS feed') + .setRequired(true)) + .addStringOption(option => + option.setName('link') + .setDescription('The link to the RSS feed') + .setRequired(true)) + .addStringOption(option => + option.setName('category') + .setDescription('The category for the RSS feed *("ALL" by default")*') + .setRequired(false)) + +// Exporting other properties +export const example = "/add [title] [https://domain.com/feed.xml] [category]"; // An example of how the command would be run in discord chat, this will be used for the help command +export const deferInitialReply = false; // If we the initial reply in discord should be deferred. This gives extra time to respond, however the method of replying is different. + +/** + * Function to give the user auto-reply suggestions + * @param {any} nodeIo The nodeIO server for manipulation of sockets + * @param {any} interaction The interaction object + */ + +// TODO - Setup autocorrect for the category +/* +export async function autocomplete(nodeIo, interaction) { + const focusedValue = interaction.options.getFocused(); + const choices = []; + const filtered = choices.filter(choice => choice.name.startsWith(focusedValue)); + console.log(focusedValue, choices, filtered); + await interaction.respond(filtered); +} +*/ + +/** + * The function to run when the command is called by a discord user + * @param {any} nodeIo The nodeIO server for manipulation of sockets + * @param {any} interaction The interaction object + */ +export const execute = async (nodeIo, interaction) => { + try { + var title = interaction.options.getString('title'); + var link = interaction.options.getString('link'); + var category = interaction.options.getString('category'); + + if (!category) category = "ALL"; + + await addSource(title, link, category, interaction.guildId, interaction.channelId, (err, result) => { + log.DEBUG("Result from adding entry", result); + + if (result) { + interaction.reply(`Adding ${title} to the list of RSS sources`); + } else { + interaction.reply(`${title} already exists in the list of RSS sources`); + } + }); + } catch (err) { + log.ERROR(err) + await interaction.reply(err.toString()); + } +} \ No newline at end of file diff --git a/discordBot/commands/triggerRss.mjs b/discordBot/commands/triggerRss.mjs new file mode 100644 index 0000000..f55b0be --- /dev/null +++ b/discordBot/commands/triggerRss.mjs @@ -0,0 +1,45 @@ +import { SlashCommandBuilder } from 'discord.js'; +import { updateFeeds } from '../../rss-manager/feedHandler.mjs' + +// Exporting data property that contains the command structure for discord including any params +export const data = new SlashCommandBuilder() + .setName('trigger-rss') + .setDescription('Manually triggers an RSS feed update'); + +// Exporting other properties +export const example = "/trigger-rss"; // An example of how the command would be run in discord chat, this will be used for the help command +export const deferInitialReply = false; // If we the initial reply in discord should be deferred. This gives extra time to respond, however the method of replying is different. + +/** + * Function to give the user auto-reply suggestions + * @param {any} nodeIo The nodeIO server for manipulation of sockets + * @param {any} interaction The interaction object + */ +/* +export async function autocomplete(nodeIo, interaction) { + const focusedValue = interaction.options.getFocused(); + const choices = []; + const filtered = choices.filter(choice => choice.name.startsWith(focusedValue)); + console.log(focusedValue, choices, filtered); + await interaction.respond(filtered); +} +*/ + +/** + * The function to run when the command is called by a discord user + * @param {any} nodeIo The nodeIO server for manipulation of sockets + * @param {any} interaction The interaction object + */ +export const execute = async (nodeIo, interaction) => { + try { + //const sockets = await nodeIo.allSockets(); + //console.log("All open sockets: ", sockets); + //await interaction.reply(`**Online Sockets: '${sockets}'**`); + await interaction.reply('Triggering RSS update'); + await updateFeeds(interaction.client); + //await interaction.channel.send('**Pong.**'); + } catch (err) { + console.error(err); + // await interaction.reply(err.toString()); + } +} \ No newline at end of file diff --git a/discordBot/modules/rssWrappers.mjs b/discordBot/modules/rssWrappers.mjs new file mode 100644 index 0000000..0f3e441 --- /dev/null +++ b/discordBot/modules/rssWrappers.mjs @@ -0,0 +1,107 @@ +// Import necessary modules +import { EmbedBuilder } from 'discord.js'; +import { DebugBuilder } from "../../modules/debugger.mjs"; +import { parse } from "node-html-parser"; +import { config } from 'dotenv'; + +// Load environment variables +config(); + +const log = new DebugBuilder("server", "libUtils"); + +const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g; +const youtubeVideoRegex = /((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)/g; + +export class EmmeliaEmbedBuilder extends EmbedBuilder { + constructor() { + super(); + this.setTimestamp(); + this.setFooter({ text: 'Brought to you by Emmelia.' }); + } +} + +export const sendPost = (post, source, channel) => { + log.DEBUG("Sending post from source: ", post, source); + const postTitle = String(post.title).substring(0, 150); + const postLink = post.link; + let postContent; + + if (post.content) { + // Reset the content parameter with the encoded parameter + post.content = parse(post['content:encoded'] ?? post.content); + // Get the post content and trim it to length or add a placeholder if necessary + let postText = String(post.content.text); + if (postText.length >= 3800) postText = `${postText.slice(0, 3800).substring(0, Math.min(postText.length, postText.lastIndexOf(" ")))} [...](${post.link})`; + else if (postText.length === 0) postText = `*This post has no content* [Direct Link](${post.link})`; + postContent = postText; + } else { + postContent = `*This post has no content* [Direct Link](${post.link})`; + } + + // Check for embedded youtube videos and add the first four as links + const ytVideos = String(post.content).match(youtubeVideoRegex); + if (ytVideos) { + for (let ytVideo of ytVideos.slice(0, 4)) { + // If the video is an embed, replace the embed to make it watchable + if (ytVideo.includes("embed")) ytVideo = ytVideo.replace("embed/", "watch?v="); + postContent += `\nEmbeded Video from Post: [YouTube](${ytVideo})`; + } + } + log.DEBUG("Post content: ", postContent); + + const postId = post.postId; + if (!post.pubDate) post.pubDate = Date.now(); + const postPubDate = new Date(post.pubDate).toISOString(); + + const postSourceLink = source.title; + let postImage = post.image ?? undefined; + + if (!postImage) { + if (post.content) { + const linksInPost = post.content.querySelectorAll("a"); + if (linksInPost) { + log.DEBUG("Found links in post:", linksInPost); + for (const link of linksInPost) { + // Check to see if this link is a youtube video that was already found, if so skip it + if (ytVideos?.includes(link)) continue; + const images = String(link.getAttribute("href")).match(imageRegex); + log.DEBUG("Images found in post:", images); + if (images) { + postImage = images[0]; + } + } + } + } + } + + log.DEBUG("Sending an RSS post to discord", postTitle, postId, postContent); + try { + const rssMessage = new EmmeliaEmbedBuilder() + .setColor(0x0099FF) + .setTitle(postTitle) + .setURL(postLink) + .addFields({ name: 'Source', value: postSourceLink, inline: true }) + .addFields({ name: 'Published', value: postPubDate, inline: true }); + + // TODO - If there is more than one image, create a canvas and post the created canvas + if (postImage) { + log.DEBUG("Image from post:", postImage); + rssMessage.setImage(postImage); + } + + // Add the main content if it's present + postContent = postContent.slice(0, 4090).trim(); + if (postContent) rssMessage.setDescription(postContent); + + const channelResponse = rssMessage; + + //const channelResponse = channel.send({ embeds: [rssMessage] }); + + log.DEBUG("Channel send response", channelResponse); + + return channelResponse; + } catch (err) { + log.ERROR("Error sending message: ", postTitle, postId, postContent, postPubDate, err); + return err; + } +}; diff --git a/modules/mongo-wrappers/mongoFeedsWrappers.mjs b/modules/mongo-wrappers/mongoFeedsWrappers.mjs new file mode 100644 index 0000000..4238ed1 --- /dev/null +++ b/modules/mongo-wrappers/mongoFeedsWrappers.mjs @@ -0,0 +1,88 @@ +import { + insertDocument, + getDocuments, + getDocumentByField, + updateDocumentByField, + deleteDocumentByField, + } from "./mongoHandler.mjs"; + + const feedCollectionName = 'feeds'; + const postCollectionName = 'posts'; + + // Wrapper for inserting a feed + export const createFeed = async (feed) => { + try { + const insertedId = await insertDocument(feedCollectionName, feed); + return insertedId; + } catch (error) { + console.error('Error creating feed:', error); + throw error; + } + }; + + // Wrapper for retrieving all feeds + export const getAllFeeds = async () => { + try { + const feeds = await getDocuments(feedCollectionName); + return feeds; + } catch (error) { + console.error('Error getting all feeds:', error); + throw error; + } + }; + + // Wrapper for retrieving a feed by link + export const getFeedByLink = async (link) => { + try { + const feed = await getDocumentByField(feedCollectionName, 'link', link); + return feed; + } catch (error) { + console.error('Error getting feed by link:', error); + throw error; + } + }; + + // Wrapper for updating a feed by link + export const updateFeedByLink = async (link, updatedFields) => { + try { + const modifiedCount = await updateDocumentByField(feedCollectionName, 'link', link, updatedFields); + return modifiedCount; + } catch (error) { + console.error('Error updating feed by link:', error); + throw error; + } + }; + + // Wrapper for deleting a feed by link + export const deleteFeedByLink = async (link) => { + try { + const deletedCount = await deleteDocumentByField(feedCollectionName, 'link', link); + return deletedCount; + } catch (error) { + console.error('Error deleting feed by link:', error); + throw error; + } + }; + + // Wrapper for inserting a post + export const createPost = async (post) => { + try { + const insertedId = await insertDocument(postCollectionName, post); + return insertedId; + } catch (error) { + console.error('Error creating post:', error); + throw error; + } + }; + + // Wrapper for retrieving a post by postId + export const getPostByPostId = async (postId) => { + try { + const post = await getDocumentByField(postCollectionName, 'postId', postId); + return post; + } catch (error) { + console.error('Error getting post by postId:', error); + throw error; + } + }; + \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index f8609bb..f2dee62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,10 @@ "express": "^4.18.2", "mongodb": "^6.3.0", "morgan": "^1.10.0", - "socket.io": "^4.7.2" + "node-html-parser": "^6.1.13", + "rss-parser": "^3.13.0", + "socket.io": "^4.7.2", + "user-agents": "^1.1.208" }, "devDependencies": { "chai": "^5.1.0", @@ -385,6 +388,11 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -606,6 +614,32 @@ "node": ">= 0.10" } }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -723,6 +757,57 @@ } } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dotenv": { "version": "16.3.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", @@ -846,6 +931,17 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/escalade": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", @@ -1155,7 +1251,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "dev": true, "bin": { "he": "bin/he" } @@ -1313,6 +1408,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" + }, "node_modules/lodash.snakecase": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz", @@ -1573,6 +1673,15 @@ "node": ">= 0.6" } }, + "node_modules/node-html-parser": { + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.13.tgz", + "integrity": "sha512-qIsTMOY4C/dAa5Q5vsobRpOOvPfC4pB61UVW2uSwZNUp0QU/jCekTal1vMmbO0DgdHeLUJpv/ARmDqErVxA3Sg==", + "dependencies": { + "css-select": "^5.1.0", + "he": "1.2.0" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -1582,6 +1691,17 @@ "node": ">=0.10.0" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -1785,6 +1905,23 @@ "node": ">=0.10.0" } }, + "node_modules/rss-parser": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.13.0.tgz", + "integrity": "sha512-7jWUBV5yGN3rqMMj7CZufl/291QAhvrrGpDNE4k/02ZchL0npisiYYqULF71jCEKoIiHvK/Q2e6IkDwPziT7+w==", + "dependencies": { + "entities": "^2.0.3", + "xml2js": "^0.5.0" + } + }, + "node_modules/rss-parser/node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1809,6 +1946,11 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/sax": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz", + "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==" + }, "node_modules/send": { "version": "0.18.0", "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", @@ -2155,6 +2297,14 @@ "node": ">= 0.8" } }, + "node_modules/user-agents": { + "version": "1.1.208", + "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.1.208.tgz", + "integrity": "sha512-OMDd2qJF3g9HVEzMGv9Zi8Fp9hF2YqfhDZRacbn/x4YknW2YGdhqrLW8MCiWgUEDB3eDLf7IvuzXGoOwWhnaBw==", + "dependencies": { + "lodash.clonedeep": "^4.5.0" + } + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -2240,6 +2390,26 @@ } } }, + "node_modules/xml2js": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", + "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/xmlhttprequest-ssl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz", diff --git a/package.json b/package.json index bb85c79..1ab0952 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,9 @@ "express": "^4.18.2", "mongodb": "^6.3.0", "morgan": "^1.10.0", - "socket.io": "^4.7.2" + "node-html-parser": "^6.1.13", + "rss-parser": "^3.13.0", + "socket.io": "^4.7.2", + "user-agents": "^1.1.208" } } diff --git a/rss-manager/feedHandler.mjs b/rss-manager/feedHandler.mjs new file mode 100644 index 0000000..babcdd1 --- /dev/null +++ b/rss-manager/feedHandler.mjs @@ -0,0 +1,156 @@ +import { createFeed, getAllFeeds, getFeedByLink, updateFeedByLink, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs'; +import crypto from 'crypto'; +import { sendPost } from '../discordBot/modules/rssWrappers.mjs'; +import { DebugBuilder } from "../modules/debugger.mjs"; +import UserAgent from "user-agents"; + +process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString(); + +// Initiate the parser +import Parser from 'rss-parser'; +const parser = new Parser({ + headers: { + 'User-Agent': process.env.USER_AGENT_STRING, + "Accept": "application/rss+xml,application/xhtml+xml,application/xml" + } +}); + +const log = new DebugBuilder("server", "feedHandler"); +const runningPostsToRemove = {}; // Assuming this is a global state variable +const sourceFailureLimit = 5; // Define your source failure limit here + +export const returnHash = (...stringsIncluded) => { + return crypto.createHash('sha1').update(`${stringsIncluded.join("-<>-")}`).digest("base64"); +}; + +export const updateFeeds = async (client) => { + if (!client) throw new Error("Client object not passed"); + + try { + const records = await getAllFeeds(); + + const sourcePromiseArray = records.map(async (source) => { + log.DEBUG('Record title:', source.title); + log.DEBUG('Record link:', source.link); + log.DEBUG('Record category:', source.category); + log.DEBUG('Record guild ID:', source.guild_id); + log.DEBUG('Record channel ID:', source.channel_id); + + try { + const parsedFeed = await parser.parseURL(source.link); + + if (parsedFeed?.items) { + await Promise.all(parsedFeed.items.reverse().map(async (post) => { + log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title); + log.VERBOSE("Post from feed:", post); + + if (!post.title || !post.link) throw new Error("Missing information from the post"); + if (!post.content || !post['content:encoded']) log.WARN("There is no content for post:", post.title); + + post.postId = post.postId ?? post.guid ?? post.id ?? returnHash(post.title, post.link, post.pubDate ?? Date.now()); + + const existingRecord = await getPostByPostId(post.postId); + log.DEBUG("Existing post record:", existingRecord); + + if (!existingRecord) { + const channel = client.channels.cache.get(source.channel_id); + const sendResults = await sendPost(post, source, channel); + + if (!sendResults) throw new Error("No sending results from sending a post"); + + log.DEBUG("Saving post to database:", sendResults, post.title, source.channel_id); + //await createPost(post); + log.DEBUG("Saved post:", post); + } + })); + } else { + await deleteFeedByLink(source.link); + } + } catch (err) { + log.ERROR("Parser Error:", source, err); + await removeSource(source.link); + throw err; + } + }); + + await Promise.all(sourcePromiseArray); + log.DEBUG("All sources finished"); + } catch (error) { + log.ERROR("Error updating feeds:", error); + throw error; + } +}; + +/** + * Adds or updates new source URL to configured storage. + * @param {string} title - Title/Name of the RSS feed. + * @param {string} link - URL of RSS feed. + * @param {string} category - Category of RSS feed. + * @param {string} guildId - Guild ID of RSS feed. + * @param {string} channelId - Channel ID of RSS feed. + * @param {function} callback - Callback function. + */ +export const addSource = async (title, link, category, guildId, channelId, callback) => { + try { + const feed = { + title, + link, + category, + guild_id: guildId, + channel_id: channelId + }; + const record = await createFeed(feed); + log.DEBUG("Record ID:", record); + return callback(null, record); + } catch (err) { + log.ERROR("Error in create:", err); + return callback(err, null); + } +}; + +/** + * Wrapper for feeds that cause errors. By default it will wait over a day for the source to come back online before deleting it. + * @param {string} sourceURL - The URL of the feed source causing issues. + */ +export const removeSource = async (sourceURL) => { + log.INFO("Removing source URL:", sourceURL); + + if (!runningPostsToRemove[sourceURL]) { + runningPostsToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 }; + return; + } + + const backoffDateTimeDifference = (Date.now() - runningPostsToRemove[sourceURL].timestamp); + const backoffWaitTime = (runningPostsToRemove[sourceURL].count * 30000); + + log.DEBUG("Datetime", runningPostsToRemove[sourceURL], backoffDateTimeDifference, backoffWaitTime); + + if (backoffDateTimeDifference <= backoffWaitTime) { + runningPostsToRemove[sourceURL].ignoredAttempts += 1; + return; + } + + if (runningPostsToRemove[sourceURL].count < sourceFailureLimit) { + runningPostsToRemove[sourceURL].count += 1; + runningPostsToRemove[sourceURL].timestamp = Date.now(); + return; + } + + try { + const record = await getFeedByLink(sourceURL); + if (!record) { + log.ERROR("No source returned from feedStorage"); + return; + } + + const results = await deleteFeedByLink(sourceURL); + if (!results) { + log.WARN("No results from remove entry"); + return; + } + + log.DEBUG("Source exceeded the limit of retries and has been removed", sourceURL); + } catch (err) { + log.ERROR("Error removing source from feedStorage", err); + } +}; diff --git a/rss-manager/rssController.mjs b/rss-manager/rssController.mjs new file mode 100644 index 0000000..e99d22c --- /dev/null +++ b/rss-manager/rssController.mjs @@ -0,0 +1,37 @@ +//Will handle updating feeds in all channels + +import { DebugBuilder } from "../modules/debugger.mjs"; +import { updateFeeds } from "./feedHandler.mjs"; +import dotenv from 'dotenv'; +dotenv.config() + +const log = new DebugBuilder("server", "rssController"); + +const refreshInterval = process.env.RSS_REFRESH_INTERVAL ?? 300000; + +export class RSSController { + constructor(client) { + this.client = client; + } + + async start() { + // Wait for the refresh period before starting RSS feeds, so the rest of the bot can start + await new Promise(resolve => setTimeout(resolve, refreshInterval)); + + log.INFO("Starting RSS Controller"); + // Get initial feeds before starting the infinite loop + await updateFeeds(this.client); + + while(true) { + // Wait for the refresh interval, then wait for the posts to return, then wait a quarter of the refresh interval to make sure everything is cleared up + await new Promise(resolve => setTimeout(resolve, refreshInterval)); + await this.collectLatestPosts(); + await new Promise(resolve => setTimeout(resolve, refreshInterval / 4)); + } + } + + async collectLatestPosts() { + log.INFO("Updating sources"); + await updateFeeds(this.client); + } +} From 429ddd333331dcbe27812ce8217411d170c15b7d Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Wed, 22 May 2024 02:15:56 -0400 Subject: [PATCH 3/9] Code improvements - Better error handling - More output - Better logic --- discordBot/modules/rssWrappers.mjs | 82 ++++++++++--------------- rss-manager/feedHandler.mjs | 99 +++++++++++++----------------- 2 files changed, 75 insertions(+), 106 deletions(-) diff --git a/discordBot/modules/rssWrappers.mjs b/discordBot/modules/rssWrappers.mjs index 0f3e441..9b769e0 100644 --- a/discordBot/modules/rssWrappers.mjs +++ b/discordBot/modules/rssWrappers.mjs @@ -22,59 +22,49 @@ export class EmmeliaEmbedBuilder extends EmbedBuilder { export const sendPost = (post, source, channel) => { log.DEBUG("Sending post from source: ", post, source); + const postTitle = String(post.title).substring(0, 150); const postLink = post.link; - let postContent; - - if (post.content) { - // Reset the content parameter with the encoded parameter - post.content = parse(post['content:encoded'] ?? post.content); - // Get the post content and trim it to length or add a placeholder if necessary - let postText = String(post.content.text); - if (postText.length >= 3800) postText = `${postText.slice(0, 3800).substring(0, Math.min(postText.length, postText.lastIndexOf(" ")))} [...](${post.link})`; - else if (postText.length === 0) postText = `*This post has no content* [Direct Link](${post.link})`; + let postContent = `*This post has no content* [Direct Link](${post.link})`; + + if (post.content || post['content:encoded']) { + const content = post['content:encoded'] ?? post.content; + const parsedContent = parse(content); + let postText = parsedContent.text.trim(); + + if (postText.length >= 3800) { + postText = `${postText.slice(0, 3800).substring(0, postText.lastIndexOf(" "))} [...](${post.link})`; + } else if (postText.length === 0) { + postText = `*This post has no content* [Direct Link](${post.link})`; + } postContent = postText; - } else { - postContent = `*This post has no content* [Direct Link](${post.link})`; - } - // Check for embedded youtube videos and add the first four as links - const ytVideos = String(post.content).match(youtubeVideoRegex); - if (ytVideos) { - for (let ytVideo of ytVideos.slice(0, 4)) { - // If the video is an embed, replace the embed to make it watchable - if (ytVideo.includes("embed")) ytVideo = ytVideo.replace("embed/", "watch?v="); - postContent += `\nEmbeded Video from Post: [YouTube](${ytVideo})`; + // Check for embedded YouTube videos and add the first four as links + const ytVideos = content.match(youtubeVideoRegex); + if (ytVideos) { + ytVideos.slice(0, 4).forEach((ytVideo) => { + if (ytVideo.includes("embed")) ytVideo = ytVideo.replace("embed/", "watch?v="); + postContent += `\nEmbedded Video from Post: [YouTube](${ytVideo})`; + }); + } + + // Extract the first image link if available + const imageLinks = parsedContent.querySelectorAll("a") + .map(link => link.getAttribute("href")) + .filter(href => href && href.match(imageRegex)); + + if (imageLinks.length > 0) { + post.image = imageLinks[0]; } } - log.DEBUG("Post content: ", postContent); const postId = post.postId; - if (!post.pubDate) post.pubDate = Date.now(); - const postPubDate = new Date(post.pubDate).toISOString(); - + const postPubDate = new Date(post.pubDate || Date.now()).toISOString(); const postSourceLink = source.title; - let postImage = post.image ?? undefined; + const postImage = post.image; - if (!postImage) { - if (post.content) { - const linksInPost = post.content.querySelectorAll("a"); - if (linksInPost) { - log.DEBUG("Found links in post:", linksInPost); - for (const link of linksInPost) { - // Check to see if this link is a youtube video that was already found, if so skip it - if (ytVideos?.includes(link)) continue; - const images = String(link.getAttribute("href")).match(imageRegex); - log.DEBUG("Images found in post:", images); - if (images) { - postImage = images[0]; - } - } - } - } - } + log.DEBUG("Post content: ", postContent); - log.DEBUG("Sending an RSS post to discord", postTitle, postId, postContent); try { const rssMessage = new EmmeliaEmbedBuilder() .setColor(0x0099FF) @@ -83,20 +73,16 @@ export const sendPost = (post, source, channel) => { .addFields({ name: 'Source', value: postSourceLink, inline: true }) .addFields({ name: 'Published', value: postPubDate, inline: true }); - // TODO - If there is more than one image, create a canvas and post the created canvas if (postImage) { log.DEBUG("Image from post:", postImage); rssMessage.setImage(postImage); } - // Add the main content if it's present postContent = postContent.slice(0, 4090).trim(); if (postContent) rssMessage.setDescription(postContent); - const channelResponse = rssMessage; - - //const channelResponse = channel.send({ embeds: [rssMessage] }); - + const channelResponse = channel.send({ embeds: [rssMessage] }); + log.DEBUG("Channel send response", channelResponse); return channelResponse; diff --git a/rss-manager/feedHandler.mjs b/rss-manager/feedHandler.mjs index babcdd1..aae743b 100644 --- a/rss-manager/feedHandler.mjs +++ b/rss-manager/feedHandler.mjs @@ -3,11 +3,11 @@ import crypto from 'crypto'; import { sendPost } from '../discordBot/modules/rssWrappers.mjs'; import { DebugBuilder } from "../modules/debugger.mjs"; import UserAgent from "user-agents"; +import Parser from 'rss-parser'; +// Initialize the User-Agent string process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString(); -// Initiate the parser -import Parser from 'rss-parser'; const parser = new Parser({ headers: { 'User-Agent': process.env.USER_AGENT_STRING, @@ -16,11 +16,11 @@ const parser = new Parser({ }); const log = new DebugBuilder("server", "feedHandler"); -const runningPostsToRemove = {}; // Assuming this is a global state variable -const sourceFailureLimit = 5; // Define your source failure limit here +const runningPostsToRemove = {}; +const sourceFailureLimit = 5; export const returnHash = (...stringsIncluded) => { - return crypto.createHash('sha1').update(`${stringsIncluded.join("-<>-")}`).digest("base64"); + return crypto.createHash('sha1').update(stringsIncluded.join("-<>-")).digest("base64"); }; export const updateFeeds = async (client) => { @@ -30,102 +30,85 @@ export const updateFeeds = async (client) => { const records = await getAllFeeds(); const sourcePromiseArray = records.map(async (source) => { - log.DEBUG('Record title:', source.title); - log.DEBUG('Record link:', source.link); - log.DEBUG('Record category:', source.category); - log.DEBUG('Record guild ID:', source.guild_id); - log.DEBUG('Record channel ID:', source.channel_id); + log.DEBUG('Processing source:', source.title); try { const parsedFeed = await parser.parseURL(source.link); if (parsedFeed?.items) { await Promise.all(parsedFeed.items.reverse().map(async (post) => { - log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title); - log.VERBOSE("Post from feed:", post); + log.DEBUG("Processing post:", post.title); - if (!post.title || !post.link) throw new Error("Missing information from the post"); - if (!post.content || !post['content:encoded']) log.WARN("There is no content for post:", post.title); + if (!post.title || !post.link) throw new Error("Missing title or link in the post"); + if (!post.content && !post['content:encoded']) log.WARN("No content for post:", post.title); post.postId = post.postId ?? post.guid ?? post.id ?? returnHash(post.title, post.link, post.pubDate ?? Date.now()); const existingRecord = await getPostByPostId(post.postId); - log.DEBUG("Existing post record:", existingRecord); - if (!existingRecord) { - const channel = client.channels.cache.get(source.channel_id); + const channel = client.channels.cache.get(source.channel_id); const sendResults = await sendPost(post, source, channel); + if (!sendResults) throw new Error("Failed to send post"); - if (!sendResults) throw new Error("No sending results from sending a post"); + log.DEBUG("Saving post to database:", post.title, source.channel_id); - log.DEBUG("Saving post to database:", sendResults, post.title, source.channel_id); - //await createPost(post); - log.DEBUG("Saved post:", post); + const postToSave = { + title: post.title, + link: post.link, + pubDate: post.pubDate, + author: post.author, + contentSnippet: post.contentSnippet, + id: post.id, + isoDate: post.isoDate, + postId: post.postId + }; + + await createPost(postToSave); + log.DEBUG("Post saved:", postToSave); } })); } else { await deleteFeedByLink(source.link); } } catch (err) { - log.ERROR("Parser Error:", source, err); + log.ERROR("Error processing source:", source.title, err); await removeSource(source.link); throw err; } }); await Promise.all(sourcePromiseArray); - log.DEBUG("All sources finished"); + log.DEBUG("All sources processed"); } catch (error) { log.ERROR("Error updating feeds:", error); throw error; } }; -/** - * Adds or updates new source URL to configured storage. - * @param {string} title - Title/Name of the RSS feed. - * @param {string} link - URL of RSS feed. - * @param {string} category - Category of RSS feed. - * @param {string} guildId - Guild ID of RSS feed. - * @param {string} channelId - Channel ID of RSS feed. - * @param {function} callback - Callback function. - */ export const addSource = async (title, link, category, guildId, channelId, callback) => { try { - const feed = { - title, - link, - category, - guild_id: guildId, - channel_id: channelId - }; + const feed = { title, link, category, guild_id: guildId, channel_id: channelId }; const record = await createFeed(feed); - log.DEBUG("Record ID:", record); - return callback(null, record); + log.DEBUG("Source added:", record); + callback(null, record); } catch (err) { - log.ERROR("Error in create:", err); - return callback(err, null); + log.ERROR("Error adding source:", err); + callback(err, null); } }; -/** - * Wrapper for feeds that cause errors. By default it will wait over a day for the source to come back online before deleting it. - * @param {string} sourceURL - The URL of the feed source causing issues. - */ export const removeSource = async (sourceURL) => { - log.INFO("Removing source URL:", sourceURL); - + log.INFO("Removing source:", sourceURL); + if (!runningPostsToRemove[sourceURL]) { runningPostsToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 }; return; } - const backoffDateTimeDifference = (Date.now() - runningPostsToRemove[sourceURL].timestamp); - const backoffWaitTime = (runningPostsToRemove[sourceURL].count * 30000); + const elapsedTime = Date.now() - runningPostsToRemove[sourceURL].timestamp; + const waitTime = runningPostsToRemove[sourceURL].count * 30000; - log.DEBUG("Datetime", runningPostsToRemove[sourceURL], backoffDateTimeDifference, backoffWaitTime); - - if (backoffDateTimeDifference <= backoffWaitTime) { + if (elapsedTime <= waitTime) { runningPostsToRemove[sourceURL].ignoredAttempts += 1; return; } @@ -139,18 +122,18 @@ export const removeSource = async (sourceURL) => { try { const record = await getFeedByLink(sourceURL); if (!record) { - log.ERROR("No source returned from feedStorage"); + log.ERROR("Source not found in storage"); return; } const results = await deleteFeedByLink(sourceURL); if (!results) { - log.WARN("No results from remove entry"); + log.WARN("Failed to remove source"); return; } - log.DEBUG("Source exceeded the limit of retries and has been removed", sourceURL); + log.DEBUG("Source removed after exceeding failure limit:", sourceURL); } catch (err) { - log.ERROR("Error removing source from feedStorage", err); + log.ERROR("Error removing source from storage:", err); } }; From 4831bb817c6885ac4717050dce4d0aab2d1f8401 Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Wed, 22 May 2024 02:20:52 -0400 Subject: [PATCH 4/9] #5 Improved logging on mongo handler core --- modules/mongo-wrappers/mongoHandler.mjs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/mongo-wrappers/mongoHandler.mjs b/modules/mongo-wrappers/mongoHandler.mjs index 359d16d..a22ba04 100644 --- a/modules/mongo-wrappers/mongoHandler.mjs +++ b/modules/mongo-wrappers/mongoHandler.mjs @@ -1,5 +1,7 @@ // Import necessary modules import { MongoClient } from 'mongodb'; +import { DebugBuilder } from '../debugger.mjs'; +const log = new DebugBuilder("server", 'mongoHandler'); import dotenv from 'dotenv'; dotenv.config() @@ -21,10 +23,11 @@ export const connectToDatabase = async () => { // Function to insert a document into the collection export const insertDocument = async (collectionName, document) => { const db = await connectToDatabase(); + log.DEBUG("Inserting document:", collectionName, document); try { const collection = db.db().collection(collectionName); const result = await collection.insertOne(document); - console.log('Document inserted:', result.insertedId); + log.DEBUG('Document inserted:', result.insertedId); return result.insertedId; } catch (error) { console.error('Error inserting document:', error); @@ -37,11 +40,12 @@ export const insertDocument = async (collectionName, document) => { // Function to retrieve documents from the collection export const getDocuments = async (collectionName) => { + log.DEBUG("Getting all documents:", collectionName); const db = await connectToDatabase(); try { const collection = db.db().collection(collectionName); const documents = await collection.find({}).toArray(); - console.log('Documents retrieved:', documents); + log.DEBUG('Documents retrieved:', documents); return documents; } catch (error) { console.error('Error retrieving documents:', error); @@ -54,6 +58,7 @@ export const getDocuments = async (collectionName) => { // Function to retrieve a document by a specific field export const getDocumentByField = async (collectionName, field, value) => { + log.DEBUG("Getting document by field:", collectionName, field, value); const db = await connectToDatabase(); try { const collection = db.db().collection(collectionName); @@ -69,11 +74,12 @@ export const getDocumentByField = async (collectionName, field, value) => { // Function to update a document by a specific field export const updateDocumentByField = async (collectionName, field, value, updatedFields) => { + log.DEBUG("Update document by field:", collectionName, field, value, updatedFields); const db = await connectToDatabase(); try { const collection = db.db().collection(collectionName); const result = await collection.updateOne({ [field]: value }, { $set: updatedFields }); - console.log('Document updated:', result.modifiedCount); + log.DEBUG('Document updated:', result.modifiedCount); return result.modifiedCount; } catch (error) { console.error('Error updating document:', error); @@ -85,11 +91,12 @@ export const updateDocumentByField = async (collectionName, field, value, update // Function to delete a document by a specific field export const deleteDocumentByField = async (collectionName, field, value) => { + log.DEBUG("Delete document by field:", collectionName, field, value); const db = await connectToDatabase(); try { const collection = db.db().collection(collectionName); const result = await collection.deleteOne({ [field]: value }); - console.log('Document deleted:', result.deletedCount); + log.DEBUG('Document deleted:', result.deletedCount); return result.deletedCount; } catch (error) { console.error('Error deleting document:', error); From 81a215f048c9976aa18c0e838b54bd417b53b430 Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Wed, 22 May 2024 02:44:56 -0400 Subject: [PATCH 5/9] Implement auto-building on push to main --- .gitea/workflows/DRBv3_server_build.yaml | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .gitea/workflows/DRBv3_server_build.yaml diff --git a/.gitea/workflows/DRBv3_server_build.yaml b/.gitea/workflows/DRBv3_server_build.yaml new file mode 100644 index 0000000..9f00202 --- /dev/null +++ b/.gitea/workflows/DRBv3_server_build.yaml @@ -0,0 +1,56 @@ +name: release-tag + +on: + push: + branches: + - main + +jobs: + release-image: + runs-on: ubuntu-latest + container: + image: catthehacker/ubuntu:act-latest + env: + DOCKER_ORG: teacup + DOCKER_LATEST: nightly + RUNNER_TOOL_CACHE: /toolcache + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker BuildX + uses: docker/setup-buildx-action@v2 + with: # replace it with your local IP + config-inline: | + [registry."${{ secrets.LOCAL_GITEA_IP}}:3000"] + http = true + insecure = true + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + registry: ${{ secrets.LOCAL_GITEA_IP}}:3000 # replace it with your local IP + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Get Meta + id: meta + run: | + echo REPO_NAME=$(echo ${GITHUB_REPOSITORY} | awk -F"/" '{print $2}') >> $GITHUB_OUTPUT + echo REPO_VERSION=$(git describe --tags --always | sed 's/^v//') >> $GITHUB_OUTPUT + + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + file: ./Dockerfile + platforms: | + linux/amd64 + linux/arm64 + push: true + tags: | # replace it with your local IP and tags + ${{ secrets.LOCAL_GITEA_IP}}:3000/${{ env.DOCKER_ORG }}/${{ steps.meta.outputs.REPO_NAME }}:${{ steps.meta.outputs.REPO_VERSION }} + ${{ secrets.LOCAL_GITEA_IP}}:3000/${{ env.DOCKER_ORG }}/${{ steps.meta.outputs.REPO_NAME }}:${{ env.DOCKER_LATEST }} \ No newline at end of file From 2ab5a181bd58d80ab80d62e700f7e4746fb4c6ab Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sat, 25 May 2024 23:52:18 -0400 Subject: [PATCH 6/9] #5 replace all `console.logs` with debugger --- addons/example/index.js | 6 ++- discordBot/commands/add.mjs | 4 +- discordBot/commands/join.mjs | 18 ++++--- discordBot/commands/leave.mjs | 8 +-- discordBot/commands/ping.mjs | 6 ++- discordBot/commands/triggerRss.mjs | 7 +-- discordBot/commands/update.mjs | 4 +- discordBot/discordBot.mjs | 29 +++++++---- discordBot/events/interactionCreate.mjs | 8 +-- discordBot/modules/registerCommands.mjs | 24 ++++----- discordBot/modules/rssWrappers.mjs | 2 +- discordBot/modules/wrappers.mjs | 8 +-- modules/addonManager.mjs | 6 ++- .../mongo-wrappers/mongoDiscordIDWrappers.mjs | 16 +++--- modules/mongo-wrappers/mongoFeedsWrappers.mjs | 16 +++--- modules/mongo-wrappers/mongoNodesWrappers.mjs | 16 +++--- .../mongo-wrappers/mongoSystemsWrappers.mjs | 18 ++++--- modules/socketServer.mjs | 4 +- modules/socketServerWrappers.mjs | 49 ++++++++++--------- rss-manager/feedHandler.mjs | 24 +++++---- rss-manager/rssController.mjs | 48 +++++++++++------- server.js | 4 +- 22 files changed, 192 insertions(+), 133 deletions(-) diff --git a/addons/example/index.js b/addons/example/index.js index 2ba6e20..384e9ad 100644 --- a/addons/example/index.js +++ b/addons/example/index.js @@ -1,8 +1,10 @@ // addons/addon1/index.js +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "server"); // Function called by the main application to initialize the addon export function initialize(nodeIo, config) { - console.log(`Initializing ${config.name}`); + log.INFO(`Initializing ${config.name}`); // Call other functions within the addon module registerSocketEvents(nodeIo, config); @@ -12,6 +14,6 @@ export function initialize(nodeIo, config) { // Function to register Socket.IO event handlers function registerSocketEvents(nodeIo, config) { nodeIo.on(config.options.eventName, (data) => { - console.log(`Received event "${config.options.eventName}" from client:`, data); + log.DEBUG(`Received event "${config.options.eventName}" from client:`, data); }); } diff --git a/discordBot/commands/add.mjs b/discordBot/commands/add.mjs index 0259c85..abe7107 100644 --- a/discordBot/commands/add.mjs +++ b/discordBot/commands/add.mjs @@ -2,7 +2,7 @@ import { SlashCommandBuilder } from 'discord.js'; import { DebugBuilder } from "../../modules/debugger.mjs"; import { addSource } from '../../rss-manager/feedHandler.mjs' -const log = new DebugBuilder("server", "add"); +const log = new DebugBuilder("server", "discordBot.command.add"); // Exporting data property that contains the command structure for discord including any params export const data = new SlashCommandBuilder() @@ -37,7 +37,7 @@ export async function autocomplete(nodeIo, interaction) { const focusedValue = interaction.options.getFocused(); const choices = []; const filtered = choices.filter(choice => choice.name.startsWith(focusedValue)); - console.log(focusedValue, choices, filtered); + log.DEBUG(focusedValue, choices, filtered); await interaction.respond(filtered); } */ diff --git a/discordBot/commands/join.mjs b/discordBot/commands/join.mjs index 0942a78..a7784eb 100644 --- a/discordBot/commands/join.mjs +++ b/discordBot/commands/join.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "discordBot.command.join"); import { SlashCommandBuilder, ActionRowBuilder, ButtonBuilder, ButtonStyle } from 'discord.js'; import { requestNodeJoinSystem, checkIfNodeIsConnectedToVC, checkIfNodeHasOpenDiscordClient, getNodeCurrentListeningSystem } from '../../modules/socketServerWrappers.mjs'; import { getSystemsByNuid, getAllSystems, getSystemByName } from '../../modules/mongo-wrappers/mongoSystemsWrappers.mjs'; @@ -27,7 +29,7 @@ export async function autocomplete(nodeIo, interaction) { const choices = await getAllSystems(); const filtered = choices.filter(choice => choice.name.startsWith(focusedValue)); - console.log(focusedValue, choices, filtered); + log.DEBUG(focusedValue, choices, filtered); await interaction.respond( filtered.map(choice => ({ name: choice.name, value: choice.name })), @@ -44,7 +46,7 @@ export async function execute(nodeIo, interaction) { if (!interaction.member.voice.channel) { return await interaction.editReply({ content: `<@${interaction.member.id}>, you need to enter a voice channel before you use this command`, ephemeral: true }) } // Grab the channel if the user is connected to VC const channelToJoin = interaction.member.voice.channel; - console.log(`The user '${interaction.member.id}' is in the voice channel '${channelToJoin}'`); + log.INFO(`The user '${interaction.member.id}' is in the voice channel '${channelToJoin}'`); // Get the selected system option from the command interaction const selectedSystem = interaction.options.getString('system'); @@ -58,11 +60,11 @@ export async function execute(nodeIo, interaction) { const openSocket = await nodeIo.sockets.sockets.get(selectedNodeSocketId); // Get the open ID for this connection\ const discordTokens = await getAvailableTokensInGuild(nodeIo, interaction.guild.id); - console.log("Available discord tokens: ", discordTokens); + log.DEBUG("Available discord tokens: ", discordTokens); if (discordTokens.length >= 1) { // TODO - Implement a method to have preferred tokens (bot users) for specific systems - console.log("Joining selected open socket:", selectedNodeSocketId, system.name, channelToJoin.id, openSocket.node.name, discordTokens[0].token); + log.INFO("Joining selected open socket:", selectedNodeSocketId, system.name, channelToJoin.id, openSocket.node.name, discordTokens[0].token); // Ask the node to join the selected channel and system await requestNodeJoinSystem(openSocket, system.name, channelToJoin.id, discordTokens[0].token); @@ -74,7 +76,7 @@ export async function execute(nodeIo, interaction) { // Get all open socket nodes const openSockets = [...await nodeIo.allSockets()]; // TODO - Filter the returned nodes to only nodes that have the radio capability - console.log("All open sockets: ", openSockets); + log.DEBUG("All open sockets: ", openSockets); var availableNodes = []; // Check each open socket to see if the node has the requested system @@ -85,14 +87,14 @@ export async function execute(nodeIo, interaction) { if (hasOpenClient) { let currentSystem = await getNodeCurrentListeningSystem(openSocket); if (currentSystem != system.name) { - console.log("Node is listening to a different system than requested", openSocket.node.name); + log.INFO("Node is listening to a different system than requested", openSocket.node.name); return; } } // Check if the bot has an open voice connection in the requested server already const connected = await checkIfNodeIsConnectedToVC(nodeIo, interaction.guild.id, openSocket.node.nuid); - console.log("Connected:", connected); + log.INFO("Connected:", connected); if (!connected) { // Check if this node has the requested system, if so add it to the availble array if (system.nodes.includes(openSocket.node.nuid)) { @@ -102,7 +104,7 @@ export async function execute(nodeIo, interaction) { })); - console.log("Availble nodes:", availableNodes.map(socket => socket.node.name)); + log.DEBUG("Availble nodes:", availableNodes.map(socket => socket.node.name)); // If there are no available nodes, let the user know there are none available if (availableNodes.length == 0) { diff --git a/discordBot/commands/leave.mjs b/discordBot/commands/leave.mjs index 2539175..a993e13 100644 --- a/discordBot/commands/leave.mjs +++ b/discordBot/commands/leave.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "discordBot.command.leave"); import { SlashCommandBuilder } from 'discord.js'; import { requestBotLeaveServer, getSocketIdByNuid } from '../../modules/socketServerWrappers.mjs'; import { checkOnlineBotsInGuild } from '../modules/wrappers.mjs' @@ -25,11 +27,11 @@ export async function autocomplete(nodeIo, interaction) { const focusedValue = interaction.options.getFocused(); const choices = (await checkOnlineBotsInGuild(nodeIo, interaction.guild.id)); - console.log(choices); + log.DEBUG(choices); const filtered = choices.filter(choice => choice.name.startsWith(focusedValue)).map(choice => choice = {name: choice.name, value: choice.nuid}); - console.log(focusedValue, choices, filtered); + log.DEBUG(focusedValue, choices, filtered); await interaction.respond(filtered); } @@ -44,7 +46,7 @@ export async function execute(nodeIo, interaction) { // Get the requested bot const selectedNode = interaction.options.getString('bot'); const socket = await getSocketIdByNuid(nodeIo, selectedNode); - console.log("All open sockets:", socket, selectedNode); + log.DEBUG("All open sockets:", socket, selectedNode); await requestBotLeaveServer(socket, interaction.guild.id); //await interaction.reply(`**Online Sockets: '${sockets}'**`); await interaction.editReply(`Ok <@${interaction.member.id}>, the bot is leaving shortly`); diff --git a/discordBot/commands/ping.mjs b/discordBot/commands/ping.mjs index 6b4cc2e..4ab5e80 100644 --- a/discordBot/commands/ping.mjs +++ b/discordBot/commands/ping.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "discordBot.command.ping"); import { SlashCommandBuilder } from 'discord.js'; // Exporting data property that contains the command structure for discord including any params @@ -19,7 +21,7 @@ export async function autocomplete(nodeIo, interaction) { const focusedValue = interaction.options.getFocused(); const choices = []; const filtered = choices.filter(choice => choice.name.startsWith(focusedValue)); - console.log(focusedValue, choices, filtered); + log.INFO(focusedValue, choices, filtered); await interaction.respond(filtered); } */ @@ -32,7 +34,7 @@ export async function autocomplete(nodeIo, interaction) { export const execute = async (nodeIo, interaction) => { try { const sockets = await nodeIo.allSockets(); - console.log("All open sockets: ",sockets); + log.DEBUG("All open sockets: ",sockets); //await interaction.reply(`**Online Sockets: '${sockets}'**`); await interaction.reply('**Pong.**'); //await interaction.channel.send('**Pong.**'); diff --git a/discordBot/commands/triggerRss.mjs b/discordBot/commands/triggerRss.mjs index f55b0be..1827575 100644 --- a/discordBot/commands/triggerRss.mjs +++ b/discordBot/commands/triggerRss.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "discordBot.command.triggerRss"); import { SlashCommandBuilder } from 'discord.js'; import { updateFeeds } from '../../rss-manager/feedHandler.mjs' @@ -20,7 +22,7 @@ export async function autocomplete(nodeIo, interaction) { const focusedValue = interaction.options.getFocused(); const choices = []; const filtered = choices.filter(choice => choice.name.startsWith(focusedValue)); - console.log(focusedValue, choices, filtered); + log.INFO(focusedValue, choices, filtered); await interaction.respond(filtered); } */ @@ -32,8 +34,7 @@ export async function autocomplete(nodeIo, interaction) { */ export const execute = async (nodeIo, interaction) => { try { - //const sockets = await nodeIo.allSockets(); - //console.log("All open sockets: ", sockets); + //const sockets = await nodeIo.allSockets(); //await interaction.reply(`**Online Sockets: '${sockets}'**`); await interaction.reply('Triggering RSS update'); await updateFeeds(interaction.client); diff --git a/discordBot/commands/update.mjs b/discordBot/commands/update.mjs index b9f4150..d5245d5 100644 --- a/discordBot/commands/update.mjs +++ b/discordBot/commands/update.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "discordBot.command.update"); import { SlashCommandBuilder } from 'discord.js'; import { requestNodeUpdate } from '../../modules/socketServerWrappers.mjs'; @@ -18,7 +20,7 @@ export const deferInitialReply = false; // If we the initial reply in discord sh export const execute = async (nodeIo, interaction) => { try { const openSockets = [...await nodeIo.allSockets()]; // TODO - Filter the returned nodes to only nodes that have the radio capability - console.log("All open sockets: ", openSockets); + log.DEBUG("All open sockets: ", openSockets); // Check each open socket to see if the node has the requested system await Promise.all(openSockets.map(openSocket => { diff --git a/discordBot/discordBot.mjs b/discordBot/discordBot.mjs index d09feeb..aea3d4e 100644 --- a/discordBot/discordBot.mjs +++ b/discordBot/discordBot.mjs @@ -1,14 +1,19 @@ +import { DebugBuilder } from "../modules/debugger.mjs"; import { Client, GatewayIntentBits, Collection } from 'discord.js'; import { registerActiveCommands, unregisterAllCommands } from './modules/registerCommands.mjs' +import { RSSController } from '../rss-manager/rssController.mjs' import { join, dirname } from 'path'; import { readdirSync } from 'fs'; import { fileURLToPath } from 'url'; +import dotenv from 'dotenv'; +dotenv.config() + +const log = new DebugBuilder("server", "discordBot"); + const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); -import dotenv from 'dotenv'; -dotenv.config() /** * Add the enabled commands to the bot to be used by users in discord @@ -25,18 +30,18 @@ export const addEnabledCommands = async (serverClient, _commandsPath = "./comman for (const file of commandFiles) { const filePath = await join(commandsPath, file); - console.log(`Adding enabled command: ${filePath}`); + log.INFO(`Adding enabled command: ${filePath}`); await import(`file://${filePath}`).then(command => { if (command.data instanceof Promise) { command.data.then(async (builder) => { command.data = builder; - console.log("Importing command: ", command.data.name, command); + log.DEBUG("Importing command: ", command.data.name, command); // Set a new item in the Collection // With the key as the command name and the value as the exported module serverClient.commands.set(command.data.name, command); }); } else { - console.log("Importing command: ", command.data.name, command); + log.DEBUG("Importing command: ", command.data.name, command); // Set a new item in the Collection // With the key as the command name and the value as the exported module serverClient.commands.set(command.data.name, command); @@ -61,9 +66,9 @@ export function addEnabledEventListeners(serverClient, _eventsPath = "./events") for (const file of eventFiles) { const filePath = join(eventsPath, file); - console.log(`Adding enabled event listener: ${filePath}`); + log.INFO(`Adding enabled event listener: ${filePath}`); import(`file://${filePath}`).then(event => { - console.log("Adding event: ", event); + log.DEBUG("Adding event: ", event); if (event.once) { serverClient.once(event.name, (...args) => event.execute(serverClient.nodeIo, ...args)); } else { @@ -78,15 +83,21 @@ export const serverClient = new Client({ intents: [GatewayIntentBits.Guilds, Gat // Run when the bot is ready serverClient.on('ready', async () => { - console.log(`Logged in as ${serverClient.user.tag}!`); + log.INFO(`Logged in as ${serverClient.user.tag}!`); // Add and register commands await addEnabledCommands(serverClient); // Config the discord bot with events await addEnabledEventListeners(serverClient); + + // Start the RSS Controller + serverClient.RSSController = await new RSSController(serverClient); + serverClient.RSSController.start(); + + log.INFO("RSS Controller:", serverClient.RSSController); }); // Startup the discord bot -console.log(`Logging into discord with ID: ${process.env.DISCORD_TOKEN}`); +log.INFO(`Logging into discord with ID: ${process.env.DISCORD_TOKEN}`); serverClient.login(process.env.DISCORD_TOKEN); diff --git a/discordBot/events/interactionCreate.mjs b/discordBot/events/interactionCreate.mjs index 0aa7482..9084d88 100644 --- a/discordBot/events/interactionCreate.mjs +++ b/discordBot/events/interactionCreate.mjs @@ -1,14 +1,16 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "discordBot.events.interactionCreate"); import { Events } from 'discord.js'; export const name = Events.InteractionCreate; export async function execute(nodeIo, interaction) { const command = interaction.client.commands.get(interaction.commandName); - console.log("Interaction created for command: ", command); + log.INFO("Interaction created for command: ", command); // Execute autocomplete if the user is checking autocomplete if (interaction.isAutocomplete()) { - console.log("Running autocomplete for command: ", command.data.name); + log.INFO("Running autocomplete for command: ", command.data.name); return await command.autocomplete(nodeIo, interaction); } @@ -20,7 +22,7 @@ export async function execute(nodeIo, interaction) { return; } - console.log(`${interaction.member.user} is running '${interaction.commandName}'`); + log.INFO(`${interaction.member.user} is running '${interaction.commandName}'`); // Defer the initial reply if the command has the parameter set if (command.deferInitialReply) { diff --git a/discordBot/modules/registerCommands.mjs b/discordBot/modules/registerCommands.mjs index cfadb0e..f8d027a 100644 --- a/discordBot/modules/registerCommands.mjs +++ b/discordBot/modules/registerCommands.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "discordBot.modules.registerCommands"); import { REST, Routes } from 'discord.js'; import dotenv from 'dotenv'; @@ -15,21 +17,21 @@ export const registerActiveCommands = async (serverClient) => { // and deploy your commands! guildIDs.forEach(guild => { - console.log("Deploying commands for: ", guild.id); - console.log("Commands", commands); + log.INFO("Deploying commands for: ", guild.id); + log.DEBUG("Commands", commands); (async () => { try { - console.log(`Started refreshing application (/) commands for guild ID: ${guild.id}.`); + log.DEBUG(`Started refreshing application (/) commands for guild ID: ${guild.id}.`); // The put method is used to fully refresh all commands in the guild with the current set const data = await rest.put( Routes.applicationGuildCommands(clientId, guild.id), { body: commands }, ); - console.log(`Successfully reloaded ${data.length} application (/) commands for guild ID: ${guild.id}.`); + log.DEBUG(`Successfully reloaded ${data.length} application (/) commands for guild ID: ${guild.id}.`); } catch (error) { // And of course, make sure you catch and log any errors! - console.log("ERROR Deploying commands: ", error, "Body from error: ", commands); + log.ERROR("ERROR Deploying commands: ", error, "Body from error: ", commands); } })() }) @@ -47,20 +49,20 @@ export const unregisterAllCommands = async (serverClient) => { const rest = new REST({ version: '10' }).setToken(discordToken); guildIDs.forEach(guild => { - console.log("Removing commands for: ", clientId, guild.id); + log.INFO("Removing commands for: ", clientId, guild.id); (async () => { try { - console.log(`Started removal of ${commands.length} application (/) commands for guild ID: ${guild.id}.`); + log.DEBUG(`Started removal of ${commands.length} application (/) commands for guild ID: ${guild.id}.`); // The put method is used to fully refresh all commands in the guild with the current set const data = await rest.put( Routes.applicationGuildCommands(clientId, guild.id), { body: commands }, ); - console.log(`Successfully removed ${data.length} application (/) commands for guild ID: ${guild.id}.`); + log.DEBUG(`Successfully removed ${data.length} application (/) commands for guild ID: ${guild.id}.`); } catch (error) { // And of course, make sure you catch and log any errors! - console.log("ERROR removing commands: ", error, "Body from error: ", commands); + log.ERROR("ERROR removing commands: ", error, "Body from error: ", commands); } })() }) @@ -74,10 +76,10 @@ export const unregisterAllCommands = async (serverClient) => { */ export const refreshActiveCommandsWrapper = async (serverClient) => { // Remove all commands - console.log("Removing/Unregistering all commands from all connected servers/guilds"); + log.INFO("Removing/Unregistering all commands from all connected servers/guilds"); await unregisterAllCommands(serverClient); // Deploy the active commands - console.log("Adding commands to all connected servers/guilds"); + log.INFO("Adding commands to all connected servers/guilds"); await registerActiveCommands(serverClient); return; } \ No newline at end of file diff --git a/discordBot/modules/rssWrappers.mjs b/discordBot/modules/rssWrappers.mjs index 9b769e0..91c1397 100644 --- a/discordBot/modules/rssWrappers.mjs +++ b/discordBot/modules/rssWrappers.mjs @@ -7,7 +7,7 @@ import { config } from 'dotenv'; // Load environment variables config(); -const log = new DebugBuilder("server", "libUtils"); +const log = new DebugBuilder("server", "discordBot.modules.rssWrappers"); const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g; const youtubeVideoRegex = /((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)/g; diff --git a/discordBot/modules/wrappers.mjs b/discordBot/modules/wrappers.mjs index 4661ec2..652b6b3 100644 --- a/discordBot/modules/wrappers.mjs +++ b/discordBot/modules/wrappers.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "discordBot.modules.wrappers"); import { checkIfNodeIsConnectedToVC, getNodeDiscordID, getNodeDiscordUsername } from '../../modules/socketServerWrappers.mjs'; import { getAllDiscordIDs } from '../../modules/mongo-wrappers/mongoDiscordIDWrappers.mjs' @@ -8,7 +10,7 @@ export const checkOnlineBotsInGuild = async (nodeIo, guildId) => { await Promise.all(openSockets.map(async openSocket => { openSocket = await nodeIo.sockets.sockets.get(openSocket); const connected = await checkIfNodeIsConnectedToVC(nodeIo, guildId, openSocket.node.nuid); - console.log("Connected:", connected); + log.INFO("Connected:", connected); if (connected) { const username = await getNodeDiscordUsername(openSocket, guildId); const discordID = await getNodeDiscordID(openSocket); @@ -33,8 +35,8 @@ export const checkOnlineBotsInGuild = async (nodeIo, guildId) => { ]); // Use the results of both promises here - console.log("Available Discord IDs:", discordIDs); - console.log("Online bots in the guild:", onlineBots); + log.INFO("Available Discord IDs:", discordIDs); + log.INFO("Online bots in the guild:", onlineBots); // Filter any discordIDs that are not active const availableDiscordIDs = discordIDs.filter(discordID => discordID.active == true).filter(discordID => !onlineBots.some(bot => Number(bot.discord_id) == discordID.discord_id)); diff --git a/modules/addonManager.mjs b/modules/addonManager.mjs index ca63c51..fbc3f40 100644 --- a/modules/addonManager.mjs +++ b/modules/addonManager.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../modules/debugger.mjs"; +const log = new DebugBuilder("server", "addonManager"); import { fileURLToPath } from 'url'; import fs from 'fs'; import path from 'path'; @@ -20,9 +22,9 @@ export const loadAddons = async (nodeIo) => { if (addonConfig.enabled) { const addonIndexPath = path.join(addonsDir, addonDir.name, 'index.js'); import(`file://${addonIndexPath}`).then(addonModule => { - console.log("Loading addon: ", addonModule); + log.DEBUG("Loading addon: ", addonModule); addonModule.initialize(nodeIo, addonConfig); - console.log(`Addon ${addonConfig.name} loaded.`); + log.DEBUG(`Addon ${addonConfig.name} loaded.`); }); } } diff --git a/modules/mongo-wrappers/mongoDiscordIDWrappers.mjs b/modules/mongo-wrappers/mongoDiscordIDWrappers.mjs index 1249a62..feb477c 100644 --- a/modules/mongo-wrappers/mongoDiscordIDWrappers.mjs +++ b/modules/mongo-wrappers/mongoDiscordIDWrappers.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "mongoDiscordIDWrappers"); import { insertDocument, getDocuments, connectToDatabase } from "./mongoHandler.mjs"; const collectionName = 'discord-ids'; @@ -8,7 +10,7 @@ export const createDiscordID = async (discordID) => { const insertedId = await insertDocument(collectionName, discordID); return insertedId; } catch (error) { - console.error('Error creating Discord ID:', error); + log.ERROR('Error creating Discord ID:', error); throw error; } }; @@ -19,7 +21,7 @@ export const getAllDiscordIDs = async () => { const discordIDs = await getDocuments(collectionName); return discordIDs; } catch (error) { - console.error('Error getting all Discord IDs:', error); + log.ERROR('Error getting all Discord IDs:', error); throw error; } }; @@ -37,7 +39,7 @@ export const getDiscordID = async (identifier) => { }); return discordID; } catch (error) { - console.error('Error getting Discord ID:', error); + log.ERROR('Error getting Discord ID:', error); throw error; } finally { // Close the connection @@ -56,10 +58,10 @@ export const updateDiscordID = async (identifier, updatedFields) => { { discord_id: identifier } ] }, { $set: updatedFields }); - console.log('Discord ID updated:', result.modifiedCount); + log.INFO('Discord ID updated:', result.modifiedCount); return result.modifiedCount; } catch (error) { - console.error('Error updating Discord ID:', error); + log.ERROR('Error updating Discord ID:', error); throw error; } finally { // Close the connection @@ -78,10 +80,10 @@ export const deleteDiscordID = async (identifier) => { { discord_id: identifier } ] }); - console.log('Discord ID deleted:', result.deletedCount); + log.INFO('Discord ID deleted:', result.deletedCount); return result.deletedCount; } catch (error) { - console.error('Error deleting Discord ID:', error); + log.ERROR('Error deleting Discord ID:', error); throw error; } finally { // Close the connection diff --git a/modules/mongo-wrappers/mongoFeedsWrappers.mjs b/modules/mongo-wrappers/mongoFeedsWrappers.mjs index 4238ed1..540967d 100644 --- a/modules/mongo-wrappers/mongoFeedsWrappers.mjs +++ b/modules/mongo-wrappers/mongoFeedsWrappers.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "mongoFeedsWrappers"); import { insertDocument, getDocuments, @@ -15,7 +17,7 @@ import { const insertedId = await insertDocument(feedCollectionName, feed); return insertedId; } catch (error) { - console.error('Error creating feed:', error); + log.ERROR('Error creating feed:', error); throw error; } }; @@ -26,7 +28,7 @@ import { const feeds = await getDocuments(feedCollectionName); return feeds; } catch (error) { - console.error('Error getting all feeds:', error); + log.ERROR('Error getting all feeds:', error); throw error; } }; @@ -37,7 +39,7 @@ import { const feed = await getDocumentByField(feedCollectionName, 'link', link); return feed; } catch (error) { - console.error('Error getting feed by link:', error); + log.ERROR('Error getting feed by link:', error); throw error; } }; @@ -48,7 +50,7 @@ import { const modifiedCount = await updateDocumentByField(feedCollectionName, 'link', link, updatedFields); return modifiedCount; } catch (error) { - console.error('Error updating feed by link:', error); + log.ERROR('Error updating feed by link:', error); throw error; } }; @@ -59,7 +61,7 @@ import { const deletedCount = await deleteDocumentByField(feedCollectionName, 'link', link); return deletedCount; } catch (error) { - console.error('Error deleting feed by link:', error); + log.ERROR('Error deleting feed by link:', error); throw error; } }; @@ -70,7 +72,7 @@ import { const insertedId = await insertDocument(postCollectionName, post); return insertedId; } catch (error) { - console.error('Error creating post:', error); + log.ERROR('Error creating post:', error); throw error; } }; @@ -81,7 +83,7 @@ import { const post = await getDocumentByField(postCollectionName, 'postId', postId); return post; } catch (error) { - console.error('Error getting post by postId:', error); + log.ERROR('Error getting post by postId:', error); throw error; } }; diff --git a/modules/mongo-wrappers/mongoNodesWrappers.mjs b/modules/mongo-wrappers/mongoNodesWrappers.mjs index f12fbbf..13bbc66 100644 --- a/modules/mongo-wrappers/mongoNodesWrappers.mjs +++ b/modules/mongo-wrappers/mongoNodesWrappers.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "mongoNodesWrappers"); import { insertDocument, getDocuments, connectToDatabase } from "./mongoHandler.mjs"; const collectionName = 'nodes'; @@ -8,7 +10,7 @@ export const createNode = async (node) => { const insertedId = await insertDocument(collectionName, node); return insertedId; } catch (error) { - console.error('Error creating node:', error); + log.ERROR('Error creating node:', error); throw error; } }; @@ -19,7 +21,7 @@ export const getAllNodes = async () => { const nodes = await getDocuments(collectionName); return nodes; } catch (error) { - console.error('Error getting all nodes:', error); + log.ERROR('Error getting all nodes:', error); throw error; } }; @@ -32,7 +34,7 @@ export const getNodeByNuid = async (nuid) => { const node = await collection.findOne({ nuid }); return node; } catch (error) { - console.error('Error getting node by NUID:', error); + log.ERROR('Error getting node by NUID:', error); throw error; } finally { // Close the connection @@ -46,10 +48,10 @@ export const updateNodeByNuid = async (nuid, updatedFields) => { try { const collection = db.db().collection(collectionName); const result = await collection.updateOne({ nuid }, { $set: updatedFields }); - console.log('Node updated:', result.modifiedCount); + log.INFO('Node updated:', result.modifiedCount); return result.modifiedCount; } catch (error) { - console.error('Error updating node by NUID:', error); + log.ERROR('Error updating node by NUID:', error); throw error; } finally { // Close the connection @@ -63,10 +65,10 @@ export const deleteNodeByNuid = async (nuid) => { try { const collection = db.db().collection(collectionName); const result = await collection.deleteOne({ nuid }); - console.log('Node deleted:', result.deletedCount); + log.INFO('Node deleted:', result.deletedCount); return result.deletedCount; } catch (error) { - console.error('Error deleting node by NUID:', error); + log.ERROR('Error deleting node by NUID:', error); throw error; } finally { // Close the connection diff --git a/modules/mongo-wrappers/mongoSystemsWrappers.mjs b/modules/mongo-wrappers/mongoSystemsWrappers.mjs index f353073..212d516 100644 --- a/modules/mongo-wrappers/mongoSystemsWrappers.mjs +++ b/modules/mongo-wrappers/mongoSystemsWrappers.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../../modules/debugger.mjs"; +const log = new DebugBuilder("server", "mongoSystemsWrappers"); import { insertDocument, getDocuments, connectToDatabase } from "./mongoHandler.mjs"; const collectionName = 'radio-systems'; @@ -21,7 +23,7 @@ export const createSystem = async (name, system, nuid) => { const insertedId = await insertDocument(collectionName, system); return insertedId; } catch (error) { - console.error('Error creating system:', error); + log.ERROR('Error creating system:', error); throw error; } }; @@ -32,7 +34,7 @@ export const getAllSystems = async () => { const systems = await getDocuments(collectionName); return systems; } catch (error) { - console.error('Error getting all systems:', error); + log.ERROR('Error getting all systems:', error); throw error; } }; @@ -45,7 +47,7 @@ export const getSystemByName = async (name) => { const system = await collection.findOne({ name }); return system; } catch (error) { - console.error('Error getting system by name:', error); + log.ERROR('Error getting system by name:', error); throw error; } finally { // Close the connection @@ -65,7 +67,7 @@ export const getSystemsByNuid = async (nuid) => { return systems; } catch (error) { - console.error('Error finding entries:', error); + log.ERROR('Error finding entries:', error); throw error; } finally { // Close the connection @@ -82,10 +84,10 @@ export const updateSystemByName = async (name, updatedSystem) => { try { const collection = db.db().collection(collectionName); const result = await collection.updateOne({ name }, { $set: updatedSystem }); - console.log('System updated:', result.modifiedCount); + log.INFO('System updated:', result.modifiedCount); return result.modifiedCount; } catch (error) { - console.error('Error updating system by name:', error); + log.ERROR('Error updating system by name:', error); throw error; } finally { // Close the connection @@ -99,10 +101,10 @@ export const deleteSystemByName = async (name) => { try { const collection = db.db().collection(collectionName); const result = await collection.deleteOne({ name }); - console.log('System deleted:', result.deletedCount); + log.INFO('System deleted:', result.deletedCount); return result.deletedCount; } catch (error) { - console.error('Error deleting system by name:', error); + log.ERROR('Error deleting system by name:', error); throw error; } finally { // Close the connection diff --git a/modules/socketServer.mjs b/modules/socketServer.mjs index 848ad9f..26c83ee 100644 --- a/modules/socketServer.mjs +++ b/modules/socketServer.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../modules/debugger.mjs"; +const log = new DebugBuilder("server", "socketServer"); import express from 'express'; import { createServer } from 'node:http'; import { Server } from 'socket.io'; @@ -15,7 +17,7 @@ app.get('/', (req, res) => { }); nodeIo.on('connection', (socket) => { - console.log('a user connected', socket.id); + log.INFO('a user connected', socket.id); socket.on('node-login', async (data) => { await nodeLoginWrapper(data, socket); diff --git a/modules/socketServerWrappers.mjs b/modules/socketServerWrappers.mjs index f7c0608..ccc4e16 100644 --- a/modules/socketServerWrappers.mjs +++ b/modules/socketServerWrappers.mjs @@ -1,3 +1,5 @@ +import { DebugBuilder } from "../modules/debugger.mjs"; +const log = new DebugBuilder("server", "socketServerWrappers"); import { createNode, getNodeByNuid, updateNodeByNuid } from "./mongo-wrappers/mongoNodesWrappers.mjs" import { createSystem, getSystemByName, updateSystemByName, getSystemsByNuid, deleteSystemByName } from "./mongo-wrappers/mongoSystemsWrappers.mjs" @@ -22,17 +24,16 @@ const sendNodeCommand = async (socket, command, data) => { * @returns {any} */ export const nodeLoginWrapper = async (data, socket) => { - console.log(`Login requested from node: ${data.nuid}`, data); + log.INFO(`Login requested from node: ${data.nuid}`, data); // Check to see if node exists - var node = await getNodeByNuid(data.nuid); - console.log("After grabbing", node); + var node = await getNodeByNuid(data.nuid); if (!node) { const insertedId = await createNode(data); - console.log("Added new node to the database:", insertedId); + log.DEBUG("Added new node to the database:", insertedId); } else { // Check for updates const updatedNode = await updateNodeByNuid(data.nuid, data) - console.log("Updated node:", updatedNode); + log.DEBUG("Updated node:", updatedNode); } node = await getNodeByNuid(data.nuid); @@ -59,7 +60,7 @@ export const nodeDisconnectWrapper = async (socketId) => { * @returns {any} */ export const nodeUpdateWrapper = async (nodeData) => { - console.log("Data update sent by node: ", nodeData); + log.DEBUG("Data update sent by node: ", nodeData); const updateResults = await updateNodeByNuid(nodeData.nuid, nodeData); return; } @@ -70,10 +71,10 @@ export const nodeUpdateWrapper = async (nodeData) => { * @param {object} nearbySystems The nearby systems object passed from the node to be updated */ export const nearbySystemsUpdateWraper = async (nuid, nearbySystems) => { - console.log("System updates sent by node: ", nuid, nearbySystems); + log.DEBUG("System updates sent by node: ", nuid, nearbySystems); // Check to see if the node removed any systems const existingSystems = await getSystemsByNuid(nuid); - console.log("Existing systems:", existingSystems); + log.DEBUG("Existing systems:", existingSystems); if (existingSystems !== nearbySystems) { for (const existingSystem of existingSystems) { if (existingSystem.name in nearbySystems) { @@ -81,17 +82,17 @@ export const nearbySystemsUpdateWraper = async (nuid, nearbySystems) => { continue; } - console.log("System exists that was not given by node", existingSystem); + log.DEBUG("System exists that was not given by node", existingSystem); // Check if this node was the only node on this system if (existingSystem.nodes.filter(node => node !== nuid).length === 0) { // Remove the system if so - console.log("Given node was the only node on this system, removing the system..."); + log.INFO("Given node was the only node on this system, removing the system..."); await deleteSystemByName(existingSystem.name); } else { // Remove the node from the array if there are other nodes with this system - console.log("Other nodes found on this system, removing the given NUID"); + log.INFO("Other nodes found on this system, removing the given NUID"); existingSystem.nodes = existingSystem.nodes.filter(node => node !== nuid); - console.log(existingSystem); + log.DEBUG(existingSystem); await updateSystemByName(existingSystem.name, existingSystem); } } @@ -111,7 +112,7 @@ export const nearbySystemsUpdateWraper = async (nuid, nearbySystems) => { existingSystem.nodes.push(nuid); // Update the system with the added node const updateResults = await updateSystemByName(nearbySystem, existingSystem); - if (updateResults) console.log("System updated", nearbySystem); + if (updateResults) log.INFO("System updated", nearbySystem); } } else { // The systems are not the same @@ -125,13 +126,13 @@ export const nearbySystemsUpdateWraper = async (nuid, nearbySystems) => { // Update the system with the added node const updateResults = await updateSystemByName(nearbySystem, nearbySystems[nearbySystem]); - if (updateResults) console.log("System updated", nearbySystem); + if (updateResults) log.INFO("System updated", nearbySystem); } } else { // Create a new system const newSystem = await createSystem(nearbySystem, nearbySystems[nearbySystem], nuid); - console.log("New system created", nearbySystem, newSystem); + log.INFO("New system created", nearbySystem, newSystem); } } return; @@ -146,7 +147,7 @@ export const nearbySystemsUpdateWraper = async (nuid, nearbySystems) => { export const getSocketIdByNuid = async (nodeIo, nuid) => { const openSockets = await nodeIo.allSockets(); for (const openSocketId of openSockets) { - console.log(openSockets) + log.DEBUG(openSockets) const openSocket = await nodeIo.sockets.sockets.get(openSocketId); if (openSocket.node.nuid == nuid) return openSocket; @@ -171,10 +172,10 @@ export const getAllSocketsConnectedToVC = async (nodeIo, guildId) => { await new Promise((res) => { openSocket.emit('node-check-connected-status', guildId, (status) => { if (status) { - console.log("Socket is connected to VC:", openSocket.node.name, status); + log.INFO("Socket is connected to VC:", openSocket.node.name, status); socketsConnectedToVC.push(openSocket); } else { - console.log("Socket is NOT connected to VC:", openSocket.node.name); + log.INFO("Socket is NOT connected to VC:", openSocket.node.name); } res(); }) @@ -196,10 +197,10 @@ export const checkIfNodeHasOpenDiscordClient = async (openSocket) => { await new Promise((res) => { openSocket.emit('node-check-discord-open-client', (status) => { if (status) { - console.log("Socket has an open discord client:", openSocket.node.name, status); + log.INFO("Socket has an open discord client:", openSocket.node.name, status); hasOpenDiscordClient = true; } else { - console.log("Socket does NOT have an open discord client:", openSocket.node.name); + log.INFO("Socket does NOT have an open discord client:", openSocket.node.name); } res(); }) @@ -217,10 +218,10 @@ export const getNodeCurrentListeningSystem = async (openSocket) => { await new Promise((res) => { openSocket.emit('node-check-current-system', (system) => { if (system) { - console.log("Socket is listening to system:", openSocket.node.name, system); + log.INFO("Socket is listening to system:", openSocket.node.name, system); currentSystem = system; } else { - console.log("Socket is not currently listening to a system:", openSocket.node.name); + log.INFO("Socket is not currently listening to a system:", openSocket.node.name); } res(); }) @@ -307,9 +308,9 @@ export const requestBotLeaveServer = async (socket, guildId) => { export const requestNodeUpdate = async (socket) => { await sendNodeCommand(socket, 'node-update', (status) => { if (status) { - console.log("Node is out of date, updating now", socket.node.name); + log.INFO("Node is out of date, updating now", socket.node.name); } else { - console.log("Node is up to date", socket.node.name); + log.INFO("Node is up to date", socket.node.name); } }); } \ No newline at end of file diff --git a/rss-manager/feedHandler.mjs b/rss-manager/feedHandler.mjs index aae743b..b0f81b8 100644 --- a/rss-manager/feedHandler.mjs +++ b/rss-manager/feedHandler.mjs @@ -5,6 +5,9 @@ import { DebugBuilder } from "../modules/debugger.mjs"; import UserAgent from "user-agents"; import Parser from 'rss-parser'; +import dotenv from 'dotenv'; +dotenv.config() + // Initialize the User-Agent string process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString(); @@ -16,8 +19,9 @@ const parser = new Parser({ }); const log = new DebugBuilder("server", "feedHandler"); -const runningPostsToRemove = {}; -const sourceFailureLimit = 5; +const sourceFailureLimit = process.env.RSS_SOURCE_FAILURE_LIMIT ?? 5; +const runningSourcesToRemove = {}; // This holds the sources that are pending removal (they've failed to load, return data, etc.) + export const returnHash = (...stringsIncluded) => { return crypto.createHash('sha1').update(stringsIncluded.join("-<>-")).digest("base64"); @@ -100,22 +104,22 @@ export const addSource = async (title, link, category, guildId, channelId, callb export const removeSource = async (sourceURL) => { log.INFO("Removing source:", sourceURL); - if (!runningPostsToRemove[sourceURL]) { - runningPostsToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 }; + if (!runningSourcesToRemove[sourceURL]) { + runningSourcesToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 }; return; } - const elapsedTime = Date.now() - runningPostsToRemove[sourceURL].timestamp; - const waitTime = runningPostsToRemove[sourceURL].count * 30000; + const elapsedTime = Date.now() - runningSourcesToRemove[sourceURL].timestamp; + const waitTime = runningSourcesToRemove[sourceURL].count * 30000; if (elapsedTime <= waitTime) { - runningPostsToRemove[sourceURL].ignoredAttempts += 1; + runningSourcesToRemove[sourceURL].ignoredAttempts += 1; return; } - if (runningPostsToRemove[sourceURL].count < sourceFailureLimit) { - runningPostsToRemove[sourceURL].count += 1; - runningPostsToRemove[sourceURL].timestamp = Date.now(); + if (runningSourcesToRemove[sourceURL].count < sourceFailureLimit) { + runningSourcesToRemove[sourceURL].count += 1; + runningSourcesToRemove[sourceURL].timestamp = Date.now(); return; } diff --git a/rss-manager/rssController.mjs b/rss-manager/rssController.mjs index e99d22c..fb64698 100644 --- a/rss-manager/rssController.mjs +++ b/rss-manager/rssController.mjs @@ -1,37 +1,49 @@ -//Will handle updating feeds in all channels +// Will handle updating feeds in all channels import { DebugBuilder } from "../modules/debugger.mjs"; import { updateFeeds } from "./feedHandler.mjs"; import dotenv from 'dotenv'; -dotenv.config() +dotenv.config(); const log = new DebugBuilder("server", "rssController"); -const refreshInterval = process.env.RSS_REFRESH_INTERVAL ?? 300000; +const refreshInterval = parseInt(process.env.RSS_REFRESH_INTERVAL) || 300000; export class RSSController { constructor(client) { - this.client = client; + this.client = client; + this.intervalId = null; } async start() { - // Wait for the refresh period before starting RSS feeds, so the rest of the bot can start - await new Promise(resolve => setTimeout(resolve, refreshInterval)); - - log.INFO("Starting RSS Controller"); - // Get initial feeds before starting the infinite loop - await updateFeeds(this.client); - - while(true) { - // Wait for the refresh interval, then wait for the posts to return, then wait a quarter of the refresh interval to make sure everything is cleared up - await new Promise(resolve => setTimeout(resolve, refreshInterval)); + try { + log.INFO("Starting RSS Controller"); + // Get initial feeds before starting the interval loop await this.collectLatestPosts(); - await new Promise(resolve => setTimeout(resolve, refreshInterval / 4)); - } + + // Start the interval loop for updating feeds + this.intervalId = setInterval(async () => { + await this.collectLatestPosts(); + }, refreshInterval); + + } catch (error) { + log.ERROR(`Failed to start RSS Controller: ${error.message}`); + } + } + + async stop() { + if (this.intervalId) { + clearInterval(this.intervalId); + log.INFO("RSS Controller stopped"); + } } async collectLatestPosts() { - log.INFO("Updating sources"); - await updateFeeds(this.client); + try { + log.INFO("Updating sources"); + await updateFeeds(this.client); + } catch (error) { + log.ERROR(`Error updating feeds: ${error.message}`); + } } } diff --git a/server.js b/server.js index 04769c3..8f35323 100644 --- a/server.js +++ b/server.js @@ -1,3 +1,5 @@ +import { DebugBuilder } from "./modules/debugger.mjs"; +const log = new DebugBuilder("server", "server"); import { nodeIo, app, server } from './modules/socketServer.mjs'; import { loadAddons } from './modules/addonManager.mjs'; import { serverClient, addEnabledEventListeners } from './discordBot/discordBot.mjs'; @@ -7,7 +9,7 @@ dotenv.config() // Startup the node server server.listen(process.env.SERVER_PORT || 3000, () => { - console.log(`server running at http://localhost:${process.env.SERVER_PORT}`); + log.INFO(`server running at http://localhost:${process.env.SERVER_PORT}`); }); // Add objects to the others From 702e291fcb26904bac4fc03cbeb18b84fdbf88f7 Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sun, 26 May 2024 01:22:47 -0400 Subject: [PATCH 7/9] #5 improve the debugger - Update the logic - Will now check to make sure the given file exists before writing --- modules/debugger.mjs | 84 +++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/modules/debugger.mjs b/modules/debugger.mjs index 8a3f524..c43e13f 100644 --- a/modules/debugger.mjs +++ b/modules/debugger.mjs @@ -1,25 +1,35 @@ // Import necessary modules import debug from 'debug'; import { config } from 'dotenv'; -import { writeFile } from 'fs'; +config(); +import { promises as fs } from 'fs'; +import { join, dirname } from 'path'; import { inspect } from 'util'; -// Load environment variables -config(); - -const logLocation = process.env.LOG_LOCATION; - +/** + * Write a given message to the log file + * @param {any} logMessage The message to write to the log file + * @param {string} appName The app name that created the log entry + */ const writeToLog = async (logMessage, appName) => { + const logLocation = join(process.env.LOG_LOCATION ?? `./logs/${appName}.log`); + + // Ensure the log directory exists + try { + await fs.mkdir(dirname(logLocation), { recursive: true }); + } catch (err) { + console.error(err); + } + + // Ensure the message is a string logMessage = `${String(logMessage)}\n`; - writeFile( - logLocation ?? `./${appName}.log`, - logMessage, - { encoding: "utf-8", flag: 'a+' }, - (err) => { - if (err) console.error(err); - } - ); + // Write to the file + try { + await fs.writeFile(logLocation, logMessage, { encoding: 'utf-8', flag: 'a+' }); + } catch (err) { + console.error(err); + } }; /** @@ -30,37 +40,31 @@ const writeToLog = async (logMessage, appName) => { */ export class DebugBuilder { constructor(appName, fileName) { - this.INFO = (...messageParts) => { - const _info = debug(`${appName}:${fileName}:INFO`); - _info(messageParts); - writeToLog(`${new Date().toLocaleString('en-US', { timeZone: 'America/New_York' })} - ${appName}:${fileName}:INFO\t-\t${messageParts.map(messagePart => inspect(messagePart))}`, appName); - }; - - this.DEBUG = (...messageParts) => { - const _debug = debug(`${appName}:${fileName}:DEBUG`); - _debug(messageParts); - writeToLog(`${new Date().toLocaleString('en-US', { timeZone: 'America/New_York' })} - ${appName}:${fileName}:DEBUG\t-\t${messageParts.map(messagePart => inspect(messagePart))}`, appName); - }; - - this.VERBOSE = (...messageParts) => { - const _verbose = debug(`${appName}:${fileName}:VERBOSE`); - _verbose(messageParts); - writeToLog(`${new Date().toLocaleString('en-US', { timeZone: 'America/New_York' })} - ${appName}:${fileName}:VERBOSE\t-\t${messageParts.map(messagePart => inspect(messagePart))}`, appName); - }; - - this.WARN = (...messageParts) => { - const _warn = debug(`${appName}:${fileName}:WARNING`); - _warn(messageParts); - writeToLog(`${new Date().toLocaleString('en-US', { timeZone: 'America/New_York' })} - ${appName}:${fileName}:WARNING\t-\t${messageParts.map(messagePart => inspect(messagePart))}`, appName); + const buildLogger = (level) => (...messageParts) => { + const logger = debug(`${appName}:${fileName}:${level}`); + logger(messageParts); + + const timeStamp = new Date().toLocaleString('en-US', { timeZone: 'America/New_York' }); + const message = `${timeStamp} - ${appName}:${fileName}:${level}\t-\t${messageParts.map(part => inspect(part)).join(' ')}`; + + // Write to console + console.log(message); + + // Write to logfile + writeToLog(message, appName); }; + this.INFO = buildLogger('INFO'); + this.DEBUG = buildLogger('DEBUG'); + this.VERBOSE = buildLogger('VERBOSE'); + this.WARN = buildLogger('WARNING'); this.ERROR = (...messageParts) => { - const _error = debug(`${appName}:${fileName}:ERROR`); - _error(messageParts); - writeToLog(`${new Date().toLocaleString('en-US', { timeZone: 'America/New_York' })} - ${appName}:${fileName}:ERROR\t-\t${messageParts.map(messagePart => inspect(messagePart))}`, appName); + buildLogger('ERROR')(...messageParts); + if (process.env.EXIT_ON_ERROR && process.env.EXIT_ON_ERROR > 0) { writeToLog("!--- EXITING ---!", appName); - setTimeout(() => process.exit(), process.env.EXIT_ON_ERROR_DELAY ?? 0); + const exitDelay = parseInt(process.env.EXIT_ON_ERROR_DELAY, 10) || 0; + setTimeout(() => process.exit(1), exitDelay); } }; } From 8f56fd9b9233c3ac397e1c24c5a01ab22776f59b Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sun, 26 May 2024 01:23:33 -0400 Subject: [PATCH 8/9] Updated default ping command with AC defaults --- discordBot/commands/ping.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/discordBot/commands/ping.mjs b/discordBot/commands/ping.mjs index 4ab5e80..92e9978 100644 --- a/discordBot/commands/ping.mjs +++ b/discordBot/commands/ping.mjs @@ -19,10 +19,10 @@ export const deferInitialReply = false; // If we the initial reply in discord sh /* export async function autocomplete(nodeIo, interaction) { const focusedValue = interaction.options.getFocused(); - const choices = []; + const choices = []; // The array to be filled with the autocorrect values const filtered = choices.filter(choice => choice.name.startsWith(focusedValue)); log.INFO(focusedValue, choices, filtered); - await interaction.respond(filtered); + await interaction.respond(filtered.map(choice => ({name: choice.name, value: choice.name}))); } */ From 43dfa7b5ad7fe26f8145956c7b381f463a9962ca Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sun, 26 May 2024 01:26:12 -0400 Subject: [PATCH 9/9] RSS improvements - Updated rss discord command name scheme - Implemented new sourceManager for handling feed sources - Added wrappers to delete/get feed sources by title --- discordBot/commands/{add.mjs => rssAdd.mjs} | 10 +-- discordBot/commands/rssRemove.mjs | 58 ++++++++++++++ .../{triggerRss.mjs => rssTrigger.mjs} | 6 +- discordBot/modules/rssWrappers.mjs | 4 +- modules/mongo-wrappers/mongoFeedsWrappers.mjs | 22 ++++++ rss-manager/feedHandler.mjs | 66 ++-------------- rss-manager/sourceManager.mjs | 75 +++++++++++++++++++ 7 files changed, 173 insertions(+), 68 deletions(-) rename discordBot/commands/{add.mjs => rssAdd.mjs} (84%) create mode 100644 discordBot/commands/rssRemove.mjs rename discordBot/commands/{triggerRss.mjs => rssTrigger.mjs} (91%) create mode 100644 rss-manager/sourceManager.mjs diff --git a/discordBot/commands/add.mjs b/discordBot/commands/rssAdd.mjs similarity index 84% rename from discordBot/commands/add.mjs rename to discordBot/commands/rssAdd.mjs index abe7107..33aefec 100644 --- a/discordBot/commands/add.mjs +++ b/discordBot/commands/rssAdd.mjs @@ -1,12 +1,12 @@ import { SlashCommandBuilder } from 'discord.js'; import { DebugBuilder } from "../../modules/debugger.mjs"; -import { addSource } from '../../rss-manager/feedHandler.mjs' -const log = new DebugBuilder("server", "discordBot.command.add"); +import { addSource } from '../../rss-manager/sourceManager.mjs' +const log = new DebugBuilder("server", "discordBot.command.rssAdd"); // Exporting data property that contains the command structure for discord including any params export const data = new SlashCommandBuilder() - .setName('add') + .setName('rss-add') .setDescription('Add RSS Source') .addStringOption(option => option.setName('title') @@ -22,7 +22,7 @@ export const data = new SlashCommandBuilder() .setRequired(false)) // Exporting other properties -export const example = "/add [title] [https://domain.com/feed.xml] [category]"; // An example of how the command would be run in discord chat, this will be used for the help command +export const example = "/rss-add [title] [https://domain.com/feed.xml] [category]"; // An example of how the command would be run in discord chat, this will be used for the help command export const deferInitialReply = false; // If we the initial reply in discord should be deferred. This gives extra time to respond, however the method of replying is different. /** @@ -59,7 +59,7 @@ export const execute = async (nodeIo, interaction) => { log.DEBUG("Result from adding entry", result); if (result) { - interaction.reply(`Adding ${title} to the list of RSS sources`); + interaction.reply(`Successfully added ${title} to the list of RSS sources`); } else { interaction.reply(`${title} already exists in the list of RSS sources`); } diff --git a/discordBot/commands/rssRemove.mjs b/discordBot/commands/rssRemove.mjs new file mode 100644 index 0000000..1e40593 --- /dev/null +++ b/discordBot/commands/rssRemove.mjs @@ -0,0 +1,58 @@ + +import { SlashCommandBuilder } from 'discord.js'; +import { DebugBuilder } from "../../modules/debugger.mjs"; +import { removeSource } from '../../rss-manager/sourceManager.mjs' +import { getAllFeeds, deleteFeedByTitle } from '../../modules/mongo-wrappers/mongoFeedsWrappers.mjs' +const log = new DebugBuilder("server", "discordBot.command.rssRemove"); + +// Exporting data property that contains the command structure for discord including any params +export const data = new SlashCommandBuilder() + .setName('rss-remove') + .setDescription('Add RSS Source') + .addStringOption(option => + option.setName('title') + .setDescription('The title of the RSS feed') + .setRequired(true) + .setAutocomplete(true)) + +// Exporting other properties +export const example = "/rss-remove [title]"; // An example of how the command would be run in discord chat, this will be used for the help command +export const deferInitialReply = false; // If we the initial reply in discord should be deferred. This gives extra time to respond, however the method of replying is different. + +/** + * Function to give the user auto-reply suggestions + * @param {any} nodeIo The nodeIO server for manipulation of sockets + * @param {any} interaction The interaction object + */ + +export async function autocomplete(nodeIo, interaction) { + const focusedValue = interaction.options.getFocused(); + const choices = await getAllFeeds() ?? []; + log.INFO("RSS Remove Choices:", choices); + const filtered = choices.filter(choice => choice.title.startsWith(focusedValue)); + log.DEBUG(focusedValue, choices, filtered); + await interaction.respond(filtered.map(choice => ({ name: choice.title, value: choice.title }))); +} + +/** + * The function to run when the command is called by a discord user + * @param {any} nodeIo The nodeIO server for manipulation of sockets + * @param {any} interaction The interaction object + */ +export const execute = async (nodeIo, interaction) => { + try { + var title = interaction.options.getString('title'); + interaction.reply(`Removing ${title} from the list of RSS sources, please wait...`); + + const results = await deleteFeedByTitle(title); + if (!results) { + log.WARN(`Failed to remove source: ${title}`); + interaction.editReply(`Failed to remove source: '${title}'`); + return; + } + interaction.editReply(`${title} was successfully removed from the RSS sources.`) + } catch (err) { + log.ERROR(err) + interaction.editReply(err.toString()); + } +} \ No newline at end of file diff --git a/discordBot/commands/triggerRss.mjs b/discordBot/commands/rssTrigger.mjs similarity index 91% rename from discordBot/commands/triggerRss.mjs rename to discordBot/commands/rssTrigger.mjs index 1827575..a18880b 100644 --- a/discordBot/commands/triggerRss.mjs +++ b/discordBot/commands/rssTrigger.mjs @@ -1,15 +1,15 @@ import { DebugBuilder } from "../../modules/debugger.mjs"; -const log = new DebugBuilder("server", "discordBot.command.triggerRss"); +const log = new DebugBuilder("server", "discordBot.command.rssTrigger"); import { SlashCommandBuilder } from 'discord.js'; import { updateFeeds } from '../../rss-manager/feedHandler.mjs' // Exporting data property that contains the command structure for discord including any params export const data = new SlashCommandBuilder() - .setName('trigger-rss') + .setName('rss-trigger') .setDescription('Manually triggers an RSS feed update'); // Exporting other properties -export const example = "/trigger-rss"; // An example of how the command would be run in discord chat, this will be used for the help command +export const example = "/rss-trigger"; // An example of how the command would be run in discord chat, this will be used for the help command export const deferInitialReply = false; // If we the initial reply in discord should be deferred. This gives extra time to respond, however the method of replying is different. /** diff --git a/discordBot/modules/rssWrappers.mjs b/discordBot/modules/rssWrappers.mjs index 91c1397..7fe5996 100644 --- a/discordBot/modules/rssWrappers.mjs +++ b/discordBot/modules/rssWrappers.mjs @@ -12,7 +12,7 @@ const log = new DebugBuilder("server", "discordBot.modules.rssWrappers"); const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g; const youtubeVideoRegex = /((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)/g; -export class EmmeliaEmbedBuilder extends EmbedBuilder { +export class DRBEmbedBuilder extends EmbedBuilder { constructor() { super(); this.setTimestamp(); @@ -66,7 +66,7 @@ export const sendPost = (post, source, channel) => { log.DEBUG("Post content: ", postContent); try { - const rssMessage = new EmmeliaEmbedBuilder() + const rssMessage = new DRBEmbedBuilder() .setColor(0x0099FF) .setTitle(postTitle) .setURL(postLink) diff --git a/modules/mongo-wrappers/mongoFeedsWrappers.mjs b/modules/mongo-wrappers/mongoFeedsWrappers.mjs index 540967d..1155dc9 100644 --- a/modules/mongo-wrappers/mongoFeedsWrappers.mjs +++ b/modules/mongo-wrappers/mongoFeedsWrappers.mjs @@ -43,6 +43,17 @@ import { throw error; } }; + + // Wrapper for retrieving a feed by the title + export const getFeedByTitle = async (title) => { + try { + const feed = await getDocumentByField(feedCollectionName, 'title', title); + return feed; + } catch (error) { + log.ERROR('Error getting feed by link:', error); + throw error; + } + }; // Wrapper for updating a feed by link export const updateFeedByLink = async (link, updatedFields) => { @@ -65,6 +76,17 @@ import { throw error; } }; + + // Wrapper for deleting a feed by title + export const deleteFeedByTitle = async (title) => { + try { + const deletedCount = await deleteDocumentByField(feedCollectionName, 'title', title); + return deletedCount; + } catch (error) { + log.ERROR('Error deleting feed by link:', error); + throw error; + } + }; // Wrapper for inserting a post export const createPost = async (post) => { diff --git a/rss-manager/feedHandler.mjs b/rss-manager/feedHandler.mjs index b0f81b8..b884a31 100644 --- a/rss-manager/feedHandler.mjs +++ b/rss-manager/feedHandler.mjs @@ -1,7 +1,8 @@ -import { createFeed, getAllFeeds, getFeedByLink, updateFeedByLink, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs'; +import { getAllFeeds, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs'; import crypto from 'crypto'; import { sendPost } from '../discordBot/modules/rssWrappers.mjs'; import { DebugBuilder } from "../modules/debugger.mjs"; +import { removeSource } from './sourceManager.mjs' import UserAgent from "user-agents"; import Parser from 'rss-parser'; @@ -19,14 +20,16 @@ const parser = new Parser({ }); const log = new DebugBuilder("server", "feedHandler"); -const sourceFailureLimit = process.env.RSS_SOURCE_FAILURE_LIMIT ?? 5; -const runningSourcesToRemove = {}; // This holds the sources that are pending removal (they've failed to load, return data, etc.) - export const returnHash = (...stringsIncluded) => { return crypto.createHash('sha1').update(stringsIncluded.join("-<>-")).digest("base64"); }; +/** + * Update the active RSS feeds and send any new posts to their discord channels + * @param {any} client The discord client to send posts with + * @returns {any} + */ export const updateFeeds = async (client) => { if (!client) throw new Error("Client object not passed"); @@ -87,57 +90,4 @@ export const updateFeeds = async (client) => { log.ERROR("Error updating feeds:", error); throw error; } -}; - -export const addSource = async (title, link, category, guildId, channelId, callback) => { - try { - const feed = { title, link, category, guild_id: guildId, channel_id: channelId }; - const record = await createFeed(feed); - log.DEBUG("Source added:", record); - callback(null, record); - } catch (err) { - log.ERROR("Error adding source:", err); - callback(err, null); - } -}; - -export const removeSource = async (sourceURL) => { - log.INFO("Removing source:", sourceURL); - - if (!runningSourcesToRemove[sourceURL]) { - runningSourcesToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 }; - return; - } - - const elapsedTime = Date.now() - runningSourcesToRemove[sourceURL].timestamp; - const waitTime = runningSourcesToRemove[sourceURL].count * 30000; - - if (elapsedTime <= waitTime) { - runningSourcesToRemove[sourceURL].ignoredAttempts += 1; - return; - } - - if (runningSourcesToRemove[sourceURL].count < sourceFailureLimit) { - runningSourcesToRemove[sourceURL].count += 1; - runningSourcesToRemove[sourceURL].timestamp = Date.now(); - return; - } - - try { - const record = await getFeedByLink(sourceURL); - if (!record) { - log.ERROR("Source not found in storage"); - return; - } - - const results = await deleteFeedByLink(sourceURL); - if (!results) { - log.WARN("Failed to remove source"); - return; - } - - log.DEBUG("Source removed after exceeding failure limit:", sourceURL); - } catch (err) { - log.ERROR("Error removing source from storage:", err); - } -}; +}; \ No newline at end of file diff --git a/rss-manager/sourceManager.mjs b/rss-manager/sourceManager.mjs new file mode 100644 index 0000000..643bb3f --- /dev/null +++ b/rss-manager/sourceManager.mjs @@ -0,0 +1,75 @@ +import { createFeed, getFeedByLink, deleteFeedByLink } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs'; + +class SourceManager { + constructor(sourceFailureLimit) { + this.sourceFailureLimit = sourceFailureLimit; + this.runningSourcesToRemove = {}; + } + + async removeSource(sourceURL) { + log.INFO(`Removing source: ${sourceURL}`); + + const currentTime = Date.now(); + const sourceData = this.runningSourcesToRemove[sourceURL]; + + if (!sourceData) { + this.runningSourcesToRemove[sourceURL] = { count: 1, timestamp: currentTime, ignoredAttempts: 0 }; + return; + } + + const elapsedTimeSinceLastAttempt = currentTime - sourceData.timestamp; + const waitTime = sourceData.count * 30000; + + if (elapsedTimeSinceLastAttempt <= waitTime) { + sourceData.ignoredAttempts += 1; + return; + } + + if (sourceData.count < this.sourceFailureLimit) { + sourceData.count += 1; + sourceData.timestamp = currentTime; + return; + } + + try { + const record = await getFeedByLink(sourceURL); + if (!record) { + log.ERROR(`Source not found in storage: ${sourceURL}`); + return; + } + + const results = await deleteFeedByLink(sourceURL); + if (!results) { + log.WARN(`Failed to remove source: ${sourceURL}`); + return; + } + + log.DEBUG(`Source removed after exceeding failure limit: ${sourceURL}`); + // Optionally, clean up the entry from runningSourcesToRemove + delete this.runningSourcesToRemove[sourceURL]; + } catch (err) { + log.ERROR(`Error removing source from storage: ${sourceURL}`, err); + } + } + + async addSource(title, link, category, guildId, channelId, callback) { + try { + const feed = { title, link, category, guild_id: guildId, channel_id: channelId }; + const record = await createFeed(feed); + log.DEBUG("Source added:", record); + if (callback) callback(null, record); + } catch (err) { + log.ERROR("Error adding source:", err); + if (callback) callback(err, null); + } + } +} + + +// Create a default instance of SourceManager +const defaultSourceManager = new SourceManager(); + +// Export the class and default instance methods +export { SourceManager }; +export const addSource = defaultSourceManager.addSource.bind(defaultSourceManager); +export const removeSource = defaultSourceManager.removeSource.bind(defaultSourceManager); \ No newline at end of file