From 43dfa7b5ad7fe26f8145956c7b381f463a9962ca Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sun, 26 May 2024 01:26:12 -0400 Subject: [PATCH] RSS improvements - Updated rss discord command name scheme - Implemented new sourceManager for handling feed sources - Added wrappers to delete/get feed sources by title --- discordBot/commands/{add.mjs => rssAdd.mjs} | 10 +-- discordBot/commands/rssRemove.mjs | 58 ++++++++++++++ .../{triggerRss.mjs => rssTrigger.mjs} | 6 +- discordBot/modules/rssWrappers.mjs | 4 +- modules/mongo-wrappers/mongoFeedsWrappers.mjs | 22 ++++++ rss-manager/feedHandler.mjs | 66 ++-------------- rss-manager/sourceManager.mjs | 75 +++++++++++++++++++ 7 files changed, 173 insertions(+), 68 deletions(-) rename discordBot/commands/{add.mjs => rssAdd.mjs} (84%) create mode 100644 discordBot/commands/rssRemove.mjs rename discordBot/commands/{triggerRss.mjs => rssTrigger.mjs} (91%) create mode 100644 rss-manager/sourceManager.mjs diff --git a/discordBot/commands/add.mjs b/discordBot/commands/rssAdd.mjs similarity index 84% rename from discordBot/commands/add.mjs rename to discordBot/commands/rssAdd.mjs index abe7107..33aefec 100644 --- a/discordBot/commands/add.mjs +++ b/discordBot/commands/rssAdd.mjs @@ -1,12 +1,12 @@ import { SlashCommandBuilder } from 'discord.js'; import { DebugBuilder } from "../../modules/debugger.mjs"; -import { addSource } from '../../rss-manager/feedHandler.mjs' -const log = new DebugBuilder("server", "discordBot.command.add"); +import { addSource } from '../../rss-manager/sourceManager.mjs' +const log = new DebugBuilder("server", "discordBot.command.rssAdd"); // Exporting data property that contains the command structure for discord including any params export const data = new SlashCommandBuilder() - .setName('add') + .setName('rss-add') .setDescription('Add RSS Source') .addStringOption(option => option.setName('title') @@ -22,7 +22,7 @@ export const data = new SlashCommandBuilder() .setRequired(false)) // Exporting other properties -export const example = "/add [title] [https://domain.com/feed.xml] [category]"; // An example of how the command would be run in discord chat, this will be used for the help command +export const example = "/rss-add [title] [https://domain.com/feed.xml] [category]"; // An example of how the command would be run in discord chat, this will be used for the help command export const deferInitialReply = false; // If we the initial reply in discord should be deferred. This gives extra time to respond, however the method of replying is different. /** @@ -59,7 +59,7 @@ export const execute = async (nodeIo, interaction) => { log.DEBUG("Result from adding entry", result); if (result) { - interaction.reply(`Adding ${title} to the list of RSS sources`); + interaction.reply(`Successfully added ${title} to the list of RSS sources`); } else { interaction.reply(`${title} already exists in the list of RSS sources`); } diff --git a/discordBot/commands/rssRemove.mjs b/discordBot/commands/rssRemove.mjs new file mode 100644 index 0000000..1e40593 --- /dev/null +++ b/discordBot/commands/rssRemove.mjs @@ -0,0 +1,58 @@ + +import { SlashCommandBuilder } from 'discord.js'; +import { DebugBuilder } from "../../modules/debugger.mjs"; +import { removeSource } from '../../rss-manager/sourceManager.mjs' +import { getAllFeeds, deleteFeedByTitle } from '../../modules/mongo-wrappers/mongoFeedsWrappers.mjs' +const log = new DebugBuilder("server", "discordBot.command.rssRemove"); + +// Exporting data property that contains the command structure for discord including any params +export const data = new SlashCommandBuilder() + .setName('rss-remove') + .setDescription('Add RSS Source') + .addStringOption(option => + option.setName('title') + .setDescription('The title of the RSS feed') + .setRequired(true) + .setAutocomplete(true)) + +// Exporting other properties +export const example = "/rss-remove [title]"; // An example of how the command would be run in discord chat, this will be used for the help command +export const deferInitialReply = false; // If we the initial reply in discord should be deferred. This gives extra time to respond, however the method of replying is different. + +/** + * Function to give the user auto-reply suggestions + * @param {any} nodeIo The nodeIO server for manipulation of sockets + * @param {any} interaction The interaction object + */ + +export async function autocomplete(nodeIo, interaction) { + const focusedValue = interaction.options.getFocused(); + const choices = await getAllFeeds() ?? []; + log.INFO("RSS Remove Choices:", choices); + const filtered = choices.filter(choice => choice.title.startsWith(focusedValue)); + log.DEBUG(focusedValue, choices, filtered); + await interaction.respond(filtered.map(choice => ({ name: choice.title, value: choice.title }))); +} + +/** + * The function to run when the command is called by a discord user + * @param {any} nodeIo The nodeIO server for manipulation of sockets + * @param {any} interaction The interaction object + */ +export const execute = async (nodeIo, interaction) => { + try { + var title = interaction.options.getString('title'); + interaction.reply(`Removing ${title} from the list of RSS sources, please wait...`); + + const results = await deleteFeedByTitle(title); + if (!results) { + log.WARN(`Failed to remove source: ${title}`); + interaction.editReply(`Failed to remove source: '${title}'`); + return; + } + interaction.editReply(`${title} was successfully removed from the RSS sources.`) + } catch (err) { + log.ERROR(err) + interaction.editReply(err.toString()); + } +} \ No newline at end of file diff --git a/discordBot/commands/triggerRss.mjs b/discordBot/commands/rssTrigger.mjs similarity index 91% rename from discordBot/commands/triggerRss.mjs rename to discordBot/commands/rssTrigger.mjs index 1827575..a18880b 100644 --- a/discordBot/commands/triggerRss.mjs +++ b/discordBot/commands/rssTrigger.mjs @@ -1,15 +1,15 @@ import { DebugBuilder } from "../../modules/debugger.mjs"; -const log = new DebugBuilder("server", "discordBot.command.triggerRss"); +const log = new DebugBuilder("server", "discordBot.command.rssTrigger"); import { SlashCommandBuilder } from 'discord.js'; import { updateFeeds } from '../../rss-manager/feedHandler.mjs' // Exporting data property that contains the command structure for discord including any params export const data = new SlashCommandBuilder() - .setName('trigger-rss') + .setName('rss-trigger') .setDescription('Manually triggers an RSS feed update'); // Exporting other properties -export const example = "/trigger-rss"; // An example of how the command would be run in discord chat, this will be used for the help command +export const example = "/rss-trigger"; // An example of how the command would be run in discord chat, this will be used for the help command export const deferInitialReply = false; // If we the initial reply in discord should be deferred. This gives extra time to respond, however the method of replying is different. /** diff --git a/discordBot/modules/rssWrappers.mjs b/discordBot/modules/rssWrappers.mjs index 91c1397..7fe5996 100644 --- a/discordBot/modules/rssWrappers.mjs +++ b/discordBot/modules/rssWrappers.mjs @@ -12,7 +12,7 @@ const log = new DebugBuilder("server", "discordBot.modules.rssWrappers"); const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g; const youtubeVideoRegex = /((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)/g; -export class EmmeliaEmbedBuilder extends EmbedBuilder { +export class DRBEmbedBuilder extends EmbedBuilder { constructor() { super(); this.setTimestamp(); @@ -66,7 +66,7 @@ export const sendPost = (post, source, channel) => { log.DEBUG("Post content: ", postContent); try { - const rssMessage = new EmmeliaEmbedBuilder() + const rssMessage = new DRBEmbedBuilder() .setColor(0x0099FF) .setTitle(postTitle) .setURL(postLink) diff --git a/modules/mongo-wrappers/mongoFeedsWrappers.mjs b/modules/mongo-wrappers/mongoFeedsWrappers.mjs index 540967d..1155dc9 100644 --- a/modules/mongo-wrappers/mongoFeedsWrappers.mjs +++ b/modules/mongo-wrappers/mongoFeedsWrappers.mjs @@ -43,6 +43,17 @@ import { throw error; } }; + + // Wrapper for retrieving a feed by the title + export const getFeedByTitle = async (title) => { + try { + const feed = await getDocumentByField(feedCollectionName, 'title', title); + return feed; + } catch (error) { + log.ERROR('Error getting feed by link:', error); + throw error; + } + }; // Wrapper for updating a feed by link export const updateFeedByLink = async (link, updatedFields) => { @@ -65,6 +76,17 @@ import { throw error; } }; + + // Wrapper for deleting a feed by title + export const deleteFeedByTitle = async (title) => { + try { + const deletedCount = await deleteDocumentByField(feedCollectionName, 'title', title); + return deletedCount; + } catch (error) { + log.ERROR('Error deleting feed by link:', error); + throw error; + } + }; // Wrapper for inserting a post export const createPost = async (post) => { diff --git a/rss-manager/feedHandler.mjs b/rss-manager/feedHandler.mjs index b0f81b8..b884a31 100644 --- a/rss-manager/feedHandler.mjs +++ b/rss-manager/feedHandler.mjs @@ -1,7 +1,8 @@ -import { createFeed, getAllFeeds, getFeedByLink, updateFeedByLink, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs'; +import { getAllFeeds, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs'; import crypto from 'crypto'; import { sendPost } from '../discordBot/modules/rssWrappers.mjs'; import { DebugBuilder } from "../modules/debugger.mjs"; +import { removeSource } from './sourceManager.mjs' import UserAgent from "user-agents"; import Parser from 'rss-parser'; @@ -19,14 +20,16 @@ const parser = new Parser({ }); const log = new DebugBuilder("server", "feedHandler"); -const sourceFailureLimit = process.env.RSS_SOURCE_FAILURE_LIMIT ?? 5; -const runningSourcesToRemove = {}; // This holds the sources that are pending removal (they've failed to load, return data, etc.) - export const returnHash = (...stringsIncluded) => { return crypto.createHash('sha1').update(stringsIncluded.join("-<>-")).digest("base64"); }; +/** + * Update the active RSS feeds and send any new posts to their discord channels + * @param {any} client The discord client to send posts with + * @returns {any} + */ export const updateFeeds = async (client) => { if (!client) throw new Error("Client object not passed"); @@ -87,57 +90,4 @@ export const updateFeeds = async (client) => { log.ERROR("Error updating feeds:", error); throw error; } -}; - -export const addSource = async (title, link, category, guildId, channelId, callback) => { - try { - const feed = { title, link, category, guild_id: guildId, channel_id: channelId }; - const record = await createFeed(feed); - log.DEBUG("Source added:", record); - callback(null, record); - } catch (err) { - log.ERROR("Error adding source:", err); - callback(err, null); - } -}; - -export const removeSource = async (sourceURL) => { - log.INFO("Removing source:", sourceURL); - - if (!runningSourcesToRemove[sourceURL]) { - runningSourcesToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 }; - return; - } - - const elapsedTime = Date.now() - runningSourcesToRemove[sourceURL].timestamp; - const waitTime = runningSourcesToRemove[sourceURL].count * 30000; - - if (elapsedTime <= waitTime) { - runningSourcesToRemove[sourceURL].ignoredAttempts += 1; - return; - } - - if (runningSourcesToRemove[sourceURL].count < sourceFailureLimit) { - runningSourcesToRemove[sourceURL].count += 1; - runningSourcesToRemove[sourceURL].timestamp = Date.now(); - return; - } - - try { - const record = await getFeedByLink(sourceURL); - if (!record) { - log.ERROR("Source not found in storage"); - return; - } - - const results = await deleteFeedByLink(sourceURL); - if (!results) { - log.WARN("Failed to remove source"); - return; - } - - log.DEBUG("Source removed after exceeding failure limit:", sourceURL); - } catch (err) { - log.ERROR("Error removing source from storage:", err); - } -}; +}; \ No newline at end of file diff --git a/rss-manager/sourceManager.mjs b/rss-manager/sourceManager.mjs new file mode 100644 index 0000000..643bb3f --- /dev/null +++ b/rss-manager/sourceManager.mjs @@ -0,0 +1,75 @@ +import { createFeed, getFeedByLink, deleteFeedByLink } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs'; + +class SourceManager { + constructor(sourceFailureLimit) { + this.sourceFailureLimit = sourceFailureLimit; + this.runningSourcesToRemove = {}; + } + + async removeSource(sourceURL) { + log.INFO(`Removing source: ${sourceURL}`); + + const currentTime = Date.now(); + const sourceData = this.runningSourcesToRemove[sourceURL]; + + if (!sourceData) { + this.runningSourcesToRemove[sourceURL] = { count: 1, timestamp: currentTime, ignoredAttempts: 0 }; + return; + } + + const elapsedTimeSinceLastAttempt = currentTime - sourceData.timestamp; + const waitTime = sourceData.count * 30000; + + if (elapsedTimeSinceLastAttempt <= waitTime) { + sourceData.ignoredAttempts += 1; + return; + } + + if (sourceData.count < this.sourceFailureLimit) { + sourceData.count += 1; + sourceData.timestamp = currentTime; + return; + } + + try { + const record = await getFeedByLink(sourceURL); + if (!record) { + log.ERROR(`Source not found in storage: ${sourceURL}`); + return; + } + + const results = await deleteFeedByLink(sourceURL); + if (!results) { + log.WARN(`Failed to remove source: ${sourceURL}`); + return; + } + + log.DEBUG(`Source removed after exceeding failure limit: ${sourceURL}`); + // Optionally, clean up the entry from runningSourcesToRemove + delete this.runningSourcesToRemove[sourceURL]; + } catch (err) { + log.ERROR(`Error removing source from storage: ${sourceURL}`, err); + } + } + + async addSource(title, link, category, guildId, channelId, callback) { + try { + const feed = { title, link, category, guild_id: guildId, channel_id: channelId }; + const record = await createFeed(feed); + log.DEBUG("Source added:", record); + if (callback) callback(null, record); + } catch (err) { + log.ERROR("Error adding source:", err); + if (callback) callback(err, null); + } + } +} + + +// Create a default instance of SourceManager +const defaultSourceManager = new SourceManager(); + +// Export the class and default instance methods +export { SourceManager }; +export const addSource = defaultSourceManager.addSource.bind(defaultSourceManager); +export const removeSource = defaultSourceManager.removeSource.bind(defaultSourceManager); \ No newline at end of file