RSS improvements
All checks were successful
DRB Tests / drb_mocha_tests (pull_request) Successful in 31s

- Updated rss discord command name scheme
- Implemented new sourceManager for handling feed sources
- Added wrappers to delete/get feed sources by title
This commit is contained in:
Logan Cusano
2024-05-26 01:26:12 -04:00
parent 8f56fd9b92
commit 43dfa7b5ad
7 changed files with 173 additions and 68 deletions

View File

@@ -1,7 +1,8 @@
import { createFeed, getAllFeeds, getFeedByLink, updateFeedByLink, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs';
import { getAllFeeds, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs';
import crypto from 'crypto';
import { sendPost } from '../discordBot/modules/rssWrappers.mjs';
import { DebugBuilder } from "../modules/debugger.mjs";
import { removeSource } from './sourceManager.mjs'
import UserAgent from "user-agents";
import Parser from 'rss-parser';
@@ -19,14 +20,16 @@ const parser = new Parser({
});
const log = new DebugBuilder("server", "feedHandler");
const sourceFailureLimit = process.env.RSS_SOURCE_FAILURE_LIMIT ?? 5;
const runningSourcesToRemove = {}; // This holds the sources that are pending removal (they've failed to load, return data, etc.)
export const returnHash = (...stringsIncluded) => {
return crypto.createHash('sha1').update(stringsIncluded.join("-<<??//\\\\??>>-")).digest("base64");
};
/**
* Update the active RSS feeds and send any new posts to their discord channels
* @param {any} client The discord client to send posts with
* @returns {any}
*/
export const updateFeeds = async (client) => {
if (!client) throw new Error("Client object not passed");
@@ -87,57 +90,4 @@ export const updateFeeds = async (client) => {
log.ERROR("Error updating feeds:", error);
throw error;
}
};
export const addSource = async (title, link, category, guildId, channelId, callback) => {
try {
const feed = { title, link, category, guild_id: guildId, channel_id: channelId };
const record = await createFeed(feed);
log.DEBUG("Source added:", record);
callback(null, record);
} catch (err) {
log.ERROR("Error adding source:", err);
callback(err, null);
}
};
export const removeSource = async (sourceURL) => {
log.INFO("Removing source:", sourceURL);
if (!runningSourcesToRemove[sourceURL]) {
runningSourcesToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 };
return;
}
const elapsedTime = Date.now() - runningSourcesToRemove[sourceURL].timestamp;
const waitTime = runningSourcesToRemove[sourceURL].count * 30000;
if (elapsedTime <= waitTime) {
runningSourcesToRemove[sourceURL].ignoredAttempts += 1;
return;
}
if (runningSourcesToRemove[sourceURL].count < sourceFailureLimit) {
runningSourcesToRemove[sourceURL].count += 1;
runningSourcesToRemove[sourceURL].timestamp = Date.now();
return;
}
try {
const record = await getFeedByLink(sourceURL);
if (!record) {
log.ERROR("Source not found in storage");
return;
}
const results = await deleteFeedByLink(sourceURL);
if (!results) {
log.WARN("Failed to remove source");
return;
}
log.DEBUG("Source removed after exceeding failure limit:", sourceURL);
} catch (err) {
log.ERROR("Error removing source from storage:", err);
}
};
};