RSS improvements
All checks were successful
DRB Tests / drb_mocha_tests (pull_request) Successful in 31s
All checks were successful
DRB Tests / drb_mocha_tests (pull_request) Successful in 31s
- Updated rss discord command name scheme - Implemented new sourceManager for handling feed sources - Added wrappers to delete/get feed sources by title
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
import { createFeed, getAllFeeds, getFeedByLink, updateFeedByLink, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs';
|
||||
import { getAllFeeds, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs';
|
||||
import crypto from 'crypto';
|
||||
import { sendPost } from '../discordBot/modules/rssWrappers.mjs';
|
||||
import { DebugBuilder } from "../modules/debugger.mjs";
|
||||
import { removeSource } from './sourceManager.mjs'
|
||||
import UserAgent from "user-agents";
|
||||
import Parser from 'rss-parser';
|
||||
|
||||
@@ -19,14 +20,16 @@ const parser = new Parser({
|
||||
});
|
||||
|
||||
const log = new DebugBuilder("server", "feedHandler");
|
||||
const sourceFailureLimit = process.env.RSS_SOURCE_FAILURE_LIMIT ?? 5;
|
||||
const runningSourcesToRemove = {}; // This holds the sources that are pending removal (they've failed to load, return data, etc.)
|
||||
|
||||
|
||||
export const returnHash = (...stringsIncluded) => {
|
||||
return crypto.createHash('sha1').update(stringsIncluded.join("-<<??//\\\\??>>-")).digest("base64");
|
||||
};
|
||||
|
||||
/**
|
||||
* Update the active RSS feeds and send any new posts to their discord channels
|
||||
* @param {any} client The discord client to send posts with
|
||||
* @returns {any}
|
||||
*/
|
||||
export const updateFeeds = async (client) => {
|
||||
if (!client) throw new Error("Client object not passed");
|
||||
|
||||
@@ -87,57 +90,4 @@ export const updateFeeds = async (client) => {
|
||||
log.ERROR("Error updating feeds:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const addSource = async (title, link, category, guildId, channelId, callback) => {
|
||||
try {
|
||||
const feed = { title, link, category, guild_id: guildId, channel_id: channelId };
|
||||
const record = await createFeed(feed);
|
||||
log.DEBUG("Source added:", record);
|
||||
callback(null, record);
|
||||
} catch (err) {
|
||||
log.ERROR("Error adding source:", err);
|
||||
callback(err, null);
|
||||
}
|
||||
};
|
||||
|
||||
export const removeSource = async (sourceURL) => {
|
||||
log.INFO("Removing source:", sourceURL);
|
||||
|
||||
if (!runningSourcesToRemove[sourceURL]) {
|
||||
runningSourcesToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 };
|
||||
return;
|
||||
}
|
||||
|
||||
const elapsedTime = Date.now() - runningSourcesToRemove[sourceURL].timestamp;
|
||||
const waitTime = runningSourcesToRemove[sourceURL].count * 30000;
|
||||
|
||||
if (elapsedTime <= waitTime) {
|
||||
runningSourcesToRemove[sourceURL].ignoredAttempts += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (runningSourcesToRemove[sourceURL].count < sourceFailureLimit) {
|
||||
runningSourcesToRemove[sourceURL].count += 1;
|
||||
runningSourcesToRemove[sourceURL].timestamp = Date.now();
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const record = await getFeedByLink(sourceURL);
|
||||
if (!record) {
|
||||
log.ERROR("Source not found in storage");
|
||||
return;
|
||||
}
|
||||
|
||||
const results = await deleteFeedByLink(sourceURL);
|
||||
if (!results) {
|
||||
log.WARN("Failed to remove source");
|
||||
return;
|
||||
}
|
||||
|
||||
log.DEBUG("Source removed after exceeding failure limit:", sourceURL);
|
||||
} catch (err) {
|
||||
log.ERROR("Error removing source from storage:", err);
|
||||
}
|
||||
};
|
||||
};
|
||||
Reference in New Issue
Block a user