logan/drb-server
1
0
Fork 0
Code Issues 16 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Code improvements

Browse Source 
- Better error handling
- More output
- Better logic
This commit is contained in:
Logan Cusano
2024-05-22 02:15:56 -04:00
parent 4e71c7b167
commit 429ddd3333
2 changed files with 75 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
rss-manager/feedHandler.mjs
View File

@@ -3,11 +3,11 @@ import crypto from 'crypto';
import { sendPost } from '../discordBot/modules/rssWrappers.mjs';
import { DebugBuilder } from "../modules/debugger.mjs";
import UserAgent from "user-agents";
import Parser from 'rss-parser';
// Initialize the User-Agent string
process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
// Initiate the parser
import Parser from 'rss-parser';
const parser = new Parser({
headers: {
'User-Agent': process.env.USER_AGENT_STRING,
@@ -16,11 +16,11 @@ const parser = new Parser({
});
const log = new DebugBuilder("server", "feedHandler");
const runningPostsToRemove = {}; // Assuming this is a global state variable
const sourceFailureLimit = 5; // Define your source failure limit here
const runningPostsToRemove = {};
const sourceFailureLimit = 5;
export const returnHash = (...stringsIncluded) => {
return crypto.createHash('sha1').update(`${stringsIncluded.join("-<<??//\\\\??>>-")}`).digest("base64");
return crypto.createHash('sha1').update(stringsIncluded.join("-<<??//\\\\??>>-")).digest("base64");
};
export const updateFeeds = async (client) => {
@@ -30,102 +30,85 @@ export const updateFeeds = async (client) => {
const records = await getAllFeeds();
const sourcePromiseArray = records.map(async (source) => {
log.DEBUG('Record title:', source.title);
log.DEBUG('Record link:', source.link);
log.DEBUG('Record category:', source.category);
log.DEBUG('Record guild ID:', source.guild_id);
log.DEBUG('Record channel ID:', source.channel_id);
log.DEBUG('Processing source:', source.title);
try {
const parsedFeed = await parser.parseURL(source.link);
if (parsedFeed?.items) {
await Promise.all(parsedFeed.items.reverse().map(async (post) => {
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
log.VERBOSE("Post from feed:", post);
log.DEBUG("Processing post:", post.title);
if (!post.title || !post.link) throw new Error("Missing information from the post");
if (!post.content || !post['content:encoded']) log.WARN("There is no content for post:", post.title);
if (!post.title || !post.link) throw new Error("Missing title or link in the post");
if (!post.content && !post['content:encoded']) log.WARN("No content for post:", post.title);
post.postId = post.postId ?? post.guid ?? post.id ?? returnHash(post.title, post.link, post.pubDate ?? Date.now());
const existingRecord = await getPostByPostId(post.postId);
log.DEBUG("Existing post record:", existingRecord);
if (!existingRecord) {
const channel = client.channels.cache.get(source.channel_id);
const channel = client.channels.cache.get(source.channel_id);
const sendResults = await sendPost(post, source, channel);
if (!sendResults) throw new Error("Failed to send post");
if (!sendResults) throw new Error("No sending results from sending a post");
log.DEBUG("Saving post to database:", post.title, source.channel_id);
log.DEBUG("Saving post to database:", sendResults, post.title, source.channel_id);
//await createPost(post);
log.DEBUG("Saved post:", post);
const postToSave = {
title: post.title,
link: post.link,
pubDate: post.pubDate,
author: post.author,
contentSnippet: post.contentSnippet,
id: post.id,
isoDate: post.isoDate,
postId: post.postId
};
await createPost(postToSave);
log.DEBUG("Post saved:", postToSave);
}
}));
} else {
await deleteFeedByLink(source.link);
}
} catch (err) {
log.ERROR("Parser Error:", source, err);
log.ERROR("Error processing source:", source.title, err);
await removeSource(source.link);
throw err;
}
});
await Promise.all(sourcePromiseArray);
log.DEBUG("All sources finished");
log.DEBUG("All sources processed");
} catch (error) {
log.ERROR("Error updating feeds:", error);
throw error;
}
};
/**
* Adds or updates new source URL to configured storage.
* @param {string} title - Title/Name of the RSS feed.
* @param {string} link - URL of RSS feed.
* @param {string} category - Category of RSS feed.
* @param {string} guildId - Guild ID of RSS feed.
* @param {string} channelId - Channel ID of RSS feed.
* @param {function} callback - Callback function.
*/
export const addSource = async (title, link, category, guildId, channelId, callback) => {
try {
const feed = {
title,
link,
category,
guild_id: guildId,
channel_id: channelId
};
const feed = { title, link, category, guild_id: guildId, channel_id: channelId };
const record = await createFeed(feed);
log.DEBUG("Record ID:", record);
return callback(null, record);
log.DEBUG("Source added:", record);
callback(null, record);
} catch (err) {
log.ERROR("Error in create:", err);
return callback(err, null);
log.ERROR("Error adding source:", err);
callback(err, null);
}
};
/**
* Wrapper for feeds that cause errors. By default it will wait over a day for the source to come back online before deleting it.
* @param {string} sourceURL - The URL of the feed source causing issues.
*/
export const removeSource = async (sourceURL) => {
log.INFO("Removing source URL:", sourceURL);
log.INFO("Removing source:", sourceURL);
if (!runningPostsToRemove[sourceURL]) {
runningPostsToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 };
return;
}
const backoffDateTimeDifference = (Date.now() - runningPostsToRemove[sourceURL].timestamp);
const backoffWaitTime = (runningPostsToRemove[sourceURL].count * 30000);
const elapsedTime = Date.now() - runningPostsToRemove[sourceURL].timestamp;
const waitTime = runningPostsToRemove[sourceURL].count * 30000;
log.DEBUG("Datetime", runningPostsToRemove[sourceURL], backoffDateTimeDifference, backoffWaitTime);
if (backoffDateTimeDifference <= backoffWaitTime) {
if (elapsedTime <= waitTime) {
runningPostsToRemove[sourceURL].ignoredAttempts += 1;
return;
}
@@ -139,18 +122,18 @@ export const removeSource = async (sourceURL) => {
try {
const record = await getFeedByLink(sourceURL);
if (!record) {
log.ERROR("No source returned from feedStorage");
log.ERROR("Source not found in storage");
return;
}
const results = await deleteFeedByLink(sourceURL);
if (!results) {
log.WARN("No results from remove entry");
log.WARN("Failed to remove source");
return;
}
log.DEBUG("Source exceeded the limit of retries and has been removed", sourceURL);
log.DEBUG("Source removed after exceeding failure limit:", sourceURL);
} catch (err) {
log.ERROR("Error removing source from feedStorage", err);
log.ERROR("Error removing source from storage:", err);
}
};