logan/Emmelia-Link-Flayer-Rewrite
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects 2 Releases Wiki Activity

Error handling different RSS feeds

Browse Source
This commit is contained in:
Logan Cusano
2023-03-17 23:27:12 -04:00
parent c8b90ca220
commit edef8a4c7d
3 changed files with 115 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
libCore.js
View File

@@ -25,6 +25,52 @@ let parser = new Parser({
var feedStorage = new FeedStorage();
var postStorage = new PostStorage();
// Initiate a running array of objects to keep track of sources that have no feeds/posts
/*
var runningPostsToRemove = [{
"{SOURCE URL}": {NUMBER OF TIMES IT'S BEEN REMOVED}
}]
*/
var runningPostsToRemove = {};
const sourceFailureLimit = process.env.SOURCE_FAILURE_LIMIT ?? 3;
/**
*
* @param {*} sourceURL
*/
exports.removeSource = function removeSource(sourceURL) {
log.INFO("Removing source URL: ", sourceURL);
if (!sourceURL in runningPostsToRemove) {runningPostsToRemove[sourceURL] = 1; return;}
if (runningPostsToRemove[sourceURL] < sourceFailureLimit) {runningPostsToRemove[sourceURL] += 1; return;}
feedStorage.getRecordBy('link', sourceURL, (err, record) => {
if (err) log.ERROR("Error getting record from feedStorage", err);
if (!record) log.ERROR("No source returned from feedStorage");
feedStorage.destroy(record.id, (err, results) => {
if (err) log.ERROR("Error removing ID from results", err);
if (!results) log.WARN("No results from remove entry");
log.DEBUG("Source exceeded the limit of retries and has been removed", sourceURL);
return;
})
})
}
/**
* Unset a source URL from deletion if the source has not already been deleted
* @param {*} sourceURL The source URL to be unset from deletion
* @returns {*}
*/
exports.unsetRemoveSource = function unsetRemoveSource(sourceURL) {
log.INFO("Unsetting source URL from deletion (if not already deleted): ", sourceURL);
if (!sourceURL in runningPostsToRemove) return;
if (runningPostsToRemove[sourceURL] > sourceFailureLimit) return delete runningPostsToRemove[sourceURL];
}
/**
* Adds or updates new source url to configured storage
* @constructor
@@ -105,29 +151,31 @@ exports.updateFeeds = (client) => {
tempFeedStorage.getAllRecords(async (err, records) => {
// Load the posts from each RSS source
for (const source of records) {
sourcePromiseArray.push(new Promise((resolve, reject) => {
sourcePromiseArray.push(new Promise((resolve, reject) => {
log.DEBUG('Record title: ', source.title);
log.DEBUG('Record link: ', source.link);
log.DEBUG('Record category: ', source.category);
log.DEBUG('Record guild ID: ', source.guild_id);
log.DEBUG('Record channel ID: ', source.channel_id);
log.DEBUG('Record channel ID: ', source.channel_id);
// Parse the RSS feed
parser.parseURL(source.link, async (err, parsedFeed) => {
if (err) {
log.ERROR("Parser Error: ", source, err);
log.ERROR("Parser Error: ", runningPostsToRemove, source, err);
// Call the wrapper to make sure the site isn't just down at the time it checks and is back up the next time
this.removeSource(source.link);
reject;
}
try {
log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
if (parsedFeed?.items){
try {
if (parsedFeed?.items){
this.unsetRemoveSource(source.link);
for (const post of parsedFeed.items){
recordPromiseArray.push(new Promise((recordResolve, recordReject) => {
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
log.VERBOSE("Post from feed: ", post);
if (!post.title || !post.link || !post.pubDate) return recordReject("Missing information from the post");
if (!post.title || !post.link) return recordReject("Missing information from the post");
if (!post.content || !post['content:encoded']) log.WARN("There is no content for post: ", post.title);
post.postId = post.postId ?? post.guid ?? post.id ?? libUtils.returnHash(post.title, post.link, post.pubDate);
post.postId = post.postId ?? post.guid ?? post.id ?? libUtils.returnHash(post.title, post.link, post.pubDate ?? Date.now());
tempPostStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
if (err) throw err;
@@ -152,15 +200,19 @@ exports.updateFeeds = (client) => {
log.DEBUG("Saved results: ", saveResults);
return recordResolve("Saved results", saveResults);
}
});
})
})
});
})
})
}))
}
}
}
}
else {
this.removeSource(source.link);
}
}
catch (err) {
log.ERROR("Error Parsing Feed: ", source.link, err);
this.removeSource(source.link);
throw err;
}
Promise.all(recordPromiseArray).then((values) => {