logan/Emmelia-Link-Flayer-Rewrite
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects 2 Releases Wiki Activity

Potential fix fo #2

Browse Source
This commit is contained in:
Logan Cusano
2023-03-12 03:47:49 -04:00
parent 72134b1b7b
commit ffacd19883
3 changed files with 102 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
libCore.js
View File

@@ -84,7 +84,7 @@ exports.deleteSource = function (title, callback) {
/**
* Update channels with new posts from sources
*/
exports.updateFeeds = async (client) => {
exports.updateFeeds = (client) => {
if (!client) throw new Error("Client object not passed");
// Create a temp pool to use for all connections while updating the feed
var tempConnection = mysql.createPool({
@@ -95,69 +95,90 @@ exports.updateFeeds = async (client) => {
connectionLimit: 10
});
const feedStorage = new FeedStorage(tempConnection);
const postStorage = new PostStorage(tempConnection);
const tempFeedStorage = new FeedStorage(tempConnection);
const tempPostStorage = new PostStorage(tempConnection);
feedStorage.getAllRecords(async (err, records) => {
// Array of promises to wait on before closing the connection
var recordPromiseArray = [];
var sourcePromiseArray = [];
tempFeedStorage.getAllRecords(async (err, records) => {
// Load the posts from each RSS source
for (const source of records) {
log.DEBUG('Record title: ', source.title);
log.DEBUG('Record link: ', source.link);
log.DEBUG('Record category: ', source.category);
log.DEBUG('Record guild ID: ', source.guild_id);
log.DEBUG('Record channel ID: ', source.channel_id);
for (const source of records) {
sourcePromiseArray.push(new Promise((resolve, reject) => {
log.DEBUG('Record title: ', source.title);
log.DEBUG('Record link: ', source.link);
log.DEBUG('Record category: ', source.category);
log.DEBUG('Record guild ID: ', source.guild_id);
log.DEBUG('Record channel ID: ', source.channel_id);
// Parse the RSS feed
parser.parseURL(source.link, async (err, parsedFeed) => {
if (err) {
log.ERROR("Parser Error: ", source, err);
reject;
}
try {
log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
if (parsedFeed?.items){
for (const post of parsedFeed.items){
recordPromiseArray.push(new Promise((recordResolve, recordReject) => {
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
log.VERBOSE("Post from feed: ", post);
if (!post.title || !post.link || !post.pubDate) return recordReject("Missing information from the post");
if (!post.content || !post['content:encoded']) log.WARN("There is no content for post: ", post.title);
await parser.parseURL(source.link, async (err, parsedFeed) => {
if (err) {
log.ERROR("Parser Error: ", source, err);
return;
}
post.postId = post.postId ?? post.guid ?? post.id ?? libUtils.returnHash(post.title, post.link, post.pubDate);
tempPostStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
if (err) throw err;
log.DEBUG("Existing post record: ", existingRecord);
if (existingRecord) return recordResolve("Existing record found for this post");
try{
log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
if (parsedFeed?.items){
for (const post of parsedFeed.items){
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
log.VERBOSE("Post from feed: ", post);
if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){
post.postId = post.postId ?? post.guid ?? post.id;
postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
if (err) throw err;
log.DEBUG("Existing post record: ", existingRecord);
if (!existingRecord){
const channel = client.channels.cache.get(source.channel_id);
libUtils.sendPost(post, source, channel, (err, sendResults) =>{
if (err) throw err;
if (!sendResults) {
log.ERROR("No sending results from sending a post: ", sendResults, existingRecord, post);
return recordReject();
}
if (sendResults){
log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id);
log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id);
postStorage.savePost(post, (err, saveResults) => {
if(err) throw err;
tempPostStorage.savePost(post, (err, saveResults) => {
if(err) throw err;
if (saveResults) {
log.DEBUG("Saved results: ", saveResults);
return;
}
});
}
})
}
})
}
}
}
}catch (err) {
log.ERROR("Error Parsing Feed: ", source.link, err);
throw err;
}
});
if (saveResults) {
log.DEBUG("Saved results: ", saveResults);
return recordResolve();
}
});
})
})
}))
}
}
}
catch (err) {
log.ERROR("Error Parsing Feed: ", source.link, err);
throw err;
}
Promise.all(recordPromiseArray).then((values) => {
log.DEBUG("All posts finished for: ", source.title, values);
return resolve();
});
});
}))
}
});
// Close the temp connections
feedStorage.closeConnection();
postStorage.closeConnection();
// Wait for all connections to finish then close the temp connections
Promise.all(sourcePromiseArray).then((values) => {
log.DEBUG("Closing temp connections: ", values);
tempFeedStorage.closeConnection();
tempPostStorage.closeConnection();
});
});
}
/**