Working on #2

This commit is contained in:
Logan Cusano
2023-03-11 23:05:50 -05:00
parent 8ed0b969dd
commit 72134b1b7b
3 changed files with 66 additions and 37 deletions

View File

@@ -5,6 +5,7 @@ const { FeedStorage, PostStorage } = require("./libStorage");
const libUtils = require("./libUtils");
const { DebugBuilder } = require("./utilities/debugBuilder");
const log = new DebugBuilder("server", "libCore");
const mysql = require("mysql");
const UserAgent = require("user-agents");
process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
@@ -85,6 +86,18 @@ exports.deleteSource = function (title, callback) {
*/
exports.updateFeeds = async (client) => {
if (!client) throw new Error("Client object not passed");
// Create a temp pool to use for all connections while updating the feed
var tempConnection = mysql.createPool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASS,
database: process.env.DB_NAME,
connectionLimit: 10
});
const feedStorage = new FeedStorage(tempConnection);
const postStorage = new PostStorage(tempConnection);
feedStorage.getAllRecords(async (err, records) => {
// Load the posts from each RSS source
for (const source of records) {
@@ -103,9 +116,9 @@ exports.updateFeeds = async (client) => {
try{
log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
if (parsedFeed?.items){
for (const post of parsedFeed.items){
for (const post of parsedFeed.items){
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
//log.VERBOSE("Post from feed: ", post);
log.VERBOSE("Post from feed: ", post);
if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){
post.postId = post.postId ?? post.guid ?? post.id;
postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
@@ -140,7 +153,11 @@ exports.updateFeeds = async (client) => {
}
});
}
});
});
// Close the temp connections
feedStorage.closeConnection();
postStorage.closeConnection();
}
/**
@@ -290,4 +307,4 @@ exports.getCategories = async (callback) => {
return callback(undefined, results);
});
}
}