From 72134b1b7ba2e8e151e9b9ff8b7de0a1af6aa51e Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sat, 11 Mar 2023 23:05:50 -0500 Subject: [PATCH 1/2] Working on #2 --- libCore.js | 25 +++++++++++++++++---- libStorage.js | 60 ++++++++++++++++++++++++++++++--------------------- package.json | 18 ++++++++-------- 3 files changed, 66 insertions(+), 37 deletions(-) diff --git a/libCore.js b/libCore.js index f7ef6de..0cd040b 100644 --- a/libCore.js +++ b/libCore.js @@ -5,6 +5,7 @@ const { FeedStorage, PostStorage } = require("./libStorage"); const libUtils = require("./libUtils"); const { DebugBuilder } = require("./utilities/debugBuilder"); const log = new DebugBuilder("server", "libCore"); +const mysql = require("mysql"); const UserAgent = require("user-agents"); process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString(); @@ -85,6 +86,18 @@ exports.deleteSource = function (title, callback) { */ exports.updateFeeds = async (client) => { if (!client) throw new Error("Client object not passed"); + // Create a temp pool to use for all connections while updating the feed + var tempConnection = mysql.createPool({ + host: process.env.DB_HOST, + user: process.env.DB_USER, + password: process.env.DB_PASS, + database: process.env.DB_NAME, + connectionLimit: 10 + }); + + const feedStorage = new FeedStorage(tempConnection); + const postStorage = new PostStorage(tempConnection); + feedStorage.getAllRecords(async (err, records) => { // Load the posts from each RSS source for (const source of records) { @@ -103,9 +116,9 @@ exports.updateFeeds = async (client) => { try{ log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title); if (parsedFeed?.items){ - for (const post of parsedFeed.items){ + for (const post of parsedFeed.items){ log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title); - //log.VERBOSE("Post from feed: ", post); + log.VERBOSE("Post from feed: ", post); if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){ post.postId = post.postId ?? post.guid ?? post.id; postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => { @@ -140,7 +153,11 @@ exports.updateFeeds = async (client) => { } }); } - }); + }); + +// Close the temp connections +feedStorage.closeConnection(); +postStorage.closeConnection(); } /** @@ -290,4 +307,4 @@ exports.getCategories = async (callback) => { return callback(undefined, results); }); -} \ No newline at end of file +} diff --git a/libStorage.js b/libStorage.js index 3ee75f0..1af8a3a 100644 --- a/libStorage.js +++ b/libStorage.js @@ -16,7 +16,7 @@ const accountsTable = process.env.DB_ACCOUNTS_TABLE; const transactionsTable = process.env.DB_TRANSACTIONS_TABLE; const pricingTable = process.env.DB_PRICING_TABLE; -var connection = mysql.createPool({ +var Connection = mysql.createPool({ host: process.env.DB_HOST, user: process.env.DB_USER, password: process.env.DB_PASS, @@ -26,14 +26,15 @@ var connection = mysql.createPool({ // Helper Functions // Function to run and handle SQL errors -function runSQL(sqlQuery, callback = (err, rows) => { +function runSQL(sqlQuery, connection, callback = (err, rows) => { log.ERROR(err); throw err; }) { - // Start the MySQL Connection + // Start the MySQL Connection + if (!connection) connection = Connection; connection.query(sqlQuery, (err, rows) => { if (err) { - log.ERROR("SQL Error:", err) + log.ERROR("SQL Error on query:", sqlQuery, err); return callback(err, undefined); } log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows); @@ -51,13 +52,14 @@ function returnMysqlTime(){ } class Storage { - constructor(_dbTable) { + constructor(_dbTable, connection = undefined) { this.dbTable = _dbTable; + this.connection = connection this.validKeys = []; var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`; - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return log.ERROR("Error getting column names: ", err); if (rows){ for (const validKey of rows){ @@ -96,7 +98,7 @@ class Storage { const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = "${keyValue}"`; - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); if (rows[0]?.[key]) return callback(undefined, rows[0]); else return callback(undefined, false); @@ -113,7 +115,7 @@ class Storage { let records = []; - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); for (const row of rows) { if (this.dbTable == rssFeedsTable){ @@ -139,7 +141,7 @@ class Storage { let records = []; - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); for (const row of rows) { if (this.dbTable == rssFeedsTable){ @@ -153,11 +155,21 @@ class Storage { return callback(undefined, records); }); } + + closeConnection() { + try { + this.connection.end(); + } + catch (err) { + log.ERROR("Error closing connection :", this.connection, err); + throw err; + } + } } exports.UserStorage = class UserStorage extends Storage { - constructor() { - super(accountsTable); + constructor(connection = undefined) { + super(accountsTable, connection); } /** @@ -171,7 +183,7 @@ exports.UserStorage = class UserStorage extends Storage { log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); if (rows?.affectedRows > 0) return callback(undefined, rows); return callback(undefined, undefined); @@ -231,7 +243,7 @@ exports.UserStorage = class UserStorage extends Storage { log.DEBUG("Updating Balance with SQL Query: ", sqlQuery); - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); if (!rows?.affectedRows > 0) return callback(new Error("Error updating Balance", rows), undefined); return callback(undefined, rows); @@ -240,8 +252,8 @@ exports.UserStorage = class UserStorage extends Storage { } exports.TransactionStorage = class TransactionStorage extends Storage { - constructor() { - super(transactionsTable); + constructor(connection = undefined) { + super(transactionsTable, connection); } createTransaction(transaction, callback){ @@ -249,7 +261,7 @@ exports.TransactionStorage = class TransactionStorage extends Storage { log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); if (rows?.affectedRows > 0) return callback(undefined, rows); return callback(undefined, undefined); @@ -258,8 +270,8 @@ exports.TransactionStorage = class TransactionStorage extends Storage { } exports.FeedStorage = class FeedStorage extends Storage { - constructor() { - super(rssFeedsTable); + constructor(connection = undefined) { + super(rssFeedsTable, connection); } /** @@ -316,7 +328,7 @@ exports.FeedStorage = class FeedStorage extends Storage { log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); return callback(undefined, rows); }) @@ -355,7 +367,7 @@ exports.FeedStorage = class FeedStorage extends Storage { log.DEBUG(`Updating entry with SQL query: '${sqlQuery}'`) - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); return callback(undefined, rows); }) @@ -373,7 +385,7 @@ exports.FeedStorage = class FeedStorage extends Storage { const sqlQuery = `DELETE FROM ${this.dbTable} WHERE id = "${id}";`; - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); return callback(undefined, rows[0]); }) @@ -439,8 +451,8 @@ exports.FeedStorage = class FeedStorage extends Storage { } exports.PostStorage = class PostStorage extends Storage { - constructor() { - super(rssPostsTable); + constructor(connection = undefined) { + super(rssPostsTable, connection); } savePost(_postObject, callback){ @@ -456,7 +468,7 @@ exports.PostStorage = class PostStorage extends Storage { log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`) - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); return callback(undefined, rows); }) diff --git a/package.json b/package.json index 1d3b406..b023e06 100644 --- a/package.json +++ b/package.json @@ -8,25 +8,25 @@ "@discordjs/rest": "~1.5.0", "axios": "~1.3.4", "chatgpt": "~4.7.2", + "cookie-parser": "~1.4.4", + "debug": "~2.6.9", "discord-api-types": "~0.37.35", "discord.js": "~14.7.1", "dotenv": "~16.0.3", + "ejs": "~2.6.1", "express": "~4.18.2", "fs": "~0.0.1-security", + "gpt-3-encoder": "~1.1.4", + "http-errors": "~1.6.3", "jsdoc": "~3.6.7", "jsonfile": "~6.1.0", + "morgan": "~1.9.1", + "mysql": "~2.18.1", + "node-html-markdown": "~1.3.0", + "node-html-parser": "~6.1.5", "openai": "~3.1.0", "parse-files": "~0.1.1", "rss-parser": "~3.12.0", - "mysql": "~2.18.1", - "cookie-parser": "~1.4.4", - "debug": "~2.6.9", - "ejs": "~2.6.1", - "http-errors": "~1.6.3", - "morgan": "~1.9.1", - "node-html-markdown": "~1.3.0", - "node-html-parser": "~6.1.5", - "gpt-3-encoder": "~1.1.4", "user-agents": "~1.0.1303" }, "scripts": { From ffacd198836ca6089447857e9a7b12dbab301f4a Mon Sep 17 00:00:00 2001 From: Logan Cusano Date: Sun, 12 Mar 2023 03:47:49 -0400 Subject: [PATCH 2/2] Potential fix fo #2 --- libCore.js | 125 +++++++++++++++++++++++++++++--------------------- libStorage.js | 4 +- libUtils.js | 32 +++++++++++-- 3 files changed, 102 insertions(+), 59 deletions(-) diff --git a/libCore.js b/libCore.js index 0cd040b..5eb6deb 100644 --- a/libCore.js +++ b/libCore.js @@ -84,7 +84,7 @@ exports.deleteSource = function (title, callback) { /** * Update channels with new posts from sources */ -exports.updateFeeds = async (client) => { +exports.updateFeeds = (client) => { if (!client) throw new Error("Client object not passed"); // Create a temp pool to use for all connections while updating the feed var tempConnection = mysql.createPool({ @@ -95,69 +95,90 @@ exports.updateFeeds = async (client) => { connectionLimit: 10 }); - const feedStorage = new FeedStorage(tempConnection); - const postStorage = new PostStorage(tempConnection); + const tempFeedStorage = new FeedStorage(tempConnection); + const tempPostStorage = new PostStorage(tempConnection); - feedStorage.getAllRecords(async (err, records) => { + // Array of promises to wait on before closing the connection + var recordPromiseArray = []; + var sourcePromiseArray = []; + + tempFeedStorage.getAllRecords(async (err, records) => { // Load the posts from each RSS source - for (const source of records) { - log.DEBUG('Record title: ', source.title); - log.DEBUG('Record link: ', source.link); - log.DEBUG('Record category: ', source.category); - log.DEBUG('Record guild ID: ', source.guild_id); - log.DEBUG('Record channel ID: ', source.channel_id); + for (const source of records) { + sourcePromiseArray.push(new Promise((resolve, reject) => { + log.DEBUG('Record title: ', source.title); + log.DEBUG('Record link: ', source.link); + log.DEBUG('Record category: ', source.category); + log.DEBUG('Record guild ID: ', source.guild_id); + log.DEBUG('Record channel ID: ', source.channel_id); + // Parse the RSS feed + parser.parseURL(source.link, async (err, parsedFeed) => { + if (err) { + log.ERROR("Parser Error: ", source, err); + reject; + } + try { + log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title); + if (parsedFeed?.items){ + for (const post of parsedFeed.items){ + recordPromiseArray.push(new Promise((recordResolve, recordReject) => { + log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title); + log.VERBOSE("Post from feed: ", post); + if (!post.title || !post.link || !post.pubDate) return recordReject("Missing information from the post"); + if (!post.content || !post['content:encoded']) log.WARN("There is no content for post: ", post.title); - await parser.parseURL(source.link, async (err, parsedFeed) => { - if (err) { - log.ERROR("Parser Error: ", source, err); - return; - } + post.postId = post.postId ?? post.guid ?? post.id ?? libUtils.returnHash(post.title, post.link, post.pubDate); + tempPostStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => { + if (err) throw err; + + log.DEBUG("Existing post record: ", existingRecord); + if (existingRecord) return recordResolve("Existing record found for this post"); - try{ - log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title); - if (parsedFeed?.items){ - for (const post of parsedFeed.items){ - log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title); - log.VERBOSE("Post from feed: ", post); - if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){ - post.postId = post.postId ?? post.guid ?? post.id; - postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => { - if (err) throw err; - log.DEBUG("Existing post record: ", existingRecord); - if (!existingRecord){ const channel = client.channels.cache.get(source.channel_id); libUtils.sendPost(post, source, channel, (err, sendResults) =>{ if (err) throw err; + + if (!sendResults) { + log.ERROR("No sending results from sending a post: ", sendResults, existingRecord, post); + return recordReject(); + } - if (sendResults){ - log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id); + log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id); - postStorage.savePost(post, (err, saveResults) => { - if(err) throw err; + tempPostStorage.savePost(post, (err, saveResults) => { + if(err) throw err; - if (saveResults) { - log.DEBUG("Saved results: ", saveResults); - return; - } - }); - } - }) - } - }) - } - } - } - }catch (err) { - log.ERROR("Error Parsing Feed: ", source.link, err); - throw err; - } - }); + if (saveResults) { + log.DEBUG("Saved results: ", saveResults); + return recordResolve(); + } + }); + }) + }) + })) + } + } + } + catch (err) { + log.ERROR("Error Parsing Feed: ", source.link, err); + throw err; + } + Promise.all(recordPromiseArray).then((values) => { + log.DEBUG("All posts finished for: ", source.title, values); + return resolve(); + }); + }); + })) } - }); -// Close the temp connections -feedStorage.closeConnection(); -postStorage.closeConnection(); + // Wait for all connections to finish then close the temp connections + + Promise.all(sourcePromiseArray).then((values) => { + log.DEBUG("Closing temp connections: ", values); + tempFeedStorage.closeConnection(); + tempPostStorage.closeConnection(); + }); + }); } /** diff --git a/libStorage.js b/libStorage.js index 1af8a3a..90f429f 100644 --- a/libStorage.js +++ b/libStorage.js @@ -52,9 +52,9 @@ function returnMysqlTime(){ } class Storage { - constructor(_dbTable, connection = undefined) { + constructor(_dbTable, _connection) { this.dbTable = _dbTable; - this.connection = connection + this.connection = _connection; this.validKeys = []; var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`; diff --git a/libUtils.js b/libUtils.js index b5febb3..2f29023 100644 --- a/libUtils.js +++ b/libUtils.js @@ -3,8 +3,10 @@ const { DebugBuilder } = require("./utilities/debugBuilder"); const log = new DebugBuilder("server", "libUtils"); const { NodeHtmlMarkdown } = require('node-html-markdown'); const { parse } = require("node-html-parser"); +const crypto = require("crypto"); const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g; +const youtubeVideoRegex = /((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)/g exports.EmmeliaEmbedBuilder = class PostEmbedBuilder extends EmbedBuilder { constructor() { @@ -74,10 +76,24 @@ exports.onError = (error) => { exports.sendPost = (post, source, channel, callback) => { log.DEBUG("Sending post from source: ", post, source); - post.content = parse(post.content); + // Reset the content parameter with the encoded parameter + post.content = parse(post['content:encoded'] ?? post.content); const postTitle = post.title; const postLink = post.link; - const postContent = NodeHtmlMarkdown.translate(post.content.text); + // Get the post content and trim it to length or add a placeholder if necessary + var postText = String(post.content.text); + if (postText.length >= 300) postText = `${postText.slice(0, 300).substring(0, Math.min(String(post.content.text).length, String(post.content.text).lastIndexOf(" ")))}...`; + else if (postText.length === 0) postText = `*This post has no content* [Direct Link](${post.link})`; + var postContent = postText; + // Check for embedded youtube videos and add the first four as links + const ytVideos = String(post.content).match(youtubeVideoRegex); + if (ytVideos) { + for (const ytVideo of ytVideos.slice(0,4)){ + // If the video is an embed, replace the embed to make it watchable + if (ytVideo.includes("embed")) ytVideo = ytVideo.replace("embed/", "watch?v="); + postContent += `\nEmbeded Video from Post: [YouTube](${ytVideo})` + } + } log.DEBUG("Post content: ", postContent); const postId = post.postId; @@ -89,7 +105,9 @@ exports.sendPost = (post, source, channel, callback) => { const linksInPost = post.content.querySelectorAll("a"); if (linksInPost) { log.DEBUG("Found links in post:", linksInPost); - for (const link of linksInPost) { + for (const link of linksInPost) { + // Check to see if this link is a youtube video that was already found, if so skip it + if (ytVideos?.includes(link)) continue; const images = String(link.getAttribute("href")).match(imageRegex); log.DEBUG("Images found in post:", images); if (images) { @@ -99,13 +117,13 @@ exports.sendPost = (post, source, channel, callback) => { } } - log.DEBUG("Sending an RSS post to discord", postTitle, postId) + log.DEBUG("Sending an RSS post to discord", postTitle, postId, postContent) try{ const rssMessage = new this.EmmeliaEmbedBuilder() .setColor(0x0099FF) .setTitle(postTitle) .setURL(postLink) - .addFields({ name: "Post Content", value: postContent.slice(0,1024), inline: false }) + .addFields({ name: "Post Content", value: postContent, inline: false }) .addFields({ name: 'Published', value: postPubDate, inline: true }) .addFields({ name: 'Source', value: postSourceLink, inline: true }); @@ -125,4 +143,8 @@ exports.sendPost = (post, source, channel, callback) => { log.ERROR("Error sending message: ", err); return callback(err, undefined); } +} + +exports.returnHash = (...stringsIncluded) => { + return crypto.createHash('sha1').update(`${stringsIncluded.join("-<>-")}`).digest("base64"); } \ No newline at end of file