diff --git a/libCore.js b/libCore.js index f7ef6de..0cd040b 100644 --- a/libCore.js +++ b/libCore.js @@ -5,6 +5,7 @@ const { FeedStorage, PostStorage } = require("./libStorage"); const libUtils = require("./libUtils"); const { DebugBuilder } = require("./utilities/debugBuilder"); const log = new DebugBuilder("server", "libCore"); +const mysql = require("mysql"); const UserAgent = require("user-agents"); process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString(); @@ -85,6 +86,18 @@ exports.deleteSource = function (title, callback) { */ exports.updateFeeds = async (client) => { if (!client) throw new Error("Client object not passed"); + // Create a temp pool to use for all connections while updating the feed + var tempConnection = mysql.createPool({ + host: process.env.DB_HOST, + user: process.env.DB_USER, + password: process.env.DB_PASS, + database: process.env.DB_NAME, + connectionLimit: 10 + }); + + const feedStorage = new FeedStorage(tempConnection); + const postStorage = new PostStorage(tempConnection); + feedStorage.getAllRecords(async (err, records) => { // Load the posts from each RSS source for (const source of records) { @@ -103,9 +116,9 @@ exports.updateFeeds = async (client) => { try{ log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title); if (parsedFeed?.items){ - for (const post of parsedFeed.items){ + for (const post of parsedFeed.items){ log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title); - //log.VERBOSE("Post from feed: ", post); + log.VERBOSE("Post from feed: ", post); if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){ post.postId = post.postId ?? post.guid ?? post.id; postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => { @@ -140,7 +153,11 @@ exports.updateFeeds = async (client) => { } }); } - }); + }); + +// Close the temp connections +feedStorage.closeConnection(); +postStorage.closeConnection(); } /** @@ -290,4 +307,4 @@ exports.getCategories = async (callback) => { return callback(undefined, results); }); -} \ No newline at end of file +} diff --git a/libStorage.js b/libStorage.js index 3ee75f0..1af8a3a 100644 --- a/libStorage.js +++ b/libStorage.js @@ -16,7 +16,7 @@ const accountsTable = process.env.DB_ACCOUNTS_TABLE; const transactionsTable = process.env.DB_TRANSACTIONS_TABLE; const pricingTable = process.env.DB_PRICING_TABLE; -var connection = mysql.createPool({ +var Connection = mysql.createPool({ host: process.env.DB_HOST, user: process.env.DB_USER, password: process.env.DB_PASS, @@ -26,14 +26,15 @@ var connection = mysql.createPool({ // Helper Functions // Function to run and handle SQL errors -function runSQL(sqlQuery, callback = (err, rows) => { +function runSQL(sqlQuery, connection, callback = (err, rows) => { log.ERROR(err); throw err; }) { - // Start the MySQL Connection + // Start the MySQL Connection + if (!connection) connection = Connection; connection.query(sqlQuery, (err, rows) => { if (err) { - log.ERROR("SQL Error:", err) + log.ERROR("SQL Error on query:", sqlQuery, err); return callback(err, undefined); } log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows); @@ -51,13 +52,14 @@ function returnMysqlTime(){ } class Storage { - constructor(_dbTable) { + constructor(_dbTable, connection = undefined) { this.dbTable = _dbTable; + this.connection = connection this.validKeys = []; var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`; - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return log.ERROR("Error getting column names: ", err); if (rows){ for (const validKey of rows){ @@ -96,7 +98,7 @@ class Storage { const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = "${keyValue}"`; - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); if (rows[0]?.[key]) return callback(undefined, rows[0]); else return callback(undefined, false); @@ -113,7 +115,7 @@ class Storage { let records = []; - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); for (const row of rows) { if (this.dbTable == rssFeedsTable){ @@ -139,7 +141,7 @@ class Storage { let records = []; - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); for (const row of rows) { if (this.dbTable == rssFeedsTable){ @@ -153,11 +155,21 @@ class Storage { return callback(undefined, records); }); } + + closeConnection() { + try { + this.connection.end(); + } + catch (err) { + log.ERROR("Error closing connection :", this.connection, err); + throw err; + } + } } exports.UserStorage = class UserStorage extends Storage { - constructor() { - super(accountsTable); + constructor(connection = undefined) { + super(accountsTable, connection); } /** @@ -171,7 +183,7 @@ exports.UserStorage = class UserStorage extends Storage { log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); if (rows?.affectedRows > 0) return callback(undefined, rows); return callback(undefined, undefined); @@ -231,7 +243,7 @@ exports.UserStorage = class UserStorage extends Storage { log.DEBUG("Updating Balance with SQL Query: ", sqlQuery); - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); if (!rows?.affectedRows > 0) return callback(new Error("Error updating Balance", rows), undefined); return callback(undefined, rows); @@ -240,8 +252,8 @@ exports.UserStorage = class UserStorage extends Storage { } exports.TransactionStorage = class TransactionStorage extends Storage { - constructor() { - super(transactionsTable); + constructor(connection = undefined) { + super(transactionsTable, connection); } createTransaction(transaction, callback){ @@ -249,7 +261,7 @@ exports.TransactionStorage = class TransactionStorage extends Storage { log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); if (rows?.affectedRows > 0) return callback(undefined, rows); return callback(undefined, undefined); @@ -258,8 +270,8 @@ exports.TransactionStorage = class TransactionStorage extends Storage { } exports.FeedStorage = class FeedStorage extends Storage { - constructor() { - super(rssFeedsTable); + constructor(connection = undefined) { + super(rssFeedsTable, connection); } /** @@ -316,7 +328,7 @@ exports.FeedStorage = class FeedStorage extends Storage { log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); return callback(undefined, rows); }) @@ -355,7 +367,7 @@ exports.FeedStorage = class FeedStorage extends Storage { log.DEBUG(`Updating entry with SQL query: '${sqlQuery}'`) - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); return callback(undefined, rows); }) @@ -373,7 +385,7 @@ exports.FeedStorage = class FeedStorage extends Storage { const sqlQuery = `DELETE FROM ${this.dbTable} WHERE id = "${id}";`; - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); return callback(undefined, rows[0]); }) @@ -439,8 +451,8 @@ exports.FeedStorage = class FeedStorage extends Storage { } exports.PostStorage = class PostStorage extends Storage { - constructor() { - super(rssPostsTable); + constructor(connection = undefined) { + super(rssPostsTable, connection); } savePost(_postObject, callback){ @@ -456,7 +468,7 @@ exports.PostStorage = class PostStorage extends Storage { log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`) - runSQL(sqlQuery, (err, rows) => { + runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); return callback(undefined, rows); }) diff --git a/package.json b/package.json index 1d3b406..b023e06 100644 --- a/package.json +++ b/package.json @@ -8,25 +8,25 @@ "@discordjs/rest": "~1.5.0", "axios": "~1.3.4", "chatgpt": "~4.7.2", + "cookie-parser": "~1.4.4", + "debug": "~2.6.9", "discord-api-types": "~0.37.35", "discord.js": "~14.7.1", "dotenv": "~16.0.3", + "ejs": "~2.6.1", "express": "~4.18.2", "fs": "~0.0.1-security", + "gpt-3-encoder": "~1.1.4", + "http-errors": "~1.6.3", "jsdoc": "~3.6.7", "jsonfile": "~6.1.0", + "morgan": "~1.9.1", + "mysql": "~2.18.1", + "node-html-markdown": "~1.3.0", + "node-html-parser": "~6.1.5", "openai": "~3.1.0", "parse-files": "~0.1.1", "rss-parser": "~3.12.0", - "mysql": "~2.18.1", - "cookie-parser": "~1.4.4", - "debug": "~2.6.9", - "ejs": "~2.6.1", - "http-errors": "~1.6.3", - "morgan": "~1.9.1", - "node-html-markdown": "~1.3.0", - "node-html-parser": "~6.1.5", - "gpt-3-encoder": "~1.1.4", "user-agents": "~1.0.1303" }, "scripts": {