Working on #2

libCore.js

@@ -5,6 +5,7 @@ const { FeedStorage, PostStorage } = require("./libStorage");
 const libUtils = require("./libUtils");
 const { DebugBuilder } = require("./utilities/debugBuilder");
 const log = new DebugBuilder("server", "libCore");
+const mysql = require("mysql");
 
 const UserAgent = require("user-agents");
 process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
@@ -85,6 +86,18 @@ exports.deleteSource = function (title, callback) {
  */
 exports.updateFeeds = async (client) => {
   if (!client) throw new Error("Client object not passed");
+  // Create a temp pool to use for all connections while updating the feed
+  var tempConnection = mysql.createPool({
+    host: process.env.DB_HOST,
+    user: process.env.DB_USER,
+    password: process.env.DB_PASS,
+    database: process.env.DB_NAME,
+    connectionLimit: 10
+  });
+  
+  const feedStorage = new FeedStorage(tempConnection);
+  const postStorage = new PostStorage(tempConnection);
+
   feedStorage.getAllRecords(async (err, records) => {
     // Load the posts from each RSS source
     for (const source of records) {
@@ -103,9 +116,9 @@ exports.updateFeeds = async (client) => {
         try{
           log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);        
           if (parsedFeed?.items){                      
-            for (const post of parsedFeed.items){     
+              for (const post of parsedFeed.items){     
               log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);     
-              //log.VERBOSE("Post from feed: ", post);
+              log.VERBOSE("Post from feed: ", post);
               if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){
                 post.postId = post.postId ?? post.guid ?? post.id;
                 postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
@@ -141,6 +154,10 @@ exports.updateFeeds = async (client) => {
       });
     }
   });
+
+// Close the temp connections
+feedStorage.closeConnection();
+postStorage.closeConnection();
 }
 
 /**

libStorage.js

@@ -16,7 +16,7 @@ const accountsTable = process.env.DB_ACCOUNTS_TABLE;
 const transactionsTable = process.env.DB_TRANSACTIONS_TABLE;
 const pricingTable = process.env.DB_PRICING_TABLE;
 
-var connection = mysql.createPool({
+var Connection = mysql.createPool({
     host: process.env.DB_HOST,
     user: process.env.DB_USER,
     password: process.env.DB_PASS,
@@ -26,14 +26,15 @@ var connection = mysql.createPool({
 
 // Helper Functions
 // Function to run and handle SQL errors
-function runSQL(sqlQuery, callback = (err, rows) => {
+function runSQL(sqlQuery, connection, callback = (err, rows) => {
     log.ERROR(err);
     throw err;
 }) {
-     // Start the MySQL Connection
+    // Start the MySQL Connection
+    if (!connection) connection = Connection;
     connection.query(sqlQuery, (err, rows) => {
         if (err) {
-            log.ERROR("SQL Error:", err)
+            log.ERROR("SQL Error on query:", sqlQuery, err);
             return callback(err, undefined);
         } 
         log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows);        
@@ -51,13 +52,14 @@ function returnMysqlTime(){
 }
 
 class Storage {
-    constructor(_dbTable) {       
+    constructor(_dbTable, connection = undefined) {       
         this.dbTable = _dbTable;
+	this.connection = connection
         this.validKeys = [];
         
         var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`; 
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection,  (err, rows) => {
             if (err) return log.ERROR("Error getting column names: ", err);
             if (rows){
                 for (const validKey of rows){
@@ -96,7 +98,7 @@ class Storage {
 
         const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = "${keyValue}"`;
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection,  (err, rows) => {
             if (err) return callback(err, undefined);
             if (rows[0]?.[key]) return callback(undefined, rows[0]);
             else return callback(undefined, false);
@@ -113,7 +115,7 @@ class Storage {
 
         let records = [];
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
             for (const row of rows) {                
                 if (this.dbTable == rssFeedsTable){
@@ -139,7 +141,7 @@ class Storage {
 
         let records = [];
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
             for (const row of rows) {                
                 if (this.dbTable == rssFeedsTable){
@@ -153,11 +155,21 @@ class Storage {
             return callback(undefined, records);
         });
     }
+
+    closeConnection() {
+        try {
+            this.connection.end();
+        }
+        catch (err) {
+            log.ERROR("Error closing connection :", this.connection, err);
+            throw err;
+        }
+    }
 }
 
 exports.UserStorage = class UserStorage extends Storage {
-    constructor() {
+    constructor(connection = undefined) {
-        super(accountsTable);
+        super(accountsTable, connection);
     }
 
     /**
@@ -171,7 +183,7 @@ exports.UserStorage = class UserStorage extends Storage {
 
         log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
             if (rows?.affectedRows > 0) return callback(undefined, rows);
             return callback(undefined, undefined);
@@ -231,7 +243,7 @@ exports.UserStorage = class UserStorage extends Storage {
 
         log.DEBUG("Updating Balance with SQL Query: ", sqlQuery);
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
             if (!rows?.affectedRows > 0) return callback(new Error("Error updating Balance", rows), undefined);
             return callback(undefined, rows);
@@ -240,8 +252,8 @@ exports.UserStorage = class UserStorage extends Storage {
 }
 
 exports.TransactionStorage = class TransactionStorage extends Storage {
-    constructor() {
+    constructor(connection = undefined) {
-        super(transactionsTable);
+        super(transactionsTable, connection);
     }
     
     createTransaction(transaction, callback){
@@ -249,7 +261,7 @@ exports.TransactionStorage = class TransactionStorage extends Storage {
 
         log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
             if (rows?.affectedRows > 0) return callback(undefined, rows);
             return callback(undefined, undefined);
@@ -258,8 +270,8 @@ exports.TransactionStorage = class TransactionStorage extends Storage {
 }
 
 exports.FeedStorage = class FeedStorage extends Storage {
-    constructor() {
+    constructor(connection = undefined) {
-        super(rssFeedsTable);
+        super(rssFeedsTable, connection);
     }
 
     /**
@@ -316,7 +328,7 @@ exports.FeedStorage = class FeedStorage extends Storage {
 
         log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
             return callback(undefined, rows);
         })
@@ -355,7 +367,7 @@ exports.FeedStorage = class FeedStorage extends Storage {
 
         log.DEBUG(`Updating entry with SQL query: '${sqlQuery}'`)
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
             return callback(undefined, rows);
         })
@@ -373,7 +385,7 @@ exports.FeedStorage = class FeedStorage extends Storage {
     
         const sqlQuery  = `DELETE FROM ${this.dbTable} WHERE id = "${id}";`;
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
             return callback(undefined, rows[0]);
         })
@@ -439,8 +451,8 @@ exports.FeedStorage = class FeedStorage extends Storage {
 }
 
 exports.PostStorage = class PostStorage extends Storage {
-    constructor() {
+    constructor(connection = undefined) {
-        super(rssPostsTable);
+        super(rssPostsTable, connection);
     }    
 
     savePost(_postObject, callback){
@@ -456,7 +468,7 @@ exports.PostStorage = class PostStorage extends Storage {
 
         log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`)
 
-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
             return callback(undefined, rows);
         })

package.json

@@ -8,25 +8,25 @@
     "@discordjs/rest": "~1.5.0",
     "axios": "~1.3.4",
     "chatgpt": "~4.7.2",
+    "cookie-parser": "~1.4.4",
+    "debug": "~2.6.9",
     "discord-api-types": "~0.37.35",
     "discord.js": "~14.7.1",
     "dotenv": "~16.0.3",
+    "ejs": "~2.6.1",
     "express": "~4.18.2",
     "fs": "~0.0.1-security",
+    "gpt-3-encoder": "~1.1.4",
+    "http-errors": "~1.6.3",
     "jsdoc": "~3.6.7",
     "jsonfile": "~6.1.0",
+    "morgan": "~1.9.1",
+    "mysql": "~2.18.1",
+    "node-html-markdown": "~1.3.0",
+    "node-html-parser": "~6.1.5",
     "openai": "~3.1.0",
     "parse-files": "~0.1.1",
     "rss-parser": "~3.12.0",
-    "mysql": "~2.18.1",
-    "cookie-parser": "~1.4.4",
-    "debug": "~2.6.9",
-    "ejs": "~2.6.1",
-    "http-errors": "~1.6.3",
-    "morgan": "~1.9.1",
-    "node-html-markdown": "~1.3.0",
-    "node-html-parser": "~6.1.5",
-    "gpt-3-encoder": "~1.1.4",
     "user-agents": "~1.0.1303"
   },
   "scripts": {