Working on #2

2023-03-11 23:05:50 -05:00
parent 8ed0b969dd
commit 72134b1b7b
3 changed files with 66 additions and 37 deletions
--- a/libCore.js
+++ b/libCore.js
@@ -5,6 +5,7 @@ const { FeedStorage, PostStorage } = require("./libStorage");
 const libUtils = require("./libUtils");
 const { DebugBuilder } = require("./utilities/debugBuilder");
 const log = new DebugBuilder("server", "libCore");
+const mysql = require("mysql");

 const UserAgent = require("user-agents");
 process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
@@ -85,6 +86,18 @@ exports.deleteSource = function (title, callback) {
 */
 exports.updateFeeds = async (client) => {
  if (!client) throw new Error("Client object not passed");
+  // Create a temp pool to use for all connections while updating the feed
+  var tempConnection = mysql.createPool({
+    host: process.env.DB_HOST,
+    user: process.env.DB_USER,
+    password: process.env.DB_PASS,
+    database: process.env.DB_NAME,
+    connectionLimit: 10
+  });
+  
+  const feedStorage = new FeedStorage(tempConnection);
+  const postStorage = new PostStorage(tempConnection);
+
  feedStorage.getAllRecords(async (err, records) => {
    // Load the posts from each RSS source
    for (const source of records) {
@@ -103,9 +116,9 @@ exports.updateFeeds = async (client) => {
        try{
          log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);        
          if (parsedFeed?.items){                      
-            for (const post of parsedFeed.items){     
+              for (const post of parsedFeed.items){     
              log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);     
-              //log.VERBOSE("Post from feed: ", post);
+              log.VERBOSE("Post from feed: ", post);
              if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){
                post.postId = post.postId ?? post.guid ?? post.id;
                postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
@@ -140,7 +153,11 @@ exports.updateFeeds = async (client) => {
        }        
      });
    }
-  });  
+  });
+
+// Close the temp connections
+feedStorage.closeConnection();
+postStorage.closeConnection();
 }

 /**
@@ -290,4 +307,4 @@ exports.getCategories = async (callback) => {
    
    return callback(undefined, results);    
  });
-}
+}
--- a/libStorage.js
+++ b/libStorage.js
@@ -16,7 +16,7 @@ const accountsTable = process.env.DB_ACCOUNTS_TABLE;
 const transactionsTable = process.env.DB_TRANSACTIONS_TABLE;
 const pricingTable = process.env.DB_PRICING_TABLE;

-var connection = mysql.createPool({
+var Connection = mysql.createPool({
    host: process.env.DB_HOST,
    user: process.env.DB_USER,
    password: process.env.DB_PASS,
@@ -26,14 +26,15 @@ var connection = mysql.createPool({

 // Helper Functions
 // Function to run and handle SQL errors
-function runSQL(sqlQuery, callback = (err, rows) => {
+function runSQL(sqlQuery, connection, callback = (err, rows) => {
    log.ERROR(err);
    throw err;
 }) {
-     // Start the MySQL Connection
+    // Start the MySQL Connection
+    if (!connection) connection = Connection;
    connection.query(sqlQuery, (err, rows) => {
        if (err) {
-            log.ERROR("SQL Error:", err)
+            log.ERROR("SQL Error on query:", sqlQuery, err);
            return callback(err, undefined);
        } 
        log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows);        
@@ -51,13 +52,14 @@ function returnMysqlTime(){
 }

 class Storage {
-    constructor(_dbTable) {       
+    constructor(_dbTable, connection = undefined) {       
        this.dbTable = _dbTable;
+	this.connection = connection
        this.validKeys = [];
        
        var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`; 

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection,  (err, rows) => {
            if (err) return log.ERROR("Error getting column names: ", err);
            if (rows){
                for (const validKey of rows){
@@ -96,7 +98,7 @@ class Storage {

        const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = "${keyValue}"`;

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection,  (err, rows) => {
            if (err) return callback(err, undefined);
            if (rows[0]?.[key]) return callback(undefined, rows[0]);
            else return callback(undefined, false);
@@ -113,7 +115,7 @@ class Storage {

        let records = [];

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
            if (err) return callback(err, undefined);
            for (const row of rows) {                
                if (this.dbTable == rssFeedsTable){
@@ -139,7 +141,7 @@ class Storage {

        let records = [];

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
            if (err) return callback(err, undefined);
            for (const row of rows) {                
                if (this.dbTable == rssFeedsTable){
@@ -153,11 +155,21 @@ class Storage {
            return callback(undefined, records);
        });
    }
+
+    closeConnection() {
+        try {
+            this.connection.end();
+        }
+        catch (err) {
+            log.ERROR("Error closing connection :", this.connection, err);
+            throw err;
+        }
+    }
 }

 exports.UserStorage = class UserStorage extends Storage {
-    constructor() {
-        super(accountsTable);
+    constructor(connection = undefined) {
+        super(accountsTable, connection);
    }

    /**
@@ -171,7 +183,7 @@ exports.UserStorage = class UserStorage extends Storage {

        log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
            if (err) return callback(err, undefined);
            if (rows?.affectedRows > 0) return callback(undefined, rows);
            return callback(undefined, undefined);
@@ -231,7 +243,7 @@ exports.UserStorage = class UserStorage extends Storage {

        log.DEBUG("Updating Balance with SQL Query: ", sqlQuery);

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
            if (err) return callback(err, undefined);
            if (!rows?.affectedRows > 0) return callback(new Error("Error updating Balance", rows), undefined);
            return callback(undefined, rows);
@@ -240,8 +252,8 @@ exports.UserStorage = class UserStorage extends Storage {
 }

 exports.TransactionStorage = class TransactionStorage extends Storage {
-    constructor() {
-        super(transactionsTable);
+    constructor(connection = undefined) {
+        super(transactionsTable, connection);
    }
    
    createTransaction(transaction, callback){
@@ -249,7 +261,7 @@ exports.TransactionStorage = class TransactionStorage extends Storage {

        log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
            if (err) return callback(err, undefined);
            if (rows?.affectedRows > 0) return callback(undefined, rows);
            return callback(undefined, undefined);
@@ -258,8 +270,8 @@ exports.TransactionStorage = class TransactionStorage extends Storage {
 }

 exports.FeedStorage = class FeedStorage extends Storage {
-    constructor() {
-        super(rssFeedsTable);
+    constructor(connection = undefined) {
+        super(rssFeedsTable, connection);
    }

    /**
@@ -316,7 +328,7 @@ exports.FeedStorage = class FeedStorage extends Storage {

        log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
            if (err) return callback(err, undefined);
            return callback(undefined, rows);
        })
@@ -355,7 +367,7 @@ exports.FeedStorage = class FeedStorage extends Storage {

        log.DEBUG(`Updating entry with SQL query: '${sqlQuery}'`)

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
            if (err) return callback(err, undefined);
            return callback(undefined, rows);
        })
@@ -373,7 +385,7 @@ exports.FeedStorage = class FeedStorage extends Storage {
    
        const sqlQuery  = `DELETE FROM ${this.dbTable} WHERE id = "${id}";`;

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
            if (err) return callback(err, undefined);
            return callback(undefined, rows[0]);
        })
@@ -439,8 +451,8 @@ exports.FeedStorage = class FeedStorage extends Storage {
 }

 exports.PostStorage = class PostStorage extends Storage {
-    constructor() {
-        super(rssPostsTable);
+    constructor(connection = undefined) {
+        super(rssPostsTable, connection);
    }    

    savePost(_postObject, callback){
@@ -456,7 +468,7 @@ exports.PostStorage = class PostStorage extends Storage {

        log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`)

-        runSQL(sqlQuery,  (err, rows) => {
+        runSQL(sqlQuery, this.connection, (err, rows) => {
            if (err) return callback(err, undefined);
            return callback(undefined, rows);
        })
--- a/package.json
+++ b/package.json
@@ -8,25 +8,25 @@
    "@discordjs/rest": "~1.5.0",
    "axios": "~1.3.4",
    "chatgpt": "~4.7.2",
+    "cookie-parser": "~1.4.4",
+    "debug": "~2.6.9",
    "discord-api-types": "~0.37.35",
    "discord.js": "~14.7.1",
    "dotenv": "~16.0.3",
+    "ejs": "~2.6.1",
    "express": "~4.18.2",
    "fs": "~0.0.1-security",
+    "gpt-3-encoder": "~1.1.4",
+    "http-errors": "~1.6.3",
    "jsdoc": "~3.6.7",
    "jsonfile": "~6.1.0",
+    "morgan": "~1.9.1",
+    "mysql": "~2.18.1",
+    "node-html-markdown": "~1.3.0",
+    "node-html-parser": "~6.1.5",
    "openai": "~3.1.0",
    "parse-files": "~0.1.1",
    "rss-parser": "~3.12.0",
-    "mysql": "~2.18.1",
-    "cookie-parser": "~1.4.4",
-    "debug": "~2.6.9",
-    "ejs": "~2.6.1",
-    "http-errors": "~1.6.3",
-    "morgan": "~1.9.1",
-    "node-html-markdown": "~1.3.0",
-    "node-html-parser": "~6.1.5",
-    "gpt-3-encoder": "~1.1.4",
    "user-agents": "~1.0.1303"
  },
  "scripts": {