logan/Emmelia-Link-Flayer-Rewrite
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects 2 Releases Wiki Activity

Working on #2

Browse Source
This commit is contained in:
Logan Cusano
2023-03-11 23:05:50 -05:00
parent 8ed0b969dd
commit 72134b1b7b
3 changed files with 66 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
libCore.js
View File

@@ -5,6 +5,7 @@ const { FeedStorage, PostStorage } = require("./libStorage");
const libUtils = require("./libUtils"); const libUtils = require("./libUtils");
const { DebugBuilder } = require("./utilities/debugBuilder"); const { DebugBuilder } = require("./utilities/debugBuilder");
const log = new DebugBuilder("server", "libCore"); const log = new DebugBuilder("server", "libCore");
const mysql = require("mysql");
const UserAgent = require("user-agents"); const UserAgent = require("user-agents");
process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString(); process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
@@ -85,6 +86,18 @@ exports.deleteSource = function (title, callback) {
*/ */
exports.updateFeeds = async (client) => { exports.updateFeeds = async (client) => {
if (!client) throw new Error("Client object not passed"); if (!client) throw new Error("Client object not passed");
// Create a temp pool to use for all connections while updating the feed
var tempConnection = mysql.createPool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASS,
database: process.env.DB_NAME,
connectionLimit: 10
});
const feedStorage = new FeedStorage(tempConnection);
const postStorage = new PostStorage(tempConnection);
feedStorage.getAllRecords(async (err, records) => { feedStorage.getAllRecords(async (err, records) => {
// Load the posts from each RSS source // Load the posts from each RSS source
for (const source of records) { for (const source of records) {
@@ -103,9 +116,9 @@ exports.updateFeeds = async (client) => {
try{ try{
log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title); log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
if (parsedFeed?.items){ if (parsedFeed?.items){
for (const post of parsedFeed.items){ for (const post of parsedFeed.items){
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title); log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
//log.VERBOSE("Post from feed: ", post); log.VERBOSE("Post from feed: ", post);
if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){ if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){
post.postId = post.postId ?? post.guid ?? post.id; post.postId = post.postId ?? post.guid ?? post.id;
postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => { postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
@@ -140,7 +153,11 @@ exports.updateFeeds = async (client) => {
} }
}); });
} }
}); });
// Close the temp connections
feedStorage.closeConnection();
postStorage.closeConnection();
} }
/** /**
@@ -290,4 +307,4 @@ exports.getCategories = async (callback) => {
return callback(undefined, results); return callback(undefined, results);
}); });
} }

60
libStorage.js
View File

@@ -16,7 +16,7 @@ const accountsTable = process.env.DB_ACCOUNTS_TABLE;
const transactionsTable = process.env.DB_TRANSACTIONS_TABLE; const transactionsTable = process.env.DB_TRANSACTIONS_TABLE;
const pricingTable = process.env.DB_PRICING_TABLE; const pricingTable = process.env.DB_PRICING_TABLE;
var connection = mysql.createPool({ var Connection = mysql.createPool({
host: process.env.DB_HOST, host: process.env.DB_HOST,
user: process.env.DB_USER, user: process.env.DB_USER,
password: process.env.DB_PASS, password: process.env.DB_PASS,
@@ -26,14 +26,15 @@ var connection = mysql.createPool({
// Helper Functions // Helper Functions
// Function to run and handle SQL errors // Function to run and handle SQL errors
function runSQL(sqlQuery, callback = (err, rows) => { function runSQL(sqlQuery, connection, callback = (err, rows) => {
log.ERROR(err); log.ERROR(err);
throw err; throw err;
}) { }) {
// Start the MySQL Connection // Start the MySQL Connection
if (!connection) connection = Connection;
connection.query(sqlQuery, (err, rows) => { connection.query(sqlQuery, (err, rows) => {
if (err) { if (err) {
log.ERROR("SQL Error:", err) log.ERROR("SQL Error on query:", sqlQuery, err);
return callback(err, undefined); return callback(err, undefined);
} }
log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows); log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows);
@@ -51,13 +52,14 @@ function returnMysqlTime(){
} }
class Storage { class Storage {
constructor(_dbTable) { constructor(_dbTable, connection = undefined) {
this.dbTable = _dbTable; this.dbTable = _dbTable;
this.connection = connection
this.validKeys = []; this.validKeys = [];
var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`; var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`;
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return log.ERROR("Error getting column names: ", err); if (err) return log.ERROR("Error getting column names: ", err);
if (rows){ if (rows){
for (const validKey of rows){ for (const validKey of rows){
@@ -96,7 +98,7 @@ class Storage {
const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = "${keyValue}"`; const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = "${keyValue}"`;
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
if (rows[0]?.[key]) return callback(undefined, rows[0]); if (rows[0]?.[key]) return callback(undefined, rows[0]);
else return callback(undefined, false); else return callback(undefined, false);
@@ -113,7 +115,7 @@ class Storage {
let records = []; let records = [];
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
for (const row of rows) { for (const row of rows) {
if (this.dbTable == rssFeedsTable){ if (this.dbTable == rssFeedsTable){
@@ -139,7 +141,7 @@ class Storage {
let records = []; let records = [];
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
for (const row of rows) { for (const row of rows) {
if (this.dbTable == rssFeedsTable){ if (this.dbTable == rssFeedsTable){
@@ -153,11 +155,21 @@ class Storage {
return callback(undefined, records); return callback(undefined, records);
}); });
} }
closeConnection() {
try {
this.connection.end();
}
catch (err) {
log.ERROR("Error closing connection :", this.connection, err);
throw err;
}
}
} }
exports.UserStorage = class UserStorage extends Storage { exports.UserStorage = class UserStorage extends Storage {
constructor() { constructor(connection = undefined) {
super(accountsTable); super(accountsTable, connection);
} }
/** /**
@@ -171,7 +183,7 @@ exports.UserStorage = class UserStorage extends Storage {
log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
if (rows?.affectedRows > 0) return callback(undefined, rows); if (rows?.affectedRows > 0) return callback(undefined, rows);
return callback(undefined, undefined); return callback(undefined, undefined);
@@ -231,7 +243,7 @@ exports.UserStorage = class UserStorage extends Storage {
log.DEBUG("Updating Balance with SQL Query: ", sqlQuery); log.DEBUG("Updating Balance with SQL Query: ", sqlQuery);
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
if (!rows?.affectedRows > 0) return callback(new Error("Error updating Balance", rows), undefined); if (!rows?.affectedRows > 0) return callback(new Error("Error updating Balance", rows), undefined);
return callback(undefined, rows); return callback(undefined, rows);
@@ -240,8 +252,8 @@ exports.UserStorage = class UserStorage extends Storage {
} }
exports.TransactionStorage = class TransactionStorage extends Storage { exports.TransactionStorage = class TransactionStorage extends Storage {
constructor() { constructor(connection = undefined) {
super(transactionsTable); super(transactionsTable, connection);
} }
createTransaction(transaction, callback){ createTransaction(transaction, callback){
@@ -249,7 +261,7 @@ exports.TransactionStorage = class TransactionStorage extends Storage {
log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
if (rows?.affectedRows > 0) return callback(undefined, rows); if (rows?.affectedRows > 0) return callback(undefined, rows);
return callback(undefined, undefined); return callback(undefined, undefined);
@@ -258,8 +270,8 @@ exports.TransactionStorage = class TransactionStorage extends Storage {
} }
exports.FeedStorage = class FeedStorage extends Storage { exports.FeedStorage = class FeedStorage extends Storage {
constructor() { constructor(connection = undefined) {
super(rssFeedsTable); super(rssFeedsTable, connection);
} }
/** /**
@@ -316,7 +328,7 @@ exports.FeedStorage = class FeedStorage extends Storage {
log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
return callback(undefined, rows); return callback(undefined, rows);
}) })
@@ -355,7 +367,7 @@ exports.FeedStorage = class FeedStorage extends Storage {
log.DEBUG(`Updating entry with SQL query: '${sqlQuery}'`) log.DEBUG(`Updating entry with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
return callback(undefined, rows); return callback(undefined, rows);
}) })
@@ -373,7 +385,7 @@ exports.FeedStorage = class FeedStorage extends Storage {
const sqlQuery = `DELETE FROM ${this.dbTable} WHERE id = "${id}";`; const sqlQuery = `DELETE FROM ${this.dbTable} WHERE id = "${id}";`;
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
return callback(undefined, rows[0]); return callback(undefined, rows[0]);
}) })
@@ -439,8 +451,8 @@ exports.FeedStorage = class FeedStorage extends Storage {
} }
exports.PostStorage = class PostStorage extends Storage { exports.PostStorage = class PostStorage extends Storage {
constructor() { constructor(connection = undefined) {
super(rssPostsTable); super(rssPostsTable, connection);
} }
savePost(_postObject, callback){ savePost(_postObject, callback){
@@ -456,7 +468,7 @@ exports.PostStorage = class PostStorage extends Storage {
log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`) log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
return callback(undefined, rows); return callback(undefined, rows);
}) })

18
package.json
View File

@@ -8,25 +8,25 @@
"@discordjs/rest": "~1.5.0", "@discordjs/rest": "~1.5.0",
"axios": "~1.3.4", "axios": "~1.3.4",
"chatgpt": "~4.7.2", "chatgpt": "~4.7.2",
"cookie-parser": "~1.4.4",
"debug": "~2.6.9",
"discord-api-types": "~0.37.35", "discord-api-types": "~0.37.35",
"discord.js": "~14.7.1", "discord.js": "~14.7.1",
"dotenv": "~16.0.3", "dotenv": "~16.0.3",
"ejs": "~2.6.1",
"express": "~4.18.2", "express": "~4.18.2",
"fs": "~0.0.1-security", "fs": "~0.0.1-security",
"gpt-3-encoder": "~1.1.4",
"http-errors": "~1.6.3",
"jsdoc": "~3.6.7", "jsdoc": "~3.6.7",
"jsonfile": "~6.1.0", "jsonfile": "~6.1.0",
"morgan": "~1.9.1",
"mysql": "~2.18.1",
"node-html-markdown": "~1.3.0",
"node-html-parser": "~6.1.5",
"openai": "~3.1.0", "openai": "~3.1.0",
"parse-files": "~0.1.1", "parse-files": "~0.1.1",
"rss-parser": "~3.12.0", "rss-parser": "~3.12.0",
"mysql": "~2.18.1",
"cookie-parser": "~1.4.4",
"debug": "~2.6.9",
"ejs": "~2.6.1",
"http-errors": "~1.6.3",
"morgan": "~1.9.1",
"node-html-markdown": "~1.3.0",
"node-html-parser": "~6.1.5",
"gpt-3-encoder": "~1.1.4",
"user-agents": "~1.0.1303" "user-agents": "~1.0.1303"
}, },
"scripts": { "scripts": {