diff --git a/libCore.js b/libCore.js index 87a0266..370a2b3 100644 --- a/libCore.js +++ b/libCore.js @@ -19,31 +19,21 @@ const openai = new OpenAIApi(configuration); // Data Structures var feeds = []; -let rssFeedMap = []; -let rssFeedCategories = []; +var rssFeedMap = []; +var rssFeedCategories = []; -// Precess Vars -var userTable = process.env.DB_TABLE - -// Setup Storage handler -var storageSource = new storageHandler.Storage(userTable); +// Setup Storage handlers +var feedStorage = new storageHandler.feedStorage(); /** - * Adds a new source url to configured storage + * Adds or updates new source url to configured storage * @constructor * @param {string} title - Title/Name of the RSS feed. * @param {string} link - URL of RSS feed. * @param {string} category - Category of RSS feed. */ -exports.addSource = function (title, link, category, guildId, channelId, callback) { - - for (i = 0; i < feeds.length; i++) { - if (feeds[i].link == link) { - return; - } - } - - storageSource.create([{ +exports.addSource = async (title, link, category, guildId, channelId, callback) => { + feedStorage.create([{ "fields": { "title": title, "link": link, @@ -54,9 +44,11 @@ exports.addSource = function (title, link, category, guildId, channelId, callbac }], function (err, record) { if (err) { log.ERROR("Error in create:", err); - callback(err, undefined); + return callback(err, undefined); } + if (!record) return callback(undefined, false); + log.DEBUG("Record ID:", record.getId()); var linkData = { @@ -71,7 +63,7 @@ exports.addSource = function (title, link, category, guildId, channelId, callbac feeds.push(linkData); log.DEBUG("pushed item to feeds"); - callback(undefined, true); + return callback(undefined, record); }); } @@ -88,13 +80,13 @@ exports.deleteSource = function (title, callback) { deleteRecord = feeds[i].id; } } - storageSource.destroy(deleteRecord, function (err, deletedRecord) { + feedStorage.destroy(deleteRecord, function (err, deletedRecord) { if (err) { log.ERROR(err); - callback(err, undefined); + return callback(err, undefined); } log.DEBUG(deletedRecord.id); - callback(undefined, deletedRecord); + return callback(undefined, deletedRecord); }); } @@ -104,17 +96,17 @@ exports.deleteSource = function (title, callback) { * @param {string} feedType - Category to select Feed by. */ exports.getFeeds = function (feedType) { - var linkFlayerFilteredMap = []; + var rssFeedFilteredMap = []; if (feedType == null || feedType == undefined || feedType == "" || feedType.toLowerCase() == "all") { return rssFeedMap; } else { - rssFeedMap.forEach(linkFlay => { - if (linkFlay.category.toLowerCase().indexOf(feedType.toLowerCase()) > -1) { - linkFlayerFilteredMap.push(linkFlay); + rssFeedMap.forEach(rssFeed => { + if (rssFeed.category.toLowerCase().indexOf(feedType.toLowerCase()) > -1) { + rssFeedFilteredMap.push(rssFeed); } }); - return linkFlayerFilteredMap; + return rssFeedFilteredMap; } } @@ -126,47 +118,39 @@ exports.loadFeeds = function () { rssFeedMap = []; rssFeedCategories = []; - storageSource.getAllRecords(function (err, records) { - records.forEach(function (record) { - try { - log.DEBUG('Retrieved title: ', record.get('title')); - log.DEBUG('Retrieved link:', record.get('link')); - log.DEBUG('Retrieved category:', record.get('category')); - - var feedData = { - title: `${unescape(record.get('title'))}`, - link: `${unescape(record.get('link'))}`, - category: `${unescape(record.get('category'))}`, - id: record.getId() - } - - var foundMatch = false; - feeds.forEach(feedBlock => { - if (feedBlock.link == feedData.link) { - foundMatch = true; - } - }); - - if (!foundMatch) { - feeds.push(feedData); + feedStorage.getAllRecords((err, records) => { + for (const record of records){ + try { + log.DEBUG('Retrieved title: ', record.title); + log.DEBUG('Retrieved link: ', record.link); + log.DEBUG('Retrieved category: ', record.category); + log.DEBUG('Retrieved guild ID: ', record.guild_id); + log.DEBUG('Retrieved channel ID: ', record.channel_id); + + // Check to see if this source has been loaded already + feeds.forEach(feedSource => { + if (feedSource.link != record.link) { + log.DEBUG("Loading new source: ", feedSource) + feeds.push(feedSource); } + }); - let foundCat = false; - rssFeedCategories.forEach(cat => { - if (cat == record.get('category')) { - foundCat = true; - } - }); - - if (!foundCat) { - rssFeedCategories.push(record.get('category')); - } - - } catch (error) { - log.DEBUG(error); + // Update the known categories if new category is not present + if (!rssFeedCategories.includes(record.category)) { + log.DEBUG("Added new category to running known categories: ", record.category) + rssFeedCategories.push(record.category); } - }); + + } catch (error) { + log.DEBUG(error); + } + } + }); +} + + + /* feeds.forEach(feedBlock => { (async () => { try { @@ -177,10 +161,12 @@ exports.loadFeeds = function () { } if (feed != undefined && feed.items != undefined) { + //log.DEBUG("Feed: ", feed); feed.items.forEach(item => { var foundFeed = false; - rssFeedMap.forEach(linkFlay => { - if (linkFlay.link == item.link) { + rssFeedMap.forEach(rssFeed => { + if (rssFeed.link == item.link) { + log.DEBUG("Found matching post?: ", rssFeed) foundFeed = true; } }); @@ -202,14 +188,9 @@ exports.loadFeeds = function () { }) } catch (error) { log.DEBUG(error); - } - })().then(); - }); - return; - //fetchNextPage(); - }, function done(error) { - log.DEBUG(error); - }); + } + + //log.DEBUG("RSS Feed Map: ", rssFeedMap); } /** @@ -372,4 +353,12 @@ exports.getQuotes = async function (quote_url) { */ exports.getCategories = function () { return rssFeedCategories; +} + +exports.addPost = (postObject, callback) => { + if(!postObject.post_link_hash) return callback(new Error("No hash included"), undefined); +} + +exports.checkIfPostExists = (postObject, callback) => { + if(!postObject.post_id || !postObject.post_link_hash) return callback(new Error("No hash included"), undefined); } \ No newline at end of file diff --git a/libStorage.js b/libStorage.js index fc87d85..58429f2 100644 --- a/libStorage.js +++ b/libStorage.js @@ -4,11 +4,14 @@ // Import modules const { DebugBuilder } = require("./utilities/debugBuilder"); const log = new DebugBuilder("server", "libStorage"); +const { RSSSourceRecord, RSSPostRecord } = require("./utilities/recordHelper"); // Storage Specific Modules // MySQL const mysql = require("mysql"); +const rssFeedsTable = process.env.DB_RSS_FEEDS_TABLE; +const rssPostsTable = process.env.DB_RSS_POSTS_TABLE; // Helper Functions // Function to run and handle SQL errors @@ -17,87 +20,100 @@ function runSQL(sqlQuery, connection, callback = (err, rows) => { throw err; }) { // Start the MySQL Connection - //connection.connect(); connection.query(sqlQuery, (err, rows) => { if (err) { log.ERROR("SQL Error:", err) - callback(err, undefined); + return callback(err, undefined); } - log.DEBUG("RunSQL Returned Rows:", rows); - callback(undefined, rows); + log.VERBOSE("RunSQL Returned Rows:", rows); + return callback(undefined, rows); }) - //connection. } -class RSSRecord { - /** - * - * @param {*} _id - * @param {*} _title - * @param {*} _link - * @param {*} _category - * @param {*} _guild_id - * @param {*} _channel_id - */ - constructor(_id, _title, _link, _category, _guild_id, _channel_id) { - this.id = _id; - this.title = _title; - this.link= _link; - this.category = _category; - this.guild_id = _guild_id; - this.channel_id = _channel_id; - } - - getId() { - return this.id; - } - - get(key) { - if (!Object.keys(this).includes(key)) throw new Error("Key is invalid", key); - return this[key] - } -} - -exports.RSSRecord = RSSRecord; - -exports.Storage = class Storage { +class Storage { constructor(_dbTable) { this.connection = mysql.createPool({ host: process.env.DB_HOST, user: process.env.DB_USER, password: process.env.DB_PASS, database: process.env.DB_NAME - }); - - // Set the DB Table for later use + }); + this.dbTable = _dbTable } + /** + * Get a record by a specified key + * @param {*} key The key to search for + * @param {*} keyValue The value of the key to search for + * @param {*} callback The callback function + */ + getRecordBy(key, keyValue, callback) { + const validKeys = ["link", "title", "category", "id"]; + if (!validKeys.includes(key)) return callback(new Error("Given key not valid"), undefined); + + const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = '${keyValue}'`; + + runSQL(sqlQuery, this.connection, (err, rows) => { + if (err) return callback(err, undefined); + if (rows[0]?.title) return callback(undefined, rows[0]); + else return callback(undefined, false); + }) + } + + /** + * Get all records stored + * @param {function} callback + */ + getAllRecords(callback) { + log.INFO("Getting all records"); + const sqlQuery = `SELECT * FROM ${this.dbTable}` + + let rssRecords = []; + + runSQL(sqlQuery, this.connection, (err, rows) => { + if (err) return callback(err, undefined); + for (const row of rows) { + log.VERBOSE("Row from SQL query:", row); + rssRecords.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id)); + } + log.VERBOSE("All records:", rssRecords); + return callback(undefined, rssRecords); + }); + } +} + +exports.feedStorage = class feedStorage extends Storage { + constructor() { + super(rssFeedsTable); + } + /** * Wrapper to save a new entry using the storage method configured * @param {Array} toBeSaved Entry or Entries to be added * @param {function} callback The callback function to be called with the record when saved */ - create(toBeSaved, callback) { + create(toBeSaved, callback) { log.DEBUG("To be saved:", toBeSaved); log.DEBUG("to be saved length:", toBeSaved.length); - if (!toBeSaved[0].fields?.title) callback(Error("No title given"), undefined); + // If the request was for the Feeds Table + if (!toBeSaved[0].fields?.title) return callback(Error("No title given"), undefined); let newRecords = [] for (var entry of toBeSaved) { - entry = entry.fields + entry = entry.fields; log.DEBUG("Entry:", entry); this.returnRecord(undefined, entry.title, entry.link, entry.category, entry.guild_id, entry.channel_id, (err, record) => { - if (err) callback(err, undefined); + if (err) return callback(err, undefined); newRecords.push(record); if (toBeSaved.length === 1) { log.DEBUG("One record to callback with:", record); - callback(undefined, record); + return callback(undefined, record); } - }) + }, false) } if (!toBeSaved.length === 1) { - callback(undefined, newRecords); - } + return callback(undefined, newRecords); + } } /** @@ -106,12 +122,12 @@ exports.Storage = class Storage { * @param {function} callback The callback function to be called with the record when deleted */ destroy(entryID, callback) { - if (!entryID) callback(Error("No entry ID given"), undefined); + if (!entryID) return callback(Error("No entry ID given"), undefined); this.getRecordBy('id', entryID, (err, entryRecord) => { this.removeEntry(entryRecord.id, (err, results) => { - if (err) callback(err, undefined); - callback(undefined, results); + if (err) return callback(err, undefined); + return callback(undefined, results); }); }); } @@ -122,29 +138,11 @@ exports.Storage = class Storage { * @returns {true|false|*} */ checkForTitle(title, callback) { - if (!title) callback(new Error("No title given when checking for title"), undefined); + if (!title) return callback(new Error("No title given when checking for title"), undefined) this.getRecordBy("title", title, callback); } - /** - * Get a record by a specified key - * @param {*} key The key to search for - * @param {*} keyValue The value of the key to search for - */ - getRecordBy(key, keyValue, callback) { - const validKeys = ["link", "title", "category", "id"]; - if (!validKeys.includes(key)) callback(new Error("Given key not valid"), undefined); - - const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = '${keyValue}'`; - - runSQL(sqlQuery, this.connection, (err, rows) => { - if (err) callback(err, undefined); - if (rows[0]?.title) callback(undefined, rows[0]); - else callback(undefined, false); - }) - } - /** * Save the given entry to the storage medium * @param {Object} entryObject The entry object to be saved @@ -153,7 +151,7 @@ exports.Storage = class Storage { saveEntry(entryObject, callback) { log.DEBUG("Saving entry:", entryObject); if (!entryObject?.title || !entryObject?.link || !entryObject?.category) { - callback(new Error("Entry object malformed, check the object before saving it"), undefined) + return callback(new Error("Entry object malformed, check the object before saving it"), undefined) } const sqlQuery = `INSERT INTO ${this.dbTable} (title, link, category, guild_id, channel_id) VALUES ('${entryObject.title}', '${entryObject.link}', '${entryObject.category}', '${entryObject.guild_id}', '${entryObject.channel_id}');`; @@ -161,8 +159,8 @@ exports.Storage = class Storage { log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) runSQL(sqlQuery, this.connection, (err, rows) => { - if (err) callback(err, undefined); - callback(undefined, rows); + if (err) return callback(err, undefined); + return callback(undefined, rows); }) } @@ -173,9 +171,9 @@ exports.Storage = class Storage { */ updateEntry(entryObject, callback) { let queryParams = []; - if (!entryObject.title) callback(new Error("No title given before updating"), undefined); + if (!entryObject.title) return callback(new Error("No title given before updating"), undefined); queryParams.push(`title = '${entryObject.title}'`); - if (!entryObject.link) callback(new Error("No link given before updating"), undefined); + if (!entryObject.link) return callback(new Error("No link given before updating"), undefined); queryParams.push(`link = '${entryObject.link}'`); if (entryObject.category) queryParams.push(`category = '${entryObject.category}'`); if (entryObject.guild_id) queryParams.push(`guild_id = '${entryObject.guild_id}'`); @@ -200,8 +198,8 @@ exports.Storage = class Storage { log.DEBUG(`Updating entry with SQL query: '${sqlQuery}'`) runSQL(sqlQuery, this.connection, (err, rows) => { - if (err) callback(err, undefined); - callback(undefined, rows); + if (err) return callback(err, undefined); + return callback(undefined, rows); }) } @@ -212,14 +210,14 @@ exports.Storage = class Storage { */ removeEntry(title, callback) { if (!title) { - callback(new Error("No entry title given before deleting"), undefined) + return callback(new Error("No entry title given before deleting"), undefined) } const sqlQuery = `DELETE FROM ${this.dbTable} WHERE title = '${title}';`; runSQL(sqlQuery, this.connection, (err, rows) => { - if (err) callback(err, undefined); - callback(undefined, rows[0]); + if (err) return callback(err, undefined); + return callback(undefined, rows[0]); }) } @@ -231,9 +229,9 @@ exports.Storage = class Storage { * @param {*} _category The category of the record * @param {*} callback Callback function to return an error or the record */ - returnRecord(_id, _title, _link, _category, _guild_id, _channel_id, callback) { - log.DEBUG(`Return record for these values: ID: '${_id}', Title: '${_title}', Category: '${_category}', Link: '${_link}', Guild: '${_guild_id}', Channel:'${_channel_id}'`) - if (!_link && !_title && !_guild_id && !_channel_id) callback(new Error("No link or title given when creating a record"), undefined); + returnRecord(_id, _title, _link, _category, _guild_id, _channel_id, callback, updateEnabled = true) { + log.DEBUG(`Return record for these values: ID: '${_id}', Title: '${_title}', Category: '${_category}', Link: '${_link}', Guild: '${_guild_id}', Channel:'${_channel_id}', Update Enabled: `, updateEnabled) + if (!_link && !_title && !_guild_id && !_channel_id) return callback(new Error("No link or title given when creating a record"), undefined); let entryObject = { "title": _title, "link": _link, @@ -244,58 +242,41 @@ exports.Storage = class Storage { if (_id) { entryObject.id = _id; + if (!updateEnabled) return callback(undefined, undefined); + this.updateEntry(entryObject, (err, rows) => { - if (err) callback(err, undefined); + if (err) return callback(err, undefined); this.getRecordBy('id', entryObject.id, (err, record) => { - if (err) callback(err, undefined); - callback(undefined, new RSSRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id)); + if (err) return callback(err, undefined); + return callback(undefined, new RSSSourceRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id)); }) - }) + }) } else { this.checkForTitle(_title, (err, titleExists) => { if (!titleExists) { log.DEBUG("Entry doesn't exist, making one now", entryObject); this.saveEntry(entryObject, (err, rows) => { - if (err) callback(err, undefined); + if (err) return callback(err, undefined); this.getRecordBy("title", entryObject.title, (err, record) => { - if (err) callback(err, undefined); - callback(undefined, new RSSRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id)); + if (err) return callback(err, undefined); + return callback(undefined, new RSSSourceRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id)); }) }); } else{ + if (!updateEnabled) return callback(undefined, undefined); + this.updateEntry(entryObject, (err, rows) => { - if (err) callback(err, undefined); + if (err) return callback(err, undefined); this.getRecordBy('title', entryObject.title, (err, record) => { - if (err) callback(err, undefined); - callback(undefined, new RSSRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id)); + if (err) return callback(err, undefined); + return callback(undefined, new RSSSourceRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id)); }) - }) + }) } }) - } - } - - /** - * Get all records stored - * @param {function} callback - */ - getAllRecords(callback) { - log.INFO("Getting all records"); - const sqlQuery = `SELECT * FROM ${this.dbTable}` - - let rssRecords = []; - - runSQL(sqlQuery, this.connection, (err, rows) => { - if (err) callback(err, undefined); - for (const row of rows) { - log.DEBUG("Row from SQL query:", row); - rssRecords.push(new RSSRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id)); - } - log.DEBUG("All records:", rssRecords); - callback(undefined, rssRecords); - }); - } + } + } } diff --git a/utilities/recordHelper.js b/utilities/recordHelper.js new file mode 100644 index 0000000..640c47f --- /dev/null +++ b/utilities/recordHelper.js @@ -0,0 +1,72 @@ +// This is for record builders + +const { DebugBuilder } = require("./debugBuilder"); +const log = new DebugBuilder("server", "recordHelper"); + +class baseRSSRecord { + /** + * + * @param {*} _id The ID of the post from the DB + * @param {*} _title The title of the post + * @param {*} _link The link to the post + * @param {*} _category The category of the post + */ + constructor(_id, _title, _link, _category) { + this.id = _id; + this.title = _title; + this.link= _link; + this.category = _category; + } + + getId() { + return this.id; + } + + get(key) { + log.DEBUG(`Getting key '${key}' from: `, Object(this)); + if (!Object.keys(this).includes(key)) throw new Error(`Key is invalid ${key}`); + return this[key] + } +} + +exports.baseRSSRecord = baseRSSRecord; + +/** + * Build a Source record. + * + * A source record is the record of an RSS feed itself. Something like "https://www.leafly.com/feed". + */ +exports.RSSSourceRecord = class RSSSourceRecord extends baseRSSRecord{ + /** + * @param {*} _id The ID of the post from the DB + * @param {*} _title The title of the post + * @param {*} _link The link to the post + * @param {*} _category The category of the post + * @param {*} _guild_id The guild id to receive updates on this source + * @param {*} _channel_id The channel to send updates from this source + */ + constructor(_id, _title, _link, _category, _guild_id, _channel_id) { + super(_id, _title, _link, _category); + this.guild_id = _guild_id; + this.channel_id = _channel_id; + } +} + +/** + * Build a Post record. + * + * A post is an individual article/post/link from an RSS feed. Each individual post will be added to the database to be recalled and sent later. + */ +exports.RSSPostRecord = class RSSPostRecord extends baseRSSRecord{ + /** + * @param {*} _id The ID of the post from the DB + * @param {*} _title The title of the post + * @param {*} _link The link to the post + * @param {*} _category The category of the post + * @param {*} _rssSourceID The ID of the RSS source that created this post + */ + constructor(_id, _title, _link, _category, _rssSourceID) { + super(_id, _title, _link, _category); + this.source_id = _rssSourceID; + } +} \ No newline at end of file