logan/Emmelia-Link-Flayer-Rewrite
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects 2 Releases Wiki Activity

Updated storage for feeds and posts

Browse Source
This commit is contained in:
Logan Cusano
2023-02-25 14:47:45 -05:00
parent e157fc94e0
commit 12b4a566c7
3 changed files with 233 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
libCore.js
View File

@@ -19,31 +19,21 @@ const openai = new OpenAIApi(configuration);
// Data Structures
var feeds = [];
let rssFeedMap = [];
let rssFeedCategories = [];
var rssFeedMap = [];
var rssFeedCategories = [];
// Precess Vars
var userTable = process.env.DB_TABLE
// Setup Storage handler
var storageSource = new storageHandler.Storage(userTable);
// Setup Storage handlers
var feedStorage = new storageHandler.feedStorage();
/**
* Adds a new source url to configured storage
* Adds or updates new source url to configured storage
* @constructor
* @param {string} title - Title/Name of the RSS feed.
* @param {string} link - URL of RSS feed.
* @param {string} category - Category of RSS feed.
*/
exports.addSource = function (title, link, category, guildId, channelId, callback) {
for (i = 0; i < feeds.length; i++) {
if (feeds[i].link == link) {
return;
}
}
storageSource.create([{
exports.addSource = async (title, link, category, guildId, channelId, callback) => {
feedStorage.create([{
"fields": {
"title": title,
"link": link,
@@ -54,9 +44,11 @@ exports.addSource = function (title, link, category, guildId, channelId, callbac
}], function (err, record) {
if (err) {
log.ERROR("Error in create:", err);
callback(err, undefined);
return callback(err, undefined);
}
if (!record) return callback(undefined, false);
log.DEBUG("Record ID:", record.getId());
var linkData = {
@@ -71,7 +63,7 @@ exports.addSource = function (title, link, category, guildId, channelId, callbac
feeds.push(linkData);
log.DEBUG("pushed item to feeds");
callback(undefined, true);
return callback(undefined, record);
});
}
@@ -88,13 +80,13 @@ exports.deleteSource = function (title, callback) {
deleteRecord = feeds[i].id;
}
}
storageSource.destroy(deleteRecord, function (err, deletedRecord) {
feedStorage.destroy(deleteRecord, function (err, deletedRecord) {
if (err) {
log.ERROR(err);
callback(err, undefined);
return callback(err, undefined);
}
log.DEBUG(deletedRecord.id);
callback(undefined, deletedRecord);
return callback(undefined, deletedRecord);
});
}
@@ -104,17 +96,17 @@ exports.deleteSource = function (title, callback) {
* @param {string} feedType - Category to select Feed by.
*/
exports.getFeeds = function (feedType) {
var linkFlayerFilteredMap = [];
var rssFeedFilteredMap = [];
if (feedType == null || feedType == undefined || feedType == "" || feedType.toLowerCase() == "all") {
return rssFeedMap;
} else {
rssFeedMap.forEach(linkFlay => {
if (linkFlay.category.toLowerCase().indexOf(feedType.toLowerCase()) > -1) {
linkFlayerFilteredMap.push(linkFlay);
rssFeedMap.forEach(rssFeed => {
if (rssFeed.category.toLowerCase().indexOf(feedType.toLowerCase()) > -1) {
rssFeedFilteredMap.push(rssFeed);
}
});
return linkFlayerFilteredMap;
return rssFeedFilteredMap;
}
}
@@ -126,47 +118,39 @@ exports.loadFeeds = function () {
rssFeedMap = [];
rssFeedCategories = [];
storageSource.getAllRecords(function (err, records) {
records.forEach(function (record) {
try {
log.DEBUG('Retrieved title: ', record.get('title'));
log.DEBUG('Retrieved link:', record.get('link'));
log.DEBUG('Retrieved category:', record.get('category'));
var feedData = {
title: `${unescape(record.get('title'))}`,
link: `${unescape(record.get('link'))}`,
category: `${unescape(record.get('category'))}`,
id: record.getId()
}
var foundMatch = false;
feeds.forEach(feedBlock => {
if (feedBlock.link == feedData.link) {
foundMatch = true;
}
});
if (!foundMatch) {
feeds.push(feedData);
feedStorage.getAllRecords((err, records) => {
for (const record of records){
try {
log.DEBUG('Retrieved title: ', record.title);
log.DEBUG('Retrieved link: ', record.link);
log.DEBUG('Retrieved category: ', record.category);
log.DEBUG('Retrieved guild ID: ', record.guild_id);
log.DEBUG('Retrieved channel ID: ', record.channel_id);
// Check to see if this source has been loaded already
feeds.forEach(feedSource => {
if (feedSource.link != record.link) {
log.DEBUG("Loading new source: ", feedSource)
feeds.push(feedSource);
}
});
let foundCat = false;
rssFeedCategories.forEach(cat => {
if (cat == record.get('category')) {
foundCat = true;
}
});
if (!foundCat) {
rssFeedCategories.push(record.get('category'));
}
} catch (error) {
log.DEBUG(error);
// Update the known categories if new category is not present
if (!rssFeedCategories.includes(record.category)) {
log.DEBUG("Added new category to running known categories: ", record.category)
rssFeedCategories.push(record.category);
}
});
} catch (error) {
log.DEBUG(error);
}
}
});
}
/*
feeds.forEach(feedBlock => {
(async () => {
try {
@@ -177,10 +161,12 @@ exports.loadFeeds = function () {
}
if (feed != undefined && feed.items != undefined) {
//log.DEBUG("Feed: ", feed);
feed.items.forEach(item => {
var foundFeed = false;
rssFeedMap.forEach(linkFlay => {
if (linkFlay.link == item.link) {
rssFeedMap.forEach(rssFeed => {
if (rssFeed.link == item.link) {
log.DEBUG("Found matching post?: ", rssFeed)
foundFeed = true;
}
});
@@ -202,14 +188,9 @@ exports.loadFeeds = function () {
})
} catch (error) {
log.DEBUG(error);
}
})().then();
});
return;
//fetchNextPage();
}, function done(error) {
log.DEBUG(error);
});
}
//log.DEBUG("RSS Feed Map: ", rssFeedMap);
}
/**
@@ -372,4 +353,12 @@ exports.getQuotes = async function (quote_url) {
*/
exports.getCategories = function () {
return rssFeedCategories;
}
exports.addPost = (postObject, callback) => {
if(!postObject.post_link_hash) return callback(new Error("No hash included"), undefined);
}
exports.checkIfPostExists = (postObject, callback) => {
if(!postObject.post_id || !postObject.post_link_hash) return callback(new Error("No hash included"), undefined);
}

215
libStorage.js
View File

@@ -4,11 +4,14 @@
// Import modules
const { DebugBuilder } = require("./utilities/debugBuilder");
const log = new DebugBuilder("server", "libStorage");
const { RSSSourceRecord, RSSPostRecord } = require("./utilities/recordHelper");
// Storage Specific Modules
// MySQL
const mysql = require("mysql");
const rssFeedsTable = process.env.DB_RSS_FEEDS_TABLE;
const rssPostsTable = process.env.DB_RSS_POSTS_TABLE;
// Helper Functions
// Function to run and handle SQL errors
@@ -17,87 +20,100 @@ function runSQL(sqlQuery, connection, callback = (err, rows) => {
throw err;
}) {
// Start the MySQL Connection
//connection.connect();
connection.query(sqlQuery, (err, rows) => {
if (err) {
log.ERROR("SQL Error:", err)
callback(err, undefined);
return callback(err, undefined);
}
log.DEBUG("RunSQL Returned Rows:", rows);
callback(undefined, rows);
log.VERBOSE("RunSQL Returned Rows:", rows);
return callback(undefined, rows);
})
//connection.
}
class RSSRecord {
/**
*
* @param {*} _id
* @param {*} _title
* @param {*} _link
* @param {*} _category
* @param {*} _guild_id
* @param {*} _channel_id
*/
constructor(_id, _title, _link, _category, _guild_id, _channel_id) {
this.id = _id;
this.title = _title;
this.link= _link;
this.category = _category;
this.guild_id = _guild_id;
this.channel_id = _channel_id;
}
getId() {
return this.id;
}
get(key) {
if (!Object.keys(this).includes(key)) throw new Error("Key is invalid", key);
return this[key]
}
}
exports.RSSRecord = RSSRecord;
exports.Storage = class Storage {
class Storage {
constructor(_dbTable) {
this.connection = mysql.createPool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASS,
database: process.env.DB_NAME
});
// Set the DB Table for later use
});
this.dbTable = _dbTable
}
/**
* Get a record by a specified key
* @param {*} key The key to search for
* @param {*} keyValue The value of the key to search for
* @param {*} callback The callback function
*/
getRecordBy(key, keyValue, callback) {
const validKeys = ["link", "title", "category", "id"];
if (!validKeys.includes(key)) return callback(new Error("Given key not valid"), undefined);
const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = '${keyValue}'`;
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined);
if (rows[0]?.title) return callback(undefined, rows[0]);
else return callback(undefined, false);
})
}
/**
* Get all records stored
* @param {function} callback
*/
getAllRecords(callback) {
log.INFO("Getting all records");
const sqlQuery = `SELECT * FROM ${this.dbTable}`
let rssRecords = [];
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined);
for (const row of rows) {
log.VERBOSE("Row from SQL query:", row);
rssRecords.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id));
}
log.VERBOSE("All records:", rssRecords);
return callback(undefined, rssRecords);
});
}
}
exports.feedStorage = class feedStorage extends Storage {
constructor() {
super(rssFeedsTable);
}
/**
* Wrapper to save a new entry using the storage method configured
* @param {Array} toBeSaved Entry or Entries to be added
* @param {function} callback The callback function to be called with the record when saved
*/
create(toBeSaved, callback) {
create(toBeSaved, callback) {
log.DEBUG("To be saved:", toBeSaved);
log.DEBUG("to be saved length:", toBeSaved.length);
if (!toBeSaved[0].fields?.title) callback(Error("No title given"), undefined);
// If the request was for the Feeds Table
if (!toBeSaved[0].fields?.title) return callback(Error("No title given"), undefined);
let newRecords = []
for (var entry of toBeSaved) {
entry = entry.fields
entry = entry.fields;
log.DEBUG("Entry:", entry);
this.returnRecord(undefined, entry.title, entry.link, entry.category, entry.guild_id, entry.channel_id, (err, record) => {
if (err) callback(err, undefined);
if (err) return callback(err, undefined);
newRecords.push(record);
if (toBeSaved.length === 1) {
log.DEBUG("One record to callback with:", record);
callback(undefined, record);
return callback(undefined, record);
}
})
}, false)
}
if (!toBeSaved.length === 1) {
callback(undefined, newRecords);
}
return callback(undefined, newRecords);
}
}
/**
@@ -106,12 +122,12 @@ exports.Storage = class Storage {
* @param {function} callback The callback function to be called with the record when deleted
*/
destroy(entryID, callback) {
if (!entryID) callback(Error("No entry ID given"), undefined);
if (!entryID) return callback(Error("No entry ID given"), undefined);
this.getRecordBy('id', entryID, (err, entryRecord) => {
this.removeEntry(entryRecord.id, (err, results) => {
if (err) callback(err, undefined);
callback(undefined, results);
if (err) return callback(err, undefined);
return callback(undefined, results);
});
});
}
@@ -122,29 +138,11 @@ exports.Storage = class Storage {
* @returns {true|false|*}
*/
checkForTitle(title, callback) {
if (!title) callback(new Error("No title given when checking for title"), undefined);
if (!title) return callback(new Error("No title given when checking for title"), undefined)
this.getRecordBy("title", title, callback);
}
/**
* Get a record by a specified key
* @param {*} key The key to search for
* @param {*} keyValue The value of the key to search for
*/
getRecordBy(key, keyValue, callback) {
const validKeys = ["link", "title", "category", "id"];
if (!validKeys.includes(key)) callback(new Error("Given key not valid"), undefined);
const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = '${keyValue}'`;
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) callback(err, undefined);
if (rows[0]?.title) callback(undefined, rows[0]);
else callback(undefined, false);
})
}
/**
* Save the given entry to the storage medium
* @param {Object} entryObject The entry object to be saved
@@ -153,7 +151,7 @@ exports.Storage = class Storage {
saveEntry(entryObject, callback) {
log.DEBUG("Saving entry:", entryObject);
if (!entryObject?.title || !entryObject?.link || !entryObject?.category) {
callback(new Error("Entry object malformed, check the object before saving it"), undefined)
return callback(new Error("Entry object malformed, check the object before saving it"), undefined)
}
const sqlQuery = `INSERT INTO ${this.dbTable} (title, link, category, guild_id, channel_id) VALUES ('${entryObject.title}', '${entryObject.link}', '${entryObject.category}', '${entryObject.guild_id}', '${entryObject.channel_id}');`;
@@ -161,8 +159,8 @@ exports.Storage = class Storage {
log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) callback(err, undefined);
callback(undefined, rows);
if (err) return callback(err, undefined);
return callback(undefined, rows);
})
}
@@ -173,9 +171,9 @@ exports.Storage = class Storage {
*/
updateEntry(entryObject, callback) {
let queryParams = [];
if (!entryObject.title) callback(new Error("No title given before updating"), undefined);
if (!entryObject.title) return callback(new Error("No title given before updating"), undefined);
queryParams.push(`title = '${entryObject.title}'`);
if (!entryObject.link) callback(new Error("No link given before updating"), undefined);
if (!entryObject.link) return callback(new Error("No link given before updating"), undefined);
queryParams.push(`link = '${entryObject.link}'`);
if (entryObject.category) queryParams.push(`category = '${entryObject.category}'`);
if (entryObject.guild_id) queryParams.push(`guild_id = '${entryObject.guild_id}'`);
@@ -200,8 +198,8 @@ exports.Storage = class Storage {
log.DEBUG(`Updating entry with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) callback(err, undefined);
callback(undefined, rows);
if (err) return callback(err, undefined);
return callback(undefined, rows);
})
}
@@ -212,14 +210,14 @@ exports.Storage = class Storage {
*/
removeEntry(title, callback) {
if (!title) {
callback(new Error("No entry title given before deleting"), undefined)
return callback(new Error("No entry title given before deleting"), undefined)
}
const sqlQuery = `DELETE FROM ${this.dbTable} WHERE title = '${title}';`;
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) callback(err, undefined);
callback(undefined, rows[0]);
if (err) return callback(err, undefined);
return callback(undefined, rows[0]);
})
}
@@ -231,9 +229,9 @@ exports.Storage = class Storage {
* @param {*} _category The category of the record
* @param {*} callback Callback function to return an error or the record
*/
returnRecord(_id, _title, _link, _category, _guild_id, _channel_id, callback) {
log.DEBUG(`Return record for these values: ID: '${_id}', Title: '${_title}', Category: '${_category}', Link: '${_link}', Guild: '${_guild_id}', Channel:'${_channel_id}'`)
if (!_link && !_title && !_guild_id && !_channel_id) callback(new Error("No link or title given when creating a record"), undefined);
returnRecord(_id, _title, _link, _category, _guild_id, _channel_id, callback, updateEnabled = true) {
log.DEBUG(`Return record for these values: ID: '${_id}', Title: '${_title}', Category: '${_category}', Link: '${_link}', Guild: '${_guild_id}', Channel:'${_channel_id}', Update Enabled: `, updateEnabled)
if (!_link && !_title && !_guild_id && !_channel_id) return callback(new Error("No link or title given when creating a record"), undefined);
let entryObject = {
"title": _title,
"link": _link,
@@ -244,58 +242,41 @@ exports.Storage = class Storage {
if (_id) {
entryObject.id = _id;
if (!updateEnabled) return callback(undefined, undefined);
this.updateEntry(entryObject, (err, rows) => {
if (err) callback(err, undefined);
if (err) return callback(err, undefined);
this.getRecordBy('id', entryObject.id, (err, record) => {
if (err) callback(err, undefined);
callback(undefined, new RSSRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id));
if (err) return callback(err, undefined);
return callback(undefined, new RSSSourceRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id));
})
})
})
}
else {
this.checkForTitle(_title, (err, titleExists) => {
if (!titleExists) {
log.DEBUG("Entry doesn't exist, making one now", entryObject);
this.saveEntry(entryObject, (err, rows) => {
if (err) callback(err, undefined);
if (err) return callback(err, undefined);
this.getRecordBy("title", entryObject.title, (err, record) => {
if (err) callback(err, undefined);
callback(undefined, new RSSRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id));
if (err) return callback(err, undefined);
return callback(undefined, new RSSSourceRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id));
})
});
}
else{
if (!updateEnabled) return callback(undefined, undefined);
this.updateEntry(entryObject, (err, rows) => {
if (err) callback(err, undefined);
if (err) return callback(err, undefined);
this.getRecordBy('title', entryObject.title, (err, record) => {
if (err) callback(err, undefined);
callback(undefined, new RSSRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id));
if (err) return callback(err, undefined);
return callback(undefined, new RSSSourceRecord(record.id, record.title, record.link, record.category, record.guild_id, record.channel_id));
})
})
})
}
})
}
}
/**
* Get all records stored
* @param {function} callback
*/
getAllRecords(callback) {
log.INFO("Getting all records");
const sqlQuery = `SELECT * FROM ${this.dbTable}`
let rssRecords = [];
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) callback(err, undefined);
for (const row of rows) {
log.DEBUG("Row from SQL query:", row);
rssRecords.push(new RSSRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id));
}
log.DEBUG("All records:", rssRecords);
callback(undefined, rssRecords);
});
}
}
}
}

72
utilities/recordHelper.js Normal file
View File

@@ -0,0 +1,72 @@
// This is for record builders
const { DebugBuilder } = require("./debugBuilder");
const log = new DebugBuilder("server", "recordHelper");
class baseRSSRecord {
/**
*
* @param {*} _id The ID of the post from the DB
* @param {*} _title The title of the post
* @param {*} _link The link to the post
* @param {*} _category The category of the post
*/
constructor(_id, _title, _link, _category) {
this.id = _id;
this.title = _title;
this.link= _link;
this.category = _category;
}
getId() {
return this.id;
}
get(key) {
log.DEBUG(`Getting key '${key}' from: `, Object(this));
if (!Object.keys(this).includes(key)) throw new Error(`Key is invalid ${key}`);
return this[key]
}
}
exports.baseRSSRecord = baseRSSRecord;
/**
* Build a Source record.
*
* A source record is the record of an RSS feed itself. Something like "https://www.leafly.com/feed".
*/
exports.RSSSourceRecord = class RSSSourceRecord extends baseRSSRecord{
/**
* @param {*} _id The ID of the post from the DB
* @param {*} _title The title of the post
* @param {*} _link The link to the post
* @param {*} _category The category of the post
* @param {*} _guild_id The guild id to receive updates on this source
* @param {*} _channel_id The channel to send updates from this source
*/
constructor(_id, _title, _link, _category, _guild_id, _channel_id) {
super(_id, _title, _link, _category);
this.guild_id = _guild_id;
this.channel_id = _channel_id;
}
}
/**
* Build a Post record.
*
* A post is an individual article/post/link from an RSS feed. Each individual post will be added to the database to be recalled and sent later.
*/
exports.RSSPostRecord = class RSSPostRecord extends baseRSSRecord{
/**
* @param {*} _id The ID of the post from the DB
* @param {*} _title The title of the post
* @param {*} _link The link to the post
* @param {*} _category The category of the post
* @param {*} _rssSourceID The ID of the RSS source that created this post
*/
constructor(_id, _title, _link, _category, _rssSourceID) {
super(_id, _title, _link, _category);
this.source_id = _rssSourceID;
}
}