diff --git a/commands/add.js b/commands/add.js index cedb327..7c1e7aa 100644 --- a/commands/add.js +++ b/commands/add.js @@ -43,6 +43,5 @@ module.exports = { log.ERROR(err) await interaction.reply(err.toString()); } - libCore.loadFeeds(); } }; \ No newline at end of file diff --git a/commands/get.js b/commands/get.js deleted file mode 100644 index e1b9e52..0000000 --- a/commands/get.js +++ /dev/null @@ -1,28 +0,0 @@ -const libCore = require("../libCore.js"); -const { SlashCommandBuilder } = require('discord.js'); -const { DebugBuilder } = require("../utilities/debugBuilder"); -const log = new DebugBuilder("server", "get"); - -module.exports = { - data: new SlashCommandBuilder() - .setName('get') - .setDescription('Get RSS link by number') - .addNumberOption(option => - option.setName('position') - .setDescription('The index position of the RSS link') - .setRequired(true)), - example: "get [1]", - isPrivileged: false, - requiresTokens: false, - async execute(interaction) { - try { - var search = args[0]; - var catName = "All"; - var feedArray = libCore.getFeeds(); - await interaction.reply(`**Retrieving**: [${catName}] (${feedArray[search].link})`); - }catch(err){ - log.ERROR(err) - //await interaction.reply(err.toString()); - } - } -}; \ No newline at end of file diff --git a/commands/random.js b/commands/random.js deleted file mode 100644 index d8dcc11..0000000 --- a/commands/random.js +++ /dev/null @@ -1,32 +0,0 @@ -var libCore = require("../libCore.js"); - -const { SlashCommandBuilder } = require('discord.js'); -const { DebugBuilder } = require("../utilities/debugBuilder"); -const log = new DebugBuilder("server", "random"); - -module.exports = { - data: new SlashCommandBuilder() - .setName('random') - .setDescription('Get a random link from one of the RSS feeds.') - .addStringOption(option => - option.setName('category') - .setDescription('Select the category to grab from *(default is "ALL")*') - .setRequired(false)), - example: "random [category]", - isPrivileged: false, - requiresTokens: false, - async execute(interaction) { - try { - let category = interaction.options.getString('category'); - if (!category) category = "ALL"; - - var feedArray = libCore.getFeeds(category); - var i = Math.floor(Math.random() * (feedArray.length - 0) + 0); - - await interaction.reply(`**Retrieved**: [${category}](${feedArray[i].link})`); - } catch (err) { - log.ERROR(err) - //await interaction.reply(err.toString()); - } - } -}; \ No newline at end of file diff --git a/commands/reload.js b/commands/reload.js deleted file mode 100644 index 1f692ef..0000000 --- a/commands/reload.js +++ /dev/null @@ -1,23 +0,0 @@ -var libCore = require("../libCore.js"); - -const { SlashCommandBuilder } = require('discord.js'); -const { DebugBuilder } = require("../utilities/debugBuilder"); -const log = new DebugBuilder("server", "update"); - -module.exports = { - data: new SlashCommandBuilder() - .setName('reload') - .setDescription('Reloads all RSS feeds'), - example: "reload", - isPrivileged: false, - requiresTokens: false, - async execute(interaction) { - try{ - libCore.loadFeeds(); - await interaction.reply("Reloaded all RSS feeds"); - }catch(err){ - log.ERROR(err) - //await interaction.reply(err.toString()); - } - } -}; diff --git a/commands/remove.js b/commands/remove.js index 69bf6b6..42e31a1 100644 --- a/commands/remove.js +++ b/commands/remove.js @@ -20,14 +20,13 @@ module.exports = { var title = interaction.options.getString("title"); libCore.deleteSource(title, (err, result) => { - console.log("Result from removing entry", result); + log.DEBUG("Result from removing entry", result); if (result) { interaction.reply(`Removing ${title} from the list of RSS sources`); } else { interaction.reply(`${title} does not exist in the list of RSS sources`); } - libCore.loadFeeds(); }); }catch(err){ log.ERROR(err) diff --git a/index.js b/index.js index 419c8fb..a8304e6 100644 --- a/index.js +++ b/index.js @@ -91,12 +91,7 @@ function runHTTPServer() { server.on('error', libUtils.onError); server.on('listening', () => { - log.INFO("HTTP server started!"); - try { - libCore.feedArray = libCore.getFeeds(); - } catch (error) { - log.ERROR(error); - } + log.INFO("HTTP server started!"); }) } @@ -123,10 +118,13 @@ client.on('ready', () => { // Deploy slash commands log.DEBUG("Deploying slash commands"); - deployCommands.deploy(client.guilds.cache.map(guild => guild.id)); + deployCommands.deploy(client.guilds.cache.map(guild => guild.id)); log.DEBUG(`Starting HTTP Server`); runHTTPServer(); + + log.DEBUG("Loading new posts"); + libCore.updateFeeds(client); }); // Setup any additional event handlers @@ -143,10 +141,4 @@ for (const file of eventFiles) { } } -client.login(discordToken); //Load Client Discord Token -try { - log.INFO("Loading initial startup feeds"); - libCore.loadFeeds(); -} catch (error) { - log.ERROR(error); -} \ No newline at end of file +client.login(discordToken); //Load Client Discord Token \ No newline at end of file diff --git a/libCore.js b/libCore.js index 370a2b3..a47f3f1 100644 --- a/libCore.js +++ b/libCore.js @@ -3,8 +3,11 @@ let Parser = require('rss-parser'); const axios = require('axios'); let parser = new Parser(); const storageHandler = require("./libStorage"); +const libUtils = require("./libUtils"); +const { createHash } = require("crypto"); const { DebugBuilder } = require("./utilities/debugBuilder"); +const { all } = require('axios'); const log = new DebugBuilder("server", "libCore"); /* OpenAI config @@ -18,12 +21,13 @@ const openai = new OpenAIApi(configuration); */ // Data Structures -var feeds = []; +var feeds = {}; var rssFeedMap = []; var rssFeedCategories = []; // Setup Storage handlers var feedStorage = new storageHandler.feedStorage(); +var postStorage = new storageHandler.postStorage(); /** * Adds or updates new source url to configured storage @@ -51,18 +55,6 @@ exports.addSource = async (title, link, category, guildId, channelId, callback) log.DEBUG("Record ID:", record.getId()); - var linkData = { - title: `${title}`, - link: `${link}`, - category: `${category}`, - id: record.getId() - } - - log.DEBUG("Link Data:", linkData); - - feeds.push(linkData); - - log.DEBUG("pushed item to feeds"); return callback(undefined, record); }); @@ -74,123 +66,99 @@ exports.addSource = async (title, link, category, guildId, channelId, callback) * @param {string} title - Title/Name of the RSS feed. */ exports.deleteSource = function (title, callback) { - var deleteRecord = ""; - for (i = 0; i < feeds.length; i++) { - if (feeds[i].title == title) { - deleteRecord = feeds[i].id; + feedStorage.getRecordBy('title', title, (err, results) => { + if (err) return callback(err, undefined); + + if (!results?.id) { + log.DEBUG("No record found for title: ", title) + return callback(undefined, undefined); } - } - feedStorage.destroy(deleteRecord, function (err, deletedRecord) { - if (err) { - log.ERROR(err); - return callback(err, undefined); - } - log.DEBUG(deletedRecord.id); - return callback(undefined, deletedRecord); + + feedStorage.destroy(results.id, function (err, deletedRecord) { + if (err) { + log.ERROR(err); + return callback(err, undefined); + } + log.DEBUG("Deleted Record: ", deletedRecord); + return callback(undefined, deletedRecord ?? true); + }); }); -} +} /** * Adds a new source url to configured storage * @constructor - * @param {string} feedType - Category to select Feed by. + * @param {string} category - Category to select Feed by, defaults to all. */ -exports.getFeeds = function (feedType) { - var rssFeedFilteredMap = []; - if (feedType == null || feedType == undefined || feedType == "" || feedType.toLowerCase() == "all") { - return rssFeedMap; - } else { - - rssFeedMap.forEach(rssFeed => { - if (rssFeed.category.toLowerCase().indexOf(feedType.toLowerCase()) > -1) { - rssFeedFilteredMap.push(rssFeed); +exports.getPosts = async (category) => { + postStorage.getAllRecords((err, results) => { + if (category == null || category == undefined || category == "" || category.toLowerCase() == "all") { + if (err) throw err; + if (results.length > 1) + return results; + } + + var rssFilteredFeed; + for (const rssFeed of results){ + if (rssFeed.category.toLowerCase() == category.toLowerCase()) { + rssFilteredFeed.push(rssFeed); } - }); - return rssFeedFilteredMap; - } + } + return rssFilteredFeed; + }); } /** - * Load the RSS feeds + * Update channels with new posts from sources */ -exports.loadFeeds = function () { - feeds = []; - rssFeedMap = []; - rssFeedCategories = []; - +exports.updateFeeds = async (client) => { + if (!client) throw new Error("Client object not passed"); feedStorage.getAllRecords((err, records) => { - for (const record of records){ - try { - log.DEBUG('Retrieved title: ', record.title); - log.DEBUG('Retrieved link: ', record.link); - log.DEBUG('Retrieved category: ', record.category); - log.DEBUG('Retrieved guild ID: ', record.guild_id); - log.DEBUG('Retrieved channel ID: ', record.channel_id); + // Load the posts from each RSS source + for (const source of records) { + log.DEBUG('Record title: ', source.title); + log.DEBUG('Record link: ', source.link); + log.DEBUG('Record category: ', source.category); + log.DEBUG('Record guild ID: ', source.guild_id); + log.DEBUG('Record channel ID: ', source.channel_id); + + parser.parseURL(source.link, (err, parsedFeed) => { + if (err) { + log.ERROR("Parser Error: ", source.link, err); + //return; + } - // Check to see if this source has been loaded already - feeds.forEach(feedSource => { - if (feedSource.link != record.link) { - log.DEBUG("Loading new source: ", feedSource) - feeds.push(feedSource); + if (parsedFeed?.items){ + for (const post of parsedFeed.items){ + if (post.title && post.link && post.content && post.guid && post.pubDate){ + postStorage.getRecordBy('post_guid', post.guid, (err, results) => { + if (err) throw err; + log.DEBUG("Results from asdasdasd: ", results); + if (!results){ + const channel = client.channels.cache.get(source.channel_id); + libUtils.sendPost(post, channel, (err, results) =>{ + if (err) throw err; + + if (results){ + log.DEBUG("Saving post to database: ", results, post.title, source.channel_id); + + postStorage.savePost(post, (err, results) => { + if(err) throw err; + + if (results) { + log.DEBUG("Saved results: ", results); + } + }); + } + }) + } + }) + } } - }); - - // Update the known categories if new category is not present - if (!rssFeedCategories.includes(record.category)) { - log.DEBUG("Added new category to running known categories: ", record.category) - rssFeedCategories.push(record.category); - } - - - } catch (error) { - log.DEBUG(error); - } + } + }); } - }); -} - - - /* - feeds.forEach(feedBlock => { - (async () => { - try { - const feed = parser.parseURL(feedBlock.link, function (err, feed) { - if (err) { - log.DEBUG(err + " " + feedBlock.link); - //return; - } - - if (feed != undefined && feed.items != undefined) { - //log.DEBUG("Feed: ", feed); - feed.items.forEach(item => { - var foundFeed = false; - rssFeedMap.forEach(rssFeed => { - if (rssFeed.link == item.link) { - log.DEBUG("Found matching post?: ", rssFeed) - foundFeed = true; - } - }); - - if (!foundFeed) { - var linkData = { - title: `${unescape(item.title)}`, - link: `${unescape(item.link)}`, - category: `${unescape(feedBlock.category)}` - } - rssFeedMap.push(linkData); - } - - }); - } else { - log.DEBUG('error parsing :' + feedBlock.link); - } - - }) - } catch (error) { - log.DEBUG(error); - } - - //log.DEBUG("RSS Feed Map: ", rssFeedMap); + }); } /** diff --git a/libStorage.js b/libStorage.js index 58429f2..8c6d42a 100644 --- a/libStorage.js +++ b/libStorage.js @@ -25,7 +25,7 @@ function runSQL(sqlQuery, connection, callback = (err, rows) => { log.ERROR("SQL Error:", err) return callback(err, undefined); } - log.VERBOSE("RunSQL Returned Rows:", rows); + log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows); return callback(undefined, rows); }) } @@ -39,7 +39,37 @@ class Storage { database: process.env.DB_NAME }); - this.dbTable = _dbTable + this.dbTable = _dbTable; + this.validKeys = []; + + var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`; + + runSQL(sqlQuery, this.connection, (err, rows) => { + if (err) return log.ERROR("Error getting column names: ", err); + if (rows){ + for (const validKey of rows){ + this.validKeys.push(validKey.Field); + } + log.VERBOSE(`Database rows for '${this.dbTable}': `, rows); + log.DEBUG(`Keys for '${this.dbTable}': `, this.validKeys); + } + }) + } + + /** + * Wrapper to delete an entry using the storage method configured + * @param {} entryID The ID of the entry to be deleted + * @param {function} callback The callback function to be called with the record when deleted + */ + destroy(entryID, callback) { + if (!entryID) return callback(Error("No entry ID given"), undefined); + + this.getRecordBy('id', entryID, (err, entryRecord) => { + this.removeEntry(entryRecord.id, (err, results) => { + if (err) return callback(err, undefined); + return callback(undefined, results); + }); + }); } /** @@ -48,15 +78,14 @@ class Storage { * @param {*} keyValue The value of the key to search for * @param {*} callback The callback function */ - getRecordBy(key, keyValue, callback) { - const validKeys = ["link", "title", "category", "id"]; - if (!validKeys.includes(key)) return callback(new Error("Given key not valid"), undefined); + getRecordBy(key, keyValue, callback) { + if (!this.validKeys.includes(key)) return callback(new Error("Given key not valid", key), undefined); const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = '${keyValue}'`; runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); - if (rows[0]?.title) return callback(undefined, rows[0]); + if (rows[0]?.[key]) return callback(undefined, rows[0]); else return callback(undefined, false); }) } @@ -66,19 +95,49 @@ class Storage { * @param {function} callback */ getAllRecords(callback) { - log.INFO("Getting all records"); + log.INFO("Getting all records from: ", this.dbTable); const sqlQuery = `SELECT * FROM ${this.dbTable}` - let rssRecords = []; + let records = []; runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); - for (const row of rows) { - log.VERBOSE("Row from SQL query:", row); - rssRecords.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id)); + for (const row of rows) { + if (this.dbTable == rssFeedsTable){ + records.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id)); + } + if (this.dbTable == rssPostsTable){ + records.push(rows); + } } - log.VERBOSE("All records:", rssRecords); - return callback(undefined, rssRecords); + log.VERBOSE("All records:", records); + return callback(undefined, records); + }); + } + + /** + * Gets all unique rows in the given key + * @param {*} key + * @param {*} callback + */ + getUniqueByKey(key, callback){ + log.INFO("Getting all unique values in column: ", key); + const sqlQuery = `SELECT DISTINCT ${key} FROM ${this.dbTable}` + + let records = []; + + runSQL(sqlQuery, this.connection, (err, rows) => { + if (err) return callback(err, undefined); + for (const row of rows) { + if (this.dbTable == rssFeedsTable){ + records.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id)); + } + if (this.dbTable == rssPostsTable){ + records.push(rows); + } + } + log.VERBOSE("All records:", records); + return callback(undefined, records); }); } } @@ -109,28 +168,12 @@ exports.feedStorage = class feedStorage extends Storage { log.DEBUG("One record to callback with:", record); return callback(undefined, record); } - }, false) + }, false) // Do not update the if it exists } if (!toBeSaved.length === 1) { return callback(undefined, newRecords); } - } - - /** - * Wrapper to delete an entry using the storage method configured - * @param {} entryID The ID of the entry to be deleted - * @param {function} callback The callback function to be called with the record when deleted - */ - destroy(entryID, callback) { - if (!entryID) return callback(Error("No entry ID given"), undefined); - - this.getRecordBy('id', entryID, (err, entryRecord) => { - this.removeEntry(entryRecord.id, (err, results) => { - if (err) return callback(err, undefined); - return callback(undefined, results); - }); - }); - } + } /** * Check to see if an entry exists in the storage method configured @@ -205,15 +248,15 @@ exports.feedStorage = class feedStorage extends Storage { /** * Delete the given entry from the storage medium - * @param {string} title The title of the entry to be deleted + * @param {*} id The title of the entry to be deleted * @param {function} callback The callback to be called with either an error or undefined if successful */ - removeEntry(title, callback) { - if (!title) { - return callback(new Error("No entry title given before deleting"), undefined) + removeEntry(id, callback) { + if (!id) { + return callback(new Error("No entry id given before deleting"), undefined) } - const sqlQuery = `DELETE FROM ${this.dbTable} WHERE title = '${title}';`; + const sqlQuery = `DELETE FROM ${this.dbTable} WHERE id = '${id}';`; runSQL(sqlQuery, this.connection, (err, rows) => { if (err) return callback(err, undefined); @@ -280,3 +323,25 @@ exports.feedStorage = class feedStorage extends Storage { } } +exports.postStorage = class postStorage extends Storage { + constructor() { + super(rssPostsTable); + } + + savePost(_postObject, callback){ + const tempCreationDate = new Date().toISOString().slice(0, 19).replace('T', ' '); + log.DEBUG("Saving Post Object:", _postObject); + if (!_postObject?.guid || !_postObject?.link) { + return callback(new Error("Post object malformed, check the object before saving it"), undefined) + } + + const sqlQuery = `INSERT INTO ${this.dbTable} (post_guid, post_link, post_sent_date) VALUES ('${_postObject.guid}','${_postObject.link}','${tempCreationDate}');`; + + log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`) + + runSQL(sqlQuery, this.connection, (err, rows) => { + if (err) return callback(err, undefined); + return callback(undefined, rows); + }) + } +} diff --git a/libUtils.js b/libUtils.js index b90f07d..50ce568 100644 --- a/libUtils.js +++ b/libUtils.js @@ -1,5 +1,7 @@ +const { EmbedBuilder } = require('discord.js'); const { DebugBuilder } = require("./utilities/debugBuilder"); const log = new DebugBuilder("server", "libUtils"); +const { NodeHtmlMarkdown } = require('node-html-markdown'); /** * sleep - sleep/wait @@ -57,4 +59,30 @@ exports.onError = (error) => { default: throw error; } +} + +exports.sendPost = (post, channel, callback) => { + const title = post.title; + const link = post.link; + const content = NodeHtmlMarkdown.translate(post.content); + const guid = post.guid; + const pubDate = post.pubDate ?? new Date(); + log.DEBUG("Sending an RSS post to discord", title, guid) + + const rssMessage = new EmbedBuilder() + .setColor(0x0099FF) + .setTitle(title) + .setURL(link) + .setDescription(content) + .setTimestamp(pubDate) + .setFooter({ text: 'Brought to you by Emmelia.' }); + + try{ + channel.send({ embeds: [rssMessage] }); + return callback(undefined, true); + } + catch (err){ + log.ERROR("Error sending message: ", err); + return callback(err, undefined); + } } \ No newline at end of file