Working RSS Feeds

- Need to get worker in the background to update the sources

commands/add.js

@@ -43,6 +43,5 @@ module.exports = {
 			log.ERROR(err)        
       await interaction.reply(err.toString());
 		}
-    libCore.loadFeeds(); 
 	}
 };

commands/get.js

@@ -1,28 +0,0 @@
-const libCore = require("../libCore.js");
-const { SlashCommandBuilder } = require('discord.js');
-const { DebugBuilder } = require("../utilities/debugBuilder");
-const log = new DebugBuilder("server", "get");
-
-module.exports = {
-	data: new SlashCommandBuilder()
-	.setName('get')
-	.setDescription('Get RSS link by number')
-	.addNumberOption(option =>
-		option.setName('position')
-			.setDescription('The index position of the RSS link')
-			.setRequired(true)),
-	example: "get [1]",
-	isPrivileged: false,
-	requiresTokens: false,
-	async execute(interaction) {
-		try {
-            var search = args[0];
-            var catName = "All";
-            var feedArray = libCore.getFeeds();
-            await interaction.reply(`**Retrieving**: [${catName}] (${feedArray[search].link})`);
-        }catch(err){
-			log.ERROR(err)        
-            //await interaction.reply(err.toString());
-		}
-	}
-};

commands/random.js

@@ -1,32 +0,0 @@
-var libCore = require("../libCore.js");
-
-const { SlashCommandBuilder } = require('discord.js');
-const { DebugBuilder } = require("../utilities/debugBuilder");
-const log = new DebugBuilder("server", "random");
-
-module.exports = {
-	data: new SlashCommandBuilder()
-	.setName('random')
-	.setDescription('Get a random link from one of the RSS feeds.')
-	.addStringOption(option =>
-		option.setName('category')
-			.setDescription('Select the category to grab from *(default is "ALL")*')
-			.setRequired(false)),
-	example: "random [category]",
-	isPrivileged: false,
-	requiresTokens: false,
-	async execute(interaction) {
-		try {
-            let category = interaction.options.getString('category');
-            if (!category) category = "ALL";
-
-            var feedArray = libCore.getFeeds(category);
-            var i = Math.floor(Math.random() * (feedArray.length - 0) + 0);
-
-            await interaction.reply(`**Retrieved**: [${category}](${feedArray[i].link})`);
-        } catch (err) {    
-            log.ERROR(err)        
-            //await interaction.reply(err.toString());
-        }
-	}
-};

commands/reload.js

@@ -1,23 +0,0 @@
-var libCore = require("../libCore.js");
-
-const { SlashCommandBuilder } = require('discord.js');
-const { DebugBuilder } = require("../utilities/debugBuilder");
-const log = new DebugBuilder("server", "update");
-
-module.exports = {
-	data: new SlashCommandBuilder()
-	.setName('reload')
-	.setDescription('Reloads all RSS feeds'),
-	example: "reload",
-	isPrivileged: false,
-	requiresTokens: false,
-	async execute(interaction) {
-		try{
-			libCore.loadFeeds();
-            await interaction.reply("Reloaded all RSS feeds");
-		}catch(err){
-			log.ERROR(err)        
-            //await interaction.reply(err.toString());
-		}
-	}
-};

commands/remove.js

@@ -20,14 +20,13 @@ module.exports = {
       var title = interaction.options.getString("title");
 
       libCore.deleteSource(title, (err, result) => {
-        console.log("Result from removing entry", result);
+        log.DEBUG("Result from removing entry", result);
 
         if (result) {
           interaction.reply(`Removing ${title} from the list of RSS sources`);
         } else {
           interaction.reply(`${title} does not exist in the list of RSS sources`);
         }
-        libCore.loadFeeds();
       });
 		}catch(err){
 			log.ERROR(err)        

index.js

@@ -91,12 +91,7 @@ function runHTTPServer() {
 	server.on('error', libUtils.onError);
 
 	server.on('listening', () => {
-		log.INFO("HTTP server started!");
+		log.INFO("HTTP server started!");		
-		try {
-			libCore.feedArray = libCore.getFeeds();
-		} catch (error) {
-			log.ERROR(error);
-		}		
 	})
 }
 
@@ -123,10 +118,13 @@ client.on('ready', () => {
 
 	// Deploy slash commands
 	log.DEBUG("Deploying slash commands");	
-	deployCommands.deploy(client.guilds.cache.map(guild => guild.id));
+	deployCommands.deploy(client.guilds.cache.map(guild => guild.id));	
 
 	log.DEBUG(`Starting HTTP Server`);
 	runHTTPServer();
+
+	log.DEBUG("Loading new posts");	
+	libCore.updateFeeds(client);
 });
 
 // Setup any additional event handlers
@@ -143,10 +141,4 @@ for (const file of eventFiles) {
 	}
 }
 
-client.login(discordToken); //Load Client Discord Token
+client.login(discordToken); //Load Client Discord Token
-try {
-	log.INFO("Loading initial startup feeds");
-	libCore.loadFeeds();
-} catch (error) {
-	log.ERROR(error);
-}

libCore.js

@@ -3,8 +3,11 @@ let Parser = require('rss-parser');
 const axios = require('axios');
 let parser = new Parser();
 const storageHandler = require("./libStorage");
+const libUtils = require("./libUtils");
+const { createHash } = require("crypto");
 
 const { DebugBuilder } = require("./utilities/debugBuilder");
+const { all } = require('axios');
 const log = new DebugBuilder("server", "libCore");
 
 /* OpenAI config
@@ -18,12 +21,13 @@ const openai = new OpenAIApi(configuration);
 */
 
 // Data Structures
-var feeds = [];
+var feeds = {};
 var rssFeedMap = [];
 var rssFeedCategories = [];
 
 // Setup Storage handlers
 var feedStorage = new storageHandler.feedStorage();
+var postStorage = new storageHandler.postStorage();
 
 /**
  * Adds or updates new source url to configured storage
@@ -51,18 +55,6 @@ exports.addSource = async (title, link, category, guildId, channelId, callback)
 
     log.DEBUG("Record ID:", record.getId());
 
-    var linkData = {
-      title: `${title}`,
-      link: `${link}`,
-      category: `${category}`,
-      id: record.getId()
-    }
-
-    log.DEBUG("Link Data:", linkData);
-
-    feeds.push(linkData);
-
-    log.DEBUG("pushed item to feeds");
     return callback(undefined, record);
   });
 
@@ -74,123 +66,99 @@ exports.addSource = async (title, link, category, guildId, channelId, callback)
  * @param {string} title - Title/Name of the RSS feed.
  */
 exports.deleteSource = function (title, callback) {
-  var deleteRecord = "";
+  feedStorage.getRecordBy('title', title, (err, results) => {
-  for (i = 0; i < feeds.length; i++) {
+    if (err) return callback(err, undefined);
-    if (feeds[i].title == title) {
+
-      deleteRecord = feeds[i].id;
+    if (!results?.id) {
+      log.DEBUG("No record found for title: ", title)
+      return callback(undefined, undefined);
     }
-  }
+
-  feedStorage.destroy(deleteRecord, function (err, deletedRecord) {
+    feedStorage.destroy(results.id, function (err, deletedRecord) {
-    if (err) {
+      if (err) {
-      log.ERROR(err);
+        log.ERROR(err);
-      return callback(err, undefined);
+        return callback(err, undefined);
-    }
+      }
-    log.DEBUG(deletedRecord.id);
+      log.DEBUG("Deleted Record: ", deletedRecord);
-    return callback(undefined, deletedRecord);
+      return callback(undefined, deletedRecord ?? true);
+    });
   });
-}
+}  
 
 /**
  * Adds a new source url to configured storage
  * @constructor
- * @param {string} feedType - Category to select Feed by.
+ * @param {string} category - Category to select Feed by, defaults to all.
  */
-exports.getFeeds = function (feedType) {
+exports.getPosts = async (category) => {
-  var rssFeedFilteredMap = [];
+  postStorage.getAllRecords((err, results) => {
-  if (feedType == null || feedType == undefined || feedType == "" || feedType.toLowerCase() == "all") {
+    if (category == null || category == undefined || category == "" || category.toLowerCase() == "all") {
-    return rssFeedMap;
+        if (err) throw err;
-  } else {
+        if (results.length > 1)
-
+        return results;
-    rssFeedMap.forEach(rssFeed => {
+      } 
-      if (rssFeed.category.toLowerCase().indexOf(feedType.toLowerCase()) > -1) {
+    
-        rssFeedFilteredMap.push(rssFeed);
+    var rssFilteredFeed;    
+    for (const rssFeed of results){
+      if (rssFeed.category.toLowerCase() == category.toLowerCase()) {
+        rssFilteredFeed.push(rssFeed);
       }
-    });
+    }
-    return rssFeedFilteredMap;
+    return rssFilteredFeed;
-  }
+  });
 }
 
 /**
- * Load the RSS feeds
+ * Update channels with new posts from sources
  */
-exports.loadFeeds = function () {
+exports.updateFeeds = async (client) => {
-  feeds = [];
+  if (!client) throw new Error("Client object not passed");
-  rssFeedMap = [];
-  rssFeedCategories = [];
-
   feedStorage.getAllRecords((err, records) => {
-    for (const record of records){
+    // Load the posts from each RSS source
-      try {          
+    for (const source of records) {
-        log.DEBUG('Retrieved title: ', record.title);
+      log.DEBUG('Record title: ', source.title);
-        log.DEBUG('Retrieved link: ', record.link);
+      log.DEBUG('Record link: ', source.link);
-        log.DEBUG('Retrieved category: ', record.category);
+      log.DEBUG('Record category: ', source.category);
-        log.DEBUG('Retrieved guild ID: ', record.guild_id);
+      log.DEBUG('Record guild ID: ', source.guild_id);
-        log.DEBUG('Retrieved channel ID: ', record.channel_id);        
+      log.DEBUG('Record channel ID: ', source.channel_id);  
+
+      parser.parseURL(source.link, (err, parsedFeed) => {
+        if (err) {
+          log.ERROR("Parser Error: ", source.link, err);
+          //return;
+        }    
         
-        // Check to see if this source has been loaded already
+        if (parsedFeed?.items){                      
-        feeds.forEach(feedSource => {            
+          for (const post of parsedFeed.items){            
-          if (feedSource.link != record.link) {              
+            if (post.title && post.link && post.content && post.guid && post.pubDate){
-            log.DEBUG("Loading new source: ", feedSource)
+              postStorage.getRecordBy('post_guid', post.guid, (err, results) => {
-            feeds.push(feedSource);              
+                if (err) throw err;
+                log.DEBUG("Results from asdasdasd: ", results);
+                if (!results){
+                  const channel = client.channels.cache.get(source.channel_id);
+                  libUtils.sendPost(post, channel, (err, results) =>{
+                    if (err) throw err;
+
+                    if (results){
+                      log.DEBUG("Saving post to database: ", results, post.title, source.channel_id);
+
+                      postStorage.savePost(post, (err, results) => {
+                        if(err) throw err;
+
+                        if (results) {
+                          log.DEBUG("Saved results: ", results);
+                        }
+                      });                  
+                    }                
+                  })                   
+                }                
+              })                           
+            }
           }
-        });          
+        }    
-
+      });
-        // Update the known categories if new category is not present        
-        if (!rssFeedCategories.includes(record.category)) {
-          log.DEBUG("Added new category to running known categories: ", record.category)
-          rssFeedCategories.push(record.category);
-        }
-        
-
-      } catch (error) {
-        log.DEBUG(error);
-      }
     }
-    });
+  });  
-}
-
-
-      /*
-      feeds.forEach(feedBlock => {
-        (async () => {
-          try {
-            const feed = parser.parseURL(feedBlock.link, function (err, feed) {
-              if (err) {
-                log.DEBUG(err + " " + feedBlock.link);
-                //return;
-              }              
-
-              if (feed != undefined && feed.items != undefined) {
-                //log.DEBUG("Feed: ", feed);
-                feed.items.forEach(item => {
-                  var foundFeed = false;
-                  rssFeedMap.forEach(rssFeed => {
-                    if (rssFeed.link == item.link) {
-                      log.DEBUG("Found matching post?: ", rssFeed)
-                      foundFeed = true;
-                    }
-                  });
-
-                  if (!foundFeed) {
-                    var linkData = {
-                      title: `${unescape(item.title)}`,
-                      link: `${unescape(item.link)}`,
-                      category: `${unescape(feedBlock.category)}`
-                    }
-                    rssFeedMap.push(linkData);
-                  }
-
-                });
-              } else {
-                log.DEBUG('error parsing :' + feedBlock.link);
-              }
-
-            })
-          } catch (error) {
-            log.DEBUG(error);
-          }          
-
-    //log.DEBUG("RSS Feed Map: ", rssFeedMap);
 }
 
 /**

libStorage.js

@@ -25,7 +25,7 @@ function runSQL(sqlQuery, connection, callback = (err, rows) => {
             log.ERROR("SQL Error:", err)
             return callback(err, undefined);
         } 
-        log.VERBOSE("RunSQL Returned Rows:", rows);        
+        log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows);        
         return callback(undefined, rows);
     })
 }
@@ -39,7 +39,37 @@ class Storage {
             database: process.env.DB_NAME
         });             
         
-        this.dbTable = _dbTable
+        this.dbTable = _dbTable;
+        this.validKeys = [];
+        
+        var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`; 
+
+        runSQL(sqlQuery, this.connection, (err, rows) => {
+            if (err) return log.ERROR("Error getting column names: ", err);
+            if (rows){
+                for (const validKey of rows){
+                    this.validKeys.push(validKey.Field);
+                }
+                log.VERBOSE(`Database rows for '${this.dbTable}': `, rows);
+                log.DEBUG(`Keys for '${this.dbTable}': `, this.validKeys);
+            }            
+        })
+    }
+
+    /**
+     * Wrapper to delete an entry using the storage method configured
+     * @param {} entryID The ID of the entry to be deleted
+     * @param {function} callback The callback function to be called with the record when deleted     
+     */
+    destroy(entryID, callback) {
+        if (!entryID) return callback(Error("No entry ID given"), undefined);
+
+        this.getRecordBy('id', entryID, (err, entryRecord) => {
+            this.removeEntry(entryRecord.id, (err, results) => {
+                if (err) return callback(err, undefined);
+                return callback(undefined, results);
+            });
+        });        
     }
 
     /**
@@ -48,15 +78,14 @@ class Storage {
      * @param {*} keyValue The value of the key to search for
      * @param {*} callback The callback function
      */
-    getRecordBy(key, keyValue, callback) {
+    getRecordBy(key, keyValue, callback) {                
-        const validKeys = ["link", "title", "category", "id"];
+        if (!this.validKeys.includes(key)) return callback(new Error("Given key not valid", key), undefined);
-        if (!validKeys.includes(key)) return callback(new Error("Given key not valid"), undefined);
 
         const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = '${keyValue}'`;
 
         runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
-            if (rows[0]?.title) return callback(undefined, rows[0]);
+            if (rows[0]?.[key]) return callback(undefined, rows[0]);
             else return callback(undefined, false);
         })
     }
@@ -66,19 +95,49 @@ class Storage {
      * @param {function} callback 
      */
     getAllRecords(callback) {
-        log.INFO("Getting all records");
+        log.INFO("Getting all records from: ", this.dbTable);
         const sqlQuery = `SELECT * FROM ${this.dbTable}`
 
-        let rssRecords = [];
+        let records = [];
 
         runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
-            for (const row of rows) {
+            for (const row of rows) {                
-                log.VERBOSE("Row from SQL query:", row);
+                if (this.dbTable == rssFeedsTable){
-                rssRecords.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id));
+                    records.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id));
+                }
+                if (this.dbTable == rssPostsTable){                    
+                    records.push(rows);
+                }
             }
-            log.VERBOSE("All records:", rssRecords);
+            log.VERBOSE("All records:", records);
-            return callback(undefined, rssRecords);
+            return callback(undefined, records);
+        });
+    }
+
+    /**
+     * Gets all unique rows in the given key
+     * @param {*} key 
+     * @param {*} callback 
+     */
+    getUniqueByKey(key, callback){
+        log.INFO("Getting all unique values in column: ", key);
+        const sqlQuery = `SELECT DISTINCT ${key} FROM ${this.dbTable}`
+
+        let records = [];
+
+        runSQL(sqlQuery, this.connection, (err, rows) => {
+            if (err) return callback(err, undefined);
+            for (const row of rows) {                
+                if (this.dbTable == rssFeedsTable){
+                    records.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id));
+                }
+                if (this.dbTable == rssPostsTable){                    
+                    records.push(rows);
+                }
+            }
+            log.VERBOSE("All records:", records);
+            return callback(undefined, records);
         });
     }
 }
@@ -109,28 +168,12 @@ exports.feedStorage = class feedStorage extends Storage {
                     log.DEBUG("One record to callback with:", record);
                     return callback(undefined, record);
                 }
-            }, false)
+            }, false) // Do not update the if it exists
         }    
         if (!toBeSaved.length === 1) {
             return callback(undefined, newRecords);
         }                
-    }
+    }    
-
-    /**
-     * Wrapper to delete an entry using the storage method configured
-     * @param {} entryID The ID of the entry to be deleted
-     * @param {function} callback The callback function to be called with the record when deleted     
-     */
-    destroy(entryID, callback) {
-        if (!entryID) return callback(Error("No entry ID given"), undefined);
-
-        this.getRecordBy('id', entryID, (err, entryRecord) => {
-            this.removeEntry(entryRecord.id, (err, results) => {
-                if (err) return callback(err, undefined);
-                return callback(undefined, results);
-            });
-        });        
-    }
 
     /**
      * Check to see if an entry exists in the storage method configured
@@ -205,15 +248,15 @@ exports.feedStorage = class feedStorage extends Storage {
 
     /**
      * Delete the given entry from the storage medium
-     * @param {string} title The title of the entry to be deleted
+     * @param {*} id The title of the entry to be deleted
      * @param {function} callback The callback to be called with either an error or undefined if successful
      */
-    removeEntry(title, callback) {
+    removeEntry(id, callback) {
-        if (!title) {
+        if (!id) {
-            return callback(new Error("No entry title given before deleting"), undefined)
+            return callback(new Error("No entry id given before deleting"), undefined)
         }
     
-        const sqlQuery  = `DELETE FROM ${this.dbTable} WHERE title = '${title}';`;
+        const sqlQuery  = `DELETE FROM ${this.dbTable} WHERE id = '${id}';`;
 
         runSQL(sqlQuery, this.connection, (err, rows) => {
             if (err) return callback(err, undefined);
@@ -280,3 +323,25 @@ exports.feedStorage = class feedStorage extends Storage {
     }    
 }
 
+exports.postStorage = class postStorage extends Storage {
+    constructor() {
+        super(rssPostsTable);
+    }    
+
+    savePost(_postObject, callback){
+        const tempCreationDate = new Date().toISOString().slice(0, 19).replace('T', ' ');
+        log.DEBUG("Saving Post Object:", _postObject);
+        if (!_postObject?.guid || !_postObject?.link) {
+            return callback(new Error("Post object malformed, check the object before saving it"), undefined)
+        }
+    
+        const sqlQuery  = `INSERT INTO ${this.dbTable} (post_guid, post_link, post_sent_date) VALUES ('${_postObject.guid}','${_postObject.link}','${tempCreationDate}');`;
+
+        log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`)
+
+        runSQL(sqlQuery, this.connection, (err, rows) => {
+            if (err) return callback(err, undefined);
+            return callback(undefined, rows);
+        })
+    }
+}

libUtils.js

@@ -1,5 +1,7 @@
+const { EmbedBuilder } = require('discord.js');
 const { DebugBuilder } = require("./utilities/debugBuilder");
 const log = new DebugBuilder("server", "libUtils");
+const { NodeHtmlMarkdown } = require('node-html-markdown');
 
 /**
  * sleep - sleep/wait
@@ -57,4 +59,30 @@ exports.onError = (error) => {
     default:
       throw error;
   }
+}
+
+exports.sendPost = (post, channel, callback) => {
+  const title = post.title;
+  const link = post.link;
+  const content = NodeHtmlMarkdown.translate(post.content);
+  const guid = post.guid;
+  const pubDate = post.pubDate ?? new Date();
+  log.DEBUG("Sending an RSS post to discord", title, guid)
+
+  const rssMessage = new EmbedBuilder()
+    .setColor(0x0099FF)
+    .setTitle(title)
+    .setURL(link)
+    .setDescription(content)
+    .setTimestamp(pubDate)
+    .setFooter({ text: 'Brought to you by Emmelia.' });
+
+  try{
+    channel.send({ embeds: [rssMessage] });
+    return callback(undefined, true);
+  }  
+  catch (err){
+    log.ERROR("Error sending message: ", err);
+    return callback(err, undefined);
+  }  
 }