Fixed bug in RSS engine

index.js

@@ -7,7 +7,6 @@ var http = require('http');
 
 const fs = require('fs');
 require('dotenv').config();
-const libCore = require("./libCore");
 const { RSSController } = require("./controllers/rssController");
 const libUtils = require("./libUtils");
 const deployCommands = require("./utilities/deployCommands");
@@ -15,10 +14,6 @@ const deployCommands = require("./utilities/deployCommands");
 const { DebugBuilder } = require("./utilities/debugBuilder");
 const log = new DebugBuilder("server", "index");
 
-const {
-	Routes
-} = require('discord-api-types/v9');
-
 //const Discord = require('discord.js');Client, Collection, Intents
 const {
 	Client,
@@ -36,6 +31,7 @@ const client = new Client({
 
 prefix = process.env.PREFIX 
 discordToken = process.env.TOKEN;
+rssTimeoutValue = process.env.RSS_TIMEOUT_VALUE ?? 300000;
 
 var indexRouter = require('./routes/index');
 var nodesRouter = require('./routes/nodes');
@@ -85,7 +81,7 @@ app.use((err, req, res, next) => {
 /**
  * Start the HTTP background server
  */
-function runHTTPServer() {
+async function runHTTPServer() {
 	var server = http.createServer(app);
 	server.listen(port);
 
@@ -96,6 +92,14 @@ function runHTTPServer() {
 	})
 }
 
+/**
+ * Start the RSS background process
+ */
+async function runRssService() {
+	const rssController = new RSSController(client);
+	rssController.start();
+}
+
 // Discord bot config
 
 // Setup commands for the Discord bot
@@ -124,9 +128,8 @@ client.on('ready', () => {
 	log.DEBUG(`Starting HTTP Server`);
 	runHTTPServer();
 
-	log.DEBUG("Loading new posts");
+	log.DEBUG("Starting RSS watcher");
-	const rssManager = new RSSController(client);
+	runRssService();
-	rssManager.start();
 });
 
 // Setup any additional event handlers

libCore.js

@@ -70,35 +70,12 @@ exports.deleteSource = function (title, callback) {
   });
 }  
 
-/**
- * Adds a new source url to configured storage
- * @constructor
- * @param {string} category - Category to select Feed by, defaults to all.
- */
-exports.getPosts = async (category) => {
-  postStorage.getAllRecords((err, results) => {
-    if (category == null || category == undefined || category == "" || category.toLowerCase() == "all") {
-        if (err) throw err;
-        if (results.length > 1)
-        return results;
-      } 
-    
-    var rssFilteredFeed;    
-    for (const rssFeed of results){
-      if (rssFeed.category.toLowerCase() == category.toLowerCase()) {
-        rssFilteredFeed.push(rssFeed);
-      }
-    }
-    return rssFilteredFeed;
-  });
-}
-
 /**
  * Update channels with new posts from sources
  */
 exports.updateFeeds = async (client) => {
   if (!client) throw new Error("Client object not passed");
-  feedStorage.getAllRecords((err, records) => {
+  feedStorage.getAllRecords(async (err, records) => {
     // Load the posts from each RSS source
     for (const source of records) {
       log.DEBUG('Record title: ', source.title);
@@ -107,31 +84,35 @@ exports.updateFeeds = async (client) => {
       log.DEBUG('Record guild ID: ', source.guild_id);
       log.DEBUG('Record channel ID: ', source.channel_id);  
 
-      parser.parseURL(source.link, (err, parsedFeed) => {
+      await parser.parseURL(source.link, async (err, parsedFeed) => {
         if (err) {
           log.ERROR("Parser Error: ", source.link, err);
           //return;
-        }    
+        } 
+        
+        log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
         
         if (parsedFeed?.items){                      
-          for (const post of parsedFeed.items){            
+          for (const post of parsedFeed.items){        
-            if (post.title && post.link && post.content && post.guid && post.pubDate){
+            log.VERBOSE("Post from feed: ", post);
-              postStorage.getRecordBy('post_guid', post.guid, (err, results) => {
+            if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){
+              post.postId = post.postId ?? post.guid ?? post.id;
+              postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
                 if (err) throw err;
-                log.DEBUG("Results from asdasdasd: ", results);
+                log.DEBUG("Existing post record: ", existingRecord);
-                if (!results){
+                if (!existingRecord){
                   const channel = client.channels.cache.get(source.channel_id);
-                  libUtils.sendPost(post, channel, (err, results) =>{
+                  libUtils.sendPost(post, channel, (err, sendResults) =>{
                     if (err) throw err;
 
-                    if (results){
+                    if (sendResults){
-                      log.DEBUG("Saving post to database: ", results, post.title, source.channel_id);
+                      log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id);
 
-                      postStorage.savePost(post, (err, results) => {
+                      postStorage.savePost(post, (err, saveResults) => {
                         if(err) throw err;
 
-                        if (results) {
+                        if (saveResults) {
-                          log.DEBUG("Saved results: ", results);
+                          log.DEBUG("Saved results: ", saveResults);
                         }
                       });                  
                     }                

libStorage.js

@@ -430,11 +430,11 @@ exports.PostStorage = class PostStorage extends Storage {
     savePost(_postObject, callback){
         const tempCreationDate = returnMysqlTime();
         log.DEBUG("Saving Post Object:", _postObject);
-        if (!_postObject?.guid || !_postObject?.link) {
+        if (!_postObject?.postId || !_postObject?.link) {
             return callback(new Error("Post object malformed, check the object before saving it"), undefined)
         }
     
-        const sqlQuery  = `INSERT INTO ${this.dbTable} (post_guid, post_link, post_sent_date) VALUES ('${_postObject.guid}','${_postObject.link}','${tempCreationDate}');`;
+        const sqlQuery  = `INSERT INTO ${this.dbTable} (post_guid, post_link, post_sent_date) VALUES ('${_postObject.postId}','${_postObject.link}','${tempCreationDate}');`;
 
         log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`)
 

libUtils.js

@@ -65,16 +65,17 @@ exports.sendPost = (post, channel, callback) => {
   const title = post.title;
   const link = post.link;
   const content = NodeHtmlMarkdown.translate(post.content);
-  const guid = post.guid;
+  const postId = post.postId;
-  const pubDate = post.pubDate ?? new Date();
+  const pubDate = new Date(post.pubDate).toISOString() ?? new Date().toISOString();
-  log.DEBUG("Sending an RSS post to discord", title, guid)
+  log.DEBUG("Sending an RSS post to discord", title, postId)
 
   const rssMessage = new EmbedBuilder()
     .setColor(0x0099FF)
     .setTitle(title)
     .setURL(link)
-    .setDescription(content)
+    .setDescription(`${content}`)
-    .setTimestamp(pubDate)
+    .addFields({ name: 'Published', value: pubDate, inline: true })
+    .setTimestamp()
     .setFooter({ text: 'Brought to you by Emmelia.' });
 
   try{