Improved RSS engine

libCore.js

@@ -1,15 +1,25 @@
-// JD Requires
-let Parser = require('rss-parser');
+const { all } = require('axios');
 const axios = require('axios');
-let parser = new Parser();
+
 const { FeedStorage, PostStorage } = require("./libStorage");
 const libUtils = require("./libUtils");
-const { createHash } = require("crypto");
-const { all } = require('axios');
-
 const { DebugBuilder } = require("./utilities/debugBuilder");
 const log = new DebugBuilder("server", "libCore");
 
+const UserAgent = require("user-agents");
+process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
+
+log.DEBUG("Generated User Agent string:", process.env.USER_AGENT_STRING);
+
+// Initiate the parser
+let Parser = require('rss-parser');
+let parser = new Parser({
+  headers: {
+    'User-Agent': process.env.USER_AGENT_STRING,
+    "Accept": "application/rss+xml,application/xhtml+xml,application/xml"
+  }
+});
+
 // Setup Storage handlers
 var feedStorage = new FeedStorage();
 var postStorage = new PostStorage();
@@ -86,13 +96,12 @@ exports.updateFeeds = async (client) => {
 
       await parser.parseURL(source.link, async (err, parsedFeed) => {
         if (err) {
-          log.ERROR("Parser Error: ", source.link, err);
-          //return;
+          log.ERROR("Parser Error: ", source, err);
+          return;
         } 
 
         try{
-          log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
-        
+          log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);        
           if (parsedFeed?.items){                      
             for (const post of parsedFeed.items){        
               //log.VERBOSE("Post from feed: ", post);
@@ -114,6 +123,7 @@ exports.updateFeeds = async (client) => {
 
                           if (saveResults) {
                             log.DEBUG("Saved results: ", saveResults);
+                            return;
                           }
                         });                  
                       }                
@@ -125,6 +135,7 @@ exports.updateFeeds = async (client) => {
           }    
         }catch (err) {
           log.ERROR("Error Parsing Feed: ", source.link, err);
+          throw err;
         }        
       });
     }