logan/Emmelia-Link-Flayer-Rewrite
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects 2 Releases Wiki Activity

Improved RSS engine

Browse Source
This commit is contained in:
Logan Cusano
2023-02-26 17:09:31 -05:00
parent 086c910197
commit 716ed2325e
1 changed files with 21 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
libCore.js
View File

@@ -1,15 +1,25 @@
// JD Requires const { all } = require('axios');
let Parser = require('rss-parser');
const axios = require('axios'); const axios = require('axios');
let parser = new Parser();
const { FeedStorage, PostStorage } = require("./libStorage"); const { FeedStorage, PostStorage } = require("./libStorage");
const libUtils = require("./libUtils"); const libUtils = require("./libUtils");
const { createHash } = require("crypto");
const { all } = require('axios');
const { DebugBuilder } = require("./utilities/debugBuilder"); const { DebugBuilder } = require("./utilities/debugBuilder");
const log = new DebugBuilder("server", "libCore"); const log = new DebugBuilder("server", "libCore");
const UserAgent = require("user-agents");
process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
log.DEBUG("Generated User Agent string:", process.env.USER_AGENT_STRING);
// Initiate the parser
let Parser = require('rss-parser');
let parser = new Parser({
headers: {
'User-Agent': process.env.USER_AGENT_STRING,
"Accept": "application/rss+xml,application/xhtml+xml,application/xml"
}
});
// Setup Storage handlers // Setup Storage handlers
var feedStorage = new FeedStorage(); var feedStorage = new FeedStorage();
var postStorage = new PostStorage(); var postStorage = new PostStorage();
@@ -86,13 +96,12 @@ exports.updateFeeds = async (client) => {
await parser.parseURL(source.link, async (err, parsedFeed) => { await parser.parseURL(source.link, async (err, parsedFeed) => {
if (err) { if (err) {
log.ERROR("Parser Error: ", source.link, err); log.ERROR("Parser Error: ", source, err);
//return; return;
} }
try{ try{
log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title); log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
if (parsedFeed?.items){ if (parsedFeed?.items){
for (const post of parsedFeed.items){ for (const post of parsedFeed.items){
//log.VERBOSE("Post from feed: ", post); //log.VERBOSE("Post from feed: ", post);
@@ -114,6 +123,7 @@ exports.updateFeeds = async (client) => {
if (saveResults) { if (saveResults) {
log.DEBUG("Saved results: ", saveResults); log.DEBUG("Saved results: ", saveResults);
return;
} }
}); });
} }
@@ -125,6 +135,7 @@ exports.updateFeeds = async (client) => {
} }
}catch (err) { }catch (err) {
log.ERROR("Error Parsing Feed: ", source.link, err); log.ERROR("Error Parsing Feed: ", source.link, err);
throw err;
} }
}); });
} }