Working RSS Feeds
- Need to get worker in the background to update the sources
This commit is contained in:
Logan Cusano
196
libCore.js
196
libCore.js
@@ -3,8 +3,11 @@ let Parser = require('rss-parser');
|
||||
const axios = require('axios');
|
||||
let parser = new Parser();
|
||||
const storageHandler = require("./libStorage");
|
||||
const libUtils = require("./libUtils");
|
||||
const { createHash } = require("crypto");
|
||||
|
||||
const { DebugBuilder } = require("./utilities/debugBuilder");
|
||||
const { all } = require('axios');
|
||||
const log = new DebugBuilder("server", "libCore");
|
||||
|
||||
/* OpenAI config
|
||||
@@ -18,12 +21,13 @@ const openai = new OpenAIApi(configuration);
|
||||
*/
|
||||
|
||||
// Data Structures
|
||||
var feeds = [];
|
||||
var feeds = {};
|
||||
var rssFeedMap = [];
|
||||
var rssFeedCategories = [];
|
||||
|
||||
// Setup Storage handlers
|
||||
var feedStorage = new storageHandler.feedStorage();
|
||||
var postStorage = new storageHandler.postStorage();
|
||||
|
||||
/**
|
||||
* Adds or updates new source url to configured storage
|
||||
@@ -51,18 +55,6 @@ exports.addSource = async (title, link, category, guildId, channelId, callback)
|
||||
|
||||
log.DEBUG("Record ID:", record.getId());
|
||||
|
||||
var linkData = {
|
||||
title: `${title}`,
|
||||
link: `${link}`,
|
||||
category: `${category}`,
|
||||
id: record.getId()
|
||||
}
|
||||
|
||||
log.DEBUG("Link Data:", linkData);
|
||||
|
||||
feeds.push(linkData);
|
||||
|
||||
log.DEBUG("pushed item to feeds");
|
||||
return callback(undefined, record);
|
||||
});
|
||||
|
||||
@@ -74,123 +66,99 @@ exports.addSource = async (title, link, category, guildId, channelId, callback)
|
||||
* @param {string} title - Title/Name of the RSS feed.
|
||||
*/
|
||||
exports.deleteSource = function (title, callback) {
|
||||
var deleteRecord = "";
|
||||
for (i = 0; i < feeds.length; i++) {
|
||||
if (feeds[i].title == title) {
|
||||
deleteRecord = feeds[i].id;
|
||||
feedStorage.getRecordBy('title', title, (err, results) => {
|
||||
if (err) return callback(err, undefined);
|
||||
|
||||
if (!results?.id) {
|
||||
log.DEBUG("No record found for title: ", title)
|
||||
return callback(undefined, undefined);
|
||||
}
|
||||
}
|
||||
feedStorage.destroy(deleteRecord, function (err, deletedRecord) {
|
||||
if (err) {
|
||||
log.ERROR(err);
|
||||
return callback(err, undefined);
|
||||
}
|
||||
log.DEBUG(deletedRecord.id);
|
||||
return callback(undefined, deletedRecord);
|
||||
|
||||
feedStorage.destroy(results.id, function (err, deletedRecord) {
|
||||
if (err) {
|
||||
log.ERROR(err);
|
||||
return callback(err, undefined);
|
||||
}
|
||||
log.DEBUG("Deleted Record: ", deletedRecord);
|
||||
return callback(undefined, deletedRecord ?? true);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new source url to configured storage
|
||||
* @constructor
|
||||
* @param {string} feedType - Category to select Feed by.
|
||||
* @param {string} category - Category to select Feed by, defaults to all.
|
||||
*/
|
||||
exports.getFeeds = function (feedType) {
|
||||
var rssFeedFilteredMap = [];
|
||||
if (feedType == null || feedType == undefined || feedType == "" || feedType.toLowerCase() == "all") {
|
||||
return rssFeedMap;
|
||||
} else {
|
||||
|
||||
rssFeedMap.forEach(rssFeed => {
|
||||
if (rssFeed.category.toLowerCase().indexOf(feedType.toLowerCase()) > -1) {
|
||||
rssFeedFilteredMap.push(rssFeed);
|
||||
exports.getPosts = async (category) => {
|
||||
postStorage.getAllRecords((err, results) => {
|
||||
if (category == null || category == undefined || category == "" || category.toLowerCase() == "all") {
|
||||
if (err) throw err;
|
||||
if (results.length > 1)
|
||||
return results;
|
||||
}
|
||||
|
||||
var rssFilteredFeed;
|
||||
for (const rssFeed of results){
|
||||
if (rssFeed.category.toLowerCase() == category.toLowerCase()) {
|
||||
rssFilteredFeed.push(rssFeed);
|
||||
}
|
||||
});
|
||||
return rssFeedFilteredMap;
|
||||
}
|
||||
}
|
||||
return rssFilteredFeed;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the RSS feeds
|
||||
* Update channels with new posts from sources
|
||||
*/
|
||||
exports.loadFeeds = function () {
|
||||
feeds = [];
|
||||
rssFeedMap = [];
|
||||
rssFeedCategories = [];
|
||||
|
||||
exports.updateFeeds = async (client) => {
|
||||
if (!client) throw new Error("Client object not passed");
|
||||
feedStorage.getAllRecords((err, records) => {
|
||||
for (const record of records){
|
||||
try {
|
||||
log.DEBUG('Retrieved title: ', record.title);
|
||||
log.DEBUG('Retrieved link: ', record.link);
|
||||
log.DEBUG('Retrieved category: ', record.category);
|
||||
log.DEBUG('Retrieved guild ID: ', record.guild_id);
|
||||
log.DEBUG('Retrieved channel ID: ', record.channel_id);
|
||||
// Load the posts from each RSS source
|
||||
for (const source of records) {
|
||||
log.DEBUG('Record title: ', source.title);
|
||||
log.DEBUG('Record link: ', source.link);
|
||||
log.DEBUG('Record category: ', source.category);
|
||||
log.DEBUG('Record guild ID: ', source.guild_id);
|
||||
log.DEBUG('Record channel ID: ', source.channel_id);
|
||||
|
||||
parser.parseURL(source.link, (err, parsedFeed) => {
|
||||
if (err) {
|
||||
log.ERROR("Parser Error: ", source.link, err);
|
||||
//return;
|
||||
}
|
||||
|
||||
// Check to see if this source has been loaded already
|
||||
feeds.forEach(feedSource => {
|
||||
if (feedSource.link != record.link) {
|
||||
log.DEBUG("Loading new source: ", feedSource)
|
||||
feeds.push(feedSource);
|
||||
if (parsedFeed?.items){
|
||||
for (const post of parsedFeed.items){
|
||||
if (post.title && post.link && post.content && post.guid && post.pubDate){
|
||||
postStorage.getRecordBy('post_guid', post.guid, (err, results) => {
|
||||
if (err) throw err;
|
||||
log.DEBUG("Results from asdasdasd: ", results);
|
||||
if (!results){
|
||||
const channel = client.channels.cache.get(source.channel_id);
|
||||
libUtils.sendPost(post, channel, (err, results) =>{
|
||||
if (err) throw err;
|
||||
|
||||
if (results){
|
||||
log.DEBUG("Saving post to database: ", results, post.title, source.channel_id);
|
||||
|
||||
postStorage.savePost(post, (err, results) => {
|
||||
if(err) throw err;
|
||||
|
||||
if (results) {
|
||||
log.DEBUG("Saved results: ", results);
|
||||
}
|
||||
});
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Update the known categories if new category is not present
|
||||
if (!rssFeedCategories.includes(record.category)) {
|
||||
log.DEBUG("Added new category to running known categories: ", record.category)
|
||||
rssFeedCategories.push(record.category);
|
||||
}
|
||||
|
||||
|
||||
} catch (error) {
|
||||
log.DEBUG(error);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
feeds.forEach(feedBlock => {
|
||||
(async () => {
|
||||
try {
|
||||
const feed = parser.parseURL(feedBlock.link, function (err, feed) {
|
||||
if (err) {
|
||||
log.DEBUG(err + " " + feedBlock.link);
|
||||
//return;
|
||||
}
|
||||
|
||||
if (feed != undefined && feed.items != undefined) {
|
||||
//log.DEBUG("Feed: ", feed);
|
||||
feed.items.forEach(item => {
|
||||
var foundFeed = false;
|
||||
rssFeedMap.forEach(rssFeed => {
|
||||
if (rssFeed.link == item.link) {
|
||||
log.DEBUG("Found matching post?: ", rssFeed)
|
||||
foundFeed = true;
|
||||
}
|
||||
});
|
||||
|
||||
if (!foundFeed) {
|
||||
var linkData = {
|
||||
title: `${unescape(item.title)}`,
|
||||
link: `${unescape(item.link)}`,
|
||||
category: `${unescape(feedBlock.category)}`
|
||||
}
|
||||
rssFeedMap.push(linkData);
|
||||
}
|
||||
|
||||
});
|
||||
} else {
|
||||
log.DEBUG('error parsing :' + feedBlock.link);
|
||||
}
|
||||
|
||||
})
|
||||
} catch (error) {
|
||||
log.DEBUG(error);
|
||||
}
|
||||
|
||||
//log.DEBUG("RSS Feed Map: ", rssFeedMap);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user