diff --git a/index.js b/index.js index 78a11bc..a99a8f2 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,6 @@ var http = require('http'); const fs = require('fs'); require('dotenv').config(); -const libCore = require("./libCore"); const { RSSController } = require("./controllers/rssController"); const libUtils = require("./libUtils"); const deployCommands = require("./utilities/deployCommands"); @@ -15,10 +14,6 @@ const deployCommands = require("./utilities/deployCommands"); const { DebugBuilder } = require("./utilities/debugBuilder"); const log = new DebugBuilder("server", "index"); -const { - Routes -} = require('discord-api-types/v9'); - //const Discord = require('discord.js');Client, Collection, Intents const { Client, @@ -36,6 +31,7 @@ const client = new Client({ prefix = process.env.PREFIX discordToken = process.env.TOKEN; +rssTimeoutValue = process.env.RSS_TIMEOUT_VALUE ?? 300000; var indexRouter = require('./routes/index'); var nodesRouter = require('./routes/nodes'); @@ -85,7 +81,7 @@ app.use((err, req, res, next) => { /** * Start the HTTP background server */ -function runHTTPServer() { +async function runHTTPServer() { var server = http.createServer(app); server.listen(port); @@ -96,6 +92,14 @@ function runHTTPServer() { }) } +/** + * Start the RSS background process + */ +async function runRssService() { + const rssController = new RSSController(client); + rssController.start(); +} + // Discord bot config // Setup commands for the Discord bot @@ -124,9 +128,8 @@ client.on('ready', () => { log.DEBUG(`Starting HTTP Server`); runHTTPServer(); - log.DEBUG("Loading new posts"); - const rssManager = new RSSController(client); - rssManager.start(); + log.DEBUG("Starting RSS watcher"); + runRssService(); }); // Setup any additional event handlers diff --git a/libCore.js b/libCore.js index 108b5d8..7852e07 100644 --- a/libCore.js +++ b/libCore.js @@ -70,35 +70,12 @@ exports.deleteSource = function (title, callback) { }); } -/** - * Adds a new source url to configured storage - * @constructor - * @param {string} category - Category to select Feed by, defaults to all. - */ -exports.getPosts = async (category) => { - postStorage.getAllRecords((err, results) => { - if (category == null || category == undefined || category == "" || category.toLowerCase() == "all") { - if (err) throw err; - if (results.length > 1) - return results; - } - - var rssFilteredFeed; - for (const rssFeed of results){ - if (rssFeed.category.toLowerCase() == category.toLowerCase()) { - rssFilteredFeed.push(rssFeed); - } - } - return rssFilteredFeed; - }); -} - /** * Update channels with new posts from sources */ exports.updateFeeds = async (client) => { if (!client) throw new Error("Client object not passed"); - feedStorage.getAllRecords((err, records) => { + feedStorage.getAllRecords(async (err, records) => { // Load the posts from each RSS source for (const source of records) { log.DEBUG('Record title: ', source.title); @@ -107,31 +84,35 @@ exports.updateFeeds = async (client) => { log.DEBUG('Record guild ID: ', source.guild_id); log.DEBUG('Record channel ID: ', source.channel_id); - parser.parseURL(source.link, (err, parsedFeed) => { + await parser.parseURL(source.link, async (err, parsedFeed) => { if (err) { log.ERROR("Parser Error: ", source.link, err); //return; - } + } + + log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title); if (parsedFeed?.items){ - for (const post of parsedFeed.items){ - if (post.title && post.link && post.content && post.guid && post.pubDate){ - postStorage.getRecordBy('post_guid', post.guid, (err, results) => { + for (const post of parsedFeed.items){ + log.VERBOSE("Post from feed: ", post); + if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){ + post.postId = post.postId ?? post.guid ?? post.id; + postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => { if (err) throw err; - log.DEBUG("Results from asdasdasd: ", results); - if (!results){ + log.DEBUG("Existing post record: ", existingRecord); + if (!existingRecord){ const channel = client.channels.cache.get(source.channel_id); - libUtils.sendPost(post, channel, (err, results) =>{ + libUtils.sendPost(post, channel, (err, sendResults) =>{ if (err) throw err; - if (results){ - log.DEBUG("Saving post to database: ", results, post.title, source.channel_id); + if (sendResults){ + log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id); - postStorage.savePost(post, (err, results) => { + postStorage.savePost(post, (err, saveResults) => { if(err) throw err; - if (results) { - log.DEBUG("Saved results: ", results); + if (saveResults) { + log.DEBUG("Saved results: ", saveResults); } }); } diff --git a/libStorage.js b/libStorage.js index af4792e..1ce3cb7 100644 --- a/libStorage.js +++ b/libStorage.js @@ -430,11 +430,11 @@ exports.PostStorage = class PostStorage extends Storage { savePost(_postObject, callback){ const tempCreationDate = returnMysqlTime(); log.DEBUG("Saving Post Object:", _postObject); - if (!_postObject?.guid || !_postObject?.link) { + if (!_postObject?.postId || !_postObject?.link) { return callback(new Error("Post object malformed, check the object before saving it"), undefined) } - const sqlQuery = `INSERT INTO ${this.dbTable} (post_guid, post_link, post_sent_date) VALUES ('${_postObject.guid}','${_postObject.link}','${tempCreationDate}');`; + const sqlQuery = `INSERT INTO ${this.dbTable} (post_guid, post_link, post_sent_date) VALUES ('${_postObject.postId}','${_postObject.link}','${tempCreationDate}');`; log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`) diff --git a/libUtils.js b/libUtils.js index 50ce568..d252e7d 100644 --- a/libUtils.js +++ b/libUtils.js @@ -65,16 +65,17 @@ exports.sendPost = (post, channel, callback) => { const title = post.title; const link = post.link; const content = NodeHtmlMarkdown.translate(post.content); - const guid = post.guid; - const pubDate = post.pubDate ?? new Date(); - log.DEBUG("Sending an RSS post to discord", title, guid) + const postId = post.postId; + const pubDate = new Date(post.pubDate).toISOString() ?? new Date().toISOString(); + log.DEBUG("Sending an RSS post to discord", title, postId) const rssMessage = new EmbedBuilder() .setColor(0x0099FF) .setTitle(title) .setURL(link) - .setDescription(content) - .setTimestamp(pubDate) + .setDescription(`${content}`) + .addFields({ name: 'Published', value: pubDate, inline: true }) + .setTimestamp() .setFooter({ text: 'Brought to you by Emmelia.' }); try{