logan/drb-server
1
0
Fork 0
Code Issues 16 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Initial RSS implementation

Browse Source 
- Added debug command to trigger RSS feed update from discord
This commit is contained in:
Logan Cusano
2024-05-22 00:17:36 -04:00
parent fac5274715
commit 4e71c7b167
8 changed files with 680 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
discordBot/commands/add.mjs Normal file
View File

@@ -0,0 +1,71 @@
import { SlashCommandBuilder } from 'discord.js';
import { DebugBuilder } from "../../modules/debugger.mjs";
import { addSource } from '../../rss-manager/feedHandler.mjs'
const log = new DebugBuilder("server", "add");
// Exporting data property that contains the command structure for discord including any params
export const data = new SlashCommandBuilder()
.setName('add')
.setDescription('Add RSS Source')
.addStringOption(option =>
option.setName('title')
.setDescription('The title of the RSS feed')
.setRequired(true))
.addStringOption(option =>
option.setName('link')
.setDescription('The link to the RSS feed')
.setRequired(true))
.addStringOption(option =>
option.setName('category')
.setDescription('The category for the RSS feed *("ALL" by default")*')
.setRequired(false))
// Exporting other properties
export const example = "/add [title] [https://domain.com/feed.xml] [category]"; // An example of how the command would be run in discord chat, this will be used for the help command
export const deferInitialReply = false; // If we the initial reply in discord should be deferred. This gives extra time to respond, however the method of replying is different.
/**
* Function to give the user auto-reply suggestions
* @param {any} nodeIo The nodeIO server for manipulation of sockets
* @param {any} interaction The interaction object
*/
// TODO - Setup autocorrect for the category
/*
export async function autocomplete(nodeIo, interaction) {
const focusedValue = interaction.options.getFocused();
const choices = [];
const filtered = choices.filter(choice => choice.name.startsWith(focusedValue));
console.log(focusedValue, choices, filtered);
await interaction.respond(filtered);
}
*/
/**
* The function to run when the command is called by a discord user
* @param {any} nodeIo The nodeIO server for manipulation of sockets
* @param {any} interaction The interaction object
*/
export const execute = async (nodeIo, interaction) => {
try {
var title = interaction.options.getString('title');
var link = interaction.options.getString('link');
var category = interaction.options.getString('category');
if (!category) category = "ALL";
await addSource(title, link, category, interaction.guildId, interaction.channelId, (err, result) => {
log.DEBUG("Result from adding entry", result);
if (result) {
interaction.reply(`Adding ${title} to the list of RSS sources`);
} else {
interaction.reply(`${title} already exists in the list of RSS sources`);
}
});
} catch (err) {
log.ERROR(err)
await interaction.reply(err.toString());
}
}

45
discordBot/commands/triggerRss.mjs Normal file
View File

@@ -0,0 +1,45 @@
import { SlashCommandBuilder } from 'discord.js';
import { updateFeeds } from '../../rss-manager/feedHandler.mjs'
// Exporting data property that contains the command structure for discord including any params
export const data = new SlashCommandBuilder()
.setName('trigger-rss')
.setDescription('Manually triggers an RSS feed update');
// Exporting other properties
export const example = "/trigger-rss"; // An example of how the command would be run in discord chat, this will be used for the help command
export const deferInitialReply = false; // If we the initial reply in discord should be deferred. This gives extra time to respond, however the method of replying is different.
/**
* Function to give the user auto-reply suggestions
* @param {any} nodeIo The nodeIO server for manipulation of sockets
* @param {any} interaction The interaction object
*/
/*
export async function autocomplete(nodeIo, interaction) {
const focusedValue = interaction.options.getFocused();
const choices = [];
const filtered = choices.filter(choice => choice.name.startsWith(focusedValue));
console.log(focusedValue, choices, filtered);
await interaction.respond(filtered);
}
*/
/**
* The function to run when the command is called by a discord user
* @param {any} nodeIo The nodeIO server for manipulation of sockets
* @param {any} interaction The interaction object
*/
export const execute = async (nodeIo, interaction) => {
try {
//const sockets = await nodeIo.allSockets();
//console.log("All open sockets: ", sockets);
//await interaction.reply(`**Online Sockets: '${sockets}'**`);
await interaction.reply('Triggering RSS update');
await updateFeeds(interaction.client);
//await interaction.channel.send('**Pong.**');
} catch (err) {
console.error(err);
// await interaction.reply(err.toString());
}
}

107
discordBot/modules/rssWrappers.mjs Normal file
View File

@@ -0,0 +1,107 @@
// Import necessary modules
import { EmbedBuilder } from 'discord.js';
import { DebugBuilder } from "../../modules/debugger.mjs";
import { parse } from "node-html-parser";
import { config } from 'dotenv';
// Load environment variables
config();
const log = new DebugBuilder("server", "libUtils");
const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g;
const youtubeVideoRegex = /((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)/g;
export class EmmeliaEmbedBuilder extends EmbedBuilder {
constructor() {
super();
this.setTimestamp();
this.setFooter({ text: 'Brought to you by Emmelia.' });
}
}
export const sendPost = (post, source, channel) => {
log.DEBUG("Sending post from source: ", post, source);
const postTitle = String(post.title).substring(0, 150);
const postLink = post.link;
let postContent;
if (post.content) {
// Reset the content parameter with the encoded parameter
post.content = parse(post['content:encoded'] ?? post.content);
// Get the post content and trim it to length or add a placeholder if necessary
let postText = String(post.content.text);
if (postText.length >= 3800) postText = `${postText.slice(0, 3800).substring(0, Math.min(postText.length, postText.lastIndexOf(" ")))} [...](${post.link})`;
else if (postText.length === 0) postText = `*This post has no content* [Direct Link](${post.link})`;
postContent = postText;
} else {
postContent = `*This post has no content* [Direct Link](${post.link})`;
}
// Check for embedded youtube videos and add the first four as links
const ytVideos = String(post.content).match(youtubeVideoRegex);
if (ytVideos) {
for (let ytVideo of ytVideos.slice(0, 4)) {
// If the video is an embed, replace the embed to make it watchable
if (ytVideo.includes("embed")) ytVideo = ytVideo.replace("embed/", "watch?v=");
postContent += `\nEmbeded Video from Post: [YouTube](${ytVideo})`;
}
}
log.DEBUG("Post content: ", postContent);
const postId = post.postId;
if (!post.pubDate) post.pubDate = Date.now();
const postPubDate = new Date(post.pubDate).toISOString();
const postSourceLink = source.title;
let postImage = post.image ?? undefined;
if (!postImage) {
if (post.content) {
const linksInPost = post.content.querySelectorAll("a");
if (linksInPost) {
log.DEBUG("Found links in post:", linksInPost);
for (const link of linksInPost) {
// Check to see if this link is a youtube video that was already found, if so skip it
if (ytVideos?.includes(link)) continue;
const images = String(link.getAttribute("href")).match(imageRegex);
log.DEBUG("Images found in post:", images);
if (images) {
postImage = images[0];
}
}
}
}
}
log.DEBUG("Sending an RSS post to discord", postTitle, postId, postContent);
try {
const rssMessage = new EmmeliaEmbedBuilder()
.setColor(0x0099FF)
.setTitle(postTitle)
.setURL(postLink)
.addFields({ name: 'Source', value: postSourceLink, inline: true })
.addFields({ name: 'Published', value: postPubDate, inline: true });
// TODO - If there is more than one image, create a canvas and post the created canvas
if (postImage) {
log.DEBUG("Image from post:", postImage);
rssMessage.setImage(postImage);
}
// Add the main content if it's present
postContent = postContent.slice(0, 4090).trim();
if (postContent) rssMessage.setDescription(postContent);
const channelResponse = rssMessage;
//const channelResponse = channel.send({ embeds: [rssMessage] });
log.DEBUG("Channel send response", channelResponse);
return channelResponse;
} catch (err) {
log.ERROR("Error sending message: ", postTitle, postId, postContent, postPubDate, err);
return err;
}
};

88
modules/mongo-wrappers/mongoFeedsWrappers.mjs Normal file
View File

@@ -0,0 +1,88 @@
import {
insertDocument,
getDocuments,
getDocumentByField,
updateDocumentByField,
deleteDocumentByField,
} from "./mongoHandler.mjs";
const feedCollectionName = 'feeds';
const postCollectionName = 'posts';
// Wrapper for inserting a feed
export const createFeed = async (feed) => {
try {
const insertedId = await insertDocument(feedCollectionName, feed);
return insertedId;
} catch (error) {
console.error('Error creating feed:', error);
throw error;
}
};
// Wrapper for retrieving all feeds
export const getAllFeeds = async () => {
try {
const feeds = await getDocuments(feedCollectionName);
return feeds;
} catch (error) {
console.error('Error getting all feeds:', error);
throw error;
}
};
// Wrapper for retrieving a feed by link
export const getFeedByLink = async (link) => {
try {
const feed = await getDocumentByField(feedCollectionName, 'link', link);
return feed;
} catch (error) {
console.error('Error getting feed by link:', error);
throw error;
}
};
// Wrapper for updating a feed by link
export const updateFeedByLink = async (link, updatedFields) => {
try {
const modifiedCount = await updateDocumentByField(feedCollectionName, 'link', link, updatedFields);
return modifiedCount;
} catch (error) {
console.error('Error updating feed by link:', error);
throw error;
}
};
// Wrapper for deleting a feed by link
export const deleteFeedByLink = async (link) => {
try {
const deletedCount = await deleteDocumentByField(feedCollectionName, 'link', link);
return deletedCount;
} catch (error) {
console.error('Error deleting feed by link:', error);
throw error;
}
};
// Wrapper for inserting a post
export const createPost = async (post) => {
try {
const insertedId = await insertDocument(postCollectionName, post);
return insertedId;
} catch (error) {
console.error('Error creating post:', error);
throw error;
}
};
// Wrapper for retrieving a post by postId
export const getPostByPostId = async (postId) => {
try {
const post = await getDocumentByField(postCollectionName, 'postId', postId);
return post;
} catch (error) {
console.error('Error getting post by postId:', error);
throw error;
}
};

174
package-lock.json generated
View File

@@ -14,7 +14,10 @@
"express": "^4.18.2",
"mongodb": "^6.3.0",
"morgan": "^1.10.0",
"socket.io": "^4.7.2"
"node-html-parser": "^6.1.13",
"rss-parser": "^3.13.0",
"socket.io": "^4.7.2",
"user-agents": "^1.1.208"
},
"devDependencies": {
"chai": "^5.1.0",
@@ -385,6 +388,11 @@
"npm": "1.2.8000 || >= 1.4.16"
}
},
"node_modules/boolbase": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
"integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="
},
"node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
@@ -606,6 +614,32 @@
"node": ">= 0.10"
}
},
"node_modules/css-select": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
"integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
"dependencies": {
"boolbase": "^1.0.0",
"css-what": "^6.1.0",
"domhandler": "^5.0.2",
"domutils": "^3.0.1",
"nth-check": "^2.0.1"
},
"funding": {
"url": "https://github.com/sponsors/fb55"
}
},
"node_modules/css-what": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
"integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==",
"engines": {
"node": ">= 6"
},
"funding": {
"url": "https://github.com/sponsors/fb55"
}
},
"node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -723,6 +757,57 @@
}
}
},
"node_modules/dom-serializer": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
"dependencies": {
"domelementtype": "^2.3.0",
"domhandler": "^5.0.2",
"entities": "^4.2.0"
},
"funding": {
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
}
},
"node_modules/domelementtype": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
]
},
"node_modules/domhandler": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
"dependencies": {
"domelementtype": "^2.3.0"
},
"engines": {
"node": ">= 4"
},
"funding": {
"url": "https://github.com/fb55/domhandler?sponsor=1"
}
},
"node_modules/domutils": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
"integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
"dependencies": {
"dom-serializer": "^2.0.0",
"domelementtype": "^2.3.0",
"domhandler": "^5.0.3"
},
"funding": {
"url": "https://github.com/fb55/domutils?sponsor=1"
}
},
"node_modules/dotenv": {
"version": "16.3.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz",
@@ -846,6 +931,17 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/entities": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
"engines": {
"node": ">=0.12"
},
"funding": {
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
"node_modules/escalade": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz",
@@ -1155,7 +1251,6 @@
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
"integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
"dev": true,
"bin": {
"he": "bin/he"
}
@@ -1313,6 +1408,11 @@
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
},
"node_modules/lodash.clonedeep": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
"integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ=="
},
"node_modules/lodash.snakecase": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz",
@@ -1573,6 +1673,15 @@
"node": ">= 0.6"
}
},
"node_modules/node-html-parser": {
"version": "6.1.13",
"resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-6.1.13.tgz",
"integrity": "sha512-qIsTMOY4C/dAa5Q5vsobRpOOvPfC4pB61UVW2uSwZNUp0QU/jCekTal1vMmbO0DgdHeLUJpv/ARmDqErVxA3Sg==",
"dependencies": {
"css-select": "^5.1.0",
"he": "1.2.0"
}
},
"node_modules/normalize-path": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -1582,6 +1691,17 @@
"node": ">=0.10.0"
}
},
"node_modules/nth-check": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
"integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
"dependencies": {
"boolbase": "^1.0.0"
},
"funding": {
"url": "https://github.com/fb55/nth-check?sponsor=1"
}
},
"node_modules/object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -1785,6 +1905,23 @@
"node": ">=0.10.0"
}
},
"node_modules/rss-parser": {
"version": "3.13.0",
"resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.13.0.tgz",
"integrity": "sha512-7jWUBV5yGN3rqMMj7CZufl/291QAhvrrGpDNE4k/02ZchL0npisiYYqULF71jCEKoIiHvK/Q2e6IkDwPziT7+w==",
"dependencies": {
"entities": "^2.0.3",
"xml2js": "^0.5.0"
}
},
"node_modules/rss-parser/node_modules/entities": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
"integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==",
"funding": {
"url": "https://github.com/fb55/entities?sponsor=1"
}
},
"node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -1809,6 +1946,11 @@
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"node_modules/sax": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz",
"integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA=="
},
"node_modules/send": {
"version": "0.18.0",
"resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
@@ -2155,6 +2297,14 @@
"node": ">= 0.8"
}
},
"node_modules/user-agents": {
"version": "1.1.208",
"resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.1.208.tgz",
"integrity": "sha512-OMDd2qJF3g9HVEzMGv9Zi8Fp9hF2YqfhDZRacbn/x4YknW2YGdhqrLW8MCiWgUEDB3eDLf7IvuzXGoOwWhnaBw==",
"dependencies": {
"lodash.clonedeep": "^4.5.0"
}
},
"node_modules/utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
@@ -2240,6 +2390,26 @@
}
}
},
"node_modules/xml2js": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz",
"integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==",
"dependencies": {
"sax": ">=0.6.0",
"xmlbuilder": "~11.0.0"
},
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/xmlbuilder": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==",
"engines": {
"node": ">=4.0"
}
},
"node_modules/xmlhttprequest-ssl": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz",

5
package.json
View File

@@ -21,6 +21,9 @@
"express": "^4.18.2",
"mongodb": "^6.3.0",
"morgan": "^1.10.0",
"socket.io": "^4.7.2"
"node-html-parser": "^6.1.13",
"rss-parser": "^3.13.0",
"socket.io": "^4.7.2",
"user-agents": "^1.1.208"
}
}

156
rss-manager/feedHandler.mjs Normal file
View File

@@ -0,0 +1,156 @@
import { createFeed, getAllFeeds, getFeedByLink, updateFeedByLink, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs';
import crypto from 'crypto';
import { sendPost } from '../discordBot/modules/rssWrappers.mjs';
import { DebugBuilder } from "../modules/debugger.mjs";
import UserAgent from "user-agents";
process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
// Initiate the parser
import Parser from 'rss-parser';
const parser = new Parser({
headers: {
'User-Agent': process.env.USER_AGENT_STRING,
"Accept": "application/rss+xml,application/xhtml+xml,application/xml"
}
});
const log = new DebugBuilder("server", "feedHandler");
const runningPostsToRemove = {}; // Assuming this is a global state variable
const sourceFailureLimit = 5; // Define your source failure limit here
export const returnHash = (...stringsIncluded) => {
return crypto.createHash('sha1').update(`${stringsIncluded.join("-<<??//\\\\??>>-")}`).digest("base64");
};
export const updateFeeds = async (client) => {
if (!client) throw new Error("Client object not passed");
try {
const records = await getAllFeeds();
const sourcePromiseArray = records.map(async (source) => {
log.DEBUG('Record title:', source.title);
log.DEBUG('Record link:', source.link);
log.DEBUG('Record category:', source.category);
log.DEBUG('Record guild ID:', source.guild_id);
log.DEBUG('Record channel ID:', source.channel_id);
try {
const parsedFeed = await parser.parseURL(source.link);
if (parsedFeed?.items) {
await Promise.all(parsedFeed.items.reverse().map(async (post) => {
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
log.VERBOSE("Post from feed:", post);
if (!post.title || !post.link) throw new Error("Missing information from the post");
if (!post.content || !post['content:encoded']) log.WARN("There is no content for post:", post.title);
post.postId = post.postId ?? post.guid ?? post.id ?? returnHash(post.title, post.link, post.pubDate ?? Date.now());
const existingRecord = await getPostByPostId(post.postId);
log.DEBUG("Existing post record:", existingRecord);
if (!existingRecord) {
const channel = client.channels.cache.get(source.channel_id);
const sendResults = await sendPost(post, source, channel);
if (!sendResults) throw new Error("No sending results from sending a post");
log.DEBUG("Saving post to database:", sendResults, post.title, source.channel_id);
//await createPost(post);
log.DEBUG("Saved post:", post);
}
}));
} else {
await deleteFeedByLink(source.link);
}
} catch (err) {
log.ERROR("Parser Error:", source, err);
await removeSource(source.link);
throw err;
}
});
await Promise.all(sourcePromiseArray);
log.DEBUG("All sources finished");
} catch (error) {
log.ERROR("Error updating feeds:", error);
throw error;
}
};
/**
* Adds or updates new source URL to configured storage.
* @param {string} title - Title/Name of the RSS feed.
* @param {string} link - URL of RSS feed.
* @param {string} category - Category of RSS feed.
* @param {string} guildId - Guild ID of RSS feed.
* @param {string} channelId - Channel ID of RSS feed.
* @param {function} callback - Callback function.
*/
export const addSource = async (title, link, category, guildId, channelId, callback) => {
try {
const feed = {
title,
link,
category,
guild_id: guildId,
channel_id: channelId
};
const record = await createFeed(feed);
log.DEBUG("Record ID:", record);
return callback(null, record);
} catch (err) {
log.ERROR("Error in create:", err);
return callback(err, null);
}
};
/**
* Wrapper for feeds that cause errors. By default it will wait over a day for the source to come back online before deleting it.
* @param {string} sourceURL - The URL of the feed source causing issues.
*/
export const removeSource = async (sourceURL) => {
log.INFO("Removing source URL:", sourceURL);
if (!runningPostsToRemove[sourceURL]) {
runningPostsToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 };
return;
}
const backoffDateTimeDifference = (Date.now() - runningPostsToRemove[sourceURL].timestamp);
const backoffWaitTime = (runningPostsToRemove[sourceURL].count * 30000);
log.DEBUG("Datetime", runningPostsToRemove[sourceURL], backoffDateTimeDifference, backoffWaitTime);
if (backoffDateTimeDifference <= backoffWaitTime) {
runningPostsToRemove[sourceURL].ignoredAttempts += 1;
return;
}
if (runningPostsToRemove[sourceURL].count < sourceFailureLimit) {
runningPostsToRemove[sourceURL].count += 1;
runningPostsToRemove[sourceURL].timestamp = Date.now();
return;
}
try {
const record = await getFeedByLink(sourceURL);
if (!record) {
log.ERROR("No source returned from feedStorage");
return;
}
const results = await deleteFeedByLink(sourceURL);
if (!results) {
log.WARN("No results from remove entry");
return;
}
log.DEBUG("Source exceeded the limit of retries and has been removed", sourceURL);
} catch (err) {
log.ERROR("Error removing source from feedStorage", err);
}
};

37
rss-manager/rssController.mjs Normal file
View File

@@ -0,0 +1,37 @@
//Will handle updating feeds in all channels
import { DebugBuilder } from "../modules/debugger.mjs";
import { updateFeeds } from "./feedHandler.mjs";
import dotenv from 'dotenv';
dotenv.config()
const log = new DebugBuilder("server", "rssController");
const refreshInterval = process.env.RSS_REFRESH_INTERVAL ?? 300000;
export class RSSController {
constructor(client) {
this.client = client;
}
async start() {
// Wait for the refresh period before starting RSS feeds, so the rest of the bot can start
await new Promise(resolve => setTimeout(resolve, refreshInterval));
log.INFO("Starting RSS Controller");
// Get initial feeds before starting the infinite loop
await updateFeeds(this.client);
while(true) {
// Wait for the refresh interval, then wait for the posts to return, then wait a quarter of the refresh interval to make sure everything is cleared up
await new Promise(resolve => setTimeout(resolve, refreshInterval));
await this.collectLatestPosts();
await new Promise(resolve => setTimeout(resolve, refreshInterval / 4));
}
}
async collectLatestPosts() {
log.INFO("Updating sources");
await updateFeeds(this.client);
}
}