43dfa7b5ad
All checks were successful
DRB Tests / drb_mocha_tests (pull_request) Successful in 31s
- Updated rss discord command name scheme - Implemented new sourceManager for handling feed sources - Added wrappers to delete/get feed sources by title
93 lines
3.2 KiB
JavaScript
93 lines
3.2 KiB
JavaScript
import { getAllFeeds, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs';
|
|
import crypto from 'crypto';
|
|
import { sendPost } from '../discordBot/modules/rssWrappers.mjs';
|
|
import { DebugBuilder } from "../modules/debugger.mjs";
|
|
import { removeSource } from './sourceManager.mjs'
|
|
import UserAgent from "user-agents";
|
|
import Parser from 'rss-parser';
|
|
|
|
import dotenv from 'dotenv';
|
|
dotenv.config()
|
|
|
|
// Initialize the User-Agent string
|
|
process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
|
|
|
|
const parser = new Parser({
|
|
headers: {
|
|
'User-Agent': process.env.USER_AGENT_STRING,
|
|
"Accept": "application/rss+xml,application/xhtml+xml,application/xml"
|
|
}
|
|
});
|
|
|
|
const log = new DebugBuilder("server", "feedHandler");
|
|
|
|
export const returnHash = (...stringsIncluded) => {
|
|
return crypto.createHash('sha1').update(stringsIncluded.join("-<<??//\\\\??>>-")).digest("base64");
|
|
};
|
|
|
|
/**
|
|
* Update the active RSS feeds and send any new posts to their discord channels
|
|
* @param {any} client The discord client to send posts with
|
|
* @returns {any}
|
|
*/
|
|
export const updateFeeds = async (client) => {
|
|
if (!client) throw new Error("Client object not passed");
|
|
|
|
try {
|
|
const records = await getAllFeeds();
|
|
|
|
const sourcePromiseArray = records.map(async (source) => {
|
|
log.DEBUG('Processing source:', source.title);
|
|
|
|
try {
|
|
const parsedFeed = await parser.parseURL(source.link);
|
|
|
|
if (parsedFeed?.items) {
|
|
await Promise.all(parsedFeed.items.reverse().map(async (post) => {
|
|
log.DEBUG("Processing post:", post.title);
|
|
|
|
if (!post.title || !post.link) throw new Error("Missing title or link in the post");
|
|
if (!post.content && !post['content:encoded']) log.WARN("No content for post:", post.title);
|
|
|
|
post.postId = post.postId ?? post.guid ?? post.id ?? returnHash(post.title, post.link, post.pubDate ?? Date.now());
|
|
|
|
const existingRecord = await getPostByPostId(post.postId);
|
|
if (!existingRecord) {
|
|
const channel = client.channels.cache.get(source.channel_id);
|
|
const sendResults = await sendPost(post, source, channel);
|
|
if (!sendResults) throw new Error("Failed to send post");
|
|
|
|
log.DEBUG("Saving post to database:", post.title, source.channel_id);
|
|
|
|
const postToSave = {
|
|
title: post.title,
|
|
link: post.link,
|
|
pubDate: post.pubDate,
|
|
author: post.author,
|
|
contentSnippet: post.contentSnippet,
|
|
id: post.id,
|
|
isoDate: post.isoDate,
|
|
postId: post.postId
|
|
};
|
|
|
|
await createPost(postToSave);
|
|
log.DEBUG("Post saved:", postToSave);
|
|
}
|
|
}));
|
|
} else {
|
|
await deleteFeedByLink(source.link);
|
|
}
|
|
} catch (err) {
|
|
log.ERROR("Error processing source:", source.title, err);
|
|
await removeSource(source.link);
|
|
throw err;
|
|
}
|
|
});
|
|
|
|
await Promise.all(sourcePromiseArray);
|
|
log.DEBUG("All sources processed");
|
|
} catch (error) {
|
|
log.ERROR("Error updating feeds:", error);
|
|
throw error;
|
|
}
|
|
}; |