import { getAllFeeds, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs'; import crypto from 'crypto'; import { sendPost } from '../discordBot/modules/rssWrappers.mjs'; import { DebugBuilder } from "../modules/debugger.mjs"; import { removeSource } from './sourceManager.mjs' import UserAgent from "user-agents"; import Parser from 'rss-parser'; import dotenv from 'dotenv'; dotenv.config() // Initialize the User-Agent string process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString(); const parser = new Parser({ headers: { 'User-Agent': process.env.USER_AGENT_STRING, "Accept": "application/rss+xml,application/xhtml+xml,application/xml" } }); const log = new DebugBuilder("server", "feedHandler"); export const returnHash = (...stringsIncluded) => { return crypto.createHash('sha1').update(stringsIncluded.join("-<>-")).digest("base64"); }; /** * Update the active RSS feeds and send any new posts to their discord channels * @param {any} client The discord client to send posts with * @returns {any} */ export const updateFeeds = async (client) => { if (!client) throw new Error("Client object not passed"); try { const records = await getAllFeeds(); const sourcePromiseArray = records.map(async (source) => { log.DEBUG('Processing source:', source.title); try { const parsedFeed = await parser.parseURL(source.link); if (parsedFeed?.items) { await Promise.all(parsedFeed.items.reverse().map(async (post) => { log.DEBUG("Processing post:", post.title); if (!post.title || !post.link) throw new Error("Missing title or link in the post"); if (!post.content && !post['content:encoded']) log.WARN("No content for post:", post.title); post.postId = post.postId ?? post.guid ?? post.id ?? returnHash(post.title, post.link, post.pubDate ?? Date.now()); const existingRecord = await getPostByPostId(post.postId); if (!existingRecord) { const channel = client.channels.cache.get(source.channel_id); const sendResults = await sendPost(post, source, channel); if (!sendResults) throw new Error("Failed to send post"); log.DEBUG("Saving post to database:", post.title, source.channel_id); const postToSave = { title: post.title, link: post.link, pubDate: post.pubDate, author: post.author, contentSnippet: post.contentSnippet, id: post.id, isoDate: post.isoDate, postId: post.postId }; await createPost(postToSave); log.DEBUG("Post saved:", postToSave); } })); } else { await deleteFeedByLink(source.link); } } catch (err) { log.ERROR("Error processing source:", source.title, err); await removeSource(source.link); throw err; } }); await Promise.all(sourcePromiseArray); log.DEBUG("All sources processed"); } catch (error) { log.ERROR("Error updating feeds:", error); throw error; } };