import { getAllFeeds, deleteFeedByLink, createPost, getPostByPostId, } from "../modules/mongo-wrappers/mongoFeedsWrappers.mjs"; import crypto from "crypto"; import { sendPost } from "../discordBot/modules/rssWrappers.mjs"; import { DebugBuilder } from "../modules/debugger.mjs"; import { removeSource } from "./sourceManager.mjs"; import UserAgent from "user-agents"; import Parser from "rss-parser"; import dotenv from "dotenv"; dotenv.config(); // Initialize the User-Agent string process.env.USER_AGENT_STRING = new UserAgent({ platform: "Win32" }).toString(); const parser = new Parser({ headers: { "User-Agent": process.env.USER_AGENT_STRING, Accept: "application/rss+xml,application/xhtml+xml,application/xml", }, }); const log = new DebugBuilder("server", "feedHandler"); export const returnHash = (...stringsIncluded) => { return crypto .createHash("sha1") .update(stringsIncluded.join("-<>-")) .digest("base64"); }; /** * Update the active RSS feeds and send any new posts to their discord channels * @param {any} client The discord client to send posts with * @returns {any} */ export const updateFeeds = async (client) => { if (!client) throw new Error("Client object not passed"); try { const records = await getAllFeeds(); const sourcePromiseArray = records.map(async (source) => { log.DEBUG("Processing source:", source.title); try { const parsedFeed = await parser.parseURL(source.link); if (parsedFeed?.items) { await Promise.all( parsedFeed.items.reverse().map(async (post) => { log.DEBUG("Processing post:", post.title); if (!post.title || !post.link) throw new Error("Missing title or link in the post"); if (!post.content && !post["content:encoded"]) log.WARN("No content for post:", post.title); post.postId = post.postId ?? post.guid ?? post.id ?? returnHash(post.title, post.link, post.pubDate ?? Date.now()); const existingRecord = await getPostByPostId(post.postId); if (!existingRecord) { const channel = client.channels.cache.get(source.channel_id); const sendResults = await sendPost(post, source, channel); if (!sendResults) throw new Error("Failed to send post"); log.DEBUG( "Saving post to database:", post.title, source.channel_id, ); const postToSave = { title: post.title, link: post.link, pubDate: post.pubDate, author: post.author, contentSnippet: post.contentSnippet, id: post.id, isoDate: post.isoDate, postId: post.postId, }; await createPost(postToSave); log.DEBUG("Post saved:", postToSave); } }), ); } else { await deleteFeedByLink(source.link); } } catch (err) { log.ERROR("Error processing source:", source.title, err); await removeSource(source.link); throw err; } }); await Promise.all(sourcePromiseArray); log.DEBUG("All sources processed"); } catch (error) { log.ERROR("Error updating feeds:", error); throw error; } };