import { createFeed, getAllFeeds, getFeedByLink, updateFeedByLink, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs'; import crypto from 'crypto'; import { sendPost } from '../discordBot/modules/rssWrappers.mjs'; import { DebugBuilder } from "../modules/debugger.mjs"; import UserAgent from "user-agents"; import Parser from 'rss-parser'; import dotenv from 'dotenv'; dotenv.config() // Initialize the User-Agent string process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString(); const parser = new Parser({ headers: { 'User-Agent': process.env.USER_AGENT_STRING, "Accept": "application/rss+xml,application/xhtml+xml,application/xml" } }); const log = new DebugBuilder("server", "feedHandler"); const sourceFailureLimit = process.env.RSS_SOURCE_FAILURE_LIMIT ?? 5; const runningSourcesToRemove = {}; // This holds the sources that are pending removal (they've failed to load, return data, etc.) export const returnHash = (...stringsIncluded) => { return crypto.createHash('sha1').update(stringsIncluded.join("-<>-")).digest("base64"); }; export const updateFeeds = async (client) => { if (!client) throw new Error("Client object not passed"); try { const records = await getAllFeeds(); const sourcePromiseArray = records.map(async (source) => { log.DEBUG('Processing source:', source.title); try { const parsedFeed = await parser.parseURL(source.link); if (parsedFeed?.items) { await Promise.all(parsedFeed.items.reverse().map(async (post) => { log.DEBUG("Processing post:", post.title); if (!post.title || !post.link) throw new Error("Missing title or link in the post"); if (!post.content && !post['content:encoded']) log.WARN("No content for post:", post.title); post.postId = post.postId ?? post.guid ?? post.id ?? returnHash(post.title, post.link, post.pubDate ?? Date.now()); const existingRecord = await getPostByPostId(post.postId); if (!existingRecord) { const channel = client.channels.cache.get(source.channel_id); const sendResults = await sendPost(post, source, channel); if (!sendResults) throw new Error("Failed to send post"); log.DEBUG("Saving post to database:", post.title, source.channel_id); const postToSave = { title: post.title, link: post.link, pubDate: post.pubDate, author: post.author, contentSnippet: post.contentSnippet, id: post.id, isoDate: post.isoDate, postId: post.postId }; await createPost(postToSave); log.DEBUG("Post saved:", postToSave); } })); } else { await deleteFeedByLink(source.link); } } catch (err) { log.ERROR("Error processing source:", source.title, err); await removeSource(source.link); throw err; } }); await Promise.all(sourcePromiseArray); log.DEBUG("All sources processed"); } catch (error) { log.ERROR("Error updating feeds:", error); throw error; } }; export const addSource = async (title, link, category, guildId, channelId, callback) => { try { const feed = { title, link, category, guild_id: guildId, channel_id: channelId }; const record = await createFeed(feed); log.DEBUG("Source added:", record); callback(null, record); } catch (err) { log.ERROR("Error adding source:", err); callback(err, null); } }; export const removeSource = async (sourceURL) => { log.INFO("Removing source:", sourceURL); if (!runningSourcesToRemove[sourceURL]) { runningSourcesToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 }; return; } const elapsedTime = Date.now() - runningSourcesToRemove[sourceURL].timestamp; const waitTime = runningSourcesToRemove[sourceURL].count * 30000; if (elapsedTime <= waitTime) { runningSourcesToRemove[sourceURL].ignoredAttempts += 1; return; } if (runningSourcesToRemove[sourceURL].count < sourceFailureLimit) { runningSourcesToRemove[sourceURL].count += 1; runningSourcesToRemove[sourceURL].timestamp = Date.now(); return; } try { const record = await getFeedByLink(sourceURL); if (!record) { log.ERROR("Source not found in storage"); return; } const results = await deleteFeedByLink(sourceURL); if (!results) { log.WARN("Failed to remove source"); return; } log.DEBUG("Source removed after exceeding failure limit:", sourceURL); } catch (err) { log.ERROR("Error removing source from storage:", err); } };