drb-server/rss-manager/feedHandler.mjs

import { createFeed, getAllFeeds, getFeedByLink, updateFeedByLink, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs';
import crypto from 'crypto';
import { sendPost } from '../discordBot/modules/rssWrappers.mjs';
import { DebugBuilder } from "../modules/debugger.mjs";
import UserAgent from "user-agents";
import Parser from 'rss-parser';

// Initialize the User-Agent string
process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();

const parser = new Parser({
  headers: {
    'User-Agent': process.env.USER_AGENT_STRING,
    "Accept": "application/rss+xml,application/xhtml+xml,application/xml"
  }
});

const log = new DebugBuilder("server", "feedHandler");
const runningPostsToRemove = {};
const sourceFailureLimit = 5;

export const returnHash = (...stringsIncluded) => {
  return crypto.createHash('sha1').update(stringsIncluded.join("-<<??//\\\\??>>-")).digest("base64");
};

export const updateFeeds = async (client) => {
  if (!client) throw new Error("Client object not passed");

  try {
    const records = await getAllFeeds();

    const sourcePromiseArray = records.map(async (source) => {
      log.DEBUG('Processing source:', source.title);

      try {
        const parsedFeed = await parser.parseURL(source.link);

        if (parsedFeed?.items) {
          await Promise.all(parsedFeed.items.reverse().map(async (post) => {
            log.DEBUG("Processing post:", post.title);

            if (!post.title || !post.link) throw new Error("Missing title or link in the post");
            if (!post.content && !post['content:encoded']) log.WARN("No content for post:", post.title);

            post.postId = post.postId ?? post.guid ?? post.id ?? returnHash(post.title, post.link, post.pubDate ?? Date.now());

            const existingRecord = await getPostByPostId(post.postId);
            if (!existingRecord) {
              const channel = client.channels.cache.get(source.channel_id);
              const sendResults = await sendPost(post, source, channel);
              if (!sendResults) throw new Error("Failed to send post");

              log.DEBUG("Saving post to database:", post.title, source.channel_id);

              const postToSave = {
                title: post.title,
                link: post.link,
                pubDate: post.pubDate,
                author: post.author,
                contentSnippet: post.contentSnippet,
                id: post.id,
                isoDate: post.isoDate,
                postId: post.postId
              };

              await createPost(postToSave);
              log.DEBUG("Post saved:", postToSave);
            }
          }));
        } else {
          await deleteFeedByLink(source.link);
        }
      } catch (err) {
        log.ERROR("Error processing source:", source.title, err);
        await removeSource(source.link);
        throw err;
      }
    });

    await Promise.all(sourcePromiseArray);
    log.DEBUG("All sources processed");
  } catch (error) {
    log.ERROR("Error updating feeds:", error);
    throw error;
  }
};

export const addSource = async (title, link, category, guildId, channelId, callback) => {
  try {
    const feed = { title, link, category, guild_id: guildId, channel_id: channelId };
    const record = await createFeed(feed);
    log.DEBUG("Source added:", record);
    callback(null, record);
  } catch (err) {
    log.ERROR("Error adding source:", err);
    callback(err, null);
  }
};

export const removeSource = async (sourceURL) => {
  log.INFO("Removing source:", sourceURL);

  if (!runningPostsToRemove[sourceURL]) {
    runningPostsToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 };
    return;
  }

  const elapsedTime = Date.now() - runningPostsToRemove[sourceURL].timestamp;
  const waitTime = runningPostsToRemove[sourceURL].count * 30000;

  if (elapsedTime <= waitTime) {
    runningPostsToRemove[sourceURL].ignoredAttempts += 1;
    return;
  }

  if (runningPostsToRemove[sourceURL].count < sourceFailureLimit) {
    runningPostsToRemove[sourceURL].count += 1;
    runningPostsToRemove[sourceURL].timestamp = Date.now();
    return;
  }

  try {
    const record = await getFeedByLink(sourceURL);
    if (!record) {
      log.ERROR("Source not found in storage");
      return;
    }

    const results = await deleteFeedByLink(sourceURL);
    if (!results) {
      log.WARN("Failed to remove source");
      return;
    }

    log.DEBUG("Source removed after exceeding failure limit:", sourceURL);
  } catch (err) {
    log.ERROR("Error removing source from storage:", err);
  }
};