114 lines
3.5 KiB
JavaScript
114 lines
3.5 KiB
JavaScript
import {
|
|
getAllFeeds,
|
|
deleteFeedByLink,
|
|
createPost,
|
|
getPostByPostId,
|
|
} from "../modules/mongo-wrappers/mongoFeedsWrappers.mjs";
|
|
import crypto from "crypto";
|
|
import { sendPost } from "../discordBot/modules/rssWrappers.mjs";
|
|
import { DebugBuilder } from "../modules/debugger.mjs";
|
|
import { removeSource } from "./sourceManager.mjs";
|
|
import UserAgent from "user-agents";
|
|
import Parser from "rss-parser";
|
|
|
|
import dotenv from "dotenv";
|
|
dotenv.config();
|
|
|
|
// Initialize the User-Agent string
|
|
process.env.USER_AGENT_STRING = new UserAgent({ platform: "Win32" }).toString();
|
|
|
|
const parser = new Parser({
|
|
headers: {
|
|
"User-Agent": process.env.USER_AGENT_STRING,
|
|
Accept: "application/rss+xml,application/xhtml+xml,application/xml",
|
|
},
|
|
});
|
|
|
|
const log = new DebugBuilder("server", "feedHandler");
|
|
|
|
export const returnHash = (...stringsIncluded) => {
|
|
return crypto
|
|
.createHash("sha1")
|
|
.update(stringsIncluded.join("-<<??//\\\\??>>-"))
|
|
.digest("base64");
|
|
};
|
|
|
|
/**
|
|
* Update the active RSS feeds and send any new posts to their discord channels
|
|
* @param {any} client The discord client to send posts with
|
|
* @returns {any}
|
|
*/
|
|
export const updateFeeds = async (client) => {
|
|
if (!client) throw new Error("Client object not passed");
|
|
|
|
try {
|
|
const records = await getAllFeeds();
|
|
|
|
const sourcePromiseArray = records.map(async (source) => {
|
|
log.DEBUG("Processing source:", source.title);
|
|
|
|
try {
|
|
const parsedFeed = await parser.parseURL(source.link);
|
|
|
|
if (parsedFeed?.items) {
|
|
await Promise.all(
|
|
parsedFeed.items.reverse().map(async (post) => {
|
|
log.DEBUG("Processing post:", post.title);
|
|
|
|
if (!post.title || !post.link)
|
|
throw new Error("Missing title or link in the post");
|
|
if (!post.content && !post["content:encoded"])
|
|
log.WARN("No content for post:", post.title);
|
|
|
|
post.postId =
|
|
post.postId ??
|
|
post.guid ??
|
|
post.id ??
|
|
returnHash(post.title, post.link, post.pubDate ?? Date.now());
|
|
|
|
const existingRecord = await getPostByPostId(post.postId);
|
|
if (!existingRecord) {
|
|
const channel = client.channels.cache.get(source.channel_id);
|
|
const sendResults = await sendPost(post, source, channel);
|
|
if (!sendResults) throw new Error("Failed to send post");
|
|
|
|
log.DEBUG(
|
|
"Saving post to database:",
|
|
post.title,
|
|
source.channel_id,
|
|
);
|
|
|
|
const postToSave = {
|
|
title: post.title,
|
|
link: post.link,
|
|
pubDate: post.pubDate,
|
|
author: post.author,
|
|
contentSnippet: post.contentSnippet,
|
|
id: post.id,
|
|
isoDate: post.isoDate,
|
|
postId: post.postId,
|
|
};
|
|
|
|
await createPost(postToSave);
|
|
log.DEBUG("Post saved:", postToSave);
|
|
}
|
|
}),
|
|
);
|
|
} else {
|
|
await deleteFeedByLink(source.link);
|
|
}
|
|
} catch (err) {
|
|
log.ERROR("Error processing source:", source.title, err);
|
|
await removeSource(source.link);
|
|
throw err;
|
|
}
|
|
});
|
|
|
|
await Promise.all(sourcePromiseArray);
|
|
log.DEBUG("All sources processed");
|
|
} catch (error) {
|
|
log.ERROR("Error updating feeds:", error);
|
|
throw error;
|
|
}
|
|
};
|