Linting
All checks were successful
release-tag / release-image (push) Successful in 1m52s
Lint JavaScript/Node.js / lint-js (push) Successful in 11s
DRB Tests / drb_mocha_tests (push) Successful in 29s

This commit is contained in:
Logan Cusano
2024-08-11 15:57:46 -04:00
parent 5cd47378d6
commit 117cbea67f
37 changed files with 2273 additions and 1738 deletions

View File

@@ -1,28 +1,36 @@
import { getAllFeeds, deleteFeedByLink, createPost, getPostByPostId } from '../modules/mongo-wrappers/mongoFeedsWrappers.mjs';
import crypto from 'crypto';
import { sendPost } from '../discordBot/modules/rssWrappers.mjs';
import {
getAllFeeds,
deleteFeedByLink,
createPost,
getPostByPostId,
} from "../modules/mongo-wrappers/mongoFeedsWrappers.mjs";
import crypto from "crypto";
import { sendPost } from "../discordBot/modules/rssWrappers.mjs";
import { DebugBuilder } from "../modules/debugger.mjs";
import { removeSource } from './sourceManager.mjs'
import { removeSource } from "./sourceManager.mjs";
import UserAgent from "user-agents";
import Parser from 'rss-parser';
import Parser from "rss-parser";
import dotenv from 'dotenv';
dotenv.config()
import dotenv from "dotenv";
dotenv.config();
// Initialize the User-Agent string
process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
process.env.USER_AGENT_STRING = new UserAgent({ platform: "Win32" }).toString();
const parser = new Parser({
headers: {
'User-Agent': process.env.USER_AGENT_STRING,
"Accept": "application/rss+xml,application/xhtml+xml,application/xml"
}
"User-Agent": process.env.USER_AGENT_STRING,
Accept: "application/rss+xml,application/xhtml+xml,application/xml",
},
});
const log = new DebugBuilder("server", "feedHandler");
export const returnHash = (...stringsIncluded) => {
return crypto.createHash('sha1').update(stringsIncluded.join("-<<??//\\\\??>>-")).digest("base64");
return crypto
.createHash("sha1")
.update(stringsIncluded.join("-<<??//\\\\??>>-"))
.digest("base64");
};
/**
@@ -37,43 +45,55 @@ export const updateFeeds = async (client) => {
const records = await getAllFeeds();
const sourcePromiseArray = records.map(async (source) => {
log.DEBUG('Processing source:', source.title);
log.DEBUG("Processing source:", source.title);
try {
const parsedFeed = await parser.parseURL(source.link);
if (parsedFeed?.items) {
await Promise.all(parsedFeed.items.reverse().map(async (post) => {
log.DEBUG("Processing post:", post.title);
await Promise.all(
parsedFeed.items.reverse().map(async (post) => {
log.DEBUG("Processing post:", post.title);
if (!post.title || !post.link) throw new Error("Missing title or link in the post");
if (!post.content && !post['content:encoded']) log.WARN("No content for post:", post.title);
if (!post.title || !post.link)
throw new Error("Missing title or link in the post");
if (!post.content && !post["content:encoded"])
log.WARN("No content for post:", post.title);
post.postId = post.postId ?? post.guid ?? post.id ?? returnHash(post.title, post.link, post.pubDate ?? Date.now());
post.postId =
post.postId ??
post.guid ??
post.id ??
returnHash(post.title, post.link, post.pubDate ?? Date.now());
const existingRecord = await getPostByPostId(post.postId);
if (!existingRecord) {
const channel = client.channels.cache.get(source.channel_id);
const sendResults = await sendPost(post, source, channel);
if (!sendResults) throw new Error("Failed to send post");
const existingRecord = await getPostByPostId(post.postId);
if (!existingRecord) {
const channel = client.channels.cache.get(source.channel_id);
const sendResults = await sendPost(post, source, channel);
if (!sendResults) throw new Error("Failed to send post");
log.DEBUG("Saving post to database:", post.title, source.channel_id);
log.DEBUG(
"Saving post to database:",
post.title,
source.channel_id,
);
const postToSave = {
title: post.title,
link: post.link,
pubDate: post.pubDate,
author: post.author,
contentSnippet: post.contentSnippet,
id: post.id,
isoDate: post.isoDate,
postId: post.postId
};
const postToSave = {
title: post.title,
link: post.link,
pubDate: post.pubDate,
author: post.author,
contentSnippet: post.contentSnippet,
id: post.id,
isoDate: post.isoDate,
postId: post.postId,
};
await createPost(postToSave);
log.DEBUG("Post saved:", postToSave);
}
}));
await createPost(postToSave);
log.DEBUG("Post saved:", postToSave);
}
}),
);
} else {
await deleteFeedByLink(source.link);
}
@@ -90,4 +110,4 @@ export const updateFeeds = async (client) => {
log.ERROR("Error updating feeds:", error);
throw error;
}
};
};