diff --git a/libCore.js b/libCore.js index 0cd040b..5eb6deb 100644 --- a/libCore.js +++ b/libCore.js @@ -84,7 +84,7 @@ exports.deleteSource = function (title, callback) { /** * Update channels with new posts from sources */ -exports.updateFeeds = async (client) => { +exports.updateFeeds = (client) => { if (!client) throw new Error("Client object not passed"); // Create a temp pool to use for all connections while updating the feed var tempConnection = mysql.createPool({ @@ -95,69 +95,90 @@ exports.updateFeeds = async (client) => { connectionLimit: 10 }); - const feedStorage = new FeedStorage(tempConnection); - const postStorage = new PostStorage(tempConnection); + const tempFeedStorage = new FeedStorage(tempConnection); + const tempPostStorage = new PostStorage(tempConnection); - feedStorage.getAllRecords(async (err, records) => { + // Array of promises to wait on before closing the connection + var recordPromiseArray = []; + var sourcePromiseArray = []; + + tempFeedStorage.getAllRecords(async (err, records) => { // Load the posts from each RSS source - for (const source of records) { - log.DEBUG('Record title: ', source.title); - log.DEBUG('Record link: ', source.link); - log.DEBUG('Record category: ', source.category); - log.DEBUG('Record guild ID: ', source.guild_id); - log.DEBUG('Record channel ID: ', source.channel_id); + for (const source of records) { + sourcePromiseArray.push(new Promise((resolve, reject) => { + log.DEBUG('Record title: ', source.title); + log.DEBUG('Record link: ', source.link); + log.DEBUG('Record category: ', source.category); + log.DEBUG('Record guild ID: ', source.guild_id); + log.DEBUG('Record channel ID: ', source.channel_id); + // Parse the RSS feed + parser.parseURL(source.link, async (err, parsedFeed) => { + if (err) { + log.ERROR("Parser Error: ", source, err); + reject; + } + try { + log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title); + if (parsedFeed?.items){ + for (const post of parsedFeed.items){ + recordPromiseArray.push(new Promise((recordResolve, recordReject) => { + log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title); + log.VERBOSE("Post from feed: ", post); + if (!post.title || !post.link || !post.pubDate) return recordReject("Missing information from the post"); + if (!post.content || !post['content:encoded']) log.WARN("There is no content for post: ", post.title); - await parser.parseURL(source.link, async (err, parsedFeed) => { - if (err) { - log.ERROR("Parser Error: ", source, err); - return; - } + post.postId = post.postId ?? post.guid ?? post.id ?? libUtils.returnHash(post.title, post.link, post.pubDate); + tempPostStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => { + if (err) throw err; + + log.DEBUG("Existing post record: ", existingRecord); + if (existingRecord) return recordResolve("Existing record found for this post"); - try{ - log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title); - if (parsedFeed?.items){ - for (const post of parsedFeed.items){ - log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title); - log.VERBOSE("Post from feed: ", post); - if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){ - post.postId = post.postId ?? post.guid ?? post.id; - postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => { - if (err) throw err; - log.DEBUG("Existing post record: ", existingRecord); - if (!existingRecord){ const channel = client.channels.cache.get(source.channel_id); libUtils.sendPost(post, source, channel, (err, sendResults) =>{ if (err) throw err; + + if (!sendResults) { + log.ERROR("No sending results from sending a post: ", sendResults, existingRecord, post); + return recordReject(); + } - if (sendResults){ - log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id); + log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id); - postStorage.savePost(post, (err, saveResults) => { - if(err) throw err; + tempPostStorage.savePost(post, (err, saveResults) => { + if(err) throw err; - if (saveResults) { - log.DEBUG("Saved results: ", saveResults); - return; - } - }); - } - }) - } - }) - } - } - } - }catch (err) { - log.ERROR("Error Parsing Feed: ", source.link, err); - throw err; - } - }); + if (saveResults) { + log.DEBUG("Saved results: ", saveResults); + return recordResolve(); + } + }); + }) + }) + })) + } + } + } + catch (err) { + log.ERROR("Error Parsing Feed: ", source.link, err); + throw err; + } + Promise.all(recordPromiseArray).then((values) => { + log.DEBUG("All posts finished for: ", source.title, values); + return resolve(); + }); + }); + })) } - }); -// Close the temp connections -feedStorage.closeConnection(); -postStorage.closeConnection(); + // Wait for all connections to finish then close the temp connections + + Promise.all(sourcePromiseArray).then((values) => { + log.DEBUG("Closing temp connections: ", values); + tempFeedStorage.closeConnection(); + tempPostStorage.closeConnection(); + }); + }); } /** diff --git a/libStorage.js b/libStorage.js index 1af8a3a..90f429f 100644 --- a/libStorage.js +++ b/libStorage.js @@ -52,9 +52,9 @@ function returnMysqlTime(){ } class Storage { - constructor(_dbTable, connection = undefined) { + constructor(_dbTable, _connection) { this.dbTable = _dbTable; - this.connection = connection + this.connection = _connection; this.validKeys = []; var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`; diff --git a/libUtils.js b/libUtils.js index b5febb3..2f29023 100644 --- a/libUtils.js +++ b/libUtils.js @@ -3,8 +3,10 @@ const { DebugBuilder } = require("./utilities/debugBuilder"); const log = new DebugBuilder("server", "libUtils"); const { NodeHtmlMarkdown } = require('node-html-markdown'); const { parse } = require("node-html-parser"); +const crypto = require("crypto"); const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g; +const youtubeVideoRegex = /((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)/g exports.EmmeliaEmbedBuilder = class PostEmbedBuilder extends EmbedBuilder { constructor() { @@ -74,10 +76,24 @@ exports.onError = (error) => { exports.sendPost = (post, source, channel, callback) => { log.DEBUG("Sending post from source: ", post, source); - post.content = parse(post.content); + // Reset the content parameter with the encoded parameter + post.content = parse(post['content:encoded'] ?? post.content); const postTitle = post.title; const postLink = post.link; - const postContent = NodeHtmlMarkdown.translate(post.content.text); + // Get the post content and trim it to length or add a placeholder if necessary + var postText = String(post.content.text); + if (postText.length >= 300) postText = `${postText.slice(0, 300).substring(0, Math.min(String(post.content.text).length, String(post.content.text).lastIndexOf(" ")))}...`; + else if (postText.length === 0) postText = `*This post has no content* [Direct Link](${post.link})`; + var postContent = postText; + // Check for embedded youtube videos and add the first four as links + const ytVideos = String(post.content).match(youtubeVideoRegex); + if (ytVideos) { + for (const ytVideo of ytVideos.slice(0,4)){ + // If the video is an embed, replace the embed to make it watchable + if (ytVideo.includes("embed")) ytVideo = ytVideo.replace("embed/", "watch?v="); + postContent += `\nEmbeded Video from Post: [YouTube](${ytVideo})` + } + } log.DEBUG("Post content: ", postContent); const postId = post.postId; @@ -89,7 +105,9 @@ exports.sendPost = (post, source, channel, callback) => { const linksInPost = post.content.querySelectorAll("a"); if (linksInPost) { log.DEBUG("Found links in post:", linksInPost); - for (const link of linksInPost) { + for (const link of linksInPost) { + // Check to see if this link is a youtube video that was already found, if so skip it + if (ytVideos?.includes(link)) continue; const images = String(link.getAttribute("href")).match(imageRegex); log.DEBUG("Images found in post:", images); if (images) { @@ -99,13 +117,13 @@ exports.sendPost = (post, source, channel, callback) => { } } - log.DEBUG("Sending an RSS post to discord", postTitle, postId) + log.DEBUG("Sending an RSS post to discord", postTitle, postId, postContent) try{ const rssMessage = new this.EmmeliaEmbedBuilder() .setColor(0x0099FF) .setTitle(postTitle) .setURL(postLink) - .addFields({ name: "Post Content", value: postContent.slice(0,1024), inline: false }) + .addFields({ name: "Post Content", value: postContent, inline: false }) .addFields({ name: 'Published', value: postPubDate, inline: true }) .addFields({ name: 'Source', value: postSourceLink, inline: true }); @@ -125,4 +143,8 @@ exports.sendPost = (post, source, channel, callback) => { log.ERROR("Error sending message: ", err); return callback(err, undefined); } +} + +exports.returnHash = (...stringsIncluded) => { + return crypto.createHash('sha1').update(`${stringsIncluded.join("-<>-")}`).digest("base64"); } \ No newline at end of file