logan/Emmelia-Link-Flayer-Rewrite
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects 2 Releases Wiki Activity

Potential fix fo #2

Browse Source
This commit is contained in:
Logan Cusano
2023-03-12 03:47:49 -04:00
parent 72134b1b7b
commit ffacd19883
3 changed files with 102 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
libCore.js
View File

@@ -84,7 +84,7 @@ exports.deleteSource = function (title, callback) {
/**
* Update channels with new posts from sources
*/
exports.updateFeeds = async (client) => {
exports.updateFeeds = (client) => {
if (!client) throw new Error("Client object not passed");
// Create a temp pool to use for all connections while updating the feed
var tempConnection = mysql.createPool({
@@ -95,69 +95,90 @@ exports.updateFeeds = async (client) => {
connectionLimit: 10
});
const feedStorage = new FeedStorage(tempConnection);
const postStorage = new PostStorage(tempConnection);
const tempFeedStorage = new FeedStorage(tempConnection);
const tempPostStorage = new PostStorage(tempConnection);
feedStorage.getAllRecords(async (err, records) => {
// Array of promises to wait on before closing the connection
var recordPromiseArray = [];
var sourcePromiseArray = [];
tempFeedStorage.getAllRecords(async (err, records) => {
// Load the posts from each RSS source
for (const source of records) {
log.DEBUG('Record title: ', source.title);
log.DEBUG('Record link: ', source.link);
log.DEBUG('Record category: ', source.category);
log.DEBUG('Record guild ID: ', source.guild_id);
log.DEBUG('Record channel ID: ', source.channel_id);
for (const source of records) {
sourcePromiseArray.push(new Promise((resolve, reject) => {
log.DEBUG('Record title: ', source.title);
log.DEBUG('Record link: ', source.link);
log.DEBUG('Record category: ', source.category);
log.DEBUG('Record guild ID: ', source.guild_id);
log.DEBUG('Record channel ID: ', source.channel_id);
// Parse the RSS feed
parser.parseURL(source.link, async (err, parsedFeed) => {
if (err) {
log.ERROR("Parser Error: ", source, err);
reject;
}
try {
log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
if (parsedFeed?.items){
for (const post of parsedFeed.items){
recordPromiseArray.push(new Promise((recordResolve, recordReject) => {
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
log.VERBOSE("Post from feed: ", post);
if (!post.title || !post.link || !post.pubDate) return recordReject("Missing information from the post");
if (!post.content || !post['content:encoded']) log.WARN("There is no content for post: ", post.title);
await parser.parseURL(source.link, async (err, parsedFeed) => {
if (err) {
log.ERROR("Parser Error: ", source, err);
return;
}
post.postId = post.postId ?? post.guid ?? post.id ?? libUtils.returnHash(post.title, post.link, post.pubDate);
tempPostStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
if (err) throw err;
log.DEBUG("Existing post record: ", existingRecord);
if (existingRecord) return recordResolve("Existing record found for this post");
try{
log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
if (parsedFeed?.items){
for (const post of parsedFeed.items){
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
log.VERBOSE("Post from feed: ", post);
if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){
post.postId = post.postId ?? post.guid ?? post.id;
postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
if (err) throw err;
log.DEBUG("Existing post record: ", existingRecord);
if (!existingRecord){
const channel = client.channels.cache.get(source.channel_id);
libUtils.sendPost(post, source, channel, (err, sendResults) =>{
if (err) throw err;
if (!sendResults) {
log.ERROR("No sending results from sending a post: ", sendResults, existingRecord, post);
return recordReject();
}
if (sendResults){
log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id);
log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id);
postStorage.savePost(post, (err, saveResults) => {
if(err) throw err;
tempPostStorage.savePost(post, (err, saveResults) => {
if(err) throw err;
if (saveResults) {
log.DEBUG("Saved results: ", saveResults);
return;
}
});
}
})
}
})
}
}
}
}catch (err) {
log.ERROR("Error Parsing Feed: ", source.link, err);
throw err;
}
});
if (saveResults) {
log.DEBUG("Saved results: ", saveResults);
return recordResolve();
}
});
})
})
}))
}
}
}
catch (err) {
log.ERROR("Error Parsing Feed: ", source.link, err);
throw err;
}
Promise.all(recordPromiseArray).then((values) => {
log.DEBUG("All posts finished for: ", source.title, values);
return resolve();
});
});
}))
}
});
// Close the temp connections
feedStorage.closeConnection();
postStorage.closeConnection();
// Wait for all connections to finish then close the temp connections
Promise.all(sourcePromiseArray).then((values) => {
log.DEBUG("Closing temp connections: ", values);
tempFeedStorage.closeConnection();
tempPostStorage.closeConnection();
});
});
}
/**

4
libStorage.js
View File

@@ -52,9 +52,9 @@ function returnMysqlTime(){
}
class Storage {
constructor(_dbTable, connection = undefined) {
constructor(_dbTable, _connection) {
this.dbTable = _dbTable;
this.connection = connection
this.connection = _connection;
this.validKeys = [];
var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`;

32
libUtils.js
View File

@@ -3,8 +3,10 @@ const { DebugBuilder } = require("./utilities/debugBuilder");
const log = new DebugBuilder("server", "libUtils");
const { NodeHtmlMarkdown } = require('node-html-markdown');
const { parse } = require("node-html-parser");
const crypto = require("crypto");
const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g;
const youtubeVideoRegex = /((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)/g
exports.EmmeliaEmbedBuilder = class PostEmbedBuilder extends EmbedBuilder {
constructor() {
@@ -74,10 +76,24 @@ exports.onError = (error) => {
exports.sendPost = (post, source, channel, callback) => {
log.DEBUG("Sending post from source: ", post, source);
post.content = parse(post.content);
// Reset the content parameter with the encoded parameter
post.content = parse(post['content:encoded'] ?? post.content);
const postTitle = post.title;
const postLink = post.link;
const postContent = NodeHtmlMarkdown.translate(post.content.text);
// Get the post content and trim it to length or add a placeholder if necessary
var postText = String(post.content.text);
if (postText.length >= 300) postText = `${postText.slice(0, 300).substring(0, Math.min(String(post.content.text).length, String(post.content.text).lastIndexOf(" ")))}...`;
else if (postText.length === 0) postText = `*This post has no content* [Direct Link](${post.link})`;
var postContent = postText;
// Check for embedded youtube videos and add the first four as links
const ytVideos = String(post.content).match(youtubeVideoRegex);
if (ytVideos) {
for (const ytVideo of ytVideos.slice(0,4)){
// If the video is an embed, replace the embed to make it watchable
if (ytVideo.includes("embed")) ytVideo = ytVideo.replace("embed/", "watch?v=");
postContent += `\nEmbeded Video from Post: [YouTube](${ytVideo})`
}
}
log.DEBUG("Post content: ", postContent);
const postId = post.postId;
@@ -89,7 +105,9 @@ exports.sendPost = (post, source, channel, callback) => {
const linksInPost = post.content.querySelectorAll("a");
if (linksInPost) {
log.DEBUG("Found links in post:", linksInPost);
for (const link of linksInPost) {
for (const link of linksInPost) {
// Check to see if this link is a youtube video that was already found, if so skip it
if (ytVideos?.includes(link)) continue;
const images = String(link.getAttribute("href")).match(imageRegex);
log.DEBUG("Images found in post:", images);
if (images) {
@@ -99,13 +117,13 @@ exports.sendPost = (post, source, channel, callback) => {
}
}
log.DEBUG("Sending an RSS post to discord", postTitle, postId)
log.DEBUG("Sending an RSS post to discord", postTitle, postId, postContent)
try{
const rssMessage = new this.EmmeliaEmbedBuilder()
.setColor(0x0099FF)
.setTitle(postTitle)
.setURL(postLink)
.addFields({ name: "Post Content", value: postContent.slice(0,1024), inline: false })
.addFields({ name: "Post Content", value: postContent, inline: false })
.addFields({ name: 'Published', value: postPubDate, inline: true })
.addFields({ name: 'Source', value: postSourceLink, inline: true });
@@ -125,4 +143,8 @@ exports.sendPost = (post, source, channel, callback) => {
log.ERROR("Error sending message: ", err);
return callback(err, undefined);
}
}
exports.returnHash = (...stringsIncluded) => {
return crypto.createHash('sha1').update(`${stringsIncluded.join("-<<??//\\\\??>>-")}`).digest("base64");
}