logan/Emmelia-Link-Flayer-Rewrite
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects 2 Releases Wiki Activity

Merge pull request 'merge production into feature branch' (#4) from main into feature/merge-multiple-images

Browse Source 
Reviewed-on: #4
This commit is contained in:
logan
2023-03-12 04:01:51 -04:00
parent af74c7b90d ffacd19883
commit fcf61f3958
4 changed files with 158 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
libCore.js
View File

@@ -5,6 +5,7 @@ const { FeedStorage, PostStorage } = require("./libStorage");
const libUtils = require("./libUtils"); const libUtils = require("./libUtils");
const { DebugBuilder } = require("./utilities/debugBuilder"); const { DebugBuilder } = require("./utilities/debugBuilder");
const log = new DebugBuilder("server", "libCore"); const log = new DebugBuilder("server", "libCore");
const mysql = require("mysql");
const UserAgent = require("user-agents"); const UserAgent = require("user-agents");
process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString(); process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
@@ -83,64 +84,101 @@ exports.deleteSource = function (title, callback) {
/** /**
* Update channels with new posts from sources * Update channels with new posts from sources
*/ */
exports.updateFeeds = async (client) => { exports.updateFeeds = (client) => {
if (!client) throw new Error("Client object not passed"); if (!client) throw new Error("Client object not passed");
feedStorage.getAllRecords(async (err, records) => { // Create a temp pool to use for all connections while updating the feed
var tempConnection = mysql.createPool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASS,
database: process.env.DB_NAME,
connectionLimit: 10
});
const tempFeedStorage = new FeedStorage(tempConnection);
const tempPostStorage = new PostStorage(tempConnection);
// Array of promises to wait on before closing the connection
var recordPromiseArray = [];
var sourcePromiseArray = [];
tempFeedStorage.getAllRecords(async (err, records) => {
// Load the posts from each RSS source // Load the posts from each RSS source
for (const source of records) { for (const source of records) {
log.DEBUG('Record title: ', source.title); sourcePromiseArray.push(new Promise((resolve, reject) => {
log.DEBUG('Record link: ', source.link); log.DEBUG('Record title: ', source.title);
log.DEBUG('Record category: ', source.category); log.DEBUG('Record link: ', source.link);
log.DEBUG('Record guild ID: ', source.guild_id); log.DEBUG('Record category: ', source.category);
log.DEBUG('Record channel ID: ', source.channel_id); log.DEBUG('Record guild ID: ', source.guild_id);
log.DEBUG('Record channel ID: ', source.channel_id);
// Parse the RSS feed
parser.parseURL(source.link, async (err, parsedFeed) => {
if (err) {
log.ERROR("Parser Error: ", source, err);
reject;
}
try {
log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
if (parsedFeed?.items){
for (const post of parsedFeed.items){
recordPromiseArray.push(new Promise((recordResolve, recordReject) => {
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
log.VERBOSE("Post from feed: ", post);
if (!post.title || !post.link || !post.pubDate) return recordReject("Missing information from the post");
if (!post.content || !post['content:encoded']) log.WARN("There is no content for post: ", post.title);
await parser.parseURL(source.link, async (err, parsedFeed) => { post.postId = post.postId ?? post.guid ?? post.id ?? libUtils.returnHash(post.title, post.link, post.pubDate);
if (err) { tempPostStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
log.ERROR("Parser Error: ", source, err); if (err) throw err;
return;
} log.DEBUG("Existing post record: ", existingRecord);
if (existingRecord) return recordResolve("Existing record found for this post");
try{
log.DEBUG("Parsed Feed Keys", Object.keys(parsedFeed), parsedFeed?.title);
if (parsedFeed?.items){
for (const post of parsedFeed.items){
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
//log.VERBOSE("Post from feed: ", post);
if (post.title && post.link && post.content && ( post.postId || post.guid || post.id ) && post.pubDate){
post.postId = post.postId ?? post.guid ?? post.id;
postStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
if (err) throw err;
log.DEBUG("Existing post record: ", existingRecord);
if (!existingRecord){
const channel = client.channels.cache.get(source.channel_id); const channel = client.channels.cache.get(source.channel_id);
libUtils.sendPost(post, source, channel, (err, sendResults) =>{ libUtils.sendPost(post, source, channel, (err, sendResults) =>{
if (err) throw err; if (err) throw err;
if (!sendResults) {
log.ERROR("No sending results from sending a post: ", sendResults, existingRecord, post);
return recordReject();
}
if (sendResults){ log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id);
log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id);
postStorage.savePost(post, (err, saveResults) => { tempPostStorage.savePost(post, (err, saveResults) => {
if(err) throw err; if(err) throw err;
if (saveResults) { if (saveResults) {
log.DEBUG("Saved results: ", saveResults); log.DEBUG("Saved results: ", saveResults);
return; return recordResolve();
} }
}); });
} })
}) })
} }))
}) }
} }
} }
} catch (err) {
}catch (err) { log.ERROR("Error Parsing Feed: ", source.link, err);
log.ERROR("Error Parsing Feed: ", source.link, err); throw err;
throw err; }
} Promise.all(recordPromiseArray).then((values) => {
}); log.DEBUG("All posts finished for: ", source.title, values);
return resolve();
});
});
}))
} }
});
// Wait for all connections to finish then close the temp connections
Promise.all(sourcePromiseArray).then((values) => {
log.DEBUG("Closing temp connections: ", values);
tempFeedStorage.closeConnection();
tempPostStorage.closeConnection();
});
});
} }
/** /**
@@ -290,4 +328,4 @@ exports.getCategories = async (callback) => {
return callback(undefined, results); return callback(undefined, results);
}); });
} }

60
libStorage.js
View File

@@ -16,7 +16,7 @@ const accountsTable = process.env.DB_ACCOUNTS_TABLE;
const transactionsTable = process.env.DB_TRANSACTIONS_TABLE; const transactionsTable = process.env.DB_TRANSACTIONS_TABLE;
const pricingTable = process.env.DB_PRICING_TABLE; const pricingTable = process.env.DB_PRICING_TABLE;
var connection = mysql.createPool({ var Connection = mysql.createPool({
host: process.env.DB_HOST, host: process.env.DB_HOST,
user: process.env.DB_USER, user: process.env.DB_USER,
password: process.env.DB_PASS, password: process.env.DB_PASS,
@@ -26,14 +26,15 @@ var connection = mysql.createPool({
// Helper Functions // Helper Functions
// Function to run and handle SQL errors // Function to run and handle SQL errors
function runSQL(sqlQuery, callback = (err, rows) => { function runSQL(sqlQuery, connection, callback = (err, rows) => {
log.ERROR(err); log.ERROR(err);
throw err; throw err;
}) { }) {
// Start the MySQL Connection // Start the MySQL Connection
if (!connection) connection = Connection;
connection.query(sqlQuery, (err, rows) => { connection.query(sqlQuery, (err, rows) => {
if (err) { if (err) {
log.ERROR("SQL Error:", err) log.ERROR("SQL Error on query:", sqlQuery, err);
return callback(err, undefined); return callback(err, undefined);
} }
log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows); log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows);
@@ -51,13 +52,14 @@ function returnMysqlTime(){
} }
class Storage { class Storage {
constructor(_dbTable) { constructor(_dbTable, _connection) {
this.dbTable = _dbTable; this.dbTable = _dbTable;
this.connection = _connection;
this.validKeys = []; this.validKeys = [];
var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`; var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`;
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return log.ERROR("Error getting column names: ", err); if (err) return log.ERROR("Error getting column names: ", err);
if (rows){ if (rows){
for (const validKey of rows){ for (const validKey of rows){
@@ -96,7 +98,7 @@ class Storage {
const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = "${keyValue}"`; const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = "${keyValue}"`;
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
if (rows[0]?.[key]) return callback(undefined, rows[0]); if (rows[0]?.[key]) return callback(undefined, rows[0]);
else return callback(undefined, false); else return callback(undefined, false);
@@ -113,7 +115,7 @@ class Storage {
let records = []; let records = [];
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
for (const row of rows) { for (const row of rows) {
if (this.dbTable == rssFeedsTable){ if (this.dbTable == rssFeedsTable){
@@ -139,7 +141,7 @@ class Storage {
let records = []; let records = [];
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
for (const row of rows) { for (const row of rows) {
if (this.dbTable == rssFeedsTable){ if (this.dbTable == rssFeedsTable){
@@ -153,11 +155,21 @@ class Storage {
return callback(undefined, records); return callback(undefined, records);
}); });
} }
closeConnection() {
try {
this.connection.end();
}
catch (err) {
log.ERROR("Error closing connection :", this.connection, err);
throw err;
}
}
} }
exports.UserStorage = class UserStorage extends Storage { exports.UserStorage = class UserStorage extends Storage {
constructor() { constructor(connection = undefined) {
super(accountsTable); super(accountsTable, connection);
} }
/** /**
@@ -171,7 +183,7 @@ exports.UserStorage = class UserStorage extends Storage {
log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
if (rows?.affectedRows > 0) return callback(undefined, rows); if (rows?.affectedRows > 0) return callback(undefined, rows);
return callback(undefined, undefined); return callback(undefined, undefined);
@@ -231,7 +243,7 @@ exports.UserStorage = class UserStorage extends Storage {
log.DEBUG("Updating Balance with SQL Query: ", sqlQuery); log.DEBUG("Updating Balance with SQL Query: ", sqlQuery);
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
if (!rows?.affectedRows > 0) return callback(new Error("Error updating Balance", rows), undefined); if (!rows?.affectedRows > 0) return callback(new Error("Error updating Balance", rows), undefined);
return callback(undefined, rows); return callback(undefined, rows);
@@ -240,8 +252,8 @@ exports.UserStorage = class UserStorage extends Storage {
} }
exports.TransactionStorage = class TransactionStorage extends Storage { exports.TransactionStorage = class TransactionStorage extends Storage {
constructor() { constructor(connection = undefined) {
super(transactionsTable); super(transactionsTable, connection);
} }
createTransaction(transaction, callback){ createTransaction(transaction, callback){
@@ -249,7 +261,7 @@ exports.TransactionStorage = class TransactionStorage extends Storage {
log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
if (rows?.affectedRows > 0) return callback(undefined, rows); if (rows?.affectedRows > 0) return callback(undefined, rows);
return callback(undefined, undefined); return callback(undefined, undefined);
@@ -258,8 +270,8 @@ exports.TransactionStorage = class TransactionStorage extends Storage {
} }
exports.FeedStorage = class FeedStorage extends Storage { exports.FeedStorage = class FeedStorage extends Storage {
constructor() { constructor(connection = undefined) {
super(rssFeedsTable); super(rssFeedsTable, connection);
} }
/** /**
@@ -316,7 +328,7 @@ exports.FeedStorage = class FeedStorage extends Storage {
log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`) log.DEBUG(`Adding new entry with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
return callback(undefined, rows); return callback(undefined, rows);
}) })
@@ -355,7 +367,7 @@ exports.FeedStorage = class FeedStorage extends Storage {
log.DEBUG(`Updating entry with SQL query: '${sqlQuery}'`) log.DEBUG(`Updating entry with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
return callback(undefined, rows); return callback(undefined, rows);
}) })
@@ -373,7 +385,7 @@ exports.FeedStorage = class FeedStorage extends Storage {
const sqlQuery = `DELETE FROM ${this.dbTable} WHERE id = "${id}";`; const sqlQuery = `DELETE FROM ${this.dbTable} WHERE id = "${id}";`;
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
return callback(undefined, rows[0]); return callback(undefined, rows[0]);
}) })
@@ -439,8 +451,8 @@ exports.FeedStorage = class FeedStorage extends Storage {
} }
exports.PostStorage = class PostStorage extends Storage { exports.PostStorage = class PostStorage extends Storage {
constructor() { constructor(connection = undefined) {
super(rssPostsTable); super(rssPostsTable, connection);
} }
savePost(_postObject, callback){ savePost(_postObject, callback){
@@ -456,7 +468,7 @@ exports.PostStorage = class PostStorage extends Storage {
log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`) log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, (err, rows) => { runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined); if (err) return callback(err, undefined);
return callback(undefined, rows); return callback(undefined, rows);
}) })

32
libUtils.js
View File

@@ -3,8 +3,10 @@ const { DebugBuilder } = require("./utilities/debugBuilder");
const log = new DebugBuilder("server", "libUtils"); const log = new DebugBuilder("server", "libUtils");
const { NodeHtmlMarkdown } = require('node-html-markdown'); const { NodeHtmlMarkdown } = require('node-html-markdown');
const { parse } = require("node-html-parser"); const { parse } = require("node-html-parser");
const crypto = require("crypto");
const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g; const imageRegex = /(http(s?):)([/|.|\w|\s|-])*((\.(?:jpg|gif|png|webm))|(\/gallery\/(?:[/|.|\w|\s|-])*))/g;
const youtubeVideoRegex = /((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube(-nocookie)?\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)/g
exports.EmmeliaEmbedBuilder = class PostEmbedBuilder extends EmbedBuilder { exports.EmmeliaEmbedBuilder = class PostEmbedBuilder extends EmbedBuilder {
constructor() { constructor() {
@@ -74,10 +76,24 @@ exports.onError = (error) => {
exports.sendPost = (post, source, channel, callback) => { exports.sendPost = (post, source, channel, callback) => {
log.DEBUG("Sending post from source: ", post, source); log.DEBUG("Sending post from source: ", post, source);
post.content = parse(post.content); // Reset the content parameter with the encoded parameter
post.content = parse(post['content:encoded'] ?? post.content);
const postTitle = post.title; const postTitle = post.title;
const postLink = post.link; const postLink = post.link;
const postContent = NodeHtmlMarkdown.translate(post.content.text); // Get the post content and trim it to length or add a placeholder if necessary
var postText = String(post.content.text);
if (postText.length >= 300) postText = `${postText.slice(0, 300).substring(0, Math.min(String(post.content.text).length, String(post.content.text).lastIndexOf(" ")))}...`;
else if (postText.length === 0) postText = `*This post has no content* [Direct Link](${post.link})`;
var postContent = postText;
// Check for embedded youtube videos and add the first four as links
const ytVideos = String(post.content).match(youtubeVideoRegex);
if (ytVideos) {
for (const ytVideo of ytVideos.slice(0,4)){
// If the video is an embed, replace the embed to make it watchable
if (ytVideo.includes("embed")) ytVideo = ytVideo.replace("embed/", "watch?v=");
postContent += `\nEmbeded Video from Post: [YouTube](${ytVideo})`
}
}
log.DEBUG("Post content: ", postContent); log.DEBUG("Post content: ", postContent);
const postId = post.postId; const postId = post.postId;
@@ -89,7 +105,9 @@ exports.sendPost = (post, source, channel, callback) => {
const linksInPost = post.content.querySelectorAll("a"); const linksInPost = post.content.querySelectorAll("a");
if (linksInPost) { if (linksInPost) {
log.DEBUG("Found links in post:", linksInPost); log.DEBUG("Found links in post:", linksInPost);
for (const link of linksInPost) { for (const link of linksInPost) {
// Check to see if this link is a youtube video that was already found, if so skip it
if (ytVideos?.includes(link)) continue;
const images = String(link.getAttribute("href")).match(imageRegex); const images = String(link.getAttribute("href")).match(imageRegex);
log.DEBUG("Images found in post:", images); log.DEBUG("Images found in post:", images);
if (images) { if (images) {
@@ -99,13 +117,13 @@ exports.sendPost = (post, source, channel, callback) => {
} }
} }
log.DEBUG("Sending an RSS post to discord", postTitle, postId) log.DEBUG("Sending an RSS post to discord", postTitle, postId, postContent)
try{ try{
const rssMessage = new this.EmmeliaEmbedBuilder() const rssMessage = new this.EmmeliaEmbedBuilder()
.setColor(0x0099FF) .setColor(0x0099FF)
.setTitle(postTitle) .setTitle(postTitle)
.setURL(postLink) .setURL(postLink)
.addFields({ name: "Post Content", value: postContent.slice(0,1024), inline: false }) .addFields({ name: "Post Content", value: postContent, inline: false })
.addFields({ name: 'Published', value: postPubDate, inline: true }) .addFields({ name: 'Published', value: postPubDate, inline: true })
.addFields({ name: 'Source', value: postSourceLink, inline: true }); .addFields({ name: 'Source', value: postSourceLink, inline: true });
@@ -125,4 +143,8 @@ exports.sendPost = (post, source, channel, callback) => {
log.ERROR("Error sending message: ", err); log.ERROR("Error sending message: ", err);
return callback(err, undefined); return callback(err, undefined);
} }
}
exports.returnHash = (...stringsIncluded) => {
return crypto.createHash('sha1').update(`${stringsIncluded.join("-<<??//\\\\??>>-")}`).digest("base64");
} }

18
package.json
View File

@@ -8,25 +8,25 @@
"@discordjs/rest": "~1.5.0", "@discordjs/rest": "~1.5.0",
"axios": "~1.3.4", "axios": "~1.3.4",
"chatgpt": "~4.7.2", "chatgpt": "~4.7.2",
"cookie-parser": "~1.4.4",
"debug": "~2.6.9",
"discord-api-types": "~0.37.35", "discord-api-types": "~0.37.35",
"discord.js": "~14.7.1", "discord.js": "~14.7.1",
"dotenv": "~16.0.3", "dotenv": "~16.0.3",
"ejs": "~2.6.1",
"express": "~4.18.2", "express": "~4.18.2",
"fs": "~0.0.1-security", "fs": "~0.0.1-security",
"gpt-3-encoder": "~1.1.4",
"http-errors": "~1.6.3",
"jsdoc": "~3.6.7", "jsdoc": "~3.6.7",
"jsonfile": "~6.1.0", "jsonfile": "~6.1.0",
"morgan": "~1.9.1",
"mysql": "~2.18.1",
"node-html-markdown": "~1.3.0",
"node-html-parser": "~6.1.5",
"openai": "~3.1.0", "openai": "~3.1.0",
"parse-files": "~0.1.1", "parse-files": "~0.1.1",
"rss-parser": "~3.12.0", "rss-parser": "~3.12.0",
"mysql": "~2.18.1",
"cookie-parser": "~1.4.4",
"debug": "~2.6.9",
"ejs": "~2.6.1",
"http-errors": "~1.6.3",
"morgan": "~1.9.1",
"node-html-markdown": "~1.3.0",
"node-html-parser": "~6.1.5",
"gpt-3-encoder": "~1.1.4",
"user-agents": "~1.0.1303" "user-agents": "~1.0.1303"
}, },
"scripts": { "scripts": {