logan/Emmelia-Link-Flayer-Rewrite
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects 2 Releases Wiki Activity

Working RSS Feeds

Browse Source 
- Need to get worker in the background to update the sources
This commit is contained in:
Logan Cusano
2023-02-25 19:51:49 -05:00
parent ae071be8b8
commit d33f1ceccc
9 changed files with 218 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
commands/add.js
View File

@@ -43,6 +43,5 @@ module.exports = {
log.ERROR(err)
await interaction.reply(err.toString());
}
libCore.loadFeeds();
}
};

28
commands/get.js
View File

@@ -1,28 +0,0 @@
const libCore = require("../libCore.js");
const { SlashCommandBuilder } = require('discord.js');
const { DebugBuilder } = require("../utilities/debugBuilder");
const log = new DebugBuilder("server", "get");
module.exports = {
data: new SlashCommandBuilder()
.setName('get')
.setDescription('Get RSS link by number')
.addNumberOption(option =>
option.setName('position')
.setDescription('The index position of the RSS link')
.setRequired(true)),
example: "get [1]",
isPrivileged: false,
requiresTokens: false,
async execute(interaction) {
try {
var search = args[0];
var catName = "All";
var feedArray = libCore.getFeeds();
await interaction.reply(`**Retrieving**: [${catName}] (${feedArray[search].link})`);
}catch(err){
log.ERROR(err)
//await interaction.reply(err.toString());
}
}
};

32
commands/random.js
View File

@@ -1,32 +0,0 @@
var libCore = require("../libCore.js");
const { SlashCommandBuilder } = require('discord.js');
const { DebugBuilder } = require("../utilities/debugBuilder");
const log = new DebugBuilder("server", "random");
module.exports = {
data: new SlashCommandBuilder()
.setName('random')
.setDescription('Get a random link from one of the RSS feeds.')
.addStringOption(option =>
option.setName('category')
.setDescription('Select the category to grab from *(default is "ALL")*')
.setRequired(false)),
example: "random [category]",
isPrivileged: false,
requiresTokens: false,
async execute(interaction) {
try {
let category = interaction.options.getString('category');
if (!category) category = "ALL";
var feedArray = libCore.getFeeds(category);
var i = Math.floor(Math.random() * (feedArray.length - 0) + 0);
await interaction.reply(`**Retrieved**: [${category}](${feedArray[i].link})`);
} catch (err) {
log.ERROR(err)
//await interaction.reply(err.toString());
}
}
};

23
commands/reload.js
View File

@@ -1,23 +0,0 @@
var libCore = require("../libCore.js");
const { SlashCommandBuilder } = require('discord.js');
const { DebugBuilder } = require("../utilities/debugBuilder");
const log = new DebugBuilder("server", "update");
module.exports = {
data: new SlashCommandBuilder()
.setName('reload')
.setDescription('Reloads all RSS feeds'),
example: "reload",
isPrivileged: false,
requiresTokens: false,
async execute(interaction) {
try{
libCore.loadFeeds();
await interaction.reply("Reloaded all RSS feeds");
}catch(err){
log.ERROR(err)
//await interaction.reply(err.toString());
}
}
};

3
commands/remove.js
View File

@@ -20,14 +20,13 @@ module.exports = {
var title = interaction.options.getString("title");
libCore.deleteSource(title, (err, result) => {
console.log("Result from removing entry", result);
log.DEBUG("Result from removing entry", result);
if (result) {
interaction.reply(`Removing ${title} from the list of RSS sources`);
} else {
interaction.reply(`${title} does not exist in the list of RSS sources`);
}
libCore.loadFeeds();
});
}catch(err){
log.ERROR(err)

20
index.js
View File

@@ -91,12 +91,7 @@ function runHTTPServer() {
server.on('error', libUtils.onError);
server.on('listening', () => {
log.INFO("HTTP server started!");
try {
libCore.feedArray = libCore.getFeeds();
} catch (error) {
log.ERROR(error);
}
log.INFO("HTTP server started!");
})
}
@@ -123,10 +118,13 @@ client.on('ready', () => {
// Deploy slash commands
log.DEBUG("Deploying slash commands");
deployCommands.deploy(client.guilds.cache.map(guild => guild.id));
deployCommands.deploy(client.guilds.cache.map(guild => guild.id));
log.DEBUG(`Starting HTTP Server`);
runHTTPServer();
log.DEBUG("Loading new posts");
libCore.updateFeeds(client);
});
// Setup any additional event handlers
@@ -143,10 +141,4 @@ for (const file of eventFiles) {
}
}
client.login(discordToken); //Load Client Discord Token
try {
log.INFO("Loading initial startup feeds");
libCore.loadFeeds();
} catch (error) {
log.ERROR(error);
}
client.login(discordToken); //Load Client Discord Token

196
libCore.js
View File

@@ -3,8 +3,11 @@ let Parser = require('rss-parser');
const axios = require('axios');
let parser = new Parser();
const storageHandler = require("./libStorage");
const libUtils = require("./libUtils");
const { createHash } = require("crypto");
const { DebugBuilder } = require("./utilities/debugBuilder");
const { all } = require('axios');
const log = new DebugBuilder("server", "libCore");
/* OpenAI config
@@ -18,12 +21,13 @@ const openai = new OpenAIApi(configuration);
*/
// Data Structures
var feeds = [];
var feeds = {};
var rssFeedMap = [];
var rssFeedCategories = [];
// Setup Storage handlers
var feedStorage = new storageHandler.feedStorage();
var postStorage = new storageHandler.postStorage();
/**
* Adds or updates new source url to configured storage
@@ -51,18 +55,6 @@ exports.addSource = async (title, link, category, guildId, channelId, callback)
log.DEBUG("Record ID:", record.getId());
var linkData = {
title: `${title}`,
link: `${link}`,
category: `${category}`,
id: record.getId()
}
log.DEBUG("Link Data:", linkData);
feeds.push(linkData);
log.DEBUG("pushed item to feeds");
return callback(undefined, record);
});
@@ -74,123 +66,99 @@ exports.addSource = async (title, link, category, guildId, channelId, callback)
* @param {string} title - Title/Name of the RSS feed.
*/
exports.deleteSource = function (title, callback) {
var deleteRecord = "";
for (i = 0; i < feeds.length; i++) {
if (feeds[i].title == title) {
deleteRecord = feeds[i].id;
feedStorage.getRecordBy('title', title, (err, results) => {
if (err) return callback(err, undefined);
if (!results?.id) {
log.DEBUG("No record found for title: ", title)
return callback(undefined, undefined);
}
}
feedStorage.destroy(deleteRecord, function (err, deletedRecord) {
if (err) {
log.ERROR(err);
return callback(err, undefined);
}
log.DEBUG(deletedRecord.id);
return callback(undefined, deletedRecord);
feedStorage.destroy(results.id, function (err, deletedRecord) {
if (err) {
log.ERROR(err);
return callback(err, undefined);
}
log.DEBUG("Deleted Record: ", deletedRecord);
return callback(undefined, deletedRecord ?? true);
});
});
}
}
/**
* Adds a new source url to configured storage
* @constructor
* @param {string} feedType - Category to select Feed by.
* @param {string} category - Category to select Feed by, defaults to all.
*/
exports.getFeeds = function (feedType) {
var rssFeedFilteredMap = [];
if (feedType == null || feedType == undefined || feedType == "" || feedType.toLowerCase() == "all") {
return rssFeedMap;
} else {
rssFeedMap.forEach(rssFeed => {
if (rssFeed.category.toLowerCase().indexOf(feedType.toLowerCase()) > -1) {
rssFeedFilteredMap.push(rssFeed);
exports.getPosts = async (category) => {
postStorage.getAllRecords((err, results) => {
if (category == null || category == undefined || category == "" || category.toLowerCase() == "all") {
if (err) throw err;
if (results.length > 1)
return results;
}
var rssFilteredFeed;
for (const rssFeed of results){
if (rssFeed.category.toLowerCase() == category.toLowerCase()) {
rssFilteredFeed.push(rssFeed);
}
});
return rssFeedFilteredMap;
}
}
return rssFilteredFeed;
});
}
/**
* Load the RSS feeds
* Update channels with new posts from sources
*/
exports.loadFeeds = function () {
feeds = [];
rssFeedMap = [];
rssFeedCategories = [];
exports.updateFeeds = async (client) => {
if (!client) throw new Error("Client object not passed");
feedStorage.getAllRecords((err, records) => {
for (const record of records){
try {
log.DEBUG('Retrieved title: ', record.title);
log.DEBUG('Retrieved link: ', record.link);
log.DEBUG('Retrieved category: ', record.category);
log.DEBUG('Retrieved guild ID: ', record.guild_id);
log.DEBUG('Retrieved channel ID: ', record.channel_id);
// Load the posts from each RSS source
for (const source of records) {
log.DEBUG('Record title: ', source.title);
log.DEBUG('Record link: ', source.link);
log.DEBUG('Record category: ', source.category);
log.DEBUG('Record guild ID: ', source.guild_id);
log.DEBUG('Record channel ID: ', source.channel_id);
parser.parseURL(source.link, (err, parsedFeed) => {
if (err) {
log.ERROR("Parser Error: ", source.link, err);
//return;
}
// Check to see if this source has been loaded already
feeds.forEach(feedSource => {
if (feedSource.link != record.link) {
log.DEBUG("Loading new source: ", feedSource)
feeds.push(feedSource);
if (parsedFeed?.items){
for (const post of parsedFeed.items){
if (post.title && post.link && post.content && post.guid && post.pubDate){
postStorage.getRecordBy('post_guid', post.guid, (err, results) => {
if (err) throw err;
log.DEBUG("Results from asdasdasd: ", results);
if (!results){
const channel = client.channels.cache.get(source.channel_id);
libUtils.sendPost(post, channel, (err, results) =>{
if (err) throw err;
if (results){
log.DEBUG("Saving post to database: ", results, post.title, source.channel_id);
postStorage.savePost(post, (err, results) => {
if(err) throw err;
if (results) {
log.DEBUG("Saved results: ", results);
}
});
}
})
}
})
}
}
});
// Update the known categories if new category is not present
if (!rssFeedCategories.includes(record.category)) {
log.DEBUG("Added new category to running known categories: ", record.category)
rssFeedCategories.push(record.category);
}
} catch (error) {
log.DEBUG(error);
}
}
});
}
});
}
/*
feeds.forEach(feedBlock => {
(async () => {
try {
const feed = parser.parseURL(feedBlock.link, function (err, feed) {
if (err) {
log.DEBUG(err + " " + feedBlock.link);
//return;
}
if (feed != undefined && feed.items != undefined) {
//log.DEBUG("Feed: ", feed);
feed.items.forEach(item => {
var foundFeed = false;
rssFeedMap.forEach(rssFeed => {
if (rssFeed.link == item.link) {
log.DEBUG("Found matching post?: ", rssFeed)
foundFeed = true;
}
});
if (!foundFeed) {
var linkData = {
title: `${unescape(item.title)}`,
link: `${unescape(item.link)}`,
category: `${unescape(feedBlock.category)}`
}
rssFeedMap.push(linkData);
}
});
} else {
log.DEBUG('error parsing :' + feedBlock.link);
}
})
} catch (error) {
log.DEBUG(error);
}
//log.DEBUG("RSS Feed Map: ", rssFeedMap);
});
}
/**

137
libStorage.js
View File

@@ -25,7 +25,7 @@ function runSQL(sqlQuery, connection, callback = (err, rows) => {
log.ERROR("SQL Error:", err)
return callback(err, undefined);
}
log.VERBOSE("RunSQL Returned Rows:", rows);
log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows);
return callback(undefined, rows);
})
}
@@ -39,7 +39,37 @@ class Storage {
database: process.env.DB_NAME
});
this.dbTable = _dbTable
this.dbTable = _dbTable;
this.validKeys = [];
var sqlQuery = `SHOW COLUMNS FROM ${this.dbTable};`;
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return log.ERROR("Error getting column names: ", err);
if (rows){
for (const validKey of rows){
this.validKeys.push(validKey.Field);
}
log.VERBOSE(`Database rows for '${this.dbTable}': `, rows);
log.DEBUG(`Keys for '${this.dbTable}': `, this.validKeys);
}
})
}
/**
* Wrapper to delete an entry using the storage method configured
* @param {} entryID The ID of the entry to be deleted
* @param {function} callback The callback function to be called with the record when deleted
*/
destroy(entryID, callback) {
if (!entryID) return callback(Error("No entry ID given"), undefined);
this.getRecordBy('id', entryID, (err, entryRecord) => {
this.removeEntry(entryRecord.id, (err, results) => {
if (err) return callback(err, undefined);
return callback(undefined, results);
});
});
}
/**
@@ -48,15 +78,14 @@ class Storage {
* @param {*} keyValue The value of the key to search for
* @param {*} callback The callback function
*/
getRecordBy(key, keyValue, callback) {
const validKeys = ["link", "title", "category", "id"];
if (!validKeys.includes(key)) return callback(new Error("Given key not valid"), undefined);
getRecordBy(key, keyValue, callback) {
if (!this.validKeys.includes(key)) return callback(new Error("Given key not valid", key), undefined);
const sqlQuery = `SELECT * FROM ${this.dbTable} WHERE ${key} = '${keyValue}'`;
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined);
if (rows[0]?.title) return callback(undefined, rows[0]);
if (rows[0]?.[key]) return callback(undefined, rows[0]);
else return callback(undefined, false);
})
}
@@ -66,19 +95,49 @@ class Storage {
* @param {function} callback
*/
getAllRecords(callback) {
log.INFO("Getting all records");
log.INFO("Getting all records from: ", this.dbTable);
const sqlQuery = `SELECT * FROM ${this.dbTable}`
let rssRecords = [];
let records = [];
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined);
for (const row of rows) {
log.VERBOSE("Row from SQL query:", row);
rssRecords.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id));
for (const row of rows) {
if (this.dbTable == rssFeedsTable){
records.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id));
}
if (this.dbTable == rssPostsTable){
records.push(rows);
}
}
log.VERBOSE("All records:", rssRecords);
return callback(undefined, rssRecords);
log.VERBOSE("All records:", records);
return callback(undefined, records);
});
}
/**
* Gets all unique rows in the given key
* @param {*} key
* @param {*} callback
*/
getUniqueByKey(key, callback){
log.INFO("Getting all unique values in column: ", key);
const sqlQuery = `SELECT DISTINCT ${key} FROM ${this.dbTable}`
let records = [];
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined);
for (const row of rows) {
if (this.dbTable == rssFeedsTable){
records.push(new RSSSourceRecord(row.id, row.title, row.link, row.category, row.guild_id, row.channel_id));
}
if (this.dbTable == rssPostsTable){
records.push(rows);
}
}
log.VERBOSE("All records:", records);
return callback(undefined, records);
});
}
}
@@ -109,28 +168,12 @@ exports.feedStorage = class feedStorage extends Storage {
log.DEBUG("One record to callback with:", record);
return callback(undefined, record);
}
}, false)
}, false) // Do not update the if it exists
}
if (!toBeSaved.length === 1) {
return callback(undefined, newRecords);
}
}
/**
* Wrapper to delete an entry using the storage method configured
* @param {} entryID The ID of the entry to be deleted
* @param {function} callback The callback function to be called with the record when deleted
*/
destroy(entryID, callback) {
if (!entryID) return callback(Error("No entry ID given"), undefined);
this.getRecordBy('id', entryID, (err, entryRecord) => {
this.removeEntry(entryRecord.id, (err, results) => {
if (err) return callback(err, undefined);
return callback(undefined, results);
});
});
}
}
/**
* Check to see if an entry exists in the storage method configured
@@ -205,15 +248,15 @@ exports.feedStorage = class feedStorage extends Storage {
/**
* Delete the given entry from the storage medium
* @param {string} title The title of the entry to be deleted
* @param {*} id The title of the entry to be deleted
* @param {function} callback The callback to be called with either an error or undefined if successful
*/
removeEntry(title, callback) {
if (!title) {
return callback(new Error("No entry title given before deleting"), undefined)
removeEntry(id, callback) {
if (!id) {
return callback(new Error("No entry id given before deleting"), undefined)
}
const sqlQuery = `DELETE FROM ${this.dbTable} WHERE title = '${title}';`;
const sqlQuery = `DELETE FROM ${this.dbTable} WHERE id = '${id}';`;
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined);
@@ -280,3 +323,25 @@ exports.feedStorage = class feedStorage extends Storage {
}
}
exports.postStorage = class postStorage extends Storage {
constructor() {
super(rssPostsTable);
}
savePost(_postObject, callback){
const tempCreationDate = new Date().toISOString().slice(0, 19).replace('T', ' ');
log.DEBUG("Saving Post Object:", _postObject);
if (!_postObject?.guid || !_postObject?.link) {
return callback(new Error("Post object malformed, check the object before saving it"), undefined)
}
const sqlQuery = `INSERT INTO ${this.dbTable} (post_guid, post_link, post_sent_date) VALUES ('${_postObject.guid}','${_postObject.link}','${tempCreationDate}');`;
log.DEBUG(`Adding new post with SQL query: '${sqlQuery}'`)
runSQL(sqlQuery, this.connection, (err, rows) => {
if (err) return callback(err, undefined);
return callback(undefined, rows);
})
}
}

28
libUtils.js
View File

@@ -1,5 +1,7 @@
const { EmbedBuilder } = require('discord.js');
const { DebugBuilder } = require("./utilities/debugBuilder");
const log = new DebugBuilder("server", "libUtils");
const { NodeHtmlMarkdown } = require('node-html-markdown');
/**
* sleep - sleep/wait
@@ -57,4 +59,30 @@ exports.onError = (error) => {
default:
throw error;
}
}
exports.sendPost = (post, channel, callback) => {
const title = post.title;
const link = post.link;
const content = NodeHtmlMarkdown.translate(post.content);
const guid = post.guid;
const pubDate = post.pubDate ?? new Date();
log.DEBUG("Sending an RSS post to discord", title, guid)
const rssMessage = new EmbedBuilder()
.setColor(0x0099FF)
.setTitle(title)
.setURL(link)
.setDescription(content)
.setTimestamp(pubDate)
.setFooter({ text: 'Brought to you by Emmelia.' });
try{
channel.send({ embeds: [rssMessage] });
return callback(undefined, true);
}
catch (err){
log.ERROR("Error sending message: ", err);
return callback(err, undefined);
}
}