383 lines
12 KiB
JavaScript
383 lines
12 KiB
JavaScript
const { all } = require('axios');
|
|
const axios = require('axios');
|
|
|
|
const { FeedStorage, PostStorage } = require("./libStorage");
|
|
const libUtils = require("./libUtils");
|
|
const { DebugBuilder } = require("./utilities/debugBuilder");
|
|
const log = new DebugBuilder("server", "libCore");
|
|
const mysql = require("mysql");
|
|
|
|
const UserAgent = require("user-agents");
|
|
process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString();
|
|
|
|
log.DEBUG("Generated User Agent string:", process.env.USER_AGENT_STRING);
|
|
|
|
// Initiate the parser
|
|
let Parser = require('rss-parser');
|
|
let parser = new Parser({
|
|
headers: {
|
|
'User-Agent': process.env.USER_AGENT_STRING,
|
|
"Accept": "application/rss+xml,application/xhtml+xml,application/xml"
|
|
}
|
|
});
|
|
|
|
// Setup Storage handlers
|
|
var feedStorage = new FeedStorage();
|
|
var postStorage = new PostStorage();
|
|
|
|
// Initiate a running array of objects to keep track of sources that have no feeds/posts
|
|
/*
|
|
var runningPostsToRemove = [{
|
|
"{SOURCE URL}": {NUMBER OF TIMES IT'S BEEN REMOVED}
|
|
}]
|
|
*/
|
|
var runningPostsToRemove = {};
|
|
const sourceFailureLimit = process.env.SOURCE_FAILURE_LIMIT ?? 3;
|
|
|
|
/**
|
|
*
|
|
* @param {*} sourceURL
|
|
*/
|
|
exports.removeSource = function removeSource(sourceURL) {
|
|
log.INFO("Removing source URL: ", sourceURL);
|
|
if (!sourceURL in runningPostsToRemove) {runningPostsToRemove[sourceURL] = 1; return;}
|
|
|
|
if (runningPostsToRemove[sourceURL] < sourceFailureLimit) {runningPostsToRemove[sourceURL] += 1; return;}
|
|
|
|
feedStorage.getRecordBy('link', sourceURL, (err, record) => {
|
|
if (err) log.ERROR("Error getting record from feedStorage", err);
|
|
|
|
if (!record) log.ERROR("No source returned from feedStorage");
|
|
feedStorage.destroy(record.id, (err, results) => {
|
|
if (err) log.ERROR("Error removing ID from results", err);
|
|
|
|
if (!results) log.WARN("No results from remove entry");
|
|
|
|
log.DEBUG("Source exceeded the limit of retries and has been removed", sourceURL);
|
|
return;
|
|
})
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Unset a source URL from deletion if the source has not already been deleted
|
|
* @param {*} sourceURL The source URL to be unset from deletion
|
|
* @returns {*}
|
|
*/
|
|
exports.unsetRemoveSource = function unsetRemoveSource(sourceURL) {
|
|
log.INFO("Unsetting source URL from deletion (if not already deleted): ", sourceURL);
|
|
if (!sourceURL in runningPostsToRemove) return;
|
|
|
|
if (runningPostsToRemove[sourceURL] > sourceFailureLimit) return delete runningPostsToRemove[sourceURL];
|
|
}
|
|
|
|
/**
|
|
* Adds or updates new source url to configured storage
|
|
* @constructor
|
|
* @param {string} title - Title/Name of the RSS feed.
|
|
* @param {string} link - URL of RSS feed.
|
|
* @param {string} category - Category of RSS feed.
|
|
*/
|
|
exports.addSource = async (title, link, category, guildId, channelId, callback) => {
|
|
feedStorage.create([{
|
|
"fields": {
|
|
"title": title,
|
|
"link": link,
|
|
"category": category,
|
|
'guild_id': guildId,
|
|
"channel_id": channelId
|
|
}
|
|
}], function (err, record) {
|
|
if (err) {
|
|
log.ERROR("Error in create:", err);
|
|
return callback(err, undefined);
|
|
}
|
|
|
|
if (!record) return callback(undefined, false);
|
|
|
|
log.DEBUG("Record ID:", record.getId());
|
|
|
|
return callback(undefined, record);
|
|
});
|
|
|
|
}
|
|
|
|
/**
|
|
* Deletes a new source url by title
|
|
* @constructor
|
|
* @param {string} title - Title/Name of the RSS feed.
|
|
*/
|
|
exports.deleteSource = function (title, callback) {
|
|
feedStorage.getRecordBy('title', title, (err, results) => {
|
|
if (err) return callback(err, undefined);
|
|
|
|
if (!results?.id) {
|
|
log.DEBUG("No record found for title: ", title)
|
|
return callback(undefined, undefined);
|
|
}
|
|
|
|
feedStorage.destroy(results.id, function (err, deletedRecord) {
|
|
if (err) {
|
|
log.ERROR(err);
|
|
return callback(err, undefined);
|
|
}
|
|
log.DEBUG("Deleted Record: ", deletedRecord);
|
|
return callback(undefined, deletedRecord ?? true);
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Update channels with new posts from sources
|
|
*/
|
|
exports.updateFeeds = (client) => {
|
|
if (!client) throw new Error("Client object not passed");
|
|
// Create a temp pool to use for all connections while updating the feed
|
|
var tempConnection = mysql.createPool({
|
|
host: process.env.DB_HOST,
|
|
user: process.env.DB_USER,
|
|
password: process.env.DB_PASS,
|
|
database: process.env.DB_NAME,
|
|
connectionLimit: 10
|
|
});
|
|
|
|
const tempFeedStorage = new FeedStorage(tempConnection);
|
|
const tempPostStorage = new PostStorage(tempConnection);
|
|
|
|
// Array of promises to wait on before closing the connection
|
|
var recordPromiseArray = [];
|
|
var sourcePromiseArray = [];
|
|
|
|
tempFeedStorage.getAllRecords(async (err, records) => {
|
|
// Load the posts from each RSS source
|
|
for (const source of records) {
|
|
sourcePromiseArray.push(new Promise((resolve, reject) => {
|
|
log.DEBUG('Record title: ', source.title);
|
|
log.DEBUG('Record link: ', source.link);
|
|
log.DEBUG('Record category: ', source.category);
|
|
log.DEBUG('Record guild ID: ', source.guild_id);
|
|
log.DEBUG('Record channel ID: ', source.channel_id);
|
|
// Parse the RSS feed
|
|
parser.parseURL(source.link, async (err, parsedFeed) => {
|
|
if (err) {
|
|
log.ERROR("Parser Error: ", runningPostsToRemove, source, err);
|
|
// Call the wrapper to make sure the site isn't just down at the time it checks and is back up the next time
|
|
this.removeSource(source.link);
|
|
reject;
|
|
}
|
|
try {
|
|
if (parsedFeed?.items){
|
|
this.unsetRemoveSource(source.link);
|
|
for (const post of parsedFeed.items.reverse()){
|
|
recordPromiseArray.push(new Promise((recordResolve, recordReject) => {
|
|
log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title);
|
|
log.VERBOSE("Post from feed: ", post);
|
|
if (!post.title || !post.link) return recordReject("Missing information from the post");
|
|
if (!post.content || !post['content:encoded']) log.WARN("There is no content for post: ", post.title);
|
|
|
|
post.postId = post.postId ?? post.guid ?? post.id ?? libUtils.returnHash(post.title, post.link, post.pubDate ?? Date.now());
|
|
tempPostStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => {
|
|
if (err) throw err;
|
|
|
|
log.DEBUG("Existing post record: ", existingRecord);
|
|
if (existingRecord) return recordResolve("Existing record found for this post");
|
|
|
|
const channel = client.channels.cache.get(source.channel_id);
|
|
libUtils.sendPost(post, source, channel, (err, sendResults) =>{
|
|
if (err) throw err;
|
|
|
|
if (!sendResults) {
|
|
log.ERROR("No sending results from sending a post: ", sendResults, existingRecord, post);
|
|
return recordReject("No sending results from sending a post");
|
|
}
|
|
|
|
log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id);
|
|
|
|
tempPostStorage.savePost(post, (err, saveResults) => {
|
|
if(err) throw err;
|
|
|
|
if (saveResults) {
|
|
log.DEBUG("Saved results: ", saveResults);
|
|
return recordResolve("Saved results", saveResults);
|
|
}
|
|
});
|
|
})
|
|
})
|
|
}))
|
|
}
|
|
}
|
|
else {
|
|
this.removeSource(source.link);
|
|
}
|
|
}
|
|
catch (err) {
|
|
log.ERROR("Error Parsing Feed: ", source.link, err);
|
|
this.removeSource(source.link);
|
|
throw err;
|
|
}
|
|
Promise.all(recordPromiseArray).then((values) => {
|
|
log.DEBUG("All posts finished for: ", source.title, values);
|
|
return resolve(source.title);
|
|
});
|
|
});
|
|
}))
|
|
}
|
|
|
|
// Wait for all connections to finish then close the temp connections
|
|
|
|
Promise.all(sourcePromiseArray).then((values) => {
|
|
log.DEBUG("All sources finished, closing temp connections: ", values);
|
|
tempConnection.end();
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Search a state for any weather alerts
|
|
*
|
|
* @param {*} state The state to search for any weather alerts in
|
|
* @returns
|
|
*/
|
|
exports.weatherAlert = async function (state) {
|
|
|
|
var answerURL = `https://api.weather.gov/alerts/active?area=${state}`;
|
|
log.DEBUG(answerURL);
|
|
answerData = [];
|
|
|
|
await axios.get(answerURL)
|
|
.then(response => {
|
|
response.data.features.forEach(feature => {
|
|
answerData.push(feature);
|
|
})
|
|
|
|
return answerData;
|
|
})
|
|
.catch(error => {
|
|
log.DEBUG(error);
|
|
});
|
|
return answerData;
|
|
}
|
|
|
|
/**
|
|
* Gets a random food recipe
|
|
*
|
|
* @returns
|
|
*/
|
|
exports.getFood = async function () {
|
|
|
|
var answerURL = `https://www.themealdb.com/api/json/v1/1/random.php`;
|
|
log.DEBUG(answerURL);
|
|
answerData = {
|
|
text: `No answer found try using a simpler search term`,
|
|
source: ``
|
|
}
|
|
await axios.get(answerURL)
|
|
.then(response => {
|
|
//log.DEBUG(response.data.RelatedTopics[0].Text);
|
|
//log.DEBUG(response.data.RelatedTopics[0].FirstURL);
|
|
|
|
// if (response.data.meals.length != 0) {
|
|
|
|
answerData = {
|
|
strMeal: `No Data`,
|
|
strSource: `-`,
|
|
strInstructions: `-`,
|
|
strMealThumb: `-`,
|
|
strCategory: `-`
|
|
}
|
|
|
|
answerData = {
|
|
strMeal: `${unescape(response.data.meals[0].strMeal)}`,
|
|
strSource: `${unescape(response.data.meals[0].strSource)}`,
|
|
strInstructions: `${unescape(response.data.meals[0].strInstructions)}`,
|
|
strMealThumb: `${unescape(response.data.meals[0].strMealThumb)}`,
|
|
strCategory: `${unescape(response.data.meals[0].strCategory)}`
|
|
}
|
|
// } else {
|
|
|
|
//}
|
|
return answerData;
|
|
})
|
|
.catch(error => {
|
|
log.DEBUG(error);
|
|
});
|
|
return answerData;
|
|
}
|
|
|
|
/**
|
|
* Search Urban dictionary for a phrase
|
|
*
|
|
* @param {*} question The phrase to search urban dictionary for
|
|
* @returns
|
|
*/
|
|
exports.getSlang = async function (question) {
|
|
|
|
var answerURL = `https://api.urbandictionary.com/v0/define?term=${question}`;
|
|
log.DEBUG(answerURL);
|
|
slangData = {
|
|
definition: `No answer found try using a simpler search term`,
|
|
example: ``
|
|
}
|
|
await axios.get(answerURL)
|
|
.then(response => {
|
|
log.DEBUG(response.data.list[0]);
|
|
|
|
slangData = {
|
|
definition: `${unescape(response.data.list[0].definition) ? unescape(response.data.list[0].definition) : ''}`,
|
|
example: `${unescape(response.data.list[0].example) ? unescape(response.data.list[0].example) : ''}`,
|
|
thumbs_down: `${unescape(response.data.list[0].thumbs_down)}`,
|
|
thumbs_up: `${unescape(response.data.list[0].thumbs_up)}`
|
|
}
|
|
|
|
return slangData;
|
|
})
|
|
.catch(error => {
|
|
log.DEBUG(error);
|
|
});
|
|
return slangData;
|
|
}
|
|
|
|
/**
|
|
* getSources - Get the RSS sources currently in use
|
|
* @constructor
|
|
*/
|
|
exports.getSources = function () {
|
|
feedStorage.getAllRecords((err, records) => {
|
|
if (err) throw err;
|
|
return records;
|
|
})
|
|
}
|
|
|
|
/**
|
|
* getQuotes - Get a random quote from the local list
|
|
* @constructor
|
|
*/
|
|
exports.getQuotes = async function (quote_url) {
|
|
|
|
var data = [];
|
|
await axios.get(quote_url)
|
|
.then(response => {
|
|
log.DEBUG(response.data[0].q);
|
|
log.DEBUG(response.data[0].a);
|
|
data = response.data;
|
|
|
|
return data;
|
|
})
|
|
.catch(error => {
|
|
log.DEBUG(error);
|
|
});
|
|
return data;
|
|
}
|
|
|
|
/**
|
|
* getCategories - Returns feed categories
|
|
* @constructor
|
|
*/
|
|
exports.getCategories = async (callback) => {
|
|
feedStorage.getUniqueByKey("category", (err, results) => {
|
|
if (err) return callback(err, undefined);
|
|
|
|
return callback(undefined, results);
|
|
});
|
|
}
|