const { all } = require('axios'); const axios = require('axios'); const { FeedStorage, PostStorage } = require("./libStorage"); const libUtils = require("./libUtils"); const { DebugBuilder } = require("./utilities/debugBuilder"); const log = new DebugBuilder("server", "libCore"); const mysql = require("mysql"); const UserAgent = require("user-agents"); process.env.USER_AGENT_STRING = new UserAgent({ platform: 'Win32' }).toString(); log.DEBUG("Generated User Agent string:", process.env.USER_AGENT_STRING); // Initiate the parser let Parser = require('rss-parser'); let parser = new Parser({ headers: { 'User-Agent': process.env.USER_AGENT_STRING, "Accept": "application/rss+xml,application/xhtml+xml,application/xml" } }); // Setup Storage handlers var feedStorage = new FeedStorage(); var postStorage = new PostStorage(); // Initiate a running array of objects to keep track of sources that have no feeds/posts /* var runningPostsToRemove = [{ "{SOURCE URL}": {NUMBER OF TIMES IT'S BEEN REMOVED} }] */ var runningPostsToRemove = {}; const sourceFailureLimit = process.env.SOURCE_FAILURE_LIMIT ?? 3; /** * * @param {*} sourceURL */ exports.removeSource = function removeSource(sourceURL) { log.INFO("Removing source URL: ", sourceURL); if (!sourceURL in runningPostsToRemove) {runningPostsToRemove[sourceURL] = 1; return;} if (runningPostsToRemove[sourceURL] < sourceFailureLimit) {runningPostsToRemove[sourceURL] += 1; return;} feedStorage.getRecordBy('link', sourceURL, (err, record) => { if (err) log.ERROR("Error getting record from feedStorage", err); if (!record) log.ERROR("No source returned from feedStorage"); feedStorage.destroy(record.id, (err, results) => { if (err) log.ERROR("Error removing ID from results", err); if (!results) log.WARN("No results from remove entry"); log.DEBUG("Source exceeded the limit of retries and has been removed", sourceURL); return; }) }) } /** * Unset a source URL from deletion if the source has not already been deleted * @param {*} sourceURL The source URL to be unset from deletion * @returns {*} */ exports.unsetRemoveSource = function unsetRemoveSource(sourceURL) { log.INFO("Unsetting source URL from deletion (if not already deleted): ", sourceURL); if (!sourceURL in runningPostsToRemove) return; if (runningPostsToRemove[sourceURL] > sourceFailureLimit) return delete runningPostsToRemove[sourceURL]; } /** * Adds or updates new source url to configured storage * @constructor * @param {string} title - Title/Name of the RSS feed. * @param {string} link - URL of RSS feed. * @param {string} category - Category of RSS feed. */ exports.addSource = async (title, link, category, guildId, channelId, callback) => { feedStorage.create([{ "fields": { "title": title, "link": link, "category": category, 'guild_id': guildId, "channel_id": channelId } }], function (err, record) { if (err) { log.ERROR("Error in create:", err); return callback(err, undefined); } if (!record) return callback(undefined, false); log.DEBUG("Record ID:", record.getId()); return callback(undefined, record); }); } /** * Deletes a new source url by title * @constructor * @param {string} title - Title/Name of the RSS feed. */ exports.deleteSource = function (title, callback) { feedStorage.getRecordBy('title', title, (err, results) => { if (err) return callback(err, undefined); if (!results?.id) { log.DEBUG("No record found for title: ", title) return callback(undefined, undefined); } feedStorage.destroy(results.id, function (err, deletedRecord) { if (err) { log.ERROR(err); return callback(err, undefined); } log.DEBUG("Deleted Record: ", deletedRecord); return callback(undefined, deletedRecord ?? true); }); }); } /** * Update channels with new posts from sources */ exports.updateFeeds = (client) => { if (!client) throw new Error("Client object not passed"); // Create a temp pool to use for all connections while updating the feed var tempConnection = mysql.createPool({ host: process.env.DB_HOST, user: process.env.DB_USER, password: process.env.DB_PASS, database: process.env.DB_NAME, connectionLimit: 10 }); const tempFeedStorage = new FeedStorage(tempConnection); const tempPostStorage = new PostStorage(tempConnection); // Array of promises to wait on before closing the connection var recordPromiseArray = []; var sourcePromiseArray = []; tempFeedStorage.getAllRecords(async (err, records) => { // Load the posts from each RSS source for (const source of records) { sourcePromiseArray.push(new Promise((resolve, reject) => { log.DEBUG('Record title: ', source.title); log.DEBUG('Record link: ', source.link); log.DEBUG('Record category: ', source.category); log.DEBUG('Record guild ID: ', source.guild_id); log.DEBUG('Record channel ID: ', source.channel_id); // Parse the RSS feed parser.parseURL(source.link, async (err, parsedFeed) => { if (err) { log.ERROR("Parser Error: ", runningPostsToRemove, source, err); // Call the wrapper to make sure the site isn't just down at the time it checks and is back up the next time this.removeSource(source.link); reject; } try { if (parsedFeed?.items){ this.unsetRemoveSource(source.link); for (const post of parsedFeed.items.reverse()){ recordPromiseArray.push(new Promise((recordResolve, recordReject) => { log.DEBUG("Parsed Source Keys", Object.keys(post), post?.title); log.VERBOSE("Post from feed: ", post); if (!post.title || !post.link) return recordReject("Missing information from the post"); if (!post.content || !post['content:encoded']) log.WARN("There is no content for post: ", post.title); post.postId = post.postId ?? post.guid ?? post.id ?? libUtils.returnHash(post.title, post.link, post.pubDate ?? Date.now()); tempPostStorage.getRecordBy('post_guid', post.postId, (err, existingRecord) => { if (err) throw err; log.DEBUG("Existing post record: ", existingRecord); if (existingRecord) return recordResolve("Existing record found for this post"); const channel = client.channels.cache.get(source.channel_id); libUtils.sendPost(post, source, channel, (err, sendResults) =>{ if (err) throw err; if (!sendResults) { log.ERROR("No sending results from sending a post: ", sendResults, existingRecord, post); return recordReject("No sending results from sending a post"); } log.DEBUG("Saving post to database: ", sendResults, post.title, source.channel_id); tempPostStorage.savePost(post, (err, saveResults) => { if(err) throw err; if (saveResults) { log.DEBUG("Saved results: ", saveResults); return recordResolve("Saved results", saveResults); } }); }) }) })) } } else { this.removeSource(source.link); } } catch (err) { log.ERROR("Error Parsing Feed: ", source.link, err); this.removeSource(source.link); throw err; } Promise.all(recordPromiseArray).then((values) => { log.DEBUG("All posts finished for: ", source.title, values); return resolve(source.title); }); }); })) } // Wait for all connections to finish then close the temp connections Promise.all(sourcePromiseArray).then((values) => { log.DEBUG("All sources finished, closing temp connections: ", values); tempConnection.end(); }); }); } /** * Search a state for any weather alerts * * @param {*} state The state to search for any weather alerts in * @returns */ exports.weatherAlert = async function (state) { var answerURL = `https://api.weather.gov/alerts/active?area=${state}`; log.DEBUG(answerURL); answerData = []; await axios.get(answerURL) .then(response => { response.data.features.forEach(feature => { answerData.push(feature); }) return answerData; }) .catch(error => { log.DEBUG(error); }); return answerData; } /** * Gets a random food recipe * * @returns */ exports.getFood = async function () { var answerURL = `https://www.themealdb.com/api/json/v1/1/random.php`; log.DEBUG(answerURL); answerData = { text: `No answer found try using a simpler search term`, source: `` } await axios.get(answerURL) .then(response => { //log.DEBUG(response.data.RelatedTopics[0].Text); //log.DEBUG(response.data.RelatedTopics[0].FirstURL); // if (response.data.meals.length != 0) { answerData = { strMeal: `No Data`, strSource: `-`, strInstructions: `-`, strMealThumb: `-`, strCategory: `-` } answerData = { strMeal: `${unescape(response.data.meals[0].strMeal)}`, strSource: `${unescape(response.data.meals[0].strSource)}`, strInstructions: `${unescape(response.data.meals[0].strInstructions)}`, strMealThumb: `${unescape(response.data.meals[0].strMealThumb)}`, strCategory: `${unescape(response.data.meals[0].strCategory)}` } // } else { //} return answerData; }) .catch(error => { log.DEBUG(error); }); return answerData; } /** * Search Urban dictionary for a phrase * * @param {*} question The phrase to search urban dictionary for * @returns */ exports.getSlang = async function (question) { var answerURL = `https://api.urbandictionary.com/v0/define?term=${question}`; log.DEBUG(answerURL); slangData = { definition: `No answer found try using a simpler search term`, example: `` } await axios.get(answerURL) .then(response => { log.DEBUG(response.data.list[0]); slangData = { definition: `${unescape(response.data.list[0].definition) ? unescape(response.data.list[0].definition) : ''}`, example: `${unescape(response.data.list[0].example) ? unescape(response.data.list[0].example) : ''}`, thumbs_down: `${unescape(response.data.list[0].thumbs_down)}`, thumbs_up: `${unescape(response.data.list[0].thumbs_up)}` } return slangData; }) .catch(error => { log.DEBUG(error); }); return slangData; } /** * getSources - Get the RSS sources currently in use * @constructor */ exports.getSources = function () { feedStorage.getAllRecords((err, records) => { if (err) throw err; return records; }) } /** * getQuotes - Get a random quote from the local list * @constructor */ exports.getQuotes = async function (quote_url) { var data = []; await axios.get(quote_url) .then(response => { log.DEBUG(response.data[0].q); log.DEBUG(response.data[0].a); data = response.data; return data; }) .catch(error => { log.DEBUG(error); }); return data; } /** * getCategories - Returns feed categories * @constructor */ exports.getCategories = async (callback) => { feedStorage.getUniqueByKey("category", (err, results) => { if (err) return callback(err, undefined); return callback(undefined, results); }); }