diff --git a/Server/libCore.js b/Server/libCore.js index 8d95b50..4f89d06 100644 --- a/Server/libCore.js +++ b/Server/libCore.js @@ -32,17 +32,38 @@ var runningPostsToRemove = [{ }] */ var runningPostsToRemove = {}; -const sourceFailureLimit = process.env.SOURCE_FAILURE_LIMIT ?? 3; +const sourceFailureLimit = process.env.SOURCE_FAILURE_LIMIT ?? 15; /** + * Wrapper for feeds that cause errors. By default it will wait over a day for the source to come back online before deleting it. * - * @param {*} sourceURL + * @param {string} sourceURL The URL of the feed source causing issues */ exports.removeSource = function removeSource(sourceURL) { log.INFO("Removing source URL: ", sourceURL); - if (!sourceURL in runningPostsToRemove) {runningPostsToRemove[sourceURL] = 1; return;} + // Check to see if this is the first time this source has been attempted + if (!Object.keys(runningPostsToRemove).includes(sourceURL)) { + runningPostsToRemove[sourceURL] = { count: 1, timestamp: Date.now(), ignoredAttempts: 0 }; + return; + } + + const backoffDateTimeDifference = (Date.now() - new Date(runningPostsToRemove[sourceURL].timestamp)); + const backoffWaitTime = (runningPostsToRemove[sourceURL].count * 30000); + + log.DEBUG("Datetime", runningPostsToRemove[sourceURL], backoffDateTimeDifference, backoffWaitTime); + + // Check to see if the last error occurred within the backoff period or if we should try again + if (backoffDateTimeDifference <= backoffWaitTime) { + runningPostsToRemove[sourceURL].ignoredAttempts +=1; + return; + } - if (runningPostsToRemove[sourceURL] < sourceFailureLimit) {runningPostsToRemove[sourceURL] += 1; return;} + // Increase the retry counter + if (runningPostsToRemove[sourceURL].count < sourceFailureLimit) { + runningPostsToRemove[sourceURL].count += 1; + runningPostsToRemove[sourceURL].timestamp = Date.now(); + return; + } feedStorage.getRecordBy('link', sourceURL, (err, record) => { if (err) log.ERROR("Error getting record from feedStorage", err); @@ -62,13 +83,14 @@ exports.removeSource = function removeSource(sourceURL) { /** * Unset a source URL from deletion if the source has not already been deleted * @param {*} sourceURL The source URL to be unset from deletion - * @returns {*} */ exports.unsetRemoveSource = function unsetRemoveSource(sourceURL) { log.INFO("Unsetting source URL from deletion (if not already deleted): ", sourceURL); - if (!sourceURL in runningPostsToRemove) return; + if (!Object.keys(runningPostsToRemove).includes(sourceURL)) return; - if (runningPostsToRemove[sourceURL] > sourceFailureLimit) return delete runningPostsToRemove[sourceURL]; + delete runningPostsToRemove[sourceURL]; + + return } /**