Update SQL helper function to retry the database if it's unreachable

libStorage.js

@@ -25,17 +25,42 @@ var Connection = mysql.createPool({
 });
 
 // Helper Functions
-// Function to run and handle SQL errors
+/**
+ * Function to run and handle SQL errors
+ * @param {string} sqlQuery The SQL query string
+ * @param {*} connection The SQL connection to be used to query
+ * @param {function} callback The callback function to be called with an error or the results
+ * @param {object} params Hidden parameters
+ * @param {object} params.retry Retry value to increase retry time
+ */
 function runSQL(sqlQuery, connection, callback = (err, rows) => {
     log.ERROR(err);
     throw err;
-}) {
+}, {retry = 0}) {
     // Start the MySQL Connection
     if (!connection) connection = Connection;
     connection.query(sqlQuery, (err, rows) => {
         if (err) {
-            log.ERROR("SQL Error on query:", sqlQuery, err);
-            return callback(err, undefined);
+            if (err.code === "EHOSTUNREACH") {
+                // DB Connection is unavailable
+                let retryTimeout;
+                switch(retry){
+                    case 0:
+                        retryTimeout = 30000;
+                        break;
+                    case retry < 15:
+                        retryTimeout = 30000 + retry * 15000;
+                        break;
+                    default:                        
+                        log.ERROR("Retried Database 15 times over, please check connection status and restart the app", sqlQuery, err);
+                        return callback(err, undefined);                        
+                }
+                log.WARN(`Database connection is unavailable, waiting ${ retryTimeout / 1000 } seconds...`);
+                retry += 1
+                // Wait for the retry timeout before trying the query again
+                setTimeout(runSQL(sqlQuery, connection, callback, { retry: retry }));
+            }
+            else return callback(err, undefined);        
         } 
         log.VERBOSE(`SQL result for query '${sqlQuery}':`, rows);        
         return callback(undefined, rows);