Initial push for #17

- Partial functionality
- No pagination
- Only 9 results

Client/controllers/botController.js

@@ -24,7 +24,7 @@ exports.getStatus = (req, res) => {
  * Start the bot and join the server and preset specified
  */
 exports.joinServer = async (req, res) => {
-    if (!req.body.clientId || !req.body.channelId) return res.send("500").json({"message": "You must include the client ID (discord token), channel ID (The discord ID of the channel to connect to)"});
+    if (!req.body.clientId || !req.body.channelId) return res.status(500).json({"message": "You must include the client ID (discord token), channel ID (The discord ID of the channel to connect to)"});
     const deviceId = process.env.AUDIO_DEVICE_ID;
     const channelId = req.body.channelId;
     const clientId = req.body.clientId;

Server/commands/join.js

@@ -1,11 +1,9 @@
 // Modules
 const { customSlashCommandBuilder } = require('../utilities/customSlashCommandBuilder');
 const { DebugBuilder } = require("../utilities/debugBuilder");
-const { BufferToJson, getMembersInRole, getKeyByArrayValue } = require("../utilities/utils");
+const { getMembersInRole, getAllClientIds } = require("../utilities/utils");
 const { requestOptions, sendHttpRequest } = require("../utilities/httpRequests");
-const { readFileSync } = require('fs');
-const { getOnlineNodes, getNodeInfoFromId, updateNodeInfo } = require("../utilities/mysqlHandler");
-const path = require('path');
+const { getOnlineNodes, updateNodeInfo, addNodeConnection, getConnectionByNodeId } = require("../utilities/mysqlHandler");
 
 // Global Vars
 const log = new DebugBuilder("server", "join");
@@ -24,62 +22,67 @@ async function joinServerWrapper(presetName, channelId, clientIdsUsed) {
 		getOnlineNodes((nodeRows) => {
 			recordResolve(nodeRows);
 		});
-	});
+	});	
+
 	// Check which nodes have the selected preset
 	onlineNodes = onlineNodes.filter(node => node.nearbySystems.includes(presetName));
 	log.DEBUG("Filtered Online Nodes: ", onlineNodes);
+
 	// Check if any nodes with this preset are available
 	var nodesCurrentlyAvailable = [];
 	for (const node of onlineNodes) {
-		const reqOptions = new requestOptions("/bot/status", "GET", node.ip, node.port);
-		await new Promise(resolve => sendHttpRequest(reqOptions, "", (responseObj) => {
-			if (!responseObj || !responseObj.statusCode == 200) return resolve(false);
-			log.VERBOSE("Response Object from node ", node, responseObj);
-			nodesCurrentlyAvailable.push(node);
-			resolve(true);
-		}));
+		const currentConnection = await getConnectionByNodeId(node.id);
+		log.DEBUG("Checking to see if there is a connection for Node: ", node, currentConnection);
+		if(!currentConnection) nodesCurrentlyAvailable.push(node);		
 	}
 	log.DEBUG("Nodes Currently Available: ", nodesCurrentlyAvailable);
+
 	// If not, let the user know	
 	if (!nodesCurrentlyAvailable.length > 0) return Error("All nodes with this channel are unavailable, consider swapping one of the currently joined bots.");
 
 	// If so, join with the first node
-	var availableClientIds = await Object(JSON.parse(readFileSync(path.resolve(__dirname, '../clientIds.json'))));
+	var availableClientIds = await getAllClientIds();
 	log.DEBUG("All clients: ", Object.keys(availableClientIds));
 
 	var selectedClientId;
 	if (typeof clientIdsUsed === 'string') {
-		if (Object.keys(availableClientIds).includes(clientIdsUsed)) selectedClientId = availableClientIds[clientIdsUsed];
+		for (const availableClientId of availableClientIds) {
+			if (availableClientId.discordId != clientIdsUsed ) selectedClientId = availableClientId;
+		}		
 	}
 	else {
 		log.DEBUG("Client IDs Used: ", clientIdsUsed.keys());
 		for (const usedClientId of clientIdsUsed.keys()) {
 			log.DEBUG("Used Client ID: ", usedClientId);
-			if (Object.keys(availableClientIds).includes(usedClientId)) {
-				delete availableClientIds[usedClientId];
-			}
+			availableClientIds = availableClientIds.filter(cid => cid.discordId != usedClientId);			
 		}
 
 		log.DEBUG("Available Client IDs: ", availableClientIds);
 
 		if (!Object.keys(availableClientIds).length > 0) return log.ERROR("All client ID have been used, consider swapping one of the curretly joined bots or adding more Client IDs to the pool.")	
-		selectedClientId = availableClientIds[Object.keys(availableClientIds)[0]];
+		selectedClientId = availableClientIds[0];
 	}
 
 	const selectedNode = nodesCurrentlyAvailable[0];
 	
 	const reqOptions = new requestOptions("/bot/join", "POST", selectedNode.ip, selectedNode.port);	
-	sendHttpRequest(reqOptions, JSON.stringify({
+	const postObject = {
 		"channelId": channelId,
-		"clientId": selectedClientId.id,
+		"clientId": selectedClientId.clientId,
 		"presetName": presetName
-	}), async (responseObj) => {
+	};
+	log.INFO("Post Object: ", postObject);
+	sendHttpRequest(reqOptions, JSON.stringify(postObject), async (responseObj) => {
 		log.VERBOSE("Response Object from node ", selectedNode, responseObj);
 		if (!responseObj || !responseObj.statusCode == 200) return false;		
 		// Node has connected to discord
-		selectedNode.connected = true;
-		const updatedNode = await updateNodeInfo(selectedNode)
+		// Updating node Object in DB
+		const updatedNode = await updateNodeInfo(selectedNode);
 		log.DEBUG("Updated Node: ", updatedNode);
+
+		// Adding a new node connection
+		const nodeConnection = await addNodeConnection(selectedNode, selectedClientId);
+		log.DEBUG("Node Connection: ", nodeConnection);
 	});	
 }
 exports.joinServerWrapper = joinServerWrapper;
@@ -98,15 +101,16 @@ module.exports = {
 		try{
 			const guildId = interaction.guild.id;
 			const presetName = interaction.options.getString('preset');
+			if (!interaction.member.voice.channel.id) return interaction.editReply(`You need to be in a voice channel, ${interaction.user}`)
 			const channelId = interaction.member.voice.channel.id;
-			log.DEBUG(`Join requested by: ${interaction.user.username}, to: '${presetName}', in channel: ${channelId} / ${guildId}`);
-			await interaction.editReply('**Pong.**');
+			log.DEBUG(`Join requested by: ${interaction.user.username}, to: '${presetName}', in channel: ${channelId} / ${guildId}`);			
 
 			const onlineBots = await getMembersInRole(interaction);
 
 			log.DEBUG("Online Bots: ", onlineBots);
 
 			await joinServerWrapper(presetName, channelId, onlineBots.online);
+			await interaction.editReply('**Pong.**');
 			//await interaction.channel.send('**Pong.**'); // This will send a message to the channel of the interaction outside of the initial reply
 		}catch(err){
 			log.ERROR(err)

Server/commands/leave.js

@@ -0,0 +1,77 @@
+// Modules
+const { customSlashCommandBuilder } = require('../utilities/customSlashCommandBuilder');
+const { DebugBuilder } = require("../utilities/debugBuilder");
+const { getAllClientIds, getKeyByArrayValue } = require("../utilities/utils");
+const { requestOptions, sendHttpRequest } = require("../utilities/httpRequests");
+const { checkNodeConnectionByClientId, removeNodeConnectionByNodeId, updateNodeInfo, getConnectedNodes, getAllConnections } = require('../utilities/mysqlHandler');
+
+// Global Vars
+const log = new DebugBuilder("server", "leave");
+const logAC = new DebugBuilder("server", "leave_autocorrect");
+
+async function leaveServerWrapper(clientIdObject) {
+	if (!clientIdObject.clientId || !clientIdObject.name) return log.ERROR("Tried to leave server without client ID and/or Name");
+
+	const node = await checkNodeConnectionByClientId(clientIdObject);
+	
+	reqOptions = new requestOptions("/bot/leave", "POST", node.ip, node.port);
+
+	const responseObj = await new Promise((recordResolve, recordReject) => {
+		sendHttpRequest(reqOptions, JSON.stringify({}), async (responseObj) => {
+			recordResolve(responseObj);
+		});
+	});	
+
+	log.VERBOSE("Response Object from node ", node, responseObj);
+	if (!responseObj || !responseObj.statusCode == 202) return false;		
+	// Node has disconnected from discor
+	// Removing the node connection from the DB
+	const removedConnection = removeNodeConnectionByNodeId(node.id);
+	log.DEBUG("Removed Node Connection: ", removedConnection);
+
+	return;
+}
+exports.leaveServerWrapper = leaveServerWrapper;
+
+module.exports = {
+	data: new customSlashCommandBuilder()
+		.setName('leave')
+		.setDescription('Disconnect a bot from the server')
+		.addStringOption(option => 
+			option.setName("bot")
+			.setDescription("The bot to disconnect from the server")
+			.setAutocomplete(true)),
+	example: "leave",
+	isPrivileged: false,
+	requiresTokens: false,
+	defaultTokenUsage: 0,
+	deferInitialReply: true,
+	async autocomplete(interaction) {
+		const focusedValue = interaction.options.getFocused();
+		const connections = await getAllConnections();
+		const filtered = connections.filter(conn => String(conn.clientObject.name).startsWith(focusedValue)).map(conn => conn.clientObject.name);
+		logAC.DEBUG("Focused Value: ", focusedValue, connections, filtered);
+		await interaction.respond(
+			filtered.map(option => ({ name: option, value: option })),
+		);		
+	},
+	async execute(interaction) {
+		try{
+			const guildId = interaction.guild.id;
+			const botName = interaction.options.getString('bot');
+			log.DEBUG("Bot Name: ", botName)			
+			const clinetIds = await getAllClientIds();
+			log.DEBUG("Client names: ", clinetIds);
+			const clientDiscordId = getKeyByArrayValue(clinetIds, {'name': botName});
+			log.DEBUG("Selected bot: ", clinetIds[clientDiscordId]);
+			// Need to create a table in DB to keep track of what bots have what IDs or an endpoint on the clients to return what ID they are running with
+			await leaveServerWrapper(clinetIds[clientDiscordId]);
+			
+			await interaction.editReply(`**${clinetIds[clientDiscordId].name}** has been disconnected`);  // This will reply to the initial interaction
+			//await interaction.channel.send('**word.**'); // This will send a message to the channel of the interaction outside of the initial reply
+		}catch(err){
+			log.ERROR(err)
+            //await interaction.reply(err.toString());
+		}
+	}
+}

Server/commands/ping.js

@@ -18,6 +18,9 @@ module.exports = {
 	requiresTokens: false,
 	defaultTokenUsage: 0,
 	deferInitialReply: false,
+	/*async autocomplete(interaction) {
+		const focusedValue = interaction.options.getFocused();		
+	},*/
 	async execute(interaction) {
 		try{
 			await interaction.channel.send('**Pong.**'); // TODO - Add insults as the response to this command

Server/events/interactionCreate.js

@@ -8,45 +8,53 @@ const log = new DebugBuilder("server", "interactionCreate");
 module.exports = {
 	name: Events.InteractionCreate,
 	async execute(interaction) {
+		const command = interaction.client.commands.get(interaction.commandName);
+		log.VERBOSE("Interaction for command: ", command);
+
+		// Execute autocomplete if the user is checking autocomplete
+		if (interaction.isAutocomplete()) {
+			log.DEBUG("Running autocomplete for command: ", command.data.name);
+			return await command.autocomplete(interaction);
+		}
+
+		// Check if the interaction is a command
 		if (!interaction.isChatInputCommand()) return;
 
-	const command = interaction.client.commands.get(interaction.commandName);	
+		if (!command) {
+			log.ERROR(`No command matching ${interaction.commandName} was found.`);
+			return;
+		}
 
-	if (!command) {
-		log.ERROR(`No command matching ${interaction.commandName} was found.`);
-		return;
-	}
+		log.DEBUG(`${interaction.member.user} is running '${interaction.commandName}'`);
 
-	log.DEBUG(`${interaction.member.user} is running '${interaction.commandName}'`);
-
-	await authorizeCommand(interaction, command, async () => {
-		await authorizeTokenUsage(interaction, command, undefined, async () => {
-			try {
-				if (command.deferInitialReply) {
-					try {
-						if (interaction.options.getBool('public') && interaction.options.getBool('public') == false) await interaction.deferReply({ ephemeral: true });
-						else await interaction.deferReply({ ephemeral: false });
-					}
-					catch (err) {
-						if (err instanceof TypeError) {
-							// The public option doesn't exist in this command
-							await interaction.deferReply({ ephemeral: false });
-						} else {
-							throw err;
+		await authorizeCommand(interaction, command, async () => {
+			await authorizeTokenUsage(interaction, command, undefined, async () => {
+				try {
+					if (command.deferInitialReply) {
+						try {
+							if (interaction.options.getBool('public') && interaction.options.getBool('public') == false) await interaction.deferReply({ ephemeral: true });
+							else await interaction.deferReply({ ephemeral: false });
 						}
+						catch (err) {
+							if (err instanceof TypeError) {
+								// The public option doesn't exist in this command
+								await interaction.deferReply({ ephemeral: false });
+							} else {
+								throw err;
+							}
 
+						}
+					} 
+					command.execute(interaction);
+				} catch (error) {
+					log.ERROR(error);
+					if (interaction.replied || interaction.deferred) {
+						interaction.followUp({ content: 'There was an error while executing this command!', ephemeral: true });
+					} else {
+						interaction.reply({ content: 'There was an error while executing this command!', ephemeral: true });
 					}
-				} 
-				command.execute(interaction);
-			} catch (error) {
-				log.ERROR(error);
-				if (interaction.replied || interaction.deferred) {
-					interaction.followUp({ content: 'There was an error while executing this command!', ephemeral: true });
-				} else {
-					interaction.reply({ content: 'There was an error while executing this command!', ephemeral: true });
 				}
-			}
-		})		
-	});
+			})		
+		});
 	},
 };

Server/events/messageCreate.js

@@ -9,6 +9,6 @@ const log = new DebugBuilder("server", "messageCreate");
 module.exports = {
 	name: Events.MessageCreate,
 	async execute(interaction) {
-		await linkCop(interaction);
+		//await linkCop(interaction);
 	},
 };

Server/modules/mlsScraper/main.py

@@ -0,0 +1,189 @@
+import re
+import json
+import pandas as pd
+import requests
+import os
+import random
+from fake_useragent import UserAgent
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse, unquote, parse_qs
+from time import sleep
+ua = UserAgent()
+
+
+#simply scrape
+def scrape(url,**kwargs):
+    
+    session=requests.Session()
+    session.headers.update({
+        'User-Agent': ua.random,
+        "authority": "www.zillow.com",
+        "accept": "*/*",
+        "accept-language": "en-US,en;q=0.9",
+        "cache-control": "no-cache",
+        "sec-ch-ua-mobile": "?0",
+        "sec-ch-ua-platform": '^\^"Windows^\^"',
+        "sec-fetch-dest": "empty",
+        "sec-fetch-mode": "cors",
+        "sec-fetch-site": "same-origin",
+    })
+    
+    response=session.get(url,**kwargs)
+
+    return response
+
+# Return all sections with key and attributes
+def slurp(html, tag, attributes):
+    return BeautifulSoup(html, features="html.parser").findAll(tag, attributes)
+
+# Returns the first number group from a given string
+def return_numbers(string):
+    return int(re.findall(r'\d+', string)[0])
+
+
+class Listing:
+    def __init__(self, address, bedrooms, bathrooms, sqft, price, link):
+        self.address = address
+        self.bedrooms = bedrooms
+        self.bathrooms = bathrooms
+        self.sqft = sqft
+        self.price = price
+        self.link = link
+
+
+class ScrapeZillowListings:
+    def __init__(self, url):
+        self.parsed_original_url = self.init_check_url(urlparse(url))
+        self.html = scrape(url).text
+        self.listings = []
+
+    def init_check_url(self, parsed_url):
+        # Check to see if we are requesting listResults
+        print(parsed_url)
+        print(unquote(parsed_url.query))
+        print(parse_qs(parsed_url.query)['wants'])
+        for want in parse_qs(parsed_url.query)['wants']:
+            print(unquote(unquote(want)))
+        
+        return parsed_url
+
+    def run(self):
+        self.listings.extend(self.scrape_listings(self.html))
+        pages = []
+        for page_nav in slurp(self.html, "nav", {"role":"navigation", "aria-label":"Pagination"}):
+            page_nav = f"<html><head><head/><body>{page_nav}<body/><html/>"
+            pages_list = slurp(page_nav, "li", {})
+            for page in pages_list:
+                if re.match("\d{1,2}", page.text) and not page.text == "1":
+                    parsed_url = self.setup_url(page.find('a').get('href'))
+                    sleep(random.randint(0,15))
+                    temp_html = scrape(parsed_url.geturl()).text                    
+                    self.listings.extend(self.scrape_listings(temp_html)) 
+
+        return self.listings
+
+    def print_listings(self):
+        index = 0
+        for temp_listing in self.listings:
+            print("--------")
+            print(f"Listing #{index}")
+            print(temp_listing.address)
+            print(temp_listing.price)
+            print(temp_listing.bedrooms)
+            print(temp_listing.bathrooms)
+            print(temp_listing.sqft)
+            print(temp_listing.link)
+            print("--------")
+            index += 1
+
+    def scrape_listings(self, html):
+        temp_listings = []
+        for listing in slurp(html, "article", {"data-test":"property-card"}):
+            listing = f"<html><head><head/><body>{listing}<body/><html/>"
+
+            uls = slurp(listing, "li", {})
+            beds = 0
+            baths = 0
+            sqft = 0
+            for ul in uls:
+                ul = ul.get_text()
+                if ("bds" in str(ul)):
+                    beds = return_numbers(ul)
+                if ("ba" in str(ul)):
+                    baths = return_numbers(ul)
+                if ("sqft" in str(ul)):
+                    sqft = return_numbers(ul)
+
+            temp_listings.append(Listing(
+                address=slurp(listing, "address", {"data-test":"property-card-addr"})[0].get_text(),
+                bedrooms=beds,
+                bathrooms=baths,
+                sqft=sqft,
+                price=slurp(listing, "span", {"data-test":"property-card-price"})[0].get_text(),
+                link=slurp(listing, "a", {"data-test":"property-card-link"})[0].get('href'),
+            ))
+        
+        return temp_listings
+    
+    def setup_url(self, url):
+        parsed_url = urlparse(url)
+        print(parsed_url)
+        if not parsed_url.netloc:
+            return urlparse(f"{self.parsed_original_url.scheme}://{self.parsed_original_url.netloc}{parsed_url.path}{self.parsed_original_url.query}{self.parsed_original_url.params}")            
+
+#create dataframe
+def etl(response):
+
+    #regex to find the data
+    
+    for listing in listings:
+        print("--------")
+        print(listing)
+        print("--------")
+
+    print("FORCE STOP")
+    exit()
+
+    #convert text to dict via json
+    dicts=[json.loads('{'+i+'}') for i in num]
+
+    #create dataframe
+    df=pd.DataFrame()
+    for ind,val in enumerate(text):
+        df[val]=dicts[ind].values()
+    df.index=dicts[ind].keys()
+    
+    return df
+
+
+def main():    
+    #scrapper = ScrapeZillowListings('https://www.zillow.com/westchester-county-ny/?searchQueryState=%7B%22usersSearchTerm%22%3A%22Yorktown%20Heights%2C%20NY%22%2C%22mapBounds%22%3A%7B%22north%22%3A41.69948153143324%2C%22east%22%3A-72.68804025585938%2C%22south%22%3A40.83865274682678%2C%22west%22%3A-74.29479074414063%7D%2C%22isMapVisible%22%3Atrue%2C%22filterState%22%3A%7B%22price%22%3A%7B%22max%22%3A250000%7D%2C%22ah%22%3A%7B%22value%22%3Atrue%7D%2C%22sort%22%3A%7B%22value%22%3A%22days%22%7D%2C%22land%22%3A%7B%22value%22%3Afalse%7D%2C%22cmsn%22%3A%7B%22value%22%3Afalse%7D%2C%22sche%22%3A%7B%22value%22%3Afalse%7D%2C%22schm%22%3A%7B%22value%22%3Afalse%7D%2C%22schh%22%3A%7B%22value%22%3Afalse%7D%2C%22schp%22%3A%7B%22value%22%3Afalse%7D%2C%22schr%22%3A%7B%22value%22%3Afalse%7D%2C%22schc%22%3A%7B%22value%22%3Afalse%7D%2C%22schu%22%3A%7B%22value%22%3Afalse%7D%7D%2C%22isListVisible%22%3Atrue%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A3148%2C%22regionType%22%3A4%7D%2C%7B%22regionId%22%3A2694%2C%22regionType%22%3A4%7D%5D%2C%22pagination%22%3A%7B%7D%7D')
+    scrapper = ScrapeZillowListings("https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState=^%^7B^%^22pagination^%^22^%^3A^%^7B^%^7D^%^2C^%^22usersSearchTerm^%^22^%^3A^%^22Yorktown^%^20Heights^%^2C^%^20NY^%^22^%^2C^%^22mapBounds^%^22^%^3A^%^7B^%^22north^%^22^%^3A42.99146217894271^%^2C^%^22east^%^22^%^3A-70.80209903627659^%^2C^%^22south^%^22^%^3A39.549453943310084^%^2C^%^22west^%^22^%^3A-77.00937442690159^%^7D^%^2C^%^22mapZoom^%^22^%^3A8^%^2C^%^22regionSelection^%^22^%^3A^%^5B^%^7B^%^22regionId^%^22^%^3A3148^%^2C^%^22regionType^%^22^%^3A4^%^7D^%^2C^%^7B^%^22regionId^%^22^%^3A2694^%^2C^%^22regionType^%^22^%^3A4^%^7D^%^5D^%^2C^%^22isMapVisible^%^22^%^3Atrue^%^2C^%^22filterState^%^22^%^3A^%^7B^%^22price^%^22^%^3A^%^7B^%^22max^%^22^%^3A250000^%^7D^%^2C^%^22isAllHomes^%^22^%^3A^%^7B^%^22value^%^22^%^3Atrue^%^7D^%^2C^%^22sortSelection^%^22^%^3A^%^7B^%^22value^%^22^%^3A^%^22days^%^22^%^7D^%^2C^%^22isLotLand^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isMiddleSchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isHighSchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22includeUnratedSchools^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isComingSoon^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isPublicSchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isPrivateSchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isElementarySchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isCharterSchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^7D^%^2C^%^22isListVisible^%^22^%^3Atrue^%^7D&wants=^\{^%^22cat1^%^22:^\[^%^22mapResults^%^22^\]^\}&requestId=3")
+    listings = scrapper.run()
+    scrapper.print_listings()
+        
+    #df=etl(response)    
+    
+    return
+
+
+if __name__ == "__main__":
+    main()
+
+
+#curl "https://www.zillow.com/search/GetSearchPageState.htm?searchQueryState=^%^7B^%^22pagination^%^22^%^3A^%^7B^%^7D^%^2C^%^22usersSearchTerm^%^22^%^3A^%^22Yorktown^%^20Heights^%^2C^%^20NY^%^22^%^2C^%^22mapBounds^%^22^%^3A^%^7B^%^22north^%^22^%^3A42.99146217894271^%^2C^%^22east^%^22^%^3A-70.80209903627659^%^2C^%^22south^%^22^%^3A39.549453943310084^%^2C^%^22west^%^22^%^3A-77.00937442690159^%^7D^%^2C^%^22mapZoom^%^22^%^3A8^%^2C^%^22regionSelection^%^22^%^3A^%^5B^%^7B^%^22regionId^%^22^%^3A3148^%^2C^%^22regionType^%^22^%^3A4^%^7D^%^2C^%^7B^%^22regionId^%^22^%^3A2694^%^2C^%^22regionType^%^22^%^3A4^%^7D^%^5D^%^2C^%^22isMapVisible^%^22^%^3Atrue^%^2C^%^22filterState^%^22^%^3A^%^7B^%^22price^%^22^%^3A^%^7B^%^22max^%^22^%^3A250000^%^7D^%^2C^%^22isAllHomes^%^22^%^3A^%^7B^%^22value^%^22^%^3Atrue^%^7D^%^2C^%^22sortSelection^%^22^%^3A^%^7B^%^22value^%^22^%^3A^%^22days^%^22^%^7D^%^2C^%^22isLotLand^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isMiddleSchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isHighSchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22includeUnratedSchools^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isComingSoon^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isPublicSchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isPrivateSchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isElementarySchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^2C^%^22isCharterSchool^%^22^%^3A^%^7B^%^22value^%^22^%^3Afalse^%^7D^%^7D^%^2C^%^22isListVisible^%^22^%^3Atrue^%^7D&wants=^\{^%^22cat1^%^22:^\[^%^22mapResults^%^22^\]^\}&requestId=3",
+#   "authority: www.zillow.com",
+#   "accept: */*",
+#   "accept-language: en-US,en;q=0.9",
+#   "cache-control: no-cache",
+#   "cookie: JSESSIONID=97FD1EB701E102B7353E8EA4528843CE; zguid=24^|^%^24825bd6e9-4f90-46df-a475-4a9910b5847c; zgsession=1^|a6a5b7ca-c651-45a2-93c8-c5b66fea68d3; AWSALB=oQ3DGTMPgyQOTPA6zLmQ0liqJ1oax2QoQ5rUSCsORkWP52C7k6G8H1gZnlxOtgU/zzO503UHUnQ7tUeivhOnupv7aYI6+E5LxUZl4TeE0JyhvT3pZ6LYeC9iFbTw; AWSALBCORS=oQ3DGTMPgyQOTPA6zLmQ0liqJ1oax2QoQ5rUSCsORkWP52C7k6G8H1gZnlxOtgU/zzO503UHUnQ7tUeivhOnupv7aYI6+E5LxUZl4TeE0JyhvT3pZ6LYeC9iFbTw; search=6^|1689549806090^%^7Crect^%^3D42.92311815473404^%^252C-70.80209903627659^%^252C39.62142250427077^%^252C-77.00937442690159^%^26rid^%^3D2694^%^26disp^%^3Dmap^%^26mdm^%^3Dauto^%^26p^%^3D1^%^26sort^%^3Ddays^%^26z^%^3D1^%^26listPriceActive^%^3D1^%^26type^%^3Dhouse^%^252Ccondo^%^252Capartment_duplex^%^252Cmobile^%^252Ctownhouse^%^26lt^%^3Dfsba^%^252Cfsbo^%^252Cfore^%^252Cnew^%^252Cauction^%^26price^%^3D0-250000^%^26fs^%^3D1^%^26fr^%^3D0^%^26mmm^%^3D0^%^26rs^%^3D0^%^26ah^%^3D0^%^26singlestory^%^3D0^%^26housing-connector^%^3D0^%^26abo^%^3D0^%^26garage^%^3D0^%^26pool^%^3D0^%^26ac^%^3D0^%^26waterfront^%^3D0^%^26finished^%^3D0^%^26unfinished^%^3D0^%^26cityview^%^3D0^%^26mountainview^%^3D0^%^26parkview^%^3D0^%^26waterview^%^3D0^%^26hoadata^%^3D1^%^26zillow-owned^%^3D0^%^263dhome^%^3D0^%^26featuredMultiFamilyBuilding^%^3D0^%^26commuteMode^%^3Ddriving^%^26commuteTimeOfDay^%^3Dnow^%^09^%^092694^%^09^%^09^%^09^%^09^%^09^%^09",
+#   "pragma: no-cache",
+#   "sec-ch-ua: ^\^"Not.A/Brand^\^";v=^\^"8^\^", ^\^"Chromium^\^";v=^\^"114^\^", ^\^"Google Chrome^\^";v=^\^"114^\^"",
+#   "sec-ch-ua-mobile: ?0",
+#   "sec-ch-ua-platform: ^\^"Windows^\^"",
+#   "sec-fetch-dest: empty",
+#   "sec-fetch-mode: cors",
+#   "sec-fetch-site: same-origin",
+#   "user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
+#   "x-kl-ajax-request: Ajax_Request",
+#  --compressed

Server/modules/mlsScraper/requirements.txt

@@ -0,0 +1,4 @@
+pandas
+requests
+fake-useragent
+beautifulsoup4

Server/utilities/mysqlHandler.js

@@ -1,9 +1,9 @@
 require('dotenv').config();
 const mysql = require('mysql');
 const utils = require('./utils');
-const { nodeObject } = require("./recordHelper");
+const { nodeObject, clientObject, connectionObject } = require("./recordHelper");
 const { DebugBuilder } = require("../utilities/debugBuilder");
-const { BufferToJson } = require("../utilities/utils");
+const { BufferToJson, getClientObjectByClientID } = require("../utilities/utils");
 
 const log = new DebugBuilder("server", "mysSQLHandler");
  
@@ -15,6 +15,7 @@ const connection = mysql.createPool({
 });
 
 const nodesTable = `${process.env.NODE_DB_NAME}.nodes`;
+const nodeConnectionsTable = `${process.env.NODE_DB_NAME}.node_connections`;
 
 /**
  * Return a node object from a single SQL row
@@ -31,7 +32,6 @@ function returnNodeObjectFromRow(row) {
         _location: row.location,
         _nearbySystems: BufferToJson(row.nearbySystems),
         _online: (row.online === 1) ? true : false,
-        _connected: (row.connected === 1) ? true : false
         });
 }
 
@@ -53,6 +53,22 @@ function returnNodeObjectFromRows(rows) {
     return rows;
 }
 
+/**
+ * Returns a connection object from an SQL row
+ * 
+ * @param {*} row The SQL row to convert to a connection object
+ * @returns {connectionObject}
+ */
+async function returnConnectionObjectFromRow(row) {
+    if (Array.isArray(row)) row = row[0]
+    log.DEBUG("Connection row: ", row);
+    return new connectionObject({
+        _connection_id: row.connection_id,
+        _node: await getNodeInfoFromId(row.id),
+        _client_object: await getClientObjectByClientID(row.discord_client_id)
+    });
+}
+
 /** Get all nodes the server knows about regardless of status
  * @param {*} callback Callback function
  */
@@ -71,8 +87,7 @@ exports.getAllNodes = (callback) => {
  * @returns 
  */
 exports.getAllNodesSync = async () => {
-    const sqlQuery = `SELECT * FROM ${nodesTable}`
-    var returnObjects = [];
+    const sqlQuery = `SELECT * FROM ${nodesTable}`    
     const rows = await runSQL(sqlQuery);
 
     console.log("Rows: ", rows);
@@ -94,20 +109,28 @@ exports.getOnlineNodes = (callback) => {
  * @param nodeId The ID of the node 
  * @param callback Callback function
  */ 
-exports.getNodeInfoFromId = (nodeId, callback) => {
+ async function getNodeInfoFromId(nodeId, callback = undefined) {
+    if (!nodeId) throw new Error("No node ID given when trying to fetch node");
+    log.DEBUG("Getting node from ID: ", nodeId);
     const sqlQuery = `SELECT * FROM ${nodesTable} WHERE id = ${nodeId}`
-    runSQL(sqlQuery, (rows) => {
-        // Call back the first (and theoretically only) row
-        // Specify 0 so downstream functions don't have to worry about it
-        return callback(returnNodeObjectFromRow(rows[0]));
-    })
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+
+    // Call back the first (and theoretically only) row
+    // Specify 0 so downstream functions don't have to worry about it
+    return (callback) ? callback(returnNodeObjectFromRow(sqlResponse[0])) : returnNodeObjectFromRow(sqlResponse[0]);
 }
+exports.getNodeInfoFromId = getNodeInfoFromId
 
 /** Add a new node to the DB
  * @param nodeObject Node information object
  * @param callback Callback function
  */
-exports.addNewNode = (nodeObject, callback) => {
+exports.addNewNode = async (nodeObject, callback) => {
     if (!nodeObject.name) throw new Error("No name provided");
     const name = nodeObject.name,
         ip = nodeObject.ip,
@@ -118,23 +141,28 @@ exports.addNewNode = (nodeObject, callback) => {
         connected = 0;
     const sqlQuery  = `INSERT INTO ${nodesTable} (name, ip, port, location, nearbySystems, online, connected) VALUES ('${name}', '${ip}', ${port}, '${location}', '${nearbySystems}', ${online}, ${connected})`;
 
-    runSQL(sqlQuery, (rows) => {
-        return callback(returnNodeObjectFromRows(rows));
-    })
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+
+    // Call back the first (and theoretically only) row
+    // Specify 0 so downstream functions don't have to worry about it
+    return (callback) ? callback(returnNodeObjectFromRow(sqlResponse)) : returnNodeObjectFromRow(sqlResponse);
 }
 
 /** Update the known info on a node
  * @param nodeObject Node information object
  * @param callback Callback function
  */
-exports.updateNodeInfo = (nodeObject, callback = undefined) => {
+exports.updateNodeInfo = async (nodeObject, callback = undefined) => {
     if(!nodeObject.id) throw new Error("Attempted to updated node without providing ID", nodeObject);
     const name = nodeObject.name,
         ip = nodeObject.ip,
         port = nodeObject.port,
         location = nodeObject.location,
-        online = nodeObject.online,
-        connected = nodeObject.connected;
+        online = nodeObject.online
     let queryParams = [],
         nearbySystems = nodeObject.nearbySystems;
 
@@ -150,10 +178,6 @@ exports.updateNodeInfo = (nodeObject, callback = undefined) => {
         if (online || online === 1) queryParams.push(`online = 1`);
         else queryParams.push(`online = 0`);
     }
-    if (typeof connected === "boolean" || typeof connected === "number") {
-        if (online || online === 1) queryParams.push(`connected = 1`);
-        else queryParams.push(`connected = 0`);
-    }
 
     let sqlQuery = `UPDATE ${nodesTable} SET`
     if (!queryParams || queryParams.length === 0) return (callback) ? callback(undefined) : undefined;
@@ -175,21 +199,167 @@ exports.updateNodeInfo = (nodeObject, callback = undefined) => {
 
     sqlQuery = `${sqlQuery} WHERE id = ${nodeObject.id};`
 
-    runSQL(sqlQuery, (rows) => {
-        if (rows.affectedRows === 1) return (callback) ? callback(true) : true;
-        else return (callback) ? callback(returnNodeObjectFromRows(rows)) : returnNodeObjectFromRows(rows);
-    })
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+
+    if (sqlResponse.affectedRows === 1) return (callback) ? callback(true) : true;
+    else return (callback) ? callback(returnNodeObjectFromRows(sqlResponse)) : returnNodeObjectFromRows(sqlResponse);
+}
+
+/**
+ * Add a new connection to the DB when a bot has been connected to the server
+ * 
+ * @param {*} nodeObject The node object that is being used for this connection
+ * @param {*} clientId The client ID Object being used for this connection
+ * @param {*} callback [OPTIONAL] The callback function to be called with the results, will return otherwise
+ */
+exports.addNodeConnection = (nodeObject, clientObject, callback = undefined) => {
+    if (!nodeObject.id || !clientObject.clientId) throw new Error("Tried to add a connection without a client and/or node ID");
+    const sqlQuery  = `INSERT INTO ${nodeConnectionsTable} (id, discord_client_id) VALUES (${nodeObject.id}, '${clientObject.clientId}')`;
+
+    const sqlResponse = new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    
+    if (!sqlResponse) throw new Error("No result from added connection"); 
+    return (callback) ? callback(true) : true;
+}
+
+/**
+ * Check what node is connected with a given client ID object
+ * 
+ * @param {*} clientId The client ID object used to search for a connected node
+ * @param {*} callback [OPTIONAL] The callback function to be called with the results, return will be used otherwise
+ */
+exports.checkNodeConnectionByClientId = async (clientId, callback = undefined) => {
+    if (!clientId.clientId) throw new Error("Tried to check a connection without a client ID");
+    const sqlQuery  = `SELECT * FROM ${nodeConnectionsTable} WHERE discord_client_id = '${clientId.clientId}'`;
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    
+    log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
+    
+    if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
+    const newNodeObject = await getNodeInfoFromId(sqlResponse[0].id);
+    log.DEBUG("Node Object from SQL Response: ", newNodeObject);
+    return (callback) ? callback(newNodeObject) : newNodeObject;
+}
+
+/**
+ * Get a connection by node ID
+ * 
+ * @param {*} nodeId The ID to search for a connection with
+ * @param {*} callback [OPTIONAL] The callback function to be called with the results, return will be used otherwise
+ * @returns {connectionObject}
+ */
+exports.getConnectionByNodeId = async (nodeId, callback = undefined) => {
+    const sqlQuery  = `SELECT * FROM ${nodeConnectionsTable} WHERE id = '${nodeId}'`;
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    
+    log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
+    
+    if (!sqlResponse | sqlResponse.length == 0) return (callback) ? callback(undefined) : undefined;
+    const newConnectionObject = await returnConnectionObjectFromRow(sqlResponse)
+    log.DEBUG("Connection Object from SQL Response: ", newConnectionObject);
+    return (callback) ? callback(newConnectionObject) : newConnectionObject;
+}
+
+/**
+ * Remove a node connection by the node
+ * 
+ * @param {*} nodeId The node ID of the node to remove connections of
+ * @param {*} callback [OPTIONAL] The callback function to callback with the results, return will be used otherwise
+ * @returns 
+ */
+exports.removeNodeConnectionByNodeId = async (nodeId, callback = undefined) => {    
+    const sqlQuery  = `DELETE FROM ${nodeConnectionsTable} WHERE id = '${nodeId}'`;
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    log.VERBOSE("SQL Response from removing connection: ", sqlResponse);
+    
+    if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
+    return (callback) ? callback(sqlResponse) : sqlResponse;
+}
+
+/**
+ * Gets all connected nodes
+ * 
+ * @param {*} callback [OPTIONAL] The callback function to callback with the results, return will be used otherwise
+ * @returns {nodeObject}
+ */
+exports.getConnectedNodes = async (callback = undefined) =>  {
+    const sqlQuery  = `SELECT * FROM ${nodeConnectionsTable}`;
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    
+    log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
+    
+    if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
+    var nodeObjects = []
+    for (const row of sqlResponse) {
+        const newNodeObject = await getNodeInfoFromId(row.id);
+        log.DEBUG("Node Object from SQL Response: ", newNodeObject);
+        nodeObjects.push(newNodeObject);
+    }    
+    return (callback) ? callback(nodeObjects) : nodeObjects;
+}
+
+/**
+ * Returns all connections
+ * 
+ * @param {*} callback [OPTIONAL] The callback function to callback with the results, return will be used otherwise
+ * @returns {connectionObject}
+ */
+exports.getAllConnections = async (callback = undefined) => {
+    const sqlQuery  = `SELECT * FROM ${nodeConnectionsTable}`;
+
+    const sqlResponse = await new Promise((recordResolve, recordReject) => {
+		runSQL(sqlQuery, (rows) => {
+            recordResolve(rows);
+        })
+	});
+    
+    log.VERBOSE("SQL Response from checking connection: ", sqlResponse);
+    
+    if (!sqlResponse) return (callback) ? callback(undefined) : undefined;
+    var connectionObjects = []
+    for (const row of sqlResponse) {
+        connectionObjects.push(await returnConnectionObjectFromRow(row));
+    }    
+    return (callback) ? callback(connectionObjects) : connectionObjects;
 }
 
 // Function to run and handle SQL errors
-function runSQL(sqlQuery, callback, error = (err) => {
+function runSQL(sqlQuery, callback = undefined, error = (err) => {
     console.log(err);
     throw err;
 }) {
-    return connection.query(sqlQuery, (err, rows) => {
+    connection.query(sqlQuery, (err, rows) => {
         if (err) return error(err);
-        //console.log('The rows are:', rows);
-        return callback(rows);
+        //console.log('The rows are:', rows);        
+        return (callback) ? callback(rows) : rows
     })
 }
 

Server/utilities/recordHelper.js

@@ -111,9 +111,10 @@ class nodeObject {
      * @param {*} param0._location The physical location of the node
      * @param {*} param0._online True/False if the node is online or offline 
      * @param {*} param0._connected True/False if the bot is connected to discord or not
+     * @param {*} param0._connection The connection Object associated with the node, null if not checked, undefined if none exists 
      * @param {*} param0._nearbySystems An object array of nearby systems     
      */
-    constructor({ _id = null, _name = null, _ip = null, _port = null, _location = null, _nearbySystems = null, _online = null, _connected = null }) {
+    constructor({ _id = null, _name = null, _ip = null, _port = null, _location = null, _nearbySystems = null, _online = null }) {
         this.id = _id;
         this.name = _name;        
         this.ip = _ip;
@@ -121,8 +122,45 @@ class nodeObject {
         this.location = _location;
         this.nearbySystems = _nearbySystems;
         this.online = _online;
-        this.connected = _connected;
     }
 }
 
-exports.nodeObject = nodeObject;
+exports.nodeObject = nodeObject;
+
+/**
+ * This object represents a discord bot's client information
+ */
+class clientObject {
+    /**
+     * 
+     * @param {*} param0._discord_id The discord id from the node, as seen when right clicking -> copy ID
+     * @param {*} param0._name The name of the bot associated with the IDs
+     * @param {*} param0._client_id The client ID of the bot needed to connect to Discord
+     */
+    constructor({_discord_id = null, _name = null, _client_id = null,}) {
+        this.discordId = _discord_id;
+        this.name = _name;
+        this.clientId = _client_id;
+    }
+}
+
+exports.clientObject = clientObject;
+
+/**
+ * This object represents a discord node connection
+ */
+class connectionObject {
+    /**
+     * 
+     * @param {*} param0._connection_id The connection ID associated with the connection in the database
+     * @param {*} param0._node The node associated with the connection
+     * @param {*} param0._client_object The client object associated with the connection
+     */
+    constructor({_connection_id = null, _node = null, _client_object}) {
+        this.connectionId = _connection_id;
+        this.node = _node;
+        this.clientObject = _client_object;
+    }
+}
+
+exports.connectionObject = connectionObject;

Server/utilities/utils.js

@@ -1,6 +1,9 @@
 // Debug
 const { DebugBuilder } = require("../utilities/debugBuilder");
+const { clientObject } = require("./recordHelper");
+const { readFileSync } = require('fs');
 const log = new DebugBuilder("server", "utils");
+const path = require('path');
 
 // Convert a JSON object to a buffer for the DB
 exports.JsonToBuffer = (jsonObject) => {
@@ -30,8 +33,11 @@ exports.SanitizePresetName = (presetName) => {
  */
 exports.getMembersInRole = async (interaction, roleName = "Bots" ) => {
     log.DEBUG("Fetching all members");
-    await interaction.guild.members.fetch() //cache all members in the server
-    const role = await interaction.guild.roles.cache.find(role => role.name === roleName); //the role to check
+    var guild = await interaction.client.guilds.fetch({ guild: interaction.guild.id, cache: false }); //cache all members in the server
+    await guild.members.fetch({cache: false});
+    await guild.roles.fetch({cache: false});
+    log.VERBOSE("Guild: ", guild);
+    const role = await guild.roles.cache.find(role => role.name === roleName); //the role to check
     log.DEBUG("Role to check members from: ", role);
     log.DEBUG("Members of role: ", role.members);
     
@@ -56,7 +62,9 @@ exports.getMembersInRole = async (interaction, roleName = "Bots" ) => {
  * @returns The key of the object that contains the value
  */
 exports.getKeyByArrayValue = (object, value) => {
-    return Object.keys(object).find(key => object[key].includes(value));
+    if (typeof value == "string") return Object.keys(object).find(key => object[key].includes(value));
+    const valueKey = Object.keys(value)[0];    
+    return Object.keys(object).find(key => (object[key][valueKey] == value[valueKey]));
   }
 
 /**
@@ -72,4 +80,40 @@ exports.isJsonString = (str) => {
         return false;
     }
     return true;
+}
+
+/**
+ * Get all client IDs from the saved JSON file
+ * 
+ * @returns Object of Client IDs
+ */
+exports.getAllClientIds = () => {
+    const jsonClientIds = JSON.parse(readFileSync(path.resolve(__dirname, '../clientIds.json')));
+    var clientObjects = [];
+    for (const jsonClientId of Object.keys(jsonClientIds)){
+        clientObjects.push(new clientObject({
+            _discord_id: jsonClientId,
+            _name: jsonClientIds[jsonClientId].name,
+            _client_id: jsonClientIds[jsonClientId].id
+        }))
+    }
+    return clientObjects;
+}
+
+/**
+ * Gets a client object froma discord client ID
+ * 
+ * @param {*} clientId The discord client ID to get the client object of
+ * @returns {clientObject|undefined}
+ */
+exports.getClientObjectByClientID = (clientId) => {
+    const clientObjects = this.getAllClientIds();
+    log.DEBUG("All client IDs: ", clientObjects);
+    for (const clientObject of clientObjects){
+        if (clientObject.clientId == clientId) {
+            log.DEBUG("Found client ID from given ID: ", clientObject);
+            return clientObject
+        }
+    }
+    return undefined
 }