Mess about with the zapping system & add another secret geojson generator

server/Helpers/Log.mjs

@@ -26,6 +26,7 @@ class Log {
 		
 		func(`${this.a.locol}[${prefix}]${this.a.reset}`, ...items);
 	}
+	log_e(...items) { this.log_raw(console.error, ...items); }
 	log(...items) { this.log_raw(console.log, ...items); }
 	info(...items) { this.log_raw(console.info, ...items); }
 	debug(...items) { this.log_raw(console.debug, ...items); }

server/Repos.SQLite/RSSIRepo.mjs

@@ -36,7 +36,12 @@ class RSSIRepo {
 	}
 	
 	iterate() {
-		return this.db.prepare(`SELECT * FROM rssis`).iterate();
+		return this.db.prepare(`SELECT
+			rssis.*,
+			readings.latitude,
+			readings.longitude
+		FROM rssis
+		JOIN readings ON rssis.reading_id = readings.id`).iterate();
 	}
 }
 

server/bootstrap/cli.mjs

@@ -4,11 +4,16 @@ import show_help from '../help.mjs';
 import { decode_payload } from '../ttn-app-server/DecodePayload.mjs';
 import path from 'path';
 
+import {
+	unnormalise_lat,
+	unnormalise_lng,
+} from '../../common/Normalisers.mjs'
+
 // HACK
 import awilix from 'awilix';
 
 export default async function(c) {
-	let { ansi: a, log: l, settings } = c.cradle;
+	let { ansi: a, log: l, settings, DatasetFetcher } = c.cradle;
 	
 	// 2: CLI Argument Parsing
 	let args = process.argv.slice(2); // Slice out the node binary name and the filename
@@ -46,7 +51,7 @@ export default async function(c) {
 		process.exit();
 	}
 
-	l.log(`${a.fgreen}${a.hicol}*** LoRaWAN Signal Mapper ***${a.reset}`);
+	l.log_e(`${a.fgreen}${a.hicol}*** LoRaWAN Signal Mapper ***${a.reset}`);
 
 	switch(extras[0]) {
 		case "ttn-app-server":
@@ -97,6 +102,32 @@ export default async function(c) {
 			await ai_trainer.train_all();
 			break;
 		
+		case "geojson-debug":
+			let result = [];
+			for(let next_item of DatasetFetcher.fetch_all(null, true)) {
+				result.push({
+					type: "Feature",
+					geometry: {
+						type: "Point",
+						coordinates: [
+							unnormalise_lng(next_item.input.longitude),
+							unnormalise_lat(next_item.input.latitude)
+						]
+					},
+					properties: {
+						"marker-symbol": "circle",
+						"marker-color": next_item.output[0] <= 0 ? "#dd0707" : "#04a104",
+						gateway: next_item.ext.gateway,
+						rssi: next_item.ext.rssi_raw
+					}
+				})
+			}
+			console.log(JSON.stringify({
+				type: "FeatureCollection",
+				features: result
+			}, null, "\t"));
+			break;
+		
 		default:
 			l.error(`Error: Subcommand '${extras[0]}' not recognised.`);
 			l.error(`Perhaps you mistyped it, or it hasn't been implemented yet?`);

server/settings.default.toml

@@ -63,7 +63,7 @@ devices = [
 # The radius around a positive result to zap negative results.
 # This helps to remove false-nagative readings, to improve the AI's ability to 
 # accurately predict the actual signal strength.
-false_negative_zap_radius = 150
+false_negative_zap_radius = 450
 
 # Whether to actually zap false negatives or not.
 # Useful when you want to... erm.... see why the zapping is necessary?

server/train-ai/DatasetFetcher.mjs

@@ -20,12 +20,17 @@ class DatasetFetcher {
 		this.repo_reading = ReadingRepo;
 	}
 	
-	fetch_all(gateway_id) {
+	fetch_all(gateway_id, extended = false) {
 		let gateway_location = this.repo_gateway.get_by_id(gateway_id);
 		
+		let iterator = gateway_id == null ? this.repo_rssi.iterate() : this.repo_rssi.iterate_gateway(gateway_id);
 		let result = [];
-		for(let rssi of this.repo_rssi.iterate_gateway(gateway_id)) {
+		
-			result.push({
+		for(let rssi of iterator) {
+			if(gateway_id == null)
+				gateway_location = this.repo_gateway.get_by_id(rssi.gateway_id);
+			
+			let item = {
 				input: {
 					latitude: rssi.latitude,
 					longitude: rssi.longitude,
@@ -34,24 +39,41 @@ class DatasetFetcher {
 				output: [
 					rssi.rssi
 				]
-			});
+			};
+			if(extended) {
+				item.ext = {
+					gateway: rssi.gateway_id,
+					rssi_raw: rssi.rssi
+				};
+			}
+			result.push(item);
 		}
 		
 		for(let reading of this.repo_reading.iterate_unreceived()) {
-			result.push({
+			let item = {
 				input: {
 					latitude: reading.latitude,
 					longitude: reading.longitude,
 					distance: haversine(gateway_location, reading)
 				},
 				output: [ -150 ]
-			});
+			};
+			if(extended) {
+				item.ext = {
+					gateway: "(none)",
+					rssi_raw: -150
+				};
+			}
+			result.push(item);
 		}
 		shuffle_fisher_yates(result);
 		
+		console.error("*** Start ***");
 		if(this.settings.ai.do_zap_false_negatives) {
+			console.error("Item count before zapping", result.length);
 			let zap_count = this.zap_false_negatives(result, this.settings.ai.false_negative_zap_radius);
-			this.l.log(`[DatasetFetcher] Zapped ${zap_count} false negatives with a radius of ${this.settings.ai.false_negative_zap_radius}m.`);
+			console.error("Item count after zapping", result.length);
+			this.l.log_e(`[DatasetFetcher] Zapped ${zap_count} false negatives with a radius of ${this.settings.ai.false_negative_zap_radius}m.`);
 		}
 		
 		for(let item of result) {
@@ -60,6 +82,8 @@ class DatasetFetcher {
 			item.input.distance = normalise_gateway_distance(item.input.distance);
 			item.output[0] = normalise_rssi(item.output[0]);
 		}
+		console.error("Item count after normalisation", result.length);
+		console.error("*** End ***");
 		
 		return result;
 	}
@@ -91,7 +115,7 @@ class DatasetFetcher {
 					throw new Error(`Error: Got NaN when checking zapping distance.`);
 				
 				if(distance < max_distance_metres) {
-					// console.log(`Zap! (${distance})`);
+					// console.error(`Zap! (${distance})`);
 					items_to_zap.push(comp_item);
 				}
 			}