Implement initial false negative zapping system

...but it's not having the effect I'd hoped.

server/settings.default.toml

@@ -55,7 +55,19 @@ devices = [
 ]
 
 [ai]
-# Settings relating to the training of the AI. Note that a number of these settings can also be specified by environment variables, to aid with fiddling with the parameters to find the right settings.
+# Settings relating to the training of the AI.
+
+# TODO: Implement environment variable support
+# Note that a number of these settings can also be specified by environment variables, to aid with fiddling with the parameters to find the right settings.
+
+# The radius around a positive result to zap negative results.
+# This helps to remove false-nagative readings, to improve the AI's ability to 
+# accurately predict the actual signal strength.
+false_negative_zap_radius = 150
+
+# Whether to actually zap false negatives or not.
+# Useful when you want to... erm.... see why the zapping is necessary?
+do_zap_false_negatives = true
 
 # The architecture of the neural network, as an arary of integers.
 # Each integer represents the number of nodes in a layer of the neural network.

server/train-ai/AITrainer.mjs

@@ -19,7 +19,6 @@ class AITrainer {
 		let index = [];
 		for(let gateway of this.repo_gateway.iterate()) {
 			let filepath = path.join(this.root_dir, "..", this.settings.ai.output_directory, `${gateway.id}.json`);
-			console.log(filepath);
 			
 			if(!fs.existsSync(path.dirname(filepath)))
 				await fs.promises.mkdir(path.dirname(filepath), { recursive: true });
@@ -30,6 +29,9 @@ class AITrainer {
 				continue;
 			}
 			
+			
+			this.l.log(`Saved to ${filepath}`);
+			
 			index.push({
 				id: gateway.id,
 				filename: path.basename(filepath),
@@ -79,7 +81,7 @@ class AITrainer {
 			learningRate: this.settings.ai.learning_rate,
 			momentum: this.settings.ai.momentum,
 			
-			log: true,
+			log: (log_line) => this.l.log(`[brain.js] ${log_line}`),
 			logPeriod: 50,
 			
 			timeout: Infinity

server/train-ai/DatasetFetcher.mjs

@@ -12,8 +12,9 @@ import {
 } from '../../common/Normalisers.mjs';
 
 class DatasetFetcher {
-	constructor({ settings, GatewayRepo, RSSIRepo, ReadingRepo }) {
+	constructor({ settings, log, GatewayRepo, RSSIRepo, ReadingRepo }) {
 		this.settings = settings;
+		this.l = log;
 		this.repo_gateway = GatewayRepo;
 		this.repo_rssi = RSSIRepo;
 		this.repo_reading = ReadingRepo;
@@ -26,14 +27,12 @@ class DatasetFetcher {
 		for(let rssi of this.repo_rssi.iterate_gateway(gateway_id)) {
 			result.push({
 				input: {
-					latitude: normalise_lat(rssi.latitude),
+					latitude: rssi.latitude,
-					longitude: normalise_lng(rssi.longitude),
+					longitude: rssi.longitude,
-					distance: normalise_gateway_distance(
+					distance: haversine(gateway_location, rssi)
-						haversine(gateway_location, rssi)
-					)
 				},
 				output: [
-					normalise_rssi(rssi.rssi)
+					rssi.rssi
 				]
 			});
 		}
@@ -41,19 +40,70 @@ class DatasetFetcher {
 		for(let reading of this.repo_reading.iterate_unreceived()) {
 			result.push({
 				input: {
-					latitude: normalise_lat(reading.latitude),
+					latitude: reading.latitude,
-					longitude: normalise_lng(reading.longitude),
+					longitude: reading.longitude,
-					distance: normalise_gateway_distance(
+					distance: haversine(gateway_location, reading)
-						haversine(gateway_location, reading)
-					)
 				},
-				output: [ 0 ]
+				output: [ -150 ]
 			});
 		}
 		shuffle_fisher_yates(result);
 		
+		if(this.settings.ai.do_zap_false_negatives) {
+			let zap_count = this.zap_false_negatives(result, this.settings.ai.false_negative_zap_radius);
+			this.l.log(`[DatasetFetcher] Zapped ${zap_count} false negatives with a radius of ${this.settings.ai.false_negative_zap_radius}m.`);
+		}
+		
+		for(let item of result) {
+			item.input.latitude = normalise_lat(item.input.latitude);
+			item.input.longitude = normalise_lng(item.input.longitude);
+			item.input.distance = normalise_gateway_distance(item.input.distance);
+			item.output[0] = normalise_rssi(item.output[0]);
+		}
+		
 		return result;
 	}
+	
+	zap_false_negatives(readings_raw, max_distance_metres) {
+		let items_zapped = 0;
+		
+		for(let next_item of readings_raw) {
+			// Only zap for readings where we got a signal
+			if(next_item.output[0] <= -150) // -150: codename for no signal
+				continue;
+			
+			// console.log(next_item);
+			
+			// Store a list of items to zap, because changing the length of the 
+			// array while we're iterating it is a recipe for disaster
+			let items_to_zap = [];
+			
+			for(let comp_item of readings_raw) {
+				// Avoid zapping readings where we got a signal
+				if(comp_item.output[0] > -150)
+					continue;
+				
+				let distance = haversine(
+					next_item.input,
+					comp_item.input
+				);
+				if(isNaN(distance))
+					throw new Error(`Error: Got NaN when checking zapping distance.`);
+				
+				if(distance < max_distance_metres) {
+					// console.log(`Zap! (${distance})`);
+					items_to_zap.push(comp_item);
+				}
+			}
+			
+			items_zapped += items_to_zap.length;
+			for(let next_item of items_to_zap) {
+				readings_raw.splice(readings_raw.indexOf(next_item), 1);
+			}
+		}
+		
+		return items_zapped;
+	}
 }