LoRaWAN-Signal-Mapping/server/train-ai/DatasetFetcher.mjs

"use strict";

import haversine from 'haversine-distance';

import shuffle_fisher_yates from '../Helpers/FisherYates.mjs';

import {
	normalise_lat,
	normalise_lng,
	normalise_rssi,
	normalise_gateway_distance,
} from '../../common/Normalisers.mjs';

class DatasetFetcher {
	constructor({ settings, log, GatewayRepo, RSSIRepo, ReadingRepo }) {
		this.settings = settings;
		this.l = log;
		this.repo_gateway = GatewayRepo;
		this.repo_rssi = RSSIRepo;
		this.repo_reading = ReadingRepo;
	}
	
	fetch_all(gateway_id, extended = false) {
		let result = [];
		
		// Determine the location of the gateway
		let gateway_location = this.repo_gateway.get_by_id(gateway_id);
		
		// Grab an iterator for the data we want to add
		let iterator = gateway_id == null ? this.repo_rssi.iterate() : this.repo_rssi.iterate_gateway(gateway_id);
		// Add the readings where we did get a signal
		for(let rssi of iterator) {
			if(gateway_id == null)
				gateway_location = this.repo_gateway.get_by_id(rssi.gateway_id);
			
			let item = {
				input: {
					latitude: rssi.latitude,
					longitude: rssi.longitude,
					distance: haversine(gateway_location, rssi)
				},
				output: [
					rssi.rssi
				]
			};
			if(extended) {
				item.ext = {
					gateway: rssi.gateway_id,
					rssi_raw: rssi.rssi
				};
			}
			result.push(item);
		}
		
		//  Add the readings where we did not get a signal
		for(let reading of this.repo_reading.iterate_unreceived()) {
			let item = {
				input: {
					latitude: reading.latitude,
					longitude: reading.longitude,
					distance: haversine(gateway_location, reading)
				},
				output: [ -150 ]
			};
			if(extended) {
				item.ext = {
					gateway: "(none)",
					rssi_raw: -150
				};
			}
			result.push(item);
		}
		
		// Zap the false negatives, but only if we're told to
		// False neegatives are readings with not signal that are right next to 
		// a reading with a signal, within a configurable radius.
		if(this.settings.ai.do_zap_false_negatives) {
			let zap_count_before = result.length,
				zap_count = this.zap_false_negatives(
					result,
					this.settings.ai.false_negative_zap_radius
				),
				zap_count_after = result.length;
			
			this.l.log_e(`[DatasetFetcher] Zapped ${zap_count} false negatives with a radius of ${this.settings.ai.false_negative_zap_radius}m (${zap_count_before} -> ${zap_count_after} points).`);
		}
		
		// Normalise all the values
		for(let item of result) {
			item.input.latitude = normalise_lat(item.input.latitude);
			item.input.longitude = normalise_lng(item.input.longitude);
			item.input.distance = normalise_gateway_distance(item.input.distance);
			item.output[0] = normalise_rssi(item.output[0]);
		}
		
		// Shuffle the dataset
		shuffle_fisher_yates(result);
		
		// Scan the resulting dataset for invalid items
		this.scan_for_corruption(result);
		
		return result;
	}
	
	zap_false_negatives(readings_raw, max_distance_metres) {
		let items_zapped = 0;
		
		for(let next_item of readings_raw) {
			// Only zap for readings where we got a signal
			if(next_item.output[0] <= -150) // -150: codename for no signal
				continue;
			
			// console.log(next_item);
			
			// Store a list of items to zap, because changing the length of the 
			// array while we're iterating it is a recipe for disaster
			let items_to_zap = [];
			
			for(let comp_item of readings_raw) {
				// Avoid zapping readings where we got a signal
				if(comp_item.output[0] > -150)
					continue;
				
				let distance = haversine(
					next_item.input,
					comp_item.input
				);
				if(isNaN(distance))
					throw new Error(`Error: Got NaN when checking zapping distance.`);
				
				if(distance < max_distance_metres) {
					// console.error(`Zap! (${distance})`);
					items_to_zap.push(comp_item);
				}
			}
			
			items_zapped += items_to_zap.length;
			for(let next_item of items_to_zap) {
				readings_raw.splice(readings_raw.indexOf(next_item), 1);
			}
		}
		
		return items_zapped;
	}
	
	scan_for_corruption(dataset) {
		// Scan the input data to make sure it is't corrupt
		for(let row of dataset) {
			if(isNaN(row.output[0]) || isNaN(row.input.latitude) || isNaN(row.input.longitude) || isNaN(row.input.distance)) {
				console.error(row);
				throw new Error("Error: Found NaN in input data");
			}
			if(typeof row.output[0] !== "number" || typeof row.input.latitude !== "number" || typeof row.input.longitude !== "number" || typeof row.input.distance !== "number") {
				console.error(row);
				throw new Error("Error: Found item with an invalid type in the input data");
			}
		}
		
		this.l.log_e(`Scanned ${dataset.length} rows of data for invalid values.`);
	}
}


export default DatasetFetcher;