"use strict"; import haversine from 'haversine-distance'; import shuffle_fisher_yates from '../Helpers/FisherYates.mjs'; import { normalise_lat, normalise_lng, normalise_rssi, normalise_gateway_distance, } from '../../common/Normalisers.mjs'; class DatasetFetcher { constructor({ settings, log, GatewayRepo, RSSIRepo, ReadingRepo }) { this.settings = settings; this.l = log; this.repo_gateway = GatewayRepo; this.repo_rssi = RSSIRepo; this.repo_reading = ReadingRepo; } fetch_all(gateway_id, extended = false) { let result = []; // Determine the location of the gateway let gateway_location = this.repo_gateway.get_by_id(gateway_id); // Grab an iterator for the data we want to add let iterator = gateway_id == null ? this.repo_rssi.iterate() : this.repo_rssi.iterate_gateway(gateway_id); // Add the readings where we did get a signal for(let rssi of iterator) { let item = { input: { latitude: rssi.latitude, longitude: rssi.longitude }, output: [ rssi.rssi ] }; if(gateway_id !== null) item.input.distance = haversine(gateway_location, rssi); if(extended) { item.ext = { gateway: rssi.gateway_id, rssi_raw: rssi.rssi }; } result.push(item); } // Add the readings where we did not get a signal for(let reading of this.repo_reading.iterate_unreceived()) { let item = { input: { latitude: reading.latitude, longitude: reading.longitude }, output: [ -150 ] }; if(gateway_id !== null) item.input.distance = haversine(gateway_location, reading); if(extended) { item.ext = { gateway: "(none)", rssi_raw: -150 }; } result.push(item); } // Zap the false negatives, but only if we're told to // False neegatives are readings with not signal that are right next to // a reading with a signal, within a configurable radius. if(this.settings.ai.do_zap_false_negatives) { let zap_count_before = result.length, zap_count = this.zap_false_negatives( result, this.settings.ai.false_negative_zap_radius ), zap_count_after = result.length; this.l.log_e(`[DatasetFetcher] Zapped ${zap_count} false negatives with a radius of ${this.settings.ai.false_negative_zap_radius}m (${zap_count_before} -> ${zap_count_after} points).`); } // Normalise all the values for(let item of result) { item.input.latitude = normalise_lat(item.input.latitude); item.input.longitude = normalise_lng(item.input.longitude); if(typeof item.input.distance == "number") item.input.distance = normalise_gateway_distance(item.input.distance); item.output[0] = normalise_rssi(item.output[0]); } // Scan the resulting dataset for invalid items this.scan_for_corruption(result); // Shuffle the dataset shuffle_fisher_yates(result); return result; } zap_false_negatives(readings_raw, max_distance_metres) { let items_zapped = 0; for(let next_item of readings_raw) { // Only zap for readings where we got a signal if(next_item.output[0] <= -150) // -150: codename for no signal continue; // console.log(next_item); // Store a list of items to zap, because changing the length of the // array while we're iterating it is a recipe for disaster let items_to_zap = []; for(let comp_item of readings_raw) { // Avoid zapping readings where we got a signal if(comp_item.output[0] > -150) continue; let distance = haversine( next_item.input, comp_item.input ); if(Number.isNaN(distance)) throw new Error(`Error: Got NaN when checking zapping distance.`); if(distance < max_distance_metres) { // console.error(`Zap! (${distance})`); items_to_zap.push(comp_item); } } items_zapped += items_to_zap.length; for(let next_item of items_to_zap) { readings_raw.splice(readings_raw.indexOf(next_item), 1); } } return items_zapped; } scan_for_corruption(dataset) { // Scan the input data to make sure it is't corrupt for(let row of dataset) { if(Number.isNaN(row.output[0]) || Number.isNaN(row.input.latitude) || Number.isNaN(row.input.longitude) || Number.isNaN(row.input.distance)) { console.error(row); throw new Error("Error: Found invalid value in input data"); } } this.l.log_e(`Scanned ${dataset.length} rows of data for invalid values.`); } } export default DatasetFetcher;