LoRaWAN-Signal-Mapping/server/train-ai/DatasetFetcher.mjs

165 lines
4.6 KiB
JavaScript
Raw Normal View History

"use strict";
import haversine from 'haversine-distance';
import shuffle_fisher_yates from '../Helpers/FisherYates.mjs';
2019-07-30 14:42:37 +00:00
import {
normalise_lat,
normalise_lng,
normalise_rssi,
normalise_gateway_distance,
} from '../../common/Normalisers.mjs';
class DatasetFetcher {
constructor({ settings, log, GatewayRepo, RSSIRepo, ReadingRepo }) {
this.settings = settings;
this.l = log;
this.repo_gateway = GatewayRepo;
this.repo_rssi = RSSIRepo;
this.repo_reading = ReadingRepo;
}
fetch_all(gateway_id, extended = false) {
2019-08-06 11:10:47 +00:00
let result = [];
// Determine the location of the gateway
let gateway_location = this.repo_gateway.get_by_id(gateway_id);
2019-08-06 11:10:47 +00:00
// Grab an iterator for the data we want to add
let iterator = gateway_id == null ? this.repo_rssi.iterate() : this.repo_rssi.iterate_gateway(gateway_id);
2019-08-06 11:10:47 +00:00
// Add the readings where we did get a signal
for(let rssi of iterator) {
if(gateway_id == null)
gateway_location = this.repo_gateway.get_by_id(rssi.gateway_id);
let item = {
2019-07-30 14:42:37 +00:00
input: {
latitude: rssi.latitude,
longitude: rssi.longitude,
distance: haversine(gateway_location, rssi)
2019-07-30 14:42:37 +00:00
},
output: [
rssi.rssi
2019-07-30 14:42:37 +00:00
]
};
if(extended) {
item.ext = {
gateway: rssi.gateway_id,
rssi_raw: rssi.rssi
};
}
result.push(item);
}
2019-08-06 11:10:47 +00:00
// Add the readings where we did not get a signal
for(let reading of this.repo_reading.iterate_unreceived()) {
let item = {
2019-07-30 14:42:37 +00:00
input: {
latitude: reading.latitude,
longitude: reading.longitude,
distance: haversine(gateway_location, reading)
2019-07-30 14:42:37 +00:00
},
output: [ -150 ]
};
if(extended) {
item.ext = {
gateway: "(none)",
rssi_raw: -150
};
}
result.push(item);
}
2019-08-06 11:10:47 +00:00
// Zap the false negatives, but only if we're told to
// False neegatives are readings with not signal that are right next to
// a reading with a signal, within a configurable radius.
if(this.settings.ai.do_zap_false_negatives) {
2019-08-02 13:04:34 +00:00
let zap_count_before = result.length,
zap_count = this.zap_false_negatives(
result,
this.settings.ai.false_negative_zap_radius
),
zap_count_after = result.length;
this.l.log_e(`[DatasetFetcher] Zapped ${zap_count} false negatives with a radius of ${this.settings.ai.false_negative_zap_radius}m (${zap_count_before} -> ${zap_count_after} points).`);
}
2019-08-06 11:10:47 +00:00
// Normalise all the values
for(let item of result) {
item.input.latitude = normalise_lat(item.input.latitude);
item.input.longitude = normalise_lng(item.input.longitude);
item.input.distance = normalise_gateway_distance(item.input.distance);
item.output[0] = normalise_rssi(item.output[0]);
}
2019-08-06 11:10:47 +00:00
// Shuffle the dataset
shuffle_fisher_yates(result);
// Scan the resulting dataset for invalid items
this.scan_for_corruption(result);
return result;
}
zap_false_negatives(readings_raw, max_distance_metres) {
let items_zapped = 0;
for(let next_item of readings_raw) {
// Only zap for readings where we got a signal
if(next_item.output[0] <= -150) // -150: codename for no signal
continue;
// console.log(next_item);
// Store a list of items to zap, because changing the length of the
// array while we're iterating it is a recipe for disaster
let items_to_zap = [];
for(let comp_item of readings_raw) {
// Avoid zapping readings where we got a signal
if(comp_item.output[0] > -150)
continue;
let distance = haversine(
next_item.input,
comp_item.input
);
if(isNaN(distance))
throw new Error(`Error: Got NaN when checking zapping distance.`);
if(distance < max_distance_metres) {
// console.error(`Zap! (${distance})`);
items_to_zap.push(comp_item);
}
}
items_zapped += items_to_zap.length;
for(let next_item of items_to_zap) {
readings_raw.splice(readings_raw.indexOf(next_item), 1);
}
}
return items_zapped;
}
2019-08-06 11:10:47 +00:00
scan_for_corruption(dataset) {
// Scan the input data to make sure it is't corrupt
for(let row of dataset) {
if(isNaN(row.output[0]) || isNaN(row.input.latitude) || isNaN(row.input.longitude) || isNaN(row.input.distance)) {
console.error(row);
throw new Error("Error: Found NaN in input data");
}
if(typeof row.output[0] !== "number" || typeof row.input.latitude !== "number" || typeof row.input.longitude !== "number" || typeof row.input.distance !== "number") {
console.error(row);
throw new Error("Error: Found item with an invalid type in the input data");
}
}
this.l.log_e(`Scanned ${dataset.length} rows of data for invalid values.`);
}
}
export default DatasetFetcher;