sbrl
/
LoRaWAN-Signal-Mapping
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Scan the input data for invalid values

This commit is contained in:
Starbeamrainbowlabs 2019-08-06 12:10:47 +01:00
parent 25a817331f
commit 082eba42ec
2 changed files with 35 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/train-ai/AITrainer.mjs
View File

@ -100,12 +100,14 @@ class AITrainer {
async train_gateway(gateway_id, destination_filename) {
this.l.log(`${this.a.fgreen}${this.a.hicol}Training AI for gateway ${gateway_id}${this.a.reset}`);
// Create the neural network
let net_settings = {
hiddenLayers: this.settings.ai.network_arch,
activation: "sigmoid"
};
let net = new brain.NeuralNetwork(net_settings);
// Fetch the dataset
let dataset = this.dataset_fetcher.fetch_all(gateway_id);
await net.trainAsync(dataset, {

37
server/train-ai/DatasetFetcher.mjs
View File

@ -21,11 +21,14 @@ class DatasetFetcher {
}
fetch_all(gateway_id, extended = false) {
let gateway_location = this.repo_gateway.get_by_id(gateway_id);
let iterator = gateway_id == null ? this.repo_rssi.iterate() : this.repo_rssi.iterate_gateway(gateway_id);
let result = [];
// Determine the location of the gateway
let gateway_location = this.repo_gateway.get_by_id(gateway_id);
// Grab an iterator for the data we want to add
let iterator = gateway_id == null ? this.repo_rssi.iterate() : this.repo_rssi.iterate_gateway(gateway_id);
// Add the readings where we did get a signal
for(let rssi of iterator) {
if(gateway_id == null)
gateway_location = this.repo_gateway.get_by_id(rssi.gateway_id);
@ -49,6 +52,7 @@ class DatasetFetcher {
result.push(item);
}
// Add the readings where we did not get a signal
for(let reading of this.repo_reading.iterate_unreceived()) {
let item = {
input: {
@ -66,8 +70,10 @@ class DatasetFetcher {
}
result.push(item);
}
shuffle_fisher_yates(result);
// Zap the false negatives, but only if we're told to
// False neegatives are readings with not signal that are right next to
// a reading with a signal, within a configurable radius.
if(this.settings.ai.do_zap_false_negatives) {
let zap_count_before = result.length,
zap_count = this.zap_false_negatives(
@ -79,6 +85,7 @@ class DatasetFetcher {
this.l.log_e(`[DatasetFetcher] Zapped ${zap_count} false negatives with a radius of ${this.settings.ai.false_negative_zap_radius}m (${zap_count_before} -> ${zap_count_after} points).`);
}
// Normalise all the values
for(let item of result) {
item.input.latitude = normalise_lat(item.input.latitude);
item.input.longitude = normalise_lng(item.input.longitude);
@ -86,6 +93,12 @@ class DatasetFetcher {
item.output[0] = normalise_rssi(item.output[0]);
}
// Shuffle the dataset
shuffle_fisher_yates(result);
// Scan the resulting dataset for invalid items
this.scan_for_corruption(result);
return result;
}
@ -129,6 +142,22 @@ class DatasetFetcher {
return items_zapped;
}
scan_for_corruption(dataset) {
// Scan the input data to make sure it is't corrupt
for(let row of dataset) {
if(isNaN(row.output[0]) || isNaN(row.input.latitude) || isNaN(row.input.longitude) || isNaN(row.input.distance)) {
console.error(row);
throw new Error("Error: Found NaN in input data");
}
if(typeof row.output[0] !== "number" || typeof row.input.latitude !== "number" || typeof row.input.longitude !== "number" || typeof row.input.distance !== "number") {
console.error(row);
throw new Error("Error: Found item with an invalid type in the input data");
}
}
this.l.log_e(`Scanned ${dataset.length} rows of data for invalid values.`);
}
}