Implement initial false negative zapping system

...but it's not having the effect I'd hoped.
This commit is contained in:
Starbeamrainbowlabs 2019-08-02 13:17:17 +01:00
parent fad188b6d1
commit 6844cdc545
3 changed files with 80 additions and 16 deletions

View file

@ -55,7 +55,19 @@ devices = [
] ]
[ai] [ai]
# Settings relating to the training of the AI. Note that a number of these settings can also be specified by environment variables, to aid with fiddling with the parameters to find the right settings. # Settings relating to the training of the AI.
# TODO: Implement environment variable support
# Note that a number of these settings can also be specified by environment variables, to aid with fiddling with the parameters to find the right settings.
# The radius around a positive result to zap negative results.
# This helps to remove false-nagative readings, to improve the AI's ability to
# accurately predict the actual signal strength.
false_negative_zap_radius = 150
# Whether to actually zap false negatives or not.
# Useful when you want to... erm.... see why the zapping is necessary?
do_zap_false_negatives = true
# The architecture of the neural network, as an arary of integers. # The architecture of the neural network, as an arary of integers.
# Each integer represents the number of nodes in a layer of the neural network. # Each integer represents the number of nodes in a layer of the neural network.

View file

@ -19,7 +19,6 @@ class AITrainer {
let index = []; let index = [];
for(let gateway of this.repo_gateway.iterate()) { for(let gateway of this.repo_gateway.iterate()) {
let filepath = path.join(this.root_dir, "..", this.settings.ai.output_directory, `${gateway.id}.json`); let filepath = path.join(this.root_dir, "..", this.settings.ai.output_directory, `${gateway.id}.json`);
console.log(filepath);
if(!fs.existsSync(path.dirname(filepath))) if(!fs.existsSync(path.dirname(filepath)))
await fs.promises.mkdir(path.dirname(filepath), { recursive: true }); await fs.promises.mkdir(path.dirname(filepath), { recursive: true });
@ -30,6 +29,9 @@ class AITrainer {
continue; continue;
} }
this.l.log(`Saved to ${filepath}`);
index.push({ index.push({
id: gateway.id, id: gateway.id,
filename: path.basename(filepath), filename: path.basename(filepath),
@ -79,7 +81,7 @@ class AITrainer {
learningRate: this.settings.ai.learning_rate, learningRate: this.settings.ai.learning_rate,
momentum: this.settings.ai.momentum, momentum: this.settings.ai.momentum,
log: true, log: (log_line) => this.l.log(`[brain.js] ${log_line}`),
logPeriod: 50, logPeriod: 50,
timeout: Infinity timeout: Infinity

View file

@ -12,8 +12,9 @@ import {
} from '../../common/Normalisers.mjs'; } from '../../common/Normalisers.mjs';
class DatasetFetcher { class DatasetFetcher {
constructor({ settings, GatewayRepo, RSSIRepo, ReadingRepo }) { constructor({ settings, log, GatewayRepo, RSSIRepo, ReadingRepo }) {
this.settings = settings; this.settings = settings;
this.l = log;
this.repo_gateway = GatewayRepo; this.repo_gateway = GatewayRepo;
this.repo_rssi = RSSIRepo; this.repo_rssi = RSSIRepo;
this.repo_reading = ReadingRepo; this.repo_reading = ReadingRepo;
@ -26,14 +27,12 @@ class DatasetFetcher {
for(let rssi of this.repo_rssi.iterate_gateway(gateway_id)) { for(let rssi of this.repo_rssi.iterate_gateway(gateway_id)) {
result.push({ result.push({
input: { input: {
latitude: normalise_lat(rssi.latitude), latitude: rssi.latitude,
longitude: normalise_lng(rssi.longitude), longitude: rssi.longitude,
distance: normalise_gateway_distance( distance: haversine(gateway_location, rssi)
haversine(gateway_location, rssi)
)
}, },
output: [ output: [
normalise_rssi(rssi.rssi) rssi.rssi
] ]
}); });
} }
@ -41,19 +40,70 @@ class DatasetFetcher {
for(let reading of this.repo_reading.iterate_unreceived()) { for(let reading of this.repo_reading.iterate_unreceived()) {
result.push({ result.push({
input: { input: {
latitude: normalise_lat(reading.latitude), latitude: reading.latitude,
longitude: normalise_lng(reading.longitude), longitude: reading.longitude,
distance: normalise_gateway_distance( distance: haversine(gateway_location, reading)
haversine(gateway_location, reading)
)
}, },
output: [ 0 ] output: [ -150 ]
}); });
} }
shuffle_fisher_yates(result); shuffle_fisher_yates(result);
if(this.settings.ai.do_zap_false_negatives) {
let zap_count = this.zap_false_negatives(result, this.settings.ai.false_negative_zap_radius);
this.l.log(`[DatasetFetcher] Zapped ${zap_count} false negatives with a radius of ${this.settings.ai.false_negative_zap_radius}m.`);
}
for(let item of result) {
item.input.latitude = normalise_lat(item.input.latitude);
item.input.longitude = normalise_lng(item.input.longitude);
item.input.distance = normalise_gateway_distance(item.input.distance);
item.output[0] = normalise_rssi(item.output[0]);
}
return result; return result;
} }
zap_false_negatives(readings_raw, max_distance_metres) {
let items_zapped = 0;
for(let next_item of readings_raw) {
// Only zap for readings where we got a signal
if(next_item.output[0] <= -150) // -150: codename for no signal
continue;
// console.log(next_item);
// Store a list of items to zap, because changing the length of the
// array while we're iterating it is a recipe for disaster
let items_to_zap = [];
for(let comp_item of readings_raw) {
// Avoid zapping readings where we got a signal
if(comp_item.output[0] > -150)
continue;
let distance = haversine(
next_item.input,
comp_item.input
);
if(isNaN(distance))
throw new Error(`Error: Got NaN when checking zapping distance.`);
if(distance < max_distance_metres) {
// console.log(`Zap! (${distance})`);
items_to_zap.push(comp_item);
}
}
items_zapped += items_to_zap.length;
for(let next_item of items_to_zap) {
readings_raw.splice(readings_raw.indexOf(next_item), 1);
}
}
return items_zapped;
}
} }