From dea7940b2ef71ad77791b2ee54b112522851c7ce Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Tue, 30 Jul 2019 18:38:44 +0100 Subject: [PATCH] Bugfix: Fix client-side Brain.js crashes --- build | 12 ++++-- client_src/js/LayerAI.mjs | 6 +++ client_src/js/Worker/AI.worker.mjs | 1 + client_src/js/Worker/AIWrapper.mjs | 33 ++++++++------- package-lock.json | 43 ++++++++++++++++++++ package.json | 1 + rollup.main.config.js | 12 ++++++ rollup.config.js => rollup.plugins.config.js | 14 ++----- rollup.worker.config.js | 12 ++++++ server/train-ai/AITrainer.mjs | 28 ++++++------- 10 files changed, 119 insertions(+), 43 deletions(-) create mode 100644 rollup.main.config.js rename rollup.config.js => rollup.plugins.config.js (93%) create mode 100644 rollup.worker.config.js diff --git a/build b/build index 697f018..f0a983b 100755 --- a/build +++ b/build @@ -139,7 +139,8 @@ task_server-ttn() { # ██████ ███████ ██ ███████ ██ ████ ██ task_client() { task_begin "Packaging Javascript"; - execute node_modules/rollup/bin/rollup --sourcemap --config rollup.config.js; + execute node_modules/rollup/bin/rollup --sourcemap --config rollup.main.config.js; + execute node_modules/rollup/bin/rollup --sourcemap --config rollup.worker.config.js; task_end $? "Error: rollup packing failed!"; task_begin "Copying html"; @@ -150,17 +151,22 @@ task_client() { task_client-watch() { set_title "Client Watcher"; - execute node_modules/rollup/bin/rollup --watch --sourcemap --config rollup.config.js & + + # execute node_modules/rollup/bin/rollup --watch --sourcemap --config rollup.config.js & echo -e "Watching for changes."; while :; do # : = infinite loop # Wait for an update # inotifywait's non-0 exit code forces an exit for some reason :-/ - inotifywait -qr --event modify --format '%:e %f' client_src rollup.config.js; + inotifywait -qr --event modify --format '%:e %f' client_src rollup.*.config.js; task_begin "Copying html"; execute cp client_src/index.html "app/"; task_end $?; + + stage_begin "Rebuilding client code"; + set +e; tasks_run client; set -e; + stage_end $?; done } diff --git a/client_src/js/LayerAI.mjs b/client_src/js/LayerAI.mjs index b785fe8..b8327f1 100644 --- a/client_src/js/LayerAI.mjs +++ b/client_src/js/LayerAI.mjs @@ -1,5 +1,7 @@ "use strict"; +import path from 'path'; + import L from 'leaflet'; import chroma from 'chroma-js'; @@ -65,6 +67,10 @@ class LayerAI { ); console.log(this.index); + for(let gateway of this.index.index) { + gateway.frozen_net = JSON.parse(await GetFromUrl(`${window.location.href}/${path.dirname(Config.ai_index_file)}/${gateway.filename}`)) + } + // Figure out the bounds of the map we're going to generate this.map_bounds = this.gateway_bounds; this.map_bounds.north += Config.border.lat; diff --git a/client_src/js/Worker/AI.worker.mjs b/client_src/js/Worker/AI.worker.mjs index a7b0155..53b94df 100644 --- a/client_src/js/Worker/AI.worker.mjs +++ b/client_src/js/Worker/AI.worker.mjs @@ -32,5 +32,6 @@ async function handle_message(event) { self.addEventListener("message", (event) => { handle_message(event).catch((error) => { console.error(error); + throw error; }); }); diff --git a/client_src/js/Worker/AIWrapper.mjs b/client_src/js/Worker/AIWrapper.mjs index e35f98f..ad7dc57 100644 --- a/client_src/js/Worker/AIWrapper.mjs +++ b/client_src/js/Worker/AIWrapper.mjs @@ -1,6 +1,5 @@ "use strict"; -import path from 'path'; import brain from 'brain.js'; import haversine from 'haversine-distance'; @@ -13,7 +12,6 @@ import { unnormalise_rssi } from '../../../common/Normalisers.mjs'; -import GetFromUrl from '../Helpers/GetFromUrl.mjs'; class AIWrapper { constructor() { @@ -36,15 +34,14 @@ class AIWrapper { // WebGL isn't available inside WebWorkers yet :-( for(let gateway of this.index.index) { - let net = new brain.NeuralNetwork(); + let net = new brain.NeuralNetwork(/*gateway.net_settings*/); net.fromJSON( - // TODO: Move this to the UI thread & do it only once? - await GetFromUrl(`${path.dirname(self.location.href)}/${path.dirname(this.Config.ai_index_file)}/${gateway.filename}`) + gateway.frozen_net ); this.gateways.set( gateway.id, - net + { ai: net, latitude: gateway.latitude, longitude: gateway.longitude } ); } console.log("Model setup complete."); @@ -64,21 +61,27 @@ class AIWrapper { for(let lng = this.map_bounds.west; lng < this.map_bounds.east; lng += this.Config.step.lng) { let max_predicted_rssi = -Infinity; - for(let [gateway_id, ai] of this.gateways) { + for(let [gateway_id, gateway] of this.gateways) { let distance_from_gateway = haversine( { latitude: lat, longitude: lng }, - this.gateways.get(gateway_id) + gateway ); + let next_value = gateway.ai.run({ + latitude: normalise_lat(lat), + longitude: normalise_lng(lng), + distance: normalise_gateway_distance( + distance_from_gateway + ), + }); + if(isNaN(next_value[0])) { + console.log(next_value); + throw new Error("Error: Neural Network returned NaN"); + } + // console.log(next_value); max_predicted_rssi = Math.max( max_predicted_rssi, - ai.run({ - latitude: normalise_lat(lat), - longitude: normalise_lng(lng), - distance: normalise_gateway_distance( - distance_from_gateway - ), - })[0] + next_value[0] ); } diff --git a/package-lock.json b/package-lock.json index c36bc9e..a535ac9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3712,6 +3712,43 @@ "rollup-pluginutils": "^2.8.1" } }, + "rollup-plugin-node-globals": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/rollup-plugin-node-globals/-/rollup-plugin-node-globals-1.4.0.tgz", + "integrity": "sha512-xRkB+W/m1KLIzPUmG0ofvR+CPNcvuCuNdjVBVS7ALKSxr3EDhnzNceGkGi1m8MToSli13AzKFYH4ie9w3I5L3g==", + "dev": true, + "requires": { + "acorn": "^5.7.3", + "buffer-es6": "^4.9.3", + "estree-walker": "^0.5.2", + "magic-string": "^0.22.5", + "process-es6": "^0.11.6", + "rollup-pluginutils": "^2.3.1" + }, + "dependencies": { + "acorn": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz", + "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==", + "dev": true + }, + "estree-walker": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.5.2.tgz", + "integrity": "sha512-XpCnW/AE10ws/kDAs37cngSkvgIR8aN3G0MS85m7dUpuK2EREo9VJ00uvw6Dg/hXEpfsE1I1TvJOJr+Z+TL+ig==", + "dev": true + }, + "magic-string": { + "version": "0.22.5", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", + "integrity": "sha512-oreip9rJZkzvA8Qzk9HFs8fZGF/u7H/gtrE8EN6RjKJ9kh2HlC+yQ2QezifqTZfGyiuAV0dRv5a+y/8gBb1m9w==", + "dev": true, + "requires": { + "vlq": "^0.2.2" + } + } + } + }, "rollup-plugin-node-resolve": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/rollup-plugin-node-resolve/-/rollup-plugin-node-resolve-5.2.0.tgz", @@ -4444,6 +4481,12 @@ "integrity": "sha512-fOi47nsJP5Wqefa43kyWSg80qF+Q3XA6MUkgi7Hp1HQaKDQW4cQrK2D0P7mmbFtsV1N89am55Yru/nyEwRubcw==", "dev": true }, + "vlq": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/vlq/-/vlq-0.2.3.tgz", + "integrity": "sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow==", + "dev": true + }, "websocket-stream": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/websocket-stream/-/websocket-stream-5.5.0.tgz", diff --git a/package.json b/package.json index 35be201..ac216ea 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "@types/leaflet": "^1.4.6", "rollup": "^1.17.0", "rollup-plugin-commonjs": "^10.0.1", + "rollup-plugin-node-globals": "^1.4.0", "rollup-plugin-node-resolve": "^5.2.0", "rollup-plugin-postcss": "^2.0.3", "rollup-plugin-replace": "^2.2.0", diff --git a/rollup.main.config.js b/rollup.main.config.js new file mode 100644 index 0000000..0afd073 --- /dev/null +++ b/rollup.main.config.js @@ -0,0 +1,12 @@ +import plugins from './rollup.plugins.config.js'; + +export default { + input: { + "app": 'client_src/js/index.mjs', + }, + output: { + dir: 'app/', + format: 'esm' + }, + plugins +}; diff --git a/rollup.config.js b/rollup.plugins.config.js similarity index 93% rename from rollup.config.js rename to rollup.plugins.config.js index 2bbf945..96b5e66 100644 --- a/rollup.config.js +++ b/rollup.plugins.config.js @@ -8,6 +8,7 @@ import postcss from 'rollup-plugin-postcss'; import { terser } from "rollup-plugin-terser"; import replace from 'rollup-plugin-replace'; import builtins from '@joseph184/rollup-plugin-node-builtins'; +import globals from 'rollup-plugin-node-globals'; // import json from 'rollup-plugin-json'; import postcss_import from 'postcss-import'; @@ -15,6 +16,7 @@ import postcss_copy from 'postcss-copy'; let plugins = [ builtins(), + globals(), resolve({ mainFields: [ @@ -87,14 +89,4 @@ if(process.env.NODE_ENV == "production") { })); } -export default { - input: { - "app": 'client_src/js/index.mjs', - "worker": 'client_src/js/Worker/AI.worker.mjs' - }, - output: { - dir: 'app/', - format: 'esm' - }, - plugins -}; +export default plugins; diff --git a/rollup.worker.config.js b/rollup.worker.config.js new file mode 100644 index 0000000..7222695 --- /dev/null +++ b/rollup.worker.config.js @@ -0,0 +1,12 @@ +import plugins from './rollup.plugins.config.js'; + +export default { + input: { + "worker": 'client_src/js/Worker/AI.worker.mjs' + }, + output: { + dir: 'app/', + format: 'esm' + }, + plugins +}; diff --git a/server/train-ai/AITrainer.mjs b/server/train-ai/AITrainer.mjs index f0c95fb..d6cb9c7 100644 --- a/server/train-ai/AITrainer.mjs +++ b/server/train-ai/AITrainer.mjs @@ -15,16 +15,6 @@ class AITrainer { this.repo_gateway = GatewayRepo; } - generate_neural_net() { - let net = new brain.NeuralNetwork({ - hiddenLayers: this.settings.ai.network_arch, - activation: "sigmoid" - }); - - - return net; - } - async train_all() { let index = []; for(let gateway of this.repo_gateway.iterate()) { @@ -34,7 +24,8 @@ class AITrainer { if(!fs.existsSync(path.dirname(filepath))) await fs.promises.mkdir(path.dirname(filepath), { recursive: true }); - if(!await this.train_gateway(gateway.id, filepath)) { + let result = await this.train_gateway(gateway.id, filepath); + if(!result || result.success === false) { this.l.warn(`Warning: Failed to train AI for ${gateway.id}.`); continue; } @@ -43,7 +34,8 @@ class AITrainer { id: gateway.id, filename: path.basename(filepath), latitude: gateway.latitude, - longitude: gateway.longitude + longitude: gateway.longitude, + net_settings: result.net_settings }); } @@ -71,7 +63,12 @@ class AITrainer { */ async train_gateway(gateway_id, destination_filename) { this.l.log(`${this.a.fgreen}${this.a.hicol}Training AI for gateway ${gateway_id}${this.a.reset}`); - let net = this.generate_neural_net(); + + let net_settings = { + hiddenLayers: this.settings.ai.network_arch, + activation: "sigmoid" + }; + let net = new brain.NeuralNetwork(net_settings); let dataset = this.dataset_fetcher.fetch_all(gateway_id); @@ -91,7 +88,10 @@ class AITrainer { await fs.promises.writeFile(destination_filename, JSON.stringify(net.toJSON())); // console.log(result); - return true; + return { + success: true, + net_settings + }; } }