diff --git a/aimodel/src/rainfallwater_identity_TEST.ipynb b/aimodel/src/rainfallwater_identity_TEST.ipynb new file mode 100644 index 0000000..11840b9 --- /dev/null +++ b/aimodel/src/rainfallwater_identity_TEST.ipynb @@ -0,0 +1,191 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "1f6fdebf-69c5-46ab-a5a8-f9c91f000ff3", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-12-09 15:49:31.909879: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n", + "2022-12-09 15:49:31.909893: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "import tensorflow as tf\n", + "\n", + "from lib.dataset.parse_heightmap import parse_heightmap\n", + "from lib.ai.model_rainfallwater_mono import model_rainfallwater_mono\n", + "from lib.ai.helpers.make_callbacks import make_callbacks\n", + "from lib.ai.helpers.summarywriter import summarywriter" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "07093079", + "metadata": {}, + "outputs": [], + "source": [ + "filepath_heightmap=\"/mnt/research-data/main/terrain50-nimrodsized.json.gz\"\n", + "\n", + "dir_output = \"/tmp/x/mono_segment_TEST\"\n", + "if not os.path.exists(os.path.join(dir_output, \"checkpoints\")):\n", + "\tos.makedirs(os.path.join(dir_output, \"checkpoints\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f4466ac9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "RAINFALL channels 1 width 64 height 64 HEIGHTMAP_INPUT False\n", + "convnext:shape IN x (None, 64, 64, 1)\n", + "DEBUG:convnext shape x (None, 64, 64, 1)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-12-09 15:49:34.026632: E tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW\n", + "2022-12-09 15:49:34.026670: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:169] retrieving CUDA diagnostic information for host: SIMVIS-CO45428A\n", + "2022-12-09 15:49:34.026678: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:176] hostname: SIMVIS-CO45428A\n", + "2022-12-09 15:49:34.026772: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:200] libcuda reported version is: 510.108.3\n", + "2022-12-09 15:49:34.026795: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:204] kernel reported version is: 510.85.2\n", + "2022-12-09 15:49:34.026801: E tensorflow/stream_executor/cuda/cuda_diagnostics.cc:313] kernel version 510.85.2 does not match DSO version 510.108.3 -- cannot find working devices in this configuration\n", + "2022-12-09 15:49:34.027072: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\n", + "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DEBUG:model ENCODER output_shape (None, 512)\n", + "DEBUG:model BOTTLENECK:stack2image output_shape (None, 4, 4, 512)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-12-09 15:49:37.559 | WARNING | lib.ai.model_rainfallwater_mono:model_rainfallwater_mono:70 - Warning: TODO implement attention from https://ieeexplore.ieee.org/document/9076883\n", + "2022-12-09 15:49:37.613 | INFO | lib.ai.model_rainfallwater_mono:model_rainfallwater_mono:80 - learning_rate: None\n" + ] + } + ], + "source": [ + "model = model_rainfallwater_mono(\n", + "\tmetadata={ \"rainfallradar\": [ 1, 64, 64 ] },\n", + "\tmodel_arch_dec=\"convnext_i_xxtiny\"\n", + ")\n", + "\n", + "summarywriter(model, filepath_output=os.path.join(dir_output, \"summary.txt\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "78c633e1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cells 4096 cells/2 2048.0 shape+ (64, 64, 2) tf.Tensor(2401, shape=(), dtype=int64)\n" + ] + } + ], + "source": [ + "heightmap = parse_heightmap(filepath_heightmap) / 100\n", + "heightmap = tf.image.crop_to_bounding_box(tf.expand_dims(heightmap, axis=-1), 0, 0, 64, 64)\n", + "heightmap_labels = tf.one_hot(tf.cast(tf.math.greater(tf.squeeze(heightmap)/10, 0.05), dtype=tf.int32), 2)\n", + "\n", + "dataset = tf.data.Dataset.from_tensor_slices([heightmap])\n", + "dataset_labels = tf.data.Dataset.from_tensor_slices([heightmap_labels])\n", + "\n", + "for item in dataset_labels:\n", + "\tprint(\"cells\", 64*64, \"cells/2\", (64*64)/2, \"shape+\", item.shape, tf.math.reduce_sum(tf.math.argmax(item, axis=-1)))\n", + "\tbreak\n", + "dataset = tf.data.Dataset.zip((dataset, dataset_labels)) \\\n", + "\t.repeat(64 * 64) \\\n", + "\t.batch(64) \\\n", + "\t.prefetch(tf.data.AUTOTUNE)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3dbc95eb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/50\n" + ] + } + ], + "source": [ + "model.fit(\n", + "\tdataset,\n", + "\tepochs=50,\n", + "\tcallbacks=make_callbacks(\"/tmp/x/mono_segment_TEST\", model)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7f8c33f", + "metadata": {}, + "outputs": [], + "source": [ + "prediction = model.predict(tf.expand_dims(heightmap, axis=0))\n", + "print(tf.math.argmax(prediction, axis=-1))\n", + "print(tf.math.reduce_sum(tf.math.argmax(prediction, axis=-1)))\n", + "print(prediction)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + }, + "vscode": { + "interpreter": { + "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}