mirror of
https://github.com/sbrl/research-rainfallradar
synced 2024-11-22 09:13:01 +00:00
jupyter: convnext-mono identity test
This commit is contained in:
parent
2a1772a211
commit
cf9e8aa237
GPG key ID:
1BE5172E637709C2
1 changed files with 191 additions and 0 deletions
191
aimodel/src/rainfallwater_identity_TEST.ipynb
Normal file
191
aimodel/src/rainfallwater_identity_TEST.ipynb
Normal file
|
|
@ -0,0 +1,191 @@
|
||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"id": "1f6fdebf-69c5-46ab-a5a8-f9c91f000ff3",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"2022-12-09 15:49:31.909879: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
|
||||||
|
"2022-12-09 15:49:31.909893: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"\n",
|
||||||
|
"import tensorflow as tf\n",
|
||||||
|
"\n",
|
||||||
|
"from lib.dataset.parse_heightmap import parse_heightmap\n",
|
||||||
|
"from lib.ai.model_rainfallwater_mono import model_rainfallwater_mono\n",
|
||||||
|
"from lib.ai.helpers.make_callbacks import make_callbacks\n",
|
||||||
|
"from lib.ai.helpers.summarywriter import summarywriter"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"id": "07093079",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"filepath_heightmap=\"/mnt/research-data/main/terrain50-nimrodsized.json.gz\"\n",
|
||||||
|
"\n",
|
||||||
|
"dir_output = \"/tmp/x/mono_segment_TEST\"\n",
|
||||||
|
"if not os.path.exists(os.path.join(dir_output, \"checkpoints\")):\n",
|
||||||
|
"\tos.makedirs(os.path.join(dir_output, \"checkpoints\"))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"id": "f4466ac9",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"RAINFALL channels 1 width 64 height 64 HEIGHTMAP_INPUT False\n",
|
||||||
|
"convnext:shape IN x (None, 64, 64, 1)\n",
|
||||||
|
"DEBUG:convnext shape x (None, 64, 64, 1)\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"2022-12-09 15:49:34.026632: E tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW\n",
|
||||||
|
"2022-12-09 15:49:34.026670: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:169] retrieving CUDA diagnostic information for host: SIMVIS-CO45428A\n",
|
||||||
|
"2022-12-09 15:49:34.026678: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:176] hostname: SIMVIS-CO45428A\n",
|
||||||
|
"2022-12-09 15:49:34.026772: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:200] libcuda reported version is: 510.108.3\n",
|
||||||
|
"2022-12-09 15:49:34.026795: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:204] kernel reported version is: 510.85.2\n",
|
||||||
|
"2022-12-09 15:49:34.026801: E tensorflow/stream_executor/cuda/cuda_diagnostics.cc:313] kernel version 510.85.2 does not match DSO version 510.108.3 -- cannot find working devices in this configuration\n",
|
||||||
|
"2022-12-09 15:49:34.027072: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\n",
|
||||||
|
"To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"DEBUG:model ENCODER output_shape (None, 512)\n",
|
||||||
|
"DEBUG:model BOTTLENECK:stack2image output_shape (None, 4, 4, 512)\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"2022-12-09 15:49:37.559 | WARNING | lib.ai.model_rainfallwater_mono:model_rainfallwater_mono:70 - Warning: TODO implement attention from https://ieeexplore.ieee.org/document/9076883\n",
|
||||||
|
"2022-12-09 15:49:37.613 | INFO | lib.ai.model_rainfallwater_mono:model_rainfallwater_mono:80 - learning_rate: None\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"model = model_rainfallwater_mono(\n",
|
||||||
|
"\tmetadata={ \"rainfallradar\": [ 1, 64, 64 ] },\n",
|
||||||
|
"\tmodel_arch_dec=\"convnext_i_xxtiny\"\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"summarywriter(model, filepath_output=os.path.join(dir_output, \"summary.txt\"))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"id": "78c633e1",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"cells 4096 cells/2 2048.0 shape+ (64, 64, 2) tf.Tensor(2401, shape=(), dtype=int64)\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"heightmap = parse_heightmap(filepath_heightmap) / 100\n",
|
||||||
|
"heightmap = tf.image.crop_to_bounding_box(tf.expand_dims(heightmap, axis=-1), 0, 0, 64, 64)\n",
|
||||||
|
"heightmap_labels = tf.one_hot(tf.cast(tf.math.greater(tf.squeeze(heightmap)/10, 0.05), dtype=tf.int32), 2)\n",
|
||||||
|
"\n",
|
||||||
|
"dataset = tf.data.Dataset.from_tensor_slices([heightmap])\n",
|
||||||
|
"dataset_labels = tf.data.Dataset.from_tensor_slices([heightmap_labels])\n",
|
||||||
|
"\n",
|
||||||
|
"for item in dataset_labels:\n",
|
||||||
|
"\tprint(\"cells\", 64*64, \"cells/2\", (64*64)/2, \"shape+\", item.shape, tf.math.reduce_sum(tf.math.argmax(item, axis=-1)))\n",
|
||||||
|
"\tbreak\n",
|
||||||
|
"dataset = tf.data.Dataset.zip((dataset, dataset_labels)) \\\n",
|
||||||
|
"\t.repeat(64 * 64) \\\n",
|
||||||
|
"\t.batch(64) \\\n",
|
||||||
|
"\t.prefetch(tf.data.AUTOTUNE)\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"id": "3dbc95eb",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Epoch 1/50\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"model.fit(\n",
|
||||||
|
"\tdataset,\n",
|
||||||
|
"\tepochs=50,\n",
|
||||||
|
"\tcallbacks=make_callbacks(\"/tmp/x/mono_segment_TEST\", model)\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "b7f8c33f",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"prediction = model.predict(tf.expand_dims(heightmap, axis=0))\n",
|
||||||
|
"print(tf.math.argmax(prediction, axis=-1))\n",
|
||||||
|
"print(tf.math.reduce_sum(tf.math.argmax(prediction, axis=-1)))\n",
|
||||||
|
"print(prediction)\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.10.6"
|
||||||
|
},
|
||||||
|
"vscode": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue