sbrl-github/research-rainfallradar
1
0
Fork 0
mirror of https://github.com/sbrl/research-rainfallradar synced 2024-11-22 09:13:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

rr de: update to dataset_mono_predict

Browse source
This commit is contained in:
Starbeamrainbowlabs 2023-11-03 15:48:53 +00:00
parent cbb916327b
commit 76b2956703
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
1 changed files with 58 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
aimodel/src/rainfallwater_data_explorer.ipynb
View file

@ -2,40 +2,33 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 12,
"id": "1f6fdebf-69c5-46ab-a5a8-f9c91f000ff3", "id": "1f6fdebf-69c5-46ab-a5a8-f9c91f000ff3",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stderr",
"output_type": "stream",
"text": [
"2022-10-24 17:18:42.868504: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
"2022-10-24 17:18:42.868519: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
]
}
],
"source": [ "source": [
"import sys\n", "import sys\n",
"\n", "\n",
"import tensorflow as tf\n", "import tensorflow as tf\n",
"from datetime import datetime\n",
"\n", "\n",
"from lib.dataset.dataset_segmenter import dataset_predict" "from lib.dataset.dataset_mono import dataset_mono_predict"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 13,
"id": "4a6a94dd-5c4e-4481-bfae-eb5ccf6214db", "id": "4a6a94dd-5c4e-4481-bfae-eb5ccf6214db",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"dirpath=\"/home/bryan-smithl/Documents/repos/PhD-Rainfall-Radar/aimodel/output/rainfallwater_records_embed_2022-10-06_contrast_embed_umap_d512e19_tfrecord\"" "# dirpath=\"/home/bryan-smithl/Documents/repos/PhD-Rainfall-Radar/aimodel/output/rainfallwater_records_embed_2022-10-06_contrast_embed_umap_d512e19_tfrecord\"\n",
"dirpath=\"/mnt/research-data/main/rainfallwater_records_tfrecord\""
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": 14,
"id": "d1aa931a-ecf2-4134-8e70-87db4ae60736", "id": "d1aa931a-ecf2-4134-8e70-87db4ae60736",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@ -43,41 +36,33 @@
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"DEBUG:dataset ITEM rainfall:shape (512,) water:shape (94, 94, 1)\n" "DEBUG DATASET:rainfall shape [7, 174, 105] / w 105 h 174\n",
] "DEBUG DATASET:water shape [348, 210]\n",
}, "DEBUG DATASET:water_threshold 0.1\n",
{ "DEBUG DATASET:water_bins 2\n",
"name": "stderr", "DEBUG DATASET:output_size 100\n",
"output_type": "stream", "DEBUG DATASET:input_size 100\n",
"text": [ "DEBUG DATASET:water_offset x 55 y 124\n",
"2022-10-24 17:18:44.582661: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:975] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", "DEBUG DATASET:rainfall_offset x 3 y 37\n",
"2022-10-24 17:18:44.583031: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n", "DEBUG:dataset BEFORE_SQUEEZE water (100, 100, 1)\n",
"2022-10-24 17:18:44.583152: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublas.so.11'; dlerror: libcublas.so.11: cannot open shared object file: No such file or directory\n", "DEBUG:dataset AFTER_SQUEEZE water (100, 100)\n",
"2022-10-24 17:18:44.583257: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublasLt.so.11'; dlerror: libcublasLt.so.11: cannot open shared object file: No such file or directory\n", "DEBUG DATASET_OUT:rainfall shape (100, 100, 7)\n",
"2022-10-24 17:18:44.583353: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcufft.so.10'; dlerror: libcufft.so.10: cannot open shared object file: No such file or directory\n", "DEBUG DATASET_OUT:water shape (100, 100)\n"
"2022-10-24 17:18:44.583438: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcurand.so.10'; dlerror: libcurand.so.10: cannot open shared object file: No such file or directory\n",
"2022-10-24 17:18:44.583519: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusolver.so.11'; dlerror: libcusolver.so.11: cannot open shared object file: No such file or directory\n",
"2022-10-24 17:18:44.583598: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusparse.so.11'; dlerror: libcusparse.so.11: cannot open shared object file: No such file or directory\n",
"2022-10-24 17:18:44.583675: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudnn.so.8'; dlerror: libcudnn.so.8: cannot open shared object file: No such file or directory\n",
"2022-10-24 17:18:44.583688: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1850] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
"Skipping registering GPU devices...\n",
"2022-10-24 17:18:44.584534: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\n",
"To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
] ]
} }
], ],
"source": [ "source": [
"dataset = dataset_predict(\n", "dataset = dataset_mono_predict(\n",
"\tdirpath_input=dirpath,\n", "\tdirpath_input=dirpath,\n",
"\twater_threshold=0.1,\n", "\twater_threshold=0.1,\n",
"\tshape_water_desired=[94, 94],\n", "\t# shape_water_desired=[94, 94],\n",
"\tparallel_reads_multiplier=1.5 # Mangles the ordering. For counting things this doesn't matter\n", "\tparallel_reads_multiplier=1.5 # Mangles the ordering. For counting things this doesn't matter\n",
").batch(64)" ")"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 15,
"id": "456f7c8f-3f7d-4a2c-b361-900588c49612", "id": "456f7c8f-3f7d-4a2c-b361-900588c49612",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@ -93,9 +78,7 @@
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"\n", "\n",
"Complete. Counts:\n", "{msg}\n"
"0: 10238285719\n",
"1: 2854828393\n"
] ]
} }
], ],
@ -103,13 +86,44 @@
"i = 0\n", "i = 0\n",
"counts = tf.constant([0, 0], dtype=tf.int64)\n", "counts = tf.constant([0, 0], dtype=tf.int64)\n",
"for (items, label) in dataset:\n", "for (items, label) in dataset:\n",
"\tlabel = tf.cast(label, tf.int32)\n",
"\tstep_counts = tf.math.bincount(tf.reshape(label, [-1]))\n", "\tstep_counts = tf.math.bincount(tf.reshape(label, [-1]))\n",
"\tcounts += tf.cast(step_counts, dtype=tf.int64)\n", "\tcounts += tf.cast(step_counts, dtype=tf.int64)\n",
"\t# print(\"STEP counts\", counts, \"step_counts\", step_counts)\n", "\t# print(\"STEP counts\", counts, \"step_counts\", step_counts)\n",
"\ti += 1\n", "\ti += 1\n",
"\tif i % 100 == 0:\n", "\tif i % 100 == 0:\n",
"\t\tsys.stderr.write(f\"Processed {i} batches\\r\")\n", "\t\tsys.stderr.write(f\"Processed {i} batches\\r\")\n",
"print(\"\\nComplete. Counts:\\n\"+\"\\n\".join([ str(i)+\": \"+str(count) for i,count in enumerate(counts.numpy().tolist()) ]))" "\n",
"msg = f\"Complete at {datetime.now()}. Counts:\\n\"+\"\\n\".join([ str(i)+\": \"+str(count) for i,count in enumerate(counts.numpy().tolist()) ])\n",
"print(f\"\\n{msg}\")\n"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "74a94efe",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Total 14817920000 cells, Percentages:\n",
"0: 77.92587213320088%\n",
"1: 22.074127866799117%\n"
]
}
],
"source": [
"total = tf.math.reduce_sum(counts)\n",
"\n",
"percentages = (tf.cast(counts, tf.float64) / tf.cast(total, tf.float64)) * 100.0\n",
"\n",
"msg = f\"Total {total.numpy()} cells, Percentages:\\n\"+\"\\n\".join(\n",
" [str(i)+\": \"+str(count)+\"%\" for i, count in enumerate(percentages.numpy().tolist())]\n",
")\n",
"print(f\"\\n{msg}\")"
] ]
} }
], ],
@ -129,7 +143,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.10.6" "version": "3.10.12"
}, },
"vscode": { "vscode": {
"interpreter": { "interpreter": {