mirror of
https://github.com/sbrl/research-rainfallradar
synced 2024-06-28 09:34:57 +00:00
191 lines
8.7 KiB
Plaintext
191 lines
8.7 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"id": "1f6fdebf-69c5-46ab-a5a8-f9c91f000ff3",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"2023-11-17 16:04:22.523958: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\n",
|
|
"To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
|
|
"2023-11-17 16:04:23.422419: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:23.422434: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
|
|
"2023-11-17 16:04:25.937266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:25.937400: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:25.937414: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import sys\n",
|
|
"\n",
|
|
"import tensorflow as tf\n",
|
|
"from datetime import datetime\n",
|
|
"\n",
|
|
"from lib.dataset.dataset_mono import dataset_mono_predict"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"id": "4a6a94dd-5c4e-4481-bfae-eb5ccf6214db",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# dirpath=\"/home/bryan-smithl/Documents/repos/PhD-Rainfall-Radar/aimodel/output/rainfallwater_records_embed_2022-10-06_contrast_embed_umap_d512e19_tfrecord\"\n",
|
|
"dirpath=\"/mnt/research-data/main/rainfallwater_records_tfrecord\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"id": "d1aa931a-ecf2-4134-8e70-87db4ae60736",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"2023-11-17 16:04:28.057254: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:28.057395: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublas.so.11'; dlerror: libcublas.so.11: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:28.057484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublasLt.so.11'; dlerror: libcublasLt.so.11: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:28.057570: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcufft.so.10'; dlerror: libcufft.so.10: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:28.057651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcurand.so.10'; dlerror: libcurand.so.10: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:28.057733: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusolver.so.11'; dlerror: libcusolver.so.11: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:28.057826: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusparse.so.11'; dlerror: libcusparse.so.11: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:28.057954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudnn.so.8'; dlerror: libcudnn.so.8: cannot open shared object file: No such file or directory\n",
|
|
"2023-11-17 16:04:28.057973: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1934] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
|
|
"Skipping registering GPU devices...\n",
|
|
"2023-11-17 16:04:28.063663: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\n",
|
|
"To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"DEBUG DATASET:rainfall shape [7, 174, 105] / w 105 h 174\n",
|
|
"DEBUG DATASET:water shape [348, 210]\n",
|
|
"DEBUG DATASET:water_threshold 0.1\n",
|
|
"DEBUG DATASET:water_bins 2\n",
|
|
"DEBUG DATASET:output_size 100\n",
|
|
"DEBUG DATASET:input_size 100\n",
|
|
"DEBUG DATASET:water_offset x 55 y 124\n",
|
|
"DEBUG DATASET:rainfall_offset x 3 y 37\n",
|
|
"DEBUG:dataset BEFORE_SQUEEZE water (100, 100, 1)\n",
|
|
"DEBUG:dataset AFTER_SQUEEZE water (100, 100)\n",
|
|
"DEBUG DATASET_OUT:rainfall shape (100, 100, 7)\n",
|
|
"DEBUG DATASET_OUT:water shape (100, 100)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"dataset = dataset_mono_predict(\n",
|
|
"\tdirpath_input=dirpath,\n",
|
|
"\twater_threshold=0.1,\n",
|
|
"\t# shape_water_desired=[94, 94],\n",
|
|
"\tparallel_reads_multiplier=1.5 # Mangles the ordering. For counting things this doesn't matter\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"id": "456f7c8f-3f7d-4a2c-b361-900588c49612",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Processed 23100 batches\r"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"Complete at 2023-11-17 16:31:08.976090. Counts:\n",
|
|
"0: 11546993392\n",
|
|
"1: 3270926608\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"i = 0\n",
|
|
"counts = tf.constant([0, 0], dtype=tf.int64)\n",
|
|
"for (items, label) in dataset:\n",
|
|
"\tlabel = tf.cast(label, tf.int32)\n",
|
|
"\tstep_counts = tf.math.bincount(tf.reshape(label, [-1]))\n",
|
|
"\tcounts += tf.cast(step_counts, dtype=tf.int64)\n",
|
|
"\t# print(\"STEP counts\", counts, \"step_counts\", step_counts)\n",
|
|
"\ti += 1\n",
|
|
"\tif i % 100 == 0:\n",
|
|
"\t\tsys.stderr.write(f\"Processed {i} batches\\r\")\n",
|
|
"\n",
|
|
"msg = f\"Complete at {datetime.now()}. Counts:\\n\"+\"\\n\".join([ str(i)+\": \"+str(count) for i,count in enumerate(counts.numpy().tolist()) ])\n",
|
|
"print(f\"\\n{msg}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"id": "74a94efe",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\n",
|
|
"Total 14817920000 cells, Percentages:\n",
|
|
"0: 77.92587213320088%\n",
|
|
"1: 22.074127866799117%\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"total = tf.math.reduce_sum(counts)\n",
|
|
"\n",
|
|
"percentages = (tf.cast(counts, tf.float64) / tf.cast(total, tf.float64)) * 100.0\n",
|
|
"\n",
|
|
"msg = f\"Total {total.numpy()} cells, Percentages:\\n\"+\"\\n\".join(\n",
|
|
" [str(i)+\": \"+str(count)+\"%\" for i, count in enumerate(percentages.numpy().tolist())]\n",
|
|
")\n",
|
|
"print(f\"\\n{msg}\")"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3.10.6 64-bit",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.12"
|
|
},
|
|
"vscode": {
|
|
"interpreter": {
|
|
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
|
|
}
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|