From a6b07a49cb1a5ad3410108062688fae8c4cf86b6 Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Mon, 24 Oct 2022 18:05:34 +0100 Subject: [PATCH] count water/nowater pixels in Jupyter Notebook --- aimodel/src/rainfallwater_data_explorer.ipynb | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 aimodel/src/rainfallwater_data_explorer.ipynb diff --git a/aimodel/src/rainfallwater_data_explorer.ipynb b/aimodel/src/rainfallwater_data_explorer.ipynb new file mode 100644 index 0000000..de653c9 --- /dev/null +++ b/aimodel/src/rainfallwater_data_explorer.ipynb @@ -0,0 +1,142 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "1f6fdebf-69c5-46ab-a5a8-f9c91f000ff3", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-10-24 17:18:42.868504: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n", + "2022-10-24 17:18:42.868519: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n" + ] + } + ], + "source": [ + "import sys\n", + "\n", + "import tensorflow as tf\n", + "\n", + "from lib.dataset.dataset_segmenter import dataset_predict" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "4a6a94dd-5c4e-4481-bfae-eb5ccf6214db", + "metadata": {}, + "outputs": [], + "source": [ + "dirpath=\"/home/bryan-smithl/Documents/repos/PhD-Rainfall-Radar/aimodel/output/rainfallwater_records_embed_2022-10-06_contrast_embed_umap_d512e19_tfrecord\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d1aa931a-ecf2-4134-8e70-87db4ae60736", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DEBUG:dataset ITEM rainfall:shape (512,) water:shape (94, 94, 1)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-10-24 17:18:44.582661: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:975] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", + "2022-10-24 17:18:44.583031: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n", + "2022-10-24 17:18:44.583152: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublas.so.11'; dlerror: libcublas.so.11: cannot open shared object file: No such file or directory\n", + "2022-10-24 17:18:44.583257: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublasLt.so.11'; dlerror: libcublasLt.so.11: cannot open shared object file: No such file or directory\n", + "2022-10-24 17:18:44.583353: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcufft.so.10'; dlerror: libcufft.so.10: cannot open shared object file: No such file or directory\n", + "2022-10-24 17:18:44.583438: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcurand.so.10'; dlerror: libcurand.so.10: cannot open shared object file: No such file or directory\n", + "2022-10-24 17:18:44.583519: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusolver.so.11'; dlerror: libcusolver.so.11: cannot open shared object file: No such file or directory\n", + "2022-10-24 17:18:44.583598: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusparse.so.11'; dlerror: libcusparse.so.11: cannot open shared object file: No such file or directory\n", + "2022-10-24 17:18:44.583675: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudnn.so.8'; dlerror: libcudnn.so.8: cannot open shared object file: No such file or directory\n", + "2022-10-24 17:18:44.583688: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1850] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n", + "Skipping registering GPU devices...\n", + "2022-10-24 17:18:44.584534: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\n", + "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n" + ] + } + ], + "source": [ + "dataset = dataset_predict(\n", + "\tdirpath_input=dirpath,\n", + "\twater_threshold=0.1,\n", + "\tshape_water_desired=[94, 94],\n", + "\tparallel_reads_multiplier=1.5 # Mangles the ordering. For counting things this doesn't matter\n", + ").batch(64)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "456f7c8f-3f7d-4a2c-b361-900588c49612", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processed 23100 batches\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Complete. Counts:\n", + "0: 10238285719\n", + "1: 2854828393\n" + ] + } + ], + "source": [ + "i = 0\n", + "counts = tf.constant([0, 0], dtype=tf.int64)\n", + "for (items, label) in dataset:\n", + "\tstep_counts = tf.math.bincount(tf.reshape(label, [-1]))\n", + "\tcounts += tf.cast(step_counts, dtype=tf.int64)\n", + "\t# print(\"STEP counts\", counts, \"step_counts\", step_counts)\n", + "\ti += 1\n", + "\tif i % 100 == 0:\n", + "\t\tsys.stderr.write(f\"Processed {i} batches\\r\")\n", + "print(\"\\nComplete. Counts:\\n\"+\"\\n\".join([ str(i)+\": \"+str(count) for i,count in enumerate(counts.numpy().tolist()) ]))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.10.6 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + }, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}