mirror of
https://github.com/sbrl/research-rainfallradar
synced 2024-12-22 06:05:01 +00:00
scripts/crossval-stbl: finish off the script
TODO switch ou median absolute distance for something else when Nina replies
This commit is contained in:
parent
fda40b05c9
commit
5a412ddc26
GPG key ID:
1BE5172E637709C2
2 changed files with 119 additions and 76 deletions
119
aimodel/scripts/crossval-stbl.py
Executable file
119
aimodel/scripts/crossval-stbl.py
Executable file
|
|
@ -0,0 +1,119 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
import pandas as pd
|
||||||
|
import scipy
|
||||||
|
from matplotlib import pyplot as plt
|
||||||
|
|
||||||
|
# This script analyses metrics.tsv files from a series of identical experiments and reports metrics on them.
|
||||||
|
# This is sometimes known as cross-validation, but we usually use the model series code crossval-stblX, where X is an integer >0.
|
||||||
|
|
||||||
|
|
||||||
|
if len(sys.argv) <= 1:
|
||||||
|
print("""
|
||||||
|
Usage:
|
||||||
|
scripts/stbl-crossval.mjs {{path/to/directory}}
|
||||||
|
|
||||||
|
...in which the given directory contains a series of experiment root directories to include in the statistical analysis.
|
||||||
|
|
||||||
|
This script is not picky about the format of the data in metrics.tsv, so long as it's in the form:
|
||||||
|
|
||||||
|
epoch metric_A metric_B …
|
||||||
|
0 val:float val:float …
|
||||||
|
1 val:float val:float …
|
||||||
|
2 val:float val:float …
|
||||||
|
⋮
|
||||||
|
""")
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
DIRPATH = sys.argv[1] # [0] == script path
|
||||||
|
|
||||||
|
files = 0
|
||||||
|
metrics = {}
|
||||||
|
epochs = None
|
||||||
|
|
||||||
|
for filepath in os.scandir(DIRPATH):
|
||||||
|
if not os.path.isdir(filepath):
|
||||||
|
continue
|
||||||
|
tbl = pd.read_csv(os.path.join(filepath, "metrics.tsv"), sep="\t")
|
||||||
|
|
||||||
|
# metrics.append(tbl)
|
||||||
|
|
||||||
|
for column in tbl.columns:
|
||||||
|
if column == "epoch":
|
||||||
|
if epochs is None:
|
||||||
|
epochs = tbl[column]
|
||||||
|
continue # Row index implicitly retains this
|
||||||
|
|
||||||
|
if column not in metrics:
|
||||||
|
metrics[column] = []
|
||||||
|
|
||||||
|
metrics[column].append(tbl[column].values)
|
||||||
|
# print(column, tbl[column])
|
||||||
|
|
||||||
|
# print("DEBUG:metrics", tbl)
|
||||||
|
files += 1
|
||||||
|
|
||||||
|
logger.info(f"Read {files} files into crossval-stbl{files} analysis")
|
||||||
|
|
||||||
|
stats = {}
|
||||||
|
|
||||||
|
for metric in metrics.keys():
|
||||||
|
metrics[metric] = pd.DataFrame(metrics[metric]).transpose()
|
||||||
|
|
||||||
|
if metric not in stats:
|
||||||
|
stats[metric] = {}
|
||||||
|
|
||||||
|
stats[metric]["mad"] = scipy.stats.median_abs_deviation(
|
||||||
|
metrics[metric], axis=1
|
||||||
|
) # median absolute deviation
|
||||||
|
stats[metric]["stddev"] = metrics[metric].std(axis=1)
|
||||||
|
stats[metric]["mean"] = metrics[metric].mean(axis=1)
|
||||||
|
stats[metric]["min"] = metrics[metric].min(axis=1)
|
||||||
|
stats[metric]["max"] = metrics[metric].max(axis=1)
|
||||||
|
stats[metric]["agg_min"] = stats[metric]["min"].min()
|
||||||
|
stats[metric]["agg_max"] = stats[metric]["max"].max()
|
||||||
|
stats[metric]["agg_stddev"] = metrics[metric].stack().std()
|
||||||
|
stats[metric]["agg_mean"] = metrics[metric].stack().std()
|
||||||
|
stats[metric]["agg_mad"] = scipy.stats.median_abs_deviation(
|
||||||
|
metrics[metric].stack()
|
||||||
|
) # median absolute deviation
|
||||||
|
|
||||||
|
# print(stats[metric])
|
||||||
|
|
||||||
|
plt.figure(figsize=(12, 8))
|
||||||
|
plt.ylim(min(0, stats[metric]["agg_min"]), max(1, stats[metric]["agg_max"]))
|
||||||
|
plt.grid(visible=True, which="major", axis="y", linewidth=2)
|
||||||
|
plt.grid(visible=True, which="minor", axis="y", linewidth=1)
|
||||||
|
plt.minorticks_on()
|
||||||
|
plt.fill_between(
|
||||||
|
epochs,
|
||||||
|
stats[metric]["min"],
|
||||||
|
stats[metric]["max"],
|
||||||
|
alpha=0.2,
|
||||||
|
facecolor="#B7DE28",
|
||||||
|
edgecolor="#FDE724",
|
||||||
|
linestyle="dotted",
|
||||||
|
linewidth=1,
|
||||||
|
)
|
||||||
|
plt.fill_between(
|
||||||
|
epochs,
|
||||||
|
stats[metric]["mean"] - stats[metric]["mad"],
|
||||||
|
stats[metric]["mean"] + stats[metric]["mad"],
|
||||||
|
alpha=0.5,
|
||||||
|
facecolor="#228A8D",
|
||||||
|
edgecolor="#3CBB74",
|
||||||
|
linestyle="dashed",
|
||||||
|
linewidth=1,
|
||||||
|
)
|
||||||
|
plt.plot(epochs, stats[metric]["mean"], color="#450C54")
|
||||||
|
plt.title(f"{metric} // crossval-stbl{files}")
|
||||||
|
plt.xlabel("epoch")
|
||||||
|
plt.ylabel(metric)
|
||||||
|
plt.savefig(os.path.join(DIRPATH, f"crossval-stbl{files}_{metric}.png"))
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
logger.success(f"Written {len(stats.keys())} graphs to {DIRPATH}")
|
||||||
|
|
@ -1,76 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
from loguru import logger
|
|
||||||
import pandas as pd
|
|
||||||
|
|
||||||
# This script analyses metrics.tsv files from a series of identical experiments and reports metrics on them.
|
|
||||||
# This is sometimes known as cross-validation, but we usually use the model series code crossval-stblX, where X is an integer >0.
|
|
||||||
|
|
||||||
|
|
||||||
if len(sys.argv) <= 1:
|
|
||||||
print("""
|
|
||||||
Usage:
|
|
||||||
scripts/stbl-crossval.mjs {{path/to/directory}}
|
|
||||||
|
|
||||||
...in which the given directory contains a series of experiment root directories to include in the statistical analysis.
|
|
||||||
|
|
||||||
This script is not picky about the format of the data in metrics.tsv, so long as it's in the form:
|
|
||||||
|
|
||||||
epoch metric_A metric_B …
|
|
||||||
0 val:float val:float …
|
|
||||||
1 val:float val:float …
|
|
||||||
2 val:float val:float …
|
|
||||||
⋮
|
|
||||||
""")
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
DIRPATH = sys.argv[1] # [0] == script path
|
|
||||||
|
|
||||||
files = 0
|
|
||||||
metrics = {}
|
|
||||||
|
|
||||||
for filepath in os.scandir(DIRPATH):
|
|
||||||
tbl = pd.read_csv(os.path.join(filepath, "metrics.tsv"), sep="\t")
|
|
||||||
|
|
||||||
# metrics.append(tbl)
|
|
||||||
|
|
||||||
for column in tbl.columns:
|
|
||||||
if column == "epoch":
|
|
||||||
continue # Row index implicitly retains this
|
|
||||||
|
|
||||||
if column not in metrics:
|
|
||||||
metrics[column] = []
|
|
||||||
|
|
||||||
metrics[column].append(tbl[column].values)
|
|
||||||
# print(column, tbl[column])
|
|
||||||
|
|
||||||
# print("DEBUG:metrics", tbl)
|
|
||||||
files += 1
|
|
||||||
|
|
||||||
logger.info(f"Read {files} files into crossval-stbl{files} analysis")
|
|
||||||
|
|
||||||
stats = {}
|
|
||||||
|
|
||||||
for metric in metrics.keys():
|
|
||||||
metrics[metric] = pd.DataFrame(metrics[metric]).transpose()
|
|
||||||
|
|
||||||
if metric not in stats:
|
|
||||||
stats[metric] = {}
|
|
||||||
|
|
||||||
stats[metric]["aad"] = metrics[metric].max(axis=1) # mean/average absolute deviation
|
|
||||||
stats[metric]["mad"] = metrics[metric].max(axis=1) # median absolute deviation
|
|
||||||
stats[metric]["stddev"] = metrics[metric].std(axis=1)
|
|
||||||
stats[metric]["mean"] = metrics[metric].mean(axis=1)
|
|
||||||
stats[metric]["min"] = metrics[metric].min(axis=1)
|
|
||||||
stats[metric]["max"] = metrics[metric].max(axis=1)
|
|
||||||
stats[metric]["agg_min"] = stats[metric]["min"].min()
|
|
||||||
stats[metric]["agg_max"] = stats[metric]["max"].max()
|
|
||||||
stats[metric]["agg_stddev"] = metrics[metric].stack().std()
|
|
||||||
stats[metric]["agg_mean"] = metrics[metric].stack().std()
|
|
||||||
stats[metric]["agg_aad"] = metrics[metric].stack().max() # mean/average absolute deviation
|
|
||||||
|
|
||||||
print(stats[metric])
|
|
||||||
|
|
||||||
Loading…
Reference in a new issue