sbrl-github/research-rainfallradar
1
0
Fork 0
mirror of https://github.com/sbrl/research-rainfallradar synced 2024-11-22 09:13:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

slurm-process: -n28 fo uniq call

Browse source
This commit is contained in:
Starbeamrainbowlabs 2022-11-04 17:23:20 +00:00
parent 0166b4d09e
commit a762664063
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
rainfallwrangler/slurm-process.job
View file

@ -76,7 +76,7 @@ echo ">>> Converting dataset to .jsonl.gz";
/usr/bin/env time -v src/index.mjs recordify --verbose --rainfall "${RAINFALL}" --water "${WATER}" --output "${OUTPUT}" --count-file "${COUNT_FILE}" ${ARGS}; /usr/bin/env time -v src/index.mjs recordify --verbose --rainfall "${RAINFALL}" --water "${WATER}" --output "${OUTPUT}" --count-file "${COUNT_FILE}" ${ARGS};
echo ">>> Deduplicating dataset"; echo ">>> Deduplicating dataset";
# This also automatically recompresses for us - hence the source/target rather than in-place # This also automatically recompresses for us - hence the source/target rather than in-place
srun --comment 'RainUniq' --exclusive -p compute /usr/bin/env time -v src/index.mjs uniq --source "${OUTPUT}" --target "${OUTPUT_UNIQ}" --count-file "${COUNT_FILE}"; srun --comment 'RainUniq' --exclusive -p compute -n28 /usr/bin/env time -v src/index.mjs uniq --source "${OUTPUT}" --target "${OUTPUT_UNIQ}" --count-file "${COUNT_FILE}";
echo ">>> Removing intermediate output"; echo ">>> Removing intermediate output";
rm -r "${OUTPUT}"; rm -r "${OUTPUT}";
echo ">>> Queuing .jsonl.gz → tfrecord"; echo ">>> Queuing .jsonl.gz → tfrecord";