From a7626640631c376c428eafae5da88a2aec1b924d Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Fri, 4 Nov 2022 17:23:20 +0000 Subject: [PATCH] slurm-process: -n28 fo uniq call --- rainfallwrangler/slurm-process.job | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rainfallwrangler/slurm-process.job b/rainfallwrangler/slurm-process.job index 194fb56..bab0970 100755 --- a/rainfallwrangler/slurm-process.job +++ b/rainfallwrangler/slurm-process.job @@ -76,7 +76,7 @@ echo ">>> Converting dataset to .jsonl.gz"; /usr/bin/env time -v src/index.mjs recordify --verbose --rainfall "${RAINFALL}" --water "${WATER}" --output "${OUTPUT}" --count-file "${COUNT_FILE}" ${ARGS}; echo ">>> Deduplicating dataset"; # This also automatically recompresses for us - hence the source/target rather than in-place -srun --comment 'RainUniq' --exclusive -p compute /usr/bin/env time -v src/index.mjs uniq --source "${OUTPUT}" --target "${OUTPUT_UNIQ}" --count-file "${COUNT_FILE}"; +srun --comment 'RainUniq' --exclusive -p compute -n28 /usr/bin/env time -v src/index.mjs uniq --source "${OUTPUT}" --target "${OUTPUT_UNIQ}" --count-file "${COUNT_FILE}"; echo ">>> Removing intermediate output"; rm -r "${OUTPUT}"; echo ">>> Queuing .jsonl.gz → tfrecord";