From 7c6329189f5c571dbedbf0aeb2f57bd723f40216 Mon Sep 17 00:00:00 2001
From: Starbeamrainbowlabs <sbrl@starbeamrainbowlabs.com>
Date: Fri, 15 Jan 2021 17:06:42 +0000
Subject: [PATCH] Add universal --tolerant argument

---
 Changelog.md                                  | 2 ++
 src/Bootstrap/cli.mjs                         | 1 +
 src/Subcommands/analyse-frequencies/index.mjs | 2 +-
 src/Subcommands/identify/index.mjs            | 2 +-
 src/Subcommands/image/index.mjs               | 2 +-
 src/Subcommands/split/index.mjs               | 2 +-
 src/Subcommands/stream-slice/index.mjs        | 2 +-
 src/Subcommands/validate/index.mjs            | 2 +-
 src/Subcommands/validate/meta.toml            | 6 ------
 9 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/Changelog.md b/Changelog.md
index f70e276..9c4c737 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -17,6 +17,8 @@ npm install --save terrain50-cli
 ## v1.7 (unreleased)
 - `image`: Add `--stream` argument for mass-converting large numbers of images from a single stream of multiple terrain50 files
 - `image`: Add support for reading from stdin with `--input -`
+- Add universal `--tolerant` argument for increasing tolerance of errors when parsing streams
+- `validate`: Removed `--use-regex` argument in favour of new universal `--tolerant` argument
 
 
 ## v1.6.4
diff --git a/src/Bootstrap/cli.mjs b/src/Bootstrap/cli.mjs
index f885990..55b6f2f 100644
--- a/src/Bootstrap/cli.mjs
+++ b/src/Bootstrap/cli.mjs
@@ -57,6 +57,7 @@ async function get_actions_metadata() {
 export default async function() {
 	let cli = new CliParser(path.resolve(__dirname, "../../package.json"));
 	cli.set_description_extended(`With terrain50 ${await get_version()}`);
+	cli.argument("tolerant", "When parsing streams of data, be more tolerant of whitespace inconsistencies and other errors at the cost of decreased performance (otherwise it is assumed a single space separates elements on a line).", false, "boolean");
 	
 	// Disable ansi escape codes if requested
 	if(!settings.output.ansi_colour) {
diff --git a/src/Subcommands/analyse-frequencies/index.mjs b/src/Subcommands/analyse-frequencies/index.mjs
index 3e099ca..312a074 100644
--- a/src/Subcommands/analyse-frequencies/index.mjs
+++ b/src/Subcommands/analyse-frequencies/index.mjs
@@ -31,7 +31,7 @@ export default async function(settings) {
 	// ------------------------------------------------------------------------
 	
 	let result_map = await Terrain50.AnalyseFrequencies(
-		Terrain50.ParseStream(stream_in),
+		Terrain50.ParseStream(stream_in, settings.cli.tolerant ? /\s+/ : " "),
 		settings.cli.ignore_nodata,
 		settings.cli.quiet
 	);
diff --git a/src/Subcommands/identify/index.mjs b/src/Subcommands/identify/index.mjs
index 85052e6..d605482 100644
--- a/src/Subcommands/identify/index.mjs
+++ b/src/Subcommands/identify/index.mjs
@@ -34,7 +34,7 @@ export default async function(settings) {
 				stream = fs.createReadStream(settings.cli.input, "utf-8");
 			
 			let i = 0;
-			for await(let next of Terrain50.ParseStream(stream)) {
+			for await(let next of Terrain50.ParseStream(stream, settings.cli.tolerant ? /\s+/ : " ")) {
 				process.stderr.write(`${a.fgreen}>>>>> ${a.hicol}`);
 				process.stdout.write(`Item ${i}`);
 				process.stderr.write(`${a.reset}${a.fgreen} <<<<<${a.reset}`);
diff --git a/src/Subcommands/image/index.mjs b/src/Subcommands/image/index.mjs
index 7d65b91..3023d8c 100644
--- a/src/Subcommands/image/index.mjs
+++ b/src/Subcommands/image/index.mjs
@@ -36,7 +36,7 @@ export default async function(settings) {
 			reader = fs.createReadStream(settings.cli.input, "utf-8");
 		
 		let i = 0;
-		for await(let next of Terrain50.ParseStream(reader)) {
+		for await(let next of Terrain50.ParseStream(reader, settings.cli.tolerant ? /\s+/ : " ")) {
 			process.stderr.write(`${a.fgreen}>>>>> ${a.hicol} Item ${i} ${a.reset}${a.fgreen} <<<<<${a.reset}`);
 			
 			await fs.promises.writeFile(
diff --git a/src/Subcommands/split/index.mjs b/src/Subcommands/split/index.mjs
index e2b9914..e2bad4d 100644
--- a/src/Subcommands/split/index.mjs
+++ b/src/Subcommands/split/index.mjs
@@ -33,7 +33,7 @@ export default async function(settings) {
 	// ------------------------------------------------------------------------
 	
 	let i = 0;
-	for await(let next of Terrain50.ParseStream(stream_in)) {
+	for await(let next of Terrain50.ParseStream(stream_in, settings.cli.tolerant ? /\s+/ : " ")) {
 		// Determine the output filepath
 		let output_filename = `${i}.asc`;
 		if(!settings.cli.no_gzip) output_filename += `.gz`;
diff --git a/src/Subcommands/stream-slice/index.mjs b/src/Subcommands/stream-slice/index.mjs
index b20abe3..3a4139e 100644
--- a/src/Subcommands/stream-slice/index.mjs
+++ b/src/Subcommands/stream-slice/index.mjs
@@ -33,7 +33,7 @@ export default async function(settings) {
 	// ------------------------------------------------------------------------
 	
 	let i = -1;
-	for await(let next of Terrain50.ParseStream(stream_in)) {
+	for await(let next of Terrain50.ParseStream(stream_in, settings.cli.tolerant ? /\s+/ : " ")) {
 		i++;
 		
 		if(i < offset) continue;
diff --git a/src/Subcommands/validate/index.mjs b/src/Subcommands/validate/index.mjs
index 13516c6..4b0c0e4 100644
--- a/src/Subcommands/validate/index.mjs
+++ b/src/Subcommands/validate/index.mjs
@@ -17,7 +17,7 @@ export default async function(settings) {
 		
 		case "stream":
 			let i = 0, ok = 0, failed = 0;
-			for await (let next of Terrain50.ParseStream(process.stdin, settings.cli.use_regex ? /\s+/ : " ")) {
+			for await (let next of Terrain50.ParseStream(process.stdin, settings.cli.tolerant ? /\s+/ : " ")) {
 				if(!settings.cli.quiet) console.log(`>>> Item ${i} <<<`);
 				let result = next.validate();
 				if(settings.cli.quiet && result.length > 0) console.log(`>>> Item ${i} <<<`);
diff --git a/src/Subcommands/validate/meta.toml b/src/Subcommands/validate/meta.toml
index b0cb791..af7395a 100644
--- a/src/Subcommands/validate/meta.toml
+++ b/src/Subcommands/validate/meta.toml
@@ -6,12 +6,6 @@ description = "The mode to operate in. Possible values: validate (default), stre
 default_value = "validate"
 type = "string"
 
-[[arguments]]
-name = "use-regex"
-description = "When in stream mode only, setting this flag causes a regular expression to be used when parsing input data (otherwise it is assumed a single space separates elements on a line). Makes the parser more tolerant, but less performant."
-default_value = false
-type = "boolean"
-
 [[arguments]]
 name = "quiet"
 description = "Don't print validation individual success messages, only errors"
