Add universal --tolerant argument

Changelog.md

@@ -17,6 +17,8 @@ npm install --save terrain50-cli
 ## v1.7 (unreleased)
 - `image`: Add `--stream` argument for mass-converting large numbers of images from a single stream of multiple terrain50 files
 - `image`: Add support for reading from stdin with `--input -`
+- Add universal `--tolerant` argument for increasing tolerance of errors when parsing streams
+- `validate`: Removed `--use-regex` argument in favour of new universal `--tolerant` argument
 
 
 ## v1.6.4

src/Bootstrap/cli.mjs

@@ -57,6 +57,7 @@ async function get_actions_metadata() {
 export default async function() {
 	let cli = new CliParser(path.resolve(__dirname, "../../package.json"));
 	cli.set_description_extended(`With terrain50 ${await get_version()}`);
+	cli.argument("tolerant", "When parsing streams of data, be more tolerant of whitespace inconsistencies and other errors at the cost of decreased performance (otherwise it is assumed a single space separates elements on a line).", false, "boolean");
 	
 	// Disable ansi escape codes if requested
 	if(!settings.output.ansi_colour) {

src/Subcommands/analyse-frequencies/index.mjs

@@ -31,7 +31,7 @@ export default async function(settings) {
 	// ------------------------------------------------------------------------
 	
 	let result_map = await Terrain50.AnalyseFrequencies(
-		Terrain50.ParseStream(stream_in),
+		Terrain50.ParseStream(stream_in, settings.cli.tolerant ? /\s+/ : " "),
 		settings.cli.ignore_nodata,
 		settings.cli.quiet
 	);

src/Subcommands/identify/index.mjs

@@ -34,7 +34,7 @@ export default async function(settings) {
 				stream = fs.createReadStream(settings.cli.input, "utf-8");
 			
 			let i = 0;
-			for await(let next of Terrain50.ParseStream(stream)) {
+			for await(let next of Terrain50.ParseStream(stream, settings.cli.tolerant ? /\s+/ : " ")) {
 				process.stderr.write(`${a.fgreen}>>>>> ${a.hicol}`);
 				process.stdout.write(`Item ${i}`);
 				process.stderr.write(`${a.reset}${a.fgreen} <<<<<${a.reset}`);

src/Subcommands/image/index.mjs

@@ -36,7 +36,7 @@ export default async function(settings) {
 			reader = fs.createReadStream(settings.cli.input, "utf-8");
 		
 		let i = 0;
-		for await(let next of Terrain50.ParseStream(reader)) {
+		for await(let next of Terrain50.ParseStream(reader, settings.cli.tolerant ? /\s+/ : " ")) {
 			process.stderr.write(`${a.fgreen}>>>>> ${a.hicol} Item ${i} ${a.reset}${a.fgreen} <<<<<${a.reset}`);
 			
 			await fs.promises.writeFile(

src/Subcommands/split/index.mjs

@@ -33,7 +33,7 @@ export default async function(settings) {
 	// ------------------------------------------------------------------------
 	
 	let i = 0;
-	for await(let next of Terrain50.ParseStream(stream_in)) {
+	for await(let next of Terrain50.ParseStream(stream_in, settings.cli.tolerant ? /\s+/ : " ")) {
 		// Determine the output filepath
 		let output_filename = `${i}.asc`;
 		if(!settings.cli.no_gzip) output_filename += `.gz`;

src/Subcommands/stream-slice/index.mjs

@@ -33,7 +33,7 @@ export default async function(settings) {
 	// ------------------------------------------------------------------------
 	
 	let i = -1;
-	for await(let next of Terrain50.ParseStream(stream_in)) {
+	for await(let next of Terrain50.ParseStream(stream_in, settings.cli.tolerant ? /\s+/ : " ")) {
 		i++;
 		
 		if(i < offset) continue;

src/Subcommands/validate/index.mjs

@@ -17,7 +17,7 @@ export default async function(settings) {
 		
 		case "stream":
 			let i = 0, ok = 0, failed = 0;
-			for await (let next of Terrain50.ParseStream(process.stdin, settings.cli.use_regex ? /\s+/ : " ")) {
+			for await (let next of Terrain50.ParseStream(process.stdin, settings.cli.tolerant ? /\s+/ : " ")) {
 				if(!settings.cli.quiet) console.log(`>>> Item ${i} <<<`);
 				let result = next.validate();
 				if(settings.cli.quiet && result.length > 0) console.log(`>>> Item ${i} <<<`);

src/Subcommands/validate/meta.toml

@@ -6,12 +6,6 @@ description = "The mode to operate in. Possible values: validate (default), stre
 default_value = "validate"
 type = "string"
 
-[[arguments]]
-name = "use-regex"
-description = "When in stream mode only, setting this flag causes a regular expression to be used when parsing input data (otherwise it is assumed a single space separates elements on a line). Makes the parser more tolerant, but less performant."
-default_value = false
-type = "boolean"
-
 [[arguments]]
 name = "quiet"
 description = "Don't print validation individual success messages, only errors"