docs: massively improve image conversion system

more formats, memoizing, custom <picture> element generation, async 
image conversion to optimise CPU usage, and more!

Too bad that the imagemagick apt package doesn't natively support JPEG 
XL just yet (Imagemagick 7 does though), but that's an easy fix on our 
end once they add support :D

It looks like we need to implement our own alternative to the serve 
command though :-/

.docs/.eleventy.js

@@ -1,12 +1,14 @@
+"use strict";
+
 const os = require("os");
 const fs = require("fs");
 const path = require("path");
 
+const debug = require("debug");
 const htmlentities = require("html-entities");
 const phin = require("phin");
 
-const Image = require("@11ty/eleventy-img");
+const HTMLPicture = require("./lib/HTMLPicture.js");
-
 
 var nextid = 0;
 
@@ -16,46 +18,22 @@ const image_filename_format = (_id, src, width, format, _options) => {
 	return `${name}-${width}w.${format}`;
 };
 
-function image_metadata_log(metadata, source) {
+async function shortcode_image(src, alt) {
-	for(let format in metadata) {
-		for(let img of metadata[format]) {
-			console.log(`${source.padEnd(10)} ${format.padEnd(5)} ${`${img.width}x${img.height}`.padEnd(10)} ${img.outputPath}`);
-		}
-	}
-}
 	
-async function shortcode_image(src, alt, classes = "") {
+	return HTMLPicture(
-	let metadata = await Image(src, {
+		src, alt,
-		widths: [300, null],
+		`./_site/img`, `/img`
-		formats: ["avif", "jpeg"],
+	);
-		outputDir: `./_site/img/`,
-		filenameFormat: image_filename_format
-	});
-	image_metadata_log(metadata, `IMAGE`);
-	
-	let imageAttributes = {
-		class: classes,
-		alt: htmlentities.encode(alt),
-		sizes: Object.values(metadata)[0].map((el) => `${el.width}w`).join(" "),
-		loading: "lazy",
-		decoding: "async",
-	};
-
-	// You bet we throw an error on missing alt in `imageAttributes` (alt="" works okay)
-	return Image.generateHTML(metadata, imageAttributes);
 }
 
 async function shortcode_image_url(src) {
-	let metadata = await Image(src, {
+	const src_parsed = path.parse(src);
-		widths: [ null ],
+	const target = path.join(`./_site/img`, src_parsed.base);
-		formats: [ "jpeg" ],
+	if(!fs.existsSync(path.dirname(target)))
-		outputDir: `./_site/img/`,
+	await fs.promises.mkdir(target_dir, { recursive: true });
-		filenameFormat: image_filename_format
+	await fs.promises.copyFile(src, target);
-	});
-	image_metadata_log(metadata, `IMAGE_URL`);
 	
-	let data = metadata.jpeg[metadata.jpeg.length - 1];
+	return path.join(`/img`, src_parsed.base);
-	return data.url;
 }
 
 async function shortcode_image_urlpass(src) {
@@ -82,14 +60,14 @@ async function shortcode_gallerybox(content, src, idthis, idprev, idnext) {
 }
 
 async function fetch(url) {
-	let package = JSON.parse(await fs.promises.readFile(
+	const pkg_obj = JSON.parse(await fs.promises.readFile(
 		path.join(__dirname, "package.json"), "utf8"
 	));
 	
 	return (await phin({
 		url,
 		headers: {
-			"user-agent": `WorldEditAdditionsStaticBuilder/${package.version} (Node.js/${process.version}; ${os.platform()} ${os.arch()}) eleventy/${package.devDependencies["@11ty/eleventy"].replace(/\^/, "")}`
+			"user-agent": `WorldEditAdditionsStaticBuilder/${pkg_obj.version} (Node.js/${process.version}; ${os.platform()} ${os.arch()}) eleventy/${pkg_obj.devDependencies["@11ty/eleventy"].replace(/\^/, "")}`
 		},
 		followRedirects: true,
 		parse: "string"

.docs/Reference.11tydata.js

@@ -1,5 +1,12 @@
+"use strict";
+
 const fs = require("fs");
 const path = require("path");
+
+const columnify = require("columnify");
+const htmlentities = require("html-entities");
+
+const a = require("./lib/Ansi.js");
 const parse_sections = require("./lib/parse_sections.js");
 
 let { sections, categories } = parse_sections(fs.readFileSync(
@@ -15,11 +22,20 @@ sections = sections.slice(1).sort((a, b) => a.title.replace(/^\/+/g, "").localeC
 
 
 console.log(`REFERENCE SECTION TITLES`)
-console.log(sections
+console.log(columnify(sections.map(s => { return {
-	.map(s => [s.category, s.title].join(`\t`)).join(`\n`));
+	category: `${a.hicol}${a.fyellow}${s.category}${a.reset}`,
+	command: `${a.hicol}${a.fmagenta}${htmlentities.decode(s.title)}${a.reset}`
+} })));
+// console.log(sections
+// 	.map(s => `${a.fyellow}${a.hicol}${s.category}${a.reset}\t${a.fmagenta}${a.hicol}${s.title}${a.reset}`).join(`\n`));
 console.log(`************************`);
 
-console.log(`REFERENCE SECTION COLOURS`, categories);
+console.log(`REFERENCE SECTION COLOURS`);
+console.log(columnify(Array.from(categories).map(el => { return {
+	category: el[0],
+	colour: el[1]
+} })));
+
 module.exports = {
 	layout: "theme.njk",
 	title: "Reference",

.docs/lib/Ansi.js

@@ -0,0 +1,88 @@
+"use strict";
+
+/**
+ * Generates various VT100 ANSI escape sequences.
+ * Ported from C#.
+ * @licence MPL-2.0 <https://www.mozilla.org/en-US/MPL/2.0/>
+ * @source https://gist.github.com/a4edd3204a03f4eedb79785751efb0f3#file-ansi-cs
+ * @author Starbeamrainbowlabs
+ * GitHub: @sbrl | Twitter: @SBRLabs | Reddit: u/Starbeamrainbowlabs
+ ***** Changelog *****
+ * 27th March 2019:
+ *  - Initial public release
+ * 9th March 2020:
+ * 	- Add Italics (\u001b[3m])
+ * 	- Export a new instance of it by default (makes it so that there's 1 global instance)
+ * 5th September 2020:
+ *	- Add support for NO_COLOR environment variable <https://no-color.org/>
+ */
+class Ansi {
+	constructor() {
+		/**
+		 * Whether we should *actually* emit ANSI escape codes or not.
+		 * Useful when we want to output to a log file, for example
+		 * @type {Boolean}
+		 */
+		this.enabled = true;
+        
+		this.escape_codes();
+	}
+	
+	escape_codes() {
+		if(typeof process !== "undefined" && typeof process.env.NO_COLOR == "string") {
+			this.enabled = false;
+			return;
+		}
+		// Solution on how to output ANSI escape codes in C# from here:
+		// https://www.jerriepelser.com/blog/using-ansi-color-codes-in-net-console-apps
+		this.reset = this.enabled ? "\u001b[0m" : "";
+		this.hicol = this.enabled ? "\u001b[1m" : "";
+		this.locol = this.enabled ? "\u001b[2m" : "";
+		this.italics = this.enabled ? "\u001b[3m" : "";
+		this.underline = this.enabled ? "\u001b[4m" : "";
+		this.inverse = this.enabled ? "\u001b[7m" : "";
+		this.fblack = this.enabled ? "\u001b[30m" : "";
+		this.fred = this.enabled ? "\u001b[31m" : "";
+		this.fgreen = this.enabled ? "\u001b[32m" : "";
+		this.fyellow = this.enabled ? "\u001b[33m" : "";
+		this.fblue = this.enabled ? "\u001b[34m" : "";
+		this.fmagenta = this.enabled ? "\u001b[35m" : "";
+		this.fcyan = this.enabled ? "\u001b[36m" : "";
+		this.fwhite = this.enabled ? "\u001b[37m" : "";
+		this.bblack = this.enabled ? "\u001b[40m" : "";
+		this.bred = this.enabled ? "\u001b[41m" : "";
+		this.bgreen = this.enabled ? "\u001b[42m" : "";
+		this.byellow = this.enabled ? "\u001b[43m" : "";
+		this.bblue = this.enabled ? "\u001b[44m" : "";
+		this.bmagenta = this.enabled ? "\u001b[45m" : "";
+		this.bcyan = this.enabled ? "\u001b[46m" : "";
+		this.bwhite = this.enabled ? "\u001b[47m" : "";
+	}
+	
+	// Thanks to http://ascii-table.com/ansi-escape-sequences.php for the following ANSI escape sequences
+	up(lines = 1) {
+		return this.enabled ? `\u001b[${lines}A` : "";
+	}
+	down(lines = 1) {
+		return this.enabled ? `\u001b[${lines}B` : "";
+	}
+	right(lines = 1) {
+		return this.enabled ? `\u001b[${lines}C` : "";
+	}
+	left(lines = 1) {
+		return this.enabled ? `\u001b[${lines}D` : "";
+	}
+	
+	jump_to(x, y) {
+		return this.enabled ? `\u001b[${y};${x}H` : "";
+	}
+	
+	cursor_pos_save() {
+		return this.enabled ? `\u001b[s` : "";
+	}
+	cursor_pos_restore() {
+		return this.enabled ? `\u001b[u` : "";
+	}
+}
+
+module.exports = new Ansi();

.docs/lib/HTMLPicture.js

@@ -0,0 +1,154 @@
+"use strict";
+
+const os = require(`os`);
+const fs = require("fs");
+const path = require("path");
+
+const pretty_ms = require("pretty-ms");
+const debug = require("debug")("image");
+const imagickal = require("imagickal");
+const htmlentities = require("html-entities");
+
+const a = require("./Ansi.js");
+
+function calculate_size(width, height, size_spec) {
+	if(size_spec.indexOf("%") > -1) {
+		// It's a percentage
+		const multiplier = parseInt(size_spec.replace(/%/, ""), 10) / 100;
+		return {
+			width: Math.ceil(width * multiplier),
+			height: Math.ceil(height * multiplier)
+		};
+	}
+	else {
+		// It's an absolute image width
+		const new_width = parseInt(size_spec, 10);
+		return {
+			width: new_width,
+			height: Math.ceil(new_width/width * height)
+		};
+	}
+}
+
+// Main task list - we make sure it completes before exiting.
+var queue = null;
+var pMemoize = null;
+
+async function make_queue() {
+	// 1: Setup task queue
+	const PQueue = (await import("p-queue")).default;
+	let concurrency = os.cpus().length;
+	if(process.env["MAX_CONCURRENT"])
+		concurrency = parseInt(process.env["MAX_CONCURRENT"], 10);
+	debug(`Image conversion queue concurrency: `, concurrency);
+	queue = new PQueue({ concurrency });
+	queue.on("idle", () => console.log(`IMAGE ${a.fcyan}all conversions complete${a.reset}`));
+	process.on("exit", async () => {
+		debug(`Waiting for image conversions to finish...`);
+		await queue.onEmpty();
+		debug(`All image conversions complete.`);
+	});
+}
+
+async function srcset(source_image, target_dir, urlpath, format = "__AUTO__", sizes = [ "25%", "50%", "100%" ], quality = 95, strip = true) {
+	if(queue === null) await make_queue();
+	
+	const source_parsed = path.parse(source_image);
+	// ext contains the dot . already
+	const target_format = format == "__AUTO__" ? source_parsed.ext.replace(/\./g, "") : format;
+	
+	const source_size = await imagickal.dimensions(source_image);
+	
+	debug(`SOURCE_SIZE`, source_size, `TARGET_FORMAT`, target_format);
+	
+	let setitems = await Promise.all(sizes.map(async (size) => {
+		let target_filename = `${source_parsed.name}_${size}.${target_format}`;
+		let target_current = path.join(
+			target_dir,
+			target_filename
+		);
+		queue.add(async () => {
+			const start = new Date();
+			await imagickal.transform(source_image, target_current, {
+				resize: { width: size },
+				quality,
+				strip
+			});
+			console.log(`IMAGE\t${a.fcyan}${queue.size}/${queue.pending} tasks${a.reset}\t${a.fyellow}${pretty_ms(new Date() - start)}${a.reset}\t${a.fgreen}${target_current}${a.reset}`);
+		});
+		// const size_target = await imagickal.dimensions(target_current);
+		
+		const predict = calculate_size(source_size.width, source_size.height, size);
+		// debug(`size spec:`, size, `size predicted: ${predict.width}x${predict.height} actual: ${size_target.width}x${size_target.height}`);
+		return `${path.resolve(urlpath, target_filename)} ${predict.width}w`;
+	}));
+	
+	return setitems.join(", ");
+}
+
+/**
+ * Generates a string of HTML for a <picture> element, converting images to the specified formats in the process.
+ * @param	{string}	source_image					The filepath to the source image.
+ * @param	{string}	alt								The alt (alternative) text. Automatically run though htmlentities.
+ * @param	{string}	target_dir						The target directory to save converted images to.
+ * @param	{string}	urlpath							The path to the aforementionoed target directory as a URL. Image paths in the HTML will be prefixed with this value.
+ * @param	{string}	[formats="__AUTO__"]			A list of formats to convert the source image to. Defaults to automatically determining the most optimal formats based on the input format. [must be lowercase]
+ * @param	{Array}		[sizes=["25%","50%", "100%" ]]	The sizes, as imagemagick size specs, to convert the source image to.
+ * @param	{Number}	[quality=95]					The quality value to use when converting images.
+ * @param	{Boolean}	[strip=true]					Whether to strip all metadata from images when converting them [saves some space]
+ * @return	{Promise<string>}	A Promise that returns a generated string of HTML.
+ */
+async function picture(source_image, alt, target_dir, urlpath, formats = "__AUTO__", sizes = [ "25%", "50%", "100%" ], quality = 95, strip = true) {
+	const source_parsed = path.parse(source_image);
+	const source_format = source_parsed.ext.toLowerCase().replace(".", "");
+	
+	if(formats == "__AUTO__") {
+		switch(source_format) {
+			case "png":
+			case "gif": // * shudder *
+			case "bmp":
+				formats = [ "png" ];
+				break;
+			default:
+				// jxl = JPEG XL <https://jpegxl.info/> - not currently supported by the old version of imagemagick shipped via apt :-/
+				// Imagemagick v7+ does support it, but isn't shipped yet :-( 
+				formats = [ "jpeg", "webp", "avif", /*"jxl"*/ ];
+				break;
+		}
+	}
+	
+	const target_original = path.join(target_dir, source_parsed.base);
+	await fs.promises.copyFile(source_image, target_original);
+	
+	const sources = await Promise.all(formats.map(async (format) => {
+		debug(`${format} ${source_image}`);
+		
+		return {
+			mime: `image/${format}`,
+			srcset: await srcset(
+				source_image,
+				target_dir, urlpath,
+				format, sizes,
+				quality, strip
+			)
+		};
+	}));
+	
+	let result = `<picture>\n\t`;
+	result += sources.map(source => `<source srcset="${source.srcset}" type="${source.mime}" />`).join(`\n\t`);
+	result += `\n\t<img loading="lazy" decoding="async" src="${urlpath}/${source_parsed.base}" alt="${htmlentities.encode(alt)}" />\n`;
+	result += `</picture>\n`
+	return result;
+}
+
+var picture_memoize = null;
+
+async function setup_memoize() {
+	const pMemoize = (await import("p-memoize")).default;
+	picture_memoize = pMemoize(picture);
+}
+
+module.exports = async function(...args) {
+	if(picture_memoize === null) await setup_memoize();
+	return await picture_memoize(...args);
+};

.docs/lib/parse_sections.js

@@ -1,3 +1,5 @@
+"use strict";
+
 const crypto = require("crypto");
 
 const htmlentities = require("html-entities");
@@ -6,7 +8,6 @@ const markdown = require("markdown-it")({
 });
 const chroma = require("chroma-js");
 
-
 const markdown_prism = require("markdown-it-prism");
 markdown.use(markdown_prism, {
 	init: (Prism) => {

.docs/package.json

@@ -21,10 +21,15 @@
 	"homepage": "https://github.com/sbrl/Minetest-WorldEditAdditions#readme",
 	"devDependencies": {
 		"@11ty/eleventy": "^0.12.1",
-		"@11ty/eleventy-img": "^0.10.0",
 		"chroma-js": "^2.1.2",
+		"columnify": "^1.5.4",
+		"debug": "^4.3.2",
+		"imagickal": "^5.0.1",
 		"markdown-it-prism": "^2.2.1",
-		"phin": "^3.6.0"
+		"p-memoize": "^6.0.1",
+		"p-queue": "^7.1.0",
+		"phin": "^3.6.0",
+		"pretty-ms": "^7.0.1"
 	},
 	"dependencies": {
 		"html-entities": "^2.3.2"