mirror of
https://github.com/sbrl/Minetest-WorldEditAdditions.git
synced 2024-11-25 08:33:00 +00:00
docs: massively improve image conversion system
more formats, memoizing, custom <picture> element generation, async image conversion to optimise CPU usage, and more! Too bad that the imagemagick apt package doesn't natively support JPEG XL just yet (Imagemagick 7 does though), but that's an easy fix on our end once they add support :D It looks like we need to implement our own alternative to the serve command though :-/
This commit is contained in:
parent
2ae241aee5
commit
6eaa3799d3
GPG key ID:
1BE5172E637709C2
7 changed files with 541 additions and 1449 deletions
|
|
@ -1,12 +1,14 @@
|
|||
"use strict";
|
||||
|
||||
const os = require("os");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const debug = require("debug");
|
||||
const htmlentities = require("html-entities");
|
||||
const phin = require("phin");
|
||||
|
||||
const Image = require("@11ty/eleventy-img");
|
||||
|
||||
const HTMLPicture = require("./lib/HTMLPicture.js");
|
||||
|
||||
var nextid = 0;
|
||||
|
||||
|
|
@ -16,46 +18,22 @@ const image_filename_format = (_id, src, width, format, _options) => {
|
|||
return `${name}-${width}w.${format}`;
|
||||
};
|
||||
|
||||
function image_metadata_log(metadata, source) {
|
||||
for(let format in metadata) {
|
||||
for(let img of metadata[format]) {
|
||||
console.log(`${source.padEnd(10)} ${format.padEnd(5)} ${`${img.width}x${img.height}`.padEnd(10)} ${img.outputPath}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function shortcode_image(src, alt, classes = "") {
|
||||
let metadata = await Image(src, {
|
||||
widths: [300, null],
|
||||
formats: ["avif", "jpeg"],
|
||||
outputDir: `./_site/img/`,
|
||||
filenameFormat: image_filename_format
|
||||
});
|
||||
image_metadata_log(metadata, `IMAGE`);
|
||||
async function shortcode_image(src, alt) {
|
||||
|
||||
let imageAttributes = {
|
||||
class: classes,
|
||||
alt: htmlentities.encode(alt),
|
||||
sizes: Object.values(metadata)[0].map((el) => `${el.width}w`).join(" "),
|
||||
loading: "lazy",
|
||||
decoding: "async",
|
||||
};
|
||||
|
||||
// You bet we throw an error on missing alt in `imageAttributes` (alt="" works okay)
|
||||
return Image.generateHTML(metadata, imageAttributes);
|
||||
return HTMLPicture(
|
||||
src, alt,
|
||||
`./_site/img`, `/img`
|
||||
);
|
||||
}
|
||||
|
||||
async function shortcode_image_url(src) {
|
||||
let metadata = await Image(src, {
|
||||
widths: [ null ],
|
||||
formats: [ "jpeg" ],
|
||||
outputDir: `./_site/img/`,
|
||||
filenameFormat: image_filename_format
|
||||
});
|
||||
image_metadata_log(metadata, `IMAGE_URL`);
|
||||
const src_parsed = path.parse(src);
|
||||
const target = path.join(`./_site/img`, src_parsed.base);
|
||||
if(!fs.existsSync(path.dirname(target)))
|
||||
await fs.promises.mkdir(target_dir, { recursive: true });
|
||||
await fs.promises.copyFile(src, target);
|
||||
|
||||
let data = metadata.jpeg[metadata.jpeg.length - 1];
|
||||
return data.url;
|
||||
return path.join(`/img`, src_parsed.base);
|
||||
}
|
||||
|
||||
async function shortcode_image_urlpass(src) {
|
||||
|
|
@ -82,14 +60,14 @@ async function shortcode_gallerybox(content, src, idthis, idprev, idnext) {
|
|||
}
|
||||
|
||||
async function fetch(url) {
|
||||
let package = JSON.parse(await fs.promises.readFile(
|
||||
const pkg_obj = JSON.parse(await fs.promises.readFile(
|
||||
path.join(__dirname, "package.json"), "utf8"
|
||||
));
|
||||
|
||||
return (await phin({
|
||||
url,
|
||||
headers: {
|
||||
"user-agent": `WorldEditAdditionsStaticBuilder/${package.version} (Node.js/${process.version}; ${os.platform()} ${os.arch()}) eleventy/${package.devDependencies["@11ty/eleventy"].replace(/\^/, "")}`
|
||||
"user-agent": `WorldEditAdditionsStaticBuilder/${pkg_obj.version} (Node.js/${process.version}; ${os.platform()} ${os.arch()}) eleventy/${pkg_obj.devDependencies["@11ty/eleventy"].replace(/\^/, "")}`
|
||||
},
|
||||
followRedirects: true,
|
||||
parse: "string"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
"use strict";
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const columnify = require("columnify");
|
||||
const htmlentities = require("html-entities");
|
||||
|
||||
const a = require("./lib/Ansi.js");
|
||||
const parse_sections = require("./lib/parse_sections.js");
|
||||
|
||||
let { sections, categories } = parse_sections(fs.readFileSync(
|
||||
|
|
@ -15,11 +22,20 @@ sections = sections.slice(1).sort((a, b) => a.title.replace(/^\/+/g, "").localeC
|
|||
|
||||
|
||||
console.log(`REFERENCE SECTION TITLES`)
|
||||
console.log(sections
|
||||
.map(s => [s.category, s.title].join(`\t`)).join(`\n`));
|
||||
console.log(columnify(sections.map(s => { return {
|
||||
category: `${a.hicol}${a.fyellow}${s.category}${a.reset}`,
|
||||
command: `${a.hicol}${a.fmagenta}${htmlentities.decode(s.title)}${a.reset}`
|
||||
} })));
|
||||
// console.log(sections
|
||||
// .map(s => `${a.fyellow}${a.hicol}${s.category}${a.reset}\t${a.fmagenta}${a.hicol}${s.title}${a.reset}`).join(`\n`));
|
||||
console.log(`************************`);
|
||||
|
||||
console.log(`REFERENCE SECTION COLOURS`, categories);
|
||||
console.log(`REFERENCE SECTION COLOURS`);
|
||||
console.log(columnify(Array.from(categories).map(el => { return {
|
||||
category: el[0],
|
||||
colour: el[1]
|
||||
} })));
|
||||
|
||||
module.exports = {
|
||||
layout: "theme.njk",
|
||||
title: "Reference",
|
||||
|
|
|
|||
88
.docs/lib/Ansi.js
Normal file
88
.docs/lib/Ansi.js
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
"use strict";
|
||||
|
||||
/**
|
||||
* Generates various VT100 ANSI escape sequences.
|
||||
* Ported from C#.
|
||||
* @licence MPL-2.0 <https://www.mozilla.org/en-US/MPL/2.0/>
|
||||
* @source https://gist.github.com/a4edd3204a03f4eedb79785751efb0f3#file-ansi-cs
|
||||
* @author Starbeamrainbowlabs
|
||||
* GitHub: @sbrl | Twitter: @SBRLabs | Reddit: u/Starbeamrainbowlabs
|
||||
***** Changelog *****
|
||||
* 27th March 2019:
|
||||
* - Initial public release
|
||||
* 9th March 2020:
|
||||
* - Add Italics (\u001b[3m])
|
||||
* - Export a new instance of it by default (makes it so that there's 1 global instance)
|
||||
* 5th September 2020:
|
||||
* - Add support for NO_COLOR environment variable <https://no-color.org/>
|
||||
*/
|
||||
class Ansi {
|
||||
constructor() {
|
||||
/**
|
||||
* Whether we should *actually* emit ANSI escape codes or not.
|
||||
* Useful when we want to output to a log file, for example
|
||||
* @type {Boolean}
|
||||
*/
|
||||
this.enabled = true;
|
||||
|
||||
this.escape_codes();
|
||||
}
|
||||
|
||||
escape_codes() {
|
||||
if(typeof process !== "undefined" && typeof process.env.NO_COLOR == "string") {
|
||||
this.enabled = false;
|
||||
return;
|
||||
}
|
||||
// Solution on how to output ANSI escape codes in C# from here:
|
||||
// https://www.jerriepelser.com/blog/using-ansi-color-codes-in-net-console-apps
|
||||
this.reset = this.enabled ? "\u001b[0m" : "";
|
||||
this.hicol = this.enabled ? "\u001b[1m" : "";
|
||||
this.locol = this.enabled ? "\u001b[2m" : "";
|
||||
this.italics = this.enabled ? "\u001b[3m" : "";
|
||||
this.underline = this.enabled ? "\u001b[4m" : "";
|
||||
this.inverse = this.enabled ? "\u001b[7m" : "";
|
||||
this.fblack = this.enabled ? "\u001b[30m" : "";
|
||||
this.fred = this.enabled ? "\u001b[31m" : "";
|
||||
this.fgreen = this.enabled ? "\u001b[32m" : "";
|
||||
this.fyellow = this.enabled ? "\u001b[33m" : "";
|
||||
this.fblue = this.enabled ? "\u001b[34m" : "";
|
||||
this.fmagenta = this.enabled ? "\u001b[35m" : "";
|
||||
this.fcyan = this.enabled ? "\u001b[36m" : "";
|
||||
this.fwhite = this.enabled ? "\u001b[37m" : "";
|
||||
this.bblack = this.enabled ? "\u001b[40m" : "";
|
||||
this.bred = this.enabled ? "\u001b[41m" : "";
|
||||
this.bgreen = this.enabled ? "\u001b[42m" : "";
|
||||
this.byellow = this.enabled ? "\u001b[43m" : "";
|
||||
this.bblue = this.enabled ? "\u001b[44m" : "";
|
||||
this.bmagenta = this.enabled ? "\u001b[45m" : "";
|
||||
this.bcyan = this.enabled ? "\u001b[46m" : "";
|
||||
this.bwhite = this.enabled ? "\u001b[47m" : "";
|
||||
}
|
||||
|
||||
// Thanks to http://ascii-table.com/ansi-escape-sequences.php for the following ANSI escape sequences
|
||||
up(lines = 1) {
|
||||
return this.enabled ? `\u001b[${lines}A` : "";
|
||||
}
|
||||
down(lines = 1) {
|
||||
return this.enabled ? `\u001b[${lines}B` : "";
|
||||
}
|
||||
right(lines = 1) {
|
||||
return this.enabled ? `\u001b[${lines}C` : "";
|
||||
}
|
||||
left(lines = 1) {
|
||||
return this.enabled ? `\u001b[${lines}D` : "";
|
||||
}
|
||||
|
||||
jump_to(x, y) {
|
||||
return this.enabled ? `\u001b[${y};${x}H` : "";
|
||||
}
|
||||
|
||||
cursor_pos_save() {
|
||||
return this.enabled ? `\u001b[s` : "";
|
||||
}
|
||||
cursor_pos_restore() {
|
||||
return this.enabled ? `\u001b[u` : "";
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new Ansi();
|
||||
154
.docs/lib/HTMLPicture.js
Normal file
154
.docs/lib/HTMLPicture.js
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
"use strict";
|
||||
|
||||
const os = require(`os`);
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const pretty_ms = require("pretty-ms");
|
||||
const debug = require("debug")("image");
|
||||
const imagickal = require("imagickal");
|
||||
const htmlentities = require("html-entities");
|
||||
|
||||
const a = require("./Ansi.js");
|
||||
|
||||
function calculate_size(width, height, size_spec) {
|
||||
if(size_spec.indexOf("%") > -1) {
|
||||
// It's a percentage
|
||||
const multiplier = parseInt(size_spec.replace(/%/, ""), 10) / 100;
|
||||
return {
|
||||
width: Math.ceil(width * multiplier),
|
||||
height: Math.ceil(height * multiplier)
|
||||
};
|
||||
}
|
||||
else {
|
||||
// It's an absolute image width
|
||||
const new_width = parseInt(size_spec, 10);
|
||||
return {
|
||||
width: new_width,
|
||||
height: Math.ceil(new_width/width * height)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Main task list - we make sure it completes before exiting.
|
||||
var queue = null;
|
||||
var pMemoize = null;
|
||||
|
||||
async function make_queue() {
|
||||
// 1: Setup task queue
|
||||
const PQueue = (await import("p-queue")).default;
|
||||
let concurrency = os.cpus().length;
|
||||
if(process.env["MAX_CONCURRENT"])
|
||||
concurrency = parseInt(process.env["MAX_CONCURRENT"], 10);
|
||||
debug(`Image conversion queue concurrency: `, concurrency);
|
||||
queue = new PQueue({ concurrency });
|
||||
queue.on("idle", () => console.log(`IMAGE ${a.fcyan}all conversions complete${a.reset}`));
|
||||
process.on("exit", async () => {
|
||||
debug(`Waiting for image conversions to finish...`);
|
||||
await queue.onEmpty();
|
||||
debug(`All image conversions complete.`);
|
||||
});
|
||||
}
|
||||
|
||||
async function srcset(source_image, target_dir, urlpath, format = "__AUTO__", sizes = [ "25%", "50%", "100%" ], quality = 95, strip = true) {
|
||||
if(queue === null) await make_queue();
|
||||
|
||||
const source_parsed = path.parse(source_image);
|
||||
// ext contains the dot . already
|
||||
const target_format = format == "__AUTO__" ? source_parsed.ext.replace(/\./g, "") : format;
|
||||
|
||||
const source_size = await imagickal.dimensions(source_image);
|
||||
|
||||
debug(`SOURCE_SIZE`, source_size, `TARGET_FORMAT`, target_format);
|
||||
|
||||
let setitems = await Promise.all(sizes.map(async (size) => {
|
||||
let target_filename = `${source_parsed.name}_${size}.${target_format}`;
|
||||
let target_current = path.join(
|
||||
target_dir,
|
||||
target_filename
|
||||
);
|
||||
queue.add(async () => {
|
||||
const start = new Date();
|
||||
await imagickal.transform(source_image, target_current, {
|
||||
resize: { width: size },
|
||||
quality,
|
||||
strip
|
||||
});
|
||||
console.log(`IMAGE\t${a.fcyan}${queue.size}/${queue.pending} tasks${a.reset}\t${a.fyellow}${pretty_ms(new Date() - start)}${a.reset}\t${a.fgreen}${target_current}${a.reset}`);
|
||||
});
|
||||
// const size_target = await imagickal.dimensions(target_current);
|
||||
|
||||
const predict = calculate_size(source_size.width, source_size.height, size);
|
||||
// debug(`size spec:`, size, `size predicted: ${predict.width}x${predict.height} actual: ${size_target.width}x${size_target.height}`);
|
||||
return `${path.resolve(urlpath, target_filename)} ${predict.width}w`;
|
||||
}));
|
||||
|
||||
return setitems.join(", ");
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a string of HTML for a <picture> element, converting images to the specified formats in the process.
|
||||
* @param {string} source_image The filepath to the source image.
|
||||
* @param {string} alt The alt (alternative) text. Automatically run though htmlentities.
|
||||
* @param {string} target_dir The target directory to save converted images to.
|
||||
* @param {string} urlpath The path to the aforementionoed target directory as a URL. Image paths in the HTML will be prefixed with this value.
|
||||
* @param {string} [formats="__AUTO__"] A list of formats to convert the source image to. Defaults to automatically determining the most optimal formats based on the input format. [must be lowercase]
|
||||
* @param {Array} [sizes=["25%","50%", "100%" ]] The sizes, as imagemagick size specs, to convert the source image to.
|
||||
* @param {Number} [quality=95] The quality value to use when converting images.
|
||||
* @param {Boolean} [strip=true] Whether to strip all metadata from images when converting them [saves some space]
|
||||
* @return {Promise<string>} A Promise that returns a generated string of HTML.
|
||||
*/
|
||||
async function picture(source_image, alt, target_dir, urlpath, formats = "__AUTO__", sizes = [ "25%", "50%", "100%" ], quality = 95, strip = true) {
|
||||
const source_parsed = path.parse(source_image);
|
||||
const source_format = source_parsed.ext.toLowerCase().replace(".", "");
|
||||
|
||||
if(formats == "__AUTO__") {
|
||||
switch(source_format) {
|
||||
case "png":
|
||||
case "gif": // * shudder *
|
||||
case "bmp":
|
||||
formats = [ "png" ];
|
||||
break;
|
||||
default:
|
||||
// jxl = JPEG XL <https://jpegxl.info/> - not currently supported by the old version of imagemagick shipped via apt :-/
|
||||
// Imagemagick v7+ does support it, but isn't shipped yet :-(
|
||||
formats = [ "jpeg", "webp", "avif", /*"jxl"*/ ];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const target_original = path.join(target_dir, source_parsed.base);
|
||||
await fs.promises.copyFile(source_image, target_original);
|
||||
|
||||
const sources = await Promise.all(formats.map(async (format) => {
|
||||
debug(`${format} ${source_image}`);
|
||||
|
||||
return {
|
||||
mime: `image/${format}`,
|
||||
srcset: await srcset(
|
||||
source_image,
|
||||
target_dir, urlpath,
|
||||
format, sizes,
|
||||
quality, strip
|
||||
)
|
||||
};
|
||||
}));
|
||||
|
||||
let result = `<picture>\n\t`;
|
||||
result += sources.map(source => `<source srcset="${source.srcset}" type="${source.mime}" />`).join(`\n\t`);
|
||||
result += `\n\t<img loading="lazy" decoding="async" src="${urlpath}/${source_parsed.base}" alt="${htmlentities.encode(alt)}" />\n`;
|
||||
result += `</picture>\n`
|
||||
return result;
|
||||
}
|
||||
|
||||
var picture_memoize = null;
|
||||
|
||||
async function setup_memoize() {
|
||||
const pMemoize = (await import("p-memoize")).default;
|
||||
picture_memoize = pMemoize(picture);
|
||||
}
|
||||
|
||||
module.exports = async function(...args) {
|
||||
if(picture_memoize === null) await setup_memoize();
|
||||
return await picture_memoize(...args);
|
||||
};
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
"use strict";
|
||||
|
||||
const crypto = require("crypto");
|
||||
|
||||
const htmlentities = require("html-entities");
|
||||
|
|
@ -6,7 +8,6 @@ const markdown = require("markdown-it")({
|
|||
});
|
||||
const chroma = require("chroma-js");
|
||||
|
||||
|
||||
const markdown_prism = require("markdown-it-prism");
|
||||
markdown.use(markdown_prism, {
|
||||
init: (Prism) => {
|
||||
|
|
|
|||
1658
.docs/package-lock.json
generated
1658
.docs/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -21,10 +21,15 @@
|
|||
"homepage": "https://github.com/sbrl/Minetest-WorldEditAdditions#readme",
|
||||
"devDependencies": {
|
||||
"@11ty/eleventy": "^0.12.1",
|
||||
"@11ty/eleventy-img": "^0.10.0",
|
||||
"chroma-js": "^2.1.2",
|
||||
"columnify": "^1.5.4",
|
||||
"debug": "^4.3.2",
|
||||
"imagickal": "^5.0.1",
|
||||
"markdown-it-prism": "^2.2.1",
|
||||
"phin": "^3.6.0"
|
||||
"p-memoize": "^6.0.1",
|
||||
"p-queue": "^7.1.0",
|
||||
"phin": "^3.6.0",
|
||||
"pretty-ms": "^7.0.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"html-entities": "^2.3.2"
|
||||
|
|
|
|||
Loading…
Reference in a new issue