Refactor //subdivide to call minetest.emerge_blocks

....whether it actually works or not is another question entirely 
though.....

worldeditadditions/init.lua

@@ -37,3 +37,5 @@ dofile(worldeditadditions.modpath.."/lib/bonemeal.lua")
 dofile(worldeditadditions.modpath.."/lib/forest.lua")
 
 dofile(worldeditadditions.modpath.."/lib/ellipsoidapply.lua")
+
+dofile(worldeditadditions.modpath.."/lib/subdivide.lua")

worldeditadditions/lib/subdivide.lua

@@ -0,0 +1,163 @@
+-- ███████ ██    ██ ██████  ██████  ██ ██    ██ ██ ██████  ███████
+-- ██      ██    ██ ██   ██ ██   ██ ██ ██    ██ ██ ██   ██ ██
+-- ███████ ██    ██ ██████  ██   ██ ██ ██    ██ ██ ██   ██ █████
+--      ██ ██    ██ ██   ██ ██   ██ ██  ██  ██  ██ ██   ██ ██
+-- ███████  ██████  ██████  ██████  ██   ████   ██ ██████  ███████
+local wea = worldeditadditions
+
+-- Counts the number of chunks in the given area.
+-- TODO: Do the maths properly here instead of using a loop - the loop is *very* inefficient - especially for large areas
+local function count_chunks(pos1, pos2, chunk_size)
+	local count = 0
+	for z = pos2.z, pos1.z, -chunk_size.z do
+		for y = pos2.y, pos1.y, -chunk_size.y do
+			for x = pos2.x, pos1.x, -chunk_size.x do
+				count = count + 1
+			end
+		end
+	end
+	return count
+end
+
+local function merge_stats(a, b)
+	for key,value in pairs(a) do
+		if not b[key] then b[key] = 0 end
+		b[key] = a[key] + b[key]
+	end
+end
+
+local function make_stats_obj(state)
+	return {
+		chunks_total = state.chunks_total,
+		chunks_completed = state.chunks_completed,
+		chunk_size = state.chunk_size,
+		volume_nodes = stats.volume_nodes,
+		emerge = state.stats_emerge,
+		times = state.times,
+		eta = state.eta,
+		emerge_overhead = state.emerge_overhead
+	}
+end
+
+local function subdivide_step_complete(state)
+	state.times.total = wea.get_ms_time() - state.times.start
+	
+	state.callback_complete(
+		state.pos1,
+		state.pos2,
+		make_stats_obj(state)
+	)
+end
+
+local function subdivide_step_beforeload(state)
+	state.cpos.z = state.cpos.z - (chunk_size.z + 1)
+	if state.cpos.z <= state.pos1.z then
+		state.cpoz.z = state.pos2.z
+		state.cpos.y = state.cpos.y - (chunk_size.y + 1)
+		if state.cpos.y <= state.pos1.y then
+			state.cpos.y = state.pos2.y
+			state.cpos.x = state.cpos.x - (chunk_size.x + 1)
+			if state.cpos.x <= state.pos1.x then
+				subdivide_step_complete(state)
+				return
+			end
+		end
+	end
+	
+	state.cpos2 = { x = state.cpos.x, y = state.cpos.y, z = state.cpos.z }
+	state.cpos1 = {
+		x = state.cpos.x - state.chunk_size.x,
+		y = state.cpos.y - state.chunk_size.y,
+		z = state.cpos.z - state.chunk_size.z
+	}
+	-- print("c1", wea.vector.tostring(c_pos1), "c2", wea.vector.tostring(c_pos2), "volume", worldedit.volume(c_pos1, c_pos2))
+	if state.cpos1.x < state.pos1.x then state.cpos1.x = state.pos1.x end
+	if state.cpos1.y < state.pos1.y then state.cpos1.y = state.pos1.y end
+	if state.cpos1.z < state.pos1.z then state.cpos1.z = state.pos1.z end
+	
+	state.times.emerge_last = wea.get_ms_time()
+	worldeditadditions.emerge_area(state.pos1, state.pos2, subdivide_step_afterload, state)
+end
+local function subdivide_step_afterload(state_emerge, state_ours)
+	state_ours.times.emerge_last = wea.get_ms_time() - state_ours.times.emerge_last
+	table.insert(state_ours.times.emerge, state_ours.times.emerge_last)
+	
+	merge_stats(state_emerge.stats, state_ours.stats_emerge)
+	
+	local callback_last = wea.get_ms_time()
+	state_ours.callback_subblock(
+		state_ours.cpos1,
+		state_ours.cpos2,
+		make_stats_obj(state_ours)
+	)
+	state_ours.times.callback_last = wea.get_ms_time() - callback_last
+	table.insert(state_ours.times.callback, state_ours.times.callback_last)
+	state_ours.chunks_completed = state_ours.chunks_completed + 1
+	
+	state_ours.times.step_last = wea.get_ms_time() - state_ours.times.step_start_abs
+	table.insert(state_ours.times.steps, state_ours.times.step_last)
+	state_ours.times.step_start_abs = wea.get_ms_time()
+	state_ours.eta = wea.eta(state_ours.times.steps, state_ours.chunks_total)
+	if state_ours.chunks_completed > 0 then
+		local total_steps = wea.sum(state_ours.times.steps)
+		local total_emerge = wea.sum(state_ours.times.emerge)
+		state_ours.emerge_overhead = total_emerge / total_steps
+	end
+	
+	minetest.after(0, subdivide_step_beforeload, state_ours)
+end
+
+--- Calls the given callback function once for each block in the defined area.
+-- This function is asynchronous, as it also uses minetest.emerge_area() to
+-- ensure that the blocks are loaded before calling the callback function.
+-- The callback functions will be passed the following arguments: pos1, pos2, stats
+-- pos1 and pos2 refer to the defined region of just the local block.
+-- stats is an table of statistics resembling the following:
+-- { chunks_completed, chunks_total, emerge = { ... }, times = { emerge = {}, emerge_last, callback = {}, callback_last, steps = {}, step_last } }
+-- The emerge property contains a table that holds a running total of statistics
+-- about what Minetest did to emerge the requested blocks in the world.
+-- callback_complete is called at the end of the process, and pos1 + pos2 will be set to that of the entire region.
+-- @param	{Vector}	pos1		The first position defining the area to emerge.
+-- @param	{Vector}	pos2		The second position defining the area to emerge.
+-- @param	{Vector}	chunk_size	The size of the chunks to subdivide into.
+-- @param	{function}	callback	The callback to call for each block.
+-- @param	{function}	callback	The callback to call upon completion.
+function worldeditadditions.subdivide(pos1, pos2, chunk_size, callback_subblock, callback_complete)
+	pos1, pos2 = worldedit.sort_pos(pos1, pos2)
+	
+	chunk_size.x = chunk_size.x - 1 -- WorldEdit regions are inclusive
+	chunk_size.y = chunk_size.y - 1 -- WorldEdit regions are inclusive
+	chunk_size.z = chunk_size.z - 1 -- WorldEdit regions are inclusive
+	
+	local state = {
+		pos1 = pos1, pos2 = pos2,
+		cpos = { x = pos2.x, y = pos2.y, z = pos2.z },
+		-- The size of a single subblock
+		chunk_size = chunk_size,
+		-- The total number of nodes in the defined region
+		volume_nodes = worldedit.volume(pos1, pos2),
+		stats_emerge = {},
+		times = {
+			-- Total time per step
+			steps = {}, step_last = 0, step_start_abs = wea.get_ms_time(),
+			-- Time per step spent on mineteest.emerge_area()
+			emerge = {}, emerge_last = 0,
+			-- Timme per step spent running the callback
+			callback = {}, callback_last = 0,
+			-- The start time (absolute)
+			start = wea.get_ms_time(),
+			-- The eta (in ms) until we're done
+			eta = 0
+		},
+		-- The percentage of the total time spent so far waiting for Minetest to emerge blocks. Updated every step.
+		emerge_overhead = 0,
+		-- The total number of chunks
+		chunks_total = count_chunks(pos1, pos2, chunk_size),
+		-- The number of chunks processed so far
+		chunks_completed = 0,
+		callback_subblock = callback_subblock,
+		callback_complete = callback_complete
+	}
+	
+	subdivide_step_beforeload(state)
+end

worldeditadditions/utils/nodes.lua

@@ -46,3 +46,45 @@ function worldeditadditions.weighted_to_list(node_weights)
 	end
 	return result
 end
+
+
+local function emerge_callback(pos, action, num_calls_remaining, state)
+	if not state.total then
+		state.total = num_calls_remaining + 1
+		state.loaded_blocks = 0
+	end
+	
+	state.loaded_blocks = state.loaded_blocks + 1
+	
+	if state.loaded_blocks == state.total then
+		state.callback(state, callback_state)
+	else
+		if action == minetest.EMERGE_CANCELLED then
+			state.stats.cancelled = state.stats.cancelled + 1
+		elseif action == minetest.EMERGE_ERRORED then
+			state.stats.error = state.stats.error + 1
+		elseif action == minetest.EMERGE_FROM_MEMORY then
+			state.stats.from_memory = state.stats.from_memory + 1
+		elseif action == minetest.EMERGE_FROM_DISK then
+			state.stats.from_disk = state.stats.from_disk + 1
+		elseif action == minetest.EMERGE_GENERATED then
+			state.stats.generated = state.stats.generated + 1
+		end
+	end
+end
+
+--- Loads the area defined by the specified region using minetest.emerge_area.
+-- Unlike minetest.emerge_area, this command calls the specified callback only
+-- once upon completion.
+-- @param	{Vector}	pos1		The first position defining the area to emerge.
+-- @param	{Vector}	pos2		The second position defining the area to emerge.
+-- @param	{function}	callback	The callback to call when the emerging process is complete.
+-- @param	{any}		callback_state	A state object to pass to the callback as a 2nd parameter (the 1st parameter is the emerge_area progress tracking state object)
+function worldeditadditions.emerge_area(pos1, pos2, callback, callback_state)
+	local state = {
+		stats = { cancelled = 0, error = 0, from_memory = 0, from_disk = 0, generated = 0 },
+		callback = callback,
+		callback_state = callback_state
+	}
+	minetest.emerge_area(pos1, pos2, emerge_callback, states)
+end

worldeditadditions/utils/numbers.lua

@@ -10,14 +10,19 @@ function worldeditadditions.hypotenuse(x1, y1, x2, y2)
 	return math.sqrt(xSquare + ySquare);
 end
 
-
+function worldeditadditions.sum(list)
-function worldeditadditions.average(list)
     if #list == 0 then return 0 end
 	local sum = 0
 	for i,value in ipairs(list) do
 		sum = sum + value
 	end
-	return sum / #list
+    return sum
+end
+
+
+function worldeditadditions.average(list)
+	if #list == 0 then return 0 end 
+	return worldeditadditions.sum(list) / #list
 end
 
 --- Returns the minetest.get_us_time() in ms

worldeditadditions_commands/commands/meta/subdivide.lua

@@ -19,18 +19,12 @@ local function will_trigger_saferegion(name, cmd_name, args)
 	return false
 end
 
--- Counts the number of chunks in the given area.
+local function emerge_stats_tostring(tbl_emerge)
--- TODO: Do the maths properly here instead of using a loop - the loop is *very* inefficient - especially for large areas
+	local result = {}
-local function count_chunks(pos1, pos2, chunk_size)
+	for key,value in pairs(tbl_emerge) do
-	local count = 0
+		table.insert(result, string.format("%s=%d", key, value))
-	for z = pos2.z, pos1.z, -chunk_size.z do
-		for y = pos2.y, pos1.y, -chunk_size.y do
-			for x = pos2.x, pos1.x, -chunk_size.x do
-				count = count + 1
 	end
-		end
+	return table.concat(result, ", ")
-	end
-	return count
 end
 
 worldedit.register_command("subdivide", {
@@ -72,95 +66,79 @@ worldedit.register_command("subdivide", {
 		return worldedit.volume(worldedit.pos1[name], worldedit.pos2[name])
 	end,
 	func = function(name, chunk_size, cmd_name, args)
-		local time_total = worldeditadditions.get_ms_time()
+		local time_total = wea.get_ms_time()
 		
 		local pos1, pos2 = worldedit.sort_pos(worldedit.pos1[name], worldedit.pos2[name])
 		local volume = worldedit.volume(pos1, pos2)
 		
 		local cmd = worldedit.registered_commands[cmd_name]
+		-- Note that we don't need to check for //multi privileges, as it does it at runtime
 		if not minetest.check_player_privs(name, cmd.privs) then
 			return false, "Error: Your privileges are unsufficient to run '"..cmd_name.."'."
 		end
 		
-		local i = 1
 		-- local chunks_total = math.ceil((pos2.x - pos1.x) / (chunk_size.x - 1))
 		-- 	* math.ceil((pos2.y - pos1.y) / (chunk_size.y - 1))
 		-- 	* math.ceil((pos2.z - pos1.z) / (chunk_size.z - 1))
-		local chunks_total = count_chunks(pos1, pos2, chunk_size)
 		
 		local msg_prefix = "[ subdivide | "..table.concat({cmd_name, args}, " ").." ] "
+		local time_last_msg = wea.get_ms_time()
 		
-		worldedit.player_notify(name,
+		wea.subdivide(cpos1, cpos2, chunk_size, function(bpos1, bpos2, stats)
-			msg_prefix.."Starting - "
+			-- Called on every subblock
-			-- ..wea.vector.tostring(pos1).." - "..wea.vector.tostring(pos2)
+			if stats.chunks_completed == 0 then
-			.." chunk size: "..wea.vector.tostring(chunk_size)
+				worldedit.player_notify(name, string.format(
-			..", "..chunks_total.." chunks total"
+					"%sStarting - chunk size: %s, %d chunks total (%d nodes)",
-			.." ("..volume.." nodes)"
+					msg_prefix,
-		)
+					wea.vector.tostring(stats.chunk_size),
+					stats.chunks_total,
+					stats.volume_nodes
+				))
+			end
 			
-		chunk_size.x = chunk_size.x - 1 -- WorldEdit regions are inclusive
-		chunk_size.y = chunk_size.y - 1 -- WorldEdit regions are inclusive
-		chunk_size.z = chunk_size.z - 1 -- WorldEdit regions are inclusive
-		
-		
-		local time_last_msg = worldeditadditions.get_ms_time()
-		local time_chunks = {}
-		for z = pos2.z, pos1.z, -(chunk_size.z + 1) do
-			for y = pos2.y, pos1.y, -(chunk_size.y + 1) do
-				for x = pos2.x, pos1.x, -(chunk_size.x + 1) do
-					local c_pos2 = { x = x, y = y, z = z }
-					local c_pos1 = {
-						x = x - chunk_size.x,
-						y = y - chunk_size.y,
-						z = z - chunk_size.z
-					}
-					-- print("c1", wea.vector.tostring(c_pos1), "c2", wea.vector.tostring(c_pos2), "volume", worldedit.volume(c_pos1, c_pos2))
-					if c_pos1.x < pos1.x then c_pos1.x = pos1.x end
-					if c_pos1.y < pos1.y then c_pos1.y = pos1.y end
-					if c_pos1.z < pos1.z then c_pos1.z = pos1.z end
-					
-					local time_this = worldeditadditions.get_ms_time()
 			worldedit.player_notify_suppress(name)
-					worldedit.pos1[name] = c_pos1
+			worldedit.pos1[name] = cpos1
-					worldedit.pos2[name] = c_pos2
+			worldedit.pos2[name] = cpos2
 			cmd.func(name, args)
 			if will_trigger_saferegion(name, cmd_name, args) then
 				minetest.chatcommands["/y"].func()
 			end
 			worldedit.player_notify_unsuppress(name)
-					time_this = worldeditadditions.get_ms_time() - time_this
-					table.insert(time_chunks, time_this)
-					
-					local time_average = wea.average(time_chunks)
-					local eta = (chunks_total - i) * time_average
-					-- print("eta", eta, "time_average", time_average, "chunks_left", chunks_total - i)
 			
 			-- Send updates every 2 seconds, and after the first 3 chunks are done
-					if worldeditadditions.get_ms_time() - time_last_msg > 2 * 1000 or i == 3 or i == chunks_total then
+			if worldeditadditions.get_ms_time() - time_last_msg > 2 * 1000 or i == 3 or i == stats.chunks_total then
 				worldedit.player_notify(name,
-							string.format("%s%d / %d (~%.2f%%) complete | last chunk: %s, average: %s, ETA: ~%s",
+					string.format("%s%d / %d (~%.2f%%) complete | last chunk: %s, average: %s, %.2f%% emerge overhead, ETA: ~%s",
 						msg_prefix,
-								i, chunks_total,
+						stats.chunks_completed, stats.chunks_total,
-								(i / chunks_total) * 100,
+						(stats.chunks_completed / stats.chunks_total) * 100,
-								wea.human_time(time_this),
+						wea.human_time(stats.times.step_last),
-								wea.human_time(time_average),
+						wea.human_time(wea.average(stats.times.steps)),
-								wea.human_time(eta)
+						stats.emerge_overhead * 100,
+						wea.human_time(stats.eta)
 					)
 				)
-						time_last_msg = worldeditadditions.get_ms_time()
+				time_last_msg = wea.get_ms_time()
 			end
+		end, function(_, _, stats)
+			-- Called on completion
+			minetest.log("action", string.format("%s used //subdivide at %s - %s, with $d chunks and %d total nodes in %s",
+				name,
+				wea.vector.tostring(pos1),
+				wea.vector.tostring(pos2),
+				stats.chunks_completed,
+				stats.volume_nodes,
+				wea.human_time(stats.times.total)
+			))
+			return true, string.format(
+				"%sComplete: %d chunks processed in %s (%.2f%% emerge overhead, emerge totals: %s)",
+				msg_prefix,
+				stats.chunks_completed,
+				wea.human_time(stats.time.total),
+				stats.emerge_overhead * 100,
+				emerge_stats_tostring(stats.emerge)
+			)
+		end)
 		
-					i = i + 1
-				end
-			end
-		end
-		i = i - 1
-		worldedit.pos1[name] = pos1
-		worldedit.pos2[name] = pos2
-		time_total = worldeditadditions.get_ms_time() - time_total
-		
-		
-		minetest.log("action", name.." used //subdivide at "..wea.vector.tostring(pos1).." - "..wea.vector.tostring(pos2)..", with "..i.." chunks and "..worldedit.volume(pos1, pos2).." total nodes in "..time_total.."s")
-		return true, msg_prefix.."Complete: "..i.." chunks processed in "..wea.human_time(time_total)
 	end
 })