//subdivide: tally total times so far as we go

- Don't use wea.sum() on a table full of all the times we've recorded
   so far
 - Limit individual stats list lengths to 25 to avoid memory issues when
   running with a large number of chunks
 - Tally total times / metrics as we go to avoid memory issues

CHANGELOG.md

@@ -7,6 +7,8 @@ It's about time I started a changelog! This will serve from now on as the main c
      - Tip: Use a monospace font for the chat window, and the columns will be aligned!
  - Add `//hollow` for hollowing out areas (a step towards parity with Minecraft WorldEdit)
  - `//subdivide`: Improve performance of initial chunk counting algorithm - it should get started on the job _much_ quicker now (especially on large regions)
+ - `//subdivide`: Fix performance & memory usage of metrics over time on large regions
+     - No more slowdowns over time with a large number of chunks!
  - Bugfix: Fix obscure crash in calls to `human_size` ("unknown" will now be returned if passed junk)
 
 

worldeditadditions/lib/subdivide.lua

@@ -85,6 +85,10 @@ end
 local function subdivide_step_afterload(state_emerge, state_ours)
 	state_ours.times.emerge_last = wea.get_ms_time() - state_ours.times.emerge_last
 	table.insert(state_ours.times.emerge, state_ours.times.emerge_last)
+	if #state_ours.times.emerge > 25 then
+		state_ours.times.emerge = wea.table_get_last(state_ours.times.emerge, 25)
+	end
+	state_ours.times.emerge_total = state_ours.times.emerge_total + state_ours.times.emerge_last
 	
 	merge_stats(state_emerge.stats, state_ours.stats_emerge)
 	
@@ -100,12 +104,14 @@ local function subdivide_step_afterload(state_emerge, state_ours)
 	
 	state_ours.times.step_last = wea.get_ms_time() - state_ours.times.step_start_abs
 	table.insert(state_ours.times.steps, state_ours.times.step_last)
+	if #state_ours.times.steps > 25 then
+		state_ours.times.steps = wea.table_get_last(state_ours.times.steps, 25)
+	end
+	state_ours.times.steps_total = state_ours.times.steps_total + state_ours.times.step_last
 	state_ours.times.step_start_abs = wea.get_ms_time()
 	state_ours.eta = wea.eta(state_ours.times.steps, state_ours.chunks_total)
 	if state_ours.chunks_completed > 0 then
-		local total_steps = wea.sum(state_ours.times.steps)
-		local total_emerge = wea.sum(state_ours.times.emerge)
-		state_ours.emerge_overhead = total_emerge / total_steps
+		state_ours.emerge_overhead = state_ours.times.emerge_total / state_ours.times.steps_total
 	end
 	
 	minetest.after(0, state_ours.__beforeload, state_ours)
@@ -145,8 +151,10 @@ function worldeditadditions.subdivide(pos1, pos2, chunk_size, callback_subblock,
 		stats_emerge = {},
 		times = {
 			-- Total time per step
+			steps_total = 0,
 			steps = {}, step_last = 0, step_start_abs = wea.get_ms_time(),
 			-- Time per step spent on mineteest.emerge_area()
+			emerge_total = 0,
 			emerge = {}, emerge_last = 0,
 			-- Timme per step spent running the callback
 			callback = {}, callback_last = 0,

worldeditadditions/utils/numbers.lua

@@ -21,6 +21,7 @@ end
 
 
 function worldeditadditions.average(list)
+    print("[DEBUG] averaging "..#list.." numbers")
 	if #list == 0 then return 0 end 
 	return worldeditadditions.sum(list) / #list
 end
@@ -32,7 +33,10 @@ function worldeditadditions.get_ms_time()
 end
 
 function worldeditadditions.eta(existing_times, times_total_count)
-    local average = worldeditadditions.average(existing_times)
+    print("[DEBUG] eta got "..#existing_times.." numbers")
+    local average = worldeditadditions.average(
+        worldeditadditions.table_get_last(existing_times, 25)
+    )
     local times_left = times_total_count - #existing_times
     if times_left == 0 then return 0 end
     return average * times_left

worldeditadditions/utils/tables.lua

@@ -16,7 +16,7 @@ function worldeditadditions.shallowcopy(orig)
 	return copy
 end
 
---- SHALLOWLY applies the values in source to overwrite the equivalent keys in target.
+--- SHALLOW ONLY - applies the values in source to overwrite the equivalent keys in target.
 -- Warning: This function mutates target!
 -- @param	source	table	The source to take values from
 -- @param	target	table	The target to write values to
@@ -25,3 +25,24 @@ function worldeditadditions.table_apply(source, target)
 		target[key] = value
 	end
 end
+
+--- Polyfill for unpack / table.unpack.
+-- Calls unpack when available, and looks for table.unpack if unpack() isn't
+-- found.
+-- This is needed because in Lua 5.1 it's the global unpack(), but in Lua 5.4
+-- it's moved to table.unpack().
+function worldeditadditions.table_unpack(tbl, offset, count)
+	if type(unpack) == "function" then
+		return unpack(tbl, offset, count)
+	else
+		return table.unpack(tbl, offset, count)
+	end
+end
+
+--- Returns only the last count items in a given numerical table-based list.
+function worldeditadditions.table_get_last(tbl, count)
+	return {worldeditadditions.table_unpack(
+		tbl,
+		math.max(0, (#tbl) - (count - 1))
+	)}
+end