//subdivide: tally total times so far as we go

- Don't use wea.sum() on a table full of all the times we've recorded
   so far
 - Limit individual stats list lengths to 25 to avoid memory issues when
   running with a large number of chunks
 - Tally total times / metrics as we go to avoid memory issues
This commit is contained in:
Starbeamrainbowlabs 2021-02-03 03:04:33 +00:00
parent 25e659321a
commit 38e5c81d33
Signed by: sbrl
GPG key ID: 1BE5172E637709C2
4 changed files with 40 additions and 5 deletions

View file

@ -7,6 +7,8 @@ It's about time I started a changelog! This will serve from now on as the main c
- Tip: Use a monospace font for the chat window, and the columns will be aligned! - Tip: Use a monospace font for the chat window, and the columns will be aligned!
- Add `//hollow` for hollowing out areas (a step towards parity with Minecraft WorldEdit) - Add `//hollow` for hollowing out areas (a step towards parity with Minecraft WorldEdit)
- `//subdivide`: Improve performance of initial chunk counting algorithm - it should get started on the job _much_ quicker now (especially on large regions) - `//subdivide`: Improve performance of initial chunk counting algorithm - it should get started on the job _much_ quicker now (especially on large regions)
- `//subdivide`: Fix performance & memory usage of metrics over time on large regions
- No more slowdowns over time with a large number of chunks!
- Bugfix: Fix obscure crash in calls to `human_size` ("unknown" will now be returned if passed junk) - Bugfix: Fix obscure crash in calls to `human_size` ("unknown" will now be returned if passed junk)

View file

@ -85,6 +85,10 @@ end
local function subdivide_step_afterload(state_emerge, state_ours) local function subdivide_step_afterload(state_emerge, state_ours)
state_ours.times.emerge_last = wea.get_ms_time() - state_ours.times.emerge_last state_ours.times.emerge_last = wea.get_ms_time() - state_ours.times.emerge_last
table.insert(state_ours.times.emerge, state_ours.times.emerge_last) table.insert(state_ours.times.emerge, state_ours.times.emerge_last)
if #state_ours.times.emerge > 25 then
state_ours.times.emerge = wea.table_get_last(state_ours.times.emerge, 25)
end
state_ours.times.emerge_total = state_ours.times.emerge_total + state_ours.times.emerge_last
merge_stats(state_emerge.stats, state_ours.stats_emerge) merge_stats(state_emerge.stats, state_ours.stats_emerge)
@ -100,12 +104,14 @@ local function subdivide_step_afterload(state_emerge, state_ours)
state_ours.times.step_last = wea.get_ms_time() - state_ours.times.step_start_abs state_ours.times.step_last = wea.get_ms_time() - state_ours.times.step_start_abs
table.insert(state_ours.times.steps, state_ours.times.step_last) table.insert(state_ours.times.steps, state_ours.times.step_last)
if #state_ours.times.steps > 25 then
state_ours.times.steps = wea.table_get_last(state_ours.times.steps, 25)
end
state_ours.times.steps_total = state_ours.times.steps_total + state_ours.times.step_last
state_ours.times.step_start_abs = wea.get_ms_time() state_ours.times.step_start_abs = wea.get_ms_time()
state_ours.eta = wea.eta(state_ours.times.steps, state_ours.chunks_total) state_ours.eta = wea.eta(state_ours.times.steps, state_ours.chunks_total)
if state_ours.chunks_completed > 0 then if state_ours.chunks_completed > 0 then
local total_steps = wea.sum(state_ours.times.steps) state_ours.emerge_overhead = state_ours.times.emerge_total / state_ours.times.steps_total
local total_emerge = wea.sum(state_ours.times.emerge)
state_ours.emerge_overhead = total_emerge / total_steps
end end
minetest.after(0, state_ours.__beforeload, state_ours) minetest.after(0, state_ours.__beforeload, state_ours)
@ -145,8 +151,10 @@ function worldeditadditions.subdivide(pos1, pos2, chunk_size, callback_subblock,
stats_emerge = {}, stats_emerge = {},
times = { times = {
-- Total time per step -- Total time per step
steps_total = 0,
steps = {}, step_last = 0, step_start_abs = wea.get_ms_time(), steps = {}, step_last = 0, step_start_abs = wea.get_ms_time(),
-- Time per step spent on mineteest.emerge_area() -- Time per step spent on mineteest.emerge_area()
emerge_total = 0,
emerge = {}, emerge_last = 0, emerge = {}, emerge_last = 0,
-- Timme per step spent running the callback -- Timme per step spent running the callback
callback = {}, callback_last = 0, callback = {}, callback_last = 0,

View file

@ -21,6 +21,7 @@ end
function worldeditadditions.average(list) function worldeditadditions.average(list)
print("[DEBUG] averaging "..#list.." numbers")
if #list == 0 then return 0 end if #list == 0 then return 0 end
return worldeditadditions.sum(list) / #list return worldeditadditions.sum(list) / #list
end end
@ -32,7 +33,10 @@ function worldeditadditions.get_ms_time()
end end
function worldeditadditions.eta(existing_times, times_total_count) function worldeditadditions.eta(existing_times, times_total_count)
local average = worldeditadditions.average(existing_times) print("[DEBUG] eta got "..#existing_times.." numbers")
local average = worldeditadditions.average(
worldeditadditions.table_get_last(existing_times, 25)
)
local times_left = times_total_count - #existing_times local times_left = times_total_count - #existing_times
if times_left == 0 then return 0 end if times_left == 0 then return 0 end
return average * times_left return average * times_left

View file

@ -16,7 +16,7 @@ function worldeditadditions.shallowcopy(orig)
return copy return copy
end end
--- SHALLOWLY applies the values in source to overwrite the equivalent keys in target. --- SHALLOW ONLY - applies the values in source to overwrite the equivalent keys in target.
-- Warning: This function mutates target! -- Warning: This function mutates target!
-- @param source table The source to take values from -- @param source table The source to take values from
-- @param target table The target to write values to -- @param target table The target to write values to
@ -25,3 +25,24 @@ function worldeditadditions.table_apply(source, target)
target[key] = value target[key] = value
end end
end end
--- Polyfill for unpack / table.unpack.
-- Calls unpack when available, and looks for table.unpack if unpack() isn't
-- found.
-- This is needed because in Lua 5.1 it's the global unpack(), but in Lua 5.4
-- it's moved to table.unpack().
function worldeditadditions.table_unpack(tbl, offset, count)
if type(unpack) == "function" then
return unpack(tbl, offset, count)
else
return table.unpack(tbl, offset, count)
end
end
--- Returns only the last count items in a given numerical table-based list.
function worldeditadditions.table_get_last(tbl, count)
return {worldeditadditions.table_unpack(
tbl,
math.max(0, (#tbl) - (count - 1))
)}
end