Add ART loader for the LuaJIT BUILD struct loader module, 2 more examples.

The map iterator now has init/finish capability, making it possible to
write scripts that aggregate data over multiple map files.  One such example
calculates some statistics, the other loads art metadata and looks for
red walls with non-pow2 ysize tiles.

git-svn-id: https://svn.eduke32.com/eduke32@2814 1a8010ca-5511-0410-912e-c29ae57300e0

polymer/eduke32/source/lunatic/build.lua

@@ -1,14 +1,14 @@
 
 -- Loaders for various BUILD structures for LuaJIT
 
--- TODO: bound-checking, load ART
-
 
 local ffi = require "ffi"
 local io = require "io"
 
+local error = error
 local assert = assert
 local print = print
+local setmetatable = setmetatable
 
 module(...)
 
@@ -62,6 +62,8 @@ local MAX =
     SECTORS = { [7]=1024, [8]=4096, [9]=4096 },
     WALLS = { [7]=8192, [8]=16384, [9]=16384 },
     SPRITES = { [7]=4096, [8]=16384, [9]=16384 },
+
+    TILES = 30720,
 }
 
 local function doread(fh, basectype, numelts)
@@ -85,6 +87,23 @@ local function doread(fh, basectype, numelts)
     return cd
 end
 
+local function set_secwalspr_mt(structar, maxidx)
+    local mt = {
+        __index = function(tab, idx)
+            if (idx < 0 or idx >= maxidx) then
+                error("Invalid structure array read access", 2)
+            end
+            return structar[idx]
+        end,
+
+        __newindex = function(tab, idx, newval)
+            error('cannot write directly to structure array', 2)
+        end,
+    }
+
+    return setmetatable({}, mt)
+end
+
 --== LOADBOARD ==--
 -- returns:
 --  on failure, nil, errmsg
@@ -133,6 +152,9 @@ function loadboard(filename)
 
     -- sectors
     map.numsectors = cd[2]
+    if (map.numsectors == nil) then
+        return nil, "Couldn't read number of sectors"
+    end
     if (map.numsectors <= 0 or map.numsectors > MAX.SECTORS[map.version]) then
         fh:close()
         return nil, "Invalid number of sectors"
@@ -145,6 +167,9 @@ function loadboard(filename)
 
     -- walls
     cd = doread(fh, "int16_t", 1)
+    if (cd == nil) then
+        return nil, "Couldn't read number of walls"
+    end
     map.numwalls = cd[0]
     if (map.numwalls <= 0 or map.numwalls > MAX.WALLS[map.version]) then
         fh:close()
@@ -158,6 +183,9 @@ function loadboard(filename)
 
     -- sprites
     cd = doread(fh, "int16_t", 1)
+    if (cd == nil) then
+        return nil, "Couldn't read number of sprites"
+    end
     map.numsprites = cd[0]
     if (map.numsprites < 0 or map.numsprites > MAX.SPRITES[map.version]) then
         fh:close()
@@ -169,7 +197,100 @@ function loadboard(filename)
         return nil, "Couldn't read sprites"
     end
 
+    map.sector = set_secwalspr_mt(map.sector, map.numsectors)
+    map.wall = set_secwalspr_mt(map.wall, map.numwalls)
+    map.sprite = set_secwalspr_mt(map.sprite, map.numsprites)
+
     -- done
     fh:close()
     return map
 end
+
+
+local function set_sizarray_mt(sizar)
+    local mt = {
+        __index = function(tab, idx)
+            if (idx < 0 or idx >= MAX.TILES) then
+                error("Invalid tile size array read access", 2)
+            end
+            return sizar[idx]
+        end,
+
+        __newindex = function(tab, idx, newval)
+            if (idx < 0 or idx >= MAX.TILES) then
+                error("Invalid tile size array write access", 2)
+            end
+            sizar[idx] = newval
+        end,
+    }
+
+    return setmetatable({}, mt)
+end
+
+
+--== LOADARTS (currently tilesizx and tilesizy only) ==--
+-- filenames: a table with ART file names
+-- returns:
+--  on failure, nil, errmsg
+--  on success, a table
+--    {
+--      sizx = <cdata (array of length MAXTILES)>
+--      sizy = <cdata (array of length MAXTILES)>
+--    }
+function loadarts(filenames)
+    local tile = {
+        sizx = ffi.new("int16_t [?]", MAX.TILES),
+        sizy = ffi.new("int16_t [?]", MAX.TILES),
+    }
+
+    for fni=1,#filenames do
+        local fn = filenames[fni]
+        local fh, errmsg = io.open(fn)
+
+        if (fh==nil) then
+            return nil, errmsg
+        end
+
+        local cd = doread(fh, "int32_t", 4)
+        -- artversion, numtiles, localtilestart, localtileend
+        if (cd==nil) then
+            fh:close()
+            return nil, fn..": Couldn't read header"
+        end
+
+        local localtilestart = cd[2]
+        local numtileshere = cd[3]-localtilestart
+
+        if (numtileshere < 0 or localtilestart+numtileshere >= MAX.TILES) then
+            fh:close()
+            return nil, fn..": Invalid tile start/end"
+        end
+
+        if (numtileshere==0) then
+            fh:close()
+        else
+            -- X sizes
+            cd = doread(fh, "int16_t", numtileshere)
+            if (cd == nil) then
+                fh:close()
+                return nil, fn..": Couldn't read tilesizx array"
+            end
+
+            ffi.copy(tile.sizx+localtilestart, cd, numtileshere*2)
+
+            -- Y sizes
+            cd = doread(fh, "int16_t", numtileshere)
+            if (cd == nil) then
+                fh:close()
+                return nil, fn..": Couldn't read tilesizy array"
+            end
+
+            ffi.copy(tile.sizy+localtilestart, cd, numtileshere*2)
+        end
+    end
+
+    tile.sizx = set_sizarray_mt(tile.sizx)
+    tile.sizy = set_sizarray_mt(tile.sizy)
+
+    return tile
+end

polymer/eduke32/source/lunatic/foreachmap.lua

@@ -4,23 +4,35 @@
 
 -- The first cmdline arg is a name of a lua file (may be sans .lua) which must
 -- be a module and is `require'd into this script, e.g. "stats" or "stats.lua".
--- Then, for each 2nd and following argument, if map loading succeeds,
+-- First, a key named .init is looked up in the loaded module and if it exists,
--- .success(map, filename) is run, otherwise .failure(filename, errmsg) --
+-- it is run like .init(arg)  (thus allowing it to parse the command-line
--- these must be functions in the loaded module.
+-- arguments and then potentially remove the ones it used).
+-- If .init returns non-nil, this script aborts.
+-- Otherwise, for each 2nd and following argument, if map loading succeeds,
+--  .success(map, filename) is run, otherwise
+--  .failure(filename, errmsg) is run if that key exists, or a standard error
+--  message is printed to stderr.
+-- Finally, if there is a .finish field in the module, it is run with no args.
 
--- TODO: aggregate batch mode
-
-
-if (#arg < 1) then
-    error("Usage: luajit ./foreachmap <module[.lua]> <filename.map> ...")
-end
 
 local B = require "build"
 local string = require "string"
+local io = require "io"
+local os = require "os"
+
+if (#arg < 1) then
+    io.stdout:write("Usage: luajit ./foreachmap <module[.lua]> [init args...] <filename.map> ...\n\n")
+    return
+end
 
 local modname = string.gsub(arg[1], "%.lua$", "")
 local mod = require(modname)
 
+if (mod.init) then
+    if (mod.init(arg) ~= nil) then
+        os.exit(1)
+    end
+end
 
 for i=2,#arg do
     local fn = arg[i]
@@ -29,6 +41,14 @@ for i=2,#arg do
     if (map ~= nil) then
         mod.success(map, fn)
     else
-        mod.failure(fn, errmsg)
+        if (mod.failure) then
+            mod.failure(fn, errmsg)
+        else
+            io.stderr:write(string.format("--- %s: %s\n", fn, errmsg))
+        end
     end
 end
+
+if (mod.finish) then
+    mod.finish()
+end

polymer/eduke32/source/lunatic/mapastats.lua

@@ -0,0 +1,50 @@
+
+-- Print out some aggregate statistics for passed BUILD maps,
+-- foreachmap module.
+
+local string = require "string"
+local math = require "math"
+
+local print = print
+
+module(...)
+
+
+local function printf(fmt, ...)
+    print(string.format(fmt, ...))
+end
+
+
+local N = 0
+
+local sumnumsectors = 0
+local sumnumwalls = 0
+
+local sumratio = 0
+local sumsqratio = 0
+
+function success(map, fn)
+    local ns = map.numsectors
+    local nw = map.numwalls
+
+    N = N+1
+
+    sumratio = sumratio + nw/ns
+    sumsqratio = sumsqratio + (nw/ns)^2
+
+    sumnumsectors = sumnumsectors+ns
+    sumnumwalls = sumnumwalls+nw
+end
+
+function finish()
+    printf("%d maps\n", N)
+    printf("total sectors: %d", sumnumsectors)
+    printf("total walls: %d", sumnumwalls)
+    printf("total walls / total sectors: %.02f", sumnumwalls/sumnumsectors)
+    printf("")
+    printf("Walls/sector")
+
+    local meanwpers = sumratio/N
+    printf(" mean: %.02f", meanwpers)
+    printf(" stdev: %.02f", math.sqrt(sumsqratio/N - meanwpers^2))
+end

polymer/eduke32/source/lunatic/mapstats.lua

@@ -19,8 +19,7 @@ function success(map, fn)
     printf("  version: %d", map.version)
     printf("  numsectors: %d\n  numwalls: %d\n  numsprites: %d",
            map.numsectors, map.numwalls, map.numsprites)
-end
+    printf("  walls/sector: %.02f\n  sprites/sector: %.02f",
-
+          map.numwalls/map.numsectors, map.numsprites/map.numsectors)
-function failure(fn, errmsg)
+    printf("")
-    printf("--- %s: %s", fn, errmsg)
 end

polymer/eduke32/source/lunatic/mapypan.lua

@@ -0,0 +1,164 @@
+
+-- Display information about problematic wall ypannings,
+-- foreachmap module.
+
+local string = require "string"
+local table = require "table"
+local bit = require "bit"
+
+local print = print
+local pairs = pairs
+local rawget = rawget
+local setmetatable = setmetatable
+
+local B = require "build"
+
+module(...)
+
+
+local function printf(fmt, ...)
+    print(string.format(fmt, ...))
+end
+
+
+local tile
+
+function init(arg)
+    local artargend = nil
+    for i=2,#arg do
+        if (arg[i]=="--") then
+            artargend = i
+            break
+        end
+    end
+
+    if (artargend==nil or artargend==0) then
+        printf("Usage: luajit ./foreachmap.lua <tilesXXX.ART> [, ...] -- <filename1.map> ...\n")
+        return 1
+    end
+
+    local artfns = {}
+    for i=2,artargend-1 do
+        artfns[#artfns+1] = arg[i]
+    end
+
+    local i = 2
+    local j = artargend+1
+
+    while (arg[j]) do
+        arg[i] = arg[j]
+        arg[j] = nil
+
+        i = i+1
+        j = j+1
+    end
+
+    local tile_, errmsg = B.loadarts(artfns)
+    if (tile_ == nil) then
+        printf("%s", errmsg)
+        return 2
+    end
+
+    tile = tile_  -- set 'tile' file-scope var
+end
+
+local table_default0_mt = {
+    __index = function(tab, idx)
+        if (rawget(tab, idx)==nil) then
+            return 0
+        end
+        return rawget(tab, idx)
+    end
+}
+
+
+local sector, wall, sprite
+
+function success(map, fn)
+    -- set file-scope vars for convenience
+    sector = map.sector
+    wall = map.wall
+    sprite = map.sprite
+
+    -- counts of non-pow2 tiles, per tilenum
+    local np2tile = setmetatable({}, table_default0_mt)
+    -- walls/overwall indices with non-pow2 tiles, [walidx]=true
+    local np2wall = {}
+    -- [i]=wallidx
+    local np2walls = {}
+
+    local badoverpicnum = false
+
+    for i=0,map.numsectors-1 do
+        local startwall = sector[i].wallptr
+        local endwall = startwall+sector[i].wallnum-1
+
+        for w=startwall,endwall do
+            for n=1,2 do
+                local pic, ysiz
+
+                if (wall[w].nextwall < 0) then
+                    -- We don't care for white walls
+                elseif (n==1) then
+                    pic = wall[w].picnum
+                    ysiz = tile.sizy[pic]
+                else
+                    pic = wall[w].overpicnum
+                    if (pic < 0 or pic > 30720) then  -- MAXTILES
+                        badoverpicnum = true
+                    else
+                        ysiz = tile.sizy[pic]
+
+                        if (bit.band(wall[w].cstat, 16+32)==0) then
+                            -- we don't care about non-masked/1-way walls
+                            ysiz = 0
+                        end
+                    end
+                end
+
+                if (ysiz~=nil and ysiz > 0 and bit.band(ysiz, bit.bnot(ysiz-1))~=ysiz) then
+                    -- non-pow2 ysize
+
+                    np2tile[pic] = np2tile[pic]+1
+
+                    if (not np2wall[w]) then
+                        np2wall[w] = true
+                        np2walls[#np2walls+1] = w
+                    end
+                end
+            end
+        end
+    end
+
+    -- report our findings
+
+    -- sort in wall index order
+    table.sort(np2walls)
+
+    printf("--- %s:", fn)
+
+--[[
+    printf(" Walls:")
+    for i=1,#np2walls do
+        printf("  %d", np2walls[i])
+    end
+    printf("")
+--]]
+    printf(" %d red walls with non-pow2 ysize tiles", #np2walls)
+    if (badoverpicnum) then
+        printf(" (some red walls have out-of-bounds overpicnums)")
+    end
+
+    local np2tiles = {}
+    for tilenum,_ in pairs(np2tile) do
+        np2tiles[#np2tiles+1] = tilenum
+    end
+    table.sort(np2tiles)
+
+    printf(" Tiles:")
+    for i=1,#np2tiles do
+        printf("  %d", np2tiles[i])
+    end
+
+    printf("")
+end