Lunatic: major overhaul of gamevar serialization for savegames.

- Handle local gamevars.
- Restore gamevars from require('end_gamevars').

git-svn-id: https://svn.eduke32.com/eduke32@3891 1a8010ca-5511-0410-912e-c29ae57300e0

polymer/eduke32/source/lunatic/defs.ilua

@@ -9,6 +9,7 @@ local ffiC = ffi.C
 local CF = CF
 
 local bit = bit
+local coroutine = coroutine
 local string = string
 local table = table
 local math = math
@@ -578,6 +579,8 @@ int32_t g_elCONSize;
 char *g_elCON;
 void El_SetCON(const char *conluacode);
 
+char *g_elSavecode;
+void El_FreeSaveCode(void);
 const char *(*El_SerializeGamevars)(int32_t *slenptr);
 
 const char *s_buildRev;
@@ -1385,6 +1388,7 @@ local package_loaded = {}  -- [<modname>] = false/true/table
 local modname_stack = {}  -- [<depth>]=string
 local module_gamevars = {}  -- [<modname>] = { <gvname1>, <gvname2>, ... }
 local module_gvlocali = {}  -- [<modname>] = { <localidx_beg>, <localidx_end> }
+local module_thread = {}  -- [<modname>] = <module_thread>
 
 local function getcurmodname(thisfuncname)
     if (#modname_stack == 0) then
@@ -1421,7 +1425,7 @@ local function getnumlocals(l)
     for i=1,200 do
         -- level:
         -- 0 is getlocal() itself.
-        -- 1 is this function.
+        -- 1 is this function (getnumlocals).
         -- 2 is the function calling getnumlocals()
         -- 3 is the function calling that one.
         if (debug.getlocal(3, i) == nil) then
@@ -1438,6 +1442,27 @@ local required_module_mt = {
     __metatable = true,
 }
 
+-- Will contain a function to restore gamevars when running from savegame
+-- restoration. See SAVEFUNC_ARGS for its arguments.
+local g_restorefunc = nil
+
+-- Local gamevar restoration function run from
+-- our_require('end_gamevars') <- [user module].
+local function restore_local(li, lval)
+    -- level:
+    -- 0 is getlocal() itself.
+    -- 1 is this function (restore_local).
+    -- 2 is the function calling restore_local(), the savecode.
+    -- 3 is the function calling the savecode, our_require.
+    -- 4 is the function calling our_require, the module function.
+    if (ffiC._DEBUG_LUNATIC ~= 0) then
+        printf("Restoring index #%d (%s) with value %s",
+               li, debug.getlocal(4, li), tostring(lval))
+    end
+
+    assert(debug.setlocal(4, li, lval))
+end
+
 -- The "require" function accessible to Lunatic code.
 -- Base modules in allowed_modules are wrapped so that they cannot be
 -- modified, user modules are searched in the EDuke32 search
@@ -1475,11 +1500,24 @@ local function our_require(modname, ...)
         gvmodi[2] = getnumlocals()
 
         if (ffiC._DEBUG_LUNATIC ~= 0) then
-            printf("Module '%s' has %d locals, index %d to %d", thismodname,
+            local numlocals = gvmodi[2]-gvmodi[1]+1
-                   gvmodi[2]-gvmodi[1]+1, gvmodi[1], gvmodi[2])
+            if (numlocals > 0) then
+                printf("Module '%s' has %d locals, index %d to %d",
+                       thismodname, numlocals, gvmodi[1], gvmodi[2])
+            end
         end
 
-        return
+        -- Potentially restore gamevars.
+        if (g_restorefunc) then
+            local modtab = package_loaded[thismodname]
+            assert(type(modtab)=="table")
+            -- SAVEFUNC_ARGS.
+            g_restorefunc(thismodname, modtab, restore_local)
+        end
+
+        -- Return whether we're NOT running from a savegame restore in the
+        -- second outarg. (Lunatic-private!)
+        return nil, (g_restorefunc==nil)
     end
 
     -- See whether it's a base module name.
@@ -1513,6 +1551,10 @@ local function our_require(modname, ...)
         errorf(ERRLEV-1, "Couldn't open file \"%s\"", modfn)
     end
 
+    -- Implant code that yields the module thread just before it would return
+    -- otherwise.
+    str = str.."\nrequire('coroutine').yield()"
+
     local modfunc, errmsg = loadstring(str, modfn)
     if (modfunc == nil) then
         errorf(ERRLEV-1, "Couldn't load \"%s\": %s", modname, errmsg)
@@ -1521,19 +1563,49 @@ local function our_require(modname, ...)
     package_loaded[modname] = false  -- 'not yet loaded'
     table.insert(modname_stack, modname)
 
-    -- Run the module code!
+    -- Run the module code in a separate Lua thread!
-    modfunc(modname, ...)  -- TODO: call protected and report errors here later?
+    local modthread = coroutine.create(modfunc)
+    local ok, retval = coroutine.resume(modthread, modname, ...)
+
+    if (not ok) then
+        errorf(ERRLEV-1, "Failed running \"%s\": %s", modname, retval)
+    end
 
     table.remove(modname_stack)
 
     local modtab = package_loaded[modname]
 
-    if (type(modtab) == "table") then
+    if (type(modtab) ~= "table") then
-        -- Protect module table if there is one...
+        -- The module didn't call our 'module'. Check if it returned a table.
-        setmetatable(modtab, required_module_mt)
+        -- In that case, the coroutine has finished its main function and has
+        -- not reached our implanted 'yield'.
+        if (coroutine.status(modthread)=="dead" and type(retval)=="table") then
+            modtab = retval
+            package_loaded[modname] = modtab
         else
             package_loaded[modname] = true
         end
+    end
+
+    if (type(modtab) == "table") then
+        -- Protect module table in any case (i.e. either if the module used our
+        -- 'module' or if it returned a table).
+        setmetatable(modtab, required_module_mt)
+    end
+
+    local gvmodi = module_gvlocali[modname]
+
+    if (gvmodi and gvmodi[2]>=gvmodi[1]) then
+        if (coroutine.status(modthread)=="suspended") then
+            -- Save off the suspended thread so that we may get its locals later on.
+            -- It is never resumed, but only ever used for debug.getlocal().
+            module_thread[modname] = modthread
+
+            if (ffiC._DEBUG_LUNATIC ~= 0) then
+                printf("Keeping coroutine for module \"%s\"", modname)
+            end
+        end
+    end
 
     return modtab
 end
@@ -1557,7 +1629,7 @@ local function our_module()
 
     local modname = getcurmodname("module")
 
-    if (type(package_loaded[modname])=="table") then
+    if (package_loaded[modname]) then
         error("'module' must be called at most once per require'd file", 2)
     end
 
@@ -1902,12 +1974,16 @@ do
     ffiC.El_SerializeGamevars = function(slenptr)
         local sb = savegame.savebuffer()
 
-        sb:addraw("local M")
+        -- Module name, module table, restore_local. See SAVEFUNC_ARGS.
+        sb:addraw("local N,M,F=...")
+        -- A local to temporarily hold module locals.
+        sb:addraw("local L")
 
-        -- XXX: System gamevars?
+        -- XXX: System gamevars? Most of them ought to be saved with C data.
         for modname, modvars in pairs(module_gamevars) do
-            -- NOTE: when emitting 'require' code, reverse '.' -> '/' substitution.
+            sb:addrawf("if (N==%q) then", modname)
-            sb:addrawf("M=require%q", modname:gsub("/","."))
+
+            -- Handle global gamevars first.
             for i=1,#modvars do
                 local varname = modvars[i]
                 -- Serialize gamevar named 'varname' from module named 'modname'.
@@ -1920,6 +1996,29 @@ do
                     return (modname==CON_MODULE_NAME and "<CON>" or modname).."."..varname
                 end
             end
+
+            local modthread = module_thread[modname]
+
+            if (modthread) then
+                -- Handle local gamevars.
+                local gvmodi = module_gvlocali[modname]
+
+                for li=gvmodi[1],gvmodi[2] do
+                    -- Serialize local with index <li>. Get its value first.
+                    local lname, lval = debug.getlocal(modthread, 1, li)
+
+                    if (sb:add("L", lval)) then
+                        -- We couldn't serialize that gamevar.
+                        slenptr[0] = -1
+                        return "local "..modname.."."..lname
+                    end
+
+                    -- Call restore_local.
+                    sb:addrawf("F(%d,L)", li)
+                end
+            end
+
+            sb:addraw("end")
         end
 
         -- Get the whole code as a string.
@@ -2060,6 +2159,19 @@ setmetatable(
 -- environment earlier.
 setfenv(0, _G)
 
+do
+    -- If we're running from a savegame restoration, create the restoration
+    -- function. Must be here, after the above setfenv(), because it must be
+    -- created in this protected ('user') context!
+    local cstr = ffiC.g_elSavecode
+    if (cstr~=nil) then
+        local restorecode = ffi.string(cstr)
+        ffiC.El_FreeSaveCode()
+
+        g_restorefunc = assert(loadstring(restorecode))
+    end
+end
+
 -- Run the CON code translated into Lua.
 if (concode) then
     local confunc, conerrmsg = loadstring(concode, "CON")
@@ -2067,10 +2179,11 @@ if (concode) then
         error("Failure loading translated CON code: "..conerrmsg, 0)
     end
 
-    local conmodule, conlabels = confunc()
+    -- Emulate our 'require' for the CON module when running it, for
-
+    -- our_module() which is called from the generated Lua code.
-    package_loaded[CON_MODULE_NAME] = conmodule
+    table.insert(modname_stack, CON_MODULE_NAME)
-    module_gamevars[CON_MODULE_NAME] = { "A", "V" }  -- See CON_GAMEVARS is lunacon.lua.
+    local conlabels = confunc()
+    table.remove(modname_stack)
 
     -- Set up CON.DEFS module, providing access to labels defined in CON from Lua.
     local mt = { __index = conlabels, __newindex = basemodule_newindex }

polymer/eduke32/source/lunatic/dynsymlist

@@ -93,6 +93,8 @@ g_elCONSize;
 g_elCON;
 El_SetCON;
 
+g_elSavecode;
+El_FreeSaveCode;
 El_SerializeGamevars;
 
 s_buildRev;

polymer/eduke32/source/lunatic/lunacon.lua

@@ -177,6 +177,7 @@ local function new_initial_codetab()
     -- mapping system.
     return {
         -- Requires.
+        "local require=require",
         "local _con, _bit, _math = require'con', require'bit', require'math'",
         "local _xmath, _geom = require'xmath', require'geom'",
 
@@ -191,10 +192,17 @@ local function new_initial_codetab()
         "local _band, _bor, _bxor = _bit.band, _bit.bor, _bit.bxor",
         "local _lsh, _rsh, _arsh = _bit.lshift, _bit.rshift, _bit.arshift",
 
-        -- Switch function table, indexed by global switch sequence number:
+        -- * CON "states" (subroutines) and
-        "local _SW = {}",
+        -- * Switch function table, indexed by global switch sequence number:
-        -- CON "states" (subroutines), gamevars and gamearrays (see mangle_name())
+        "local _F,_SW = {},{}",
-        "local _F,_V,_A={},{},{}",
+
+        -- CON gamevars and gamearrays (see mangle_name()), set up for
+        -- restoration from savegames.
+        "module(...)",
+        "_V,_A={},{}",
+        "-- NOTE to the reader: This require's result is Lunatic-private API! DO NOT USE!",
+        "local _dummy,_S=require'end_gamevars'",
+        "local _V,_A=_V,_A",
 
         -- Static ivec3s so that no allocations need to be made.
         "local _IVEC = { _geom.ivec3(), _geom.ivec3() }",
@@ -1220,7 +1228,9 @@ function Cmd.gamearray(identifier, initsize)
     local ga = { name=mangle_name(identifier, "A"), size=initsize }
     g_gamearray[identifier] = ga
 
+    addcode("if _S then")
     addcodef("%s=_con._gamearray(%d)", ga.name, initsize)
+    addcode("end")
 end
 
 function Cmd.gamevar(identifier, initval, flags)
@@ -1272,6 +1282,7 @@ function Cmd.gamevar(identifier, initval, flags)
                 local linestr = "--"..getlinecol(g_lastkwpos)
 
                 -- Emit code to set the variable at Lua parse time.
+                -- XXX: How does this interact with savegame restoration?
                 if (bit.band(oflags, GVFLAG.PERPLAYER) ~= 0) then
                     -- Replace player index by 0. PLAYER_0.
                     -- TODO_MP: init for all players.
@@ -1300,7 +1311,8 @@ function Cmd.gamevar(identifier, initval, flags)
     local gv = { name=mangle_name(identifier, "V"), flags=flags }
     g_gamevar[identifier] = gv
 
-    -- TODO: Write gamevar system on the Lunatic side and hook it up.
+    addcode("if _S then")
+
     if (bit.band(flags, GVFLAG.PERX_MASK)==GVFLAG.PERACTOR) then
         addcodef("%s=_con.actorvar(%d)", gv.name, initval)
     elseif (bit.band(flags, GVFLAG.PERX_MASK)==GVFLAG.PERPLAYER and g_cgopt["playervar"]) then
@@ -1309,6 +1321,8 @@ function Cmd.gamevar(identifier, initval, flags)
     else
         addcodef("%s=%d", gv.name, initval)
     end
+
+    addcode("end")
 end
 
 function Cmd.dynamicremap()
@@ -3325,12 +3339,8 @@ end
 
 -- <lineinfop>: Get line info?
 local function get_code_string(codetab, lineinfop)
-    -- Finalize translated code: return table containing gamevar and gamearray
-    -- tables. CON_GAMEVARS.
-    codetab[#codetab+1] = "return { V=_V, A=_A }"
-
     -- Return defined labels in a table...
-    codetab[#codetab+1] = ",{"
+    codetab[#codetab+1] = "return {"
     for label, val in pairs(g_labeldef) do
         -- ... skipping 'NO' and those that are gamevars in C-CON.
         if (g_labeltype[label]==LABEL.NUMBER and not g_labelspecial[label]) then

polymer/eduke32/source/lunatic/savegame.lua

@@ -62,7 +62,6 @@ local savebuffer_mt = {
                     if (isSerializeable(value)) then
                         -- We have a serializeable object from Lunatic
                         -- (e.g. actorvar).
-                        -- TODO: clean up (e.g. clear default values for actorvar).
 
                         -- First, get the code necessary to create this object,
                         -- usually 'require'ing a module into a local variable.
@@ -78,9 +77,8 @@ local savebuffer_mt = {
                         -- We have a Lua table.
                         havetab = true
 
-                        -- Clear the table instead of creating a new one.
+                        -- Create a new table for this gamevar.
-                        -- This way, local references to it don't become stale.
+                        self:addrawf("%s={}", refcode)
-                        self:addrawf("ct(%s)", refcode)
 
                         for k,v in pairs(value) do
                             local keystr = basicSerialize(k)
@@ -138,11 +136,6 @@ local function sb_get_initial_strbuf()
     return {
         "local nan, inf = 0/0, 1/0",
         "local t, f = true, false",
-        "local pairs = assert(pairs)",
-        -- Clear table function:
-        "local function ct(t)",
-        "  for k in pairs(t) do t[k]=nil end",
-        "end",
     }
 end
 

polymer/eduke32/source/lunatic/test/delmusicsfx.lua

@@ -1,6 +1,8 @@
 
 local require = require
-local con = require "con"
+
+local string = require("string")
+local con = require("con")
 
 local gv = gv
 local sprite = sprite
@@ -35,6 +37,7 @@ require "end_gamevars"
 -- refer to locals defined prior to the gamevar section in it.
 local tag = tag
 
+local Q = 1200
 
 gameevent{"JUMP", actor.FLAGS.chain_beg,
 function(aci, pli)
@@ -75,3 +78,15 @@ function(aci, pli)
 
     insp = not insp
 end}
+
+-- Display the number of times we jumped on the screen.
+gameevent
+{
+    "DISPLAYREST",
+
+    function()
+        con._definequote(Q, string.format("jumped %d times", ournumjumps))
+        -- NOTE: uses INTERNAL interface, don't copy!
+        con._minitext(160, 10, Q, 0,0)
+    end
+}

polymer/eduke32/source/savegame.c

@@ -1700,17 +1700,11 @@ static uint8_t *dosaveplayer2(FILE *fil, uint8_t *mem)
     PRINTSIZE("ud");
     mem=writespecdata(svgm_secwsp, fil, mem);  // sector, wall, sprite
     PRINTSIZE("sws");
-    mem=writespecdata(svgm_script, fil, mem);  // script
+#ifdef LUNATIC
-    PRINTSIZE("script");
-    mem=writespecdata(svgm_anmisc, fil, mem);  // animates, quotes & misc.
-    PRINTSIZE("animisc");
-
-#if !defined LUNATIC
-    Gv_WriteSave(fil, 1);  // gamevars
-    mem=writespecdata(svgm_vars, 0, mem);
-    PRINTSIZE("vars");
-#else
     {
+        // Serialize Lunatic gamevars. When loading, the restoration code must
+        // be present before Lua state creation in svgm_script, so save it
+        // right before, too.
         int32_t slen, slen_ext;
         const char *svcode = El_SerializeGamevars(&slen);
 
@@ -1722,56 +1716,35 @@ static uint8_t *dosaveplayer2(FILE *fil, uint8_t *mem)
             return mem;
         }
 
-        // TODO: compress text.
         fwrite("\0\1LunaGVAR\3\4", 12, 1, fil);
         slen_ext = B_LITTLE32(slen);
         fwrite(&slen_ext, sizeof(slen_ext), 1, fil);
-        fwrite(svcode, slen, 1, fil);
+        dfwrite(svcode, 1, slen, fil);  // cnt and sz swapped
 
         g_savedOK = 1;
     }
 #endif
+    mem=writespecdata(svgm_script, fil, mem);  // script
+    PRINTSIZE("script");
+    mem=writespecdata(svgm_anmisc, fil, mem);  // animates, quotes & misc.
+    PRINTSIZE("animisc");
+
+#if !defined LUNATIC
+    Gv_WriteSave(fil, 1);  // gamevars
+    mem=writespecdata(svgm_vars, 0, mem);
+    PRINTSIZE("vars");
+#endif
 
     return mem;
 }
 
-#define LOADRD(ptr, sz, cnt) (kdfread(ptr,sz,cnt,fil)!=(cnt))
+#ifdef LUNATIC
-#define LOADRDU(ptr, sz, cnt) (kread(fil,ptr,(sz)*(cnt))!=(sz)*(cnt))
+char *g_elSavecode = NULL;
 
-static int32_t doloadplayer2(int32_t fil, uint8_t **memptr)
+static int32_t El_ReadSaveCode(int32_t fil)
 {
-    uint8_t *mem = memptr ? *memptr : NULL;
+    // Read Lua code to restore gamevar values from the savegame.
-#ifdef DEBUGGINGAIDS
+    // It will be run from Lua with its state creation later on.
-    uint8_t *tmem=mem;
-    int32_t t=getticks();
-#endif
-    if (readspecdata(svgm_udnetw, fil, &mem)) return -2;
-    PRINTSIZE("ud");
-    if (readspecdata(svgm_secwsp, fil, &mem)) return -4;
-    PRINTSIZE("sws");
-    if (readspecdata(svgm_script, fil, &mem)) return -5;
-    PRINTSIZE("script");
-    if (readspecdata(svgm_anmisc, fil, &mem)) return -6;
-    PRINTSIZE("animisc");
-
-#if !defined LUNATIC
-    if (Gv_ReadSave(fil, 1)) return -7;
-
-    if (mem)
-    {
-        int32_t i;
-
-        sv_makevarspec();
-        for (i=1; svgm_vars[i].flags!=DS_END; i++)
-        {
-            Bmemcpy(mem, svgm_vars[i].ptr, svgm_vars[i].size*svgm_vars[i].cnt);  // careful! works because there are no DS_DYNAMIC's!
-            mem += svgm_vars[i].size*svgm_vars[i].cnt;
-        }
-    }
-    PRINTSIZE("vars");
-#else
-    {
-        // Read Lua code to restore gamevar values from the savegame and run it.
 
     char header[12];
     int32_t slen;
@@ -1803,23 +1776,70 @@ static int32_t doloadplayer2(int32_t fil, uint8_t **memptr)
 
     if (slen > 0)
     {
-            char *svcode = Bmalloc(slen);
+        char *svcode = Bmalloc(slen+1);
         if (svcode == NULL)
             G_GameExit("OUT OF MEMORY in doloadplayer2().");
 
-            if (kread(fil, svcode, slen) != slen)
+        if (kdfread(svcode, 1, slen, fil) != slen)  // cnt and sz swapped
         {
             OSD_Printf("doloadplayer2: failed reading Lunatic gamevar restoration code.\n");
+            Bfree(svcode);
             return -104;
         }
 
-            if (L_RunString(&g_ElState, svcode, 0, slen, "luaload"))
+        svcode[slen] = 0;
+        g_elSavecode = svcode;
+    }
+
+    return 0;
+}
+
+void El_FreeSaveCode(void)
+{
+    // Free Lunatic gamevar savegame restoration Lua code.
+    Bfree(g_elSavecode);
+    g_elSavecode = NULL;
+}
+#endif
+
+static int32_t doloadplayer2(int32_t fil, uint8_t **memptr)
+{
+    uint8_t *mem = memptr ? *memptr : NULL;
+#ifdef DEBUGGINGAIDS
+    uint8_t *tmem=mem;
+    int32_t t=getticks();
+#endif
+    if (readspecdata(svgm_udnetw, fil, &mem)) return -2;
+    PRINTSIZE("ud");
+    if (readspecdata(svgm_secwsp, fil, &mem)) return -4;
+    PRINTSIZE("sws");
+#ifdef LUNATIC
     {
-                OSD_Printf("doloadplayer2: failed restoring Lunatic gamevars.\n");
+        int32_t ret = El_ReadSaveCode(fil);
-                return -105;
+        if (ret < 0)
-            }
+            return ret;
+    }
+#endif
+    if (readspecdata(svgm_script, fil, &mem)) return -5;
+    PRINTSIZE("script");
+    if (readspecdata(svgm_anmisc, fil, &mem)) return -6;
+    PRINTSIZE("animisc");
+
+#if !defined LUNATIC
+    if (Gv_ReadSave(fil, 1)) return -7;
+
+    if (mem)
+    {
+        int32_t i;
+
+        sv_makevarspec();
+        for (i=1; svgm_vars[i].flags!=DS_END; i++)
+        {
+            Bmemcpy(mem, svgm_vars[i].ptr, svgm_vars[i].size*svgm_vars[i].cnt);  // careful! works because there are no DS_DYNAMIC's!
+            mem += svgm_vars[i].size*svgm_vars[i].cnt;
         }
     }
+    PRINTSIZE("vars");
 #endif
 
     if (memptr)