Switched Duke3D to use new LZ4 dfwrite()/kdfread() implementations to improve compression/decompression speed.

SV_MINOR_VER is incremented as using this alternate compression algorithm is not backwards compatible with older saves/demos.

git-svn-id: https://svn.eduke32.com/eduke32@6789 1a8010ca-5511-0410-912e-c29ae57300e0
This commit is contained in:
pogokeen 2018-03-23 21:41:02 +00:00
parent b969687689
commit 136dd514e1
6 changed files with 93 additions and 43 deletions

View file

@ -73,11 +73,13 @@ void klistfree(CACHE1D_FIND_REC *rec);
CACHE1D_FIND_REC *klistpath(const char *path, const char *mask, int32_t type);
int32_t kdfread(void *buffer, bsize_t dasizeof, bsize_t count, int32_t fil);
int32_t kdfread_LZ4(void *buffer, bsize_t dasizeof, bsize_t count, int32_t fil);
#if 0
int32_t dfread(void *buffer, bsize_t dasizeof, bsize_t count, BFILE *fil);
void kdfwrite(const void *buffer, bsize_t dasizeof, bsize_t count, int32_t fil);
#endif
void dfwrite(const void *buffer, bsize_t dasizeof, bsize_t count, BFILE *fil);
void dfwrite_LZ4(const void *buffer, bsize_t dasizeof, bsize_t count, BFILE *fil);
#ifdef __cplusplus
}

View file

@ -23,6 +23,7 @@
#include "cache1d.h"
#include "pragmas.h"
#include "baselayer.h"
#include "lz4.h"
#ifdef WITHKPLIB
#include "kplib.h"
@ -1703,6 +1704,31 @@ int32_t kdfread(void *buffer, bsize_t dasizeof, bsize_t count, int32_t fil)
return c1d_read_compressed(buffer, dasizeof, count, (intptr_t)fil);
}
int32_t kdfread_LZ4(void *buffer, bsize_t dasizeof, bsize_t count, int32_t fil)
{
int32_t leng;
// read compressed data length
if (c1d_readfunc(fil, &leng, 4) != 4)
return -1;
leng = B_LITTLE32(leng);
char compressedDataStackBuf[100000];
char* pCompressedData = compressedDataStackBuf;
if (leng > 100000)
pCompressedData = (char*) Bmalloc(leng);
if (c1d_readfunc(fil, pCompressedData, leng) != leng)
return -1;
int32_t decompressedLength = LZ4_decompress_safe(pCompressedData, (char*) buffer, leng, dasizeof*count);
if (pCompressedData != compressedDataStackBuf)
free(pCompressedData);
return decompressedLength/dasizeof;
}
////////// COMPRESSED WRITE //////////
@ -1768,6 +1794,28 @@ void dfwrite(const void *buffer, bsize_t dasizeof, bsize_t count, BFILE *fil)
c1d_write_compressed(buffer, dasizeof, count, (intptr_t)fil);
}
// LZ4_COMPRESSION_ACCELERATION_VALUE can be tuned for performance/space trade-off
// (lower number = higher compression ratio, higher number = faster compression speed)
#define LZ4_COMPRESSION_ACCELERATION_VALUE 15
void dfwrite_LZ4(const void *buffer, bsize_t dasizeof, bsize_t count, BFILE *fil)
{
char compressedDataStackBuf[100000];
char* pCompressedData = compressedDataStackBuf;
int32_t maxCompressedSize = LZ4_compressBound(dasizeof*count);
if (maxCompressedSize > 100000)
pCompressedData = (char*) Bmalloc(maxCompressedSize);
const int32_t leng = LZ4_compress_fast((const char*) buffer, pCompressedData, dasizeof*count, maxCompressedSize, LZ4_COMPRESSION_ACCELERATION_VALUE);
const int32_t swleng = B_LITTLE32(leng);
c1d_writefunc((intptr_t) fil, &swleng, 4);
c1d_writefunc((intptr_t) fil, pCompressedData, leng);
if (pCompressedData != compressedDataStackBuf)
free(pCompressedData);
}
////////// CORE COMPRESSION FUNCTIONS //////////
static int32_t lzwcompress(const char *lzwinbuf, int32_t uncompleng, char *lzwoutbuf)

View file

@ -270,7 +270,7 @@ static void Demo_WriteSync()
fwrite(g_demo_seedbuf, 1, ud.reccnt, g_demo_filePtr);
if (demo_synccompress)
dfwrite(recsync, sizeof(input_t), ud.reccnt, g_demo_filePtr);
dfwrite_LZ4(recsync, sizeof(input_t), ud.reccnt, g_demo_filePtr);
else //if (demo_synccompress==0)
fwrite(recsync, sizeof(input_t), ud.reccnt, g_demo_filePtr);
@ -363,7 +363,7 @@ static int32_t Demo_ReadSync(int32_t errcode)
if (demo_synccompress)
{
if (kdfread(recsync, sizeof(input_t), i, g_demo_recFilePtr) != i)
if (kdfread_LZ4(recsync, sizeof(input_t), i, g_demo_recFilePtr) != i)
return errcode+1;
}
else

View file

@ -117,46 +117,46 @@ int Gv_ReadSave(int32_t kFile)
Gv_Free(); // nuke 'em from orbit, it's the only way to be sure...
if (kdfread(&g_gameVarCount,sizeof(g_gameVarCount),1,kFile) != 1) goto corrupt;
if (kdfread_LZ4(&g_gameVarCount,sizeof(g_gameVarCount),1,kFile) != 1) goto corrupt;
for (bssize_t i=0; i<g_gameVarCount; i++)
{
char *const olabel = aGameVars[i].szLabel;
if (kdfread(&aGameVars[i], sizeof(gamevar_t), 1, kFile) != 1)
if (kdfread_LZ4(&aGameVars[i], sizeof(gamevar_t), 1, kFile) != 1)
goto corrupt;
aGameVars[i].szLabel = (char *)Xrealloc(olabel, MAXVARLABEL * sizeof(uint8_t));
if (kdfread(aGameVars[i].szLabel, MAXVARLABEL, 1, kFile) != 1)
if (kdfread_LZ4(aGameVars[i].szLabel, MAXVARLABEL, 1, kFile) != 1)
goto corrupt;
hash_add(&h_gamevars, aGameVars[i].szLabel,i, 1);
if (aGameVars[i].flags & GAMEVAR_PERPLAYER)
{
aGameVars[i].pValues = (intptr_t*)Xaligned_alloc(PLAYER_VAR_ALIGNMENT, MAXPLAYERS * sizeof(intptr_t));
if (kdfread(aGameVars[i].pValues,sizeof(intptr_t) * MAXPLAYERS, 1, kFile) != 1) goto corrupt;
if (kdfread_LZ4(aGameVars[i].pValues,sizeof(intptr_t) * MAXPLAYERS, 1, kFile) != 1) goto corrupt;
}
else if (aGameVars[i].flags & GAMEVAR_PERACTOR)
{
aGameVars[i].pValues = (intptr_t*)Xaligned_alloc(ACTOR_VAR_ALIGNMENT, MAXSPRITES * sizeof(intptr_t));
if (kdfread(aGameVars[i].pValues,sizeof(intptr_t), MAXSPRITES, kFile) != MAXSPRITES) goto corrupt;
if (kdfread_LZ4(aGameVars[i].pValues,sizeof(intptr_t), MAXSPRITES, kFile) != MAXSPRITES) goto corrupt;
}
}
Gv_InitWeaponPointers();
if (kdfread(&g_gameArrayCount,sizeof(g_gameArrayCount),1,kFile) != 1) goto corrupt;
if (kdfread_LZ4(&g_gameArrayCount,sizeof(g_gameArrayCount),1,kFile) != 1) goto corrupt;
for (bssize_t i=0; i<g_gameArrayCount; i++)
{
char *const olabel = aGameArrays[i].szLabel;
// read for .size and .dwFlags (the rest are pointers):
if (kdfread(&aGameArrays[i], sizeof(gamearray_t), 1, kFile) != 1)
if (kdfread_LZ4(&aGameArrays[i], sizeof(gamearray_t), 1, kFile) != 1)
goto corrupt;
aGameArrays[i].szLabel = (char *) Xrealloc(olabel, MAXARRAYLABEL * sizeof(uint8_t));
if (kdfread(aGameArrays[i].szLabel,sizeof(uint8_t) * MAXARRAYLABEL, 1, kFile) != 1)
if (kdfread_LZ4(aGameArrays[i].szLabel,sizeof(uint8_t) * MAXARRAYLABEL, 1, kFile) != 1)
goto corrupt;
hash_add(&h_arrays, aGameArrays[i].szLabel, i, 1);
@ -166,7 +166,7 @@ int Gv_ReadSave(int32_t kFile)
if (asize != 0 && !(aGameArrays[i].flags & GAMEARRAY_SYSTEM))
{
aGameArrays[i].pValues = (intptr_t *)Xaligned_alloc(ARRAY_ALIGNMENT, Gv_GetArrayAllocSize(i));
if (kdfread(aGameArrays[i].pValues, Gv_GetArrayAllocSize(i), 1, kFile) < 1) goto corrupt;
if (kdfread_LZ4(aGameArrays[i].pValues, Gv_GetArrayAllocSize(i), 1, kFile) < 1) goto corrupt;
}
else
aGameArrays[i].pValues = NULL;
@ -174,12 +174,12 @@ int Gv_ReadSave(int32_t kFile)
Gv_RefreshPointers();
if (kdfread(apScriptEvents, sizeof(apScriptEvents), 1, kFile) != 1) goto corrupt;
if (kdfread_LZ4(apScriptEvents, sizeof(apScriptEvents), 1, kFile) != 1) goto corrupt;
uint8_t savedstate[MAXVOLUMES*MAXLEVELS];
Bmemset(savedstate, 0, sizeof(savedstate));
if (kdfread(savedstate, sizeof(savedstate), 1, kFile) != 1) goto corrupt;
if (kdfread_LZ4(savedstate, sizeof(savedstate), 1, kFile) != 1) goto corrupt;
for (bssize_t i = 0; i < (MAXVOLUMES * MAXLEVELS); i++)
{
@ -189,7 +189,7 @@ int Gv_ReadSave(int32_t kFile)
continue;
g_mapInfo[i].savedstate = (mapstate_t *)Xaligned_alloc(ACTOR_VAR_ALIGNMENT, sizeof(mapstate_t));
if (kdfread(g_mapInfo[i].savedstate, 1, sizeof(mapstate_t), kFile) != sizeof(mapstate_t)) return -8;
if (kdfread_LZ4(g_mapInfo[i].savedstate, 1, sizeof(mapstate_t), kFile) != sizeof(mapstate_t)) return -8;
mapstate_t &sv = *g_mapInfo[i].savedstate;
@ -199,16 +199,16 @@ int Gv_ReadSave(int32_t kFile)
if (aGameVars[j].flags & GAMEVAR_PERPLAYER)
{
sv.vars[j] = (intptr_t *) Xaligned_alloc(PLAYER_VAR_ALIGNMENT, MAXPLAYERS * sizeof(intptr_t));
if (kdfread(sv.vars[j], sizeof(intptr_t) * MAXPLAYERS, 1, kFile) != 1) return -9;
if (kdfread_LZ4(sv.vars[j], sizeof(intptr_t) * MAXPLAYERS, 1, kFile) != 1) return -9;
}
else if (aGameVars[j].flags & GAMEVAR_PERACTOR)
{
sv.vars[j] = (intptr_t *) Xaligned_alloc(ACTOR_VAR_ALIGNMENT, MAXSPRITES * sizeof(intptr_t));
if (kdfread(sv.vars[j], sizeof(intptr_t), MAXSPRITES, kFile) != MAXSPRITES) return -10;
if (kdfread_LZ4(sv.vars[j], sizeof(intptr_t), MAXSPRITES, kFile) != MAXSPRITES) return -10;
}
}
if (kdfread(sv.arraysiz, sizeof(sv.arraysiz), 1, kFile) < 1)
if (kdfread_LZ4(sv.arraysiz, sizeof(sv.arraysiz), 1, kFile) < 1)
return -11;
for (bssize_t j = 0; j < g_gameArrayCount; j++)
@ -216,7 +216,7 @@ int Gv_ReadSave(int32_t kFile)
{
size_t const siz = Gv_GetArrayAllocSizeForCount(j, sv.arraysiz[j]);
sv.arrays[j] = (intptr_t *) Xaligned_alloc(ARRAY_ALIGNMENT, siz);
if (kdfread(sv.arrays[j], siz, 1, kFile) < 1) return -12;
if (kdfread_LZ4(sv.arrays[j], siz, 1, kFile) < 1) return -12;
}
}
@ -234,32 +234,32 @@ void Gv_WriteSave(FILE *fil)
// AddLog("Saving Game Vars to File");
fwrite("BEG: EDuke32", 12, 1, fil);
dfwrite(&g_gameVarCount,sizeof(g_gameVarCount),1,fil);
dfwrite_LZ4(&g_gameVarCount,sizeof(g_gameVarCount),1,fil);
for (bssize_t i = 0; i < g_gameVarCount; i++)
{
dfwrite(&(aGameVars[i]), sizeof(gamevar_t), 1, fil);
dfwrite(aGameVars[i].szLabel, sizeof(uint8_t) * MAXVARLABEL, 1, fil);
dfwrite_LZ4(&(aGameVars[i]), sizeof(gamevar_t), 1, fil);
dfwrite_LZ4(aGameVars[i].szLabel, sizeof(uint8_t) * MAXVARLABEL, 1, fil);
if (aGameVars[i].flags & GAMEVAR_PERPLAYER)
dfwrite(aGameVars[i].pValues, sizeof(intptr_t) * MAXPLAYERS, 1, fil);
dfwrite_LZ4(aGameVars[i].pValues, sizeof(intptr_t) * MAXPLAYERS, 1, fil);
else if (aGameVars[i].flags & GAMEVAR_PERACTOR)
dfwrite(aGameVars[i].pValues, sizeof(intptr_t), MAXSPRITES, fil);
dfwrite_LZ4(aGameVars[i].pValues, sizeof(intptr_t), MAXSPRITES, fil);
}
dfwrite(&g_gameArrayCount,sizeof(g_gameArrayCount),1,fil);
dfwrite_LZ4(&g_gameArrayCount,sizeof(g_gameArrayCount),1,fil);
for (bssize_t i = 0; i < g_gameArrayCount; i++)
{
// write for .size and .dwFlags (the rest are pointers):
dfwrite(&aGameArrays[i], sizeof(gamearray_t), 1, fil);
dfwrite(aGameArrays[i].szLabel, sizeof(uint8_t) * MAXARRAYLABEL, 1, fil);
dfwrite_LZ4(&aGameArrays[i], sizeof(gamearray_t), 1, fil);
dfwrite_LZ4(aGameArrays[i].szLabel, sizeof(uint8_t) * MAXARRAYLABEL, 1, fil);
if ((aGameArrays[i].flags & GAMEARRAY_SYSTEM) != GAMEARRAY_SYSTEM)
dfwrite(aGameArrays[i].pValues, Gv_GetArrayAllocSize(i), 1, fil);
dfwrite_LZ4(aGameArrays[i].pValues, Gv_GetArrayAllocSize(i), 1, fil);
}
dfwrite(apScriptEvents, sizeof(apScriptEvents), 1, fil);
dfwrite_LZ4(apScriptEvents, sizeof(apScriptEvents), 1, fil);
uint8_t savedstate[MAXVOLUMES * MAXLEVELS];
Bmemset(savedstate, 0, sizeof(savedstate));
@ -268,7 +268,7 @@ void Gv_WriteSave(FILE *fil)
if (g_mapInfo[i].savedstate != NULL)
savedstate[i] = 1;
dfwrite(savedstate, sizeof(savedstate), 1, fil);
dfwrite_LZ4(savedstate, sizeof(savedstate), 1, fil);
for (bssize_t i = 0; i < (MAXVOLUMES * MAXLEVELS); i++)
{
@ -276,23 +276,23 @@ void Gv_WriteSave(FILE *fil)
mapstate_t &sv = *g_mapInfo[i].savedstate;
dfwrite(g_mapInfo[i].savedstate, sizeof(mapstate_t), 1, fil);
dfwrite_LZ4(g_mapInfo[i].savedstate, sizeof(mapstate_t), 1, fil);
for (bssize_t j = 0; j < g_gameVarCount; j++)
{
if (aGameVars[j].flags & GAMEVAR_NORESET) continue;
if (aGameVars[j].flags & GAMEVAR_PERPLAYER)
dfwrite(sv.vars[j], sizeof(intptr_t) * MAXPLAYERS, 1, fil);
dfwrite_LZ4(sv.vars[j], sizeof(intptr_t) * MAXPLAYERS, 1, fil);
else if (aGameVars[j].flags & GAMEVAR_PERACTOR)
dfwrite(sv.vars[j], sizeof(intptr_t), MAXSPRITES, fil);
dfwrite_LZ4(sv.vars[j], sizeof(intptr_t), MAXSPRITES, fil);
}
dfwrite(sv.arraysiz, sizeof(sv.arraysiz), 1, fil);
dfwrite_LZ4(sv.arraysiz, sizeof(sv.arraysiz), 1, fil);
for (bssize_t j = 0; j < g_gameArrayCount; j++)
if (aGameArrays[j].flags & GAMEARRAY_RESTORE)
{
dfwrite(sv.arrays[j], Gv_GetArrayAllocSizeForCount(j, sv.arraysiz[j]), 1, fil);
dfwrite_LZ4(sv.arrays[j], Gv_GetArrayAllocSizeForCount(j, sv.arraysiz[j]), 1, fil);
}
}

View file

@ -299,7 +299,7 @@ int32_t G_LoadSaveHeaderNew(char const *fn, savehead_t *saveh)
tilesiz[TILE_LOADSHOT].y = 320;
if (screenshotofs)
{
if (kdfread((char *)waloff[TILE_LOADSHOT], 320, 200, fil) != 200)
if (kdfread_LZ4((char *)waloff[TILE_LOADSHOT], 320, 200, fil) != 200)
{
OSD_Printf("G_LoadSaveHeaderNew(): failed reading screenshot in \"%s\"\n", fn);
goto corrupt;
@ -744,7 +744,7 @@ static uint8_t *writespecdata(const dataspec_t *spec, FILE *fil, uint8_t *dump)
|| (sp->flags&DS_CMP))
fwrite(ptr, sp->size, cnt, fil);
else
dfwrite((void *)ptr, sp->size, cnt, fil);
dfwrite_LZ4((void *)ptr, sp->size, cnt, fil);
}
if (dump && (sp->flags&(DS_NOCHK|DS_CMP))==0)
@ -817,7 +817,7 @@ static int32_t readspecdata(const dataspec_t *spec, int32_t fil, uint8_t **dumpv
}
else
{
i = kdfread(mem, sp->size, cnt, fil);
i = kdfread_LZ4(mem, sp->size, cnt, fil);
j = cnt;
}
if (i!=j)
@ -1507,7 +1507,7 @@ int32_t sv_saveandmakesnapshot(FILE *fil, char const *name, int8_t spot, int8_t
int32_t ofs;
// write the screenshot compressed
dfwrite((char *)waloff[TILE_SAVESHOT], 320, 200, fil);
dfwrite_LZ4((char *)waloff[TILE_SAVESHOT], 320, 200, fil);
// write the current file offset right after the header
ofs = ftell(fil);
@ -1715,7 +1715,7 @@ uint32_t sv_writediff(FILE *fil)
fwrite("dIfF",4,1,fil);
fwrite(&diffsiz, sizeof(diffsiz), 1, fil);
if (savegame_diffcompress)
dfwrite(svdiff, 1, diffsiz, fil); // cnt and sz swapped
dfwrite_LZ4(svdiff, 1, diffsiz, fil); // cnt and sz swapped
else
fwrite(svdiff, 1, diffsiz, fil);
@ -1737,7 +1737,7 @@ int32_t sv_readdiff(int32_t fil)
return -1;
if (savegame_diffcompress)
{
if (kdfread(svdiff, 1, diffsiz, fil) != diffsiz) // cnt and sz swapped
if (kdfread_LZ4(svdiff, 1, diffsiz, fil) != diffsiz) // cnt and sz swapped
return -2;
}
else
@ -2086,7 +2086,7 @@ static uint8_t *dosaveplayer2(FILE *fil, uint8_t *mem)
fwrite("\0\1LunaGVAR\3\4", 12, 1, fil);
slen_ext = B_LITTLE32(slen);
fwrite(&slen_ext, sizeof(slen_ext), 1, fil);
dfwrite(svcode, 1, slen, fil); // cnt and sz swapped
dfwrite_LZ4(svcode, 1, slen, fil); // cnt and sz swapped
g_savedOK = 1;
}
@ -2145,7 +2145,7 @@ static int32_t El_ReadSaveCode(int32_t fil)
{
char *svcode = (char *)Xmalloc(slen+1);
if (kdfread(svcode, 1, slen, fil) != slen) // cnt and sz swapped
if (kdfread_LZ4(svcode, 1, slen, fil) != slen) // cnt and sz swapped
{
OSD_Printf("doloadplayer2: failed reading Lunatic gamevar restoration code.\n");
Bfree(svcode);

View file

@ -34,7 +34,7 @@ extern "C" {
#else
# define SV_MAJOR_VER 1
#endif
#define SV_MINOR_VER 5
#define SV_MINOR_VER 6
#pragma pack(push,1)
typedef struct