eukara/fteqw
1
0
Fork 0
forked from fte/fteqw
Code Pull requests Activity
fteqw/engine/qclib/packager.c
Spoike 685404250f added cmake file. 
reduced input latency.
reworked how internal texture formats work,.
added support for LIGHTING_E5BGR9 bspx lump for HDR lighting.
updated support for srgb, no longer looks quite so weird. works on glx
vid_srgb 3 attempts to use half-float swapchains, where possible.
gl: use glTextureStorage where available.
d3d11: gave up on using dxgi for fullscreen, was just too buggy.
glx: updated gl context creation on linux.
server: fix svc_updatefrags not being passed though (fixes frikbot scores)
fs: spanned pk3s now work (fragmented files/directory will fail to open, so this needs a custom tool to be fully useful).
fixed restart_ents command (restarts the map, but preserving the players as they are)
tw: removed 'QWSKINS' featureset from tw config

git-svn-id: https://svn.code.sf.net/p/fteqw/code/trunk@5217 fc73d0e0-1445-4013-8a0c-d673dee63da5
2018-03-04 14:41:16 +00:00

1637 lines
42 KiB
C
Raw Blame History

#include "qcc.h"
#include <time.h>
void QCC_Canonicalize(char *fullname, size_t fullnamesize, const char *newfile, const char *base);
//package formats:
//pakzip - files are uncompressed, with both a pak header and a zip trailer, allowing it to be read as either type of file.
//zip - standard zips
//spanned zip - the full list of files is written into a separate central-directory-only zip, the actual file data comes from regular zips named foo.z##/.p## instead of foo.zip/foo.pk3
/*
dataset common
{
output default data.pk3
output logic textures.pk3
}
dataset desktop
{
output tex textures_pc.pk3
}
dataset mobile
{
output tex textures_mobile.pk3
}
input pak0.pk3
rule dxt1 {
dataset desktop
output tex
newext dds
command "@\"c:/program files/Compressonator/CompressonatorCLI\" -fd DXT1 $input $output"
}
rule etc2 {
dataset mobile
output tex
newext ktx
command "@\"c:/program files/Compressonator/CompressonatorCLI\" -fd ETC2 $input $output"
}
logic {
progs.dat
}
class texa0 {
output tex
desktop: dxt1
mobile: etc2
}
texa0
{
gfx/conback.txt
}
*/
#define quint64_t long long
#define qofs_t size_t
#define countof(x) (sizeof(x)/sizeof((x)[0]))
struct pkgctx_s
{
void (*messagecallback)(void *userctx, char *message, ...);
void *userctx;
char *listfile;
pbool test;
pbool readoldpacks;
char gamepath[MAX_PATH];
char sourcepath[MAX_PATH];
time_t buildtime;
//skips the file if its listed in one of these packages, unless the modification time on disk is newer.
struct oldpack_s
{
struct oldpack_s *next;
char filename[128];
size_t numfiles;
struct
{
char name[128];
unsigned short zmethod;
unsigned int zcrc;
qofs_t zhdrofs;
qofs_t rawsize;
qofs_t zipsize;
unsigned short dostime;
unsigned short dosdate;
} *file;
} *oldpacks;
struct dataset_s
{
struct dataset_s *next;
//these are the output pk3s from this package.
struct output_s
{
struct output_s *next;
char code[128];
char filename[128];
struct file_s *files;
pbool usediffs;
unsigned int numparts;
struct oldpack_s *oldparts;
} *outputs;
char name[1];
} *datasets;
struct rule_s
{
struct rule_s *next;
char name[128];
int dropfile:1;
char *newext;
char *command;
} *rules;
struct class_s
{
char name[128];
struct class_s *next;
//the output package codename to write to. class is skipped if the dataset doesn't include that name.
char outname[128];
struct
{
struct dataset_s *set;
struct rule_s *rule;
} dataset[8];
struct rule_s *defaultrule;
struct file_s
{
struct file_s *next;
char name[128];
//temp data for tracking what's getting written.
struct
{
char name[128];
struct file_s *nextwrite;
struct rule_s *rule;
unsigned int zdisk;
unsigned short zmethod;
unsigned int zcrc;
qofs_t zhdrofs;
qofs_t pakofs;
qofs_t rawsize;
qofs_t zipsize;
unsigned short dostime;
unsigned short dosdate;
time_t timestamp;
} write;
} *files;
} *classes;
};
#ifdef _WIN32
static time_t filetime_to_timet(FILETIME ft)
{
ULARGE_INTEGER ull;
ull.LowPart = ft.dwLowDateTime;
ull.HighPart = ft.dwHighDateTime;
return ull.QuadPart / 10000000ULL - 11644473600ULL;
}
#endif
static struct rule_s *PKG_FindRule(struct pkgctx_s *ctx, char *code)
{
struct rule_s *o;
for (o = ctx->rules; o; o = o->next)
{
if (!strcmp(o->name, code))
return o;
}
return NULL;
}
static struct class_s *PKG_FindClass(struct pkgctx_s *ctx, char *code)
{
struct class_s *c;
for (c = ctx->classes; c; c = c->next)
{
if (!strcmp(c->name, code))
return c;
}
return NULL;
}
static struct dataset_s *PKG_FindDataset(struct pkgctx_s *ctx, char *code)
{
struct dataset_s *o;
for (o = ctx->datasets; o; o = o->next)
{
if (!strcmp(o->name, code))
return o;
}
return NULL;
}
static struct dataset_s *PKG_GetDataset(struct pkgctx_s *ctx, char *code)
{
struct dataset_s *s = PKG_FindDataset(ctx, code);
if (!s)
{
s = malloc(sizeof(*s)+strlen(code));
strcpy(s->name, code);
s->outputs = NULL;
s->next = ctx->datasets;
ctx->datasets = s;
}
return s;
}
static pbool PKG_SkipWhite(struct pkgctx_s *ctx, pbool linebreak)
{
for(;;)
{
if (qcc_iswhite(*ctx->listfile))
{
if (qcc_islineending(ctx->listfile[0], ctx->listfile[1]) && !linebreak)
return false;
ctx->listfile++;
continue;
}
if (ctx->listfile[0] == '/' && ctx->listfile[1] == '/')
{
while (!qcc_islineending(ctx->listfile[0], ctx->listfile[1]))
ctx->listfile++;
continue;
}
if (ctx->listfile[0] == '/' && ctx->listfile[1] == '*')
{
ctx->listfile+=2;
while (*ctx->listfile)
{
if (ctx->listfile[0]=='*' && ctx->listfile[1]=='/')
{
ctx->listfile+=2;
break;
}
ctx->listfile++;
}
continue;
}
break;
}
return true;
}
static pbool PKG_GetToken(struct pkgctx_s *ctx, char *token, size_t sizeoftoken, pbool linebreak)
{
if (!PKG_SkipWhite(ctx, linebreak))
return false;
if (*ctx->listfile)
{
while(*ctx->listfile)
{
if (qcc_iswhite(*ctx->listfile))
break;
*token = *ctx->listfile++;
if (sizeoftoken > 1)
{
token++;
sizeoftoken--;
}
}
*token = 0;
return true;
}
return false;
}
static pbool PKG_GetStringToken(struct pkgctx_s *ctx, char *token, size_t sizeoftoken)
{
if (!PKG_SkipWhite(ctx, false))
return false;
if (*ctx->listfile == '\"')
{
ctx->listfile++;
while(*ctx->listfile)
{
if (*ctx->listfile == '\"')
{
ctx->listfile++;
break;
}
else if (*ctx->listfile == '\\')
{
ctx->listfile++;
switch(*ctx->listfile++)
{
case '\"': *token = '\"'; break;
case '\\': *token = '\\'; break;
case '\r': *token = '\r'; break;
case '\n': *token = '\n'; break;
case '\t': *token = '\t'; break;
default: *token = '?'; break;
}
sizeoftoken--;
}
else
*token = *ctx->listfile++;
if (sizeoftoken > 1)
{
token++;
sizeoftoken--;
}
}
*token = 0;
return true;
}
return false;
}
static pbool PKG_Expect(struct pkgctx_s *ctx, char *token)
{
char tok[128];
if (PKG_GetToken(ctx, tok, sizeof(tok), true))
{
if (!strcmp(tok, token))
return true;
}
ctx->messagecallback(ctx->userctx, "Expected '%s', found '%s'\n", token, tok);
return false;
}
static void PKG_ReplaceString(char *str, char *find, char *newpart)
{
char *oldpart;
size_t oldlen = strlen(find);
size_t nlen = strlen(newpart);
while((oldpart = strstr(str, find)))
{
memmove(oldpart+nlen, oldpart+oldlen, strlen(oldpart+oldlen)+1);
memmove(oldpart, newpart, nlen);
str = oldpart+nlen;
}
}
static void PKG_CreateOutput(struct pkgctx_s *ctx, struct dataset_s *s, const char *code, const char *filename, pbool diff)
{
char path[MAX_PATH];
char date[64];
struct output_s *o;
for (o = s->outputs; o; o = o->next)
{
if (!strcmp(o->code, code))
{
ctx->messagecallback(ctx->userctx, "Dataset '%s' defined with dupe output\n", s->name, code);
return;
}
}
if (strlen(code) >= sizeof(o->code))
{
ctx->messagecallback(ctx->userctx, "Output '%s' name too long\n", code);
return;
}
strcpy(path, filename);
strftime(date, sizeof(date), "%Y%m%d", localtime(&ctx->buildtime));
PKG_ReplaceString(path, "$date", date);
o = malloc(sizeof(*o));
memset(o, 0, sizeof(*o));
strcpy(o->code, code);
o->usediffs = diff;
QCC_Canonicalize(o->filename, sizeof(o->filename), path, ctx->gamepath);
o->next = s->outputs;
s->outputs = o;
}
static void PKG_ParseOutput(struct pkgctx_s *ctx, pbool diff)
{
struct dataset_s *s;
char name[128];
char prop[128];
char fname[128];
if (!PKG_GetToken(ctx, name, sizeof(name), false))
{
ctx->messagecallback(ctx->userctx, "Output: Expected name\n");
return;
}
if (PKG_GetStringToken(ctx, prop, sizeof(prop)))
{
s = PKG_GetDataset(ctx, "core");
PKG_CreateOutput(ctx, s, name, prop, diff);
}
else
{
if (!PKG_Expect(ctx, "{"))
return;
while(PKG_GetToken(ctx, prop, sizeof(prop), true))
{
if (!strcmp(prop, "}"))
break;
else
{
char *e = strchr(prop, ':');
if (e && !e[1])
{
*e = 0;
s = PKG_GetDataset(ctx, prop);
if (PKG_GetStringToken(ctx, fname, sizeof(fname)))
PKG_CreateOutput(ctx, s, name, fname, diff);
else
ctx->messagecallback(ctx->userctx, "Output '%s[%s]' filename omitted\n", name, prop);
}
else
ctx->messagecallback(ctx->userctx, "Output '%s' has unknown property '%s'\n", name, prop);
}
//skip any junk
while(PKG_GetToken(ctx, prop, sizeof(prop), false))
{
if (!strcmp(prop, ";"))
break;
}
}
}
}
static void PKG_AddOldPack(struct pkgctx_s *ctx, const char *fname)
{
struct oldpack_s *pack;
pack = malloc(sizeof(*pack));
strcpy(pack->filename, fname);
pack->numfiles = 0;
pack->file = NULL;
pack->next = ctx->oldpacks;
ctx->oldpacks = pack;
}
static void PKG_ParseOldPack(struct pkgctx_s *ctx)
{
char token[MAX_PATH];
char oldpack[MAX_PATH];
if (!PKG_GetStringToken(ctx, token, sizeof(token)))
return;
#ifdef _WIN32
{
WIN32_FIND_DATA fd;
HANDLE h;
QCC_Canonicalize(oldpack, sizeof(oldpack), token, ctx->gamepath);
h = FindFirstFile(oldpack, &fd);
if (h == INVALID_HANDLE_VALUE)
ctx->messagecallback(ctx->userctx, "wildcard string '%s' found no files\n", token);
else
{
do
{
QCC_Canonicalize(token, sizeof(token), fd.cFileName, oldpack);
PKG_AddOldPack(ctx, token);
} while(FindNextFile(h, &fd));
}
}
#else
ctx->messagecallback(ctx->userctx, "no wildcard support, sorry\n");
#endif
}
/*
static void PKG_ParseDataset(struct pkgctx_s *ctx)
{
struct dataset_s *s;
char name[128];
char prop[128];
if (!PKG_GetToken(ctx, name, sizeof(name), false))
{
ctx->messagecallback(ctx->userctx, "Dataset: Expected name\n");
return;
}
if (strlen(name) >= sizeof(s->name))
{
ctx->messagecallback(ctx->userctx, "Dataset '%s' name too long\n", name);
return;
}
s = malloc(sizeof(*s));
memset(s, 0, sizeof(*s));
strcpy(s->name, name);
if (PKG_Expect(ctx, "{"))
{
while(PKG_GetToken(ctx, prop, sizeof(prop), true))
{
if (!strcmp(prop, "}"))
break;
else if (!strcmp(prop, "output"))
{
if (PKG_GetToken(ctx, name, sizeof(name), false))
if (PKG_GetStringToken(ctx, prop, sizeof(prop)))
{
PKG_CreateOutput(ctx, s, name, prop);
}
}
else if (!strcmp(prop, "base"))
PKG_GetStringToken(ctx, prop, sizeof(prop));
else
ctx->messagecallback(ctx->userctx, "Dataset '%s' has unknown property '%s'\n", name, prop);
//skip any junk
while(PKG_GetToken(ctx, prop, sizeof(prop), false))
{
if (!strcmp(prop, ";"))
break;
}
}
}
if (PKG_FindDataset(ctx, name))
ctx->messagecallback(ctx->userctx, "Dataset '%s' is already defined\n", name);
else
{ //link it in!
s->next = ctx->datasets;
ctx->datasets = s;
return;
}
PKG_DestroyDataset(s);
return;
}*/
static void PKG_ParseRule(struct pkgctx_s *ctx)
{
struct rule_s *r;
char name[128];
char prop[128];
char newext[128];
char command[4096];
int dropfile = false;
if (!PKG_GetToken(ctx, name, sizeof(name), false))
return;
if (strlen(name) >= sizeof(r->name))
{
ctx->messagecallback(ctx->userctx, "Rule '%s' name too long\n", name);
return;
}
*newext = *command = 0;
if (PKG_Expect(ctx, "{"))
{
while(PKG_GetToken(ctx, prop, sizeof(prop), true))
{
if (!strcmp(prop, "}"))
break;
else if (!strcmp(prop, "newext"))
PKG_GetToken(ctx, newext, sizeof(newext), false);
else if (!strcmp(prop, "skip"))
{
if (PKG_GetToken(ctx, prop, sizeof(prop), false))
dropfile = atoi(prop);
else
dropfile = true;
}
else if (!strcmp(prop, "command"))
PKG_GetStringToken(ctx, command, sizeof(command));
else
ctx->messagecallback(ctx->userctx, "Rule '%s' has unknown property '%s'\n", name, prop);
//skip any junk
while(PKG_GetToken(ctx, prop, sizeof(prop), false))
{
if (!strcmp(prop, ";"))
break;
}
}
}
r = PKG_FindRule(ctx, name);
if (r)
{
ctx->messagecallback(ctx->userctx, "Rule %s is already defined\n", name);
return;
}
r = malloc(sizeof(*r));
memset(r, 0, sizeof(*r));
strcpy(r->name, name);
r->newext = strdup(newext);
r->command = strdup(command);
r->dropfile = dropfile;
r->next = ctx->rules;
ctx->rules = r;
}
static void PKG_AddClassFile(struct pkgctx_s *ctx, struct class_s *c, const char *fname, time_t mtime)
{
struct file_s *f;
struct tm *t;
if (strlen(fname) >= sizeof(f->name))
{
ctx->messagecallback(ctx->userctx, "File name '%s' too long in class %s\n", fname, c->name);
return;
}
f = malloc(sizeof(*f));
memset(f, 0, sizeof(*f));
strcpy(f->name, fname);
f->write.timestamp = mtime;
t = localtime(&f->write.timestamp);
f->write.dostime = (t->tm_sec>>1)|(t->tm_min<<5)|(t->tm_hour<<11);
f->write.dosdate = (t->tm_mday<<0)|(t->tm_mon<<5)|((t->tm_year+1900-1980)<<9);
f->next = c->files;
c->files = f;
}
static void PKG_AddClassFiles(struct pkgctx_s *ctx, struct class_s *c, const char *fname)
{
#ifdef _WIN32
WIN32_FIND_DATA fd;
HANDLE h;
char basepath[MAX_PATH];
QCC_Canonicalize(basepath, sizeof(basepath), fname, ctx->sourcepath);
h = FindFirstFile(basepath, &fd);
if (h == INVALID_HANDLE_VALUE)
ctx->messagecallback(ctx->userctx, "wildcard string '%s' found no files\n", fname);
else
{
do
{
QCC_Canonicalize(basepath, sizeof(basepath), fd.cFileName, fname);
PKG_AddClassFile(ctx, c, basepath, filetime_to_timet(fd.ftLastWriteTime));
} while(FindNextFile(h, &fd));
}
#else
ctx->messagecallback(ctx->userctx, "no wildcard support, sorry\n");
#endif
}
static void PKG_ParseClass(struct pkgctx_s *ctx, char *output)
{
struct class_s *c;
struct rule_s *r;
struct dataset_s *s;
char *e;
char name[128];
char prop[128];
size_t u;
if (output)
{
if (!PKG_Expect(ctx, "{"))
return;
*name = 0;
}
else if (!PKG_GetToken(ctx, name, sizeof(name), false))
return;
if (output || !strcmp(name, "{"))
{
c = malloc(sizeof(*c));
memset(c, 0, sizeof(*c));
strcpy(c->name, "");
strcpy(c->outname, (output && *output)?output:"default");
c->next = ctx->classes;
ctx->classes = c;
}
else
{
if (strlen(name) >= sizeof(c->name))
{
ctx->messagecallback(ctx->userctx, "Class '%s' name too long\n", name);
return;
}
c = PKG_FindClass(ctx, name);
if (!c)
{
c = malloc(sizeof(*c));
memset(c, 0, sizeof(*c));
strcpy(c->name, name);
strcpy(c->outname, (output && *output)?output:"default");
c->next = ctx->classes;
ctx->classes = c;
}
if (!PKG_Expect(ctx, "{"))
return;
}
{
while(PKG_GetToken(ctx, prop, sizeof(prop), true))
{
if (!strcmp(prop, "}"))
break;
else if (!strcmp(prop, "output"))
PKG_GetToken(ctx, c->outname, sizeof(c->outname), false);
else if (!strcmp(prop, "rule"))
{
if (PKG_GetToken(ctx, prop, sizeof(prop), false))
{
if (c->defaultrule)
ctx->messagecallback(ctx->userctx, "Class '%s' already has a default rule\n", name);
c->defaultrule = PKG_FindRule(ctx, prop);
if (!c->defaultrule)
ctx->messagecallback(ctx->userctx, "Class '%s' specifies unknown rule %s\n", name, prop);
}
}
else
{
e = strchr(prop, ':');
if (e && !e[1])
{
*e = 0;
s = PKG_FindDataset(ctx, prop);
PKG_GetToken(ctx, prop, sizeof(prop), false);
if (s)
{
r = PKG_FindRule(ctx, prop);
for (u = 0; ; u++)
{
if (u == countof(c->dataset))
{
ctx->messagecallback(ctx->userctx, "Class '%s' specialises for too many datasets\n", c->name, s->name);
break;
}
if (c->dataset[u].set == s)
ctx->messagecallback(ctx->userctx, "Class '%s' already defines a rule for dataset '%s'\n", c->name, s->name);
else if (!c->dataset[u].set)
{
c->dataset[u].set = s;
c->dataset[u].rule = r;
break;
}
}
}
}
else if (strchr(prop, '.'))
{
// if (strchr(prop, '*') || strchr(prop, '?'))
PKG_AddClassFiles(ctx, c, prop);
// else
// PKG_AddClassFile(ctx, c, prop);
}
else
ctx->messagecallback(ctx->userctx, "Class '%s' has unknown property '%s'\n", name, prop);
}
//skip any junk
while(PKG_GetToken(ctx, prop, sizeof(prop), false))
{
if (!strcmp(prop, ";"))
break;
}
}
}
}
static void PKG_ParseClassFiles(struct pkgctx_s *ctx, struct class_s *c)
{
char prop[128];
if (PKG_Expect(ctx, "{"))
{
while(PKG_GetToken(ctx, prop, sizeof(prop), true))
{
if (!strcmp(prop, "}"))
break;
if (!strcmp(prop, ";"))
continue;
// if (strchr(prop, '*') || strchr(prop, '?'))
PKG_AddClassFiles(ctx, c, prop);
// else
// PKG_AddClassFile(ctx, c, prop);
}
}
}
#ifdef AVAIL_ZLIB
#include <zlib.h>
static unsigned int PKG_DeflateToFile(FILE *f, unsigned int rawsize, void *in, int method)
{
char out[8192];
int i=0;
z_stream strm = {
(char *)in,
rawsize,
0,
out,
sizeof(out),
0,
NULL,
NULL,
NULL,
NULL,
NULL,
Z_BINARY,
0,
0
};
if (method == 8)
deflateInit2(&strm, 9, Z_DEFLATED, -MAX_WBITS, 9, Z_DEFAULT_STRATEGY); //zip deflate compression
else
deflateInit(&strm, Z_BEST_COMPRESSION); //zlib compression
while(deflate(&strm, Z_FINISH) == Z_OK)
{
fwrite(out, 1, sizeof(out) - strm.avail_out, f); //compress in chunks of 8192. Saves having to allocate a huge-mega-big buffer
i+=sizeof(out) - strm.avail_out;
strm.next_out = out;
strm.avail_out = sizeof(out);
}
deflateEnd(&strm);
fwrite(out, 1, sizeof(out) - strm.avail_out, f);
i+=sizeof(out) - strm.avail_out;
return i;
}
#endif
#ifdef _WIN32
static void StupidWindowsPopenAlternativeCrap(struct pkgctx_s *ctx, char *commandline)
{
PROCESS_INFORMATION piProcInfo = {0};
SECURITY_ATTRIBUTES saAttr = {sizeof(SECURITY_ATTRIBUTES), NULL, TRUE};
STARTUPINFO siStartInfo = {sizeof(STARTUPINFO)};
HANDLE readpipe = INVALID_HANDLE_VALUE;
HANDLE writepipe = INVALID_HANDLE_VALUE;
siStartInfo.hStdError = siStartInfo.hStdOutput = siStartInfo.hStdInput = INVALID_HANDLE_VALUE;
if (CreatePipe(&readpipe, &siStartInfo.hStdOutput, &saAttr, 0))
{
if (CreatePipe(&siStartInfo.hStdInput, &writepipe, &saAttr, 0))
{
SetHandleInformation(readpipe, HANDLE_FLAG_INHERIT, 0);
SetHandleInformation(writepipe, HANDLE_FLAG_INHERIT, 0);
siStartInfo.hStdError = siStartInfo.hStdOutput;
siStartInfo.dwFlags |= STARTF_USESTDHANDLES|STARTF_USESHOWWINDOW/*ZOMGWTFBBQ*/;
if (!CreateProcess(NULL, (*commandline=='@')?commandline+1:commandline, NULL, NULL, TRUE, 0, NULL, NULL, &siStartInfo, &piProcInfo))
ctx->messagecallback(ctx->userctx, "Unable to execute command %s\n", commandline);
else
{
CloseHandle(piProcInfo.hProcess);
CloseHandle(piProcInfo.hThread);
}
}
}
CloseHandle(siStartInfo.hStdOutput);
CloseHandle(siStartInfo.hStdInput);
CloseHandle(writepipe);
for (;;)
{
char buf[64];
DWORD SHOUTY;
if (!ReadFile(readpipe, buf, sizeof(buf)-1, &SHOUTY, NULL) || SHOUTY == 0)
break;
if (*commandline == '@')
continue;
buf[SHOUTY] = 0;
ctx->messagecallback(ctx->userctx, "%s", buf);
}
CloseHandle(readpipe);
}
#endif
static void *PKG_OpenSourceFile(struct pkgctx_s *ctx, struct file_s *file, size_t *fsize)
{
char fullname[1024];
FILE *f;
char *data;
size_t size;
struct rule_s *rule = file->write.rule;
*fsize = 0;
QCC_Canonicalize(fullname, sizeof(fullname), file->name, ctx->sourcepath);
strcpy(file->write.name, file->name);
f = fopen(fullname, "rb");
if (!f)
return NULL;
if (rule)
ctx->messagecallback(ctx->userctx, "\t\tProcessing %s (%s)\n", file->name, rule->name);
else
ctx->messagecallback(ctx->userctx, "\t\tCompressing %s\n", file->name);
if (rule)
{
data = strrchr(file->write.name, '.');
if (!data)
data = file->write.name+strlen(file->write.name);
if (strchr(rule->newext, '.'))
strcpy(data, rule->newext); //note: this allows weird _foo.tga postfixes.
else
{
*data = '.';
strcpy(data+1, rule->newext);
}
if (rule->command)
{
int i;
char commandline[4096];
char *cmd;
char tempname[1024];
//generate a sequenced temp filename
//run the external tool to write that file
//read the temp file.
//delete temp file...
fclose(f);
QCC_Canonicalize(tempname, sizeof(tempname), file->write.name, ctx->sourcepath);
f = fopen(tempname, "rb");
if (f)
{
fclose(f);
ctx->messagecallback(ctx->userctx, "Temp file %s already exists... not replacing+deleting\n", tempname);
return NULL;
}
for (i = 0, cmd = rule->command; *cmd && i < countof(commandline)-1; )
{
if (!strncmp(cmd, "$input", 6))
{
strcpy(&commandline[i], fullname);
i += strlen(&commandline[i]);
cmd += 6;
}
else if (!strncmp(cmd, "$output", 7))
{
strcpy(&commandline[i], tempname);
i += strlen(&commandline[i]);
cmd += 7;
}
else
commandline[i++] = *cmd++;
}
commandline[i] = 0;
// ctx->messagecallback(ctx->userctx, "Commandline is %s\n", commandline);
#ifdef _WIN32 //windows is so fucking useless sometimes. sure, _popen 'works'... its just perverse enough that its not an option, forcing system-specific crap in anything that isn't originally from unix... maybe it is just incompetence? still feels like malice to me.
StupidWindowsPopenAlternativeCrap(ctx, commandline);
#else
{
FILE *p;
p = popen((*commandline=='@')?commandline+1:commandline, "rt");
if (!p)
{
ctx->messagecallback(ctx->userctx, "Unable to execute command\n", tempname);
return NULL;
}
while(fgets(commandline, sizeof(commandline), p))
ctx->messagecallback(ctx->userctx, "%s", commandline);
if (feof(p))
ctx->messagecallback(ctx->userctx, "Process returned %d\n", _pclose( p ));
else
{
printf( "Error: Failed to read the pipe to the end.\n");
_pclose(p);
}
}
#endif
f = fopen(tempname, "rb");
if (!f)
{
ctx->messagecallback(ctx->userctx, "Temp file %s wasn't created\n", tempname);
return NULL;
}
fseek(f, 0, SEEK_END);
size = ftell(f);
fseek(f, 0, SEEK_SET);
data = malloc(size+1);
fread(data, 1, size, f);
fclose(f);
*fsize = size;
_unlink(tempname);
return data;
}
}
fseek(f, 0, SEEK_END);
size = ftell(f);
fseek(f, 0, SEEK_SET);
data = malloc(size+1);
fread(data, 1, size, f);
fclose(f);
*fsize = size;
return data;
}
static void PKG_WritePackageData(struct pkgctx_s *ctx, struct output_s *out, unsigned int index, pbool directoryonly)
{
//helpers to deal with misaligned data. writes little-endian.
#define misbyte(ptr,ofs,data) ((unsigned char*)(ptr))[ofs] = (data)&0xff
#define misshort(ptr,ofs,data) do{misbyte((ptr),(ofs),(data));misbyte((ptr),(ofs)+1,(data)>>8);}while(0)
#define misint(ptr,ofs,data) do{misshort((ptr),(ofs),(data));misshort((ptr),(ofs)+2,(data)>>16);}while(0)
#define misint64(ptr,ofs,data) do{misint((ptr),(ofs),(data));misint((ptr),(ofs)+4,((quint64_t)(data))>>32);}while(0)
qofs_t num=0;
pbool pak = false;
int startofs = 0;
struct file_s *f;
char centralheader[46+sizeof(f->write.name)];
qofs_t centraldirsize;
qofs_t centraldirofs;
qofs_t z64eocdofs;
char *filedata;
FILE *outf;
struct
{
char magic[4];
unsigned int tabofs;
unsigned int tabbytes;
} pakheader = {"PACK", 0, 0};
char *ext;
#define GPF_TRAILINGSIZE (1u<<3)
#define GPF_UTF8 (1u<<11)
#ifdef AVAIL_ZLIB
#define compmethod (pak?0:8)/*Z_DEFLATED*/
#else
#define compmethod 0/*Z_RAW*/
#endif
if (!compmethod && !directoryonly)
pak = true; //might as well boost compat...
ext = strrchr(out->filename, '.');
if (ext && !QC_strcasecmp(ext, ".pak") && !index)
pak = true;
if (out->usediffs && !directoryonly)
{
char newname[MAX_PATH];
memcpy(newname, out->filename, sizeof(newname));
if (ext)
{
ext = newname+(ext-out->filename);
ext+=1;
if (*ext)
ext++;
QC_snprintfz(ext, sizeof(newname)-(ext-newname), "%02u", index+1);
}
outf = fopen(newname, "wb");
}
else
outf = fopen(out->filename, "wb");
if (!outf)
{
ctx->messagecallback(ctx->userctx, "Unable to open %s\n", out->filename);
return;
}
if (pak) //reserve space for the pak header
fwrite(&pakheader, 1, sizeof(pakheader), outf);
if (!directoryonly)
{
for (f = out->files; f ; f=f->write.nextwrite)
{
char header[32+sizeof(f->write.name)];
size_t fnamelen = strlen(f->write.name);
size_t hofs;
unsigned short gpflags = GPF_UTF8;
if (index != f->write.zdisk)
continue; //not in this disk...
filedata = PKG_OpenSourceFile(ctx, f, &f->write.rawsize);
if (!filedata)
{
ctx->messagecallback(ctx->userctx, "Unable to open %s\n", f->name);
}
f->write.zcrc = QC_encodecrc(f->write.rawsize, filedata);
misint (header, 0, 0x04034b50);
misshort(header, 4, 45);//minver
misshort(header, 6, gpflags);//general purpose flags
misshort(header, 8, 0);//compression method, 0=store, 8=deflate
misshort(header, 10, f->write.dostime);//lastmodfiletime
misshort(header, 12, f->write.dosdate);//lastmodfiledate
misint (header, 14, f->write.zcrc);//crc32
misint (header, 18, f->write.rawsize);//compressed size
misint (header, 22, f->write.rawsize);//uncompressed size
misshort(header, 26, fnamelen);//filename length
misshort(header, 28, 0);//extradata length
strcpy(header+30, f->write.name);
hofs = 30+fnamelen;
misshort(header, 28, hofs-(30+fnamelen));//extradata length
f->write.zhdrofs = ftell(outf);
fwrite(header, 1, hofs, outf);
#ifdef AVAIL_ZLIB
if (f->write.rawsize && (compmethod == 2 || compmethod == 8))
{
gpflags |= 1u<<1;
f->write.pakofs = 0;
f->write.zmethod = compmethod;
f->write.zipsize = PKG_DeflateToFile(outf, f->write.rawsize, filedata, compmethod);
}
else
#endif
{
f->write.zmethod = 0;
f->write.pakofs = ftell(outf);
f->write.zipsize = fwrite(filedata, 1, f->write.rawsize, outf);
}
//update the header
misshort(header, 8, f->write.zmethod);//compression method, 0=store, 8=deflate
if (f->write.zipsize > 0xffffffff)
{
misint (header, 18, 0xffffffff);//compressed size
gpflags |= GPF_TRAILINGSIZE;
}
else
misint (header, 18, f->write.zipsize);//compressed size
if (f->write.rawsize > 0xffffffff)
{
misint (header, 22, 0xffffffff);//compressed size
gpflags |= GPF_TRAILINGSIZE;
}
else
misint (header, 22, f->write.rawsize);//compressed size
misshort(header, 6, gpflags);//general purpose flags
fseek(outf, f->write.zhdrofs, SEEK_SET);
fwrite(header, 1, hofs, outf);
fseek(outf, 0, SEEK_END);
if (gpflags & GPF_TRAILINGSIZE) //if (gpflags & GPF_TRAILINGSIZE)
{
misint (header, 0, 0x08074b50);
misint (header, 4, f->write.zcrc);
misint64(header, 8, f->write.zipsize);
misint64(header, 16, f->write.rawsize);
fwrite(header, 1, 24, outf);
}
free(filedata);
num++;
}
}
if (pak)
{
struct
{
char name[56];
unsigned int size;
unsigned int offset;
} pakentry;
pakheader.tabofs = ftell(outf);
//write the pak file table.
for (f = out->files,num=0; f ; f=f->write.nextwrite)
{
if (index != f->write.zdisk)
continue; //not in this disk...
memset(&pakentry, 0, sizeof(pakentry));
QC_strlcpy(pakentry.name, f->write.name, sizeof(pakentry.name));
pakentry.size = (f->write.pakofs==0)?0:f->write.rawsize;
pakentry.offset = f->write.pakofs;
fwrite(&pakentry, 1, sizeof(pakentry), outf);
num++;
}
//replace the pak header, then return to the end of the file for the zip end-of-central-directory
pakheader.tabbytes = num * sizeof(pakentry);
fseek(outf, 0, SEEK_SET);
fwrite(&pakheader, 1, sizeof(pakheader), outf);
fseek(outf, 0, SEEK_END);
}
centraldirofs = ftell(outf);
for (f = out->files,num=0; f ; f=f->write.nextwrite)
{
size_t hofs;
size_t fnamelen;
if (!directoryonly && index != f->write.zdisk)
continue;
fnamelen = strlen(f->write.name);
misint (centralheader, 0, 0x02014b50);
misshort(centralheader, 4, (3<<8)|63);//ourver
misshort(centralheader, 6, 45);//minver
misshort(centralheader, 8, GPF_UTF8);//general purpose flags
misshort(centralheader, 10, f->write.rawsize?compmethod:0);//compression method, 0=store, 8=deflate
misshort(centralheader, 12, f->write.dostime);//lastmodfiletime
misshort(centralheader, 14, f->write.dosdate);//lastmodfiledate
misint (centralheader, 16, f->write.zcrc);//crc32
misint (centralheader, 20, f->write.zipsize);//compressed size
misint (centralheader, 24, f->write.rawsize);//uncompressed size
misshort(centralheader, 28, fnamelen);//filename length
misshort(centralheader, 30, 0);//extradata length
misshort(centralheader, 32, 0);//comment length
misshort(centralheader, 34, f->write.zdisk);//first disk number
misshort(centralheader, 36, 0);//internal file attribs
misint (centralheader, 38, 0);//external file attribs
misint (centralheader, 42, f->write.zhdrofs);//local header offset
strcpy(centralheader+46, f->write.name);
hofs = 46+fnamelen;
if (f->write.zdisk >= 0xffff || f->write.zhdrofs >= 0xffffffff || f->write.rawsize >= 0xffffffff || f->write.zipsize >= 0xffffffff)
{
misshort(centralheader, hofs, 0x0001);//zip64 tagid
misshort(centralheader, hofs+2, 0x0001);//zip64 tag size
hofs+=4;
if (f->write.rawsize >= 0xffffffff)
{
misint64(centralheader, hofs, f->write.rawsize);//uncompressed size
hofs += 8;
}
if (f->write.zipsize >= 0xffffffff)
{
misint64(centralheader, hofs, f->write.zipsize);//compressed size
hofs += 8;
}
if (f->write.zhdrofs >= 0xffffffff)
{
misint64(centralheader, hofs, f->write.zhdrofs);//localheader offset
hofs += 8;
}
if (f->write.zdisk >= 0xffff)
{
misint (centralheader, hofs, f->write.zdisk);//compressed size
hofs += 4;
}
}
misshort(centralheader, 30, hofs-(46+fnamelen));//extradata length
fwrite(centralheader, 1, hofs, outf);
num++;
}
centraldirsize = ftell(outf)-centraldirofs;
//zip64 end of central dir
z64eocdofs = ftell(outf);
misint (centralheader, 0, 0x06064b50);
misint64(centralheader, 4, (qofs_t)(56-16));
misshort(centralheader, 12, (3<<8)|63); //ver made by = unix|appnote ver
misshort(centralheader, 14, 45); //ver needed
misint (centralheader, 16, index); //thisdisk number
misint (centralheader, 20, index); //centraldir start disk
misint64(centralheader, 24, num); //centraldir entry count (disk)
misint64(centralheader, 32, num); //centraldir entry count (total)
misint64(centralheader, 40, centraldirsize);//centraldir entry bytes
misint64(centralheader, 48, centraldirofs); //centraldir start offset
fwrite(centralheader, 1, 56, outf);
//zip64 end of central dir locator
misint (centralheader, 0, 0x07064b50);
misint (centralheader, 4, index); //centraldir first disk
misint64(centralheader, 8, z64eocdofs);
misint (centralheader, 16, index+1); //total disk count
fwrite(centralheader, 1, 20, outf);
// centraldirofs = ftell(outf) - centraldirofs;
//write zip end-of-central-directory
misint (centralheader, 0, 0x06054b50);
misshort(centralheader, 4, (index > 0xffff)? 0xffff:index); //this disk number
misshort(centralheader, 6, (index > 0xffff)? 0xffff:index); //centraldir first disk
misshort(centralheader, 8, (num > 0xffff)? 0xffff:num); //centraldir entries
misshort(centralheader, 10, (num > 0xffff)? 0xffff:num); //total centraldir entries
misint (centralheader, 12, (centraldirsize>0xffffffff)?0xffffffff:centraldirsize); //centraldir size
misint (centralheader, 16, (centraldirofs >0xffffffff)?0xffffffff:centraldirofs); //centraldir offset
misshort(centralheader, 20, 0); //comment length
fwrite(centralheader, 1, 22, outf);
fclose(outf);
}
#include <sys/stat.h>
static time_t PKG_GetFileTime(const char *filename)
{
struct stat s;
if (stat(filename, &s) != -1)
return s.st_mtime;
}
static void PKG_ReadPackContents(struct pkgctx_s *ctx, struct oldpack_s *old)
{
#define longfromptr(p) (((p)[0]<<0)|((p)[1]<<8)|((p)[2]<<16)|((p)[3]<<24))
#define shortfromptr(p) (((p)[0]<<0)|((p)[1]<<8))
size_t u, namelen;
unsigned int foffset;
unsigned char header[46];
int i;
FILE *f;
//ignore packages if we're going to be overwritten.
struct dataset_s *set;
struct output_s *out;
for (set = ctx->datasets; set; set = set->next)
{
for (out = set->outputs; out; out = out->next)
{
if (!strcmp(out->filename, old->filename))
return;
}
}
f = fopen(old->filename, "rb");
if (f)
{
//find end-of-central-dir
//assume no comment
fseek(f, -22, SEEK_END);
fread(header, 1, 22, f);
if (header[0] == 'P' && header[1] == 'K' && header[2] == 5 && header[3] == 6)
{
//thisdisk = shortfromptr(header+4);
//centraldirstart = shortfromptr(header+6);
old->numfiles = shortfromptr(header+8);
//numfiles_all = shortfromptr(header+10);
//centaldirsize = shortfromptr(header+12);
foffset = longfromptr(header+16);
//commentength = shortfromptr(header+20);
old->file = malloc(sizeof(*old->file)*old->numfiles);
fseek(f, foffset, SEEK_SET);
for(u = 0; u < old->numfiles; u++)
{
unsigned int extra_len, comment_len;
fread(header, 1, 46, f);
//zcrc @ 16
//version_madeby = shortfromptr(header+4);
//version_needed = shortfromptr(header+6);
//gflags = shortfromptr(header+8);
old->file[u].zmethod = shortfromptr(header+10);
old->file[u].dostime = shortfromptr(header+12);
old->file[u].dosdate = shortfromptr(header+12);
old->file[u].zcrc = longfromptr(header+16);
old->file[u].zipsize = longfromptr(header+20);
old->file[u].rawsize = longfromptr(header+24);
namelen = shortfromptr(header+28);
extra_len = shortfromptr(header+30);
comment_len = shortfromptr(header+32);
//disknum = shortfromptr(header+34);
//iattributes = shortfromptr(header+36);
//eattributes = longfromptr(header+38);
//localheaderoffset = longfromptr(header+42);
fread(old->file[u].name, 1, namelen, f);
old->file[u].name[namelen] = 0;
i = extra_len+comment_len;
if (i)
fseek(f, i, SEEK_CUR);
}
}
else
{
fseek(f, 0, SEEK_SET);
fread(header, 1, 12, f);
if (header[0] == 'P' && header[1] == 'A' && header[2] == 'C' && header[3] == 'K')
{
unsigned int ofs = longfromptr(header+4);
unsigned int dsz = longfromptr(header+8);
struct
{
char name[56];
unsigned int size;
unsigned int offset;
} *files;
files = malloc(dsz);
fseek(f, ofs, SEEK_SET);
fread(files, 1, dsz, f);
old->numfiles = dsz / sizeof(*files);
old->file = malloc(sizeof(*old->file)*old->numfiles);
for (u = 0; u < old->numfiles; u++)
{
strcpy(old->file[u].name, files[u].name);
old->file[u].rawsize = files[u].size;
}
free(files);
}
else
ctx->messagecallback(ctx->userctx, "%s does not appear to be a package\n", old->filename);
}
//walk central directory
fclose(f);
}
}
static pbool PKG_FileIsModified(struct pkgctx_s *ctx, struct oldpack_s *old, struct file_s *file)
{
size_t u;
for (u = 0; u < old->numfiles; u++)
{
//should check filesize etc, but rules and extension changes make that messy
if (!strcmp(old->file[u].name, file->name))
{
if(file->write.dosdate < old->file[u].dosdate || (file->write.dosdate == old->file[u].dosdate && file->write.dostime <= old->file[u].dostime))
{
file->write.zmethod = old->file[u].zmethod;
//char name[128];
file->write.zcrc = old->file[u].zcrc;
file->write.zhdrofs = old->file[u].zhdrofs;
file->write.pakofs = 0;
file->write.rawsize = old->file[u].rawsize;
file->write.zipsize = old->file[u].zipsize;
file->write.dostime = old->file[u].dostime;
file->write.dosdate = old->file[u].dosdate;
return false;
}
}
}
return true;
}
static void PKG_WriteDataset(struct pkgctx_s *ctx, struct dataset_s *set)
{
struct class_s *cls;
struct output_s *out;
struct file_s *file;
struct rule_s *rule;
struct oldpack_s *old;
size_t u;
if (!ctx->readoldpacks)
{
ctx->readoldpacks = true;
for(old = ctx->oldpacks; old; old = old->next)
{ //fixme: strip any wildcarded paks that match an output, to avoid weirdness.
PKG_ReadPackContents(ctx, old);
}
for (out = set->outputs; out; out = out->next)
{
if(out->usediffs)
{ //FIXME: look for old parts
}
}
}
ctx->messagecallback(ctx->userctx, "Building dataset %s\n", set->name);
for (cls = ctx->classes; cls; cls = cls->next)
{
for (out = set->outputs; out; out = out->next)
{
if (!strcmp(out->code, cls->outname))
break;
}
if (!out) //dataset doesn't name this.
continue;
rule = cls->defaultrule;
for (u = 0; u < countof(cls->dataset); u++)
{
if (cls->dataset[u].set == set)
{
rule = cls->dataset[u].rule;
break;
}
}
if (rule && rule->dropfile)
continue;
for (file = cls->files; file; file = file->next)
{
for (old = ctx->oldpacks; old; old = old->next)
{
if (!PKG_FileIsModified(ctx, old, file))
break;
}
if (old)
{
ctx->messagecallback(ctx->userctx, "\t\tFile %s found inside %s\n", file->name, old->filename);
file->write.zdisk = ~0u;
}
else
{
// ctx->messagecallback(ctx->userctx, "\t\tFile %s, rule %s\n", file->name, rule?rule->name:"");
file->write.zdisk = out->numparts;
for (old = out->oldparts; old; old = old->next)
{
if (!PKG_FileIsModified(ctx, old, file))
break;
}
file->write.nextwrite = out->files;
file->write.rule = rule;
out->files = file;
}
}
}
for (out = set->outputs; out; out = out->next)
{
if (!out->files)
{
ctx->messagecallback(ctx->userctx, "\tOutput %s[%s] \"%s\" has no files\n", out->code, set->name, out->filename);
continue;
}
if (ctx->test)
{
for (file = out->files; file; file = file->write.nextwrite)
{
if (file->write.rule)
ctx->messagecallback(ctx->userctx, "\t\tFile %s has changed (rule %s)\n", file->name, file->write.rule->name);
else
ctx->messagecallback(ctx->userctx, "\t\tFile %s has changed\n", file->name);
}
}
else
{
ctx->messagecallback(ctx->userctx, "\tGenerating %s[%s] \"%s\"\n", out->code, set->name, out->filename);
PKG_WritePackageData(ctx, out, out->numparts, false);
}
if(out->usediffs)
PKG_WritePackageData(ctx, out, out->numparts+1, true);
}
}
void Packager_WriteDataset(struct pkgctx_s *ctx, char *setname)
{
struct dataset_s *dataset;
if (setname && strcmp(setname, "*"))
{
dataset = PKG_FindDataset(ctx, setname);
if (dataset)
PKG_WriteDataset(ctx, dataset);
else
ctx->messagecallback(ctx->userctx, "Dataset %s not known\n", setname);
}
else
{
for (dataset = ctx->datasets; dataset; dataset = dataset->next)
PKG_WriteDataset(ctx, dataset);
}
}
struct pkgctx_s *Packager_Create(void (*messagecallback)(void *userctx, char *message, ...), void *userctx)
{
struct pkgctx_s *ctx;
ctx = malloc(sizeof(*ctx));
memset(ctx, 0, sizeof(*ctx));
ctx->messagecallback = messagecallback;
ctx->userctx = userctx;
ctx->test = false;
time(&ctx->buildtime);
return ctx;
}
void Packager_ParseText(struct pkgctx_s *ctx, char *scripttext)
{
char cmd[128];
ctx->listfile = scripttext;
while (PKG_GetToken(ctx, cmd, sizeof(cmd), true))
{
// if (!strcmp(cmd, "dataset"))
// PKG_ParseDataset(ctx);
if (!strcmp(cmd, "output"))
PKG_ParseOutput(ctx, false);
else if (!strcmp(cmd, "diffoutput") || !strcmp(cmd, "splitoutput"))
PKG_ParseOutput(ctx, true);
else if (!strcmp(cmd, "inputdir"))
{
char old[MAX_PATH];
memcpy(old, ctx->sourcepath, sizeof(old));
if (PKG_GetStringToken(ctx, cmd, sizeof(cmd)))
{
QC_strlcat(cmd, "/", sizeof(cmd));
QCC_Canonicalize(ctx->sourcepath, sizeof(ctx->sourcepath), cmd, old);
}
}
else if (!strcmp(cmd, "rule"))
PKG_ParseRule(ctx);
else if (!strcmp(cmd, "class"))
PKG_ParseClass(ctx, NULL);
else if (!strcmp(cmd, "ignore")||!strcmp(cmd, "oldpack"))
PKG_ParseOldPack(ctx);
else
{
char *e = strchr(cmd, ':');
if (e && !e[1])
{
*e = 0;
PKG_ParseClass(ctx, cmd);
}
else
{
struct class_s *c = PKG_FindClass(ctx, cmd);
if (c)
PKG_ParseClassFiles(ctx, c);
else
ctx->messagecallback(ctx->userctx, "Unrecognised token at global scope '%s'\n", cmd);
}
}
//skip any junk
while(PKG_GetToken(ctx, cmd, sizeof(cmd), false))
{
if (!strcmp(cmd, ";"))
break;
}
}
}
void Packager_ParseFile(struct pkgctx_s *ctx, char *scriptname)
{
size_t remaining = 0;
char *file = qccprogfuncs->funcs.parms->ReadFile(scriptname, NULL, NULL, &remaining);
strcpy(ctx->gamepath, scriptname);
strcpy(ctx->sourcepath, scriptname);
Packager_ParseText(ctx, file);
free(file);
}
void Packager_Destroy(struct pkgctx_s *ctx)
{
}
View git blame Copy permalink