qf/quakeforge
0
0
Fork 0
mirror of https://git.code.sf.net/p/quake/quakeforge synced 2025-04-06 01:11:59 +00:00
Code Issues Projects Releases Packages Wiki Activity

[qfcc] Parse the spirv grammar json files

Browse source 
I really don't like the way they're included (I'm really looking forward
to #embed, but gotta wait for gcc 15), and I'm a tad grumpy that the
documentation for them
(https://registry.khronos.org/SPIR-V/specs/unified1/MachineReadableGrammar.html)
is wrong (missing fields), but I think I like the result.

The grammars (core and glsl.std.450) are parsed into structs that should
be fairly easy to interpret: the instructions, kinds, and enumerant
values are sorted by name for search with bsearch. Having the data
parsed in means source code can refer to the items by name rather than
magic numbers, which will be very nice for intrinsics and image types
(and probably a few other things).
This commit is contained in:
Bill Currie 2025-01-07 03:26:12 +09:00
parent c0004ed7d5
commit 99b1859aac
5 changed files with 600 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
tools/qfcc/include/spirv.h
View file

@ -75,4 +75,6 @@ void spirv_set_addressing_model (module_t *module, SpvAddressingModel model);
void spirv_set_memory_model (module_t *module, SpvMemoryModel model);
bool spirv_write (struct pr_info_s *pr, const char *filename);
const struct plitem_s *spirv_operand_kind (const char *set, const char *kind);
#endif//__spirv_h

91
tools/qfcc/include/spirv_grammar.h Normal file
View file

@ -0,0 +1,91 @@
/*
spirv_grammar.c
SPIR-V grammar json embedding
Copyright (C) 2024 Bill Currie
Author: Bill Currie <bill@taniwha.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to:
Free Software Foundation, Inc.
59 Temple Place - Suite 330
Boston, MA 02111-1307, USA
*/
#ifndef __spirv_grammar_h
#define __spirv_grammar_h
typedef struct spirv_operand_s {
const char *kind;
char quantifier; // ?
const char *name; // ?
} spirv_operand_t;
typedef struct spiv_enumerant_s {
const char *enumerant;
uint32_t value;
uint32_t num_capabilities;
const char **capabilities;
const char **extensions;
uint32_t num_extensions;
uint32_t num_parameters;
spirv_operand_t *parameters;
const char *version;
const char *lastVersion;
} spirv_enumerant_t;
typedef struct spirv_kind_s {
const char *category;
const char *kind;
const char *doc;
uint32_t num;
union {
spirv_enumerant_t *enumerants;
const char **bases;
};
} spirv_kind_t;
typedef struct spirv_instruction_s {
const char *opname;
uint32_t opcode;
uint32_t num_operands;
spirv_operand_t *operands;
const char **capabilities;
uint32_t num_capabilities;
uint32_t num_extensions;
const char **extensions;
const char *version;
const char *lastVersion;
} spirv_instruction_t;
typedef struct spirv_grammar_s {
struct spirv_grammar_s *parent; // for inheriting operand kinds
const char **copyright; // array of string objects
uint32_t num_copyright;
uint32_t magic_number;
uint32_t major_version;
uint32_t minor_version;
uint32_t version;
uint32_t revision;
spirv_instruction_t *instructions;
uint32_t num_instructions;
uint32_t num_operand_kinds;
spirv_kind_t *operand_kinds;
} spirv_grammar_t;
#endif//__spirv_grammar_h

26
tools/qfcc/source/Makemodule.am
View file

@ -75,6 +75,7 @@ qfcc_SOURCES = \
tools/qfcc/source/reloc.c \
tools/qfcc/source/rua-declaration.c \
tools/qfcc/source/shared.c \
tools/qfcc/source/spirv_grammar.c \
tools/qfcc/source/statements.c \
tools/qfcc/source/strpool.c \
tools/qfcc/source/struct.c \
@ -88,6 +89,31 @@ qfcc_SOURCES = \
tools/qfcc/source/value.c \
$(tracy_src)
qfcc_source = $(top_builddir)/tools/qfcc/source
embed_py = $(srcdir)/tools/qfcc/source/embed.py
spirv_core_grammar_json = spirv/unified1/spirv.core.grammar.json
spirv_core_grammar_jinc = $(qfcc_source)/spirv.core.grammar.jinc
$(spirv_core_grammar_jinc): $(embed_py)
$(V_PY)$(PYTHON) $(embed_py) $(COMPILE) \
-o $@.t $(spirv_core_grammar_json) && \
$(am__mv) $@.t $@
extinst_glsl_std_450_grammar_json = \
spirv/unified1/extinst.glsl.std.450.grammar.json
extinst_glsl_std_450_grammar_jinc = \
$(qfcc_source)/extinst.glsl.std.450.grammar.jinc
$(extinst_glsl_std_450_grammar_jinc): $(embed_py)
$(V_PY)$(PYTHON) $(embed_py) $(COMPILE) \
-o $@.t $(extinst_glsl_std_450_grammar_json) && \
$(am__mv) $@.t $@
tools/qfcc/source/spirv_grammar.$(OBJEXT): \
tools/qfcc/source/spirv_grammar.c \
$(spirv_core_grammar_jinc) \
$(extinst_glsl_std_450_grammar_jinc)
qfcc_LDADD= $(QFCC_LIBS)
qfcc_DEPENDENCIES= $(QFCC_DEPS)

46
tools/qfcc/source/embed.py Normal file
View file

@ -0,0 +1,46 @@
import sys
import re
import subprocess
outind = sys.argv.index("-o")
flags = sys.argv[1:outind]
if "-include" in flags:
ind = flags.index("-include")
flags = flags[:ind] + flags[ind + 2:]
args = sys.argv[outind:]
outfile = args[1]
infile = args[-1]
cmd = flags + ["-E","-"]
def escape(m):
return f'\\{m[0]}'
escape_re = re.compile(r'["\\]')
STRING = r'\s*"((\\.|[^"\\])*)"\s*'
NUM=r'\s*([0-9]+)'
linemarker = re.compile(r'\s*#' + NUM + STRING + NUM + ".*")
pipe = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
pipe.stdin.write(bytes(f'#include "{infile}"\n',"utf-8"))
pipe.stdin.flush()
pipe.stdin.close()
lines=[]
dep=None
while True:
l=pipe.stdout.readline()
if not l:
break
l = "".join(map(lambda b: chr(b), l))
l = l.rstrip()
if l[:1] == '#' and infile in l:
m = linemarker.match(l)
if m[4] == "1":
dep = m[2]
if l and l[0] != '#':
l = escape_re.sub(escape, l)
lines.append(f'"{l}"\n')
output = open(outfile, "wt")
output.writelines(lines)
output.close()

435
tools/qfcc/source/spirv_grammar.c Normal file
View file

@ -0,0 +1,435 @@
/*
spirv_grammar.c
SPIR-V grammar json embedding
Copyright (C) 2024 Bill Currie
Author: Bill Currie <bill@taniwha.org>
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to:
Free Software Foundation, Inc.
59 Temple Place - Suite 330
Boston, MA 02111-1307, USA
*/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#ifdef HAVE_STRING_H
# include <string.h>
#endif
#ifdef HAVE_STRINGS_H
# include <strings.h>
#endif
#include <stdlib.h>
#include "QF/heapsort.h"
#include "QF/plist.h"
#include "QF/sys.h"
#include "QF/va.h"
#include "tools/qfcc/include/qfcc.h"
#include "tools/qfcc/include/diagnostic.h"
#include "tools/qfcc/include/expr.h"
#include "tools/qfcc/include/options.h"
#include "tools/qfcc/include/shared.h"
#include "tools/qfcc/include/strpool.h"
#include "tools/qfcc/include/struct.h"
#include "tools/qfcc/include/symtab.h"
#include "tools/qfcc/include/spirv.h"
#include "tools/qfcc/include/spirv_grammar.h"
#include "tools/qfcc/include/type.h"
#include "tools/qfcc/include/value.h"
typedef struct spirv_json_s {
struct spirv_json_s *parent;
const char *name;
const char *json;
spirv_grammar_t *grammar;
} spirv_json_t;
static spirv_json_t builtin_json[] = {
{ .name = "core",
.json =
#include "tools/qfcc/source/spirv.core.grammar.jinc"
},
{ .parent = &builtin_json[0],
.name = "GLSL.std.450",
.json =
#include "tools/qfcc/source/extinst.glsl.std.450.grammar.jinc"
},
{}
};
static void *
spvg_alloc (void *data, size_t size)
{
return malloc (size);
}
typedef struct parse_string_s {
size_t value_offset;
} parse_string_t;
typedef struct parse_array_s {
pltype_t type;
size_t stride;
plparser_t parser;
void *data;
size_t value_offset;
size_t size_offset;
__compar_fn_t cmp;
} parse_array_t;
static int
parse_char (const plfield_t *field, const plitem_t *item,
void *data, plitem_t *messages, void *context)
{
auto value = (char *) data;
const char *str = PL_String (item);
*value = *str;
return 1;
}
static int
parse_hex (const plfield_t *field, const plitem_t *item,
void *data, plitem_t *messages, void *context)
{
auto value = (uint32_t *) data;
const char *str = PL_String (item);
char *end = 0;
*value = strtol (str, &end, 0);
if (*end) {
PL_Message (messages, item, "error parsing %s: %s", field->name, str);
}
return !*end;
}
static int
parse_uint32_t (const plfield_t *field, const plitem_t *item,
void *data, plitem_t *messages, void *context)
{
auto value = (uint32_t *) data;
*value = PL_Number (item);
return 1;
}
static int
parse_value (const plfield_t *field, const plitem_t *item,
void *data, plitem_t *messages, void *context)
{
if (PL_Type (item) == QFString) {
return parse_hex (field, item, data, messages, context);
} else {
return parse_uint32_t (field, item, data, messages, context);
}
}
static parse_string_t parse_string_array = { 0 };
static int
parse_string (const plfield_t *field, const plitem_t *item,
void *data, plitem_t *messages, void *context)
{
auto string = (parse_string_t *) field->data;
auto value = (const char **) ((byte *)data + string->value_offset);
const char *str = PL_String (item);
*value = save_string (str);
return 1;
}
static int
parse_array (const plfield_t *field, const plitem_t *item,
void *data, plitem_t *messages, void *context)
{
auto array = (parse_array_t *) field->data;
auto value = (void **) ((byte *)data + array->value_offset);
auto size = (uint32_t *) ((byte *)data + array->size_offset);
plelement_t element = {
array->type,
array->stride,
spvg_alloc,
array->parser,
array->data,
};
plfield_t f = { 0, 0, 0, 0, &element };
typedef struct arr_s DARRAY_TYPE(byte) arr_t;
arr_t *arr;
if (!PL_ParseArray (&f, item, &arr, messages, context)) {
return 0;
}
*value = arr->a;
if ((void *) size >= data) {
*size = arr->size;
}
if (array->cmp) {
heapsort (*value, *size, array->stride, array->cmp);
}
return 1;
}
static int
parse_struct (const plfield_t *field, const plitem_t *dict,
void *data, plitem_t *messages, void *context)
{
return PL_ParseStruct (field->data, dict, data, messages, context);
}
static int
parse_ignore (const plfield_t *field, const plitem_t *item,
void *data, plitem_t *messages, void *context)
{
return 1;
}
static int
spirv_enumerant_cmp (const void *_a, const void *_b)
{
auto a = (const spirv_enumerant_t *) _a;
auto b = (const spirv_enumerant_t *) _b;
return strcmp (a->enumerant, b->enumerant);
}
static int
spirv_instruction_cmp (const void *_a, const void *_b)
{
auto a = (const spirv_instruction_t *) _a;
auto b = (const spirv_instruction_t *) _b;
return strcmp (a->opname, b->opname);
}
static int
spirv_kind_cmp (const void *_a, const void *_b)
{
auto a = (const spirv_kind_t *) _a;
auto b = (const spirv_kind_t *) _b;
return strcmp (a->kind, b->kind);
}
static plfield_t spirv_operand_fields[] = {
{"kind", offsetof (spirv_operand_t, kind), QFString, parse_string, &parse_string_array},
{"quantifier", offsetof (spirv_operand_t, quantifier), QFString, parse_char, nullptr},
{"name", offsetof (spirv_operand_t, name), QFString, parse_string, &parse_string_array},
{ }
};
static parse_array_t parse_ecapability_data = {
.type = QFString,
.stride = sizeof (const char *),
.parser = parse_string,
.data = &parse_string_array,
.value_offset = offsetof (spirv_enumerant_t, capabilities),
.size_offset = offsetof (spirv_enumerant_t, num_capabilities),
};
static parse_array_t parse_eextension_data = {
.type = QFString,
.stride = sizeof (const char *),
.parser = parse_string,
.data = &parse_string_array,
.value_offset = offsetof (spirv_enumerant_t, extensions),
.size_offset = offsetof (spirv_enumerant_t, num_extensions),
};
static parse_array_t parse_parameter_data = {
.type = QFDictionary,
.stride = sizeof (spirv_operand_t),
.parser = parse_struct,
.data = &spirv_operand_fields,
.value_offset = offsetof (spirv_enumerant_t, parameters),
.size_offset = offsetof (spirv_enumerant_t, num_parameters),
};
static plfield_t spirv_enumerant_fields[] = {
{"enumerant", offsetof (spirv_enumerant_t, enumerant), QFString, parse_string, &parse_string_array},
{"value", offsetof (spirv_enumerant_t, value), QFMultiType | (1 << QFString) | (1 << QFNumber), parse_value, nullptr},
{"capabilities", 0, QFArray, parse_array, &parse_ecapability_data},
{"extensions", 0, QFArray, parse_array, &parse_eextension_data},
{"parameters", 0, QFArray, parse_array, &parse_parameter_data},
{"version", offsetof (spirv_enumerant_t, version), QFString, parse_string, &parse_string_array},
{"lastVersion", offsetof (spirv_enumerant_t, lastVersion), QFString, parse_string, &parse_string_array},
{ }
};
static parse_array_t parse_enumerant_data = {
.type = QFDictionary,
.stride = sizeof (spirv_enumerant_t),
.parser = parse_struct,
.data = &spirv_enumerant_fields,
.value_offset = offsetof (spirv_kind_t, enumerants),
.size_offset = offsetof (spirv_kind_t, num),
.cmp = spirv_enumerant_cmp,
};
static parse_array_t parse_base_data = {
.type = QFString,
.stride = sizeof (const char *),
.parser = parse_string,
.data = &parse_string_array,
.value_offset = offsetof (spirv_kind_t, bases),
.size_offset = offsetof (spirv_kind_t, num),
};
static plfield_t spirv_kind_fields[] = {
{"category", offsetof (spirv_kind_t, category), QFString, parse_string, &parse_string_array},
{"kind", offsetof (spirv_kind_t, kind), QFString, parse_string, &parse_string_array},
{"doc", offsetof (spirv_kind_t, doc), QFString, parse_string, &parse_string_array},
{"enumerants", 0, QFArray, parse_array, &parse_enumerant_data},
{"bases", 0, QFArray, parse_array, &parse_base_data},
};
static parse_array_t parse_operand_data = {
.type = QFDictionary,
.stride = sizeof (spirv_operand_t),
.parser = parse_struct,
.data = &spirv_operand_fields,
.value_offset = offsetof (spirv_instruction_t, operands),
.size_offset = offsetof (spirv_instruction_t, num_operands),
};
static parse_array_t parse_capability_data = {
.type = QFString,
.stride = sizeof (const char *),
.parser = parse_string,
.data = &parse_string_array,
.value_offset = offsetof (spirv_instruction_t, capabilities),
.size_offset = offsetof (spirv_instruction_t, num_capabilities),
};
static parse_array_t parse_extension_data = {
.type = QFString,
.stride = sizeof (const char *),
.parser = parse_string,
.data = &parse_string_array,
.value_offset = offsetof (spirv_instruction_t, extensions),
.size_offset = offsetof (spirv_instruction_t, num_extensions),
};
static plfield_t spirv_instruction_fields[] = {
{"opname", offsetof (spirv_instruction_t, opname), QFString, parse_string, &parse_string_array},
{"opcode", offsetof (spirv_instruction_t, opcode), QFNumber, parse_uint32_t, nullptr},
{"operands", 0, QFArray, parse_array, &parse_operand_data},
{"capabilities", 0, QFArray, parse_array, &parse_capability_data},
{"extensions", 0, QFArray, parse_array, &parse_extension_data},
{"version", offsetof (spirv_instruction_t, version), QFString, parse_string, &parse_string_array},
{"lastVersion", offsetof (spirv_instruction_t, lastVersion), QFString, parse_string, &parse_string_array},
{"class", 0, QFString, parse_ignore, nullptr},
{ }
};
static parse_array_t parse_copyright_data = {
.type = QFString,
.stride = sizeof (const char *),
.parser = parse_string,
.data = &parse_string_array,
.value_offset = offsetof (spirv_grammar_t, copyright),
.size_offset = offsetof (spirv_grammar_t, num_copyright),
};
static parse_array_t parse_instruction_data = {
.type = QFDictionary,
.stride = sizeof (spirv_instruction_t),
.parser = parse_struct,
.data = spirv_instruction_fields,
.value_offset = offsetof (spirv_grammar_t, instructions),
.size_offset = offsetof (spirv_grammar_t, num_instructions),
.cmp = spirv_instruction_cmp,
};
static parse_array_t parse_operand_kind_data = {
.type = QFDictionary,
.stride = sizeof (spirv_kind_t),
.parser = parse_struct,
.data = spirv_kind_fields,
.value_offset = offsetof (spirv_grammar_t, operand_kinds),
.size_offset = offsetof (spirv_grammar_t, num_operand_kinds),
.cmp = spirv_kind_cmp,
};
static bool built;
static plfield_t spirv_grammar_fields[] = {
{"copyright", 0, QFArray, parse_array, &parse_copyright_data},
{"magic_number", offsetof (spirv_grammar_t, magic_number), QFString, parse_hex, nullptr},
{"major_version", offsetof (spirv_grammar_t, major_version), QFNumber, parse_uint32_t, nullptr},
{"minor_version", offsetof (spirv_grammar_t, minor_version), QFNumber, parse_uint32_t, nullptr},
{"version", offsetof (spirv_grammar_t, version), QFNumber, parse_uint32_t, nullptr},
{"revision", offsetof (spirv_grammar_t, revision), QFNumber, parse_uint32_t, nullptr},
{"instructions", 0, QFArray, parse_array, &parse_instruction_data},
{"operand_kinds", 0, QFArray, parse_array, &parse_operand_kind_data},
{"instruction_printing_class", 0, QFArray, parse_ignore, nullptr},
{ }
};
static bool
parse_grammar (plitem_t *plitem, spirv_grammar_t **grammar)
{
spirv_grammar_t g = {};
auto messages = PL_NewArray ();
bool ret = PL_ParseStruct (spirv_grammar_fields, plitem, &g, messages,
nullptr);
if (!ret) {
for (int i = 0; i < PL_A_NumObjects (messages); i++) {
fprintf (stderr, "%s\n",
PL_String (PL_ObjectAtIndex (messages, i)));
}
} else {
*grammar = malloc (sizeof (**grammar));
**grammar = g;
}
PL_Release (messages);
return ret;
}
static void
build_grammars (void)
{
for (int i = 0; builtin_json[i].name; i++) {
auto plitem = PL_ParseJSON (builtin_json[i].json, nullptr);
if (!plitem) {
internal_error (0, "could not parse JSON for %s",
builtin_json[i].name);
}
if (!parse_grammar (plitem, &builtin_json[i].grammar)) {
internal_error (0, "could not parse grammar spec for %s",
builtin_json[i].name);
}
}
for (int i = 0; builtin_json[i].name; i++) {
auto parent = builtin_json[i].parent;
if (parent) {
builtin_json[i].grammar->parent = parent->grammar;
}
}
}
const plitem_t *
spirv_operand_kind (const char *set, const char *kind)
{
if (!built) {
build_grammars ();
built = true;
}
return nullptr;
}