darkplaces/gmqcc
0
0
Fork 0
mirror of https://github.com/DarkPlacesEngine/gmqcc.git synced 2025-03-23 19:21:29 +00:00
Code Issues Projects Releases Packages Wiki Activity

-futf8 implementation

Browse source
This commit is contained in:
Wolfgang Bumiller 2012-12-22 22:34:57 +01:00
parent aec2284f45
commit a707440e52
4 changed files with 58 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
gmqcc.h
View file

@ -934,6 +934,7 @@ typedef uint32_t longbit;
/*===================================================================*/
typedef uint32_t Uchar;
bool u8_analyze (const char *_s, size_t *_start, size_t *_len, Uchar *_ch, size_t _maxlen);
size_t u8_strlen (const char*);
size_t u8_strnlen (const char*, size_t);
Uchar u8_getchar (const char*, const char**);

60
lexer.c
View file

@ -742,6 +742,9 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote)
{
int ch = 0;
int nextch;
bool hex;
char u8buf[8]; /* way more than enough */
int u8len, uc;
while (ch != EOF)
{
@ -823,13 +826,51 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote)
case ']': ch = 17; break;
case '{':
ch = 0;
nextch = lex_getch(lex);
hex = (nextch == 'x');
if (!hex)
lex_ungetch(lex, nextch);
for (nextch = lex_getch(lex); nextch != '}'; nextch = lex_getch(lex)) {
ch = ch * 10 + nextch - '0';
if (nextch < '0' || nextch > '9' || ch > 255) {
lexerror(lex, "bad character code");
if (!hex) {
if (nextch >= '0' && nextch <= '9')
ch = ch * 10 + nextch - '0';
else {
lexerror(lex, "bad character code");
return (lex->tok.ttype = TOKEN_ERROR);
}
} else {
if (nextch >= '0' || nextch <= '9')
ch = ch * 16 + nextch - '0';
else if (nextch >= 'a' && nextch <= 'f')
ch = ch * 16 + nextch - 'a' + 10;
else if (nextch >= 'A' && nextch <= 'F')
ch = ch * 16 + nextch - 'A' + 10;
else {
lexerror(lex, "bad character code");
return (lex->tok.ttype = TOKEN_ERROR);
}
}
if ( (!OPTS_FLAG(UTF8) && ch > 255) ||
( OPTS_FLAG(UTF8) && ch > 0x10FFFF) )
{
lexerror(lex, "character code out of range");
return (lex->tok.ttype = TOKEN_ERROR);
}
}
if (OPTS_FLAG(UTF8) && ch >= 128) {
u8len = u8_fromchar((Uchar)ch, u8buf, sizeof(u8buf));
if (!u8len)
ch = 0;
else {
--u8len;
for (uc = 0; uc < u8len; ++uc)
lex_tokench(lex, u8buf[uc]);
/* the last character will be inserted with the tokench() call
* below the switch
*/
ch = u8buf[uc];
}
}
break;
case '\n': ch = '\n'; break;
@ -1386,10 +1427,17 @@ int lex_do(lex_file *lex)
else
{
if (!lex->flags.preprocessing && strlen(lex->tok.value) > 1) {
if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER, "multibyte character: `%s`", lex->tok.value))
return (lex->tok.ttype = TOKEN_ERROR);
Uchar u8char;
/* check for a valid utf8 character */
if (!OPTS_FLAG(UTF8) || !u8_analyze(lex->tok.value, NULL, NULL, &u8char, 8)) {
if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER, "multibyte character: `%s`", lex->tok.value))
return (lex->tok.ttype = TOKEN_ERROR);
}
else
lex->tok.constval.i = u8char;
}
lex->tok.constval.i = lex->tok.value[0];
else
lex->tok.constval.i = lex->tok.value[0];
}
return lex->tok.ttype;

1
opts.def
View file

@ -43,6 +43,7 @@
GMQCC_DEFINE_FLAG(CORRECT_LOGIC)
GMQCC_DEFINE_FLAG(TRUE_EMPTY_STRINGS)
GMQCC_DEFINE_FLAG(FALSE_EMPTY_STRINGS)
GMQCC_DEFINE_FLAG(UTF8)
#endif
/* warning flags */

4
utf8lib.c
View file

@ -35,12 +35,12 @@ static Uchar utf8_range[5] = {
/** Analyze the next character and return various information if requested.
* @param _s An utf-8 string.
* @param _start Filled with the start byte-offset of the next valid character
* @param _len Fileed with the length of the next valid character
* @param _len Filled with the length of the next valid character
* @param _ch Filled with the unicode value of the next character
* @param _maxlen Maximum number of bytes to read from _s
* @return Whether or not another valid character is in the string
*/
static bool u8_analyze(const char *_s, size_t *_start, size_t *_len, Uchar *_ch, size_t _maxlen)
bool u8_analyze(const char *_s, size_t *_start, size_t *_len, Uchar *_ch, size_t _maxlen)
{
const unsigned char *s = (const unsigned char*)_s;
size_t i, j;