darkplaces/gmqcc
0
0
Fork 0
mirror of https://github.com/DarkPlacesEngine/gmqcc.git synced 2024-12-18 00:11:06 +00:00
Code Issues Projects Releases Packages Wiki Activity

Major utility rewrite for compiler memory utilization statistics. Cleanups everywhere, no more NOTRACK stuff, all allocates are tracked. Major identifier cleanups as well.

Browse source
This commit is contained in:
Dale Weiler 2013-06-02 08:21:06 +00:00
parent ec7bf4767a
commit 9af3c502da
7 changed files with 636 additions and 634 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
Makefile
View file

@ -44,11 +44,11 @@ ifeq ($(track), no)
CFLAGS += -DNOTRACK
endif
OBJ_D = util.o code.o ast.o ir.o conout.o ftepp.o opts.o fs.o utf8.o correct.o
OBJ_P = util.o fs.o conout.o opts.o pak.o
OBJ_T = test.o util.o conout.o fs.o
OBJ_C = main.o lexer.o parser.o fs.o
OBJ_X = exec-standalone.o util.o conout.o fs.o
OBJ_D = util.o code.o ast.o ir.o conout.o ftepp.o opts.o fs.o utf8.o correct.o stat.o
OBJ_P = util.o fs.o conout.o opts.o pak.o stat.o
OBJ_T = test.o util.o conout.o fs.o stat.o
OBJ_C = main.o lexer.o parser.o fs.o stat.o
OBJ_X = exec-standalone.o util.o conout.o fs.o stat.o
#we have duplicate object files when dealing with creating a simple list
#for dependinces. To combat this we use some clever recrusive-make to
@ -244,6 +244,7 @@ opts.o: gmqcc.h opts.def
fs.o: gmqcc.h opts.def
utf8.o: gmqcc.h opts.def
correct.o: gmqcc.h opts.def
stat.o: gmqcc.h opts.def
pak.o: gmqcc.h opts.def
test.o: gmqcc.h opts.def
main.o: gmqcc.h opts.def lexer.h

57
gmqcc.h
View file

@ -288,20 +288,47 @@ GMQCC_IND_STRING(GMQCC_VERSION_PATCH) \
# include <dirent.h>
#endif /*! _WIN32 && !defined(__MINGW32__) */
/*===================================================================*/
/*=========================== stat.c ================================*/
/*===================================================================*/
typedef struct {
size_t key;
size_t value;
} stat_size_entry_t, **stat_size_table_t;
void stat_info();
char *stat_mem_strdup (const char *, size_t, const char *, bool);
void *stat_mem_reallocate(void *, size_t, size_t, const char *);
void stat_mem_deallocate(void *);
void *stat_mem_allocate (size_t, size_t, const char *);
stat_size_table_t stat_size_new();
stat_size_entry_t *stat_size_get(stat_size_table_t, size_t);
void stat_size_del(stat_size_table_t);
void stat_size_put(stat_size_table_t, size_t, size_t);
/* getters for hashtable: */
stat_size_table_t *stat_size_hashtables_get();
uint64_t *stat_type_hashtables_get();
uint64_t *stat_used_hashtables_get();
stat_size_table_t *stat_hashtables_init();
#define mem_a(SIZE) stat_mem_allocate ((SIZE), __LINE__, __FILE__)
#define mem_d(PTRN) stat_mem_deallocate((void*)(PTRN))
#define mem_r(PTRN, SIZE) stat_mem_reallocate((void*)(PTRN), (SIZE), __LINE__, __FILE__)
#define mem_af(SIZE, FILE, LINE) stat_mem_allocate ((SIZE), (LINE), (FILE))
/* TODO: rename to mem variations */
#define util_strdup(SRC) stat_mem_strdup((char*)(SRC), __LINE__, __FILE__, false)
#define util_strdupe(SRC) stat_mem_strdup((char*)(SRC), __LINE__, __FILE__, true)
/*===================================================================*/
/*=========================== util.c ================================*/
/*===================================================================*/
void *util_memory_a (size_t, /*****/ unsigned int, const char *);
void *util_memory_r (void *, size_t, unsigned int, const char *);
void util_memory_d (void *);
void util_meminfo ();
bool util_filexists (const char *);
bool util_strupper (const char *);
bool util_strdigit (const char *);
char *_util_Estrdup (const char *, const char *, size_t);
char *_util_Estrdup_empty(const char *, const char *, size_t);
void util_debug (const char *, const char *, ...);
void util_endianswap (void *, size_t, unsigned int);
@ -326,22 +353,6 @@ char *util_strcat (char *dest, const char *src);
char *util_strncpy (char *dest, const char *src, size_t num);
const char *util_strerror (int num);
#ifdef NOTRACK
# define mem_a(x) malloc (x)
# define mem_d(x) free ((void*)x)
# define mem_r(x, n) realloc((void*)x, n)
# define mem_af(x,f,l) malloc (x)
#else
# define mem_a(x) util_memory_a((x), __LINE__, __FILE__)
# define mem_d(x) util_memory_d((void*)(x))
# define mem_r(x, n) util_memory_r((void*)(x), (n), __LINE__, __FILE__)
# define mem_af(x,f,l) util_memory_a((x), __LINE__, __FILE__)
#endif /*! NOTRACK */
#define util_strdup(X) _util_Estrdup((X), __FILE__, __LINE__)
#define util_strdupe(X) _util_Estrdup_empty((X), __FILE__, __LINE__)
/*
* A flexible vector implementation: all vector pointers contain some
* data about themselfs exactly - sizeof(vector_t) behind the pointer

3
main.c
View file

@ -796,8 +796,7 @@ cleanup:
mem_d((void*)operators);
lex_cleanup();
util_meminfo();
/*util_vecstats_destroy();*/
stat_info();
return retval;
}

5
pak.c
View file

@ -552,7 +552,8 @@ int main(int argc, char **argv) {
/* not possible */
pak_close(pak);
vec_free(files);
util_meminfo();
stat_info();
return EXIT_SUCCESS;
}
@ -575,6 +576,6 @@ int main(int argc, char **argv) {
pak_close(pak);
vec_free(files);
util_meminfo();
stat_info();
return EXIT_SUCCESS;
}

590
stat.c Normal file
View file

@ -0,0 +1,590 @@
#include "gmqcc.h"
/*
* GMQCC performs tons of allocations, constructions, and crazyness
* all around. When trying to optimizes systems, or just get fancy
* statistics out of the compiler, it's often printf mess. This file
* implements the statistics system of the compiler. I.E the allocator
* we use to track allocations, and other systems of interest.
*/
#define ST_SIZE 1024
typedef struct stat_mem_block_s {
const char *file;
size_t line;
size_t size;
struct stat_mem_block_s *next;
struct stat_mem_block_s *prev;
} stat_mem_block_t;
static uint64_t stat_mem_allocated = 0;
static uint64_t stat_mem_deallocated = 0;
static uint64_t stat_mem_allocated_total = 0;
static uint64_t stat_mem_deallocated_total = 0;
static uint64_t stat_mem_high = 0;
static uint64_t stat_mem_peak = 0;
static uint64_t stat_used_strdups = 0;
static uint64_t stat_used_vectors = 0;
static uint64_t stat_used_hashtables = 0;
static uint64_t stat_type_vectors = 0;
static uint64_t stat_type_hashtables = 0;
static stat_size_table_t stat_size_vectors = NULL;
static stat_size_table_t stat_size_hashtables = NULL;
static stat_mem_block_t *stat_mem_block_root = NULL;
/*
* A basic header of information wrapper allocator. Simply stores
* information as a header, returns the memory + 1 past it, can be
* retrieved again with - 1. Where type is stat_mem_block_t*.
*/
void *stat_mem_allocate(size_t size, size_t line, const char *file) {
stat_mem_block_t *info = (stat_mem_block_t*)malloc(sizeof(stat_mem_block_t) + size);
void *data = (void*)(info + 1);
if(!info)
return NULL;
info->line = line;
info->size = size;
info->file = file;
info->prev = NULL;
info->next = stat_mem_block_root;
if (stat_mem_block_root)
stat_mem_block_root->prev = info;
stat_mem_block_root = info;
stat_mem_allocated += size;
stat_mem_high += size;
stat_mem_allocated_total ++;
if (stat_mem_high > stat_mem_peak)
stat_mem_peak = stat_mem_high;
return data;
}
void stat_mem_deallocate(void *ptr) {
stat_mem_block_t *info = NULL;
if (!ptr)
return;
info = ((stat_mem_block_t*)ptr - 1);
stat_mem_deallocated += info->size;
stat_mem_high -= info->size;
stat_mem_deallocated_total ++;
if (info->prev) info->prev->next = info->next;
if (info->next) info->next->prev = info->prev;
/* move ahead */
if (info == stat_mem_block_root)
stat_mem_block_root = info->next;
}
void *stat_mem_reallocate(void *ptr, size_t size, size_t line, const char *file) {
stat_mem_block_t *oldinfo = NULL;
stat_mem_block_t *newinfo;
if (!ptr)
return stat_mem_allocate(size, line, file);
/* stay consistent with glic */
if (!size) {
stat_mem_deallocate(ptr);
return NULL;
}
oldinfo = ((stat_mem_block_t*)ptr - 1);
newinfo = ((stat_mem_block_t*)malloc(sizeof(stat_mem_block_t) + size));
if (!newinfo) {
stat_mem_deallocate(ptr);
return NULL;
}
memcpy(newinfo+1, oldinfo+1, oldinfo->size);
if (oldinfo->prev) oldinfo->prev->next = oldinfo->next;
if (oldinfo->next) oldinfo->next->prev = oldinfo->prev;
/* move ahead */
if (oldinfo == stat_mem_block_root)
stat_mem_block_root = oldinfo->next;
newinfo->line = line;
newinfo->size = size;
newinfo->file = file;
newinfo->prev = NULL;
newinfo->next = stat_mem_block_root;
if (stat_mem_block_root)
stat_mem_block_root->prev = newinfo;
stat_mem_block_root = newinfo;
stat_mem_allocated -= oldinfo->size;
stat_mem_high -= oldinfo->size;
stat_mem_allocated += newinfo->size;
stat_mem_high += newinfo->size;
if (stat_mem_high > stat_mem_peak)
stat_mem_peak = stat_mem_high;
free(oldinfo);
return newinfo + 1;
}
/*
* strdup does it's own malloc, we need to track malloc. We don't want
* to overwrite malloc though, infact, we can't really hook it at all
* without library specific assumptions. So we re implement strdup.
*/
char *stat_mem_strdup(const char *src, size_t line, const char *file, bool empty) {
size_t len = 0;
char *ptr = NULL;
if (!src)
return NULL;
len = strlen(src);
if (((!empty) ? len : true) && (ptr = (char*)stat_mem_allocate(len + 1, line, file))) {
memcpy(ptr, src, len);
ptr[len] = '\0';
}
stat_used_strdups ++;
return ptr;
}
/*
* The reallocate function for resizing vectors.
*/
void _util_vec_grow(void **a, size_t i, size_t s) {
vector_t *d = vec_meta(*a);
size_t m = 0;
stat_size_entry_t *e = NULL;
void *p = NULL;
if (*a) {
m = 2 * d->allocated + i;
p = mem_r(d, s * m + sizeof(vector_t));
} else {
m = i + 1;
p = mem_a(s * m + sizeof(vector_t));
((vector_t*)p)->used = 0;
stat_used_vectors++;
}
if (!stat_size_vectors)
stat_size_vectors = stat_size_new();
if ((e = stat_size_get(stat_size_vectors, s))) {
e->value ++;
} else {
stat_size_put(stat_size_vectors, s, 1); /* start off with 1 */
stat_type_vectors++;
}
*a = (vector_t*)p + 1;
vec_meta(*a)->allocated = m;
}
/*
* Hash table for generic data, based on dynamic memory allocations
* all around. This is the internal interface, please look for
* EXPOSED INTERFACE comment below
*/
typedef struct hash_node_t {
char *key; /* the key for this node in table */
void *value; /* pointer to the data as void* */
struct hash_node_t *next; /* next node (linked list) */
} hash_node_t;
GMQCC_INLINE size_t util_hthash(hash_table_t *ht, const char *key) {
const uint32_t mix = 0x5BD1E995;
const uint32_t rot = 24;
size_t size = strlen(key);
uint32_t hash = 0x1EF0 /* LICRC TAB */ ^ size;
uint32_t alias = 0;
const unsigned char *data = (const unsigned char*)key;
while (size >= 4) {
alias = (data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24));
alias *= mix;
alias ^= alias >> rot;
alias *= mix;
hash *= mix;
hash ^= alias;
data += 4;
size -= 4;
}
switch (size) {
case 3: hash ^= data[2] << 16;
case 2: hash ^= data[1] << 8;
case 1: hash ^= data[0];
hash *= mix;
}
hash ^= hash >> 13;
hash *= mix;
hash ^= hash >> 15;
return (size_t) (hash % ht->size);
}
static hash_node_t *_util_htnewpair(const char *key, void *value) {
hash_node_t *node;
if (!(node = (hash_node_t*)mem_a(sizeof(hash_node_t))))
return NULL;
if (!(node->key = util_strdupe(key))) {
mem_d(node);
return NULL;
}
node->value = value;
node->next = NULL;
return node;
}
/*
* EXPOSED INTERFACE for the hashtable implementation
* util_htnew(size) -- to make a new hashtable
* util_htset(table, key, value, sizeof(value)) -- to set something in the table
* util_htget(table, key) -- to get something from the table
* util_htdel(table) -- to delete the table
*/
hash_table_t *util_htnew(size_t size) {
hash_table_t *hashtable = NULL;
stat_size_entry_t *find = NULL;
if (size < 1)
return NULL;
if (!stat_size_hashtables)
stat_size_hashtables = stat_size_new();
if (!(hashtable = (hash_table_t*)mem_a(sizeof(hash_table_t))))
return NULL;
if (!(hashtable->table = (hash_node_t**)mem_a(sizeof(hash_node_t*) * size))) {
mem_d(hashtable);
return NULL;
}
if ((find = stat_size_get(stat_size_hashtables, size)))
find->value++;
else {
stat_used_hashtables++;
stat_size_put(stat_size_hashtables, size, 1);
}
hashtable->size = size;
memset(hashtable->table, 0, sizeof(hash_node_t*) * size);
stat_type_hashtables++;
return hashtable;
}
void util_htseth(hash_table_t *ht, const char *key, size_t bin, void *value) {
hash_node_t *newnode = NULL;
hash_node_t *next = NULL;
hash_node_t *last = NULL;
next = ht->table[bin];
while (next && next->key && strcmp(key, next->key) > 0)
last = next, next = next->next;
/* already in table, do a replace */
if (next && next->key && strcmp(key, next->key) == 0) {
next->value = value;
} else {
/* not found, grow a pair man :P */
newnode = _util_htnewpair(key, value);
if (next == ht->table[bin]) {
newnode->next = next;
ht->table[bin] = newnode;
} else if (!next) {
last->next = newnode;
} else {
newnode->next = next;
last->next = newnode;
}
}
}
void util_htset(hash_table_t *ht, const char *key, void *value) {
util_htseth(ht, key, util_hthash(ht, key), value);
}
void *util_htgeth(hash_table_t *ht, const char *key, size_t bin) {
hash_node_t *pair = ht->table[bin];
while (pair && pair->key && strcmp(key, pair->key) > 0)
pair = pair->next;
if (!pair || !pair->key || strcmp(key, pair->key) != 0)
return NULL;
return pair->value;
}
void *util_htget(hash_table_t *ht, const char *key) {
return util_htgeth(ht, key, util_hthash(ht, key));
}
void *code_util_str_htgeth(hash_table_t *ht, const char *key, size_t bin) {
hash_node_t *pair;
size_t len, keylen;
int cmp;
keylen = strlen(key);
pair = ht->table[bin];
while (pair && pair->key) {
len = strlen(pair->key);
if (len < keylen) {
pair = pair->next;
continue;
}
if (keylen == len) {
cmp = strcmp(key, pair->key);
if (cmp == 0)
return pair->value;
if (cmp < 0)
return NULL;
pair = pair->next;
continue;
}
cmp = strcmp(key, pair->key + len - keylen);
if (cmp == 0) {
uintptr_t up = (uintptr_t)pair->value;
up += len - keylen;
return (void*)up;
}
pair = pair->next;
}
return NULL;
}
/*
* Free all allocated data in a hashtable, this is quite the amount
* of work.
*/
void util_htrem(hash_table_t *ht, void (*callback)(void *data)) {
size_t i = 0;
for (; i < ht->size; i++) {
hash_node_t *n = ht->table[i];
hash_node_t *p;
/* free in list */
while (n) {
if (n->key)
mem_d(n->key);
if (callback)
callback(n->value);
p = n;
n = n->next;
mem_d(p);
}
}
/* free table */
mem_d(ht->table);
mem_d(ht);
}
void util_htrmh(hash_table_t *ht, const char *key, size_t bin, void (*cb)(void*)) {
hash_node_t **pair = &ht->table[bin];
hash_node_t *tmp;
while (*pair && (*pair)->key && strcmp(key, (*pair)->key) > 0)
pair = &(*pair)->next;
tmp = *pair;
if (!tmp || !tmp->key || strcmp(key, tmp->key) != 0)
return;
if (cb)
(*cb)(tmp->value);
*pair = tmp->next;
mem_d(tmp->key);
mem_d(tmp);
}
void util_htrm(hash_table_t *ht, const char *key, void (*cb)(void*)) {
util_htrmh(ht, key, util_hthash(ht, key), cb);
}
void util_htdel(hash_table_t *ht) {
util_htrem(ht, NULL);
}
/*
* A tiny size_t key-value hashtbale for tracking vector and hashtable
* sizes. We can use it for other things too, if we need to. This is
* very TIGHT, and efficent in terms of space though.
*/
stat_size_table_t stat_size_new() {
return (stat_size_table_t)memset(
mem_a(sizeof(stat_size_entry_t*) * ST_SIZE),
0, ST_SIZE * sizeof(stat_size_entry_t*)
);
}
void stat_size_del(stat_size_table_t table) {
size_t i = 0;
for (; i < ST_SIZE; i++) if(table[i]) mem_d(table[i]);
mem_d(table);
}
stat_size_entry_t *stat_size_get(stat_size_table_t table, size_t key) {
size_t hash = (key % ST_SIZE);
while (table[hash] && table[hash]->key != key)
hash = (hash + 1) % ST_SIZE;
return table[hash];
}
void stat_size_put(stat_size_table_t table, size_t key, size_t value) {
size_t hash = (key % ST_SIZE);
while (table[hash] && table[hash]->key != key)
hash = (hash + 1) % ST_SIZE;
table[hash] = (stat_size_entry_t*)mem_a(sizeof(stat_size_entry_t));
table[hash]->key = key;
table[hash]->value = value;
}
/*
* The following functions below implement printing / dumping of statistical
* information.
*/
static void stat_dump_mem_contents(stat_mem_block_t *memory, uint16_t cols) {
uint32_t i, j;
for (i = 0; i < memory->size + ((memory->size % cols) ? (cols - memory->size % cols) : 0); i++) {
if (i % cols == 0) con_out(" 0x%06X: ", i);
if (i < memory->size) con_out("%02X " , 0xFF & ((unsigned char*)(memory + 1))[i]);
else con_out(" ");
if ((uint16_t)(i % cols) == (cols - 1)) {
for (j = i - (cols - 1); j <= i; j++) {
con_out("%c",
(j >= memory->size)
? ' '
: (isprint(((unsigned char*)(memory + 1))[j]))
? 0xFF & ((unsigned char*)(memory + 1)) [j]
: '.'
);
}
con_out("\n");
}
}
}
static void stat_dump_mem_leaks() {
stat_mem_block_t *info;
for (info = stat_mem_block_root; info; info = info->next) {
con_out("lost: %u (bytes) at %s:%u\n",
info->size,
info->file,
info->line
);
stat_dump_mem_contents(info, OPTS_OPTION_U16(OPTION_MEMDUMPCOLS));
}
}
static void stat_dump_mem_info() {
con_out("Memory information:\n\
Total allocations: %llu\n\
Total deallocations: %llu\n\
Total allocated: %f (MB)\n\
Total deallocated: %f (MB)\n\
Total peak memory: %f (MB)\n\
Total leaked memory: %f (MB) in %llu allocations\n",
stat_mem_allocated_total,
stat_mem_deallocated_total,
(float)(stat_mem_allocated) / 1048576.0f,
(float)(stat_mem_deallocated) / 1048576.0f,
(float)(stat_mem_high) / 1048576.0f,
(float)(stat_mem_allocated - stat_mem_deallocated) / 1048576.0f,
stat_mem_allocated_total - stat_mem_deallocated_total
);
}
static void stat_dump_stats_table(stat_size_table_t table, const char *string, uint64_t *size) {
size_t i,j;
for (i = 0, j = 0; i < ST_SIZE; i++) {
stat_size_entry_t *entry;
if (!(entry = table[i]))
continue;
con_out(string, (unsigned)j, (unsigned)entry->key, (unsigned)entry->value);
j++;
if (size)
*size += entry->key * entry->value;
}
}
void stat_info() {
if (OPTS_OPTION_BOOL(OPTION_DEBUG))
stat_dump_mem_leaks();
if (OPTS_OPTION_BOOL(OPTION_DEBUG) ||
OPTS_OPTION_BOOL(OPTION_MEMCHK))
stat_dump_mem_info();
if (OPTS_OPTION_BOOL(OPTION_MEMCHK) ||
OPTS_OPTION_BOOL(OPTION_STATISTICS)) {
uint64_t mem;
con_out("\nAdditional Statistics:\n\
Total vectors allocated: %llu\n\
Total string duplicates: %llu\n\
Total hashtables allocated: %llu\n\
Total unique vector sizes: %llu\n",
stat_used_vectors,
stat_used_strdups,
stat_used_hashtables,
stat_type_vectors
);
stat_dump_stats_table (
stat_size_vectors,
" %2u| # of %4u byte vectors: %u\n",
&mem
);
con_out (
" Total unique hashtable sizes: %llu\n",
stat_type_hashtables
);
stat_dump_stats_table (
stat_size_hashtables,
" %2u| # of %4u element hashtables: %u\n",
NULL
);
con_out (
" Total vector memory: %f (MB)\n",
(float)(mem) / 1048576.0f
);
}
if (stat_size_vectors)
stat_size_del(stat_size_vectors);
if (stat_size_hashtables)
stat_size_del(stat_size_hashtables);
}
#undef ST_SIZE

2
test.c
View file

@ -1321,7 +1321,7 @@ int main(int argc, char **argv) {
}
con_change(redirout, redirerr);
succeed = test_perform("tests", defs);
util_meminfo();
stat_info();
return (succeed) ? EXIT_SUCCESS : EXIT_FAILURE;

600
util.c
View file

@ -25,338 +25,6 @@
#include <errno.h>
#include "gmqcc.h"
/* TODO: remove globals ... */
static uint64_t mem_ab = 0;
static uint64_t mem_db = 0;
static uint64_t mem_at = 0;
static uint64_t mem_dt = 0;
static uint64_t mem_pk = 0;
static uint64_t mem_hw = 0;
struct memblock_t {
const char *file;
unsigned int line;
size_t byte;
struct memblock_t *next;
struct memblock_t *prev;
};
#define PEAK_MEM \
do { \
if (mem_hw > mem_pk) \
mem_pk = mem_hw; \
} while (0)
static struct memblock_t *mem_start = NULL;
void *util_memory_a(size_t byte, unsigned int line, const char *file) {
struct memblock_t *info = (struct memblock_t*)malloc(sizeof(struct memblock_t) + byte);
void *data = (void*)(info+1);
if (!info) return NULL;
info->line = line;
info->byte = byte;
info->file = file;
info->prev = NULL;
info->next = mem_start;
if (mem_start)
mem_start->prev = info;
mem_start = info;
mem_at++;
mem_ab += info->byte;
mem_hw += info->byte;
PEAK_MEM;
return data;
}
void util_memory_d(void *ptrn) {
struct memblock_t *info = NULL;
if (!ptrn) return;
info = ((struct memblock_t*)ptrn - 1);
mem_db += info->byte;
mem_hw -= info->byte;
mem_dt++;
if (info->prev)
info->prev->next = info->next;
if (info->next)
info->next->prev = info->prev;
if (info == mem_start)
mem_start = info->next;
free(info);
}
void *util_memory_r(void *ptrn, size_t byte, unsigned int line, const char *file) {
struct memblock_t *oldinfo = NULL;
struct memblock_t *newinfo;
if (!ptrn)
return util_memory_a(byte, line, file);
if (!byte) {
util_memory_d(ptrn);
return NULL;
}
oldinfo = ((struct memblock_t*)ptrn - 1);
newinfo = ((struct memblock_t*)malloc(sizeof(struct memblock_t) + byte));
/* new data */
if (!newinfo) {
util_memory_d(oldinfo+1);
return NULL;
}
/* copy old */
memcpy(newinfo+1, oldinfo+1, oldinfo->byte);
/* free old */
if (oldinfo->prev)
oldinfo->prev->next = oldinfo->next;
if (oldinfo->next)
oldinfo->next->prev = oldinfo->prev;
if (oldinfo == mem_start)
mem_start = oldinfo->next;
/* fill info */
newinfo->line = line;
newinfo->byte = byte;
newinfo->file = file;
newinfo->prev = NULL;
newinfo->next = mem_start;
if (mem_start)
mem_start->prev = newinfo;
mem_start = newinfo;
mem_ab -= oldinfo->byte;
mem_hw -= oldinfo->byte;
mem_ab += newinfo->byte;
mem_hw += newinfo->byte;
PEAK_MEM;
free(oldinfo);
return newinfo+1;
}
static void util_dumpmem(struct memblock_t *memory, uint16_t cols) {
uint32_t i, j;
for (i = 0; i < memory->byte + ((memory->byte % cols) ? (cols - memory->byte % cols) : 0); i++) {
if (i % cols == 0) con_out(" 0x%06X: ", i);
if (i < memory->byte) con_out("%02X " , 0xFF & ((char*)(memory + 1))[i]);
else con_out(" ");
if ((uint16_t)(i % cols) == (cols - 1)) {
for (j = i - (cols - 1); j <= i; j++) {
con_out("%c",
(j >= memory->byte)
? ' '
: (isprint(((char*)(memory + 1))[j]))
? 0xFF & ((char*)(memory + 1)) [j]
: '.'
);
}
con_out("\n");
}
}
}
/*
* The following is a VERY tight, efficent, hashtable for integer
* values and keys, and for nothing more. We could make our existing
* hashtable support type-genericness through a void * pointer but,
* ideally that would make things more complicated. We also don't need
* that much of a bloat for something as basic as this.
*/
typedef struct {
size_t key;
size_t value;
} size_entry_t;
#define ST_SIZE 1024
typedef size_entry_t **size_table_t;
static size_table_t util_st_new() {
return (size_table_t)memset(
mem_a(sizeof(size_entry_t*) * ST_SIZE),
0, ST_SIZE * sizeof(size_entry_t*)
);
}
static void util_st_del(size_table_t table) {
size_t i = 0;
for (; i < ST_SIZE; i++) if(table[i]) mem_d(table[i]);
mem_d(table);
}
static size_entry_t *util_st_get(size_table_t table, size_t key) {
size_t hash = (key % ST_SIZE);
while (table[hash] && table[hash]->key != key)
hash = (hash + 1) % ST_SIZE;
return table[hash];
}
static void util_st_put(size_table_t table, size_t key, size_t value) {
size_t hash = (key % ST_SIZE);
while (table[hash] && table[hash]->key != key)
hash = (hash + 1) % ST_SIZE;
table[hash] = (size_entry_t*)mem_a(sizeof(size_entry_t));
table[hash]->key = key;
table[hash]->value = value;
}
static uint64_t strdups = 0;
static uint64_t vectors = 0;
static uint64_t vector_sizes = 0;
static uint64_t hashtables = 0;
static uint64_t hashtable_sizes = 0;
static size_table_t vector_usage = NULL;
static size_table_t hashtable_usage = NULL;
void util_meminfo() {
struct memblock_t *info;
if (OPTS_OPTION_BOOL(OPTION_DEBUG)) {
for (info = mem_start; info; info = info->next) {
con_out("lost: %u (bytes) at %s:%u\n",
info->byte,
info->file,
info->line);
util_dumpmem(info, OPTS_OPTION_U16(OPTION_MEMDUMPCOLS));
}
}
if (OPTS_OPTION_BOOL(OPTION_DEBUG) ||
OPTS_OPTION_BOOL(OPTION_MEMCHK)) {
con_out("Memory information:\n\
Total allocations: %llu\n\
Total deallocations: %llu\n\
Total allocated: %f (MB)\n\
Total deallocated: %f (MB)\n\
Total peak memory: %f (MB)\n\
Total leaked memory: %f (MB) in %llu allocations\n",
mem_at,
mem_dt,
(float)(mem_ab) / 1048576.0f,
(float)(mem_db) / 1048576.0f,
(float)(mem_pk) / 1048576.0f,
(float)(mem_ab - mem_db) / 1048576.0f,
/* could be more clever */
(mem_at - mem_dt)
);
}
if (OPTS_OPTION_BOOL(OPTION_STATISTICS) ||
OPTS_OPTION_BOOL(OPTION_MEMCHK)) {
size_t i = 0;
size_t e = 1;
uint64_t vectormem = 0;
con_out("\nAdditional Statistics:\n\
Total vectors allocated: %llu\n\
Total string duplicates: %llu\n\
Total hashtables allocated: %llu\n\
Total unique vector sizes: %llu\n",
vectors,
strdups,
hashtables,
vector_sizes
);
for (; i < ST_SIZE; i++) {
size_entry_t *entry;
if (!(entry = vector_usage[i]))
continue;
con_out(" %2u| # of %4u byte vectors: %u\n",
(unsigned)e,
(unsigned)entry->key,
(unsigned)entry->value
);
e++;
vectormem += entry->key * entry->value;
}
con_out("\
Total unique hashtable sizes: %llu\n",
hashtable_sizes
);
for (i = 0, e = 1; i < ST_SIZE; i++) {
size_entry_t *entry;
if (!(entry = hashtable_usage[i]))
continue;
con_out(" %2u| # of %4u element hashtables: %u\n",
(unsigned)e,
(unsigned)entry->key,
(unsigned)entry->value
);
e++;
}
con_out(" Total vector memory: %f (MB)\n",
(float)(vectormem) / 1048576.0f
);
}
if (vector_usage)
util_st_del(vector_usage);
if (hashtable_usage)
util_st_del(hashtable_usage);
}
/*
* Some string utility functions, because strdup uses malloc, and we want
* to track all memory (without replacing malloc).
*/
char *_util_Estrdup(const char *s, const char *file, size_t line) {
size_t len = 0;
char *ptr = NULL;
/* in case of -DNOTRACK */
(void)file;
(void)line;
if (!s)
return NULL;
if ((len = strlen(s)) && (ptr = (char*)mem_af(len+1, line, file))) {
memcpy(ptr, s, len);
ptr[len] = '\0';
}
strdups++;
return ptr;
}
char *_util_Estrdup_empty(const char *s, const char *file, size_t line) {
size_t len = 0;
char *ptr = NULL;
/* in case of -DNOTRACK */
(void)file;
(void)line;
if (!s)
return NULL;
len = strlen(s);
if ((ptr = (char*)mem_af(len+1, line, file))) {
memcpy(ptr, s, len);
ptr[len] = '\0';
}
strdups++;
return ptr;
}
void util_debug(const char *area, const char *ms, ...) {
va_list va;
if (!OPTS_OPTION_BOOL(OPTION_DEBUG))
@ -544,274 +212,6 @@ size_t util_strtononcmd(const char *in, char *out, size_t outsz) {
return sz-1;
}
/* TODO: rewrite ... when I redo the ve cleanup */
void _util_vec_grow(void **a, size_t i, size_t s) {
vector_t *d = vec_meta(*a);
size_t m = 0;
size_entry_t *e = NULL;
void *p = NULL;
if (*a) {
m = 2 * d->allocated + i;
p = mem_r(d, s * m + sizeof(vector_t));
} else {
m = i + 1;
p = mem_a(s * m + sizeof(vector_t));
((vector_t*)p)->used = 0;
vectors++;
}
if (!vector_usage)
vector_usage = util_st_new();
if ((e = util_st_get(vector_usage, s))) {
e->value ++;
} else {
util_st_put(vector_usage, s, 1); /* start off with 1 */
vector_sizes++;
}
*a = (vector_t*)p + 1;
vec_meta(*a)->allocated = m;
}
/*
* Hash table for generic data, based on dynamic memory allocations
* all around. This is the internal interface, please look for
* EXPOSED INTERFACE comment below
*/
typedef struct hash_node_t {
char *key; /* the key for this node in table */
void *value; /* pointer to the data as void* */
struct hash_node_t *next; /* next node (linked list) */
} hash_node_t;
GMQCC_INLINE size_t util_hthash(hash_table_t *ht, const char *key) {
const uint32_t mix = 0x5BD1E995;
const uint32_t rot = 24;
size_t size = strlen(key);
uint32_t hash = 0x1EF0 /* LICRC TAB */ ^ size;
uint32_t alias = 0;
const unsigned char *data = (const unsigned char*)key;
while (size >= 4) {
alias = (data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24));
alias *= mix;
alias ^= alias >> rot;
alias *= mix;
hash *= mix;
hash ^= alias;
data += 4;
size -= 4;
}
switch (size) {
case 3: hash ^= data[2] << 16;
case 2: hash ^= data[1] << 8;
case 1: hash ^= data[0];
hash *= mix;
}
hash ^= hash >> 13;
hash *= mix;
hash ^= hash >> 15;
return (size_t) (hash % ht->size);
}
static hash_node_t *_util_htnewpair(const char *key, void *value) {
hash_node_t *node;
if (!(node = (hash_node_t*)mem_a(sizeof(hash_node_t))))
return NULL;
if (!(node->key = util_strdupe(key))) {
mem_d(node);
return NULL;
}
node->value = value;
node->next = NULL;
return node;
}
/*
* EXPOSED INTERFACE for the hashtable implementation
* util_htnew(size) -- to make a new hashtable
* util_htset(table, key, value, sizeof(value)) -- to set something in the table
* util_htget(table, key) -- to get something from the table
* util_htdel(table) -- to delete the table
*/
hash_table_t *util_htnew(size_t size) {
hash_table_t *hashtable = NULL;
size_entry_t *find;
if (size < 1)
return NULL;
if (!hashtable_usage)
hashtable_usage = util_st_new();
if (!(hashtable = (hash_table_t*)mem_a(sizeof(hash_table_t))))
return NULL;
if (!(hashtable->table = (hash_node_t**)mem_a(sizeof(hash_node_t*) * size))) {
mem_d(hashtable);
return NULL;
}
if ((find = util_st_get(hashtable_usage, size)))
find->value++;
else {
hashtable_sizes++;
util_st_put(hashtable_usage, size, 1);
}
hashtable->size = size;
memset(hashtable->table, 0, sizeof(hash_node_t*) * size);
hashtables++;
return hashtable;
}
void util_htseth(hash_table_t *ht, const char *key, size_t bin, void *value) {
hash_node_t *newnode = NULL;
hash_node_t *next = NULL;
hash_node_t *last = NULL;
next = ht->table[bin];
while (next && next->key && strcmp(key, next->key) > 0)
last = next, next = next->next;
/* already in table, do a replace */
if (next && next->key && strcmp(key, next->key) == 0) {
next->value = value;
} else {
/* not found, grow a pair man :P */
newnode = _util_htnewpair(key, value);
if (next == ht->table[bin]) {
newnode->next = next;
ht->table[bin] = newnode;
} else if (!next) {
last->next = newnode;
} else {
newnode->next = next;
last->next = newnode;
}
}
}
void util_htset(hash_table_t *ht, const char *key, void *value) {
util_htseth(ht, key, util_hthash(ht, key), value);
}
void *util_htgeth(hash_table_t *ht, const char *key, size_t bin) {
hash_node_t *pair = ht->table[bin];
while (pair && pair->key && strcmp(key, pair->key) > 0)
pair = pair->next;
if (!pair || !pair->key || strcmp(key, pair->key) != 0)
return NULL;
return pair->value;
}
void *util_htget(hash_table_t *ht, const char *key) {
return util_htgeth(ht, key, util_hthash(ht, key));
}
void *code_util_str_htgeth(hash_table_t *ht, const char *key, size_t bin) {
hash_node_t *pair;
size_t len, keylen;
int cmp;
keylen = strlen(key);
pair = ht->table[bin];
while (pair && pair->key) {
len = strlen(pair->key);
if (len < keylen) {
pair = pair->next;
continue;
}
if (keylen == len) {
cmp = strcmp(key, pair->key);
if (cmp == 0)
return pair->value;
if (cmp < 0)
return NULL;
pair = pair->next;
continue;
}
cmp = strcmp(key, pair->key + len - keylen);
if (cmp == 0) {
uintptr_t up = (uintptr_t)pair->value;
up += len - keylen;
return (void*)up;
}
pair = pair->next;
}
return NULL;
}
/*
* Free all allocated data in a hashtable, this is quite the amount
* of work.
*/
void util_htrem(hash_table_t *ht, void (*callback)(void *data)) {
size_t i = 0;
for (; i < ht->size; i++) {
hash_node_t *n = ht->table[i];
hash_node_t *p;
/* free in list */
while (n) {
if (n->key)
mem_d(n->key);
if (callback)
callback(n->value);
p = n;
n = n->next;
mem_d(p);
}
}
/* free table */
mem_d(ht->table);
mem_d(ht);
}
void util_htrmh(hash_table_t *ht, const char *key, size_t bin, void (*cb)(void*)) {
hash_node_t **pair = &ht->table[bin];
hash_node_t *tmp;
while (*pair && (*pair)->key && strcmp(key, (*pair)->key) > 0)
pair = &(*pair)->next;
tmp = *pair;
if (!tmp || !tmp->key || strcmp(key, tmp->key) != 0)
return;
if (cb)
(*cb)(tmp->value);
*pair = tmp->next;
mem_d(tmp->key);
mem_d(tmp);
}
void util_htrm(hash_table_t *ht, const char *key, void (*cb)(void*)) {
util_htrmh(ht, key, util_hthash(ht, key), cb);
}
void util_htdel(hash_table_t *ht) {
util_htrem(ht, NULL);
}
/*
* Portable implementation of vasprintf/asprintf. Assumes vsnprintf
* exists, otherwise compiler error.