Major utility rewrite for compiler memory utilization statistics. Cleanups everywhere, no more NOTRACK stuff, all allocates are tracked. Major identifier cleanups as well.

Makefile

@@ -44,11 +44,11 @@ ifeq ($(track), no)
 	CFLAGS += -DNOTRACK
 endif
 
-OBJ_D = util.o code.o ast.o ir.o conout.o ftepp.o opts.o fs.o utf8.o correct.o
+OBJ_D = util.o code.o ast.o ir.o conout.o ftepp.o opts.o fs.o utf8.o correct.o stat.o
-OBJ_P = util.o fs.o conout.o opts.o pak.o
+OBJ_P = util.o fs.o conout.o opts.o pak.o stat.o
-OBJ_T = test.o util.o conout.o fs.o
+OBJ_T = test.o util.o conout.o fs.o stat.o
-OBJ_C = main.o lexer.o parser.o fs.o
+OBJ_C = main.o lexer.o parser.o fs.o stat.o
-OBJ_X = exec-standalone.o util.o conout.o fs.o
+OBJ_X = exec-standalone.o util.o conout.o fs.o stat.o
 
 #we have duplicate object files when dealing with creating a simple list
 #for dependinces. To combat this we use some clever recrusive-make to
@@ -244,6 +244,7 @@ opts.o: gmqcc.h opts.def
 fs.o: gmqcc.h opts.def
 utf8.o: gmqcc.h opts.def
 correct.o: gmqcc.h opts.def
+stat.o: gmqcc.h opts.def
 pak.o: gmqcc.h opts.def
 test.o: gmqcc.h opts.def
 main.o: gmqcc.h opts.def lexer.h

gmqcc.h

@@ -288,20 +288,47 @@ GMQCC_IND_STRING(GMQCC_VERSION_PATCH) \
 #   include <dirent.h>
 #endif /*! _WIN32 && !defined(__MINGW32__) */
 
+/*===================================================================*/
+/*=========================== stat.c ================================*/
+/*===================================================================*/
+typedef struct {
+    size_t key;
+    size_t value;
+} stat_size_entry_t, **stat_size_table_t;
+
+void  stat_info();
+
+char *stat_mem_strdup    (const char *, size_t,         const char *, bool);
+void *stat_mem_reallocate(void *,       size_t, size_t, const char *);
+void  stat_mem_deallocate(void *);
+void *stat_mem_allocate  (size_t, size_t, const char *);
+
+stat_size_table_t  stat_size_new();
+stat_size_entry_t *stat_size_get(stat_size_table_t, size_t);
+void               stat_size_del(stat_size_table_t);
+void               stat_size_put(stat_size_table_t, size_t, size_t);
+
+/* getters for hashtable: */
+stat_size_table_t *stat_size_hashtables_get();
+uint64_t          *stat_type_hashtables_get();
+uint64_t          *stat_used_hashtables_get();
+stat_size_table_t *stat_hashtables_init();
+
+#define mem_a(SIZE)              stat_mem_allocate  ((SIZE), __LINE__, __FILE__)
+#define mem_d(PTRN)              stat_mem_deallocate((void*)(PTRN))
+#define mem_r(PTRN, SIZE)        stat_mem_reallocate((void*)(PTRN), (SIZE), __LINE__, __FILE__)
+#define mem_af(SIZE, FILE, LINE) stat_mem_allocate  ((SIZE), (LINE), (FILE))
+
+/* TODO: rename to mem variations */
+#define util_strdup(SRC)         stat_mem_strdup((char*)(SRC), __LINE__, __FILE__, false)
+#define util_strdupe(SRC)        stat_mem_strdup((char*)(SRC), __LINE__, __FILE__, true)
 
 /*===================================================================*/
 /*=========================== util.c ================================*/
 /*===================================================================*/
-void *util_memory_a      (size_t, /*****/ unsigned int, const char *);
-void *util_memory_r      (void *, size_t, unsigned int, const char *);
-void  util_memory_d      (void *);
-void  util_meminfo       ();
-
 bool  util_filexists     (const char *);
 bool  util_strupper      (const char *);
 bool  util_strdigit      (const char *);
-char *_util_Estrdup      (const char *, const char *, size_t);
-char *_util_Estrdup_empty(const char *, const char *, size_t);
 void  util_debug         (const char *, const char *, ...);
 void  util_endianswap    (void *,  size_t, unsigned int);
 
@@ -326,22 +353,6 @@ char       *util_strcat   (char *dest, const char *src);
 char       *util_strncpy  (char *dest, const char *src, size_t num);
 const char *util_strerror (int num);
 
-
-#ifdef NOTRACK
-#    define mem_a(x)      malloc (x)
-#    define mem_d(x)      free   ((void*)x)
-#    define mem_r(x, n)   realloc((void*)x, n)
-#    define mem_af(x,f,l) malloc (x)
-#else
-#    define mem_a(x)      util_memory_a((x), __LINE__, __FILE__)
-#    define mem_d(x)      util_memory_d((void*)(x))
-#    define mem_r(x, n)   util_memory_r((void*)(x), (n), __LINE__, __FILE__)
-#    define mem_af(x,f,l) util_memory_a((x), __LINE__, __FILE__)
-#endif /*! NOTRACK */
-
-#define util_strdup(X)  _util_Estrdup((X), __FILE__, __LINE__)
-#define util_strdupe(X) _util_Estrdup_empty((X), __FILE__, __LINE__)
-
 /*
  * A flexible vector implementation: all vector pointers contain some
  * data about themselfs exactly - sizeof(vector_t) behind the pointer

main.c

@@ -796,8 +796,7 @@ cleanup:
         mem_d((void*)operators);
 
     lex_cleanup();
-    util_meminfo();
+    stat_info();
-    /*util_vecstats_destroy();*/
 
     return retval;
 }

pak.c

@@ -552,7 +552,8 @@ int main(int argc, char **argv) {
         /* not possible */
         pak_close(pak);
         vec_free(files);
-        util_meminfo();
+        stat_info();
+        
         return EXIT_SUCCESS;
     }
 
@@ -575,6 +576,6 @@ int main(int argc, char **argv) {
     pak_close(pak);
     vec_free(files);
 
-    util_meminfo();
+    stat_info();
     return EXIT_SUCCESS;
 }

stat.c

@@ -0,0 +1,590 @@
+#include "gmqcc.h"
+
+/*
+ * GMQCC performs tons of allocations, constructions, and crazyness
+ * all around. When trying to optimizes systems, or just get fancy
+ * statistics out of the compiler, it's often printf mess. This file
+ * implements the statistics system of the compiler. I.E the allocator
+ * we use to track allocations, and other systems of interest.
+ */
+#define ST_SIZE 1024
+
+typedef struct stat_mem_block_s {
+    const char              *file;
+    size_t                   line;
+    size_t                   size;
+    struct stat_mem_block_s *next;
+    struct stat_mem_block_s *prev;
+} stat_mem_block_t;
+
+static uint64_t          stat_mem_allocated         = 0;
+static uint64_t          stat_mem_deallocated       = 0;
+static uint64_t          stat_mem_allocated_total   = 0;
+static uint64_t          stat_mem_deallocated_total = 0;
+static uint64_t          stat_mem_high              = 0;
+static uint64_t          stat_mem_peak              = 0;
+static uint64_t          stat_used_strdups          = 0;
+static uint64_t          stat_used_vectors          = 0;
+static uint64_t          stat_used_hashtables       = 0;
+static uint64_t          stat_type_vectors          = 0;
+static uint64_t          stat_type_hashtables       = 0;
+static stat_size_table_t stat_size_vectors          = NULL;
+static stat_size_table_t stat_size_hashtables       = NULL;
+static stat_mem_block_t *stat_mem_block_root        = NULL;
+
+/*
+ * A basic header of information wrapper allocator. Simply stores
+ * information as a header, returns the memory + 1 past it, can be
+ * retrieved again with - 1. Where type is stat_mem_block_t*.
+ */
+void *stat_mem_allocate(size_t size, size_t line, const char *file) {
+    stat_mem_block_t *info = (stat_mem_block_t*)malloc(sizeof(stat_mem_block_t) + size);
+    void             *data = (void*)(info + 1);
+    
+    if(!info)
+        return NULL;
+        
+    info->line = line;
+    info->size = size;
+    info->file = file;
+    info->prev = NULL;
+    info->next = stat_mem_block_root;
+    
+    if (stat_mem_block_root)
+        stat_mem_block_root->prev = info;
+        
+    stat_mem_block_root       = info;
+    stat_mem_allocated       += size;
+    stat_mem_high            += size;
+    stat_mem_allocated_total ++;
+    
+    if (stat_mem_high > stat_mem_peak)
+        stat_mem_peak = stat_mem_high;
+
+    return data;
+}
+
+void stat_mem_deallocate(void *ptr) {
+    stat_mem_block_t *info = NULL;
+    
+    if (!ptr)
+        return;
+        
+    info = ((stat_mem_block_t*)ptr - 1);
+    
+    stat_mem_deallocated       += info->size;
+    stat_mem_high              -= info->size;
+    stat_mem_deallocated_total ++;
+    
+    if (info->prev) info->prev->next = info->next;
+    if (info->next) info->next->prev = info->prev;
+    
+    /* move ahead */
+    if (info == stat_mem_block_root)
+        stat_mem_block_root = info->next;
+}
+
+void *stat_mem_reallocate(void *ptr, size_t size, size_t line, const char *file) {
+    stat_mem_block_t *oldinfo = NULL;
+    stat_mem_block_t *newinfo;
+    
+    if (!ptr)
+        return stat_mem_allocate(size, line, file);
+    
+    /* stay consistent with glic */
+    if (!size) {
+        stat_mem_deallocate(ptr);
+        return NULL;
+    }
+    
+    oldinfo = ((stat_mem_block_t*)ptr - 1);
+    newinfo = ((stat_mem_block_t*)malloc(sizeof(stat_mem_block_t) + size));
+    
+    if (!newinfo) {
+        stat_mem_deallocate(ptr);
+        return NULL;
+    }
+    
+    memcpy(newinfo+1, oldinfo+1, oldinfo->size);
+    
+    if (oldinfo->prev) oldinfo->prev->next = oldinfo->next;
+    if (oldinfo->next) oldinfo->next->prev = oldinfo->prev;
+    
+    /* move ahead */
+    if (oldinfo == stat_mem_block_root)
+        stat_mem_block_root = oldinfo->next;
+        
+    newinfo->line = line;
+    newinfo->size = size;
+    newinfo->file = file;
+    newinfo->prev = NULL;
+    newinfo->next = stat_mem_block_root;
+    
+    if (stat_mem_block_root)
+        stat_mem_block_root->prev = newinfo;
+    
+    stat_mem_block_root = newinfo;
+    stat_mem_allocated -= oldinfo->size;
+    stat_mem_high      -= oldinfo->size;
+    stat_mem_allocated += newinfo->size;
+    stat_mem_high      += newinfo->size;
+    
+    if (stat_mem_high > stat_mem_peak)
+        stat_mem_peak = stat_mem_high;
+        
+    free(oldinfo);
+    
+    return newinfo + 1;
+}
+
+/*
+ * strdup does it's own malloc, we need to track malloc. We don't want
+ * to overwrite malloc though, infact, we can't really hook it at all
+ * without library specific assumptions. So we re implement strdup.
+ */
+char *stat_mem_strdup(const char *src, size_t line, const char *file, bool empty) {
+    size_t len = 0;
+    char  *ptr = NULL;
+    
+    if (!src)
+        return NULL;
+    
+    len = strlen(src);
+    if (((!empty) ? len : true) && (ptr = (char*)stat_mem_allocate(len + 1, line, file))) {
+        memcpy(ptr, src, len);
+        ptr[len] = '\0';
+    }
+    
+    stat_used_strdups ++;
+    return ptr;
+}
+
+/*
+ * The reallocate function for resizing vectors.
+ */
+void _util_vec_grow(void **a, size_t i, size_t s) {
+    vector_t          *d = vec_meta(*a);
+    size_t             m = 0;
+    stat_size_entry_t *e = NULL;
+    void              *p = NULL;
+    
+    if (*a) {
+        m = 2 * d->allocated + i;
+        p = mem_r(d, s * m + sizeof(vector_t));
+    } else {
+        m = i + 1;
+        p = mem_a(s * m + sizeof(vector_t));
+        ((vector_t*)p)->used = 0;
+        stat_used_vectors++;
+    }
+    
+    if (!stat_size_vectors)
+        stat_size_vectors = stat_size_new();
+
+    if ((e = stat_size_get(stat_size_vectors, s))) {
+        e->value ++;
+    } else {
+        stat_size_put(stat_size_vectors, s, 1); /* start off with 1 */
+        stat_type_vectors++;
+    }
+
+    *a = (vector_t*)p + 1;
+    vec_meta(*a)->allocated = m;
+}
+
+/*
+ * Hash table for generic data, based on dynamic memory allocations
+ * all around.  This is the internal interface, please look for
+ * EXPOSED INTERFACE comment below
+ */
+typedef struct hash_node_t {
+    char               *key;   /* the key for this node in table */
+    void               *value; /* pointer to the data as void*   */
+    struct hash_node_t *next;  /* next node (linked list)        */
+} hash_node_t;
+
+GMQCC_INLINE size_t util_hthash(hash_table_t *ht, const char *key) {
+    const uint32_t       mix   = 0x5BD1E995;
+    const uint32_t       rot   = 24;
+    size_t               size  = strlen(key);
+    uint32_t             hash  = 0x1EF0 /* LICRC TAB */  ^ size;
+    uint32_t             alias = 0;
+    const unsigned char *data  = (const unsigned char*)key;
+
+    while (size >= 4) {
+        alias  = (data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24));
+        alias *= mix;
+        alias ^= alias >> rot;
+        alias *= mix;
+
+        hash  *= mix;
+        hash  ^= alias;
+
+        data += 4;
+        size -= 4;
+    }
+
+    switch (size) {
+        case 3: hash ^= data[2] << 16;
+        case 2: hash ^= data[1] << 8;
+        case 1: hash ^= data[0];
+                hash *= mix;
+    }
+
+    hash ^= hash >> 13;
+    hash *= mix;
+    hash ^= hash >> 15;
+
+    return (size_t) (hash % ht->size);
+}
+
+static hash_node_t *_util_htnewpair(const char *key, void *value) {
+    hash_node_t *node;
+    if (!(node = (hash_node_t*)mem_a(sizeof(hash_node_t))))
+        return NULL;
+
+    if (!(node->key = util_strdupe(key))) {
+        mem_d(node);
+        return NULL;
+    }
+
+    node->value = value;
+    node->next  = NULL;
+
+    return node;
+}
+
+/*
+ * EXPOSED INTERFACE for the hashtable implementation
+ * util_htnew(size)                             -- to make a new hashtable
+ * util_htset(table, key, value, sizeof(value)) -- to set something in the table
+ * util_htget(table, key)                       -- to get something from the table
+ * util_htdel(table)                            -- to delete the table
+ */
+hash_table_t *util_htnew(size_t size) {
+    hash_table_t      *hashtable = NULL;
+    stat_size_entry_t *find      = NULL;
+    
+    if (size < 1)
+        return NULL;
+        
+    if (!stat_size_hashtables)
+        stat_size_hashtables = stat_size_new();
+
+    if (!(hashtable = (hash_table_t*)mem_a(sizeof(hash_table_t))))
+        return NULL;
+
+    if (!(hashtable->table = (hash_node_t**)mem_a(sizeof(hash_node_t*) * size))) {
+        mem_d(hashtable);
+        return NULL;
+    }
+    
+    if ((find = stat_size_get(stat_size_hashtables, size)))
+        find->value++;
+    else {
+        stat_used_hashtables++;
+        stat_size_put(stat_size_hashtables, size, 1);
+    }
+
+    hashtable->size = size;
+    memset(hashtable->table, 0, sizeof(hash_node_t*) * size);
+
+    stat_type_hashtables++;
+    return hashtable;
+}
+
+void util_htseth(hash_table_t *ht, const char *key, size_t bin, void *value) {
+    hash_node_t *newnode = NULL;
+    hash_node_t *next    = NULL;
+    hash_node_t *last    = NULL;
+
+    next = ht->table[bin];
+
+    while (next && next->key && strcmp(key, next->key) > 0)
+        last = next, next = next->next;
+
+    /* already in table, do a replace */
+    if (next && next->key && strcmp(key, next->key) == 0) {
+        next->value = value;
+    } else {
+        /* not found, grow a pair man :P */
+        newnode = _util_htnewpair(key, value);
+        if (next == ht->table[bin]) {
+            newnode->next  = next;
+            ht->table[bin] = newnode;
+        } else if (!next) {
+            last->next = newnode;
+        } else {
+            newnode->next = next;
+            last->next = newnode;
+        }
+    }
+}
+
+void util_htset(hash_table_t *ht, const char *key, void *value) {
+    util_htseth(ht, key, util_hthash(ht, key), value);
+}
+
+void *util_htgeth(hash_table_t *ht, const char *key, size_t bin) {
+    hash_node_t *pair = ht->table[bin];
+
+    while (pair && pair->key && strcmp(key, pair->key) > 0)
+        pair = pair->next;
+
+    if (!pair || !pair->key || strcmp(key, pair->key) != 0)
+        return NULL;
+
+    return pair->value;
+}
+
+void *util_htget(hash_table_t *ht, const char *key) {
+    return util_htgeth(ht, key, util_hthash(ht, key));
+}
+
+void *code_util_str_htgeth(hash_table_t *ht, const char *key, size_t bin) {
+    hash_node_t *pair;
+    size_t len, keylen;
+    int cmp;
+
+    keylen = strlen(key);
+
+    pair = ht->table[bin];
+    while (pair && pair->key) {
+        len = strlen(pair->key);
+        if (len < keylen) {
+            pair = pair->next;
+            continue;
+        }
+        if (keylen == len) {
+            cmp = strcmp(key, pair->key);
+            if (cmp == 0)
+                return pair->value;
+            if (cmp < 0)
+                return NULL;
+            pair = pair->next;
+            continue;
+        }
+        cmp = strcmp(key, pair->key + len - keylen);
+        if (cmp == 0) {
+            uintptr_t up = (uintptr_t)pair->value;
+            up += len - keylen;
+            return (void*)up;
+        }
+        pair = pair->next;
+    }
+    return NULL;
+}
+
+/*
+ * Free all allocated data in a hashtable, this is quite the amount
+ * of work.
+ */
+void util_htrem(hash_table_t *ht, void (*callback)(void *data)) {
+    size_t i = 0;
+    for (; i < ht->size; i++) {
+        hash_node_t *n = ht->table[i];
+        hash_node_t *p;
+
+        /* free in list */
+        while (n) {
+            if (n->key)
+                mem_d(n->key);
+            if (callback)
+                callback(n->value);
+            p = n;
+            n = n->next;
+            mem_d(p);
+        }
+
+    }
+    /* free table */
+    mem_d(ht->table);
+    mem_d(ht);
+}
+
+void util_htrmh(hash_table_t *ht, const char *key, size_t bin, void (*cb)(void*)) {
+    hash_node_t **pair = &ht->table[bin];
+    hash_node_t *tmp;
+
+    while (*pair && (*pair)->key && strcmp(key, (*pair)->key) > 0)
+        pair = &(*pair)->next;
+
+    tmp = *pair;
+    if (!tmp || !tmp->key || strcmp(key, tmp->key) != 0)
+        return;
+
+    if (cb)
+        (*cb)(tmp->value);
+
+    *pair = tmp->next;
+    mem_d(tmp->key);
+    mem_d(tmp);
+}
+
+void util_htrm(hash_table_t *ht, const char *key, void (*cb)(void*)) {
+    util_htrmh(ht, key, util_hthash(ht, key), cb);
+}
+
+void util_htdel(hash_table_t *ht) {
+    util_htrem(ht, NULL);
+}
+
+/*
+ * A tiny size_t key-value hashtbale for tracking vector and hashtable
+ * sizes. We can use it for other things too, if we need to. This is
+ * very TIGHT, and efficent in terms of space though.
+ */
+stat_size_table_t stat_size_new() {
+    return (stat_size_table_t)memset(
+        mem_a(sizeof(stat_size_entry_t*) * ST_SIZE),
+        0, ST_SIZE * sizeof(stat_size_entry_t*)
+    );
+}
+
+void stat_size_del(stat_size_table_t table) {
+    size_t i = 0;
+    for (; i < ST_SIZE; i++) if(table[i]) mem_d(table[i]);
+    mem_d(table);
+}
+
+stat_size_entry_t *stat_size_get(stat_size_table_t table, size_t key) {
+    size_t hash = (key % ST_SIZE);
+    while (table[hash] && table[hash]->key != key)
+        hash = (hash + 1) % ST_SIZE;
+    return table[hash];
+}
+void stat_size_put(stat_size_table_t table, size_t key, size_t value) {
+    size_t hash = (key % ST_SIZE);
+    while (table[hash] && table[hash]->key != key)
+        hash = (hash + 1) % ST_SIZE;
+    table[hash] = (stat_size_entry_t*)mem_a(sizeof(stat_size_entry_t));
+    table[hash]->key = key;
+    table[hash]->value = value;
+}
+
+/*
+ * The following functions below implement printing / dumping of statistical
+ * information.
+ */
+static void stat_dump_mem_contents(stat_mem_block_t *memory, uint16_t cols) {
+    uint32_t i, j;
+    for (i = 0; i < memory->size + ((memory->size % cols) ? (cols - memory->size % cols) : 0); i++) {
+        if (i % cols == 0)    con_out(" 0x%06X: ", i);
+        if (i < memory->size) con_out("%02X " , 0xFF & ((unsigned char*)(memory + 1))[i]);
+        else                  con_out(" ");
+
+        if ((uint16_t)(i % cols) == (cols - 1)) {
+            for (j = i - (cols - 1); j <= i; j++) {
+                con_out("%c",
+                    (j >= memory->size)
+                        ? ' '
+                        : (isprint(((unsigned char*)(memory + 1))[j]))
+                            ? 0xFF & ((unsigned char*)(memory + 1)) [j]
+                            : '.'
+                );
+            }
+            con_out("\n");
+        }
+    }
+}
+
+static void stat_dump_mem_leaks() {
+    stat_mem_block_t *info;
+    for (info = stat_mem_block_root; info; info = info->next) {
+        con_out("lost: %u (bytes) at %s:%u\n",
+            info->size,
+            info->file,
+            info->line
+        );
+        
+        stat_dump_mem_contents(info, OPTS_OPTION_U16(OPTION_MEMDUMPCOLS));
+    }
+}
+
+static void stat_dump_mem_info() {
+    con_out("Memory information:\n\
+    Total allocations:   %llu\n\
+    Total deallocations: %llu\n\
+    Total allocated:     %f (MB)\n\
+    Total deallocated:   %f (MB)\n\
+    Total peak memory:   %f (MB)\n\
+    Total leaked memory: %f (MB) in %llu allocations\n",
+        stat_mem_allocated_total,
+        stat_mem_deallocated_total,
+        (float)(stat_mem_allocated)                        / 1048576.0f,
+        (float)(stat_mem_deallocated)                      / 1048576.0f,
+        (float)(stat_mem_high)                             / 1048576.0f,
+        (float)(stat_mem_allocated - stat_mem_deallocated) / 1048576.0f,
+        stat_mem_allocated_total - stat_mem_deallocated_total
+    );
+}
+
+static void stat_dump_stats_table(stat_size_table_t table, const char *string, uint64_t *size) {
+    size_t i,j;
+    
+    for (i = 0, j = 0; i < ST_SIZE; i++) {
+        stat_size_entry_t *entry;
+
+        if (!(entry = table[i]))
+            continue;
+
+        con_out(string, (unsigned)j, (unsigned)entry->key, (unsigned)entry->value);
+        j++;
+        
+        if (size)
+            *size += entry->key * entry->value;
+    }
+}
+
+void stat_info() {
+    if (OPTS_OPTION_BOOL(OPTION_DEBUG))
+        stat_dump_mem_leaks();
+
+    if (OPTS_OPTION_BOOL(OPTION_DEBUG) ||
+        OPTS_OPTION_BOOL(OPTION_MEMCHK))
+        stat_dump_mem_info();
+
+    if (OPTS_OPTION_BOOL(OPTION_MEMCHK) ||
+        OPTS_OPTION_BOOL(OPTION_STATISTICS)) {
+        uint64_t mem;
+        
+        con_out("\nAdditional Statistics:\n\
+    Total vectors allocated:    %llu\n\
+    Total string duplicates:    %llu\n\
+    Total hashtables allocated: %llu\n\
+    Total unique vector sizes:  %llu\n",
+            stat_used_vectors,
+            stat_used_strdups,
+            stat_used_hashtables,
+            stat_type_vectors
+        );
+        
+        stat_dump_stats_table (
+            stat_size_vectors,
+            "        %2u| # of %4u byte vectors: %u\n",
+            &mem
+        );
+        
+        con_out (
+            "    Total unique hashtable sizes: %llu\n",
+            stat_type_hashtables
+        );
+        
+        stat_dump_stats_table (
+            stat_size_hashtables,
+            "        %2u| # of %4u element hashtables: %u\n",
+            NULL
+        );
+        
+        con_out (
+            "    Total vector memory:          %f (MB)\n",
+            (float)(mem) / 1048576.0f
+        );
+    }
+
+    if (stat_size_vectors)
+        stat_size_del(stat_size_vectors);
+    if (stat_size_hashtables)
+        stat_size_del(stat_size_hashtables);
+}
+#undef ST_SIZE

test.c

@@ -1321,7 +1321,7 @@ int main(int argc, char **argv) {
     }
     con_change(redirout, redirerr);
     succeed = test_perform("tests", defs);
-    util_meminfo();
+    stat_info();
 
 
     return (succeed) ? EXIT_SUCCESS : EXIT_FAILURE;

util.c

@@ -25,338 +25,6 @@
 #include <errno.h>
 #include "gmqcc.h"
 
-/* TODO: remove globals ... */
-static uint64_t mem_ab = 0;
-static uint64_t mem_db = 0;
-static uint64_t mem_at = 0;
-static uint64_t mem_dt = 0;
-static uint64_t mem_pk = 0;
-static uint64_t mem_hw = 0;
-
-struct memblock_t {
-    const char  *file;
-    unsigned int line;
-    size_t       byte;
-    struct memblock_t *next;
-    struct memblock_t *prev;
-};
-
-#define PEAK_MEM             \
-    do {                     \
-        if (mem_hw > mem_pk) \
-            mem_pk = mem_hw; \
-    } while (0)
-
-static struct memblock_t *mem_start = NULL;
-
-void *util_memory_a(size_t byte, unsigned int line, const char *file) {
-    struct memblock_t *info = (struct memblock_t*)malloc(sizeof(struct memblock_t) + byte);
-    void              *data = (void*)(info+1);
-    if (!info) return NULL;
-    info->line = line;
-    info->byte = byte;
-    info->file = file;
-    info->prev = NULL;
-    info->next = mem_start;
-    if (mem_start)
-        mem_start->prev = info;
-    mem_start = info;
-
-    mem_at++;
-    mem_ab += info->byte;
-    mem_hw += info->byte;
-
-    PEAK_MEM;
-
-    return data;
-}
-
-void util_memory_d(void *ptrn) {
-    struct memblock_t *info = NULL;
-
-    if (!ptrn) return;
-    info = ((struct memblock_t*)ptrn - 1);
-
-    mem_db += info->byte;
-    mem_hw -= info->byte;
-    mem_dt++;
-
-    if (info->prev)
-        info->prev->next = info->next;
-    if (info->next)
-        info->next->prev = info->prev;
-    if (info == mem_start)
-        mem_start = info->next;
-
-    free(info);
-}
-
-void *util_memory_r(void *ptrn, size_t byte, unsigned int line, const char *file) {
-    struct memblock_t *oldinfo = NULL;
-
-    struct memblock_t *newinfo;
-
-    if (!ptrn)
-        return util_memory_a(byte, line, file);
-    if (!byte) {
-        util_memory_d(ptrn);
-        return NULL;
-    }
-
-    oldinfo = ((struct memblock_t*)ptrn - 1);
-    newinfo = ((struct memblock_t*)malloc(sizeof(struct memblock_t) + byte));
-
-    /* new data */
-    if (!newinfo) {
-        util_memory_d(oldinfo+1);
-        return NULL;
-    }
-
-    /* copy old */
-    memcpy(newinfo+1, oldinfo+1, oldinfo->byte);
-
-    /* free old */
-    if (oldinfo->prev)
-        oldinfo->prev->next = oldinfo->next;
-    if (oldinfo->next)
-        oldinfo->next->prev = oldinfo->prev;
-    if (oldinfo == mem_start)
-        mem_start = oldinfo->next;
-
-    /* fill info */
-    newinfo->line = line;
-    newinfo->byte = byte;
-    newinfo->file = file;
-    newinfo->prev = NULL;
-    newinfo->next = mem_start;
-    if (mem_start)
-        mem_start->prev = newinfo;
-    mem_start = newinfo;
-
-    mem_ab -= oldinfo->byte;
-    mem_hw -= oldinfo->byte;
-    mem_ab += newinfo->byte;
-    mem_hw += newinfo->byte;
-
-    PEAK_MEM;
-
-    free(oldinfo);
-
-    return newinfo+1;
-}
-
-static void util_dumpmem(struct memblock_t *memory, uint16_t cols) {
-    uint32_t i, j;
-    for (i = 0; i < memory->byte + ((memory->byte % cols) ? (cols - memory->byte % cols) : 0); i++) {
-        if (i % cols == 0)    con_out("    0x%06X: ", i);
-        if (i < memory->byte) con_out("%02X "   , 0xFF & ((char*)(memory + 1))[i]);
-        else                  con_out("    ");
-
-        if ((uint16_t)(i % cols) == (cols - 1)) {
-            for (j = i - (cols - 1); j <= i; j++) {
-                con_out("%c",
-                    (j >= memory->byte)
-                        ? ' '
-                        : (isprint(((char*)(memory + 1))[j]))
-                            ? 0xFF & ((char*)(memory + 1)) [j]
-                            : '.'
-                );
-            }
-            con_out("\n");
-        }
-    }
-}
-
-/*
- * The following is a VERY tight, efficent, hashtable for integer
- * values and keys, and for nothing more. We could make our existing
- * hashtable support type-genericness through a void * pointer but,
- * ideally that would make things more complicated. We also don't need
- * that much of a bloat for something as basic as this.
- */
-typedef struct {
-    size_t key;
-    size_t value;
-} size_entry_t;
-#define ST_SIZE 1024
-
-typedef size_entry_t **size_table_t;
-
-static size_table_t util_st_new() {
-    return (size_table_t)memset(
-        mem_a(sizeof(size_entry_t*) * ST_SIZE),
-        0, ST_SIZE * sizeof(size_entry_t*)
-    );
-}
-static void util_st_del(size_table_t table) {
-    size_t i = 0;
-    for (; i < ST_SIZE; i++) if(table[i]) mem_d(table[i]);
-    mem_d(table);
-}
-static size_entry_t *util_st_get(size_table_t table, size_t key) {
-    size_t hash = (key % ST_SIZE);
-    while (table[hash] && table[hash]->key != key)
-        hash = (hash + 1) % ST_SIZE;
-    return table[hash];
-}
-static void util_st_put(size_table_t table, size_t key, size_t value) {
-    size_t hash = (key % ST_SIZE);
-    while (table[hash] && table[hash]->key != key)
-        hash = (hash + 1) % ST_SIZE;
-    table[hash]        = (size_entry_t*)mem_a(sizeof(size_entry_t));
-    table[hash]->key   = key;
-    table[hash]->value = value;
-}
-
-static uint64_t      strdups           = 0;
-static uint64_t      vectors           = 0;
-static uint64_t      vector_sizes      = 0;
-static uint64_t      hashtables        = 0;
-static uint64_t      hashtable_sizes   = 0;
-static size_table_t  vector_usage      = NULL;
-static size_table_t  hashtable_usage   = NULL;
-
-void util_meminfo() {
-    struct memblock_t *info;
-
-    if (OPTS_OPTION_BOOL(OPTION_DEBUG)) {
-        for (info = mem_start; info; info = info->next) {
-            con_out("lost: %u (bytes) at %s:%u\n",
-                info->byte,
-                info->file,
-                info->line);
-
-            util_dumpmem(info, OPTS_OPTION_U16(OPTION_MEMDUMPCOLS));
-        }
-    }
-
-    if (OPTS_OPTION_BOOL(OPTION_DEBUG) ||
-        OPTS_OPTION_BOOL(OPTION_MEMCHK)) {
-        con_out("Memory information:\n\
-            Total allocations:   %llu\n\
-            Total deallocations: %llu\n\
-            Total allocated:     %f (MB)\n\
-            Total deallocated:   %f (MB)\n\
-            Total peak memory:   %f (MB)\n\
-            Total leaked memory: %f (MB) in %llu allocations\n",
-                mem_at,
-                mem_dt,
-                (float)(mem_ab)           / 1048576.0f,
-                (float)(mem_db)           / 1048576.0f,
-                (float)(mem_pk)           / 1048576.0f,
-                (float)(mem_ab -  mem_db) / 1048576.0f,
-
-                /* could be more clever */
-                (mem_at -  mem_dt)
-        );
-    }
-    
-    if (OPTS_OPTION_BOOL(OPTION_STATISTICS) ||
-        OPTS_OPTION_BOOL(OPTION_MEMCHK)) {
-        size_t   i         = 0;
-        size_t   e         = 1;
-        uint64_t vectormem = 0;
-        
-        con_out("\nAdditional Statistics:\n\
-            Total vectors allocated:      %llu\n\
-            Total string duplicates:      %llu\n\
-            Total hashtables allocated:   %llu\n\
-            Total unique vector sizes:    %llu\n",
-            vectors,
-            strdups,
-            hashtables,
-            vector_sizes
-        );
-        
-        for (; i < ST_SIZE; i++) {
-            size_entry_t *entry;
-            
-            if (!(entry = vector_usage[i]))
-                continue;
-            
-            con_out("                %2u| # of %4u byte vectors: %u\n",
-                (unsigned)e,
-                (unsigned)entry->key,
-                (unsigned)entry->value
-            );
-            e++;
-            
-            vectormem += entry->key * entry->value;
-        }
-
-        con_out("\
-            Total unique hashtable sizes: %llu\n",
-            hashtable_sizes
-        );
-        
-        for (i = 0, e = 1; i < ST_SIZE; i++) {
-            size_entry_t *entry;
-            
-            if (!(entry = hashtable_usage[i]))
-                continue;
-                
-            con_out("                %2u| # of %4u element hashtables: %u\n",
-                (unsigned)e,
-                (unsigned)entry->key,
-                (unsigned)entry->value
-            );
-            e++;
-        }
-        
-        con_out("            Total vector memory:          %f (MB)\n",
-            (float)(vectormem) / 1048576.0f
-        );
-    }
-
-    if (vector_usage)
-        util_st_del(vector_usage);
-    if (hashtable_usage)
-        util_st_del(hashtable_usage);
-}
-
-/*
- * Some string utility functions, because strdup uses malloc, and we want
- * to track all memory (without replacing malloc).
- */
-char *_util_Estrdup(const char *s, const char *file, size_t line) {
-    size_t  len = 0;
-    char   *ptr = NULL;
-
-    /* in case of -DNOTRACK */
-    (void)file;
-    (void)line;
-
-    if (!s)
-        return NULL;
-
-    if ((len = strlen(s)) && (ptr = (char*)mem_af(len+1, line, file))) {
-        memcpy(ptr, s, len);
-        ptr[len] = '\0';
-    }
-    strdups++;
-    return ptr;
-}
-
-char *_util_Estrdup_empty(const char *s, const char *file, size_t line) {
-    size_t  len = 0;
-    char   *ptr = NULL;
-
-    /* in case of -DNOTRACK */
-    (void)file;
-    (void)line;
-
-    if (!s)
-        return NULL;
-
-    len = strlen(s);
-    if ((ptr = (char*)mem_af(len+1, line, file))) {
-        memcpy(ptr, s, len);
-        ptr[len] = '\0';
-    }
-    strdups++;
-    return ptr;
-}
-
 void util_debug(const char *area, const char *ms, ...) {
     va_list  va;
     if (!OPTS_OPTION_BOOL(OPTION_DEBUG))
@@ -544,274 +212,6 @@ size_t util_strtononcmd(const char *in, char *out, size_t outsz) {
     return sz-1;
 }
 
-/* TODO: rewrite ... when I redo the ve cleanup */
-void _util_vec_grow(void **a, size_t i, size_t s) {
-    vector_t     *d = vec_meta(*a);
-    size_t        m = 0;
-    size_entry_t *e = NULL;
-    void         *p = NULL;
-    
-    if (*a) {
-        m = 2 * d->allocated + i;
-        p = mem_r(d, s * m + sizeof(vector_t));
-    } else {
-        m = i + 1;
-        p = mem_a(s * m + sizeof(vector_t));
-        ((vector_t*)p)->used = 0;
-        vectors++;
-    }
-    
-    if (!vector_usage)
-        vector_usage = util_st_new();
-
-    if ((e = util_st_get(vector_usage, s))) {
-        e->value ++;
-    } else {
-        util_st_put(vector_usage, s, 1); /* start off with 1 */
-        vector_sizes++;
-    }
-
-    *a = (vector_t*)p + 1;
-    vec_meta(*a)->allocated = m;
-}
-
-/*
- * Hash table for generic data, based on dynamic memory allocations
- * all around.  This is the internal interface, please look for
- * EXPOSED INTERFACE comment below
- */
-typedef struct hash_node_t {
-    char               *key;   /* the key for this node in table */
-    void               *value; /* pointer to the data as void*   */
-    struct hash_node_t *next;  /* next node (linked list)        */
-} hash_node_t;
-
-GMQCC_INLINE size_t util_hthash(hash_table_t *ht, const char *key) {
-    const uint32_t       mix   = 0x5BD1E995;
-    const uint32_t       rot   = 24;
-    size_t               size  = strlen(key);
-    uint32_t             hash  = 0x1EF0 /* LICRC TAB */  ^ size;
-    uint32_t             alias = 0;
-    const unsigned char *data  = (const unsigned char*)key;
-
-    while (size >= 4) {
-        alias  = (data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24));
-        alias *= mix;
-        alias ^= alias >> rot;
-        alias *= mix;
-
-        hash  *= mix;
-        hash  ^= alias;
-
-        data += 4;
-        size -= 4;
-    }
-
-    switch (size) {
-        case 3: hash ^= data[2] << 16;
-        case 2: hash ^= data[1] << 8;
-        case 1: hash ^= data[0];
-                hash *= mix;
-    }
-
-    hash ^= hash >> 13;
-    hash *= mix;
-    hash ^= hash >> 15;
-
-    return (size_t) (hash % ht->size);
-}
-
-static hash_node_t *_util_htnewpair(const char *key, void *value) {
-    hash_node_t *node;
-    if (!(node = (hash_node_t*)mem_a(sizeof(hash_node_t))))
-        return NULL;
-
-    if (!(node->key = util_strdupe(key))) {
-        mem_d(node);
-        return NULL;
-    }
-
-    node->value = value;
-    node->next  = NULL;
-
-    return node;
-}
-
-/*
- * EXPOSED INTERFACE for the hashtable implementation
- * util_htnew(size)                             -- to make a new hashtable
- * util_htset(table, key, value, sizeof(value)) -- to set something in the table
- * util_htget(table, key)                       -- to get something from the table
- * util_htdel(table)                            -- to delete the table
- */
-hash_table_t *util_htnew(size_t size) {
-    hash_table_t *hashtable = NULL;
-    size_entry_t *find;
-    
-    if (size < 1)
-        return NULL;
-        
-    if (!hashtable_usage)
-        hashtable_usage = util_st_new();
-
-    if (!(hashtable = (hash_table_t*)mem_a(sizeof(hash_table_t))))
-        return NULL;
-
-    if (!(hashtable->table = (hash_node_t**)mem_a(sizeof(hash_node_t*) * size))) {
-        mem_d(hashtable);
-        return NULL;
-    }
-    
-    if ((find = util_st_get(hashtable_usage, size)))
-        find->value++;
-    else {
-        hashtable_sizes++;
-        util_st_put(hashtable_usage, size, 1);
-    }
-
-    hashtable->size = size;
-    memset(hashtable->table, 0, sizeof(hash_node_t*) * size);
-
-    hashtables++;
-    return hashtable;
-}
-
-void util_htseth(hash_table_t *ht, const char *key, size_t bin, void *value) {
-    hash_node_t *newnode = NULL;
-    hash_node_t *next    = NULL;
-    hash_node_t *last    = NULL;
-
-    next = ht->table[bin];
-
-    while (next && next->key && strcmp(key, next->key) > 0)
-        last = next, next = next->next;
-
-    /* already in table, do a replace */
-    if (next && next->key && strcmp(key, next->key) == 0) {
-        next->value = value;
-    } else {
-        /* not found, grow a pair man :P */
-        newnode = _util_htnewpair(key, value);
-        if (next == ht->table[bin]) {
-            newnode->next  = next;
-            ht->table[bin] = newnode;
-        } else if (!next) {
-            last->next = newnode;
-        } else {
-            newnode->next = next;
-            last->next = newnode;
-        }
-    }
-}
-
-void util_htset(hash_table_t *ht, const char *key, void *value) {
-    util_htseth(ht, key, util_hthash(ht, key), value);
-}
-
-void *util_htgeth(hash_table_t *ht, const char *key, size_t bin) {
-    hash_node_t *pair = ht->table[bin];
-
-    while (pair && pair->key && strcmp(key, pair->key) > 0)
-        pair = pair->next;
-
-    if (!pair || !pair->key || strcmp(key, pair->key) != 0)
-        return NULL;
-
-    return pair->value;
-}
-
-void *util_htget(hash_table_t *ht, const char *key) {
-    return util_htgeth(ht, key, util_hthash(ht, key));
-}
-
-void *code_util_str_htgeth(hash_table_t *ht, const char *key, size_t bin) {
-    hash_node_t *pair;
-    size_t len, keylen;
-    int cmp;
-
-    keylen = strlen(key);
-
-    pair = ht->table[bin];
-    while (pair && pair->key) {
-        len = strlen(pair->key);
-        if (len < keylen) {
-            pair = pair->next;
-            continue;
-        }
-        if (keylen == len) {
-            cmp = strcmp(key, pair->key);
-            if (cmp == 0)
-                return pair->value;
-            if (cmp < 0)
-                return NULL;
-            pair = pair->next;
-            continue;
-        }
-        cmp = strcmp(key, pair->key + len - keylen);
-        if (cmp == 0) {
-            uintptr_t up = (uintptr_t)pair->value;
-            up += len - keylen;
-            return (void*)up;
-        }
-        pair = pair->next;
-    }
-    return NULL;
-}
-
-/*
- * Free all allocated data in a hashtable, this is quite the amount
- * of work.
- */
-void util_htrem(hash_table_t *ht, void (*callback)(void *data)) {
-    size_t i = 0;
-    for (; i < ht->size; i++) {
-        hash_node_t *n = ht->table[i];
-        hash_node_t *p;
-
-        /* free in list */
-        while (n) {
-            if (n->key)
-                mem_d(n->key);
-            if (callback)
-                callback(n->value);
-            p = n;
-            n = n->next;
-            mem_d(p);
-        }
-
-    }
-    /* free table */
-    mem_d(ht->table);
-    mem_d(ht);
-}
-
-void util_htrmh(hash_table_t *ht, const char *key, size_t bin, void (*cb)(void*)) {
-    hash_node_t **pair = &ht->table[bin];
-    hash_node_t *tmp;
-
-    while (*pair && (*pair)->key && strcmp(key, (*pair)->key) > 0)
-        pair = &(*pair)->next;
-
-    tmp = *pair;
-    if (!tmp || !tmp->key || strcmp(key, tmp->key) != 0)
-        return;
-
-    if (cb)
-        (*cb)(tmp->value);
-
-    *pair = tmp->next;
-    mem_d(tmp->key);
-    mem_d(tmp);
-}
-
-void util_htrm(hash_table_t *ht, const char *key, void (*cb)(void*)) {
-    util_htrmh(ht, key, util_hthash(ht, key), cb);
-}
-
-void util_htdel(hash_table_t *ht) {
-    util_htrem(ht, NULL);
-}
-
 /*
  * Portable implementation of vasprintf/asprintf. Assumes vsnprintf
  * exists, otherwise compiler error.