[ecs] Add support for subpools

Subpools are for grouping components by some criterion. Any component
that has a rangeid callback will be grouped with other components that
return the same render id. Note that the ordering of components within a
group will be affected by adding a component into a group that comes
before that group (or removing a component).

Component pools can have multiple groups, added and removed dynamically,
but removing a group should (currently) be done only when empty.

include/QF/ecs.h

@@ -48,6 +48,7 @@
 
 #define ENT_GROW 1024
 #define COMP_GROW 128
+#define RANGE_GROW 32
 #define ENT_IDBITS 20
 #define nullent (~0u)
 
@@ -61,14 +62,29 @@ typedef struct ecs_pool_s {
 
 typedef struct DARRAY_TYPE(component_t) componentset_t;
 
+typedef struct ecs_range_s {
+	uint32_t    end;
+} ecs_range_t;
+
+typedef struct ecs_subpool_s {
+	ecs_range_t *ranges;
+	uint32_t   *rangeids;
+	uint32_t   *sorted;
+	uint32_t    next;
+	uint32_t    available;
+	uint32_t    num_ranges;
+	uint32_t    max_ranges;
+} ecs_subpool_t;
+
 typedef struct ecs_registry_s {
+	ecs_pool_t *comp_pools;
 	uint32_t   *entities;
+	ecs_subpool_t *subpools;
 	uint32_t    next;
 	uint32_t    available;
 	uint32_t    num_entities;
 	uint32_t    max_entities;
 	componentset_t components;
-	ecs_pool_t *comp_pools;
 	PR_RESMAP (hierarchy_t) hierarchies;//FIXME find a better way
 } ecs_registry_t;
 
@@ -101,6 +117,10 @@ uint32_t ECS_NewEntity (ecs_registry_t *registry);
 void ECS_DelEntity (ecs_registry_t *registry, uint32_t ent);
 void ECS_RemoveEntities (ecs_registry_t *registry, uint32_t component);
 
+uint32_t ECS_NewSubpoolRange (ecs_registry_t *registry, uint32_t component);
+void ECS_DelSubpoolRange (ecs_registry_t *registry, uint32_t component,
+						  uint32_t id);
+
 #undef ECSINLINE
 
 ///@}

include/QF/ecs/component.h

@@ -42,10 +42,14 @@
 */
 ///@{
 
+struct ecs_registry_s;
 typedef struct component_s {
 	size_t      size;
 	void      (*create) (void *);
 	void      (*destroy) (void *);
+	// comp is the registry component id (base + system component id)
+	uint32_t  (*rangeid) (struct ecs_registry_s *reg, uint32_t ent,
+						  uint32_t comp);
 	const char *name;
 } component_t;
 

include/QF/ecs/hierarchy.h

@@ -51,7 +51,6 @@ typedef struct hierref_s {
 } hierref_t;
 
 typedef struct hierarchy_s {
-	struct ecs_registry_s *reg;
 	uint32_t    num_objects;
 	uint32_t    max_objects;
 	uint32_t   *ent;
@@ -60,6 +59,7 @@ typedef struct hierarchy_s {
 	uint32_t   *parentIndex;
 	const hierarchy_type_t *type;
 	void      **components;
+	struct ecs_registry_s *reg;
 	uint32_t    href_comp;
 } hierarchy_t;
 

include/QF/scene/scene.h

@@ -49,9 +49,6 @@ enum scene_components {
 	scene_old_origin,	//XXX FIXME XXX should not be here
 	scene_colormap,
 
-	scene_canvas,
-	scene_vref,			//view hierarchy reference
-
 	//FIXME these should probably be private to the sw renderer (and in a
 	//group, which needs to be implemented), but need to sort out a good
 	//scheme for semi-dynamic components

libs/ecs/Makemodule.am

@@ -11,4 +11,5 @@ libs_ecs_libQFecs_la_SOURCES= \
 	libs/ecs/component.c \
 	libs/ecs/ecs.c \
 	libs/ecs/entity.c \
-	libs/ecs/hierarchy.c
+	libs/ecs/hierarchy.c \
+	libs/ecs/subpool.c

libs/ecs/ecs.c

@@ -51,7 +51,11 @@ ECS_DelRegistry (ecs_registry_t *registry)
 		free (registry->comp_pools[i].sparse);
 		free (registry->comp_pools[i].dense);
 		free (registry->comp_pools[i].data);
+
+		free (registry->subpools[i].ranges);
+		free (registry->subpools[i].rangeids);
 	}
+	free (registry->subpools);
 	free (registry->comp_pools);
 	free (registry);
 }
@@ -77,6 +81,7 @@ ECS_CreateComponentPools (ecs_registry_t *registry)
 		registry->comp_pools[i].sparse = malloc (size);
 		memset (registry->comp_pools[i].sparse, nullent, size);
 	}
+	registry->subpools = calloc (count, sizeof (ecs_subpool_t));
 }
 
 typedef struct {

libs/ecs/entity.c

@@ -34,10 +34,21 @@
 #define IMPLEMENT_ECS_ENTITY_Funcs
 #include "QF/ecs.h"
 
+#include "compat.h"
+
+static void swap_inds (uint32_t *a, uint32_t *b)
+{
+	uint32_t    t = *a;
+	*a = *b;
+	*b = t;
+}
+
 VISIBLE void *
 Ent_AddComponent (uint32_t ent, uint32_t comp, ecs_registry_t *registry)
 {
 	ecs_pool_t *pool = &registry->comp_pools[comp];
+	ecs_subpool_t *subpool = &registry->subpools[comp];
+	component_t *c = &registry->components.a[comp];
 	uint32_t    id = Ent_Index (ent);
 	uint32_t    ind = pool->sparse[id];
 	if (ind >= pool->count || pool->dense[ind] != ent) {
@@ -45,31 +56,85 @@ Ent_AddComponent (uint32_t ent, uint32_t comp, ecs_registry_t *registry)
 			pool->max_count += COMP_GROW;
 			pool->dense = realloc (pool->dense,
 								   pool->max_count * sizeof (uint32_t));
-			Component_ResizeArray (&registry->components.a[comp], &pool->data,
+			Component_ResizeArray (c, &pool->data, pool->max_count);
-								   pool->max_count);
 		}
 		uint32_t    ind = pool->count++;
 		pool->sparse[id] = ind;
 		pool->dense[ind] = ent;
+		uint32_t    rind = subpool->num_ranges - subpool->available;
+		if (rind && c->rangeid) {
+			uint32_t    rangeid = c->rangeid (registry, ent, comp);
+			uint32_t    rangeind = subpool->sorted[Ent_Index (rangeid)];
+			printf ("ent:%d rangeid:%d rangeind:%d\n", ent, rangeid, rangeind);
+			while (rind-- > rangeind) {
+				if (subpool->ranges[rind].end == ind) {
+					subpool->ranges[rind].end++;
+					continue;
+				}
+				uint32_t    end = subpool->ranges[rind].end++;
+				Component_MoveElements (c, pool->data, ind, end, 1);
+				swap_inds (&pool->sparse[Ent_Index (pool->dense[end])],
+						   &pool->sparse[Ent_Index (pool->dense[ind])]);
+				swap_inds (&pool->dense[ind], &pool->dense[end]);
+				ind = end;
+			}
+		}
 	}
 	return Ent_GetComponent (ent, comp, registry);
 }
 
+static int
+range_cmp (const void *_key, const void *_range, void *_subpool)
+{
+	const uint32_t *key = _key;
+	const ecs_range_t *range = _range;
+	ecs_subpool_t *subpool = _subpool;
+
+	if (*key >= range->end) {
+		return -1;
+	}
+	if (range - subpool->ranges > 0) {
+		return *key >= range[-1].end ? 0 : -1;
+	}
+	return 0;
+}
+
+static ecs_range_t *
+find_range (ecs_subpool_t *subpool, uint32_t ind)
+{
+	return bsearch_r (&ind, subpool->ranges,
+					  subpool->num_ranges - subpool->available,
+					  sizeof (ecs_range_t), range_cmp, subpool);
+}
+
 VISIBLE void
 Ent_RemoveComponent (uint32_t ent, uint32_t comp, ecs_registry_t *registry)
 {
 	uint32_t    id = Ent_Index (ent);
 	ecs_pool_t *pool = &registry->comp_pools[comp];
+	ecs_subpool_t *subpool = &registry->subpools[comp];
 	uint32_t    ind = pool->sparse[id];
+	component_t *c = &registry->components.a[comp];
 	if (ind < pool->count && pool->dense[ind] == ent) {
 		uint32_t    last = pool->count - 1;
-		Component_DestroyElements (&registry->components.a[comp], pool->data,
+		Component_DestroyElements (c, pool->data,
 								   ind, 1);
+		if (subpool->num_ranges - subpool->available) {
+			uint32_t    range_count = subpool->num_ranges - subpool->available;
+			ecs_range_t *range = find_range (subpool, ind);
+			while (range - subpool->ranges < range_count) {
+				uint32_t    end = --range->end;
+				range++;
+				pool->sparse[Ent_Index (pool->dense[end])] = ind;
+				pool->dense[ind] = pool->dense[end];
+				Component_MoveElements (c, pool->data, ind, end, 1);
+				ind = end;
+			}
+		}
 		if (last > ind) {
 			pool->sparse[Ent_Index (pool->dense[last])] = ind;
 			pool->dense[ind] = pool->dense[last];
-			Component_MoveElements (&registry->components.a[comp], pool->data,
+			Component_MoveElements (c, pool->data, ind, last, 1);
-									ind, last, 1);
 		}
 		pool->count--;
 		pool->sparse[id] = nullent;

libs/ecs/subpool.c

@@ -0,0 +1,92 @@
+/*
+	subpool.c
+
+	ECS subpool management
+
+	Copyright (C) 2022 Bill Currke
+
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation; either version 2
+	of the License, or (at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+	See the GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to:
+
+		Free Software Foundation, Inc.
+		59 Temple Place - Suite 330
+		Boston, MA  02111-1307, USA
+
+*/
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include "QF/sys.h"
+
+#include "QF/ecs.h"
+
+VISIBLE uint32_t
+ECS_NewSubpoolRange (ecs_registry_t *registry, uint32_t component)
+{
+	ecs_subpool_t *subpool = &registry->subpools[component];
+	uint32_t    id;
+	uint32_t    num_ranges = subpool->num_ranges - subpool->available;
+	if (subpool->available) {
+		subpool->available--;
+		uint32_t    next = subpool->next;
+		id = next | Ent_Generation (subpool->rangeids[next]);
+		subpool->next = Ent_Index (subpool->rangeids[next]);
+		subpool->rangeids[next] = id;
+		subpool->sorted[next] = num_ranges;
+	} else {
+		if (subpool->num_ranges == Ent_Index (nullent)) {
+			Sys_Error ("ECS_NewEntity: out of rangeids");
+		}
+		if (subpool->num_ranges == subpool->max_ranges) {
+			subpool->max_ranges += RANGE_GROW;
+			size_t      idsize = subpool->max_ranges * sizeof (uint32_t);
+			size_t      rsize = subpool->max_ranges * sizeof (ecs_range_t);
+			subpool->rangeids = realloc (subpool->rangeids, idsize);
+			subpool->sorted = realloc (subpool->sorted, idsize);
+			subpool->ranges = realloc (subpool->ranges, rsize);
+		}
+		id = subpool->num_ranges++;
+		// id starts out with generation 0
+		subpool->rangeids[id] = id;
+		subpool->sorted[id] = num_ranges;
+	}
+	uint32_t    end = 0;
+	if (num_ranges) {
+		end = subpool->ranges[num_ranges - 1].end;
+	}
+	subpool->ranges[Ent_Index (id)] = (ecs_range_t) {
+		.end = end,
+	};
+	return id;
+}
+
+VISIBLE void
+ECS_DelSubpoolRange (ecs_registry_t *registry, uint32_t component, uint32_t id)
+{
+	ecs_subpool_t *subpool = &registry->subpools[component];
+	uint32_t    next = subpool->next | Ent_NextGen (Ent_Generation (id));
+	uint32_t    ind = Ent_Index (id);
+	uint32_t    count = subpool->num_ranges - subpool->available;
+	subpool->rangeids[ind] = next;
+	subpool->next = ind;
+	subpool->available++;
+	memmove (subpool->ranges + ind, subpool->ranges + ind + 1,
+			 (count - 1 - ind) * sizeof (ecs_range_t));
+	for (uint32_t i = 0; i < count; i++) {
+		if (subpool->sorted[i] > ind) {
+			subpool->sorted[i]--;
+		}
+	}
+}

libs/ecs/test/Makemodule.am

@@ -1,7 +1,8 @@
 libs_ecs_tests = \
 	libs/ecs/test/test-components \
 	libs/ecs/test/test-hierarchy \
-	libs/ecs/test/test-registry
+	libs/ecs/test/test-registry \
+	libs/ecs/test/test-subpools
 
 TESTS += $(libs_ecs_tests)
 
@@ -31,3 +32,10 @@ libs_ecs_test_test_registry_LDADD=			\
 	$(libs_ecs_test_libs)
 libs_ecs_test_test_registry_DEPENDENCIES=	\
 	$(libs_ecs_test_libs)
+
+libs_ecs_test_test_subpools_SOURCES= 		\
+	libs/ecs/test/test-subpools.c
+libs_ecs_test_test_subpools_LDADD=			\
+	$(libs_ecs_test_libs)
+libs_ecs_test_test_subpools_DEPENDENCIES=	\
+	$(libs_ecs_test_libs)

libs/ecs/test/test-subpools.c

@@ -0,0 +1,256 @@
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "QF/simd/types.h"
+#include "QF/mathlib.h"
+#include "QF/ecs.h"
+
+enum test_components {
+	test_subpool,
+	test_obj,
+
+	test_num_components
+};
+
+static uint32_t
+obj_rangeid (ecs_registry_t *reg, uint32_t ent, uint32_t comp)
+{
+	uint32_t    sp_comp = comp - (test_obj - test_subpool);
+	return *(uint32_t *) Ent_GetComponent (ent, sp_comp, reg);
+}
+
+static const component_t test_components[] = {
+	[test_subpool] = {
+		.size = sizeof (uint32_t),
+		.name = "subpool",
+	},
+	[test_obj] = {
+		.size = sizeof (uint32_t),
+		.name = "obj",
+		.rangeid = obj_rangeid,
+	},
+};
+
+static void
+dump_sp_ids (ecs_registry_t *reg, uint32_t comp)
+{
+	ecs_pool_t *pool = &reg->comp_pools[comp];
+	uint32_t   *ent = pool->dense;
+	uint32_t   *id = pool->data;
+
+	for (uint32_t i = 0; i < pool->count; i++) {
+		printf ("ent[%d]: %d, %d\n", i, ent[i], id[i]);
+	}
+}
+
+static int
+check_subpool_ranges (ecs_subpool_t *subpool, uint32_t *expect)
+{
+	uint32_t    count = subpool->num_ranges - subpool->available;
+	ecs_range_t *range = subpool->ranges;
+
+	while (count--) {
+		ecs_range_t *r = range++;
+		uint32_t     e = *expect++;
+		printf ("%d: %d %d\n", (int)(r - subpool->ranges), r->end, e);
+		if (r->end != e++) {
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static int
+check_obj_comps (ecs_registry_t *reg, uint32_t comp, uint32_t *expect)
+{
+	ecs_pool_t *pool = &reg->comp_pools[comp];
+	uint32_t   *val = pool->data;
+	int         fail = 0;
+
+	for (uint32_t i = 0; i < pool->count; i++) {
+		printf ("val[%d]: %d %d\n", i, val[i], expect[i]);
+		if (val[i] != expect[i]) {
+			fail = 1;
+		}
+	}
+	return fail;
+}
+
+int
+main (void)
+{
+	ecs_registry_t *reg = ECS_NewRegistry ();
+	uint32_t base = ECS_RegisterComponents (reg, test_components,
+											test_num_components);
+	ECS_CreateComponentPools (reg);
+
+	uint32_t    sp1 = ECS_NewSubpoolRange (reg, base + test_obj);
+	uint32_t    sp2 = ECS_NewSubpoolRange (reg, base + test_obj);
+	uint32_t    sp3 = ECS_NewSubpoolRange (reg, base + test_obj);
+
+	printf ("%d %d %d\n", sp1, sp2, sp3);
+	if (reg->subpools[base + test_subpool].num_ranges != 0
+		|| reg->subpools[base + test_subpool].available != 0) {
+		printf ("subpool not 0 count: %d %d\n",
+				reg->subpools[base + test_subpool].num_ranges,
+				reg->subpools[base + test_subpool].available);
+		return 1;
+	}
+	if (reg->subpools[base + test_obj].num_ranges != 3
+		|| reg->subpools[base + test_obj].available != 0) {
+		printf ("obj not 3 count: %d %d\n",
+				reg->subpools[base + test_obj].num_ranges,
+				reg->subpools[base + test_obj].available);
+		return 1;
+	}
+	for (uint32_t i = 0; i < reg->subpools[base + test_obj].num_ranges; i++) {
+		if (reg->subpools[base + test_obj].ranges[i].end != 0) {
+			printf ("end %d not 0 count: %d\n", i,
+					reg->subpools[base + test_obj].ranges[i].end);
+			return 1;
+		}
+	}
+
+	uint32_t    enta = ECS_NewEntity (reg);
+	uint32_t    entb = ECS_NewEntity (reg);
+	uint32_t    entc = ECS_NewEntity (reg);
+	uint32_t    entd = ECS_NewEntity (reg);
+	uint32_t    ente = ECS_NewEntity (reg);
+	uint32_t    entf = ECS_NewEntity (reg);
+	uint32_t    entg = ECS_NewEntity (reg);
+	uint32_t    enth = ECS_NewEntity (reg);
+
+	Ent_SetComponent (enta, base + test_subpool, reg, &sp1);
+	Ent_SetComponent (entb, base + test_subpool, reg, &sp1);
+	Ent_SetComponent (entc, base + test_subpool, reg, &sp2);
+	Ent_SetComponent (entd, base + test_subpool, reg, &sp3);
+	Ent_SetComponent (ente, base + test_subpool, reg, &sp3);
+	Ent_SetComponent (entf, base + test_subpool, reg, &sp2);
+	Ent_SetComponent (entg, base + test_subpool, reg, &sp2);
+	Ent_SetComponent (enth, base + test_subpool, reg, &sp1);
+
+	dump_sp_ids (reg, base + test_subpool);
+	if (check_subpool_ranges (&reg->subpools[base + test_obj],
+							  (uint32_t[]) { 0, 0, 0 })) {
+		printf ("oops\n");
+		return 1;
+	}
+
+	uint32_t val = 0;
+	Ent_SetComponent (enta, base + test_obj, reg, &val); val++;
+	Ent_SetComponent (entb, base + test_obj, reg, &val); val++;
+	Ent_SetComponent (entc, base + test_obj, reg, &val); val++;
+	Ent_SetComponent (entd, base + test_obj, reg, &val); val++;
+	Ent_SetComponent (ente, base + test_obj, reg, &val); val++;
+	Ent_SetComponent (entf, base + test_obj, reg, &val); val++;
+	Ent_SetComponent (entg, base + test_obj, reg, &val); val++;
+	Ent_SetComponent (enth, base + test_obj, reg, &val); val++;
+
+	if (check_subpool_ranges (&reg->subpools[base + test_obj],
+							  (uint32_t[]) { 3, 6, 8 })) {
+		printf ("oops\n");
+		return 1;
+	}
+	if (check_obj_comps (reg, base + test_obj,
+						 (uint32_t[]) { 0, 1, 7, 5, 6, 2, 4, 3 })) {
+		printf ("oops\n");
+		return 1;
+	}
+
+	Ent_RemoveComponent (entb, base + test_obj, reg);
+	if (check_subpool_ranges (&reg->subpools[base + test_obj],
+							  (uint32_t[]) { 2, 5, 7 })) {
+		printf ("oops\n");
+		return 1;
+	}
+	if (check_obj_comps (reg, base + test_obj,
+						 (uint32_t[]) { 0, 7, 2, 5, 6, 3, 4 })) {
+		printf ("oops\n");
+		return 1;
+	}
+
+	Ent_RemoveComponent (entd, base + test_obj, reg);
+	if (check_subpool_ranges (&reg->subpools[base + test_obj],
+							  (uint32_t[]) { 2, 5, 6 })) {
+		printf ("oops\n");
+		return 1;
+	}
+	if (check_obj_comps (reg, base + test_obj,
+						 (uint32_t[]) { 0, 7, 2, 5, 6, 4 })) {
+		printf ("oops\n");
+		return 1;
+	}
+
+	Ent_RemoveComponent (ente, base + test_obj, reg);
+	if (check_subpool_ranges (&reg->subpools[base + test_obj],
+							  (uint32_t[]) { 2, 5, 5 })) {
+		printf ("oops\n");
+		return 1;
+	}
+	if (check_obj_comps (reg, base + test_obj,
+						 (uint32_t[]) { 0, 7, 2, 5, 6 })) {
+		printf ("oops\n");
+		return 1;
+	}
+
+	Ent_SetComponent (entd, base + test_obj, reg, &val); val++;
+	Ent_SetComponent (ente, base + test_obj, reg, &val); val++;
+	if (check_subpool_ranges (&reg->subpools[base + test_obj],
+							  (uint32_t[]) { 2, 5, 7 })) {
+		printf ("oops\n");
+		return 1;
+	}
+	if (check_obj_comps (reg, base + test_obj,
+						 (uint32_t[]) { 0, 7, 2, 5, 6, 8, 9 })) {
+		printf ("oops\n");
+		return 1;
+	}
+
+	Ent_RemoveComponent (entc, base + test_obj, reg);
+	Ent_RemoveComponent (entf, base + test_obj, reg);
+	Ent_RemoveComponent (entg, base + test_obj, reg);
+	if (check_subpool_ranges (&reg->subpools[base + test_obj],
+							  (uint32_t[]) { 2, 2, 4 })) {
+		printf ("oops\n");
+		return 1;
+	}
+	if (check_obj_comps (reg, base + test_obj,
+						 (uint32_t[]) { 0, 7, 9, 8 })) {
+		printf ("oops\n");
+		return 1;
+	}
+
+	ECS_DelSubpoolRange (reg, base + test_obj, sp2);
+	if (check_subpool_ranges (&reg->subpools[base + test_obj],
+							  (uint32_t[]) { 2, 4 })) {
+		printf ("oops\n");
+		return 1;
+	}
+
+	sp2 = ECS_NewSubpoolRange (reg, base + test_obj);
+	printf ("sp2: %d\n", sp2);
+	Ent_SetComponent (entc, base + test_subpool, reg, &sp2);
+	Ent_SetComponent (entf, base + test_subpool, reg, &sp2);
+	Ent_SetComponent (entg, base + test_subpool, reg, &sp2);
+	Ent_SetComponent (entc, base + test_obj, reg, &val); val++;
+	Ent_SetComponent (entf, base + test_obj, reg, &val); val++;
+	Ent_SetComponent (entg, base + test_obj, reg, &val); val++;
+	if (check_subpool_ranges (&reg->subpools[base + test_obj],
+							  (uint32_t[]) { 2, 4, 7})) {
+		printf ("oops\n");
+		return 1;
+	}
+	if (check_obj_comps (reg, base + test_obj,
+						 (uint32_t[]) { 0, 7, 9, 8, 10, 11, 12 })) {
+		printf ("oops\n");
+		return 1;
+	}
+
+	ECS_DelRegistry (reg);
+	return 0;
+}