[scene] Start work on a full-on ECS system

That does feel a little redundant, but I think the System in ECS is
referring to the systems that run on the components, while the other
system is the support code for the ECS. Anyway...

This is based heavily on the information provided by @skipjack in his
github blog about EnTT. Currently, entity recycling and sparse arrays
for component pools have been implemented, and adding components to an
entity has been tested.

include/QF/scene/component.h

@@ -36,12 +36,22 @@
 
 #include "QF/qtypes.h"
 
-/** \defgroup entity Hierarchy management
+/** \defgroup component Entity Component System
 	\ingroup utils
 */
 ///@{
 
-#define null_transform (~0u)
+#define ENT_GROW 1024
+#define COMP_GROW 128
+#define ENT_IDBITS 20
+
+typedef struct ecs_pool_s {
+	uint32_t   *sparse;		// indexed by entity id, holds index to dense/data
+	uint32_t   *dense;		// holds entity id
+	void       *data;		// component data
+	uint32_t    count;		// number of components/dense entity ids
+	uint32_t    max_count;	// current capacity for components/entity ids
+} ecs_pool_t;
 
 typedef struct component_s {
 	size_t      size;
@@ -50,6 +60,17 @@ typedef struct component_s {
 	const char *name;
 } component_t;
 
+typedef struct ecs_registry_s {
+	uint32_t   *entities;
+	uint32_t    next;
+	uint32_t    available;
+	uint32_t    num_entities;
+	uint32_t    max_entities;
+	const component_t *components;
+	ecs_pool_t *comp_pools;
+	uint32_t    num_components;
+} ecs_registry_t;
+
 #define COMPINLINE GNU89INLINE inline
 
 COMPINLINE void Component_ResizeArray (const component_t *component,
@@ -64,6 +85,12 @@ COMPINLINE void Component_CopyElements (const component_t *component,
 COMPINLINE void Component_CreateElements (const component_t *component,
 										  void *array,
 										  uint32_t index, uint32_t count);
+COMPINLINE void Component_DestroyElements (const component_t *component,
+										   void *array,
+										   uint32_t index, uint32_t count);
+COMPINLINE uint32_t Ent_Index (uint32_t id);
+COMPINLINE uint32_t Ent_Generation (uint32_t id);
+COMPINLINE uint32_t Ent_NextGen(uint32_t id);
 
 #undef COMPINLINE
 #ifndef IMPLEMENT_COMPONENT_Funcs
@@ -72,16 +99,14 @@ COMPINLINE void Component_CreateElements (const component_t *component,
 #define COMPINLINE VISIBLE
 #endif
 
-COMPINLINE
-void
+COMPINLINE void
 Component_ResizeArray (const component_t *component,
 					   void **array, uint32_t count)
 {
 	*array = realloc (*array, count * component->size);
 }
 
-COMPINLINE
-void
+COMPINLINE void
 Component_MoveElements (const component_t *component,
 						void *array, uint32_t dstIndex, uint32_t srcIndex,
 						uint32_t count)
@@ -91,8 +116,7 @@ Component_MoveElements (const component_t *component,
 	memmove (dst, src, count * component->size);
 }
 
-COMPINLINE
-void
+COMPINLINE void
 Component_CopyElements (const component_t *component,
 						void *dstArray, uint32_t dstIndex,
 						const void *srcArray, uint32_t srcIndex,
@@ -103,8 +127,7 @@ Component_CopyElements (const component_t *component,
 	memcpy (dst, src, count * component->size);
 }
 
-COMPINLINE
-void
+COMPINLINE void
 Component_CreateElements (const component_t *component, void *array,
 						  uint32_t index, uint32_t count)
 {
@@ -119,6 +142,48 @@ Component_CreateElements (const component_t *component, void *array,
 	}
 }
 
+COMPINLINE void
+Component_DestroyElements (const component_t *component, void *array,
+						   uint32_t index, uint32_t count)
+{
+	if (component->destroy) {
+		for (uint32_t i = index; count-- > 0; i++) {
+			__auto_type dst = (byte *) array + i * component->size;
+			component->destroy (dst);
+		}
+	}
+}
+
+COMPINLINE uint32_t
+Ent_Index (uint32_t id)
+{
+	return id & ((1 << ENT_IDBITS) - 1);
+}
+
+COMPINLINE uint32_t
+Ent_Generation (uint32_t id)
+{
+	return id & ~((1 << ENT_IDBITS) - 1);
+}
+
+COMPINLINE uint32_t
+Ent_NextGen(uint32_t id)
+{
+	return id + (1 << ENT_IDBITS);
+}
+
+ecs_registry_t *ECS_NewRegistry (void);
+void ECS_DelRegistry (ecs_registry_t *registry);
+void ECS_RegisterComponents (ecs_registry_t *registry,
+							 const component_t *components, uint32_t count);
+
+uint32_t ECS_NewEntity (ecs_registry_t *registry);
+void ECS_DelEntity (ecs_registry_t *registry, uint32_t ent);
+
+void Ent_AddComponent (uint32_t ent, uint32_t comp, ecs_registry_t *registry);
+void Ent_RemoveComponent (uint32_t ent, uint32_t comp,
+						  ecs_registry_t *registry);
+
 ///@}
 
 #endif//__QF_scene_component_h

libs/scene/component.c

@@ -28,5 +28,133 @@
 # include "config.h"
 #endif
 
+#include "QF/sys.h"
+
 #define IMPLEMENT_COMPONENT_Funcs
 #include "QF/scene/component.h"
+
+VISIBLE ecs_registry_t *
+ECS_NewRegistry (void)
+{
+	ecs_registry_t *reg = calloc (1, sizeof (ecs_registry_t));
+	reg->next = Ent_Index (~0);
+	return reg;
+}
+
+VISIBLE void
+ECS_DelRegistry (ecs_registry_t *registry)
+{
+	free (registry->entities);
+	for (uint32_t i = 0; i < registry->num_components; i++) {
+		free (registry->comp_pools[i].sparse);
+		free (registry->comp_pools[i].dense);
+		free (registry->comp_pools[i].data);
+	}
+	free (registry->comp_pools);
+	free (registry);
+}
+
+VISIBLE void
+ECS_RegisterComponents (ecs_registry_t *registry,
+						const component_t *components, uint32_t count)
+{
+	registry->num_components = count;
+	registry->components = components;
+	registry->comp_pools = calloc (count, sizeof (ecs_pool_t));
+	size_t      size = registry->max_entities * sizeof (uint32_t);
+	for (uint32_t i = 0; i < registry->num_components; i++) {
+		registry->comp_pools[i].sparse = malloc (size);
+		memset (registry->comp_pools[i].sparse, ~0, size);
+	}
+}
+
+VISIBLE void
+Ent_AddComponent (uint32_t ent, uint32_t comp, ecs_registry_t *registry)
+{
+	uint32_t    id = Ent_Index (ent);
+	ecs_pool_t *pool = &registry->comp_pools[comp];
+	if (pool->sparse[id] < pool->count) {
+		//Sys_Error ("Ent_AddComponent: component %s already on entity %x\n",
+		//		   registry->components[i].name, ent);
+		return;
+	}
+	if (pool->count == pool->max_count) {
+		pool->max_count += COMP_GROW;
+		pool->dense = realloc (pool->dense,
+							   pool->max_count * sizeof (uint32_t));
+		Component_ResizeArray (&registry->components[comp], &pool->data,
+							   pool->max_count);
+	}
+	uint32_t    ind = pool->count++;
+	pool->sparse[id] = ind;
+	pool->dense[ind] = ent;
+	Component_CreateElements (&registry->components[comp], pool->data, ind, 1);
+}
+
+VISIBLE void
+Ent_RemoveComponent (uint32_t ent, uint32_t comp, ecs_registry_t *registry)
+{
+	uint32_t    id = Ent_Index (ent);
+	ecs_pool_t *pool = &registry->comp_pools[comp];
+	uint32_t    ind = pool->sparse[id];
+	if (ind < pool->count) {
+		uint32_t    last = pool->count - 1;
+		Component_DestroyElements (&registry->components[comp], pool->data,
+								   ind, 1);
+		if (last > ind) {
+			pool->sparse[Ent_Index (pool->dense[last])] = ind;
+			pool->dense[ind] = pool->dense[last];
+			Component_MoveElements (&registry->components[comp], pool->data,
+									ind, last, 1);
+		}
+		pool->count--;
+		pool->sparse[id] = ~0;
+	}
+}
+
+VISIBLE uint32_t
+ECS_NewEntity (ecs_registry_t *registry)
+{
+	uint32_t    ent;
+	if (registry->available) {
+		registry->available--;
+		uint32_t    next = registry->next;
+		ent = next | Ent_Generation (registry->entities[next]);
+		registry->next = Ent_Index (registry->entities[next]);
+		registry->entities[next] = ent;
+	} else {
+		if (registry->num_entities == Ent_Index (~0)) {
+			Sys_Error ("ECS_NewEntity: out of entities");
+		}
+		if (registry->num_entities == registry->max_entities) {
+			registry->max_entities += ENT_GROW;
+			size_t      size = registry->max_entities * sizeof (uint32_t);
+			registry->entities = realloc (registry->entities, size);
+			for (uint32_t i = 0; i < registry->num_components; i++) {
+				uint32_t   *sparse = registry->comp_pools[i].sparse;
+				sparse = realloc (sparse, size);
+				memset (sparse + registry->max_entities - ENT_GROW, ~0,
+						ENT_GROW * sizeof (uint32_t));
+				registry->comp_pools[i].sparse = sparse;
+			}
+		}
+		ent = registry->num_entities++;
+		// ent starts out with generation 0
+		registry->entities[ent] = ent;
+	}
+	return ent;
+}
+
+VISIBLE void
+ECS_DelEntity (ecs_registry_t *registry, uint32_t ent)
+{
+	uint32_t    next = registry->next | Ent_NextGen (Ent_Generation (ent));
+	uint32_t    id = Ent_Index (ent);
+	registry->entities[id] = next;
+	registry->next = id;
+	registry->available++;
+
+	for (uint32_t i = 0; i < registry->num_components; i++) {
+		Ent_RemoveComponent (ent, i, registry);
+	}
+}

libs/scene/test/Makemodule.am

@@ -1,5 +1,7 @@
 libs_scene_tests = \
-	libs/scene/test/test-hierarchy
+	libs/scene/test/test-components \
+	libs/scene/test/test-hierarchy \
+	libs/scene/test/test-registry
 
 TESTS += $(libs_scene_tests)
 
@@ -9,9 +11,23 @@ libs_scene_test_libs= \
 	libs/scene/libQFscene.la \
 	libs/util/libQFutil.la
 
+libs_scene_test_test_components_SOURCES= 		\
+	libs/scene/test/test-components.c
+libs_scene_test_test_components_LDADD=			\
+	$(libs_scene_test_libs)
+libs_scene_test_test_components_DEPENDENCIES=	\
+	$(libs_scene_test_libs)
+
 libs_scene_test_test_hierarchy_SOURCES= 		\
 	libs/scene/test/test-hierarchy.c
 libs_scene_test_test_hierarchy_LDADD=			\
 	$(libs_scene_test_libs)
 libs_scene_test_test_hierarchy_DEPENDENCIES=	\
 	$(libs_scene_test_libs)
+
+libs_scene_test_test_registry_SOURCES= 		\
+	libs/scene/test/test-registry.c
+libs_scene_test_test_registry_LDADD=			\
+	$(libs_scene_test_libs)
+libs_scene_test_test_registry_DEPENDENCIES=	\
+	$(libs_scene_test_libs)

libs/scene/test/test-components.c

@@ -0,0 +1,133 @@
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "QF/simd/types.h"
+#include "QF/mathlib.h"
+#include "QF/scene/component.h"
+
+enum test_components {
+	test_position,
+	test_scale,
+	test_rotation,
+
+	test_num_components
+};
+
+static void
+create_position (void *data)
+{
+	vec4f_t    *pos = data;
+	*pos = (vec4f_t) { 0, 0, 0, 1 };
+}
+
+static void
+create_scale (void *data)
+{
+	vec_t     *scale = data;
+	VectorSet (1, 1, 1, scale);
+}
+
+static void
+create_rotation (void *data)
+{
+	vec4f_t    *rot = data;
+	*rot = (vec4f_t) { 0, 0, 0, 1 };
+}
+
+static const component_t test_components[] = {
+	[test_position] = {
+		.size = sizeof (vec4f_t),
+		.create = create_position,
+		.name = "position",
+	},
+	[test_scale] = {
+		.size = sizeof (vec3_t),
+		.create = create_scale,
+		.name = "scale",
+	},
+	[test_rotation] = {
+		.size = sizeof (vec4f_t),
+		.create = create_rotation,
+		.name = "rotation",
+	},
+};
+
+static int
+check_ent_components (const uint32_t *ents, uint32_t count, uint32_t comp,
+					  ecs_registry_t *reg)
+{
+	ecs_pool_t *pool = &reg->comp_pools[comp];
+	const component_t *component = &reg->components[comp];
+	if (pool->count != count) {
+		printf ("%s pool has wrong object count: %d %d\n", component->name,
+				pool->count, count);
+		return 0;
+	}
+	uint32_t    num_entities = 0;
+	for (uint32_t i = 0; i < reg->max_entities; i++) {
+		if (pool->sparse[i] == ~0u) {
+			continue;
+		}
+		if (pool->sparse[i] < pool->count) {
+			num_entities++;
+			continue;
+		}
+		printf ("invalid index in sparse array of %s: %d %d\n", component->name,
+				pool->sparse[i], pool->count);
+		return 0;
+	}
+	if (num_entities != count) {
+		printf ("%s sparse has wrong entity count: %d %d\n", component->name,
+				num_entities, count);
+		return 0;
+	}
+
+	for (uint32_t i = 0; i < count; i++) {
+		if (pool->sparse[Ent_Index (ents[i])] == ~0u) {
+			printf ("ent not in %s sparse: %x\n", component->name, ents[i]);
+			return 0;
+		}
+		if (pool->dense[pool->sparse[Ent_Index (ents[i])]] != ents[i]) {
+			printf ("indexed %s dense does have ent: %x %x\n", component->name,
+					pool->dense[pool->sparse[Ent_Index (ents[i])]], ents[i]);
+			return 0;
+		}
+	}
+	return 1;
+}
+
+int
+main (void)
+{
+	ecs_registry_t *reg = ECS_NewRegistry ();
+	ECS_RegisterComponents (reg, test_components, test_num_components);
+	uint32_t    enta = ECS_NewEntity (reg);
+	uint32_t    entb = ECS_NewEntity (reg);
+	uint32_t    entc = ECS_NewEntity (reg);
+	Ent_AddComponent (enta, test_position, reg);
+	Ent_AddComponent (entb, test_position, reg);
+	Ent_AddComponent (entc, test_position, reg);
+	Ent_AddComponent (enta, test_rotation, reg);
+	Ent_AddComponent (entb, test_rotation, reg);
+	Ent_AddComponent (enta, test_scale, reg);
+
+	if (!check_ent_components ((uint32_t[]){enta, entb, entc}, 3,
+							   test_position, reg)) {
+		return 1;
+	}
+	if (!check_ent_components ((uint32_t[]){enta, entb}, 2,
+							   test_rotation, reg)) {
+		return 1;
+	}
+	if (!check_ent_components ((uint32_t[]){enta}, 1, test_scale, reg)) {
+		return 1;
+	}
+
+	ECS_DelRegistry (reg);
+	return 0;
+}

libs/scene/test/test-registry.c

@@ -0,0 +1,151 @@
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "QF/scene/component.h"
+
+static int
+test_new_del (void)
+{
+	ecs_registry_t *reg = ECS_NewRegistry ();
+	if (!reg) {
+		printf ("could not create registry\n");
+		return 0;
+	}
+	if (reg->max_entities && !reg->entities) {
+		printf ("Non-zero max_entities with null entities pointer\n");
+		return 0;
+	}
+	if (!reg->max_entities && reg->entities) {
+		printf ("Zero max_entities with non-null entities pointer\n");
+		return 0;
+	}
+	if (reg->num_entities > reg->max_entities) {
+		printf ("num_entities > max_entities\n");
+		return 0;
+	}
+	if (reg->num_entities) {
+		printf ("Fresh registry has entities\n");
+		return 0;
+	}
+	if (reg->available) {
+		printf ("Fresh registry has recycled entities\n");
+		return 0;
+	}
+	ECS_DelRegistry (reg);
+	return 1;
+}
+
+#define NUM_ENTS 5
+
+static int
+test_entities (void)
+{
+	ecs_registry_t *reg = ECS_NewRegistry ();
+
+	uint32_t    entities[NUM_ENTS];
+
+	for (int i = 0; i < NUM_ENTS; i++) {
+		entities[i] = ECS_NewEntity (reg);
+	}
+	if (!reg->max_entities || !reg->entities) {
+		printf ("Zero max_entities or non-null entities pointer\n");
+		return 0;
+	}
+	if (reg->available) {
+		printf ("unexpected recycled entities\n");
+		return 0;
+	}
+	if (reg->num_entities != NUM_ENTS) {
+		printf ("wrong number of entities in registry: %d vs %d\n", NUM_ENTS,
+				reg->num_entities);
+		return 0;
+	}
+	for (int i = 0; i < NUM_ENTS; i++) {
+		if (Ent_Index (entities[i]) > reg->num_entities) {
+			printf ("bad entity returned: %d > %d\n", Ent_Index (entities[i]),
+					reg->num_entities);
+			return 0;
+		}
+		if (Ent_Generation (entities[i])) {
+			printf ("fresh entity not generation 0\n");
+			return 0;
+		}
+		for (int j = 0; j < i; j++) {
+			if (Ent_Index (entities[j]) == Ent_Index (entities[i])) {
+				printf ("duplicate entity id\n");
+				return 0;
+			}
+		}
+		if (reg->entities[Ent_Index (entities[i])] != entities[i]) {
+			printf ("wrong entity in entities array: %d %d\n",
+					entities[i], reg->entities[Ent_Index (entities[i])]);
+			return 0;
+		}
+	}
+	ECS_DelEntity (reg, entities[2]);
+	ECS_DelEntity (reg, entities[0]);
+	if (reg->entities[Ent_Index (entities[2])] == entities[2]) {
+		printf ("deleted entity not deleted: %d %d\n",
+				entities[2], reg->entities[Ent_Index (entities[2])]);
+		return 0;
+	}
+	if (reg->entities[Ent_Index (entities[0])] == entities[0]) {
+		printf ("deleted entity not deleted: %d %d\n",
+				entities[0], reg->entities[Ent_Index (entities[0])]);
+		return 0;
+	}
+	if (reg->available != 2) {
+		printf ("not 2 available entity for recycling\n");
+		return 0;
+	}
+	for (uint32_t i = reg->next, c = 0; i != Ent_Index (~0);
+		 i = Ent_Index (reg->entities[i]), c++) {
+		if (c >= reg->available || i >= reg->num_entities) {
+			printf ("invalid deleted entity chain\n");
+			return 0;
+		}
+		if (Ent_Index (reg->entities[i]) == i) {
+			printf ("deleted entity points to itself\n");
+			return 0;
+		}
+		//printf ("%x\n", reg->entities[i]);
+	}
+	entities[2] = ECS_NewEntity (reg);
+	if (reg->available != 1) {
+		printf ("not 1 available entity for recycling\n");
+		return 0;
+	}
+	entities[0] = ECS_NewEntity (reg);
+	if (reg->available != 0) {
+		printf ("not 0 available entities for recycling\n");
+		return 0;
+	}
+	if (reg->num_entities != NUM_ENTS) {
+		printf ("wrong number of entities in registry: %d vs %d\n", NUM_ENTS,
+				reg->num_entities);
+		return 0;
+	}
+	if (!Ent_Generation (entities[2]) || !Ent_Generation (entities[0])) {
+		printf ("recycled entity generations not updated\n");
+		return 0;
+	}
+	//for (int i = 0; i < NUM_ENTS; i++) {
+	//	printf ("%x\n", entities[i]);
+	//}
+
+	ECS_DelRegistry (reg);
+	return 1;
+}
+
+int
+main (void)
+{
+	if (!test_new_del ()) { return 1; }
+	if (!test_entities ()) { return 1; }
+	return 0;
+}