- Use MAP_FAILED error code for check whether mmap() was successful

- Replace various malloc() with Z_Malloc
- Fix several memory leaks when VM compilation failed and Com_Error is called
- Make failed mmap/VirtualAlloc/malloc calls fatal

code/qcommon/vm_powerpc.c

@@ -1837,7 +1837,7 @@ PPC_ComputeCode( vm_t *vm )
 	unsigned char *dataAndCode = mmap( NULL, codeLength,
 		PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0 );
 
-	if ( ! dataAndCode )
+	if (dataAndCode == MAP_FAILED)
 		DIE( "Not enough memory" );
 
 	ppc_instruction_t *codeNow, *codeBegin;

code/qcommon/vm_sparc.c

@@ -1427,7 +1427,7 @@ static void sparc_compute_code(vm_t *vm, struct func_info * const fp)
 
 	data_and_code = mmap(NULL, code_length, PROT_READ | PROT_WRITE,
 			     MAP_SHARED | MAP_ANONYMOUS, -1, 0);
-	if (!data_and_code)
+	if (data_and_code == MAP_FAILED)
 		DIE("Not enough memory");
 
 	code_now = code_begin = (unsigned int *)

code/qcommon/vm_x86.c

@@ -53,6 +53,7 @@ static void VM_Destroy_Compiled(vm_t* self);
 
 */
 
+#define VMFREE_BUFFERS() {Z_Free(buf); Z_Free(jused);}
 static	byte	*buf = NULL;
 static	byte	*jused = NULL;
 static	int		compiledOfs = 0;
@@ -289,6 +290,7 @@ static int Hex( int c ) {
 		return c - '0';
 	}
 
+	VMFREE_BUFFERS();
 	Com_Error( ERR_DROP, "Hex: bad char '%c'", c );
 
 	return 0;
@@ -408,6 +410,7 @@ qboolean EmitMovEBXEDI(vm_t *vm, int andit) {
 #define JUSED(x) \
 	do { \
 		if (x < 0 || x >= jusedSize) { \
+		        VMFREE_BUFFERS(); \
 			Com_Error( ERR_DROP, \
 					"VM_CompileX86: jump target out of range at offset %d", pc ); \
 		} \
@@ -454,16 +457,21 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) {
 
 	LastCommand = LAST_COMMAND_NONE;
 
-	while ( instruction < header->instructionCount ) {
-		if ( compiledOfs > maxLength - 16 ) {
-			Com_Error( ERR_FATAL, "VM_CompileX86: maxLength exceeded" );
+	while(instruction < header->instructionCount)
+	{
+		if(compiledOfs > maxLength - 16)
+		{
+	        	VMFREE_BUFFERS();
+			Com_Error(ERR_DROP, "VM_CompileX86: maxLength exceeded");
 		}
 
 		vm->instructionPointers[ instruction ] = compiledOfs;
 		instruction++;
 
-		if ( pc > header->codeLength ) {
-			Com_Error( ERR_FATAL, "VM_CompileX86: pc > header->codeLength" );
+		if(pc > header->codeLength)
+		{
+		        VMFREE_BUFFERS();
+			Com_Error(ERR_DROP, "VM_CompileX86: pc > header->codeLength");
 		}
 
 		op = code[ pc ];
@@ -1075,6 +1083,7 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) {
 			Emit4( (int)vm->instructionPointers );
 			break;
 		default:
+		        VMFREE_BUFFERS();
 			Com_Error(ERR_DROP, "VM_CompileX86: bad opcode %i at offset %i", op, pc);
 		}
 		pop0 = pop1;
@@ -1086,13 +1095,13 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) {
 	vm->codeLength = compiledOfs;
 #ifdef VM_X86_MMAP
 	vm->codeBase = mmap(NULL, compiledOfs, PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0);
-	if(vm->codeBase == (void*)-1)
-		Com_Error(ERR_DROP, "VM_CompileX86: can't mmap memory");
+	if(vm->codeBase == MAP_FAILED)
+		Com_Error(ERR_FATAL, "VM_CompileX86: can't mmap memory");
 #elif _WIN32
 	// allocate memory with EXECUTE permissions under windows.
 	vm->codeBase = VirtualAlloc(NULL, compiledOfs, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
 	if(!vm->codeBase)
-		Com_Error(ERR_DROP, "VM_CompileX86: VirtualAlloc failed");
+		Com_Error(ERR_FATAL, "VM_CompileX86: VirtualAlloc failed");
 #else
 	vm->codeBase = malloc(compiledOfs);
 #endif
@@ -1101,14 +1110,14 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) {
 
 #ifdef VM_X86_MMAP
 	if(mprotect(vm->codeBase, compiledOfs, PROT_READ|PROT_EXEC))
-		Com_Error(ERR_DROP, "VM_CompileX86: mprotect failed");
+		Com_Error(ERR_FATAL, "VM_CompileX86: mprotect failed");
 #elif _WIN32
 	{
 		DWORD oldProtect = 0;
 		
 		// remove write permissions.
 		if(!VirtualProtect(vm->codeBase, compiledOfs, PAGE_EXECUTE_READ, &oldProtect))
-			Com_Error(ERR_DROP, "VM_CompileX86: VirtualProtect failed");
+			Com_Error(ERR_FATAL, "VM_CompileX86: VirtualProtect failed");
 	}
 #endif
 

code/qcommon/vm_x86_64.c

@@ -54,6 +54,8 @@ static FILE* qdasmout;
 #define Dfprintf(args...)
 #endif
 
+#define VM_FREEBUFFERS(vm) {assembler_init(0); VM_Destroy_Compiled(vm);}
+
 void assembler_set_output(char* buf);
 size_t assembler_get_code_size(void);
 void assembler_init(int pass);
@@ -251,6 +253,7 @@ void emit(const char* fmt, ...)
 
 #define CHECK_INSTR(nr) \
 	do { if(nr < 0 || nr >= header->instructionCount) { \
+		VM_FREEBUFFERS(vm); \
 		Com_Error( ERR_DROP, \
 			"%s: jump target 0x%x out of range at offset %d", __func__, nr, pc ); \
 	} } while(0)
@@ -430,6 +433,8 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) {
 	// const optimization
 	unsigned got_const = 0, const_value = 0;
 	
+	vm->codeBase = NULL;
+
 	gettimeofday(&tvstart, NULL);
 
 	for (pass = 0; pass < 2; ++pass) {
@@ -441,15 +446,17 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) {
 
 		#ifdef VM_X86_64_MMAP
 			vm->codeBase = mmap(NULL, compiledOfs, PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0);
-			if(vm->codeBase == (void*)-1)
-				Com_Error(ERR_DROP, "VM_CompileX86: can't mmap memory");
+			if(vm->codeBase == MAP_FAILED)
+				Com_Error(ERR_FATAL, "VM_CompileX86_64: can't mmap memory");
 		#elif __WIN64__
 			// allocate memory with write permissions under windows.
 			vm->codeBase = VirtualAlloc(NULL, compiledOfs, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
 			if(!vm->codeBase)
-				Com_Error(ERR_DROP, "VM_CompileX86: VirtualAlloc failed");
+				Com_Error(ERR_FATAL, "VM_CompileX86_64: VirtualAlloc failed");
 		#else
 			vm->codeBase = malloc(compiledOfs);
+			if(!vm_codeBase)
+				Com_Error(ERR_FATAL, "VM_CompileX86_64: Failed to allocate memory");
 		#endif
 
 		assembler_set_output((char*)vm->codeBase);
@@ -930,7 +937,9 @@ emit_do_syscall:
 
 	}
 
-	if(got_const) {
+	if(got_const)
+	{
+		VM_FREEBUFFERS(vm);
 		Com_Error(ERR_DROP, "leftover const\n");
 	}
 
@@ -943,14 +952,14 @@ emit_do_syscall:
 
 	#ifdef VM_X86_64_MMAP
 		if(mprotect(vm->codeBase, compiledOfs, PROT_READ|PROT_EXEC))
-			Com_Error(ERR_DROP, "VM_CompileX86: mprotect failed");
+			Com_Error(ERR_FATAL, "VM_CompileX86_64: mprotect failed");
 	#elif __WIN64__
 		{
 			DWORD oldProtect = 0;
 			
 			// remove write permissions; give exec permision
 			if(!VirtualProtect(vm->codeBase, compiledOfs, PAGE_EXECUTE_READ, &oldProtect))
-				Com_Error(ERR_DROP, "VM_CompileX86: VirtualProtect failed");
+				Com_Error(ERR_FATAL, "VM_CompileX86_64: VirtualProtect failed");
 		}
 	#endif
 
@@ -987,12 +996,17 @@ emit_do_syscall:
 
 void VM_Destroy_Compiled(vm_t* self)
 {
-#ifdef _WIN32
+	if(self && self->codeBase)
+	{
+#ifdef VM_X86_64_MMAP
+		munmap(self->codeBase, self->codeLength);
+#elif __WIN64__
 		VirtualFree(self->codeBase, 0, MEM_RELEASE);
 #else
-	munmap(self->codeBase, self->codeLength);
+		free(self->codeBase);
 #endif
 	}
+}
 
 /*
 ==============

code/qcommon/vm_x86_64_assembler.c

@@ -244,10 +244,10 @@ static void hash_add_label(const char* label, unsigned address)
 	int labellen;
 	
 	i %= sizeof(labelhash)/sizeof(labelhash[0]);
-	h = malloc(sizeof(struct hashentry));
+	h = Z_Malloc(sizeof(struct hashentry));
 	
 	labellen = strlen(label) + 1;
-	h->label = malloc(labellen);
+	h->label = Z_Malloc(labellen);
 	memcpy(h->label, label, labellen);
 	
 	h->address = address;
@@ -282,8 +282,8 @@ static void labelhash_free(void)
 		while(h)
 		{
 			struct hashentry* next = h->next;
-			free(h->label);
-			free(h);
+			Z_Free(h->label);
+			Z_Free(h);
 			h = next;
 			++n;
 		}