use mmap to allocate memory for generated code to be able to set PROT_EXEC

code/qcommon/vm.c

@@ -615,6 +615,9 @@ VM_Free
 */
 void VM_Free( vm_t *vm ) {
 
+	if(vm->destroy)
+		vm->destroy(vm);
+
 	if ( vm->dllHandle ) {
 		Sys_UnloadDll( vm->dllHandle );
 		Com_Memset( vm, 0, sizeof( *vm ) );

code/qcommon/vm_x86.c

@@ -31,6 +31,14 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #include <sys/mman.h> // for PROT_ stuff
 #endif
 
+/* need this on NX enabled systems (i386 with PAE kernel or
+ * noexec32=on x86_64) */
+#ifdef __linux__
+#define VM_X86_MMAP
+#endif
+
+static void VM_Destroy_Compiled(vm_t* self);
+
 /*
 
   eax	scratch
@@ -1069,34 +1077,40 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) {
 
 	// copy to an exact size buffer on the hunk
 	vm->codeLength = compiledOfs;
-	vm->codeBase = Hunk_Alloc( compiledOfs, h_low );
+#ifdef VM_X86_MMAP
+	vm->codeBase = mmap(NULL, compiledOfs, PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0);
+	if(vm->codeBase == (void*)-1)
+		Com_Error(ERR_DROP, "VM_CompileX86: can't mmap memory");
+#else
+	vm->codeBase = malloc(compiledOfs);
+#endif
+
 	Com_Memcpy( vm->codeBase, buf, compiledOfs );
+
+#ifdef VM_X86_MMAP
+	if(mprotect(vm->codeBase, compiledOfs, PROT_READ|PROT_EXEC))
+		Com_Error(ERR_DROP, "VM_CompileX86: mprotect failed");
+#endif
+
 	Z_Free( buf );
 	Z_Free( jused );
 	Com_Printf( "VM file %s compiled to %i bytes of code\n", vm->name, compiledOfs );
 
+	vm->destroy = VM_Destroy_Compiled;
+
 	// offset all the instruction pointers for the new location
 	for ( i = 0 ; i < header->instructionCount ; i++ ) {
 		vm->instructionPointers[i] += (int)vm->codeBase;
 	}
+}
 
-#if 0 // ndef _WIN32
-	// Must make the newly generated code executable
-	{
-		int r;
-		unsigned long addr;
-		int psize = getpagesize();
-
-		addr = ((int)vm->codeBase & ~(psize-1)) - psize;
-
-		r = mprotect((char*)addr, vm->codeLength + (int)vm->codeBase - addr + psize, 
-			PROT_READ | PROT_WRITE | PROT_EXEC );
-
-		if (r < 0)
-			Com_Error( ERR_FATAL, "mprotect failed to change PROT_EXEC" );
-	}
+void VM_Destroy_Compiled(vm_t* self)
+{
+#ifdef VM_X86_MMAP
+	munmap(self->codeBase, self->codeLength);
+#else
+	free(self->codeBase);
 #endif
-
 }
 
 /*

code/qcommon/vm_x86_64.c

@@ -39,6 +39,8 @@ static FILE* qdasmout;
 #define Dfprintf(args...)
 #endif
 
+static void VM_Destroy_Compiled(vm_t* self);
+
 /*
 
   eax	scratch
@@ -308,10 +310,8 @@ out:
 
 static int doas(char* in, char* out, unsigned char** compiledcode)
 {
-	char* buf;
-	char* mem;
-	size_t size = -1, allocsize;
-	int ps;
+	unsigned char* mem;
+	size_t size = -1;
 	pid_t pid;
 
 	Com_Printf("running assembler < %s > %s\n", in, out);
@@ -358,42 +358,16 @@ static int doas(char* in, char* out, unsigned char** compiledcode)
 
 	Com_Printf("done\n");
 
-	mem = mmapfile(out, &size);
+	mem = (unsigned char*)mmapfile(out, &size);
 	if(!mem)
 	{
 		Com_Printf(S_COLOR_RED "can't mmap object file %s: %s\n", out, strerror(errno));
 		return -1;
 	}
 
-	ps = sysconf(_SC_PAGE_SIZE);
-	if(ps == -1)
-	{
-		Com_Printf(S_COLOR_RED "can't determine page size: %s\n", strerror(errno));
-		return -1;
-	}
+	*compiledcode = mem;
 
-	--ps;
-
-	allocsize = (size+ps)&~ps;
-	buf = Hunk_Alloc(allocsize, h_high);
-
-	buf = (void*)(((unsigned long)buf+ps)&~ps);
-
-	memcpy(buf, mem, size);
-
-	munmap(mem, 0);
-
-	if((*compiledcode = (unsigned char*)buf))
-	{
-		// need to be able to exec code
-		if(mprotect(buf, allocsize, PROT_READ|PROT_WRITE|PROT_EXEC) == -1)
-		{
-			Com_Error(ERR_FATAL, "mprotect failed on %p+%x: %s\n", buf, allocsize, strerror(errno));
-		}
-		return size;
-	}
-
-	return -1;
+	return size;
 }
 
 static void block_copy_vm(unsigned dest, unsigned src, unsigned count)
@@ -897,6 +871,8 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) {
 
 	vm->codeBase   = compiledcode; // remember to skip ELF header!
 	vm->codeLength = compiledsize;
+
+	vm->destroy = VM_Destroy_Compiled;
 	
 	entryPoint = getentrypoint(vm);
 
@@ -930,6 +906,12 @@ out:
 	}
 }
 
+
+void VM_Destroy_Compiled(vm_t* self)
+{
+	munmap(self->codeBase, self->codeLength);
+}
+
 /*
 ==============
 VM_CallCompiled