From 544a1c0c1ab533ca09074b155a92cca783339d6f Mon Sep 17 00:00:00 2001 From: Thilo Schulz Date: Thu, 10 Feb 2011 18:45:28 +0000 Subject: [PATCH] - Use MAP_FAILED error code for check whether mmap() was successful - Replace various malloc() with Z_Malloc - Fix several memory leaks when VM compilation failed and Com_Error is called - Make failed mmap/VirtualAlloc/malloc calls fatal --- code/qcommon/vm_powerpc.c | 2 +- code/qcommon/vm_sparc.c | 2 +- code/qcommon/vm_x86.c | 33 +++++++++++++++++++----------- code/qcommon/vm_x86_64.c | 32 +++++++++++++++++++++-------- code/qcommon/vm_x86_64_assembler.c | 8 ++++---- 5 files changed, 50 insertions(+), 27 deletions(-) diff --git a/code/qcommon/vm_powerpc.c b/code/qcommon/vm_powerpc.c index 76bb984a..18a449f4 100644 --- a/code/qcommon/vm_powerpc.c +++ b/code/qcommon/vm_powerpc.c @@ -1837,7 +1837,7 @@ PPC_ComputeCode( vm_t *vm ) unsigned char *dataAndCode = mmap( NULL, codeLength, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0 ); - if ( ! dataAndCode ) + if (dataAndCode == MAP_FAILED) DIE( "Not enough memory" ); ppc_instruction_t *codeNow, *codeBegin; diff --git a/code/qcommon/vm_sparc.c b/code/qcommon/vm_sparc.c index 39f2202a..06119d56 100644 --- a/code/qcommon/vm_sparc.c +++ b/code/qcommon/vm_sparc.c @@ -1427,7 +1427,7 @@ static void sparc_compute_code(vm_t *vm, struct func_info * const fp) data_and_code = mmap(NULL, code_length, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); - if (!data_and_code) + if (data_and_code == MAP_FAILED) DIE("Not enough memory"); code_now = code_begin = (unsigned int *) diff --git a/code/qcommon/vm_x86.c b/code/qcommon/vm_x86.c index f3de65db..ba590635 100644 --- a/code/qcommon/vm_x86.c +++ b/code/qcommon/vm_x86.c @@ -53,6 +53,7 @@ static void VM_Destroy_Compiled(vm_t* self); */ +#define VMFREE_BUFFERS() {Z_Free(buf); Z_Free(jused);} static byte *buf = NULL; static byte *jused = NULL; static int compiledOfs = 0; @@ -289,6 +290,7 @@ static int Hex( int c ) { return c - '0'; } + VMFREE_BUFFERS(); Com_Error( ERR_DROP, "Hex: bad char '%c'", c ); return 0; @@ -408,6 +410,7 @@ qboolean EmitMovEBXEDI(vm_t *vm, int andit) { #define JUSED(x) \ do { \ if (x < 0 || x >= jusedSize) { \ + VMFREE_BUFFERS(); \ Com_Error( ERR_DROP, \ "VM_CompileX86: jump target out of range at offset %d", pc ); \ } \ @@ -429,7 +432,7 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) { // allocate a very large temp buffer, we will shrink it later maxLength = header->codeLength * 8; - buf = Z_Malloc( maxLength ); + buf = Z_Malloc(maxLength); jused = Z_Malloc(jusedSize); Com_Memset(jused, 0, jusedSize); @@ -454,16 +457,21 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) { LastCommand = LAST_COMMAND_NONE; - while ( instruction < header->instructionCount ) { - if ( compiledOfs > maxLength - 16 ) { - Com_Error( ERR_FATAL, "VM_CompileX86: maxLength exceeded" ); + while(instruction < header->instructionCount) + { + if(compiledOfs > maxLength - 16) + { + VMFREE_BUFFERS(); + Com_Error(ERR_DROP, "VM_CompileX86: maxLength exceeded"); } vm->instructionPointers[ instruction ] = compiledOfs; instruction++; - if ( pc > header->codeLength ) { - Com_Error( ERR_FATAL, "VM_CompileX86: pc > header->codeLength" ); + if(pc > header->codeLength) + { + VMFREE_BUFFERS(); + Com_Error(ERR_DROP, "VM_CompileX86: pc > header->codeLength"); } op = code[ pc ]; @@ -1075,7 +1083,8 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) { Emit4( (int)vm->instructionPointers ); break; default: - Com_Error( ERR_DROP, "VM_CompileX86: bad opcode %i at offset %i", op, pc ); + VMFREE_BUFFERS(); + Com_Error(ERR_DROP, "VM_CompileX86: bad opcode %i at offset %i", op, pc); } pop0 = pop1; pop1 = op; @@ -1086,13 +1095,13 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) { vm->codeLength = compiledOfs; #ifdef VM_X86_MMAP vm->codeBase = mmap(NULL, compiledOfs, PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); - if(vm->codeBase == (void*)-1) - Com_Error(ERR_DROP, "VM_CompileX86: can't mmap memory"); + if(vm->codeBase == MAP_FAILED) + Com_Error(ERR_FATAL, "VM_CompileX86: can't mmap memory"); #elif _WIN32 // allocate memory with EXECUTE permissions under windows. vm->codeBase = VirtualAlloc(NULL, compiledOfs, MEM_COMMIT, PAGE_EXECUTE_READWRITE); if(!vm->codeBase) - Com_Error(ERR_DROP, "VM_CompileX86: VirtualAlloc failed"); + Com_Error(ERR_FATAL, "VM_CompileX86: VirtualAlloc failed"); #else vm->codeBase = malloc(compiledOfs); #endif @@ -1101,14 +1110,14 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) { #ifdef VM_X86_MMAP if(mprotect(vm->codeBase, compiledOfs, PROT_READ|PROT_EXEC)) - Com_Error(ERR_DROP, "VM_CompileX86: mprotect failed"); + Com_Error(ERR_FATAL, "VM_CompileX86: mprotect failed"); #elif _WIN32 { DWORD oldProtect = 0; // remove write permissions. if(!VirtualProtect(vm->codeBase, compiledOfs, PAGE_EXECUTE_READ, &oldProtect)) - Com_Error(ERR_DROP, "VM_CompileX86: VirtualProtect failed"); + Com_Error(ERR_FATAL, "VM_CompileX86: VirtualProtect failed"); } #endif diff --git a/code/qcommon/vm_x86_64.c b/code/qcommon/vm_x86_64.c index 2e23f4a9..4c824c1b 100644 --- a/code/qcommon/vm_x86_64.c +++ b/code/qcommon/vm_x86_64.c @@ -54,6 +54,8 @@ static FILE* qdasmout; #define Dfprintf(args...) #endif +#define VM_FREEBUFFERS(vm) {assembler_init(0); VM_Destroy_Compiled(vm);} + void assembler_set_output(char* buf); size_t assembler_get_code_size(void); void assembler_init(int pass); @@ -251,6 +253,7 @@ void emit(const char* fmt, ...) #define CHECK_INSTR(nr) \ do { if(nr < 0 || nr >= header->instructionCount) { \ + VM_FREEBUFFERS(vm); \ Com_Error( ERR_DROP, \ "%s: jump target 0x%x out of range at offset %d", __func__, nr, pc ); \ } } while(0) @@ -429,6 +432,8 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) { // const optimization unsigned got_const = 0, const_value = 0; + + vm->codeBase = NULL; gettimeofday(&tvstart, NULL); @@ -441,15 +446,17 @@ void VM_Compile( vm_t *vm, vmHeader_t *header ) { #ifdef VM_X86_64_MMAP vm->codeBase = mmap(NULL, compiledOfs, PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); - if(vm->codeBase == (void*)-1) - Com_Error(ERR_DROP, "VM_CompileX86: can't mmap memory"); + if(vm->codeBase == MAP_FAILED) + Com_Error(ERR_FATAL, "VM_CompileX86_64: can't mmap memory"); #elif __WIN64__ // allocate memory with write permissions under windows. vm->codeBase = VirtualAlloc(NULL, compiledOfs, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); if(!vm->codeBase) - Com_Error(ERR_DROP, "VM_CompileX86: VirtualAlloc failed"); + Com_Error(ERR_FATAL, "VM_CompileX86_64: VirtualAlloc failed"); #else vm->codeBase = malloc(compiledOfs); + if(!vm_codeBase) + Com_Error(ERR_FATAL, "VM_CompileX86_64: Failed to allocate memory"); #endif assembler_set_output((char*)vm->codeBase); @@ -930,7 +937,9 @@ emit_do_syscall: } - if(got_const) { + if(got_const) + { + VM_FREEBUFFERS(vm); Com_Error(ERR_DROP, "leftover const\n"); } @@ -943,14 +952,14 @@ emit_do_syscall: #ifdef VM_X86_64_MMAP if(mprotect(vm->codeBase, compiledOfs, PROT_READ|PROT_EXEC)) - Com_Error(ERR_DROP, "VM_CompileX86: mprotect failed"); + Com_Error(ERR_FATAL, "VM_CompileX86_64: mprotect failed"); #elif __WIN64__ { DWORD oldProtect = 0; // remove write permissions; give exec permision if(!VirtualProtect(vm->codeBase, compiledOfs, PAGE_EXECUTE_READ, &oldProtect)) - Com_Error(ERR_DROP, "VM_CompileX86: VirtualProtect failed"); + Com_Error(ERR_FATAL, "VM_CompileX86_64: VirtualProtect failed"); } #endif @@ -987,11 +996,16 @@ emit_do_syscall: void VM_Destroy_Compiled(vm_t* self) { -#ifdef _WIN32 - VirtualFree(self->codeBase, 0, MEM_RELEASE); + if(self && self->codeBase) + { +#ifdef VM_X86_64_MMAP + munmap(self->codeBase, self->codeLength); +#elif __WIN64__ + VirtualFree(self->codeBase, 0, MEM_RELEASE); #else - munmap(self->codeBase, self->codeLength); + free(self->codeBase); #endif + } } /* diff --git a/code/qcommon/vm_x86_64_assembler.c b/code/qcommon/vm_x86_64_assembler.c index 9c6c5c69..a8abb075 100644 --- a/code/qcommon/vm_x86_64_assembler.c +++ b/code/qcommon/vm_x86_64_assembler.c @@ -244,10 +244,10 @@ static void hash_add_label(const char* label, unsigned address) int labellen; i %= sizeof(labelhash)/sizeof(labelhash[0]); - h = malloc(sizeof(struct hashentry)); + h = Z_Malloc(sizeof(struct hashentry)); labellen = strlen(label) + 1; - h->label = malloc(labellen); + h->label = Z_Malloc(labellen); memcpy(h->label, label, labellen); h->address = address; @@ -282,8 +282,8 @@ static void labelhash_free(void) while(h) { struct hashentry* next = h->next; - free(h->label); - free(h); + Z_Free(h->label); + Z_Free(h); h = next; ++n; }