/* =========================================================================== Doom 3 GPL Source Code Copyright (C) 1999-2011 id Software LLC, a ZeniMax Media company. This file is part of the Doom 3 GPL Source Code ("Doom 3 Source Code"). Doom 3 Source Code is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. Doom 3 Source Code is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with Doom 3 Source Code. If not, see . In addition, the Doom 3 Source Code is also subject to certain additional terms. You should have received a copy of these additional terms immediately following the terms and conditions of the GNU General Public License which accompanied the Doom 3 Source Code. If not, please request a copy in writing from id Software at the address below. If you have questions concerning this license or the applicable additional terms, you may contact in writing id Software LLC, c/o ZeniMax Media Inc., Suite 120, Rockville, Maryland 20850 USA. =========================================================================== */ #include "sys/platform.h" #include "idlib/math/Simd_3DNow.h" //=============================================================== // // 3DNow! implementation of idSIMDProcessor // //=============================================================== #if defined(_MSC_VER) && defined(_M_IX86) /* ============ idSIMD_3DNow::GetName ============ */ const char * idSIMD_3DNow::GetName( void ) const { return "MMX & 3DNow!"; } // Very optimized memcpy() routine for all AMD Athlon and Duron family. // This code uses any of FOUR different basic copy methods, depending // on the transfer size. // NOTE: Since this code uses MOVNTQ (also known as "Non-Temporal MOV" or // "Streaming Store"), and also uses the software prefetchnta instructions, // be sure you're running on Athlon/Duron or other recent CPU before calling! #define TINY_BLOCK_COPY 64 // upper limit for movsd type copy // The smallest copy uses the X86 "movsd" instruction, in an optimized // form which is an "unrolled loop". #define IN_CACHE_COPY 64 * 1024 // upper limit for movq/movq copy w/SW prefetch // Next is a copy that uses the MMX registers to copy 8 bytes at a time, // also using the "unrolled loop" optimization. This code uses // the software prefetch instruction to get the data into the cache. #define UNCACHED_COPY 197 * 1024 // upper limit for movq/movntq w/SW prefetch // For larger blocks, which will spill beyond the cache, it's faster to // use the Streaming Store instruction MOVNTQ. This write instruction // bypasses the cache and writes straight to main memory. This code also // uses the software prefetch instruction to pre-read the data. // USE 64 * 1024 FOR THIS VALUE IF YOU'RE ALWAYS FILLING A "CLEAN CACHE" #define BLOCK_PREFETCH_COPY infinity // no limit for movq/movntq w/block prefetch #define CACHEBLOCK 80h // number of 64-byte blocks (cache lines) for block prefetch // For the largest size blocks, a special technique called Block Prefetch // can be used to accelerate the read operations. Block Prefetch reads // one address per cache line, for a series of cache lines, in a short loop. // This is faster than using software prefetch. The technique is great for // getting maximum read bandwidth, especially in DDR memory systems. /* ================ idSIMD_3DNow::Memcpy optimized memory copy routine that handles all alignment cases and block sizes efficiently ================ */ void VPCALL idSIMD_3DNow::Memcpy( void *dest, const void *src, const int n ) { __asm { mov ecx, [n] // number of bytes to copy mov edi, [dest] // destination mov esi, [src] // source mov ebx, ecx // keep a copy of count cld cmp ecx, TINY_BLOCK_COPY jb $memcpy_ic_3 // tiny? skip mmx copy cmp ecx, 32*1024 // don't align between 32k-64k because jbe $memcpy_do_align // it appears to be slower cmp ecx, 64*1024 jbe $memcpy_align_done $memcpy_do_align: mov ecx, 8 // a trick that's faster than rep movsb... sub ecx, edi // align destination to qword and ecx, 111b // get the low bits sub ebx, ecx // update copy count neg ecx // set up to jump into the array add ecx, offset $memcpy_align_done jmp ecx // jump to array of movsb's align 4 movsb movsb movsb movsb movsb movsb movsb movsb $memcpy_align_done: // destination is dword aligned mov ecx, ebx // number of bytes left to copy shr ecx, 6 // get 64-byte block count jz $memcpy_ic_2 // finish the last few bytes cmp ecx, IN_CACHE_COPY/64 // too big 4 cache? use uncached copy jae $memcpy_uc_test // This is small block copy that uses the MMX registers to copy 8 bytes // at a time. It uses the "unrolled loop" optimization, and also uses // the software prefetch instruction to get the data into the cache. align 16 $memcpy_ic_1: // 64-byte block copies, in-cache copy prefetchnta [esi + (200*64/34+192)] // start reading ahead movq mm0, [esi+0] // read 64 bits movq mm1, [esi+8] movq [edi+0], mm0 // write 64 bits movq [edi+8], mm1 // note: the normal movq writes the movq mm2, [esi+16] // data to cache; a cache line will be movq mm3, [esi+24] // allocated as needed, to store the data movq [edi+16], mm2 movq [edi+24], mm3 movq mm0, [esi+32] movq mm1, [esi+40] movq [edi+32], mm0 movq [edi+40], mm1 movq mm2, [esi+48] movq mm3, [esi+56] movq [edi+48], mm2 movq [edi+56], mm3 add esi, 64 // update source pointer add edi, 64 // update destination pointer dec ecx // count down jnz $memcpy_ic_1 // last 64-byte block? $memcpy_ic_2: mov ecx, ebx // has valid low 6 bits of the byte count $memcpy_ic_3: shr ecx, 2 // dword count and ecx, 1111b // only look at the "remainder" bits neg ecx // set up to jump into the array add ecx, offset $memcpy_last_few jmp ecx // jump to array of movsd's $memcpy_uc_test: cmp ecx, UNCACHED_COPY/64 // big enough? use block prefetch copy jae $memcpy_bp_1 $memcpy_64_test: or ecx, ecx // tail end of block prefetch will jump here jz $memcpy_ic_2 // no more 64-byte blocks left // For larger blocks, which will spill beyond the cache, it's faster to // use the Streaming Store instruction MOVNTQ. This write instruction // bypasses the cache and writes straight to main memory. This code also // uses the software prefetch instruction to pre-read the data. align 16 $memcpy_uc_1: // 64-byte blocks, uncached copy prefetchnta [esi + (200*64/34+192)] // start reading ahead movq mm0,[esi+0] // read 64 bits add edi,64 // update destination pointer movq mm1,[esi+8] add esi,64 // update source pointer movq mm2,[esi-48] movntq [edi-64], mm0 // write 64 bits, bypassing the cache movq mm0,[esi-40] // note: movntq also prevents the CPU movntq [edi-56], mm1 // from READING the destination address movq mm1,[esi-32] // into the cache, only to be over-written movntq [edi-48], mm2 // so that also helps performance movq mm2,[esi-24] movntq [edi-40], mm0 movq mm0,[esi-16] movntq [edi-32], mm1 movq mm1,[esi-8] movntq [edi-24], mm2 movntq [edi-16], mm0 dec ecx movntq [edi-8], mm1 jnz $memcpy_uc_1 // last 64-byte block? jmp $memcpy_ic_2 // almost done // For the largest size blocks, a special technique called Block Prefetch // can be used to accelerate the read operations. Block Prefetch reads // one address per cache line, for a series of cache lines, in a short loop. // This is faster than using software prefetch, in this case. // The technique is great for getting maximum read bandwidth, // especially in DDR memory systems. $memcpy_bp_1: // large blocks, block prefetch copy cmp ecx, CACHEBLOCK // big enough to run another prefetch loop? jl $memcpy_64_test // no, back to regular uncached copy mov eax, CACHEBLOCK / 2 // block prefetch loop, unrolled 2X add esi, CACHEBLOCK * 64 // move to the top of the block align 16 $memcpy_bp_2: mov edx, [esi-64] // grab one address per cache line mov edx, [esi-128] // grab one address per cache line sub esi, 128 // go reverse order dec eax // count down the cache lines jnz $memcpy_bp_2 // keep grabbing more lines into cache mov eax, CACHEBLOCK // now that it's in cache, do the copy align 16 $memcpy_bp_3: movq mm0, [esi ] // read 64 bits movq mm1, [esi+ 8] movq mm2, [esi+16] movq mm3, [esi+24] movq mm4, [esi+32] movq mm5, [esi+40] movq mm6, [esi+48] movq mm7, [esi+56] add esi, 64 // update source pointer movntq [edi ], mm0 // write 64 bits, bypassing cache movntq [edi+ 8], mm1 // note: movntq also prevents the CPU movntq [edi+16], mm2 // from READING the destination address movntq [edi+24], mm3 // into the cache, only to be over-written, movntq [edi+32], mm4 // so that also helps performance movntq [edi+40], mm5 movntq [edi+48], mm6 movntq [edi+56], mm7 add edi, 64 // update dest pointer dec eax // count down jnz $memcpy_bp_3 // keep copying sub ecx, CACHEBLOCK // update the 64-byte block count jmp $memcpy_bp_1 // keep processing chunks // The smallest copy uses the X86 "movsd" instruction, in an optimized // form which is an "unrolled loop". Then it handles the last few bytes. align 4 movsd movsd // perform last 1-15 dword copies movsd movsd movsd movsd movsd movsd movsd movsd // perform last 1-7 dword copies movsd movsd movsd movsd movsd movsd $memcpy_last_few: // dword aligned from before movsd's mov ecx, ebx // has valid low 2 bits of the byte count and ecx, 11b // the last few cows must come home jz $memcpy_final // no more, let's leave rep movsb // the last 1, 2, or 3 bytes $memcpy_final: emms // clean up the MMX state sfence // flush the write buffer mov eax, [dest] // ret value = destination pointer } } #endif /* _MSC_VER */