From 922f9eddefd28b928beddbb95f32493b4579e627 Mon Sep 17 00:00:00 2001 From: Stephen Saunders Date: Fri, 18 Feb 2022 12:57:29 -0500 Subject: [PATCH 1/9] Cinematic Video and Audio memory management fixes (cherry picked from commit 40a9190283d30180a3b7e4ab9b77f3c3728dad4f) --- neo/renderer/Cinematic.cpp | 8 +++++--- neo/sound/OpenAL/AL_CinematicAudio.cpp | 6 +++--- neo/sound/XAudio2/XA2_CinematicAudio.cpp | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/neo/renderer/Cinematic.cpp b/neo/renderer/Cinematic.cpp index e4242b14..c142b84f 100644 --- a/neo/renderer/Cinematic.cpp +++ b/neo/renderer/Cinematic.cpp @@ -790,7 +790,9 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping ) frameRate = av_q2d( fmt_ctx->streams[video_stream_index]->avg_frame_rate ); common->Printf( "Loaded FFMPEG file: '%s', looping=%d, %dx%d, %f FPS, %f sec\n", qpath, looping, CIN_WIDTH, CIN_HEIGHT, frameRate, durationSec ); - image = ( byte* )Mem_Alloc( CIN_WIDTH * CIN_HEIGHT * 4 * 2, TAG_CINEMATIC ); + // SRS - Get number of image bytes needed by querying with NULL first, then allocate image and fill with correct parameters + int img_bytes = av_image_fill_arrays( frame2->data, frame2->linesize, NULL, AV_PIX_FMT_BGR32, CIN_WIDTH, CIN_HEIGHT, 1 ); + image = ( byte* )Mem_Alloc( img_bytes, TAG_CINEMATIC ); av_image_fill_arrays( frame2->data, frame2->linesize, image, AV_PIX_FMT_BGR32, CIN_WIDTH, CIN_HEIGHT, 1 ); //GK: Straight out of the FFMPEG source code if( img_convert_ctx ) { @@ -1586,7 +1588,7 @@ cinData_t idCinematicLocal::ImageForTimeBinkDec( int thisTime ) if( audioTracks > 0 ) { - audioBuffer = ( int16_t* )malloc( binkInfo.idealBufferSize ); + audioBuffer = ( int16_t* )Mem_Alloc( binkInfo.idealBufferSize, TAG_AUDIO ); num_bytes = Bink_GetAudioData( binkHandle, trackIndex, audioBuffer ); // SRS - If we have cinematic audio data, start playing it now @@ -1598,7 +1600,7 @@ cinData_t idCinematicLocal::ImageForTimeBinkDec( int thisTime ) else { // SRS - Even though we have no audio data to play, still need to free the audio buffer - free( audioBuffer ); + Mem_Free( audioBuffer ); } } diff --git a/neo/sound/OpenAL/AL_CinematicAudio.cpp b/neo/sound/OpenAL/AL_CinematicAudio.cpp index 25f23133..5b45f870 100644 --- a/neo/sound/OpenAL/AL_CinematicAudio.cpp +++ b/neo/sound/OpenAL/AL_CinematicAudio.cpp @@ -129,7 +129,7 @@ void CinematicAudio_OpenAL::PlayAudio( uint8_t* data, int size ) #if defined(USE_FFMPEG) av_freep( &tempdata ); #elif defined(USE_BINKDEC) - free( tempdata ); + Mem_Free( tempdata ); #endif alSourceQueueBuffers( alMusicSourceVoicecin, 1, &bufid ); ALenum error = alGetError(); @@ -149,7 +149,7 @@ void CinematicAudio_OpenAL::PlayAudio( uint8_t* data, int size ) #if defined(USE_FFMPEG) av_freep( &data ); #elif defined(USE_BINKDEC) - free( data ); + Mem_Free( data ); #endif offset++; if( offset == NUM_BUFFERS ) @@ -221,7 +221,7 @@ void CinematicAudio_OpenAL::ShutdownAudio() #if defined(USE_FFMPEG) av_freep( &tempdata ); #elif defined(USE_BINKDEC) - free( tempdata ); + Mem_Free( tempdata ); #endif buffersize--; } diff --git a/neo/sound/XAudio2/XA2_CinematicAudio.cpp b/neo/sound/XAudio2/XA2_CinematicAudio.cpp index ac5f1090..f1af99d5 100644 --- a/neo/sound/XAudio2/XA2_CinematicAudio.cpp +++ b/neo/sound/XAudio2/XA2_CinematicAudio.cpp @@ -52,7 +52,7 @@ public: #if defined(USE_FFMPEG) av_freep( &data ); #elif defined(USE_BINKDEC) - free( data ); + Mem_Free( data ); #endif } //Unused methods are stubs From 22cd4205138e54de90636cbc74562a594980f82e Mon Sep 17 00:00:00 2001 From: Stephen Saunders Date: Mon, 21 Feb 2022 23:31:05 -0500 Subject: [PATCH 2/9] More cinematic audio memory mgmt fixes, support cinematic audio looping (cherry picked from commit bb0b260baa187d2dd88ef2a631e7b7f163095b10) --- neo/framework/common_frame.cpp | 8 ++- neo/renderer/Cinematic.cpp | 76 ++++++++++---------- neo/sound/CinematicAudio.h | 1 + neo/sound/OpenAL/AL_CinematicAudio.cpp | 88 +++++++++++++++++------- neo/sound/OpenAL/AL_CinematicAudio.h | 6 +- neo/sound/XAudio2/XA2_CinematicAudio.cpp | 9 +++ neo/sound/XAudio2/XA2_CinematicAudio.h | 1 + 7 files changed, 125 insertions(+), 64 deletions(-) diff --git a/neo/framework/common_frame.cpp b/neo/framework/common_frame.cpp index 2c2b1351..d7df5d6e 100644 --- a/neo/framework/common_frame.cpp +++ b/neo/framework/common_frame.cpp @@ -862,7 +862,7 @@ void idCommonLocal::Frame() // RB begin #if defined(USE_DOOMCLASSIC) // If we're in Doom or Doom 2, run tics and upload the new texture. - // SRS - Add check for com_pause cvar to make sure window is in focus - if not classic game should be paused (FIXME: but classic music still plays in background) + // SRS - Add check for com_pause cvar to make sure window is in focus - if not classic game should be paused if( ( GetCurrentGame() == DOOM_CLASSIC || GetCurrentGame() == DOOM2_CLASSIC ) && !( Dialog().IsDialogPausing() || session->IsSystemUIShowing() || com_pause.GetInteger() ) ) { RunDoomClassicFrame(); @@ -922,16 +922,18 @@ void idCommonLocal::Frame() { soundWorld->Pause(); soundSystem->SetPlayingSoundWorld( menuSoundWorld ); + soundSystem->SetMute( false ); } else { soundWorld->UnPause(); soundSystem->SetPlayingSoundWorld( soundWorld ); + soundSystem->SetMute( false ); } - // SRS - Play silence when dialog waiting or window not in focus + // SRS - Mute all sound output when dialog waiting or window not in focus (mutes Doom3, Classic, Cinematic Audio) if( Dialog().IsDialogPausing() || session->IsSystemUIShowing() || com_pause.GetInteger() ) { - soundSystem->SetPlayingSoundWorld( NULL ); + soundSystem->SetMute( true ); } soundSystem->Render(); diff --git a/neo/renderer/Cinematic.cpp b/neo/renderer/Cinematic.cpp index c142b84f..e9caf5b5 100644 --- a/neo/renderer/Cinematic.cpp +++ b/neo/renderer/Cinematic.cpp @@ -548,30 +548,9 @@ idCinematicLocal::~idCinematicLocal() av_freep( &frame2 ); av_freep( &frame3 ); #endif - - // SRS - Free any lagged cinematic audio buffers - for( int i = 0; i < NUM_LAG_FRAMES; i++ ) - { - av_freep( &lagBuffer[ i ] ); - } - - if( fmt_ctx ) - { - avformat_free_context( fmt_ctx ); - } - - if( img_convert_ctx ) - { - sws_freeContext( img_convert_ctx ); - } #endif #ifdef USE_BINKDEC - if( binkHandle.isValid ) - { - Bink_Close( binkHandle ); - } - delete imgY; imgY = NULL; delete imgCr; @@ -794,10 +773,6 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping ) int img_bytes = av_image_fill_arrays( frame2->data, frame2->linesize, NULL, AV_PIX_FMT_BGR32, CIN_WIDTH, CIN_HEIGHT, 1 ); image = ( byte* )Mem_Alloc( img_bytes, TAG_CINEMATIC ); av_image_fill_arrays( frame2->data, frame2->linesize, image, AV_PIX_FMT_BGR32, CIN_WIDTH, CIN_HEIGHT, 1 ); //GK: Straight out of the FFMPEG source code - if( img_convert_ctx ) - { - sws_freeContext( img_convert_ctx ); - } img_convert_ctx = sws_getContext( dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt, CIN_WIDTH, CIN_HEIGHT, AV_PIX_FMT_BGR32, SWS_BICUBIC, NULL, NULL, NULL ); buf = NULL; @@ -823,6 +798,21 @@ void idCinematicLocal::FFMPEGReset() //startTime = 0; framePos = -1; + + // SRS - If we have an ffmpeg audio context and are not looping, reset audio to release any stale buffers + if( dec_ctx2 && ! ( looping && status == FMV_EOF ) ) + { + cinematicAudio->ResetAudio(); + + for( int i = 0; i < NUM_LAG_FRAMES; i++ ) + { + lagBufSize[ i ] = 0; + if( lagBuffer[ i ] ) + { + av_freep( &lagBuffer[ i ] ); + } + } + } if( av_seek_frame( fmt_ctx, video_stream_index, 0, 0 ) >= 0 ) { @@ -915,6 +905,13 @@ bool idCinematicLocal::InitFromBinkDecFile( const char* qpath, bool amilooping ) void idCinematicLocal::BinkDecReset() { framePos = -1; + + // SRS - If we have bink audio tracks, reset audio to release any stale buffers (even if looping) + if( audioTracks > 0 ) + { + cinematicAudio->ResetAudio(); + } + Bink_GotoFrame( binkHandle, 0 ); status = FMV_LOOPED; } @@ -961,16 +958,12 @@ bool idCinematicLocal::InitFromFile( const char* qpath, bool amilooping ) //idLib::Warning( "Original Doom 3 RoQ Cinematic not found: '%s'\n", fileName.c_str() ); idStr temp = fileName.StripFileExtension() + ".bik"; animationLength = 0; - hasFrame = false; - RoQShutdown(); fileName = temp; //idLib::Warning( "New filename: '%s'\n", fileName.c_str() ); return InitFromFFMPEGFile( fileName.c_str(), amilooping ); #elif defined(USE_BINKDEC) idStr temp = fileName.StripFileExtension() + ".bik"; animationLength = 0; - hasFrame = false; - RoQShutdown(); fileName = temp; //idLib::Warning( "New filename: '%s'\n", fileName.c_str() ); return InitFromBinkDecFile( fileName.c_str(), amilooping ); @@ -1032,32 +1025,45 @@ void idCinematicLocal::Close() RoQShutdown(); #if defined(USE_FFMPEG) - hasFrame = false; - if( !isRoQ ) { if( img_convert_ctx ) { sws_freeContext( img_convert_ctx ); + img_convert_ctx = NULL; } - img_convert_ctx = NULL; + // SRS - Free audio codec context and any lagged audio buffers + if( dec_ctx2 ) + { + avcodec_close( dec_ctx2 ); + dec_ctx2 = NULL; + + for( int i = 0; i < NUM_LAG_FRAMES; i++ ) + { + lagBufSize[ i ] = 0; + if( lagBuffer[ i ] ) + { + av_freep( &lagBuffer[ i ] ); + } + } + } if( dec_ctx ) { avcodec_close( dec_ctx ); + dec_ctx = NULL; } if( fmt_ctx ) { avformat_close_input( &fmt_ctx ); + fmt_ctx = NULL; } status = FMV_EOF; } #endif #ifdef USE_BINKDEC - hasFrame = false; - if( !isRoQ ) { if( binkHandle.isValid ) @@ -1322,7 +1328,6 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime ) { desiredFrame = 0; FFMPEGReset(); - framePos = -1; startTime = thisTime; if( av_read_frame( fmt_ctx, &packet ) < 0 ) { @@ -1493,7 +1498,6 @@ cinData_t idCinematicLocal::ImageForTimeBinkDec( int thisTime ) { desiredFrame = 0; BinkDecReset(); - framePos = -1; startTime = thisTime; status = FMV_PLAY; } diff --git a/neo/sound/CinematicAudio.h b/neo/sound/CinematicAudio.h index 08042f37..009de0a1 100644 --- a/neo/sound/CinematicAudio.h +++ b/neo/sound/CinematicAudio.h @@ -30,6 +30,7 @@ class CinematicAudio public: virtual void InitAudio( void* audioContext ) = 0; virtual void PlayAudio( uint8_t* data, int size ) = 0; + virtual void ResetAudio() = 0; virtual void ShutdownAudio() = 0; }; diff --git a/neo/sound/OpenAL/AL_CinematicAudio.cpp b/neo/sound/OpenAL/AL_CinematicAudio.cpp index 5b45f870..4584e4f1 100644 --- a/neo/sound/OpenAL/AL_CinematicAudio.cpp +++ b/neo/sound/OpenAL/AL_CinematicAudio.cpp @@ -108,22 +108,29 @@ void CinematicAudio_OpenAL::PlayAudio( uint8_t* data, int size ) if( trigger ) { - tBuffer->push( data ); - sizes->push( size ); + tBuffer.push( data ); + sizes.push( size ); while( processed > 0 ) { ALuint bufid; - alSourceUnqueueBuffers( alMusicSourceVoicecin, 1, &bufid ); - processed--; - if( !tBuffer->empty() ) + // SRS - Only unqueue an alBuffer if we don't already have a free bufid to use + if( bufids.size() == 0 ) { - int tempSize = sizes->front(); - sizes->pop(); - uint8_t* tempdata = tBuffer->front(); - tBuffer->pop(); + alSourceUnqueueBuffers( alMusicSourceVoicecin, 1, &bufid ); + bufids.push( bufid ); + processed--; + } + if( !tBuffer.empty() ) + { + uint8_t* tempdata = tBuffer.front(); + tBuffer.pop(); + int tempSize = sizes.front(); + sizes.pop(); if( tempSize > 0 ) { + bufid = bufids.front(); + bufids.pop(); alBufferData( bufid, av_sample_cin, tempdata, tempSize, av_rate_cin ); // SRS - We must free the audio buffer once it has been copied into an alBuffer #if defined(USE_FFMPEG) @@ -140,6 +147,10 @@ void CinematicAudio_OpenAL::PlayAudio( uint8_t* data, int size ) } } } + else // SRS - When no new audio frames left to queue, break and continue playing + { + break; + } } } else @@ -186,18 +197,49 @@ void CinematicAudio_OpenAL::PlayAudio( uint8_t* data, int size ) } } +void CinematicAudio_OpenAL::ResetAudio() +{ + if( alIsSource( alMusicSourceVoicecin ) ) + { + alSourceRewind( alMusicSourceVoicecin ); + alSourcei( alMusicSourceVoicecin, AL_BUFFER, 0 ); + } + + if( !tBuffer.empty() ) + { + int buffersize = tBuffer.size(); + while( buffersize > 0 ) + { + uint8_t* tempdata = tBuffer.front(); + tBuffer.pop(); + // SRS - We must free any audio buffers that have not been copied into an alBuffer +#if defined(USE_FFMPEG) + av_freep( &tempdata ); +#elif defined(USE_BINKDEC) + Mem_Free( tempdata ); +#endif + buffersize--; + } + } + if( !sizes.empty() ) + { + int buffersize = sizes.size(); + while( buffersize > 0 ) + { + sizes.pop(); + buffersize--; + } + } + + offset = 0; + trigger = false; +} + void CinematicAudio_OpenAL::ShutdownAudio() { if( alIsSource( alMusicSourceVoicecin ) ) { alSourceStop( alMusicSourceVoicecin ); - // SRS - Make sure we don't try to unqueue buffers that were never processed - ALint processed; - alGetSourcei( alMusicSourceVoicecin, AL_BUFFERS_PROCESSED, &processed ); - if( processed > 0 ) - { - alSourceUnqueueBuffers( alMusicSourceVoicecin, processed, alMusicBuffercin ); - } alSourcei( alMusicSourceVoicecin, AL_BUFFER, 0 ); alDeleteSources( 1, &alMusicSourceVoicecin ); if( CheckALErrors() == AL_NO_ERROR ) @@ -210,13 +252,13 @@ void CinematicAudio_OpenAL::ShutdownAudio() { alDeleteBuffers( NUM_BUFFERS, alMusicBuffercin ); } - if( !tBuffer->empty() ) + if( !tBuffer.empty() ) { - int buffersize = tBuffer->size(); + int buffersize = tBuffer.size(); while( buffersize > 0 ) { - uint8_t* tempdata = tBuffer->front(); - tBuffer->pop(); + uint8_t* tempdata = tBuffer.front(); + tBuffer.pop(); // SRS - We must free any audio buffers that have not been copied into an alBuffer #if defined(USE_FFMPEG) av_freep( &tempdata ); @@ -226,12 +268,12 @@ void CinematicAudio_OpenAL::ShutdownAudio() buffersize--; } } - if( !sizes->empty() ) + if( !sizes.empty() ) { - int buffersize = sizes->size(); + int buffersize = sizes.size(); while( buffersize > 0 ) { - sizes->pop(); + sizes.pop(); buffersize--; } } diff --git a/neo/sound/OpenAL/AL_CinematicAudio.h b/neo/sound/OpenAL/AL_CinematicAudio.h index a9ed7200..cc0ca6b9 100644 --- a/neo/sound/OpenAL/AL_CinematicAudio.h +++ b/neo/sound/OpenAL/AL_CinematicAudio.h @@ -41,6 +41,7 @@ public: CinematicAudio_OpenAL(); void InitAudio( void* audioContext ); void PlayAudio( uint8_t* data, int size ); + void ResetAudio(); void ShutdownAudio(); private: ALuint alMusicSourceVoicecin; @@ -55,8 +56,9 @@ private: // So, what happens if there are no freely available buffers but we still geting audio frames ? Loss of data. // That why now I am using two queues in order to store the frames (and their sizes) and when we have available buffers, // then start popping those frames instead of the current, so we don't lose any audio frames and the sound doesn't crack anymore. - std::queue tBuffer[NUM_BUFFERS]; - std::queue sizes[NUM_BUFFERS]; + std::queue tBuffer; + std::queue sizes; + std::queue bufids; // SRS - Added queue of free alBuffer ids to handle audio frame underflow (starvation) case }; #endif diff --git a/neo/sound/XAudio2/XA2_CinematicAudio.cpp b/neo/sound/XAudio2/XA2_CinematicAudio.cpp index f1af99d5..488d8b1b 100644 --- a/neo/sound/XAudio2/XA2_CinematicAudio.cpp +++ b/neo/sound/XAudio2/XA2_CinematicAudio.cpp @@ -179,6 +179,15 @@ void CinematicAudio_XAudio2::PlayAudio( uint8_t* data, int size ) } } +void CinematicAudio_XAudio2::ResetAudio() +{ + if( pMusicSourceVoice1 ) + { + pMusicSourceVoice1->Stop(); + pMusicSourceVoice1->FlushSourceBuffers(); + } +} + void CinematicAudio_XAudio2::ShutdownAudio() { if( pMusicSourceVoice1 ) diff --git a/neo/sound/XAudio2/XA2_CinematicAudio.h b/neo/sound/XAudio2/XA2_CinematicAudio.h index 5c64a972..998a6594 100644 --- a/neo/sound/XAudio2/XA2_CinematicAudio.h +++ b/neo/sound/XAudio2/XA2_CinematicAudio.h @@ -33,6 +33,7 @@ public: CinematicAudio_XAudio2(); void InitAudio( void* audioContext ); void PlayAudio( uint8_t* data, int size ); + void ResetAudio(); void ShutdownAudio(); private: WAVEFORMATEX voiceFormatcine = { 0 }; From f24d7ecc168514669f1d86cd18cfbc1c1cb1986d Mon Sep 17 00:00:00 2001 From: Stephen Saunders Date: Tue, 22 Feb 2022 15:38:56 -0500 Subject: [PATCH 3/9] Fix XAudio2 and OpenAL errors on shutdown, improve cinematic audio queue mgmt (cherry picked from commit 4bfdf622f94b52eae52faece058c0e3d6139551e) --- neo/framework/Common.cpp | 11 ++++--- neo/sound/OpenAL/AL_CinematicAudio.cpp | 43 ++++++++++---------------- neo/sound/OpenAL/AL_SoundVoice.cpp | 7 +++-- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/neo/framework/Common.cpp b/neo/framework/Common.cpp index df3303a0..fd5f7c55 100644 --- a/neo/framework/Common.cpp +++ b/neo/framework/Common.cpp @@ -1571,10 +1571,6 @@ void idCommonLocal::Shutdown() printf( "uiManager->Shutdown();\n" ); uiManager->Shutdown(); - // shut down the sound system - printf( "soundSystem->Shutdown();\n" ); - soundSystem->Shutdown(); - // shut down the user command input code printf( "usercmdGen->Shutdown();\n" ); usercmdGen->Shutdown(); @@ -1584,9 +1580,16 @@ void idCommonLocal::Shutdown() eventLoop->Shutdown(); // shutdown the decl manager + // SRS - Note this also shuts down all cinematic resources, including cinematic audio voices printf( "declManager->Shutdown();\n" ); declManager->Shutdown(); + // shut down the sound system + // SRS - Shut down sound system after decl manager so cinematic audio voices are destroyed first + // Important for XAudio2 where the mastering voice cannot be destroyed if any other voices exist + printf( "soundSystem->Shutdown();\n" ); + soundSystem->Shutdown(); + // shut down the renderSystem printf( "renderSystem->Shutdown();\n" ); renderSystem->Shutdown(); diff --git a/neo/sound/OpenAL/AL_CinematicAudio.cpp b/neo/sound/OpenAL/AL_CinematicAudio.cpp index 4584e4f1..36176f03 100644 --- a/neo/sound/OpenAL/AL_CinematicAudio.cpp +++ b/neo/sound/OpenAL/AL_CinematicAudio.cpp @@ -205,30 +205,25 @@ void CinematicAudio_OpenAL::ResetAudio() alSourcei( alMusicSourceVoicecin, AL_BUFFER, 0 ); } - if( !tBuffer.empty() ) + while( !tBuffer.empty() ) { - int buffersize = tBuffer.size(); - while( buffersize > 0 ) + uint8_t* tempdata = tBuffer.front(); + tBuffer.pop(); + sizes.pop(); + if( tempdata ) { - uint8_t* tempdata = tBuffer.front(); - tBuffer.pop(); // SRS - We must free any audio buffers that have not been copied into an alBuffer #if defined(USE_FFMPEG) av_freep( &tempdata ); #elif defined(USE_BINKDEC) Mem_Free( tempdata ); #endif - buffersize--; } } - if( !sizes.empty() ) + + while( !bufids.empty() ) { - int buffersize = sizes.size(); - while( buffersize > 0 ) - { - sizes.pop(); - buffersize--; - } + bufids.pop(); } offset = 0; @@ -252,29 +247,25 @@ void CinematicAudio_OpenAL::ShutdownAudio() { alDeleteBuffers( NUM_BUFFERS, alMusicBuffercin ); } - if( !tBuffer.empty() ) + + while( !tBuffer.empty() ) { - int buffersize = tBuffer.size(); - while( buffersize > 0 ) + uint8_t* tempdata = tBuffer.front(); + tBuffer.pop(); + sizes.pop(); + if( tempdata ) { - uint8_t* tempdata = tBuffer.front(); - tBuffer.pop(); // SRS - We must free any audio buffers that have not been copied into an alBuffer #if defined(USE_FFMPEG) av_freep( &tempdata ); #elif defined(USE_BINKDEC) Mem_Free( tempdata ); #endif - buffersize--; } } - if( !sizes.empty() ) + + while( !bufids.empty() ) { - int buffersize = sizes.size(); - while( buffersize > 0 ) - { - sizes.pop(); - buffersize--; - } + bufids.pop(); } } diff --git a/neo/sound/OpenAL/AL_SoundVoice.cpp b/neo/sound/OpenAL/AL_SoundVoice.cpp index d5e09644..39ff1186 100644 --- a/neo/sound/OpenAL/AL_SoundVoice.cpp +++ b/neo/sound/OpenAL/AL_SoundVoice.cpp @@ -206,11 +206,14 @@ void idSoundVoice_OpenAL::DestroyInternal() idLib::Printf( "%dms: %i destroyed\n", Sys_Milliseconds(), openalSource ); } + // SRS - Make sure the source is stopped before detaching buffers + alSourceStop( openalSource ); + alSourcei( openalSource, AL_BUFFER, 0 ); + + // SRS - Delete source only after detaching buffers above alDeleteSources( 1, &openalSource ); openalSource = 0; - alSourcei( openalSource, AL_BUFFER, 0 ); - if( openalStreamingBuffer[0] && openalStreamingBuffer[1] && openalStreamingBuffer[2] ) { CheckALErrors(); From 27c1fe476a82b8e61633ca7ff00eb35c04d8e236 Mon Sep 17 00:00:00 2001 From: Stephen Saunders Date: Thu, 24 Feb 2022 02:04:30 -0500 Subject: [PATCH 4/9] Close all ffmpeg contexts, remove packet queue, support ffmpeg RoQ decoding with audio --- neo/renderer/Cinematic.cpp | 73 ++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/neo/renderer/Cinematic.cpp b/neo/renderer/Cinematic.cpp index e9caf5b5..6ff9df43 100644 --- a/neo/renderer/Cinematic.cpp +++ b/neo/renderer/Cinematic.cpp @@ -71,11 +71,7 @@ extern "C" #include #include } -// SRS - For handling cinematic audio packets -#include -#define NUM_PACKETS 4 -#define NUM_LAG_FRAMES 15 // SRS - Lag cinematic audio by 15 frames (~1/2 sec at 30 fps) to sync with FFMPEG video -bool hasplanar = true; +#define NUM_LAG_FRAMES 15 // SRS - Lag audio by 15 frames (~1/2 sec at 30 fps) for ffmpeg bik decoder AV sync #endif #ifdef USE_BINKDEC @@ -127,14 +123,15 @@ private: bool hasFrame; long framePos; AVSampleFormat dst_smp; + bool hasplanar; SwrContext* swr_ctx; cinData_t ImageForTimeFFMPEG( int milliseconds ); bool InitFromFFMPEGFile( const char* qpath, bool looping ); void FFMPEGReset(); - std::queue packets[NUM_PACKETS]; uint8_t* lagBuffer[NUM_LAG_FRAMES] = {}; int lagBufSize[NUM_LAG_FRAMES] = {}; int lagIndex; + bool skipLag; #endif #ifdef USE_BINKDEC BinkHandle binkHandle; @@ -436,7 +433,6 @@ idCinematicLocal::idCinematicLocal() isRoQ = false; // SRS - Initialize isRoQ for all cases, not just FFMPEG #if defined(USE_FFMPEG) // Carl: ffmpeg stuff, for bink and normal video files: -// fmt_ctx = avformat_alloc_context(); #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55,28,1) frame = av_frame_alloc(); frame2 = av_frame_alloc(); @@ -451,10 +447,13 @@ idCinematicLocal::idCinematicLocal() fmt_ctx = NULL; video_stream_index = -1; audio_stream_index = -1; + hasplanar = false; + swr_ctx = NULL; img_convert_ctx = NULL; hasFrame = false; framePos = -1; lagIndex = 0; + skipLag = false; #endif #ifdef USE_BINKDEC @@ -926,7 +925,8 @@ bool idCinematicLocal::InitFromFile( const char* qpath, bool amilooping ) { unsigned short RoQID; - Close(); + // SRS - Don't need to call Close() here, all initialization is handled by constructor + //Close(); inMemory = 0; animationLength = 100000; @@ -941,28 +941,41 @@ bool idCinematicLocal::InitFromFile( const char* qpath, bool amilooping ) sprintf( fileName, "%s", qpath ); } // Carl: Look for original Doom 3 RoQ files first: - idStr ext; - fileName.ExtractFileExtension( ext ); - fileName = fileName.StripFileExtension(); - fileName = fileName + ".roq"; - //if (fileName == "video\\loadvideo.roq") { - // fileName = "video\\idlogo.roq"; + idStr temp = fileName.StripFileExtension() + ".roq"; + + // SRS - Cool legacy support, but leaving this disabled since it might break existing mods + //if( temp == "video\\loadvideo.roq" ) + //{ + // temp = "video\\idlogo.roq"; //} - iFile = fileSystem->OpenFileRead( fileName ); + iFile = fileSystem->OpenFileRead( temp ); - // Carl: If the RoQ file doesn't exist, try using ffmpeg instead: + // Carl: If the RoQ file doesn't exist, try using bik file extension instead: if( !iFile ) { + //idLib::Warning( "Original Doom 3 RoQ Cinematic not found: '%s'\n", temp.c_str() ); #if defined(USE_FFMPEG) - //idLib::Warning( "Original Doom 3 RoQ Cinematic not found: '%s'\n", fileName.c_str() ); - idStr temp = fileName.StripFileExtension() + ".bik"; + temp = fileName.StripFileExtension() + ".bik"; + skipLag = false; // SRS - Enable lag buffer for ffmpeg bik decoder AV sync + + // SRS - Support RoQ cinematic playback via ffmpeg decoder - better quality plus audio support + } + else + { + fileSystem->CloseFile( iFile ); // SRS - Close the RoQ file and let ffmpeg reopen it + iFile = NULL; + skipLag = true; // SRS - Disable lag buffer for ffmpeg RoQ decoder AV sync + } + { + // SRS End + animationLength = 0; fileName = temp; //idLib::Warning( "New filename: '%s'\n", fileName.c_str() ); return InitFromFFMPEGFile( fileName.c_str(), amilooping ); #elif defined(USE_BINKDEC) - idStr temp = fileName.StripFileExtension() + ".bik"; + temp = fileName.StripFileExtension() + ".bik"; animationLength = 0; fileName = temp; //idLib::Warning( "New filename: '%s'\n", fileName.c_str() ); @@ -1033,11 +1046,16 @@ void idCinematicLocal::Close() img_convert_ctx = NULL; } - // SRS - Free audio codec context and any lagged audio buffers + // SRS - Free audio codec context, resample context, and any lagged audio buffers if( dec_ctx2 ) { - avcodec_close( dec_ctx2 ); - dec_ctx2 = NULL; + avcodec_free_context( &dec_ctx2 ); + + // SRS - Free resample context if we were decoding planar audio + if( swr_ctx ) + { + swr_free( &swr_ctx ); + } for( int i = 0; i < NUM_LAG_FRAMES; i++ ) { @@ -1051,14 +1069,12 @@ void idCinematicLocal::Close() if( dec_ctx ) { - avcodec_close( dec_ctx ); - dec_ctx = NULL; + avcodec_free_context( &dec_ctx ); } if( fmt_ctx ) { avformat_close_input( &fmt_ctx ); - fmt_ctx = NULL; } status = FMV_EOF; } @@ -1365,8 +1381,7 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime ) //GK:Begin else if( packet.stream_index == audio_stream_index ) //Check if it found any audio data { - packets->push( packet ); - res = avcodec_send_packet( dec_ctx2, &packets->front() ); + res = avcodec_send_packet( dec_ctx2, &packet ); if( res != 0 && res != AVERROR( EAGAIN ) ) { char* error = new char[256]; @@ -1375,8 +1390,6 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime ) } else { - packet = packets->front(); - packets->pop(); if( ( frameFinished1 = avcodec_receive_frame( dec_ctx2, frame3 ) ) != 0 ) { char* error = new char[256]; @@ -1443,7 +1456,7 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime ) lagBuffer[ lagIndex ] = audioBuffer; lagBufSize[ lagIndex ] = num_bytes; - lagIndex = ( lagIndex + 1 ) % NUM_LAG_FRAMES; + lagIndex = ( lagIndex + 1 ) % ( skipLag ? 1 : NUM_LAG_FRAMES ); } return cinData; From c4bc217d269da4c874a7a104e15d2cf7dd14be34 Mon Sep 17 00:00:00 2001 From: Stephen Saunders Date: Fri, 25 Feb 2022 01:13:54 -0500 Subject: [PATCH 5/9] Linux case sensitivity hack for opening RoQ files with ffmpeg --- neo/renderer/Cinematic.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/neo/renderer/Cinematic.cpp b/neo/renderer/Cinematic.cpp index 6ff9df43..6c0d76b7 100644 --- a/neo/renderer/Cinematic.cpp +++ b/neo/renderer/Cinematic.cpp @@ -655,8 +655,20 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping ) if( ( ret = avformat_open_input( &fmt_ctx, fullpath, NULL, NULL ) ) < 0 ) { - common->Warning( "idCinematic: Cannot open FFMPEG video file: '%s', %d\n", qpath, looping ); - return false; + // SRS - another case sensitivity hack for Linux, this time for ffmpeg and RoQ files + idStr ext; + fullpath.ExtractFileExtension( ext ); + if( idStr::Cmp( ext.c_str(), "roq" ) == 0 ) + { + // SRS - If ffmpeg can't open .roq file, then try again with .RoQ extension instead + fullpath.Replace( ".roq", ".RoQ" ); + ret = avformat_open_input( &fmt_ctx, fullpath, NULL, NULL ); + } + if( ret < 0 ) + { + common->Warning( "idCinematic: Cannot open FFMPEG video file: '%s', %d\n", qpath, looping ); + return false; + } } if( ( ret = avformat_find_stream_info( fmt_ctx, NULL ) ) < 0 ) { From 69be2f1e05fcb0d0bb254f6abf4cf3487c7763be Mon Sep 17 00:00:00 2001 From: Stephen Saunders Date: Sat, 26 Feb 2022 18:28:22 -0500 Subject: [PATCH 6/9] Enabled RoQ & ffmpeg decoders in testVideo cmd, fixed RoQ looping with ffmpeg decoder --- neo/renderer/Cinematic.cpp | 57 ++++++++++++++++++-------- neo/renderer/OpenGL/RenderDebug_GL.cpp | 9 +++- neo/renderer/RenderSystem_init.cpp | 3 +- neo/renderer/Vulkan/RenderDebug_VK.cpp | 7 +++- neo/sound/snd_system.cpp | 1 + 5 files changed, 57 insertions(+), 20 deletions(-) diff --git a/neo/renderer/Cinematic.cpp b/neo/renderer/Cinematic.cpp index 6c0d76b7..b16e9fa1 100644 --- a/neo/renderer/Cinematic.cpp +++ b/neo/renderer/Cinematic.cpp @@ -114,8 +114,10 @@ private: AVFrame* frame3; //GK: make extra frame for audio #if LIBAVCODEC_VERSION_MAJOR > 58 const AVCodec* dec; + const AVCodec* dec2; // SRS - Separate decoder for audio #else AVCodec* dec; + AVCodec* dec2; // SRS - Separate decoder for audio #endif AVCodecContext* dec_ctx; AVCodecContext* dec_ctx2; @@ -702,12 +704,12 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping ) return false; } //GK:Begin - //After the video decoder is open then try to open audio decoder since it will re-bind the main decoder from video to audio - ret2 = av_find_best_stream( fmt_ctx, AVMEDIA_TYPE_AUDIO, -1, -1, &dec, 0 ); + //After the video decoder is open then try to open audio decoder + ret2 = av_find_best_stream( fmt_ctx, AVMEDIA_TYPE_AUDIO, -1, -1, &dec2, 0 ); if( ret2 >= 0 ) //Make audio optional (only intro video has audio no other) { audio_stream_index = ret2; - dec_ctx2 = avcodec_alloc_context3( dec ); + dec_ctx2 = avcodec_alloc_context3( dec2 ); if( ( ret2 = avcodec_parameters_to_context( dec_ctx2, fmt_ctx->streams[audio_stream_index]->codecpar ) ) < 0 ) { char* error = new char[256]; @@ -717,7 +719,7 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping ) dec_ctx2->time_base = fmt_ctx->streams[audio_stream_index]->time_base; dec_ctx2->framerate = fmt_ctx->streams[audio_stream_index]->avg_frame_rate; dec_ctx2->pkt_timebase = fmt_ctx->streams[audio_stream_index]->time_base; - if( ( ret2 = avcodec_open2( dec_ctx2, dec, NULL ) ) < 0 ) + if( ( ret2 = avcodec_open2( dec_ctx2, dec2, NULL ) ) < 0 ) { common->Warning( "idCinematic: Cannot open audio decoder for: '%s', %d\n", qpath, looping ); //return false; @@ -767,13 +769,20 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping ) //GK: No duration is given. Check if we get at least bitrate to calculate the length, otherwise set it to a fixed 100 seconds (should it be lower ?) if( durationSec < 0 ) { - if( dec_ctx->bit_rate > 0 ) + // SRS - First check the file context bit rate and estimate duration using file size and overall bit rate + if( fmt_ctx->bit_rate > 0 ) { - durationSec = file_size / dec_ctx->bit_rate; + durationSec = file_size * 8.0 / fmt_ctx->bit_rate; } + // SRS - Likely an RoQ file, so use the video bit rate tolerance plus audio bit rate to estimate duration, then add 10% to correct for variable bit rate + else if( dec_ctx->bit_rate_tolerance > 0 ) + { + durationSec = file_size * 8.0 / ( dec_ctx->bit_rate_tolerance + ( dec_ctx2 ? dec_ctx2->bit_rate : 0 ) ) * 1.1; + } + // SRS - Otherwise just set a large max duration else { - durationSec = 100; + durationSec = 100.0; } } animationLength = durationSec * 1000; @@ -810,8 +819,8 @@ void idCinematicLocal::FFMPEGReset() framePos = -1; - // SRS - If we have an ffmpeg audio context and are not looping, reset audio to release any stale buffers - if( dec_ctx2 && ! ( looping && status == FMV_EOF ) ) + // SRS - If we have an ffmpeg audio context and are not looping, or skipLag is true, reset audio to release any stale buffers + if( dec_ctx2 && ( !( looping && status == FMV_EOF ) || skipLag ) ) { cinematicAudio->ResetAudio(); @@ -829,7 +838,17 @@ void idCinematicLocal::FFMPEGReset() { status = FMV_LOOPED; } - else if( av_seek_frame( fmt_ctx, audio_stream_index, 0, 0 ) < 0 && av_seek_frame( fmt_ctx, video_stream_index, 0, 0 ) < 0 ) + // SRS - Special handling for RoQ files: only frame byte seek works and ffmpeg RoQ decoder needs reset + else if( av_seek_frame( fmt_ctx, video_stream_index, 0, AVSEEK_FLAG_BYTE ) >= 0 && dec_ctx->codec_id == AV_CODEC_ID_ROQ ) + { + // Close and reopen the ffmpeg RoQ codec without clearing the context - this seems to reset the decoder properly + avcodec_close( dec_ctx ); + avcodec_open2( dec_ctx, dec, NULL ); + + status = FMV_LOOPED; + } + // SRS - Can't rewind the stream so we really are at EOF + else { status = FMV_EOF; } @@ -1047,10 +1066,12 @@ void idCinematicLocal::Close() status = FMV_EOF; } - RoQShutdown(); - + if( isRoQ ) + { + RoQShutdown(); + } #if defined(USE_FFMPEG) - if( !isRoQ ) + else //if( !isRoQ ) { if( img_convert_ctx ) { @@ -1092,7 +1113,7 @@ void idCinematicLocal::Close() } #endif #ifdef USE_BINKDEC - if( !isRoQ ) + else //if( !isRoQ ) { if( binkHandle.isValid ) { @@ -1264,7 +1285,8 @@ cinData_t idCinematicLocal::ImageForTime( int thisTime ) else { status = FMV_IDLE; - RoQShutdown(); + // SRS - When status == FMV_IDLE this is a no-op, disable since shutdown not needed here anyways + //RoQShutdown(); } } @@ -3184,11 +3206,14 @@ idCinematicLocal::RoQShutdown */ void idCinematicLocal::RoQShutdown() { + // SRS - Depending on status, this could prevent closing of iFile on shutdown, disable it + /* if( status == FMV_IDLE ) { return; } - status = FMV_IDLE; + */ + status = FMV_EOF; // SRS - Changed from FMV_IDLE to FMV_EOF for shutdown consistency if( iFile ) { diff --git a/neo/renderer/OpenGL/RenderDebug_GL.cpp b/neo/renderer/OpenGL/RenderDebug_GL.cpp index de7b0077..53191ccf 100644 --- a/neo/renderer/OpenGL/RenderDebug_GL.cpp +++ b/neo/renderer/OpenGL/RenderDebug_GL.cpp @@ -3263,6 +3263,11 @@ void idRenderBackend::DBG_TestImage() imageCr = cin.imageCr; imageCb = cin.imageCb; } + // SRS - Also handle ffmpeg and original RoQ decoders for test videos (using cin.image) + else if( cin.image != NULL ) + { + image = cin.image; + } else { tr.testImage = NULL; @@ -3302,9 +3307,9 @@ void idRenderBackend::DBG_TestImage() float scale[16] = { 0 }; scale[0] = w; // scale - scale[5] = -h; // scale + scale[5] = h; // scale (SRS - changed h from -ve to +ve so video plays right side up) scale[12] = halfScreenWidth - ( halfScreenWidth * w ); // translate - scale[13] = halfScreenHeight - ( halfScreenHeight * h ); // translate + scale[13] = halfScreenHeight - ( halfScreenHeight * h ) - h; // translate (SRS - moved up by h) scale[10] = 1.0f; scale[15] = 1.0f; diff --git a/neo/renderer/RenderSystem_init.cpp b/neo/renderer/RenderSystem_init.cpp index f7590849..76b91d55 100644 --- a/neo/renderer/RenderSystem_init.cpp +++ b/neo/renderer/RenderSystem_init.cpp @@ -617,7 +617,8 @@ void R_TestVideo_f( const idCmdArgs& args ) cinData_t cin; cin = tr.testVideo->ImageForTime( 0 ); - if( cin.imageY == NULL ) + // SRS - Also handle ffmpeg and original RoQ decoders for test videos (using cin.image) + if( cin.imageY == NULL && cin.image == NULL ) { delete tr.testVideo; tr.testVideo = NULL; diff --git a/neo/renderer/Vulkan/RenderDebug_VK.cpp b/neo/renderer/Vulkan/RenderDebug_VK.cpp index 93f46a33..11cf7ce1 100644 --- a/neo/renderer/Vulkan/RenderDebug_VK.cpp +++ b/neo/renderer/Vulkan/RenderDebug_VK.cpp @@ -570,6 +570,11 @@ void idRenderBackend::DBG_TestImage() imageCr = cin.imageCr; imageCb = cin.imageCb; } + // SRS - Also handle ffmpeg and original RoQ decoders for test videos (using cin.image) + else if( cin.image != NULL ) + { + image = cin.image; + } else { tr.testImage = NULL; @@ -610,7 +615,7 @@ void idRenderBackend::DBG_TestImage() scale[0] = w; // scale scale[5] = h; // scale scale[12] = halfScreenWidth - ( halfScreenWidth * w ); // translate - scale[13] = halfScreenHeight - ( halfScreenHeight * h ); // translate + scale[13] = halfScreenHeight - ( halfScreenHeight * h ) - h; // translate (SRS - moved up by h) scale[10] = 1.0f; scale[15] = 1.0f; diff --git a/neo/sound/snd_system.cpp b/neo/sound/snd_system.cpp index 76b789a3..9cf15661 100644 --- a/neo/sound/snd_system.cpp +++ b/neo/sound/snd_system.cpp @@ -532,6 +532,7 @@ cinData_t idSoundSystemLocal::ImageForTime( const int milliseconds, const bool w cd.imageY = NULL; cd.imageCr = NULL; cd.imageCb = NULL; + cd.image = NULL; cd.imageWidth = 0; cd.imageHeight = 0; cd.status = FMV_IDLE; From bd3c82c9308cb397816d5f08a9c7642a856ec07a Mon Sep 17 00:00:00 2001 From: Stephen Saunders Date: Wed, 2 Mar 2022 20:49:26 -0500 Subject: [PATCH 7/9] Align frame logic for Cinematic decoders (ffmpeg, Bink, RoQ), use correct shaders, fix bugs in RoQ looping and testVideo cmd --- neo/renderer/Cinematic.cpp | 115 +++++++++++++------------ neo/renderer/OpenGL/RenderDebug_GL.cpp | 8 +- neo/renderer/RenderBackend.cpp | 5 +- neo/renderer/RenderProgs.cpp | 3 + neo/renderer/RenderProgs.h | 7 ++ neo/renderer/RenderSystem_init.cpp | 3 +- neo/renderer/Vulkan/RenderDebug_VK.cpp | 8 +- 7 files changed, 88 insertions(+), 61 deletions(-) diff --git a/neo/renderer/Cinematic.cpp b/neo/renderer/Cinematic.cpp index b16e9fa1..2f5fa368 100644 --- a/neo/renderer/Cinematic.cpp +++ b/neo/renderer/Cinematic.cpp @@ -787,7 +787,7 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping ) } animationLength = durationSec * 1000; frameRate = av_q2d( fmt_ctx->streams[video_stream_index]->avg_frame_rate ); - common->Printf( "Loaded FFMPEG file: '%s', looping=%d, %dx%d, %f FPS, %f sec\n", qpath, looping, CIN_WIDTH, CIN_HEIGHT, frameRate, durationSec ); + common->Printf( "Loaded FFMPEG file: '%s', looping=%d, %dx%d, %3.2f FPS, %4.1f sec\n", qpath, looping, CIN_WIDTH, CIN_HEIGHT, frameRate, durationSec ); // SRS - Get number of image bytes needed by querying with NULL first, then allocate image and fill with correct parameters int img_bytes = av_image_fill_arrays( frame2->data, frame2->linesize, NULL, AV_PIX_FMT_BGR32, CIN_WIDTH, CIN_HEIGHT, 1 ); @@ -795,7 +795,6 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping ) av_image_fill_arrays( frame2->data, frame2->linesize, image, AV_PIX_FMT_BGR32, CIN_WIDTH, CIN_HEIGHT, 1 ); //GK: Straight out of the FFMPEG source code img_convert_ctx = sws_getContext( dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt, CIN_WIDTH, CIN_HEIGHT, AV_PIX_FMT_BGR32, SWS_BICUBIC, NULL, NULL, NULL ); - buf = NULL; status = FMV_PLAY; hasFrame = false; framePos = -1; @@ -834,12 +833,13 @@ void idCinematicLocal::FFMPEGReset() } } - if( av_seek_frame( fmt_ctx, video_stream_index, 0, 0 ) >= 0 ) + // SRS - For non-RoQ (i.e. bik) files, use standard frame seek to rewind the stream + if( dec_ctx->codec_id != AV_CODEC_ID_ROQ && av_seek_frame( fmt_ctx, video_stream_index, 0, 0 ) >= 0 ) { status = FMV_LOOPED; } - // SRS - Special handling for RoQ files: only frame byte seek works and ffmpeg RoQ decoder needs reset - else if( av_seek_frame( fmt_ctx, video_stream_index, 0, AVSEEK_FLAG_BYTE ) >= 0 && dec_ctx->codec_id == AV_CODEC_ID_ROQ ) + // SRS - Special handling for RoQ files: only byte seek works and ffmpeg RoQ decoder needs reset + else if( dec_ctx->codec_id == AV_CODEC_ID_ROQ && av_seek_frame( fmt_ctx, video_stream_index, 0, AVSEEK_FLAG_BYTE ) >= 0 ) { // Close and reopen the ffmpeg RoQ codec without clearing the context - this seems to reset the decoder properly avcodec_close( dec_ctx ); @@ -918,11 +918,10 @@ bool idCinematicLocal::InitFromBinkDecFile( const char* qpath, bool amilooping ) numFrames = Bink_GetNumFrames( binkHandle ); float durationSec = numFrames / frameRate; // SRS - fixed Bink durationSec calculation animationLength = durationSec * 1000; // SRS - animationLength is in milliseconds - common->Printf( "Loaded BinkDec file: '%s', looping=%d, %dx%d, %f FPS, %f sec\n", qpath, looping, CIN_WIDTH, CIN_HEIGHT, frameRate, durationSec ); + common->Printf( "Loaded BinkDec file: '%s', looping=%d, %dx%d, %3.2f FPS, %4.1f sec\n", qpath, looping, CIN_WIDTH, CIN_HEIGHT, frameRate, durationSec ); memset( yuvBuffer, 0, sizeof( yuvBuffer ) ); - buf = NULL; status = FMV_PLAY; hasFrame = false; // SRS - Implemented hasFrame for BinkDec behaviour consistency with FFMPEG framePos = -1; @@ -1043,6 +1042,7 @@ bool idCinematicLocal::InitFromFile( const char* qpath, bool amilooping ) RoQ_init(); status = FMV_PLAY; ImageForTime( 0 ); + common->Printf( "Loaded RoQ file: '%s', looping=%d, %dx%d, %3.2f FPS\n", qpath, looping, CIN_WIDTH, CIN_HEIGHT, frameRate ); status = ( looping ) ? FMV_PLAY : FMV_IDLE; return true; } @@ -1111,8 +1111,7 @@ void idCinematicLocal::Close() } status = FMV_EOF; } -#endif -#ifdef USE_BINKDEC +#elif defined(USE_BINKDEC) else //if( !isRoQ ) { if( binkHandle.isValid ) @@ -1178,8 +1177,7 @@ cinData_t idCinematicLocal::ImageForTime( int thisTime ) { return ImageForTimeFFMPEG( thisTime ); } -#endif -#ifdef USE_BINKDEC // DG: libbinkdec support +#elif defined(USE_BINKDEC) // DG: libbinkdec support if( !isRoQ ) { return ImageForTimeBinkDec( thisTime ); @@ -1189,15 +1187,16 @@ cinData_t idCinematicLocal::ImageForTime( int thisTime ) // Carl: Handle original Doom 3 RoQ video files cinData_t cinData; - if( thisTime == 0 ) + // SRS - Changed from == 0 to <= 0 to match behaviour of FFMPEG and BinkDec decoders + if( thisTime <= 0 ) { thisTime = Sys_Milliseconds(); } - if( thisTime < 0 ) - { - thisTime = 0; - } + //if( thisTime < 0 ) + //{ + // thisTime = 0; + //} memset( &cinData, 0, sizeof( cinData ) ); @@ -1234,7 +1233,8 @@ cinData_t idCinematicLocal::ImageForTime( int thisTime ) tfps = 0; } - if( tfps < numQuads ) + // SRS - Need to use numQuads - 1 for frame position (otherwise get into reset loop at start) + if( tfps < numQuads - 1 ) { RoQReset(); buf = NULL; @@ -1243,6 +1243,7 @@ cinData_t idCinematicLocal::ImageForTime( int thisTime ) if( buf == NULL ) { + // SRS - This frame init loop is not really necessary, but leaving in to avoid breakage while( buf == NULL ) { RoQInterrupt(); @@ -1250,32 +1251,31 @@ cinData_t idCinematicLocal::ImageForTime( int thisTime ) } else { - while( ( tfps != numQuads && status == FMV_PLAY ) ) + // SRS - This frame loop is really all we need and could handle the above case as well + while( ( numQuads - 1 < tfps && status == FMV_PLAY ) ) { RoQInterrupt(); } } - if( status == FMV_LOOPED ) - { - status = FMV_PLAY; - while( buf == NULL && status == FMV_PLAY ) - { - RoQInterrupt(); - } - startTime = thisTime; - } + // SRS - This is redundant code, virtually identical logic correctly handles looping below + //if( status == FMV_LOOPED ) + //{ + // status = FMV_PLAY; + // while( buf == NULL && status == FMV_PLAY ) + // { + // RoQInterrupt(); + // } + // startTime = thisTime; + //} - if( status == FMV_EOF ) + if( status == FMV_LOOPED || status == FMV_EOF ) { if( looping ) { - RoQReset(); + //RoQReset(); // SRS - RoQReset() already called by RoQInterrupt() when looping buf = NULL; - if( status == FMV_LOOPED ) - { - status = FMV_PLAY; - } + status = FMV_PLAY; while( buf == NULL && status == FMV_PLAY ) { RoQInterrupt(); @@ -1284,9 +1284,10 @@ cinData_t idCinematicLocal::ImageForTime( int thisTime ) } else { + buf = NULL; status = FMV_IDLE; - // SRS - When status == FMV_IDLE this is a no-op, disable since shutdown not needed here anyways - //RoQShutdown(); + //RoQShutdown(); //SRS - RoQShutdown() not needed on EOF, return null data instead + return cinData; } } @@ -1346,6 +1347,8 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime ) if( desiredFrame < framePos ) { FFMPEGReset(); + hasFrame = false; + status = FMV_PLAY; } if( hasFrame && desiredFrame == framePos ) @@ -1378,6 +1381,7 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime ) { desiredFrame = 0; FFMPEGReset(); + hasFrame = false; startTime = thisTime; if( av_read_frame( fmt_ctx, &packet ) < 0 ) { @@ -1388,6 +1392,7 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime ) } else { + hasFrame = false; status = FMV_IDLE; return cinData; } @@ -1538,27 +1543,12 @@ cinData_t idCinematicLocal::ImageForTimeBinkDec( int thisTime ) desiredFrame = 0; } - if( desiredFrame >= numFrames ) - { - status = FMV_EOF; - if( looping ) - { - desiredFrame = 0; - BinkDecReset(); - startTime = thisTime; - status = FMV_PLAY; - } - else - { - status = FMV_IDLE; - return cinData; - } - } - // SRS - Enable video replay within PDAs if( desiredFrame < framePos ) { BinkDecReset(); + hasFrame = false; + status = FMV_PLAY; } // SRS end @@ -1574,6 +1564,25 @@ cinData_t idCinematicLocal::ImageForTimeBinkDec( int thisTime ) return cinData; } + if( desiredFrame >= numFrames ) + { + status = FMV_EOF; + if( looping ) + { + desiredFrame = 0; + BinkDecReset(); + hasFrame = false; + startTime = thisTime; + status = FMV_PLAY; + } + else + { + hasFrame = false; + status = FMV_IDLE; + return cinData; + } + } + // Bink_GotoFrame(binkHandle, desiredFrame); // apparently Bink_GotoFrame() doesn't work super well, so skip frames // (if necessary) by calling Bink_GetNextFrame() @@ -3130,7 +3139,7 @@ redump: // // read in next frame data // - if( RoQPlayed >= ROQSize ) + if( RoQPlayed >= ROQSize || status == FMV_EOF ) // SRS - handle FMV_EOF case { if( looping ) { diff --git a/neo/renderer/OpenGL/RenderDebug_GL.cpp b/neo/renderer/OpenGL/RenderDebug_GL.cpp index 53191ccf..22effb11 100644 --- a/neo/renderer/OpenGL/RenderDebug_GL.cpp +++ b/neo/renderer/OpenGL/RenderDebug_GL.cpp @@ -3256,7 +3256,9 @@ void idRenderBackend::DBG_TestImage() { cinData_t cin; - cin = tr.testVideo->ImageForTime( viewDef->renderView.time[1] - tr.testVideoStartTime ); + // SRS - Don't need calibrated time for testing cinematics, so just call ImageForTime( 0 ) for current system time + // This simplification allows cinematic test playback to work over both 2D and 3D background scenes + cin = tr.testVideo->ImageForTime( 0 /*viewDef->renderView.time[1] - tr.testVideoStartTime*/ ); if( cin.imageY != NULL ) { image = cin.imageY; @@ -3345,7 +3347,9 @@ void idRenderBackend::DBG_TestImage() imageCr->Bind(); GL_SelectTexture( 2 ); imageCb->Bind(); - renderProgManager.BindShader_Bink(); + // SRS - Use Bink shader without sRGB to linear conversion, otherwise cinematic colours may be wrong + // BindShader_BinkGUI() does not seem to work here - perhaps due to vertex shader input dependencies? + renderProgManager.BindShader_Bink_sRGB(); } else { diff --git a/neo/renderer/RenderBackend.cpp b/neo/renderer/RenderBackend.cpp index de55e786..a8df8fd4 100644 --- a/neo/renderer/RenderBackend.cpp +++ b/neo/renderer/RenderBackend.cpp @@ -602,8 +602,8 @@ void idRenderBackend::BindVariableStageImage( const textureStage_t* texture, con GL_SelectTexture( 0 ); cin.image->Bind(); - /* - if( backEnd.viewDef->is2Dgui ) + // SRS - Reenable shaders so ffmpeg and RoQ decoder cinematics are rendered with correct colour + if( viewDef->is2Dgui ) { renderProgManager.BindShader_TextureVertexColor_sRGB(); } @@ -611,7 +611,6 @@ void idRenderBackend::BindVariableStageImage( const textureStage_t* texture, con { renderProgManager.BindShader_TextureVertexColor(); } - */ } else { diff --git a/neo/renderer/RenderProgs.cpp b/neo/renderer/RenderProgs.cpp index 58856cee..deabe6d8 100644 --- a/neo/renderer/RenderProgs.cpp +++ b/neo/renderer/RenderProgs.cpp @@ -213,6 +213,9 @@ void idRenderProgManager::Init() { BUILTIN_STEREO_DEGHOST, "builtin/VR/stereoDeGhost", "", 0, false, SHADER_STAGE_DEFAULT, LAYOUT_DRAW_VERT }, { BUILTIN_STEREO_WARP, "builtin/VR/stereoWarp", "", 0, false, SHADER_STAGE_DEFAULT, LAYOUT_DRAW_VERT }, { BUILTIN_BINK, "builtin/video/bink", "", 0, false, SHADER_STAGE_DEFAULT, LAYOUT_DRAW_VERT }, + // SRS - Added Bink shader without sRGB to linear conversion for testVideo cmd + { BUILTIN_BINK_SRGB, "builtin/video/bink", "_sRGB", BIT( USE_SRGB ), false, SHADER_STAGE_DEFAULT, LAYOUT_DRAW_VERT }, + // SRS end { BUILTIN_BINK_GUI, "builtin/video/bink_gui", "", 0, false, SHADER_STAGE_DEFAULT, LAYOUT_DRAW_VERT }, { BUILTIN_STEREO_INTERLACE, "builtin/VR/stereoInterlace", "", 0, false, SHADER_STAGE_DEFAULT, LAYOUT_DRAW_VERT }, { BUILTIN_MOTION_BLUR, "builtin/post/motionBlur", "", 0, false, SHADER_STAGE_DEFAULT, LAYOUT_DRAW_VERT }, diff --git a/neo/renderer/RenderProgs.h b/neo/renderer/RenderProgs.h index 3709606e..2cd68966 100644 --- a/neo/renderer/RenderProgs.h +++ b/neo/renderer/RenderProgs.h @@ -693,6 +693,12 @@ public: BindShader_Builtin( BUILTIN_BINK ); } + // SRS - Added Bink shader without sRGB to linear conversion for testVideo cmd + void BindShader_Bink_sRGB() + { + BindShader_Builtin( BUILTIN_BINK_SRGB ); + } + void BindShader_BinkGUI() { BindShader_Builtin( BUILTIN_BINK_GUI ); @@ -854,6 +860,7 @@ private: BUILTIN_STEREO_DEGHOST, BUILTIN_STEREO_WARP, BUILTIN_BINK, + BUILTIN_BINK_SRGB, // SRS - Added Bink shader without sRGB to linear conversion for testVideo cmd BUILTIN_BINK_GUI, BUILTIN_STEREO_INTERLACE, BUILTIN_MOTION_BLUR, diff --git a/neo/renderer/RenderSystem_init.cpp b/neo/renderer/RenderSystem_init.cpp index 76b91d55..5e7c0d8d 100644 --- a/neo/renderer/RenderSystem_init.cpp +++ b/neo/renderer/RenderSystem_init.cpp @@ -631,7 +631,8 @@ void R_TestVideo_f( const idCmdArgs& args ) int len = tr.testVideo->AnimationLength(); common->Printf( "%5.1f seconds of video\n", len * 0.001 ); - tr.testVideoStartTime = tr.primaryRenderView.time[1]; + // SRS - Not needed or used since InitFromFile() sets the correct start time automatically + //tr.testVideoStartTime = tr.primaryRenderView.time[1]; // try to play the matching wav file idStr wavString = args.Argv( ( args.Argc() == 2 ) ? 1 : 2 ); diff --git a/neo/renderer/Vulkan/RenderDebug_VK.cpp b/neo/renderer/Vulkan/RenderDebug_VK.cpp index 11cf7ce1..305dd4bb 100644 --- a/neo/renderer/Vulkan/RenderDebug_VK.cpp +++ b/neo/renderer/Vulkan/RenderDebug_VK.cpp @@ -563,7 +563,9 @@ void idRenderBackend::DBG_TestImage() { cinData_t cin; - cin = tr.testVideo->ImageForTime( viewDef->renderView.time[1] - tr.testVideoStartTime ); + // SRS - Don't need calibrated time for testing cinematics, so just call ImageForTime( 0 ) for current system time + // This simplification allows cinematic test playback to work over both 2D and 3D background scenes + cin = tr.testVideo->ImageForTime( 0 /*viewDef->renderView.time[1] - tr.testVideoStartTime*/ ); if( cin.imageY != NULL ) { image = cin.imageY; @@ -651,7 +653,9 @@ void idRenderBackend::DBG_TestImage() GL_SelectTexture( 2 ); imageCb->Bind(); - renderProgManager.BindShader_Bink(); + // SRS - Use Bink shader with no sRGB to linear conversion, otherwise cinematic colours may be wrong + // BindShader_BinkGUI() does not seem to work here - perhaps due to vertex shader input dependencies? + renderProgManager.BindShader_Bink_sRGB(); } else { From 8f474b9cb4181e0ab57f5eb8e39ed9d96397ea67 Mon Sep 17 00:00:00 2001 From: Stephen Saunders Date: Thu, 3 Mar 2022 11:59:32 -0500 Subject: [PATCH 8/9] Tiny fix for printing RoQ cinematic filename --- neo/renderer/Cinematic.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neo/renderer/Cinematic.cpp b/neo/renderer/Cinematic.cpp index 2f5fa368..d15426c9 100644 --- a/neo/renderer/Cinematic.cpp +++ b/neo/renderer/Cinematic.cpp @@ -1017,6 +1017,7 @@ bool idCinematicLocal::InitFromFile( const char* qpath, bool amilooping ) } // Carl: The rest of this function is for original Doom 3 RoQ files: isRoQ = true; + fileName = temp; ROQSize = iFile->Length(); looping = amilooping; @@ -1042,7 +1043,7 @@ bool idCinematicLocal::InitFromFile( const char* qpath, bool amilooping ) RoQ_init(); status = FMV_PLAY; ImageForTime( 0 ); - common->Printf( "Loaded RoQ file: '%s', looping=%d, %dx%d, %3.2f FPS\n", qpath, looping, CIN_WIDTH, CIN_HEIGHT, frameRate ); + common->Printf( "Loaded RoQ file: '%s', looping=%d, %dx%d, %3.2f FPS\n", fileName.c_str(), looping, CIN_WIDTH, CIN_HEIGHT, frameRate ); status = ( looping ) ? FMV_PLAY : FMV_IDLE; return true; } From 1ac9baca5d4ea4e9863f462ed4794864d1157e73 Mon Sep 17 00:00:00 2001 From: Stephen Saunders Date: Thu, 3 Mar 2022 16:31:49 -0500 Subject: [PATCH 9/9] Tiny fix for printing Bink audio stream sample format --- neo/renderer/Cinematic.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/renderer/Cinematic.cpp b/neo/renderer/Cinematic.cpp index d15426c9..16c45225 100644 --- a/neo/renderer/Cinematic.cpp +++ b/neo/renderer/Cinematic.cpp @@ -910,7 +910,7 @@ bool idCinematicLocal::InitFromBinkDecFile( const char* qpath, bool amilooping ) { trackIndex = 0; // SRS - Use the first audio track - is this reasonable? binkInfo = Bink_GetAudioTrackDetails( binkHandle, trackIndex ); - common->Printf( "Cinematic audio stream found: Sample Rate=%d Hz, Channels=%d\n", binkInfo.sampleRate, binkInfo.nChannels ); + common->Printf( "Cinematic audio stream found: Sample Rate=%d Hz, Channels=%d, Format=16-bit\n", binkInfo.sampleRate, binkInfo.nChannels ); cinematicAudio->InitAudio( &binkInfo ); }