id/doom3-bfg
0
0
Fork 0
mirror of https://github.com/id-Software/DOOM-3-BFG.git synced 2025-04-24 02:32:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'SRSaunders/cinematic-updates'

Browse source
This commit is contained in:
Robert Beckebans 2023-01-18 10:04:00 +01:00
commit 0a983d32ad
15 changed files with 319 additions and 208 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
neo/framework/Common.cpp
View file

@ -1584,16 +1584,17 @@ void idCommonLocal::Shutdown()
printf( "declManager->Shutdown();\n" );
declManager->Shutdown();
// shut down the renderSystem
// SRS - Note this also shuts down any testVideo resources, including cinematic audio voices
printf( "renderSystem->Shutdown();\n" );
renderSystem->Shutdown();
// shut down the sound system
// SRS - Shut down sound system after decl manager so cinematic audio voices are destroyed first
// SRS - Shut down sound system after decl manager and render system so cinematic audio voices are destroyed first
// Important for XAudio2 where the mastering voice cannot be destroyed if any other voices exist
printf( "soundSystem->Shutdown();\n" );
soundSystem->Shutdown();
// shut down the renderSystem
printf( "renderSystem->Shutdown();\n" );
renderSystem->Shutdown();
printf( "commonDialog.Shutdown();\n" );
commonDialog.Shutdown();

377
neo/renderer/Cinematic.cpp
View file

@ -37,7 +37,8 @@ If you have questions concerning this license or the applicable additional terms
#endif
extern idCVar s_noSound;
// SRS - Add cvar to control whether cinematic audio is played: default is ON
idCVar s_playCinematicAudio( "s_playCinematicAudio", "1", CVAR_BOOL, "Play audio if available in cinematic video files" );
//extern "C" {
#include <jpeglib.h>
@ -220,7 +221,7 @@ private:
// RB end
//GK:Also init variables for XAudio2 or OpenAL (SRS - this must be an instance variable)
CinematicAudio* cinematicAudio;
CinematicAudio* cinematicAudio = NULL;
};
// Carl: ROQ files from original Doom 3
@ -259,7 +260,7 @@ static unsigned short* vq8 = NULL;
/*
==============
idCinematicLocal::InitCinematic
idCinematic::InitCinematic
==============
*/
// RB: 64 bit fixes, changed long to int
@ -294,7 +295,7 @@ void idCinematic::InitCinematic()
/*
==============
idCinematicLocal::ShutdownCinematic
idCinematic::ShutdownCinematic
==============
*/
void idCinematic::ShutdownCinematic()
@ -412,12 +413,27 @@ void idCinematic::Close()
}
// RB begin
/*
==============
idCinematic::IsPlaying
==============
*/
bool idCinematic::IsPlaying() const
{
return false;
}
// RB end
// SRS begin
/*
==============
CinematicAudio::~CinematicAudio
==============
*/
CinematicAudio::~CinematicAudio()
{
}
// SRS end
//===========================================
@ -509,13 +525,6 @@ idCinematicLocal::idCinematicLocal()
opts.numLevels = 1;
img->AllocImage( opts, TF_LINEAR, TR_REPEAT );
}
//GK: Make sure the cinematic voices are the first to be initialized
#if defined(_MSC_VER) && !defined(USE_OPENAL)
cinematicAudio = new( TAG_AUDIO ) CinematicAudio_XAudio2;
#else
cinematicAudio = new( TAG_AUDIO ) CinematicAudio_OpenAL;
#endif
}
/*
@ -562,14 +571,19 @@ idCinematicLocal::~idCinematicLocal()
delete img;
img = NULL;
//GK: Properly close local XAudio2 or OpenAL voice
cinematicAudio->ShutdownAudio();
// GK/SRS - Properly close and delete XAudio2 or OpenAL voice if instantiated
if( cinematicAudio )
{
cinematicAudio->ShutdownAudio();
delete cinematicAudio;
cinematicAudio = NULL;
}
}
#if defined(USE_FFMPEG)
/*
==============
idCinematicLocal::GetSampleFormat
GetSampleFormat
==============
*/
const char* GetSampleFormat( AVSampleFormat sample_fmt )
@ -618,6 +632,7 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping, n
int ret;
int ret2;
int file_size;
char error[64];
looping = amilooping;
startTime = 0;
isRoQ = false;
@ -656,7 +671,7 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping, n
if( ( ret = avformat_open_input( &fmt_ctx, fullpath, NULL, NULL ) ) < 0 )
{
// SRS - another case sensitivity hack for Linux, this time for ffmpeg and RoQ files
// SRS - another case sensitivity HACK for Linux, this time for ffmpeg and RoQ files
idStr ext;
fullpath.ExtractFileExtension( ext );
if( idStr::Cmp( ext.c_str(), "roq" ) == 0 )
@ -687,37 +702,34 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping, n
dec_ctx = avcodec_alloc_context3( dec );
if( ( ret = avcodec_parameters_to_context( dec_ctx, fmt_ctx->streams[video_stream_index]->codecpar ) ) < 0 )
{
char* error = new char[256];
av_strerror( ret, error, 256 );
av_strerror( ret, error, sizeof( error ) );
common->Warning( "idCinematic: Failed to create video codec context from codec parameters with error: %s\n", error );
}
dec_ctx->time_base = fmt_ctx->streams[video_stream_index]->time_base;
//dec_ctx->time_base = fmt_ctx->streams[video_stream_index]->time_base; // SRS - decoder timebase is set by avcodec_open2()
dec_ctx->framerate = fmt_ctx->streams[video_stream_index]->avg_frame_rate;
dec_ctx->pkt_timebase = fmt_ctx->streams[video_stream_index]->time_base;
dec_ctx->pkt_timebase = fmt_ctx->streams[video_stream_index]->time_base; // SRS - packet timebase for frame->pts timestamps
/* init the video decoder */
if( ( ret = avcodec_open2( dec_ctx, dec, NULL ) ) < 0 )
{
char* error = new char[256];
av_strerror( ret, error, 256 );
av_strerror( ret, error, sizeof( error ) );
common->Warning( "idCinematic: Cannot open video decoder for: '%s', %d, with error: %s\n", qpath, looping, error );
return false;
}
//GK:Begin
//After the video decoder is open then try to open audio decoder
ret2 = av_find_best_stream( fmt_ctx, AVMEDIA_TYPE_AUDIO, -1, -1, &dec2, 0 );
if( ret2 >= 0 ) //Make audio optional (only intro video has audio no other)
if( ret2 >= 0 && s_playCinematicAudio.GetBool() ) //Make audio optional (only intro video has audio no other)
{
audio_stream_index = ret2;
dec_ctx2 = avcodec_alloc_context3( dec2 );
if( ( ret2 = avcodec_parameters_to_context( dec_ctx2, fmt_ctx->streams[audio_stream_index]->codecpar ) ) < 0 )
{
char* error = new char[256];
av_strerror( ret2, error, 256 );
av_strerror( ret2, error, sizeof( error ) );
common->Warning( "idCinematic: Failed to create audio codec context from codec parameters with error: %s\n", error );
}
dec_ctx2->time_base = fmt_ctx->streams[audio_stream_index]->time_base;
//dec_ctx2->time_base = fmt_ctx->streams[audio_stream_index]->time_base; // SRS - decoder timebase is set by avcodec_open2()
dec_ctx2->framerate = fmt_ctx->streams[audio_stream_index]->avg_frame_rate;
dec_ctx2->pkt_timebase = fmt_ctx->streams[audio_stream_index]->time_base;
dec_ctx2->pkt_timebase = fmt_ctx->streams[audio_stream_index]->time_base; // SRS - packet timebase for frame3->pts timestamps
if( ( ret2 = avcodec_open2( dec_ctx2, dec2, NULL ) ) < 0 )
{
common->Warning( "idCinematic: Cannot open audio decoder for: '%s', %d\n", qpath, looping );
@ -726,8 +738,16 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping, n
if( dec_ctx2->sample_fmt >= AV_SAMPLE_FMT_U8P ) // SRS - Planar formats start at AV_SAMPLE_FMT_U8P
{
dst_smp = static_cast<AVSampleFormat>( dec_ctx2->sample_fmt - AV_SAMPLE_FMT_U8P ); // SRS - Setup context to convert from planar to packed
#if LIBSWRESAMPLE_VERSION_INT >= AV_VERSION_INT(4,7,100)
if( ( ret2 = swr_alloc_set_opts2( &swr_ctx, &dec_ctx2->ch_layout, dst_smp, dec_ctx2->sample_rate, &dec_ctx2->ch_layout, dec_ctx2->sample_fmt, dec_ctx2->sample_rate, 0, NULL ) ) < 0 )
{
av_strerror( ret2, error, sizeof( error ) );
common->Warning( "idCinematic: Failed to create audio resample context with error: %s\n", error );
}
#else
swr_ctx = swr_alloc_set_opts( NULL, dec_ctx2->channel_layout, dst_smp, dec_ctx2->sample_rate, dec_ctx2->channel_layout, dec_ctx2->sample_fmt, dec_ctx2->sample_rate, 0, NULL );
int res = swr_init( swr_ctx );
#endif
ret2 = swr_init( swr_ctx );
hasplanar = true;
}
else
@ -735,14 +755,21 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping, n
dst_smp = dec_ctx2->sample_fmt; // SRS - Must always define the destination format
hasplanar = false;
}
common->Printf( "Cinematic audio stream found: Sample Rate=%d Hz, Channels=%d, Format=%s, Planar=%d\n", dec_ctx2->sample_rate, dec_ctx2->channels, GetSampleFormat( dec_ctx2->sample_fmt ), hasplanar );
common->Printf( "Cinematic audio stream found: Sample Rate=%d Hz, Channels=%d, Format=%s, Planar=%d\n", dec_ctx2->sample_rate,
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(59,37,100)
dec_ctx2->ch_layout.nb_channels,
#else
dec_ctx2->channels,
#endif
GetSampleFormat( dec_ctx2->sample_fmt ), hasplanar );
#if defined(_MSC_VER) && !defined(USE_OPENAL)
cinematicAudio = new( TAG_AUDIO ) CinematicAudio_XAudio2;
#else
cinematicAudio = new( TAG_AUDIO ) CinematicAudio_OpenAL;
#endif
cinematicAudio->InitAudio( dec_ctx2 );
}
else
{
// SRS - Most cinematics have no audio, so disable the warning to reduce distracting log messages
//common->Warning("idCinematic: Cannot find an audio stream in: '%s', %d\n", qpath, looping);
}
//GK:End
CIN_WIDTH = dec_ctx->width;
CIN_HEIGHT = dec_ctx->height;
@ -754,17 +781,8 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping, n
* - encoding: MUST be set by user.
* - decoding: Set by libavcodec.
*/
AVRational avr = dec_ctx->time_base;
/**
* For some codecs, the time base is closer to the field rate than the frame rate.
* Most notably, H.264 and MPEG-2 specify time_base as half of frame duration
* if no telecine is used ...
*
* Set to time_base ticks per frame. Default 1, e.g., H.264/MPEG-2 set it to 2.
*/
int ticksPerFrame = dec_ctx->ticks_per_frame;
// SRS - In addition to ticks, must also use time_base numerator (not always 1) and denominator in the duration calculation
float durationSec = static_cast<double>( fmt_ctx->streams[video_stream_index]->duration ) * static_cast<double>( ticksPerFrame ) * static_cast<double>( avr.num ) / static_cast<double>( avr.den );
// SRS - Must use consistent duration and timebase parameters in the durationSec calculation (don't mix fmt_ctx duration with dec_ctx timebase)
float durationSec = static_cast<double>( fmt_ctx->streams[video_stream_index]->duration ) * av_q2d( fmt_ctx->streams[video_stream_index]->time_base );
//GK: No duration is given. Check if we get at least bitrate to calculate the length, otherwise set it to a fixed 100 seconds (should it be lower ?)
if( durationSec < 0 )
{
@ -788,8 +806,10 @@ bool idCinematicLocal::InitFromFFMPEGFile( const char* qpath, bool amilooping, n
frameRate = av_q2d( fmt_ctx->streams[video_stream_index]->avg_frame_rate );
common->Printf( "Loaded FFMPEG file: '%s', looping=%d, %dx%d, %3.2f FPS, %4.1f sec\n", qpath, looping, CIN_WIDTH, CIN_HEIGHT, frameRate, durationSec );
// SRS - Get number of image bytes needed by querying with NULL first, then allocate image and fill with correct parameters
int img_bytes = av_image_fill_arrays( frame2->data, frame2->linesize, NULL, AV_PIX_FMT_BGR32, CIN_WIDTH, CIN_HEIGHT, 1 );
// SRS - Get image buffer size (dimensions mod 32 for bik & webm codecs, subsumes mod 16 for mp4 codec), then allocate image and fill with correct parameters
int bufWidth = ( CIN_WIDTH + 31 ) & ~31;
int bufHeight = ( CIN_HEIGHT + 31 ) & ~31;
int img_bytes = av_image_get_buffer_size( AV_PIX_FMT_BGR32, bufWidth, bufHeight, 1 );
image = ( byte* )Mem_Alloc( img_bytes, TAG_CINEMATIC );
av_image_fill_arrays( frame2->data, frame2->linesize, image, AV_PIX_FMT_BGR32, CIN_WIDTH, CIN_HEIGHT, 1 ); //GK: Straight out of the FFMPEG source code
img_convert_ctx = sws_getContext( dec_ctx->width, dec_ctx->height, dec_ctx->pix_fmt, CIN_WIDTH, CIN_HEIGHT, AV_PIX_FMT_BGR32, SWS_BICUBIC, NULL, NULL, NULL );
@ -817,11 +837,12 @@ void idCinematicLocal::FFMPEGReset()
framePos = -1;
// SRS - If we have an ffmpeg audio context and are not looping, or skipLag is true, reset audio to release any stale buffers
if( dec_ctx2 && ( !( looping && status == FMV_EOF ) || skipLag ) )
// SRS - If we have cinematic audio, reset audio to release any stale buffers and avoid AV drift if looping
if( cinematicAudio )
{
cinematicAudio->ResetAudio();
lagIndex = 0;
for( int i = 0; i < NUM_LAG_FRAMES; i++ )
{
lagBufSize[ i ] = 0;
@ -832,7 +853,7 @@ void idCinematicLocal::FFMPEGReset()
}
}
// SRS - For non-RoQ (i.e. bik) files, use standard frame seek to rewind the stream
// SRS - For non-RoQ (i.e. bik, mp4, webm, etc) files, use standard frame seek to rewind the stream
if( dec_ctx->codec_id != AV_CODEC_ID_ROQ && av_seek_frame( fmt_ctx, video_stream_index, 0, 0 ) >= 0 )
{
status = FMV_LOOPED;
@ -854,6 +875,11 @@ void idCinematicLocal::FFMPEGReset()
}
#endif
/*
==============
idCinematicLocal::InitFromBinkDecFile
==============
*/
#ifdef USE_BINKDEC
bool idCinematicLocal::InitFromBinkDecFile( const char* qpath, bool amilooping, nvrhi::ICommandList* commandList )
{
@ -905,11 +931,16 @@ bool idCinematicLocal::InitFromBinkDecFile( const char* qpath, bool amilooping,
// SRS - Support Bink Audio for cinematic playback
audioTracks = Bink_GetNumAudioTracks( binkHandle );
if( audioTracks > 0 )
if( audioTracks > 0 && s_playCinematicAudio.GetBool() )
{
trackIndex = 0; // SRS - Use the first audio track - is this reasonable?
binkInfo = Bink_GetAudioTrackDetails( binkHandle, trackIndex );
common->Printf( "Cinematic audio stream found: Sample Rate=%d Hz, Channels=%d, Format=16-bit\n", binkInfo.sampleRate, binkInfo.nChannels );
#if defined(_MSC_VER) && !defined(USE_OPENAL)
cinematicAudio = new( TAG_AUDIO ) CinematicAudio_XAudio2;
#else
cinematicAudio = new( TAG_AUDIO ) CinematicAudio_OpenAL;
#endif
cinematicAudio->InitAudio( &binkInfo );
}
@ -930,12 +961,17 @@ bool idCinematicLocal::InitFromBinkDecFile( const char* qpath, bool amilooping,
return true;
}
/*
==============
idCinematicLocal::BinkDecReset
==============
*/
void idCinematicLocal::BinkDecReset()
{
framePos = -1;
// SRS - If we have bink audio tracks, reset audio to release any stale buffers (even if looping)
if( audioTracks > 0 )
// SRS - If we have cinematic audio, reset audio to release any stale buffers (even if looping)
if( cinematicAudio )
{
cinematicAudio->ResetAudio();
}
@ -970,23 +1006,68 @@ bool idCinematicLocal::InitFromFile( const char* qpath, bool amilooping, nvrhi::
sprintf( fileName, "%s", qpath );
}
// Carl: Look for original Doom 3 RoQ files first:
idStr temp = fileName.StripFileExtension() + ".roq";
idStr temp, ext;
// SRS - Since loadvideo.bik is hardcoded, first check for legacy idlogo then later for mp4/webm mods
// If neither idlogo.roq nor loadvideo.mp4/.webm are found, then fall back to stock loadvideo.bik
if( fileName == "video\\loadvideo.bik" )
{
temp = "video\\idlogo.roq"; // Check for legacy idlogo.roq video
}
else
{
temp = fileName;
}
// SRS - Cool legacy support, but leaving this disabled since it might break existing mods
//if( temp == "video\\loadvideo.roq" )
//{
// temp = "video\\idlogo.roq";
//}
iFile = fileSystem->OpenFileRead( temp );
temp.ExtractFileExtension( ext );
ext.ToLower();
if( ext == "roq" )
{
temp = temp.StripFileExtension() + ".roq"; // Force lower case .roq extension
iFile = fileSystem->OpenFileRead( temp );
}
else
{
iFile = NULL;
}
// Carl: If the RoQ file doesn't exist, try using bik file extension instead:
if( !iFile )
{
//idLib::Warning( "Original Doom 3 RoQ Cinematic not found: '%s'\n", temp.c_str() );
#if defined(USE_FFMPEG)
temp = fileName.StripFileExtension() + ".bik";
skipLag = false; // SRS - Enable lag buffer for ffmpeg bik decoder AV sync
if( fileName == "video\\loadvideo.bik" )
{
temp = "video\\loadvideo"; // Check for loadvideo.xxx mod videos
if( ( iFile = fileSystem->OpenFileRead( temp + ".mp4" ) ) )
{
fileSystem->CloseFile( iFile ); // Close the mp4 file and reopen in ffmpeg
iFile = NULL;
temp = temp + ".mp4";
ext = "mp4";
}
else if( ( iFile = fileSystem->OpenFileRead( temp + ".webm" ) ) )
{
fileSystem->CloseFile( iFile ); // Close the webm file and reopen in ffmpeg
iFile = NULL;
temp = temp + ".webm";
ext = "webm";
}
else
{
temp = temp + ".bik"; // Fall back to bik if no mod videos found
ext = "bik";
}
}
if( ext == "roq" || ext == "bik" ) // Check for stock/legacy cinematic types
{
temp = fileName.StripFileExtension() + ".bik";
skipLag = false; // SRS - Enable lag buffer for ffmpeg bik decoder AV sync
}
else // Enable mp4/webm cinematic types for mods
{
skipLag = true; // SRS - Disable lag buffer for ffmpeg mp4/webm decoder AV sync
}
// SRS - Support RoQ cinematic playback via ffmpeg decoder - better quality plus audio support
}
@ -1079,10 +1160,13 @@ void idCinematicLocal::Close()
img_convert_ctx = NULL;
}
// SRS - Free audio codec context, resample context, and any lagged audio buffers
if( dec_ctx2 )
// SRS - If we have cinematic audio, free audio codec context, resample context, and any lagged audio buffers
if( cinematicAudio )
{
avcodec_free_context( &dec_ctx2 );
if( dec_ctx2 )
{
avcodec_free_context( &dec_ctx2 );
}
// SRS - Free resample context if we were decoding planar audio
if( swr_ctx )
@ -1090,6 +1174,7 @@ void idCinematicLocal::Close()
swr_free( &swr_ctx );
}
lagIndex = 0;
for( int i = 0; i < NUM_LAG_FRAMES; i++ )
{
lagBufSize[ i ] = 0;
@ -1135,6 +1220,11 @@ int idCinematicLocal::AnimationLength()
}
// RB begin
/*
==============
idCinematicLocal::IsPlaying
==============
*/
bool idCinematicLocal::IsPlaying() const
{
return ( status == FMV_PLAY );
@ -1171,6 +1261,20 @@ idCinematicLocal::ImageForTime
*/
cinData_t idCinematicLocal::ImageForTime( int thisTime, nvrhi::ICommandList* commandList )
{
cinData_t cinData;
if( thisTime <= 0 )
{
thisTime = Sys_Milliseconds();
}
memset( &cinData, 0, sizeof( cinData ) );
// SRS - also return if commandList is null, typically when called from within InitFromFile()
if( r_skipDynamicTextures.GetBool() || status == FMV_EOF || status == FMV_IDLE || !commandList )
{
return cinData;
}
#if defined(USE_FFMPEG)
// Carl: Handle BFG format BINK videos separately
if( !isRoQ )
@ -1184,39 +1288,12 @@ cinData_t idCinematicLocal::ImageForTime( int thisTime, nvrhi::ICommandList* com
}
#endif
// Carl: Handle original Doom 3 RoQ video files
cinData_t cinData;
// SRS - Changed from == 0 to <= 0 to match behaviour of FFMPEG and BinkDec decoders
if( thisTime <= 0 )
{
thisTime = Sys_Milliseconds();
}
//if( thisTime < 0 )
//{
// thisTime = 0;
//}
memset( &cinData, 0, sizeof( cinData ) );
// if ( r_skipROQ.GetBool() ) {
if( r_skipDynamicTextures.GetBool() )
{
return cinData;
}
if( !iFile )
{
// RB: neither .bik or .roq found
return cinData;
}
if( status == FMV_EOF || status == FMV_IDLE )
{
return cinData;
}
if( buf == NULL || startTime == -1 )
{
if( startTime == -1 )
@ -1258,17 +1335,6 @@ cinData_t idCinematicLocal::ImageForTime( int thisTime, nvrhi::ICommandList* com
}
}
// SRS - This is redundant code, virtually identical logic correctly handles looping below
//if( status == FMV_LOOPED )
//{
// status = FMV_PLAY;
// while( buf == NULL && status == FMV_PLAY )
// {
// RoQInterrupt();
// }
// startTime = thisTime;
//}
if( status == FMV_LOOPED || status == FMV_EOF )
{
if( looping )
@ -1309,20 +1375,11 @@ idCinematicLocal::ImageForTimeFFMPEG
cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime, nvrhi::ICommandList* commandList )
{
cinData_t cinData;
char error[64];
uint8_t* audioBuffer = NULL;
int num_bytes = 0;
if( thisTime <= 0 )
{
thisTime = Sys_Milliseconds();
}
memset( &cinData, 0, sizeof( cinData ) );
if( r_skipDynamicTextures.GetBool() || status == FMV_EOF || status == FMV_IDLE )
{
return cinData;
}
if( !fmt_ctx )
{
// RB: .bik requested but not found
@ -1364,9 +1421,6 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime, nvrhi::ICommandLis
while( framePos < desiredFrame )
{
int frameFinished = -1;
//GK: Separate frame finisher for audio in order to not have the video lagging
int frameFinished1 = -1;
int res = 0;
// Do a single frame by getting packets until we have a full frame
@ -1403,42 +1457,49 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime, nvrhi::ICommandLis
// Decode video frame
if( ( res = avcodec_send_packet( dec_ctx, &packet ) ) != 0 )
{
char* error = new char[256];
av_strerror( res, error, 256 );
av_strerror( res, error, sizeof( error ) );
common->Warning( "idCinematic: Failed to send video packet for decoding with error: %s\n", error );
}
else
{
if( ( frameFinished = avcodec_receive_frame( dec_ctx, frame ) ) != 0 )
frameFinished = avcodec_receive_frame( dec_ctx, frame );
if( frameFinished != 0 && frameFinished != AVERROR( EAGAIN ) )
{
char* error = new char[256];
av_strerror( frameFinished, error, 256 );
av_strerror( frameFinished, error, sizeof( error ) );
common->Warning( "idCinematic: Failed to receive video frame from decoding with error: %s\n", error );
}
}
}
//GK:Begin
else if( packet.stream_index == audio_stream_index ) //Check if it found any audio data
else if( cinematicAudio && packet.stream_index == audio_stream_index ) //Check if it found any audio data
{
res = avcodec_send_packet( dec_ctx2, &packet );
if( res != 0 && res != AVERROR( EAGAIN ) )
{
char* error = new char[256];
av_strerror( res, error, 256 );
av_strerror( res, error, sizeof( error ) );
common->Warning( "idCinematic: Failed to send audio packet for decoding with error: %s\n", error );
}
else
//SRS - Separate frame finisher for audio since there can be multiple audio frames per video frame (e.g. at bik startup)
int frameFinished1 = 0;
while( frameFinished1 == 0 )
{
if( ( frameFinished1 = avcodec_receive_frame( dec_ctx2, frame3 ) ) != 0 )
{
char* error = new char[256];
av_strerror( frameFinished1, error, 256 );
common->Warning( "idCinematic: Failed to receive audio frame from decoding with error: %s\n", error );
if( frameFinished1 != AVERROR( EAGAIN ) )
{
av_strerror( frameFinished1, error, sizeof( error ) );
common->Warning( "idCinematic: Failed to receive audio frame from decoding with error: %s\n", error );
}
}
else
// SRS - For the final (desired) frame only: allocate audio buffers, convert to packed format, and play the synced audio frames
else if( framePos + 1 == desiredFrame )
{
// SRS - Since destination sample format is packed (non-planar), returned bufflinesize equals num_bytes
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(59,37,100)
res = av_samples_alloc( &audioBuffer, &num_bytes, frame3->ch_layout.nb_channels, frame3->nb_samples, dst_smp, 0 );
#else
res = av_samples_alloc( &audioBuffer, &num_bytes, frame3->channels, frame3->nb_samples, dst_smp, 0 );
#endif
if( res < 0 || res != num_bytes )
{
common->Warning( "idCinematic: Failed to allocate audio buffer with result: %d\n", res );
@ -1460,6 +1521,29 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime, nvrhi::ICommandLis
memcpy( audioBuffer, frame3->extended_data[0], num_bytes );
}
}
// SRS - If we have cinematic audio data, play a lagged frame (for bik video sync) and save the current frame
if( num_bytes > 0 )
{
// SRS - If we have a lagged cinematic audio frame, then play it now
if( lagBufSize[ lagIndex ] > 0 )
{
// SRS - Note that PlayAudio() is responsible for releasing any audio buffers sent to it
cinematicAudio->PlayAudio( lagBuffer[ lagIndex ], lagBufSize[ lagIndex ] );
}
// SRS - Save the current (new) audio buffer and its size to play in the future
lagBuffer[ lagIndex ] = audioBuffer;
lagBufSize[ lagIndex ] = num_bytes;
// SRS - If skipLag is true (e.g. RoQ, mp4, webm), reduce lag to 1 frame since AV playback is already synced
lagIndex = ( lagIndex + 1 ) % ( skipLag ? 1 : NUM_LAG_FRAMES );
}
// SRS - Not sure if an audioBuffer can ever be allocated on failure, but check and free just in case
else if( audioBuffer )
{
av_freep( &audioBuffer );
}
//common->Printf( "idCinematic: video pts = %7.3f, audio pts = %7.3f, samples = %4d, num_bytes = %5d\n", static_cast<double>( frame->pts ) * av_q2d( dec_ctx->pkt_timebase ), static_cast<double>( frame3->pts ) * av_q2d( dec_ctx2->pkt_timebase ), frame3->nb_samples, num_bytes );
}
}
}
@ -1481,28 +1565,16 @@ cinData_t idCinematicLocal::ImageForTimeFFMPEG( int thisTime, nvrhi::ICommandLis
hasFrame = true;
cinData.image = img;
// SRS - If we have cinematic audio data, play a lagged frame (for FFMPEG video sync) and save the current frame
if( num_bytes > 0 )
{
// SRS - If we have a lagged cinematic audio frame, then play it now
if( lagBufSize[ lagIndex ] > 0 )
{
// SRS - Note that PlayAudio() is responsible for releasing any audio buffers sent to it
cinematicAudio->PlayAudio( lagBuffer[ lagIndex ], lagBufSize[ lagIndex ] );
}
// SRS - Save the current (new) audio buffer and its size to play NUM_LAG_FRAMES in the future
lagBuffer[ lagIndex ] = audioBuffer;
lagBufSize[ lagIndex ] = num_bytes;
lagIndex = ( lagIndex + 1 ) % ( skipLag ? 1 : NUM_LAG_FRAMES );
}
return cinData;
}
#endif
/*
==============
idCinematicLocal::ImageForTimeBinkDec
==============
*/
#ifdef USE_BINKDEC
cinData_t idCinematicLocal::ImageForTimeBinkDec( int thisTime, nvrhi::ICommandList* commandList )
{
@ -1510,17 +1582,7 @@ cinData_t idCinematicLocal::ImageForTimeBinkDec( int thisTime, nvrhi::ICommandLi
int16_t* audioBuffer = NULL;
uint32_t num_bytes = 0;
if( thisTime <= 0 )
{
thisTime = Sys_Milliseconds();
}
memset( &cinData, 0, sizeof( cinData ) );
if( r_skipDynamicTextures.GetBool() || status == FMV_EOF || status == FMV_IDLE )
{
return cinData;
}
if( !binkHandle.isValid )
{
// RB: .bik requested but not found
@ -1650,7 +1712,7 @@ cinData_t idCinematicLocal::ImageForTimeBinkDec( int thisTime, nvrhi::ICommandLi
cinData.imageCr = imgCr;
cinData.imageCb = imgCb;
if( audioTracks > 0 )
if( cinematicAudio )
{
audioBuffer = ( int16_t* )Mem_Alloc( binkInfo.idealBufferSize, TAG_AUDIO );
num_bytes = Bink_GetAudioData( binkHandle, trackIndex, audioBuffer );
@ -1665,6 +1727,7 @@ cinData_t idCinematicLocal::ImageForTimeBinkDec( int thisTime, nvrhi::ICommandLi
{
// SRS - Even though we have no audio data to play, still need to free the audio buffer
Mem_Free( audioBuffer );
audioBuffer = NULL;
}
}

5
neo/renderer/Image.h
View file

@ -365,6 +365,9 @@ public:
// Adds the image to the list of images to load on the main thread to the gpu.
void DeferredLoadImage();
// Removes the image from the list of images to load on the main thread to the gpu.
void DeferredPurgeImage();
//---------------------------------------------
// Platform specific implementations
//---------------------------------------------
@ -792,4 +795,4 @@ IMAGEPROGRAM
void R_LoadImageProgram( const char* name, byte** pic, int* width, int* height, ID_TIME_T* timestamp, textureUsage_t* usage = NULL );
const char* R_ParsePastImageProgram( idLexer& src );
#endif
#endif

7
neo/renderer/Image_load.cpp
View file

@ -743,6 +743,11 @@ void idImage::DeferredLoadImage()
globalImages->imagesToLoad.AddUnique( this );
}
void idImage::DeferredPurgeImage()
{
globalImages->imagesToLoad.Remove( this );
}
/*
==================
StorageSize
@ -1293,4 +1298,4 @@ void idImage::UploadScratch( const byte* data, int cols, int rows, nvrhi::IComma
}
isLoaded = true;
}
}

3
neo/renderer/NVRHI/Image_NVRHI.cpp
View file

@ -76,6 +76,9 @@ idImage::~idImage
idImage::~idImage()
{
PurgeImage();
// SRS - if image found in deferred load list, remove it now to avoid problems later
DeferredPurgeImage();
}
/*

11
neo/renderer/NVRHI/RenderDebug_NVRHI.cpp
View file

@ -2238,7 +2238,7 @@ Display a single image over most of the screen
*/
void idRenderBackend::DBG_TestImage()
{
idImage* image = NULL;
idImage* image = NULL;
idImage* imageCr = NULL;
idImage* imageCb = NULL;
int max;
@ -2254,9 +2254,9 @@ void idRenderBackend::DBG_TestImage()
{
cinData_t cin;
// SRS - Don't need calibrated time for testing cinematics, so just call ImageForTime( 0 ) for current system time
// SRS - Don't need calibrated time for testing cinematics, so just call ImageForTime( ) with current system time
// This simplification allows cinematic test playback to work over both 2D and 3D background scenes
cin = tr.testVideo->ImageForTime( 0 /*viewDef->renderView.time[1] - tr.testVideoStartTime*/, commandList );
cin = tr.testVideo->ImageForTime( Sys_Milliseconds() /*viewDef->renderView.time[1] - tr.testVideoStartTime*/, commandList );
if( cin.imageY != NULL )
{
image = cin.imageY;
@ -2308,9 +2308,10 @@ void idRenderBackend::DBG_TestImage()
float scale[16] = { 0 };
scale[0] = w; // scale
scale[5] = h; // scale
scale[12] = halfScreenWidth - ( halfScreenWidth * w ); // translate
scale[13] = halfScreenHeight - ( halfScreenHeight * h ); // translate
scale[10] = 1.0f;
scale[12] = halfScreenWidth - ( halfScreenWidth * w ); // translate to center x
scale[13] = halfScreenHeight / 2 - ( halfScreenHeight * h ); // translate to center y of console dropdown
scale[14] = -0.5f; // translate to center z
scale[15] = 1.0f;
float ortho[16] = { 0 };

6
neo/renderer/OpenGL/RenderDebug_GL.cpp
View file

@ -3256,9 +3256,9 @@ void idRenderBackend::DBG_TestImage()
{
cinData_t cin;
// SRS - Don't need calibrated time for testing cinematics, so just call ImageForTime( 0 ) for current system time
// SRS - Don't need calibrated time for testing cinematics, so just call ImageForTime() with current system time
// This simplification allows cinematic test playback to work over both 2D and 3D background scenes
cin = tr.testVideo->ImageForTime( 0 /*viewDef->renderView.time[1] - tr.testVideoStartTime*/ );
cin = tr.testVideo->ImageForTime( Sys_Milliseconds() /*viewDef->renderView.time[1] - tr.testVideoStartTime*/ );
if( cin.imageY != NULL )
{
image = cin.imageY;
@ -3311,7 +3311,7 @@ void idRenderBackend::DBG_TestImage()
scale[0] = w; // scale
scale[5] = h; // scale (SRS - changed h from -ve to +ve so video plays right side up)
scale[12] = halfScreenWidth - ( halfScreenWidth * w ); // translate
scale[13] = halfScreenHeight - ( halfScreenHeight * h ) - h; // translate (SRS - moved up by h)
scale[13] = halfScreenHeight / 2 - ( halfScreenHeight * h ); // translate (SRS - center of console dropdown)
scale[10] = 1.0f;
scale[15] = 1.0f;

23
neo/renderer/RenderSystem_init.cpp
View file

@ -704,14 +704,9 @@ void R_TestVideo_f( const idCmdArgs& args )
tr.testImage = globalImages->ImageFromFile( "_scratch", TF_DEFAULT, TR_REPEAT, TD_DEFAULT );
tr.testVideo = idCinematic::Alloc();
tr.testVideo->InitFromFile( args.Argv( 1 ), true, NULL );
cinData_t cin;
// FIXME commandList
cin = tr.testVideo->ImageForTime( 0, NULL );
// SRS - Also handle ffmpeg and original RoQ decoders for test videos (using cin.image)
if( cin.imageY == NULL && cin.image == NULL )
// SRS - make sure we have a valid bink, ffmpeg, or RoQ video file, otherwise delete testVideo and return
// SRS - no need to call ImageForTime() here, playback is handled within idRenderBackend::DBG_TestImage()
if( !tr.testVideo->InitFromFile( args.Argv( 1 ), true, NULL ) )
{
delete tr.testVideo;
tr.testVideo = NULL;
@ -719,11 +714,6 @@ void R_TestVideo_f( const idCmdArgs& args )
return;
}
common->Printf( "%i x %i images\n", cin.imageWidth, cin.imageHeight );
int len = tr.testVideo->AnimationLength();
common->Printf( "%5.1f seconds of video\n", len * 0.001 );
// try to play the matching wav file
idStr wavString = args.Argv( ( args.Argc() == 2 ) ? 1 : 2 );
wavString.StripFileExtension();
@ -2258,6 +2248,13 @@ void idRenderSystemLocal::Shutdown()
renderModelManager->Shutdown();
// SRS - if testVideo is currently playing, make sure cinematic is deleted before ShutdownCinematic()
if( tr.testVideo )
{
delete tr.testVideo;
tr.testVideo = NULL;
}
idCinematic::ShutdownCinematic();
globalImages->Shutdown();

3
neo/renderer/Vulkan/Image_VK.cpp
View file

@ -590,7 +590,8 @@ void idImage::AllocImage()
viewCreateInfo.viewType = ( opts.textureType == TT_CUBIC ) ? VK_IMAGE_VIEW_TYPE_CUBE : VK_IMAGE_VIEW_TYPE_2D;
viewCreateInfo.format = internalFormat;
viewCreateInfo.components = VK_GetComponentMappingFromTextureFormat( opts.format, opts.colorFormat );
viewCreateInfo.subresourceRange.aspectMask = ( opts.format == FMT_DEPTH ) ? VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT : VK_IMAGE_ASPECT_COLOR_BIT;
// SRS - Added FMT_DEPTH_STENCIL case
viewCreateInfo.subresourceRange.aspectMask = ( opts.format == FMT_DEPTH || opts.format == FMT_DEPTH_STENCIL ) ? VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT : VK_IMAGE_ASPECT_COLOR_BIT;
viewCreateInfo.subresourceRange.levelCount = opts.numLevels;
viewCreateInfo.subresourceRange.layerCount = ( opts.textureType == TT_CUBIC ) ? 6 : 1;
viewCreateInfo.subresourceRange.baseMipLevel = 0;

6
neo/renderer/Vulkan/RenderDebug_VK.cpp
View file

@ -546,9 +546,9 @@ void idRenderBackend::DBG_TestImage()
{
cinData_t cin;
// SRS - Don't need calibrated time for testing cinematics, so just call ImageForTime( 0 ) for current system time
// SRS - Don't need calibrated time for testing cinematics, so just call ImageForTime() with current system time
// This simplification allows cinematic test playback to work over both 2D and 3D background scenes
cin = tr.testVideo->ImageForTime( 0 /*viewDef->renderView.time[1] - tr.testVideoStartTime*/ );
cin = tr.testVideo->ImageForTime( Sys_Milliseconds() /*viewDef->renderView.time[1] - tr.testVideoStartTime*/ );
if( cin.imageY != NULL )
{
image = cin.imageY;
@ -600,7 +600,7 @@ void idRenderBackend::DBG_TestImage()
scale[0] = w; // scale
scale[5] = h; // scale
scale[12] = halfScreenWidth - ( halfScreenWidth * w ); // translate
scale[13] = halfScreenHeight - ( halfScreenHeight * h ) - h; // translate (SRS - moved up by h)
scale[13] = halfScreenHeight / 2 - ( halfScreenHeight * h ); // translate (SRS - center of console dropdown)
scale[10] = 1.0f;
scale[15] = 1.0f;

1
neo/sound/CinematicAudio.h
View file

@ -28,6 +28,7 @@
class CinematicAudio
{
public:
virtual ~CinematicAudio();
virtual void InitAudio( void* audioContext ) = 0;
virtual void PlayAudio( uint8_t* data, int size ) = 0;
virtual void ResetAudio() = 0;

53
neo/sound/OpenAL/AL_CinematicAudio.cpp
View file

@ -66,19 +66,31 @@ void CinematicAudio_OpenAL::InitAudio( void* audioContext )
case AV_SAMPLE_FMT_U8:
case AV_SAMPLE_FMT_U8P:
{
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(59,37,100)
av_sample_cin = dec_ctx2->ch_layout.nb_channels == 2 ? AL_FORMAT_STEREO8 : AL_FORMAT_MONO8;
#else
av_sample_cin = dec_ctx2->channels == 2 ? AL_FORMAT_STEREO8 : AL_FORMAT_MONO8;
#endif
break;
}
case AV_SAMPLE_FMT_S16:
case AV_SAMPLE_FMT_S16P:
{
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(59,37,100)
av_sample_cin = dec_ctx2->ch_layout.nb_channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16;
#else
av_sample_cin = dec_ctx2->channels == 2 ? AL_FORMAT_STEREO16 : AL_FORMAT_MONO16;
#endif
break;
}
case AV_SAMPLE_FMT_FLT:
case AV_SAMPLE_FMT_FLTP:
{
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(59,37,100)
av_sample_cin = dec_ctx2->ch_layout.nb_channels == 2 ? AL_FORMAT_STEREO_FLOAT32 : AL_FORMAT_MONO_FLOAT32;
#else
av_sample_cin = dec_ctx2->channels == 2 ? AL_FORMAT_STEREO_FLOAT32 : AL_FORMAT_MONO_FLOAT32;
#endif
break;
}
default:
@ -105,29 +117,32 @@ void CinematicAudio_OpenAL::PlayAudio( uint8_t* data, int size )
alGetSourcei( alMusicSourceVoicecin, AL_SOURCE_STATE, &state );
alGetSourcei( alMusicSourceVoicecin, AL_BUFFERS_PROCESSED, &processed );
//common->Printf( "AL_CinematicAudio: processed = %2d, bufids = %2d, tBuffers = %2d, state = %d\n", processed, bufids.size(), tBuffer.size(), state );
if( trigger )
{
tBuffer.push( data );
sizes.push( size );
ALuint bufid;
// SRS - Unqueue all processed alBuffers and place them on the free bufids queue
while( processed > 0 )
{
ALuint bufid;
alSourceUnqueueBuffers( alMusicSourceVoicecin, 1, &bufid );
bufids.push( bufid );
processed--;
}
// SRS - Only unqueue an alBuffer if we don't already have a free bufid to use
if( bufids.size() == 0 )
{
alSourceUnqueueBuffers( alMusicSourceVoicecin, 1, &bufid );
bufids.push( bufid );
processed--;
}
if( !tBuffer.empty() )
tBuffer.push( data );
sizes.push( size );
while( !tBuffer.empty() )
{
// SRS - If we have an audio buffer ready to play and a free bufid, then queue it up
if( !bufids.empty() )
{
uint8_t* tempdata = tBuffer.front();
tBuffer.pop();
int tempSize = sizes.front();
sizes.pop();
if( tempSize > 0 )
if( tempdata )
{
bufid = bufids.front();
bufids.pop();
@ -147,7 +162,8 @@ void CinematicAudio_OpenAL::PlayAudio( uint8_t* data, int size )
}
}
}
else // SRS - When no new audio frames left to queue, break and continue playing
// SRS - If there are no available bufids remaining, break and continue playing
else
{
break;
}
@ -161,17 +177,24 @@ void CinematicAudio_OpenAL::PlayAudio( uint8_t* data, int size )
av_freep( &data );
#elif defined(USE_BINKDEC)
Mem_Free( data );
data = NULL;
#endif
offset++;
if( offset == NUM_BUFFERS )
// SRS - Initiate playback trigger once we have MIN_BUFFERS filled: limit startup latency
if( offset == MIN_BUFFERS )
{
alSourceQueueBuffers( alMusicSourceVoicecin, offset, alMusicBuffercin );
alSourceQueueBuffers( alMusicSourceVoicecin, MIN_BUFFERS, &alMusicBuffercin[0] );
ALenum error = alGetError();
if( error != AL_NO_ERROR )
{
common->Warning( "OpenAL Cinematic: %s\n", alGetString( error ) );
return;
}
// SRS - Prepare additional free buffers to handle variable packet rate codecs (e.g. webm vorbis)
for( int i = MIN_BUFFERS; i < NUM_BUFFERS; i++ )
{
bufids.push( alMusicBuffercin[ i ] );
}
trigger = true;
}
}

5
neo/sound/OpenAL/AL_CinematicAudio.h
View file

@ -33,7 +33,8 @@
#define __CINEMATIC_AUDIO_AL_H__
#include <queue>
#define NUM_BUFFERS 4
#define MIN_BUFFERS 4 // SRS - Minimum buffers to fill before triggering playback
#define NUM_BUFFERS 16 // SRS - Total buffers available to support variable rate codecs
class CinematicAudio_OpenAL: public CinematicAudio
{
@ -58,7 +59,7 @@ private:
// then start popping those frames instead of the current, so we don't lose any audio frames and the sound doesn't crack anymore.
std::queue<uint8_t*> tBuffer;
std::queue<int> sizes;
std::queue<ALuint> bufids; // SRS - Added queue of free alBuffer ids to handle audio frame underflow (starvation) case
std::queue<ALuint> bufids; // SRS - Added queue of free alBuffer ids to handle variable rate codecs
};
#endif

12
neo/sound/XAudio2/XA2_CinematicAudio.cpp
View file

@ -53,6 +53,7 @@ public:
av_freep( &data );
#elif defined(USE_BINKDEC)
Mem_Free( data );
data = NULL;
#endif
}
//Unused methods are stubs
@ -107,7 +108,11 @@ void CinematicAudio_XAudio2::InitAudio( void* audioContext )
return;
}
}
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(59,37,100)
voiceFormatcine.nChannels = dec_ctx2->ch_layout.nb_channels; //fixed
#else
voiceFormatcine.nChannels = dec_ctx2->channels; //fixed
#endif
voiceFormatcine.nSamplesPerSec = dec_ctx2->sample_rate; //fixed
#elif defined(USE_BINKDEC)
AudioInfo* binkInfo = ( AudioInfo* )audioContext;
@ -172,6 +177,13 @@ void CinematicAudio_XAudio2::PlayAudio( uint8_t* data, int size )
HRESULT hr;
if( FAILED( hr = pMusicSourceVoice1->SubmitSourceBuffer( &Packet ) ) )
{
// SRS - We should free the audio buffer if XAudio2 buffer submit failed
#if defined(USE_FFMPEG)
av_freep( &data );
#elif defined(USE_BINKDEC)
Mem_Free( data );
data = NULL;
#endif
int fail = 1;
}

4
neo/sound/snd_decoder.cpp
View file

@ -330,7 +330,7 @@ bool idSoundDecoder_Vorbis::Open( const char* fileName )
return false;
}
this->sample = sample;
//this->sample = sample; // SRS - self assignment not needed here
return true;
}
}