zdoom/qzdoom
0
0
Fork 0
mirror of https://github.com/ZDoom/qzdoom.git synced 2025-02-09 01:01:05 +00:00
Code Issues Projects Releases Packages Wiki Activity

Sync movie video playback to the audio, when possible

Browse source
This commit is contained in:
Chris Robinson 2022-10-13 16:13:43 -07:00 committed by Christoph Oelckers
parent f34ba48e3e
commit 9aee7415eb
1 changed files with 63 additions and 224 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

287
src/common/cutscenes/movieplayer.cpp
View file

@ -85,21 +85,9 @@ static constexpr double AudioSyncThreshold{0.03};
class MovieAudioTrack class MovieAudioTrack
{ {
SoundStream *AudioStream = nullptr; SoundStream *AudioStream = nullptr;
std::atomic<uint64_t> ClockTime = 0;
int64_t AudioOffset = 0;
double ClockDiffAvg = 0;
int SampleRate = 0; int SampleRate = 0;
int FrameSize = 0; int FrameSize = 0;
int64_t EndClockDiff = 0;
double FilterDelay(double diff)
{
ClockDiffAvg = ClockDiffAvg*AudioAvgFilterCoeff + diff;
diff = ClockDiffAvg*(1.0 - AudioAvgFilterCoeff);
if(diff < AudioSyncThreshold/2.0 && diff > -AudioSyncThreshold)
return 0.0;
return diff;
}
public: public:
MovieAudioTrack() = default; MovieAudioTrack() = default;
@ -125,27 +113,29 @@ public:
AudioStream = nullptr; AudioStream = nullptr;
} }
void SetClock(uint64_t clock) noexcept uint64_t GetClockTime(uint64_t clock)
{ {
ClockTime.store(clock, std::memory_order_release); // If there's no stream playing, report the frame clock adjusted by the audio
} // end time. This ensures the returned clock keeps incrementing even after
// the audio stopped.
if(!AudioStream || EndClockDiff != 0)
return clock + EndClockDiff;
// NOTE: The callback (and thus GetClockDiff) can be called before auto pos = AudioStream->GetPlayPosition();
// S_CreateCustomStream returns, and thus before AudioStream gets set. So the int64_t postime = static_cast<int64_t>(pos.samplesplayed / double(SampleRate) * 1'000'000'000.0);
// caller needs to pass in the SoundStream given to the callback. postime = std::max<int64_t>(0, postime - pos.latency.count());
double GetClockDiff(SoundStream *stream)
{
// Calculate the difference in time between the movie clock and the audio position.
auto pos = stream->GetPlayPosition();
uint64_t clock = ClockTime.load(std::memory_order_acquire);
return FilterDelay(clock / 1'000'000'000.0 -
double(int64_t(pos.samplesplayed)+AudioOffset) / SampleRate +
pos.latency.count() / 1'000'000'000.0);
}
void AdjustOffset(int adjust) noexcept if(AudioStream->IsEnded())
{ {
AudioOffset += adjust; // If the stream just ended, get the difference between the frame clock and
// the audio end time, so future calls keep incrementing the clock from this
// point. An alternative option may be to allow the AudioStream to hook into
// the audio device clock, which can keep incrementing at the same rate
// without the stream itself actually playing.
EndClockDiff = postime - clock;
}
return static_cast<uint64_t>(postime);
} }
SoundStream *GetAudioStream() const noexcept { return AudioStream; } SoundStream *GetAudioStream() const noexcept { return AudioStream; }
@ -263,52 +253,8 @@ class MvePlayer : public MoviePlayer
MovieAudioTrack audioTrack; MovieAudioTrack audioTrack;
bool failed = false; bool failed = false;
bool StreamCallback(SoundStream *stream, void *buff, int len) bool StreamCallback(SoundStream*, void *buff, int len)
{ {
const double delay = audioTrack.GetClockDiff(stream);
const int samplerate = audioTrack.GetSampleRate();
const int framesize = audioTrack.GetFrameSize();
if(delay > 0.0)
{
// If diff > 0, skip samples. Don't skip more than a full update at once.
int skip = std::min(int(delay*samplerate)*framesize, len);
if(!decoder.FillSamples(buff, len))
return false;
// Offset the measured audio position to account for the skipped samples.
audioTrack.AdjustOffset(skip/framesize);
if(skip == len)
return decoder.FillSamples(buff, len);
memmove(buff, (char*)buff+skip, len-skip);
if(!decoder.FillSamples((char*)buff+len-skip, skip))
memset((char*)buff+len-skip, 0, skip);
return true;
}
if(delay < 0.0)
{
// If diff < 0, duplicate samples. Don't duplicate a full update (we need at
// least one new sample frame to duplicate).
int dup = std::min(int(-delay*samplerate)*framesize, len-framesize);
if(!decoder.FillSamples((char*)buff+dup, len-dup))
return false;
if(framesize == 1)
memset(buff, ((char*)buff)[dup], dup);
else
{
for(int i=0;i < dup;++i)
((char*)buff)[i] = ((char*)buff+dup)[i%framesize];
}
// Offset the measured audio position to account for the duplicated samples.
audioTrack.AdjustOffset(-dup/framesize);
return true;
}
return decoder.FillSamples(buff, len); return decoder.FillSamples(buff, len);
} }
static bool StreamCallbackC(SoundStream *stream, void *buff, int len, void *userdata) static bool StreamCallbackC(SoundStream *stream, void *buff, int len, void *userdata)
@ -332,19 +278,15 @@ public:
{ {
if (failed) return false; if (failed) return false;
audioTrack.SetClock(clock); if (!audioTrack.GetAudioStream() && decoder.HasAudio() && clock != 0)
bool playon = decoder.RunFrame(clock);
if (playon)
{ {
if (!audioTrack.GetAudioStream() && decoder.HasAudio()) S_StopMusic(true);
{ // start audio playback
S_StopMusic(true); if (!audioTrack.Start(decoder.GetSampleRate(), decoder.NumChannels(), MusicSamples16bit, StreamCallbackC, this))
// start audio playback decoder.DisableAudio();
if (!audioTrack.Start(decoder.GetSampleRate(), decoder.NumChannels(), MusicSamples16bit, StreamCallbackC, this))
decoder.DisableAudio();
}
} }
bool playon = decoder.RunFrame(audioTrack.GetClockTime(clock));
return playon; return playon;
} }
@ -396,52 +338,8 @@ class VpxPlayer : public MoviePlayer
public: public:
int soundtrack = -1; int soundtrack = -1;
bool StreamCallback(SoundStream *stream, void *buff, int len) bool StreamCallback(SoundStream*, void *buff, int len)
{ {
const double delay = AudioTrack.GetClockDiff(stream);
const int samplerate = AudioTrack.GetSampleRate();
const int framesize = AudioTrack.GetFrameSize();
if(delay > 0.0)
{
// If diff > 0, skip samples. Don't skip more than a full update at once.
int skip = std::min(int(delay*samplerate)*framesize, len);
if(!ZMusic_FillStream(MusicStream, buff, len))
return false;
// Offset the measured audio position to account for the skipped samples.
AudioTrack.AdjustOffset(skip/framesize);
if(skip == len)
return ZMusic_FillStream(MusicStream, buff, len);
memmove(buff, (char*)buff+skip, len-skip);
if(!ZMusic_FillStream(MusicStream, (char*)buff+len-skip, skip))
memset((char*)buff+len-skip, 0, skip);
return true;
}
if(delay < 0.0)
{
// If diff < 0, duplicate samples. Don't duplicate a full update (we need at
// least one new sample frame to duplicate).
int dup = std::min(int(-delay*samplerate)*framesize, len-framesize);
if(!ZMusic_FillStream(MusicStream, (char*)buff+dup, len-dup))
return false;
if(framesize == 1)
memset(buff, ((char*)buff)[dup], dup);
else
{
for(int i=0;i < dup;++i)
((char*)buff)[i] = ((char*)buff+dup)[i%framesize];
}
// Offset the measured audio position to account for the duplicated samples.
AudioTrack.AdjustOffset(-dup/framesize);
return true;
}
return ZMusic_FillStream(MusicStream, buff, len); return ZMusic_FillStream(MusicStream, buff, len);
} }
static bool StreamCallbackC(SoundStream *stream, void *buff, int len, void *userdata) static bool StreamCallbackC(SoundStream *stream, void *buff, int len, void *userdata)
@ -617,7 +515,6 @@ public:
} }
else if (soundtrack >= 0) else if (soundtrack >= 0)
{ {
S_StopMusic(true);
FileReader reader = fileSystem.OpenFileReader(soundtrack); FileReader reader = fileSystem.OpenFileReader(soundtrack);
if (reader.isOpen()) if (reader.isOpen())
{ {
@ -639,7 +536,28 @@ public:
bool Frame(uint64_t clock) override bool Frame(uint64_t clock) override
{ {
AudioTrack.SetClock(clock); if (!AudioTrack.GetAudioStream() && MusicStream && clock != 0)
{
S_StopMusic(true);
bool ok = false;
SoundStreamInfo info{};
ZMusic_GetStreamInfo(MusicStream, &info);
// if mBufferSize == 0, the music stream is played externally (e.g.
// Windows' MIDI synth), which we can't keep synced. Play anyway?
if (info.mBufferSize > 0 && ZMusic_Start(MusicStream, 0, false))
{
ok = AudioTrack.Start(info.mSampleRate, abs(info.mNumChannels),
(info.mNumChannels < 0) ? MusicSamples16bit : MusicSamplesFloat, &StreamCallbackC, this);
}
if (!ok)
{
ZMusic_Close(MusicStream);
MusicStream = nullptr;
}
}
clock = AudioTrack.GetClockTime(clock);
bool stop = false; bool stop = false;
if (clock >= nextframetime) if (clock >= nextframetime)
@ -658,25 +576,6 @@ public:
framenum++; framenum++;
if (framenum >= numframes) stop = true; if (framenum >= numframes) stop = true;
if (!AudioTrack.GetAudioStream() && MusicStream)
{
bool ok = false;
SoundStreamInfo info{};
ZMusic_GetStreamInfo(MusicStream, &info);
// if mBufferSize == 0, the music stream is played externally (e.g.
// Windows' MIDI synth), which we can't keep synced. Play anyway?
if (info.mBufferSize > 0 && ZMusic_Start(MusicStream, 0, false))
{
ok = AudioTrack.Start(info.mSampleRate, abs(info.mNumChannels),
(info.mNumChannels < 0) ? MusicSamples16bit : MusicSamplesFloat, &StreamCallbackC, this);
}
if (!ok)
{
ZMusic_Close(MusicStream);
MusicStream = nullptr;
}
}
bool nostopsound = (flags & NOSOUNDCUTOFF); bool nostopsound = (flags & NOSOUNDCUTOFF);
int soundframe = convdenom ? Scale(framenum, convnumer, convdenom) : framenum; int soundframe = convdenom ? Scale(framenum, convnumer, convdenom) : framenum;
if (soundframe > lastsoundframe) if (soundframe > lastsoundframe)
@ -764,71 +663,11 @@ public:
bool StreamCallback(SoundStream* stream, void* buff, int len) bool StreamCallback(SoundStream* stream, void* buff, int len)
{ {
const double delay = AudioTrack.GetClockDiff(stream);
const int samplerate = AudioTrack.GetSampleRate(); const int samplerate = AudioTrack.GetSampleRate();
const int framesize = AudioTrack.GetFrameSize(); const int framesize = AudioTrack.GetFrameSize();
int avail = (adata.nWrite - adata.nRead) * 2; int avail = (adata.nWrite - adata.nRead) * 2;
if(delay > 0.0)
{
// If diff > 0, skip samples. Don't skip more than a full update at once.
int skip = std::min(int(delay*samplerate)*framesize, len);
while (skip > 0)
{
if (avail >= skip)
{
AudioTrack.AdjustOffset(skip / framesize);
if (avail > skip)
{
adata.nRead += skip / 2;
avail -= skip;
}
else
adata.nRead = adata.nWrite = avail = 0;
break;
}
AudioTrack.AdjustOffset(avail / framesize);
adata.nWrite = 0;
adata.nRead = 0;
skip -= avail;
avail = 0;
auto read = Smacker_GetAudioData(hSMK, 0, audioBuffer.Data());
if (read == 0) break;
adata.nWrite = read / 2;
avail = read;
}
}
else if(delay < 0.0)
{
// If diff < 0, duplicate samples. Don't duplicate more than a full update.
int dup = std::min(int(-delay*samplerate)*framesize, len-framesize);
if(avail == 0)
{
auto read = Smacker_GetAudioData(hSMK, 0, audioBuffer.Data());
if (read == 0) return false;
adata.nWrite = read / 2;
avail = read;
}
// Offset the measured audio position to account for the duplicated samples.
AudioTrack.AdjustOffset(-dup/framesize);
char *src = (char*)&audioBuffer[adata.nRead];
char *dst = (char*)buff;
for(int i=0;i < dup;++i)
*(dst++) = src[i%framesize];
buff = dst;
len -= dup;
}
int wrote = 0; int wrote = 0;
while(wrote < len) while(wrote < len)
{ {
@ -923,7 +762,18 @@ public:
bool Frame(uint64_t clock) override bool Frame(uint64_t clock) override
{ {
AudioTrack.SetClock(clock); if (!AudioTrack.GetAudioStream() && numAudioTracks && clock != 0)
{
S_StopMusic(true);
if (!AudioTrack.Start(adata.inf.sampleRate, adata.inf.nChannels, MusicSamples16bit, StreamCallbackC, this))
{
Smacker_DisableAudioTrack(hSMK, 0);
numAudioTracks = 0;
}
}
clock = AudioTrack.GetClockTime(clock);
int frame = int(clock / nFrameNs); int frame = int(clock / nFrameNs);
twod->ClearScreen(); twod->ClearScreen();
@ -935,17 +785,6 @@ public:
Smacker_GetFrame(hSMK, pFrame.Data()); Smacker_GetFrame(hSMK, pFrame.Data());
animtex.SetFrame(palette, pFrame.Data()); animtex.SetFrame(palette, pFrame.Data());
if (!AudioTrack.GetAudioStream() && numAudioTracks)
{
S_StopMusic(true);
if (!AudioTrack.Start(adata.inf.sampleRate, adata.inf.nChannels, MusicSamples16bit, StreamCallbackC, this))
{
Smacker_DisableAudioTrack(hSMK, 0);
numAudioTracks = 0;
}
}
bool nostopsound = (flags & NOSOUNDCUTOFF); bool nostopsound = (flags & NOSOUNDCUTOFF);
if (!hassound) for (unsigned i = 0; i < animSnd.Size(); i += 2) if (!hassound) for (unsigned i = 0; i < animSnd.Size(); i += 2)
{ {