From 5d00b96e5f8bc7971e58ca7c0dff02fe4e910147 Mon Sep 17 00:00:00 2001 From: Chris Robinson Date: Sun, 2 Oct 2022 02:54:36 -0700 Subject: [PATCH] Read Smacker video and audio data separately --- src/common/cutscenes/movieplayer.cpp | 63 +++++-- .../libsmackerdec/include/SmackerDecoder.h | 21 ++- .../libsmackerdec/src/SmackerDecoder.cpp | 162 ++++++++++++++---- 3 files changed, 192 insertions(+), 54 deletions(-) diff --git a/src/common/cutscenes/movieplayer.cpp b/src/common/cutscenes/movieplayer.cpp index b6ec526176..72e4705a20 100644 --- a/src/common/cutscenes/movieplayer.cpp +++ b/src/common/cutscenes/movieplayer.cpp @@ -640,14 +640,39 @@ class SmkPlayer : public MoviePlayer public: bool isvalid() { return hSMK.isValid; } - static bool StreamCallbackFunc(SoundStream* stream, void* buff, int len, void* userdata) + bool StreamCallback(SoundStream* stream, void* buff, int len) { - SmkPlayer* pId = (SmkPlayer*)userdata; - memcpy(buff, &pId->adata.samples[pId->adata.nRead], len); - pId->adata.nRead += len / 2; - if (pId->adata.nRead >= (int)countof(pId->adata.samples)) pId->adata.nRead = 0; + int avail = (adata.nWrite >= adata.nRead) ? + adata.nWrite - adata.nRead : + (std::size(adata.samples) + adata.nWrite - adata.nRead); + avail *= 2; + + while (avail < len) + { + auto read = Smacker_GetAudioData(hSMK, 0, (int16_t*)audioBuffer.Data()); + if (read == 0) break; + + if (adata.inf.bitsPerSample == 8) copy8bitSamples(read); + else copy16bitSamples(read); + avail += read; + } + + if (avail == 0) + return false; + + if (avail > 0) + { + int remaining = std::min(len, avail); + memcpy(buff, &adata.samples[adata.nRead], remaining); + adata.nRead += remaining / 2; + if (adata.nRead >= (int)std::size(adata.samples)) adata.nRead = 0; + } + if (len > avail) + memset((char*)buff+avail, 0, len-avail); return true; } + static bool StreamCallbackC(SoundStream* stream, void* buff, int len, void* userdata) + { return static_cast(userdata)->StreamCallback(stream, buff, len); } void copy8bitSamples(unsigned count) { @@ -685,7 +710,7 @@ public: Smacker_GetPalette(hSMK, palette); numAudioTracks = Smacker_GetNumAudioTracks(hSMK); - if (numAudioTracks) + if (numAudioTracks && SoundEnabled()) { adata.nWrite = 0; adata.nRead = 0; @@ -693,17 +718,16 @@ public: if (adata.inf.idealBufferSize > 0) { audioBuffer.Resize(adata.inf.idealBufferSize); - auto read = Smacker_GetAudioData(hSMK, 0, (int16_t*)audioBuffer.Data()); - if (adata.inf.bitsPerSample == 8) copy8bitSamples(read); - else copy16bitSamples(read); hassound = true; } } if (!hassound) { adata.inf = {}; + Smacker_DisableAudioTrack(hSMK, 0); } - + for (int i = 1;i < numAudioTracks;++i) + Smacker_DisableAudioTrack(hSMK, i); } //--------------------------------------------------------------------------- @@ -715,6 +739,7 @@ public: void Start() override { animtex.SetSize(AnimTexture::Paletted, nWidth, nHeight); + if (stream) stream->SetPaused(false); } //--------------------------------------------------------------------------- @@ -733,16 +758,17 @@ public: Smacker_GetPalette(hSMK, palette); Smacker_GetFrame(hSMK, pFrame.Data()); animtex.SetFrame(palette, pFrame.Data()); - if (numAudioTracks && SoundEnabled()) + if (!stream && hassound) { - auto read = Smacker_GetAudioData(hSMK, 0, (int16_t*)audioBuffer.Data()); - if (adata.inf.bitsPerSample == 8) copy8bitSamples(read); - else copy16bitSamples(read); - if (!stream && read) // the sound may not start in the first frame, but the stream cannot start without any sound data present. - stream = S_CreateCustomStream(6000, adata.inf.sampleRate, adata.inf.nChannels, MusicSamples16bit, StreamCallbackFunc, this); + S_StopMusic(true); + stream = S_CreateCustomStream(6000, adata.inf.sampleRate, adata.inf.nChannels, MusicSamples16bit, StreamCallbackC, this); + if (!stream) + { + Smacker_DisableAudioTrack(hSMK, 0); + hassound = false; + } } - } if (frame > nFrame) @@ -768,13 +794,14 @@ public: void Stop() override { - if (stream) S_StopCustomStream(stream); + if (stream) stream->SetPaused(true); bool nostopsound = (flags & NOSOUNDCUTOFF); if (!nostopsound && !hassound) soundEngine->StopAllChannels(); } ~SmkPlayer() { + if (stream) S_StopCustomStream(stream); Smacker_Close(hSMK); animtex.Clean(); } diff --git a/src/common/thirdparty/libsmackerdec/include/SmackerDecoder.h b/src/common/thirdparty/libsmackerdec/include/SmackerDecoder.h index df890d8c21..5e7e436419 100644 --- a/src/common/thirdparty/libsmackerdec/include/SmackerDecoder.h +++ b/src/common/thirdparty/libsmackerdec/include/SmackerDecoder.h @@ -48,6 +48,9 @@ #include #include "FileStream.h" #include "BitReader.h" +#include +#include +#include #include // exportable interface @@ -71,6 +74,7 @@ void Smacker_Close (SmackerHandle &handle); uint32_t Smacker_GetNumAudioTracks (SmackerHandle &handle); SmackerAudioInfo Smacker_GetAudioTrackDetails (SmackerHandle &handle, uint32_t trackIndex); uint32_t Smacker_GetAudioData (SmackerHandle &handle, uint32_t trackIndex, int16_t *data); +void Smacker_DisableAudioTrack (SmackerHandle &handle, uint32_t trackIndex); uint32_t Smacker_GetNumFrames (SmackerHandle &handle); void Smacker_GetFrameSize (SmackerHandle &handle, uint32_t &width, uint32_t &height); uint32_t Smacker_GetCurrentFrameNum (SmackerHandle &handle); @@ -86,6 +90,12 @@ const int kMaxAudioTracks = 7; struct HuffContext; struct DBCtx; +struct SmackerPacket +{ + size_t size = 0; + std::unique_ptr data; +}; + struct SmackerAudioTrack { uint32_t sizeInBytes; @@ -99,6 +109,7 @@ struct SmackerAudioTrack uint32_t bufferSize; uint32_t bytesReadThisFrame; + std::deque packetData; }; class SmackerDecoder @@ -116,6 +127,7 @@ class SmackerDecoder SmackerAudioInfo GetAudioTrackDetails(uint32_t trackIndex); uint32_t GetAudioData(uint32_t trackIndex, int16_t *audioBuffer); + void DisableAudioTrack(uint32_t trackIndex); uint32_t GetNumFrames(); uint32_t GetCurrentFrameNum(); float GetFrameRate(); @@ -125,6 +137,7 @@ class SmackerDecoder private: SmackerCommon::FileStream file; char signature[4]; + std::mutex fileMutex; // video related members uint32_t nFrames; @@ -135,8 +148,10 @@ class SmackerDecoder bool isVer4; + uint32_t currentReadFrame; std::vector packetData; - uint8_t *packetDataPtr = nullptr; + + std::deque framePacketData; SmackerAudioTrack audioTracks[kMaxAudioTracks]; uint32_t treeSize; @@ -162,9 +177,9 @@ class SmackerDecoder int DecodeBigTree(SmackerCommon::BitReader &bits, HuffContext *hc, DBCtx *ctx); int GetCode(SmackerCommon::BitReader &bits, std::vector &recode, int *last); int ReadPacket(); - int DecodeFrame(uint32_t frameSize); + int DecodeFrame(const uint8_t *dataPtr, uint32_t frameSize); void GetFrameSize(uint32_t &width, uint32_t &height); - int DecodeAudio(uint32_t size, SmackerAudioTrack &track); + int DecodeAudio(const uint8_t *dataPtr, uint32_t size, SmackerAudioTrack &track); }; #endif diff --git a/src/common/thirdparty/libsmackerdec/src/SmackerDecoder.cpp b/src/common/thirdparty/libsmackerdec/src/SmackerDecoder.cpp index 18f5014457..5539181d45 100644 --- a/src/common/thirdparty/libsmackerdec/src/SmackerDecoder.cpp +++ b/src/common/thirdparty/libsmackerdec/src/SmackerDecoder.cpp @@ -116,6 +116,18 @@ uint32_t Smacker_GetAudioData(SmackerHandle &handle, uint32_t trackIndex, int16_ return classInstances[handle.instanceIndex]->GetAudioData(trackIndex, data); } +/* Disables an audio track if it's enabled. + * + * When getting video or audio data, encoded data is stored for other tracks so + * they can be read and decoded later. This function prevents a build-up of + * encoded audio data for the specified track if such data isn't going to be + * decoded and read by the caller. + */ +void Smacker_DisableAudioTrack(SmackerHandle &handle, uint32_t trackIndex) +{ + classInstances[handle.instanceIndex]->DisableAudioTrack(trackIndex); +} + uint32_t Smacker_GetNumFrames(SmackerHandle &handle) { return classInstances[handle.instanceIndex]->GetNumFrames(); @@ -166,6 +178,7 @@ void Smacker_GotoFrame(SmackerHandle &handle, uint32_t frameNum) SmackerDecoder::SmackerDecoder() { isVer4 = false; + currentReadFrame = 0; currentFrame = 0; picture = 0; nextPos = 0; @@ -224,6 +237,7 @@ static const uint8_t smk_pal[64] = { enum SAudFlags { SMK_AUD_PACKED = 0x80000000, + SMK_AUD_PRESENT = 0x40000000, SMK_AUD_16BITS = 0x20000000, SMK_AUD_STEREO = 0x10000000, SMK_AUD_BINKAUD = 0x08000000, @@ -384,6 +398,10 @@ bool SmackerDecoder::Open(const char *fileName) audioTracks[i].bufferSize = 0; audioTracks[i].bytesReadThisFrame = 0; + // FIXME: Only disable non-consecutive enabled tracks + if (i > 0 /*&& !(audioTracks[i-1].flags & SMK_AUD_PRESENT)*/) + audioTracks[i].flags &= ~SMK_AUD_PRESENT; + if (audioTracks[i].flags & 0xFFFFFF) { /* @@ -696,24 +714,22 @@ bool SmackerDecoder::DecodeHeaderTrees() void SmackerDecoder::GetNextFrame() { - ReadPacket(); -} - -int SmackerDecoder::ReadPacket() -{ - // test-remove if (currentFrame >= nFrames) - return 1; + return; - // seek to next frame position - file.Seek(nextPos, SmackerCommon::FileStream::kSeekStart); + std::unique_lock flock(fileMutex); + while (framePacketData.empty()) + { + if (ReadPacket() > 0) + return; + } + SmackerPacket pkt = std::move(framePacketData.front()); + framePacketData.pop_front(); - uint32_t frameSize = frameSizes[currentFrame] & (~3); + uint32_t frameSize = pkt.size; uint8_t frameFlag = frameFlags[currentFrame]; - packetData.resize(frameSize); - packetDataPtr = packetData.data(); - file.ReadBytes(packetDataPtr, frameSize); + const uint8_t *packetDataPtr = pkt.data.get(); // handle palette change if (frameFlag & kSMKpal) @@ -721,7 +737,7 @@ int SmackerDecoder::ReadPacket() int size, sz, t, off, j; uint8_t *pal = palette; uint8_t oldpal[768]; - uint8_t *dataEnd; + const uint8_t *dataEnd; memcpy(oldpal, pal, 768); size = *(packetDataPtr++); @@ -754,44 +770,87 @@ int SmackerDecoder::ReadPacket() sz++; } } - + packetDataPtr = dataEnd; } - frameFlag >>= 1; + if (frameSize > 0) + DecodeFrame(packetDataPtr, frameSize); - // check for and handle audio - for (int i = 0; i < kMaxAudioTracks; i++) + currentFrame++; +} + +int SmackerDecoder::ReadPacket() +{ + // test-remove + if (currentReadFrame >= nFrames) + return 1; + + // seek to next frame position + file.Seek(nextPos, SmackerCommon::FileStream::kSeekStart); + + uint32_t frameSize = frameSizes[currentReadFrame] & (~3); + uint8_t frameFlag = frameFlags[currentReadFrame]; + + packetData.resize(frameSize); + file.ReadBytes(packetData.data(), frameSize); + + const uint8_t *packetDataPtr = packetData.data(); + + // skip pal data for later, after getting audio data + if (frameFlag & kSMKpal) { - audioTracks[i].bytesReadThisFrame = 0; + int size = (*packetDataPtr * 4); + packetDataPtr += size; + frameSize -= size; + } - if (frameFlag & 1) + frameFlag >>= 1; + for (int i = 0; i < kMaxAudioTracks; i++) + { + if (frameFlag & 1) { uint32_t size = *(packetDataPtr++); size |= *(packetDataPtr++) << 8; size |= *(packetDataPtr++) << 16; size |= *(packetDataPtr++) << 24; frameSize -= size; + size -= 4; - DecodeAudio(size-4, audioTracks[i]); + if (audioTracks[i].flags & SMK_AUD_PRESENT) + { + SmackerPacket pkt; + pkt.size = size; + pkt.data = std::make_unique(size); + memcpy(pkt.data.get(), packetDataPtr, size); + audioTracks[i].packetData.emplace_back(std::move(pkt)); + } + packetDataPtr += size; } frameFlag >>= 1; } - if (frameSize == 0) { - return -1; + SmackerPacket pkt; + frameFlag = frameFlags[currentReadFrame]; + if (frameSize != 0 || (frameFlag & kSMKpal)) + { + int palsize = (frameFlag & kSMKpal) ? packetData[0] * 4 : 0; + int totalSize = frameSize + palsize; + pkt.size = totalSize; + pkt.data = std::make_unique(totalSize); + memcpy(pkt.data.get(), packetData.data(), palsize); + memcpy(pkt.data.get()+palsize, packetDataPtr, frameSize); } + framePacketData.emplace_back(std::move(pkt)); - DecodeFrame(frameSize); - - currentFrame++; + ++currentReadFrame; nextPos = file.GetPosition(); return 0; } -int SmackerDecoder::DecodeFrame(uint32_t frameSize) +int SmackerDecoder::DecodeFrame(const uint8_t *dataPtr, uint32_t frameSize) { last_reset(mmap_tbl, mmap_last); last_reset(mclr_tbl, mclr_last); @@ -811,8 +870,8 @@ int SmackerDecoder::DecodeFrame(uint32_t frameSize) stride = frameWidth; - SmackerCommon::BitReader bits(packetDataPtr, frameSize); - packetDataPtr += frameSize; + SmackerCommon::BitReader bits(dataPtr, frameSize); + dataPtr += frameSize; while (blk < blocks) { @@ -948,7 +1007,7 @@ int SmackerDecoder::DecodeFrame(uint32_t frameSize) /** * Decode Smacker audio data */ -int SmackerDecoder::DecodeAudio(uint32_t size, SmackerAudioTrack &track) +int SmackerDecoder::DecodeAudio(const uint8_t *dataPtr, uint32_t size, SmackerAudioTrack &track) { SmackerCommon::VLCtable vlc[4]; int val; @@ -967,8 +1026,7 @@ int SmackerDecoder::DecodeAudio(uint32_t size, SmackerAudioTrack &track) return -1; } - SmackerCommon::BitReader bits(packetDataPtr, size); - packetDataPtr += size; + SmackerCommon::BitReader bits(dataPtr, size); unpackedSize = bits.GetBits(32); @@ -1108,11 +1166,22 @@ void SmackerDecoder::GotoFrame(uint32_t frameNum) // file.Seek(firstFrameFilePos, SmackerCommon::FileStream::kSeekStart); + currentReadFrame = 0; currentFrame = 0; nextPos = firstFrameFilePos; - for (unsigned i = 0; i < frameNum + 1; i++) + framePacketData.clear(); + for (unsigned j = 0; j < kMaxAudioTracks; j++) + audioTracks[j].packetData.clear(); + + for (unsigned i = 0; i < frameNum; i++) + { GetNextFrame(); + for (unsigned j = 0; j < kMaxAudioTracks; j++) + audioTracks[j].packetData.clear(); + } + + GetNextFrame(); } SmackerAudioInfo SmackerDecoder::GetAudioTrackDetails(uint32_t trackIndex) @@ -1138,9 +1207,36 @@ uint32_t SmackerDecoder::GetAudioData(uint32_t trackIndex, int16_t *audioBuffer) SmackerAudioTrack *track = &audioTracks[trackIndex]; + std::unique_lock flock(fileMutex); + if (!(track->flags & SMK_AUD_PRESENT)) + return 0; + + while (track->packetData.empty()) + { + if (ReadPacket() > 0) + return 0; + } + SmackerPacket pkt = std::move(track->packetData.front()); + track->packetData.pop_front(); + + track->bytesReadThisFrame = 0; + + const uint8_t *packetDataPtr = pkt.data.get(); + uint32_t size = pkt.size; + + DecodeAudio(packetDataPtr, size, *track); + if (track->bytesReadThisFrame) { memcpy(audioBuffer, track->buffer, std::min(track->bufferSize, track->bytesReadThisFrame)); } return track->bytesReadThisFrame; } + +void SmackerDecoder::DisableAudioTrack(uint32_t trackIndex) +{ + SmackerAudioTrack *track = &audioTracks[trackIndex]; + std::unique_lock flock(fileMutex); + track->flags &= ~SMK_AUD_PRESENT; + track->packetData.clear(); +}