zdoom/qzdoom
0
0
Fork 0
mirror of https://github.com/ZDoom/qzdoom.git synced 2024-11-24 04:51:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

- Added input buffering to the Implode and Shrink routines for a marked

Browse source 
speedup.
- Replaced the Shanno-Fano/Huffman reading routines from FZipExploder with
  ones of my own devising, based solely on the specs in the APPNOTE.


SVN r1590 (trunk)
This commit is contained in:
Randy Heit 2009-05-17 03:05:26 +00:00
parent c0d2f78b2a
commit 2e87ebff22
7 changed files with 348 additions and 305 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
docs/rh-log.txt
View file

@ -1,9 +1,16 @@
May 15, 2009
May 16, 2009
- Added input buffering to the Implode and Shrink routines for a marked
speedup.
- Replaced the Shanno-Fano/Huffman reading routines from FZipExploder with
ones of my own devising, based solely on the specs in the APPNOTE.
May 15, 2009
- Found a copy of PKZIP 1.1 and verified that Implode support works with
files that use a literal table and 8k dictionary, and that the just-added
Shrink support works at all.
- Replaced the bit-at-a-time Shannon-Fano decoder from GunZip.c64 with the
word-at-a-time one from 7-Zip.
word-at-a-time one from 7-Zip for a slight speedup when working with
Imploded files.
May 15, 2009 (Changes by Graf Zahl)
- Fixed: Monsters should not check the inventory for damage absorbtion when

2
src/CMakeLists.txt
View file

@ -656,7 +656,7 @@ add_executable( zdoom WIN32
oplsynth/music_opldumper_mididevice.cpp
oplsynth/music_opl_mididevice.cpp
oplsynth/opl_mus_player.cpp
resourcefiles/explode.cpp
resourcefiles/ancientzip.cpp
resourcefiles/file_7z.cpp
resourcefiles/file_grp.cpp
resourcefiles/file_lump.cpp

537
src/resourcefiles/explode.cpp → src/resourcefiles/ancientzip.cpp
View file

@ -1,272 +1,303 @@
/*
gunzip.c by Pasi Ojala, a1bert@iki.fi
http://www.iki.fi/a1bert/
A hopefully easier to understand guide to GZip
(deflate) decompression routine than the GZip
source code.
*/
/*----------------------------------------------------------------------*/
#include <stdlib.h>
#include "explode.h"
static const unsigned char BitReverse[] = {
0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0,
0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0,
0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8,
0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8,
0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4,
0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4,
0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec,
0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc,
0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2,
0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2,
0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea,
0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa,
0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6,
0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6,
0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee,
0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe,
0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1,
0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1,
0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9,
0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9,
0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5,
0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5,
0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed,
0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd,
0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3,
0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3,
0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb,
0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb,
0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7,
0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7,
0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef,
0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff
};
/*
gunzip.c by Pasi Ojala, a1bert@iki.fi
http://www.iki.fi/a1bert/
A hopefully easier to understand guide to GZip
(deflate) decompression routine than the GZip
source code.
*/
/*----------------------------------------------------------------------*/
#include <stdlib.h>
#include "ancientzip.h"
/****************************************************************
Bit-I/O variables and routines/macros
These routines work in the bit level because the target
environment does not have a barrel shifter. Trying to
handle several bits at once would've only made the code
slower.
If the environment supports multi-bit shifts, you should
write these routines again (see e.g. the GZIP sources).
[RH] Since the target environment is not a C64, I did as
suggested and rewrote these using zlib as a reference.
****************************************************************/
#define READBYTE(c) \
do { \
c = 0; \
if (InLeft) { \
InLeft--; \
if (bs < be) \
c = ReadBuf[bs++]; \
else { \
be = In->Read(&ReadBuf, sizeof(ReadBuf)); \
c = ReadBuf[0]; \
bs = 1; \
} \
} \
} while (0)
/* Get a byte of input into the bit accumulator. */
#define PULLBYTE() \
do { \
unsigned char next; \
READBYTE(next); \
Hold += (unsigned int)(next) << Bits; \
Bits += 8; \
} while (0)
/* Assure that there are at least n bits in the bit accumulator. */
#define NEEDBITS(n) \
do { \
while (Bits < (unsigned)(n)) \
PULLBYTE(); \
} while (0)
/* Return the low n bits of the bit accumulator (n < 16) */
#define BITS(n) \
((unsigned)Hold & ((1U << (n)) - 1))
/* Remove n bits from the bit accumulator */
#define DROPBITS(n) \
do { \
Hold >>= (n); \
Bits -= (unsigned)(n); \
} while (0)
#define READBITS(c, a) \
do { \
NEEDBITS(a); \
c = BITS(a); \
DROPBITS(a); \
} while (0)
/****************************************************************
Bit-I/O variables and routines/macros
These routines work in the bit level because the target
environment does not have a barrel shifter. Trying to
handle several bits at once would've only made the code
slower.
If the environment supports multi-bit shifts, you should
write these routines again (see e.g. the GZIP sources).
[RH] Since the target environment is not a C64, I did as
suggested and rewrote these using zlib as a reference.
Shannon-Fano tree routines
****************************************************************/
#define READBYTE(c) \
do { \
c = 0; \
if (InLeft) { \
InLeft--; \
In->Read(&c, 1); \
} \
} while (0)
/* Get a byte of input into the bit accumulator. */
#define PULLBYTE() \
do { \
unsigned char next; \
READBYTE(next); \
Hold += (unsigned int)(next) << Bits; \
Bits += 8; \
} while (0)
/* Assure that there are at least n bits in the bit accumulator. */
#define NEEDBITS(n) \
do { \
while (Bits < (unsigned)(n)) \
PULLBYTE(); \
} while (0)
/* Return the low n bits of the bit accumulator (n < 16) */
#define BITS(n) \
((unsigned)Hold & ((1U << (n)) - 1))
/* Remove n bits from the bit accumulator */
#define DROPBITS(n) \
do { \
Hold >>= (n); \
Bits -= (unsigned)(n); \
} while (0)
#define READBITS(c, a) \
do { \
NEEDBITS(a); \
c = BITS(a); \
DROPBITS(a); \
} while (0)
/* SetCodeLengths() and DecodeSFValue() are from 7-Zip, which is LGPL. */
bool FZipExploder::DecoderBase::SetCodeLengths(const Byte *codeLengths, const int kNumSymbols)
{
int lenCounts[kNumBitsInLongestCode + 2], tmpPositions[kNumBitsInLongestCode + 1];
int i;
for(i = 0; i <= kNumBitsInLongestCode; i++)
lenCounts[i] = 0;
int symbolIndex;
for (symbolIndex = 0; symbolIndex < kNumSymbols; symbolIndex++)
lenCounts[codeLengths[symbolIndex]]++;
Limits[kNumBitsInLongestCode + 1] = 0;
Positions[kNumBitsInLongestCode + 1] = 0;
lenCounts[kNumBitsInLongestCode + 1] = 0;
int startPos = 0;
static const UInt32 kMaxValue = (1 << kNumBitsInLongestCode);
static const unsigned char BitReverse4[] = {
0x00, 0x08, 0x04, 0x0c, 0x02, 0x0a, 0x06, 0x0e,
0x01, 0x09, 0x05, 0x0d, 0x03, 0x0b, 0x07, 0x0f
};
for (i = kNumBitsInLongestCode; i > 0; i--)
{
startPos += lenCounts[i] << (kNumBitsInLongestCode - i);
if (startPos > kMaxValue)
return false;
Limits[i] = startPos;
Positions[i] = Positions[i + 1] + lenCounts[i + 1];
tmpPositions[i] = Positions[i] + lenCounts[i];
}
#define FIRST_BIT_LEN 8
#define REST_BIT_LEN 4
if (startPos != kMaxValue)
return false;
void FZipExploder::InsertCode(TArray<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value)
{
assert(len > 0);
unsigned int node = pos + (code & ((1 << bits) - 1));
for (symbolIndex = 0; symbolIndex < kNumSymbols; symbolIndex++)
if (codeLengths[symbolIndex] != 0)
Symbols[--tmpPositions[codeLengths[symbolIndex]]] = symbolIndex;
return true;
}
int FZipExploder::DecodeSFValue(const DecoderBase &decoder, const unsigned int kNumSymbols)
{
unsigned int numBits = 0;
unsigned int value;
int i;
NEEDBITS(kNumBitsInLongestCode);
// value = BITS(kNumBitsInLongestCode);
// TODO: Rewrite this so it doesn't need the BitReverse table.
// (It should be possible, right?)
value = (BitReverse[Hold & 0xFF] << 8) | BitReverse[(Hold >> 8) & 0xFF];
for(i = kNumBitsInLongestCode; i > 0; i--)
{
if (value < decoder.Limits[i])
if (len > bits)
{ // This code uses more bits than this level has room for. Store the bottom bits
// in this table, then proceed to the next one.
unsigned int child = decoder[node].ChildTable;
if (child == 0)
{ // Need to create child table.
child = InitTable(decoder, 1 << REST_BIT_LEN);
decoder[node].ChildTable = child;
decoder[node].Length = bits;
decoder[node].Value = 0;
}
else
{
numBits = i;
break;
assert(decoder[node].Length == bits);
assert(decoder[node].Value == 0);
}
InsertCode(decoder, child, REST_BIT_LEN, code >> bits, len - bits, value);
}
else
{ // If this code uses fewer bits than this level of the table, it is
// inserted repeatedly for each value that matches it at its lower
// bits.
for (int i = 1 << (bits - len); --i >= 0; node += 1 << len)
{
decoder[node].Length = len;
decoder[node].Value = value;
assert(decoder[node].ChildTable == 0);
}
}
if (i == 0)
return -1;
DROPBITS(numBits);
unsigned int index = decoder.Positions[numBits] +
((value - decoder.Limits[numBits + 1]) >> (kNumBitsInLongestCode - numBits));
if (index >= kNumSymbols)
return -1;
return decoder.Symbols[index];
}
int FZipExploder::DecodeSF(unsigned char *table)
}
unsigned int FZipExploder::InitTable(TArray<HuffNode> &decoder, int numspots)
{
unsigned int start = decoder.Size();
decoder.Reserve(numspots);
memset(&decoder[start], 0, sizeof(HuffNode)*numspots);
return start;
}
int STACK_ARGS FZipExploder::buildercmp(const void *a, const void *b)
{
const TableBuilder *v1 = (const TableBuilder *)a;
const TableBuilder *v2 = (const TableBuilder *)b;
int d = v1->Length - v2->Length;
if (d == 0) {
d = v1->Value - v2->Value;
}
return d;
}
int FZipExploder::BuildDecoder(TArray<HuffNode> &decoder, TableBuilder *values, int numvals)
{
unsigned char a, c;
int i, n, v = 0;
int i;
READBYTE(c);
n = c + 1;
for (i = 0; i < n; i++) {
int nv, bl;
READBYTE(a);
nv = ((a >> 4) & 15) + 1;
bl = (a & 15) + 1;
while (nv--) {
table[v++] = bl;
}
}
return v; /* entries used */
}
qsort(values, numvals, sizeof(*values), buildercmp);
int FZipExploder::Explode(unsigned char *out, unsigned int outsize,
FileReader *in, unsigned int insize,
int flags)
{
int c, i, minMatchLen = 3, len, dist;
int lowDistanceBits;
unsigned char ll[256];
unsigned int bIdx = 0;
Hold = 0;
Bits = 0;
In = in;
InLeft = insize;
if ((flags & 4)) {
/* 3 trees: literals, lengths, distance top 6 */
minMatchLen = 3;
if (!LiteralDecoder.SetCodeLengths(ll, DecodeSF(ll)))
return 1;
} else {
/* 2 trees: lengths, distance top 6 */
minMatchLen = 2;
}
if (!LengthDecoder.SetCodeLengths(ll, DecodeSF(ll)))
return 1;
if (!DistanceDecoder.SetCodeLengths(ll, DecodeSF(ll)))
return 1;
lowDistanceBits = (flags & 2) ? /* 8k dictionary */ 7 : /* 4k dictionary */ 6;
while (bIdx < outsize)
// Generate the Shannon-Fano tree:
unsigned short code = 0;
unsigned short code_increment = 0;
unsigned short last_bit_length = 0;
for (i = numvals - 1; i >= 0; --i)
{
READBITS(c, 1);
if (c) {
/* literal data */
if ((flags & 4)) {
c = DecodeSFValue(LiteralDecoder, 256);
} else {
READBITS(c, 8);
}
out[bIdx++] = c;
} else {
READBITS(dist, lowDistanceBits);
c = DecodeSFValue(DistanceDecoder, 64);
dist |= (c << lowDistanceBits);
len = DecodeSFValue(LengthDecoder, 64);
if (len == 63) {
READBITS(c, 8);
len += c;
}
len += minMatchLen;
dist++;
if (bIdx + len > outsize) {
throw CExplosionError("Not enough output space");
}
if ((unsigned int)dist > bIdx) {
/* Anything before the first input byte is zero. */
int zeros = dist - bIdx;
if (len < zeros)
zeros = len;
for(i = zeros; i; i--)
out[bIdx++] = 0;
len -= zeros;
}
for(i = len; i; i--, bIdx++) {
out[bIdx] = out[bIdx - dist];
}
code += code_increment;
if (values[i].Length != last_bit_length)
{
last_bit_length = values[i].Length;
code_increment = 1 << (16 - last_bit_length);
}
// Reverse the order of the bits in the code before storing it.
values[i].Code = BitReverse4[code >> 12] |
(BitReverse4[(code >> 8) & 0xf] << 4) |
(BitReverse4[(code >> 4) & 0xf] << 8) |
(BitReverse4[code & 0xf] << 12);
}
// Insert each code into the hierarchical table. The top level is FIRST_BIT_LEN bits,
// and the other levels are REST_BIT_LEN bits. If a code does not completely fill
// a level, every permutation for the remaining bits is filled in to
// match this one.
InitTable(decoder, 1 << FIRST_BIT_LEN); // Set up the top level.
for (i = 0; i < numvals; ++i)
{
InsertCode(decoder, 0, FIRST_BIT_LEN, values[i].Code, values[i].Length, values[i].Value);
}
return 0;
}
int FZipExploder::DecodeSFValue(const TArray<HuffNode> &decoder)
{
unsigned int bits = FIRST_BIT_LEN, table = 0, code;
const HuffNode *pos;
do
{
NEEDBITS(bits);
code = BITS(bits);
bits = REST_BIT_LEN;
pos = &decoder[table + code];
DROPBITS(pos->Length);
table = pos->ChildTable;
}
while (table != 0);
return pos->Value;
}
int FZipExploder::DecodeSF(TArray<HuffNode> &decoder, int numvals)
{
TableBuilder builder[256];
unsigned char a, c;
int i, n, v = 0;
READBYTE(c);
n = c + 1;
for (i = 0; i < n; i++) {
int nv, bl;
READBYTE(a);
nv = ((a >> 4) & 15) + 1;
bl = (a & 15) + 1;
while (nv--) {
builder[v].Length = bl;
builder[v].Value = v;
v++;
}
}
if (v != numvals)
return 1; /* bad table */
return BuildDecoder(decoder, builder, v);
}
int FZipExploder::Explode(unsigned char *out, unsigned int outsize,
FileReader *in, unsigned int insize,
int flags)
{
int c, i, minMatchLen = 3, len, dist;
int lowDistanceBits;
unsigned int bIdx = 0;
Hold = 0;
Bits = 0;
In = in;
InLeft = insize;
bs = be = 0;
if ((flags & 4)) {
/* 3 trees: literals, lengths, distance top 6 */
minMatchLen = 3;
if (DecodeSF(LiteralDecoder, 256))
return 1;
} else {
/* 2 trees: lengths, distance top 6 */
minMatchLen = 2;
}
if (DecodeSF(LengthDecoder, 64))
return 1;
if (DecodeSF(DistanceDecoder, 64))
return 1;
lowDistanceBits = (flags & 2) ? /* 8k dictionary */ 7 : /* 4k dictionary */ 6;
while (bIdx < outsize)
{
READBITS(c, 1);
if (c) {
/* literal data */
if ((flags & 4)) {
c = DecodeSFValue(LiteralDecoder);
} else {
READBITS(c, 8);
}
out[bIdx++] = c;
} else {
READBITS(dist, lowDistanceBits);
c = DecodeSFValue(DistanceDecoder);
dist |= (c << lowDistanceBits);
len = DecodeSFValue(LengthDecoder);
if (len == 63) {
READBITS(c, 8);
len += c;
}
len += minMatchLen;
dist++;
if (bIdx + len > outsize) {
throw CExplosionError("Not enough output space");
}
if ((unsigned int)dist > bIdx) {
/* Anything before the first input byte is zero. */
int zeros = dist - bIdx;
if (len < zeros)
zeros = len;
for(i = zeros; i; i--)
out[bIdx++] = 0;
len -= zeros;
}
for(i = len; i; i--, bIdx++) {
out[bIdx] = out[bIdx - dist];
}
}
}
return 0;
}
/* HSIZE is defined as 2^13 (8192) in unzip.h */
#define HSIZE 8192
#define BOGUSCODE 256
@ -277,6 +308,7 @@ int FZipExploder::Explode(unsigned char *out, unsigned int outsize,
int ShrinkLoop(unsigned char *out, unsigned int outsize,
FileReader *In, unsigned int InLeft)
{
unsigned char ReadBuf[256];
unsigned short Parent[HSIZE];
unsigned char Value[HSIZE], Stack[HSIZE];
unsigned char *newstr;
@ -285,6 +317,7 @@ int ShrinkLoop(unsigned char *out, unsigned int outsize,
int code, oldcode, freecode, curcode;
unsigned int Bits = 0, Hold = 0;
unsigned int size = 0;
unsigned int bs = 0, be = 0;
freecode = BOGUSCODE;
for (code = 0; code < BOGUSCODE; code++)

50
src/resourcefiles/ancientzip.h Normal file
View file

@ -0,0 +1,50 @@
#include "files.h"
#include "doomerrors.h"
class FZipExploder
{
unsigned int Hold, Bits;
FileReader *In;
unsigned int InLeft;
/****************************************************************
Shannon-Fano tree structures, variables and related routines
****************************************************************/
struct HuffNode
{
unsigned char Value;
unsigned char Length;
unsigned short ChildTable;
};
struct TableBuilder
{
unsigned char Value;
unsigned char Length;
unsigned short Code;
};
TArray<HuffNode> LiteralDecoder;
TArray<HuffNode> DistanceDecoder;
TArray<HuffNode> LengthDecoder;
unsigned char ReadBuf[256];
unsigned int bs, be;
static int STACK_ARGS buildercmp(const void *a, const void *b);
void InsertCode(TArray<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value);
unsigned int InitTable(TArray<HuffNode> &decoder, int numspots);
int BuildDecoder(TArray<HuffNode> &decoder, TableBuilder *values, int numvals);
int DecodeSFValue(const TArray<HuffNode> &currentTree);
int DecodeSF(TArray<HuffNode> &decoder, int numvals);
public:
int Explode(unsigned char *out, unsigned int outsize, FileReader *in, unsigned int insize, int flags);
};
class CExplosionError : CRecoverableError
{
public:
CExplosionError(const char *message) : CRecoverableError(message) {}
};
int ShrinkLoop(unsigned char *out, unsigned int outsize, FileReader *in, unsigned int insize);

47
src/resourcefiles/explode.h
View file

@ -1,47 +0,0 @@
#include "files.h"
#include "doomerrors.h"
class FZipExploder
{
unsigned int Hold, Bits;
FileReader *In;
unsigned int InLeft;
/****************************************************************
Shannon-Fano tree structures, variables and related routines
****************************************************************/
enum { kNumBitsInLongestCode = 16 };
struct DecoderBase
{
unsigned int Limits[kNumBitsInLongestCode + 2]; // Limits[i] = value limit for symbols with length = i
unsigned char Positions[kNumBitsInLongestCode + 2]; // Positions[i] = index in Symbols[] of first symbol with length = i
unsigned char Symbols[1];
bool SetCodeLengths(const unsigned char *codeLengths, const int kNumSymbols);
};
template<int kNumSymbols>
struct Decoder : DecoderBase
{
unsigned char RestOfSymbols[kNumSymbols];
};
Decoder<256> LiteralDecoder;
Decoder<64> DistanceDecoder;
Decoder<64> LengthDecoder;
int DecodeSFValue(const DecoderBase &currentTree, const unsigned int kNumSymbols);
int DecodeSF(unsigned char *table);
public:
int Explode(unsigned char *out, unsigned int outsize, FileReader *in, unsigned int insize, int flags);
};
class CExplosionError : CRecoverableError
{
public:
CExplosionError(const char *message) : CRecoverableError(message) {}
};
int ShrinkLoop(unsigned char *out, unsigned int outsize, FileReader *in, unsigned int insize);

2
src/resourcefiles/file_zip.cpp
View file

@ -40,7 +40,7 @@
#include "w_wad.h"
#include "w_zip.h"
#include "i_system.h"
#include "explode.h"
#include "ancientzip.h"
#define BUFREADCOMMENT (0x400)

4
zdoom.vcproj
View file

@ -6366,11 +6366,11 @@
Name="Resource Files"
>
<File
RelativePath=".\src\resourcefiles\explode.cpp"
RelativePath=".\src\resourcefiles\ancientzip.cpp"
>
</File>
<File
RelativePath=".\src\resourcefiles\explode.h"
RelativePath=".\src\resourcefiles\ancientzip.h"
>
</File>
<File