- Added input buffering to the Implode and Shrink routines for a marked

speedup.
- Replaced the Shanno-Fano/Huffman reading routines from FZipExploder with
  ones of my own devising, based solely on the specs in the APPNOTE.


SVN r1590 (trunk)

docs/rh-log.txt

@@ -1,9 +1,16 @@
-May 15, 2009
+May 16, 2009
+- Added input buffering to the Implode and Shrink routines for a marked
+  speedup.
+- Replaced the Shanno-Fano/Huffman reading routines from FZipExploder with
+  ones of my own devising, based solely on the specs in the APPNOTE.
+
+May 15, 2009
 - Found a copy of PKZIP 1.1 and verified that Implode support works with
   files that use a literal table and 8k dictionary, and that the just-added
   Shrink support works at all.
 - Replaced the bit-at-a-time Shannon-Fano decoder from GunZip.c64 with the
-  word-at-a-time one from 7-Zip.
+  word-at-a-time one from 7-Zip for a slight speedup when working with
+  Imploded files.
 
 May 15, 2009  (Changes by Graf Zahl)
 - Fixed: Monsters should not check the inventory for damage absorbtion when

src/CMakeLists.txt

@@ -656,7 +656,7 @@ add_executable( zdoom WIN32
 	oplsynth/music_opldumper_mididevice.cpp
 	oplsynth/music_opl_mididevice.cpp
 	oplsynth/opl_mus_player.cpp
-	resourcefiles/explode.cpp
+	resourcefiles/ancientzip.cpp
 	resourcefiles/file_7z.cpp
 	resourcefiles/file_grp.cpp
 	resourcefiles/file_lump.cpp

src/resourcefiles/ancientzip.cpp

@@ -12,42 +12,7 @@
 
 
 #include <stdlib.h>
-#include "explode.h"
+#include "ancientzip.h"
-
-static const unsigned char BitReverse[] = {
-    0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0,
-    0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0,
-    0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8,
-    0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8,
-    0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4,
-    0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4,
-    0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec,
-    0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc,
-    0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2,
-    0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2,
-    0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea,
-    0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa,
-    0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6,
-    0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6,
-    0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee,
-    0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe,
-    0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1,
-    0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1,
-    0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9,
-    0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9,
-    0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5,
-    0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5,
-    0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed,
-    0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd,
-    0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3,
-    0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3,
-    0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb,
-    0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb,
-    0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7,
-    0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7,
-    0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef,
-    0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff
-};
 
 /****************************************************************
     Bit-I/O variables and routines/macros
@@ -70,7 +35,13 @@ static const unsigned char BitReverse[] = {
 		c = 0; \
 		if (InLeft) { \
 			InLeft--; \
-			In->Read(&c, 1); \
+			if (bs < be) \
+				c = ReadBuf[bs++]; \
+			else { \
+				be = In->Read(&ReadBuf, sizeof(ReadBuf)); \
+				c = ReadBuf[0]; \
+				bs = 1; \
+			} \
 		} \
 	} while (0)
 
@@ -108,76 +79,132 @@ static const unsigned char BitReverse[] = {
 		DROPBITS(a); \
 	} while (0)
 
+/****************************************************************
+   Shannon-Fano tree routines
+ ****************************************************************/
 
-/* SetCodeLengths() and DecodeSFValue() are from 7-Zip, which is LGPL. */
+static const unsigned char BitReverse4[] = {
+	0x00, 0x08, 0x04, 0x0c, 0x02, 0x0a, 0x06, 0x0e,
+	0x01, 0x09, 0x05, 0x0d, 0x03, 0x0b, 0x07, 0x0f
+};
 
-bool FZipExploder::DecoderBase::SetCodeLengths(const Byte *codeLengths, const int kNumSymbols)
+#define FIRST_BIT_LEN	8
+#define REST_BIT_LEN	4
+
+void FZipExploder::InsertCode(TArray<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value)
 {
-	int lenCounts[kNumBitsInLongestCode + 2], tmpPositions[kNumBitsInLongestCode + 1];
+	assert(len > 0);
-	int i;
+	unsigned int node = pos + (code & ((1 << bits) - 1));
-	for(i = 0; i <= kNumBitsInLongestCode; i++)
-		lenCounts[i] = 0;
-	int symbolIndex;
-	for (symbolIndex = 0; symbolIndex < kNumSymbols; symbolIndex++)
-		lenCounts[codeLengths[symbolIndex]]++;
 
-	Limits[kNumBitsInLongestCode + 1] = 0;
+	if (len > bits)
-	Positions[kNumBitsInLongestCode + 1] = 0;
+	{ // This code uses more bits than this level has room for. Store the bottom bits
-	lenCounts[kNumBitsInLongestCode + 1] =  0;
+	  // in this table, then proceed to the next one.
-
+		unsigned int child = decoder[node].ChildTable;
-	int startPos = 0;
+		if (child == 0)
-	static const UInt32 kMaxValue = (1 << kNumBitsInLongestCode);
+		{ // Need to create child table.
-
+			child = InitTable(decoder, 1 << REST_BIT_LEN);
-	for (i = kNumBitsInLongestCode; i > 0; i--)
+			decoder[node].ChildTable = child;
-	{
+			decoder[node].Length = bits;
-		startPos += lenCounts[i] << (kNumBitsInLongestCode - i);
+			decoder[node].Value = 0;
-		if (startPos > kMaxValue)
+		}
-			return false;
+		else
-		Limits[i] = startPos;
-		Positions[i] = Positions[i + 1] + lenCounts[i + 1];
-		tmpPositions[i] = Positions[i] + lenCounts[i];
-	}
-
-	if (startPos != kMaxValue)
-		return false;
-
-	for (symbolIndex = 0; symbolIndex < kNumSymbols; symbolIndex++)
-		if (codeLengths[symbolIndex] != 0)
-			Symbols[--tmpPositions[codeLengths[symbolIndex]]] = symbolIndex;
-	return true;
-}
-
-int FZipExploder::DecodeSFValue(const DecoderBase &decoder, const unsigned int kNumSymbols)
-{
-	unsigned int numBits = 0;
-	unsigned int value;
-	int i;
-	NEEDBITS(kNumBitsInLongestCode);
-//	value = BITS(kNumBitsInLongestCode);
-// TODO: Rewrite this so it doesn't need the BitReverse table.
-//		 (It should be possible, right?)
-	value = (BitReverse[Hold & 0xFF] << 8) | BitReverse[(Hold >> 8) & 0xFF];
-	for(i = kNumBitsInLongestCode; i > 0; i--)
-	{
-		if (value < decoder.Limits[i])
 		{
-			numBits = i;
+			assert(decoder[node].Length == bits);
-			break;
+			assert(decoder[node].Value == 0);
+		}
+		InsertCode(decoder, child, REST_BIT_LEN, code >> bits, len - bits, value);
+	}
+	else
+	{ // If this code uses fewer bits than this level of the table, it is
+	  // inserted repeatedly for each value that matches it at its lower
+	  // bits.
+		for (int i = 1 << (bits - len); --i >= 0; node += 1 << len)
+		{
+			decoder[node].Length = len;
+			decoder[node].Value = value;
+			assert(decoder[node].ChildTable == 0);
 		}
 	}
-	if (i == 0)
+}
-		return -1;
+
-	DROPBITS(numBits);
+unsigned int FZipExploder::InitTable(TArray<HuffNode> &decoder, int numspots)
-	unsigned int index = decoder.Positions[numBits] +
+{
-		((value - decoder.Limits[numBits + 1]) >> (kNumBitsInLongestCode - numBits));
+	unsigned int start = decoder.Size();
-	if (index >= kNumSymbols)
+	decoder.Reserve(numspots);
-		return -1;
+	memset(&decoder[start], 0, sizeof(HuffNode)*numspots);
-	return decoder.Symbols[index];
+	return start;
+}
+
+int STACK_ARGS FZipExploder::buildercmp(const void *a, const void *b)
+{
+	const TableBuilder *v1 = (const TableBuilder *)a;
+	const TableBuilder *v2 = (const TableBuilder *)b;
+	int d = v1->Length - v2->Length;
+	if (d == 0) {
+		d = v1->Value - v2->Value;
+	}
+	return d;
+}
+
+int FZipExploder::BuildDecoder(TArray<HuffNode> &decoder, TableBuilder *values, int numvals)
+{
+	int i;
+
+	qsort(values, numvals, sizeof(*values), buildercmp);
+
+	// Generate the Shannon-Fano tree:
+	unsigned short code = 0;
+	unsigned short code_increment = 0;
+	unsigned short last_bit_length = 0;
+	for (i = numvals - 1; i >= 0; --i)
+	{
+		code += code_increment;
+		if (values[i].Length != last_bit_length)
+		{
+			last_bit_length = values[i].Length;
+			code_increment = 1 << (16 - last_bit_length);
+		}
+		// Reverse the order of the bits in the code before storing it.
+		values[i].Code = BitReverse4[code >> 12] |
+						(BitReverse4[(code >> 8) & 0xf] << 4) |
+						(BitReverse4[(code >> 4) & 0xf] << 8) |
+						(BitReverse4[code & 0xf] << 12);
+	}
+
+	// Insert each code into the hierarchical table. The top level is FIRST_BIT_LEN bits,
+	// and the other levels are REST_BIT_LEN bits. If a code does not completely fill
+	// a level, every permutation for the remaining bits is filled in to
+	// match this one.
+	InitTable(decoder, 1 << FIRST_BIT_LEN);		// Set up the top level.
+	for (i = 0; i < numvals; ++i)
+	{
+		InsertCode(decoder, 0, FIRST_BIT_LEN, values[i].Code, values[i].Length, values[i].Value);
+	}
+	return 0;
 }
 
 
-int FZipExploder::DecodeSF(unsigned char *table)
+int FZipExploder::DecodeSFValue(const TArray<HuffNode> &decoder)
 {
+	unsigned int bits = FIRST_BIT_LEN, table = 0, code;
+	const HuffNode *pos;
+	do
+	{
+		NEEDBITS(bits);
+		code = BITS(bits);
+		bits = REST_BIT_LEN;
+		pos = &decoder[table + code];
+		DROPBITS(pos->Length);
+		table = pos->ChildTable;
+	}
+	while (table != 0);
+	return pos->Value;
+}
+
+
+int FZipExploder::DecodeSF(TArray<HuffNode> &decoder, int numvals)
+{
+	TableBuilder builder[256];
 	unsigned char a, c;
 	int i, n, v = 0;
 
@@ -189,10 +216,14 @@ int FZipExploder::DecodeSF(unsigned char *table)
 		nv = ((a >> 4) & 15) + 1;
 		bl = (a & 15) + 1;
 		while (nv--) {
-			table[v++] = bl;
+			builder[v].Length = bl;
+			builder[v].Value = v;
+			v++;
 		}
 	}
-	return v; /* entries used */
+	if (v != numvals)
+		return 1;	/* bad table */
+	return BuildDecoder(decoder, builder, v);
 }
 
 int FZipExploder::Explode(unsigned char *out, unsigned int outsize,
@@ -201,26 +232,26 @@ int FZipExploder::Explode(unsigned char *out, unsigned int outsize,
 {
 	int c, i, minMatchLen = 3, len, dist;
 	int lowDistanceBits;
-	unsigned char ll[256];
 	unsigned int bIdx = 0;
 
 	Hold = 0;
 	Bits = 0;
 	In = in;
 	InLeft = insize;
+	bs = be = 0;
 
 	if ((flags & 4)) {
 		/* 3 trees: literals, lengths, distance top 6 */
 		minMatchLen = 3;
-		if (!LiteralDecoder.SetCodeLengths(ll, DecodeSF(ll)))
+		if (DecodeSF(LiteralDecoder, 256))
 			return 1;
 	} else {
 		/* 2 trees: lengths, distance top 6 */
 		minMatchLen = 2;
 	}
-	if (!LengthDecoder.SetCodeLengths(ll, DecodeSF(ll)))
+	if (DecodeSF(LengthDecoder, 64))
 		return 1;
-	if (!DistanceDecoder.SetCodeLengths(ll, DecodeSF(ll)))
+	if (DecodeSF(DistanceDecoder, 64))
 		return 1;
 
 	lowDistanceBits = (flags & 2) ? /* 8k dictionary */ 7 : /* 4k dictionary */ 6;
@@ -230,16 +261,16 @@ int FZipExploder::Explode(unsigned char *out, unsigned int outsize,
 		if (c) {
 			/* literal data */
 			if ((flags & 4)) {
-				c = DecodeSFValue(LiteralDecoder, 256);
+				c = DecodeSFValue(LiteralDecoder);
 			} else {
 				READBITS(c, 8);
 			}
 			out[bIdx++] = c;
 		} else {
 			READBITS(dist, lowDistanceBits);
-			c = DecodeSFValue(DistanceDecoder, 64);
+			c = DecodeSFValue(DistanceDecoder);
 			dist |= (c << lowDistanceBits);
-			len = DecodeSFValue(LengthDecoder, 64);
+			len = DecodeSFValue(LengthDecoder);
 			if (len == 63) {
 				READBITS(c, 8);
 				len += c;
@@ -277,6 +308,7 @@ int FZipExploder::Explode(unsigned char *out, unsigned int outsize,
 int ShrinkLoop(unsigned char *out, unsigned int outsize,
 			   FileReader *In, unsigned int InLeft)
 {
+	unsigned char ReadBuf[256];
 	unsigned short Parent[HSIZE];
 	unsigned char Value[HSIZE], Stack[HSIZE];
 	unsigned char *newstr;
@@ -285,6 +317,7 @@ int ShrinkLoop(unsigned char *out, unsigned int outsize,
 	int code, oldcode, freecode, curcode;
 	unsigned int Bits = 0, Hold = 0;
 	unsigned int size = 0;
+	unsigned int bs = 0, be = 0;
 
 	freecode = BOGUSCODE;
 	for (code = 0; code < BOGUSCODE; code++)

src/resourcefiles/ancientzip.h

@@ -0,0 +1,50 @@
+#include "files.h"
+#include "doomerrors.h"
+
+class FZipExploder
+{
+	unsigned int Hold, Bits;
+	FileReader *In;
+	unsigned int InLeft;
+
+	/****************************************************************
+	   Shannon-Fano tree structures, variables and related routines
+	 ****************************************************************/
+
+	struct HuffNode
+	{
+		unsigned char Value;
+		unsigned char Length;
+		unsigned short ChildTable;
+	};
+
+	struct TableBuilder
+	{
+		unsigned char Value;
+		unsigned char Length;
+		unsigned short Code;
+	};
+
+	TArray<HuffNode> LiteralDecoder;
+	TArray<HuffNode> DistanceDecoder;
+	TArray<HuffNode> LengthDecoder;
+	unsigned char ReadBuf[256];
+	unsigned int bs, be;
+
+	static int STACK_ARGS buildercmp(const void *a, const void *b);
+	void InsertCode(TArray<HuffNode> &decoder, unsigned int pos, int bits, unsigned short code, int len, unsigned char value);
+	unsigned int InitTable(TArray<HuffNode> &decoder, int numspots);
+	int BuildDecoder(TArray<HuffNode> &decoder, TableBuilder *values, int numvals);
+	int DecodeSFValue(const TArray<HuffNode> &currentTree);
+	int DecodeSF(TArray<HuffNode> &decoder, int numvals);
+public:
+	int Explode(unsigned char *out, unsigned int outsize, FileReader *in, unsigned int insize, int flags);
+};
+
+class CExplosionError : CRecoverableError
+{
+public:
+	CExplosionError(const char *message) : CRecoverableError(message) {}
+};
+
+int ShrinkLoop(unsigned char *out, unsigned int outsize, FileReader *in, unsigned int insize);

src/resourcefiles/explode.h

@@ -1,47 +0,0 @@
-#include "files.h"
-#include "doomerrors.h"
-
-class FZipExploder
-{
-	unsigned int Hold, Bits;
-	FileReader *In;
-	unsigned int InLeft;
-
-	/****************************************************************
-	   Shannon-Fano tree structures, variables and related routines
-	 ****************************************************************/
-
-	enum { kNumBitsInLongestCode = 16 };
-
-	struct DecoderBase
-	{
-		unsigned int Limits[kNumBitsInLongestCode + 2];		// Limits[i] = value limit for symbols with length = i
-		unsigned char Positions[kNumBitsInLongestCode + 2];	// Positions[i] = index in Symbols[] of first symbol with length = i
-		unsigned char Symbols[1];
-
-		bool SetCodeLengths(const unsigned char *codeLengths, const int kNumSymbols);
-	};
-
-	template<int kNumSymbols>
-	struct Decoder : DecoderBase
-	{
-		unsigned char RestOfSymbols[kNumSymbols];
-	};
-
-	Decoder<256> LiteralDecoder;
-	Decoder<64> DistanceDecoder;
-	Decoder<64> LengthDecoder;
-
-	int DecodeSFValue(const DecoderBase &currentTree, const unsigned int kNumSymbols);
-	int DecodeSF(unsigned char *table);
-public:
-	int Explode(unsigned char *out, unsigned int outsize, FileReader *in, unsigned int insize, int flags);
-};
-
-class CExplosionError : CRecoverableError
-{
-public:
-	CExplosionError(const char *message) : CRecoverableError(message) {}
-};
-
-int ShrinkLoop(unsigned char *out, unsigned int outsize, FileReader *in, unsigned int insize);

src/resourcefiles/file_zip.cpp

@@ -40,7 +40,7 @@
 #include "w_wad.h"
 #include "w_zip.h"
 #include "i_system.h"
-#include "explode.h"
+#include "ancientzip.h"
 
 #define BUFREADCOMMENT (0x400)
 

zdoom.vcproj

@@ -6366,11 +6366,11 @@
 			Name="Resource Files"
 			>
 			<File
-				RelativePath=".\src\resourcefiles\explode.cpp"
+				RelativePath=".\src\resourcefiles\ancientzip.cpp"
 				>
 			</File>
 			<File
-				RelativePath=".\src\resourcefiles\explode.h"
+				RelativePath=".\src\resourcefiles\ancientzip.h"
 				>
 			</File>
 			<File