diff --git a/polymer/eduke32/Makefile b/polymer/eduke32/Makefile index cd77c8d0a..72c4432c4 100644 --- a/polymer/eduke32/Makefile +++ b/polymer/eduke32/Makefile @@ -35,7 +35,7 @@ else endif OURCOMMONFLAGS=$(BASECOMMONFLAGS) \ - -I$(INC) -I$(EINC) -I$(SRC)/jmact -I$(JAUDIOLIBDIR)/include -I$(ENETDIR)/include -I$(SRC)/xdelta3 + -I$(INC) -I$(EINC) -I$(SRC)/jmact -I$(JAUDIOLIBDIR)/include -I$(ENETDIR)/include OURCFLAGS=$(OURCOMMONFLAGS) $(BASECFLAGS) OURCXXFLAGS=$(BASECXXFLAGS) OURCONLYFLAGS=$(BASECONLYFLAGS) @@ -134,7 +134,6 @@ GAMEOBJS=$(OBJ)/game.$o \ $(OBJ)/osdcmds.$o \ $(OBJ)/grpscan.$o \ $(OBJ)/sounds.$o \ - $(OBJ)/xdelta3.$o \ $(JMACTOBJ) EDITOROBJS=$(OBJ)/astub.$o \ @@ -450,10 +449,6 @@ $(OBJ)/%.$o: $(SRC)/lunatic/%.c $(COMPILE_STATUS) if $(COMPILER) $(OURCFLAGS) -c $< -o $@; then $(COMPILE_OK); else $(COMPILE_FAILED); fi -$(OBJ)/%.$o: $(SRC)/xdelta3/%.c - $(COMPILE_STATUS) - if $(COMPILER) $(OURCFLAGS) -c $< -o $@; then $(COMPILE_OK); else $(COMPILE_FAILED); fi - # TODO: _m32 # List of exported symbols, OS X $(SRC)/lunatic/dynsymlist_osx: $(SRC)/lunatic/dynsymlist diff --git a/polymer/eduke32/Makefile.deps b/polymer/eduke32/Makefile.deps index 88bbb89d1..090fc281c 100644 --- a/polymer/eduke32/Makefile.deps +++ b/polymer/eduke32/Makefile.deps @@ -55,7 +55,6 @@ $(OBJ)/config.$o: $(SRC)/config.c $(duke3d_h) $(SRC)/jmact/scriplib.h $(INC)/_fu $(OBJ)/winbits.$o: $(SRC)/winbits.c $(OBJ)/osdfuncs.$o: $(SRC)/names.h $(EINC)/build.h $(EINC)/osd.h $(OBJ)/osdcmds.$o: $(SRC)/osdcmds.c $(INC)/osdcmds.h $(EINC)/osd.h $(duke3d_h) -$(OBJ)/xdelta3.$o: $(SRC)/xdelta3/xdelta3.c $(SRC)/xdelta3/xdelta3.h $(OBJ)/animvpx.$o: $(SRC)/animvpx.c $(SRC)/animvpx.h $(duke3d_h) $(EINC)/glbuild.h $(OBJ)/lunatic_game.$o: $(EINC)/lunatic.h $(SRC)/lunatic/lunatic_game.c $(SRC)/lunatic/lunatic_game.h $(SRC)/gamedef.h $(SRC)/gameexec.h $(EINC)/cache1d.h $(EINC)/osd.h diff --git a/polymer/eduke32/Makefile.msvc b/polymer/eduke32/Makefile.msvc index da401019a..275481298 100644 --- a/polymer/eduke32/Makefile.msvc +++ b/polymer/eduke32/Makefile.msvc @@ -44,7 +44,7 @@ AS=ml LINK=link /nologo /opt:ref MT=mt CFLAGS= /MT /J /nologo $(flags_cl) \ - /I$(INC) /I$(EINC)\msvc /I$(EINC)\ /I$(SRC)\jmact /I$(SRC)\xdelta3 /I$(JAUDIOLIBDIR)\include /I$(ENETDIR)\include \ + /I$(INC) /I$(EINC)\msvc /I$(EINC)\ /I$(SRC)\jmact /I$(JAUDIOLIBDIR)\include /I$(ENETDIR)\include \ /W2 $(ENGINEOPTS) \ /I$(DXROOT)\include /DRENDERTYPEWIN=1 @@ -106,7 +106,6 @@ GAMEOBJS=$(OBJ)\game.$o \ $(JMACTOBJ) \ $(AUDIOLIBOBJ) \ $(OBJ)\sounds.$o \ - $(OBJ)\xdelta3.$o \ !ifdef DEBUG $(OBJ)\mdump.$o !endif @@ -134,9 +133,6 @@ EDITOROBJS=$(OBJ)\astub.$o \ {$(SRC)\jmact}.c{$(OBJ)\}.$o: $(CC) /c $(CFLAGS) /Fo$@ $< -{$(SRC)\xdelta3}.c{$(OBJ)\}.$o: - $(CC) /c $(CFLAGS) /Fo$@ $< - {$(SRC)\util}.c{$(OBJ)\}.$o: $(CC) /c $(CFLAGS) /Fo$@ $< diff --git a/polymer/eduke32/source/net.c b/polymer/eduke32/source/net.c index 457ecefbd..88242b2d6 100644 --- a/polymer/eduke32/source/net.c +++ b/polymer/eduke32/source/net.c @@ -31,7 +31,6 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. #include "enet/enet.h" #include "quicklz.h" #include "crc32.h" -#include "xdelta3.h" /* this should be lower than the MTU size by at least the size of the UDP and ENet headers @@ -1313,7 +1312,7 @@ void Net_SendMapUpdate(void) int32_t pi; int32_t siz = 0; netmapstate_t *currentstate; - usize_t osize = sizeof(netmapstate_t); + size_t osize = sizeof(netmapstate_t); if (!g_netServer || numplayers < 2) return; @@ -1346,11 +1345,13 @@ void Net_SendMapUpdate(void) packbuf[0] = PACKET_MAP_STREAM; *(uint32_t *)&packbuf[1] = g_player[playeridx].revision; // base revision for this update +/* xd3_encode_memory((const uint8_t *) currentstate, sizeof(netmapstate_t), (const uint8_t *) g_multiMapState[playeridx], sizeof(netmapstate_t), (uint8_t *)tempnetbuf, &osize, sizeof(netmapstate_t), XD3_COMPLEVEL_1|XD3_NOCOMPRESS); Bmemcpy(g_pendingMapState[playeridx], currentstate, sizeof(netmapstate_t)); +*/ siz = qlz_compress((char *)tempnetbuf, packbuf+1+sizeof(uint32_t), osize, state_compress); @@ -1370,7 +1371,7 @@ void Net_SendMapUpdate(void) void Net_ReceiveMapUpdate(uint8_t *pbuf, int32_t packbufleng) { int ret; - usize_t osize = 0; + size_t osize = 0; netmapstate_t *receivedstate; packbufleng = qlz_size_decompressed((char *)&pbuf[5]); @@ -1379,6 +1380,7 @@ void Net_ReceiveMapUpdate(uint8_t *pbuf, int32_t packbufleng) initprintf("packbufleng: %d\n", packbufleng); +/* ret = xd3_decode_memory((const uint8_t *)pbuf, packbufleng, (const uint8_t *)g_multiMapState[0], sizeof(netmapstate_t), (uint8_t *)tempnetbuf, &osize, sizeof(netmapstate_t), XD3_COMPLEVEL_1|XD3_NOCOMPRESS); @@ -1389,7 +1391,7 @@ void Net_ReceiveMapUpdate(uint8_t *pbuf, int32_t packbufleng) Bfree(pbuf); return; } - +*/ if (sizeof(netmapstate_t) != osize) { initprintf("decompressed data size mismatch!\n"); diff --git a/polymer/eduke32/source/xdelta3/xdelta3-cfgs.h b/polymer/eduke32/source/xdelta3/xdelta3-cfgs.h deleted file mode 100644 index b13f7b049..000000000 --- a/polymer/eduke32/source/xdelta3/xdelta3-cfgs.h +++ /dev/null @@ -1,173 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/****************************************************************** - SOFT string matcher - ******************************************************************/ - -#if XD3_BUILD_SOFT - -#define TEMPLATE soft -#define LLOOK stream->smatcher.large_look -#define LSTEP stream->smatcher.large_step -#define SLOOK stream->smatcher.small_look -#define SCHAIN stream->smatcher.small_chain -#define SLCHAIN stream->smatcher.small_lchain -#define MAXLAZY stream->smatcher.max_lazy -#define LONGENOUGH stream->smatcher.long_enough - -#define SOFTCFG 1 -#include "xdelta3.c" -#undef SOFTCFG - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif - -#define SOFTCFG 0 - -/************************************************************ - FASTEST string matcher - **********************************************************/ -#if XD3_BUILD_FASTEST -#define TEMPLATE fastest -#define LLOOK 9 -#define LSTEP 26 -#define SLOOK 4U -#define SCHAIN 1 -#define SLCHAIN 1 -#define MAXLAZY 6 -#define LONGENOUGH 6 - -#include "xdelta3.c" - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif - -/************************************************************ - FASTER string matcher - **********************************************************/ -#if XD3_BUILD_FASTER -#define TEMPLATE faster -#define LLOOK 9 -#define LSTEP 15 -#define SLOOK 4U -#define SCHAIN 1 -#define SLCHAIN 1 -#define MAXLAZY 18 -#define LONGENOUGH 18 - -#include "xdelta3.c" - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif - -/****************************************************** - FAST string matcher - ********************************************************/ -#if XD3_BUILD_FAST -#define TEMPLATE fast -#define LLOOK 9 -#define LSTEP 8 -#define SLOOK 4U -#define SCHAIN 4 -#define SLCHAIN 1 -#define MAXLAZY 18 -#define LONGENOUGH 35 - -#include "xdelta3.c" - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif - -/************************************************** - SLOW string matcher - **************************************************************/ -#if XD3_BUILD_SLOW -#define TEMPLATE slow -#define LLOOK 9 -#define LSTEP 2 -#define SLOOK 4U -#define SCHAIN 44 -#define SLCHAIN 13 -#define MAXLAZY 90 -#define LONGENOUGH 70 - -#include "xdelta3.c" - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif - -/******************************************************** - DEFAULT string matcher - ************************************************************/ -#if XD3_BUILD_DEFAULT -#define TEMPLATE default -#define LLOOK 9 -#define LSTEP 3 -#define SLOOK 4U -#define SCHAIN 8 -#define SLCHAIN 2 -#define MAXLAZY 36 -#define LONGENOUGH 70 - -#include "xdelta3.c" - -#undef TEMPLATE -#undef LLOOK -#undef SLOOK -#undef LSTEP -#undef SCHAIN -#undef SLCHAIN -#undef MAXLAZY -#undef LONGENOUGH -#endif diff --git a/polymer/eduke32/source/xdelta3/xdelta3-decode.h b/polymer/eduke32/source/xdelta3/xdelta3-decode.h deleted file mode 100644 index 37e790bc4..000000000 --- a/polymer/eduke32/source/xdelta3/xdelta3-decode.h +++ /dev/null @@ -1,1181 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _XDELTA3_DECODE_H_ -#define _XDELTA3_DECODE_H_ - -#include "xdelta3-internal.h" - -#define SRCORTGT(x) ((((x) & VCD_SRCORTGT) == VCD_SOURCE) ? \ - VCD_SOURCE : ((((x) & VCD_SRCORTGT) == \ - VCD_TARGET) ? VCD_TARGET : 0)) - -/* Initialize the decoder for a new window. The dec_tgtlen value is - * preserved across successive window decodings, and the update to - * dec_winstart is delayed until a new window actually starts. This - * is to avoid throwing an error due to overflow until the last - * possible moment. This makes it possible to encode exactly 4GB - * through a 32-bit encoder. */ -static int -xd3_decode_init_window (xd3_stream *stream) -{ - stream->dec_cpylen = 0; - stream->dec_cpyoff = 0; - stream->dec_cksumbytes = 0; - - xd3_init_cache (& stream->acache); - - return 0; -} - -/* Allocates buffer space for the target window and possibly the - * VCD_TARGET copy-window. Also sets the base of the two copy - * segments. */ -static int -xd3_decode_setup_buffers (xd3_stream *stream) -{ - /* If VCD_TARGET is set then the previous buffer may be reused. */ - if (stream->dec_win_ind & VCD_TARGET) - { - /* But this implementation only supports copying from the last - * target window. If the offset is outside that range, it can't - * be done. */ - if (stream->dec_cpyoff < stream->dec_laststart) - { - stream->msg = "unsupported VCD_TARGET offset"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - /* See if the two windows are the same. This indicates the - * first time VCD_TARGET is used. This causes a second buffer - * to be allocated, after that the two are swapped in the - * DEC_FINISH case. */ - if (stream->dec_lastwin == stream->next_out) - { - stream->next_out = NULL; - stream->space_out = 0; - } - - // TODO: VCD_TARGET mode, this is broken - stream->dec_cpyaddrbase = stream->dec_lastwin + - (usize_t) (stream->dec_cpyoff - stream->dec_laststart); - } - - /* See if the current output window is large enough. */ - if (stream->space_out < stream->dec_tgtlen) - { - xd3_free (stream, stream->dec_buffer); - - stream->space_out = - xd3_round_blksize (stream->dec_tgtlen, XD3_ALLOCSIZE); - - if ((stream->dec_buffer = - (uint8_t*) xd3_alloc (stream, stream->space_out, 1)) == NULL) - { - return ENOMEM; - } - - stream->next_out = stream->dec_buffer; - } - - /* dec_tgtaddrbase refers to an invalid base address, but it is - * always used with a sufficiently large instruction offset (i.e., - * beyond the copy window). This condition is enforced by - * xd3_decode_output_halfinst. */ - stream->dec_tgtaddrbase = stream->next_out - stream->dec_cpylen; - - return 0; -} - -static int -xd3_decode_allocate (xd3_stream *stream, - usize_t size, - uint8_t **buf_ptr, - usize_t *buf_alloc) -{ - if (*buf_ptr != NULL && *buf_alloc < size) - { - xd3_free (stream, *buf_ptr); - *buf_ptr = NULL; - } - - if (*buf_ptr == NULL) - { - *buf_alloc = xd3_round_blksize (size, XD3_ALLOCSIZE); - - if ((*buf_ptr = (uint8_t*) xd3_alloc (stream, *buf_alloc, 1)) == NULL) - { - return ENOMEM; - } - } - - return 0; -} - -static int -xd3_decode_section (xd3_stream *stream, - xd3_desect *section, - xd3_decode_state nstate, - int copy) -{ - XD3_ASSERT (section->pos <= section->size); - XD3_ASSERT (stream->dec_state != nstate); - - if (section->pos < section->size) - { - usize_t sect_take; - - if (stream->avail_in == 0) - { - return XD3_INPUT; - } - - if ((copy == 0) && (section->pos == 0)) - { - /* No allocation/copy needed */ - section->buf = stream->next_in; - sect_take = section->size; - } - else - { - usize_t sect_need = section->size - section->pos; - - /* Allocate and copy */ - sect_take = min (sect_need, stream->avail_in); - - if (section->pos == 0) - { - int ret; - - if ((ret = xd3_decode_allocate (stream, - section->size, - & section->copied1, - & section->alloc1))) - { - return ret; - } - - section->buf = section->copied1; - } - - memcpy (section->copied1 + section->pos, - stream->next_in, - sect_take); - } - - section->pos += sect_take; - - stream->dec_winbytes += sect_take; - - DECODE_INPUT (sect_take); - } - - if (section->pos < section->size) - { - stream->msg = "further input required"; - return XD3_INPUT; - } - - XD3_ASSERT (section->pos == section->size); - - stream->dec_state = nstate; - section->buf_max = section->buf + section->size; - section->pos = 0; - return 0; -} - -/* Decode the size and address for half of an instruction (i.e., a - * single opcode). This updates the stream->dec_position, which are - * bytes already output prior to processing this instruction. Perform - * bounds checking for sizes and copy addresses, which uses the - * dec_position (which is why these checks are done here). */ -static int -xd3_decode_parse_halfinst (xd3_stream *stream, xd3_hinst *inst) -{ - int ret; - - /* If the size from the instruction table is zero then read a size value. */ - if ((inst->size == 0) && - (ret = xd3_read_size (stream, - & stream->inst_sect.buf, - stream->inst_sect.buf_max, - & inst->size))) - { - return XD3_INVALID_INPUT; - } - - /* For copy instructions, read address. */ - if (inst->type >= XD3_CPY) - { - IF_DEBUG2 ({ - static int cnt = 0; - XPR(NT "DECODE:%u: COPY at %"Q"u (winoffset %u) size %u winaddr %u\n", - cnt++, - stream->total_out + (stream->dec_position - - stream->dec_cpylen), - (stream->dec_position - stream->dec_cpylen), - inst->size, - inst->addr); - }); - - if ((ret = xd3_decode_address (stream, - stream->dec_position, - inst->type - XD3_CPY, - & stream->addr_sect.buf, - stream->addr_sect.buf_max, - & inst->addr))) - { - return ret; - } - - /* Cannot copy an address before it is filled-in. */ - if (inst->addr >= stream->dec_position) - { - stream->msg = "address too large"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - /* Check: a VCD_TARGET or VCD_SOURCE copy cannot exceed the remaining - * buffer space in its own segment. */ - if (inst->addr < stream->dec_cpylen && - inst->addr + inst->size > stream->dec_cpylen) - { - stream->msg = "size too large"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - } - else - { - IF_DEBUG2 ({ - if (inst->type == XD3_ADD) - { - static int cnt; - XPR(NT "DECODE:%d: ADD at %"Q"u (winoffset %u) size %u\n", - cnt++, - (stream->total_out + stream->dec_position - stream->dec_cpylen), - stream->dec_position - stream->dec_cpylen, - inst->size); - } - else - { - static int cnt; - XD3_ASSERT (inst->type == XD3_RUN); - XPR(NT "DECODE:%d: RUN at %"Q"u (winoffset %u) size %u\n", - cnt++, - stream->total_out + stream->dec_position - stream->dec_cpylen, - stream->dec_position - stream->dec_cpylen, - inst->size); - } - }); - } - - /* Check: The instruction will not overflow the output buffer. */ - if (stream->dec_position + inst->size > stream->dec_maxpos) - { - stream->msg = "size too large"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - stream->dec_position += inst->size; - return 0; -} - -/* Decode a single opcode and then decode the two half-instructions. */ -static int -xd3_decode_instruction (xd3_stream *stream) -{ - int ret; - const xd3_dinst *inst; - - if (stream->inst_sect.buf == stream->inst_sect.buf_max) - { - stream->msg = "instruction underflow"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - inst = &stream->code_table[*stream->inst_sect.buf++]; - - stream->dec_current1.type = inst->type1; - stream->dec_current2.type = inst->type2; - stream->dec_current1.size = inst->size1; - stream->dec_current2.size = inst->size2; - - /* For each instruction with a real operation, decode the - * corresponding size and addresses if necessary. Assume a - * code-table may have NOOP in either position, although this is - * unlikely. */ - if (inst->type1 != XD3_NOOP && - (ret = xd3_decode_parse_halfinst (stream, & stream->dec_current1))) - { - return ret; - } - if (inst->type2 != XD3_NOOP && - (ret = xd3_decode_parse_halfinst (stream, & stream->dec_current2))) - { - return ret; - } - return 0; -} - -/* Output the result of a single half-instruction. OPT: This the - decoder hotspot. Modifies "hinst", see below. */ -static int -xd3_decode_output_halfinst (xd3_stream *stream, xd3_hinst *inst) -{ - /* This method is reentrant for copy instructions which may return - * XD3_GETSRCBLK to the caller. Each time through a copy takes the - * minimum of inst->size and the available space on whichever block - * supplies the data */ - usize_t take = inst->size; - - XD3_ASSERT (inst->type != XD3_NOOP); - - switch (inst->type) - { - case XD3_RUN: - { - /* Only require a single data byte. */ - if (stream->data_sect.buf == stream->data_sect.buf_max) - { - stream->msg = "data underflow"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - memset (stream->next_out + stream->avail_out, - stream->data_sect.buf[0], - take); - - stream->data_sect.buf += 1; - stream->avail_out += take; - inst->type = XD3_NOOP; - break; - } - case XD3_ADD: - { - /* Require at least TAKE data bytes. */ - if (stream->data_sect.buf + take > stream->data_sect.buf_max) - { - stream->msg = "data underflow"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - memcpy (stream->next_out + stream->avail_out, - stream->data_sect.buf, - take); - - stream->data_sect.buf += take; - stream->avail_out += take; - inst->type = XD3_NOOP; - break; - } - default: - { - usize_t i; - const uint8_t *src; - uint8_t *dst; - int overlap; - - /* See if it copies from the VCD_TARGET/VCD_SOURCE window or - * the target window. Out-of-bounds checks for the addresses - * and sizes are performed in xd3_decode_parse_halfinst. This - * if/else must set "overlap", "src", and "dst". */ - if (inst->addr < stream->dec_cpylen) - { - /* In both branches we are copying from outside the - * current decoder window, the first (VCD_TARGET) is - * unimplemented. */ - overlap = 0; - - /* This branch sets "src". As a side-effect, we modify - * "inst" so that if we reenter this method after a - * XD3_GETSRCBLK response the state is correct. So if the - * instruction can be fulfilled by a contiguous block of - * memory then we will set: - * - * inst->type = XD3_NOOP; - * inst->size = 0; - */ - if (stream->dec_win_ind & VCD_TARGET) - { - /* TODO: Users have requested long-distance copies of - * similar material within a target (e.g., for dup - * supression in backups). */ - inst->size = 0; - inst->type = XD3_NOOP; - stream->msg = "VCD_TARGET not implemented"; - return XD3_UNIMPLEMENTED; - } - else - { - /* In this case we have to read a source block, which - * could return control to the caller. We need to - * know the first block number needed for this - * copy. */ - xd3_source *source = stream->src; - xoff_t block = source->cpyoff_blocks; - usize_t blkoff = source->cpyoff_blkoff; - const usize_t blksize = source->blksize; - int ret; - - xd3_blksize_add (&block, &blkoff, source, inst->addr); - XD3_ASSERT (blkoff < blksize); - - if ((ret = xd3_getblk (stream, block))) - { - /* could be a XD3_GETSRCBLK failure. */ - if (ret == XD3_TOOFARBACK) - { - stream->msg = "non-seekable source in decode"; - ret = XD3_INTERNAL; - } - return ret; - } - - src = source->curblk + blkoff; - - /* This block is either full, or a partial block that - * must contain enough bytes. */ - if ((source->onblk != blksize) && - (blkoff + take > source->onblk)) - { - IF_DEBUG1 (XPR(NT "[srcfile] short at blkno %"Q"u onblk " - "%u blksize %u blkoff %u take %u\n", - block, - source->onblk, - blksize, - blkoff, - take)); - stream->msg = "source file too short"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - XD3_ASSERT (blkoff != blksize); - - /* Check if we have enough data on this block to - * finish the instruction. */ - if (blkoff + take <= blksize) - { - inst->type = XD3_NOOP; - inst->size = 0; - } - else - { - take = blksize - blkoff; - inst->size -= take; - inst->addr += take; - - /* because (blkoff + take > blksize), above */ - XD3_ASSERT (inst->size != 0); - } - } - } - else - { - /* TODO: the memcpy/overlap optimization, etc. Overlap - * here could be more specific, it's whether (inst->addr - - * srclen) + inst->size > input_pos ? And is the system - * memcpy really any good? */ - overlap = 1; - - /* For a target-window copy, we know the entire range is - * in-memory. The dec_tgtaddrbase is negatively offset by - * dec_cpylen because the addresses start beyond that - * point. */ - src = stream->dec_tgtaddrbase + inst->addr; - inst->type = XD3_NOOP; - inst->size = 0; - } - - dst = stream->next_out + stream->avail_out; - - stream->avail_out += take; - - if (overlap) - { - /* Can't just memcpy here due to possible overlap. */ - for (i = take; i != 0; i -= 1) - { - *dst++ = *src++; - } - } - else - { - memcpy (dst, src, take); - } - } - } - - return 0; -} - -static int -xd3_decode_finish_window (xd3_stream *stream) -{ - stream->dec_winbytes = 0; - stream->dec_state = DEC_FINISH; - - stream->data_sect.pos = 0; - stream->inst_sect.pos = 0; - stream->addr_sect.pos = 0; - - return XD3_OUTPUT; -} - -static int -xd3_decode_secondary_sections (xd3_stream *secondary_stream -#if !defined(SECONDARY_ANY) || SECONDARY_ANY == 0 - ATTRIBUTE((unused)) -#endif - ) -{ -#if SECONDARY_ANY - int ret; -#define DECODE_SECONDARY_SECTION(UPPER,LOWER) \ - ((secondary_stream->dec_del_ind & VCD_ ## UPPER ## COMP) && \ - (ret = xd3_decode_secondary (secondary_stream, \ - & secondary_stream-> LOWER ## _sect, \ - & xd3_sec_ ## LOWER (secondary_stream)))) - - if (DECODE_SECONDARY_SECTION (DATA, data) || - DECODE_SECONDARY_SECTION (INST, inst) || - DECODE_SECONDARY_SECTION (ADDR, addr)) - { - return ret; - } -#undef DECODE_SECONDARY_SECTION -#endif - return 0; -} - -static int -xd3_decode_sections (xd3_stream *stream) -{ - usize_t need, more, take; - int copy, ret; - - if ((stream->flags & XD3_JUST_HDR) != 0) - { - /* Nothing left to do. */ - return xd3_decode_finish_window (stream); - } - - /* To avoid copying, need this much data available */ - need = (stream->inst_sect.size + - stream->addr_sect.size + - stream->data_sect.size); - - /* The window may be entirely processed. */ - XD3_ASSERT (stream->dec_winbytes <= need); - - /* Compute how much more input is needed. */ - more = (need - stream->dec_winbytes); - - /* How much to consume. */ - take = min (more, stream->avail_in); - - /* See if the input is completely available, to avoid copy. */ - copy = (take != more); - - /* If the window is skipped... */ - if ((stream->flags & XD3_SKIP_WINDOW) != 0) - { - /* Skip the available input. */ - DECODE_INPUT (take); - - stream->dec_winbytes += take; - - if (copy) - { - stream->msg = "further input required"; - return XD3_INPUT; - } - - return xd3_decode_finish_window (stream); - } - - /* Process all but the DATA section. */ - switch (stream->dec_state) - { - default: - stream->msg = "internal error"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - - case DEC_DATA: - if ((ret = xd3_decode_section (stream, & stream->data_sect, - DEC_INST, copy))) { return ret; } - case DEC_INST: - if ((ret = xd3_decode_section (stream, & stream->inst_sect, - DEC_ADDR, copy))) { return ret; } - case DEC_ADDR: - if ((ret = xd3_decode_section (stream, & stream->addr_sect, - DEC_EMIT, copy))) { return ret; } - } - - XD3_ASSERT (stream->dec_winbytes == need); - - if ((ret = xd3_decode_secondary_sections (stream))) { return ret; } - - if (stream->flags & XD3_SKIP_EMIT) - { - return xd3_decode_finish_window (stream); - } - - /* OPT: A possible optimization is to avoid allocating memory in - * decode_setup_buffers and to avoid a large memcpy when the window - * consists of a single VCD_SOURCE copy instruction. */ - if ((ret = xd3_decode_setup_buffers (stream))) { return ret; } - - return 0; -} - -static int -xd3_decode_emit (xd3_stream *stream) -{ - int ret; - - /* Produce output: originally structured to allow reentrant code - * that fills as much of the output buffer as possible, but VCDIFF - * semantics allows to copy from anywhere from the target window, so - * instead allocate a sufficiently sized buffer after the target - * window length is decoded. - * - * This code still needs to be reentrant to allow XD3_GETSRCBLK to - * return control. This is handled by setting the - * stream->dec_currentN instruction types to XD3_NOOP after they - * have been processed. */ - XD3_ASSERT (! (stream->flags & XD3_SKIP_EMIT)); - XD3_ASSERT (stream->dec_tgtlen <= stream->space_out); - - while (stream->inst_sect.buf != stream->inst_sect.buf_max || - stream->dec_current1.type != XD3_NOOP || - stream->dec_current2.type != XD3_NOOP) - { - /* Decode next instruction pair. */ - if ((stream->dec_current1.type == XD3_NOOP) && - (stream->dec_current2.type == XD3_NOOP) && - (ret = xd3_decode_instruction (stream))) { return ret; } - - /* Output dec_current1 */ - while ((stream->dec_current1.type != XD3_NOOP)) - { - if ((ret = xd3_decode_output_halfinst (stream, & stream->dec_current1))) - { - return ret; - } - } - /* Output dec_current2 */ - while (stream->dec_current2.type != XD3_NOOP) - { - if ((ret = xd3_decode_output_halfinst (stream, & stream->dec_current2))) - { - return ret; - } - } - } - - if (stream->avail_out != stream->dec_tgtlen) - { - IF_DEBUG2 (DP(RINT "AVAIL_OUT(%d) != DEC_TGTLEN(%d)\n", - stream->avail_out, stream->dec_tgtlen)); - stream->msg = "wrong window length"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - if (stream->data_sect.buf != stream->data_sect.buf_max) - { - stream->msg = "extra data section"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - if (stream->addr_sect.buf != stream->addr_sect.buf_max) - { - stream->msg = "extra address section"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - /* OPT: Should cksum computation be combined with the above loop? */ - if ((stream->dec_win_ind & VCD_ADLER32) != 0 && - (stream->flags & XD3_ADLER32_NOVER) == 0) - { - uint32_t a32 = adler32 (1L, stream->next_out, stream->avail_out); - - if (a32 != stream->dec_adler32) - { - stream->msg = "target window checksum mismatch"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - } - - /* Finished with a window. */ - return xd3_decode_finish_window (stream); -} - -int -xd3_decode_input (xd3_stream *stream) -{ - int ret; - - if (stream->enc_state != 0) - { - stream->msg = "encoder/decoder transition"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - -#define BYTE_CASE(expr,x,nstate) \ - do { \ - if ( (expr) && \ - ((ret = xd3_decode_byte (stream, & (x))) != 0) ) { return ret; } \ - stream->dec_state = (nstate); \ - } while (0) - -#define OFFSET_CASE(expr,x,nstate) \ - do { \ - if ( (expr) && \ - ((ret = xd3_decode_offset (stream, & (x))) != 0) ) { return ret; } \ - stream->dec_state = (nstate); \ - } while (0) - -#define SIZE_CASE(expr,x,nstate) \ - do { \ - if ( (expr) && \ - ((ret = xd3_decode_size (stream, & (x))) != 0) ) { return ret; } \ - stream->dec_state = (nstate); \ - } while (0) - - switch (stream->dec_state) - { - case DEC_VCHEAD: - { - if ((ret = xd3_decode_bytes (stream, stream->dec_magic, - & stream->dec_magicbytes, 4))) - { - return ret; - } - - if (stream->dec_magic[0] != VCDIFF_MAGIC1 || - stream->dec_magic[1] != VCDIFF_MAGIC2 || - stream->dec_magic[2] != VCDIFF_MAGIC3) - { - stream->msg = "not a VCDIFF input"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - if (stream->dec_magic[3] != 0) - { - stream->msg = "VCDIFF input version > 0 is not supported"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - stream->dec_state = DEC_HDRIND; - } - case DEC_HDRIND: - { - if ((ret = xd3_decode_byte (stream, & stream->dec_hdr_ind))) - { - return ret; - } - - if ((stream->dec_hdr_ind & VCD_INVHDR) != 0) - { - stream->msg = "unrecognized header indicator bits set"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - stream->dec_state = DEC_SECONDID; - } - - case DEC_SECONDID: - /* Secondary compressor ID: only if VCD_SECONDARY is set */ - if ((stream->dec_hdr_ind & VCD_SECONDARY) != 0) - { - BYTE_CASE (1, stream->dec_secondid, DEC_TABLEN); - - switch (stream->dec_secondid) - { - case VCD_FGK_ID: - FGK_CASE (stream); - case VCD_DJW_ID: - DJW_CASE (stream); - case VCD_LZMA_ID: - LZMA_CASE (stream); - default: - stream->msg = "unknown secondary compressor ID"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - } - - case DEC_TABLEN: - /* Length of code table data: only if VCD_CODETABLE is set */ - SIZE_CASE ((stream->dec_hdr_ind & VCD_CODETABLE) != 0, - stream->dec_codetblsz, DEC_NEAR); - - /* The codetblsz counts the two NEAR/SAME bytes */ - if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) { - if (stream->dec_codetblsz <= 2) { - stream->msg = "invalid code table size"; - return ENOMEM; - } - stream->dec_codetblsz -= 2; - } - case DEC_NEAR: - /* Near modes: only if VCD_CODETABLE is set */ - BYTE_CASE((stream->dec_hdr_ind & VCD_CODETABLE) != 0, - stream->acache.s_near, DEC_SAME); - case DEC_SAME: - /* Same modes: only if VCD_CODETABLE is set */ - BYTE_CASE((stream->dec_hdr_ind & VCD_CODETABLE) != 0, - stream->acache.s_same, DEC_TABDAT); - case DEC_TABDAT: - /* Compressed code table data */ - - if ((stream->dec_hdr_ind & VCD_CODETABLE) != 0) - { - /* Get the code table data. */ - if ((stream->dec_codetbl == NULL) && - (stream->dec_codetbl = - (uint8_t*) xd3_alloc (stream, - stream->dec_codetblsz, 1)) == NULL) - { - return ENOMEM; - } - - if ((ret = xd3_decode_bytes (stream, stream->dec_codetbl, - & stream->dec_codetblbytes, - stream->dec_codetblsz))) - { - return ret; - } - - if ((ret = xd3_apply_table_encoding (stream, stream->dec_codetbl, - stream->dec_codetblbytes))) - { - return ret; - } - } - else - { - /* Use the default table. */ - stream->acache.s_near = __rfc3284_code_table_desc.near_modes; - stream->acache.s_same = __rfc3284_code_table_desc.same_modes; - stream->code_table = xd3_rfc3284_code_table (); - } - - if ((ret = xd3_alloc_cache (stream))) { return ret; } - - stream->dec_state = DEC_APPLEN; - - case DEC_APPLEN: - /* Length of application data */ - SIZE_CASE((stream->dec_hdr_ind & VCD_APPHEADER) != 0, - stream->dec_appheadsz, DEC_APPDAT); - - case DEC_APPDAT: - /* Application data */ - if (stream->dec_hdr_ind & VCD_APPHEADER) - { - /* Note: we add an additional byte for padding, to allow - 0-termination. */ - if ((stream->dec_appheader == NULL) && - (stream->dec_appheader = - (uint8_t*) xd3_alloc (stream, - stream->dec_appheadsz+1, 1)) == NULL) - { - return ENOMEM; - } - - stream->dec_appheader[stream->dec_appheadsz] = 0; - - if ((ret = xd3_decode_bytes (stream, stream->dec_appheader, - & stream->dec_appheadbytes, - stream->dec_appheadsz))) - { - return ret; - } - } - - /* xoff_t -> usize_t is safe because this is the first block. */ - stream->dec_hdrsize = (usize_t) stream->total_in; - stream->dec_state = DEC_WININD; - - case DEC_WININD: - { - /* Start of a window: the window indicator */ - if ((ret = xd3_decode_byte (stream, & stream->dec_win_ind))) - { - return ret; - } - - stream->current_window = stream->dec_window_count; - - if (XOFF_T_OVERFLOW (stream->dec_winstart, stream->dec_tgtlen)) - { - stream->msg = "decoder file offset overflow"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - stream->dec_winstart += stream->dec_tgtlen; - - if ((stream->dec_win_ind & VCD_INVWIN) != 0) - { - stream->msg = "unrecognized window indicator bits set"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - if ((ret = xd3_decode_init_window (stream))) { return ret; } - - stream->dec_state = DEC_CPYLEN; - - IF_DEBUG2 (DP(RINT "--------- TARGET WINDOW %"Q"u -----------\n", - stream->current_window)); - } - - case DEC_CPYLEN: - /* Copy window length: only if VCD_SOURCE or VCD_TARGET is set */ - SIZE_CASE(SRCORTGT (stream->dec_win_ind), stream->dec_cpylen, - DEC_CPYOFF); - - /* Set the initial, logical decoder position (HERE address) in - * dec_position. This is set to just after the source/copy - * window, as we are just about to output the first byte of - * target window. */ - stream->dec_position = stream->dec_cpylen; - - case DEC_CPYOFF: - /* Copy window offset: only if VCD_SOURCE or VCD_TARGET is set */ - OFFSET_CASE(SRCORTGT (stream->dec_win_ind), stream->dec_cpyoff, - DEC_ENCLEN); - - /* Copy offset and copy length may not overflow. */ - if (XOFF_T_OVERFLOW (stream->dec_cpyoff, stream->dec_cpylen)) - { - stream->msg = "decoder copy window overflows a file offset"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - /* Check copy window bounds: VCD_TARGET window may not exceed - current position. */ - if ((stream->dec_win_ind & VCD_TARGET) && - (stream->dec_cpyoff + (xoff_t) stream->dec_cpylen > - stream->dec_winstart)) - { - stream->msg = "VCD_TARGET window out of bounds"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - case DEC_ENCLEN: - /* Length of the delta encoding */ - SIZE_CASE(1, stream->dec_enclen, DEC_TGTLEN); - case DEC_TGTLEN: - /* Length of target window */ - SIZE_CASE(1, stream->dec_tgtlen, DEC_DELIND); - - /* Set the maximum decoder position, beyond which we should not - * decode any data. This is the maximum value for dec_position. - * This may not exceed the size of a usize_t. */ - if (USIZE_T_OVERFLOW (stream->dec_cpylen, stream->dec_tgtlen)) - { - stream->msg = "decoder target window overflows a usize_t"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - /* Check for malicious files. */ - if (stream->dec_tgtlen > XD3_HARDMAXWINSIZE) - { - stream->msg = "hard window size exceeded"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - stream->dec_maxpos = stream->dec_cpylen + stream->dec_tgtlen; - - case DEC_DELIND: - /* Delta indicator */ - BYTE_CASE(1, stream->dec_del_ind, DEC_DATALEN); - - if ((stream->dec_del_ind & VCD_INVDEL) != 0) - { - stream->msg = "unrecognized delta indicator bits set"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - /* Delta indicator is only used with secondary compression. */ - if ((stream->dec_del_ind != 0) && (stream->sec_type == NULL)) - { - stream->msg = "invalid delta indicator bits set"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - /* Section lengths */ - case DEC_DATALEN: - SIZE_CASE(1, stream->data_sect.size, DEC_INSTLEN); - case DEC_INSTLEN: - SIZE_CASE(1, stream->inst_sect.size, DEC_ADDRLEN); - case DEC_ADDRLEN: - SIZE_CASE(1, stream->addr_sect.size, DEC_CKSUM); - - case DEC_CKSUM: - /* Window checksum. */ - if ((stream->dec_win_ind & VCD_ADLER32) != 0) - { - int i; - - if ((ret = xd3_decode_bytes (stream, stream->dec_cksum, - & stream->dec_cksumbytes, 4))) - { - return ret; - } - - for (i = 0; i < 4; i += 1) - { - stream->dec_adler32 = - (stream->dec_adler32 << 8) | stream->dec_cksum[i]; - } - } - - stream->dec_state = DEC_DATA; - - /* Check dec_enclen for redundency, otherwise it is not really used. */ - { - usize_t enclen_check = - (1 + (xd3_sizeof_size (stream->dec_tgtlen) + - xd3_sizeof_size (stream->data_sect.size) + - xd3_sizeof_size (stream->inst_sect.size) + - xd3_sizeof_size (stream->addr_sect.size)) + - stream->data_sect.size + - stream->inst_sect.size + - stream->addr_sect.size + - ((stream->dec_win_ind & VCD_ADLER32) ? 4 : 0)); - - if (stream->dec_enclen != enclen_check) - { - stream->msg = "incorrect encoding length (redundent)"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - } - - /* Returning here gives the application a chance to inspect the - * header, skip the window, etc. */ - if (stream->current_window == 0) { return XD3_GOTHEADER; } - else { return XD3_WINSTART; } - - case DEC_DATA: - case DEC_INST: - case DEC_ADDR: - /* Next read the three sections. */ - if ((ret = xd3_decode_sections (stream))) { return ret; } - - case DEC_EMIT: - - /* To speed VCD_SOURCE block-address calculations, the source - * cpyoff_blocks and cpyoff_blkoff are pre-computed. */ - if (stream->dec_win_ind & VCD_SOURCE) - { - xd3_source *src = stream->src; - - if (src == NULL) - { - stream->msg = "source input required"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - xd3_blksize_div(stream->dec_cpyoff, src, - &src->cpyoff_blocks, - &src->cpyoff_blkoff); - - IF_DEBUG1(DP(RINT - "decode cpyoff %"Q"u " - "cpyblkno %"Q"u " - "cpyblkoff %u " - "blksize %u\n", - stream->dec_cpyoff, - src->cpyoff_blocks, - src->cpyoff_blkoff, - src->blksize)); - } - - /* xd3_decode_emit returns XD3_OUTPUT on every success. */ - if ((ret = xd3_decode_emit (stream)) == XD3_OUTPUT) - { - stream->total_out += (xoff_t) stream->avail_out; - } - - return ret; - - case DEC_FINISH: - { - if (stream->dec_win_ind & VCD_TARGET) - { - if (stream->dec_lastwin == NULL) - { - stream->dec_lastwin = stream->next_out; - stream->dec_lastspace = stream->space_out; - } - else - { - xd3_swap_uint8p (& stream->dec_lastwin, - & stream->next_out); - xd3_swap_usize_t (& stream->dec_lastspace, - & stream->space_out); - } - } - - stream->dec_lastlen = stream->dec_tgtlen; - stream->dec_laststart = stream->dec_winstart; - stream->dec_window_count += 1; - - /* Note: the updates to dec_winstart & current_window are - * deferred until after the next DEC_WININD byte is read. */ - stream->dec_state = DEC_WININD; - return XD3_WINFINISH; - } - - default: - stream->msg = "invalid state"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } -} - -#endif // _XDELTA3_DECODE_H_ diff --git a/polymer/eduke32/source/xdelta3/xdelta3-hash.h b/polymer/eduke32/source/xdelta3/xdelta3-hash.h deleted file mode 100644 index 0f0b456da..000000000 --- a/polymer/eduke32/source/xdelta3/xdelta3-hash.h +++ /dev/null @@ -1,187 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001, 2003, 2004, 2005, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _XDELTA3_HASH_H_ -#define _XDELTA3_HASH_H_ - -#if XD3_DEBUG -#define SMALL_HASH_DEBUG1(s,inp) \ - usize_t debug_state; \ - usize_t debug_hval = xd3_checksum_hash (& (s)->small_hash, \ - xd3_scksum (&debug_state, (inp), (s)->smatcher.small_look)) -#define SMALL_HASH_DEBUG2(s,inp) \ - XD3_ASSERT (debug_hval == xd3_checksum_hash (& (s)->small_hash, \ - xd3_scksum (&debug_state, (inp), (s)->smatcher.small_look))) -#else -#define SMALL_HASH_DEBUG1(s,inp) -#define SMALL_HASH_DEBUG2(s,inp) -#endif /* XD3_DEBUG */ - -/* This is a good hash multiplier for 32-bit LCGs: see "linear - * congruential generators of different sizes and good lattice - * structure" */ -static const uint32_t hash_multiplier = 1597334677U; - -/*********************************************************************** - Permute stuff - ***********************************************************************/ - -#if HASH_PERMUTE == 0 -#define PERMUTE(x) (x) -#else -#define PERMUTE(x) (__single_hash[(uint32_t)x]) - -extern const uint16_t __single_hash[256]; -#endif - -/* Update the checksum state. */ -#if ADLER_LARGE_CKSUM -inline uint32_t -xd3_large_cksum_update (uint32_t cksum, - const uint8_t *base, - usize_t look) { - uint32_t old_c = PERMUTE(base[0]); - uint32_t new_c = PERMUTE(base[look]); - uint32_t low = ((cksum & 0xffff) - old_c + new_c) & 0xffff; - uint32_t high = ((cksum >> 16) - (old_c * look) + low) & 0xffff; - return (high << 16) | low; -} -#else -/* TODO: revisit this topic */ -#endif - -/* Note: small cksum is hard-coded for 4 bytes */ -#if UNALIGNED_OK -static inline uint32_t -xd3_scksum (uint32_t *state, - const uint8_t *base, - const usize_t look ATTRIBUTE((unused))) -{ - (*state) = *(uint32_t*)base; - return (*state) * hash_multiplier; -} -static inline uint32_t -xd3_small_cksum_update (uint32_t *state, - const uint8_t *base, - usize_t look ATTRIBUTE((unused))) -{ - (*state) = *(uint32_t*)(base+1); - return (*state) * hash_multiplier; -} -#else -static inline uint32_t -xd3_scksum (uint32_t *state, - const uint8_t *base, - const usize_t look) -{ - (*state) = (base[0] << 24 | - base[1] << 16 | - base[2] << 8 | - base[3]); - return (*state) * hash_multiplier; -} -static inline uint32_t -xd3_small_cksum_update (uint32_t *state, - const uint8_t *base, - const usize_t look) -{ - (*state) <<= 8; - (*state) |= base[4]; - return (*state) * hash_multiplier; -} -#endif - -/*********************************************************************** - Ctable stuff - ***********************************************************************/ - -static inline usize_t -xd3_checksum_hash (const xd3_hash_cfg *cfg, const usize_t cksum) -{ - return (cksum >> cfg->shift) ^ (cksum & cfg->mask); -} - -/*********************************************************************** - Cksum function - ***********************************************************************/ - -#if ADLER_LARGE_CKSUM -static inline uint32_t -xd3_lcksum (const uint8_t *seg, const usize_t ln) -{ - usize_t i = 0; - uint32_t low = 0; - uint32_t high = 0; - - for (; i < ln; i += 1) - { - low += PERMUTE(*seg++); - high += low; - } - - return ((high & 0xffff) << 16) | (low & 0xffff); -} -#else -static inline uint32_t -xd3_lcksum (const uint8_t *seg, const usize_t ln) -{ - usize_t i, j; - uint32_t h = 0; - for (i = 0, j = ln - 1; i < ln; ++i, --j) { - h += PERMUTE(seg[i]) * hash_multiplier_powers[j]; - } - return h; -} -#endif - -#if XD3_ENCODER -static usize_t -xd3_size_log2 (usize_t slots) -{ - int bits = 28; /* This should not be an unreasonable limit. */ - int i; - - for (i = 3; i <= bits; i += 1) - { - if (slots < (1U << i)) - { - /* TODO: this is compaction=1 in checksum_test.cc and maybe should - * not be fixed at -1. */ - bits = i - 1; - break; - } - } - - return bits; -} - -static void -xd3_size_hashtable (xd3_stream *stream ATTRIBUTE((unused)), - usize_t slots, - xd3_hash_cfg *cfg) -{ - int bits = xd3_size_log2 (slots); - - /* TODO: there's a 32-bit assumption here */ - cfg->size = (1 << bits); - cfg->mask = (cfg->size - 1); - cfg->shift = 32 - bits; -} -#endif - -#endif diff --git a/polymer/eduke32/source/xdelta3/xdelta3-internal.h b/polymer/eduke32/source/xdelta3/xdelta3-internal.h deleted file mode 100644 index 10ecb3852..000000000 --- a/polymer/eduke32/source/xdelta3/xdelta3-internal.h +++ /dev/null @@ -1,142 +0,0 @@ -/* xdelta3 - delta compression tools and library - * Copyright (C) 2011, 2012 Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ -#ifndef XDELTA3_INTERNAL_H__ -#define XDELTA3_INTERNAL_H__ - -#include "xdelta3.h" - -typedef struct _main_file main_file; -typedef struct _main_extcomp main_extcomp; - -void main_file_init (main_file *xfile); -int main_file_close (main_file *xfile); -void main_file_cleanup (main_file *xfile); -int main_file_isopen (main_file *xfile); -int main_file_open (main_file *xfile, const char* name, int mode); -int main_file_exists (main_file *xfile); -int xd3_whole_append_window (xd3_stream *stream); -int xd3_main_cmdline (int argc, char **argv); -int main_file_read (main_file *ifile, - uint8_t *buf, - usize_t size, - usize_t *nread, - const char *msg); -int main_file_write (main_file *ofile, uint8_t *buf, - usize_t size, const char *msg); -usize_t xd3_bytes_on_srcblk (xd3_source *src, xoff_t blkno); -xoff_t xd3_source_eof(const xd3_source *src); -uint32_t xd3_large_cksum_update (uint32_t cksum, - const uint8_t *base, - usize_t look); -int xd3_encode_init_full (xd3_stream *stream); -#if PYTHON_MODULE || SWIG_MODULE || NOT_MAIN -int xd3_main_cmdline (int argc, char **argv); -#endif - -/* main_file->mode values */ -typedef enum -{ - XO_READ = 0, - XO_WRITE = 1 -} main_file_modes; - -struct _main_file -{ -#if XD3_STDIO - FILE *file; -#elif XD3_POSIX - int file; -#elif XD3_WIN32 - HANDLE file; -#endif - - int mode; /* XO_READ and XO_WRITE */ - const char *filename; /* File name or /dev/stdin, - * /dev/stdout, /dev/stderr. */ - char *filename_copy; /* File name or /dev/stdin, - * /dev/stdout, /dev/stderr. */ - const char *realname; /* File name or /dev/stdin, - * /dev/stdout, /dev/stderr. */ - const main_extcomp *compressor; /* External compression struct. */ - int flags; /* RD_FIRST, RD_NONEXTERNAL, ... */ - xoff_t nread; /* for input position */ - xoff_t nwrite; /* for output position */ - uint8_t *snprintf_buf; /* internal snprintf() use */ - int size_known; /* Set by main_set_souze */ - xoff_t source_position; /* for avoiding seek in getblk_func */ - int seek_failed; /* after seek fails once, try FIFO */ -}; - -/* According to the internet, Windows vsnprintf() differs from most - * Unix implementations regarding the terminating 0 when the boundary - * condition is met. It doesn't matter here, we don't rely on the - * trailing 0. Besides, both Windows and DJGPP vsnprintf return -1 - * upon truncation, which isn't C99 compliant. To overcome this, - * recent MinGW runtimes provided their own vsnprintf (notice the - * absence of the '_' prefix) but they were initially buggy. So, - * always use the native '_'-prefixed version with Win32. */ -#ifdef _WIN32 -#define vsnprintf_func(str,size,fmt,args) \ - _vsnprintf_s(str,size,size-1,fmt,args) -#define snprintf_func(str,size,fmt,...) \ - _snprintf_s(str,size,size-1,fmt,__VA_ARGS__) -#else -#define vsnprintf_func vsnprintf -#define snprintf_func snprintf -#endif -#define short_sprintf(sb,fmt,...) \ - snprintf_func((sb).buf,sizeof((sb).buf),fmt,__VA_ARGS__) - -/* Type used for short snprintf calls. */ -typedef struct { - char buf[48]; -} shortbuf; - -/* Prior to SVN 303 this function was only defined in DJGPP and WIN32 - * environments and other platforms would use the builtin snprintf() - * with an arrangement of macros below. In OS X 10.6, Apply made - * snprintf() a macro, which defeated those macros (since snprintf - * would be evaluated before its argument macros were expanded, - * therefore always define xsnprintf_func. */ -#undef PRINTF_ATTRIBUTE -#ifdef __GNUC__ -/* Let's just assume no one uses gcc 2.x! */ -#define PRINTF_ATTRIBUTE(x,y) __attribute__ ((__format__ (__printf__, x, y))) -#else -#define PRINTF_ATTRIBUTE(x,y) -#endif - -/* Underlying xprintf() */ -int xsnprintf_func (char *str, int n, const char *fmt, ...) - PRINTF_ATTRIBUTE(3,4); - -/* XPR(NT "", ...) (used by main) prefixes an "xdelta3: " to the output. */ -void xprintf(const char *fmt, ...) PRINTF_ATTRIBUTE(1,2); -#define XPR xprintf -#define NT "xdelta3: " -#define NTR "" - -#ifndef UINT32_MAX -#define UINT32_MAX 4294967295U -#endif - -#ifndef UINT64_MAX -#define UINT64_MAX 18446744073709551615ULL -#endif - -#endif // XDELTA3_INTERNAL_H__ diff --git a/polymer/eduke32/source/xdelta3/xdelta3-list.h b/polymer/eduke32/source/xdelta3/xdelta3-list.h deleted file mode 100644 index 6e3125fe3..000000000 --- a/polymer/eduke32/source/xdelta3/xdelta3-list.h +++ /dev/null @@ -1,130 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2002, 2006, 2007. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef __XDELTA3_LIST__ -#define __XDELTA3_LIST__ - -#define XD3_MAKELIST(LTYPE,ETYPE,LNAME) \ - \ -static inline ETYPE* \ -LTYPE ## _entry (LTYPE* l) \ -{ \ - return (ETYPE*) ((char*) l - (ptrdiff_t) &((ETYPE*) 0)->LNAME); \ -} \ - \ -static inline void \ -LTYPE ## _init (LTYPE *l) \ -{ \ - l->next = l; \ - l->prev = l; \ -} \ - \ -static inline void \ -LTYPE ## _add (LTYPE *prev, LTYPE *next, LTYPE *ins) \ -{ \ - next->prev = ins; \ - prev->next = ins; \ - ins->next = next; \ - ins->prev = prev; \ -} \ - \ -static inline void \ -LTYPE ## _push_back (LTYPE *l, ETYPE *i) \ -{ \ - LTYPE ## _add (l->prev, l, & i->LNAME); \ -} \ - \ -static inline void \ -LTYPE ## _del (LTYPE *next, \ - LTYPE *prev) \ -{ \ - next->prev = prev; \ - prev->next = next; \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _remove (ETYPE *f) \ -{ \ - LTYPE *i = f->LNAME.next; \ - LTYPE ## _del (f->LNAME.next, f->LNAME.prev); \ - return LTYPE ## _entry (i); \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _pop_back (LTYPE *l) \ -{ \ - LTYPE *i = l->prev; \ - LTYPE ## _del (i->next, i->prev); \ - return LTYPE ## _entry (i); \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _pop_front (LTYPE *l) \ -{ \ - LTYPE *i = l->next; \ - LTYPE ## _del (i->next, i->prev); \ - return LTYPE ## _entry (i); \ -} \ - \ -static inline int \ -LTYPE ## _empty (LTYPE *l) \ -{ \ - return l == l->next; \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _front (LTYPE *f) \ -{ \ - return LTYPE ## _entry (f->next); \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _back (LTYPE *f) \ -{ \ - return LTYPE ## _entry (f->prev); \ -} \ - \ -static inline int \ -LTYPE ## _end (LTYPE *f, ETYPE *i) \ -{ \ - return f == & i->LNAME; \ -} \ - \ -static inline ETYPE* \ -LTYPE ## _next (ETYPE *f) \ -{ \ - return LTYPE ## _entry (f->LNAME.next); \ -} \ - \ -static inline usize_t \ -LTYPE ## _length (LTYPE *l) \ -{ \ - LTYPE *p; \ - int c = 0; \ - \ - for (p = l->next; p != l; p = p->next) \ - { \ - c += 1; \ - } \ - \ - return c; \ -} \ - \ -typedef int unused_ ## LTYPE - -#endif diff --git a/polymer/eduke32/source/xdelta3/xdelta3.c b/polymer/eduke32/source/xdelta3/xdelta3.c deleted file mode 100644 index 70453d436..000000000 --- a/polymer/eduke32/source/xdelta3/xdelta3.c +++ /dev/null @@ -1,5536 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001, 2003, 2004, 2005, 2006, 2007, - * 2008, 2009, 2010. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - ------------------------------------------------------------------- - - Xdelta 3 - - The goal of this library is to to implement both the (stand-alone) - data-compression and delta-compression aspects of VCDIFF encoding, and - to support a programming interface that works like Zlib - (http://www.gzip.org/zlib.html). See RFC3284: The VCDIFF Generic - Differencing and Compression Data Format. - - VCDIFF is a unified encoding that combines data-compression and - delta-encoding ("differencing"). - - VCDIFF has a detailed byte-code instruction set with many features. - The instruction format supports an immediate size operand for small - COPYs and ADDs (e.g., under 18 bytes). There are also instruction - "modes", which are used to compress COPY addresses by using two - address caches. An instruction mode refers to slots in the NEAR - and SAME caches for recent addresses. NEAR remembers the - previous 4 (by default) COPY addresses, and SAME catches - frequent re-uses of the same address using a 3-way (by default) - 256-entry associative cache of [ADDR mod 256], the encoded byte. - A hit in the NEAR/SAME cache requires 0/1 ADDR bytes. - - VCDIFF has a default instruction table, but an alternate - instruction tables may themselves be be delta-compressed and - included in the encoding header. This allows even more freedom. - There are 9 instruction modes in the default code table, 4 near, 3 - same, VCD_SELF (absolute encoding) and VCD_HERE (relative to the - current position). - - ---------------------------------------------------------------------- - - Algorithms - - Aside from the details of encoding and decoding, there are a bunch - of algorithms needed. - - 1. STRING-MATCH. A two-level fingerprinting approach is used. A - single loop computes the two checksums -- small and large -- at - successive offsets in the TARGET file. The large checksum is more - accurate and is used to discover SOURCE matches, which are - potentially very long. The small checksum is used to discover - copies within the TARGET. Small matching, which is more expensive, - usually dominates the large STRING-MATCH costs in this code - the - more exhaustive the search, the better the results. Either of the - two string-matching mechanisms may be disabled. - - 2. INSTRUCTION SELECTION. The IOPT buffer here represents a queue - used to store overlapping copy instructions. There are two possible - optimizations that go beyond a greedy search. Both of these fall - into the category of "non-greedy matching" optimizations. - - The first optimization stems from backward SOURCE-COPY matching. - When a new SOURCE-COPY instruction covers a previous instruction in - the target completely, it is erased from the queue. Randal Burns - originally analyzed these algorithms and did a lot of related work - (\cite the 1.5-pass algorithm). - - The second optimization comes by the encoding of common very-small - COPY and ADD instructions, for which there are special DOUBLE-code - instructions, which code two instructions in a single byte. - - The cost of bad instruction-selection overhead is relatively high - for data-compression, relative to delta-compression, so this second - optimization is fairly important. With "lazy" matching (the name - used in Zlib for a similar optimization), the string-match - algorithm searches after a match for potential overlapping copy - instructions. In Xdelta and by default, VCDIFF, the minimum match - size is 4 bytes, whereas Zlib searches with a 3-byte minimum. This - feature, combined with double instructions, provides a nice - challenge. Search in this file for "black magic", a heuristic. - - 3. STREAM ALIGNMENT. Stream alignment is needed to compress large - inputs in constant space. See xd3_srcwin_move_point(). - - 4. WINDOW SELECTION. When the IOPT buffer flushes, in the first call - to xd3_iopt_finish_encoding containing any kind of copy instruction, - the parameters of the source window must be decided: the offset into - the source and the length of the window. Since the IOPT buffer is - finite, the program may be forced to fix these values before knowing - the best offset/length. - - 5. SECONDARY COMPRESSION. VCDIFF supports a secondary encoding to - be applied to the individual sections of the data format, which are - ADDRess, INSTruction, and DATA. Several secondary compressor - variations are implemented here, although none is standardized yet. - - One is an adaptive huffman algorithm -- the FGK algorithm (Faller, - Gallager, and Knuth, 1985). This compressor is extremely slow. - - The other is a simple static Huffman routine, which is the base - case of a semi-adaptive scheme published by D.J. Wheeler and first - widely used in bzip2 (by Julian Seward). This is a very - interesting algorithm, originally published in nearly cryptic form - by D.J. Wheeler. !!!NOTE!!! Because these are not standardized, - secondary compression remains off by default. - ftp://ftp.cl.cam.ac.uk/users/djw3/bred3.{c,ps} - -------------------------------------------------------------------- - - Other Features - - 1. USER CONVENIENCE - - For user convenience, it is essential to recognize Gzip-compressed - files and automatically Gzip-decompress them prior to - delta-compression (or else no delta-compression will be achieved - unless the user manually decompresses the inputs). The compressed - represention competes with Xdelta, and this must be hidden from the - command-line user interface. The Xdelta-1.x encoding was simple, not - compressed itself, so Xdelta-1.x uses Zlib internally to compress the - representation. - - This implementation supports external compression, which implements - the necessary fork() and pipe() mechanics. There is a tricky step - involved to support automatic detection of a compressed input in a - non-seekable input. First you read a bit of the input to detect - magic headers. When a compressed format is recognized, exec() the - external compression program and create a second child process to - copy the original input stream. [Footnote: There is a difficulty - related to using Gzip externally. It is not possible to decompress - and recompress a Gzip file transparently. If FILE.GZ had a - cryptographic signature, then, after: (1) Gzip-decompression, (2) - Xdelta-encoding, (3) Gzip-compression the signature could be - broken. The only way to solve this problem is to guess at Gzip's - compression level or control it by other means. I recommend that - specific implementations of any compression scheme store - information needed to exactly re-compress the input, that way - external compression is transparent - however, this won't happen - here until it has stabilized.] - - 2. APPLICATION-HEADER - - This feature was introduced in RFC3284. It allows any application - to include a header within the VCDIFF file format. This allows - general inter-application data exchange with support for - application-specific extensions to communicate metadata. - - 3. VCDIFF CHECKSUM - - An optional checksum value is included with each window, which can - be used to validate the final result. This verifies the correct source - file was used for decompression as well as the obvious advantage: - checking the implementation (and underlying) correctness. - - 4. LIGHT WEIGHT - - The code makes efforts to avoid copying data more than necessary. - The code delays many initialization tasks until the first use, it - optimizes for identical (perfectly matching) inputs. It does not - compute any checksums until the first lookup misses. Memory usage - is reduced. String-matching is templatized (by slightly gross use - of CPP) to hard-code alternative compile-time defaults. The code - has few outside dependencies. - ---------------------------------------------------------------------- - - The default rfc3284 instruction table: - (see RFC for the explanation) - - TYPE SIZE MODE TYPE SIZE MODE INDEX - -------------------------------------------------------------------- - 1. Run 0 0 Noop 0 0 0 - 2. Add 0, [1,17] 0 Noop 0 0 [1,18] - 3. Copy 0, [4,18] 0 Noop 0 0 [19,34] - 4. Copy 0, [4,18] 1 Noop 0 0 [35,50] - 5. Copy 0, [4,18] 2 Noop 0 0 [51,66] - 6. Copy 0, [4,18] 3 Noop 0 0 [67,82] - 7. Copy 0, [4,18] 4 Noop 0 0 [83,98] - 8. Copy 0, [4,18] 5 Noop 0 0 [99,114] - 9. Copy 0, [4,18] 6 Noop 0 0 [115,130] - 10. Copy 0, [4,18] 7 Noop 0 0 [131,146] - 11. Copy 0, [4,18] 8 Noop 0 0 [147,162] - 12. Add [1,4] 0 Copy [4,6] 0 [163,174] - 13. Add [1,4] 0 Copy [4,6] 1 [175,186] - 14. Add [1,4] 0 Copy [4,6] 2 [187,198] - 15. Add [1,4] 0 Copy [4,6] 3 [199,210] - 16. Add [1,4] 0 Copy [4,6] 4 [211,222] - 17. Add [1,4] 0 Copy [4,6] 5 [223,234] - 18. Add [1,4] 0 Copy 4 6 [235,238] - 19. Add [1,4] 0 Copy 4 7 [239,242] - 20. Add [1,4] 0 Copy 4 8 [243,246] - 21. Copy 4 [0,8] Add 1 0 [247,255] - -------------------------------------------------------------------- - - Reading the source: Overview - - This file includes itself in several passes to macro-expand certain - sections with variable forms. Just read ahead, there's only a - little confusion. I know this sounds ugly, but hard-coding some of - the string-matching parameters results in a 10-15% increase in - string-match performance. The only time this hurts is when you have - unbalanced #if/endifs. - - A single compilation unit tames the Makefile. In short, this is to - allow the above-described hack without an explodingMakefile. The - single compilation unit includes the core library features, - configurable string-match templates, optional main() command-line - tool, misc optional features, and a regression test. Features are - controled with CPP #defines, see Makefile.am. - - The initial __XDELTA3_C_HEADER_PASS__ starts first, the _INLINE_ and - _TEMPLATE_ sections follow. Easy stuff first, hard stuff last. - - Optional features include: - - xdelta3-main.h The command-line interface, external compression - support, POSIX-specific, info & VCDIFF-debug tools. - xdelta3-second.h The common secondary compression routines. - xdelta3-decoder.h All decoding routines. - xdelta3-djw.h The semi-adaptive huffman secondary encoder. - xdelta3-fgk.h The adaptive huffman secondary encoder. - xdelta3-test.h The unit test covers major algorithms, - encoding and decoding. There are single-bit - error decoding tests. There are 32/64-bit file size - boundary tests. There are command-line tests. - There are compression tests. There are external - compression tests. There are string-matching tests. - There should be more tests... - - Additional headers include: - - xdelta3.h The public header file. - xdelta3-cfgs.h The default settings for default, built-in - encoders. These are hard-coded at - compile-time. There is also a single - soft-coded string matcher for experimenting - with arbitrary values. - xdelta3-list.h A cyclic list template - - Misc little debug utilities: - - badcopy.c Randomly modifies an input file based on two - parameters: (1) the probability that a byte in - the file is replaced with a pseudo-random value, - and (2) the mean change size. Changes are - generated using an expoential distribution - which approximates the expected error_prob - distribution. - -------------------------------------------------------------------- - - This file itself is unusually large. I hope to defend this layout - with lots of comments. Everything in this file is related to - encoding and decoding. I like it all together - the template stuff - is just a hack. */ - -#ifndef __XDELTA3_C_HEADER_PASS__ -#define __XDELTA3_C_HEADER_PASS__ - -#include "xdelta3.h" -#include "baselayer.h" - -/*********************************************************************** - STATIC CONFIGURATION - ***********************************************************************/ - -#ifndef XD3_MAIN /* the main application */ -#define XD3_MAIN 0 -#endif - -#ifndef VCDIFF_TOOLS -#define VCDIFF_TOOLS XD3_MAIN -#endif - -#ifndef SECONDARY_FGK /* one from the algorithm preservation department: */ -#define SECONDARY_FGK 0 /* adaptive Huffman routines */ -#endif - -#ifndef SECONDARY_DJW /* semi-adaptive/static Huffman for the eventual */ -#define SECONDARY_DJW 0 /* standardization, off by default until such time. */ -#endif - -#ifdef HAVE_LZMA_H -#define SECONDARY_LZMA 1 -#else -#define SECONDARY_LZMA 0 -#endif - -#ifndef GENERIC_ENCODE_TABLES /* These three are the RFC-spec app-specific */ -#define GENERIC_ENCODE_TABLES 0 /* code features. This is tested but not */ -#endif /* recommended unless there's a real use. */ -#ifndef GENERIC_ENCODE_TABLES_COMPUTE -#define GENERIC_ENCODE_TABLES_COMPUTE 0 -#endif -#ifndef GENERIC_ENCODE_TABLES_COMPUTE_PRINT -#define GENERIC_ENCODE_TABLES_COMPUTE_PRINT 0 -#endif - -#if XD3_ENCODER -#define IF_ENCODER(x) x -#else -#define IF_ENCODER(x) -#endif - -/***********************************************************************/ - - /* header indicator bits */ -#define VCD_SECONDARY (1U << 0) /* uses secondary compressor */ -#define VCD_CODETABLE (1U << 1) /* supplies code table data */ -#define VCD_APPHEADER (1U << 2) /* supplies application data */ -#define VCD_INVHDR (~0x7U) - - /* window indicator bits */ -#define VCD_SOURCE (1U << 0) /* copy window in source file */ -#define VCD_TARGET (1U << 1) /* copy window in target file */ -#define VCD_ADLER32 (1U << 2) /* has adler32 checksum */ -#define VCD_INVWIN (~0x7U) - -#define VCD_SRCORTGT (VCD_SOURCE | VCD_TARGET) - - /* delta indicator bits */ -#define VCD_DATACOMP (1U << 0) -#define VCD_INSTCOMP (1U << 1) -#define VCD_ADDRCOMP (1U << 2) -#define VCD_INVDEL (~0x7U) - -typedef enum { - VCD_DJW_ID = 1, - VCD_LZMA_ID = 2, - VCD_FGK_ID = 16 /* Note: these are not standard IANA-allocated IDs! */ -} xd3_secondary_ids; - -typedef enum { - SEC_NOFLAGS = 0, - - /* Note: SEC_COUNT_FREQS Not implemented (to eliminate 1st Huffman pass) */ - SEC_COUNT_FREQS = (1 << 0) -} xd3_secondary_flags; - -typedef enum { - DATA_SECTION, /* These indicate which section to the secondary - * compressor. */ - INST_SECTION, /* The header section is not compressed, therefore not - * listed here. */ - ADDR_SECTION -} xd3_section_type; - -typedef unsigned int xd3_rtype; - -/***********************************************************************/ - -#include "xdelta3-list.h" - -XD3_MAKELIST(xd3_rlist, xd3_rinst, link); - -/***********************************************************************/ - -#define SECONDARY_MIN_SAVINGS 2 /* Secondary compression has to save - at least this many bytes. */ -#define SECONDARY_MIN_INPUT 10 /* Secondary compression needs at - least this many bytes. */ - -#define VCDIFF_MAGIC1 0xd6 /* 1st file byte */ -#define VCDIFF_MAGIC2 0xc3 /* 2nd file byte */ -#define VCDIFF_MAGIC3 0xc4 /* 3rd file byte */ -#define VCDIFF_VERSION 0x00 /* 4th file byte */ - -#define VCD_SELF 0 /* 1st address mode */ -#define VCD_HERE 1 /* 2nd address mode */ - -#define CODE_TABLE_STRING_SIZE (6 * 256) /* Should fit a code table string. */ -#define CODE_TABLE_VCDIFF_SIZE (6 * 256) /* Should fit a compressed code - * table string */ - -#define SECONDARY_ANY (SECONDARY_DJW || SECONDARY_FGK || SECONDARY_LZMA) - -#define ALPHABET_SIZE 256 /* Used in test code--size of the secondary - * compressor alphabet. */ - -#define HASH_PERMUTE 1 /* The input is permuted by random nums */ -#define ADLER_LARGE_CKSUM 1 /* Adler checksum vs. RK checksum */ - -#define HASH_CKOFFSET 1U /* Table entries distinguish "no-entry" from - * offset 0 using this offset. */ - -#define MIN_SMALL_LOOK 2U /* Match-optimization stuff. */ -#define MIN_LARGE_LOOK 2U -#define MIN_MATCH_OFFSET 1U -#define MAX_MATCH_SPLIT 18U /* VCDIFF code table: 18 is the default limit - * for direct-coded ADD sizes */ - -#define LEAST_MATCH_INCR 0 /* The least number of bytes an overlapping - * match must beat the preceding match by. This - * is a bias for the lazy match optimization. A - * non-zero value means that an adjacent match - * has to be better by more than the step - * between them. 0. */ - -#define MIN_MATCH 4U /* VCDIFF code table: MIN_MATCH=4 */ -#define MIN_ADD 1U /* 1 */ -#define MIN_RUN 8U /* The shortest run, if it is shorter than this - * an immediate add/copy will be just as good. - * ADD1/COPY6 = 1I+1D+1A bytes, RUN18 = - * 1I+1D+1A. */ - -#define MAX_MODES 9 /* Maximum number of nodes used for - * compression--does not limit decompression. */ - -#define ENC_SECTS 4 /* Number of separate output sections. */ - -#define HDR_TAIL(s) ((s)->enc_tails[0]) -#define DATA_TAIL(s) ((s)->enc_tails[1]) -#define INST_TAIL(s) ((s)->enc_tails[2]) -#define ADDR_TAIL(s) ((s)->enc_tails[3]) - -#define HDR_HEAD(s) ((s)->enc_heads[0]) -#define DATA_HEAD(s) ((s)->enc_heads[1]) -#define INST_HEAD(s) ((s)->enc_heads[2]) -#define ADDR_HEAD(s) ((s)->enc_heads[3]) - -#define TOTAL_MODES(x) (2+(x)->acache.s_same+(x)->acache.s_near) - -/* Template instances. */ -#if XD3_BUILD_SLOW -#define IF_BUILD_SLOW(x) x -#else -#define IF_BUILD_SLOW(x) -#endif -#if XD3_BUILD_FAST -#define IF_BUILD_FAST(x) x -#else -#define IF_BUILD_FAST(x) -#endif -#if XD3_BUILD_FASTER -#define IF_BUILD_FASTER(x) x -#else -#define IF_BUILD_FASTER(x) -#endif -#if XD3_BUILD_FASTEST -#define IF_BUILD_FASTEST(x) x -#else -#define IF_BUILD_FASTEST(x) -#endif -#if XD3_BUILD_SOFT -#define IF_BUILD_SOFT(x) x -#else -#define IF_BUILD_SOFT(x) -#endif -#if XD3_BUILD_DEFAULT -#define IF_BUILD_DEFAULT(x) x -#else -#define IF_BUILD_DEFAULT(x) -#endif - -/* Consume N bytes of input, only used by the decoder. */ -#define DECODE_INPUT(n) \ - do { \ - stream->total_in += (xoff_t) (n); \ - stream->avail_in -= (n); \ - stream->next_in += (n); \ - } while (0) - -/* Update the run-length state */ -#define NEXTRUN(c) do { if ((c) == run_c) { run_l += 1; } \ - else { run_c = (c); run_l = 1; } } while (0) - -/* This CPP-conditional stuff can be cleaned up... */ -#if REGRESSION_TEST -#define IF_REGRESSION(x) x -#else -#define IF_REGRESSION(x) -#endif - -/* custom EDuke32 "not" variants created to help avoid compiler warnings */ -#if XD3_DEBUG -#define IFN_DEBUG(x) -#else -#define IFN_DEBUG(x) x -#endif -#if XD3_DEBUG > 1 -#define IFN_DEBUG1(x) -#else -#define IFN_DEBUG1(x) x -#endif -#if XD3_DEBUG > 2 -#define IFN_DEBUG2(x) -#else -#define IFN_DEBUG2(x) x -#endif -#if REGRESSION_TEST -#define IFN_REGRESSION(x) -#else -#define IFN_REGRESSION(x) x -#endif - -/***********************************************************************/ - -#if XD3_ENCODER -static void* xd3_alloc0 (xd3_stream *stream, - usize_t elts, - usize_t size); - - -static xd3_output* xd3_alloc_output (xd3_stream *stream, - xd3_output *old_output); - -static int xd3_alloc_iopt (xd3_stream *stream, usize_t elts); - -static void xd3_free_output (xd3_stream *stream, - xd3_output *output); - -static int xd3_emit_byte (xd3_stream *stream, - xd3_output **outputp, - uint8_t code); - -static int xd3_emit_bytes (xd3_stream *stream, - xd3_output **outputp, - const uint8_t *base, - usize_t size); - -static int xd3_emit_double (xd3_stream *stream, xd3_rinst *first, - xd3_rinst *second, usize_t code); -static int xd3_emit_single (xd3_stream *stream, xd3_rinst *single, - usize_t code); - -static usize_t xd3_sizeof_output (xd3_output *output); -static void xd3_encode_reset (xd3_stream *stream); - -static int xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos); -static int xd3_source_extend_match (xd3_stream *stream); -static int xd3_srcwin_setup (xd3_stream *stream); -static usize_t xd3_iopt_last_matched (xd3_stream *stream); -static int xd3_emit_uint32_t (xd3_stream *stream, xd3_output **output, - uint32_t num); - -static usize_t xd3_smatch (xd3_stream *stream, - usize_t base, - usize_t scksum, - usize_t *match_offset); -static int xd3_string_match_init (xd3_stream *stream); -static uint32_t xd3_scksum (uint32_t *state, const uint8_t *seg, - const usize_t ln); -static usize_t xd3_comprun (const uint8_t *seg, usize_t slook, uint8_t *run_cp); -static int xd3_srcwin_move_point (xd3_stream *stream, - usize_t *next_move_point); - -static int xd3_emit_run (xd3_stream *stream, usize_t pos, - usize_t size, uint8_t *run_c); -static usize_t xd3_checksum_hash (const xd3_hash_cfg *cfg, - const usize_t cksum); -static xoff_t xd3_source_cksum_offset(xd3_stream *stream, usize_t low); -static void xd3_scksum_insert (xd3_stream *stream, - usize_t inx, - usize_t scksum, - usize_t pos); - - -#if XD3_DEBUG -static void xd3_verify_run_state (xd3_stream *stream, - const uint8_t *inp, - usize_t x_run_l, - uint8_t *x_run_c); -static void xd3_verify_large_state (xd3_stream *stream, - const uint8_t *inp, - uint32_t x_cksum); -static void xd3_verify_small_state (xd3_stream *stream, - const uint8_t *inp, - uint32_t x_cksum); - -#endif /* XD3_DEBUG */ -#endif /* XD3_ENCODER */ - -static int xd3_decode_allocate (xd3_stream *stream, usize_t size, - uint8_t **copied1, usize_t *alloc1); - -static void xd3_compute_code_table_string (const xd3_dinst *code_table, - uint8_t *str); -static void* xd3_alloc (xd3_stream *stream, usize_t elts, usize_t size); -static void xd3_free (xd3_stream *stream, void *ptr); - -static int xd3_read_uint32_t (xd3_stream *stream, const uint8_t **inpp, - const uint8_t *max, uint32_t *valp); - -#if REGRESSION_TEST -static int xd3_selftest (void); -#endif - -/***********************************************************************/ - -#define UINT32_OFLOW_MASK 0xfe000000U -#define UINT64_OFLOW_MASK 0xfe00000000000000ULL - -#if SIZEOF_USIZE_T == 4 -#define USIZE_T_MAX UINT32_MAX -#define xd3_decode_size xd3_decode_uint32_t -#define xd3_emit_size xd3_emit_uint32_t -#define xd3_sizeof_size xd3_sizeof_uint32_t -#define xd3_read_size xd3_read_uint32_t -#elif SIZEOF_USIZE_T == 8 -#define USIZE_T_MAX UINT64_MAX -#define xd3_decode_size xd3_decode_uint64_t -#define xd3_emit_size xd3_emit_uint64_t -#define xd3_sizeof_size xd3_sizeof_uint64_t -#define xd3_read_size xd3_read_uint64_t -#endif - -#if SIZEOF_XOFF_T == 4 -#define XOFF_T_MAX UINT32_MAX -#define xd3_decode_offset xd3_decode_uint32_t -#define xd3_emit_offset xd3_emit_uint32_t -#elif SIZEOF_XOFF_T == 8 -#define XOFF_T_MAX UINT64_MAX -#define xd3_decode_offset xd3_decode_uint64_t -#define xd3_emit_offset xd3_emit_uint64_t -#endif - -#define USIZE_T_OVERFLOW(a,b) ((USIZE_T_MAX - (usize_t) (a)) < (usize_t) (b)) -#define XOFF_T_OVERFLOW(a,b) ((XOFF_T_MAX - (xoff_t) (a)) < (xoff_t) (b)) - -const char* xd3_strerror (int ret) -{ - switch (ret) - { - case XD3_INPUT: return "XD3_INPUT"; - case XD3_OUTPUT: return "XD3_OUTPUT"; - case XD3_GETSRCBLK: return "XD3_GETSRCBLK"; - case XD3_GOTHEADER: return "XD3_GOTHEADER"; - case XD3_WINSTART: return "XD3_WINSTART"; - case XD3_WINFINISH: return "XD3_WINFINISH"; - case XD3_TOOFARBACK: return "XD3_TOOFARBACK"; - case XD3_INTERNAL: return "XD3_INTERNAL"; - case XD3_INVALID: return "XD3_INVALID"; - case XD3_INVALID_INPUT: return "XD3_INVALID_INPUT"; - case XD3_NOSECOND: return "XD3_NOSECOND"; - case XD3_UNIMPLEMENTED: return "XD3_UNIMPLEMENTED"; - } - return NULL; -} - -/***********************************************************************/ - -#define xd3_sec_data(s) ((s)->sec_stream_d) -#define xd3_sec_inst(s) ((s)->sec_stream_i) -#define xd3_sec_addr(s) ((s)->sec_stream_a) - -struct _xd3_sec_type -{ - int id; - const char *name; - xd3_secondary_flags flags; - - /* xd3_sec_stream is opaque to the generic code */ - xd3_sec_stream* (*alloc) (xd3_stream *stream); - void (*destroy) (xd3_stream *stream, - xd3_sec_stream *sec); - int (*init) (xd3_stream *stream, - xd3_sec_stream *sec_stream, - int is_encode); - int (*decode) (xd3_stream *stream, - xd3_sec_stream *sec_stream, - const uint8_t **input, - const uint8_t *input_end, - uint8_t **output, - const uint8_t *output_end); -#if XD3_ENCODER - int (*encode) (xd3_stream *stream, - xd3_sec_stream *sec_stream, - xd3_output *input, - xd3_output *output, - xd3_sec_cfg *cfg); -#endif -}; - -#define BIT_STATE_ENCODE_INIT { 0, 1 } -#define BIT_STATE_DECODE_INIT { 0, 0x100 } - -typedef struct _bit_state bit_state; -struct _bit_state -{ - usize_t cur_byte; - usize_t cur_mask; -}; - -#if SECONDARY_ANY == 0 -#define IF_SEC(x) -#define IF_NSEC(x) x -#else /* yuck */ -#define IF_SEC(x) x -#define IF_NSEC(x) -static int -xd3_decode_secondary (xd3_stream *stream, - xd3_desect *sect, - xd3_sec_stream **sec_streamp); -#if XD3_ENCODER -static int -xd3_encode_secondary (xd3_stream *stream, - xd3_output **head, - xd3_output **tail, - xd3_sec_stream **sec_streamp, - xd3_sec_cfg *cfg, - int *did_it); -#endif -#endif /* SECONDARY_ANY */ - -#if SECONDARY_FGK -extern const xd3_sec_type fgk_sec_type; -#define IF_FGK(x) x -#define FGK_CASE(s) \ - s->sec_type = & fgk_sec_type; \ - break; -#else -#define IF_FGK(x) -#define FGK_CASE(s) \ - s->msg = "unavailable secondary compressor: FGK Adaptive Huffman"; \ - initprintf("xdelta3: %s\n",s->msg); \ - return XD3_INTERNAL; -#endif - -#if SECONDARY_DJW -extern const xd3_sec_type djw_sec_type; -#define IF_DJW(x) x -#define DJW_CASE(s) \ - s->sec_type = & djw_sec_type; \ - break; -#else -#define IF_DJW(x) -#define DJW_CASE(s) \ - s->msg = "unavailable secondary compressor: DJW Static Huffman"; \ - initprintf("xdelta3: %s\n",s->msg); \ - return XD3_INTERNAL; -#endif - -#if SECONDARY_LZMA -extern const xd3_sec_type lzma_sec_type; -#define IF_LZMA(x) x -#define LZMA_CASE(s) \ - s->sec_type = & lzma_sec_type; \ - break; -#else -#define IF_LZMA(x) -#define LZMA_CASE(s) \ - s->msg = "unavailable secondary compressor: LZMA"; \ - return XD3_INTERNAL; -#endif - -/***********************************************************************/ - -#include "xdelta3-hash.h" - -/* Process template passes - this includes xdelta3.c several times. */ -#define __XDELTA3_C_TEMPLATE_PASS__ -#include "xdelta3-cfgs.h" -#undef __XDELTA3_C_TEMPLATE_PASS__ - -/* Process the inline pass. */ -#define __XDELTA3_C_INLINE_PASS__ -#include "xdelta3.c" -#undef __XDELTA3_C_INLINE_PASS__ - -/* Secondary compression */ -#if SECONDARY_ANY -#include "xdelta3-second.h" -#endif - -#if SECONDARY_FGK -#include "xdelta3-fgk.h" -const xd3_sec_type fgk_sec_type = -{ - VCD_FGK_ID, - "FGK Adaptive Huffman", - SEC_NOFLAGS, - (xd3_sec_stream* (*)(xd3_stream*)) fgk_alloc, - (void (*)(xd3_stream*, xd3_sec_stream*)) fgk_destroy, - (int (*)(xd3_stream*, xd3_sec_stream*, int)) fgk_init, - (int (*)(xd3_stream*, xd3_sec_stream*, const uint8_t**, const uint8_t*, - uint8_t**, const uint8_t*)) xd3_decode_fgk, - IF_ENCODER((int (*)(xd3_stream*, xd3_sec_stream*, xd3_output*, - xd3_output*, xd3_sec_cfg*)) xd3_encode_fgk) -}; -#endif - -#if SECONDARY_DJW -#include "xdelta3-djw.h" -const xd3_sec_type djw_sec_type = -{ - VCD_DJW_ID, - "Static Huffman", - SEC_COUNT_FREQS, - (xd3_sec_stream* (*)(xd3_stream*)) djw_alloc, - (void (*)(xd3_stream*, xd3_sec_stream*)) djw_destroy, - (int (*)(xd3_stream*, xd3_sec_stream*, int)) djw_init, - (int (*)(xd3_stream*, xd3_sec_stream*, const uint8_t**, const uint8_t*, - uint8_t**, const uint8_t*)) xd3_decode_huff, - IF_ENCODER((int (*)(xd3_stream*, xd3_sec_stream*, xd3_output*, - xd3_output*, xd3_sec_cfg*)) xd3_encode_huff) -}; -#endif - -#if SECONDARY_LZMA -#include "xdelta3-lzma.h" -const xd3_sec_type lzma_sec_type = -{ - VCD_LZMA_ID, - "lzma", - SEC_NOFLAGS, - (xd3_sec_stream* (*)(xd3_stream*)) xd3_lzma_alloc, - (void (*)(xd3_stream*, xd3_sec_stream*)) xd3_lzma_destroy, - (int (*)(xd3_stream*, xd3_sec_stream*, int)) xd3_lzma_init, - (int (*)(xd3_stream*, xd3_sec_stream*, const uint8_t**, const uint8_t*, - uint8_t**, const uint8_t*)) xd3_decode_lzma, - IF_ENCODER((int (*)(xd3_stream*, xd3_sec_stream*, xd3_output*, - xd3_output*, xd3_sec_cfg*)) xd3_encode_lzma) -}; -#endif - -#if XD3_MAIN || PYTHON_MODULE || SWIG_MODULE || NOT_MAIN -#include "xdelta3-main.h" -#endif - -#if REGRESSION_TEST -#include "xdelta3-test.h" -#endif - -#endif /* __XDELTA3_C_HEADER_PASS__ */ -#ifdef __XDELTA3_C_INLINE_PASS__ - -const uint16_t __single_hash[256] = -{ - /* Random numbers generated using SLIB's pseudo-random number generator. - * This hashes the input alphabet. */ - 0xbcd1, 0xbb65, 0x42c2, 0xdffe, 0x9666, 0x431b, 0x8504, 0xeb46, - 0x6379, 0xd460, 0xcf14, 0x53cf, 0xdb51, 0xdb08, 0x12c8, 0xf602, - 0xe766, 0x2394, 0x250d, 0xdcbb, 0xa678, 0x02af, 0xa5c6, 0x7ea6, - 0xb645, 0xcb4d, 0xc44b, 0xe5dc, 0x9fe6, 0x5b5c, 0x35f5, 0x701a, - 0x220f, 0x6c38, 0x1a56, 0x4ca3, 0xffc6, 0xb152, 0x8d61, 0x7a58, - 0x9025, 0x8b3d, 0xbf0f, 0x95a3, 0xe5f4, 0xc127, 0x3bed, 0x320b, - 0xb7f3, 0x6054, 0x333c, 0xd383, 0x8154, 0x5242, 0x4e0d, 0x0a94, - 0x7028, 0x8689, 0x3a22, 0x0980, 0x1847, 0xb0f1, 0x9b5c, 0x4176, - 0xb858, 0xd542, 0x1f6c, 0x2497, 0x6a5a, 0x9fa9, 0x8c5a, 0x7743, - 0xa8a9, 0x9a02, 0x4918, 0x438c, 0xc388, 0x9e2b, 0x4cad, 0x01b6, - 0xab19, 0xf777, 0x365f, 0x1eb2, 0x091e, 0x7bf8, 0x7a8e, 0x5227, - 0xeab1, 0x2074, 0x4523, 0xe781, 0x01a3, 0x163d, 0x3b2e, 0x287d, - 0x5e7f, 0xa063, 0xb134, 0x8fae, 0x5e8e, 0xb7b7, 0x4548, 0x1f5a, - 0xfa56, 0x7a24, 0x900f, 0x42dc, 0xcc69, 0x02a0, 0x0b22, 0xdb31, - 0x71fe, 0x0c7d, 0x1732, 0x1159, 0xcb09, 0xe1d2, 0x1351, 0x52e9, - 0xf536, 0x5a4f, 0xc316, 0x6bf9, 0x8994, 0xb774, 0x5f3e, 0xf6d6, - 0x3a61, 0xf82c, 0xcc22, 0x9d06, 0x299c, 0x09e5, 0x1eec, 0x514f, - 0x8d53, 0xa650, 0x5c6e, 0xc577, 0x7958, 0x71ac, 0x8916, 0x9b4f, - 0x2c09, 0x5211, 0xf6d8, 0xcaaa, 0xf7ef, 0x287f, 0x7a94, 0xab49, - 0xfa2c, 0x7222, 0xe457, 0xd71a, 0x00c3, 0x1a76, 0xe98c, 0xc037, - 0x8208, 0x5c2d, 0xdfda, 0xe5f5, 0x0b45, 0x15ce, 0x8a7e, 0xfcad, - 0xaa2d, 0x4b5c, 0xd42e, 0xb251, 0x907e, 0x9a47, 0xc9a6, 0xd93f, - 0x085e, 0x35ce, 0xa153, 0x7e7b, 0x9f0b, 0x25aa, 0x5d9f, 0xc04d, - 0x8a0e, 0x2875, 0x4a1c, 0x295f, 0x1393, 0xf760, 0x9178, 0x0f5b, - 0xfa7d, 0x83b4, 0x2082, 0x721d, 0x6462, 0x0368, 0x67e2, 0x8624, - 0x194d, 0x22f6, 0x78fb, 0x6791, 0xb238, 0xb332, 0x7276, 0xf272, - 0x47ec, 0x4504, 0xa961, 0x9fc8, 0x3fdc, 0xb413, 0x007a, 0x0806, - 0x7458, 0x95c6, 0xccaa, 0x18d6, 0xe2ae, 0x1b06, 0xf3f6, 0x5050, - 0xc8e8, 0xf4ac, 0xc04c, 0xf41c, 0x992f, 0xae44, 0x5f1b, 0x1113, - 0x1738, 0xd9a8, 0x19ea, 0x2d33, 0x9698, 0x2fe9, 0x323f, 0xcde2, - 0x6d71, 0xe37d, 0xb697, 0x2c4f, 0x4373, 0x9102, 0x075d, 0x8e25, - 0x1672, 0xec28, 0x6acb, 0x86cc, 0x186e, 0x9414, 0xd674, 0xd1a5 -}; - -/**************************************************************** - Instruction tables - *****************************************************************/ - -/* The following code implements a parametrized description of the - * code table given above for a few reasons. It is not necessary for - * implementing the standard, to support compression with variable - * tables, so an implementation is only required to know the default - * code table to begin decompression. (If the encoder uses an - * alternate table, the table is included in compressed form inside - * the VCDIFF file.) - * - * Before adding variable-table support there were two functions which - * were hard-coded to the default table above. - * xd3_compute_default_table() would create the default table by - * filling a 256-elt array of xd3_dinst values. The corresponding - * function, xd3_choose_instruction(), would choose an instruction - * based on the hard-coded parameters of the default code table. - * - * Notes: The parametrized code table description here only generates - * tables of a certain regularity similar to the default table by - * allowing to vary the distribution of single- and - * double-instructions and change the number of near and same copy - * modes. More exotic tables are only possible by extending this - * code. - * - * For performance reasons, both the parametrized and non-parametrized - * versions of xd3_choose_instruction remain. The parametrized - * version is only needed for testing multi-table decoding support. - * If ever multi-table encoding is required, this can be optimized by - * compiling static functions for each table. - */ - -/* The XD3_CHOOSE_INSTRUCTION calls xd3_choose_instruction with the - * table description when GENERIC_ENCODE_TABLES are in use. The - * IF_GENCODETBL macro enables generic-code-table specific code. */ -#if GENERIC_ENCODE_TABLES -#define XD3_CHOOSE_INSTRUCTION(stream,prev,inst) xd3_choose_instruction (stream->code_table_desc, prev, inst) -#define IF_GENCODETBL(x) x -#else -#define XD3_CHOOSE_INSTRUCTION(stream,prev,inst) xd3_choose_instruction (prev, inst) -#define IF_GENCODETBL(x) -#endif - -/* This structure maintains information needed by - * xd3_choose_instruction to compute the code for a double instruction - * by first indexing an array of code_table_sizes by copy mode, then - * using (offset + (muliplier * X)) */ -struct _xd3_code_table_sizes { - uint8_t cpy_max; - uint8_t offset; - uint8_t mult; -}; - -/* This contains a complete description of a code table. */ -struct _xd3_code_table_desc -{ - /* Assumes a single RUN instruction */ - /* Assumes that MIN_MATCH is 4 */ - - uint8_t add_sizes; /* Number of immediate-size single adds (default 17) */ - uint8_t near_modes; /* Number of near copy modes (default 4) */ - uint8_t same_modes; /* Number of same copy modes (default 3) */ - uint8_t cpy_sizes; /* Number of immediate-size single copies (default 15) */ - - uint8_t addcopy_add_max; /* Maximum add size for an add-copy double instruction, - all modes (default 4) */ - uint8_t addcopy_near_cpy_max; /* Maximum cpy size for an add-copy double instruction, - up through VCD_NEAR modes (default 6) */ - uint8_t addcopy_same_cpy_max; /* Maximum cpy size for an add-copy double instruction, - VCD_SAME modes (default 4) */ - - uint8_t copyadd_add_max; /* Maximum add size for a copy-add double instruction, - all modes (default 1) */ - uint8_t copyadd_near_cpy_max; /* Maximum cpy size for a copy-add double instruction, - up through VCD_NEAR modes (default 4) */ - uint8_t copyadd_same_cpy_max; /* Maximum cpy size for a copy-add double instruction, - VCD_SAME modes (default 4) */ - - xd3_code_table_sizes addcopy_max_sizes[MAX_MODES]; - xd3_code_table_sizes copyadd_max_sizes[MAX_MODES]; -}; - -/* The rfc3284 code table is represented: */ -static const xd3_code_table_desc __rfc3284_code_table_desc = { - 17, /* add sizes */ - 4, /* near modes */ - 3, /* same modes */ - 15, /* copy sizes */ - - 4, /* add-copy max add */ - 6, /* add-copy max cpy, near */ - 4, /* add-copy max cpy, same */ - - 1, /* copy-add max add */ - 4, /* copy-add max cpy, near */ - 4, /* copy-add max cpy, same */ - - /* addcopy */ - { {6,163,3},{6,175,3},{6,187,3},{6,199,3},{6,211,3},{6,223,3},{4,235,1},{4,239,1},{4,243,1} }, - /* copyadd */ - { {4,247,1},{4,248,1},{4,249,1},{4,250,1},{4,251,1},{4,252,1},{4,253,1},{4,254,1},{4,255,1} }, -}; - -#if GENERIC_ENCODE_TABLES -/* An alternate code table for testing (5 near, 0 same): - * - * TYPE SIZE MODE TYPE SIZE MODE INDEX - * --------------------------------------------------------------- - * 1. Run 0 0 Noop 0 0 0 - * 2. Add 0, [1,23] 0 Noop 0 0 [1,24] - * 3. Copy 0, [4,20] 0 Noop 0 0 [25,42] - * 4. Copy 0, [4,20] 1 Noop 0 0 [43,60] - * 5. Copy 0, [4,20] 2 Noop 0 0 [61,78] - * 6. Copy 0, [4,20] 3 Noop 0 0 [79,96] - * 7. Copy 0, [4,20] 4 Noop 0 0 [97,114] - * 8. Copy 0, [4,20] 5 Noop 0 0 [115,132] - * 9. Copy 0, [4,20] 6 Noop 0 0 [133,150] - * 10. Add [1,4] 0 Copy [4,6] 0 [151,162] - * 11. Add [1,4] 0 Copy [4,6] 1 [163,174] - * 12. Add [1,4] 0 Copy [4,6] 2 [175,186] - * 13. Add [1,4] 0 Copy [4,6] 3 [187,198] - * 14. Add [1,4] 0 Copy [4,6] 4 [199,210] - * 15. Add [1,4] 0 Copy [4,6] 5 [211,222] - * 16. Add [1,4] 0 Copy [4,6] 6 [223,234] - * 17. Copy 4 [0,6] Add [1,3] 0 [235,255] - * --------------------------------------------------------------- */ -static const xd3_code_table_desc __alternate_code_table_desc = { - 23, /* add sizes */ - 5, /* near modes */ - 0, /* same modes */ - 17, /* copy sizes */ - - 4, /* add-copy max add */ - 6, /* add-copy max cpy, near */ - 0, /* add-copy max cpy, same */ - - 3, /* copy-add max add */ - 4, /* copy-add max cpy, near */ - 0, /* copy-add max cpy, same */ - - /* addcopy */ - { {6,151,3},{6,163,3},{6,175,3},{6,187,3},{6,199,3},{6,211,3},{6,223,3},{0,0,0},{0,0,0} }, - /* copyadd */ - { {4,235,1},{4,238,1},{4,241,1},{4,244,1},{4,247,1},{4,250,1},{4,253,1},{0,0,0},{0,0,0} }, -}; -#endif - -/* Computes code table entries of TBL using the specified description. */ -static void -xd3_build_code_table (const xd3_code_table_desc *desc, xd3_dinst *tbl) -{ - usize_t size1, size2, mode; - usize_t cpy_modes = 2 + desc->near_modes + desc->same_modes; - xd3_dinst *d = tbl; - - (d++)->type1 = XD3_RUN; - (d++)->type1 = XD3_ADD; - - for (size1 = 1; size1 <= desc->add_sizes; size1 += 1, d += 1) - { - d->type1 = XD3_ADD; - d->size1 = size1; - } - - for (mode = 0; mode < cpy_modes; mode += 1) - { - (d++)->type1 = XD3_CPY + mode; - - for (size1 = MIN_MATCH; size1 < MIN_MATCH + desc->cpy_sizes; size1 += 1, d += 1) - { - d->type1 = XD3_CPY + mode; - d->size1 = size1; - } - } - - for (mode = 0; mode < cpy_modes; mode += 1) - { - for (size1 = 1; size1 <= desc->addcopy_add_max; size1 += 1) - { - usize_t max = (mode < 2U + desc->near_modes) ? - desc->addcopy_near_cpy_max : - desc->addcopy_same_cpy_max; - - for (size2 = MIN_MATCH; size2 <= max; size2 += 1, d += 1) - { - d->type1 = XD3_ADD; - d->size1 = size1; - d->type2 = XD3_CPY + mode; - d->size2 = size2; - } - } - } - - for (mode = 0; mode < cpy_modes; mode += 1) - { - usize_t max = (mode < 2U + desc->near_modes) ? - desc->copyadd_near_cpy_max : - desc->copyadd_same_cpy_max; - - for (size1 = MIN_MATCH; size1 <= max; size1 += 1) - { - for (size2 = 1; size2 <= desc->copyadd_add_max; size2 += 1, d += 1) - { - d->type1 = XD3_CPY + mode; - d->size1 = size1; - d->type2 = XD3_ADD; - d->size2 = size2; - } - } - } - - XD3_ASSERT (d - tbl == 256); -} - -/* This function generates the static default code table. */ -static const xd3_dinst* -xd3_rfc3284_code_table (void) -{ - static xd3_dinst __rfc3284_code_table[256]; - - if (__rfc3284_code_table[0].type1 != XD3_RUN) - { - xd3_build_code_table (& __rfc3284_code_table_desc, __rfc3284_code_table); - } - - return __rfc3284_code_table; -} - -#if XD3_ENCODER -#if GENERIC_ENCODE_TABLES -/* This function generates the alternate code table. */ -static const xd3_dinst* -xd3_alternate_code_table (void) -{ - static xd3_dinst __alternate_code_table[256]; - - if (__alternate_code_table[0].type1 != XD3_RUN) - { - xd3_build_code_table (& __alternate_code_table_desc, __alternate_code_table); - } - - return __alternate_code_table; -} - -/* This function computes the ideal second instruction INST based on - * preceding instruction PREV. If it is possible to issue a double - * instruction based on this pair it sets PREV->code2, otherwise it - * sets INST->code1. */ -static void -xd3_choose_instruction (const xd3_code_table_desc *desc, xd3_rinst *prev, xd3_rinst *inst) -{ - switch (inst->type) - { - case XD3_RUN: - /* The 0th instruction is RUN */ - inst->code1 = 0; - break; - - case XD3_ADD: - - if (inst->size > desc->add_sizes) - { - /* The first instruction is non-immediate ADD */ - inst->code1 = 1; - } - else - { - /* The following ADD_SIZES instructions are immediate ADDs */ - inst->code1 = 1 + inst->size; - - /* Now check for a possible COPY-ADD double instruction */ - if (prev != NULL) - { - int prev_mode = prev->type - XD3_CPY; - - /* If previous is a copy. Note: as long as the previous - * is not a RUN instruction, it should be a copy because - * it cannot be an add. This check is more clear. */ - if (prev_mode >= 0 && inst->size <= desc->copyadd_add_max) - { - const xd3_code_table_sizes *sizes = & desc->copyadd_max_sizes[prev_mode]; - - /* This check and the inst->size-<= above are == in - the default table. */ - if (prev->size <= sizes->cpy_max) - { - /* The second and third exprs are 0 in the - default table. */ - prev->code2 = sizes->offset + - (sizes->mult * (prev->size - MIN_MATCH)) + - (inst->size - MIN_ADD); - } - } - } - } - break; - - default: - { - int mode = inst->type - XD3_CPY; - - /* The large copy instruction is offset by the run, large add, - * and immediate adds, then multipled by the number of - * immediate copies plus one (the large copy) (i.e., if there - * are 15 immediate copy instructions then there are 16 copy - * instructions per mode). */ - inst->code1 = 2 + desc->add_sizes + (1 + desc->cpy_sizes) * mode; - - /* Now if the copy is short enough for an immediate instruction. */ - if (inst->size < MIN_MATCH + desc->cpy_sizes && - /* TODO: there needs to be a more comprehensive test for this - * boundary condition, merge is now exercising code in which - * size < MIN_MATCH is possible and it's unclear if the above - * size < (MIN_MATCH + cpy_sizes) should be a <= from inspection - * of the default table version below. */ - inst->size >= MIN_MATCH) - { - inst->code1 += inst->size + 1 - MIN_MATCH; - - /* Now check for a possible ADD-COPY double instruction. */ - if ( (prev != NULL) && - (prev->type == XD3_ADD) && - (prev->size <= desc->addcopy_add_max) ) - { - const xd3_code_table_sizes *sizes = & desc->addcopy_max_sizes[mode]; - - if (inst->size <= sizes->cpy_max) - { - prev->code2 = sizes->offset + - (sizes->mult * (prev->size - MIN_ADD)) + - (inst->size - MIN_MATCH); - } - } - } - } - } -} -#else /* GENERIC_ENCODE_TABLES */ - -/* This version of xd3_choose_instruction is hard-coded for the default - table. */ -static void -xd3_choose_instruction (xd3_rinst *prev, xd3_rinst *inst) -{ - switch (inst->type) - { - case XD3_RUN: - inst->code1 = 0; - break; - - case XD3_ADD: - inst->code1 = 1; - - if (inst->size <= 17) - { - inst->code1 += inst->size; - - if ( (inst->size == 1) && - (prev != NULL) && - (prev->size == 4) && - (prev->type >= XD3_CPY) ) - { - prev->code2 = 247 + (prev->type - XD3_CPY); - } - } - - break; - - default: - { - int mode = inst->type - XD3_CPY; - - XD3_ASSERT (inst->type >= XD3_CPY && inst->type < 12); - - inst->code1 = 19 + 16 * mode; - - if (inst->size <= 18 && inst->size >= 4) - { - inst->code1 += inst->size - 3; - - if ( (prev != NULL) && - (prev->type == XD3_ADD) && - (prev->size <= 4) ) - { - if ( (inst->size <= 6) && - (mode <= 5) ) - { - prev->code2 = 163 + (mode * 12) + (3 * (prev->size - 1)) + (inst->size - 4); - - XD3_ASSERT (prev->code2 <= 234); - } - else if ( (inst->size == 4) && - (mode >= 6) ) - { - prev->code2 = 235 + ((mode - 6) * 4) + (prev->size - 1); - - XD3_ASSERT (prev->code2 <= 246); - } - } - } - - XD3_ASSERT (inst->code1 <= 162); - } - break; - } -} -#endif /* GENERIC_ENCODE_TABLES */ - -/*********************************************************************** - Instruction table encoder/decoder - ***********************************************************************/ - -#if GENERIC_ENCODE_TABLES -#if GENERIC_ENCODE_TABLES_COMPUTE == 0 - -/* In this case, we hard-code the result of - * compute_code_table_encoding for each alternate code table, - * presuming that saves time/space. This has been 131 bytes, but - * secondary compression was turned off. */ -static const uint8_t __alternate_code_table_compressed[178] = -{0xd6,0xc3,0xc4,0x00,0x00,0x01,0x8a,0x6f,0x40,0x81,0x27,0x8c,0x00,0x00,0x4a,0x4a,0x0d,0x02,0x01,0x03, -0x01,0x03,0x00,0x01,0x00,0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e, -0x0f,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x00,0x01,0x01,0x01,0x02,0x02,0x02,0x03,0x03,0x03,0x04, -0x04,0x04,0x04,0x00,0x04,0x05,0x06,0x01,0x02,0x03,0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x05,0x05,0x05, -0x06,0x06,0x06,0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x00,0x02,0x00,0x18,0x13,0x63,0x00,0x1b,0x00,0x54, -0x00,0x15,0x23,0x6f,0x00,0x28,0x13,0x54,0x00,0x15,0x01,0x1a,0x31,0x23,0x6c,0x0d,0x23,0x48,0x00,0x15, -0x93,0x6f,0x00,0x28,0x04,0x23,0x51,0x04,0x32,0x00,0x2b,0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x12,0x00, -0x12,0x00,0x12,0x53,0x57,0x9c,0x07,0x43,0x6f,0x00,0x34,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00, -0x0c,0x00,0x0c,0x00,0x15,0x00,0x82,0x6f,0x00,0x15,0x12,0x0c,0x00,0x03,0x03,0x00,0x06,0x00,}; - -static int -xd3_compute_alternate_table_encoding (xd3_stream *stream, const uint8_t **data, usize_t *size) -{ - (*data) = __alternate_code_table_compressed; - (*size) = sizeof (__alternate_code_table_compressed); - return 0; -} - -#else - -/* The alternate code table will be computed and stored here. */ -static uint8_t __alternate_code_table_compressed[CODE_TABLE_VCDIFF_SIZE]; -static usize_t __alternate_code_table_compressed_size; - -/* This function generates a delta describing the code table for - * encoding within a VCDIFF file. This function is NOT thread safe - * because it is only intended that this function is used to generate - * statically-compiled strings. "comp_string" must be sized - * CODE_TABLE_VCDIFF_SIZE. */ -int xd3_compute_code_table_encoding (xd3_stream *in_stream, - const xd3_dinst *code_table, - uint8_t *comp_string, - usize_t *comp_string_size) -{ - /* Use DJW secondary compression if it is on by default. This saves - * about 20 bytes. */ - uint8_t dflt_string[CODE_TABLE_STRING_SIZE]; - uint8_t code_string[CODE_TABLE_STRING_SIZE]; - - xd3_compute_code_table_string (xd3_rfc3284_code_table (), dflt_string); - xd3_compute_code_table_string (code_table, code_string); - - return xd3_encode_memory (code_string, CODE_TABLE_STRING_SIZE, - dflt_string, CODE_TABLE_STRING_SIZE, - comp_string, comp_string_size, - CODE_TABLE_VCDIFF_SIZE, - /* flags */ 0); -} - -/* Compute a delta between alternate and rfc3284 tables. As soon as - * another alternate table is added, this code should become generic. - * For now there is only one alternate table for testing. */ -static int -xd3_compute_alternate_table_encoding (xd3_stream *stream, const uint8_t **data, usize_t *size) -{ - int ret; - - if (__alternate_code_table_compressed[0] == 0) - { - if ((ret = xd3_compute_code_table_encoding (stream, xd3_alternate_code_table (), - __alternate_code_table_compressed, - & __alternate_code_table_compressed_size))) - { - return ret; - } - - /* During development of a new code table, enable this variable to print - * the new static contents and determine its size. At run time the - * table will be filled in appropriately, but at least it should have - * the proper size beforehand. */ -#if GENERIC_ENCODE_TABLES_COMPUTE_PRINT - { - int i; - - DP(RINT, "\nstatic const usize_t __alternate_code_table_compressed_size = %u;\n", - __alternate_code_table_compressed_size); - - DP(RINT, "static const uint8_t __alternate_code_table_compressed[%u] =\n{", - __alternate_code_table_compressed_size); - - for (i = 0; i < __alternate_code_table_compressed_size; i += 1) - { - DP(RINT, "0x%02x,", __alternate_code_table_compressed[i]); - if ((i % 20) == 19) { DP(RINT, "\n"); } - } - - DP(RINT, "};\n"); - } -#endif - } - - (*data) = __alternate_code_table_compressed; - (*size) = __alternate_code_table_compressed_size; - - return 0; -} -#endif /* GENERIC_ENCODE_TABLES_COMPUTE != 0 */ -#endif /* GENERIC_ENCODE_TABLES */ - -#endif /* XD3_ENCODER */ - -/* This function generates the 1536-byte string specified in sections 5.4 and - * 7 of rfc3284, which is used to represent a code table within a VCDIFF - * file. */ -void xd3_compute_code_table_string (const xd3_dinst *code_table, uint8_t *str) -{ - int i, s; - - XD3_ASSERT (CODE_TABLE_STRING_SIZE == 6 * 256); - - for (s = 0; s < 6; s += 1) - { - for (i = 0; i < 256; i += 1) - { - switch (s) - { - case 0: *str++ = (code_table[i].type1 >= XD3_CPY ? XD3_CPY : code_table[i].type1); break; - case 1: *str++ = (code_table[i].type2 >= XD3_CPY ? XD3_CPY : code_table[i].type2); break; - case 2: *str++ = (code_table[i].size1); break; - case 3: *str++ = (code_table[i].size2); break; - case 4: *str++ = (code_table[i].type1 >= XD3_CPY ? code_table[i].type1 - XD3_CPY : 0); break; - case 5: *str++ = (code_table[i].type2 >= XD3_CPY ? code_table[i].type2 - XD3_CPY : 0); break; - } - } - } -} - -/* This function translates the code table string into the internal representation. The - * stream's near and same-modes should already be set. */ -static int -xd3_apply_table_string (xd3_stream *stream, const uint8_t *code_string) -{ - int i, s; - int modes = TOTAL_MODES (stream); - xd3_dinst *code_table; - - if ((code_table = stream->code_table_alloc = - (xd3_dinst*) xd3_alloc (stream, - (usize_t) sizeof (xd3_dinst), - 256)) == NULL) - { - return ENOMEM; - } - - for (s = 0; s < 6; s += 1) - { - for (i = 0; i < 256; i += 1) - { - switch (s) - { - case 0: - if (*code_string > XD3_CPY) - { - stream->msg = "invalid code-table opcode"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - code_table[i].type1 = *code_string++; - break; - case 1: - if (*code_string > XD3_CPY) - { - stream->msg = "invalid code-table opcode"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - code_table[i].type2 = *code_string++; - break; - case 2: - if (*code_string != 0 && code_table[i].type1 == XD3_NOOP) - { - stream->msg = "invalid code-table size"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - code_table[i].size1 = *code_string++; - break; - case 3: - if (*code_string != 0 && code_table[i].type2 == XD3_NOOP) - { - stream->msg = "invalid code-table size"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - code_table[i].size2 = *code_string++; - break; - case 4: - if (*code_string >= modes) - { - stream->msg = "invalid code-table mode"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - if (*code_string != 0 && code_table[i].type1 != XD3_CPY) - { - stream->msg = "invalid code-table mode"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - code_table[i].type1 += *code_string++; - break; - case 5: - if (*code_string >= modes) - { - stream->msg = "invalid code-table mode"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - if (*code_string != 0 && code_table[i].type2 != XD3_CPY) - { - stream->msg = "invalid code-table mode"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - code_table[i].type2 += *code_string++; - break; - } - } - } - - stream->code_table = code_table; - return 0; -} - -/* This function applies a code table delta and returns an actual code table. */ -static int -xd3_apply_table_encoding (xd3_stream *in_stream, const uint8_t *data, usize_t size) -{ - uint8_t dflt_string[CODE_TABLE_STRING_SIZE]; - uint8_t code_string[CODE_TABLE_STRING_SIZE]; - usize_t code_size; - int ret; - - xd3_compute_code_table_string (xd3_rfc3284_code_table (), dflt_string); - - if ((ret = xd3_decode_memory (data, size, - dflt_string, CODE_TABLE_STRING_SIZE, - code_string, &code_size, - CODE_TABLE_STRING_SIZE, - 0))) { return ret; } - - if (code_size != sizeof (code_string)) - { - in_stream->msg = "corrupt code-table encoding"; - initprintf("xdelta3: %s\n",in_stream->msg); - return XD3_INTERNAL; - } - - return xd3_apply_table_string (in_stream, code_string); -} - -/***********************************************************************/ - -static inline void -xd3_swap_uint8p (uint8_t** p1, uint8_t** p2) -{ - uint8_t *t = (*p1); - (*p1) = (*p2); - (*p2) = t; -} - -static inline void -xd3_swap_usize_t (usize_t* p1, usize_t* p2) -{ - usize_t t = (*p1); - (*p1) = (*p2); - (*p2) = t; -} - -/* It's not constant time, but it computes the log. */ -static int -xd3_check_pow2 (usize_t value, usize_t *logof) -{ - usize_t x = 1; - usize_t nolog; - if (logof == NULL) { - logof = &nolog; - } - - *logof = 0; - - for (; x != 0; x <<= 1, *logof += 1) - { - if (x == value) - { - return 0; - } - } - - return XD3_INTERNAL; -} - -static usize_t -xd3_pow2_roundup (usize_t x) -{ - usize_t i = 1; - while (x > i) { - i <<= 1U; - } - return i; -} - -static usize_t -xd3_round_blksize (usize_t sz, usize_t blksz) -{ - usize_t mod = sz & (blksz-1); - - XD3_ASSERT (xd3_check_pow2 (blksz, NULL) == 0); - - return mod ? (sz + (blksz - mod)) : sz; -} - -/*********************************************************************** - Adler32 stream function: code copied from Zlib, defined in RFC1950 - ***********************************************************************/ - -#define A32_BASE 65521L /* Largest prime smaller than 2^16 */ -#define A32_NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ - -#define A32_DO1(buf,i) {s1 += buf[i]; s2 += s1;} -#define A32_DO2(buf,i) A32_DO1(buf,i); A32_DO1(buf,i+1); -#define A32_DO4(buf,i) A32_DO2(buf,i); A32_DO2(buf,i+2); -#define A32_DO8(buf,i) A32_DO4(buf,i); A32_DO4(buf,i+4); -#define A32_DO16(buf) A32_DO8(buf,0); A32_DO8(buf,8); - -static unsigned long adler32 (unsigned long adler, const uint8_t *buf, - usize_t len) -{ - unsigned long s1 = adler & 0xffff; - unsigned long s2 = (adler >> 16) & 0xffff; - int k; - - while (len > 0) - { - k = (len < A32_NMAX) ? len : A32_NMAX; - len -= k; - - while (k >= 16) - { - A32_DO16(buf); - buf += 16; - k -= 16; - } - - if (k != 0) - { - do - { - s1 += *buf++; - s2 += s1; - } - while (--k); - } - - s1 %= A32_BASE; - s2 %= A32_BASE; - } - - return (s2 << 16) | s1; -} - -/*********************************************************************** - Run-length function - ***********************************************************************/ - -#if XD3_ENCODER -static usize_t -xd3_comprun (const uint8_t *seg, usize_t slook, uint8_t *run_cp) -{ - usize_t i; - usize_t run_l = 0; - uint8_t run_c = 0; - - for (i = 0; i < slook; i += 1) - { - NEXTRUN(seg[i]); - } - - (*run_cp) = run_c; - - return run_l; -} -#endif - -/*********************************************************************** - Basic encoder/decoder functions - ***********************************************************************/ - -static inline int -xd3_decode_byte (xd3_stream *stream, usize_t *val) -{ - if (stream->avail_in == 0) - { - stream->msg = "further input required"; - return XD3_INPUT; - } - - (*val) = stream->next_in[0]; - - DECODE_INPUT (1); - return 0; -} - -static inline int -xd3_decode_bytes (xd3_stream *stream, uint8_t *buf, usize_t *pos, usize_t size) -{ - usize_t want; - usize_t take; - - /* Note: The case where (*pos == size) happens when a zero-length - * appheader or code table is transmitted, but there is nothing in - * the standard against that. */ - while (*pos < size) - { - if (stream->avail_in == 0) - { - stream->msg = "further input required"; - return XD3_INPUT; - } - - want = size - *pos; - take = min (want, stream->avail_in); - - memcpy (buf + *pos, stream->next_in, (size_t) take); - - DECODE_INPUT (take); - (*pos) += take; - } - - return 0; -} - -#if XD3_ENCODER -static inline int -xd3_emit_byte (xd3_stream *stream, - xd3_output **outputp, - uint8_t code) -{ - xd3_output *output = (*outputp); - - if (output->next == output->avail) - { - xd3_output *aoutput; - - if ((aoutput = xd3_alloc_output (stream, output)) == NULL) - { - return ENOMEM; - } - - output = (*outputp) = aoutput; - } - - output->base[output->next++] = code; - - return 0; -} - -static inline int -xd3_emit_bytes (xd3_stream *stream, - xd3_output **outputp, - const uint8_t *base, - usize_t size) -{ - xd3_output *output = (*outputp); - - do - { - usize_t take; - - if (output->next == output->avail) - { - xd3_output *aoutput; - - if ((aoutput = xd3_alloc_output (stream, output)) == NULL) - { - return ENOMEM; - } - - output = (*outputp) = aoutput; - } - - take = min (output->avail - output->next, size); - - memcpy (output->base + output->next, base, (size_t) take); - - output->next += take; - size -= take; - base += take; - } - while (size > 0); - - return 0; -} -#endif /* XD3_ENCODER */ - -/********************************************************************* - Integer encoder/decoder functions - **********************************************************************/ - -#define DECODE_INTEGER_TYPE(PART,OFLOW) \ - while (stream->avail_in != 0) \ - { \ - usize_t next = stream->next_in[0]; \ - \ - DECODE_INPUT(1); \ - \ - if (PART & OFLOW) \ - { \ - stream->msg = "overflow in decode_integer"; \ - initprintf("xdelta3: %s\n", stream->msg); \ - return XD3_INVALID_INPUT; \ - } \ - \ - PART = (PART << 7) | (next & 127); \ - \ - if ((next & 128) == 0) \ - { \ - (*val) = PART; \ - PART = 0; \ - return 0; \ - } \ - } \ - \ - stream->msg = "further input required"; \ - return XD3_INPUT - -#define READ_INTEGER_TYPE(TYPE, OFLOW) \ - TYPE val = 0; \ - const uint8_t *inp = (*inpp); \ - usize_t next; \ - \ - do \ - { \ - if (inp == max) \ - { \ - stream->msg = "end-of-input in read_integer"; \ - initprintf("xdelta3: %s\n", stream->msg); \ - return XD3_INVALID_INPUT; \ - } \ - \ - if (val & OFLOW) \ - { \ - stream->msg = "overflow in read_intger"; \ - initprintf("xdelta3: %s\n", stream->msg); \ - return XD3_INVALID_INPUT; \ - } \ - \ - next = (*inp++); \ - val = (val << 7) | (next & 127); \ - } \ - while (next & 128); \ - \ - (*valp) = val; \ - (*inpp) = inp; \ - \ - return 0 - -#define EMIT_INTEGER_TYPE() \ - /* max 64-bit value in base-7 encoding is 9.1 bytes */ \ - uint8_t buf[10]; \ - usize_t bufi = 10; \ - \ - /* This loop performs division and turns on all MSBs. */ \ - do \ - { \ - buf[--bufi] = (num & 127) | 128; \ - num >>= 7U; \ - } \ - while (num != 0); \ - \ - /* Turn off MSB of the last byte. */ \ - buf[9] &= 127; \ - \ - return xd3_emit_bytes (stream, output, buf + bufi, 10 - bufi) - -#define IF_SIZEOF32(x) if (num < (1U << (7 * (x)))) return (x); -#define IF_SIZEOF64(x) if (num < (1ULL << (7 * (x)))) return (x); - -#if USE_UINT32 -static inline uint32_t -xd3_sizeof_uint32_t (uint32_t num) -{ - IF_SIZEOF32(1); - IF_SIZEOF32(2); - IF_SIZEOF32(3); - IF_SIZEOF32(4); - return 5; -} - -static inline int -xd3_decode_uint32_t (xd3_stream *stream, uint32_t *val) -{ DECODE_INTEGER_TYPE (stream->dec_32part, UINT32_OFLOW_MASK); } - -static inline int -xd3_read_uint32_t (xd3_stream *stream, const uint8_t **inpp, - const uint8_t *max, uint32_t *valp) -{ READ_INTEGER_TYPE (uint32_t, UINT32_OFLOW_MASK); } - -#if XD3_ENCODER -static inline int -xd3_emit_uint32_t (xd3_stream *stream, xd3_output **output, uint32_t num) -{ EMIT_INTEGER_TYPE (); } -#endif -#endif - -#if USE_UINT64 -static inline int -xd3_decode_uint64_t (xd3_stream *stream, uint64_t *val) -{ DECODE_INTEGER_TYPE (stream->dec_64part, UINT64_OFLOW_MASK); } - -#if XD3_ENCODER -static inline int -xd3_emit_uint64_t (xd3_stream *stream, xd3_output **output, uint64_t num) -{ EMIT_INTEGER_TYPE (); } -#endif - -/* These are tested but not used */ -#if REGRESSION_TEST -static int -xd3_read_uint64_t (xd3_stream *stream, const uint8_t **inpp, - const uint8_t *max, uint64_t *valp) -{ READ_INTEGER_TYPE (uint64_t, UINT64_OFLOW_MASK); } - -static uint32_t -xd3_sizeof_uint64_t (uint64_t num) -{ - IF_SIZEOF64(1); - IF_SIZEOF64(2); - IF_SIZEOF64(3); - IF_SIZEOF64(4); - IF_SIZEOF64(5); - IF_SIZEOF64(6); - IF_SIZEOF64(7); - IF_SIZEOF64(8); - IF_SIZEOF64(9); - - return 10; -} -#endif - -#endif - -/*********************************************************************** - Address cache stuff - ***********************************************************************/ - -static int -xd3_alloc_cache (xd3_stream *stream) -{ - if (stream->acache.near_array != NULL) - { - xd3_free (stream, stream->acache.near_array); - } - - if (stream->acache.same_array != NULL) - { - xd3_free (stream, stream->acache.same_array); - } - - if (((stream->acache.s_near > 0) && - (stream->acache.near_array = (usize_t*) - xd3_alloc (stream, stream->acache.s_near, - (usize_t) sizeof (usize_t))) - == NULL) || - ((stream->acache.s_same > 0) && - (stream->acache.same_array = (usize_t*) - xd3_alloc (stream, stream->acache.s_same * 256, - (usize_t) sizeof (usize_t))) - == NULL)) - { - return ENOMEM; - } - - return 0; -} - -void -xd3_init_cache (xd3_addr_cache* acache) -{ - if (acache->s_near > 0) - { - memset (acache->near_array, 0, acache->s_near * sizeof (usize_t)); - acache->next_slot = 0; - } - - if (acache->s_same > 0) - { - memset (acache->same_array, 0, acache->s_same * 256 * sizeof (usize_t)); - } -} - -static void -xd3_update_cache (xd3_addr_cache* acache, usize_t addr) -{ - if (acache->s_near > 0) - { - acache->near_array[acache->next_slot] = addr; - acache->next_slot = (acache->next_slot + 1) % acache->s_near; - } - - if (acache->s_same > 0) - { - acache->same_array[addr % (acache->s_same*256)] = addr; - } -} - -#if XD3_ENCODER -/* OPT: this gets called a lot, can it be optimized? */ -static int -xd3_encode_address (xd3_stream *stream, - usize_t addr, - usize_t here, - uint8_t* mode) -{ - usize_t d, bestd; - usize_t i, bestm, ret; - xd3_addr_cache* acache = & stream->acache; - -#define SMALLEST_INT(x) do { if (((x) & ~127U) == 0) { goto good; } } while (0) - - /* Attempt to find the address mode that yields the smallest integer value - * for "d", the encoded address value, thereby minimizing the encoded size - * of the address. */ - bestd = addr; - bestm = VCD_SELF; - - XD3_ASSERT (addr < here); - - SMALLEST_INT (bestd); - - if ((d = here-addr) < bestd) - { - bestd = d; - bestm = VCD_HERE; - - SMALLEST_INT (bestd); - } - - for (i = 0; i < acache->s_near; i += 1) - { - /* Note: If we used signed computation here, we'd could compte d - * and then check (d >= 0 && d < bestd). */ - if (addr >= acache->near_array[i]) - { - d = addr - acache->near_array[i]; - - if (d < bestd) - { - bestd = d; - bestm = i+2; /* 2 counts the VCD_SELF, VCD_HERE modes */ - - SMALLEST_INT (bestd); - } - } - } - - if (acache->s_same > 0 && - acache->same_array[d = addr%(acache->s_same*256)] == addr) - { - bestd = d%256; - /* 2 + s_near offsets past the VCD_NEAR modes */ - bestm = acache->s_near + 2 + d/256; - - if ((ret = xd3_emit_byte (stream, & ADDR_TAIL (stream), bestd))) - { - return ret; - } - } - else - { - good: - - if ((ret = xd3_emit_size (stream, & ADDR_TAIL (stream), bestd))) - { - return ret; - } - } - - xd3_update_cache (acache, addr); - - (*mode) += bestm; - - return 0; -} -#endif - -static int -xd3_decode_address (xd3_stream *stream, usize_t here, - usize_t mode, const uint8_t **inpp, - const uint8_t *max, uint32_t *valp) -{ - int ret; - usize_t same_start = 2 + stream->acache.s_near; - - if (mode < same_start) - { - if ((ret = xd3_read_size (stream, inpp, max, valp))) { return ret; } - - switch (mode) - { - case VCD_SELF: - break; - case VCD_HERE: - (*valp) = here - (*valp); - break; - default: - (*valp) += stream->acache.near_array[mode - 2]; - break; - } - } - else - { - if (*inpp == max) - { - stream->msg = "address underflow"; - initprintf("xdelta3: %s\n", stream->msg); - return XD3_INVALID_INPUT; - } - - mode -= same_start; - - (*valp) = stream->acache.same_array[mode*256 + (**inpp)]; - - (*inpp) += 1; - } - - xd3_update_cache (& stream->acache, *valp); - - return 0; -} - -/*********************************************************************** - Alloc/free -***********************************************************************/ - -static void* -__xd3_alloc_func (void* opaque ATTRIBUTE((unused)), usize_t items, usize_t size) -{ - return malloc ((size_t) items * (size_t) size); -} - -static void -__xd3_free_func (void* opaque ATTRIBUTE((unused)), void* address) -{ - free (address); -} - -static void* -xd3_alloc (xd3_stream *stream, - usize_t elts, - usize_t size) -{ - void *a = stream->alloc (stream->opaque, elts, size); - - if (a != NULL) - { - IF_DEBUG (stream->alloc_cnt += 1); - IF_DEBUG2 (DP(RINT "[stream %p malloc] size %u ptr %p\n", - stream, elts * size, a)); - } - else - { - stream->msg = "out of memory"; - } - - return a; -} - -static void -xd3_free (xd3_stream *stream, - void *ptr) -{ - if (ptr != NULL) - { - IF_DEBUG (stream->free_cnt += 1); - XD3_ASSERT (stream->free_cnt <= stream->alloc_cnt); - IF_DEBUG2 (DP(RINT "[stream %p free] %p\n", - stream, ptr)); - stream->free (stream->opaque, ptr); - } -} - -#if XD3_ENCODER -static void* -xd3_alloc0 (xd3_stream *stream, - usize_t elts, - usize_t size) -{ - void *a = xd3_alloc (stream, elts, size); - - if (a != NULL) - { - memset (a, 0, (size_t) (elts * size)); - } - - return a; -} - -static xd3_output* -xd3_alloc_output (xd3_stream *stream, - xd3_output *old_output) -{ - xd3_output *output; - uint8_t *base; - - if (stream->enc_free != NULL) - { - output = stream->enc_free; - stream->enc_free = output->next_page; - } - else - { - if ((output = (xd3_output*) xd3_alloc (stream, 1, - (usize_t) sizeof (xd3_output))) - == NULL) - { - return NULL; - } - - if ((base = (uint8_t*) xd3_alloc (stream, XD3_ALLOCSIZE, - sizeof (uint8_t))) == NULL) - { - xd3_free (stream, output); - return NULL; - } - - output->base = base; - output->avail = XD3_ALLOCSIZE; - } - - output->next = 0; - - if (old_output) - { - old_output->next_page = output; - } - - output->next_page = NULL; - - return output; -} - -static usize_t -xd3_sizeof_output (xd3_output *output) -{ - usize_t s = 0; - - for (; output; output = output->next_page) - { - s += output->next; - } - - return s; -} - -static void -xd3_freelist_output (xd3_stream *stream, - xd3_output *output) -{ - xd3_output *tmp; - - while (output) - { - tmp = output; - output = output->next_page; - - tmp->next = 0; - tmp->next_page = stream->enc_free; - stream->enc_free = tmp; - } -} - -static void -xd3_free_output (xd3_stream *stream, - xd3_output *output) -{ - xd3_output *next; - - again: - if (output == NULL) - { - return; - } - - next = output->next_page; - - xd3_free (stream, output->base); - xd3_free (stream, output); - - output = next; - goto again; -} -#endif /* XD3_ENCODER */ - -void -xd3_free_stream (xd3_stream *stream) -{ - xd3_iopt_buflist *blist = stream->iopt_alloc; - - while (blist != NULL) - { - xd3_iopt_buflist *tmp = blist; - blist = blist->next; - xd3_free (stream, tmp->buffer); - xd3_free (stream, tmp); - } - - xd3_free (stream, stream->large_table); - xd3_free (stream, stream->small_table); - xd3_free (stream, stream->small_prev); - -#if XD3_ENCODER - { - int i; - for (i = 0; i < ENC_SECTS; i += 1) - { - xd3_free_output (stream, stream->enc_heads[i]); - } - xd3_free_output (stream, stream->enc_free); - } -#endif - - xd3_free (stream, stream->acache.near_array); - xd3_free (stream, stream->acache.same_array); - - xd3_free (stream, stream->inst_sect.copied1); - xd3_free (stream, stream->addr_sect.copied1); - xd3_free (stream, stream->data_sect.copied1); - - xd3_free (stream, stream->dec_buffer); - xd3_free (stream, (uint8_t*) stream->dec_lastwin); - - xd3_free (stream, stream->buf_in); - xd3_free (stream, stream->dec_appheader); - xd3_free (stream, stream->dec_codetbl); - xd3_free (stream, stream->code_table_alloc); - -#if SECONDARY_ANY - xd3_free (stream, stream->inst_sect.copied2); - xd3_free (stream, stream->addr_sect.copied2); - xd3_free (stream, stream->data_sect.copied2); - - if (stream->sec_type != NULL) - { - stream->sec_type->destroy (stream, stream->sec_stream_d); - stream->sec_type->destroy (stream, stream->sec_stream_i); - stream->sec_type->destroy (stream, stream->sec_stream_a); - } -#endif - - xd3_free (stream, stream->whole_target.adds); - xd3_free (stream, stream->whole_target.inst); - xd3_free (stream, stream->whole_target.wininfo); - - XD3_ASSERT (stream->alloc_cnt == stream->free_cnt); - - memset (stream, 0, sizeof (xd3_stream)); -} - -#if (XD3_DEBUG > 1 || VCDIFF_TOOLS) -static const char* -xd3_rtype_to_string (xd3_rtype type, int print_mode) -{ - switch (type) - { - case XD3_NOOP: - return "NOOP "; - case XD3_RUN: - return "RUN "; - case XD3_ADD: - return "ADD "; - default: break; - } - if (! print_mode) - { - return "CPY "; - } - switch (type) - { - case XD3_CPY + 0: return "CPY_0"; - case XD3_CPY + 1: return "CPY_1"; - case XD3_CPY + 2: return "CPY_2"; - case XD3_CPY + 3: return "CPY_3"; - case XD3_CPY + 4: return "CPY_4"; - case XD3_CPY + 5: return "CPY_5"; - case XD3_CPY + 6: return "CPY_6"; - case XD3_CPY + 7: return "CPY_7"; - case XD3_CPY + 8: return "CPY_8"; - case XD3_CPY + 9: return "CPY_9"; - default: return "CPY>9"; - } -} -#endif - -/**************************************************************** - Stream configuration - ******************************************************************/ - -int -xd3_config_stream(xd3_stream *stream, - xd3_config *config) -{ - int ret; - xd3_config defcfg; - xd3_smatcher *smatcher = &stream->smatcher; - - if (config == NULL) - { - config = & defcfg; - memset (config, 0, sizeof (*config)); - } - - /* Initial setup: no error checks yet */ - memset (stream, 0, sizeof (*stream)); - - stream->winsize = config->winsize ? config->winsize : XD3_DEFAULT_WINSIZE; - stream->sprevsz = config->sprevsz ? config->sprevsz : XD3_DEFAULT_SPREVSZ; - stream->srcwin_maxsz = config->srcwin_maxsz ? - config->srcwin_maxsz : XD3_DEFAULT_SRCWINSZ; - - if (config->iopt_size == 0) - { - stream->iopt_size = XD3_ALLOCSIZE / sizeof(xd3_rinst); - stream->iopt_unlimited = 1; - } - else - { - stream->iopt_size = config->iopt_size; - } - - stream->getblk = config->getblk; - stream->alloc = config->alloc ? config->alloc : __xd3_alloc_func; - stream->free = config->freef ? config->freef : __xd3_free_func; - stream->opaque = config->opaque; - stream->flags = config->flags; - - /* Secondary setup. */ - stream->sec_data = config->sec_data; - stream->sec_inst = config->sec_inst; - stream->sec_addr = config->sec_addr; - - stream->sec_data.data_type = DATA_SECTION; - stream->sec_inst.data_type = INST_SECTION; - stream->sec_addr.data_type = ADDR_SECTION; - - /* Check static sizes. */ - if (sizeof (usize_t) != SIZEOF_USIZE_T || - sizeof (xoff_t) != SIZEOF_XOFF_T || - (ret = xd3_check_pow2(XD3_ALLOCSIZE, NULL))) - { - stream->msg = "incorrect compilation: wrong integer sizes"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - - /* Check/set secondary compressor. */ - switch (stream->flags & XD3_SEC_TYPE) - { - case 0: - if (stream->flags & XD3_SEC_NOALL) - { - stream->msg = "XD3_SEC flags require a secondary compressor type"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - break; - case XD3_SEC_FGK: - FGK_CASE (stream); - case XD3_SEC_DJW: - DJW_CASE (stream); - case XD3_SEC_LZMA: - LZMA_CASE (stream); - default: - stream->msg = "too many secondary compressor types set"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - - /* Check/set encoder code table. */ - switch (stream->flags & XD3_ALT_CODE_TABLE) { - case 0: - stream->code_table_desc = & __rfc3284_code_table_desc; - stream->code_table_func = xd3_rfc3284_code_table; - break; -#if GENERIC_ENCODE_TABLES - case XD3_ALT_CODE_TABLE: - stream->code_table_desc = & __alternate_code_table_desc; - stream->code_table_func = xd3_alternate_code_table; - stream->comp_table_func = xd3_compute_alternate_table_encoding; - break; -#endif - default: - stream->msg = "alternate code table support was not compiled"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - - /* Check sprevsz */ - if (smatcher->small_chain == 1 && - smatcher->small_lchain == 1) - { - stream->sprevsz = 0; - } - else - { - if ((ret = xd3_check_pow2 (stream->sprevsz, NULL))) - { - stream->msg = "sprevsz is required to be a power of two"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - - stream->sprevmask = stream->sprevsz - 1; - } - - /* Default scanner settings. */ -#if XD3_ENCODER - switch (config->smatch_cfg) - { - IF_BUILD_SOFT(case XD3_SMATCH_SOFT: - { - *smatcher = config->smatcher_soft; - smatcher->string_match = __smatcher_soft.string_match; - smatcher->name = __smatcher_soft.name; - if (smatcher->large_look < MIN_MATCH || - smatcher->large_step < 1 || - smatcher->small_look < MIN_MATCH) - { - stream->msg = "invalid soft string-match config"; - return XD3_INVALID; - } - break; - }) - - IF_BUILD_DEFAULT(case XD3_SMATCH_DEFAULT: - *smatcher = __smatcher_default; - break;) - IF_BUILD_SLOW(case XD3_SMATCH_SLOW: - *smatcher = __smatcher_slow; - break;) - IF_BUILD_FASTEST(case XD3_SMATCH_FASTEST: - *smatcher = __smatcher_fastest; - break;) - IF_BUILD_FASTER(case XD3_SMATCH_FASTER: - *smatcher = __smatcher_faster; - break;) - IF_BUILD_FAST(case XD3_SMATCH_FAST: - *smatcher = __smatcher_fast; - break;) - default: - stream->msg = "invalid string match config type"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - - if (config->smatch_cfg == XD3_SMATCH_DEFAULT && - (stream->flags & XD3_COMPLEVEL_MASK) != 0) - { - int level = (stream->flags & XD3_COMPLEVEL_MASK) >> XD3_COMPLEVEL_SHIFT; - - switch (level) - { - case 1: - IF_BUILD_FASTEST(*smatcher = __smatcher_fastest; - break;) - case 2: - IF_BUILD_FASTER(*smatcher = __smatcher_faster; - break;) - case 3: case 4: case 5: - IF_BUILD_FAST(*smatcher = __smatcher_fast; - break;) - case 6: - IF_BUILD_DEFAULT(*smatcher = __smatcher_default; - break;) - default: - IF_BUILD_SLOW(*smatcher = __smatcher_slow; - break;) - IF_BUILD_DEFAULT(*smatcher = __smatcher_default; - break;) - IF_BUILD_FAST(*smatcher = __smatcher_fast; - break;) - IF_BUILD_FASTER(*smatcher = __smatcher_faster; - break;) - IF_BUILD_FASTEST(*smatcher = __smatcher_fastest; - break;) - } - } -#endif - - return 0; -} - -/*********************************************************** - Getblk interface - ***********************************************************/ - -inline -xoff_t xd3_source_eof(const xd3_source *src) -{ - xoff_t r = (src->blksize * src->max_blkno) + (xoff_t)src->onlastblk; - return r; -} - -inline -usize_t xd3_bytes_on_srcblk (xd3_source *src, xoff_t blkno) -{ - usize_t r = (blkno == src->max_blkno ? - src->onlastblk : - src->blksize); - return r; -} - -/* This function interfaces with the client getblk function, checks - * its results, updates frontier_blkno, max_blkno, onlastblk, eof_known. */ -static int -xd3_getblk (xd3_stream *stream, xoff_t blkno) -{ - int ret; - xd3_source *source = stream->src; - - if (source->curblk == NULL || blkno != source->curblkno) - { - source->getblkno = blkno; - - if (stream->getblk == NULL) - { - stream->msg = "getblk source input"; - return XD3_GETSRCBLK; - } - - ret = stream->getblk (stream, source, blkno); - if (ret != 0) - { - IF_DEBUG1 (DP(RINT "[getblk] app error blkno %"Q"u: %s\n", - blkno, xd3_strerror (ret))); - return ret; - } - } - - if (blkno >= source->frontier_blkno) - { - if (blkno > source->max_blkno) - { - source->max_blkno = blkno; - source->onlastblk = source->onblk; - } - - if (source->onblk == source->blksize) - { - source->frontier_blkno = blkno + 1; - - IF_DEBUG2 (DP(RINT "[getblk] full source blkno %"Q"u: " - "source length unknown %"Q"u\n", - blkno, - xd3_source_eof (source))); - } - else - { - if (!source->eof_known) - { - IF_DEBUG2 (DP(RINT "[getblk] eof block has %d bytes; " - "source length known %"Q"u\n", - xd3_bytes_on_srcblk (source, blkno), - xd3_source_eof (source))); - source->eof_known = 1; - } - - source->frontier_blkno = blkno; - } - } - - XD3_ASSERT (source->curblk != NULL); - IF_DEBUG2 (DP(RINT "[getblk] read source block %"Q"u onblk %u blksize %u\n", - blkno, source->onblk, source->blksize)); - - if (blkno == source->max_blkno) - { - /* In case the application sets the source as 1 block w/ a - preset buffer. */ - source->onlastblk = source->onblk; - - if (source->onblk == source->blksize) - { - source->frontier_blkno = blkno + 1; - } - } - return 0; -} - -/*********************************************************** - Stream open/close - ***************************************************************/ - -int -xd3_set_source (xd3_stream *stream, - xd3_source *src) -{ - usize_t shiftby; - - stream->src = src; - src->srclen = 0; - src->srcbase = 0; - - /* Enforce power-of-two blocksize so that source-block number - * calculations are cheap. */ - if (!xd3_check_pow2 (src->blksize, &shiftby) == 0) - { - src->blksize = xd3_pow2_roundup(src->blksize); - xd3_check_pow2 (src->blksize, &shiftby); - IF_DEBUG1 (DP(RINT "raising srcblksz to %u\n", src->blksize)); - } - - src->shiftby = shiftby; - src->maskby = (1 << shiftby) - 1; - return 0; -} - -int -xd3_set_source_and_size (xd3_stream *stream, - xd3_source *user_source, - xoff_t source_size) { - int ret = xd3_set_source (stream, user_source); - if (ret == 0) - { - stream->src->eof_known = 1; - IF_DEBUG2 (DP(RINT "[set source] size known %"Q"u\n", - source_size)); - - xd3_blksize_div(source_size, - stream->src, - &stream->src->max_blkno, - &stream->src->onlastblk); - } - return ret; -} - -void -xd3_abort_stream (xd3_stream *stream) -{ - stream->dec_state = DEC_ABORTED; - stream->enc_state = ENC_ABORTED; -} - -int -xd3_close_stream (xd3_stream *stream) -{ - if (stream->enc_state != 0 && stream->enc_state != ENC_ABORTED) - { - if (stream->buf_leftover != NULL) - { - stream->msg = "encoding is incomplete"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - - if (stream->enc_state == ENC_POSTWIN) - { -#if XD3_ENCODER - xd3_encode_reset (stream); -#endif - stream->current_window += 1; - stream->enc_state = ENC_INPUT; - } - - /* If encoding, should be ready for more input but not actually - have any. */ - if (stream->enc_state != ENC_INPUT || stream->avail_in != 0) - { - stream->msg = "encoding is incomplete"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - } - else - { - switch (stream->dec_state) - { - case DEC_VCHEAD: - case DEC_WININD: - /* TODO: Address the zero-byte ambiguity. Does the encoder - * emit a window or not? If so, then catch an error here. - * If not, need another routine to say - * decode_at_least_one_if_empty. */ - case DEC_ABORTED: - break; - default: - /* If decoding, should be ready for the next window. */ - stream->msg = "EOF in decode"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - } - - return 0; -} - -/************************************************************** - Application header - ****************************************************************/ - -int -xd3_get_appheader (xd3_stream *stream, - uint8_t **data, - usize_t *size) -{ - if (stream->dec_state < DEC_WININD) - { - stream->msg = "application header not available"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - - (*data) = stream->dec_appheader; - (*size) = stream->dec_appheadsz; - return 0; -} - -/********************************************************** - Decoder stuff - *************************************************/ - -#include "xdelta3-decode.h" - -/**************************************************************** - Encoder stuff - *****************************************************************/ - -#if XD3_ENCODER -void -xd3_set_appheader (xd3_stream *stream, - const uint8_t *data, - usize_t size) -{ - stream->enc_appheader = data; - stream->enc_appheadsz = size; -} - -#if XD3_DEBUG -static int -xd3_iopt_check (xd3_stream *stream) -{ - usize_t ul = xd3_rlist_length (& stream->iopt_used); - usize_t fl = xd3_rlist_length (& stream->iopt_free); - - return (ul + fl + (stream->iout ? 1 : 0)) == stream->iopt_size; -} -#endif - -static xd3_rinst* -xd3_iopt_free (xd3_stream *stream, xd3_rinst *i) -{ - xd3_rinst *n = xd3_rlist_remove (i); - xd3_rlist_push_back (& stream->iopt_free, i); - return n; -} - -static void -xd3_iopt_free_nonadd (xd3_stream *stream, xd3_rinst *i) -{ - if (i->type != XD3_ADD) - { - xd3_rlist_push_back (& stream->iopt_free, i); - } -} - -/* When an instruction is ready to flush from the iopt buffer, this - * function is called to produce an encoding. It writes the - * instruction plus size, address, and data to the various encoding - * sections. */ -static int -xd3_iopt_finish_encoding (xd3_stream *stream, xd3_rinst *inst) -{ - int ret; - - /* Check for input overflow. */ - XD3_ASSERT (inst->pos + inst->size <= stream->avail_in); - - switch (inst->type) - { - case XD3_CPY: - { - /* the address may have an offset if there is a source window. */ - usize_t addr; - xd3_source *src = stream->src; - - if (src != NULL) - { - /* If there is a source copy, the source must have its - * source window decided before we can encode. This can - * be bad -- we have to make this decision even if no - * source matches have been found. */ - if (stream->srcwin_decided == 0) - { - if ((ret = xd3_srcwin_setup (stream))) { return ret; } - } - else - { - stream->srcwin_decided_early = (!stream->src->eof_known || - (stream->srcwin_cksum_pos < - xd3_source_eof (stream->src))); - } - - /* xtra field indicates the copy is from the source */ - if (inst->xtra) - { - XD3_ASSERT (inst->addr >= src->srcbase); - XD3_ASSERT (inst->addr + inst->size <= - src->srcbase + src->srclen); - addr = (usize_t)(inst->addr - src->srcbase); - stream->n_scpy += 1; - stream->l_scpy += (xoff_t) inst->size; - } - else - { - /* with source window: target copy address is offset - * by taroff. */ - addr = stream->taroff + (usize_t) inst->addr; - stream->n_tcpy += 1; - stream->l_tcpy += (xoff_t) inst->size; - } - } - else - { - addr = (usize_t) inst->addr; - stream->n_tcpy += 1; - stream->l_tcpy += inst->size; - } - - /* Note: used to assert inst->size >= MIN_MATCH, but not true - * for merge operations & identical match heuristics. */ - /* the "here" position is always offset by taroff */ - if ((ret = xd3_encode_address (stream, addr, inst->pos + stream->taroff, - & inst->type))) - { - return ret; - } - - IF_DEBUG2 ({ - static int cnt; - DP(RINT "[iopt copy:%d] pos %"Q"u-%"Q"u addr %"Q"u-%"Q"u size %u\n", - cnt++, - stream->total_in + inst->pos, - stream->total_in + inst->pos + inst->size, - inst->addr, inst->addr + inst->size, inst->size); - }); - break; - } - case XD3_RUN: - { - XD3_ASSERT (inst->size >= MIN_MATCH); - - if ((ret = xd3_emit_byte (stream, & DATA_TAIL (stream), inst->xtra))) { return ret; } - - stream->n_run += 1; - stream->l_run += inst->size; - - IF_DEBUG2 ({ - static int cnt; - DP(RINT "[iopt run:%d] pos %"Q"u size %u\n", cnt++, stream->total_in + inst->pos, inst->size); - }); - break; - } - case XD3_ADD: - { - if ((ret = xd3_emit_bytes (stream, & DATA_TAIL (stream), - stream->next_in + inst->pos, inst->size))) { return ret; } - - stream->n_add += 1; - stream->l_add += inst->size; - - IF_DEBUG2 ({ - static int cnt; - DP(RINT "[iopt add:%d] pos %"Q"u size %u\n", cnt++, stream->total_in + inst->pos, inst->size); - }); - - break; - } - } - - /* This is the only place stream->unencoded_offset is incremented. */ - XD3_ASSERT (stream->unencoded_offset == inst->pos); - stream->unencoded_offset += inst->size; - - inst->code2 = 0; - - XD3_CHOOSE_INSTRUCTION (stream, stream->iout, inst); - - if (stream->iout != NULL) - { - if (stream->iout->code2 != 0) - { - if ((ret = xd3_emit_double (stream, stream->iout, inst, stream->iout->code2))) { return ret; } - - xd3_iopt_free_nonadd (stream, stream->iout); - xd3_iopt_free_nonadd (stream, inst); - stream->iout = NULL; - return 0; - } - else - { - if ((ret = xd3_emit_single (stream, stream->iout, stream->iout->code1))) { return ret; } - - xd3_iopt_free_nonadd (stream, stream->iout); - } - } - - stream->iout = inst; - - return 0; -} - -/* This possibly encodes an add instruction, iadd, which must remain - * on the stack until the following call to - * xd3_iopt_finish_encoding. */ -static int -xd3_iopt_add (xd3_stream *stream, usize_t pos, xd3_rinst *iadd) -{ - int ret; - usize_t off = stream->unencoded_offset; - - if (pos > off) - { - iadd->type = XD3_ADD; - iadd->pos = off; - iadd->size = pos - off; - - if ((ret = xd3_iopt_finish_encoding (stream, iadd))) { return ret; } - } - - return 0; -} - -/* This function calls xd3_iopt_finish_encoding to finish encoding an - * instruction, and it may also produce an add instruction for an - * unmatched region. */ -static int -xd3_iopt_add_encoding (xd3_stream *stream, xd3_rinst *inst) -{ - int ret; - xd3_rinst iadd; - - if ((ret = xd3_iopt_add (stream, inst->pos, & iadd))) { return ret; } - - if ((ret = xd3_iopt_finish_encoding (stream, inst))) { return ret; } - - return 0; -} - -/* Generates a final add instruction to encode the remaining input. */ -static int -xd3_iopt_add_finalize (xd3_stream *stream) -{ - int ret; - xd3_rinst iadd; - - if ((ret = xd3_iopt_add (stream, stream->avail_in, & iadd))) { return ret; } - - if (stream->iout) - { - if ((ret = xd3_emit_single (stream, stream->iout, stream->iout->code1))) { return ret; } - - xd3_iopt_free_nonadd (stream, stream->iout); - stream->iout = NULL; - } - - return 0; -} - -/* Compact the instruction buffer by choosing the best non-overlapping - * instructions when lazy string-matching. There are no ADDs in the - * iopt buffer because those are synthesized in xd3_iopt_add_encoding - * and during xd3_iopt_add_finalize. */ -static int -xd3_iopt_flush_instructions (xd3_stream *stream, int force) -{ - xd3_rinst *r1 = xd3_rlist_front (& stream->iopt_used); - xd3_rinst *r2; - xd3_rinst *r3; - usize_t r1end; - usize_t r2end; - usize_t r2off; - usize_t r2moff; - usize_t gap; - usize_t flushed; - int ret; - - XD3_ASSERT (xd3_iopt_check (stream)); - - /* Note: once tried to skip this step if it's possible to assert - * there are no overlapping instructions. Doesn't work because - * xd3_opt_erase leaves overlapping instructions. */ - while (! xd3_rlist_end (& stream->iopt_used, r1) && - ! xd3_rlist_end (& stream->iopt_used, r2 = xd3_rlist_next (r1))) - { - r1end = r1->pos + r1->size; - - /* If the instructions do not overlap, continue. */ - if (r1end <= r2->pos) - { - r1 = r2; - continue; - } - - r2end = r2->pos + r2->size; - - /* The min_match adjustments prevent this. */ - XD3_ASSERT (r2end > (r1end + LEAST_MATCH_INCR)); - - /* If r3 is available... */ - if (! xd3_rlist_end (& stream->iopt_used, r3 = xd3_rlist_next (r2))) - { - /* If r3 starts before r1 finishes or just about, r2 is irrelevant */ - if (r3->pos <= r1end + 1) - { - xd3_iopt_free (stream, r2); - continue; - } - } - else if (! force) - { - /* Unless force, end the loop when r3 is not available. */ - break; - } - - r2off = r2->pos - r1->pos; - r2moff = r2end - r1end; - gap = r2end - r1->pos; - - /* If the two matches overlap almost entirely, choose the better match - * and discard the other. The else branch can still create inefficient - * copies, e.g., a 4-byte copy that takes 4 bytes to encode, which - * xd3_smatch() wouldn't allow by its crude efficiency check. However, - * in this case there are adjacent copies which mean the add would cost - * one extra byte. Allow the inefficiency here. */ - if (gap < 2*MIN_MATCH || r2moff <= 2 || r2off <= 2) - { - /* Only one match should be used, choose the longer one. */ - if (r1->size < r2->size) - { - xd3_iopt_free (stream, r1); - r1 = r2; - } - else - { - /* We are guaranteed that r1 does not overlap now, so advance past r2 */ - r1 = xd3_iopt_free (stream, r2); - } - continue; - } - else - { - /* Shorten one of the instructions -- could be optimized - * based on the address cache. */ - usize_t average; - usize_t newsize; - usize_t adjust1; - - XD3_ASSERT (r1end > r2->pos && r2end > r1->pos); - - /* Try to balance the length of both instructions, but avoid - * making both longer than MAX_MATCH_SPLIT . */ - average = gap / 2; - newsize = min (MAX_MATCH_SPLIT, gap - average); - - /* Should be possible to simplify this code. */ - if (newsize > r1->size) - { - /* shorten r2 */ - adjust1 = r1end - r2->pos; - } - else if (newsize > r2->size) - { - /* shorten r1 */ - adjust1 = r1end - r2->pos; - - XD3_ASSERT (r1->size > adjust1); - - r1->size -= adjust1; - - /* don't shorten r2 */ - adjust1 = 0; - } - else - { - /* shorten r1 */ - adjust1 = r1->size - newsize; - - if (r2->pos > r1end - adjust1) - { - adjust1 -= r2->pos - (r1end - adjust1); - } - - XD3_ASSERT (r1->size > adjust1); - - r1->size -= adjust1; - - /* shorten r2 */ - XD3_ASSERT (r1->pos + r1->size >= r2->pos); - - adjust1 = r1->pos + r1->size - r2->pos; - } - - /* Fallthrough above if-else, shorten r2 */ - XD3_ASSERT (r2->size > adjust1); - - r2->size -= adjust1; - r2->pos += adjust1; - r2->addr += adjust1; - - XD3_ASSERT (r1->size >= MIN_MATCH); - XD3_ASSERT (r2->size >= MIN_MATCH); - - r1 = r2; - } - } - - XD3_ASSERT (xd3_iopt_check (stream)); - - /* If forcing, pick instructions until the list is empty, otherwise - * this empties 50% of the queue. */ - for (flushed = 0; ! xd3_rlist_empty (& stream->iopt_used); ) - { - xd3_rinst *renc = xd3_rlist_pop_front (& stream->iopt_used); - if ((ret = xd3_iopt_add_encoding (stream, renc))) - { - return ret; - } - - if (! force) - { - if (++flushed > stream->iopt_size / 2) - { - break; - } - - /* If there are only two instructions remaining, break, - * because they were not optimized. This means there were - * more than 50% eliminated by the loop above. */ - r1 = xd3_rlist_front (& stream->iopt_used); - if (xd3_rlist_end(& stream->iopt_used, r1) || - xd3_rlist_end(& stream->iopt_used, r2 = xd3_rlist_next (r1)) || - xd3_rlist_end(& stream->iopt_used, r3 = xd3_rlist_next (r2))) - { - break; - } - } - } - - XD3_ASSERT (xd3_iopt_check (stream)); - - XD3_ASSERT (!force || xd3_rlist_length (& stream->iopt_used) == 0); - - return 0; -} - -static int -xd3_iopt_get_slot (xd3_stream *stream, xd3_rinst** iptr) -{ - xd3_rinst *i; - int ret; - - if (xd3_rlist_empty (& stream->iopt_free)) - { - if (stream->iopt_unlimited) - { - usize_t elts = XD3_ALLOCSIZE / sizeof(xd3_rinst); - - if ((ret = xd3_alloc_iopt (stream, elts))) - { - return ret; - } - - stream->iopt_size += elts; - } - else - { - if ((ret = xd3_iopt_flush_instructions (stream, 0))) { return ret; } - - XD3_ASSERT (! xd3_rlist_empty (& stream->iopt_free)); - } - } - - i = xd3_rlist_pop_back (& stream->iopt_free); - - xd3_rlist_push_back (& stream->iopt_used, i); - - (*iptr) = i; - - ++stream->i_slots_used; - - return 0; -} - -/* A copy is about to be emitted that extends backwards to POS, - * therefore it may completely cover some existing instructions in the - * buffer. If an instruction is completely covered by this new match, - * erase it. If the new instruction is covered by the previous one, - * return 1 to skip it. */ -static void -xd3_iopt_erase (xd3_stream *stream, usize_t pos, usize_t size ATTRIBUTE((unused))) -{ - while (! xd3_rlist_empty (& stream->iopt_used)) - { - xd3_rinst *r = xd3_rlist_back (& stream->iopt_used); - - /* Verify that greedy is working. The previous instruction - * should end before the new one begins. */ - XD3_ASSERT ((stream->flags & XD3_BEGREEDY) == 0 || (r->pos + r->size <= pos)); - /* Verify that min_match is working. The previous instruction - * should end before the new one ends. */ - XD3_ASSERT ((stream->flags & XD3_BEGREEDY) != 0 || (r->pos + r->size < pos + size)); - - /* See if the last instruction starts before the new - * instruction. If so, there is nothing to erase. */ - if (r->pos < pos) - { - return; - } - - /* Otherwise, the new instruction covers the old one, delete it - and repeat. */ - xd3_rlist_remove (r); - xd3_rlist_push_back (& stream->iopt_free, r); - --stream->i_slots_used; - } -} - -/* This function tells the last matched input position. */ -static usize_t -xd3_iopt_last_matched (xd3_stream *stream) -{ - xd3_rinst *r; - - if (xd3_rlist_empty (& stream->iopt_used)) - { - return 0; - } - - r = xd3_rlist_back (& stream->iopt_used); - - return r->pos + r->size; -} - -/********************************************************* - Emit routines - ***********************************************************/ - -static int -xd3_emit_single (xd3_stream *stream, xd3_rinst *single, usize_t code) -{ - int has_size = stream->code_table[code].size1 == 0; - int ret; - - IF_DEBUG2 (DP(RINT "[emit1] %u %s (%u) code %u\n", - single->pos, - xd3_rtype_to_string ((xd3_rtype) single->type, 0), - single->size, - code)); - - if ((ret = xd3_emit_byte (stream, & INST_TAIL (stream), code))) - { - return ret; - } - - if (has_size) - { - if ((ret = xd3_emit_size (stream, & INST_TAIL (stream), single->size))) - { - return ret; - } - } - - return 0; -} - -static int -xd3_emit_double (xd3_stream *stream, xd3_rinst *first IFN_DEBUG2(ATTRIBUTE((unused))), - xd3_rinst *second IFN_DEBUG2(ATTRIBUTE((unused))), usize_t code) -{ - int ret; - - /* All double instructions use fixed sizes, so all we need to do is - * output the instruction code, no sizes. */ - XD3_ASSERT (stream->code_table[code].size1 != 0 && - stream->code_table[code].size2 != 0); - - if ((ret = xd3_emit_byte (stream, & INST_TAIL (stream), code))) - { - return ret; - } - - IF_DEBUG2 (DP(RINT "[emit2]: %u %s (%u) %s (%u) code %u\n", - first->pos, - xd3_rtype_to_string ((xd3_rtype) first->type, 0), - first->size, - xd3_rtype_to_string ((xd3_rtype) second->type, 0), - second->size, - code)); - - return 0; -} - -/* This enters a potential run instruction into the iopt buffer. The - * position argument is relative to the target window. */ -static int -xd3_emit_run (xd3_stream *stream, usize_t pos, usize_t size, uint8_t *run_c) -{ - xd3_rinst* ri; - int ret; - - if ((ret = xd3_iopt_get_slot (stream, & ri))) { return ret; } - - ri->type = XD3_RUN; - ri->xtra = *run_c; - ri->pos = pos; - ri->size = size; - - return 0; -} - -/* This enters a potential copy instruction into the iopt buffer. The - * position argument is relative to the target window.. */ -int -xd3_found_match (xd3_stream *stream, usize_t pos, - usize_t size, xoff_t addr, int is_source) -{ - xd3_rinst* ri; - int ret; - - if ((ret = xd3_iopt_get_slot (stream, & ri))) { return ret; } - - ri->type = XD3_CPY; - ri->xtra = is_source; - ri->pos = pos; - ri->size = size; - ri->addr = addr; - - return 0; -} - -static int -xd3_emit_hdr (xd3_stream *stream) -{ - int ret; - int use_secondary = stream->sec_type != NULL; - int use_adler32 = stream->flags & (XD3_ADLER32 | XD3_ADLER32_RECODE); - int vcd_source = xd3_encoder_used_source (stream); - usize_t win_ind = 0; - usize_t del_ind = 0; - usize_t enc_len; - usize_t tgt_len; - usize_t data_len; - usize_t inst_len; - usize_t addr_len; - - if (stream->current_window == 0) - { - usize_t hdr_ind = 0; - int use_appheader = stream->enc_appheader != NULL; - int use_gencodetbl = GENERIC_ENCODE_TABLES && - (stream->code_table_desc != & __rfc3284_code_table_desc); - - if (use_secondary) { hdr_ind |= VCD_SECONDARY; } - if (use_gencodetbl) { hdr_ind |= VCD_CODETABLE; } - if (use_appheader) { hdr_ind |= VCD_APPHEADER; } - - if ((ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - VCDIFF_MAGIC1)) != 0 || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - VCDIFF_MAGIC2)) != 0 || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - VCDIFF_MAGIC3)) != 0 || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - VCDIFF_VERSION)) != 0 || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), hdr_ind)) != 0) - { - return ret; - } - - /* Secondary compressor ID */ -#if SECONDARY_ANY - if (use_secondary && - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - stream->sec_type->id))) - { - return ret; - } -#endif - - /* Compressed code table */ - if (use_gencodetbl) - { - usize_t code_table_size; - const uint8_t *code_table_data; - - if ((ret = stream->comp_table_func (stream, & code_table_data, - & code_table_size))) - { - return ret; - } - - if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), - code_table_size + 2)) || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - stream->code_table_desc->near_modes)) || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), - stream->code_table_desc->same_modes)) || - (ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), - code_table_data, code_table_size))) - { - return ret; - } - } - - /* Application header */ - if (use_appheader) - { - if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), - stream->enc_appheadsz)) || - (ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), - stream->enc_appheader, - stream->enc_appheadsz))) - { - return ret; - } - } - } - - /* try to compress this window */ -#if SECONDARY_ANY - if (use_secondary) - { - int data_sec = 0; - int inst_sec = 0; - int addr_sec = 0; - -# define ENCODE_SECONDARY_SECTION(UPPER,LOWER) \ - ((stream->flags & XD3_SEC_NO ## UPPER) == 0 && \ - (ret = xd3_encode_secondary (stream, \ - & UPPER ## _HEAD (stream), \ - & UPPER ## _TAIL (stream), \ - & xd3_sec_ ## LOWER (stream), \ - & stream->sec_ ## LOWER, \ - & LOWER ## _sec))) - - if (ENCODE_SECONDARY_SECTION (DATA, data) || - ENCODE_SECONDARY_SECTION (INST, inst) || - ENCODE_SECONDARY_SECTION (ADDR, addr)) - { - return ret; - } - - del_ind |= (data_sec ? VCD_DATACOMP : 0); - del_ind |= (inst_sec ? VCD_INSTCOMP : 0); - del_ind |= (addr_sec ? VCD_ADDRCOMP : 0); - } -#endif - - /* if (vcd_target) { win_ind |= VCD_TARGET; } */ - if (vcd_source) { win_ind |= VCD_SOURCE; } - if (use_adler32) { win_ind |= VCD_ADLER32; } - - /* window indicator */ - if ((ret = xd3_emit_byte (stream, & HDR_TAIL (stream), win_ind))) - { - return ret; - } - - /* source window */ - if (vcd_source) - { - /* or (vcd_target) { ... } */ - if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), - stream->src->srclen)) || - (ret = xd3_emit_offset (stream, & HDR_TAIL (stream), - stream->src->srcbase))) { return ret; } - } - - tgt_len = stream->avail_in; - data_len = xd3_sizeof_output (DATA_HEAD (stream)); - inst_len = xd3_sizeof_output (INST_HEAD (stream)); - addr_len = xd3_sizeof_output (ADDR_HEAD (stream)); - - /* The enc_len field is a redundency for future extensions.*/ - enc_len = (1 + (xd3_sizeof_size (tgt_len) + - xd3_sizeof_size (data_len) + - xd3_sizeof_size (inst_len) + - xd3_sizeof_size (addr_len)) + - data_len + - inst_len + - addr_len + - (use_adler32 ? 4 : 0)); - - if ((ret = xd3_emit_size (stream, & HDR_TAIL (stream), enc_len)) || - (ret = xd3_emit_size (stream, & HDR_TAIL (stream), tgt_len)) || - (ret = xd3_emit_byte (stream, & HDR_TAIL (stream), del_ind)) || - (ret = xd3_emit_size (stream, & HDR_TAIL (stream), data_len)) || - (ret = xd3_emit_size (stream, & HDR_TAIL (stream), inst_len)) || - (ret = xd3_emit_size (stream, & HDR_TAIL (stream), addr_len))) - { - return ret; - } - - if (use_adler32) - { - uint8_t send[4]; - uint32_t a32; - - if (stream->flags & XD3_ADLER32) - { - a32 = adler32 (1L, stream->next_in, stream->avail_in); - } - else - { - a32 = stream->recode_adler32; - } - - /* Four bytes. */ - send[0] = (uint8_t) (a32 >> 24); - send[1] = (uint8_t) (a32 >> 16); - send[2] = (uint8_t) (a32 >> 8); - send[3] = (uint8_t) (a32 & 0x000000FFU); - - if ((ret = xd3_emit_bytes (stream, & HDR_TAIL (stream), send, 4))) - { - return ret; - } - } - - return 0; -} - -/**************************************************************** - Encode routines - ****************************************************************/ - -static int -xd3_encode_buffer_leftover (xd3_stream *stream) -{ - usize_t take; - usize_t room; - - /* Allocate the buffer. */ - if (stream->buf_in == NULL && - (stream->buf_in = (uint8_t*) xd3_alloc (stream, stream->winsize, 1)) == NULL) - { - return ENOMEM; - } - - IF_DEBUG2 (DP(RINT "[leftover] flush?=%s\n", (stream->flags & XD3_FLUSH) ? "yes" : "no")); - - /* Take leftover input first. */ - if (stream->buf_leftover != NULL) - { - XD3_ASSERT (stream->buf_avail == 0); - XD3_ASSERT (stream->buf_leftavail < stream->winsize); - - IF_DEBUG2 (DP(RINT "[leftover] previous %u avail %u\n", stream->buf_leftavail, stream->avail_in)); - - memcpy (stream->buf_in, stream->buf_leftover, stream->buf_leftavail); - - stream->buf_leftover = NULL; - stream->buf_avail = stream->buf_leftavail; - } - - /* Copy into the buffer. */ - room = stream->winsize - stream->buf_avail; - take = min (room, stream->avail_in); - - memcpy (stream->buf_in + stream->buf_avail, stream->next_in, take); - - stream->buf_avail += take; - - if (take < stream->avail_in) - { - /* Buffer is full */ - stream->buf_leftover = stream->next_in + take; - stream->buf_leftavail = stream->avail_in - take; - } - else if ((stream->buf_avail < stream->winsize) && !(stream->flags & XD3_FLUSH)) - { - /* Buffer has space */ - IF_DEBUG2 (DP(RINT "[leftover] emptied %u\n", take)); - return XD3_INPUT; - } - - /* Use the buffer: */ - IF_DEBUG2 (DP(RINT "[leftover] take %u remaining %u\n", take, stream->buf_leftavail)); - stream->next_in = stream->buf_in; - stream->avail_in = stream->buf_avail; - stream->buf_avail = 0; - - return 0; -} - -/* Allocates one block of xd3_rlist elements */ -static int -xd3_alloc_iopt (xd3_stream *stream, usize_t elts) -{ - usize_t i; - xd3_iopt_buflist* last = - (xd3_iopt_buflist*) xd3_alloc (stream, sizeof (xd3_iopt_buflist), 1); - - if (last == NULL || - (last->buffer = (xd3_rinst*) xd3_alloc (stream, sizeof (xd3_rinst), elts)) == NULL) - { - return ENOMEM; - } - - last->next = stream->iopt_alloc; - stream->iopt_alloc = last; - - for (i = 0; i < elts; i += 1) - { - xd3_rlist_push_back (& stream->iopt_free, & last->buffer[i]); - } - - return 0; -} - -/* This function allocates all memory initially used by the encoder. */ -static int -xd3_encode_init (xd3_stream *stream, int full_init) -{ - int i; - - if (full_init) - { - int large_comp = (stream->src != NULL); - int small_comp = ! (stream->flags & XD3_NOCOMPRESS); - - /* Memory allocations for checksum tables are delayed until - * xd3_string_match_init in the first call to string_match--that way - * identical or short inputs require no table allocation. */ - if (large_comp) - { - usize_t hash_values = (stream->srcwin_maxsz / - stream->smatcher.large_step); - - xd3_size_hashtable (stream, - hash_values, - & stream->large_hash); - } - - if (small_comp) - { - /* TODO: This is under devel: used to have min(sprevsz) here, which sort - * of makes sense, but observed fast performance w/ larger tables, which - * also sort of makes sense. @@@ */ - usize_t hash_values = stream->winsize; - - xd3_size_hashtable (stream, - hash_values, - & stream->small_hash); - } - } - - /* data buffers */ - for (i = 0; i < ENC_SECTS; i += 1) - { - if ((stream->enc_heads[i] = - stream->enc_tails[i] = - xd3_alloc_output (stream, NULL)) == NULL) - { - return ENOMEM; - } - } - - /* iopt buffer */ - xd3_rlist_init (& stream->iopt_used); - xd3_rlist_init (& stream->iopt_free); - - if (xd3_alloc_iopt (stream, stream->iopt_size) != 0) { goto fail; } - - XD3_ASSERT (xd3_rlist_length (& stream->iopt_free) == stream->iopt_size); - XD3_ASSERT (xd3_rlist_length (& stream->iopt_used) == 0); - - /* address cache, code table */ - stream->acache.s_near = stream->code_table_desc->near_modes; - stream->acache.s_same = stream->code_table_desc->same_modes; - stream->code_table = stream->code_table_func (); - - return xd3_alloc_cache (stream); - - fail: - - return ENOMEM; -} - -int -xd3_encode_init_full (xd3_stream *stream) -{ - return xd3_encode_init (stream, 1); -} - -int -xd3_encode_init_partial (xd3_stream *stream) -{ - return xd3_encode_init (stream, 0); -} - -/* Called after the ENC_POSTOUT state, this puts the output buffers - * back into separate lists and re-initializes some variables. (The - * output lists were spliced together during the ENC_FLUSH state.) */ -static void -xd3_encode_reset (xd3_stream *stream) -{ - int i; - xd3_output *olist; - - stream->avail_in = 0; - stream->small_reset = 1; - stream->i_slots_used = 0; - - if (stream->src != NULL) - { - stream->src->srcbase = 0; - stream->src->srclen = 0; - stream->srcwin_decided = 0; - stream->srcwin_decided_early = 0; - stream->match_minaddr = 0; - stream->match_maxaddr = 0; - stream->taroff = 0; - } - - /* Reset output chains. */ - olist = stream->enc_heads[0]; - - for (i = 0; i < ENC_SECTS; i += 1) - { - XD3_ASSERT (olist != NULL); - - stream->enc_heads[i] = olist; - stream->enc_tails[i] = olist; - olist = olist->next_page; - - stream->enc_heads[i]->next = 0; - stream->enc_heads[i]->next_page = NULL; - - stream->enc_tails[i]->next_page = NULL; - stream->enc_tails[i] = stream->enc_heads[i]; - } - - xd3_freelist_output (stream, olist); -} - -/* The main encoding routine. */ -int -xd3_encode_input (xd3_stream *stream) -{ - int ret, i; - - if (stream->dec_state != 0) - { - stream->msg = "encoder/decoder transition"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - - switch (stream->enc_state) - { - case ENC_INIT: - /* Only reached on first time through: memory setup. */ - if ((ret = xd3_encode_init_full (stream))) { return ret; } - - stream->enc_state = ENC_INPUT; - - case ENC_INPUT: - - /* If there is no input yet, just return. This checks for - * next_in == NULL, not avail_in == 0 since zero bytes is a - * valid input. There is an assertion in xd3_avail_input() that - * next_in != NULL for this reason. By returning right away we - * avoid creating an input buffer before the caller has supplied - * its first data. It is possible for xd3_avail_input to be - * called both before and after the first call to - * xd3_encode_input(). */ - if (stream->next_in == NULL) - { - return XD3_INPUT; - } - - enc_flush: - /* See if we should buffer the input: either if there is already - * a leftover buffer, or if the input is short of winsize - * without flush. The label at this point is reached by a goto - * below, when there is leftover input after postout. */ - if ((stream->buf_leftover != NULL) || - (stream->buf_avail != 0) || - (stream->avail_in < stream->winsize && ! (stream->flags & XD3_FLUSH))) - { - if ((ret = xd3_encode_buffer_leftover (stream))) { return ret; } - } - - /* Initalize the address cache before each window. */ - xd3_init_cache (& stream->acache); - - stream->input_position = 0; - stream->min_match = MIN_MATCH; - stream->unencoded_offset = 0; - - stream->enc_state = ENC_SEARCH; - - IF_DEBUG2 (DP(RINT "[WINSTART:%"Q"u] input bytes %u offset %"Q"u\n", - stream->current_window, stream->avail_in, - stream->total_in)); - return XD3_WINSTART; - - case ENC_SEARCH: - IF_DEBUG2 (DP(RINT "[SEARCH] match_state %d avail_in %u %s\n", - stream->match_state, stream->avail_in, - stream->src ? "source" : "no source")); - - /* Reentrant matching. */ - if (stream->src != NULL) - { - switch (stream->match_state) - { - case MATCH_TARGET: - /* Try matching forward at the start of the target. - * This is entered the first time through, to check for - * a perfect match, and whenever there is a source match - * that extends to the end of the previous window. The - * match_srcpos field is initially zero and later set - * during xd3_source_extend_match. */ - - if (stream->avail_in > 0) - { - /* This call can't fail because the source window is - * unrestricted. */ - ret = xd3_source_match_setup (stream, stream->match_srcpos); - XD3_ASSERT (ret == 0); - stream->match_state = MATCH_FORWARD; - } - else - { - stream->match_state = MATCH_SEARCHING; - stream->match_fwd = 0; - } - XD3_ASSERT (stream->match_fwd == 0); - - case MATCH_FORWARD: - case MATCH_BACKWARD: - if (stream->avail_in != 0) - { - if ((ret = xd3_source_extend_match (stream)) != 0) - { - return ret; - } - - /* The search has to make forward progress here - * or else it can get stuck in a match-backward - * (getsrcblk) then match-forward (getsrcblk), - * find insufficient match length, then repeat - * exactly the same search. - */ - stream->input_position += stream->match_fwd; - } - - case MATCH_SEARCHING: - /* Continue string matching. (It's possible that the - * initial match continued through the entire input, in - * which case we're still in MATCH_FORWARD and should - * remain so for the next input window.) */ - break; - } - } - - /* String matching... */ - if (stream->avail_in != 0 && - (ret = stream->smatcher.string_match (stream))) - { - return ret; - } - - stream->enc_state = ENC_INSTR; - - case ENC_INSTR: - /* Note: Jump here to encode VCDIFF deltas w/o using this - * string-matching code. Merging code code enters here. */ - - /* Flush the instrution buffer, then possibly add one more - * instruction, then emit the header. */ - if ((ret = xd3_iopt_flush_instructions (stream, 1)) || - (ret = xd3_iopt_add_finalize (stream))) - { - return ret; - } - - stream->enc_state = ENC_FLUSH; - - case ENC_FLUSH: - /* Note: main_recode_func() bypasses string-matching by setting - * ENC_FLUSH. */ - if ((ret = xd3_emit_hdr (stream))) - { - return ret; - } - - /* Begin output. */ - stream->enc_current = HDR_HEAD (stream); - - /* Chain all the outputs together. After doing this, it looks - * as if there is only one section. The other enc_heads are set - * to NULL to avoid freeing them more than once. */ - for (i = 1; i < ENC_SECTS; i += 1) - { - stream->enc_tails[i-1]->next_page = stream->enc_heads[i]; - stream->enc_heads[i] = NULL; - } - - enc_output: - - stream->enc_state = ENC_POSTOUT; - stream->next_out = stream->enc_current->base; - stream->avail_out = stream->enc_current->next; - stream->total_out += (xoff_t) stream->avail_out; - - /* If there is any output in this buffer, return it, otherwise - * fall through to handle the next buffer or finish the window - * after all buffers have been output. */ - if (stream->avail_out > 0) - { - /* This is the only place xd3_encode returns XD3_OUTPUT */ - return XD3_OUTPUT; - } - - case ENC_POSTOUT: - - if (stream->avail_out != 0) - { - stream->msg = "missed call to consume output"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - - /* Continue outputting one buffer at a time, until the next is NULL. */ - if ((stream->enc_current = stream->enc_current->next_page) != NULL) - { - goto enc_output; - } - - stream->total_in += (xoff_t) stream->avail_in; - stream->enc_state = ENC_POSTWIN; - - IF_DEBUG2 (DP(RINT "[WINFINISH:%"Q"u] in=%"Q"u\n", - stream->current_window, - stream->total_in)); - return XD3_WINFINISH; - - case ENC_POSTWIN: - - xd3_encode_reset (stream); - - stream->current_window += 1; - stream->enc_state = ENC_INPUT; - - /* If there is leftover input to flush, repeat. */ - if (stream->buf_leftover != NULL) - { - goto enc_flush; - } - - /* Ready for more input. */ - return XD3_INPUT; - - default: - stream->msg = "invalid state"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } -} -#endif /* XD3_ENCODER */ - -/***************************************************************** - Client convenience functions - ******************************************************************/ - -static int -xd3_process_stream (int is_encode ATTRIBUTE((unused)), - xd3_stream *stream, - int (*func) (xd3_stream *), - int close_stream, - const uint8_t *input, - usize_t input_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max) -{ - usize_t ipos = 0; - usize_t n = min(stream->winsize, input_size); - - (*output_size) = 0; - - stream->flags |= XD3_FLUSH; - - xd3_avail_input (stream, input + ipos, n); - ipos += n; - - for (;;) - { - int ret; - switch((ret = func (stream))) - { - case XD3_OUTPUT: { /* memcpy below */ break; } - case XD3_INPUT: { - n = min(stream->winsize, input_size - ipos); - if (n == 0) { - goto done; - } - xd3_avail_input (stream, input + ipos, n); - ipos += n; - continue; - } - case XD3_GOTHEADER: { /* ignore */ continue; } - case XD3_WINSTART: { /* ignore */ continue; } - case XD3_WINFINISH: { /* ignore */ continue; } - case XD3_GETSRCBLK: - { - stream->msg = "stream requires source input"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - case 0: - { - /* xd3_encode_input/xd3_decode_input never return 0 */ - stream->msg = "invalid return: 0"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - default: - return ret; - } - - if (*output_size + stream->avail_out > output_size_max) - { - stream->msg = "insufficient output space"; - return ENOSPC; - } - - memcpy (output + *output_size, stream->next_out, stream->avail_out); - - *output_size += stream->avail_out; - - xd3_consume_output (stream); - } - done: - return (close_stream == 0) ? 0 : xd3_close_stream (stream); -} - -static int -xd3_process_memory (int is_encode, - int (*func) (xd3_stream *), - int close_stream ATTRIBUTE((unused)), - const uint8_t *input, - usize_t input_size, - const uint8_t *source, - usize_t source_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max, - int flags) { - xd3_stream stream; - xd3_config config; - xd3_source src; - int ret; - - memset (& stream, 0, sizeof (stream)); - memset (& config, 0, sizeof (config)); - - if (input == NULL || output == NULL) { - stream.msg = "invalid input/output buffer"; - initprintf("xdelta3: %s\n",stream.msg); - ret = XD3_INTERNAL; - goto exit; - } - - config.flags = flags; - - if (is_encode) - { - config.srcwin_maxsz = source_size; - config.winsize = min(input_size, (usize_t) XD3_DEFAULT_WINSIZE); - config.iopt_size = min(input_size / 32, XD3_DEFAULT_IOPT_SIZE); - config.iopt_size = max(config.iopt_size, 128U); - config.sprevsz = xd3_pow2_roundup (config.winsize); - } - - if ((ret = xd3_config_stream (&stream, &config)) != 0) - { - goto exit; - } - - if (source != NULL) - { - memset (& src, 0, sizeof (src)); - - src.blksize = source_size; - src.onblk = source_size; - src.curblk = source; - src.curblkno = 0; - - if ((ret = xd3_set_source_and_size (&stream, &src, source_size)) != 0) - { - goto exit; - } - } - - if ((ret = xd3_process_stream (is_encode, - & stream, - func, 1, - input, input_size, - output, - output_size, - output_size_max)) != 0) - { - goto exit; - } - - exit: - if (ret != 0) - { - IF_DEBUG2 (DP(RINT "process_memory: %d: %s\n", ret, stream.msg)); - } - xd3_free_stream(&stream); - return ret; -} - -int -xd3_decode_stream (xd3_stream *stream, - const uint8_t *input, - usize_t input_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max) -{ - return xd3_process_stream (0, stream, & xd3_decode_input, 1, - input, input_size, - output, output_size, output_size_max); -} - -int -xd3_decode_memory (const uint8_t *input, - usize_t input_size, - const uint8_t *source, - usize_t source_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max, - int flags) { - return xd3_process_memory (0, & xd3_decode_input, 1, - input, input_size, - source, source_size, - output, output_size, output_size_max, - flags); -} - - -#if XD3_ENCODER -int -xd3_encode_stream (xd3_stream *stream, - const uint8_t *input, - usize_t input_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max) -{ - return xd3_process_stream (1, stream, & xd3_encode_input, 1, - input, input_size, - output, output_size, output_size_max); -} - -int -xd3_encode_memory (const uint8_t *input, - usize_t input_size, - const uint8_t *source, - usize_t source_size, - uint8_t *output, - usize_t *output_size, - usize_t output_size_max, - int flags) { - return xd3_process_memory (1, & xd3_encode_input, 1, - input, input_size, - source, source_size, - output, output_size, output_size_max, - flags); -} -#endif - - -/************************************************************* - String matching helpers - *************************************************************/ - -#if XD3_ENCODER -/* Do the initial xd3_string_match() checksum table setup. - * Allocations are delayed until first use to avoid allocation - * sometimes (e.g., perfect matches, zero-length inputs). */ -static int -xd3_string_match_init (xd3_stream *stream) -{ - const int DO_SMALL = ! (stream->flags & XD3_NOCOMPRESS); - const int DO_LARGE = (stream->src != NULL); - - if (DO_LARGE && stream->large_table == NULL) - { - if ((stream->large_table = - (usize_t*) xd3_alloc0 (stream, stream->large_hash.size, sizeof (usize_t))) == NULL) - { - return ENOMEM; - } - } - - if (DO_SMALL) - { - /* Subsequent calls can return immediately after checking reset. */ - if (stream->small_table != NULL) - { - /* The target hash table is reinitialized once per window. */ - /* TODO: This would not have to be reinitialized if absolute - * offsets were being stored. */ - if (stream->small_reset) - { - stream->small_reset = 0; - memset (stream->small_table, 0, - sizeof (usize_t) * stream->small_hash.size); - } - - return 0; - } - - if ((stream->small_table = - (usize_t*) xd3_alloc0 (stream, - stream->small_hash.size, - sizeof (usize_t))) == NULL) - { - return ENOMEM; - } - - /* If there is a previous table needed. */ - if (stream->smatcher.small_lchain > 1 || - stream->smatcher.small_chain > 1) - { - if ((stream->small_prev = - (xd3_slist*) xd3_alloc (stream, - stream->sprevsz, - sizeof (xd3_slist))) == NULL) - { - return ENOMEM; - } - } - } - - return 0; -} - -#if XD3_USE_LARGEFILE64 -/* This function handles the 32/64bit ambiguity -- file positions are 64bit - * but the hash table for source-offsets is 32bit. */ -static xoff_t -xd3_source_cksum_offset(xd3_stream *stream, usize_t low) -{ - xoff_t scp = stream->srcwin_cksum_pos; - xoff_t s0 = scp >> 32; - - usize_t sr = (usize_t) scp; - - if (s0 == 0) { - return low; - } - - /* This should not be >= because srcwin_cksum_pos is the next - * position to index. */ - if (low > sr) { - return (--s0 << 32) | low; - } - - return (s0 << 32) | low; -} -#else -static xoff_t -xd3_source_cksum_offset(xd3_stream *stream, usize_t low) -{ - return (xoff_t) low; -} -#endif - -/* This function sets up the stream->src fields srcbase, srclen. The - * call is delayed until these values are needed to encode a copy - * address. At this point the decision has to be made. */ -static int -xd3_srcwin_setup (xd3_stream *stream) -{ - xd3_source *src = stream->src; - xoff_t length, x; - - /* Check the undecided state. */ - XD3_ASSERT (src->srclen == 0 && src->srcbase == 0); - - /* Avoid repeating this call. */ - stream->srcwin_decided = 1; - - /* If the stream is flushing, then the iopt buffer was able to - * contain the complete encoding. If no copies were issued no - * source window is actually needed. This prevents the VCDIFF - * header from including source base/len. xd3_emit_hdr checks for - * srclen == 0. */ - if (stream->enc_state == ENC_INSTR && stream->match_maxaddr == 0) - { - goto done; - } - - /* Check for overflow, srclen is usize_t - this can't happen unless - * XD3_DEFAULT_SRCBACK and related parameters are extreme - should - * use smaller windows. */ - length = stream->match_maxaddr - stream->match_minaddr; - - x = (xoff_t) USIZE_T_MAX; - if (length > x) - { - stream->msg = "source window length overflow (not 64bit)"; - initprintf("xdelta3: %s\n",stream->msg); - return XD3_INTERNAL; - } - - /* If ENC_INSTR, then we know the exact source window to use because - * no more copies can be issued. */ - if (stream->enc_state == ENC_INSTR) - { - src->srcbase = stream->match_minaddr; - src->srclen = (usize_t) length; - XD3_ASSERT (src->srclen); - goto done; - } - - /* Otherwise, we have to make a guess. More copies may still be - * issued, but we have to decide the source window base and length - * now. */ - src->srcbase = stream->match_minaddr; - src->srclen = max ((usize_t) length, - stream->avail_in + (stream->avail_in >> 2)); - - /* OPT: If we know the source size, it might be possible to reduce - * srclen. */ - XD3_ASSERT (src->srclen); - done: - /* Set the taroff. This convenience variable is used even when - stream->src == NULL. */ - stream->taroff = src->srclen; - return 0; -} - -/* Sets the bounding region for a newly discovered source match, prior - * to calling xd3_source_extend_match(). This sets the match_maxfwd, - * match_maxback variables. Note: srcpos is an absolute position - * (xoff_t) but the match_maxfwd, match_maxback variables are usize_t. - * Returns 0 if the setup succeeds, or 1 if the source position lies - * outside an already-decided srcbase/srclen window. */ -static int -xd3_source_match_setup (xd3_stream *stream, xoff_t srcpos) -{ - xd3_source *src = stream->src; - usize_t greedy_or_not; - xoff_t frontier_pos; - - stream->match_maxback = 0; - stream->match_maxfwd = 0; - stream->match_back = 0; - stream->match_fwd = 0; - - /* This avoids a non-blocking endless loop caused by scanning - * backwards across a block boundary, only to find not enough - * matching bytes to beat the current min_match due to a better lazy - * target match: the re-entry to xd3_string_match() repeats the same - * long match because the input position hasn't changed. TODO: if - * ever duplicates are added to the source hash table, this logic - * won't suffice to avoid loops. See testing/regtest.cc's - * TestNonBlockingProgress test! */ - if (srcpos != 0 && srcpos == stream->match_last_srcpos) - { - IF_DEBUG2(DP(RINT "[match_setup] looping failure\n")); - goto bad; - } - - /* Implement srcwin_maxsz, which prevents the encoder from seeking - * back further than the LRU cache maintaining FIFO discipline, (to - * avoid seeking). */ - frontier_pos = - stream->src->frontier_blkno * stream->src->blksize; - IF_DEBUG1(DP(RINT "[match_setup] frontier_pos %"Q"u, srcpos %"Q"u, " - "srcwin_maxsz %u\n", - frontier_pos, srcpos, stream->srcwin_maxsz)); - if (srcpos < frontier_pos && - frontier_pos - srcpos > stream->srcwin_maxsz) { - IF_DEBUG1(DP(RINT "[match_setup] rejected due to srcwin_maxsz " - "distance eof=%"Q"u srcpos=%"Q"u maxsz=%u\n", - xd3_source_eof (stream->src), - srcpos, stream->srcwin_maxsz)); - goto bad; - } - - /* Going backwards, the 1.5-pass algorithm allows some - * already-matched input may be covered by a longer source match. - * The greedy algorithm does not allow this. */ - if (stream->flags & XD3_BEGREEDY) - { - /* The greedy algorithm allows backward matching to the last - matched position. */ - greedy_or_not = xd3_iopt_last_matched (stream); - } - else - { - /* The 1.5-pass algorithm allows backward matching to go back as - * far as the unencoded offset, which is updated as instructions - * pass out of the iopt buffer. If this (default) is chosen, it - * means xd3_iopt_erase may be called to eliminate instructions - * when a covering source match is found. */ - greedy_or_not = stream->unencoded_offset; - } - - /* Backward target match limit. */ - XD3_ASSERT (stream->input_position >= greedy_or_not); - stream->match_maxback = stream->input_position - greedy_or_not; - - /* Forward target match limit. */ - XD3_ASSERT (stream->avail_in > stream->input_position); - stream->match_maxfwd = stream->avail_in - stream->input_position; - - /* Now we take the source position into account. It depends whether - * the srclen/srcbase have been decided yet. */ - if (stream->srcwin_decided == 0) - { - /* Unrestricted case: the match can cover the entire source, - * 0--src->size. We compare the usize_t - * match_maxfwd/match_maxback against the xoff_t - * src->size/srcpos values and take the min. */ - if (srcpos < (xoff_t) stream->match_maxback) - { - stream->match_maxback = (usize_t) srcpos; - } - - if (stream->src->eof_known) - { - xoff_t srcavail = xd3_source_eof (stream->src) - srcpos; - - if (srcavail < (xoff_t) stream->match_maxfwd) - { - stream->match_maxfwd = (usize_t) srcavail; - } - } - - IF_DEBUG1(DP(RINT - "[match_setup] srcpos %"Q"u (tgtpos %"Q"u) " - "unrestricted maxback %u maxfwd %u\n", - srcpos, - stream->total_in + stream->input_position, - stream->match_maxback, - stream->match_maxfwd)); - goto good; - } - - /* Decided some source window. */ - XD3_ASSERT (src->srclen > 0); - - /* Restricted case: fail if the srcpos lies outside the source window */ - if ((srcpos < src->srcbase) || - (srcpos > (src->srcbase + (xoff_t) src->srclen))) - { - IF_DEBUG1(DP(RINT "[match_setup] restricted source window failure\n")); - goto bad; - } - else - { - usize_t srcavail; - - srcavail = (usize_t) (srcpos - src->srcbase); - if (srcavail < stream->match_maxback) - { - stream->match_maxback = srcavail; - } - - srcavail = (usize_t) (src->srcbase + (xoff_t) src->srclen - srcpos); - if (srcavail < stream->match_maxfwd) - { - stream->match_maxfwd = srcavail; - } - - IF_DEBUG1(DP(RINT - "[match_setup] srcpos %"Q"u (tgtpos %"Q"u) " - "restricted maxback %u maxfwd %u\n", - srcpos, - stream->total_in + stream->input_position, - stream->match_maxback, - stream->match_maxfwd)); - goto good; - } - - good: - stream->match_state = MATCH_BACKWARD; - stream->match_srcpos = srcpos; - stream->match_last_srcpos = srcpos; - return 0; - - bad: - stream->match_state = MATCH_SEARCHING; - return 1; -} - -static inline int -xd3_forward_match(const uint8_t *s1c, const uint8_t *s2c, int n) -{ - int i = 0; -#if UNALIGNED_OK - int nint = n / sizeof(int); - - if (nint >> 3) - { - int j = 0; - const int *s1 = (const int*)s1c; - const int *s2 = (const int*)s2c; - int nint_8 = nint - 8; - - while (i <= nint_8 && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++] && - s1[i++] == s2[j++]) { } - - i = (i - 1) * sizeof(int); - } -#endif - - while (i < n && s1c[i] == s2c[i]) - { - i++; - } - return i; -} - -/* This function expands the source match backward and forward. It is - * reentrant, since xd3_getblk may return XD3_GETSRCBLK, so most - * variables are kept in xd3_stream. There are two callers of this - * function, the string_matching routine when a checksum match is - * discovered, and xd3_encode_input whenever a continuing (or initial) - * match is suspected. The two callers do different things with the - * input_position, thus this function leaves that variable untouched. - * If a match is taken the resulting stream->match_fwd is left - * non-zero. */ -static int -xd3_source_extend_match (xd3_stream *stream) -{ - int ret; - xd3_source *src = stream->src; - xoff_t matchoff; /* matchoff is the current right/left-boundary of - the source match being tested. */ - usize_t streamoff; /* streamoff is the current right/left-boundary - of the input match being tested. */ - xoff_t tryblk; /* tryblk, tryoff are the block, offset position - of matchoff */ - usize_t tryoff; - usize_t tryrem; /* tryrem is the number of matchable bytes */ - usize_t matched; - - IF_DEBUG2(DP(RINT "[extend match] srcpos %"Q"u\n", - stream->match_srcpos)); - - XD3_ASSERT (src != NULL); - - /* Does it make sense to compute backward match AFTER forward match? */ - if (stream->match_state == MATCH_BACKWARD) - { - /* Note: this code is practically duplicated below, substituting - * match_fwd/match_back and direction. TODO: Consolidate? */ - matchoff = stream->match_srcpos - stream->match_back; - streamoff = stream->input_position - stream->match_back; - xd3_blksize_div (matchoff, src, &tryblk, &tryoff); - - /* this loops backward over source blocks */ - while (stream->match_back < stream->match_maxback) - { - /* see if we're backing across a source block boundary */ - if (tryoff == 0) - { - tryoff = src->blksize; - tryblk -= 1; - } - - if ((ret = xd3_getblk (stream, tryblk))) - { - /* if search went too far back, continue forward. */ - if (ret == XD3_TOOFARBACK) - { - break; - } - - /* could be a XD3_GETSRCBLK failure. */ - return ret; - } - - tryrem = min (tryoff, stream->match_maxback - stream->match_back); - - IF_DEBUG2(DP(RINT "[maxback] maxback %u trysrc %"Q"u/%u tgt %u tryrem %u\n", - stream->match_maxback, tryblk, tryoff, streamoff, tryrem)); - - /* TODO: This code can be optimized similar to xd3_match_forward() */ - for (; tryrem != 0; tryrem -= 1, stream->match_back += 1) - { - if (src->curblk[tryoff-1] != stream->next_in[streamoff-1]) - { - goto doneback; - } - - tryoff -= 1; - streamoff -= 1; - } - } - - doneback: - stream->match_state = MATCH_FORWARD; - } - - XD3_ASSERT (stream->match_state == MATCH_FORWARD); - - matchoff = stream->match_srcpos + stream->match_fwd; - streamoff = stream->input_position + stream->match_fwd; - xd3_blksize_div (matchoff, src, & tryblk, & tryoff); - - /* Note: practically the same code as backwards case above: same comments */ - while (stream->match_fwd < stream->match_maxfwd) - { - if (tryoff == src->blksize) - { - tryoff = 0; - tryblk += 1; - } - - if ((ret = xd3_getblk (stream, tryblk))) - { - /* if search went too far back, continue forward. */ - if (ret == XD3_TOOFARBACK) - { - break; - } - - /* could be a XD3_GETSRCBLK failure. */ - return ret; - } - - tryrem = min(stream->match_maxfwd - stream->match_fwd, - src->onblk - tryoff); - - if (tryrem == 0) - { - /* Generally, this means we have a power-of-two size source - * and we just found the end-of-file, in this case it's an - * empty block. */ - XD3_ASSERT (src->onblk < src->blksize); - break; - } - - matched = xd3_forward_match(src->curblk + tryoff, - stream->next_in + streamoff, - tryrem); - tryoff += matched; - streamoff += matched; - stream->match_fwd += matched; - - if (tryrem != matched) - { - break; - } - } - - stream->match_state = MATCH_SEARCHING; - - /* If the match ends short of the last instruction end, we probably - * don't want it. There is the possibility that a copy ends short - * of the last copy but also goes further back, in which case we - * might want it. This code does not implement such: if so we would - * need more complicated xd3_iopt_erase logic. */ - if (stream->match_fwd < stream->min_match) - { - stream->match_fwd = 0; - } - else - { - usize_t total = stream->match_fwd + stream->match_back; - - /* Correct the variables to remove match_back from the equation. */ - usize_t target_position = stream->input_position - stream->match_back; - usize_t match_length = stream->match_back + stream->match_fwd; - xoff_t match_position = stream->match_srcpos - stream->match_back; - xoff_t match_end = stream->match_srcpos + stream->match_fwd; - - /* At this point we may have to erase any iopt-buffer - * instructions that are fully covered by a backward-extending - * copy. */ - if (stream->match_back > 0) - { - xd3_iopt_erase (stream, target_position, total); - } - - stream->match_back = 0; - - /* Update ranges. The first source match occurs with both - values set to 0. */ - if (stream->match_maxaddr == 0 || - match_position < stream->match_minaddr) - { - stream->match_minaddr = match_position; - } - - if (match_end > stream->match_maxaddr) - { - /* Note: per-window */ - stream->match_maxaddr = match_end; - } - - if (match_end > stream->maxsrcaddr) - { - /* Note: across windows */ - stream->maxsrcaddr = match_end; - } - - IF_DEBUG1 ({ - static int x = 0; - DP(RINT "[source match:%d] (%s) [ %u bytes ]\n", - x++, - stream->total_in + target_position, - stream->total_in + target_position + match_length, - match_position, - match_position + match_length, - (stream->total_in + target_position == match_position) ? "same" : "diff", - match_length); - }); - - if ((ret = xd3_found_match (stream, - /* decoder position */ target_position, - /* length */ match_length, - /* address */ match_position, - /* is_source */ 1))) - { - return ret; - } - - /* If the match ends with the available input: */ - if (target_position + match_length == stream->avail_in) - { - /* Setup continuing match for the next window. */ - stream->match_state = MATCH_TARGET; - stream->match_srcpos = match_end; - } - } - - return 0; -} - -/* Update the small hash. Values in the small_table are offset by - * HASH_CKOFFSET (1) to distinguish empty buckets from real offsets. */ -static void -xd3_scksum_insert (xd3_stream *stream, - usize_t inx, - usize_t scksum ATTRIBUTE((unused)), - usize_t pos) -{ - /* If we are maintaining previous duplicates. */ - if (stream->small_prev) - { - usize_t last_pos = stream->small_table[inx]; - xd3_slist *pos_list = & stream->small_prev[pos & stream->sprevmask]; - - /* Note last_pos is offset by HASH_CKOFFSET. */ - pos_list->last_pos = last_pos; - } - - /* Enter the new position into the hash bucket. */ - stream->small_table[inx] = pos + HASH_CKOFFSET; -} - -#if XD3_DEBUG -static int -xd3_check_smatch (const uint8_t *ref0, const uint8_t *inp0, - const uint8_t *inp_max, usize_t cmp_len) -{ - usize_t i; - - for (i = 0; i < cmp_len; i += 1) - { - XD3_ASSERT (ref0[i] == inp0[i]); - } - - if (inp0 + cmp_len < inp_max) - { - XD3_ASSERT (inp0[i] != ref0[i]); - } - - return 1; -} -#endif /* XD3_DEBUG */ - -/* When the hash table indicates a possible small string match, it - * calls this routine to find the best match. The first matching - * position is taken from the small_table, HASH_CKOFFSET is subtracted - * to get the actual position. After checking that match, if previous - * linked lists are in use (because stream->smatcher.small_chain > 1), - * previous matches are tested searching for the longest match. If - * (stream->min_match > MIN_MATCH) then a lazy match is in effect. - */ -static usize_t -xd3_smatch (xd3_stream *stream, - usize_t base, - usize_t scksum IFN_DEBUG2(ATTRIBUTE((unused))), - usize_t *match_offset) -{ - usize_t cmp_len; - usize_t match_length = 0; - usize_t chain = (stream->min_match == MIN_MATCH ? - stream->smatcher.small_chain : - stream->smatcher.small_lchain); - const uint8_t *inp_max = stream->next_in + stream->avail_in; - const uint8_t *inp; - const uint8_t *ref; - - SMALL_HASH_DEBUG1 (stream, stream->next_in + stream->input_position); - - XD3_ASSERT (stream->min_match + stream->input_position <= stream->avail_in); - - base -= HASH_CKOFFSET; - - again: - - IF_DEBUG2 (DP(RINT "smatch at base=%u inp=%u cksum=%u\n", base, - stream->input_position, scksum)); - - /* For small matches, we can always go to the end-of-input because - * the matching position must be less than the input position. */ - XD3_ASSERT (base < stream->input_position); - - ref = stream->next_in + base; - inp = stream->next_in + stream->input_position; - - SMALL_HASH_DEBUG2 (stream, ref); - - /* Expand potential match forward. */ - while (inp < inp_max && *inp == *ref) - { - ++inp; - ++ref; - } - - cmp_len = (usize_t)(inp - (stream->next_in + stream->input_position)); - - /* Verify correctness */ - XD3_ASSERT (xd3_check_smatch (stream->next_in + base, - stream->next_in + stream->input_position, - inp_max, cmp_len)); - - /* Update longest match */ - if (cmp_len > match_length) - { - ( match_length) = cmp_len; - (*match_offset) = base; - - /* Stop if we match the entire input or have a long_enough match. */ - if (inp == inp_max || cmp_len >= stream->smatcher.long_enough) - { - goto done; - } - } - - /* If we have not reached the chain limit, see if there is another - previous position. */ - while (--chain != 0) - { - /* Calculate the previous offset. */ - usize_t prev_pos = stream->small_prev[base & stream->sprevmask].last_pos; - usize_t diff_pos; - - if (prev_pos == 0) - { - break; - } - - prev_pos -= HASH_CKOFFSET; - - if (prev_pos > base) - { - break; - } - - base = prev_pos; - - XD3_ASSERT (stream->input_position > base); - diff_pos = stream->input_position - base; - - /* Stop searching if we go beyond sprevsz, since those entries - * are for unrelated checksum entries. */ - if (diff_pos & ~stream->sprevmask) - { - break; - } - - goto again; - } - - done: - /* Crude efficiency test: if the match is very short and very far back, it's - * unlikely to help, but the exact calculation requires knowing the state of - * the address cache and adjacent instructions, which we can't do here. - * Rather than encode a probably inefficient copy here and check it later - * (which complicates the code a lot), do this: - */ - if (match_length == 4 && stream->input_position - (*match_offset) >= 1<<14) - { - /* It probably takes >2 bytes to encode an address >= 2^14 from here */ - return 0; - } - if (match_length == 5 && stream->input_position - (*match_offset) >= 1<<21) - { - /* It probably takes >3 bytes to encode an address >= 2^21 from here */ - return 0; - } - - /* It's unlikely that a window is large enough for the (match_length == 6 && - * address >= 2^28) check */ - return match_length; -} - -#if XD3_DEBUG -static void -xd3_verify_small_state (xd3_stream *stream, - const uint8_t *inp, - uint32_t x_cksum) -{ - uint32_t state; - uint32_t cksum = xd3_scksum (&state, inp, stream->smatcher.small_look); - - XD3_ASSERT (cksum == x_cksum); -} - -static void -xd3_verify_large_state (xd3_stream *stream, - const uint8_t *inp, - uint32_t x_cksum) -{ - uint32_t cksum = xd3_lcksum (inp, stream->smatcher.large_look); - XD3_ASSERT (cksum == x_cksum); -} -static void -xd3_verify_run_state (xd3_stream *stream, - const uint8_t *inp, - usize_t x_run_l, - uint8_t *x_run_c) -{ - usize_t slook = stream->smatcher.small_look; - uint8_t run_c; - usize_t run_l = xd3_comprun (inp, slook, &run_c); - - XD3_ASSERT (run_l == 0 || run_c == *x_run_c); - XD3_ASSERT (x_run_l > slook || run_l == x_run_l); -} -#endif /* XD3_DEBUG */ - -/* This function computes more source checksums to advance the window. - * Called at every entrance to the string-match loop and each time - * stream->input_position reaches the value returned as - * *next_move_point. NB: this is one of the most expensive functions - * in this code and also the most critical for good compression. - * TODO: optimize the inner loop - */ -static int -xd3_srcwin_move_point (xd3_stream *stream, usize_t *next_move_point) -{ - xoff_t logical_input_cksum_pos; - xoff_t source_size; - - if (stream->src->eof_known) - { - source_size = xd3_source_eof (stream->src); - XD3_ASSERT(stream->srcwin_cksum_pos <= source_size); - - if (stream->srcwin_cksum_pos == source_size) - { - *next_move_point = USIZE_T_MAX; - return 0; - } - } - - /* Begin by advancing at twice the input rate, up to half the - * maximum window size. */ - logical_input_cksum_pos = min((stream->total_in + stream->input_position) * 2, - (stream->total_in + stream->input_position) + - (stream->srcwin_maxsz / 2)); - - /* If srcwin_cksum_pos is already greater, wait until the difference - * is met. */ - if (stream->srcwin_cksum_pos > logical_input_cksum_pos) - { - *next_move_point = stream->input_position + - (usize_t)(stream->srcwin_cksum_pos - logical_input_cksum_pos); - return 0; - } - - /* A long match may have extended past srcwin_cksum_pos. Don't - * start checksumming already-matched source data. */ - if (stream->maxsrcaddr > stream->srcwin_cksum_pos) - { - stream->srcwin_cksum_pos = stream->maxsrcaddr; - } - - if (logical_input_cksum_pos < stream->srcwin_cksum_pos) - { - logical_input_cksum_pos = stream->srcwin_cksum_pos; - } - - /* Advance at least one source block. With the command-line - * defaults this means: - * - * if (src->size <= srcwin_maxsz), index the entire source at once - * using the position of the first non-match. This is good for - * small inputs, especially when the content may have moved anywhere - * in the file (e.g., tar files). - * - * if (src->size > srcwin_maxsz), index at least one block (which - * the command-line sets to 1/32 of srcwin_maxsz) ahead of the - * logical position. This is good for different reasons: when a - * long match spanning several source blocks is encountered, this - * avoids computing checksums for those blocks. If the data can - * move anywhere, this is bad. - */ - logical_input_cksum_pos += stream->src->blksize; - - while (stream->srcwin_cksum_pos < logical_input_cksum_pos && - (!stream->src->eof_known || - stream->srcwin_cksum_pos < xd3_source_eof (stream->src))) - { - xoff_t blkno; - xoff_t blkbaseoffset; - usize_t blkrem; - ssize_t oldpos; /* Using ssize_t because of a */ - ssize_t blkpos; /* do { blkpos-- } - while (blkpos >= oldpos); */ - int ret; - xd3_blksize_div (stream->srcwin_cksum_pos, - stream->src, &blkno, &blkrem); - oldpos = blkrem; - - if ((ret = xd3_getblk (stream, blkno))) - { - /* TOOFARBACK should never occur here, since we read forward. */ - if (ret == XD3_TOOFARBACK) - { - ret = XD3_INTERNAL; - } - IF_DEBUG1 (DP(RINT - "[srcwin_move_point] async getblk return for %"Q"u\n", - blkno)); - return ret; - } - - IF_DEBUG1 (DP(RINT - "[srcwin_move_point] T=%"Q"u{%"Q"u} S=%"Q"u EOF=%"Q"u %s\n", - stream->total_in + stream->input_position, - logical_input_cksum_pos, - stream->srcwin_cksum_pos, - xd3_source_eof (stream->src), - stream->src->eof_known ? "known" : "unknown")); - - blkpos = xd3_bytes_on_srcblk (stream->src, blkno); - - if (blkpos < (ssize_t) stream->smatcher.large_look) - { - stream->srcwin_cksum_pos = (blkno + 1) * stream->src->blksize; - IF_DEBUG1 (DP(RINT "[srcwin_move_point] continue (end-of-block)\n")); - continue; - } - - /* This inserts checksums for the entire block, in reverse, - * starting from the end of the block. This logic does not test - * stream->srcwin_cksum_pos because it always advances it to the - * start of the next block. - * - * oldpos is the srcwin_cksum_pos within this block. blkpos is - * the number of bytes available. Each iteration inspects - * large_look bytes then steps back large_step bytes. The - * if-stmt above ensures at least one large_look of data. */ - blkpos -= stream->smatcher.large_look; - blkbaseoffset = stream->src->blksize * blkno; - - do - { - uint32_t cksum = xd3_lcksum (stream->src->curblk + blkpos, - stream->smatcher.large_look); - usize_t hval = xd3_checksum_hash (& stream->large_hash, cksum); - - stream->large_table[hval] = - (usize_t) (blkbaseoffset + - (xoff_t)(blkpos + HASH_CKOFFSET)); - - IF_DEBUG (stream->large_ckcnt += 1); - - blkpos -= stream->smatcher.large_step; - } - while (blkpos >= oldpos); - - stream->srcwin_cksum_pos = (blkno + 1) * stream->src->blksize; - } - - IF_DEBUG1 (DP(RINT - "[srcwin_move_point] exited loop T=%"Q"u{%"Q"u} " - "S=%"Q"u EOF=%"Q"u %s\n", - stream->total_in + stream->input_position, - logical_input_cksum_pos, - stream->srcwin_cksum_pos, - xd3_source_eof (stream->src), - stream->src->eof_known ? "known" : "unknown")); - - if (stream->src->eof_known) - { - source_size = xd3_source_eof (stream->src); - - if (stream->srcwin_cksum_pos >= source_size) - { - /* This invariant is needed for xd3_source_cksum_offset() */ - stream->srcwin_cksum_pos = source_size; - *next_move_point = USIZE_T_MAX; - IF_DEBUG1 (DP(RINT - "[srcwin_move_point] finished with source input\n")); - return 0; - } - } - - /* How long until this function should be called again. */ - XD3_ASSERT(stream->srcwin_cksum_pos >= logical_input_cksum_pos); - *next_move_point = stream->input_position + 1 + - (usize_t)(stream->srcwin_cksum_pos - logical_input_cksum_pos); - return 0; -} - -#endif /* XD3_ENCODER */ - -/******************************************************************** - TEMPLATE pass - *********************************************************************/ - -#endif /* __XDELTA3_C_INLINE_PASS__ */ -#ifdef __XDELTA3_C_TEMPLATE_PASS__ - -#if XD3_ENCODER - -/******************************************************************** - Templates - *******************************************************************/ - -/* Template macros */ -#define XD3_TEMPLATE(x) XD3_TEMPLATE2(x,TEMPLATE) -#define XD3_TEMPLATE2(x,n) XD3_TEMPLATE3(x,n) -#define XD3_TEMPLATE3(x,n) x ## n -#define XD3_STRINGIFY(x) XD3_STRINGIFY2(x) -#define XD3_STRINGIFY2(x) #x - -static int XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream); - -static const xd3_smatcher XD3_TEMPLATE(__smatcher_) = -{ - XD3_STRINGIFY(TEMPLATE), - XD3_TEMPLATE(xd3_string_match_), -#if SOFTCFG == 1 - 0, 0, 0, 0, 0, 0, 0 -#else - LLOOK, LSTEP, SLOOK, SCHAIN, SLCHAIN, MAXLAZY, LONGENOUGH -#endif -}; - -static int -XD3_TEMPLATE(xd3_string_match_) (xd3_stream *stream) -{ - const int DO_SMALL = ! (stream->flags & XD3_NOCOMPRESS); - const int DO_LARGE = (stream->src != NULL); - const int DO_RUN = (1); - - const uint8_t *inp; - uint32_t scksum = 0; - uint32_t scksum_state = 0; - uint32_t lcksum = 0; - usize_t sinx; - usize_t linx; - uint8_t run_c; - usize_t run_l; - int ret; - usize_t match_length; - usize_t match_offset = 0; - usize_t next_move_point; - - /* If there will be no compression due to settings or short input, - * skip it entirely. */ - if (! (DO_SMALL || DO_LARGE || DO_RUN) || - stream->input_position + SLOOK > stream->avail_in) { goto loopnomore; } - - if ((ret = xd3_string_match_init (stream))) { return ret; } - - /* The restartloop label is reached when the incremental loop state - * needs to be reset. */ - restartloop: - - /* If there is not enough input remaining for any kind of match, - skip it. */ - if (stream->input_position + SLOOK > stream->avail_in) { goto loopnomore; } - - /* Now reset the incremental loop state: */ - - /* The min_match variable is updated to avoid matching the same lazy - * match over and over again. For example, if you find a (small) - * match of length 9 at one position, you will likely find a match - * of length 8 at the next position. */ - if (xd3_iopt_last_matched (stream) > stream->input_position) - { - stream->min_match = max(MIN_MATCH, - 1 + xd3_iopt_last_matched(stream) - - stream->input_position); - } - else - { - stream->min_match = MIN_MATCH; - } - - /* The current input byte. */ - inp = stream->next_in + stream->input_position; - - /* Small match state. */ - if (DO_SMALL) - { - scksum = xd3_scksum (&scksum_state, inp, SLOOK); - } - - /* Run state. */ - if (DO_RUN) - { - run_l = xd3_comprun (inp, SLOOK, & run_c); - } - - /* Large match state. We continue the loop even after not enough - * bytes for LLOOK remain, so always check stream->input_position in - * DO_LARGE code. */ - if (DO_LARGE && (stream->input_position + LLOOK <= stream->avail_in)) - { - /* Source window: next_move_point is the point that - * stream->input_position must reach before computing more - * source checksum. */ - if ((ret = xd3_srcwin_move_point (stream, & next_move_point))) - { - return ret; - } - - lcksum = xd3_lcksum (inp, LLOOK); - } - - /* TRYLAZYLEN: True if a certain length match should be followed by - * lazy search. This checks that LEN is shorter than MAXLAZY and - * that there is enough leftover data to consider lazy matching. - * "Enough" is set to 2 since the next match will start at the next - * offset, it must match two extra characters. */ -#define TRYLAZYLEN(LEN,POS,MAX) ((MAXLAZY) > 0 && (LEN) < (MAXLAZY) \ - && (POS) + (LEN) <= (MAX) - 2) - - /* HANDLELAZY: This statement is called each time an instruciton is - * emitted (three cases). If the instruction is large enough, the - * loop is restarted, otherwise lazy matching may ensue. */ -#define HANDLELAZY(mlen) \ - if (TRYLAZYLEN ((mlen), (stream->input_position), (stream->avail_in))) \ - { stream->min_match = (mlen) + LEAST_MATCH_INCR; goto updateone; } \ - else \ - { stream->input_position += (mlen); goto restartloop; } - - /* Now loop over one input byte at a time until a match is found... */ - for (;; inp += 1, stream->input_position += 1) - { - /* Now we try three kinds of string match in order of expense: - * run, large match, small match. */ - - /* Expand the start of a RUN. The test for (run_l == SLOOK) - * avoids repeating this check when we pass through a run area - * performing lazy matching. The run is only expanded once when - * the min_match is first reached. If lazy matching is - * performed, the run_l variable will remain inconsistent until - * the first non-running input character is reached, at which - * time the run_l may then again grow to SLOOK. */ - if (DO_RUN && run_l == SLOOK) - { - usize_t max_len = stream->avail_in - stream->input_position; - - IF_DEBUG (xd3_verify_run_state (stream, inp, run_l, &run_c)); - - while (run_l < max_len && inp[run_l] == run_c) { run_l += 1; } - - /* Output a RUN instruction. */ - if (run_l >= stream->min_match && run_l >= MIN_RUN) - { - if ((ret = xd3_emit_run (stream, stream->input_position, - run_l, &run_c))) { return ret; } - - HANDLELAZY (run_l); - } - } - - /* If there is enough input remaining. */ - if (DO_LARGE && (stream->input_position + LLOOK <= stream->avail_in)) - { - if ((stream->input_position >= next_move_point) && - (ret = xd3_srcwin_move_point (stream, & next_move_point))) - { - return ret; - } - - linx = xd3_checksum_hash (& stream->large_hash, lcksum); - - IF_DEBUG (xd3_verify_large_state (stream, inp, lcksum)); - - if (stream->large_table[linx] != 0) - { - /* the match_setup will fail if the source window has - * been decided and the match lies outside it. - * OPT: Consider forcing a window at this point to - * permit a new source window. */ - xoff_t adj_offset = - xd3_source_cksum_offset(stream, - stream->large_table[linx] - - HASH_CKOFFSET); - if (xd3_source_match_setup (stream, adj_offset) == 0) - { - if ((ret = xd3_source_extend_match (stream))) - { - return ret; - } - - /* Update stream position. match_fwd is zero if no - * match. */ - if (stream->match_fwd > 0) - { - HANDLELAZY (stream->match_fwd); - } - } - } - } - - /* Small matches. */ - if (DO_SMALL) - { - sinx = xd3_checksum_hash (& stream->small_hash, scksum); - - /* Verify incremental state in debugging mode. */ - IF_DEBUG (xd3_verify_small_state (stream, inp, scksum)); - - /* Search for the longest match */ - if (stream->small_table[sinx] != 0) - { - match_length = xd3_smatch (stream, - stream->small_table[sinx], - scksum, - & match_offset); - } - else - { - match_length = 0; - } - - /* Insert a hash for this string. */ - xd3_scksum_insert (stream, sinx, scksum, stream->input_position); - - /* Maybe output a COPY instruction */ - if (match_length >= stream->min_match) - { - IF_DEBUG2 ({ - static int x = 0; - DP(RINT "[target match:%d] " - "(-%d) [ %u bytes ]\n", - x++, - stream->input_position, - stream->input_position + match_length, - match_offset, - match_offset + match_length, - stream->input_position - match_offset, - match_length); - }); - - if ((ret = xd3_found_match (stream, - /* decoder position */ - stream->input_position, - /* length */ match_length, - /* address */ (xoff_t) match_offset, - /* is_source */ 0))) - { - return ret; - } - - /* Copy instruction. */ - HANDLELAZY (match_length); - } - } - - /* The logic above prevents excess work during lazy matching by - * increasing min_match to avoid smaller matches. Each time we - * advance stream->input_position by one, the minimum match - * shortens as well. */ - if (stream->min_match > MIN_MATCH) - { - stream->min_match -= 1; - } - - updateone: - - /* See if there are no more incremental cksums to compute. */ - if (stream->input_position + SLOOK == stream->avail_in) - { - goto loopnomore; - } - - /* Compute next RUN, CKSUM */ - if (DO_RUN) - { - NEXTRUN (inp[SLOOK]); - } - - if (DO_SMALL) - { - scksum = xd3_small_cksum_update (&scksum_state, inp, SLOOK); - } - - if (DO_LARGE && (stream->input_position + LLOOK < stream->avail_in)) - { - lcksum = xd3_large_cksum_update (lcksum, inp, LLOOK); - } - } - - loopnomore: - return 0; -} - -#endif /* XD3_ENCODER */ -#endif /* __XDELTA3_C_TEMPLATE_PASS__ */ diff --git a/polymer/eduke32/source/xdelta3/xdelta3.h b/polymer/eduke32/source/xdelta3/xdelta3.h deleted file mode 100644 index b2288f403..000000000 --- a/polymer/eduke32/source/xdelta3/xdelta3.h +++ /dev/null @@ -1,1377 +0,0 @@ -/* xdelta 3 - delta compression tools and library - * Copyright (C) 2001, 2003, 2004, 2005, 2006, 2007, - * 2008, 2009, 2010. Joshua P. MacDonald - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -/* To know more about Xdelta, start by reading xdelta3.c. If you are - * ready to use the API, continue reading here. There are two - * interfaces -- xd3_encode_input and xd3_decode_input -- plus a dozen - * or so related calls. This interface is styled after Zlib. */ - -#ifndef _XDELTA3_H_ -#define _XDELTA3_H_ - -/* -#define _POSIX_SOURCE -#define _ISOC99_SOURCE -#define _C99_SOURCE -*/ - -#include -#include -#include -#include -#include -#include -#include - -/****************************************************************/ - -/* Default configured value of stream->winsize. If the program - * supplies xd3_encode_input() with data smaller than winsize the - * stream will automatically buffer the input, otherwise the input - * buffer is used directly. - */ -#ifndef XD3_DEFAULT_WINSIZE -#define XD3_DEFAULT_WINSIZE (1U << 23) -#endif - -/* Default total size of the source window used in xdelta3-main.h */ -#ifndef XD3_DEFAULT_SRCWINSZ -#define XD3_DEFAULT_SRCWINSZ (1U << 26) -#endif - -/* When Xdelta requests a memory allocation for certain buffers, it - * rounds up to units of at least this size. The code assumes (and - * asserts) that this is a power-of-two. */ -#ifndef XD3_ALLOCSIZE -#define XD3_ALLOCSIZE (1U<<14) -#endif - -/* The XD3_HARDMAXWINSIZE parameter is a safety mechanism to protect - * decoders against malicious files. The decoder will never decode a - * window larger than this. If the file specifies VCD_TARGET the - * decoder may require two buffers of this size. - * - * 8-16MB is reasonable, probably don't need to go larger. */ -#ifndef XD3_HARDMAXWINSIZE -#define XD3_HARDMAXWINSIZE (1U<<24) -#endif -/* The IOPT_SIZE value sets the size of a buffer used to batch - * overlapping copy instructions before they are optimized by picking - * the best non-overlapping ranges. The larger this buffer, the - * longer a forced xd3_srcwin_setup() decision is held off. Setting - * this value to 0 causes an unlimited buffer to be used. */ -#ifndef XD3_DEFAULT_IOPT_SIZE -#define XD3_DEFAULT_IOPT_SIZE (1U<<15) -#endif - -/* The maximum distance backward to search for small matches */ -#ifndef XD3_DEFAULT_SPREVSZ -#define XD3_DEFAULT_SPREVSZ (1U<<18) -#endif - -/* The default compression level - */ -#ifndef XD3_DEFAULT_LEVEL -#define XD3_DEFAULT_LEVEL 1 -#endif - -#ifndef XD3_DEFAULT_SECONDARY_LEVEL -#define XD3_DEFAULT_SECONDARY_LEVEL 1 -#endif - -#ifndef XD3_USE_LARGEFILE64 -#define XD3_USE_LARGEFILE64 1 -#endif - -/* Sizes and addresses within VCDIFF windows are represented as usize_t - * - * For source-file offsets and total file sizes, total input and - * output counts, the xoff_t type is used. The decoder and encoder - * generally check for overflow of the xoff_t size (this is tested at - * the 32bit boundary [xdelta3-test.h]). - */ -#ifndef _WIN32 -#include -#else -#define WIN32_LEAN_AND_MEAN -#if XD3_USE_LARGEFILE64 -/* 64 bit file offsets: uses GetFileSizeEx and SetFilePointerEx. - * requires Win2000 or newer version of WinNT */ - #ifndef WINVER -#define WINVER 0x0500 - #endif - #ifndef _WIN32_WINNT -#define _WIN32_WINNT 0x0500 - #endif -#else -/* 32 bit (DWORD) file offsets: uses GetFileSize and - * SetFilePointer. compatible with win9x-me and WinNT4 */ - #ifndef WINVER -#define WINVER 0x0400 - #endif - #ifndef _WIN32_WINNT -#define _WIN32_WINNT 0x0400 -#endif - #endif -#include -#ifdef _MSC_VER -#define inline -typedef signed int ssize_t; -typedef unsigned char uint8_t; -typedef unsigned short uint16_t; -typedef unsigned __int32 uint32_t; -typedef ULONGLONG uint64_t; -#else -/* mingw32, lcc and watcom provide a proper header */ -#include -#endif -#endif -typedef unsigned int usize_t; - -/* TODO: note that SIZEOF_USIZE_T is never set to 8, although it should be for - * a 64bit platform. OTOH, may be that using 32bits is appropriate even on a - * 64bit platform because we allocate large arrays of these values. */ -#if XD3_USE_LARGEFILE64 -#define __USE_FILE_OFFSET64 1 /* GLIBC: for 64bit fileops, ... ? */ -#ifndef _LARGEFILE_SOURCE -#define _LARGEFILE_SOURCE -#endif -#ifndef _FILE_OFFSET_BITS -#define _FILE_OFFSET_BITS 64 -#endif - -typedef uint64_t xoff_t; -#define SIZEOF_XOFF_T 8 -#define SIZEOF_USIZE_T 4 -#ifndef WIN32 -#define Q "ll" -#else -#define Q "I64" -#endif -#else -typedef uint32_t xoff_t; -#define SIZEOF_XOFF_T 4 -#define SIZEOF_USIZE_T 4 -#define Q -#endif - -#define USE_UINT32 (SIZEOF_USIZE_T == 4 || \ - SIZEOF_XOFF_T == 4 || REGRESSION_TEST) -#define USE_UINT64 (SIZEOF_USIZE_T == 8 || \ - SIZEOF_XOFF_T == 8 || REGRESSION_TEST) - -/* TODO: probably should do something better here. */ -#ifndef UNALIGNED_OK -#if defined(__i386__) || defined(__i486__) || defined(__i586__) || \ - defined(__i686__) || defined(_X86_) || defined(__x86_64__) -#define UNALIGNED_OK 1 -#else -#define UNALIGNED_OK 0 -#endif -#endif - -/**********************************************************************/ - -/* Whether to build the encoder, otherwise only build the decoder. */ -#ifndef XD3_ENCODER -#define XD3_ENCODER 1 -#endif - -/* The code returned when main() fails, also defined in system - includes. */ -#ifndef EXIT_FAILURE -#define EXIT_FAILURE 1 -#endif - -/* REGRESSION TEST enables the "xdelta3 test" command, which runs a - series of self-tests. */ -#ifndef REGRESSION_TEST -#define REGRESSION_TEST 0 -#endif - -/* XD3_DEBUG=1 enables assertions and various statistics. Levels > 1 - * enable some additional output only useful during development and - * debugging. */ -#ifndef XD3_DEBUG -#define XD3_DEBUG 0 -#endif - -#ifndef PYTHON_MODULE -#define PYTHON_MODULE 0 -#endif - -#ifndef SWIG_MODULE -#define SWIG_MODULE 0 -#endif - -/* There are three string matching functions supplied: one fast, one - * slow (default), and one soft-configurable. To disable any of - * these, use the following definitions. */ -#ifndef XD3_BUILD_SLOW -#define XD3_BUILD_SLOW 1 -#endif -#ifndef XD3_BUILD_FAST -#define XD3_BUILD_FAST 1 -#endif -#ifndef XD3_BUILD_FASTER -#define XD3_BUILD_FASTER 1 -#endif -#ifndef XD3_BUILD_FASTEST -#define XD3_BUILD_FASTEST 1 -#endif -#ifndef XD3_BUILD_SOFT -#define XD3_BUILD_SOFT 1 -#endif -#ifndef XD3_BUILD_DEFAULT -#define XD3_BUILD_DEFAULT 1 -#endif - -#if XD3_DEBUG -#include -#endif - -/* XPRINT. Debug output and VCDIFF_TOOLS functions report to stderr. - * I have used an irregular style to abbreviate [fprintf(stderr, "] as - * [DP(RINT "]. */ -#define DP fprintf -#define RINT stderr, - -typedef struct _xd3_stream xd3_stream; -typedef struct _xd3_source xd3_source; -typedef struct _xd3_hash_cfg xd3_hash_cfg; -typedef struct _xd3_smatcher xd3_smatcher; -typedef struct _xd3_rinst xd3_rinst; -typedef struct _xd3_dinst xd3_dinst; -typedef struct _xd3_hinst xd3_hinst; -typedef struct _xd3_winst xd3_winst; -typedef struct _xd3_rpage xd3_rpage; -typedef struct _xd3_addr_cache xd3_addr_cache; -typedef struct _xd3_output xd3_output; -typedef struct _xd3_desect xd3_desect; -typedef struct _xd3_iopt_buflist xd3_iopt_buflist; -typedef struct _xd3_rlist xd3_rlist; -typedef struct _xd3_sec_type xd3_sec_type; -typedef struct _xd3_sec_cfg xd3_sec_cfg; -typedef struct _xd3_sec_stream xd3_sec_stream; -typedef struct _xd3_config xd3_config; -typedef struct _xd3_code_table_desc xd3_code_table_desc; -typedef struct _xd3_code_table_sizes xd3_code_table_sizes; -typedef struct _xd3_slist xd3_slist; -typedef struct _xd3_whole_state xd3_whole_state; -typedef struct _xd3_wininfo xd3_wininfo; - -/* The stream configuration has three callbacks functions, all of - * which may be supplied with NULL values. If config->getblk is - * provided as NULL, the stream returns XD3_GETSRCBLK. */ - -typedef void* (xd3_alloc_func) (void *opaque, - usize_t items, - usize_t size); -typedef void (xd3_free_func) (void *opaque, - void *address); - -typedef int (xd3_getblk_func) (xd3_stream *stream, - xd3_source *source, - xoff_t blkno); - -/* These are internal functions to delay construction of encoding - * tables and support alternate code tables. See the comments & code - * enabled by GENERIC_ENCODE_TABLES. */ - -typedef const xd3_dinst* (xd3_code_table_func) (void); -typedef int (xd3_comp_table_func) (xd3_stream *stream, - const uint8_t **data, - usize_t *size); - - - -#if XD3_DEBUG -#define XD3_ASSERT(x) \ - do { if (! (x)) { DP(RINT "%s:%d: XD3 assertion failed: %s\n", __FILE__, __LINE__, #x); \ - abort (); } } while (0) -#else -#define XD3_ASSERT(x) (void)0 -#endif /* XD3_DEBUG */ - -#ifndef max -#define max(x,y) ((x) < (y) ? (y) : (x)) -#endif -#ifndef min -#define min(x,y) ((x) < (y) ? (x) : (y)) -#endif - -/**************************************************************** - PUBLIC ENUMS - ******************************************************************/ - -/* These are the five ordinary status codes returned by the - * xd3_encode_input() and xd3_decode_input() state machines. */ -typedef enum { - - /* An application must be prepared to handle these five return - * values from either xd3_encode_input or xd3_decode_input, except - * in the case of no-source compression, in which case XD3_GETSRCBLK - * is never returned. More detailed comments for these are given in - * xd3_encode_input and xd3_decode_input comments, below. */ - XD3_INPUT = -17703, /* need input */ - XD3_OUTPUT = -17704, /* have output */ - XD3_GETSRCBLK = -17705, /* need a block of source input (with no - * xd3_getblk function), a chance to do - * non-blocking read. */ - XD3_GOTHEADER = -17706, /* (decode-only) after the initial VCDIFF & - first window header */ - XD3_WINSTART = -17707, /* notification: returned before a window is - * processed, giving a chance to - * XD3_SKIP_WINDOW or not XD3_SKIP_EMIT that - * window. */ - XD3_WINFINISH = -17708, /* notification: returned after - encode/decode & output for a window */ - XD3_TOOFARBACK = -17709, /* (encoder only) may be returned by - getblk() if the block is too old */ - XD3_INTERNAL = -17710, /* internal error */ - XD3_INVALID = -17711, /* invalid config */ - XD3_INVALID_INPUT = -17712, /* invalid input/decoder error */ - XD3_NOSECOND = -17713, /* when secondary compression finds no - improvement. */ - XD3_UNIMPLEMENTED = -17714 /* currently VCD_TARGET */ -} xd3_rvalues; - -/* special values in config->flags */ -typedef enum -{ - XD3_JUST_HDR = (1 << 1), /* used by VCDIFF tools, see - xdelta3-main.h. */ - XD3_SKIP_WINDOW = (1 << 2), /* used by VCDIFF tools, see - xdelta3-main.h. */ - XD3_SKIP_EMIT = (1 << 3), /* used by VCDIFF tools, see - xdelta3-main.h. */ - XD3_FLUSH = (1 << 4), /* flush the stream buffer to - prepare for - xd3_stream_close(). */ - - XD3_SEC_DJW = (1 << 5), /* use DJW static huffman */ - XD3_SEC_FGK = (1 << 6), /* use FGK adaptive huffman */ - XD3_SEC_LZMA = (1 << 24), /* use LZMA secondary */ - - XD3_SEC_TYPE = (XD3_SEC_DJW | XD3_SEC_FGK | XD3_SEC_LZMA), - - XD3_SEC_NODATA = (1 << 7), /* disable secondary compression of - the data section. */ - XD3_SEC_NOINST = (1 << 8), /* disable secondary compression of - the inst section. */ - XD3_SEC_NOADDR = (1 << 9), /* disable secondary compression of - the addr section. */ - - XD3_SEC_NOALL = (XD3_SEC_NODATA | XD3_SEC_NOINST | XD3_SEC_NOADDR), - - XD3_ADLER32 = (1 << 10), /* enable checksum computation in - the encoder. */ - XD3_ADLER32_NOVER = (1 << 11), /* disable checksum verification in - the decoder. */ - - XD3_ALT_CODE_TABLE = (1 << 12), /* for testing th - e alternate code table encoding. */ - - XD3_NOCOMPRESS = (1 << 13), /* disable ordinary data - * compression feature, only search - * the source, not the target. */ - XD3_BEGREEDY = (1 << 14), /* disable the "1.5-pass - * algorithm", instead use greedy - * matching. Greedy is off by - * default. */ - XD3_ADLER32_RECODE = (1 << 15), /* used by "recode". */ - - /* 4 bits to set the compression level the same as the command-line - * setting -1 through -9 (-0 corresponds to the XD3_NOCOMPRESS flag, - * and is independent of compression level). This is for - * convenience, especially with xd3_encode_memory(). */ - - XD3_COMPLEVEL_SHIFT = 20, /* 20 - 23 */ - XD3_COMPLEVEL_MASK = (0xF << XD3_COMPLEVEL_SHIFT), - XD3_COMPLEVEL_1 = (1 << XD3_COMPLEVEL_SHIFT), - XD3_COMPLEVEL_2 = (2 << XD3_COMPLEVEL_SHIFT), - XD3_COMPLEVEL_3 = (3 << XD3_COMPLEVEL_SHIFT), - XD3_COMPLEVEL_6 = (6 << XD3_COMPLEVEL_SHIFT), - XD3_COMPLEVEL_9 = (9 << XD3_COMPLEVEL_SHIFT) - -} xd3_flags; - -/* The values of this enumeration are set in xd3_config using the - * smatch_cfg variable. It can be set to default, slow, fast, etc., - * and soft. */ -typedef enum -{ - XD3_SMATCH_DEFAULT = 0, /* Flags may contain XD3_COMPLEVEL bits, - else default. */ - XD3_SMATCH_SLOW = 1, - XD3_SMATCH_FAST = 2, - XD3_SMATCH_FASTER = 3, - XD3_SMATCH_FASTEST = 4, - XD3_SMATCH_SOFT = 5 -} xd3_smatch_cfg; - -/********************************************************************* - PRIVATE ENUMS -**********************************************************************/ - -/* stream->match_state is part of the xd3_encode_input state machine - * for source matching: - * - * 1. the XD3_GETSRCBLK block-read mechanism means reentrant matching - * 2. this state spans encoder windows: a match and end-of-window - * will continue in the next 3. the initial target byte and source - * byte are a presumed match, to avoid some computation in case the - * inputs are identical. - */ -typedef enum { - - MATCH_TARGET = 0, /* in this state, attempt to match the start of - * the target with the previously set source - * address (initially 0). */ - MATCH_BACKWARD = 1, /* currently expanding a match backward in the - source/target. */ - MATCH_FORWARD = 2, /* currently expanding a match forward in the - source/target. */ - MATCH_SEARCHING = 3 /* currently searching for a match. */ - -} xd3_match_state; - -/* The xd3_encode_input state machine steps through these states in - * the following order. The matcher is reentrant and returns - * XD3_INPUT whenever it requires more data. After receiving - * XD3_INPUT, if the application reads EOF it should call - * xd3_stream_close(). - */ -typedef enum { - - ENC_INIT = 0, /* xd3_encode_input has never been called. */ - ENC_INPUT = 1, /* waiting for xd3_avail_input () to be called. */ - ENC_SEARCH = 2, /* currently searching for matches. */ - ENC_INSTR = 3, /* currently formatting output. */ - ENC_FLUSH = 4, /* currently emitting output. */ - ENC_POSTOUT = 5, /* after an output section. */ - ENC_POSTWIN = 6, /* after all output sections. */ - ENC_ABORTED = 7 /* abort. */ -} xd3_encode_state; - -/* The xd3_decode_input state machine steps through these states in - * the following order. The matcher is reentrant and returns - * XD3_INPUT whenever it requires more data. After receiving - * XD3_INPUT, if the application reads EOF it should call - * xd3_stream_close(). - * - * 0-8: the VCDIFF header - * 9-18: the VCDIFF window header - * 19-21: the three primary sections: data, inst, addr - * 22: producing output: returns XD3_OUTPUT, possibly XD3_GETSRCBLK, - * 23: return XD3_WINFINISH, set state=9 to decode more input - */ -typedef enum { - - DEC_VCHEAD = 0, /* VCDIFF header */ - DEC_HDRIND = 1, /* header indicator */ - - DEC_SECONDID = 2, /* secondary compressor ID */ - - DEC_TABLEN = 3, /* code table length */ - DEC_NEAR = 4, /* code table near */ - DEC_SAME = 5, /* code table same */ - DEC_TABDAT = 6, /* code table data */ - - DEC_APPLEN = 7, /* application data length */ - DEC_APPDAT = 8, /* application data */ - - DEC_WININD = 9, /* window indicator */ - - DEC_CPYLEN = 10, /* copy window length */ - DEC_CPYOFF = 11, /* copy window offset */ - - DEC_ENCLEN = 12, /* length of delta encoding */ - DEC_TGTLEN = 13, /* length of target window */ - DEC_DELIND = 14, /* delta indicator */ - - DEC_DATALEN = 15, /* length of ADD+RUN data */ - DEC_INSTLEN = 16, /* length of instruction data */ - DEC_ADDRLEN = 17, /* length of address data */ - - DEC_CKSUM = 18, /* window checksum */ - - DEC_DATA = 19, /* data section */ - DEC_INST = 20, /* instruction section */ - DEC_ADDR = 21, /* address section */ - - DEC_EMIT = 22, /* producing data */ - - DEC_FINISH = 23, /* window finished */ - - DEC_ABORTED = 24 /* xd3_abort_stream */ -} xd3_decode_state; - -/************************************************************ - internal types - ************************************************************/ - -/* instruction lists used in the IOPT buffer */ -struct _xd3_rlist -{ - xd3_rlist *next; - xd3_rlist *prev; -}; - -/* the raw encoding of an instruction used in the IOPT buffer */ -struct _xd3_rinst -{ - uint8_t type; - uint8_t xtra; - uint8_t code1; - uint8_t code2; - usize_t pos; - usize_t size; - xoff_t addr; - xd3_rlist link; -}; - -/* the code-table form of an single- or double-instruction */ -struct _xd3_dinst -{ - uint8_t type1; - uint8_t size1; - uint8_t type2; - uint8_t size2; -}; - -/* the decoded form of a single (half) instruction. */ -struct _xd3_hinst -{ - uint8_t type; - uint32_t size; /* TODO: why decode breaks if this is usize_t? */ - uint32_t addr; /* TODO: why decode breaks if this is usize_t? */ -}; - -/* the form of a whole-file instruction */ -struct _xd3_winst -{ - uint8_t type; /* RUN, ADD, COPY */ - uint8_t mode; /* 0, VCD_SOURCE, VCD_TARGET */ - usize_t size; - xoff_t addr; - xoff_t position; /* absolute position of this inst */ -}; - -/* used by the encoder to buffer output in sections. list of blocks. */ -struct _xd3_output -{ - uint8_t *base; - usize_t next; - usize_t avail; - xd3_output *next_page; -}; - -/* used by the decoder to buffer input in sections. */ -struct _xd3_desect -{ - const uint8_t *buf; - const uint8_t *buf_max; - uint32_t size; /* TODO: why decode breaks if this is usize_t? */ - usize_t pos; - - /* used in xdelta3-decode.h */ - uint8_t *copied1; - usize_t alloc1; - - /* used in xdelta3-second.h */ - uint8_t *copied2; - usize_t alloc2; -}; - -/* the VCDIFF address cache, see the RFC */ -struct _xd3_addr_cache -{ - usize_t s_near; - usize_t s_same; - usize_t next_slot; /* the circular index for near */ - usize_t *near_array; /* array of size s_near */ - usize_t *same_array; /* array of size s_same*256 */ -}; - -/* the IOPT buffer list is just a list of buffers, which may be allocated - * during encode when using an unlimited buffer. */ -struct _xd3_iopt_buflist -{ - xd3_rinst *buffer; - xd3_iopt_buflist *next; -}; - -/* This is the record of a pre-compiled configuration, a subset of - xd3_config. */ -struct _xd3_smatcher -{ - const char *name; - int (*string_match) (xd3_stream *stream); - usize_t large_look; - usize_t large_step; - usize_t small_look; - usize_t small_chain; - usize_t small_lchain; - usize_t max_lazy; - usize_t long_enough; -}; - -/* hash table size & power-of-two hash function. */ -struct _xd3_hash_cfg -{ - usize_t size; - usize_t shift; - usize_t mask; -}; - -/* the sprev list */ -struct _xd3_slist -{ - usize_t last_pos; -}; - -/* window info (for whole state) */ -struct _xd3_wininfo { - xoff_t offset; - usize_t length; - uint32_t adler32; -}; - -/* whole state for, e.g., merge */ -struct _xd3_whole_state { - usize_t addslen; - uint8_t *adds; - usize_t adds_alloc; - - usize_t instlen; - xd3_winst *inst; - usize_t inst_alloc; - - usize_t wininfolen; - xd3_wininfo *wininfo; - usize_t wininfo_alloc; - - xoff_t length; -}; - -/******************************************************************** - public types - *******************************************************************/ - -/* Settings for the secondary compressor. */ -struct _xd3_sec_cfg -{ - int data_type; /* Which section. (set automatically) */ - usize_t ngroups; /* Number of DJW Huffman groups. */ - usize_t sector_size; /* Sector size. */ - int inefficient; /* If true, ignore efficiency check [avoid XD3_NOSECOND]. */ -}; - -/* This is the user-visible stream configuration. */ -struct _xd3_config -{ - usize_t winsize; /* The encoder window size. */ - usize_t sprevsz; /* How far back small string - matching goes */ - usize_t iopt_size; /* entries in the - instruction-optimizing - buffer */ - usize_t srcwin_maxsz; /* srcwin_size grows by a factor - of 2 when no matches are - found. encoder will not seek - back further than this. */ - - xd3_getblk_func *getblk; /* The three callbacks. */ - xd3_alloc_func *alloc; - xd3_free_func *freef; - void *opaque; /* Not used. */ - int flags; /* stream->flags are initialized - * from xd3_config & never - * modified by the library. Use - * xd3_set_flags to modify flags - * settings mid-stream. */ - - xd3_sec_cfg sec_data; /* Secondary compressor config: data */ - xd3_sec_cfg sec_inst; /* Secondary compressor config: inst */ - xd3_sec_cfg sec_addr; /* Secondary compressor config: addr */ - - xd3_smatch_cfg smatch_cfg; /* See enum: use fields below for - soft config */ - xd3_smatcher smatcher_soft; -}; - -/* The primary source file object. You create one of these objects and - * initialize the first four fields. This library maintains the next - * 5 fields. The configured getblk implementation is responsible for - * setting the final 3 fields when called (and/or when XD3_GETSRCBLK - * is returned). - */ -struct _xd3_source -{ - /* you set */ - usize_t blksize; /* block size */ - const char *name; /* its name, for debug/print - purposes */ - void *ioh; /* opaque handle */ - - /* getblk sets */ - xoff_t curblkno; /* current block number: client - sets after getblk request */ - usize_t onblk; /* number of bytes on current - block: client sets, must be >= 0 - and <= blksize */ - const uint8_t *curblk; /* current block array: client - sets after getblk request */ - - /* xd3 sets */ - usize_t srclen; /* length of this source window */ - xoff_t srcbase; /* offset of this source window - in the source itself */ - int shiftby; /* for power-of-two blocksizes */ - int maskby; /* for power-of-two blocksizes */ - xoff_t cpyoff_blocks; /* offset of dec_cpyoff in blocks */ - usize_t cpyoff_blkoff; /* offset of copy window in - blocks, remainder */ - xoff_t getblkno; /* request block number: xd3 sets - current getblk request */ - - /* See xd3_getblk() */ - xoff_t max_blkno; /* Maximum block, if eof is known, - * otherwise, equals frontier_blkno - * (initially 0). */ - xoff_t frontier_blkno; /* If eof is unknown, the next - * source position to be read. - * Otherwise, equal to - * max_blkno. */ - usize_t onlastblk; /* Number of bytes on max_blkno */ - int eof_known; /* Set to true when the first - * partial block is read. */ -}; - -/* The primary xd3_stream object, used for encoding and decoding. You - * may access only two fields: avail_out, next_out. Use the methods - * above to operate on xd3_stream. */ -struct _xd3_stream -{ - /* input state */ - const uint8_t *next_in; /* next input byte */ - usize_t avail_in; /* number of bytes available at - next_in */ - xoff_t total_in; /* how many bytes in */ - - /* output state */ - uint8_t *next_out; /* next output byte */ - usize_t avail_out; /* number of bytes available at - next_out */ - usize_t space_out; /* total out space */ - xoff_t current_window; /* number of windows encoded/decoded */ - xoff_t total_out; /* how many bytes out */ - - /* to indicate an error, xd3 sets */ - const char *msg; /* last error message, NULL if - no error */ - - /* source configuration */ - xd3_source *src; /* source array */ - - /* encoder memory configuration */ - usize_t winsize; /* suggested window size */ - usize_t sprevsz; /* small string, previous window - size (power of 2) */ - usize_t sprevmask; /* small string, previous window - size mask */ - usize_t iopt_size; - usize_t iopt_unlimited; - usize_t srcwin_maxsz; - - /* general configuration */ - xd3_getblk_func *getblk; /* set nxtblk, nxtblkno to scanblkno */ - xd3_alloc_func *alloc; /* malloc function */ - xd3_free_func *free; /* free function */ - void* opaque; /* private data object passed to - alloc, free, and getblk */ - int flags; /* various options */ - - /* secondary compressor configuration */ - xd3_sec_cfg sec_data; /* Secondary compressor config: data */ - xd3_sec_cfg sec_inst; /* Secondary compressor config: inst */ - xd3_sec_cfg sec_addr; /* Secondary compressor config: addr */ - - xd3_smatcher smatcher; - - usize_t *large_table; /* table of large checksums */ - xd3_hash_cfg large_hash; /* large hash config */ - - usize_t *small_table; /* table of small checksums */ - xd3_slist *small_prev; /* table of previous offsets, - circular linked list */ - int small_reset; /* true if small table should - be reset */ - - xd3_hash_cfg small_hash; /* small hash config */ - xd3_addr_cache acache; /* the vcdiff address cache */ - xd3_encode_state enc_state; /* state of the encoder */ - - usize_t taroff; /* base offset of the target input */ - usize_t input_position; /* current input position */ - usize_t min_match; /* current minimum match - length, avoids redundent - matches */ - usize_t unencoded_offset; /* current input, first - * unencoded offset. this value - * is <= the first instruction's - * position in the iopt buffer, - * if there is at least one - * match in the buffer. */ - - /* SRCWIN: these variables plus srcwin_maxsz above (set by config) */ - int srcwin_decided; /* boolean: true if srclen and - srcbase have been - decided. */ - int srcwin_decided_early; /* boolean: true if srclen - and srcbase were - decided early. */ - xoff_t srcwin_cksum_pos; /* Source checksum position */ - - /* MATCH */ - xd3_match_state match_state; /* encoder match state */ - xoff_t match_srcpos; /* current match source - position relative to - srcbase */ - xoff_t match_last_srcpos; /* previously attempted - * srcpos, to avoid loops. */ - xoff_t match_minaddr; /* smallest matching address to - * set window params (reset each - * window xd3_encode_reset) */ - xoff_t match_maxaddr; /* largest matching address to - * set window params (reset each - * window xd3_encode_reset) */ - usize_t match_back; /* match extends back so far */ - usize_t match_maxback; /* match extends back maximum */ - usize_t match_fwd; /* match extends forward so far */ - usize_t match_maxfwd; /* match extends forward maximum */ - - xoff_t maxsrcaddr; /* address of the last source - match (across windows) */ - - uint8_t *buf_in; /* for saving buffered input */ - usize_t buf_avail; /* amount of saved input */ - const uint8_t *buf_leftover; /* leftover content of next_in - (i.e., user's buffer) */ - usize_t buf_leftavail; /* amount of leftover content */ - - xd3_output *enc_current; /* current output buffer */ - xd3_output *enc_free; /* free output buffers */ - xd3_output *enc_heads[4]; /* array of encoded outputs: - head of chain */ - xd3_output *enc_tails[4]; /* array of encoded outputs: - tail of chain */ - uint32_t recode_adler32; /* set the adler32 checksum - * during "recode". */ - - xd3_rlist iopt_used; /* instruction optimizing buffer */ - xd3_rlist iopt_free; - xd3_rinst *iout; /* next single instruction */ - xd3_iopt_buflist *iopt_alloc; - - const uint8_t *enc_appheader; /* application header to encode */ - usize_t enc_appheadsz; /* application header size */ - - /* decoder stuff */ - xd3_decode_state dec_state; /* current DEC_XXX value */ - usize_t dec_hdr_ind; /* VCDIFF header indicator */ - usize_t dec_win_ind; /* VCDIFF window indicator */ - usize_t dec_del_ind; /* VCDIFF delta indicator */ - - uint8_t dec_magic[4]; /* First four bytes */ - usize_t dec_magicbytes; /* Magic position. */ - - usize_t dec_secondid; /* Optional secondary compressor ID. */ - - /* TODO: why decode breaks if this is usize_t? */ - uint32_t dec_codetblsz; /* Optional code table: length. */ - uint8_t *dec_codetbl; /* Optional code table: storage. */ - usize_t dec_codetblbytes; /* Optional code table: position. */ - - /* TODO: why decode breaks if this is usize_t? */ - uint32_t dec_appheadsz; /* Optional application header: - size. */ - uint8_t *dec_appheader; /* Optional application header: - storage */ - usize_t dec_appheadbytes; /* Optional application header: - position. */ - - usize_t dec_cksumbytes; /* Optional checksum: position. */ - uint8_t dec_cksum[4]; /* Optional checksum: storage. */ - uint32_t dec_adler32; /* Optional checksum: value. */ - - /* TODO: why decode breaks if this is usize_t? */ - uint32_t dec_cpylen; /* length of copy window - (VCD_SOURCE or VCD_TARGET) */ - xoff_t dec_cpyoff; /* offset of copy window - (VCD_SOURCE or VCD_TARGET) */ - /* TODO: why decode breaks if this is usize_t? */ - uint32_t dec_enclen; /* length of delta encoding */ - /* TODO: why decode breaks if this is usize_t? */ - uint32_t dec_tgtlen; /* length of target window */ - -#if USE_UINT64 - uint64_t dec_64part; /* part of a decoded uint64_t */ -#endif -#if USE_UINT32 - uint32_t dec_32part; /* part of a decoded uint32_t */ -#endif - - xoff_t dec_winstart; /* offset of the start of - current target window */ - xoff_t dec_window_count; /* == current_window + 1 in - DEC_FINISH */ - usize_t dec_winbytes; /* bytes of the three sections - so far consumed */ - usize_t dec_hdrsize; /* VCDIFF + app header size */ - - const uint8_t *dec_tgtaddrbase; /* Base of decoded target - addresses (addr >= - dec_cpylen). */ - const uint8_t *dec_cpyaddrbase; /* Base of decoded copy - addresses (addr < - dec_cpylen). */ - - usize_t dec_position; /* current decoder position - counting the cpylen - offset */ - usize_t dec_maxpos; /* maximum decoder position - counting the cpylen - offset */ - xd3_hinst dec_current1; /* current instruction */ - xd3_hinst dec_current2; /* current instruction */ - - uint8_t *dec_buffer; /* Decode buffer */ - uint8_t *dec_lastwin; /* In case of VCD_TARGET, the - last target window. */ - usize_t dec_lastlen; /* length of the last target - window */ - xoff_t dec_laststart; /* offset of the start of last - target window */ - usize_t dec_lastspace; /* allocated space of last - target window, for reuse */ - - xd3_desect inst_sect; /* staging area for decoding - window sections */ - xd3_desect addr_sect; - xd3_desect data_sect; - - xd3_code_table_func *code_table_func; - xd3_comp_table_func *comp_table_func; - const xd3_dinst *code_table; - const xd3_code_table_desc *code_table_desc; - xd3_dinst *code_table_alloc; - - /* secondary compression */ - const xd3_sec_type *sec_type; - xd3_sec_stream *sec_stream_d; - xd3_sec_stream *sec_stream_i; - xd3_sec_stream *sec_stream_a; - - /* state for reconstructing whole files (e.g., for merge), this only - * supports loading USIZE_T_MAX instructions, adds, etc. */ - xd3_whole_state whole_target; - - /* statistics */ - xoff_t n_scpy; - xoff_t n_tcpy; - xoff_t n_add; - xoff_t n_run; - - xoff_t l_scpy; - xoff_t l_tcpy; - xoff_t l_add; - xoff_t l_run; - - usize_t i_slots_used; - -#if XD3_DEBUG - usize_t large_ckcnt; - - /* memory usage */ - usize_t alloc_cnt; - usize_t free_cnt; -#endif -}; - -/************************************************************************** - PUBLIC FUNCTIONS - **************************************************************************/ - -/* This function configures an xd3_stream using the provided in-memory - * input buffer, source buffer, output buffer, and flags. The output - * array must be large enough or else ENOSPC will be returned. This - * is the simplest in-memory encoding interface. */ -int xd3_encode_memory (const uint8_t *input, - usize_t input_size, - const uint8_t *source, - usize_t source_size, - uint8_t *output_buffer, - usize_t *output_size, - usize_t avail_output, - int flags); - -/* The reverse of xd3_encode_memory. */ -int xd3_decode_memory (const uint8_t *input, - usize_t input_size, - const uint8_t *source, - usize_t source_size, - uint8_t *output_buf, - usize_t *output_size, - usize_t avail_output, - int flags); - -/* This function encodes an in-memory input using a pre-configured - * xd3_stream. This allows the caller to set a variety of options - * which are not available in the xd3_encode/decode_memory() - * functions. - * - * The output array must be large enough to hold the output or else - * ENOSPC is returned. The source (if any) should be set using - * xd3_set_source_and_size() with a single-block xd3_source. This - * calls the underlying non-blocking interfaces, - * xd3_encode/decode_input(), handling the necessary input/output - * states. This method may be considered a reference for any - * application using xd3_encode_input() directly. - * - * xd3_stream stream; - * xd3_config config; - * xd3_source src; - * - * memset (& src, 0, sizeof (src)); - * memset (& stream, 0, sizeof (stream)); - * memset (& config, 0, sizeof (config)); - * - * if (source != NULL) - * { - * src.size = source_size; - * src.blksize = source_size; - * src.curblkno = 0; - * src.onblk = source_size; - * src.curblk = source; - * xd3_set_source(&stream, &src); - * } - * - * config.flags = flags; - * config.srcwin_maxsz = source_size; - * config.winsize = input_size; - * - * ... set smatcher, appheader, encoding-table, compression-level, etc. - * - * xd3_config_stream(&stream, &config); - * xd3_encode_stream(&stream, ...); - * xd3_free_stream(&stream); - */ -int xd3_encode_stream (xd3_stream *stream, - const uint8_t *input, - usize_t input_size, - uint8_t *output, - usize_t *output_size, - usize_t avail_output); - -/* The reverse of xd3_encode_stream. */ -int xd3_decode_stream (xd3_stream *stream, - const uint8_t *input, - usize_t input_size, - uint8_t *output, - usize_t *output_size, - usize_t avail_size); - -/* This is the non-blocking interface. - * - * Handling input and output states is the same for encoding or - * decoding using the xd3_avail_input() and xd3_consume_output() - * routines, inlined below. - * - * Return values: - * - * XD3_INPUT: the process requires more input: call - * xd3_avail_input() then repeat - * - * XD3_OUTPUT: the process has more output: read stream->next_out, - * stream->avail_out, then call xd3_consume_output(), - * then repeat - * - * XD3_GOTHEADER: (decoder-only) notification returned following the - * VCDIFF header and first window header. the decoder - * may use the header to configure itself. - * - * XD3_WINSTART: a general notification returned once for each - * window except the 0-th window, which is implied by - * XD3_GOTHEADER. It is recommended to use a - * switch-stmt such as: - * - * ... - * again: - * switch ((ret = xd3_decode_input (stream))) { - * case XD3_GOTHEADER: { - * assert(stream->current_window == 0); - * stuff; - * } - * // fallthrough - * case XD3_WINSTART: { - * something(stream->current_window); - * goto again; - * } - * ... - * - * XD3_WINFINISH: a general notification, following the complete - * input & output of a window. at this point, - * stream->total_in and stream->total_out are consistent - * for either encoding or decoding. - * - * XD3_GETSRCBLK: If the xd3_getblk() callback is NULL, this value - * is returned to initiate a non-blocking source read. - */ -int xd3_decode_input (xd3_stream *stream); -int xd3_encode_input (xd3_stream *stream); - -/* The xd3_config structure is used to initialize a stream - all data - * is copied into stream so config may be a temporary variable. See - * the [documentation] or comments on the xd3_config structure. */ -int xd3_config_stream (xd3_stream *stream, - xd3_config *config); - -/* Since Xdelta3 doesn't open any files, xd3_close_stream is just an - * error check that the stream is in a proper state to be closed: this - * means the encoder is flushed and the decoder is at a window - * boundary. The application is responsible for freeing any of the - * resources it supplied. */ -int xd3_close_stream (xd3_stream *stream); - -/* This arranges for closes the stream to succeed. Does not free the - * stream.*/ -void xd3_abort_stream (xd3_stream *stream); - -/* xd3_free_stream frees all memory allocated for the stream. The - * application is responsible for freeing any of the resources it - * supplied. */ -void xd3_free_stream (xd3_stream *stream); - -/* This function informs the encoder or decoder that source matching - * (i.e., delta-compression) is possible. For encoding, this should - * be called before the first xd3_encode_input. A NULL source is - * ignored. For decoding, this should be called before the first - * window is decoded, but the appheader may be read first - * (XD3_GOTHEADER). After decoding the header, call xd3_set_source() - * if you have a source file. Note: if (stream->dec_win_ind & VCD_SOURCE) - * is true, it means the first window expects there to be a source file. - */ -int xd3_set_source (xd3_stream *stream, - xd3_source *source); - -/* If the source size is known, call this instead of xd3_set_source(). - * to avoid having stream->getblk called (and/or to avoid XD3_GETSRCBLK). - * - * Follow these steps: - xd3_source source; - memset(&source, 0, sizeof(source)); - source.blksize = size; - source.onblk = size; - source.curblk = buf; - source.curblkno = 0; - int ret = xd3_set_source_and_size(&stream, &source, size); - ... - */ -int xd3_set_source_and_size (xd3_stream *stream, - xd3_source *source, - xoff_t source_size); - -/* This should be called before the first call to xd3_encode_input() - * to include application-specific data in the VCDIFF header. */ -void xd3_set_appheader (xd3_stream *stream, - const uint8_t *data, - usize_t size); - -/* xd3_get_appheader may be called in the decoder after XD3_GOTHEADER. - * For convenience, the decoder always adds a single byte padding to - * the end of the application header, which is set to zero in case the - * application header is a string. */ -int xd3_get_appheader (xd3_stream *stream, - uint8_t **data, - usize_t *size); - -/* To generate a VCDIFF encoded delta with xd3_encode_init() from - * another format, use: - * - * xd3_encode_init_partial() -- initialze encoder state (w/o hash tables) - * xd3_init_cache() -- reset VCDIFF address cache - * xd3_found_match() -- to report a copy instruction - * - * set stream->enc_state to ENC_INSTR and call xd3_encode_input as usual. - */ -int xd3_encode_init_partial (xd3_stream *stream); -void xd3_init_cache (xd3_addr_cache* acache); -int xd3_found_match (xd3_stream *stream, - usize_t pos, usize_t size, - xoff_t addr, int is_source); - -/* Gives an error string for xdelta3-speficic errors, returns NULL for - system errors */ -const char* xd3_strerror (int ret); - -/* For convenience, zero & initialize the xd3_config structure with - specified flags. */ -static inline -void xd3_init_config (xd3_config *config, - int flags) -{ - memset (config, 0, sizeof (*config)); - config->flags = flags; -} - -/* This supplies some input to the stream. - * - * For encoding, if the input is larger than the configured window - * size (xd3_config.winsize), the entire input will be consumed and - * encoded anyway. If you wish to strictly limit the window size, - * limit the buffer passed to xd3_avail_input to the window size. - * - * For encoding, if the input is smaller than the configured window - * size (xd3_config.winsize), the library will create a window-sized - * buffer and accumulate input until a full-sized window can be - * encoded. XD3_INPUT will be returned. The input must remain valid - * until the next time xd3_encode_input() returns XD3_INPUT. - * - * For decoding, the input will be consumed entirely before XD3_INPUT - * is returned again. - */ -static inline -void xd3_avail_input (xd3_stream *stream, - const uint8_t *idata, - usize_t isize) -{ - /* Even if isize is zero, the code expects a non-NULL idata. Why? - * It uses this value to determine whether xd3_avail_input has ever - * been called. If xd3_encode_input is called before - * xd3_avail_input it will return XD3_INPUT right away without - * allocating a stream->winsize buffer. This is to avoid an - * unwanted allocation. */ - XD3_ASSERT (idata != NULL || isize == 0); - - stream->next_in = idata; - stream->avail_in = isize; -} - -/* This acknowledges receipt of output data, must be called after any - * XD3_OUTPUT return. */ -static inline -void xd3_consume_output (xd3_stream *stream) -{ - stream->avail_out = 0; -} - -/* These are set for each XD3_WINFINISH return. */ -static inline -int xd3_encoder_used_source (xd3_stream *stream) { - return stream->src != NULL && stream->src->srclen > 0; -} -static inline -xoff_t xd3_encoder_srcbase (xd3_stream *stream) { - return stream->src->srcbase; -} -static inline -usize_t xd3_encoder_srclen (xd3_stream *stream) { - return stream->src->srclen; -} - -/* Checks for legal flag changes. */ -static inline -void xd3_set_flags (xd3_stream *stream, int flags) -{ - /* The bitwise difference should contain only XD3_FLUSH or - XD3_SKIP_WINDOW */ - XD3_ASSERT(((flags ^ stream->flags) & ~(XD3_FLUSH | XD3_SKIP_WINDOW)) == 0); - stream->flags = flags; -} - -/* Gives some extra information about the latest library error, if any - * is known. */ -static inline -const char* xd3_errstring (xd3_stream *stream) -{ - return stream->msg ? stream->msg : ""; -} - - -/* 64-bit divisions are expensive, which is why we require a - * power-of-two source->blksize. To relax this restriction is - * relatively easy, see the history for xd3_blksize_div(). */ -static inline -void xd3_blksize_div (const xoff_t offset, - const xd3_source *source, - xoff_t *blkno, - usize_t *blkoff) { - *blkno = (xoff_t) (offset >> source->shiftby); - *blkoff = (usize_t) (offset & source->maskby); - XD3_ASSERT (*blkoff < source->blksize); -} - -static inline -void xd3_blksize_add (xoff_t *blkno, - usize_t *blkoff, - const xd3_source *source, - const usize_t add) -{ - usize_t blkdiff; - - /* Does not check for overflow, checked in xdelta3-decode.h. */ - *blkoff += add; - blkdiff = *blkoff >> source->shiftby; - - if (blkdiff) - { - *blkno += blkdiff; - *blkoff &= source->maskby; - } - - XD3_ASSERT (*blkoff < source->blksize); -} - -#define XD3_NOOP 0U -#define XD3_ADD 1U -#define XD3_RUN 2U -#define XD3_CPY 3U /* XD3_CPY rtypes are represented as (XD3_CPY + - * copy-mode value) */ - -#if XD3_DEBUG -#define IF_DEBUG(x) x -#else -#define IF_DEBUG(x) -#endif -#if XD3_DEBUG > 1 -#define IF_DEBUG1(x) x -#else -#define IF_DEBUG1(x) -#endif -#if XD3_DEBUG > 2 -#define IF_DEBUG2(x) x -#else -#define IF_DEBUG2(x) -#endif - -#define SIZEOF_ARRAY(x) (sizeof(x) / sizeof(x[0])) - -#endif /* _XDELTA3_H_ */