added the new client HTTP download system

changelog.txt

@@ -1,6 +1,26 @@
 
 DD Mmm 17 - 1.49
 
+add: new CNQ3 download system that uses checksums to get the right pak files
+  the CNQ3 download system will use the WorldSpawn map server for inexact queries
+  (when loading demos or using /dlmap) when the CNQ3 map server doesn't have the file
+
+  system | speed | cases               | source(s)     | file availability | server requirements
+  -------|-------|---------------------|---------------|-------------------|--------------------
+  CNQ3   | fast  | pure servers, demos | map server(s) | not guaranteed    | none
+  id     | slow  | pure servers only   | the Q3 server | guaranteed        | sv_allowDownload 1
+
+chg: cl_allowDownload can be used to select the download system
+  cl_allowDownload  0 = downloads disabled
+  cl_allowDownload  1 (default) = CNQ3 download system
+  cl_allowDownload -1 = id's original download system
+
+add: new commands for manually initiating and canceling fast downloads:
+  dlpak  <checksum> to download a pak by its checksum if the pak doesn't exist locally
+  dlmap  <map_name> to download a map by name if no map with such a name exists locally
+  dlmapf <map_name> to force download a map by name
+  dlstop            to cancel the download in progress, if any
+
 fix: r_fullbright is no longer latched and actually does its job
 
 fix: r_lightmap is now archived and actually does its job

code/client/cl_cgame.cpp

@@ -280,7 +280,7 @@ rescan:
 
 static void CL_CM_LoadMap( const char* mapname )
 {
-	int checksum;
+	unsigned checksum;
 	CM_LoadMap( mapname, qtrue, &checksum );
 }
 
@@ -290,6 +290,7 @@ void CL_ShutdownCGame()
 	cls.keyCatchers &= ~KEYCATCH_CGAME;
 	cls.cgameStarted = qfalse;
 	cls.cgameForwardInput = 0;
+	CL_MapDownload_Cancel();
 	if ( !cgvm ) {
 		return;
 	}

code/client/cl_console.cpp

@@ -591,6 +591,7 @@ static void Con_DrawSolidConsole( float frac )
 	}
 
 	Con_DrawInput();
+	CL_MapDownload_DrawConsole( con.cw, con.ch );
 
 	re.SetColor( NULL );
 }

code/client/cl_download.cpp

@@ -0,0 +1,912 @@
+#include "client.h"
+#include <stdio.h>
+#include <stdarg.h>
+#if defined(_WIN32)
+#include <io.h>
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#if defined(_WIN32)
+#include <winsock2.h>
+#include <Windows.h>
+#include <ws2tcpip.h>
+#else
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <netdb.h>
+#include <string.h>
+#endif
+
+#if defined(_WIN32)
+typedef ADDRINFOA		addrInfo_t;
+#define Q_closesocket	closesocket
+#define Q_unlink		_unlink
+#else
+typedef addrinfo		addrInfo_t;
+typedef int				SOCKET;
+#define INVALID_SOCKET	(-1)
+#define SOCKET_ERROR	(-1)
+#define MAX_PATH		(256)
+#define Q_closesocket	close
+#define Q_unlink		unlink
+#endif
+
+
+struct mapDownload_t {
+	char recBuffer[1 << 20];		// for both download data and checksumming
+	char tempMessage[MAXPRINTMSG];	// for PrintError
+	char tempMessage2[MAXPRINTMSG];	// for PrintSocketError
+	char errorMessage[MAXPRINTMSG];
+	char mapName[MAX_PATH];		// only used if the server doesn't give us a .pk3 name
+	char tempPath[MAX_PATH];	// full path of the temp file being written to
+	char finalName[MAX_PATH];	// file name with extension suggested by the server
+	char finalPath[MAX_PATH];	// full path of the new .pk3 file
+	char httpHeaderValue[128];	// only set when BadResponse is qtrue
+	SOCKET socket;
+	FILE* file;
+	int startTimeMS;
+	unsigned int bytesHeader;		// header only
+	unsigned int bytesContent;		// file content only
+	unsigned int bytesTotal;		// message header + file content
+	unsigned int bytesDownloaded;	// total downloaded, including the header
+	unsigned int crc32;
+	qbool fromCommand;		// qtrue if started by a console command
+	qbool headerParsed;		// qtrue if we're done parsing the header
+	qbool badResponse;		// qtrue if we need to read more packets for the custom error message
+	int timeOutStartTimeMS;	// when the recv timeout started
+	qbool lastErrorTimeOut;	// qtrue if the last recv error was a timeout
+	int sourceIndex;		// index into the cl_mapDLSources array
+	qbool exactMatch;		// qtrue if an exact match is required
+};
+
+
+enum mapDownloadStatus_t {
+	MDLS_NOTHING,
+	MDLS_ERROR,		// just finished unsuccessfully
+	MDLS_SUCCESS,	// just finished successfully
+	MDLS_IN_PROGRESS,
+	MDLS_COUNT
+};
+
+
+typedef void (*mapdlQueryFormatter_t)( char* query, int querySize, const char* mapName );
+
+
+// map download source for queries by map name only
+struct mapDownloadSource_t {
+	const char* name;
+	const char* hostName; // don't put in the scheme (e.g. "http://")
+	int port;
+	mapdlQueryFormatter_t formatQuery;
+};
+
+
+static mapDownload_t cl_mapDL;
+
+
+static void FormatQueryCNQ3( char* query, int querySize, const char* mapName );
+static void FormatQueryWS( char* query, int querySize, const char* mapName );
+
+
+static const mapDownloadSource_t cl_mapDLSources[2] = {
+	{ "CNQ3",		"maps.playmorepromode.org",	8000,	&FormatQueryCNQ3 },
+	{ "WorldSpawn",	"ws.q3df.org",				80,		&FormatQueryWS }
+};
+
+
+static void FormatQueryCNQ3( char* query, int querySize, const char* mapName )
+{
+	Com_sprintf(query, querySize, "map?n=%s", mapName);
+}
+
+
+static void FormatQueryWS( char* query, int querySize, const char* mapName )
+{
+	Com_sprintf(query, querySize, "getpk3bymapname.php/%s", mapName);
+}
+
+
+static void PrintError( mapDownload_t* dl, const char* format, ... )
+{
+	va_list ap;
+	va_start(ap, format);
+	Q_vsnprintf(dl->tempMessage, sizeof(dl->tempMessage), format, ap);
+	va_end(ap);
+
+	Q_strncpyz(dl->errorMessage, "^bMap DL failed: ^7", sizeof(dl->errorMessage));
+	Q_strcat(dl->errorMessage, sizeof(dl->errorMessage), dl->tempMessage);
+	if (dl->errorMessage[strlen(dl->errorMessage) - 1] != '\n')
+		Q_strcat(dl->errorMessage, sizeof(dl->errorMessage), "\n");
+
+	Com_Printf(dl->errorMessage);
+}
+
+
+#if defined(_WIN32)
+static void PrintSocketError( mapDownload_t* dl, const char* functionName, int ec )
+{
+	const int bufferSize = sizeof(dl->tempMessage2);
+	const int bw = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+								  NULL, (DWORD)ec, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+								  dl->tempMessage2, (DWORD)bufferSize, NULL);
+	if (bw <= 0) {
+		*dl->tempMessage2 = '\0';
+		PrintError(dl, "%s failed: %d", functionName, ec);
+		return;
+	}
+
+	int lastByte = bw;
+	if (lastByte >= bufferSize)
+		lastByte = bufferSize - 1;
+	dl->tempMessage2[lastByte] = '\0';
+
+	PrintError(dl, "%s failed: %d -> %s", functionName, ec, dl->tempMessage2);
+}
+#else
+static void PrintSocketError( mapDownload_t* dl, const char* functionName, int ec )
+{
+#if (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE
+	// XSI strerror_r
+	const int serec = strerror_r(ec, dl->tempMessage2, sizeof(dl->tempMessage2));
+	const char* const errorMsg = dl->tempMessage2;
+	if (serec != 0) {
+#else
+	// GNU strerror_r
+	const char* const errorMsg = strerror_r(ec, dl->tempMessage2, sizeof(dl->tempMessage2));
+	if (errorMsg == NULL) {
+#endif
+		PrintError(dl, "%s failed with error %d", functionName, ec);
+		return;
+	}
+	
+	PrintError(dl, "%s failed with error %d (%s)", functionName, ec, errorMsg);
+}
+#endif
+
+
+static void PrintSocketError( mapDownload_t* dl, const char* functionName )
+{
+#if defined(_WIN32)
+	PrintSocketError(dl, functionName, WSAGetLastError());
+#else
+	PrintSocketError(dl, functionName, errno);
+#endif
+}
+
+
+static qbool IsSocketTimeoutError()
+{
+#if defined(_WIN32)
+	const int ec = WSAGetLastError();
+	return ec == WSAEWOULDBLOCK || ec == WSAETIMEDOUT;
+#else
+	const int ec = errno;
+	return ec == EAGAIN || ec == EWOULDBLOCK;
+#endif
+}
+
+
+static void Download_Clear( mapDownload_t* dl )
+{
+	*dl->tempPath = '\0';
+	*dl->finalName = '\0';
+	*dl->errorMessage = '\0';
+	dl->socket = INVALID_SOCKET;
+	dl->file = NULL;
+	dl->startTimeMS = INT_MIN;
+	dl->crc32 = 0;
+	dl->bytesHeader = 0;
+	dl->bytesTotal = 0;
+	dl->bytesDownloaded = 0;
+	dl->headerParsed = qfalse;
+	dl->badResponse = qfalse;
+	dl->timeOutStartTimeMS = INT_MIN;
+	dl->lastErrorTimeOut = qfalse;
+	dl->sourceIndex = 0;
+	dl->exactMatch = qfalse;
+}
+
+
+static qbool Download_IsFileValid( mapDownload_t* dl )
+{
+	const unsigned int fileSize = dl->bytesTotal - dl->bytesHeader;
+	if (fileSize == 0)
+		return qfalse;
+
+	FILE* const file = fopen(dl->tempPath, "rb");
+	if (file == NULL)
+		return qfalse;
+
+	const unsigned int maxBlockSize = (unsigned int)sizeof(dl->recBuffer);
+	const unsigned int fullBlockCount = fileSize / maxBlockSize;
+	const unsigned int lastBlockSize = fileSize - fullBlockCount * maxBlockSize;
+
+	unsigned int crc32;
+	CRC32_Begin(&crc32);
+	for (unsigned int i = 0; i < fullBlockCount; ++i) {
+		if (fread(dl->recBuffer, maxBlockSize, 1, file) != 1) {
+			fclose(file);
+			return qfalse;
+		}
+		CRC32_ProcessBlock(&crc32, dl->recBuffer, maxBlockSize);
+	}
+	if (lastBlockSize > 0) {
+		if (fread(dl->recBuffer, lastBlockSize, 1, file) != 1) {
+			fclose(file);
+			return qfalse;
+		}
+		CRC32_ProcessBlock(&crc32, dl->recBuffer, lastBlockSize);
+	}
+	CRC32_End(&crc32);
+
+	fclose(file);
+
+	if (crc32 != dl->crc32) {
+		PrintError(dl, "The CRC32 for %s was %08X instead of %08X", dl->finalName, crc32, dl->crc32);
+		return qfalse;
+	}
+
+	return qtrue;
+}
+
+
+// fails if the dest path already exists
+static qbool RenameFile( const char* source, const char* dest )
+{
+	// note: the rename function behaves differently on Windows and Linux
+	// Windows: dest path cannot exist
+	// Linux: if dest path exists, old file gets erased
+#if defined(_WIN32)
+	return MoveFileA(source, dest) != 0;
+#else
+	struct stat destInfo;  
+	if (stat(dest, &destInfo) == 0)
+		return qfalse;
+
+	return rename(source, dest) == 0;
+#endif
+}
+
+
+static qbool Download_Rename( mapDownload_t* dl )
+{
+	char dir[MAX_PATH];
+#if defined(_WIN32)
+	Q_strncpyz(dir, "baseq3\\", sizeof(dir));
+#else
+	Q_strncpyz(dir, "baseq3/", sizeof(dir));
+#endif
+
+	char name[MAX_PATH];
+	if (*dl->finalName == '\0') {
+		Q_strncpyz(name, dl->mapName, sizeof(name));
+	} else {
+		Q_strncpyz(name, dl->finalName, sizeof(name));
+		const int l = strlen(dl->finalName);
+		if (Q_stricmp(name + l - 4, ".pk3") == 0)
+			name[l - 4] = '\0';
+	}
+
+	// try with the desired name
+	Com_sprintf(dl->finalPath, sizeof(dl->finalPath), "%s%s.pk3", dir, name);
+	if (RenameFile(dl->tempPath, dl->finalPath))
+		return qtrue;
+
+	// try a few more times with random name suffixes
+	for (int i = 0; i < 4; ++i) {
+		// the suffix is 24 bits long, i.e. 6 characters
+		const unsigned int suffix0 = (unsigned int)rand() % (1 << 12);
+		const unsigned int suffix1 = (unsigned int)rand() % (1 << 12);
+		const unsigned int suffix = suffix0 | (suffix1 << 12);
+		Com_sprintf(dl->finalPath, sizeof(dl->finalPath), "%s%s_%06x.pk3", dir, name, suffix);
+		if (RenameFile(dl->tempPath, dl->finalPath))
+			return qtrue;
+	}
+
+	PrintError(dl, "Failed to rename '%s' to '%s' or a similar name", dl->tempPath, name);
+
+	return qfalse;
+}
+
+
+static qbool Download_CleanUp( mapDownload_t* dl, qbool rename )
+{
+	if (dl->socket != INVALID_SOCKET) {
+		Q_closesocket(dl->socket);
+		dl->socket = INVALID_SOCKET;
+	}
+
+	if (dl->file != NULL) {
+		fclose(dl->file);
+		dl->file = NULL;
+	}
+
+	qbool success = qtrue;
+	if (rename) {
+		if (dl->crc32 != 0)
+			success = Download_IsFileValid(dl);
+
+		if (success)
+			success = Download_Rename(dl);
+	}
+
+	if (Q_unlink(dl->tempPath) != 0 && errno != ENOENT) {
+		// ENOENT means the file wasn't found, which is good because
+		// it means our previous code was successful
+		PrintError(dl, "Failed to delete file '%s'", dl->tempPath);
+	}
+
+	return success;
+}
+
+
+static qbool Download_Begin( mapDownload_t* dl, int port, const char* server, const char* file )
+{
+	Download_CleanUp(dl, qfalse);
+	Download_Clear(dl);
+
+	char portString[16];
+	Com_sprintf(portString, sizeof(portString), "%d", port);
+
+	addrInfo_t* address;
+	const int ec = getaddrinfo(server, portString, NULL, &address);
+	if (ec != 0) {
+		// EAI* errors map to WSA* errors, so it's ok to call this
+		PrintSocketError(dl, "getaddrinfo", ec);
+		return qfalse;
+	}
+
+	qbool connected = qfalse;
+	for (addrInfo_t* a = address; a != NULL; a = a->ai_next) {
+		dl->socket = socket(a->ai_family, a->ai_socktype, a->ai_protocol);
+		if (dl->socket == INVALID_SOCKET) {
+			PrintSocketError(dl, "socket");
+			continue;
+		}
+
+		if (connect(dl->socket, address->ai_addr, address->ai_addrlen) == SOCKET_ERROR) {
+			PrintSocketError(dl, "connect");
+			if (dl->socket != INVALID_SOCKET) {
+				Q_closesocket(dl->socket);
+				dl->socket = INVALID_SOCKET;
+			}
+			continue;
+		}
+
+		connected = qtrue;
+		break;
+	}
+
+	freeaddrinfo(address);
+
+	if (!connected) {
+		PrintError(dl, "Failed to connect to the host");
+		return qfalse;
+	}
+
+#if defined(_WIN32)
+	const DWORD timeoutMs = 1;
+	if (setsockopt(dl->socket, SOL_SOCKET, SO_RCVTIMEO, (const char*)&timeoutMs, sizeof(timeoutMs)) == SOCKET_ERROR) {
+		PrintSocketError(dl, "setsockopt");
+		return qfalse;
+	}
+#else
+	timeval timeout;      
+	timeout.tv_sec = 0;
+	timeout.tv_usec = 1000;
+	if (setsockopt(dl->socket, SOL_SOCKET, SO_RCVTIMEO, (const char*)&timeout, sizeof(timeout)) == SOCKET_ERROR) {
+		PrintSocketError(dl, "setsockopt");
+		return qfalse;
+	}
+#endif
+
+	char request[256];
+	Com_sprintf(request, sizeof(request), "GET /%s HTTP/1.0\r\nHost: %s:%d\r\n\r\n", file, server, port);
+	const int requestLength = strlen(request);
+	if (send(dl->socket, request, requestLength, 0) != requestLength) {
+		PrintSocketError(dl, "send");
+		return qfalse;
+	}
+
+#if defined(_WIN32)
+	if (GetTempFileNameA("baseq3", "", 0, dl->tempPath) == 0) {
+		PrintError(dl, "Couldn't create a file name");
+		return qfalse;
+	}
+	dl->file = fopen(dl->tempPath, "wb");
+#else
+	Q_strncpyz(dl->tempPath, "baseq3/XXXXXX.tmp", sizeof(dl->tempPath));
+	const int fd = mkstemps(dl->tempPath, 4);
+	dl->file = fd != -1 ? fdopen(fd, "wb") : NULL;
+#endif
+	if (dl->file == NULL) {
+		const int error = errno;
+		char* const errorString = strerror(error);
+		PrintError(dl, "Failed to open file '%s': %s (%d)", dl->tempPath, errorString ? errorString : "?", error);
+		return qfalse;
+	}
+
+	dl->startTimeMS = Sys_Milliseconds();
+
+	return qtrue;
+}
+
+
+static qbool IsWhiteSpace( char c )
+{
+	return c == '\r' || c == '\n' || c == '\t' || c == ' ';
+}
+
+
+static void RemoveTrailingWhiteSpace( char* s )
+{
+	const int l = strlen(s);
+	int i = l;
+	while (i--) {
+		if (!IsWhiteSpace(s[i])) {
+			if (i + 1 < l)
+				s[i + 1] = '\0';
+			break;
+		}
+	}
+}
+
+
+static qbool ParseHeader( unsigned int* headerLength, mapDownload_t* dl )
+{
+	if (dl->badResponse) {
+		RemoveTrailingWhiteSpace(dl->recBuffer);
+		if (*dl->recBuffer != '\0')
+			PrintError(dl, "HTTP status %s - %s", dl->httpHeaderValue, dl->recBuffer);
+		else
+			PrintError(dl, "HTTP status %s", dl->httpHeaderValue);
+		return qfalse;
+	}
+
+// note: sscanf %s with the width specifier will null terminate in overflow cases too
+#define		MAXSTRLEN	512
+#define		STR(X)		#X
+#define		XSTR(X)		STR(X)
+#define		WSPEC		XSTR(MAXSTRLEN)
+
+	static char httpHeaderValue[MAXSTRLEN];
+	static char header[MAXSTRLEN];
+	static char value[MAXSTRLEN];
+	static char fileName[MAXSTRLEN];
+
+	qbool badResponse = qfalse;
+	const char* s = dl->recBuffer;
+	for (;;) {
+		qbool httpHeader = qfalse;
+		int bytesRead = 0;
+		if (sscanf(s, "HTTP/1.1 %"WSPEC"[^\r]\r\n%n", httpHeaderValue, &bytesRead) == 1)
+			httpHeader = qtrue;
+		else if (sscanf(s, "%"WSPEC"[^:]: %"WSPEC"[^\r]\r\n%n", header, value, &bytesRead) != 2)
+			break;
+
+		s += bytesRead;
+
+		if (badResponse)
+			continue;
+
+		if (httpHeader) {
+			int code;
+			if (sscanf(httpHeaderValue, "%d", &code) == 1 && code != 200)
+				badResponse = qtrue;
+		} else if (Q_stricmp(header, "Content-Length") == 0) {
+			int temp;
+			if (sscanf(value, "%d", &temp) == 1)
+				dl->bytesContent = temp;
+		} else if (Q_stricmp(header, "Content-Disposition") == 0) {
+			const size_t l = strlen("filename=");
+			const char* valueFileName = strstr(value, "filename=");
+			if (valueFileName != NULL) {
+				valueFileName += l;
+				if (*valueFileName == '\"')
+					valueFileName++;
+
+				if (sscanf(valueFileName, "%"WSPEC"[^\";\r]", fileName) == 1)
+					Q_strncpyz(dl->finalName, fileName, sizeof(dl->finalName));
+			}
+		} else if (Q_stricmp(header, "X-CNQ3-CRC32") == 0) {
+			unsigned int temp;
+			if (sscanf(value, "%x", &temp) == 1)
+				dl->crc32 = temp;
+		}
+	}
+
+	if (badResponse) {
+		if (Q_stricmpn(s - 4, "\r\n\r\n", 4) == 0) {
+			RemoveTrailingWhiteSpace((char*)s);
+			if (*s != '\0') {
+				PrintError(dl, "%s - %s", httpHeaderValue, s);
+				return qfalse;
+			} else {
+				dl->badResponse = qtrue;
+				Q_strncpyz(dl->httpHeaderValue, httpHeaderValue, sizeof(dl->httpHeaderValue));
+				return qtrue;
+			}
+		}
+		return qfalse;
+	}
+
+	if (Q_stricmpn(s - 4, "\r\n\r\n", 4) == 0) {
+		if (dl->bytesContent == 0) {
+			PrintError(dl, "Content-Length wasn't defined or was invalid");
+			return qfalse;
+		}
+
+		if (dl->finalName[0] == '\0') {
+			PrintError(dl, "Content-Disposition wasn't defined or was missing the filename field");
+			return qfalse;
+		}
+
+		dl->headerParsed = qtrue;
+	}
+
+	const unsigned int headerBytes = s - dl->recBuffer;
+	dl->bytesHeader += headerBytes;
+	if (dl->headerParsed) {
+		dl->bytesTotal = dl->bytesHeader + dl->bytesContent;
+		*headerLength = headerBytes;
+	}
+
+	return qtrue;
+
+#undef	WSPEC
+#undef	XSTR
+#undef	STR
+#undef	MAXSTRLEN
+}
+
+
+int Download_Continue( mapDownload_t* dl )
+{
+	if (dl->socket == INVALID_SOCKET)
+		return MDLS_NOTHING;
+
+	// the -1 is necessary because we need to be able to safely null-terminate
+	// the buffer for ParseHeader without stomping another buffer's memory
+	const int ec = recv(dl->socket, dl->recBuffer, sizeof(dl->recBuffer) - 1, 0);
+	if (ec < 0) {
+		if (IsSocketTimeoutError()) {
+			const int now = Sys_Milliseconds();
+			if (!dl->lastErrorTimeOut)
+				dl->timeOutStartTimeMS = now;
+			if (now - dl->timeOutStartTimeMS >= 1000) {
+				PrintError(dl, "Timed out for more than a full second");
+				Download_CleanUp(dl, qfalse);
+				return MDLS_ERROR;
+			}
+			dl->lastErrorTimeOut = qtrue;
+			return MDLS_IN_PROGRESS;
+		}
+		PrintSocketError(dl, "recv");
+		Download_CleanUp(dl, qfalse);
+		return MDLS_ERROR;
+	}
+
+	if (ec == 0) {
+		if (dl->bytesDownloaded == dl->bytesTotal)
+			return Download_CleanUp(dl, qtrue) ? MDLS_SUCCESS : MDLS_ERROR;
+
+		if (dl->badResponse)
+			PrintError(dl, "HTTP status %s", dl->httpHeaderValue);
+		else
+			PrintError(dl, "Connection closed too early");
+		Download_CleanUp(dl, qfalse);
+		return MDLS_ERROR;
+	}
+
+	dl->bytesDownloaded += ec;
+
+	unsigned int offset = 0;
+	if (!dl->headerParsed) {
+		// make sure ParseHeader can read the buffer as a C string
+		dl->recBuffer[ec] = '\0';
+		if (!ParseHeader(&offset, dl)) {
+			Download_CleanUp(dl, qfalse);
+			return MDLS_ERROR;
+		}
+	}
+
+	const unsigned int bytesToWrite = ec - offset;
+	if (dl->headerParsed && bytesToWrite > 0) {
+		if (fwrite(dl->recBuffer + offset, bytesToWrite, 1, dl->file) != 1) {
+			const int error = errno;
+			char* const errorString = strerror(error);
+			PrintError(dl, "Failed to write %d bytes to '%s': %s (%d)",
+					   (int)(ec - offset), dl->tempPath, errorString ? errorString : "?", error);
+			Download_CleanUp(dl, qfalse);
+			return MDLS_ERROR;
+		}
+	}
+
+	if (dl->bytesDownloaded == dl->bytesTotal)
+		return Download_CleanUp(dl, qtrue) ? MDLS_SUCCESS : MDLS_ERROR;
+
+	dl->timeOutStartTimeMS = INT_MIN;
+	dl->lastErrorTimeOut = qfalse;
+
+	return MDLS_IN_PROGRESS;
+}
+
+
+static qbool CL_MapDownload_StartImpl( const char* mapName, int source, const char* query, qbool fromCommand, qbool exactMatch )
+{
+	Com_Printf("Attempting download from the %s map server...\n", cl_mapDLSources[source].name);
+
+	Q_strncpyz(cl_mapDL.mapName, mapName, sizeof(cl_mapDL.mapName));
+
+	const qbool success = Download_Begin(&cl_mapDL, cl_mapDLSources[source].port, cl_mapDLSources[source].hostName, query);	
+	if (!success) {
+		if (!fromCommand)
+			Com_Error(ERR_DROP, cl_mapDL.errorMessage);
+		return qfalse;
+	}
+
+	// We set these after the Download_Begin call since it clears cl_mapDL.
+	cl_mapDL.fromCommand = fromCommand;
+	cl_mapDL.sourceIndex = source;
+	cl_mapDL.exactMatch = exactMatch;
+
+	if (!fromCommand) {
+		Cvar_Set("cl_downloadName", mapName);
+		Cvar_Set("cl_downloadSize", "0");
+		Cvar_Set("cl_downloadCount", "0");
+		Cvar_SetValue("cl_downloadTime", cls.realtime);
+	}
+
+	return qtrue;
+}
+
+
+static qbool CL_MapDownload_CheckActive()
+{
+	if (!CL_MapDownload_Active())
+		return qfalse;
+
+	PrintError(&cl_mapDL, "Download already in progress for map %s", cl_mapDL.mapName);
+	return qtrue;
+}
+
+
+qbool CL_MapDownload_Start( const char* mapName, qbool fromCommand )
+{
+	if (CL_MapDownload_CheckActive())
+		return qfalse;
+
+	char query[256];
+	(*cl_mapDLSources[0].formatQuery)(query, sizeof(query), mapName);
+	if (CL_MapDownload_StartImpl(mapName, 0, query, fromCommand, qfalse))
+		return qtrue;
+
+	(*cl_mapDLSources[1].formatQuery)(query, sizeof(query), mapName);
+	return CL_MapDownload_StartImpl(mapName, 1, query, fromCommand, qfalse);
+}
+
+
+qbool CL_MapDownload_Start_MapChecksum( const char* mapName, unsigned int mapCrc32, qbool exactMatch )
+{
+	if (mapCrc32 == 0 || CL_MapDownload_CheckActive())
+		return qfalse;
+
+	char query[256];
+	Com_sprintf(query, sizeof(query), "map?n=%s&m=%x", mapName, mapCrc32);
+	if (!exactMatch)
+		Q_strcat(query, sizeof(query), "&e=0");
+
+	return CL_MapDownload_StartImpl(mapName, 0, query, qfalse, exactMatch);
+}
+
+
+qbool CL_MapDownload_Start_PakChecksums( const char* mapName, unsigned int* pakChecksums, int pakCount, qbool exactMatch )
+{
+	if (pakCount == 0 || CL_MapDownload_CheckActive())
+		return qfalse;
+
+	static char query[1024];
+	Com_sprintf(query, sizeof(query), "map?n=%s&p=", mapName);
+	Q_strcat(query, sizeof(query), va("%x", pakChecksums[0]));
+	for (int i = 1; i < pakCount; ++i) {
+		Q_strcat(query, sizeof(query), va(",%x", pakChecksums[i]));
+	}
+	if (!exactMatch)
+		Q_strcat(query, sizeof(query), "&e=0");
+
+	return CL_MapDownload_StartImpl(mapName, 0, query, qfalse, exactMatch);
+}
+
+
+qbool CL_PakDownload_Start( unsigned int checksum, qbool fromCommand )
+{
+	if (checksum == 0 || CL_MapDownload_CheckActive())
+		return qfalse;
+
+	char query[64];
+	Com_sprintf(query, sizeof(query), "pak?%x", checksum);
+
+	char mapName[64];
+	Com_sprintf(mapName, sizeof(mapName), "%x", checksum);
+
+	return CL_MapDownload_StartImpl(mapName, 0, query, fromCommand, qtrue);
+}
+
+
+static void CL_MapDownload_ClearCvars()
+{
+	Cvar_Set("cl_downloadSize", "0");
+	Cvar_Set("cl_downloadCount", "0");
+	Cvar_Set("cl_downloadName", "");
+	Cvar_SetValue("cl_downloadTime", cls.realtime);
+}
+
+
+void CL_MapDownload_Continue()
+{
+	const int status = Download_Continue(&cl_mapDL);
+	if (status == MDLS_SUCCESS) {
+		FS_Restart(clc.checksumFeed);
+		if (!cl_mapDL.fromCommand) {
+			CL_DownloadsComplete();
+			CL_MapDownload_ClearCvars();
+		}
+		Com_Printf("'%s' downloaded successfully to '%s'\n", cl_mapDL.finalName, cl_mapDL.finalPath);
+	} else if (status == MDLS_IN_PROGRESS) {
+		if (!cl_mapDL.fromCommand) {
+			Cvar_Set("cl_downloadSize", va("%d", cl_mapDL.bytesTotal));
+			Cvar_Set("cl_downloadCount", va("%d", cl_mapDL.bytesDownloaded));
+		}
+	} else if (status == MDLS_ERROR) {
+		if (!cl_mapDL.fromCommand) {
+			if (clc.demoplaying)
+				CL_DemoCompleted();
+			CL_MapDownload_ClearCvars();
+			Com_Error(ERR_DROP, cl_mapDL.errorMessage);
+		} else if (cl_mapDL.sourceIndex == 0 && !cl_mapDL.exactMatch) {
+			char query[256];
+			(*cl_mapDLSources[1].formatQuery)(query, sizeof(query), cl_mapDL.mapName);
+			CL_MapDownload_StartImpl(cl_mapDL.mapName, 1, query, cl_mapDL.fromCommand, qfalse);
+		}
+	}
+}
+
+
+void CL_MapDownload_Init()
+{
+	Download_Clear(&cl_mapDL);
+}
+
+
+qbool CL_MapDownload_Active()
+{
+	return cl_mapDL.socket != INVALID_SOCKET;
+}
+
+
+void CL_MapDownload_Cancel()
+{
+	if (!CL_MapDownload_Active())
+		return;
+
+	Download_CleanUp(&cl_mapDL, qfalse);
+	CL_MapDownload_ClearCvars();
+}
+
+
+// negative if nothing's going on
+static float CL_MapDownload_Progress()
+{
+	if (!CL_MapDownload_Active() || !cl_mapDL.fromCommand || 
+		cl_mapDL.bytesTotal < 1 || cl_mapDL.bytesDownloaded < 0)
+		return -1;
+
+	const uint64_t t = (uint64_t)cl_mapDL.bytesTotal;
+	const uint64_t d = (uint64_t)cl_mapDL.bytesDownloaded;
+	const double p = ((double)d * 100.0) / (double)t;
+	const float progress = min((float)p, 100.0f);
+	
+	return progress;
+}
+
+
+// bytes/s, negative if nothing's going on
+static int CL_MapDownload_Speed()
+{
+	if (!CL_MapDownload_Active() || !cl_mapDL.fromCommand ||
+		cl_mapDL.bytesTotal < 1 || cl_mapDL.bytesDownloaded < 0)
+		return -1;
+
+	const int startMS = cl_mapDL.startTimeMS;
+	const int nowMS = Sys_Milliseconds();
+	const int elapsedMS = nowMS - startMS;
+	if (startMS == INT_MIN || elapsedMS <= 0)
+		return -1;
+
+	return (int)(((uint64_t)cl_mapDL.bytesDownloaded * 1000) / (uint64_t)elapsedMS);
+}
+
+
+static void FormatSize( char* buffer, int bufferSize, unsigned int bytes )
+{
+	unsigned int m = (unsigned int)(-1);
+	unsigned int x = bytes;
+	while (x) {
+		x >>= 10;
+		m++;
+	}
+
+	if (m == 0)
+		Com_sprintf(buffer, bufferSize, "%u bytes", bytes);
+	else if (m == 1)
+		Com_sprintf(buffer, bufferSize, "%u KB", bytes >> 10);
+	else if (m == 2)
+		Com_sprintf(buffer, bufferSize, "%.1f MB", (float)(bytes >> 10) / 1024.0f);
+	else
+		Com_sprintf(buffer, bufferSize, "%.2f GB", (float)(bytes >> 20) / 1024.0f);
+}
+
+
+static void FormatTime( char* buffer, int bufferSize, unsigned int seconds )
+{
+	const int s = seconds % 60;
+	const int m = seconds / 60;
+	if (m > 0)
+		Com_sprintf(buffer, bufferSize, "%dm %2ds", m, s);
+	else
+		Com_sprintf(buffer, bufferSize, "%2ds", s);
+}
+
+
+void CL_MapDownload_DrawConsole( float cw, float ch )
+{
+	const float progress = CL_MapDownload_Progress();
+	if (progress < 0.0f)
+		return;
+
+	const float vw = cls.glconfig.vidWidth;
+
+	char msg0[128];
+	const char* fileName = cl_mapDL.finalName[0] != '\0' ? cl_mapDL.finalName : "pk3";
+	Com_sprintf(msg0, sizeof(msg0), "Downloading %s: %2d%%", fileName, (int)progress);
+	const float wl0 = cw * strlen(msg0);
+	re.SetColor(colorWhite);
+	SCR_DrawString(vw - wl0 - cw / 2, ch / 2, cw, ch, msg0, qtrue);
+
+	const int speed = CL_MapDownload_Speed();
+	if (speed <= 0)
+		return;
+
+	char size[64];
+	FormatSize(size, sizeof(size), speed);
+	char msg1[128];
+	Com_sprintf(msg1, sizeof(msg1), "Speed: %s/s", size);
+	const float wl1 = cw * strlen(msg1);
+	SCR_DrawString(vw - wl1 - cw / 2, 3 * ch / 2, cw, ch, msg1, qtrue);
+
+	if (progress <= 0.0f)
+		return;
+
+	const int elapsedMS = Sys_Milliseconds() - cl_mapDL.startTimeMS;
+	const int totalMS = (int)((elapsedMS * 100.0f) / progress);
+	const int remainingMS = max(totalMS - elapsedMS, 0);
+	char time[64];
+	FormatTime(time, sizeof(time), remainingMS / 1000);
+	char msg2[128];
+	Com_sprintf(msg2, sizeof(msg2), "Time left: %s", time);
+	const float wl2 = cw * strlen(msg2);
+	SCR_DrawString(vw - wl2 - cw / 2, 5 * ch / 2, cw, ch, msg2, qtrue);
+}
+
+
+void CL_MapDownload_CrashCleanUp()
+{
+	if (cl_mapDL.file != NULL && cl_mapDL.tempPath[0] != '\0') {
+		fclose(cl_mapDL.file);
+		Q_unlink(cl_mapDL.tempPath);
+	}
+}

code/client/cl_main.cpp

@@ -275,7 +275,7 @@ CLIENT SIDE DEMO PLAYBACK
 */
 
 
-static void CL_DemoCompleted()
+void CL_DemoCompleted()
 {
 	if (cl_timedemo && cl_timedemo->integer) {
 		int time = Sys_Milliseconds() - clc.timeDemoStart;
@@ -419,7 +419,7 @@ void CL_PlayDemo_f()
 	Q_strncpyz( cls.servername, Cmd_Argv(1), sizeof( cls.servername ) );
 
 	// read demo messages until connected
-	while ( cls.state >= CA_CONNECTED && cls.state < CA_PRIMED ) {
+	while (cls.state >= CA_CONNECTED && cls.state < CA_PRIMED && !CL_MapDownload_Active()) {
 		CL_ReadDemoMessage();
 	}
 	// don't get the first snapshot this frame, to prevent the long
@@ -1067,7 +1067,7 @@ static void CL_Clientinfo_f( void )
 ///////////////////////////////////////////////////////////////
 
 
-static void CL_DownloadsComplete()
+void CL_DownloadsComplete()
 {
 	// if we downloaded files we need to restart the file system
 	if (clc.downloadRestart) {
@@ -1177,14 +1177,10 @@ void CL_NextDownload(void) {
 		else
 			s = localName + strlen(localName); // point at the nul byte
 
-		if( !cl_allowDownload->integer ) {
+		if( cl_allowDownload->integer != -1 ) {
-			Com_Error(ERR_DROP, "UDP Downloads are "
+			Com_Error(ERR_DROP, "The id download system is disabled (cl_allowDownload must be -1)");
-				"disabled on your client. "
+			return;
-				"(cl_allowDownload is %d)",
+		} else {
-				cl_allowDownload->integer);
-		return;
-		}
-		else {
 			CL_BeginDownload( localName, remoteName );
 		}
 
@@ -1200,6 +1196,91 @@ void CL_NextDownload(void) {
 }
 
 
+// returns qtrue if a download started
+static qbool CL_StartDownloads()
+{
+	int mode = cl_allowDownload->integer;
+	if (mode < -1 || mode > 1) {
+		mode = 1;
+	}
+
+	// downloads disabled
+	if (mode == 0) {
+		// autodownload is disabled on the client
+		// but it's possible that some referenced files on the server are missing
+		char missingfiles[1024];
+		if (FS_ComparePaks(missingfiles, sizeof(missingfiles), qfalse)) {
+			// NOTE TTimo I would rather have that printed as a modal message box
+			// but at this point while joining the game we don't know whether we will successfully join or not
+			Com_Printf("\nWARNING: You are missing some files referenced by the server:\n%s"
+					   "To enable downloads, set cl_allowDownload to 1 (new) or -1 (old)\n\n", missingfiles);
+		}
+		return qfalse;
+	}
+
+	// legacy id downloads
+	if (mode == -1) {
+		if (FS_ComparePaks(clc.downloadList, sizeof(clc.downloadList), qtrue)) {
+			Com_Printf("Need paks: %s\n", clc.downloadList);
+			if (*clc.downloadList) {
+				// if autodownloading is not enabled on the server
+				cls.state = CA_CONNECTED;
+				CL_NextDownload();
+				return qtrue;
+			}
+		}
+		return qfalse;
+	}
+
+	//
+	// CNQ3 downloads
+	//
+
+	// note: the use of FS_FileIsInPAK works here because it rejects paks that aren't in the pure list
+	const qbool pureServer = Cvar_VariableIntegerValue("sv_pure"); // the cvar is in CS_SYSTEMINFO
+	const qbool exactMatch = !clc.demoplaying && pureServer;
+	const char* const serverInfo = cl.gameState.stringData + cl.gameState.stringOffsets[CS_SERVERINFO];
+	const char* const mapName = Info_ValueForKey(serverInfo, "mapname");
+	const char* const mapPath = va("maps/%s.bsp", mapName);
+	if ((!exactMatch && FS_FileExists(mapPath)) || FS_FileIsInPAK(mapPath, NULL, NULL))
+		return qfalse;
+
+	// generate a checksum list of all the pure paks we're missing
+	unsigned int pakChecksums[256];
+	int pakCount;
+	const char* const pakChecksumString = Info_ValueForKey(serverInfo, "sv_currentPak");
+	unsigned int pakChecksum;
+	if (sscanf(pakChecksumString, "%d", &pakChecksum) == 1 && pakChecksum != 0) {
+		pakChecksums[0] = pakChecksum;
+		pakCount = 1;
+	} else {
+		FS_MissingPaks(pakChecksums, &pakCount, ARRAY_LEN(pakChecksums));
+	}
+	
+	if (pakCount > 0) {
+		const qbool dlStarted = pakCount == 1 ?
+			// we know exactly which pk3 we need, so no need to send a map name
+			CL_PakDownload_Start(pakChecksums[0], qfalse) :
+			// we send the map's name and a list of pk3 checksums (qmd4)
+			CL_MapDownload_Start_PakChecksums(mapName, pakChecksums, pakCount, exactMatch);
+
+		if (dlStarted) {
+			cls.state = CA_CONNECTED;
+			return qtrue;
+		}
+		return qfalse;
+	}
+
+	// we send the map's name only because we have no additional data and
+	// an exact match isn't needed
+	if (!exactMatch && CL_MapDownload_Start(mapName, qfalse)) {
+		cls.state = CA_CONNECTED;
+		return qtrue;
+	}
+	return qfalse;
+}
+
+
 /*
 =================
 CL_InitDownloads
@@ -1209,35 +1290,11 @@ and determine if we need to download them
 =================
 */
 void CL_InitDownloads(void) {
-  char missingfiles[1024];
+	
-
+	if(!CL_StartDownloads())
-  if ( !cl_allowDownload->integer )
+	{
-  {
+		CL_DownloadsComplete();
-    // autodownload is disabled on the client
-    // but it's possible that some referenced files on the server are missing
-    if (FS_ComparePaks( missingfiles, sizeof( missingfiles ), qfalse ) )
-    {      
-      // NOTE TTimo I would rather have that printed as a modal message box
-      //   but at this point while joining the game we don't know wether we will successfully join or not
-      Com_Printf( "\nWARNING: You are missing some files referenced by the server:\n%s"
-                  "You might not be able to join the game\n"
-                  "Go to the setting menu to turn on autodownload, or get the file elsewhere\n\n", missingfiles );
-    }
-  }
-  else if ( FS_ComparePaks( clc.downloadList, sizeof( clc.downloadList ) , qtrue ) ) {
-
-    Com_Printf("Need paks: %s\n", clc.downloadList );
-
-		if ( *clc.downloadList ) {
-			// if autodownloading is not enabled on the server
-			cls.state = CA_CONNECTED;
-			CL_NextDownload();
-			return;
-		}
-
 	}
-
-	CL_DownloadsComplete();
 }
 
 
@@ -1536,6 +1593,9 @@ void CL_Frame( int msec )
 		SCR_DebugGraph ( cls.realFrametime * 0.25, 0 );
 	}
 
+	// advance the current map download, if any
+	CL_MapDownload_Continue();
+
 	// see if we need to update any userinfo
 	CL_CheckUserinfo();
 
@@ -1903,6 +1963,88 @@ static void CL_CompleteCallVote_f( int startArg, int compArg )
 }
 
 
+static void CL_PrintDownloadPakUsage()
+{
+	Com_Printf( "Usage: %s checksum (signed decimal, '0x' or '0X' prefix for hex)\n", Cmd_Argv(0) );
+}
+
+
+static void CL_DownloadPak_f()
+{
+	unsigned int checksum;
+	if ( Cmd_Argc() != 2 ) {
+		CL_PrintDownloadPakUsage();
+		return;
+	}
+
+	const char* const arg1 = Cmd_Argv(1);
+	const int length = strlen( arg1 );
+	if ( length > 2 && arg1[0] == '0' && (arg1[1] == 'x' || arg1[1] == 'X') ) {
+		if ( sscanf(arg1 + 2, "%x", &checksum) != 1 ) {
+			CL_PrintDownloadPakUsage();
+			return;
+		}
+	} else {
+		int crc32;
+		if ( sscanf(arg1, "%d", &crc32) != 1 ) {
+			CL_PrintDownloadPakUsage();
+			return;
+		}
+		checksum = (unsigned int)crc32;
+	}
+
+	if ( checksum == 0 ) {
+		Com_Printf( "%s: invalid checksum 0\n", Cmd_Argv(0) );
+		return;
+	}
+
+	if ( FS_PakExists(checksum) ) {
+		Com_Printf( "%s: pk3 with checksum 0x%08x (%d) already present\n", Cmd_Argv(0), checksum, (int)checksum );
+		return;
+	}
+
+	CL_PakDownload_Start( checksum, qtrue );
+}
+
+
+static void CL_DownloadMap( qbool forceDL )
+{
+	if ( Cmd_Argc() != 2 ) {
+		Com_Printf( "Usage: %s mapname\n", Cmd_Argv(0) );
+		return;
+	}
+
+	const char* const mapName = Cmd_Argv(1);
+	if ( !forceDL ) {
+		const char* const mapPath = va( "maps/%s.bsp", mapName );
+		if ( FS_FileExists(mapPath) || FS_FileIsInPAK(mapPath, NULL, NULL) ) {
+			Com_Printf( "Map already exists! To force the download, use /%sf\n", Cmd_Argv(0) );
+			return;
+		}
+	}
+
+	CL_MapDownload_Start(mapName, qtrue);
+}
+
+
+static void CL_DownloadMap_f()
+{
+	CL_DownloadMap( qfalse );
+}
+
+
+static void CL_ForceDownloadMap_f()
+{
+	CL_DownloadMap( qtrue );
+}
+
+
+static void CL_CancelDownload_f()
+{
+	CL_MapDownload_Cancel();
+}
+
+
 void CL_Init()
 {
 	//QSUBSYSTEM_INIT_START( "Client" );
@@ -1936,7 +2078,7 @@ void CL_Init()
 	cl_maxpackets = Cvar_Get ("cl_maxpackets", "30", CVAR_ARCHIVE );
 	cl_packetdup = Cvar_Get ("cl_packetdup", "1", CVAR_ARCHIVE );
 
-	cl_allowDownload = Cvar_Get ("cl_allowDownload", "0", CVAR_ARCHIVE);
+	cl_allowDownload = Cvar_Get ("cl_allowDownload", "1", CVAR_ARCHIVE);
 
 #ifdef MACOS_X
 	// In game video is REALLY slow in Mac OS X right now due to driver slowness
@@ -1987,6 +2129,10 @@ void CL_Init()
 	Cmd_AddCommand ("model", CL_SetModel_f );
 	Cmd_AddCommand ("video", CL_Video_f );
 	Cmd_AddCommand ("stopvideo", CL_StopVideo_f );
+	Cmd_AddCommand ("dlpak", CL_DownloadPak_f );
+	Cmd_AddCommand ("dlmap", CL_DownloadMap_f );
+	Cmd_AddCommand ("dlmapf", CL_ForceDownloadMap_f );
+	Cmd_AddCommand ("dlstop", CL_CancelDownload_f );
 
 	// we use these until we get proper handling on the mod side
 	Cmd_AddCommand ("cv", CL_CallVote_f );
@@ -2002,6 +2148,8 @@ void CL_Init()
 
 	Cvar_Set( "cl_running", "1" );
 
+	CL_MapDownload_Init();
+
 	//QSUBSYSTEM_INIT_DONE( "Client" );
 }
 
@@ -2030,6 +2178,7 @@ void CL_Shutdown()
 	CL_ShutdownUI();
 
 	CL_SaveCommandHistory();
+	CL_MapDownload_Cancel();
 
 	Cmd_RemoveCommand ("cmd");
 	Cmd_RemoveCommand ("configstrings");
@@ -2052,6 +2201,10 @@ void CL_Shutdown()
 	Cmd_RemoveCommand ("model");
 	Cmd_RemoveCommand ("video");
 	Cmd_RemoveCommand ("stopvideo");
+	Cmd_RemoveCommand ("dlpak");
+	Cmd_RemoveCommand ("dlmap");
+	Cmd_RemoveCommand ("dlmapf");
+	Cmd_RemoveCommand ("dlstop");
 
 	// we use these until we get proper handling on the mod side
 	Cmd_RemoveCommand ("cv");

code/client/client.h

@@ -326,7 +326,7 @@ extern	cvar_t	*cl_timedemo;
 extern	cvar_t	*cl_aviFrameRate;
 extern	cvar_t	*cl_aviMotionJpeg;
 
-extern	cvar_t	*cl_allowDownload;
+extern	cvar_t	*cl_allowDownload;	// 0=off, 1=CNQ3, -1=id
 extern	cvar_t	*cl_inGameVideo;
 
 //=================================================
@@ -352,6 +352,9 @@ void CL_NextDownload(void);
 qbool CL_CDKeyValidate( const char *key, const char *checksum );
 int CL_ServerStatus( char *serverAddress, char *serverStatusString, int maxLen );
 
+void CL_DownloadsComplete();
+void CL_DemoCompleted();
+
 
 // cl_browser
 
@@ -475,3 +478,17 @@ void CL_WriteAVIVideoFrame( const byte *imageBuffer, int size );
 void CL_WriteAVIAudioFrame( const byte *pcmBuffer, int size );
 qbool CL_CloseAVI( void );
 qbool CL_VideoRecording( void );
+
+//
+// cl_download.cpp
+//
+qbool CL_MapDownload_Start( const char* mapName, qbool fromCommand );
+qbool CL_MapDownload_Start_MapChecksum( const char* mapName, unsigned int mapCrc32, qbool exactMatch );
+qbool CL_MapDownload_Start_PakChecksums( const char* mapName, unsigned int* pakChecksums, int pakCount, qbool exactMatch );
+qbool CL_PakDownload_Start( unsigned int pakChecksum, qbool fromCommand );
+void CL_MapDownload_Continue();
+void CL_MapDownload_Init();
+qbool CL_MapDownload_Active();
+void CL_MapDownload_Cancel();
+void CL_MapDownload_DrawConsole( float cw, float ch );
+void CL_MapDownload_CrashCleanUp();

code/qcommon/cm_load.cpp

@@ -439,7 +439,7 @@ void CM_ClearMap()
 
 // loads in the map and all submodels
 
-void CM_LoadMap( const char* name, qbool clientload, int* checksum )
+void CM_LoadMap( const char* name, qbool clientload, unsigned* checksum )
 {
 	if ( !name || !name[0] )
 		Com_Error( ERR_DROP, "CM_LoadMap: NULL name" );

code/qcommon/cm_public.h

@@ -23,7 +23,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #include "qfiles.h"
 
 
-void CM_LoadMap( const char* name, qbool clientload, int* checksum );
+void CM_LoadMap( const char* name, qbool clientload, unsigned* checksum );
 void CM_ClearMap();
 clipHandle_t CM_InlineModel( int index );		// 0 = world, 1 + are bmodels
 clipHandle_t CM_TempBoxModel( const vec3_t mins, const vec3_t maxs, int capsule );

code/qcommon/common.cpp

@@ -2423,6 +2423,13 @@ void Com_Frame()
 	if ( !com_dedicated->integer && com_maxfps->integer > 0 && !com_timedemo->integer )
 		minMsec = 1000 / com_maxfps->integer;
 
+#ifndef DEDICATED
+	// let's not limit the download speed by sleeping too much
+	qbool CL_MapDownload_Active(); // in client.h
+	if ( CL_MapDownload_Active() )
+		minMsec = 1;
+#endif
+
 	static int lastTime = 0;
 	int msec;
 	do {
@@ -2576,48 +2583,43 @@ float Q_acos(float c)
 }
 
 
-/*
+static unsigned int CRC32_table[256];
-==================
+static qbool CRC32_tableCreated = qfalse;
-crc32 routines
-==================
-*/
 
-static unsigned int crc32_table[256];
-static qboolean crc32_inited = qfalse;
 
-void crc32_init( unsigned int *crc )
+void CRC32_Begin( unsigned int* crc )
-{
+{
-	unsigned int c;
+	if ( !CRC32_tableCreated )
-	int i, j;
-
-	if ( !crc32_inited )
 	{
-		for (i = 0; i < 256; i++)
+		for ( int i = 0; i < 256; i++ )
 		{
-			c = i;
+			unsigned int c = i;
-			for ( j = 0; j < 8; j++ )
+			for ( int j = 0; j < 8; j++ )
 				c = c & 1 ? (c >> 1) ^ 0xEDB88320UL : c >> 1;
-			crc32_table[i] = c;
+			CRC32_table[i] = c;
 		}
-		crc32_inited = qtrue;
+		CRC32_tableCreated = qtrue;
-	}
+	}
-
+
-	*crc = 0xFFFFFFFFUL;
+	*crc = 0xFFFFFFFFUL;
-}
+}
-
+
-
+
-void crc32_update( unsigned int *crc, unsigned char *buf, unsigned int len )
+void CRC32_ProcessBlock( unsigned int* crc, const void* buffer, unsigned int length )
 {
-	while ( len-- )
+	unsigned int hash = *crc;
+	const unsigned char* buf = (const unsigned char*)buffer;
+	while ( length-- )
 	{
-		*crc = crc32_table[(*crc ^ *buf++) & 0xFF] ^ (*crc >> 8);
+		hash = CRC32_table[(hash ^ *buf++) & 0xFF] ^ (hash >> 8);
-	}
+	}
-}
+	*crc = hash;
-
+}
-
+
-void crc32_final( unsigned int *crc )
+
-{
+void CRC32_End( unsigned int* crc )
-	*crc = *crc ^ 0xFFFFFFFFUL;
+{
+	*crc ^= 0xFFFFFFFFUL;
 }
 
 

code/qcommon/crash.cpp

@@ -156,14 +156,14 @@ static unsigned int CRC32_HashFile(const char* filePath)
 	const unsigned int lastBlockSize = fileSize - fullBlocks * (unsigned int)BUFFER_SIZE;
 
 	unsigned int crc32 = 0;
-	crc32_init(&crc32);
+	CRC32_Begin(&crc32);
 
 	for(unsigned int i = 0; i < fullBlocks; ++i) {
 		if (fread(buffer, BUFFER_SIZE, 1, file) != 1) {
 			fclose(file);
 			return 0;
 		}
-		crc32_update(&crc32, buffer, BUFFER_SIZE);
+		CRC32_ProcessBlock(&crc32, buffer, BUFFER_SIZE);
 	}
 
 	if(lastBlockSize > 0) {
@@ -171,10 +171,10 @@ static unsigned int CRC32_HashFile(const char* filePath)
 			fclose(file);
 			return 0;
 		}
-		crc32_update(&crc32, buffer, lastBlockSize);
+		CRC32_ProcessBlock(&crc32, buffer, lastBlockSize);
 	}
 
-	crc32_final(&crc32);
+	CRC32_End(&crc32);
 
 	fclose(file);
 

code/qcommon/files.cpp

@@ -1266,7 +1266,7 @@ CONVENIENCE FUNCTIONS FOR ENTIRE FILES
 */
 
 
-qbool FS_FileIsInPAK( const char* filename, int* pChecksum )
+qbool FS_FileIsInPAK( const char* filename, int* pureChecksum, int* checksum )
 {
 	if ( !fs_searchpaths ) {
 		Com_Error( ERR_FATAL, "Filesystem call made without initialization\n" );
@@ -1307,8 +1307,11 @@ qbool FS_FileIsInPAK( const char* filename, int* pChecksum )
 			do {
 				// case and separator insensitive comparisons
 				if ( !FS_FilenameCompare( pakFile->name, filename ) ) {
-					if (pChecksum) {
+					if (pureChecksum) {
-						*pChecksum = search->pack->pure_checksum;
+						*pureChecksum = search->pack->pure_checksum;
+					}
+					if (checksum) {
+						*checksum = search->pack->checksum;
 					}
 					return qtrue;
 				}
@@ -2412,6 +2415,49 @@ qbool FS_ComparePaks( char *neededpaks, int len, qbool dlstring ) {
 	return qfalse; // We have them all
 }
 
+
+void FS_MissingPaks( unsigned int* checksums, int* checksumCount, int maxChecksums )
+{
+	int count = 0;
+	for (int i = 0; i < fs_numServerReferencedPaks; i++) {
+		// ignore official paks
+		if (FS_idPak(fs_serverReferencedPakNames[i], BASEGAME))
+			continue;
+
+		qbool gotIt = qfalse;
+		for (searchpath_t* sp = fs_searchpaths; sp; sp = sp->next) {
+			if (sp->pack && sp->pack->checksum == fs_serverReferencedPaks[i]) {
+				gotIt = qtrue;
+				break;
+			}
+		}
+
+		if (gotIt)
+			continue;
+
+		checksums[count++] = fs_serverReferencedPaks[i];
+
+		if (count == maxChecksums) {
+			Com_Printf("^3WARNING: FS_MissingPaks has too many checksums for the array given (over %d)\n", maxChecksums);
+			break;
+		}
+	}
+
+	*checksumCount = count;
+}
+
+
+qbool FS_PakExists( unsigned int checksum )
+{
+	for (searchpath_t* sp = fs_searchpaths; sp; sp = sp->next) {
+		if (sp->pack && sp->pack->checksum == checksum)
+			return qtrue;
+	}
+
+	return qfalse;
+}
+
+
 /*
 ================
 FS_Shutdown
@@ -2564,9 +2610,6 @@ static void FS_Startup( const char *gameName )
 	// reorder the pure pk3 files according to server order
 	FS_ReorderPurePaks();
 
-	// print the current search paths
-	FS_Path_f();
-
 	fs_gamedirvar->modified = qfalse; // We just loaded, it's not modified
 
 #ifdef FS_MISSING
@@ -3007,6 +3050,7 @@ void FS_Restart( int checksumFeed ) {
 
 }
 
+
 /*
 =================
 FS_ConditionalRestart

code/qcommon/qcommon.h

@@ -487,9 +487,9 @@ extern	int			cvar_modifiedFlags;
 // etc, variables have been modified since the last check.  The bit
 // can then be cleared to allow another change detection.
 
-void crc32_init( unsigned int *crc );
+void CRC32_Begin( unsigned int* crc );
-void crc32_update( unsigned int *crc, unsigned char *buf, unsigned int len );
+void CRC32_ProcessBlock( unsigned int* crc, const void* buffer, unsigned int length );
-void crc32_final( unsigned int *crc );
+void CRC32_End( unsigned int* crc );
 
 /*
 ==============================================================
@@ -556,7 +556,7 @@ int		FS_FOpenFileRead( const char *qpath, fileHandle_t *file, qbool uniqueFILE )
 // It is generally safe to always set uniqueFILE to qtrue, because the majority of
 // file IO goes through FS_ReadFile, which Does The Right Thing already.
 
-qbool	FS_FileIsInPAK( const char* filename, int* pChecksum );
+qbool	FS_FileIsInPAK( const char* filename, int* pureChecksum, int* checksum );
 
 int		FS_Write( const void *buffer, int len, fileHandle_t f );
 
@@ -628,6 +628,8 @@ void FS_PureServerSetLoadedPaks( const char *pakSums );
 qbool FS_CheckDirTraversal(const char *checkdir);
 qbool FS_idPak( const char* pak, const char* base );
 qbool FS_ComparePaks( char *neededpaks, int len, qbool dlstring );
+void FS_MissingPaks( unsigned int* checksums, int* checksumCount, int maxChecksums );
+qbool FS_PakExists( unsigned int checksum );
 
 void FS_Rename( const char *from, const char *to );
 

code/qcommon/vm.cpp

@@ -409,7 +409,6 @@ vmHeader_t *VM_LoadQVM( vm_t *vm, qboolean alloc ) {
 	int					dataLength;
 	int					i;
 	char				filename[MAX_QPATH], *errorMsg;
-	unsigned int		crc32sum;
 	vmHeader_t			*header;
 
 	// load the image
@@ -422,10 +421,6 @@ vmHeader_t *VM_LoadQVM( vm_t *vm, qboolean alloc ) {
 		return NULL;
 	}
 
-	crc32_init( &crc32sum );
-	crc32_update( &crc32sum, (unsigned char*)header, length );
-	crc32_final( &crc32sum );
-
 	// will also swap header
 	errorMsg = VM_ValidateHeader( header, length );
 	if ( errorMsg ) {
@@ -435,9 +430,6 @@ vmHeader_t *VM_LoadQVM( vm_t *vm, qboolean alloc ) {
 		return NULL;
 	}
 
-	vm->crc32sum = crc32sum;
-	Crash_SaveQVMChecksum( vm->index, crc32sum );
-
 	dataLength = header->dataLength + header->litLength + header->bssLength;
 	vm->dataLength = dataLength;
 
@@ -471,6 +463,12 @@ vmHeader_t *VM_LoadQVM( vm_t *vm, qboolean alloc ) {
 		*(int *)(vm->dataBase + i) = LittleLong( *(int *)(vm->dataBase + i ) );
 	}
 
+	unsigned int crc32;
+	CRC32_Begin( &crc32 );
+	CRC32_ProcessBlock( &crc32, header, length );
+	CRC32_End( &crc32 );
+	Crash_SaveQVMChecksum( vm->index, crc32 );
+
 	return header;
 }
 

code/qcommon/vm_local.h

@@ -202,7 +202,6 @@ struct vm_s {
 	byte		*jumpTableTargets;
 	int			numJumpTableTargets;
 
-	uint32_t	crc32sum;
 	vmIndex_t	index;
 
 	int			callStackDepth;

code/server/sv_client.cpp

@@ -1030,11 +1030,11 @@ static void SV_VerifyPaks_f( client_t* cl )
 	// namely the ui and cgame that we think they should have
 
 	pArg = Cmd_Argv(nCurArg++);
-	if ( !FS_FileIsInPAK("vm/cgame.qvm", &checksum) || !pArg || *pArg == '@' || atoi(pArg) != checksum )
+	if ( !FS_FileIsInPAK("vm/cgame.qvm", &checksum, NULL) || !pArg || *pArg == '@' || atoi(pArg) != checksum )
 		goto impure;
 
 	pArg = Cmd_Argv(nCurArg++);
-	if ( !FS_FileIsInPAK("vm/ui.qvm", &checksum) || !pArg || *pArg == '@' || atoi(pArg) != checksum )
+	if ( !FS_FileIsInPAK("vm/ui.qvm", &checksum, NULL) || !pArg || *pArg == '@' || atoi(pArg) != checksum )
 		goto impure;
 
 	// should be sitting at the delimiter now

code/server/sv_init.cpp

@@ -324,11 +324,6 @@ static void SV_TouchCGame()
 
 void SV_SpawnServer( const char* mapname )
 {
-	int			i;
-	int			checksum;
-	char		systemInfo[16384];
-	const char	*p;
-
 	// shut down the existing game if it is running
 	SV_ShutdownGameProgs();
 
@@ -376,7 +371,7 @@ void SV_SpawnServer( const char* mapname )
 	//Cvar_Set( "nextmap", "map_restart 0");
 	Cvar_Set( "nextmap", va("map %s", mapname) );
 
-	for (i=0 ; i<sv_maxclients->integer ; i++) {
+	for (int i=0 ; i<sv_maxclients->integer ; i++) {
 		// save when the server started for each client already connected
         if (svs.clients[i].state >= CS_CONNECTED) {
             svs.clients[i].oldServerTime = svs.time;
@@ -385,7 +380,7 @@ void SV_SpawnServer( const char* mapname )
 
 	// wipe the entire per-level structure
 	SV_ClearServer();
-	for ( i = 0 ; i < MAX_CONFIGSTRINGS ; i++ ) {
+	for ( int i = 0 ; i < MAX_CONFIGSTRINGS ; i++ ) {
 		sv.configstrings[i] = CopyString("");
 	}
 
@@ -397,12 +392,17 @@ void SV_SpawnServer( const char* mapname )
 	sv.checksumFeed = ( ((int) rand() << 16) ^ rand() ) ^ Com_Milliseconds();
 	FS_Restart( sv.checksumFeed );
 
+	unsigned checksum;
 	CM_LoadMap( va("maps/%s.bsp", mapname), qfalse, &checksum );
 
 	// set serverinfo visible name
 	Cvar_Set( "mapname", mapname );
 
-	Cvar_Set( "sv_mapChecksum", va("%i",checksum) );
+	Cvar_Set( "sv_mapChecksum", va("%d", (int)checksum) );
+
+	int pakChecksum = 0;
+	FS_FileIsInPAK( va("maps/%s.bsp", mapname), NULL, &pakChecksum );
+	Cvar_Set( "sv_currentPak", va("%d", (int)pakChecksum) );
 
 	// serverid should be different each time
 	sv.serverId = com_frameTime;
@@ -425,7 +425,7 @@ void SV_SpawnServer( const char* mapname )
 	sv_gametype->modified = qfalse;
 
 	// run a few frames to allow everything to settle
-	for (i = 0;i < 3; i++)
+	for (int i = 0;i < 3; i++)
 	{
 		VM_Call (gvm, GAME_RUN_FRAME, svs.time);
 		SV_BotFrame (svs.time);
@@ -435,7 +435,7 @@ void SV_SpawnServer( const char* mapname )
 	// create a baseline for more efficient communications
 	SV_CreateBaseline ();
 
-	for (i=0 ; i<sv_maxclients->integer ; i++) {
+	for (int i=0 ; i<sv_maxclients->integer ; i++) {
 		// send the new gamestate to all connected clients
 		if (svs.clients[i].state >= CS_CONNECTED) {
 			qbool isBot = ( svs.clients[i].netchan.remoteAddress.type == NA_BOT );
@@ -472,6 +472,7 @@ void SV_SpawnServer( const char* mapname )
 	SV_BotFrame (svs.time);
 	svs.time += 100;
 
+	const char *p;
 	if ( sv_pure->integer ) {
 		// the server sends these to the clients so they will only
 		// load pk3s also loaded at the server
@@ -503,6 +504,7 @@ void SV_SpawnServer( const char* mapname )
 	Cvar_Set( "sv_referencedPakNames", p );
 
 	// save systeminfo and serverinfo strings
+	char systemInfo[16384];
 	Q_strncpyz( systemInfo, Cvar_InfoString_Big( CVAR_SYSTEMINFO ), sizeof( systemInfo ) );
 	cvar_modifiedFlags &= ~CVAR_SYSTEMINFO;
 	SV_SetConfigstring( CS_SYSTEMINFO, systemInfo );
@@ -540,6 +542,7 @@ void SV_Init()
 	sv_gametype = Cvar_Get ("g_gametype", "0", CVAR_SERVERINFO | CVAR_LATCH );
 	Cvar_Get ("protocol", va("%i", PROTOCOL_VERSION), CVAR_SERVERINFO | CVAR_ROM);
 	sv_mapname = Cvar_Get ("mapname", "nomap", CVAR_SERVERINFO | CVAR_ROM);
+	Cvar_Get ("sv_currentPak", "", CVAR_SERVERINFO | CVAR_ROM);
 	sv_privateClients = Cvar_Get ("sv_privateClients", "0", CVAR_SERVERINFO);
 	sv_hostname = Cvar_Get ("sv_hostname", "noname", CVAR_SERVERINFO | CVAR_ARCHIVE );
 	sv_maxclients = Cvar_Get ("sv_maxclients", "8", CVAR_SERVERINFO | CVAR_LATCH);

code/win32/win_exception.cpp

@@ -1,6 +1,9 @@
 #include "../qcommon/q_shared.h"
 #include "../qcommon/qcommon.h"
 #include "../qcommon/crash.h"
+#ifndef DEDICATED
+#include "../client/client.h"
+#endif
 #include "win_local.h"
 #include "glw_win.h"
 #include <DbgHelp.h>
@@ -493,6 +496,12 @@ LONG CALLBACK WIN_HandleException( EXCEPTION_POINTERS* ep )
 	}
 #endif
 
+#ifndef DEDICATED
+	__try {
+		CL_MapDownload_CrashCleanUp();
+	} __except(EXCEPTION_EXECUTE_HANDLER) {}
+#endif
+
 	if (exc_exitCalled || IsDebuggerPresent())
 		return EXCEPTION_CONTINUE_SEARCH;
 

makefiles/gmake/cnq3.make

@@ -140,6 +140,7 @@ OBJECTS := \
 	$(OBJDIR)/cl_cgame.o \
 	$(OBJDIR)/cl_cin.o \
 	$(OBJDIR)/cl_console.o \
+	$(OBJDIR)/cl_download.o \
 	$(OBJDIR)/cl_input.o \
 	$(OBJDIR)/cl_keys.o \
 	$(OBJDIR)/cl_main.o \
@@ -265,6 +266,9 @@ $(OBJDIR)/cl_cin.o: ../../code/client/cl_cin.cpp
 $(OBJDIR)/cl_console.o: ../../code/client/cl_console.cpp
 	@echo $(notdir $<)
 	$(SILENT) $(CXX) $(ALL_CXXFLAGS) $(FORCE_INCLUDE) -o "$@" -MF "$(@:%.o=%.d)" -c "$<"
+$(OBJDIR)/cl_download.o: ../../code/client/cl_download.cpp
+	@echo $(notdir $<)
+	$(SILENT) $(CXX) $(ALL_CXXFLAGS) $(FORCE_INCLUDE) -o "$@" -MF "$(@:%.o=%.d)" -c "$<"
 $(OBJDIR)/cl_input.o: ../../code/client/cl_input.cpp
 	@echo $(notdir $<)
 	$(SILENT) $(CXX) $(ALL_CXXFLAGS) $(FORCE_INCLUDE) -o "$@" -MF "$(@:%.o=%.d)" -c "$<"

makefiles/premake5.lua

@@ -333,6 +333,7 @@ local function ApplyExeProjectSettings(exeName, server)
 		"client/cl_cgame.cpp",
 		"client/cl_cin.cpp",
 		"client/cl_console.cpp",
+		"client/cl_download.cpp",
 		"client/cl_input.cpp",
 		"client/cl_keys.cpp",
 		"client/cl_main.cpp",

makefiles/vs2013/cnq3.vcxproj

@@ -315,6 +315,7 @@ copy "..\..\.bin\release_x64\cnq3-x64.pdb" "$(QUAKE3DIR)"</Command>
     <ClCompile Include="..\..\code\client\cl_cgame.cpp" />
     <ClCompile Include="..\..\code\client\cl_cin.cpp" />
     <ClCompile Include="..\..\code\client\cl_console.cpp" />
+    <ClCompile Include="..\..\code\client\cl_download.cpp" />
     <ClCompile Include="..\..\code\client\cl_input.cpp" />
     <ClCompile Include="..\..\code\client\cl_keys.cpp" />
     <ClCompile Include="..\..\code\client\cl_main.cpp" />

makefiles/vs2013/cnq3.vcxproj.filters

@@ -254,6 +254,9 @@
     <ClCompile Include="..\..\code\client\cl_console.cpp">
       <Filter>client</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\code\client\cl_download.cpp">
+      <Filter>client</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\code\client\cl_input.cpp">
       <Filter>client</Filter>
     </ClCompile>