fteqw/engine/http/httpclient.c

#include "quakedef.h"

#include "iweb.h"

#include "netinc.h"
#include "fs.h"

#if defined(WEBCLIENT)

#if defined(FTE_TARGET_WEB)


#define MYJS 1
#if MYJS
#include "web/ftejslib.h"
#else
#include <emscripten/emscripten.h>
#endif

static void DL_Abort(struct dl_download *dl)
{
	dl->ctx = NULL;
}
static void DL_OnLoad(void *c, void *data, int datasize)
{
	struct dl_download *dl = c;

	//make sure the file is 'open'.
	if (!dl->file)
	{
		if (*dl->localname)
		{
			FS_CreatePath(dl->localname, FS_GAME);
			dl->file = FS_OpenVFS(dl->localname, "w+b", FS_GAME);
		}
		else
		{
			//emscripten does not close the file. plus we seem to end up with infinite loops.
			dl->file = FS_OpenTemp();
		}
	}

	if (dl->file)
	{
		VFS_WRITE(dl->file, data, datasize);
		VFS_SEEK(dl->file, 0);
		dl->status = DL_FINISHED;
	}
	else
		dl->status = DL_FAILED;

	dl->replycode = 200;
#if !MYJS
	dl->completed += datasize;
#endif
}
#if MYJS
static void DL_OnError(void *c, int ecode)
#else
static void DL_OnError(void *c)
#endif
{
	struct dl_download *dl = c;

#if MYJS
	dl->replycode = ecode;
#else
	dl->replycode = 404;	//we don't actually know. should we not do this?
#endif
	Con_Printf("download %p: error %i\n", dl, dl->replycode);
	dl->status = DL_FAILED;
}
static void DL_OnProgress(void *c, int position, int totalsize)
{
	struct dl_download *dl = c;
	dl->completed = position;
	dl->totalsize = totalsize;
}

//this becomes a poll function. the main thread will call this once a frame or so.
qboolean DL_Decide(struct dl_download *dl)
{
	const char *url = dl->redir;
	if (!*url)
		url = dl->url;

	if (dl->postdata)
	{
		DL_Abort(dl);
		return false;	//safe to destroy it now
	}

	if (dl->ctx)
	{
		if (dl->status == DL_FAILED || dl->status == DL_FINISHED)
		{
			DL_Abort(dl);
			return false;	//safe to destroy it now
		}
	}
	else
	{
		dl->status = DL_ACTIVE;
		dl->abort = DL_Abort;
		dl->ctx = dl;

#if MYJS
		emscriptenfte_async_wget_data2(url, dl, DL_OnLoad, DL_OnError, DL_OnProgress);
#else
		//annoyingly, emscripten doesn't provide an onprogress callback, unlike firefox etc, so we can't actually tell how far its got.
		//we'd need to provide our own js library to fix this. it can be done, I'm just lazy.
		emscripten_async_wget_data(url, dl, DL_OnLoad, DL_OnError);
#endif
	}

	return true;
}
#elif defined(NACL)
#include <ppapi/c/pp_errors.h>
#include <ppapi/c/ppb_core.h>
#include <ppapi/c/pp_file_info.h>
#include <ppapi/c/ppb_file_system.h>
#include <ppapi/c/ppb_file_ref.h>
#include <ppapi/c/ppb_url_request_info.h>
#include <ppapi/c/ppb_url_response_info.h>
#include <ppapi/c/pp_var.h>
#include <ppapi/c/ppb_var.h>
#include <ppapi/c/ppb_file_io.h>
#include <ppapi/c/ppb_url_loader.h>

extern PPB_Core *ppb_core;
extern PPB_URLRequestInfo *urlrequestinfo;
extern PPB_URLLoader  *urlloader;
extern PP_Instance pp_instance;
extern PPB_Var *ppb_var_interface;

struct nacl_dl {
	char buffer[65536];
	PP_Resource req;
};

static void readfinished(void* user_data, int32_t result)
{
	struct dl_download *f = user_data;
	struct nacl_dl *ctx = f->ctx;
	struct PP_CompletionCallback ccb = {readfinished, f, PP_COMPLETIONCALLBACK_FLAG_NONE};

	//trying to clean up
	if (!ctx)
		return;

//	Sys_Printf("lastresult: %i\n", result);

	if (result == PP_OK)
	{
//		Sys_Printf("%s completed\n", f->url);
		ppb_core->ReleaseResource(ctx->req);
		ctx->req = 0;

		f->status = DL_FINISHED;
		return;
	}

	for (; result > 0; result = urlloader->ReadResponseBody(ctx->req, ctx->buffer, sizeof(ctx->buffer), ccb))
	{
		//make sure the file is 'open'.
		if (!f->file)
		{
			if (*f->localname)
			{
				FS_CreatePath(f->localname, FS_GAME);
				f->file = FS_OpenVFS(f->localname, "w+b", FS_GAME);
			}
			else
				f->file = FS_OpenTemp();
		}

//		Sys_Printf("write: %i\n", result);
		VFS_WRITE(f->file, ctx->buffer, result);

		f->completed += result;
	}

//	Sys_Printf("result: %i\n", result);
	if (result != PP_OK_COMPLETIONPENDING)
	{
		Sys_Printf("file %s failed or something\n", f->url);
		ppb_core->ReleaseResource(ctx->req);
		ctx->req = 0;
		f->status = DL_FAILED;
	}
}
//urloader->open completed
static void dlstarted(void* user_data, int32_t result)
{
	struct dl_download *f = user_data;
	struct nacl_dl *ctx = f->ctx;

	struct PP_CompletionCallback ccb = {readfinished, f, PP_COMPLETIONCALLBACK_FLAG_NONE};
	readfinished(user_data, urlloader->ReadResponseBody(ctx->req, ctx->buffer, sizeof(ctx->buffer), ccb));
}

static void nadl_cleanup_cb(void* user_data, int32_t result)
{
	struct nacl_dl *ctx = user_data;

	if (ctx->req)
		ppb_core->ReleaseResource(ctx->req);
	free(ctx);
}

void NADL_Cleanup(struct dl_download *dl)
{
	struct nacl_dl *ctx = dl->ctx;

	//we can't free the ctx memory etc, in case the browser still has requests pending on it before it handles our close.
	//so set up a callback to do it later

	dl->ctx = NULL;	//orphan
	struct PP_CompletionCallback ccb = {nadl_cleanup_cb, ctx, PP_COMPLETIONCALLBACK_FLAG_NONE};
	ppb_core->CallOnMainThread(1000, ccb, 0);
}

qboolean DL_Decide(struct dl_download *dl)
{
	const char *url = dl->redir;
	struct nacl_dl *ctx;
	if (!*url)
		url = dl->url;

	if (dl->postdata)
	{
		NADL_Cleanup(dl);
		return false;	//safe to destroy it now
	}

	if (dl->ctx)
	{
		if (dl->status == DL_FAILED || dl->status == DL_FINISHED)
		{
			NADL_Cleanup(dl);
			return false;	//safe to destroy it now
		}
	}
	else
	{
		PP_Resource dlri;

		dl->status = DL_ACTIVE;
		dl->abort = NADL_Cleanup;
		dl->ctx = ctx = Z_Malloc(sizeof(*ctx));

		/*everything goes via nacl, so we might as well just init that here*/
		ctx->req = urlloader->Create(pp_instance);
		dlri = urlrequestinfo->Create(pp_instance);
		urlrequestinfo->SetProperty(dlri, PP_URLREQUESTPROPERTY_ALLOWCROSSORIGINREQUESTS, ppb_var_interface->VarFromUtf8(url, strlen(url)));
		urlrequestinfo->SetProperty(dlri, PP_URLREQUESTPROPERTY_URL, ppb_var_interface->VarFromUtf8(url, strlen(url)));

		struct PP_CompletionCallback ccb = {dlstarted, dl, PP_COMPLETIONCALLBACK_FLAG_NONE};
		urlloader->Open(ctx->req, dlri, ccb);
		ppb_core->ReleaseResource(dlri);
	}

	return true;
}
#else
qboolean DL_Decide(struct dl_download *dl);

/*
This file does one thing. Connects to servers and grabs the specified file. It doesn't do any uploading whatsoever. Live with it.
It doesn't use persistant connections.

*/

struct http_dl_ctx_s {
	struct dl_download *dlctx;

	SOCKET sock;	//FIXME: support https.

	char *buffer;

	int bufferused;
	int bufferlen;

	int totalreceived;	//useful when we're just dumping to a file.

	struct vfsfile_s *file;	//if gzipping, this is a temporary file. we'll write to the real file from this after the transfer is complete.
	qboolean gzip;
	qboolean chunking;
	int chunksize;
	int chunked;

	enum {HC_REQUESTING, HC_GETTINGHEADER, HC_GETTING} state;

	int contentlength;
};

void HTTP_Cleanup(struct dl_download *dl)
{
	struct http_dl_ctx_s *con = dl->ctx;
	dl->ctx = NULL;

	if (con->sock != INVALID_SOCKET)
		closesocket(con->sock);
	con->sock = INVALID_SOCKET;
	free(con->buffer);
	free(con);

	dl->abort = NULL;
	dl->status = DL_PENDING;
	dl->completed = 0;
	dl->totalsize = 0;
}

static void ExpandBuffer(struct http_dl_ctx_s *con, int quant)
{
	int newlen;
	newlen = con->bufferlen + quant;
	con->buffer = realloc(con->buffer, newlen);
	con->bufferlen = newlen;
}

static qboolean HTTP_DL_Work(struct dl_download *dl)
{
	struct http_dl_ctx_s *con = dl->ctx;
	char buffer[256];
	char Location[256];
	char mimetype[256];
	char *nl;
	char *msg;
	int ammount;
	switch(con->state)
	{
	case HC_REQUESTING:
		ammount = send(con->sock, con->buffer, con->bufferused, 0);
		if (!ammount)
			return false;

		if (ammount < 0)
		{
			if (neterrno() != NET_EWOULDBLOCK)
				return false;
			return true;
		}

		con->bufferused -= ammount;
		memmove(con->buffer, con->buffer+ammount, con->bufferused);
		if (!con->bufferused)	//that's it, all sent.
			con->state = HC_GETTINGHEADER;
		break;

	case HC_GETTINGHEADER:
		if (con->bufferlen - con->bufferused < 1530)
			ExpandBuffer(con, 1530);

		ammount = recv(con->sock, con->buffer+con->bufferused, con->bufferlen-con->bufferused-15, 0);
		if (!ammount)
			return false;
		if (ammount < 0)
		{
			if (neterrno() != NET_EWOULDBLOCK)
				return false;
			return true;
		}

		con->bufferused+=ammount;
		con->buffer[con->bufferused] = '\0';
		//have we got the entire thing yet?

		msg = con->buffer;
		con->chunking = false;
		con->contentlength = 0;
		con->gzip = false;
		*mimetype = 0;
		*Location = 0;
		if (strnicmp(msg, "HTTP/", 5))
		{	//pre version 1. (lame servers.
			con->state = HC_GETTING;
			dl->status = DL_ACTIVE;
			con->contentlength = -1;	//meaning end of stream.
			dl->replycode = 200;
		}
		else
		{
			qboolean hcomplete = false;
			while(*msg)
			{
				if (*msg == '\n')
				{
					if (msg[1] == '\n')
					{	//tut tut, not '\r'? that's not really allowed...
						msg+=2;
						hcomplete = true;
						break;
					}
					if (msg[1] == '\r' && msg[2] == '\n')
					{
						msg+=3;
						hcomplete = true;
						break;
					}
					msg++;
				}
				while (*msg == ' ' || *msg == '\t')
					msg++;

				nl = strchr(msg, '\n');
				if (!nl)
					break;//not complete, don't bother trying to parse it.
				if (!strnicmp(msg, "Content-Length: ", 16))
					con->contentlength = atoi(msg+16);
				else if (!strnicmp(msg, "Content-Type:", 13))
				{
					*nl = '\0';
					Q_strncpyz(mimetype, COM_TrimString(msg+13), sizeof(mimetype));
					*nl = '\n';
				}
				else if (!strnicmp(msg, "Location: ", 10))
				{
					*nl = '\0';
					Q_strncpyz(Location, COM_TrimString(msg+10), sizeof(Location));
					*nl = '\n';
				}
				else if (!strnicmp(msg, "Content-Encoding: ", 18))
				{
					char *chunk = strstr(msg, "gzip");
					if (chunk < nl)
						con->gzip = true;
				}
				else if (!strnicmp(msg, "Transfer-Encoding: ", 19))
				{
					char *chunk = strstr(msg, "chunked");
					if (chunk < nl)
						con->chunking = true;
				}
				msg = nl;
			}
			if (!hcomplete)
				break;//headers not complete. break out of switch

			ammount = msg - con->buffer;

			msg = COM_ParseOut(con->buffer, buffer, sizeof(buffer));
			msg = COM_ParseOut(msg, buffer, sizeof(buffer));

			dl->replycode = atoi(buffer);

			if (!stricmp(buffer, "100"))
			{	//http/1.1 servers can give this. We ignore it.
				con->bufferused -= ammount;
				memmove(con->buffer, con->buffer+ammount, con->bufferused);
				return true;
			}

			if (!stricmp(buffer, "301") || !stricmp(buffer, "302") || !stricmp(buffer, "303"))
			{
				nl = strchr(msg, '\n');
				if (nl)
					*nl = '\0';
				Con_Printf("HTTP: %s %s\n", buffer, COM_TrimString(msg));
				if (!*Location)
					Con_Printf("Server redirected to null location\n");
				else
				{
					HTTP_Cleanup(dl);
					if (*Location == '/')
					{
						char *cur = *dl->redir?dl->redir:dl->url;
						char *curserver = cur;
						char *curpath;
						/*same server+protocol*/
						if (!strncmp(curserver, "http://", 7))
							curserver += 7;
						curpath = strchr(curserver, '/');
						if (!curpath)
							curpath = curserver + strlen(curserver);
						if (cur == dl->redir)
							*curpath = 0;
						else
							Q_strncpyz(dl->redir, cur, (curpath-cur) + 1);
						Q_strncatz(dl->redir, Location, sizeof(dl->redir));
					}
					else
						Q_strncpyz(dl->redir, Location, sizeof(dl->redir));
					dl->poll = DL_Decide;
					dl->status = DL_PENDING;
				}
				return true;
			}

			if (stricmp(buffer, "200"))
			{
				nl = strchr(msg, '\n');
				if (!nl)
					return false;	//eh?
				if (nl>msg&&nl[-1] == '\r')
					nl--;
				*nl = '\0';
				Con_Printf("HTTP: %s%s\n", buffer, msg);
				return false;	//something went wrong.
			}

			con->bufferused -= ammount;

			dl->totalsize = con->contentlength;

			memmove(con->buffer, con->buffer+ammount, con->bufferused);
		}

		if (dl->notifystarted)
			dl->notifystarted(dl, *mimetype?mimetype:NULL);

		if (!dl->file)
		{
#ifndef NPFTE
			if (*dl->localname)
			{
				FS_CreatePath(dl->localname, FS_GAME);
				dl->file = FS_OpenVFS(dl->localname, "w+b", FS_GAME);
			}
			else
				dl->file = FS_OpenTemp();
#endif
			if (!dl->file)
			{
				if (*dl->localname)
					Con_Printf("HTTP: Couldn't open file \"%s\"\n", dl->localname);
				else
					Con_Printf("HTTP: Couldn't open temporary file\n");
				dl->status = DL_FAILED;
				return false;
			}
		}

		if (con->gzip)
		{
#if !defined(NPFTE) && defined(AVAIL_ZLIB)
			con->file = FS_OpenTemp();
#else
			Con_Printf("HTTP: no support for gzipped files \"%s\"\n", dl->localname);
			dl->status = DL_FAILED;
			return false;
#endif
		}
		else
			con->file = dl->file;
		con->state = HC_GETTING;
		dl->status = DL_ACTIVE;
		//Fall through
	case HC_GETTING:
		if (con->bufferlen - con->bufferused < 1530)
			ExpandBuffer(con, 1530);

		ammount = recv(con->sock, con->buffer+con->bufferused, con->bufferlen-con->bufferused-1, 0);
		if (ammount < 0)
		{
			if (neterrno() != NET_EWOULDBLOCK)
				return false;
			return true;
		}

		con->bufferused+=ammount;

		if (con->chunking)	//FIXME: NEEDS TESTING!!!
		{
			int trim;
			char *nl;
			con->buffer[con->bufferused] = '\0';
			for(;;)
			{	//work out as we go.
				if (con->chunksize)//we are trying to parse a chunk.
				{
					trim = con->bufferused - con->chunked;
					if (trim > con->chunksize)
						trim = con->chunksize;	//don't go into the next size field.
					con->chunksize -= trim;
					con->chunked += trim;

					if (!con->chunksize)
					{	//we need to find the next \n and trim it.
						nl = strchr(con->buffer+con->chunked, '\n');
						if (!nl)
							break;
						nl++;
						trim = nl - (con->buffer+con->chunked);
						memmove(con->buffer + con->chunked, nl, con->buffer+con->bufferused-nl+1);
						con->bufferused -= trim;
					}
					if (!(con->bufferused - con->chunked))
						break;
				}
				else
				{
					nl = strchr(con->buffer+con->chunked, '\n');
					if (!nl)
						break;
					con->chunksize = strtol(con->buffer+con->chunked, NULL, 16);	//it's hex.
					nl++;
					trim = nl - (con->buffer+con->chunked);
					memmove(con->buffer + con->chunked, nl, con->buffer+con->bufferused-nl+1);
					con->bufferused -= trim;
				}
			}

			con->totalreceived+=con->chunked;
			if (con->file && con->chunked)	//we've got a chunk in the buffer
			{	//write it
				if (VFS_WRITE(con->file, con->buffer, con->chunked) != con->chunked)
				{
					Con_Printf("Write error whilst downloading %s\nDisk full?\n", dl->localname);
					return false;
				}

				//and move the unparsed chunk to the front.
				con->bufferused -= con->chunked;
				memmove(con->buffer, con->buffer+con->chunked, con->bufferused);
				con->chunked = 0;
			}
		}
		else
		{
			con->totalreceived+=ammount;
			if (con->file)	//we've got a chunk in the buffer
			{	//write it
				if (VFS_WRITE(con->file, con->buffer, con->bufferused) != con->bufferused)
				{
					Con_Printf("Write error whilst downloading %s\nDisk full?\n", dl->localname);
					return false;
				}
				con->bufferused = 0;
			}
		}

		if (!ammount)
		{	//server closed off the connection.
			if (con->chunksize)
				dl->status = DL_FAILED;
			else
			{
#if !defined(NPFTE) && defined(AVAIL_ZLIB)
				if (con->gzip)
				{
					VFS_SEEK(con->file, 0);
					dl->file = FS_DecompressGZip(con->file, dl->file);
					con->file = NULL;
				}
#endif
				dl->status = (dl->replycode == 200)?DL_FINISHED:DL_FAILED; 
			}
			return false;
		}
		dl->completed = con->totalreceived;

		break;
	}

	return true;
}

void HTTPDL_Establish(struct dl_download *dl)
{
	unsigned long _true = true;
	struct sockaddr_qstorage	serveraddr;
	struct http_dl_ctx_s *con;
	int addressfamily;
	int addresssize;

	char server[128];
	char uri[MAX_OSPATH];
	char *slash;
	const char *url = dl->redir;
	if (!*url)
		url = dl->url;

	if (!strnicmp(url, "http://", 7))
		url+=7;

	slash = strchr(url, '/');
	if (!slash)
	{
		Q_strncpyz(server, url, sizeof(server));
		Q_strncpyz(uri, "/", sizeof(uri));
	}
	else
	{
		Q_strncpyz(uri, slash, sizeof(uri));
		Q_strncpyz(server, url, sizeof(server));
		server[slash-url] = '\0';
	}

	con = malloc(sizeof(*con));
	memset(con, 0, sizeof(*con));
	dl->ctx = con;
	dl->abort = HTTP_Cleanup;

	dl->status = DL_RESOLVING;

	if (!NET_StringToSockaddr(server, 80, &serveraddr, &addressfamily, &addresssize))
	{
		dl->status = DL_FAILED;
		return;
	}

	dl->status = DL_QUERY;

	if ((con->sock = socket (addressfamily, SOCK_STREAM, IPPROTO_TCP)) == -1)
	{
		dl->status = DL_FAILED;
		return;
	}

	//don't bother binding. its optional.

	//not yet blocking.
	if (connect(con->sock, (struct sockaddr *)&serveraddr, addresssize) == -1)
	{
		int err = neterrno();
		switch(err)
		{
		case NET_EACCES:
			Con_Printf("HTTP: connect(%s): access denied. Check firewall.\n", server);
			break;
		default:
			Con_Printf("HTTP: connect(%s): %s", server, strerror(neterrno()));
			break;
		}
		dl->status = DL_FAILED;
		return;
	}

	if (ioctlsocket (con->sock, FIONBIO, &_true) == -1)	//now make it non blocking.
	{
		dl->status = DL_FAILED;
		return;
	}

	if (dl->postdata)
	{
		ExpandBuffer(con, 1024 + strlen(uri) + strlen(server) + strlen(con->dlctx->postmimetype) + dl->postlen);
		Q_snprintfz(con->buffer, con->bufferlen,
			"POST %s HTTP/1.1\r\n"
			"Host: %s\r\n"
			"Content-Length: %u\r\n"
			"Content-Type: %s\r\n"
			"Connection: close\r\n"
#if !defined(NPFTE) && defined(AVAIL_ZLIB)
			"Accept-Encoding: gzip\r\n"
#endif
			"User-Agent: "FULLENGINENAME"\r\n"
			"\r\n", uri, server, (unsigned int)dl->postlen, dl->postmimetype);
		con->bufferused = strlen(con->buffer);
		memcpy(con->buffer + con->bufferused, dl->postdata, dl->postlen);
		con->bufferused += dl->postlen;
	}
	else
	{
		ExpandBuffer(con, 512*1024);
		Q_snprintfz(con->buffer, con->bufferlen,
			"GET %s HTTP/1.1\r\n"
			"Host: %s\r\n"
			"Connection: close\r\n"
#if !defined(NPFTE) && defined(AVAIL_ZLIB)
			"Accept-Encoding: gzip\r\n"
#endif
			"User-Agent: "FULLENGINENAME"\r\n"
			"\r\n", uri, server);
		con->bufferused = strlen(con->buffer);
	}
	con->contentlength = -1;
}

qboolean HTTPDL_Poll(struct dl_download *dl)
{
	/*failed previously*/
	if (dl->status == DL_FAILED)
		return false;

	if (!dl->ctx)
	{
		HTTPDL_Establish(dl);
		if (dl->status == DL_FAILED)
		{
			HTTP_Cleanup(dl);
			dl->status = DL_FAILED;
			return false;
		}
	}

	if (dl->ctx)
	{
		if (!HTTP_DL_Work(dl))
			if (dl->status != DL_FINISHED)
				dl->status = DL_FAILED;
		if (dl->status == DL_FAILED)
		{
			HTTP_Cleanup(dl);
			dl->status = DL_FAILED;
			return false;
		}
		if (dl->status == DL_FINISHED)
			return false;
	}

	return true;
}

qboolean DL_Decide(struct dl_download *dl)
{
	const char *url = dl->redir;
	if (!*url)
		url = dl->url;

	if (!strnicmp(url, "http://", 7))
		dl->poll = HTTPDL_Poll;
	else
	{
		dl->status = DL_FAILED;
		return false;
	}
	return true;
}
#endif	/*!defined(NACL)*/

#ifdef MULTITHREAD
static int DL_Thread_Work(void *arg)
{
	struct dl_download *dl = arg;

	while (!dl->threaddie)
	{
		if (!dl->poll(dl))
		{
#ifdef NPFTE
			//the plugin doesn't have a download loop
			if (dl->notifycomplete)
				dl->notifycomplete(dl);
			if (dl->file)
				VFS_CLOSE(dl->file);
#else
			if (dl->status != DL_FAILED && dl->status != DL_FINISHED)
				dl->status = DL_FAILED;
#endif
			break;
		}
	}
	return 0;
}

/*create a thread to perform the given download
to use: call DL_Create (not HTTP_CL_Get!) to get a context, then call this.
note that you need to call DL_Close from another thread, NOT IN THE NOTIFY FUNC.
the file handle must be safe to write to in threads.
*/
qboolean DL_CreateThread(struct dl_download *dl, vfsfile_t *file, void (*NotifyFunction)(struct dl_download *dl))
{
	if (!dl)
		return false;

	if (file)
		dl->file = file;
	if (NotifyFunction)
		dl->notifycomplete = NotifyFunction;

	dl->threadctx = Sys_CreateThread("download", DL_Thread_Work, dl, THREADP_NORMAL, 0);
	if (!dl->threadctx)
		return false;

	return true;
}
#endif

/*create a standalone download context*/
struct dl_download *DL_Create(const char *url)
{
	struct dl_download *newdl;
	newdl = malloc(sizeof(*newdl));
	if (!newdl)
		return NULL;
	memset(newdl, 0, sizeof(*newdl));
	Q_strncpyz(newdl->url, url, sizeof(newdl->url));
	newdl->poll = DL_Decide;

	if (!newdl->poll(newdl))
	{
		free(newdl);
		newdl = NULL;
	}

	return newdl;
}
static struct dl_download *showndownload;

/*destroys an entire download context*/
void DL_Close(struct dl_download *dl)
{
#ifndef NPFTE
	if (showndownload == dl)
	{
		if (cls.downloadmethod == DL_HTTP)
		{
			cls.downloadmethod = DL_NONE;
			*cls.downloadlocalname = *cls.downloadremotename = 0;
		}
		showndownload = NULL;
	}
#endif

#ifdef MULTITHREAD
	dl->threaddie = true;
	if (dl->threadctx)
		Sys_WaitOnThread(dl->threadctx);
#endif
	if (dl->abort)
		dl->abort(dl);
	if (dl->file)
		VFS_CLOSE(dl->file);
	if (dl->postdata)
		BZ_Free(dl->postdata);
	free(dl);
}


/*updates pending downloads*/
#ifndef NPFTE

static struct dl_download *activedownloads;
unsigned int shownbytestart;
/*create a download context and add it to the list, for lazy people. not threaded*/
struct dl_download *HTTP_CL_Get(const char *url, const char *localfile, void (*NotifyFunction)(struct dl_download *dl))
{
	struct dl_download *newdl = DL_Create(url);
	if (!newdl)
		return newdl;

	newdl->notifycomplete = NotifyFunction;
	if (localfile)
		Q_strncpyz(newdl->localname, localfile, sizeof(newdl->localname));

	newdl->next = activedownloads;
	activedownloads = newdl;

	return newdl;
}

struct dl_download *HTTP_CL_Put(const char *url, const char *mime, const char *data, size_t datalen, void (*NotifyFunction)(struct dl_download *dl))
{
	struct dl_download *dl;
	if (!*mime)
		return NULL;

	dl = HTTP_CL_Get(url, NULL, NotifyFunction);
	Q_strncpyz(dl->postmimetype, mime, sizeof(dl->postmimetype));
	dl->postdata = BZ_Malloc(datalen);
	memcpy(dl->postdata, data, datalen);
	dl->postlen = datalen;
	return dl;
}

void HTTP_CL_Think(void)
{
	struct dl_download *dl = activedownloads;
	struct dl_download **link = NULL;

	link = &activedownloads;
	while (*link)
	{
		dl = *link;
#ifdef MULTITHREAD
		if (dl->threadctx)
		{
			if (dl->status == DL_FINISHED || dl->status == DL_FAILED)
			{
				Sys_WaitOnThread(dl->threadctx);
				dl->threadctx = NULL;
				continue;
			}
		}
		else 
#endif
		if (!dl->poll(dl))
		{
			*link = dl->next;
			if (dl->file && dl->file->Seek)
				VFS_SEEK(dl->file, 0);
			if (dl->notifycomplete)
				dl->notifycomplete(dl);
			DL_Close(dl);
			continue;
		}
		link = &dl->next;

		if (!cls.downloadmethod)
		{
			cls.downloadmethod = DL_HTTP;
			showndownload = dl;
			if (*dl->localname)
				strcpy(cls.downloadlocalname, dl->localname);
			else
				strcpy(cls.downloadlocalname, dl->url);
			strcpy(cls.downloadremotename, dl->url);
			cls.downloadstarttime = Sys_DoubleTime();
			cls.downloadedbytes = 0;
			shownbytestart = dl->completed;
		}
		if (cls.downloadmethod == DL_HTTP)
		{
			if (showndownload == dl)
			{
				if (dl->status == DL_FINISHED)
					cls.downloadpercent = 100;
				else if (dl->status != DL_ACTIVE)
					cls.downloadpercent = 0;
				else if (dl->totalsize <= 0)
					cls.downloadpercent = 50;
				else
					cls.downloadpercent = dl->completed*100.0f/dl->totalsize;
				cls.downloadedbytes = dl->completed;
			}
		}
	}
}
#endif
#endif	/*WEBCLIENT*/


typedef struct 
{
	vfsfile_t funcs;

	char *data;
	int maxlen;
	int writepos;
	int readpos;
} vfspipe_t;

static qboolean QDECL VFSPIPE_Close(vfsfile_t *f)
{
	vfspipe_t *p = (vfspipe_t*)f;
	free(p->data);
	free(p);
	return true;
}
static qofs_t QDECL VFSPIPE_GetLen(vfsfile_t *f)
{
	vfspipe_t *p = (vfspipe_t*)f;
	return p->writepos - p->readpos;
}
//static unsigned long QDECL VFSPIPE_Tell(vfsfile_t *f)
//{
//	return 0;
//}
//static qboolean QDECL VFSPIPE_Seek(vfsfile_t *f, unsigned long offset)
//{
//	Con_Printf("Seeking is a bad plan, mmkay?\n");
//	return false;
//}
static int QDECL VFSPIPE_ReadBytes(vfsfile_t *f, void *buffer, int len)
{
	vfspipe_t *p = (vfspipe_t*)f;
	if (len > p->writepos - p->readpos)
		len = p->writepos - p->readpos;
	memcpy(buffer, p->data+p->readpos, len);
	p->readpos += len;

	if (p->readpos > 8192)
	{
		//shift the memory down periodically
		//fixme: use cyclic buffer? max size, etc?
		memmove(p->data, p->data+p->readpos, p->writepos-p->readpos);

		p->writepos -= p->readpos;
		p->readpos = 0;
	}
	return len;
}
static int QDECL VFSPIPE_WriteBytes(vfsfile_t *f, const void *buffer, int len)
{
	vfspipe_t *p = (vfspipe_t*)f;
	if (p->writepos + len > p->maxlen)
	{
		p->maxlen = p->writepos + len;
		p->data = realloc(p->data, p->maxlen);
	}
	memcpy(p->data+p->writepos, buffer, len);
	p->writepos += len;
	return len;
}

vfsfile_t *VFSPIPE_Open(void)
{
	vfspipe_t *newf;
	newf = malloc(sizeof(*newf));
	newf->data = NULL;
	newf->maxlen = 0;
	newf->readpos = 0;
	newf->writepos = 0;
	newf->funcs.Close = VFSPIPE_Close;
	newf->funcs.Flush = NULL;
	newf->funcs.GetLen = VFSPIPE_GetLen;
	newf->funcs.ReadBytes = VFSPIPE_ReadBytes;
	newf->funcs.Seek = NULL;//VFSPIPE_Seek;
	newf->funcs.Tell = NULL;//VFSPIPE_Tell;
	newf->funcs.WriteBytes = VFSPIPE_WriteBytes;
	newf->funcs.seekingisabadplan = true;

	return &newf->funcs;
}