fteqw/engine/http/httpclient.c

943 lines
20 KiB
C

#include "quakedef.h"
#include "iweb.h"
#include "netinc.h"
#include "fs.h"
#if defined(WEBCLIENT)
#if defined(FTE_TARGET_WEB)
#define MYJS 1
#if MYJS
#include "web/ftejslib.h"
#else
#include <emscripten/emscripten.h>
#endif
static void DL_Abort(struct dl_download *dl)
{
dl->ctx = NULL;
}
static void DL_OnLoad(void *c, void *data, int datasize)
{
struct dl_download *dl = c;
//make sure the file is 'open'.
if (!dl->file)
{
if (*dl->localname)
{
FS_CreatePath(dl->localname, FS_GAME);
dl->file = FS_OpenVFS(dl->localname, "w+b", FS_GAME);
}
else
{
//emscripten does not close the file. plus we seem to end up with infinite loops.
dl->file = FS_OpenTemp();
}
}
if (dl->file)
{
VFS_WRITE(dl->file, data, datasize);
VFS_SEEK(dl->file, 0);
dl->status = DL_FINISHED;
}
else
dl->status = DL_FAILED;
dl->replycode = 200;
#if !MYJS
dl->completed += datasize;
#endif
}
#if MYJS
static void DL_OnError(void *c, int ecode)
#else
static void DL_OnError(void *c)
#endif
{
struct dl_download *dl = c;
#if MYJS
dl->replycode = ecode;
#else
dl->replycode = 404; //we don't actually know. should we not do this?
#endif
Con_Printf("download %p: error %i\n", dl, dl->replycode);
dl->status = DL_FAILED;
}
static void DL_OnProgress(void *c, int position, int totalsize)
{
struct dl_download *dl = c;
dl->completed = position;
dl->totalsize = totalsize;
}
//this becomes a poll function. the main thread will call this once a frame or so.
qboolean DL_Decide(struct dl_download *dl)
{
const char *url = dl->redir;
if (!*url)
url = dl->url;
if (dl->ctx)
{
if (dl->status == DL_FAILED || dl->status == DL_FINISHED)
{
DL_Abort(dl);
return false; //safe to destroy it now
}
}
else
{
dl->status = DL_ACTIVE;
dl->abort = DL_Abort;
dl->ctx = dl;
#if MYJS
emscriptenfte_async_wget_data2(url, dl, DL_OnLoad, DL_OnError, DL_OnProgress);
#else
//annoyingly, emscripten doesn't provide an onprogress callback, unlike firefox etc, so we can't actually tell how far its got.
//we'd need to provide our own js library to fix this. it can be done, I'm just lazy.
emscripten_async_wget_data(url, dl, DL_OnLoad, DL_OnError);
#endif
}
return true;
}
#elif defined(NACL)
#include <ppapi/c/pp_errors.h>
#include <ppapi/c/ppb_core.h>
#include <ppapi/c/pp_file_info.h>
#include <ppapi/c/ppb_file_system.h>
#include <ppapi/c/ppb_file_ref.h>
#include <ppapi/c/ppb_url_request_info.h>
#include <ppapi/c/ppb_url_response_info.h>
#include <ppapi/c/pp_var.h>
#include <ppapi/c/ppb_var.h>
#include <ppapi/c/ppb_file_io.h>
#include <ppapi/c/ppb_url_loader.h>
extern PPB_Core *ppb_core;
extern PPB_URLRequestInfo *urlrequestinfo;
extern PPB_URLLoader *urlloader;
extern PP_Instance pp_instance;
extern PPB_Var *ppb_var_interface;
struct nacl_dl {
char buffer[65536];
PP_Resource req;
};
static void readfinished(void* user_data, int32_t result)
{
struct dl_download *f = user_data;
struct nacl_dl *ctx = f->ctx;
struct PP_CompletionCallback ccb = {readfinished, f, PP_COMPLETIONCALLBACK_FLAG_NONE};
//trying to clean up
if (!ctx)
return;
// Sys_Printf("lastresult: %i\n", result);
if (result == PP_OK)
{
// Sys_Printf("%s completed\n", f->url);
ppb_core->ReleaseResource(ctx->req);
ctx->req = 0;
f->status = DL_FINISHED;
return;
}
for (; result > 0; result = urlloader->ReadResponseBody(ctx->req, ctx->buffer, sizeof(ctx->buffer), ccb))
{
//make sure the file is 'open'.
if (!f->file)
{
if (*f->localname)
{
FS_CreatePath(f->localname, FS_GAME);
f->file = FS_OpenVFS(f->localname, "w+b", FS_GAME);
}
else
f->file = FS_OpenTemp();
}
// Sys_Printf("write: %i\n", result);
VFS_WRITE(f->file, ctx->buffer, result);
f->completed += result;
}
// Sys_Printf("result: %i\n", result);
if (result != PP_OK_COMPLETIONPENDING)
{
Sys_Printf("file %s failed or something\n", f->url);
ppb_core->ReleaseResource(ctx->req);
ctx->req = 0;
f->status = DL_FAILED;
}
}
//urloader->open completed
static void dlstarted(void* user_data, int32_t result)
{
struct dl_download *f = user_data;
struct nacl_dl *ctx = f->ctx;
struct PP_CompletionCallback ccb = {readfinished, f, PP_COMPLETIONCALLBACK_FLAG_NONE};
readfinished(user_data, urlloader->ReadResponseBody(ctx->req, ctx->buffer, sizeof(ctx->buffer), ccb));
}
static void nadl_cleanup(void* user_data, int32_t result)
{
struct nacl_dl *ctx = user_data;
if (ctx->req)
ppb_core->ReleaseResource(ctx->req);
free(ctx);
}
void NADL_Cleanup(struct dl_download *dl)
{
struct nacl_dl *ctx = dl->ctx;
//we can't free the ctx memory etc, in case the browser still has requests pending on it before it handles our close.
//so set up a callback to do it later
dl->ctx = NULL; //orphan
struct PP_CompletionCallback ccb = {nadl_cleanup, ctx, PP_COMPLETIONCALLBACK_FLAG_NONE};
ppb_core->CallOnMainThread(1000, ccb, 0);
}
qboolean DL_Decide(struct dl_download *dl)
{
const char *url = dl->redir;
struct nacl_dl *ctx;
if (!*url)
url = dl->url;
if (dl->ctx)
{
if (dl->status == DL_FAILED || dl->status == DL_FINISHED)
{
NADL_Cleanup(dl);
return false; //safe to destroy it now
}
}
else
{
PP_Resource dlri;
dl->status = DL_ACTIVE;
dl->abort = NADL_Cleanup;
dl->ctx = ctx = Z_Malloc(sizeof(*ctx));
/*everything goes via nacl, so we might as well just init that here*/
ctx->req = urlloader->Create(pp_instance);
dlri = urlrequestinfo->Create(pp_instance);
urlrequestinfo->SetProperty(dlri, PP_URLREQUESTPROPERTY_ALLOWCROSSORIGINREQUESTS, ppb_var_interface->VarFromUtf8(url, strlen(url)));
urlrequestinfo->SetProperty(dlri, PP_URLREQUESTPROPERTY_URL, ppb_var_interface->VarFromUtf8(url, strlen(url)));
struct PP_CompletionCallback ccb = {dlstarted, dl, PP_COMPLETIONCALLBACK_FLAG_NONE};
urlloader->Open(ctx->req, dlri, ccb);
ppb_core->ReleaseResource(dlri);
}
return true;
}
#else
qboolean DL_Decide(struct dl_download *dl);
/*
This file does one thing. Connects to servers and grabs the specified file. It doesn't do any uploading whatsoever. Live with it.
It doesn't use persistant connections.
*/
struct http_dl_ctx_s {
struct dl_download *dlctx;
SOCKET sock;
char *buffer;
int bufferused;
int bufferlen;
int totalreceived; //useful when we're just dumping to a file.
struct vfsfile_s *file; //if gzipping, this is a temporary file. we'll write to the real file from this after the transfer is complete.
qboolean gzip;
qboolean chunking;
int chunksize;
int chunked;
enum {HC_REQUESTING, HC_GETTINGHEADER, HC_GETTING} state;
int contentlength;
};
void HTTP_Cleanup(struct dl_download *dl)
{
struct http_dl_ctx_s *con = dl->ctx;
dl->ctx = NULL;
if (con->sock != INVALID_SOCKET)
closesocket(con->sock);
con->sock = INVALID_SOCKET;
free(con->buffer);
free(con);
dl->abort = NULL;
dl->status = DL_PENDING;
dl->completed = 0;
dl->totalsize = 0;
}
static void ExpandBuffer(struct http_dl_ctx_s *con, int quant)
{
int newlen;
newlen = con->bufferlen + quant;
con->buffer = realloc(con->buffer, newlen);
con->bufferlen = newlen;
}
static qboolean HTTP_DL_Work(struct dl_download *dl)
{
struct http_dl_ctx_s *con = dl->ctx;
char buffer[256];
char Location[256];
char *nl;
char *msg;
int ammount;
switch(con->state)
{
case HC_REQUESTING:
ammount = send(con->sock, con->buffer, con->bufferused, 0);
if (!ammount)
return false;
if (ammount < 0)
{
if (qerrno != EWOULDBLOCK)
return false;
return true;
}
con->bufferused -= ammount;
memmove(con->buffer, con->buffer+ammount, con->bufferused);
if (!con->bufferused) //that's it, all sent.
con->state = HC_GETTINGHEADER;
break;
case HC_GETTINGHEADER:
if (con->bufferlen - con->bufferused < 1530)
ExpandBuffer(con, 1530);
ammount = recv(con->sock, con->buffer+con->bufferused, con->bufferlen-con->bufferused-15, 0);
if (!ammount)
return false;
if (ammount < 0)
{
if (qerrno != EWOULDBLOCK)
return false;
return true;
}
con->bufferused+=ammount;
con->buffer[con->bufferused] = '\0';
//have we got the entire thing yet?
msg = con->buffer;
con->chunking = false;
if (strnicmp(msg, "HTTP/", 5))
{ //pre version 1. (lame servers.
con->state = HC_GETTING;
dl->status = DL_ACTIVE;
con->contentlength = -1; //meaning end of stream.
dl->replycode = 200;
}
else
{
qboolean hcomplete = false;
while(*msg)
{
if (*msg == '\n')
{
if (msg[1] == '\n')
{ //tut tut, not '\r'? that's not really allowed...
msg+=2;
hcomplete = true;
break;
}
if (msg[1] == '\r' && msg[2] == '\n')
{
msg+=3;
hcomplete = true;
break;
}
msg++;
}
while (*msg == ' ' || *msg == '\t')
msg++;
nl = strchr(msg, '\n');
if (!nl)
break;//not complete, don't bother trying to parse it.
if (!strnicmp(msg, "Content-Length: ", 16))
con->contentlength = atoi(msg+16);
else if (!strnicmp(msg, "Location: ", 10))
{
*nl = '\0';
Q_strncpyz(Location, COM_TrimString(msg+10), sizeof(Location));
*nl = '\n';
}
else if (!strnicmp(msg, "Content-Encoding: ", 18))
{
char *chunk = strstr(msg, "gzip");
if (chunk < nl)
con->gzip = true;
}
else if (!strnicmp(msg, "Transfer-Encoding: ", 19))
{
char *chunk = strstr(msg, "chunked");
if (chunk < nl)
con->chunking = true;
}
msg = nl;
}
if (!hcomplete)
break;//headers not complete. break out of switch
ammount = msg - con->buffer;
msg = COM_ParseOut(con->buffer, buffer, sizeof(buffer));
msg = COM_ParseOut(msg, buffer, sizeof(buffer));
dl->replycode = atoi(buffer);
if (!stricmp(buffer, "100"))
{ //http/1.1 servers can give this. We ignore it.
con->bufferused -= ammount;
memmove(con->buffer, con->buffer+ammount, con->bufferused);
return true;
}
if (!stricmp(buffer, "301") || !stricmp(buffer, "302") || !stricmp(buffer, "303"))
{
nl = strchr(msg, '\n');
if (nl)
*nl = '\0';
Con_Printf("HTTP: %s %s\n", buffer, COM_TrimString(msg));
if (!*Location)
Con_Printf("Server redirected to null location\n");
else
{
HTTP_Cleanup(dl);
if (*Location == '/')
{
char *cur = *dl->redir?dl->redir:dl->url;
char *curserver = cur;
char *curpath;
/*same server+protocol*/
if (!strncmp(curserver, "http://", 7))
curserver += 7;
curpath = strchr(curserver, '/');
if (!curpath)
curpath = curserver + strlen(curserver);
if (cur == dl->redir)
*curpath = 0;
else
Q_strncpyz(dl->redir, cur, (curpath-cur) + 1);
Q_strncatz(dl->redir, Location, sizeof(dl->redir));
}
else
Q_strncpyz(dl->redir, Location, sizeof(dl->redir));
dl->poll = DL_Decide;
dl->status = DL_PENDING;
}
return true;
}
if (stricmp(buffer, "200"))
{
nl = strchr(msg, '\n');
if (!nl)
return false; //eh?
if (nl>msg&&nl[-1] == '\r')
nl--;
*nl = '\0';
Con_Printf("HTTP: %s%s\n", buffer, msg);
return false; //something went wrong.
}
con->bufferused -= ammount;
dl->totalsize = con->contentlength;
memmove(con->buffer, con->buffer+ammount, con->bufferused);
}
if (!dl->file)
{
#ifndef NPFTE
if (*dl->localname)
{
FS_CreatePath(dl->localname, FS_GAME);
dl->file = FS_OpenVFS(dl->localname, "w+b", FS_GAME);
}
else
dl->file = FS_OpenTemp();
#endif
if (!dl->file)
{
Con_Printf("HTTP: Couldn't open file \"%s\"\n", dl->localname);
dl->status = DL_FAILED;
return false;
}
}
if (con->gzip)
{
#ifdef NPFTE
Con_Printf("HTTP: no support for gzipped files \"%s\"\n", dl->localname);
#else
con->file = FS_OpenTemp();
#endif
}
else
con->file = dl->file;
con->state = HC_GETTING;
dl->status = DL_ACTIVE;
//Fall through
case HC_GETTING:
if (con->bufferlen - con->bufferused < 1530)
ExpandBuffer(con, 1530);
ammount = recv(con->sock, con->buffer+con->bufferused, con->bufferlen-con->bufferused-1, 0);
if (ammount < 0)
{
if (qerrno != EWOULDBLOCK)
return false;
return true;
}
con->bufferused+=ammount;
if (con->chunking) //FIXME: NEEDS TESTING!!!
{
int trim;
char *nl;
con->buffer[con->bufferused] = '\0';
for(;;)
{ //work out as we go.
if (con->chunksize)//we are trying to parse a chunk.
{
trim = con->bufferused - con->chunked;
if (trim > con->chunksize)
trim = con->chunksize; //don't go into the next size field.
con->chunksize -= trim;
con->chunked += trim;
if (!con->chunksize)
{ //we need to find the next \n and trim it.
nl = strchr(con->buffer+con->chunked, '\n');
if (!nl)
break;
nl++;
trim = nl - (con->buffer+con->chunked);
memmove(con->buffer + con->chunked, nl, con->buffer+con->bufferused-nl+1);
con->bufferused -= trim;
}
if (!(con->bufferused - con->chunked))
break;
}
else
{
nl = strchr(con->buffer+con->chunked, '\n');
if (!nl)
break;
con->chunksize = strtol(con->buffer+con->chunked, NULL, 16); //it's hex.
nl++;
trim = nl - (con->buffer+con->chunked);
memmove(con->buffer + con->chunked, nl, con->buffer+con->bufferused-nl+1);
con->bufferused -= trim;
}
}
con->totalreceived+=con->chunked;
if (con->file && con->chunked) //we've got a chunk in the buffer
{ //write it
if (VFS_WRITE(con->file, con->buffer, con->chunked) != con->chunked)
{
Con_Printf("Write error whilst downloading %s\nDisk full?\n", dl->localname);
return false;
}
//and move the unparsed chunk to the front.
con->bufferused -= con->chunked;
memmove(con->buffer, con->buffer+con->chunked, con->bufferused);
con->chunked = 0;
}
}
else
{
con->totalreceived+=ammount;
if (con->file) //we've got a chunk in the buffer
{ //write it
if (VFS_WRITE(con->file, con->buffer, con->bufferused) != con->bufferused)
{
Con_Printf("Write error whilst downloading %s\nDisk full?\n", dl->localname);
return false;
}
con->bufferused = 0;
}
}
if (!ammount)
{ //server closed off the connection.
if (con->chunksize)
dl->status = DL_FAILED;
else
{
#ifndef NPFTE
if (con->gzip)
{
VFS_SEEK(con->file, 0);
dl->file = FS_DecompressGZip(con->file, dl->file);
con->file = NULL;
}
#endif
dl->status = (dl->replycode == 200)?DL_FINISHED:DL_FAILED;
}
return false;
}
dl->completed = con->totalreceived;
break;
}
return true;
}
void HTTPDL_Establish(struct dl_download *dl)
{
unsigned long _true = true;
struct sockaddr_qstorage serveraddr;
struct http_dl_ctx_s *con;
int addressfamily;
int addresssize;
char server[128];
char uri[MAX_OSPATH];
char *slash;
const char *url = dl->redir;
if (!*url)
url = dl->url;
if (!strnicmp(url, "http://", 7))
url+=7;
slash = strchr(url, '/');
if (!slash)
{
Q_strncpyz(server, url, sizeof(server));
Q_strncpyz(uri, "/", sizeof(uri));
}
else
{
Q_strncpyz(uri, slash, sizeof(uri));
Q_strncpyz(server, url, sizeof(server));
server[slash-url] = '\0';
}
con = malloc(sizeof(*con));
memset(con, 0, sizeof(*con));
dl->ctx = con;
dl->abort = HTTP_Cleanup;
dl->status = DL_RESOLVING;
if (!NET_StringToSockaddr(server, 80, &serveraddr, &addressfamily, &addresssize))
{
dl->status = DL_FAILED;
return;
}
dl->status = DL_QUERY;
if ((con->sock = socket (addressfamily, SOCK_STREAM, IPPROTO_TCP)) == -1)
{
dl->status = DL_FAILED;
return;
}
//don't bother binding. its optional.
//not yet blocking.
if (connect(con->sock, (struct sockaddr *)&serveraddr, addresssize) == -1)
{
dl->status = DL_FAILED;
return;
}
if (ioctlsocket (con->sock, FIONBIO, &_true) == -1) //now make it non blocking.
{
dl->status = DL_FAILED;
return;
}
ExpandBuffer(con, 512*1024);
sprintf(con->buffer,
"GET %s HTTP/1.1\r\n"
"Host: %s\r\n"
"Connection: close\r\n"
"Accept-Encoding: gzip\r\n"
"User-Agent: "FULLENGINENAME"\r\n"
"\r\n", uri, server);
con->bufferused = strlen(con->buffer);
con->contentlength = -1;
}
qboolean HTTPDL_Poll(struct dl_download *dl)
{
/*failed previously*/
if (dl->status == DL_FAILED)
return false;
if (!dl->ctx)
{
HTTPDL_Establish(dl);
if (dl->status == DL_FAILED)
{
HTTP_Cleanup(dl);
dl->status = DL_FAILED;
return false;
}
}
if (dl->ctx)
{
if (!HTTP_DL_Work(dl))
if (dl->status != DL_FINISHED)
dl->status = DL_FAILED;
if (dl->status == DL_FAILED)
{
HTTP_Cleanup(dl);
dl->status = DL_FAILED;
return false;
}
if (dl->status == DL_FINISHED)
return false;
}
return true;
}
qboolean DL_Decide(struct dl_download *dl)
{
const char *url = dl->redir;
if (!*url)
url = dl->url;
if (!strnicmp(url, "http://", 7))
dl->poll = HTTPDL_Poll;
else
{
dl->status = DL_FAILED;
return false;
}
return true;
}
#endif /*!defined(NACL)*/
#ifdef MULTITHREAD
static int DL_Thread_Work(void *arg)
{
struct dl_download *dl = arg;
while (!dl->threaddie)
{
if (!dl->poll(dl))
{
#ifdef NPFTE
//the plugin doesn't have a download loop
if (dl->notify)
dl->notify(dl);
if (dl->file)
VFS_CLOSE(dl->file);
#else
if (dl->status != DL_FAILED && dl->status != DL_FINISHED)
dl->status = DL_FAILED;
#endif
break;
}
}
return 0;
}
/*create a thread to perform the given download
to use: call DL_Create (not HTTP_CL_Get!) to get a context, then call this.
note that you need to call DL_Close from another thread, NOT IN THE NOTIFY FUNC.
the file handle must be safe to write to in threads.
*/
qboolean DL_CreateThread(struct dl_download *dl, vfsfile_t *file, void (*NotifyFunction)(struct dl_download *dl))
{
if (!dl)
return false;
if (file)
dl->file = file;
if (NotifyFunction)
dl->notify = NotifyFunction;
dl->threadctx = Sys_CreateThread("download", DL_Thread_Work, dl, THREADP_NORMAL, 0);
if (!dl->threadctx)
return false;
return true;
}
#endif
/*create a standalone download context*/
struct dl_download *DL_Create(const char *url)
{
struct dl_download *newdl;
newdl = malloc(sizeof(*newdl));
if (!newdl)
return NULL;
memset(newdl, 0, sizeof(*newdl));
Q_strncpyz(newdl->url, url, sizeof(newdl->url));
newdl->poll = DL_Decide;
if (!newdl->poll(newdl))
{
free(newdl);
newdl = NULL;
}
return newdl;
}
static struct dl_download *showndownload;
/*destroys an entire download context*/
void DL_Close(struct dl_download *dl)
{
#ifndef NPFTE
if (showndownload == dl)
{
if (cls.downloadmethod == DL_HTTP)
{
cls.downloadmethod = DL_NONE;
*cls.downloadlocalname = *cls.downloadremotename = 0;
}
showndownload = NULL;
}
#endif
#ifdef MULTITHREAD
dl->threaddie = true;
if (dl->threadctx)
Sys_WaitOnThread(dl->threadctx);
#endif
if (dl->abort)
dl->abort(dl);
if (dl->file)
VFS_CLOSE(dl->file);
free(dl);
}
/*updates pending downloads*/
#ifndef NPFTE
static struct dl_download *activedownloads;
unsigned int shownbytestart;
/*create a download context and add it to the list, for lazy people*/
struct dl_download *HTTP_CL_Get(const char *url, const char *localfile, void (*NotifyFunction)(struct dl_download *dl))
{
struct dl_download *newdl = DL_Create(url);
if (!newdl)
return newdl;
newdl->notify = NotifyFunction;
if (localfile)
Q_strncpyz(newdl->localname, localfile, sizeof(newdl->localname));
newdl->next = activedownloads;
activedownloads = newdl;
return newdl;
}
void HTTP_CL_Think(void)
{
struct dl_download *dl = activedownloads;
struct dl_download **link = NULL;
link = &activedownloads;
while (*link)
{
dl = *link;
#ifdef MULTITHREAD
if (dl->threadctx)
{
if (dl->status == DL_FINISHED || dl->status == DL_FAILED)
{
Sys_WaitOnThread(dl->threadctx);
dl->threadctx = NULL;
continue;
}
}
else
#endif
if (!dl->poll(dl))
{
*link = dl->next;
if (dl->file)
VFS_SEEK(dl->file, 0);
if (dl->notify)
dl->notify(dl);
DL_Close(dl);
continue;
}
link = &dl->next;
if (!cls.downloadmethod)
{
cls.downloadmethod = DL_HTTP;
showndownload = dl;
if (*dl->localname)
strcpy(cls.downloadlocalname, dl->localname);
else
strcpy(cls.downloadlocalname, dl->url);
strcpy(cls.downloadremotename, dl->url);
cls.downloadstarttime = Sys_DoubleTime();
cls.downloadedbytes = 0;
shownbytestart = dl->completed;
}
if (cls.downloadmethod == DL_HTTP)
{
if (showndownload == dl)
{
if (dl->status == DL_FINISHED)
cls.downloadpercent = 100;
else if (dl->status != DL_ACTIVE)
cls.downloadpercent = 0;
else if (dl->totalsize <= 0)
cls.downloadpercent = 50;
else
cls.downloadpercent = dl->completed*100.0f/dl->totalsize;
cls.downloadedbytes = dl->completed;
}
}
}
}
#endif
#endif /*WEBCLIENT*/