From 73d5ad3b5c937afceb95569dd55ce8603da17357 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Nov 2014 16:12:54 -0800 Subject: [PATCH 1/5] Make the HTTP client more generic. This way we can send other headers and use HEAD requests. --- base/buffer.cpp | 5 + base/buffer.h | 1 + net/http_client.cpp | 253 +++++++++++++++++++++++++------------------- net/http_client.h | 8 ++ 4 files changed, 158 insertions(+), 109 deletions(-) diff --git a/base/buffer.cpp b/base/buffer.cpp index 6de2ea48a8..0c2cb830d5 100644 --- a/base/buffer.cpp +++ b/base/buffer.cpp @@ -57,11 +57,16 @@ void Buffer::Take(size_t length, std::string *dest) { } dest->resize(length); if (length > 0) { + Take(length, &(*dest)[0]); memcpy(&(*dest)[0], &data_[0], length); data_.erase(data_.begin(), data_.begin() + length); } } +void Buffer::Take(size_t length, char *dest) { + memcpy(dest, &data_[0], length); +} + int Buffer::TakeLineCRLF(std::string *dest) { int after_next_line = OffsetToAfterNextCRLF(); if (after_next_line < 0) diff --git a/base/buffer.h b/base/buffer.h index 73fbe898fb..6021c76699 100644 --- a/base/buffer.h +++ b/base/buffer.h @@ -38,6 +38,7 @@ class Buffer { // Takers void Take(size_t length, std::string *dest); + void Take(size_t length, char *dest); void TakeAll(std::string *dest) { Take(size(), dest); } // On failure, return value < 0 and *dest is unchanged. // Strips off the actual CRLF from the result. diff --git a/net/http_client.cpp b/net/http_client.cpp index 59b6257529..88f18c5d24 100644 --- a/net/http_client.cpp +++ b/net/http_client.cpp @@ -105,17 +105,18 @@ void Connection::Disconnect() { namespace http { -Client::Client() { +// TODO: do something sane here +#define USERAGENT "NATIVEAPP 1.0" +Client::Client() { + httpVersion_ = "1.1"; + userAgent_ = USERAGENT; } Client::~Client() { Disconnect(); } -// TODO: do something sane here -#define USERAGENT "NATIVEAPP 1.0" - void DeChunk(Buffer *inbuffer, Buffer *outbuffer, int contentLength, float *progress) { int dechunkedBytes = 0; @@ -144,118 +145,25 @@ void DeChunk(Buffer *inbuffer, Buffer *outbuffer, int contentLength, float *prog } int Client::GET(const char *resource, Buffer *output, float *progress) { - if (progress) { - *progress = 0.01f; - } - - Buffer buffer; - const char *tpl = - "GET %s HTTP/1.1\r\n" - "Host: %s\r\n" - "User-Agent: " USERAGENT "\r\n" + const char *otherHeaders = "Accept: */*\r\n" - "Accept-Encoding: gzip\r\n" - "Connection: close\r\n" - "\r\n"; - - buffer.Printf(tpl, resource, host_.c_str()); - bool flushed = buffer.FlushSocket(sock()); - if (!flushed) { - return -1; // TODO error code. + "Accept-Encoding: gzip\r\n"; + int err = SendRequest("GET", resource, otherHeaders, progress); + if (err < 0) { + return err; } Buffer readbuf; - - // Snarf all the data we can into RAM. A little unsafe but hey. - if (readbuf.Read(sock(), 4096) < 0) { - ELOG("Failed to read HTTP headers :("); - return -1; + std::vector responseHeaders; + int code = ReadResponseHeaders(&readbuf, responseHeaders, progress); + if (code < 0) { + return code; } - // Grab the first header line that contains the http code. - - // Skip the header. TODO: read HTTP code and file size so we can make progress bars. - - std::string line; - readbuf.TakeLineCRLF(&line); - - int code; - size_t code_pos = line.find(' '); - if (code_pos != line.npos) { - code_pos = line.find_first_not_of(' ', code_pos); + err = ReadResponseEntity(&readbuf, responseHeaders, output, progress); + if (err < 0) { + return err; } - - if (code_pos != line.npos) { - code = atoi(&line[code_pos]); - } else { - return -1; - } - - bool gzip = false; - bool chunked = false; - int contentLength = 0; - while (true) { - int sz = readbuf.TakeLineCRLF(&line); - if (!sz) - break; - // TODO: Case folding. - if (startsWith(line, "Content-Length:")) { - size_t size_pos = line.find_first_of(' '); - if (size_pos != line.npos) { - size_pos = line.find_first_not_of(' ', size_pos); - } - if (size_pos != line.npos) { - contentLength = atoi(&line[size_pos]); - chunked = false; - } - } else if (startsWith(line, "Content-Encoding:")) { - if (line.find("gzip") != std::string::npos) { - gzip = true; - } - } else if (startsWith(line, "Transfer-Encoding:")) { - if (line.find("chunked") != std::string::npos) { - chunked = true; - } - } - } - - if (!contentLength && progress) { - // Content length is unknown. - // Set progress to 1% so it looks like something is happening... - *progress = 0.1f; - } - - if (!contentLength) { - // No way to know how far along we are. Let's just not update the progress counter. - if (!readbuf.ReadAll(sock())) - return -1; - } else { - // Let's read in chunks, updating progress between each. - if (!readbuf.ReadAllWithProgress(sock(), contentLength, progress)) - return -1; - } - - // output now contains the rest of the reply. Dechunk it. - if (chunked) { - DeChunk(&readbuf, output, contentLength, progress); - } else { - output->Append(readbuf); - } - - // If it's gzipped, we decompress it and put it back in the buffer. - if (gzip) { - std::string compressed, decompressed; - output->TakeAll(&compressed); - bool result = decompress_string(compressed, &decompressed); - if (!result) { - ELOG("Error decompressing using zlib"); - *progress = 0.0f; - return -1; - } - output->Append(decompressed); - } - - *progress = 1.0f; return code; } @@ -305,6 +213,133 @@ int Client::POST(const char *resource, const std::string &data, Buffer *output) return POST(resource, data, "", output); } +int Client::SendRequest(const char *method, const char *resource, const char *otherHeaders, float *progress) { + if (progress) { + *progress = 0.01f; + } + + Buffer buffer; + const char *tpl = + "%s %s HTTP/1.1\r\n" + "Host: %s\r\n" + "User-Agent: " USERAGENT "\r\n" + "Connection: close\r\n" + "%s" + "\r\n"; + + buffer.Printf(tpl, method, resource, host_.c_str(), otherHeaders ? otherHeaders : ""); + bool flushed = buffer.FlushSocket(sock()); + if (!flushed) { + return -1; // TODO error code. + } + return 0; +} + +int Client::ReadResponseHeaders(Buffer *readbuf, std::vector &responseHeaders, float *progress) { + // Snarf all the data we can into RAM. A little unsafe but hey. + if (readbuf->Read(sock(), 4096) < 0) { + ELOG("Failed to read HTTP headers :("); + return -1; + } + + // Grab the first header line that contains the http code. + + // Skip the header. TODO: read HTTP code and file size so we can make progress bars. + + std::string line; + readbuf->TakeLineCRLF(&line); + + int code; + size_t code_pos = line.find(' '); + if (code_pos != line.npos) { + code_pos = line.find_first_not_of(' ', code_pos); + } + + if (code_pos != line.npos) { + code = atoi(&line[code_pos]); + } else { + return -1; + } + + // TODO + while (true) { + int sz = readbuf->TakeLineCRLF(&line); + if (!sz) + break; + responseHeaders.push_back(line); + } + + return code; +} + +int Client::ReadResponseEntity(Buffer *readbuf, const std::vector &responseHeaders, Buffer *output, float *progress) { + bool gzip = false; + bool chunked = false; + int contentLength = 0; + for (std::string line : responseHeaders) { + // TODO: Case folding. + if (startsWith(line, "Content-Length:")) { + size_t size_pos = line.find_first_of(' '); + if (size_pos != line.npos) { + size_pos = line.find_first_not_of(' ', size_pos); + } + if (size_pos != line.npos) { + contentLength = atoi(&line[size_pos]); + chunked = false; + } + } else if (startsWith(line, "Content-Encoding:")) { + if (line.find("gzip") != std::string::npos) { + gzip = true; + } + } else if (startsWith(line, "Transfer-Encoding:")) { + if (line.find("chunked") != std::string::npos) { + chunked = true; + } + } + } + + if (!contentLength && progress) { + // Content length is unknown. + // Set progress to 1% so it looks like something is happening... + *progress = 0.1f; + } + + if (!contentLength || !progress) { + // No way to know how far along we are. Let's just not update the progress counter. + if (!readbuf->ReadAll(sock())) + return -1; + } else { + // Let's read in chunks, updating progress between each. + if (!readbuf->ReadAllWithProgress(sock(), contentLength, progress)) + return -1; + } + + // output now contains the rest of the reply. Dechunk it. + if (chunked) { + DeChunk(readbuf, output, contentLength, progress); + } else { + output->Append(*readbuf); + } + + // If it's gzipped, we decompress it and put it back in the buffer. + if (gzip) { + std::string compressed, decompressed; + output->TakeAll(&compressed); + bool result = decompress_string(compressed, &decompressed); + if (!result) { + ELOG("Error decompressing using zlib"); + *progress = 0.0f; + return -1; + } + output->Append(decompressed); + } + + if (progress) { + *progress = 1.0f; + } + return 0; +} + Download::Download(const std::string &url, const std::string &outfile) : progress_(0.0f), url_(url), outfile_(outfile), resultCode_(0), completed_(false), failed_(false), cancelled_(false), hidden_(false) { } diff --git a/net/http_client.h b/net/http_client.h index cc6c607520..7425115785 100644 --- a/net/http_client.h +++ b/net/http_client.h @@ -66,6 +66,14 @@ public: int POST(const char *resource, const std::string &data, Buffer *output); // HEAD, PUT, DELETE aren't implemented yet. + + int SendRequest(const char *method, const char *resource, const char *otherHeaders = nullptr, float *progress = nullptr); + int ReadResponseHeaders(Buffer *readbuf, std::vector &responseHeaders, float *progress = nullptr); + // If your response contains a response, you must read it. + int ReadResponseEntity(Buffer *readbuf, const std::vector &responseHeaders, Buffer *output, float *progress = nullptr); + + const char *userAgent_; + const char *httpVersion_; }; // Not particularly efficient, but hey - it's a background download, that's pretty cool :P From c85fb55502feaea7c8a36004957229df4dc2b334 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Mon, 24 Nov 2014 07:47:57 -0800 Subject: [PATCH 2/5] Fix Take() methods. --- base/buffer.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/base/buffer.cpp b/base/buffer.cpp index 0c2cb830d5..54d062036d 100644 --- a/base/buffer.cpp +++ b/base/buffer.cpp @@ -58,13 +58,12 @@ void Buffer::Take(size_t length, std::string *dest) { dest->resize(length); if (length > 0) { Take(length, &(*dest)[0]); - memcpy(&(*dest)[0], &data_[0], length); - data_.erase(data_.begin(), data_.begin() + length); } } void Buffer::Take(size_t length, char *dest) { memcpy(dest, &data_[0], length); + data_.erase(data_.begin(), data_.begin() + length); } int Buffer::TakeLineCRLF(std::string *dest) { From 3de556b248f0c83405451e46ce5e56168d0ed0d1 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Tue, 25 Nov 2014 00:30:13 -0800 Subject: [PATCH 3/5] Handle commonly mixed-case headers better. Sometimes you'll see Content-length, etc. --- base/stringutil.h | 15 +++++++++++++++ net/http_client.cpp | 9 +++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/base/stringutil.h b/base/stringutil.h index 3cd9a9f3d4..24b6092096 100644 --- a/base/stringutil.h +++ b/base/stringutil.h @@ -12,6 +12,7 @@ #ifdef _MSC_VER #pragma warning (disable:4996) +#define strncasecmp _strnicmp #endif #ifdef BLACKBERRY @@ -74,6 +75,20 @@ inline bool endsWith(const std::string &str, const std::string &what) { return str.substr(str.size() - what.size()) == what; } +// Only use on strings where you're only concerned about ASCII. +inline bool startsWithNoCase(const std::string &str, const std::string &what) { + if (str.size() < what.size()) + return false; + return strncasecmp(str.c_str(), what.c_str(), what.size()) == 0; +} + +inline bool endsWithNoCase(const std::string &str, const std::string &what) { + if (str.size() < what.size()) + return false; + const size_t offset = str.size() - what.size(); + return strncasecmp(str.c_str() + offset, what.c_str(), what.size()) == 0; +} + void DataToHexString(const uint8 *data, size_t size, std::string *output); inline void StringToHexString(const std::string &data, std::string *output) { DataToHexString((uint8_t *)(&data[0]), data.size(), output); diff --git a/net/http_client.cpp b/net/http_client.cpp index 88f18c5d24..081958b00b 100644 --- a/net/http_client.cpp +++ b/net/http_client.cpp @@ -277,8 +277,7 @@ int Client::ReadResponseEntity(Buffer *readbuf, const std::vector & bool chunked = false; int contentLength = 0; for (std::string line : responseHeaders) { - // TODO: Case folding. - if (startsWith(line, "Content-Length:")) { + if (startsWithNoCase(line, "Content-Length:")) { size_t size_pos = line.find_first_of(' '); if (size_pos != line.npos) { size_pos = line.find_first_not_of(' ', size_pos); @@ -287,11 +286,13 @@ int Client::ReadResponseEntity(Buffer *readbuf, const std::vector & contentLength = atoi(&line[size_pos]); chunked = false; } - } else if (startsWith(line, "Content-Encoding:")) { + } else if (startsWithNoCase(line, "Content-Encoding:")) { + // TODO: Case folding... if (line.find("gzip") != std::string::npos) { gzip = true; } - } else if (startsWith(line, "Transfer-Encoding:")) { + } else if (startsWithNoCase(line, "Transfer-Encoding:")) { + // TODO: Case folding... if (line.find("chunked") != std::string::npos) { chunked = true; } From b88c792f0ed2239a98cb190298e63ef52af026d1 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Tue, 25 Nov 2014 08:37:59 -0800 Subject: [PATCH 4/5] Actually use httpVersion_ and userAgent_. --- net/http_client.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/net/http_client.cpp b/net/http_client.cpp index 081958b00b..84879441d0 100644 --- a/net/http_client.cpp +++ b/net/http_client.cpp @@ -220,14 +220,14 @@ int Client::SendRequest(const char *method, const char *resource, const char *ot Buffer buffer; const char *tpl = - "%s %s HTTP/1.1\r\n" + "%s %s HTTP/%s\r\n" "Host: %s\r\n" - "User-Agent: " USERAGENT "\r\n" + "User-Agent: %s\r\n" "Connection: close\r\n" "%s" "\r\n"; - buffer.Printf(tpl, method, resource, host_.c_str(), otherHeaders ? otherHeaders : ""); + buffer.Printf(tpl, method, resource, httpVersion_, host_.c_str(), userAgent_, otherHeaders ? otherHeaders : ""); bool flushed = buffer.FlushSocket(sock()); if (!flushed) { return -1; // TODO error code. From 1150598ce463c99fbb08ba4177a7937692fe9bc0 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Tue, 25 Nov 2014 08:49:05 -0800 Subject: [PATCH 5/5] Migrate POST to use HTTP/1.1, etc. --- net/http_client.cpp | 66 +++++++++++++++++++-------------------------- net/http_client.h | 9 ++++--- 2 files changed, 33 insertions(+), 42 deletions(-) diff --git a/net/http_client.cpp b/net/http_client.cpp index 84879441d0..2d7dfa0db5 100644 --- a/net/http_client.cpp +++ b/net/http_client.cpp @@ -167,53 +167,41 @@ int Client::GET(const char *resource, Buffer *output, float *progress) { return code; } -int Client::POST(const char *resource, const std::string &data, const std::string &mime, Buffer *output) { - Buffer buffer; - const char *tpl = "POST %s HTTP/1.0\r\nHost: %s\r\nUser-Agent: " USERAGENT "\r\nContent-Length: %d\r\n"; - buffer.Printf(tpl, resource, host_.c_str(), (int)data.size()); - if (!mime.empty()) { - buffer.Printf("Content-Type: %s\r\n", mime.c_str()); +int Client::POST(const char *resource, const std::string &data, const std::string &mime, Buffer *output, float *progress) { + char otherHeaders[2048]; + if (mime.empty()) { + snprintf(otherHeaders, sizeof(otherHeaders), "Content-Length: %lld\r\n", (long long)data.size()); + } else { + snprintf(otherHeaders, sizeof(otherHeaders), "Content-Length: %lld\r\nContent-Type: %s\r\n", (long long)data.size(), mime.c_str()); } - buffer.Append("\r\n"); - buffer.Append(data); - if (!buffer.FlushSocket(sock())) { - ELOG("Failed posting"); + int err = SendRequestWithData("POST", resource, data, otherHeaders, progress); + if (err < 0) { + return err; } - // I guess we could add a deadline here. - output->ReadAll(sock()); - - if (output->size() == 0) { - // The connection was closed. - ELOG("POST failed."); - return -1; + Buffer readbuf; + std::vector responseHeaders; + int code = ReadResponseHeaders(&readbuf, responseHeaders, progress); + if (code < 0) { + return code; } - std::string debug_data; - output->PeekAll(&debug_data); - - //VLOG(1) << "Reply size (before stripping headers): " << debug_data.size(); - std::string debug_str; - StringToHexString(debug_data, &debug_str); - // Tear off the http headers, leaving the actual response data. - std::string firstline; - CHECK_GT(output->TakeLineCRLF(&firstline), 0); - int code = atoi(&firstline[9]); - //VLOG(1) << "HTTP result code: " << code; - while (true) { - int skipped = output->SkipLineCRLF(); - if (skipped == 0) - break; + err = ReadResponseEntity(&readbuf, responseHeaders, output, progress); + if (err < 0) { + return err; } - output->PeekAll(&debug_data); return code; } -int Client::POST(const char *resource, const std::string &data, Buffer *output) { - return POST(resource, data, "", output); +int Client::POST(const char *resource, const std::string &data, Buffer *output, float *progress) { + return POST(resource, data, "", output, progress); } int Client::SendRequest(const char *method, const char *resource, const char *otherHeaders, float *progress) { + return SendRequestWithData(method, resource, "", otherHeaders, progress); +} + +int Client::SendRequestWithData(const char *method, const char *resource, const std::string &data, const char *otherHeaders, float *progress) { if (progress) { *progress = 0.01f; } @@ -228,6 +216,7 @@ int Client::SendRequest(const char *method, const char *resource, const char *ot "\r\n"; buffer.Printf(tpl, method, resource, httpVersion_, host_.c_str(), userAgent_, otherHeaders ? otherHeaders : ""); + buffer.Append(data); bool flushed = buffer.FlushSocket(sock()); if (!flushed) { return -1; // TODO error code. @@ -244,8 +233,6 @@ int Client::ReadResponseHeaders(Buffer *readbuf, std::vector &respo // Grab the first header line that contains the http code. - // Skip the header. TODO: read HTTP code and file size so we can make progress bars. - std::string line; readbuf->TakeLineCRLF(&line); @@ -261,7 +248,6 @@ int Client::ReadResponseHeaders(Buffer *readbuf, std::vector &respo return -1; } - // TODO while (true) { int sz = readbuf->TakeLineCRLF(&line); if (!sz) @@ -269,6 +255,10 @@ int Client::ReadResponseHeaders(Buffer *readbuf, std::vector &respo responseHeaders.push_back(line); } + if (responseHeaders.size() == 0) { + return -1; + } + return code; } diff --git a/net/http_client.h b/net/http_client.h index 7425115785..ece8add041 100644 --- a/net/http_client.h +++ b/net/http_client.h @@ -59,15 +59,16 @@ public: ~Client(); // Return value is the HTTP return code. 200 means OK. < 0 means some local error. - int GET(const char *resource, Buffer *output, float *progress = 0); + int GET(const char *resource, Buffer *output, float *progress = nullptr); // Return value is the HTTP return code. - int POST(const char *resource, const std::string &data, const std::string &mime, Buffer *output); - int POST(const char *resource, const std::string &data, Buffer *output); + int POST(const char *resource, const std::string &data, const std::string &mime, Buffer *output, float *progress = nullptr); + int POST(const char *resource, const std::string &data, Buffer *output, float *progress = nullptr); - // HEAD, PUT, DELETE aren't implemented yet. + // HEAD, PUT, DELETE aren't implemented yet, but can be done with SendRequest. int SendRequest(const char *method, const char *resource, const char *otherHeaders = nullptr, float *progress = nullptr); + int SendRequestWithData(const char *method, const char *resource, const std::string &data, const char *otherHeaders = nullptr, float *progress = nullptr); int ReadResponseHeaders(Buffer *readbuf, std::vector &responseHeaders, float *progress = nullptr); // If your response contains a response, you must read it. int ReadResponseEntity(Buffer *readbuf, const std::vector &responseHeaders, Buffer *output, float *progress = nullptr);