From 17bc6a5a2dd8d6e6a7ee16380dcc8fa5d276cf53 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Jun 2019 12:04:59 -0700 Subject: [PATCH 1/4] http: Support redirects for downloads. --- ext/native/net/http_client.cpp | 117 ++++++++++++++++++++++----------- ext/native/net/http_client.h | 3 + 2 files changed, 82 insertions(+), 38 deletions(-) diff --git a/ext/native/net/http_client.cpp b/ext/native/net/http_client.cpp index 22e5fb9c56..af9b5593e4 100644 --- a/ext/native/net/http_client.cpp +++ b/ext/native/net/http_client.cpp @@ -421,6 +421,52 @@ void Download::SetFailed(int code) { completed_ = true; } +int Download::PerformGET(const std::string &url) { + Url fileUrl(url); + if (!fileUrl.Valid()) { + return -1; + } + + http::Client client; + if (!client.Resolve(fileUrl.Host().c_str(), fileUrl.Port())) { + ELOG("Failed resolving %s", url.c_str()); + return -1; + } + + if (cancelled_) { + return -1; + } + + if (!client.Connect(2, 20.0, &cancelled_)) { + ELOG("Failed connecting to server or cancelled."); + return -1; + } + + if (cancelled_) { + return -1; + } + + return client.GET(fileUrl.Resource().c_str(), &buffer_, responseHeaders_, &progress_, &cancelled_); +} + +std::string Download::RedirectLocation(const std::string &baseUrl) { + std::string redirectUrl = ""; + for (const std::string &line : responseHeaders_) { + if (startsWithNoCase(line, "Location:")) { + size_t url_pos = strlen("Location:"); + size_t after_white = line.find_first_not_of(" \t", url_pos); + if (after_white != line.npos) + url_pos = after_white; + + redirectUrl = line.substr(url_pos); + } + } + + // TODO: Resolve relative redirect url. + + return redirectUrl; +} + void Download::Do(std::shared_ptr self) { setCurrentThreadName("Downloader::Do"); // as long as this is in scope, we won't get destructed. @@ -428,47 +474,42 @@ void Download::Do(std::shared_ptr self) { std::shared_ptr self_ = self; resultCode_ = 0; - Url fileUrl(url_); - if (!fileUrl.Valid()) { - SetFailed(-1); - return; - } - - http::Client client; - if (!client.Resolve(fileUrl.Host().c_str(), fileUrl.Port())) { - ELOG("Failed resolving %s", url_.c_str()); - SetFailed(-1); - return; - } - - if (cancelled_) { - SetFailed(-1); - return; - } - - if (!client.Connect()) { - ELOG("Failed connecting to server."); - SetFailed(-1); - return; - } - - if (cancelled_) { - SetFailed(-1); - return; - } - - // TODO: Allow cancelling during a GET somehow... - int resultCode = client.GET(fileUrl.Resource().c_str(), &buffer_, &progress_, &cancelled_); - if (resultCode == 200) { - ILOG("Completed downloading %s to %s", url_.c_str(), outfile_.empty() ? "memory" : outfile_.c_str()); - if (!outfile_.empty() && !buffer_.FlushToFile(outfile_.c_str())) { - ELOG("Failed writing download to %s", outfile_.c_str()); + std::string downloadURL = url_; + while (resultCode_ == 0) { + int resultCode = PerformGET(downloadURL); + if (resultCode == -1) { + SetFailed(resultCode); + return; } - } else { - ELOG("Error downloading %s to %s: %i", url_.c_str(), outfile_.c_str(), resultCode); + + if (resultCode == 301 || resultCode == 302 || resultCode == 303 || resultCode == 307 || resultCode == 308) { + std::string redirectURL = RedirectLocation(downloadURL); + if (redirectURL.empty()) { + ELOG("Could not find Location header for redirect"); + resultCode_ = resultCode; + } else if (redirectURL == downloadURL || redirectURL == url_) { + // Simple loop detected, bail out. + resultCode_ = resultCode; + } + + // Perform the next GET. + if (resultCode_ == 0) + ILOG("Download of %s redirected to %s", downloadURL.c_str(), redirectURL.c_str()); + downloadURL = redirectURL; + continue; + } + + if (resultCode == 200) { + ILOG("Completed downloading %s to %s", url_.c_str(), outfile_.empty() ? "memory" : outfile_.c_str()); + if (!outfile_.empty() && !buffer_.FlushToFile(outfile_.c_str())) { + ELOG("Failed writing download to %s", outfile_.c_str()); + } + } else { + ELOG("Error downloading %s to %s: %i", url_.c_str(), outfile_.c_str(), resultCode); + } + resultCode_ = resultCode; } - resultCode_ = resultCode; progress_ = 1.0f; // Set this last to ensure no race conditions when checking Done. Users must always check diff --git a/ext/native/net/http_client.h b/ext/native/net/http_client.h index 348cf33d93..1cf3a2e0ed 100644 --- a/ext/native/net/http_client.h +++ b/ext/native/net/http_client.h @@ -137,9 +137,12 @@ public: private: void Do(std::shared_ptr self); // Actually does the download. Runs on thread. + int PerformGET(const std::string &url); + std::string RedirectLocation(const std::string &baseUrl); void SetFailed(int code); float progress_; Buffer buffer_; + std::vector responseHeaders_; std::string url_; std::string outfile_; int resultCode_; From c5e2c0e6cdb48b3812b80bfba761927b079c6bfe Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Jun 2019 12:21:56 -0700 Subject: [PATCH 2/4] http: Support relative redirects too. Per HTTP/1.1. --- ext/native/net/http_client.cpp | 6 ++++- ext/native/net/url.cpp | 47 ++++++++++++++++++++++++++++++++++ ext/native/net/url.h | 4 +++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/ext/native/net/http_client.cpp b/ext/native/net/http_client.cpp index af9b5593e4..1e2d7c90b7 100644 --- a/ext/native/net/http_client.cpp +++ b/ext/native/net/http_client.cpp @@ -462,7 +462,11 @@ std::string Download::RedirectLocation(const std::string &baseUrl) { } } - // TODO: Resolve relative redirect url. + if (!redirectUrl.empty()) { + Url url(baseUrl); + url = url.Relative(redirectUrl); + redirectUrl = url.ToString(); + } return redirectUrl; } diff --git a/ext/native/net/url.cpp b/ext/native/net/url.cpp index ec32a3fe99..45175372e7 100644 --- a/ext/native/net/url.cpp +++ b/ext/native/net/url.cpp @@ -1,4 +1,5 @@ #include "base/logging.h" +#include "base/stringutil.h" #include "net/url.h" const char *UrlEncoder::unreservedChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~"; @@ -35,6 +36,52 @@ void Url::Split() { valid_ = protocol_.size() > 1 && host_.size() > 1; } +Url Url::Relative(const std::string &next) const { + if (next.size() > 2 && next[0] == '/' && next[1] == '/') { + // This means use the same protocol, but the rest is new. + return Url(protocol_ + ":" + next); + } + + // Or it could just be a fully absolute URL. + size_t colonSlashSlash = next.find("://"); + if (colonSlashSlash != std::string::npos) { + return Url(next); + } + + // Anything else should be a new resource, but it might be directory relative. + Url resolved = *this; + if (next.size() > 1 && next[0] == '/') { + // Easy, just replace the resource. + resolved.resource_ = next; + } else { + size_t last_slash = resource_.find_last_of('/'); + resolved.resource_ = resource_.substr(0, last_slash + 1) + next; + } + + resolved.url_ = resolved.ToString(); + return resolved; +} + +std::string Url::ToString() const { + if (!valid_) { + return "about:invalid-url"; + } + + std::string serialized = protocol_ + "://" + host_; + bool needsPort = true; + if (protocol_ == "https") { + needsPort = port_ != 443; + } else if (protocol_ == "http") { + needsPort = port_ != 80; + } + + if (needsPort) { + serialized += ":" + StringFromInt(port_); + } + + return serialized + resource_; +} + // UriDecode and UriEncode are from http://www.codeguru.com/cpp/cpp/string/conversions/print.php/c12759 // by jinq0123 (November 2, 2006) diff --git a/ext/native/net/url.h b/ext/native/net/url.h index bc597e8bdb..a46564d642 100644 --- a/ext/native/net/url.h +++ b/ext/native/net/url.h @@ -189,6 +189,10 @@ public: std::string Protocol() const { return protocol_; } std::string Resource() const { return resource_; } + Url Relative(const std::string &next) const; + + std::string ToString() const; + private: void Split(); bool valid_; From 94e6950d486b8341f8e88165b6f42c3b880d7bb1 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Jun 2019 13:12:13 -0700 Subject: [PATCH 3/4] http: Support redirects for load URL too. Might as well, especially to keep old recent entries working. --- Core/FileLoaders/HTTPFileLoader.cpp | 115 +++++++++++++++++++--------- Core/FileLoaders/HTTPFileLoader.h | 2 + ext/native/net/http_client.cpp | 40 ++++++---- ext/native/net/http_client.h | 2 + 4 files changed, 110 insertions(+), 49 deletions(-) diff --git a/Core/FileLoaders/HTTPFileLoader.cpp b/Core/FileLoaders/HTTPFileLoader.cpp index 0731c617fb..7d710dbe52 100644 --- a/Core/FileLoaders/HTTPFileLoader.cpp +++ b/Core/FileLoaders/HTTPFileLoader.cpp @@ -27,43 +27,53 @@ HTTPFileLoader::HTTPFileLoader(const std::string &filename) void HTTPFileLoader::Prepare() { std::call_once(preparedFlag_, [this](){ - if (!url_.Valid()) { - ERROR_LOG(LOADER, "HTTP request failed, invalid URL"); - latestError_ = "Invalid URL"; - return; - } - - if (!client_.Resolve(url_.Host().c_str(), url_.Port())) { - ERROR_LOG(LOADER, "HTTP request failed, unable to resolve: %s port %d", url_.Host().c_str(), url_.Port()); - latestError_ = "Could not connect (name not resolved)"; - return; - } - - client_.SetDataTimeout(20.0); - Connect(); - if (!connected_) { - ERROR_LOG(LOADER, "HTTP request failed, failed to connect: %s port %d", url_.Host().c_str(), url_.Port()); - latestError_ = "Could not connect (refused to connect)"; - return; - } - - int err = client_.SendRequest("HEAD", url_.Resource().c_str()); - if (err < 0) { - ERROR_LOG(LOADER, "HTTP request failed, failed to send request: %s port %d", url_.Host().c_str(), url_.Port()); - latestError_ = "Could not connect (could not request data)"; - Disconnect(); - return; - } - - Buffer readbuf; std::vector responseHeaders; - int code = client_.ReadResponseHeaders(&readbuf, responseHeaders); - if (code != 200) { - // Leave size at 0, invalid. - ERROR_LOG(LOADER, "HTTP request failed, got %03d for %s", code, filename_.c_str()); - latestError_ = "Could not connect (invalid response)"; - Disconnect(); - return; + Url resourceURL = url_; + int redirectsLeft = 20; + while (redirectsLeft > 0) { + responseHeaders.clear(); + int code = SendHEAD(resourceURL, responseHeaders); + if (code == -400) { + // Already reported the error. + return; + } + + if (code == 301 || code == 302 || code == 303 || code == 307 || code == 308) { + Disconnect(); + + std::string redirectURL; + if (http::GetHeaderValue(responseHeaders, "Location", &redirectURL)) { + Url url(resourceURL); + url = url.Relative(redirectURL); + + if (url.ToString() == url_.ToString() || url.ToString() == resourceURL.ToString()) { + ERROR_LOG(LOADER, "HTTP request failed, hit a redirect loop"); + latestError_ = "Could not connect (redirect loop)"; + return; + } + + resourceURL = url; + redirectsLeft--; + continue; + } + + // No Location header? + ERROR_LOG(LOADER, "HTTP request failed, invalid redirect"); + latestError_ = "Could not connect (invalid response)"; + return; + } + + if (code != 200) { + // Leave size at 0, invalid. + ERROR_LOG(LOADER, "HTTP request failed, got %03d for %s", code, filename_.c_str()); + latestError_ = "Could not connect (invalid response)"; + Disconnect(); + return; + } + + // We got a good, non-redirect response. + redirectsLeft = 0; + url_ = resourceURL; } // TODO: Expire cache via ETag, etc. @@ -102,6 +112,39 @@ void HTTPFileLoader::Prepare() { }); } +int HTTPFileLoader::SendHEAD(const Url &url, std::vector &responseHeaders) { + if (!url.Valid()) { + ERROR_LOG(LOADER, "HTTP request failed, invalid URL"); + latestError_ = "Invalid URL"; + return -400; + } + + if (!client_.Resolve(url.Host().c_str(), url.Port())) { + ERROR_LOG(LOADER, "HTTP request failed, unable to resolve: |%s| port %d", url.Host().c_str(), url.Port()); + latestError_ = "Could not connect (name not resolved)"; + return -400; + } + + client_.SetDataTimeout(20.0); + Connect(); + if (!connected_) { + ERROR_LOG(LOADER, "HTTP request failed, failed to connect: %s port %d", url.Host().c_str(), url.Port()); + latestError_ = "Could not connect (refused to connect)"; + return -400; + } + + int err = client_.SendRequest("HEAD", url.Resource().c_str()); + if (err < 0) { + ERROR_LOG(LOADER, "HTTP request failed, failed to send request: %s port %d", url.Host().c_str(), url.Port()); + latestError_ = "Could not connect (could not request data)"; + Disconnect(); + return -400; + } + + Buffer readbuf; + return client_.ReadResponseHeaders(&readbuf, responseHeaders); +} + HTTPFileLoader::~HTTPFileLoader() { Disconnect(); } diff --git a/Core/FileLoaders/HTTPFileLoader.h b/Core/FileLoaders/HTTPFileLoader.h index d853153177..cf3441df9c 100644 --- a/Core/FileLoaders/HTTPFileLoader.h +++ b/Core/FileLoaders/HTTPFileLoader.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include "net/http_client.h" #include "net/resolve.h" @@ -54,6 +55,7 @@ public: private: void Prepare(); + int SendHEAD(const Url &url, std::vector &responseHeaders); void Connect(); diff --git a/ext/native/net/http_client.cpp b/ext/native/net/http_client.cpp index 1e2d7c90b7..19bff66f48 100644 --- a/ext/native/net/http_client.cpp +++ b/ext/native/net/http_client.cpp @@ -174,6 +174,31 @@ Client::~Client() { Disconnect(); } +// Ignores line folding (deprecated), but respects field combining. +// Don't use for Set-Cookie, which is a special header per RFC 7230. +bool GetHeaderValue(const std::vector &responseHeaders, const std::string &header, std::string *value) { + std::string search = header + ":"; + bool found = false; + + value->clear(); + for (const std::string &line : responseHeaders) { + auto stripped = StripSpaces(line); + if (startsWithNoCase(stripped, search)) { + size_t value_pos = search.length(); + size_t after_white = stripped.find_first_not_of(" \t", value_pos); + if (after_white != stripped.npos) + value_pos = after_white; + + if (!found) + *value = stripped.substr(value_pos); + else + *value += "," + stripped.substr(value_pos); + found = true; + } + } + + return found; +} void DeChunk(Buffer *inbuffer, Buffer *outbuffer, int contentLength, float *progress) { int dechunkedBytes = 0; @@ -450,19 +475,8 @@ int Download::PerformGET(const std::string &url) { } std::string Download::RedirectLocation(const std::string &baseUrl) { - std::string redirectUrl = ""; - for (const std::string &line : responseHeaders_) { - if (startsWithNoCase(line, "Location:")) { - size_t url_pos = strlen("Location:"); - size_t after_white = line.find_first_not_of(" \t", url_pos); - if (after_white != line.npos) - url_pos = after_white; - - redirectUrl = line.substr(url_pos); - } - } - - if (!redirectUrl.empty()) { + std::string redirectUrl; + if (GetHeaderValue(responseHeaders_, "Location", &redirectUrl)) { Url url(baseUrl); url = url.Relative(redirectUrl); redirectUrl = url.ToString(); diff --git a/ext/native/net/http_client.h b/ext/native/net/http_client.h index 1cf3a2e0ed..3d2d97eb20 100644 --- a/ext/native/net/http_client.h +++ b/ext/native/net/http_client.h @@ -54,6 +54,8 @@ private: namespace http { +bool GetHeaderValue(const std::vector &responseHeaders, const std::string &header, std::string *value); + class Client : public net::Connection { public: Client(); From fb2ce46eca1ab8478c48e12e97e8c4b4ffa76adc Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Sun, 23 Jun 2019 13:34:54 -0700 Subject: [PATCH 4/4] UI: Add a note for HTML link parsing. It's a bit messy but should probably do this eventually. --- UI/RemoteISOScreen.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/UI/RemoteISOScreen.cpp b/UI/RemoteISOScreen.cpp index 783ec81f3d..6dd15835f6 100644 --- a/UI/RemoteISOScreen.cpp +++ b/UI/RemoteISOScreen.cpp @@ -159,6 +159,7 @@ static bool LoadGameList(const std::string &host, int port, std::vector responseHeaders; + // TODO: Use relative url parsing (URL::Relative) instead. std::string subdir = RemoteSubdir(); // Start by requesting the list of games from the server.