http: Separate out net buffer code from formatting.

This commit is contained in:
Unknown W. Brackets
2021-05-01 08:36:25 -07:00
parent e869a3979b
commit 1e22966984
16 changed files with 234 additions and 211 deletions
+2
View File
@@ -575,6 +575,8 @@ add_library(Common STATIC
Common/Net/HTTPHeaders.h
Common/Net/HTTPServer.cpp
Common/Net/HTTPServer.h
Common/Net/NetBuffer.cpp
Common/Net/NetBuffer.h
Common/Net/Resolve.cpp
Common/Net/Resolve.h
Common/Net/Sinks.cpp
+47 -183
View File
@@ -1,24 +1,7 @@
#include <cstdarg>
#include <cstdlib>
#include <cstring>
#include <stdarg.h>
#include <stdlib.h>
#include <algorithm>
#ifdef _WIN32
#include <winsock2.h>
#undef min
#undef max
#else
#include <sys/socket.h>
#include <unistd.h>
#endif
#ifndef MSG_NOSIGNAL
// Default value to 0x00 (do nothing) in systems where it's not supported.
#define MSG_NOSIGNAL 0x00
#endif
#include "Common/File/FileDescriptor.h"
#include "Common/TimeUtil.h"
#include "Common/Buffer.h"
#include "Common/Log.h"
@@ -36,14 +19,14 @@ char *Buffer::Append(size_t length) {
}
void Buffer::Append(const std::string &str) {
char *ptr = Append(str.size());
memcpy(ptr, str.data(), str.size());
char *ptr = Append(str.size());
memcpy(ptr, str.data(), str.size());
}
void Buffer::Append(const char *str) {
size_t len = strlen(str);
char *dest = Append(len);
memcpy(dest, str, len);
size_t len = strlen(str);
char *dest = Append(len);
memcpy(dest, str, len);
}
void Buffer::Append(const Buffer &other) {
@@ -55,10 +38,10 @@ void Buffer::Append(const Buffer &other) {
}
void Buffer::AppendValue(int value) {
char buf[16];
// This is slow.
sprintf(buf, "%i", value);
Append(buf);
char buf[16];
// This is slow.
sprintf(buf, "%i", value);
Append(buf);
}
void Buffer::Take(size_t length, std::string *dest) {
@@ -78,14 +61,14 @@ void Buffer::Take(size_t length, char *dest) {
}
int Buffer::TakeLineCRLF(std::string *dest) {
int after_next_line = OffsetToAfterNextCRLF();
if (after_next_line < 0)
return after_next_line;
else {
Take(after_next_line - 2, dest);
Skip(2); // Skip the CRLF
return after_next_line - 2;
}
int after_next_line = OffsetToAfterNextCRLF();
if (after_next_line < 0) {
return after_next_line;
} else {
Take(after_next_line - 2, dest);
Skip(2); // Skip the CRLF
return after_next_line - 2;
}
}
void Buffer::Skip(size_t length) {
@@ -97,48 +80,39 @@ void Buffer::Skip(size_t length) {
}
int Buffer::SkipLineCRLF() {
int after_next_line = OffsetToAfterNextCRLF();
if (after_next_line < 0)
return after_next_line;
else {
Skip(after_next_line);
return after_next_line - 2;
}
int after_next_line = OffsetToAfterNextCRLF();
if (after_next_line < 0) {
return after_next_line;
} else {
Skip(after_next_line);
return after_next_line - 2;
}
}
int Buffer::OffsetToAfterNextCRLF() {
for (int i = 0; i < (int)data_.size() - 1; i++) {
if (data_[i] == '\r' && data_[i + 1] == '\n') {
return i + 2;
}
}
return -1;
for (int i = 0; i < (int)data_.size() - 1; i++) {
if (data_[i] == '\r' && data_[i + 1] == '\n') {
return i + 2;
}
}
return -1;
}
void Buffer::Printf(const char *fmt, ...) {
char buffer[2048];
va_list vl;
va_start(vl, fmt);
size_t retval = vsnprintf(buffer, sizeof(buffer), fmt, vl);
if ((int)retval >= (int)sizeof(buffer)) {
// Output was truncated. TODO: Do something.
ERROR_LOG(IO, "Buffer::Printf truncated output");
}
if (retval < 0) {
ERROR_LOG(IO, "Buffer::Printf failed");
}
va_end(vl);
char *ptr = Append(retval);
memcpy(ptr, buffer, retval);
}
bool Buffer::Flush(int fd) {
// Look into using send() directly.
bool success = data_.size() == fd_util::WriteLine(fd, &data_[0], data_.size());
if (success) {
data_.resize(0);
}
return success;
char buffer[2048];
va_list vl;
va_start(vl, fmt);
size_t retval = vsnprintf(buffer, sizeof(buffer), fmt, vl);
if ((int)retval >= (int)sizeof(buffer)) {
// Output was truncated. TODO: Do something.
ERROR_LOG(IO, "Buffer::Printf truncated output");
}
if (retval < 0) {
ERROR_LOG(IO, "Buffer::Printf failed");
}
va_end(vl);
char *ptr = Append(retval);
memcpy(ptr, buffer, retval);
}
bool Buffer::FlushToFile(const char *filename) {
@@ -152,116 +126,6 @@ bool Buffer::FlushToFile(const char *filename) {
return true;
}
bool Buffer::FlushSocket(uintptr_t sock, double timeout, bool *cancelled) {
static constexpr float CANCEL_INTERVAL = 0.25f;
for (size_t pos = 0, end = data_.size(); pos < end; ) {
bool ready = false;
double leftTimeout = timeout;
while (!ready && (leftTimeout >= 0 || cancelled)) {
if (cancelled && *cancelled)
return false;
ready = fd_util::WaitUntilReady(sock, CANCEL_INTERVAL, true);
if (!ready && leftTimeout >= 0.0) {
leftTimeout -= CANCEL_INTERVAL;
if (leftTimeout < 0) {
ERROR_LOG(IO, "FlushSocket timed out");
return false;
}
}
}
int sent = send(sock, &data_[pos], (int)(end - pos), MSG_NOSIGNAL);
if (sent < 0) {
ERROR_LOG(IO, "FlushSocket failed");
return false;
}
pos += sent;
// Buffer full, don't spin.
if (sent == 0 && timeout < 0.0) {
sleep_ms(1);
}
}
data_.resize(0);
return true;
}
bool Buffer::ReadAll(int fd, int hintSize) {
std::vector<char> buf;
if (hintSize >= 65536 * 16) {
buf.resize(65536);
} else if (hintSize >= 1024 * 16) {
buf.resize(hintSize / 16);
} else {
buf.resize(4096);
}
while (true) {
int retval = recv(fd, &buf[0], (int)buf.size(), MSG_NOSIGNAL);
if (retval == 0) {
break;
} else if (retval < 0) {
ERROR_LOG(IO, "Error reading from buffer: %i", retval);
return false;
}
char *p = Append((size_t)retval);
memcpy(p, &buf[0], retval);
}
return true;
}
bool Buffer::ReadAllWithProgress(int fd, int knownSize, float *progress, bool *cancelled) {
static constexpr float CANCEL_INTERVAL = 0.25f;
std::vector<char> buf;
if (knownSize >= 65536 * 16) {
buf.resize(65536);
} else if (knownSize >= 1024 * 16) {
buf.resize(knownSize / 16);
} else {
buf.resize(1024);
}
int total = 0;
while (true) {
bool ready = false;
while (!ready && cancelled) {
if (*cancelled)
return false;
ready = fd_util::WaitUntilReady(fd, CANCEL_INTERVAL, false);
}
int retval = recv(fd, &buf[0], (int)buf.size(), MSG_NOSIGNAL);
if (retval == 0) {
return true;
} else if (retval < 0) {
ERROR_LOG(IO, "Error reading from buffer: %i", retval);
return false;
}
char *p = Append((size_t)retval);
memcpy(p, &buf[0], retval);
total += retval;
if (progress)
*progress = (float)total / (float)knownSize;
}
return true;
}
int Buffer::Read(int fd, size_t sz) {
char buf[1024];
int retval;
size_t received = 0;
while ((retval = recv(fd, buf, (int)std::min(sz, sizeof(buf)), MSG_NOSIGNAL)) > 0) {
if (retval < 0) {
return retval;
}
char *p = Append((size_t)retval);
memcpy(p, buf, retval);
sz -= retval;
received += retval;
if (sz == 0)
return 0;
}
return (int)received;
}
void Buffer::PeekAll(std::string *dest) {
dest->resize(data_.size());
memcpy(&(*dest)[0], &data_[0], data_.size());
+2 -10
View File
@@ -59,25 +59,17 @@ public:
// Writes the entire buffer to the file descriptor. Also resets the
// size to zero. On failure, data remains in buffer and nothing is
// written.
bool Flush(int fd);
bool FlushToFile(const char *filename);
bool FlushSocket(uintptr_t sock, double timeout = -1.0, bool *cancelled = nullptr); // Windows portability
bool ReadAll(int fd, int hintSize = 0);
bool ReadAllWithProgress(int fd, int knownSize, float *progress, bool *cancelled);
// < 0: error
// >= 0: number of bytes read
int Read(int fd, size_t sz);
// Utilities. Try to avoid checking for size.
size_t size() const { return data_.size(); }
bool empty() const { return size() == 0; }
void clear() { data_.resize(0); }
private:
protected:
// TODO: Find a better internal representation, like a cord.
std::vector<char> data_;
private:
DISALLOW_COPY_AND_ASSIGN(Buffer);
};
+2
View File
@@ -452,6 +452,7 @@
<ClInclude Include="Math\lin\matrix4x4.h" />
<ClInclude Include="Math\lin\vec3.h" />
<ClInclude Include="Math\math_util.h" />
<ClInclude Include="Net\NetBuffer.h" />
<ClInclude Include="Net\HTTPClient.h" />
<ClInclude Include="Net\HTTPHeaders.h" />
<ClInclude Include="Net\HTTPServer.h" />
@@ -775,6 +776,7 @@
<ClCompile Include="Math\lin\matrix4x4.cpp" />
<ClCompile Include="Math\lin\vec3.cpp" />
<ClCompile Include="Math\math_util.cpp" />
<ClCompile Include="Net\NetBuffer.cpp" />
<ClCompile Include="Net\HTTPClient.cpp" />
<ClCompile Include="Net\HTTPHeaders.cpp" />
<ClCompile Include="Net\HTTPServer.cpp" />
+8 -2
View File
@@ -68,7 +68,6 @@
</ClInclude>
<ClInclude Include="TimeUtil.h" />
<ClInclude Include="FakeEmitter.h" />
<ClInclude Include="Buffer.h" />
<ClInclude Include="SysError.h" />
<ClInclude Include="..\ext\libpng17\png.h">
<Filter>ext\libpng17</Filter>
@@ -388,6 +387,10 @@
<ClInclude Include="Data\Convert\ColorConv.h">
<Filter>Data\Convert</Filter>
</ClInclude>
<ClInclude Include="Buffer.h" />
<ClInclude Include="Net\NetBuffer.h">
<Filter>Net</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="ABI.cpp" />
@@ -434,7 +437,6 @@
</ClCompile>
<ClCompile Include="TimeUtil.cpp" />
<ClCompile Include="Log.cpp" />
<ClCompile Include="Buffer.cpp" />
<ClCompile Include="SysError.cpp" />
<ClCompile Include="..\ext\libpng17\png.c">
<Filter>ext\libpng17</Filter>
@@ -748,6 +750,10 @@
<ClCompile Include="Data\Convert\ColorConv.cpp">
<Filter>Data\Convert</Filter>
</ClCompile>
<ClCompile Include="Buffer.cpp" />
<ClCompile Include="Net\NetBuffer.cpp">
<Filter>Net</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<Filter Include="Crypto">
+6 -6
View File
@@ -31,7 +31,7 @@
#include "Common/File/FileDescriptor.h"
#include "Common/Thread/ThreadUtil.h"
#include "Common/Data/Encoding/Compression.h"
#include "Common/Buffer.h"
#include "Common/Net/NetBuffer.h"
#include "Common/Log.h"
namespace net {
@@ -253,7 +253,7 @@ int Client::GET(const char *resource, Buffer *output, std::vector<std::string> &
return err;
}
Buffer readbuf;
net::Buffer readbuf;
int code = ReadResponseHeaders(&readbuf, responseHeaders, progress, cancelled);
if (code < 0) {
return code;
@@ -284,7 +284,7 @@ int Client::POST(const char *resource, const std::string &data, const std::strin
return err;
}
Buffer readbuf;
net::Buffer readbuf;
std::vector<std::string> responseHeaders;
int code = ReadResponseHeaders(&readbuf, responseHeaders, progress);
if (code < 0) {
@@ -311,7 +311,7 @@ int Client::SendRequestWithData(const char *method, const char *resource, const
*progress = 0.01f;
}
Buffer buffer;
net::Buffer buffer;
const char *tpl =
"%s %s HTTP/%s\r\n"
"Host: %s\r\n"
@@ -333,7 +333,7 @@ int Client::SendRequestWithData(const char *method, const char *resource, const
return 0;
}
int Client::ReadResponseHeaders(Buffer *readbuf, std::vector<std::string> &responseHeaders, float *progress, bool *cancelled) {
int Client::ReadResponseHeaders(net::Buffer *readbuf, std::vector<std::string> &responseHeaders, float *progress, bool *cancelled) {
// Snarf all the data we can into RAM. A little unsafe but hey.
static constexpr float CANCEL_INTERVAL = 0.25f;
bool ready = false;
@@ -389,7 +389,7 @@ int Client::ReadResponseHeaders(Buffer *readbuf, std::vector<std::string> &respo
return code;
}
int Client::ReadResponseEntity(Buffer *readbuf, const std::vector<std::string> &responseHeaders, Buffer *output, float *progress, bool *cancelled) {
int Client::ReadResponseEntity(net::Buffer *readbuf, const std::vector<std::string> &responseHeaders, Buffer *output, float *progress, bool *cancelled) {
bool gzip = false;
bool chunked = false;
int contentLength = 0;
+3 -4
View File
@@ -5,10 +5,9 @@
#include <thread>
#include <cstdint>
#include "Common/Net/NetBuffer.h"
#include "Common/Net/Resolve.h"
#include "Common/Buffer.h"
namespace net {
class Connection {
@@ -61,9 +60,9 @@ public:
int SendRequest(const char *method, const char *resource, const char *otherHeaders = nullptr, float *progress = nullptr, bool *cancelled = nullptr);
int SendRequestWithData(const char *method, const char *resource, const std::string &data, const char *otherHeaders = nullptr, float *progress = nullptr, bool *cancelled = nullptr);
int ReadResponseHeaders(Buffer *readbuf, std::vector<std::string> &responseHeaders, float *progress = nullptr, bool *cancelled = nullptr);
int ReadResponseHeaders(net::Buffer *readbuf, std::vector<std::string> &responseHeaders, float *progress = nullptr, bool *cancelled = nullptr);
// If your response contains a response, you must read it.
int ReadResponseEntity(Buffer *readbuf, const std::vector<std::string> &responseHeaders, Buffer *output, float *progress = nullptr, bool *cancelled = nullptr);
int ReadResponseEntity(net::Buffer *readbuf, const std::vector<std::string> &responseHeaders, Buffer *output, float *progress = nullptr, bool *cancelled = nullptr);
void SetDataTimeout(double t) {
dataTimeout_ = t;
+1 -1
View File
@@ -3,7 +3,7 @@
#include <string>
#include <unordered_map>
#include "Common/Buffer.h"
#include "Common/Net/NetBuffer.h"
namespace net {
class InputSink;
+1 -2
View File
@@ -35,10 +35,9 @@
#include <cstdlib>
#include "Common/Net/HTTPServer.h"
#include "Common/Net/NetBuffer.h"
#include "Common/Net/Sinks.h"
#include "Common/File/FileDescriptor.h"
#include "Common/Buffer.h"
#include "Common/Log.h"
void NewThreadExecutor::Run(std::function<void()> &&func) {
+134
View File
@@ -0,0 +1,134 @@
#ifdef _WIN32
#include <winsock2.h>
#undef min
#undef max
#else
#include <sys/socket.h>
#include <unistd.h>
#endif
#include <algorithm>
#include <cstring>
#ifndef MSG_NOSIGNAL
// Default value to 0x00 (do nothing) in systems where it's not supported.
#define MSG_NOSIGNAL 0x00
#endif
#include "Common/File/FileDescriptor.h"
#include "Common/Log.h"
#include "Common/Net/NetBuffer.h"
#include "Common/TimeUtil.h"
namespace net {
bool Buffer::FlushSocket(uintptr_t sock, double timeout, bool *cancelled) {
static constexpr float CANCEL_INTERVAL = 0.25f;
for (size_t pos = 0, end = data_.size(); pos < end; ) {
bool ready = false;
double leftTimeout = timeout;
while (!ready && (leftTimeout >= 0 || cancelled)) {
if (cancelled && *cancelled)
return false;
ready = fd_util::WaitUntilReady(sock, CANCEL_INTERVAL, true);
if (!ready && leftTimeout >= 0.0) {
leftTimeout -= CANCEL_INTERVAL;
if (leftTimeout < 0) {
ERROR_LOG(IO, "FlushSocket timed out");
return false;
}
}
}
int sent = send(sock, &data_[pos], (int)(end - pos), MSG_NOSIGNAL);
if (sent < 0) {
ERROR_LOG(IO, "FlushSocket failed");
return false;
}
pos += sent;
// Buffer full, don't spin.
if (sent == 0 && timeout < 0.0) {
sleep_ms(1);
}
}
data_.resize(0);
return true;
}
bool Buffer::ReadAll(int fd, int hintSize) {
std::vector<char> buf;
if (hintSize >= 65536 * 16) {
buf.resize(65536);
} else if (hintSize >= 1024 * 16) {
buf.resize(hintSize / 16);
} else {
buf.resize(4096);
}
while (true) {
int retval = recv(fd, &buf[0], (int)buf.size(), MSG_NOSIGNAL);
if (retval == 0) {
break;
} else if (retval < 0) {
ERROR_LOG(IO, "Error reading from buffer: %i", retval);
return false;
}
char *p = Append((size_t)retval);
memcpy(p, &buf[0], retval);
}
return true;
}
bool Buffer::ReadAllWithProgress(int fd, int knownSize, float *progress, bool *cancelled) {
static constexpr float CANCEL_INTERVAL = 0.25f;
std::vector<char> buf;
if (knownSize >= 65536 * 16) {
buf.resize(65536);
} else if (knownSize >= 1024 * 16) {
buf.resize(knownSize / 16);
} else {
buf.resize(1024);
}
int total = 0;
while (true) {
bool ready = false;
while (!ready && cancelled) {
if (*cancelled)
return false;
ready = fd_util::WaitUntilReady(fd, CANCEL_INTERVAL, false);
}
int retval = recv(fd, &buf[0], (int)buf.size(), MSG_NOSIGNAL);
if (retval == 0) {
return true;
} else if (retval < 0) {
ERROR_LOG(IO, "Error reading from buffer: %i", retval);
return false;
}
char *p = Append((size_t)retval);
memcpy(p, &buf[0], retval);
total += retval;
if (progress)
*progress = (float)total / (float)knownSize;
}
return true;
}
int Buffer::Read(int fd, size_t sz) {
char buf[1024];
int retval;
size_t received = 0;
while ((retval = recv(fd, buf, (int)std::min(sz, sizeof(buf)), MSG_NOSIGNAL)) > 0) {
if (retval < 0) {
return retval;
}
char *p = Append((size_t)retval);
memcpy(p, buf, retval);
sz -= retval;
received += retval;
if (sz == 0)
return 0;
}
return (int)received;
}
}
+19
View File
@@ -0,0 +1,19 @@
#pragma once
#include "Common/Buffer.h"
namespace net {
class Buffer : public ::Buffer {
public:
bool FlushSocket(uintptr_t sock, double timeout = -1.0, bool *cancelled = nullptr);
bool ReadAll(int fd, int hintSize = 0);
bool ReadAllWithProgress(int fd, int knownSize, float *progress, bool *cancelled);
// < 0: error
// >= 0: number of bytes read
int Read(int fd, size_t sz);
};
}
+3 -3
View File
@@ -145,7 +145,7 @@ int HTTPFileLoader::SendHEAD(const Url &url, std::vector<std::string> &responseH
return -400;
}
Buffer readbuf;
net::Buffer readbuf;
return client_.ReadResponseHeaders(&readbuf, responseHeaders);
}
@@ -203,7 +203,7 @@ size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data, Flags f
return 0;
}
Buffer readbuf;
net::Buffer readbuf;
std::vector<std::string> responseHeaders;
int code = client_.ReadResponseHeaders(&readbuf, responseHeaders);
if (code != 206) {
@@ -235,7 +235,7 @@ size_t HTTPFileLoader::ReadAt(s64 absolutePos, size_t bytes, void *data, Flags f
}
// TODO: Would be nice to read directly.
Buffer output;
net::Buffer output;
int res = client_.ReadResponseEntity(&readbuf, responseHeaders, &output);
if (res != 0) {
ERROR_LOG(LOADER, "Unable to read HTTP response entity: %d", res);
+2
View File
@@ -386,6 +386,7 @@
<ClInclude Include="..\..\Common\BitScan.h" />
<ClInclude Include="..\..\Common\BitSet.h" />
<ClInclude Include="..\..\Common\Buffer.h" />
<ClInclude Include="..\..\Common\Net\NetBuffer.h" />
<ClInclude Include="..\..\Common\Data\Collections\ConstMap.h" />
<ClInclude Include="..\..\Common\Data\Collections\FixedSizeQueue.h" />
<ClInclude Include="..\..\Common\Data\Collections\Hashmaps.h" />
@@ -513,6 +514,7 @@
<ClCompile Include="..\..\Common\ArmCPUDetect.cpp" />
<ClCompile Include="..\..\Common\ArmEmitter.cpp" />
<ClCompile Include="..\..\Common\Buffer.cpp" />
<ClCompile Include="..\..\Common\Net\NetBuffer.cpp" />
<ClCompile Include="..\..\Common\Data\Color\RGBAUtil.cpp" />
<ClCompile Include="..\..\Common\Data\Convert\SmallDataConvert.cpp" />
<ClCompile Include="..\..\Common\Data\Encoding\Base64.cpp" />
+2
View File
@@ -124,6 +124,7 @@
<Filter>Crypto</Filter>
</ClCompile>
<ClCompile Include="..\..\Common\Buffer.cpp" />
<ClCompile Include="..\..\Common\Net\NetBuffer.cpp" />
<ClCompile Include="..\..\ext\libpng17\png.c">
<Filter>ext\libpng17</Filter>
</ClCompile>
@@ -415,6 +416,7 @@
<Filter>Crypto</Filter>
</ClInclude>
<ClInclude Include="..\..\Common\Buffer.h" />
<ClInclude Include="..\..\Common\Net\NetBuffer.h" />
<ClInclude Include="..\..\Common\Serialize\SerializeDeque.h" />
<ClInclude Include="..\..\Common\Serialize\SerializeFuncs.h" />
<ClInclude Include="..\..\Common\Serialize\SerializeList.h" />
+1
View File
@@ -274,6 +274,7 @@ EXEC_AND_LIB_FILES := \
$(SRC)/Common/Net/HTTPClient.cpp \
$(SRC)/Common/Net/HTTPHeaders.cpp \
$(SRC)/Common/Net/HTTPServer.cpp \
$(SRC)/Common/Net/NetBuffer.cpp \
$(SRC)/Common/Net/Resolve.cpp \
$(SRC)/Common/Net/Sinks.cpp \
$(SRC)/Common/Net/URL.cpp \
+1
View File
@@ -197,6 +197,7 @@ SOURCES_CXX += \
$(COMMONDIR)/Net/HTTPClient.cpp \
$(COMMONDIR)/Net/HTTPHeaders.cpp \
$(COMMONDIR)/Net/HTTPServer.cpp \
$(COMMONDIR)/Net/NetBuffer.cpp \
$(COMMONDIR)/Net/Resolve.cpp \
$(COMMONDIR)/Net/Sinks.cpp \
$(COMMONDIR)/Net/URL.cpp \