rexy712/frnetlib
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add support for chunked HTTP response parsing

Browse Source
This commit is contained in:
Fred Nicolson 2019-11-25 16:29:06 +00:00
parent 41f600cd11
commit 5b2b92835e
No known key found for this signature in database
GPG Key ID: 78C1DD87B47797D2
10 changed files with 202 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -95,6 +95,7 @@ target_link_libraries(frnetlib ${FRNETLIB_LINK_LIBRARIES})
#Build Tests if needbe
if(BUILD_TESTS)
add_definitions(-DENABLE_TESTING)
set(EXT_PROJECTS_DIR ${PROJECT_SOURCE_DIR}/ext)
add_subdirectory(${EXT_PROJECTS_DIR}/gtest)
add_subdirectory(tests)

49
include/frnetlib/Http.h
View File

@ -7,13 +7,23 @@
#include <string>
#include <vector>
#include <unordered_map>
#include <set>
#include <iostream>
#include <algorithm>
#include "Socket.h"
#include "Sendable.h"
#ifdef ENABLE_TESTING
#include <gtest/gtest.h>
#endif
namespace fr
{
class Http : public Sendable
{
#ifdef ENABLE_TESTING
FRIEND_TEST(HttpTest, test_string_split);
#endif
public:
enum class RequestVersion
{
@ -32,6 +42,16 @@ namespace fr
Unknown = 6,
Partial = 7,
};
enum class TransferEncoding
{
None = 0,
Chunked = 1,
Compress = 2,
Deflate = 3,
Gzip = 4,
Identity = 5,
EncodingCount = 6,
};
enum class RequestStatus
{
Continue = 100,
@ -267,6 +287,28 @@ namespace fr
*/
const static std::string &get_mimetype(const std::string &filename);
/*!
* Converts a string to a TransferEncoding enum.
*
* @note case insentive
* @param encoding String to convert.
* @return The encoding type, or TransferEncoding::None if no match was found
*/
static TransferEncoding string_to_transfer_encoding(std::string encoding)
{
static_assert((uint32_t)TransferEncoding::EncodingCount == 6, "Update transfer_encoding_to_string");
const static std::unordered_map<std::string, TransferEncoding> string_list = {{"none", TransferEncoding::None},
{"chunked", TransferEncoding::Chunked},
{"compress", TransferEncoding::Compress},
{"deflate", TransferEncoding::Deflate},
{"gzip", TransferEncoding::Gzip},
{"identity", TransferEncoding::Identity}};
std::transform(encoding.begin(), encoding.end(), encoding.begin(), tolower);
auto iter = string_list.find(encoding);
return iter == string_list.end() ? TransferEncoding::None : iter->second;
}
/*!
* Converts a 'RequestType' enum value to
* a printable string.
@ -340,11 +382,13 @@ namespace fr
protected:
/*!
* Splits a string by new line. Ignores escaped \n's
* Splits a string by a given token. Allows for escaping.
*
* @param token Token to split by
* @param strip_spacing True to remove ALL spacing from entries. False to leave intact.
* @return The split string
*/
static std::vector<std::string> split_string(const std::string &str);
static std::vector<std::string> split_string(const std::string &str, char token = '\n', bool strip_spacing = false);
/*!
* Converts a parameter list to a vector pair.
@ -369,6 +413,7 @@ namespace fr
std::unordered_map<std::string, std::string> header_data;
std::unordered_map<std::string, std::string> post_data;
std::unordered_map<std::string, std::string> get_data;
std::set<TransferEncoding> transfer_encodings;
std::string body;
RequestType request_type;
std::string uri;

1
include/frnetlib/HttpResponse.h
View File

@ -50,6 +50,7 @@ namespace fr
//State
bool header_ended{false};
size_t content_length{0};
size_t chunk_offset{0};
};
}

4
include/frnetlib/NetworkEncoding.h
View File

@ -34,11 +34,13 @@
#define INVALID_SOCKET 0
#define SOCKET_ERROR (-1)
#include <netdb.h>
#include <unistd.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#endif

43
src/Http.cpp
View File

@ -26,23 +26,32 @@ namespace fr
return request_type;
}
std::vector<std::string> Http::split_string(const std::string &str)
std::vector<std::string> Http::split_string(const std::string &str, char token, bool strip_spacing)
{
char last_character = '\0';
size_t line_start = 0;
std::vector<std::string> result;
std::vector<std::string> ret;
std::string buffer;
for(size_t a = 0; a < str.size(); a++)
for(char a : str)
{
if(str[a] == '\n' && last_character != '\\')
if(a == token)
{
result.emplace_back(str.substr(line_start, a - line_start));
line_start = a + 1;
if(!buffer.empty())
{
ret.emplace_back(std::move(buffer));
}
last_character = str[a];
buffer.clear();
}
result.emplace_back(str.substr(line_start, str.size() - line_start));
return result;
else if(a != ' ' || !strip_spacing)
{
buffer += a;
}
}
if(!buffer.empty())
{
ret.emplace_back(std::move(buffer));
}
return ret;
}
void Http::set_body(const std::string &body_)
@ -244,6 +253,18 @@ namespace fr
std::string header_name = str.substr(0, colon_pos);
std::string header_value = str.substr(data_begin, data_len);
std::transform(header_name.begin(), header_name.end(), header_name.begin(), ::tolower);
//Check if it contains transfer encoding, we store this in a separate set
if(header_name == "transfer-encoding")
{
auto encodings = split_string(header_value, ',', true);
for(const auto &enc : encodings)
{
transfer_encodings.emplace(string_to_transfer_encoding(enc));
}
}
//Store header for good
header_data.emplace(std::move(header_name), std::move(header_value));
}

31
src/HttpResponse.cpp
View File

@ -50,7 +50,36 @@ namespace fr
if(body.size() > MAX_HTTP_BODY_SIZE)
return fr::Socket::Status::HttpBodyTooBig;
//Cut off any data if it exceeds content length, todo: potentially an issue, could cut the next request off
//Check if chunked encoding
if(transfer_encodings.find(TransferEncoding::Chunked) != transfer_encodings.end())
{
//It's chunked, find the length of the chunk in hex!
unsigned int chunk_len = 1;
while(chunk_len > 0 && chunk_offset < body.size())
{
auto length_end = body.find("\r\n", chunk_offset);
if(length_end == std::string::npos)
return fr::Socket::Status::NotEnoughData;
std::string hex_length = body.substr(chunk_offset, length_end - chunk_offset);
chunk_len = std::stoul(hex_length, nullptr, 16);
//If the current chunk hasn't fully been received yet then ask for more data
if(body.size() - length_end < chunk_len + 4) //+4 because of the \r\n after the hex value and the chunk end
return fr::Socket::Status::NotEnoughData;
//Now we have the full chunk, erase the padding \r\n
body.erase(chunk_offset + hex_length.size() + chunk_len + 2, 2); //delete \r\n after chunk
body.erase(chunk_offset, hex_length.size() + 2); //delete both hex length and following \r\n
chunk_offset += chunk_len;
}
//If this chunk is 0, we have everything!
if(chunk_len == 0)
return fr::Socket::Status::Success;
return fr::Socket::Status::NotEnoughData;
}
//Cut off any data if it exceeds content length, provided that a content length is specified
if(content_length > 0 && body.size() > content_length)
body.resize(content_length);
else if(body.size() < content_length)

4
src/TcpSocket.cpp
View File

@ -5,8 +5,6 @@
#include <iostream>
#include <frnetlib/SocketSelector.h>
#include <frnetlib/TcpSocket.h>
#include "frnetlib/TcpSocket.h"
#define DEFAULT_SOCKET_TIMEOUT 20
namespace fr
@ -21,7 +19,7 @@ namespace fr
TcpSocket::~TcpSocket()
{
close_socket();
TcpSocket::close_socket();
}
Socket::Status TcpSocket::send_raw(const char *data, size_t size, size_t &sent)

2
tests/CMakeLists.txt
View File

@ -27,5 +27,5 @@ else()
endif()
#Link tests
target_link_libraries(${FRNETLIB_TEST} frnetlib)
target_link_libraries(${FRNETLIB_TEST} frnetlib gmock)
add_test(test1 ${FRNETLIB_TEST})

66
tests/HttpResponseTest.cpp
View File

@ -107,6 +107,72 @@ TEST(HttpResponseTest, response_partial_parse)
ASSERT_EQ(test.get_body(), response_body);
}
TEST(HttpResponseTest, parse_chunked_response_test)
{
const std::string raw_response1 =
"HTTP/1.1 200 OK\r\n"
"Content-Type: text/plain\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"7\r\n"
"Mozilla\r\n"
"9\r\n"
"Developer\r\n"
"7\r\n"
"Network\r\n"
"0\r\n"
"\r\n";
//Parse response
fr::HttpResponse test;
ASSERT_EQ(test.parse(raw_response1.c_str(), raw_response1.size()), fr::Socket::Status::Success);
//Verify it
ASSERT_EQ(test.get_status(), fr::Http::RequestStatus::Ok);
ASSERT_EQ(test.get_body(), "MozillaDeveloperNetwork");
}
TEST(HttpResponseTest, parse_partial_chunked_response_test)
{
const std::string raw_response1 =
"HTTP/1.1 200 OK\r\n"
"Content-Type: text/plain\r\n"
"Transfer-Encoding: chunked\r\n";
const std::string raw_response2 =
"\r\n"
"7\r\n"
"Mozilla\r\n";
const std::string raw_response3 =
"9\r\n";
const std::string raw_response4 =
"Developer\r\n"
"7\r\n"
"Netw";
const std::string raw_response5 =
"ork\r\n"
"0\r\n";
const std::string raw_response6 =
"\r\n";
//Parse response
fr::HttpResponse test;
ASSERT_EQ(test.parse(raw_response1.c_str(), raw_response1.size()), fr::Socket::Status::NotEnoughData);
ASSERT_EQ(test.parse(raw_response2.c_str(), raw_response2.size()), fr::Socket::Status::NotEnoughData);
ASSERT_EQ(test.parse(raw_response3.c_str(), raw_response3.size()), fr::Socket::Status::NotEnoughData);
ASSERT_EQ(test.parse(raw_response4.c_str(), raw_response4.size()), fr::Socket::Status::NotEnoughData);
ASSERT_EQ(test.parse(raw_response5.c_str(), raw_response5.size()), fr::Socket::Status::NotEnoughData);
ASSERT_EQ(test.parse(raw_response6.c_str(), raw_response6.size()), fr::Socket::Status::Success);
//Verify it
ASSERT_EQ(test.get_status(), fr::Http::RequestStatus::Ok);
ASSERT_EQ(test.get_body(), "MozillaDeveloperNetwork");
}
TEST(HttpResponseTest, header_length_test)
{
//Try data with no header end first

20
tests/HttpTest.cpp
View File

@ -3,6 +3,7 @@
//
#include <gtest/gtest.h>
#include <gmock/gmock-more-matchers.h>
#include <frnetlib/HttpResponse.h>
TEST(HttpTest, test_request_type_to_string)
@ -58,3 +59,22 @@ TEST(HttpTest, test_get_mimetype)
ASSERT_EQ(fr::Http::get_mimetype("my_file.html"), "text/html");
ASSERT_EQ(fr::Http::get_mimetype("file.some_random_type"), "application/octet-stream");
}
TEST(HttpTest, test_string_to_transfer_encoding)
{
ASSERT_EQ(fr::Http::string_to_transfer_encoding("chunked"), fr::Http::TransferEncoding::Chunked);
ASSERT_EQ(fr::Http::string_to_transfer_encoding("unknown"), fr::Http::TransferEncoding::None);
ASSERT_EQ(fr::Http::string_to_transfer_encoding("ioudhweauidhgiwuyahfiuywhafyuhgwayufhg"), fr::Http::TransferEncoding::None);
ASSERT_EQ(fr::Http::string_to_transfer_encoding("IDenTITy"), fr::Http::TransferEncoding::Identity);
}
namespace fr
{
TEST(HttpTest, test_string_split)
{
ASSERT_THAT(Http::split_string("chunked\nstuff"), ::testing::ElementsAre("chunked", "stuff"));
ASSERT_THAT(Http::split_string("chunked\n stuff"), ::testing::ElementsAre("chunked", " stuff"));
ASSERT_THAT(Http::split_string("chunked,stuff", ',', false), ::testing::ElementsAre("chunked", "stuff"));
ASSERT_THAT(Http::split_string("chunked, stuff", ',', true), ::testing::ElementsAre("chunked", "stuff"));
}
}