From 5b2b92835e52d546e595a889779bc62e83be1b72 Mon Sep 17 00:00:00 2001 From: Fred Nicolson Date: Mon, 25 Nov 2019 16:29:06 +0000 Subject: [PATCH] Add support for chunked HTTP response parsing --- CMakeLists.txt | 1 + include/frnetlib/Http.h | 49 +++++++++++++++++++++- include/frnetlib/HttpResponse.h | 1 + include/frnetlib/NetworkEncoding.h | 4 +- src/Http.cpp | 43 ++++++++++++++----- src/HttpResponse.cpp | 31 +++++++++++++- src/TcpSocket.cpp | 4 +- tests/CMakeLists.txt | 2 +- tests/HttpResponseTest.cpp | 66 ++++++++++++++++++++++++++++++ tests/HttpTest.cpp | 20 +++++++++ 10 files changed, 202 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index abed522..9c4e50e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,7 @@ target_link_libraries(frnetlib ${FRNETLIB_LINK_LIBRARIES}) #Build Tests if needbe if(BUILD_TESTS) + add_definitions(-DENABLE_TESTING) set(EXT_PROJECTS_DIR ${PROJECT_SOURCE_DIR}/ext) add_subdirectory(${EXT_PROJECTS_DIR}/gtest) add_subdirectory(tests) diff --git a/include/frnetlib/Http.h b/include/frnetlib/Http.h index 6bd0715..e10da39 100644 --- a/include/frnetlib/Http.h +++ b/include/frnetlib/Http.h @@ -7,13 +7,23 @@ #include #include #include +#include +#include +#include #include "Socket.h" #include "Sendable.h" +#ifdef ENABLE_TESTING +#include +#endif + namespace fr { class Http : public Sendable { +#ifdef ENABLE_TESTING + FRIEND_TEST(HttpTest, test_string_split); +#endif public: enum class RequestVersion { @@ -32,6 +42,16 @@ namespace fr Unknown = 6, Partial = 7, }; + enum class TransferEncoding + { + None = 0, + Chunked = 1, + Compress = 2, + Deflate = 3, + Gzip = 4, + Identity = 5, + EncodingCount = 6, + }; enum class RequestStatus { Continue = 100, @@ -267,6 +287,28 @@ namespace fr */ const static std::string &get_mimetype(const std::string &filename); + /*! + * Converts a string to a TransferEncoding enum. + * + * @note case insentive + * @param encoding String to convert. + * @return The encoding type, or TransferEncoding::None if no match was found + */ + static TransferEncoding string_to_transfer_encoding(std::string encoding) + { + static_assert((uint32_t)TransferEncoding::EncodingCount == 6, "Update transfer_encoding_to_string"); + const static std::unordered_map string_list = {{"none", TransferEncoding::None}, + {"chunked", TransferEncoding::Chunked}, + {"compress", TransferEncoding::Compress}, + {"deflate", TransferEncoding::Deflate}, + {"gzip", TransferEncoding::Gzip}, + {"identity", TransferEncoding::Identity}}; + + std::transform(encoding.begin(), encoding.end(), encoding.begin(), tolower); + auto iter = string_list.find(encoding); + return iter == string_list.end() ? TransferEncoding::None : iter->second; + } + /*! * Converts a 'RequestType' enum value to * a printable string. @@ -340,11 +382,13 @@ namespace fr protected: /*! - * Splits a string by new line. Ignores escaped \n's + * Splits a string by a given token. Allows for escaping. * + * @param token Token to split by + * @param strip_spacing True to remove ALL spacing from entries. False to leave intact. * @return The split string */ - static std::vector split_string(const std::string &str); + static std::vector split_string(const std::string &str, char token = '\n', bool strip_spacing = false); /*! * Converts a parameter list to a vector pair. @@ -369,6 +413,7 @@ namespace fr std::unordered_map header_data; std::unordered_map post_data; std::unordered_map get_data; + std::set transfer_encodings; std::string body; RequestType request_type; std::string uri; diff --git a/include/frnetlib/HttpResponse.h b/include/frnetlib/HttpResponse.h index d77963d..f91c802 100644 --- a/include/frnetlib/HttpResponse.h +++ b/include/frnetlib/HttpResponse.h @@ -50,6 +50,7 @@ namespace fr //State bool header_ended{false}; size_t content_length{0}; + size_t chunk_offset{0}; }; } diff --git a/include/frnetlib/NetworkEncoding.h b/include/frnetlib/NetworkEncoding.h index 2c74c20..23ece3f 100644 --- a/include/frnetlib/NetworkEncoding.h +++ b/include/frnetlib/NetworkEncoding.h @@ -34,11 +34,13 @@ #define INVALID_SOCKET 0 #define SOCKET_ERROR (-1) -#include #include #include #include #include +#include +#include +#include #endif diff --git a/src/Http.cpp b/src/Http.cpp index 4e0879e..be924c9 100644 --- a/src/Http.cpp +++ b/src/Http.cpp @@ -26,23 +26,32 @@ namespace fr return request_type; } - std::vector Http::split_string(const std::string &str) + std::vector Http::split_string(const std::string &str, char token, bool strip_spacing) { - char last_character = '\0'; - size_t line_start = 0; - std::vector result; + std::vector ret; + std::string buffer; - for(size_t a = 0; a < str.size(); a++) + for(char a : str) { - if(str[a] == '\n' && last_character != '\\') + if(a == token) { - result.emplace_back(str.substr(line_start, a - line_start)); - line_start = a + 1; + if(!buffer.empty()) + { + ret.emplace_back(std::move(buffer)); + } + buffer.clear(); + } + else if(a != ' ' || !strip_spacing) + { + buffer += a; } - last_character = str[a]; } - result.emplace_back(str.substr(line_start, str.size() - line_start)); - return result; + if(!buffer.empty()) + { + ret.emplace_back(std::move(buffer)); + } + + return ret; } void Http::set_body(const std::string &body_) @@ -244,6 +253,18 @@ namespace fr std::string header_name = str.substr(0, colon_pos); std::string header_value = str.substr(data_begin, data_len); std::transform(header_name.begin(), header_name.end(), header_name.begin(), ::tolower); + + //Check if it contains transfer encoding, we store this in a separate set + if(header_name == "transfer-encoding") + { + auto encodings = split_string(header_value, ',', true); + for(const auto &enc : encodings) + { + transfer_encodings.emplace(string_to_transfer_encoding(enc)); + } + } + + //Store header for good header_data.emplace(std::move(header_name), std::move(header_value)); } diff --git a/src/HttpResponse.cpp b/src/HttpResponse.cpp index d78509b..5fe390e 100644 --- a/src/HttpResponse.cpp +++ b/src/HttpResponse.cpp @@ -50,7 +50,36 @@ namespace fr if(body.size() > MAX_HTTP_BODY_SIZE) return fr::Socket::Status::HttpBodyTooBig; - //Cut off any data if it exceeds content length, todo: potentially an issue, could cut the next request off + //Check if chunked encoding + if(transfer_encodings.find(TransferEncoding::Chunked) != transfer_encodings.end()) + { + //It's chunked, find the length of the chunk in hex! + unsigned int chunk_len = 1; + while(chunk_len > 0 && chunk_offset < body.size()) + { + auto length_end = body.find("\r\n", chunk_offset); + if(length_end == std::string::npos) + return fr::Socket::Status::NotEnoughData; + std::string hex_length = body.substr(chunk_offset, length_end - chunk_offset); + chunk_len = std::stoul(hex_length, nullptr, 16); + + //If the current chunk hasn't fully been received yet then ask for more data + if(body.size() - length_end < chunk_len + 4) //+4 because of the \r\n after the hex value and the chunk end + return fr::Socket::Status::NotEnoughData; + + //Now we have the full chunk, erase the padding \r\n + body.erase(chunk_offset + hex_length.size() + chunk_len + 2, 2); //delete \r\n after chunk + body.erase(chunk_offset, hex_length.size() + 2); //delete both hex length and following \r\n + chunk_offset += chunk_len; + } + + //If this chunk is 0, we have everything! + if(chunk_len == 0) + return fr::Socket::Status::Success; + return fr::Socket::Status::NotEnoughData; + } + + //Cut off any data if it exceeds content length, provided that a content length is specified if(content_length > 0 && body.size() > content_length) body.resize(content_length); else if(body.size() < content_length) diff --git a/src/TcpSocket.cpp b/src/TcpSocket.cpp index 94209fb..f37abda 100644 --- a/src/TcpSocket.cpp +++ b/src/TcpSocket.cpp @@ -5,8 +5,6 @@ #include #include #include - -#include "frnetlib/TcpSocket.h" #define DEFAULT_SOCKET_TIMEOUT 20 namespace fr @@ -21,7 +19,7 @@ namespace fr TcpSocket::~TcpSocket() { - close_socket(); + TcpSocket::close_socket(); } Socket::Status TcpSocket::send_raw(const char *data, size_t size, size_t &sent) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b0b698f..73ea505 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,5 +27,5 @@ else() endif() #Link tests -target_link_libraries(${FRNETLIB_TEST} frnetlib) +target_link_libraries(${FRNETLIB_TEST} frnetlib gmock) add_test(test1 ${FRNETLIB_TEST}) \ No newline at end of file diff --git a/tests/HttpResponseTest.cpp b/tests/HttpResponseTest.cpp index 8161bd3..f20bbd3 100644 --- a/tests/HttpResponseTest.cpp +++ b/tests/HttpResponseTest.cpp @@ -107,6 +107,72 @@ TEST(HttpResponseTest, response_partial_parse) ASSERT_EQ(test.get_body(), response_body); } +TEST(HttpResponseTest, parse_chunked_response_test) +{ + const std::string raw_response1 = + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/plain\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "7\r\n" + "Mozilla\r\n" + "9\r\n" + "Developer\r\n" + "7\r\n" + "Network\r\n" + "0\r\n" + "\r\n"; + + //Parse response + fr::HttpResponse test; + ASSERT_EQ(test.parse(raw_response1.c_str(), raw_response1.size()), fr::Socket::Status::Success); + + //Verify it + ASSERT_EQ(test.get_status(), fr::Http::RequestStatus::Ok); + ASSERT_EQ(test.get_body(), "MozillaDeveloperNetwork"); +} + +TEST(HttpResponseTest, parse_partial_chunked_response_test) +{ + const std::string raw_response1 = + "HTTP/1.1 200 OK\r\n" + "Content-Type: text/plain\r\n" + "Transfer-Encoding: chunked\r\n"; + + const std::string raw_response2 = + "\r\n" + "7\r\n" + "Mozilla\r\n"; + + const std::string raw_response3 = + "9\r\n"; + + const std::string raw_response4 = + "Developer\r\n" + "7\r\n" + "Netw"; + const std::string raw_response5 = + "ork\r\n" + "0\r\n"; + + const std::string raw_response6 = + "\r\n"; + + //Parse response + fr::HttpResponse test; + ASSERT_EQ(test.parse(raw_response1.c_str(), raw_response1.size()), fr::Socket::Status::NotEnoughData); + ASSERT_EQ(test.parse(raw_response2.c_str(), raw_response2.size()), fr::Socket::Status::NotEnoughData); + ASSERT_EQ(test.parse(raw_response3.c_str(), raw_response3.size()), fr::Socket::Status::NotEnoughData); + ASSERT_EQ(test.parse(raw_response4.c_str(), raw_response4.size()), fr::Socket::Status::NotEnoughData); + ASSERT_EQ(test.parse(raw_response5.c_str(), raw_response5.size()), fr::Socket::Status::NotEnoughData); + ASSERT_EQ(test.parse(raw_response6.c_str(), raw_response6.size()), fr::Socket::Status::Success); + + //Verify it + + ASSERT_EQ(test.get_status(), fr::Http::RequestStatus::Ok); + ASSERT_EQ(test.get_body(), "MozillaDeveloperNetwork"); +} + TEST(HttpResponseTest, header_length_test) { //Try data with no header end first diff --git a/tests/HttpTest.cpp b/tests/HttpTest.cpp index b796b0c..ea5bb1a 100644 --- a/tests/HttpTest.cpp +++ b/tests/HttpTest.cpp @@ -3,6 +3,7 @@ // #include +#include #include TEST(HttpTest, test_request_type_to_string) @@ -57,4 +58,23 @@ TEST(HttpTest, test_get_mimetype) ASSERT_EQ(fr::Http::get_mimetype(".html"), "text/html"); ASSERT_EQ(fr::Http::get_mimetype("my_file.html"), "text/html"); ASSERT_EQ(fr::Http::get_mimetype("file.some_random_type"), "application/octet-stream"); +} + +TEST(HttpTest, test_string_to_transfer_encoding) +{ + ASSERT_EQ(fr::Http::string_to_transfer_encoding("chunked"), fr::Http::TransferEncoding::Chunked); + ASSERT_EQ(fr::Http::string_to_transfer_encoding("unknown"), fr::Http::TransferEncoding::None); + ASSERT_EQ(fr::Http::string_to_transfer_encoding("ioudhweauidhgiwuyahfiuywhafyuhgwayufhg"), fr::Http::TransferEncoding::None); + ASSERT_EQ(fr::Http::string_to_transfer_encoding("IDenTITy"), fr::Http::TransferEncoding::Identity); +} + +namespace fr +{ + TEST(HttpTest, test_string_split) + { + ASSERT_THAT(Http::split_string("chunked\nstuff"), ::testing::ElementsAre("chunked", "stuff")); + ASSERT_THAT(Http::split_string("chunked\n stuff"), ::testing::ElementsAre("chunked", " stuff")); + ASSERT_THAT(Http::split_string("chunked,stuff", ',', false), ::testing::ElementsAre("chunked", "stuff")); + ASSERT_THAT(Http::split_string("chunked, stuff", ',', true), ::testing::ElementsAre("chunked", "stuff")); + } } \ No newline at end of file