Add support for chunked HTTP response parsing

CMakeLists.txt

@@ -95,6 +95,7 @@ target_link_libraries(frnetlib ${FRNETLIB_LINK_LIBRARIES})
 
 #Build Tests if needbe
 if(BUILD_TESTS)
+    add_definitions(-DENABLE_TESTING)
     set(EXT_PROJECTS_DIR ${PROJECT_SOURCE_DIR}/ext)
     add_subdirectory(${EXT_PROJECTS_DIR}/gtest)
     add_subdirectory(tests)

include/frnetlib/Http.h

@@ -7,13 +7,23 @@
 #include <string>
 #include <vector>
 #include <unordered_map>
+#include <set>
+#include <iostream>
+#include <algorithm>
 #include "Socket.h"
 #include "Sendable.h"
 
+#ifdef ENABLE_TESTING
+#include <gtest/gtest.h>
+#endif
+
 namespace fr
 {
     class Http : public Sendable
     {
+#ifdef ENABLE_TESTING
+        FRIEND_TEST(HttpTest, test_string_split);
+#endif
     public:
         enum class RequestVersion
         {
@@ -32,6 +42,16 @@ namespace fr
             Unknown = 6,
             Partial = 7,
         };
+        enum class TransferEncoding
+        {
+            None = 0,
+            Chunked = 1,
+            Compress = 2,
+            Deflate = 3,
+            Gzip = 4,
+            Identity = 5,
+            EncodingCount = 6,
+        };
         enum class RequestStatus
         {
             Continue = 100,
@@ -267,6 +287,28 @@ namespace fr
          */
         const static std::string &get_mimetype(const std::string &filename);
 
+        /*!
+         * Converts a string to a TransferEncoding enum.
+         *
+         * @note case insentive
+         * @param encoding String to convert.
+         * @return The encoding type, or TransferEncoding::None if no match was found
+         */
+        static TransferEncoding string_to_transfer_encoding(std::string encoding)
+        {
+            static_assert((uint32_t)TransferEncoding::EncodingCount == 6, "Update transfer_encoding_to_string");
+            const static std::unordered_map<std::string, TransferEncoding> string_list = {{"none", TransferEncoding::None},
+                                                                                          {"chunked", TransferEncoding::Chunked},
+                                                                                          {"compress", TransferEncoding::Compress},
+                                                                                          {"deflate", TransferEncoding::Deflate},
+                                                                                          {"gzip", TransferEncoding::Gzip},
+                                                                                          {"identity", TransferEncoding::Identity}};
+
+            std::transform(encoding.begin(), encoding.end(), encoding.begin(), tolower);
+            auto iter = string_list.find(encoding);
+            return iter == string_list.end() ? TransferEncoding::None : iter->second;
+        }
+
         /*!
          * Converts a 'RequestType' enum value to
          * a printable string.
@@ -340,11 +382,13 @@ namespace fr
 
     protected:
         /*!
-         * Splits a string by new line. Ignores escaped \n's
+         * Splits a string by a given token. Allows for escaping.
          *
+         * @param token Token to split by
+         * @param strip_spacing True to remove ALL spacing from entries. False to leave intact.
          * @return The split string
          */
-        static std::vector<std::string> split_string(const std::string &str);
+        static std::vector<std::string> split_string(const std::string &str, char token = '\n', bool strip_spacing = false);
 
         /*!
          * Converts a parameter list to a vector pair.
@@ -369,6 +413,7 @@ namespace fr
         std::unordered_map<std::string, std::string> header_data;
         std::unordered_map<std::string, std::string> post_data;
         std::unordered_map<std::string, std::string> get_data;
+        std::set<TransferEncoding> transfer_encodings;
         std::string body;
         RequestType request_type;
         std::string uri;

include/frnetlib/HttpResponse.h

@@ -50,6 +50,7 @@ namespace fr
         //State
         bool header_ended{false};
         size_t content_length{0};
+        size_t chunk_offset{0};
     };
 }
 

include/frnetlib/NetworkEncoding.h

@@ -34,11 +34,13 @@
 #define INVALID_SOCKET 0
 #define SOCKET_ERROR (-1)
 
-#include <netdb.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <netinet/in.h>
 #include <netinet/tcp.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
 #endif
 
 

src/Http.cpp

@@ -26,23 +26,32 @@ namespace fr
         return request_type;
     }
 
-    std::vector<std::string> Http::split_string(const std::string &str)
+    std::vector<std::string> Http::split_string(const std::string &str, char token, bool strip_spacing)
     {
-        char last_character = '\0';
+        std::vector<std::string> ret;
-        size_t line_start = 0;
+        std::string buffer;
-        std::vector<std::string> result;
 
-        for(size_t a = 0; a < str.size(); a++)
+        for(char a : str)
         {
-            if(str[a] == '\n' && last_character != '\\')
+            if(a == token)
             {
-                result.emplace_back(str.substr(line_start, a - line_start));
+                if(!buffer.empty())
-                line_start = a + 1;
+                {
+                    ret.emplace_back(std::move(buffer));
                 }
-            last_character = str[a];
+                buffer.clear();
             }
-        result.emplace_back(str.substr(line_start, str.size() - line_start));
+            else if(a != ' ' || !strip_spacing)
-        return result;
+            {
+                buffer += a;
+            }
+        }
+        if(!buffer.empty())
+        {
+            ret.emplace_back(std::move(buffer));
+        }
+
+        return ret;
     }
 
     void Http::set_body(const std::string &body_)
@@ -244,6 +253,18 @@ namespace fr
         std::string header_name = str.substr(0, colon_pos);
         std::string header_value = str.substr(data_begin, data_len);
         std::transform(header_name.begin(), header_name.end(), header_name.begin(), ::tolower);
+
+        //Check if it contains transfer encoding, we store this in a separate set
+        if(header_name == "transfer-encoding")
+        {
+            auto encodings = split_string(header_value, ',', true);
+            for(const auto &enc : encodings)
+            {
+                transfer_encodings.emplace(string_to_transfer_encoding(enc));
+            }
+        }
+
+        //Store header for good
         header_data.emplace(std::move(header_name), std::move(header_value));
     }
 

src/HttpResponse.cpp

@@ -50,7 +50,36 @@ namespace fr
         if(body.size() > MAX_HTTP_BODY_SIZE)
             return fr::Socket::Status::HttpBodyTooBig;
 
-        //Cut off any data if it exceeds content length, todo: potentially an issue, could cut the next request off
+        //Check if chunked encoding
+        if(transfer_encodings.find(TransferEncoding::Chunked) != transfer_encodings.end())
+        {
+            //It's chunked, find the length of the chunk in hex!
+            unsigned int chunk_len = 1;
+            while(chunk_len > 0 && chunk_offset < body.size())
+            {
+                auto length_end = body.find("\r\n", chunk_offset);
+                if(length_end == std::string::npos)
+                    return fr::Socket::Status::NotEnoughData;
+                std::string hex_length = body.substr(chunk_offset, length_end - chunk_offset);
+                chunk_len = std::stoul(hex_length, nullptr, 16);
+
+                //If the current chunk hasn't fully been received yet then ask for more data
+                if(body.size() - length_end < chunk_len + 4) //+4 because of the \r\n after the hex value and the chunk end
+                    return fr::Socket::Status::NotEnoughData;
+
+                //Now we have the full chunk, erase the padding \r\n
+                body.erase(chunk_offset + hex_length.size() + chunk_len + 2, 2); //delete \r\n after chunk
+                body.erase(chunk_offset, hex_length.size() + 2); //delete both hex length and following \r\n
+                chunk_offset += chunk_len;
+            }
+
+            //If this chunk is 0, we have everything!
+            if(chunk_len == 0)
+                return fr::Socket::Status::Success;
+            return fr::Socket::Status::NotEnoughData;
+        }
+
+        //Cut off any data if it exceeds content length, provided that a content length is specified
         if(content_length > 0 && body.size() > content_length)
             body.resize(content_length);
         else if(body.size() < content_length)

src/TcpSocket.cpp

@@ -5,8 +5,6 @@
 #include <iostream>
 #include <frnetlib/SocketSelector.h>
 #include <frnetlib/TcpSocket.h>
-
-#include "frnetlib/TcpSocket.h"
 #define DEFAULT_SOCKET_TIMEOUT 20
 
 namespace fr
@@ -21,7 +19,7 @@ namespace fr
 
     TcpSocket::~TcpSocket()
     {
-        close_socket();
+        TcpSocket::close_socket();
     }
 
     Socket::Status TcpSocket::send_raw(const char *data, size_t size, size_t &sent)

tests/CMakeLists.txt

@@ -27,5 +27,5 @@ else()
 endif()
 
 #Link tests
-target_link_libraries(${FRNETLIB_TEST} frnetlib)
+target_link_libraries(${FRNETLIB_TEST} frnetlib gmock)
 add_test(test1 ${FRNETLIB_TEST})

tests/HttpResponseTest.cpp

@@ -107,6 +107,72 @@ TEST(HttpResponseTest, response_partial_parse)
     ASSERT_EQ(test.get_body(), response_body);
 }
 
+TEST(HttpResponseTest, parse_chunked_response_test)
+{
+    const std::string raw_response1 =
+                                    "HTTP/1.1 200 OK\r\n"
+                                    "Content-Type: text/plain\r\n"
+                                    "Transfer-Encoding: chunked\r\n"
+                                    "\r\n"
+                                    "7\r\n"
+                                    "Mozilla\r\n"
+                                    "9\r\n"
+                                    "Developer\r\n"
+                                    "7\r\n"
+                                    "Network\r\n"
+                                    "0\r\n"
+                                    "\r\n";
+
+    //Parse response
+    fr::HttpResponse test;
+    ASSERT_EQ(test.parse(raw_response1.c_str(), raw_response1.size()), fr::Socket::Status::Success);
+
+    //Verify it
+    ASSERT_EQ(test.get_status(), fr::Http::RequestStatus::Ok);
+    ASSERT_EQ(test.get_body(), "MozillaDeveloperNetwork");
+}
+
+TEST(HttpResponseTest, parse_partial_chunked_response_test)
+{
+    const std::string raw_response1 =
+            "HTTP/1.1 200 OK\r\n"
+            "Content-Type: text/plain\r\n"
+            "Transfer-Encoding: chunked\r\n";
+
+    const std::string raw_response2 =
+            "\r\n"
+            "7\r\n"
+            "Mozilla\r\n";
+
+    const std::string raw_response3 =
+            "9\r\n";
+
+    const std::string raw_response4 =
+            "Developer\r\n"
+            "7\r\n"
+            "Netw";
+    const std::string raw_response5 =
+            "ork\r\n"
+            "0\r\n";
+
+    const std::string raw_response6 =
+            "\r\n";
+
+    //Parse response
+    fr::HttpResponse test;
+    ASSERT_EQ(test.parse(raw_response1.c_str(), raw_response1.size()), fr::Socket::Status::NotEnoughData);
+    ASSERT_EQ(test.parse(raw_response2.c_str(), raw_response2.size()), fr::Socket::Status::NotEnoughData);
+    ASSERT_EQ(test.parse(raw_response3.c_str(), raw_response3.size()), fr::Socket::Status::NotEnoughData);
+    ASSERT_EQ(test.parse(raw_response4.c_str(), raw_response4.size()), fr::Socket::Status::NotEnoughData);
+    ASSERT_EQ(test.parse(raw_response5.c_str(), raw_response5.size()), fr::Socket::Status::NotEnoughData);
+    ASSERT_EQ(test.parse(raw_response6.c_str(), raw_response6.size()), fr::Socket::Status::Success);
+
+    //Verify it
+
+    ASSERT_EQ(test.get_status(), fr::Http::RequestStatus::Ok);
+    ASSERT_EQ(test.get_body(), "MozillaDeveloperNetwork");
+}
+
 TEST(HttpResponseTest, header_length_test)
 {
     //Try data with no header end first

tests/HttpTest.cpp

@@ -3,6 +3,7 @@
 //
 
 #include <gtest/gtest.h>
+#include <gmock/gmock-more-matchers.h>
 #include <frnetlib/HttpResponse.h>
 
 TEST(HttpTest, test_request_type_to_string)
@@ -58,3 +59,22 @@ TEST(HttpTest, test_get_mimetype)
     ASSERT_EQ(fr::Http::get_mimetype("my_file.html"), "text/html");
     ASSERT_EQ(fr::Http::get_mimetype("file.some_random_type"), "application/octet-stream");
 }
+
+TEST(HttpTest, test_string_to_transfer_encoding)
+{
+    ASSERT_EQ(fr::Http::string_to_transfer_encoding("chunked"), fr::Http::TransferEncoding::Chunked);
+    ASSERT_EQ(fr::Http::string_to_transfer_encoding("unknown"), fr::Http::TransferEncoding::None);
+    ASSERT_EQ(fr::Http::string_to_transfer_encoding("ioudhweauidhgiwuyahfiuywhafyuhgwayufhg"), fr::Http::TransferEncoding::None);
+    ASSERT_EQ(fr::Http::string_to_transfer_encoding("IDenTITy"), fr::Http::TransferEncoding::Identity);
+}
+
+namespace fr
+{
+    TEST(HttpTest, test_string_split)
+    {
+        ASSERT_THAT(Http::split_string("chunked\nstuff"), ::testing::ElementsAre("chunked", "stuff"));
+        ASSERT_THAT(Http::split_string("chunked\n stuff"), ::testing::ElementsAre("chunked", " stuff"));
+        ASSERT_THAT(Http::split_string("chunked,stuff", ',', false), ::testing::ElementsAre("chunked", "stuff"));
+        ASSERT_THAT(Http::split_string("chunked, stuff", ',', true), ::testing::ElementsAre("chunked", "stuff"));
+    }
+}