From df9eac120625a8ec5193f28d80d4ea4147584f4d Mon Sep 17 00:00:00 2001 From: rexy712 Date: Tue, 5 Mar 2019 14:19:32 -0800 Subject: [PATCH] video thumbnails --- TODO | 14 +-- include/matrix.hpp | 12 ++- include/raii/rjp_string.hpp | 5 +- include/raii/string.hpp | 5 +- include/raii/video_man.hpp | 190 ++++++++++++++++++++++++++++++++++++ src/matrix.cpp | 159 +++++++++++++++++++----------- src/reddit.cpp | 2 +- src/test.cpp | 7 +- 8 files changed, 314 insertions(+), 80 deletions(-) create mode 100644 include/raii/video_man.hpp diff --git a/TODO b/TODO index b80e2c8..65c4f2e 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,3 @@ -file info struct as return from upload_file so as to know some metadata about the file - uploaded url - file type if able to be figured out - file size - image - dimensions - thumbnail - - -use libmagic to determine file types -create thumbnail images somehow +use libmagic to determine file types? +//create a raii interface for libav* which allows creation of thumbnails +add av_freep cleanup to raii interface for libav* (probably replacing unique_ptr in that case) diff --git a/include/matrix.hpp b/include/matrix.hpp index 2f39bbf..b51ca26 100644 --- a/include/matrix.hpp +++ b/include/matrix.hpp @@ -37,6 +37,7 @@ namespace matrix{ size_t thumb_height; size_t thumbsize; }; + using video_info = image_info; class bot { @@ -105,21 +106,24 @@ namespace matrix{ //other networked operations raii::string create_room(const raii::string_base& name, const raii::string_base& alias); - raii::rjp_string upload_file(const raii::string_base& filename, const raii::curl_llist& header); + //upload media + file_info upload_file(const raii::string_base& filename); image_info upload_image(const raii::string_base& filename, const raii::string_base& alias); image_info upload_image(const raii::string_base& filename); - raii::rjp_string upload_video(const raii::string_base& filename); + video_info upload_video(const raii::string_base& filename); + //send messages raii::rjp_string send_image(const raii::string_base& room, const image_info& image); - raii::rjp_string send_video(const raii::string_base& room, const raii::string_base& file_url, const raii::string_base& filetype, const raii::string_base& filename); + raii::rjp_string send_video(const raii::string_base& room, const video_info& video); raii::rjp_string send_message(const raii::string_base& room, const raii::string_base& text); + raii::rjp_string send_file(const raii::string_base& room, const file_info& file); - bool send_file(const raii::string_base& room, const raii::string_base& file_url); void sync(void); void logout(void); protected: raii::rjp_string _upload_file(raii::filerd& fp, const raii::curl_llist& header); + raii::rjp_string _send_message(const raii::string_base& room, const raii::string_base& msg, const raii::curl_llist& header); static size_t _post_reply_curl_callback(char* ptr, size_t size, size_t nmemb, void* userdata); raii::string _get_curl(const raii::string_base& url); raii::string _post_curl(const raii::string_base& postdata, const raii::string_base& url, const raii::curl_llist& header); diff --git a/include/raii/rjp_string.hpp b/include/raii/rjp_string.hpp index 3d6cc3c..c2bc831 100644 --- a/include/raii/rjp_string.hpp +++ b/include/raii/rjp_string.hpp @@ -20,8 +20,9 @@ namespace raii{ return rjp_alloc(size); } static void* copy(const void* data, size_t len){ - void* tmp = allocate(len); - memcpy(tmp, data, len); + char* tmp = reinterpret_cast(allocate(len)); + memcpy(tmp, data, len-1); + tmp[len-1] = 0; return tmp; } }; diff --git a/include/raii/string.hpp b/include/raii/string.hpp index fb2047f..e7b68e1 100644 --- a/include/raii/string.hpp +++ b/include/raii/string.hpp @@ -20,8 +20,9 @@ namespace raii{ return ::operator new(size); } static void* copy(const void* c, size_t size){ - void* tmp = allocate(size); - memcpy(tmp, c, size); + char* tmp = reinterpret_cast(allocate(size)); + memcpy(tmp, c, size-1); + tmp[size-1] = 0; return tmp; } }; diff --git a/include/raii/video_man.hpp b/include/raii/video_man.hpp new file mode 100644 index 0000000..99e2d1f --- /dev/null +++ b/include/raii/video_man.hpp @@ -0,0 +1,190 @@ +#ifndef RAII_VIDEO_MAN_HPP +#define RAII_VIDEO_MAN_HPP + +#include //unique_ptr +#include //exchange +#include "raii/string.hpp" +#include "raii/static_string.hpp" + +extern "C"{ +#include +#include +#include +#include +} + +namespace raii{ + + class video_man + { + private: + static inline auto _context_deleter = [](AVCodecContext* ptr){ + avcodec_close(ptr); + avcodec_free_context(&ptr); + }; + static inline auto _frame_deleter = [](AVFrame* ptr){ + av_frame_unref(ptr); + av_frame_free(&ptr); + }; + static inline auto _packet_deleter = [](AVPacket* ptr){ + av_packet_unref(ptr); + av_packet_free(&ptr); + }; + public: + using context_type = std::unique_ptr; + using frame_type = std::unique_ptr; + using packet_type = std::unique_ptr; + + private: + AVFormatContext* m_vid; + AVCodec* m_codec; + AVCodecContext* m_context; + int m_stream_index = -1; + + + public: + video_man(const raii::string_base& filename){ + av_register_all(); + _init(filename); + } + video_man(const video_man& v) = delete; + video_man(video_man&& v): + m_vid(std::exchange(v.m_vid, nullptr)){} + ~video_man(void){ + reset(); + } + + int height(void)const{ + return m_context->height; + } + int width(void)const{ + return m_context->width; + } + raii::string get_mimetype(void)const{ + const char* first = strstr(m_vid->iformat->name, ","); + return raii::string(raii::static_string(m_vid->iformat->name, first - m_vid->iformat->name)); + } + packet_type create_jpg_thumbnail(void){ + context_type jpg_context = _get_jpeg_context(m_context); + frame_type frame(_init_frame(m_context->width, m_context->height, m_context->pix_fmt), _frame_deleter); + frame_type jpg_frame(_init_frame(m_context->width, m_context->height, jpg_context->pix_fmt), _frame_deleter); + packet_type packet(av_packet_alloc(), _packet_deleter); + SwsContext* sws_ctx = sws_getContext(m_context->width, m_context->height, m_context->pix_fmt, + m_context->width, m_context->height, AV_PIX_FMT_YUV420P, + SWS_BILINEAR, NULL, NULL, NULL); + _change_color_range(sws_ctx); + + av_image_alloc(jpg_frame->data, jpg_frame->linesize, m_context->width, m_context->height, jpg_context->pix_fmt, 32); + while(av_read_frame(m_vid, packet.get()) >= 0){ + if(packet->stream_index == m_stream_index){ + avcodec_send_packet(m_context, packet.get()); + if(avcodec_receive_frame(m_context, frame.get()) >= 0) + break; + } + av_packet_unref(packet.get()); + } + sws_scale(sws_ctx, frame->data, frame->linesize, 0, m_context->height, jpg_frame->data, jpg_frame->linesize); + + sws_freeContext(sws_ctx); + + packet_type jpg_packet(av_packet_alloc(), _packet_deleter); + if(avcodec_send_frame(jpg_context.get(), jpg_frame.get()) < 0){ + av_freep(&jpg_frame->data[0]); + return packet_type(nullptr, _packet_deleter); + } + if(avcodec_receive_packet(jpg_context.get(), jpg_packet.get()) < 0){ + return packet_type(nullptr, _packet_deleter); + } + + av_freep(&jpg_frame->data[0]); + return jpg_packet; + } + + AVFormatContext* release(void){ + avcodec_close(m_context); + avcodec_free_context(&m_context); + m_context = nullptr; + m_codec = nullptr; + m_stream_index = -1; + return std::exchange(m_vid, nullptr); + } + void reset(const raii::string_base& filename){ + reset(); + _init(filename); + } + void reset(void){ + avcodec_close(m_context); + avcodec_free_context(&m_context); + avformat_close_input(&m_vid); + m_context = nullptr; + m_vid = nullptr; + m_codec = nullptr; + m_stream_index = -1; + } + private: + void _init(const raii::string_base& filename){ + m_vid = _open_video_file(filename); + if(!m_vid) + return; + + AVCodecParameters* codec_parms = m_vid->streams[m_stream_index]->codecpar; + m_codec = avcodec_find_decoder(codec_parms->codec_id); + m_context = avcodec_alloc_context3(m_codec); + avcodec_parameters_to_context(m_context, codec_parms); + avcodec_open2(m_context, m_codec, NULL); + } + AVFormatContext* _open_video_file(const raii::string_base& filename){ + AVFormatContext* context = NULL; + avformat_open_input(&context, filename.get(), NULL, NULL); + avformat_find_stream_info(context, NULL); + for(size_t i = 0;i < context->nb_streams;++i){ + if(context->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO){ + m_stream_index = i; + break; + } + } + if(m_stream_index != -1) + return context; + avformat_close_input(&context); + return nullptr; + } + static AVFrame* _init_frame(int w, int h, int format){ + AVFrame* f = av_frame_alloc(); + f->data[0] = NULL; + f->width = w; + f->height = h; + f->format = format; + return f; + } + static void _change_color_range(SwsContext* sws_ctx){ + //we want to convert from YUVJ420P to YUV420P for sws_scale to be happy. but we actually want to output as YUVJ420P. + //so we change the color range to use the full range like YUVJ420P. dummy is used to ignore unneeded returns. + int dummy[4]; + int src_range, dest_range; + int br, con, sat; + sws_getColorspaceDetails(sws_ctx, (int**)&dummy, &src_range, (int**)&dummy, &dest_range, &br, &con, &sat); + const int* coefs = sws_getCoefficients(SWS_CS_DEFAULT); + src_range = 2; + sws_setColorspaceDetails(sws_ctx, coefs, src_range, coefs, dest_range, br, con, sat); + } + static context_type _get_jpeg_context(const AVCodecContext* decoder){ + AVCodec* jpg_codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG); + context_type jpg_context(avcodec_alloc_context3(jpg_codec), _context_deleter); + jpg_context->pix_fmt = AV_PIX_FMT_YUVJ420P; + jpg_context->width = decoder->width; + jpg_context->height = decoder->height; + jpg_context->codec_id = AV_CODEC_ID_MJPEG; + jpg_context->codec_type = AVMEDIA_TYPE_VIDEO; + if(decoder->time_base.num == 0 || decoder->time_base.den == 0) + jpg_context->time_base = AVRational{1, 30}; + else + jpg_context->time_base = decoder->time_base; + avcodec_open2(jpg_context.get(), jpg_codec, NULL); + return jpg_context; + } + }; + + +} + +#endif diff --git a/src/matrix.cpp b/src/matrix.cpp index 36af6f4..d97f188 100644 --- a/src/matrix.cpp +++ b/src/matrix.cpp @@ -1,3 +1,4 @@ +#include "raii/video_man.hpp" #include "matrix.hpp" #include "raii/curl_llist.hpp" @@ -6,12 +7,6 @@ #include "raii/filerd.hpp" #include -extern "C"{ -#include -#include -#include -} - namespace matrix{ auth_data parse_auth_data(RJP_value* root){ @@ -149,10 +144,11 @@ namespace matrix{ return _post_curl(postdata, m_urls.create_room, raii::curl_llist()); } - raii::rjp_string bot::upload_file(const raii::string_base& filename, const raii::curl_llist& header){ + file_info bot::upload_file(const raii::string_base& filename){ raii::filerd fd(filename); if(!fd) return {}; - return _upload_file(fd, header); + + return file_info{_upload_file(fd, raii::curl_llist{}), filename, {}, fd.length()}; } image_info bot::upload_image(const raii::string_base& filename){ return upload_image(filename, raii::static_string()); @@ -177,6 +173,7 @@ namespace matrix{ raii::filerd fd(filename, "rb"); if(!fd) return {}; + //save fullsize image info ret.width = image.getWidth(); ret.height = image.getHeight(); ret.filetype = formattotype(type); @@ -184,63 +181,85 @@ namespace matrix{ ret.filesize = fd.length(); raii::curl_llist header(raii::string("Content-Type: image/" + ret.filetype)); ret.fileurl = _upload_file(fd, header); - fd.reset(fopen("anewNewimage", "w+")); + fd.reset(); -//TODO remove temporary file + //create thumbnail image.makeThumbnail(500); FreeImageIO fileout; - fileout.read_proc = [](void* ptr, unsigned int size, unsigned int nmemb, void* fp) -> unsigned int{ - return fread(ptr, size, nmemb, (FILE*)fp); - }; + std::vector buffer; fileout.write_proc = [](void* ptr, unsigned int size, unsigned int nmemb, void* fp) -> unsigned int{ - return fwrite(ptr, size, nmemb, (FILE*)fp); + std::vector& buffer = *reinterpret_cast*>(fp); + buffer.insert(buffer.end(), (char*)ptr, ((char*)ptr)+size*nmemb); + return size*nmemb; }; - fileout.seek_proc = [](void* fp, long int off, int whence) -> int{ - return fseek((FILE*)fp, off, whence); - }; - fileout.tell_proc = [](void* fp) -> long int{ - return ftell((FILE*)fp); - }; - bool b; + bool b = false; switch(type){ case FIF_JPEG: - b = image.saveToHandle(type, &fileout, (fi_handle)(fd.get()), JPEG_QUALITYGOOD | JPEG_SUBSAMPLING_411); + b = image.saveToHandle(type, &fileout, (fi_handle)&buffer, JPEG_QUALITYGOOD | JPEG_SUBSAMPLING_411); break; case FIF_PNG: - b = image.saveToHandle(type, &fileout, (fi_handle)(fd.get()), PNG_Z_BEST_COMPRESSION); + b = image.saveToHandle(type, &fileout, (fi_handle)&buffer, PNG_Z_BEST_COMPRESSION); break; case FIF_GIF: - b = image.saveToHandle(type, &fileout, (fi_handle)(fd.get())); + b = image.saveToHandle(type, &fileout, (fi_handle)&buffer); break; default: ; }; - ret.thumb_width = image.getWidth(); - ret.thumb_height = image.getHeight(); - - fd.rewind(); - ret.thumburl = _upload_file(fd, header); - ret.thumbsize = fd.length(); + if(!b){ + fprintf(stderr, "Unable to create image thumbnail"); + ret.thumb_width = ret.width; + ret.thumb_height = ret.height; + ret.thumbsize = ret.filesize; + }else{ + ret.thumb_width = image.getWidth(); + ret.thumb_height = image.getHeight(); + ret.thumburl = _post_and_find(raii::static_string(buffer.data(), buffer.size()), m_urls.file_upload, header, "content_uri"_ss); + ret.thumbsize = buffer.size(); + } return ret; } - raii::rjp_string bot::upload_video(const raii::string_base& filename){ - image_info ret; + video_info bot::upload_video(const raii::string_base& filename){ + video_info ret = {}; - av_register_all(); - AVFormatContext* context = NULL; - avformat_open_input(&context, filename.get(), NULL, NULL); - avformat_find_stream_info(context, NULL); - av_dump_format(context, 0, filename.get(), false); - avformat_close_input(&context); + raii::video_man context(filename); + auto packet = context.create_jpg_thumbnail(); + raii::curl_llist header("Content-Type: image/jpeg"); + ret.thumb_width = context.width(); + ret.thumb_height = context.height(); + ret.thumbsize = packet->size; + ret.thumburl = _post_and_find(raii::static_string((char*)packet->data, packet->size), m_urls.file_upload, header, "content_uri"_ss); + raii::string mimetype = context.get_mimetype(); + + + header = raii::curl_llist(raii::string("Content-Type: video/" + mimetype)); raii::filerd fd(filename); if(!fd) return {}; - - raii::curl_llist header(raii::string("Content-Type: video/mp4")); - return _upload_file(fd, header); + ret.filesize = fd.length(); + ret.width = context.width(); + ret.height = context.height(); + ret.filetype = std::move(mimetype); + ret.fileurl = _upload_file(fd, header); + ret.filename = filename; + return ret; } + raii::rjp_string bot::send_file(const raii::string_base& room, const file_info& file){ + raii::string url = raii::json_escape(file.fileurl); + raii::string body = + "{" + "\"body\":\"" + raii::json_escape(file.filename) + "\"," + "\"info\":{" + "\"size\":" + itostr(file.filesize) + + "}," + "\"msgtype\":\"m.file\"," + "\"body\":\"" + file.filename + "\"," + "\"url\":\"" + url + "\"" + "}"; + return _send_message(room, body, raii::curl_llist{}); + } raii::rjp_string bot::send_image(const raii::string_base& room, const image_info& image){ raii::string mimetype = "\"mimetype\":\"image/" + raii::json_escape(image.filetype) + "\""; raii::string url = raii::json_escape(image.fileurl); @@ -250,6 +269,7 @@ namespace matrix{ else thumburl = &image.fileurl; + //compiler intensive raii::string body = "{" "\"body\":\"" + raii::json_escape(image.filename) + "\"," @@ -269,25 +289,36 @@ namespace matrix{ "\"msgtype\":\"m.image\"," "\"url\":\"" + url + "\"" "}"; - raii::rjp_string reply = _post_and_find( - body, - raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token), - raii::curl_llist(), - "event_id"_ss); - return reply; + return _send_message(room, body, raii::curl_llist{}); } - raii::rjp_string bot::send_video(const raii::string_base& room, const raii::string_base& file_url, const raii::string_base& filetype, const raii::string_base& filename){ - raii::rjp_string reply = _post_and_find(raii::string("{\"body\":\"" + raii::json_escape(filename) + "\",\"info\":{\"w\":854,\"mimetype\":\"video/mp4\"},\"msgtype\":\"m.video\",\"url\":\"" + raii::json_escape(file_url) + "\"}"), - raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token), - raii::curl_llist(), - "event_id"_ss); - return reply; + raii::rjp_string bot::send_video(const raii::string_base& room, const video_info& video){ + raii::string body = + "{" + "\"body\":\"" + raii::json_escape(video.filename) + "\"," + "\"info\":{" + "\"h\":" + itostr(video.height) + "," + "\"mimetype\":\"video/" + raii::json_escape(video.filetype) + "\"," + "\"size\":" + itostr(video.filesize) + "," + "\"thumnail_info\":{" + "\"h\":" + itostr(video.thumb_height) + "," + "\"mimetype\":\"image/jpeg\"," + "\"size\":" + itostr(video.thumbsize) + "," + "\"w\":" + itostr(video.thumb_width) + + "}," + "\"thumbnail_url\":\"" + video.thumburl + "\"," + "\"w\":" + itostr(video.width) + + "}," + "\"msgtype\":\"m.video\"," + "\"url\":\"" + raii::json_escape(video.fileurl) + "\"" + "}"; + return _send_message(room, + body, + raii::curl_llist()); } raii::rjp_string bot::send_message(const raii::string_base& room, const raii::string_base& text){ - return _post_and_find(raii::string("{\"body\":\""_ss + raii::json_escape(text) + "\",\"msgtype\":\"m.text\"}"_ss), - raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token), - raii::curl_llist(), - "event_id"_ss); + return _send_message(room, + raii::string("{\"body\":\""_ss + raii::json_escape(text) + "\",\"msgtype\":\"m.text\"}"_ss), + raii::curl_llist()); } void bot::logout(void){ _get_curl(raii::string("https://" + m_homeserver + "/_matrix/client/r0/logout?access_token=" + m_access_token)); @@ -295,9 +326,15 @@ namespace matrix{ } + +/******************************* + Internal functions +********************************/ + raii::rjp_string bot::_upload_file(raii::filerd& fp, const raii::curl_llist& header){ raii::string fileurl; m_curl.postreq(); + m_curl.setopt(CURLOPT_POSTFIELDS, NULL); m_curl.setopt(CURLOPT_READDATA, (void*)fp.get()); m_curl.setopt(CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)fp.length()); m_curl.setopt(CURLOPT_INFILESIZE_LARGE, (curl_off_t)fp.length()); @@ -320,6 +357,14 @@ namespace matrix{ return res.value; } + raii::rjp_string bot::_send_message(const raii::string_base& room, const raii::string_base& msg, const raii::curl_llist& header){ + raii::rjp_string reply = _post_and_find( + msg, + raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token), + header, + "event_id"_ss); + return reply; + } size_t bot::_post_reply_curl_callback(char* ptr, size_t size, size_t nmemb, void* userdata){ raii::string* data = reinterpret_cast(userdata); (*data) += ptr; diff --git a/src/reddit.cpp b/src/reddit.cpp index 34bc710..c8bcc9f 100644 --- a/src/reddit.cpp +++ b/src/reddit.cpp @@ -103,7 +103,7 @@ namespace reddit{ return *std::search(str.get(), str.get()+str.length(), gfycat, gfycat+sizeof(gfycat)-1) != 0; } static bool is_imgur_link(const raii::string_base& str){ - static const char imgur[] = "i.imgur.com"; + static const char imgur[] = "imgur.com"; return *std::search(str.get(), str.get()+str.length(), imgur, imgur+sizeof(imgur)-1) != 0; } static bool is_direct_imgur_link(const raii::string_base& str){ diff --git a/src/test.cpp b/src/test.cpp index 9ffc195..81d8257 100644 --- a/src/test.cpp +++ b/src/test.cpp @@ -148,10 +148,11 @@ int main(){ { DEBUG_PRINT("Got a video\n"); file_output_curl(curl, reply.mediaurl()); - auto vid_url = matbot.upload_video("testout"_ss); - //rudimentary video sending - auto val = matbot.send_video("!QeYfNDCRodtNohhnaI:matrix.org"_ss, vid_url, "mp4"_ss, raii::static_string("testout")); + auto vid_data = matbot.upload_video("testout"_ss); + auto val = matbot.send_video("!QeYfNDCRodtNohhnaI:matrix.org"_ss, vid_data); + DEBUG_PRINT("video event: %s\n", val.get()); val = matbot.send_message("!QeYfNDCRodtNohhnaI:matrix.org"_ss, raii::string(reply.title() + "\n" + reply.posturl())); + DEBUG_PRINT("text event: %s\n", val.get()); } else if(reply.type() == reddit::post_type::link) {