video thumbnails

TODO

@@ -1,11 +1,3 @@
-file info struct as return from upload_file so as to know some metadata about the file
+use libmagic to determine file types?
-	uploaded url
+//create a raii interface for libav* which allows creation of thumbnails
-	file type if able to be figured out
+add av_freep cleanup to raii interface for libav* (probably replacing unique_ptr in that case)
-	file size
-	image
-		dimensions
-		thumbnail
-
-
-use libmagic to determine file types
-create thumbnail images somehow

include/matrix.hpp

@@ -37,6 +37,7 @@ namespace matrix{
 		size_t thumb_height;
 		size_t thumbsize;
 	};
+	using video_info = image_info;
 
 	class bot
 	{
@@ -105,21 +106,24 @@ namespace matrix{
 		//other networked operations
 		raii::string create_room(const raii::string_base& name, const raii::string_base& alias);
 
-		raii::rjp_string upload_file(const raii::string_base& filename, const raii::curl_llist& header);
+		//upload media
+		file_info upload_file(const raii::string_base& filename);
 		image_info upload_image(const raii::string_base& filename, const raii::string_base& alias);
 		image_info upload_image(const raii::string_base& filename);
-		raii::rjp_string upload_video(const raii::string_base& filename);
+		video_info upload_video(const raii::string_base& filename);
 
+		//send messages
 		raii::rjp_string send_image(const raii::string_base& room, const image_info& image);
-		raii::rjp_string send_video(const raii::string_base& room, const raii::string_base& file_url, const raii::string_base& filetype, const raii::string_base& filename);
+		raii::rjp_string send_video(const raii::string_base& room, const video_info& video);
 		raii::rjp_string send_message(const raii::string_base& room, const raii::string_base& text);
+		raii::rjp_string send_file(const raii::string_base& room, const file_info& file);
 
-		bool send_file(const raii::string_base& room, const raii::string_base& file_url);
 		void sync(void);
 		void logout(void);
 
 	protected:
 		raii::rjp_string _upload_file(raii::filerd& fp, const raii::curl_llist& header);
+		raii::rjp_string _send_message(const raii::string_base& room, const raii::string_base& msg, const raii::curl_llist& header);
 		static size_t _post_reply_curl_callback(char* ptr, size_t size, size_t nmemb, void* userdata);
 		raii::string _get_curl(const raii::string_base& url);
 		raii::string _post_curl(const raii::string_base& postdata, const raii::string_base& url, const raii::curl_llist& header);

include/raii/rjp_string.hpp

@@ -20,8 +20,9 @@ namespace raii{
 				return rjp_alloc(size);
 			}
 			static void* copy(const void* data, size_t len){
-				void* tmp = allocate(len);
+				char* tmp = reinterpret_cast<char*>(allocate(len));
-				memcpy(tmp, data, len);
+				memcpy(tmp, data, len-1);
+				tmp[len-1] = 0;
 				return tmp;
 			}
 		};

include/raii/string.hpp

@@ -20,8 +20,9 @@ namespace raii{
 				return ::operator new(size);
 			}
 			static void* copy(const void* c, size_t size){
-				void* tmp = allocate(size);
+				char* tmp = reinterpret_cast<char*>(allocate(size));
-				memcpy(tmp, c, size);
+				memcpy(tmp, c, size-1);
+				tmp[size-1] = 0;
 				return tmp;
 			}
 		};

include/raii/video_man.hpp

@@ -0,0 +1,190 @@
+#ifndef RAII_VIDEO_MAN_HPP
+#define RAII_VIDEO_MAN_HPP
+
+#include <memory> //unique_ptr
+#include <utility> //exchange
+#include "raii/string.hpp"
+#include "raii/static_string.hpp"
+
+extern "C"{
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libswscale/swscale.h>
+#include <libavutil/imgutils.h>
+}
+
+namespace raii{
+
+	class video_man
+	{
+	private:
+		static inline auto _context_deleter = [](AVCodecContext* ptr){
+			avcodec_close(ptr);
+			avcodec_free_context(&ptr);
+		};
+		static inline auto _frame_deleter = [](AVFrame* ptr){
+			av_frame_unref(ptr);
+			av_frame_free(&ptr);
+		};
+		static inline auto _packet_deleter = [](AVPacket* ptr){
+			av_packet_unref(ptr);
+			av_packet_free(&ptr);
+		};
+	public:
+		using context_type = std::unique_ptr<AVCodecContext,decltype(_context_deleter)>;
+		using frame_type = std::unique_ptr<AVFrame,decltype(_frame_deleter)>;
+		using packet_type = std::unique_ptr<AVPacket,decltype(_packet_deleter)>;
+
+	private:
+		AVFormatContext* m_vid;
+		AVCodec*         m_codec;
+		AVCodecContext*  m_context;
+		int              m_stream_index = -1;
+
+
+	public:
+		video_man(const raii::string_base& filename){
+			av_register_all();
+			_init(filename);
+		}
+		video_man(const video_man& v) = delete;
+		video_man(video_man&& v):
+			m_vid(std::exchange(v.m_vid, nullptr)){}
+		~video_man(void){
+			reset();
+		}
+
+		int height(void)const{
+			return m_context->height;
+		}
+		int width(void)const{
+			return m_context->width;
+		}
+		raii::string get_mimetype(void)const{
+			const char* first = strstr(m_vid->iformat->name, ",");
+			return raii::string(raii::static_string(m_vid->iformat->name, first - m_vid->iformat->name));
+		}
+		packet_type create_jpg_thumbnail(void){
+			context_type jpg_context = _get_jpeg_context(m_context);
+			frame_type frame(_init_frame(m_context->width, m_context->height, m_context->pix_fmt), _frame_deleter);
+			frame_type jpg_frame(_init_frame(m_context->width, m_context->height, jpg_context->pix_fmt), _frame_deleter);
+			packet_type packet(av_packet_alloc(), _packet_deleter);
+			SwsContext* sws_ctx = sws_getContext(m_context->width, m_context->height, m_context->pix_fmt,
+			                                     m_context->width, m_context->height, AV_PIX_FMT_YUV420P,
+			                                     SWS_BILINEAR, NULL, NULL, NULL);
+			_change_color_range(sws_ctx);
+
+			av_image_alloc(jpg_frame->data, jpg_frame->linesize, m_context->width, m_context->height, jpg_context->pix_fmt, 32);
+			while(av_read_frame(m_vid, packet.get()) >= 0){
+				if(packet->stream_index == m_stream_index){
+					avcodec_send_packet(m_context, packet.get());
+					if(avcodec_receive_frame(m_context, frame.get()) >= 0)
+						break;
+				}
+				av_packet_unref(packet.get());
+			}
+			sws_scale(sws_ctx, frame->data, frame->linesize, 0, m_context->height, jpg_frame->data, jpg_frame->linesize);
+
+			sws_freeContext(sws_ctx);
+
+			packet_type jpg_packet(av_packet_alloc(), _packet_deleter);
+			if(avcodec_send_frame(jpg_context.get(), jpg_frame.get()) < 0){
+				av_freep(&jpg_frame->data[0]);
+				return packet_type(nullptr, _packet_deleter);
+			}
+			if(avcodec_receive_packet(jpg_context.get(), jpg_packet.get()) < 0){
+				return packet_type(nullptr, _packet_deleter);
+			}
+
+			av_freep(&jpg_frame->data[0]);
+			return jpg_packet;
+		}
+
+		AVFormatContext* release(void){
+			avcodec_close(m_context);
+			avcodec_free_context(&m_context);
+			m_context = nullptr;
+			m_codec = nullptr;
+			m_stream_index = -1;
+			return std::exchange(m_vid, nullptr);
+		}
+		void reset(const raii::string_base& filename){
+			reset();
+			_init(filename);
+		}
+		void reset(void){
+			avcodec_close(m_context);
+			avcodec_free_context(&m_context);
+			avformat_close_input(&m_vid);
+			m_context = nullptr;
+			m_vid = nullptr;
+			m_codec = nullptr;
+			m_stream_index = -1;
+		}
+	private:
+		void _init(const raii::string_base& filename){
+			m_vid = _open_video_file(filename);
+			if(!m_vid)
+				return;
+
+			AVCodecParameters* codec_parms = m_vid->streams[m_stream_index]->codecpar;
+			m_codec = avcodec_find_decoder(codec_parms->codec_id);
+			m_context = avcodec_alloc_context3(m_codec);
+			avcodec_parameters_to_context(m_context, codec_parms);
+			avcodec_open2(m_context, m_codec, NULL);
+		}
+		AVFormatContext* _open_video_file(const raii::string_base& filename){
+			AVFormatContext* context = NULL;
+			avformat_open_input(&context, filename.get(), NULL, NULL);
+			avformat_find_stream_info(context, NULL);
+			for(size_t i = 0;i < context->nb_streams;++i){
+				if(context->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO){
+					m_stream_index = i;
+					break;
+				}
+			}
+			if(m_stream_index != -1)
+				return context;
+			avformat_close_input(&context);
+			return nullptr;
+		}
+		static AVFrame* _init_frame(int w, int h, int format){
+			AVFrame* f = av_frame_alloc();
+			f->data[0] = NULL;
+			f->width = w;
+			f->height = h;
+			f->format = format;
+			return f;
+		}
+		static void _change_color_range(SwsContext* sws_ctx){
+			//we want to convert from YUVJ420P to YUV420P for sws_scale to be happy. but we actually want to output as YUVJ420P.
+			//so we change the color range to use the full range like YUVJ420P. dummy is used to ignore unneeded returns.
+			int dummy[4];
+			int src_range, dest_range;
+			int br, con, sat;
+			sws_getColorspaceDetails(sws_ctx, (int**)&dummy, &src_range, (int**)&dummy, &dest_range, &br, &con, &sat);
+			const int* coefs = sws_getCoefficients(SWS_CS_DEFAULT);
+			src_range = 2;
+			sws_setColorspaceDetails(sws_ctx, coefs, src_range, coefs, dest_range, br, con, sat);
+		}
+		static context_type _get_jpeg_context(const AVCodecContext* decoder){
+			AVCodec* jpg_codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
+			context_type jpg_context(avcodec_alloc_context3(jpg_codec), _context_deleter);
+			jpg_context->pix_fmt = AV_PIX_FMT_YUVJ420P;
+			jpg_context->width = decoder->width;
+			jpg_context->height = decoder->height;
+			jpg_context->codec_id = AV_CODEC_ID_MJPEG;
+			jpg_context->codec_type = AVMEDIA_TYPE_VIDEO;
+			if(decoder->time_base.num == 0 || decoder->time_base.den == 0)
+				jpg_context->time_base = AVRational{1, 30};
+			else
+				jpg_context->time_base = decoder->time_base;
+			avcodec_open2(jpg_context.get(), jpg_codec, NULL);
+			return jpg_context;
+		}
+	};
+
+
+}
+
+#endif

src/matrix.cpp

@@ -1,3 +1,4 @@
+#include "raii/video_man.hpp"
 #include "matrix.hpp"
 
 #include "raii/curl_llist.hpp"
@@ -6,12 +7,6 @@
 #include "raii/filerd.hpp"
 #include <FreeImagePlus.h>
 
-extern "C"{
-#include <libavcodec/avcodec.h>
-#include <libavformat/avformat.h>
-#include <libswscale/swscale.h>
-}
-
 namespace matrix{
 
 	auth_data parse_auth_data(RJP_value* root){
@@ -149,10 +144,11 @@ namespace matrix{
 
 		return _post_curl(postdata, m_urls.create_room, raii::curl_llist());
 	}
-	raii::rjp_string bot::upload_file(const raii::string_base& filename, const raii::curl_llist& header){
+	file_info bot::upload_file(const raii::string_base& filename){
 		raii::filerd fd(filename);
 		if(!fd) return {};
-		return _upload_file(fd, header);
+
+		return file_info{_upload_file(fd, raii::curl_llist{}), filename, {}, fd.length()};
 	}
 	image_info bot::upload_image(const raii::string_base& filename){
 		return upload_image(filename, raii::static_string());
@@ -177,6 +173,7 @@ namespace matrix{
 		raii::filerd fd(filename, "rb");
 		if(!fd) return {};
 
+		//save fullsize image info
 		ret.width    = image.getWidth();
 		ret.height   = image.getHeight();
 		ret.filetype = formattotype(type);
@@ -184,63 +181,85 @@ namespace matrix{
 		ret.filesize = fd.length();
 		raii::curl_llist header(raii::string("Content-Type: image/" + ret.filetype));
 		ret.fileurl  = _upload_file(fd, header);
-		fd.reset(fopen("anewNewimage", "w+"));
+		fd.reset();
 
-//TODO remove temporary file
+		//create thumbnail
 		image.makeThumbnail(500);
 		FreeImageIO fileout;
-		fileout.read_proc = [](void* ptr, unsigned int size, unsigned int nmemb, void* fp) -> unsigned int{
+		std::vector<char> buffer;
-			return fread(ptr, size, nmemb, (FILE*)fp);
-		};
 		fileout.write_proc = [](void* ptr, unsigned int size, unsigned int nmemb, void* fp) -> unsigned int{
-			return fwrite(ptr, size, nmemb, (FILE*)fp);
+			std::vector<char>& buffer = *reinterpret_cast<std::vector<char>*>(fp);
+			buffer.insert(buffer.end(), (char*)ptr, ((char*)ptr)+size*nmemb);
+			return size*nmemb;
 		};
-		fileout.seek_proc = [](void* fp, long int off, int whence) -> int{
+		bool b = false;
-			return fseek((FILE*)fp, off, whence);
-		};
-		fileout.tell_proc = [](void* fp) -> long int{
-			return ftell((FILE*)fp);
-		};
-		bool b;
 		switch(type){
 		case FIF_JPEG:
-			b = image.saveToHandle(type, &fileout, (fi_handle)(fd.get()), JPEG_QUALITYGOOD | JPEG_SUBSAMPLING_411);
+			b = image.saveToHandle(type, &fileout, (fi_handle)&buffer, JPEG_QUALITYGOOD | JPEG_SUBSAMPLING_411);
 			break;
 		case FIF_PNG:
-			b = image.saveToHandle(type, &fileout, (fi_handle)(fd.get()), PNG_Z_BEST_COMPRESSION);
+			b = image.saveToHandle(type, &fileout, (fi_handle)&buffer, PNG_Z_BEST_COMPRESSION);
 			break;
 		case FIF_GIF:
-			b = image.saveToHandle(type, &fileout, (fi_handle)(fd.get()));
+			b = image.saveToHandle(type, &fileout, (fi_handle)&buffer);
 			break;
 		default:
 			;
 		};
+		if(!b){
+			fprintf(stderr, "Unable to create image thumbnail");
+			ret.thumb_width = ret.width;
+			ret.thumb_height = ret.height;
+			ret.thumbsize = ret.filesize;
+		}else{
 			ret.thumb_width = image.getWidth();
 			ret.thumb_height = image.getHeight();
-
+			ret.thumburl = _post_and_find(raii::static_string(buffer.data(), buffer.size()), m_urls.file_upload, header, "content_uri"_ss);
-		fd.rewind();
+			ret.thumbsize = buffer.size();
-		ret.thumburl = _upload_file(fd, header);
+		}
-		ret.thumbsize = fd.length();
 
 		return ret;
 	}
-	raii::rjp_string bot::upload_video(const raii::string_base& filename){
+	video_info bot::upload_video(const raii::string_base& filename){
-		image_info ret;
+		video_info ret = {};
 
-		av_register_all();
+		raii::video_man context(filename);
-		AVFormatContext* context = NULL;
+		auto packet = context.create_jpg_thumbnail();
-		avformat_open_input(&context, filename.get(), NULL, NULL);
+		raii::curl_llist header("Content-Type: image/jpeg");
-		avformat_find_stream_info(context, NULL);
-		av_dump_format(context, 0, filename.get(), false);
-		avformat_close_input(&context);
 
+		ret.thumb_width = context.width();
+		ret.thumb_height = context.height();
+		ret.thumbsize = packet->size;
+		ret.thumburl = _post_and_find(raii::static_string((char*)packet->data, packet->size), m_urls.file_upload, header, "content_uri"_ss);
+		raii::string mimetype = context.get_mimetype();
+
+
+		header = raii::curl_llist(raii::string("Content-Type: video/" + mimetype));
 		raii::filerd fd(filename);
 		if(!fd) return {};
-
+		ret.filesize = fd.length();
-		raii::curl_llist header(raii::string("Content-Type: video/mp4"));
+		ret.width = context.width();
-		return _upload_file(fd, header);
+		ret.height = context.height();
+		ret.filetype = std::move(mimetype);
+		ret.fileurl = _upload_file(fd, header);
+		ret.filename = filename;
+		return ret;
 	}
 
+	raii::rjp_string bot::send_file(const raii::string_base& room, const file_info& file){
+		raii::string url = raii::json_escape(file.fileurl);
+		raii::string body =
+			"{"
+				"\"body\":\"" + raii::json_escape(file.filename) + "\","
+				"\"info\":{"
+					"\"size\":" + itostr(file.filesize) +
+				"},"
+				"\"msgtype\":\"m.file\","
+				"\"body\":\"" + file.filename + "\","
+				"\"url\":\"" + url + "\""
+			"}";
+		return _send_message(room, body, raii::curl_llist{});
+	}
 	raii::rjp_string bot::send_image(const raii::string_base& room, const image_info& image){
 		raii::string mimetype = "\"mimetype\":\"image/" + raii::json_escape(image.filetype) + "\"";
 		raii::string url = raii::json_escape(image.fileurl);
@@ -250,6 +269,7 @@ namespace matrix{
 		else
 			thumburl = &image.fileurl;
 
+		//compiler intensive
 		raii::string body =
 			"{"
 				"\"body\":\"" + raii::json_escape(image.filename) + "\","
@@ -269,25 +289,36 @@ namespace matrix{
 				"\"msgtype\":\"m.image\","
 				"\"url\":\"" + url + "\""
 			"}";
-		raii::rjp_string reply = _post_and_find(
+		return _send_message(room, body, raii::curl_llist{});
-			body,
-			raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token),
-			raii::curl_llist(),
-			"event_id"_ss);
-		return reply;
 	}
-	raii::rjp_string bot::send_video(const raii::string_base& room, const raii::string_base& file_url, const raii::string_base& filetype, const raii::string_base& filename){
+	raii::rjp_string bot::send_video(const raii::string_base& room, const video_info& video){
-		raii::rjp_string reply = _post_and_find(raii::string("{\"body\":\"" + raii::json_escape(filename) + "\",\"info\":{\"w\":854,\"mimetype\":\"video/mp4\"},\"msgtype\":\"m.video\",\"url\":\"" + raii::json_escape(file_url) + "\"}"),
+		raii::string body =
-		                                        raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token),
+			"{"
-		                                        raii::curl_llist(),
+				"\"body\":\"" + raii::json_escape(video.filename) + "\","
-		                                        "event_id"_ss);
+				"\"info\":{"
-		return reply;
+					"\"h\":" + itostr(video.height) + ","
+					"\"mimetype\":\"video/" + raii::json_escape(video.filetype) + "\","
+					"\"size\":" + itostr(video.filesize) + ","
+					"\"thumnail_info\":{"
+						"\"h\":" + itostr(video.thumb_height) + ","
+						"\"mimetype\":\"image/jpeg\","
+						"\"size\":" + itostr(video.thumbsize) + ","
+						"\"w\":" + itostr(video.thumb_width) +
+					"},"
+					"\"thumbnail_url\":\"" + video.thumburl + "\","
+					"\"w\":" + itostr(video.width) +
+				"},"
+				"\"msgtype\":\"m.video\","
+				"\"url\":\"" + raii::json_escape(video.fileurl) + "\""
+			"}";
+		return _send_message(room,
+		                     body,
+		                     raii::curl_llist());
 	}
 	raii::rjp_string bot::send_message(const raii::string_base& room, const raii::string_base& text){
-		return _post_and_find(raii::string("{\"body\":\""_ss + raii::json_escape(text) + "\",\"msgtype\":\"m.text\"}"_ss),
+		return _send_message(room,
-		                      raii::string("https://" + m_homeserver +  "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token),
+		                     raii::string("{\"body\":\""_ss + raii::json_escape(text) + "\",\"msgtype\":\"m.text\"}"_ss),
-		                      raii::curl_llist(),
+		                     raii::curl_llist());
-		                      "event_id"_ss);
 	}
 	void bot::logout(void){
 		_get_curl(raii::string("https://" + m_homeserver + "/_matrix/client/r0/logout?access_token=" + m_access_token));
@@ -295,9 +326,15 @@ namespace matrix{
 	}
 
 
+
+/*******************************
+	     Internal functions
+********************************/
+
 	raii::rjp_string bot::_upload_file(raii::filerd& fp, const raii::curl_llist& header){
 		raii::string fileurl;
 		m_curl.postreq();
+		m_curl.setopt(CURLOPT_POSTFIELDS, NULL);
 		m_curl.setopt(CURLOPT_READDATA, (void*)fp.get());
 		m_curl.setopt(CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)fp.length());
 		m_curl.setopt(CURLOPT_INFILESIZE_LARGE, (curl_off_t)fp.length());
@@ -320,6 +357,14 @@ namespace matrix{
 
 		return res.value;
 	}
+	raii::rjp_string bot::_send_message(const raii::string_base& room, const raii::string_base& msg, const raii::curl_llist& header){
+		raii::rjp_string reply = _post_and_find(
+			msg,
+			raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token),
+			header,
+			"event_id"_ss);
+		return reply;
+	}
 	size_t bot::_post_reply_curl_callback(char* ptr, size_t size, size_t nmemb, void* userdata){
 		raii::string* data = reinterpret_cast<raii::string*>(userdata);
 		(*data) += ptr;

src/reddit.cpp

@@ -103,7 +103,7 @@ namespace reddit{
 		return *std::search(str.get(), str.get()+str.length(), gfycat, gfycat+sizeof(gfycat)-1) != 0;
 	}
 	static bool is_imgur_link(const raii::string_base& str){
-		static const char imgur[] = "i.imgur.com";
+		static const char imgur[] = "imgur.com";
 		return *std::search(str.get(), str.get()+str.length(), imgur, imgur+sizeof(imgur)-1) != 0;
 	}
 	static bool is_direct_imgur_link(const raii::string_base& str){

src/test.cpp

@@ -148,10 +148,11 @@ int main(){
 	{
 		DEBUG_PRINT("Got a video\n");
 		file_output_curl(curl, reply.mediaurl());
-		auto vid_url = matbot.upload_video("testout"_ss);
+		auto vid_data = matbot.upload_video("testout"_ss);
-		//rudimentary video sending
+		auto val = matbot.send_video("!QeYfNDCRodtNohhnaI:matrix.org"_ss, vid_data);
-		auto val = matbot.send_video("!QeYfNDCRodtNohhnaI:matrix.org"_ss, vid_url, "mp4"_ss, raii::static_string("testout"));
+		DEBUG_PRINT("video event: %s\n", val.get());
 		val = matbot.send_message("!QeYfNDCRodtNohhnaI:matrix.org"_ss, raii::string(reply.title() + "\n" + reply.posturl()));
+		DEBUG_PRINT("text event: %s\n", val.get());
 	}
 	else if(reply.type() == reddit::post_type::link)
 	{