rexy712/matrix_thing
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

video thumbnails

Browse Source
This commit is contained in:
rexy712 2019-03-05 14:19:32 -08:00
parent 6f078c8495
commit df9eac1206
8 changed files with 314 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
TODO
View File

@ -1,11 +1,3 @@
file info struct as return from upload_file so as to know some metadata about the file
uploaded url
file type if able to be figured out
file size
image
dimensions
thumbnail
use libmagic to determine file types
create thumbnail images somehow
use libmagic to determine file types?
//create a raii interface for libav* which allows creation of thumbnails
add av_freep cleanup to raii interface for libav* (probably replacing unique_ptr in that case)

12
include/matrix.hpp
View File

@ -37,6 +37,7 @@ namespace matrix{
size_t thumb_height;
size_t thumbsize;
};
using video_info = image_info;
class bot
{
@ -105,21 +106,24 @@ namespace matrix{
//other networked operations
raii::string create_room(const raii::string_base& name, const raii::string_base& alias);
raii::rjp_string upload_file(const raii::string_base& filename, const raii::curl_llist& header);
//upload media
file_info upload_file(const raii::string_base& filename);
image_info upload_image(const raii::string_base& filename, const raii::string_base& alias);
image_info upload_image(const raii::string_base& filename);
raii::rjp_string upload_video(const raii::string_base& filename);
video_info upload_video(const raii::string_base& filename);
//send messages
raii::rjp_string send_image(const raii::string_base& room, const image_info& image);
raii::rjp_string send_video(const raii::string_base& room, const raii::string_base& file_url, const raii::string_base& filetype, const raii::string_base& filename);
raii::rjp_string send_video(const raii::string_base& room, const video_info& video);
raii::rjp_string send_message(const raii::string_base& room, const raii::string_base& text);
raii::rjp_string send_file(const raii::string_base& room, const file_info& file);
bool send_file(const raii::string_base& room, const raii::string_base& file_url);
void sync(void);
void logout(void);
protected:
raii::rjp_string _upload_file(raii::filerd& fp, const raii::curl_llist& header);
raii::rjp_string _send_message(const raii::string_base& room, const raii::string_base& msg, const raii::curl_llist& header);
static size_t _post_reply_curl_callback(char* ptr, size_t size, size_t nmemb, void* userdata);
raii::string _get_curl(const raii::string_base& url);
raii::string _post_curl(const raii::string_base& postdata, const raii::string_base& url, const raii::curl_llist& header);

5
include/raii/rjp_string.hpp
View File

@ -20,8 +20,9 @@ namespace raii{
return rjp_alloc(size);
}
static void* copy(const void* data, size_t len){
void* tmp = allocate(len);
memcpy(tmp, data, len);
char* tmp = reinterpret_cast<char*>(allocate(len));
memcpy(tmp, data, len-1);
tmp[len-1] = 0;
return tmp;
}
};

5
include/raii/string.hpp
View File

@ -20,8 +20,9 @@ namespace raii{
return ::operator new(size);
}
static void* copy(const void* c, size_t size){
void* tmp = allocate(size);
memcpy(tmp, c, size);
char* tmp = reinterpret_cast<char*>(allocate(size));
memcpy(tmp, c, size-1);
tmp[size-1] = 0;
return tmp;
}
};

190
include/raii/video_man.hpp Normal file
View File

@ -0,0 +1,190 @@
#ifndef RAII_VIDEO_MAN_HPP
#define RAII_VIDEO_MAN_HPP
#include <memory> //unique_ptr
#include <utility> //exchange
#include "raii/string.hpp"
#include "raii/static_string.hpp"
extern "C"{
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
#include <libavutil/imgutils.h>
}
namespace raii{
class video_man
{
private:
static inline auto _context_deleter = [](AVCodecContext* ptr){
avcodec_close(ptr);
avcodec_free_context(&ptr);
};
static inline auto _frame_deleter = [](AVFrame* ptr){
av_frame_unref(ptr);
av_frame_free(&ptr);
};
static inline auto _packet_deleter = [](AVPacket* ptr){
av_packet_unref(ptr);
av_packet_free(&ptr);
};
public:
using context_type = std::unique_ptr<AVCodecContext,decltype(_context_deleter)>;
using frame_type = std::unique_ptr<AVFrame,decltype(_frame_deleter)>;
using packet_type = std::unique_ptr<AVPacket,decltype(_packet_deleter)>;
private:
AVFormatContext* m_vid;
AVCodec* m_codec;
AVCodecContext* m_context;
int m_stream_index = -1;
public:
video_man(const raii::string_base& filename){
av_register_all();
_init(filename);
}
video_man(const video_man& v) = delete;
video_man(video_man&& v):
m_vid(std::exchange(v.m_vid, nullptr)){}
~video_man(void){
reset();
}
int height(void)const{
return m_context->height;
}
int width(void)const{
return m_context->width;
}
raii::string get_mimetype(void)const{
const char* first = strstr(m_vid->iformat->name, ",");
return raii::string(raii::static_string(m_vid->iformat->name, first - m_vid->iformat->name));
}
packet_type create_jpg_thumbnail(void){
context_type jpg_context = _get_jpeg_context(m_context);
frame_type frame(_init_frame(m_context->width, m_context->height, m_context->pix_fmt), _frame_deleter);
frame_type jpg_frame(_init_frame(m_context->width, m_context->height, jpg_context->pix_fmt), _frame_deleter);
packet_type packet(av_packet_alloc(), _packet_deleter);
SwsContext* sws_ctx = sws_getContext(m_context->width, m_context->height, m_context->pix_fmt,
m_context->width, m_context->height, AV_PIX_FMT_YUV420P,
SWS_BILINEAR, NULL, NULL, NULL);
_change_color_range(sws_ctx);
av_image_alloc(jpg_frame->data, jpg_frame->linesize, m_context->width, m_context->height, jpg_context->pix_fmt, 32);
while(av_read_frame(m_vid, packet.get()) >= 0){
if(packet->stream_index == m_stream_index){
avcodec_send_packet(m_context, packet.get());
if(avcodec_receive_frame(m_context, frame.get()) >= 0)
break;
}
av_packet_unref(packet.get());
}
sws_scale(sws_ctx, frame->data, frame->linesize, 0, m_context->height, jpg_frame->data, jpg_frame->linesize);
sws_freeContext(sws_ctx);
packet_type jpg_packet(av_packet_alloc(), _packet_deleter);
if(avcodec_send_frame(jpg_context.get(), jpg_frame.get()) < 0){
av_freep(&jpg_frame->data[0]);
return packet_type(nullptr, _packet_deleter);
}
if(avcodec_receive_packet(jpg_context.get(), jpg_packet.get()) < 0){
return packet_type(nullptr, _packet_deleter);
}
av_freep(&jpg_frame->data[0]);
return jpg_packet;
}
AVFormatContext* release(void){
avcodec_close(m_context);
avcodec_free_context(&m_context);
m_context = nullptr;
m_codec = nullptr;
m_stream_index = -1;
return std::exchange(m_vid, nullptr);
}
void reset(const raii::string_base& filename){
reset();
_init(filename);
}
void reset(void){
avcodec_close(m_context);
avcodec_free_context(&m_context);
avformat_close_input(&m_vid);
m_context = nullptr;
m_vid = nullptr;
m_codec = nullptr;
m_stream_index = -1;
}
private:
void _init(const raii::string_base& filename){
m_vid = _open_video_file(filename);
if(!m_vid)
return;
AVCodecParameters* codec_parms = m_vid->streams[m_stream_index]->codecpar;
m_codec = avcodec_find_decoder(codec_parms->codec_id);
m_context = avcodec_alloc_context3(m_codec);
avcodec_parameters_to_context(m_context, codec_parms);
avcodec_open2(m_context, m_codec, NULL);
}
AVFormatContext* _open_video_file(const raii::string_base& filename){
AVFormatContext* context = NULL;
avformat_open_input(&context, filename.get(), NULL, NULL);
avformat_find_stream_info(context, NULL);
for(size_t i = 0;i < context->nb_streams;++i){
if(context->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO){
m_stream_index = i;
break;
}
}
if(m_stream_index != -1)
return context;
avformat_close_input(&context);
return nullptr;
}
static AVFrame* _init_frame(int w, int h, int format){
AVFrame* f = av_frame_alloc();
f->data[0] = NULL;
f->width = w;
f->height = h;
f->format = format;
return f;
}
static void _change_color_range(SwsContext* sws_ctx){
//we want to convert from YUVJ420P to YUV420P for sws_scale to be happy. but we actually want to output as YUVJ420P.
//so we change the color range to use the full range like YUVJ420P. dummy is used to ignore unneeded returns.
int dummy[4];
int src_range, dest_range;
int br, con, sat;
sws_getColorspaceDetails(sws_ctx, (int**)&dummy, &src_range, (int**)&dummy, &dest_range, &br, &con, &sat);
const int* coefs = sws_getCoefficients(SWS_CS_DEFAULT);
src_range = 2;
sws_setColorspaceDetails(sws_ctx, coefs, src_range, coefs, dest_range, br, con, sat);
}
static context_type _get_jpeg_context(const AVCodecContext* decoder){
AVCodec* jpg_codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
context_type jpg_context(avcodec_alloc_context3(jpg_codec), _context_deleter);
jpg_context->pix_fmt = AV_PIX_FMT_YUVJ420P;
jpg_context->width = decoder->width;
jpg_context->height = decoder->height;
jpg_context->codec_id = AV_CODEC_ID_MJPEG;
jpg_context->codec_type = AVMEDIA_TYPE_VIDEO;
if(decoder->time_base.num == 0 || decoder->time_base.den == 0)
jpg_context->time_base = AVRational{1, 30};
else
jpg_context->time_base = decoder->time_base;
avcodec_open2(jpg_context.get(), jpg_codec, NULL);
return jpg_context;
}
};
}
#endif

155
src/matrix.cpp
View File

@ -1,3 +1,4 @@
#include "raii/video_man.hpp"
#include "matrix.hpp"
#include "raii/curl_llist.hpp"
@ -6,12 +7,6 @@
#include "raii/filerd.hpp"
#include <FreeImagePlus.h>
extern "C"{
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
}
namespace matrix{
auth_data parse_auth_data(RJP_value* root){
@ -149,10 +144,11 @@ namespace matrix{
return _post_curl(postdata, m_urls.create_room, raii::curl_llist());
}
raii::rjp_string bot::upload_file(const raii::string_base& filename, const raii::curl_llist& header){
file_info bot::upload_file(const raii::string_base& filename){
raii::filerd fd(filename);
if(!fd) return {};
return _upload_file(fd, header);
return file_info{_upload_file(fd, raii::curl_llist{}), filename, {}, fd.length()};
}
image_info bot::upload_image(const raii::string_base& filename){
return upload_image(filename, raii::static_string());
@ -177,6 +173,7 @@ namespace matrix{
raii::filerd fd(filename, "rb");
if(!fd) return {};
//save fullsize image info
ret.width = image.getWidth();
ret.height = image.getHeight();
ret.filetype = formattotype(type);
@ -184,63 +181,85 @@ namespace matrix{
ret.filesize = fd.length();
raii::curl_llist header(raii::string("Content-Type: image/" + ret.filetype));
ret.fileurl = _upload_file(fd, header);
fd.reset(fopen("anewNewimage", "w+"));
fd.reset();
//TODO remove temporary file
//create thumbnail
image.makeThumbnail(500);
FreeImageIO fileout;
fileout.read_proc = [](void* ptr, unsigned int size, unsigned int nmemb, void* fp) -> unsigned int{
return fread(ptr, size, nmemb, (FILE*)fp);
};
std::vector<char> buffer;
fileout.write_proc = [](void* ptr, unsigned int size, unsigned int nmemb, void* fp) -> unsigned int{
return fwrite(ptr, size, nmemb, (FILE*)fp);
std::vector<char>& buffer = *reinterpret_cast<std::vector<char>*>(fp);
buffer.insert(buffer.end(), (char*)ptr, ((char*)ptr)+size*nmemb);
return size*nmemb;
};
fileout.seek_proc = [](void* fp, long int off, int whence) -> int{
return fseek((FILE*)fp, off, whence);
};
fileout.tell_proc = [](void* fp) -> long int{
return ftell((FILE*)fp);
};
bool b;
bool b = false;
switch(type){
case FIF_JPEG:
b = image.saveToHandle(type, &fileout, (fi_handle)(fd.get()), JPEG_QUALITYGOOD | JPEG_SUBSAMPLING_411);
b = image.saveToHandle(type, &fileout, (fi_handle)&buffer, JPEG_QUALITYGOOD | JPEG_SUBSAMPLING_411);
break;
case FIF_PNG:
b = image.saveToHandle(type, &fileout, (fi_handle)(fd.get()), PNG_Z_BEST_COMPRESSION);
b = image.saveToHandle(type, &fileout, (fi_handle)&buffer, PNG_Z_BEST_COMPRESSION);
break;
case FIF_GIF:
b = image.saveToHandle(type, &fileout, (fi_handle)(fd.get()));
b = image.saveToHandle(type, &fileout, (fi_handle)&buffer);
break;
default:
;
};
if(!b){
fprintf(stderr, "Unable to create image thumbnail");
ret.thumb_width = ret.width;
ret.thumb_height = ret.height;
ret.thumbsize = ret.filesize;
}else{
ret.thumb_width = image.getWidth();
ret.thumb_height = image.getHeight();
fd.rewind();
ret.thumburl = _upload_file(fd, header);
ret.thumbsize = fd.length();
ret.thumburl = _post_and_find(raii::static_string(buffer.data(), buffer.size()), m_urls.file_upload, header, "content_uri"_ss);
ret.thumbsize = buffer.size();
}
return ret;
}
raii::rjp_string bot::upload_video(const raii::string_base& filename){
image_info ret;
video_info bot::upload_video(const raii::string_base& filename){
video_info ret = {};
av_register_all();
AVFormatContext* context = NULL;
avformat_open_input(&context, filename.get(), NULL, NULL);
avformat_find_stream_info(context, NULL);
av_dump_format(context, 0, filename.get(), false);
avformat_close_input(&context);
raii::video_man context(filename);
auto packet = context.create_jpg_thumbnail();
raii::curl_llist header("Content-Type: image/jpeg");
ret.thumb_width = context.width();
ret.thumb_height = context.height();
ret.thumbsize = packet->size;
ret.thumburl = _post_and_find(raii::static_string((char*)packet->data, packet->size), m_urls.file_upload, header, "content_uri"_ss);
raii::string mimetype = context.get_mimetype();
header = raii::curl_llist(raii::string("Content-Type: video/" + mimetype));
raii::filerd fd(filename);
if(!fd) return {};
raii::curl_llist header(raii::string("Content-Type: video/mp4"));
return _upload_file(fd, header);
ret.filesize = fd.length();
ret.width = context.width();
ret.height = context.height();
ret.filetype = std::move(mimetype);
ret.fileurl = _upload_file(fd, header);
ret.filename = filename;
return ret;
}
raii::rjp_string bot::send_file(const raii::string_base& room, const file_info& file){
raii::string url = raii::json_escape(file.fileurl);
raii::string body =
"{"
"\"body\":\"" + raii::json_escape(file.filename) + "\","
"\"info\":{"
"\"size\":" + itostr(file.filesize) +
"},"
"\"msgtype\":\"m.file\","
"\"body\":\"" + file.filename + "\","
"\"url\":\"" + url + "\""
"}";
return _send_message(room, body, raii::curl_llist{});
}
raii::rjp_string bot::send_image(const raii::string_base& room, const image_info& image){
raii::string mimetype = "\"mimetype\":\"image/" + raii::json_escape(image.filetype) + "\"";
raii::string url = raii::json_escape(image.fileurl);
@ -250,6 +269,7 @@ namespace matrix{
else
thumburl = &image.fileurl;
//compiler intensive
raii::string body =
"{"
"\"body\":\"" + raii::json_escape(image.filename) + "\","
@ -269,25 +289,36 @@ namespace matrix{
"\"msgtype\":\"m.image\","
"\"url\":\"" + url + "\""
"}";
raii::rjp_string reply = _post_and_find(
body,
raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token),
raii::curl_llist(),
"event_id"_ss);
return reply;
return _send_message(room, body, raii::curl_llist{});
}
raii::rjp_string bot::send_video(const raii::string_base& room, const raii::string_base& file_url, const raii::string_base& filetype, const raii::string_base& filename){
raii::rjp_string reply = _post_and_find(raii::string("{\"body\":\"" + raii::json_escape(filename) + "\",\"info\":{\"w\":854,\"mimetype\":\"video/mp4\"},\"msgtype\":\"m.video\",\"url\":\"" + raii::json_escape(file_url) + "\"}"),
raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token),
raii::curl_llist(),
"event_id"_ss);
return reply;
raii::rjp_string bot::send_video(const raii::string_base& room, const video_info& video){
raii::string body =
"{"
"\"body\":\"" + raii::json_escape(video.filename) + "\","
"\"info\":{"
"\"h\":" + itostr(video.height) + ","
"\"mimetype\":\"video/" + raii::json_escape(video.filetype) + "\","
"\"size\":" + itostr(video.filesize) + ","
"\"thumnail_info\":{"
"\"h\":" + itostr(video.thumb_height) + ","
"\"mimetype\":\"image/jpeg\","
"\"size\":" + itostr(video.thumbsize) + ","
"\"w\":" + itostr(video.thumb_width) +
"},"
"\"thumbnail_url\":\"" + video.thumburl + "\","
"\"w\":" + itostr(video.width) +
"},"
"\"msgtype\":\"m.video\","
"\"url\":\"" + raii::json_escape(video.fileurl) + "\""
"}";
return _send_message(room,
body,
raii::curl_llist());
}
raii::rjp_string bot::send_message(const raii::string_base& room, const raii::string_base& text){
return _post_and_find(raii::string("{\"body\":\""_ss + raii::json_escape(text) + "\",\"msgtype\":\"m.text\"}"_ss),
raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token),
raii::curl_llist(),
"event_id"_ss);
return _send_message(room,
raii::string("{\"body\":\""_ss + raii::json_escape(text) + "\",\"msgtype\":\"m.text\"}"_ss),
raii::curl_llist());
}
void bot::logout(void){
_get_curl(raii::string("https://" + m_homeserver + "/_matrix/client/r0/logout?access_token=" + m_access_token));
@ -295,9 +326,15 @@ namespace matrix{
}
/*******************************
Internal functions
********************************/
raii::rjp_string bot::_upload_file(raii::filerd& fp, const raii::curl_llist& header){
raii::string fileurl;
m_curl.postreq();
m_curl.setopt(CURLOPT_POSTFIELDS, NULL);
m_curl.setopt(CURLOPT_READDATA, (void*)fp.get());
m_curl.setopt(CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)fp.length());
m_curl.setopt(CURLOPT_INFILESIZE_LARGE, (curl_off_t)fp.length());
@ -320,6 +357,14 @@ namespace matrix{
return res.value;
}
raii::rjp_string bot::_send_message(const raii::string_base& room, const raii::string_base& msg, const raii::curl_llist& header){
raii::rjp_string reply = _post_and_find(
msg,
raii::string("https://" + m_homeserver + "/_matrix/client/r0/rooms/" + m_curl.encode(room) + "/send/m.room.message?access_token=" + m_access_token),
header,
"event_id"_ss);
return reply;
}
size_t bot::_post_reply_curl_callback(char* ptr, size_t size, size_t nmemb, void* userdata){
raii::string* data = reinterpret_cast<raii::string*>(userdata);
(*data) += ptr;

2
src/reddit.cpp
View File

@ -103,7 +103,7 @@ namespace reddit{
return *std::search(str.get(), str.get()+str.length(), gfycat, gfycat+sizeof(gfycat)-1) != 0;
}
static bool is_imgur_link(const raii::string_base& str){
static const char imgur[] = "i.imgur.com";
static const char imgur[] = "imgur.com";
return *std::search(str.get(), str.get()+str.length(), imgur, imgur+sizeof(imgur)-1) != 0;
}
static bool is_direct_imgur_link(const raii::string_base& str){

7
src/test.cpp
View File

@ -148,10 +148,11 @@ int main(){
{
DEBUG_PRINT("Got a video\n");
file_output_curl(curl, reply.mediaurl());
auto vid_url = matbot.upload_video("testout"_ss);
//rudimentary video sending
auto val = matbot.send_video("!QeYfNDCRodtNohhnaI:matrix.org"_ss, vid_url, "mp4"_ss, raii::static_string("testout"));
auto vid_data = matbot.upload_video("testout"_ss);
auto val = matbot.send_video("!QeYfNDCRodtNohhnaI:matrix.org"_ss, vid_data);
DEBUG_PRINT("video event: %s\n", val.get());
val = matbot.send_message("!QeYfNDCRodtNohhnaI:matrix.org"_ss, raii::string(reply.title() + "\n" + reply.posturl()));
DEBUG_PRINT("text event: %s\n", val.get());
}
else if(reply.type() == reddit::post_type::link)
{