sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor FrameReader (#21141)

Browse Source 
* refactor FrameReader

* continue

* move that

* small cleanup

* little more

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
pull/21184/head
Dean Lee 4 years ago committed by GitHub
parent 5dbf1c1e28
commit 67fe3feb09
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 175 additions and 116 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 214
      selfdrive/ui/replay/framereader.cc
  2. 52
      selfdrive/ui/replay/framereader.h
  3. 25
      selfdrive/ui/replay/replay.cc

214
selfdrive/ui/replay/framereader.cc
Unescape View File

@ -3,6 +3,9 @@
#include <assert.h> #include <assert.h>
#include <unistd.h> #include <unistd.h>
#include <QDebug>
static int ffmpeg_lockmgr_cb(void **arg, enum AVLockOp op) { static int ffmpeg_lockmgr_cb(void **arg, enum AVLockOp op) {
std::mutex *mutex = (std::mutex *)*arg; std::mutex *mutex = (std::mutex *)*arg;
switch (op) { switch (op) {
@ -21,122 +24,173 @@ static int ffmpeg_lockmgr_cb(void **arg, enum AVLockOp op) {
return 0; return 0;
} }
FrameReader::FrameReader(const std::string &fn) : url(fn) { class AVInitializer {
int ret = av_lockmgr_register(ffmpeg_lockmgr_cb); public:
assert(ret >= 0); AVInitializer() {
int ret = av_lockmgr_register(ffmpeg_lockmgr_cb);
assert(ret >= 0);
av_register_all();
avformat_network_init();
}
avformat_network_init(); ~AVInitializer() { avformat_network_deinit(); }
av_register_all(); };
static AVInitializer av_initializer;
FrameReader::FrameReader(const std::string &url, QObject *parent) : url_(url), QObject(parent) {
process_thread_ = QThread::create(&FrameReader::process, this);
connect(process_thread_, &QThread::finished, process_thread_, &QThread::deleteLater);
process_thread_->start();
} }
FrameReader::~FrameReader() { FrameReader::~FrameReader() {
// wait until thread is finished.
exit_ = true; exit_ = true;
thread.join(); process_thread_->wait();
for (auto &f : frames) { cv_decode_.notify_all();
delete f->pkt; cv_frame_.notify_all();
if (f->picture) { if (decode_thread_.joinable()) {
av_frame_free(&f->picture); decode_thread_.join();
}
// free all.
for (auto &f : frames_) {
av_free_packet(&f.pkt);
if (f.data) {
delete[] f.data;
} }
delete f;
} }
avcodec_free_context(&pCodecCtx); while (!buffer_pool.empty()) {
avformat_free_context(pFormatCtx); delete[] buffer_pool.front();
sws_freeContext(sws_ctx); buffer_pool.pop();
avformat_network_deinit(); }
av_frame_free(&frmRgb_);
avcodec_close(pCodecCtx_);
avcodec_free_context(&pCodecCtx_);
avformat_close_input(&pFormatCtx_);
sws_freeContext(sws_ctx_);
} }
void FrameReader::process() { void FrameReader::process() {
if (avformat_open_input(&pFormatCtx, url.c_str(), NULL, NULL) != 0) { if (processFrames()) {
fprintf(stderr, "error loading %s\n", url.c_str()); decode_thread_ = std::thread(&FrameReader::decodeThread, this);
valid = false;
return;
} }
avformat_find_stream_info(pFormatCtx, NULL); if (!exit_) {
av_dump_format(pFormatCtx, 0, url.c_str(), 0); emit finished();
}
}
auto pCodecCtxOrig = pFormatCtx->streams[0]->codec; bool FrameReader::processFrames() {
if (avformat_open_input(&pFormatCtx_, url_.c_str(), NULL, NULL) != 0) {
qDebug() << "error loading " << url_.c_str();
return false;
}
avformat_find_stream_info(pFormatCtx_, NULL);
av_dump_format(pFormatCtx_, 0, url_.c_str(), 0);
auto pCodecCtxOrig = pFormatCtx_->streams[0]->codec;
auto pCodec = avcodec_find_decoder(pCodecCtxOrig->codec_id); auto pCodec = avcodec_find_decoder(pCodecCtxOrig->codec_id);
assert(pCodec != NULL); assert(pCodec);
pCodecCtx = avcodec_alloc_context3(pCodec); pCodecCtx_ = avcodec_alloc_context3(pCodec);
int ret = avcodec_copy_context(pCodecCtx, pCodecCtxOrig); int ret = avcodec_copy_context(pCodecCtx_, pCodecCtxOrig);
assert(ret == 0); assert(ret == 0);
ret = avcodec_open2(pCodecCtx, pCodec, NULL); ret = avcodec_open2(pCodecCtx_, pCodec, NULL);
assert(ret >= 0); assert(ret >= 0);
width = pCodecCtxOrig->width; width = pCodecCtxOrig->width;
height = pCodecCtxOrig->height; height = pCodecCtxOrig->height;
sws_ctx = sws_getContext(width, height, AV_PIX_FMT_YUV420P, sws_ctx_ = sws_getContext(width, height, AV_PIX_FMT_YUV420P,
width, height, AV_PIX_FMT_BGR24, width, height, AV_PIX_FMT_BGR24,
SWS_BILINEAR, NULL, NULL, NULL); SWS_BILINEAR, NULL, NULL, NULL);
assert(sws_ctx != NULL); assert(sws_ctx_);
frmRgb_ = av_frame_alloc();
assert(frmRgb_);
frames_.reserve(60 * 20); // 20fps, one minute
do { do {
AVPacket *pkt = new AVPacket; Frame &frame = frames_.emplace_back();
if (av_read_frame(pFormatCtx, pkt) < 0) { if (av_read_frame(pFormatCtx_, &frame.pkt) < 0) {
delete pkt; frames_.pop_back();
break; break;
} }
Frame *frame = new Frame; } while (!exit_);
frame->pkt = pkt;
frames.push_back(frame);
} while (true);
printf("framereader download done\n"); valid_ = !exit_;
return valid_;
}
uint8_t *FrameReader::get(int idx) {
if (!valid_ || idx < 0 || idx >= frames_.size()) {
return nullptr;
}
thread = std::thread(&FrameReader::decodeThread, this); {
std::unique_lock lk(mutex_);
decode_idx_ = idx;
cv_decode_.notify_one();
cv_frame_.wait(lk, [=] { return exit_ || frames_[idx].data || frames_[idx].failed; });
}
return frames_[idx].data;
} }
void FrameReader::decodeThread() { void FrameReader::decodeThread() {
int idx = 0;
while (!exit_) { while (!exit_) {
int gop = 0; const int from = std::max(idx, 0);
{ const int to = std::min(idx + 20, (int)frames_.size());
std::unique_lock lk(mutex); for (int i = 0; i < frames_.size() && !exit_; ++i) {
cv_decode.wait(lk, [=] { return exit_ || decode_idx != -1; }); Frame &frame = frames_[i];
if (exit_) break; if (i >= from && i < to) {
if (frame.data || frame.failed) continue;
gop = std::max(decode_idx - decode_idx % 15, 0);
decode_idx = -1; uint8_t *dat = decodeFrame(&frame.pkt);
std::unique_lock lk(mutex_);
frame.data = dat;
frame.failed = !dat;
cv_frame_.notify_all();
} else if (frame.data) {
buffer_pool.push(frame.data);
frame.data = nullptr;
frame.failed = false;
}
} }
for (int i = gop; i < std::min(gop + 15, (int)frames.size()); ++i) { // sleep & wait
if (frames[i]->picture != nullptr) continue; std::unique_lock lk(mutex_);
cv_decode_.wait(lk, [=] { return exit_ || decode_idx_ != -1; });
int frameFinished; idx = decode_idx_;
AVFrame *pFrame = av_frame_alloc(); decode_idx_ = -1;
avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, frames[i]->pkt);
AVFrame *picture = toRGB(pFrame);
av_frame_free(&pFrame);
std::unique_lock lk(mutex);
frames[i]->picture = picture;
cv_frame.notify_all();
}
} }
} }
AVFrame *FrameReader::toRGB(AVFrame *pFrame) { uint8_t *FrameReader::decodeFrame(AVPacket *pkt) {
AVFrame *pFrameRGB = av_frame_alloc(); int gotFrame;
int numBytes = avpicture_get_size(AV_PIX_FMT_BGR24, pFrame->width, pFrame->height); AVFrame *f = av_frame_alloc();
uint8_t *buffer = (uint8_t *)av_malloc(numBytes * sizeof(uint8_t)); avcodec_decode_video2(pCodecCtx_, f, &gotFrame, pkt);
avpicture_fill((AVPicture *)pFrameRGB, buffer, AV_PIX_FMT_BGR24, pFrame->width, pFrame->height);
sws_scale(sws_ctx, (uint8_t const *const *)pFrame->data, uint8_t *dat = nullptr;
pFrame->linesize, 0, pFrame->height, if (gotFrame) {
pFrameRGB->data, pFrameRGB->linesize); if (!buffer_pool.empty()) {
return pFrameRGB; dat = buffer_pool.front();
} buffer_pool.pop();
} else {
dat = new uint8_t[getRGBSize()];
}
uint8_t *FrameReader::get(int idx) { int ret = avpicture_fill((AVPicture *)frmRgb_, dat, AV_PIX_FMT_BGR24, f->width, f->height);
if (!valid || idx < 0 || idx >= frames.size()) return nullptr; assert(ret > 0);
if (sws_scale(sws_ctx_, (const uint8_t **)f->data, f->linesize, 0,
std::unique_lock lk(mutex); f->height, frmRgb_->data, frmRgb_->linesize) <= 0) {
decode_idx = idx; delete[] dat;
cv_decode.notify_one(); dat = nullptr;
Frame *frame = frames[idx]; }
if (!frame->picture) {
cv_frame.wait(lk, [=] { return exit_ || frame->picture != nullptr; });
} }
return frame->picture ? frame->picture->data[0] : nullptr; av_frame_free(&f);
return dat;
} }

52
selfdrive/ui/replay/framereader.h
Unescape View File

@ -5,10 +5,13 @@
#include <atomic> #include <atomic>
#include <condition_variable> #include <condition_variable>
#include <mutex> #include <mutex>
#include <queue>
#include <string> #include <string>
#include <thread> #include <thread>
#include <vector> #include <vector>
#include <QThread>
// independent of QT, needs ffmpeg // independent of QT, needs ffmpeg
extern "C" { extern "C" {
#include <libavcodec/avcodec.h> #include <libavcodec/avcodec.h>
@ -16,38 +19,47 @@ extern "C" {
#include <libswscale/swscale.h> #include <libswscale/swscale.h>
} }
class FrameReader : public QObject {
Q_OBJECT
class FrameReader {
public: public:
FrameReader(const std::string &fn); FrameReader(const std::string &url, QObject *parent = nullptr);
~FrameReader(); ~FrameReader();
uint8_t *get(int idx); uint8_t *get(int idx);
AVFrame *toRGB(AVFrame *); int getRGBSize() { return width * height * 3; }
int getRGBSize() { return width*height*3; } bool valid() const { return valid_; }
void process();
int width = 0, height = 0; int width = 0, height = 0;
signals:
void finished();
private: private:
void process();
bool processFrames();
void decodeThread(); void decodeThread();
uint8_t *decodeFrame(AVPacket *pkt);
struct Frame{ struct Frame {
AVPacket *pkt; AVPacket pkt = {};
AVFrame *picture; uint8_t *data = nullptr;
bool failed = false;
}; };
std::vector<Frame*> frames; std::vector<Frame> frames_;
AVFormatContext *pFormatCtx = NULL; AVFormatContext *pFormatCtx_ = NULL;
AVCodecContext *pCodecCtx = NULL; AVCodecContext *pCodecCtx_ = NULL;
struct SwsContext *sws_ctx = NULL; AVFrame *frmRgb_ = nullptr;
std::queue<uint8_t *> buffer_pool;
struct SwsContext *sws_ctx_ = NULL;
std::mutex mutex; std::mutex mutex_;
std::condition_variable cv_decode; std::condition_variable cv_decode_;
std::condition_variable cv_frame; std::condition_variable cv_frame_;
int decode_idx = -1; int decode_idx_ = 0;
std::atomic<bool> exit_ = false; std::atomic<bool> exit_ = false;
std::thread thread; bool valid_ = false;
std::string url_;
bool valid = true; QThread *process_thread_;
std::string url; std::thread decode_thread_;
}; };

25
selfdrive/ui/replay/replay.cc
Unescape View File

@ -76,19 +76,11 @@ void Replay::addSegment(int n) {
QObject::connect(t, &QThread::started, lrs[n], &LogReader::process); QObject::connect(t, &QThread::started, lrs[n], &LogReader::process);
t->start(); t->start();
QThread *frame_thread = QThread::create([=]{ frs[n] = new FrameReader(qPrintable(camera_paths.at(n).toString()), this);
FrameReader *frame_reader = new FrameReader(qPrintable(camera_paths.at(n).toString()));
frame_reader->process();
frs.insert(n, frame_reader);
});
QObject::connect(frame_thread, &QThread::finished, frame_thread, &QThread::deleteLater);
frame_thread->start();
} }
void Replay::removeSegment(int n) { void Replay::removeSegment(int n) {
// TODO: fix FrameReader and LogReader destructors // TODO: fix LogReader destructors
/* /*
if (lrs.contains(n)) { if (lrs.contains(n)) {
auto lr = lrs.take(n); auto lr = lrs.take(n);
@ -255,8 +247,6 @@ void Replay::stream() {
auto pp = *it_; auto pp = *it_;
if (frs.find(pp.first) != frs.end()) { if (frs.find(pp.first) != frs.end()) {
auto frm = frs[pp.first]; auto frm = frs[pp.first];
auto data = frm->get(pp.second);
if (vipc_server == nullptr) { if (vipc_server == nullptr) {
cl_device_id device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT); cl_device_id device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT);
cl_context context = CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err)); cl_context context = CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
@ -267,10 +257,13 @@ void Replay::stream() {
vipc_server->start_listener(); vipc_server->start_listener();
} }
VisionIpcBufExtra extra = {}; uint8_t *dat = frm->get(pp.second);
VisionBuf *buf = vipc_server->get_buffer(VisionStreamType::VISION_STREAM_RGB_BACK); if (dat) {
memcpy(buf->addr, data, frm->getRGBSize()); VisionIpcBufExtra extra = {};
vipc_server->send(buf, &extra, false); VisionBuf *buf = vipc_server->get_buffer(VisionStreamType::VISION_STREAM_RGB_BACK);
memcpy(buf->addr, dat, frm->getRGBSize());
vipc_server->send(buf, &extra, false);
}
} }
} }
} }

Write Preview
Loading…
Cancel
Save