From 1e67fc25f1019517b695ad76fa22f64d65d4a9d3 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Wed, 10 Nov 2021 06:08:24 +0800 Subject: [PATCH] replay: add qcam and YUV flags (#22788) * common flags * cleanup * remove double semicolon * camera * apply reviews old-commit-hash: fb8ba34f3103513705de2cb6397796b38a0c0689 --- selfdrive/ui/replay/camera.cc | 23 ++++++++++++----------- selfdrive/ui/replay/camera.h | 3 ++- selfdrive/ui/replay/framereader.cc | 16 +++++++++------- selfdrive/ui/replay/main.cc | 2 ++ selfdrive/ui/replay/replay.cc | 10 +++++----- selfdrive/ui/replay/replay.h | 3 ++- selfdrive/ui/replay/route.cc | 11 ++++++----- selfdrive/ui/replay/route.h | 5 +++-- selfdrive/ui/replay/tests/test_replay.cc | 8 ++------ 9 files changed, 43 insertions(+), 38 deletions(-) diff --git a/selfdrive/ui/replay/camera.cc b/selfdrive/ui/replay/camera.cc index 7ff07e3044..b37332ae7e 100644 --- a/selfdrive/ui/replay/camera.cc +++ b/selfdrive/ui/replay/camera.cc @@ -5,9 +5,9 @@ const int YUV_BUF_COUNT = 50; -CameraServer::CameraServer(std::pair camera_size[MAX_CAMERAS]) { - for (auto &cam : cameras_) { - std::tie(cam.width, cam.height) = camera_size[cam.type]; +CameraServer::CameraServer(std::pair camera_size[MAX_CAMERAS], bool send_yuv) : send_yuv(send_yuv) { + for (int i = 0; i < MAX_CAMERAS; ++i) { + std::tie(cameras_[i].width, cameras_[i].height) = camera_size[i]; } startVipcServer(); } @@ -28,7 +28,9 @@ void CameraServer::startVipcServer() { if (cam.width > 0 && cam.height > 0) { std::cout << "camera[" << cam.type << "] frame size " << cam.width << "x" << cam.height << std::endl; vipc_server_->create_buffers(cam.rgb_type, UI_BUF_COUNT, true, cam.width, cam.height); - vipc_server_->create_buffers(cam.yuv_type, YUV_BUF_COUNT, false, cam.width, cam.height); + if (send_yuv) { + vipc_server_->create_buffers(cam.yuv_type, YUV_BUF_COUNT, false, cam.width, cam.height); + } if (!cam.thread.joinable()) { cam.thread = std::thread(&CameraServer::cameraThread, this, std::ref(cam)); } @@ -40,8 +42,8 @@ void CameraServer::startVipcServer() { void CameraServer::cameraThread(Camera &cam) { auto read_frame = [&](FrameReader *fr, int frame_id) { VisionBuf *rgb_buf = vipc_server_->get_buffer(cam.rgb_type); - VisionBuf *yuv_buf = vipc_server_->get_buffer(cam.yuv_type); - bool ret = fr->get(frame_id, (uint8_t *)rgb_buf->addr, (uint8_t *)yuv_buf->addr); + VisionBuf *yuv_buf = send_yuv ? vipc_server_->get_buffer(cam.yuv_type) : nullptr; + bool ret = fr->get(frame_id, (uint8_t *)rgb_buf->addr, yuv_buf ? (uint8_t *)yuv_buf->addr : nullptr); return ret ? std::pair{rgb_buf, yuv_buf} : std::pair{nullptr, nullptr}; }; @@ -50,17 +52,16 @@ void CameraServer::cameraThread(Camera &cam) { if (!fr) break; const int id = eidx.getSegmentId(); - bool prefetched = (id == cam.cached_id && eidx.getSegmentNum() == cam.cached_seg && cam.cached_buf.first && cam.cached_buf.second); + bool prefetched = (id == cam.cached_id && eidx.getSegmentNum() == cam.cached_seg); auto [rgb, yuv] = prefetched ? cam.cached_buf : read_frame(fr, id); - - if (rgb && yuv) { + if (rgb || yuv) { VisionIpcBufExtra extra = { .frame_id = eidx.getFrameId(), .timestamp_sof = eidx.getTimestampSof(), .timestamp_eof = eidx.getTimestampEof(), }; - vipc_server_->send(rgb, &extra, false); - vipc_server_->send(yuv, &extra, false); + if (rgb) vipc_server_->send(rgb, &extra, false); + if (yuv) vipc_server_->send(yuv, &extra, false); } else { std::cout << "camera[" << cam.type << "] failed to get frame:" << eidx.getSegmentId() << std::endl; } diff --git a/selfdrive/ui/replay/camera.h b/selfdrive/ui/replay/camera.h index e813b85d3d..8b1fa7620b 100644 --- a/selfdrive/ui/replay/camera.h +++ b/selfdrive/ui/replay/camera.h @@ -8,7 +8,7 @@ class CameraServer { public: - CameraServer(std::pair camera_size[MAX_CAMERAS] = nullptr); + CameraServer(std::pair camera_size[MAX_CAMERAS] = nullptr, bool send_yuv = false); ~CameraServer(); void pushFrame(CameraType type, FrameReader* fr, const cereal::EncodeIndex::Reader& eidx); inline void waitFinish() { @@ -38,4 +38,5 @@ protected: }; std::atomic publishing_ = 0; std::unique_ptr vipc_server_; + bool send_yuv; }; diff --git a/selfdrive/ui/replay/framereader.cc b/selfdrive/ui/replay/framereader.cc index 12dda678f2..51a99cea63 100644 --- a/selfdrive/ui/replay/framereader.cc +++ b/selfdrive/ui/replay/framereader.cc @@ -31,7 +31,7 @@ int readFunction(void *opaque, uint8_t *buf, int buf_size) { return iss.gcount() ? iss.gcount() : AVERROR_EOF; } -} // namespace +} // namespace FrameReader::FrameReader(bool local_cache, int chunk_size, int retries) : FileReader(local_cache, chunk_size, retries) { static std::once_flag once_flag; @@ -43,7 +43,7 @@ FrameReader::FrameReader(bool local_cache, int chunk_size, int retries) : FileRe pFormatCtx_ = avformat_alloc_context(); av_frame_ = av_frame_alloc(); rgb_frame_ = av_frame_alloc(); - yuv_frame_ = av_frame_alloc();; + yuv_frame_ = av_frame_alloc(); } FrameReader::~FrameReader() { @@ -126,7 +126,7 @@ bool FrameReader::load(const std::string &url, std::atomic *abort) { } bool FrameReader::get(int idx, uint8_t *rgb, uint8_t *yuv) { - assert(rgb != nullptr && yuv != nullptr); + assert(rgb || yuv); if (!valid_ || idx < 0 || idx >= frames_.size()) { return false; } @@ -162,11 +162,13 @@ bool FrameReader::decode(int idx, uint8_t *rgb, uint8_t *yuv) { bool FrameReader::decodeFrame(AVFrame *f, uint8_t *rgb, uint8_t *yuv) { // images is going to be written to output buffers, no alignment (align = 1) - av_image_fill_arrays(yuv_frame_->data, yuv_frame_->linesize, yuv, AV_PIX_FMT_YUV420P, width, height, 1); - int ret = sws_scale(yuv_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, yuv_frame_->data, yuv_frame_->linesize); - if (ret < 0) return false; + if (yuv) { + av_image_fill_arrays(yuv_frame_->data, yuv_frame_->linesize, yuv, AV_PIX_FMT_YUV420P, width, height, 1); + int ret = sws_scale(yuv_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, yuv_frame_->data, yuv_frame_->linesize); + if (ret < 0) return false; + } av_image_fill_arrays(rgb_frame_->data, rgb_frame_->linesize, rgb, AV_PIX_FMT_BGR24, width, height, 1); - ret = sws_scale(rgb_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, rgb_frame_->data, rgb_frame_->linesize); + int ret = sws_scale(rgb_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, rgb_frame_->data, rgb_frame_->linesize); return ret >= 0; } diff --git a/selfdrive/ui/replay/main.cc b/selfdrive/ui/replay/main.cc index fea351c4b5..d4fc8ff13a 100644 --- a/selfdrive/ui/replay/main.cc +++ b/selfdrive/ui/replay/main.cc @@ -100,6 +100,8 @@ int main(int argc, char *argv[]) { {"ecam", REPLAY_FLAG_ECAM, "load wide road camera"}, {"no-loop", REPLAY_FLAG_NO_LOOP, "stop at the end of the route"}, {"no-cache", REPLAY_FLAG_NO_FILE_CACHE, "turn off local cache"}, + {"qcam", REPLAY_FLAG_QCAMERA, "load qcamera"}, + {"yuv", REPLAY_FLAG_SEND_YUV, "send yuv frame"}, }; QCommandLineParser parser; diff --git a/selfdrive/ui/replay/replay.cc b/selfdrive/ui/replay/replay.cc index d6637b09b0..e3753ccf20 100644 --- a/selfdrive/ui/replay/replay.cc +++ b/selfdrive/ui/replay/replay.cc @@ -151,7 +151,7 @@ void Replay::queueSegment() { if (!it->second) { if (it == cur || std::prev(it)->second->isLoaded()) { auto &[n, seg] = *it; - seg = std::make_unique(n, route_->at(n), hasFlag(REPLAY_FLAG_DCAM), hasFlag(REPLAY_FLAG_ECAM), !hasFlag(REPLAY_FLAG_NO_FILE_CACHE)); + seg = std::make_unique(n, route_->at(n), flags_); QObject::connect(seg.get(), &Segment::loadFinished, this, &Replay::segmentLoadFinished); qDebug() << "loading segment" << n << "..."; } @@ -230,7 +230,7 @@ void Replay::startStream(const Segment *cur_segment) { camera_size[type] = {fr->width, fr->height}; } } - camera_server_ = std::make_unique(camera_size); + camera_server_ = std::make_unique(camera_size, flags_ & REPLAY_FLAG_SEND_YUV); // start stream thread stream_thread_ = new QThread(); @@ -258,8 +258,8 @@ void Replay::publishFrame(const Event *e) { {cereal::Event::DRIVER_ENCODE_IDX, DriverCam}, {cereal::Event::WIDE_ROAD_ENCODE_IDX, WideRoadCam}, }; - if ((e->which == cereal::Event::DRIVER_ENCODE_IDX && !hasFlag(REPLAY_FLAG_DCAM)) || - (e->which == cereal::Event::WIDE_ROAD_ENCODE_IDX && !hasFlag(REPLAY_FLAG_ECAM))) { + if ((e->which == cereal::Event::DRIVER_ENCODE_IDX && !(flags_ & REPLAY_FLAG_DCAM)) || + (e->which == cereal::Event::WIDE_ROAD_ENCODE_IDX && !(flags_ & REPLAY_FLAG_ECAM))) { return; } auto eidx = capnp::AnyStruct::Reader(e->event).getPointerSection()[0].getAs(); @@ -334,7 +334,7 @@ void Replay::stream() { // wait for frame to be sent before unlock.(frameReader may be deleted after unlock) camera_server_->waitFinish(); - if (eit == events_->end() && !hasFlag(REPLAY_FLAG_NO_LOOP)) { + if (eit == events_->end() && !(flags_ & REPLAY_FLAG_NO_LOOP)) { int last_segment = segments_.rbegin()->first; if (current_segment_ >= last_segment && isSegmentMerged(last_segment)) { qInfo() << "reaches the end of route, restart from beginning"; diff --git a/selfdrive/ui/replay/replay.h b/selfdrive/ui/replay/replay.h index 57d9b582d3..320afa505b 100644 --- a/selfdrive/ui/replay/replay.h +++ b/selfdrive/ui/replay/replay.h @@ -14,6 +14,8 @@ enum REPLAY_FLAGS { REPLAY_FLAG_ECAM = 0x0004, REPLAY_FLAG_NO_LOOP = 0x0010, REPLAY_FLAG_NO_FILE_CACHE = 0x0020, + REPLAY_FLAG_QCAMERA = 0x0040, + REPLAY_FLAG_SEND_YUV = 0x0080, }; class Replay : public QObject { @@ -28,7 +30,6 @@ public: void stop(); void pause(bool pause); bool isPaused() const { return paused_; } - inline bool hasFlag(REPLAY_FLAGS flag) { return flags_ & flag; }; signals: void segmentChanged(); diff --git a/selfdrive/ui/replay/route.cc b/selfdrive/ui/replay/route.cc index 04ad7b2144..d834d67ab6 100644 --- a/selfdrive/ui/replay/route.cc +++ b/selfdrive/ui/replay/route.cc @@ -9,6 +9,7 @@ #include "selfdrive/hardware/hw.h" #include "selfdrive/ui/qt/api.h" +#include "selfdrive/ui/replay/replay.h" #include "selfdrive/ui/replay/util.h" Route::Route(const QString &route, const QString &data_dir) : data_dir_(data_dir) { @@ -90,18 +91,18 @@ void Route::addFileToSegment(int n, const QString &file) { // class Segment -Segment::Segment(int n, const SegmentFile &files, bool load_dcam, bool load_ecam, bool local_cache) : seg_num(n) { +Segment::Segment(int n, const SegmentFile &files, uint32_t flags) : seg_num(n) { // [RoadCam, DriverCam, WideRoadCam, log]. fallback to qcamera/qlog const QString file_list[] = { - files.road_cam.isEmpty() ? files.qcamera : files.road_cam, - load_dcam ? files.driver_cam : "", - load_ecam ? files.wide_road_cam : "", + (flags & REPLAY_FLAG_QCAMERA) || files.road_cam.isEmpty() ? files.qcamera : files.road_cam, + flags & REPLAY_FLAG_DCAM ? files.driver_cam : "", + flags & REPLAY_FLAG_ECAM ? files.wide_road_cam : "", files.rlog.isEmpty() ? files.qlog : files.rlog, }; for (int i = 0; i < std::size(file_list); i++) { if (!file_list[i].isEmpty()) { loading_++; - synchronizer_.addFuture(QtConcurrent::run([=] { loadFile(i, file_list[i].toStdString(), local_cache); })); + synchronizer_.addFuture(QtConcurrent::run([=] { loadFile(i, file_list[i].toStdString(), !(flags & REPLAY_FLAG_NO_FILE_CACHE)); })); } } } diff --git a/selfdrive/ui/replay/route.h b/selfdrive/ui/replay/route.h index a77e8ad334..b11607f250 100644 --- a/selfdrive/ui/replay/route.h +++ b/selfdrive/ui/replay/route.h @@ -4,6 +4,7 @@ #include "selfdrive/ui/replay/framereader.h" #include "selfdrive/ui/replay/logreader.h" +#include "selfdrive/ui/replay/util.h" struct RouteIdentifier { QString dongle_id; @@ -28,7 +29,7 @@ public: inline const QString &name() const { return route_.str; } inline const RouteIdentifier &identifier() const { return route_; } inline const std::map &segments() const { return segments_; } - inline SegmentFile &at(int n) { return segments_.at(n); } + inline const SegmentFile &at(int n) { return segments_.at(n); } static RouteIdentifier parseRoute(const QString &str); protected: @@ -45,7 +46,7 @@ class Segment : public QObject { Q_OBJECT public: - Segment(int n, const SegmentFile &files, bool load_dcam, bool load_ecam, bool local_cache); + Segment(int n, const SegmentFile &files, uint32_t flags); ~Segment(); inline bool isLoaded() const { return !loading_ && !abort_; } diff --git a/selfdrive/ui/replay/tests/test_replay.cc b/selfdrive/ui/replay/tests/test_replay.cc index bda6da2670..4b776e7403 100644 --- a/selfdrive/ui/replay/tests/test_replay.cc +++ b/selfdrive/ui/replay/tests/test_replay.cc @@ -57,17 +57,13 @@ TEST_CASE("FileReader") { } TEST_CASE("Segment") { - auto test_qlog = GENERATE(false, true); + auto flags = GENERATE(REPLAY_FLAG_NONE, REPLAY_FLAG_QCAMERA); Route demo_route(DEMO_ROUTE); REQUIRE(demo_route.load()); REQUIRE(demo_route.segments().size() == 11); - if (test_qlog) { - demo_route.at(0).road_cam = ""; - demo_route.at(0).rlog = ""; - } QEventLoop loop; - Segment segment(0, demo_route.at(0), false, false, false); + Segment segment(0, demo_route.at(0), flags); QObject::connect(&segment, &Segment::loadFinished, [&]() { REQUIRE(segment.isLoaded() == true); REQUIRE(segment.log != nullptr);