From 816fbec7ab69bad1b49bf3f569c119e3d427ab57 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Thu, 18 Nov 2021 12:04:45 +0800 Subject: [PATCH] replay: fix swsscale crash on nv12->yuv conversion (#22961) old-commit-hash: ddea21e6482a16cb80c772f5a4fd0a19a038d393 --- selfdrive/ui/SConscript | 2 +- selfdrive/ui/replay/framereader.cc | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index d4c8fe71e2..bc21022dd9 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -116,7 +116,7 @@ if arch in ['x86_64', 'Darwin'] or GetOption('extras'): replay_lib_src = ["replay/replay.cc", "replay/camera.cc", "replay/filereader.cc", "replay/logreader.cc", "replay/framereader.cc", "replay/route.cc", "replay/util.cc"] replay_lib = qt_env.Library("qt_replay", replay_lib_src, LIBS=base_libs) - replay_libs = [replay_lib, 'avutil', 'avcodec', 'avformat', 'bz2', 'curl', 'swscale'] + qt_libs + replay_libs = [replay_lib, 'avutil', 'avcodec', 'avformat', 'bz2', 'curl', 'swscale', 'yuv'] + qt_libs qt_env.Program("replay/replay", ["replay/main.cc"], LIBS=replay_libs) qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'json11']) diff --git a/selfdrive/ui/replay/framereader.cc b/selfdrive/ui/replay/framereader.cc index 1524ad9748..fdbfc0672c 100644 --- a/selfdrive/ui/replay/framereader.cc +++ b/selfdrive/ui/replay/framereader.cc @@ -1,6 +1,7 @@ #include "selfdrive/ui/replay/framereader.h" #include +#include "libyuv.h" namespace { @@ -227,13 +228,26 @@ AVFrame *FrameReader::decodeFrame(AVPacket *pkt) { } bool FrameReader::copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv) { - // images is going to be written to output buffers, no alignment (align = 1) if (yuv) { - av_image_fill_arrays(sws_frame->data, sws_frame->linesize, yuv, AV_PIX_FMT_YUV420P, width, height, 1); - int ret = sws_scale(yuv_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, sws_frame->data, sws_frame->linesize); - if (ret < 0) return false; + if (sws_src_format == AV_PIX_FMT_NV12) { + // libswscale crash if height is not 16 bytes aligned for NV12->YUV420 conversion + assert(sws_src_format == AV_PIX_FMT_NV12); + uint8_t *u = yuv + width * height; + uint8_t *v = u + (width / 2) * (height / 2); + libyuv::NV12ToI420(f->data[0], f->linesize[0], + f->data[1], f->linesize[1], + yuv, width, + u, width / 2, + v, width / 2, + width, height); + } else { + av_image_fill_arrays(sws_frame->data, sws_frame->linesize, yuv, AV_PIX_FMT_YUV420P, width, height, 1); + int ret = sws_scale(yuv_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, sws_frame->data, sws_frame->linesize); + if (ret < 0) return false; + } } + // images is going to be written to output buffers, no alignment (align = 1) av_image_fill_arrays(sws_frame->data, sws_frame->linesize, rgb, AV_PIX_FMT_BGR24, width, height, 1); int ret = sws_scale(rgb_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, sws_frame->data, sws_frame->linesize); return ret >= 0;