sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add msm_vidc hw decoder interface

Browse Source
sync
Test User 2 months ago
parent 57425a15eb
commit 61be81cd43
2 changed files with 472 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 380
      tools/replay/qcom/decoder.cc
  2. 92
      tools/replay/qcom/decoder.h

380
tools/replay/qcom/decoder.cc
Unescape View File

@ -0,0 +1,380 @@
#include "decoder.h"
#include <assert.h>
#include "third_party/linux/include/v4l2-controls.h"
#include <linux/videodev2.h>
#include <sys/signalfd.h>
#include "common/swaglog.h"
#include "common/util.h"
static void request_buffers(int fd, v4l2_buf_type buf_type, unsigned int count) {
struct v4l2_requestbuffers reqbuf = {
.count = count,
.type = buf_type,
.memory = V4L2_MEMORY_USERPTR
};
util::safe_ioctl(fd, VIDIOC_REQBUFS, &reqbuf, "VIDIOC_REQBUFS failed");
}
MsmVidc::~MsmVidc() {
if (fd > 0) {
close(fd);
}
if (sigfd > 0) {
close(sigfd);
}
}
bool MsmVidc::init(const char* dev, size_t width, size_t height, uint64_t codec) {
LOG("Initializing msm_vidc device %s", dev);
this->w = width;
this->h = height;
this->fd = open(dev, O_RDWR, 0);
if (fd < 0) {
LOGE("failed to open video device %s", dev);
return false;
}
subscribeEvents();
v4l2_buf_type out_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
setPlaneFormat(out_type, V4L2_PIX_FMT_HEVC); // Also allocates the output buffer
setFPS(FPS);
request_buffers(fd, out_type, OUTPUT_BUFFER_COUNT);
util::safe_ioctl(fd, VIDIOC_STREAMON, &out_type, "VIDIOC_STREAMON OUTPUT failed");
restartCapture();
setupPolling();
rotator.init();
rotator.configUBWCtoNV12(width, height);
this->initialized = true;
return true;
}
VisionBuf* MsmVidc::decodeFrame(AVPacket *pkt, VisionBuf *buf) {
assert(initialized && (pkt != nullptr) && (buf != nullptr));
this->frame_ready = false;
this->current_output_buf = buf;
bool sent_packet = false;
while (!this->frame_ready) {
if (!sent_packet) {
int buf_index = getBufferUnlocked();
if (buf_index >= 0) {
assert(buf_index < out_buf_cnt);
sendPacket(buf_index, pkt);
sent_packet = true;
}
}
if (poll(pfd, nfds, -1) < 0) {
LOGE("poll() error: %d", errno);
return nullptr;
}
if (VisionBuf* result = processEvents()) {
return result;
}
}
return buf;
}
VisionBuf* MsmVidc::processEvents() {
for (int idx = 0; idx < nfds; idx++) {
short revents = pfd[idx].revents;
if (!revents) continue;
if (idx == ev[EV_VIDEO]) {
if (revents & (POLLIN | POLLRDNORM)) {
VisionBuf *result = handleCapture();
if (result == this->current_output_buf) {
this->frame_ready = true;
}
}
if (revents & (POLLOUT | POLLWRNORM)) {
handleOutput();
}
if (revents & POLLPRI) {
handleEvent();
}
} else if (idx == ev[EV_SIGNAL]) {
handleSignal();
return nullptr; // Signal received, abort
} else {
LOGE("Unexpected event on fd %d", pfd[idx].fd);
}
}
return nullptr;
}
VisionBuf* MsmVidc::handleCapture() {
struct v4l2_buffer buf = {0};
struct v4l2_plane planes[1] = {0};
buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
buf.memory = V4L2_MEMORY_USERPTR;
buf.m.planes = planes;
buf.length = 1;
util::safe_ioctl(this->fd, VIDIOC_DQBUF, &buf, "VIDIOC_DQBUF CAPTURE failed");
if (buf.m.planes[0].bytesused) {
static size_t cap_cnt = 0;
cap_cnt++;
if (cap_cnt % 240 == 0) {
LOGD("Dequeued %zu capture buffers", cap_cnt);
}
if (!this->reconfigure_pending) {
rotator.putFrame(&cap_bufs[buf.index]);
VisionBuf *rotated = rotator.getFrame(100);
queueCaptureBuffer(buf.index);
if (rotated) {
rotator.convertStride(rotated, this->current_output_buf);
return this->current_output_buf;
}
}
} else {
LOGE("Dequeued empty capture buffer %d", buf.index);
}
return nullptr;
}
bool MsmVidc::subscribeEvents() {
for (uint32_t event : subscriptions) {
struct v4l2_event_subscription sub = { .type = event};
util::safe_ioctl(fd, VIDIOC_SUBSCRIBE_EVENT, &sub, "VIDIOC_SUBSCRIBE_EVENT failed");
}
return true;
}
bool MsmVidc::setPlaneFormat(enum v4l2_buf_type type, uint32_t fourcc) {
struct v4l2_format fmt = {.type = type};
struct v4l2_pix_format_mplane *pix = &fmt.fmt.pix_mp;
*pix = { .width = (__u32)this->w, .height = (__u32)this->h, .pixelformat = fourcc };
util::safe_ioctl(fd, VIDIOC_S_FMT, &fmt, "VIDIOC_S_FMT failed");
if (type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
this->out_buf_size = pix->plane_fmt[0].sizeimage;
int ion_size = this->out_buf_size * OUTPUT_BUFFER_COUNT; // Output (input) buffers are ION buffer.
this->out_buf.allocate_no_cache(ion_size); // mmap rw
for (int i = 0; i < OUTPUT_BUFFER_COUNT; i++) {
this->out_buf_off[i] = i * this->out_buf_size;
this->out_buf_addr[i] = (char *)this->out_buf.addr + this->out_buf_off[i];
this->out_buf_flag[i] = false;
}
LOGD("Set output buffer size to %d, count %d, addr %p", this->out_buf_size, OUTPUT_BUFFER_COUNT, this->out_buf.addr);
} else if (type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
request_buffers(this->fd, type, CAPTURE_BUFFER_COUNT);
util::safe_ioctl(fd, VIDIOC_G_FMT, &fmt, "VIDIOC_G_FMT failed");
for (int i = 0; i < CAPTURE_BUFFER_COUNT; i++) {
this->cap_bufs[i].allocate_no_cache(pix->plane_fmt[0].sizeimage);
this->cap_bufs[i].init_yuv(pix->width, pix->height, pix->plane_fmt[0].bytesperline, 0);
}
this->cap_buf_format = pix->pixelformat;
LOGD("Set capture buffer size to %d, count %d, addr %p, extradata size %d",
pix->plane_fmt[0].sizeimage, CAPTURE_BUFFER_COUNT, this->cap_bufs[0].addr, pix->plane_fmt[1].sizeimage);
}
return true;
}
bool MsmVidc::setFPS(uint32_t fps) {
struct v4l2_streamparm streamparam = {
.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
.parm.output.timeperframe = {1, fps}
};
util::safe_ioctl(fd, VIDIOC_S_PARM, &streamparam, "VIDIOC_S_PARM failed");
return true;
}
bool MsmVidc::restartCapture() {
// stop if already initialized
enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
if (this->initialized) {
LOGD("Restarting capture, flushing buffers...");
util::safe_ioctl(this->fd, VIDIOC_STREAMOFF, &type, "VIDIOC_STREAMOFF CAPTURE failed");
struct v4l2_requestbuffers reqbuf = {.type = type, .memory = V4L2_MEMORY_USERPTR};
util::safe_ioctl(this->fd, VIDIOC_REQBUFS, &reqbuf, "VIDIOC_REQBUFS failed");
for (size_t i = 0; i < CAPTURE_BUFFER_COUNT; ++i) {
this->cap_bufs[i].free();
this->cap_buf_flag[i] = false; // mark as not queued
cap_bufs[i].~VisionBuf();
new (&cap_bufs[i]) VisionBuf();
}
}
// setup, start and queue capture buffers
setDBP();
setPlaneFormat(type, V4L2_PIX_FMT_NV12_UBWC);
util::safe_ioctl(this->fd, VIDIOC_STREAMON, &type, "VIDIOC_STREAMON CAPTURE failed");
for (size_t i = 0; i < CAPTURE_BUFFER_COUNT; ++i) {
queueCaptureBuffer(i);
}
return true;
}
bool MsmVidc::queueCaptureBuffer(int i) {
struct v4l2_buffer buf = {0};
struct v4l2_plane planes[1] = {0};
buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
buf.memory = V4L2_MEMORY_USERPTR;
buf.index = i;
buf.m.planes = planes;
buf.length = 1;
// decoded frame plane
planes[0].m.userptr = (unsigned long)this->cap_bufs[i].addr; // no security
planes[0].length = this->cap_bufs[i].len;
planes[0].reserved[0] = this->cap_bufs[i].fd; // ION fd
planes[0].reserved[1] = 0;
planes[0].bytesused = this->cap_bufs[i].len;
planes[0].data_offset = 0;
util::safe_ioctl(this->fd, VIDIOC_QBUF, &buf, "VIDIOC_QBUF failed");
this->cap_buf_flag[i] = true; // mark as queued
return true;
}
bool MsmVidc::queueOutputBuffer(int i, size_t size) {
struct v4l2_buffer buf = {0};
struct v4l2_plane planes[1] = {0};
buf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
buf.memory = V4L2_MEMORY_USERPTR;
buf.index = i;
buf.m.planes = planes;
buf.length = 1;
// decoded frame plane
planes[0].m.userptr = (unsigned long)this->out_buf_off[i]; // check this
planes[0].length = this->out_buf_size;
planes[0].reserved[0] = this->out_buf.fd; // ION fd
planes[0].reserved[1] = 0;
planes[0].bytesused = size;
planes[0].data_offset = 0;
assert((this->out_buf_off[i] & 0xfff) == 0); // must be 4 KiB aligned
assert(this->out_buf_size % 4096 == 0); // ditto for size
util::safe_ioctl(this->fd, VIDIOC_QBUF, &buf, "VIDIOC_QBUF failed");
this->out_buf_flag[i] = true; // mark as queued
return true;
}
bool MsmVidc::setDBP() {
struct v4l2_ext_control control[2] = {0};
struct v4l2_ext_controls controls = {0};
control[0].id = V4L2_CID_MPEG_VIDC_VIDEO_STREAM_OUTPUT_MODE;
control[0].value = 0; // V4L2_CID_MPEG_VIDC_VIDEO_STREAM_OUTPUT_PRIMARY
control[1].id = V4L2_CID_MPEG_VIDC_VIDEO_DPB_COLOR_FORMAT;
control[1].value = 0; // V4L2_MPEG_VIDC_VIDEO_DPB_COLOR_FMT_NONE
controls.count = 2;
controls.ctrl_class = V4L2_CTRL_CLASS_MPEG;
controls.controls = control;
util::safe_ioctl(fd, VIDIOC_S_EXT_CTRLS, &controls, "VIDIOC_S_EXT_CTRLS failed");
return true;
}
bool MsmVidc::setupPolling() {
// Setup signals
sigset_t sigmask;
sigemptyset(&sigmask);
sigaddset(&sigmask, SIGINT);
sigaddset(&sigmask, SIGTERM);
sigprocmask(SIG_BLOCK, &sigmask, nullptr);
sigfd = signalfd(-1, &sigmask, SFD_CLOEXEC);
assert(sigfd > 0);
// Initialize poll array
pfd[EV_VIDEO] = {fd, POLLIN | POLLOUT | POLLWRNORM | POLLRDNORM | POLLPRI, 0};
pfd[EV_SIGNAL] = {sigfd, POLLIN, 0};
ev[EV_VIDEO] = EV_VIDEO;
ev[EV_SIGNAL] = EV_SIGNAL;
nfds = 2;
LOGD("Setup polling: video_fd=%d, signal_fd=%d", fd, sigfd);
return true;
}
bool MsmVidc::sendPacket(int buf_index, AVPacket *pkt) {
assert(buf_index >= 0 && buf_index < out_buf_cnt);
assert(pkt != nullptr && pkt->data != nullptr && pkt->size > 0);
// Prepare output buffer
memset(this->out_buf_addr[buf_index], 0, this->out_buf_size);
uint8_t * data = (uint8_t *)this->out_buf_addr[buf_index];
memcpy(data, pkt->data, pkt->size);
queueOutputBuffer(buf_index, pkt->size);
return true;
}
int MsmVidc::getBufferUnlocked() {
for (int i = 0; i < this->out_buf_cnt; i++) {
if (!out_buf_flag[i]) {
return i;
}
}
return -1;
}
int MsmVidc::handleSignal() {
struct signalfd_siginfo siginfo;
sigset_t sigmask;
if (read(this->sigfd, &siginfo, sizeof (siginfo)) < 0) {
return -1;
}
sigemptyset(&sigmask);
sigaddset(&sigmask, siginfo.ssi_signo);
sigprocmask(SIG_UNBLOCK, &sigmask, NULL);
LOGD("Received signal %d, cleaning up", siginfo.ssi_signo);
if (fd > 0) {
close(fd);
}
if (sigfd > 0) {
close(sigfd);
}
return 0;
}
bool MsmVidc::handleOutput() {
struct v4l2_buffer buf = {0};
struct v4l2_plane planes[1];
buf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
buf.memory = V4L2_MEMORY_USERPTR;
buf.m.planes = planes;
buf.length = 1;
util::safe_ioctl(this->fd, VIDIOC_DQBUF, &buf, "VIDIOC_DQBUF OUTPUT failed");
this->out_buf_flag[buf.index] = false; // mark as not queued
return true;
}
bool MsmVidc::handleEvent() {
// dequeue event
struct v4l2_event event = {0};
util::safe_ioctl(this->fd, VIDIOC_DQEVENT, &event, "VIDIOC_DQEVENT failed");
switch (event.type) {
case V4L2_EVENT_MSM_VIDC_PORT_SETTINGS_CHANGED_INSUFFICIENT: {
unsigned int *ptr = (unsigned int *)event.u.data;
unsigned int height = ptr[0];
unsigned int width = ptr[1];
this->w = width;
this->h = height;
LOGD("Port Reconfig received insufficient, new size %ux%u, flushing capture bufs...", width, height); // This is normal
struct v4l2_decoder_cmd dec;
dec.flags = V4L2_QCOM_CMD_FLUSH_CAPTURE;
dec.cmd = V4L2_QCOM_CMD_FLUSH;
util::safe_ioctl(this->fd, VIDIOC_DECODER_CMD, &dec, "VIDIOC_DECODER_CMD FLUSH_CAPTURE failed");
this->reconfigure_pending = true;
LOGD("Waiting for flush done event to reconfigure capture queue");
break;
}
case V4L2_EVENT_MSM_VIDC_FLUSH_DONE: {
unsigned int *ptr = (unsigned int *)event.u.data;
unsigned int flags = ptr[0];
if (flags & V4L2_QCOM_CMD_FLUSH_CAPTURE) {
if (this->reconfigure_pending) {
this->restartCapture();
this->reconfigure_pending = false;
}
}
break;
}
default:
break;
}
return true;
}

92
tools/replay/qcom/decoder.h
Unescape View File

@ -0,0 +1,92 @@
#pragma once
#include "sde_rotator.h"
#include "third_party/linux/include/media/msm_vidc.h"
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
}
#define V4L2_EVENT_MSM_VIDC_START (V4L2_EVENT_PRIVATE_START + 0x00001000)
#define V4L2_EVENT_MSM_VIDC_FLUSH_DONE (V4L2_EVENT_MSM_VIDC_START + 1)
#define V4L2_EVENT_MSM_VIDC_PORT_SETTINGS_CHANGED_INSUFFICIENT (V4L2_EVENT_MSM_VIDC_START + 3)
#define V4L2_CID_MPEG_MSM_VIDC_BASE 0x00992000
#define V4L2_CID_MPEG_VIDC_VIDEO_DPB_COLOR_FORMAT (V4L2_CID_MPEG_MSM_VIDC_BASE + 44)
#define V4L2_CID_MPEG_VIDC_VIDEO_STREAM_OUTPUT_MODE (V4L2_CID_MPEG_MSM_VIDC_BASE + 22)
#define V4L2_QCOM_CMD_FLUSH_CAPTURE (1 << 1)
#define V4L2_QCOM_CMD_FLUSH (4)
#define VIDEO_DEVICE "/dev/video32"
#define OUTPUT_BUFFER_COUNT 8
#define CAPTURE_BUFFER_COUNT 4
#define FPS 20
class MsmVidc {
public:
MsmVidc() = default;
~MsmVidc();
bool init(const char* dev, size_t width, size_t height, uint64_t codec);
VisionBuf* decodeFrame(AVPacket* pkt, VisionBuf* buf);
AVFormatContext* avctx = nullptr;
int fd = 0;
int sigfd = 0;
private:
bool initialized = false;
bool reconfigure_pending = false;
bool frame_ready = false;
VisionBuf* current_output_buf = nullptr;
VisionBuf out_buf; // Single input buffer
VisionBuf cap_bufs[CAPTURE_BUFFER_COUNT]; // Capture (output) buffers
size_t w = 1928, h = 1208;
size_t cap_height = 0, cap_width = 0;
int cap_buf_size = 0;
int out_buf_size = 0;
size_t cap_plane_off[CAPTURE_BUFFER_COUNT] = {0};
size_t cap_plane_stride[CAPTURE_BUFFER_COUNT] = {0};
bool cap_buf_flag[CAPTURE_BUFFER_COUNT] = {false};
size_t out_buf_off[OUTPUT_BUFFER_COUNT] = {0};
void* out_buf_addr[OUTPUT_BUFFER_COUNT] = {0};
bool out_buf_flag[OUTPUT_BUFFER_COUNT] = {false};
const int out_buf_cnt = OUTPUT_BUFFER_COUNT;
uint32_t cap_buf_format = 0;
const int subscriptions[2] = {
V4L2_EVENT_MSM_VIDC_FLUSH_DONE,
V4L2_EVENT_MSM_VIDC_PORT_SETTINGS_CHANGED_INSUFFICIENT
};
enum { EV_VIDEO, EV_SIGNAL, EV_COUNT };
struct pollfd pfd[EV_COUNT] = {0};
int ev[EV_COUNT] = {-1, -1};
int nfds = 0;
VisionBuf* processEvents();
bool setupOutput();
bool subscribeEvents();
bool setPlaneFormat(v4l2_buf_type type, uint32_t fourcc);
bool setFPS(uint32_t fps);
bool restartCapture();
bool queueCaptureBuffer(int i);
bool queueOutputBuffer(int i, size_t size);
bool setDBP();
bool setupPolling();
bool sendPacket(int buf_index, AVPacket* pkt);
int getBufferUnlocked();
VisionBuf* handleCapture();
int handleSignal();
bool handleOutput();
bool handleEvent();
SdeRotator rotator;
};
Write Preview
Loading…
Cancel
Save