Build openpilot with webcam support in CI (#2070)

old-commit-hash: f4e18e0b74

.github/workflows/test.yaml

@@ -10,16 +10,13 @@ env:
   CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm tmppilotci /bin/bash -c
   UNIT_TEST: coverage run --append -m unittest discover
   BUILD: |
-      # build the openpilot docker base image
       docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
       docker pull docker.io/commaai/openpilot-base:latest || true
       docker build --cache-from docker.io/commaai/openpilot-base:latest -t commaai/openpilot-base:latest -f Dockerfile.openpilot_base .
 
-      # build the final CI image
       docker pull docker.io/commaai/openpilotci:latest || true
       docker build --cache-from docker.io/commaai/openpilotci:latest -t tmppilot -f Dockerfile.openpilotci .
 
-
 jobs:
   build_release:
     name: build release
@@ -78,6 +75,27 @@ jobs:
     - name: Brew cleanup
       run: brew cleanup || true # keeps our cache small
 
+  build_webcam:
+    name: build webcam
+    runs-on: ubuntu-16.04
+    timeout-minutes: 90
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Build Docker image
+      run: |
+        docker pull docker.io/commaai/openpilotwebcamci:latest
+        docker build --cache-from docker.io/commaai/openpilotwebcamci:latest -t tmppilotwebcam -f tools/webcam/Dockerfile .
+    - name: Build openpilot
+      run: docker run --shm-size 1G --rm tmppilotwebcam /bin/sh -c "cd /tmp/openpilot && USE_WEBCAM=1 scons -j$(nproc)"
+    - name: Push to dockerhub
+      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+      run: |
+          docker login -u wmelching -p ${{ secrets.COMMA_DOCKERHUB_TOKEN}}
+          docker tag tmppilotwebcam docker.io/commaai/openpilotwebcamci:latest
+          docker push docker.io/commaai/openpilotwebcamci:latest
+  
   docker_push:
     name: docker push
     runs-on: ubuntu-16.04

SConstruct

@@ -28,7 +28,7 @@ if platform.system() == "Darwin":
 if arch == "aarch64" and TICI:
   arch = "larch64"
 
-webcam = bool(ARGUMENTS.get("use_webcam", 0))
+USE_WEBCAM = os.getenv("USE_WEBCAM") is not None
 QCOM_REPLAY = arch == "aarch64" and os.getenv("QCOM_REPLAY") is not None
 
 if arch == "aarch64" or arch == "larch64":
@@ -261,7 +261,7 @@ def abspath(x):
 
 # still needed for apks
 zmq = 'zmq'
-Export('env', 'qt_env', 'arch', 'zmq', 'SHARED', 'webcam', 'QCOM_REPLAY')
+Export('env', 'qt_env', 'arch', 'zmq', 'SHARED', 'USE_WEBCAM', 'QCOM_REPLAY')
 
 # cereal and messaging are shared with the system
 SConscript(['cereal/SConscript'])

selfdrive/camerad/SConscript

@@ -1,4 +1,4 @@
-Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'webcam', 'QCOM_REPLAY')
+Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'USE_WEBCAM', 'QCOM_REPLAY')
 
 libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', cereal, messaging, 'czmq', 'zmq', 'capnp', 'kj', visionipc, gpucommon]
 
@@ -16,7 +16,7 @@ elif arch == "larch64":
   # env.Append(CXXFLAGS = '-DNOSCREEN')
   # env.Append(CFLAGS = '-DNOSCREEN')
 else:
-  if webcam:
+  if USE_WEBCAM:
     libs += ['opencv_core', 'opencv_highgui', 'opencv_imgproc', 'opencv_videoio']
     cameras = ['cameras/camera_webcam.cc']
     env = env.Clone()

selfdrive/camerad/cameras/camera_webcam.cc

@@ -10,10 +10,14 @@
 #include "common/swaglog.h"
 #include "buffering.h"
 
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wundefined-inline"
 #include <opencv2/opencv.hpp>
 #include <opencv2/highgui.hpp>
 #include <opencv2/core.hpp>
 #include <opencv2/videoio.hpp>
+#pragma clang diagnostic pop
+
 
 extern volatile sig_atomic_t do_exit;
 
@@ -244,7 +248,6 @@ void cameras_open(MultiCameraState *s, VisionBuf *camera_bufs_rear,
                   VisionBuf *camera_bufs_front) {
   assert(camera_bufs_rear);
   assert(camera_bufs_front);
-  int err;
 
   // LOG("*** open front ***");
   camera_open(&s->front, camera_bufs_front, false);
@@ -260,7 +263,7 @@ void cameras_close(MultiCameraState *s) {
 
 void cameras_run(MultiCameraState *s) {
   set_thread_name("webcam_thread");
-
+  int err;
   pthread_t rear_thread_handle;
   err = pthread_create(&rear_thread_handle, NULL,
                         rear_thread, &s->rear);

selfdrive/camerad/main.cc

@@ -191,7 +191,12 @@ void* frontview_thread(void *arg) {
   // TODO: the loop is bad, ideally models shouldn't affect sensors
   SubMaster sm({"driverState"});
 
+  #ifdef __APPLE__
     cl_command_queue q = clCreateCommandQueue(s->context, s->device_id, 0, &err);
+  #else
+    const cl_queue_properties props[] = {0}; //CL_QUEUE_PRIORITY_KHR, CL_QUEUE_PRIORITY_HIGH_KHR, 0};
+    cl_command_queue q = clCreateCommandQueueWithProperties(s->context, s->device_id, props, &err);
+  #endif
   assert(err == 0);
 
   for (int cnt = 0; !do_exit; cnt++) {

tools/webcam/Dockerfile

@@ -0,0 +1,45 @@
+FROM commaai/openpilot-base:latest
+
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH}
+
+# install opencv
+RUN export OPENCV_VERSION='4.2.0' && \
+    sudo apt-get -y update && \
+    sudo apt-get install -y build-essential cmake && \
+    sudo apt-get install -y qt5-default libvtk6-dev && \
+    sudo apt-get install -y libdc1394-22-dev libavcodec-dev libavformat-dev libswscale-dev \
+                            libtheora-dev libvorbis-dev libxvidcore-dev libx264-dev yasm \
+                            libopencore-amrnb-dev libopencore-amrwb-dev libv4l-dev libxine2-dev && \
+    sudo apt-get install -y libtbb-dev libeigen3-dev && \
+
+    wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.tar.gz && \
+    tar -xvf ${OPENCV_VERSION}.tar.gz && rm ${OPENCV_VERSION}.tar.gz && \
+    mv opencv-${OPENCV_VERSION} OpenCV && \
+    cd OpenCV && mkdir build && cd build && \
+    cmake -DWITH_OPENGL=ON -DFORCE_VTK=ON -DWITH_TBB=ON -DWITH_GDAL=ON \
+          -DWITH_XINE=ON -DENABLE_PRECOMPILED_HEADERS=OFF .. && \
+    make -j8 && \
+    sudo make install && \
+    sudo ldconfig
+
+
+RUN mkdir -p /tmp/openpilot
+
+COPY SConstruct \
+     .pylintrc \
+     .pre-commit-config.yaml \
+     /tmp/openpilot/
+
+COPY ./pyextra /tmp/openpilot/pyextra
+COPY ./phonelibs /tmp/openpilot/phonelibs
+COPY ./laika /tmp/openpilot/laika
+COPY ./laika_repo /tmp/openpilot/laika_repo
+COPY ./rednose /tmp/openpilot/rednose
+COPY ./tools /tmp/openpilot/tools
+COPY ./release /tmp/openpilot/release
+COPY ./common /tmp/openpilot/common
+COPY ./opendbc /tmp/openpilot/opendbc
+COPY ./cereal /tmp/openpilot/cereal
+COPY ./panda /tmp/openpilot/panda
+COPY ./selfdrive /tmp/openpilot/selfdrive

tools/webcam/README.md

@@ -27,8 +27,7 @@ cd ~/openpilot
 ```
 - check out selfdrive/camerad/cameras/camera_webcam.cc lines 72 and 146 before building if any camera is upside down
 ```
-scons use_webcam=1
-touch prebuilt
+USE_WEBCAM=1 scons -j$(nproc)
 ```
 
 ## Connect the hardware