Merge remote-tracking branch 'upstream/master' into fuzzy-panda-carstate

.github/workflows/badges.yaml

@@ -7,7 +7,7 @@ on:
 env:
   BASE_IMAGE: openpilot-base
   DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
 
 jobs:
   badges:

.github/workflows/docs.yaml

@@ -15,7 +15,7 @@ env:
 
   BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
 jobs:
   docs:

.github/workflows/selfdrive_tests.yaml

@@ -20,11 +20,11 @@ env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
   BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
   BUILD_CL: selfdrive/test/docker_build.sh cl
 
-  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
+  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/bash -c
 
   PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0
 
@@ -175,7 +175,9 @@ jobs:
     - name: Run unit tests
       timeout-minutes: 15
       run: |
-        ${{ env.RUN }} "$PYTEST --timeout 30 -m 'not slow' && \
+        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+                        export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
+                        $PYTEST --timeout 30 -m 'not slow' && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                         ./selfdrive/ui/tests/test_translations.py && \

.github/workflows/tools_tests.yaml

@@ -17,11 +17,11 @@ env:
 
   BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
   BUILD_CL: selfdrive/test/docker_build.sh cl
 
-  RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
+  RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/bash -c
 
 
 jobs:

Dockerfile.openpilot_base

@@ -40,7 +40,9 @@ COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
 RUN cd /tmp && \
     tools/install_python_dependencies.sh && \
     rm -rf /tmp/* && \
-    rm -rf /home/$USER/.cache
+    rm -rf /home/$USER/.cache && \
+    find /home/$USER/pyenv -type d -name ".git" | xargs rm -rf && \
+    rm -rf /home/$USER/pyenv/versions/3.11.4/lib/python3.11/test
 
 USER root
 RUN sudo git config --global --add safe.directory /tmp/openpilot

Jenkinsfile

@@ -67,13 +67,14 @@ END"""
   }
 }
 
-def deviceStage(String stageName, String deviceType, List env, def steps) {
+def deviceStage(String stageName, String deviceType, List extra_env, def steps) {
   stage(stageName) {
     if (currentBuild.result != null) {
         return
     }
 
-    def extra = env.collect { "export ${it}" }.join('\n');
+    def extra = extra_env.collect { "export ${it}" }.join('\n');
+    def branch = env.BRANCH_NAME ?: 'master';
 
     docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
       lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1) {
@@ -82,6 +83,10 @@ def deviceStage(String stageName, String deviceType, List env, def steps) {
             device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
           }
           steps.each { item ->
+            if (branch != "master" && item.size() == 3 && !hasDirectoryChanged(item[2])) {
+              println "Skipped '${item[0]}', no relevant changes were detected."
+              return;
+            }
             device(device_ip, item[0], item[1])
           }
         }
@@ -104,7 +109,7 @@ def pcStage(String stageName, Closure body) {
     def openpilot_base = retryWithDelay (3, 15) {
       return docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .")
     }
-    
+
     openpilot_base.inside(dockerArgs) {
       timeout(time: 20, unit: 'MINUTES') {
         try {
@@ -134,6 +139,21 @@ def setupCredentials() {
   }
 }
 
+def hasDirectoryChanged(List<String> paths) {
+  for (change in currentBuild.changeSets) {
+    for (item in change.items) {
+      for (affectedPath in item.affectedPaths) {
+        for (path in paths) {
+          if (affectedPath.startsWith(path)) {
+            return true
+          }
+        }
+      }
+    }
+  }
+  return false
+}
+
 node {
   env.CI = "1"
   env.PYTHONWARNINGS = "error"
@@ -171,8 +191,9 @@ node {
     parallel (
       // tici tests
       'onroad tests': {
-        deviceStage("onroad", "tici-needs-can", ["SKIP_COPY=1"], [
+        deviceStage("onroad", "tici-needs-can", [], [
-          ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"],
+          // TODO: ideally, this test runs in master-ci, but it takes 5+m to build it
+          //["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"],
           ["build openpilot", "cd selfdrive/manager && ./build.py"],
           ["check dirty", "release/check-dirty.sh"],
           ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"],
@@ -182,12 +203,11 @@ node {
       'HW + Unit Tests': {
         deviceStage("tici", "tici-common", ["UNSAFE=1"], [
           ["build", "cd selfdrive/manager && ./build.py"],
-          ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"],
+          ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py", ["panda/", "selfdrive/boardd/"]],
           ["test power draw", "./system/hardware/tici/tests/test_power_draw.py"],
           ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"],
           ["test pigeond", "pytest system/sensord/tests/test_pigeond.py"],
           ["test manager", "pytest selfdrive/manager/test/test_manager.py"],
-          ["test nav", "pytest selfdrive/navd/tests/"],
         ])
       },
       'loopback': {
@@ -228,10 +248,10 @@ node {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
           ["build openpilot", "cd selfdrive/manager && ./build.py"],
           ["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"],
-          ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"],
+          ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py", ["panda/", "selfdrive/boardd/"]],
           ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"],
           ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"],
-          ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"],
+          ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", ["system/qcomgpsd/"]],
         ])
       },
 

docs/CARS.md

@@ -97,7 +97,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Hyundai|Kona Electric 2018-21|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai G connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric 2018-21">Buy Here</a></sub></details>||
 |Hyundai|Kona Electric 2022-23|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai O connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric 2022-23">Buy Here</a></sub></details>||
 |Hyundai|Kona Electric (with HDA II, Korea only) 2023[<sup>6</sup>](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai R connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric (with HDA II, Korea only) 2023">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=U2fOCmcQ8hw" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
-|Hyundai|Kona Hybrid 2020|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai I connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Hybrid 2020">Buy Here</a></sub></details>|<a href="https://youtu.be/0dwpAHiZgFo" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
+|Hyundai|Kona Hybrid 2020|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai I connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Hybrid 2020">Buy Here</a></sub></details>||
 |Hyundai|Palisade 2020-22|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Palisade 2020-22">Buy Here</a></sub></details>|<a href="https://youtu.be/TAnDqjF4fDY?t=456" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Hyundai|Santa Cruz 2022-23[<sup>6</sup>](#footnotes)|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai N connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Santa Cruz 2022-23">Buy Here</a></sub></details>||
 |Hyundai|Santa Fe 2019-20|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai D connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Santa Fe 2019-20">Buy Here</a></sub></details>|<a href="https://youtu.be/bjDR0YjM__s" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|

pyproject.toml

@@ -17,6 +17,7 @@ testpaths = [
   "selfdrive/controls",
   "selfdrive/locationd",
   "selfdrive/monitoring",
+  "selfdrive/navd/tests",
   "selfdrive/thermald",
   "selfdrive/test/longitudinal_maneuvers",
   "system/hardware/tici",

selfdrive/athena/tests/test_athenad.py

@@ -13,7 +13,6 @@ from datetime import datetime, timedelta
 from parameterized import parameterized
 from typing import Optional
 
-from pympler.tracker import SummaryTracker
 from unittest import mock
 from websocket import ABNF
 from websocket._exceptions import WebSocketConnectionClosedException
@@ -144,11 +143,6 @@ class TestAthenadMethods(unittest.TestCase):
     # random bytes to ensure rather large object post-compression
     fn = self._create_file('qlog', data=os.urandom(10000 * 1024))
 
-    # warm up object tracker
-    tracker = SummaryTracker()
-    for _ in range(5):
-      tracker.diff()
-
     upload_fn = fn + ('.bz2' if compress else '')
     item = athenad.UploadItem(path=upload_fn, url="http://localhost:1238", headers={}, created_at=int(time.time()*1000), id='')
     with self.assertRaises(requests.exceptions.ConnectionError):
@@ -158,11 +152,6 @@ class TestAthenadMethods(unittest.TestCase):
     resp = athenad._do_upload(item)
     self.assertEqual(resp.status_code, 201)
 
-    # assert memory cleaned up
-    for _type, num_objects, total_size in tracker.diff():
-      with self.subTest(_type=_type):
-        self.assertLess(total_size / 1024, 10, f'Object {_type} ({num_objects=}) grew larger than 10 kB while uploading file')
-
   @with_http_server
   def test_uploadFileToUrl(self, host):
     fn = self._create_file('qlog.bz2')

selfdrive/car/hyundai/values.py

@@ -191,8 +191,7 @@ CAR_INFO: Dict[str, Optional[Union[HyundaiCarInfo, List[HyundaiCarInfo]]]] = {
   CAR.KONA: HyundaiCarInfo("Hyundai Kona 2020", car_parts=CarParts.common([CarHarness.hyundai_b])),
   CAR.KONA_EV: HyundaiCarInfo("Hyundai Kona Electric 2018-21", car_parts=CarParts.common([CarHarness.hyundai_g])),
   CAR.KONA_EV_2022: HyundaiCarInfo("Hyundai Kona Electric 2022-23", car_parts=CarParts.common([CarHarness.hyundai_o])),
-  CAR.KONA_HEV: HyundaiCarInfo("Hyundai Kona Hybrid 2020", video_link="https://youtu.be/0dwpAHiZgFo",
+  CAR.KONA_HEV: HyundaiCarInfo("Hyundai Kona Hybrid 2020", car_parts=CarParts.common([CarHarness.hyundai_i])),  # TODO: check packages
-                               car_parts=CarParts.common([CarHarness.hyundai_i])),  # TODO: check packages
   # TODO: this is the 2024 US MY, not yet released
   CAR.KONA_EV_2ND_GEN: HyundaiCarInfo("Hyundai Kona Electric (with HDA II, Korea only) 2023", video_link="https://www.youtube.com/watch?v=U2fOCmcQ8hw",
                                       car_parts=CarParts.common([CarHarness.hyundai_r])),

selfdrive/navd/tests/test_map_renderer.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import time
 import numpy as np
 import os
 import pytest
@@ -10,7 +11,7 @@ import cereal.messaging as messaging
 
 from typing import Any
 from cereal.visionipc import VisionIpcClient, VisionStreamType
-from openpilot.selfdrive.manager.process_config import managed_processes
+from openpilot.selfdrive.test.helpers import with_processes
 
 LLK_DECIMATION = 10
 CACHE_PATH = "/data/mbgl-cache-navd.db"
@@ -18,7 +19,8 @@ CACHE_PATH = "/data/mbgl-cache-navd.db"
 LOCATION1 = (32.7174, -117.16277)
 LOCATION2 = (32.7558, -117.2037)
 
-DEFAULT_ITERATIONS = 30 * LLK_DECIMATION
+RENDER_FRAMES = 15
+DEFAULT_ITERATIONS = RENDER_FRAMES * LLK_DECIMATION
 
 LOCATION1_REPEATED = [LOCATION1] * DEFAULT_ITERATIONS
 LOCATION2_REPEATED = [LOCATION2] * DEFAULT_ITERATIONS
@@ -60,7 +62,8 @@ class MapBoxInternetDisabledRequestHandler(http.server.BaseHTTPRequestHandler):
 
 class MapBoxInternetDisabledServer(threading.Thread):
   def run(self):
-    self.server = http.server.HTTPServer(("127.0.0.1", 5000), MapBoxInternetDisabledRequestHandler)
+    self.server = http.server.HTTPServer(("127.0.0.1", 0), MapBoxInternetDisabledRequestHandler)
+    self.port = self.server.server_port
     self.server.serve_forever()
 
   def stop(self):
@@ -74,13 +77,15 @@ class MapBoxInternetDisabledServer(threading.Thread):
 
 
 class TestMapRenderer(unittest.TestCase):
-  server = MapBoxInternetDisabledServer()
+  server: MapBoxInternetDisabledServer
 
   @classmethod
   def setUpClass(cls):
     assert "MAPBOX_TOKEN" in os.environ
     cls.original_token = os.environ["MAPBOX_TOKEN"]
+    cls.server = MapBoxInternetDisabledServer()
     cls.server.start()
+    time.sleep(0.5) # wait for server to startup
 
   @classmethod
   def tearDownClass(cls) -> None:
@@ -88,7 +93,7 @@ class TestMapRenderer(unittest.TestCase):
 
   def setUp(self):
     self.server.enable_internet()
-    os.environ['MAPS_HOST'] = 'http://localhost:5000'
+    os.environ['MAPS_HOST'] = f'http://localhost:{self.server.port}'
 
     self.sm = messaging.SubMaster(['mapRenderState'])
     self.pm = messaging.PubMaster(['liveLocationKalman'])
@@ -97,15 +102,11 @@ class TestMapRenderer(unittest.TestCase):
     if os.path.exists(CACHE_PATH):
       os.remove(CACHE_PATH)
 
-  def tearDown(self):
-    managed_processes['mapsd'].stop()
-
   def _setup_test(self):
-    # start + sync up
-    managed_processes['mapsd'].start()
-
     assert self.pm.wait_for_readers_to_update("liveLocationKalman", 10)
 
+    time.sleep(0.5)
+
     assert VisionIpcClient.available_streams("navd", False) == {VisionStreamType.VISION_STREAM_MAP, }
     assert self.vipc.connect(False)
     self.vipc.recv()
@@ -164,17 +165,22 @@ class TestMapRenderer(unittest.TestCase):
       assert self.vipc.timestamp_sof == llk.logMonoTime
       assert self.vipc.frame_id == self.sm['mapRenderState'].frameId
 
+    assert frames_since_test_start >= RENDER_FRAMES
+
     return render_times
 
+  @with_processes(["mapsd"])
   def test_with_internet(self):
     self._setup_test()
     self._run_test(True)
 
+  @with_processes(["mapsd"])
   def test_with_no_internet(self):
     self.server.disable_internet()
     self._setup_test()
     self._run_test(False)
 
+  @with_processes(["mapsd"])
   def test_recover_from_no_internet(self):
     self._setup_test()
     self._run_test(True)
@@ -187,8 +193,7 @@ class TestMapRenderer(unittest.TestCase):
     self.server.enable_internet()
     self._run_test(True, LOCATION2_REPEATED)
 
-    self._run_test(True, LOCATION2_REPEATED)
+  @with_processes(["mapsd"])
-
   @pytest.mark.tici
   def test_render_time_distribution(self):
     self._setup_test()

selfdrive/test/setup_device_ci.sh

@@ -79,9 +79,7 @@ safe_checkout() {
   echo "git checkout done, t=$SECONDS"
   du -hs $SOURCE_DIR $SOURCE_DIR/.git
 
-  if [ -z "SKIP_COPY" ]; then
+  rsync -a --delete $SOURCE_DIR $TEST_DIR
-    rsync -a --delete $SOURCE_DIR $TEST_DIR
-  fi
 }
 
 unsafe_checkout() {

selfdrive/test/setup_xvfb.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Sets up a virtual display for running map renderer and simulator without an X11 display
+
+DISP_ID=99
+export DISPLAY=:$DISP_ID
+
+sudo Xvfb $DISPLAY -screen 0 2160x1080x24 &
+
+# check for x11 socket for the specified display ID
+while [ ! -S /tmp/.X11-unix/X$DISP_ID ]
+do
+  echo "Waiting for Xvfb..."
+  sleep 1
+done

selfdrive/test/test_onroad.py

@@ -168,6 +168,15 @@ class TestOnroad(unittest.TestCase):
     cls.lr = list(LogReader(os.path.join(str(cls.segments[1]), "rlog")))
     cls.log_path = cls.segments[1]
 
+    cls.log_sizes = {}
+    for f in cls.log_path.iterdir():
+      assert f.is_file()
+      cls.log_sizes[f]  = f.stat().st_size / 1e6
+      if f.name in ("qlog", "rlog"):
+        with open(f, 'rb') as ff:
+          cls.log_sizes[f] = len(bz2.compress(ff.read())) / 1e6
+
+
   @cached_property
   def service_msgs(self):
     msgs = defaultdict(list)
@@ -198,14 +207,7 @@ class TestOnroad(unittest.TestCase):
     self.assertEqual(len(big_logs), 0, f"Log spam: {big_logs}")
 
   def test_log_sizes(self):
-    for f in self.log_path.iterdir():
+    for f, sz in self.log_sizes.items():
-      assert f.is_file()
-
-      sz = f.stat().st_size / 1e6
-      if f.name in ("qlog", "rlog"):
-        with open(f, 'rb') as ff:
-          sz = len(bz2.compress(ff.read())) / 1e6
-
       if f.name == "qcamera.ts":
         assert 2.15 < sz < 2.35
       elif f.name == "qlog":

system/loggerd/uploader.py

@@ -25,7 +25,7 @@ NetworkType = log.DeviceState.NetworkType
 UPLOAD_ATTR_NAME = 'user.upload'
 UPLOAD_ATTR_VALUE = b'1'
 
-UPLOAD_QLOG_QCAM_MAX_SIZE = 100 * 1e6  # MB
+UPLOAD_QLOG_QCAM_MAX_SIZE = 5 * 1e6  # MB
 
 allow_sleep = bool(os.getenv("UPLOADER_SLEEP", "1"))
 force_wifi = os.getenv("FORCEWIFI") is not None

tools/sim/bridge/metadrive/metadrive_bridge.py

@@ -36,7 +36,8 @@ class RGBCameraWide(CopyRamRGBCamera):
     cam = self.get_cam()
     cam.setPos(C3_POSITION)
     lens = self.get_lens()
-    lens.setFov(160)
+    lens.setFov(120)
+    lens.setNear(0.1)
 
 class RGBCameraRoad(CopyRamRGBCamera):
   def __init__(self, *args, **kwargs):
@@ -45,6 +46,7 @@ class RGBCameraRoad(CopyRamRGBCamera):
     cam.setPos(C3_POSITION)
     lens = self.get_lens()
     lens.setFov(40)
+    lens.setNear(0.1)
 
 
 def straight_block(length):
@@ -116,4 +118,4 @@ class MetaDriveBridge(SimulatorBridge):
       preload_models=False
     )
 
-    return MetaDriveWorld(config)
+    return MetaDriveWorld(config, self.dual_camera)

tools/sim/bridge/metadrive/metadrive_process.py

@@ -39,10 +39,13 @@ def apply_metadrive_patches():
 
   MetaDriveEnv._is_arrive_destination = arrive_destination_patch
 
-def metadrive_process(dual_camera: bool, config: dict, camera_array, controls_recv: Connection, state_send: Connection, exit_event):
+def metadrive_process(dual_camera: bool, config: dict, camera_array, wide_camera_array, controls_recv: Connection, state_send: Connection, exit_event):
   apply_metadrive_patches()
 
   road_image = np.frombuffer(camera_array.get_obj(), dtype=np.uint8).reshape((H, W, 3))
+  if dual_camera:
+    assert wide_camera_array is not None
+    wide_road_image = np.frombuffer(wide_camera_array.get_obj(), dtype=np.uint8).reshape((H, W, 3))
 
   env = MetaDriveEnv(config)
 
@@ -92,8 +95,8 @@ def metadrive_process(dual_camera: bool, config: dict, camera_array, controls_re
       if terminated:
         reset()
 
-      #if dual_camera:
+      if dual_camera:
-      #  wide_road_image = get_cam_as_rgb("rgb_wide")
+        wide_road_image[...] = get_cam_as_rgb("rgb_wide")
       road_image[...] = get_cam_as_rgb("rgb_road")
 
     rk.keep_time()

tools/sim/bridge/metadrive/metadrive_world.py

@@ -15,6 +15,10 @@ class MetaDriveWorld(World):
     super().__init__(dual_camera)
     self.camera_array = Array(ctypes.c_uint8, W*H*3)
     self.road_image = np.frombuffer(self.camera_array.get_obj(), dtype=np.uint8).reshape((H, W, 3))
+    self.wide_camera_array = None
+    if dual_camera:
+      self.wide_camera_array = Array(ctypes.c_uint8, W*H*3)
+      self.wide_road_image = np.frombuffer(self.wide_camera_array.get_obj(), dtype=np.uint8).reshape((H, W, 3))
 
     self.controls_send, self.controls_recv = Pipe()
     self.state_send, self.state_recv = Pipe()
@@ -23,7 +27,7 @@ class MetaDriveWorld(World):
 
     self.metadrive_process = multiprocessing.Process(name="metadrive process", target=
                               functools.partial(metadrive_process, dual_camera, config,
-                                                self.camera_array, self.controls_recv, self.state_send, self.exit_event))
+                                                self.camera_array, self.wide_camera_array, self.controls_recv, self.state_send, self.exit_event))
     self.metadrive_process.start()
 
     print("----------------------------------------------------------")

tools/sim/lib/simulated_car.py

@@ -5,6 +5,7 @@ from opendbc.can.parser import CANParser
 from openpilot.selfdrive.boardd.boardd_api_impl import can_list_to_can_capnp
 from openpilot.selfdrive.car import crc8_pedal
 from openpilot.tools.sim.lib.common import SimulatorState
+from panda.python import Panda
 
 
 class SimulatedCar:
@@ -106,7 +107,8 @@ class SimulatedCar:
       'pandaType': "blackPanda",
       'controlsAllowed': True,
       'safetyModel': 'hondaNidec',
-      'alternativeExperience': self.sm["carParams"].alternativeExperience
+      'alternativeExperience': self.sm["carParams"].alternativeExperience,
+      'safetyParam': Panda.FLAG_HONDA_GAS_INTERCEPTOR
     }
     self.pm.send('pandaStates', dat)