Merge remote-tracking branch 'upstream/master' into toyota-fuzzy-v2

.devcontainer/Dockerfile

@@ -3,8 +3,8 @@ FROM ghcr.io/commaai/openpilot-base:latest
 RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux
 RUN pip install ipython jupyter jupyterlab
 
+USER batman
+
 RUN cd $HOME && \
     curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.tmux.conf && \
-    curl -O https://github.com/commaai/agnos-builder/blob/master/userspace/home/.vimrc
+    curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.vimrc
-
-ENV CARLA_HOST="host.docker.internal"

.devcontainer/container_post_create.sh

@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 
+TARGET_USER=batman
 source .devcontainer/.host/.env
 
 # override display flag for mac
 if [[ $HOST_OS == darwin ]]; then
   echo "Setting up DISPLAY override for macOS..."
-  cat <<EOF >> /root/.bashrc
+  cat <<EOF >> /home/$TARGET_USER/.bashrc
 if [ -n "\$DISPLAY" ]; then
   DISPLAY_NUM=\$(echo "\$DISPLAY" | awk -F: '{print \$NF}')
   export DISPLAY=host.docker.internal:\$DISPLAY_NUM
@@ -15,6 +16,5 @@ fi
 
 # These lines are temporary, to remain backwards compatible with old devcontainers
 # that were running as root and therefore had their caches written as root
-USER=batman
+sudo chown -R $TARGET_USER: /tmp/scons_cache
-sudo chown -R $USER: /tmp/scons_cache
+sudo chown -R $TARGET_USER: /tmp/comma_download_cache
-sudo chown -R $USER: /tmp/comma_download_cache

.devcontainer/devcontainer.json

@@ -10,6 +10,8 @@
   "containerEnv": {
     "DISPLAY": "${localEnv:DISPLAY}",
     "PYTHONPATH": "${containerWorkspaceFolder}",
+    "TERM": "xterm-256color",
+    "CARLA_HOST": "host.docker.internal",
     "force_color_prompt": "1"
   },
   "runArgs": [

.gitattributes

@@ -1,5 +1,16 @@
 *.dlc filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
-selfdrive/assets/fonts/* filter=lfs diff=lfs -text
-selfdrive/assets/training/* filter=lfs diff=lfs -text
 selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
+selfdrive/assets/fonts/*.ttf filter=lfs diff=lfs merge=lfs -text
+selfdrive/assets/training/*.png filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
+selfdrive/ui/qt/spinner_larch64 filter=lfs diff=lfs merge=lfs -text
+selfdrive/ui/qt/text_larch64 filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.dylib filter=lfs diff=lfs merge=lfs -text
+third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
+third_party/bootstrap/bootstrap-icons.svg filter=lfs diff=lfs merge=lfs -text
+third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
+third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
+third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text

.github/workflows/selfdrive_tests.yaml

@@ -38,6 +38,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
+    - run: git lfs pull
     - name: Build devel
       timeout-minutes: 1
       run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
@@ -65,7 +66,7 @@ jobs:
         cp pyproject.toml $STRIPPED_DIR
         cp poetry.lock $STRIPPED_DIR
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files pre-commit run --all"
 
   build:
     strategy:
@@ -95,6 +96,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
+    - run: git lfs pull
     - name: Determine pre-existing Homebrew packages
       if: steps.dependency-cache.outputs.cache-hit != 'true'
       run: |
@@ -198,7 +200,6 @@ jobs:
         $DOCKER_LOGIN
     - uses: ./.github/workflows/setup-with-retry
       with:
-        git_lfs: false
         docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
     - name: Build and push CL Docker image
       if: matrix.arch == 'x86_64'
@@ -230,8 +231,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - name: Build Docker image
+    - uses: ./.github/workflows/setup-with-retry
-      run: eval "$BUILD"
     - name: pre-commit
       timeout-minutes: 4
       run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all"
@@ -394,6 +394,7 @@ jobs:
         with:
           submodules: true
           ref: ${{ github.event.pull_request.base.ref }}
+      - run: git lfs pull
       - uses: ./.github/workflows/setup-with-retry
       - name: Get base car info
         run: |
@@ -403,6 +404,7 @@ jobs:
         with:
           submodules: true
           path: current
+      - run: cd current && git lfs pull
       - name: Save car docs diff
         id: save_diff
         run: |

.github/workflows/setup-with-retry/action.yaml

@@ -5,10 +5,6 @@ inputs:
     description: 'Auth token for Docker Hub, required for BuildJet jobs'
     required: false
     default: ''
-  git_lfs:
-    description: 'Whether or not to pull the git lfs'
-    required: false
-    default: 'true'
   cache_key_prefix:
     description: 'Prefix for caching key'
     required: false
@@ -26,7 +22,6 @@ runs:
       continue-on-error: true
       with:
         docker_hub_pat: ${{ inputs.docker_hub_pat }}
-        git_lfs: ${{ inputs.git_lfs }}
         cache_key_prefix: ${{ inputs.cache_key_prefix }}
         is_retried: true
     - if: steps.setup1.outcome == 'failure'
@@ -38,7 +33,6 @@ runs:
       continue-on-error: true
       with:
         docker_hub_pat: ${{ inputs.docker_hub_pat }}
-        git_lfs: ${{ inputs.git_lfs }}
         cache_key_prefix: ${{ inputs.cache_key_prefix }}
         is_retried: true
     - if: steps.setup2.outcome == 'failure'
@@ -49,6 +43,5 @@ runs:
       uses: ./.github/workflows/setup
       with:
         docker_hub_pat: ${{ inputs.docker_hub_pat }}
-        git_lfs: ${{ inputs.git_lfs }}
         cache_key_prefix: ${{ inputs.cache_key_prefix }}
         is_retried: true

.github/workflows/setup/action.yaml

@@ -5,10 +5,6 @@ inputs:
     description: 'Auth token for Docker Hub, required for BuildJet jobs'
     required: true
     default: ''
-  git_lfs:
-    description: 'Whether or not to pull the git lfs'
-    required: true
-    default: 'true'
   cache_key_prefix:
     description: 'Prefix for caching key'
     required: true
@@ -30,7 +26,6 @@ runs:
 
     # do this after checkout to ensure our custom LFS config is used to pull from GitLab
     - shell: bash
-      if: ${{ inputs.git_lfs == 'true' }}
       run: git lfs pull
 
     # on BuildJet runners, must be logged into DockerHub to avoid rate limiting

.github/workflows/tools_tests.yaml

@@ -72,8 +72,6 @@ jobs:
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
-      with:
-        git_lfs: false
     - name: Setup to push to repo
       if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
       run: |
@@ -91,8 +89,6 @@ jobs:
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
-      with:
-        git_lfs: false
     - name: Use local image for testing devcontainer with latest base image
       run: |
         echo "USE_LOCAL_IMAGE=true" >> "$GITHUB_ENV"

.pre-commit-config.yaml

@@ -19,7 +19,9 @@ repos:
     -   id: check-executables-have-shebangs
     -   id: check-shebang-scripts-are-executable
     -   id: check-added-large-files
-        args: ['--maxkb=100']
+        args:
+        - --maxkb=500
+        - --enforce-all
 -   repo: https://github.com/codespell-project/codespell
     rev: v2.2.5
     hooks:

selfdrive/assets/fonts/.gitattributes

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9b19422590f5a1995f18f5097aa43f1d721e15a3980aa23236f612511bbb1247
-size 542

selfdrive/assets/training/.gitattributes

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:949bc01262fb0c894157a2f0d2bb01297a276c11acf55460d4c87c84b7832af4
-size 883

selfdrive/car/hyundai/values.py

@@ -182,7 +182,9 @@ CAR_INFO: Dict[str, Optional[Union[HyundaiCarInfo, List[HyundaiCarInfo]]]] = {
   CAR.KONA_EV_2022: HyundaiCarInfo("Hyundai Kona Electric 2022-23", car_parts=CarParts.common([CarHarness.hyundai_o])),
   CAR.KONA_HEV: HyundaiCarInfo("Hyundai Kona Hybrid 2020", video_link="https://youtu.be/0dwpAHiZgFo",
                                car_parts=CarParts.common([CarHarness.hyundai_i])),  # TODO: check packages
-  CAR.KONA_EV_2ND_GEN: HyundaiCarInfo("Hyundai Kona Electric (with HDA II, Korea only) 2023", video_link="https://www.youtube.com/watch?v=U2fOCmcQ8hw", car_parts=CarParts.common([CarHarness.hyundai_r])),
+  # TODO: this is the 2024 US MY, not yet released
+  CAR.KONA_EV_2ND_GEN: HyundaiCarInfo("Hyundai Kona Electric (with HDA II, Korea only) 2023", video_link="https://www.youtube.com/watch?v=U2fOCmcQ8hw",
+                                      car_parts=CarParts.common([CarHarness.hyundai_r])),
   CAR.SANTA_FE: HyundaiCarInfo("Hyundai Santa Fe 2019-20", "All", car_parts=CarParts.common([CarHarness.hyundai_d])),
   CAR.SANTA_FE_2022: HyundaiCarInfo("Hyundai Santa Fe 2021-23", "All", video_link="https://youtu.be/VnHzSTygTS4",
                                     car_parts=CarParts.common([CarHarness.hyundai_l])),

selfdrive/car/toyota/values.py

@@ -1381,17 +1381,21 @@ FW_VERSIONS = {
   CAR.LEXUS_IS_TSS2: {
     (Ecu.engine, 0x700, None): [
       b'\x018966353S1000\x00\x00\x00\x00',
+      b'\x018966353S2000\x00\x00\x00\x00',
     ],
     (Ecu.abs, 0x7b0, None): [
+      b'\x01F15265337200\x00\x00\x00\x00',
       b'\x01F15265342000\x00\x00\x00\x00',
     ],
     (Ecu.eps, 0x7a1, None): [
       b'8965B53450\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.fwdRadar, 0x750, 0xf): [
+      b'\x018821F6201200\x00\x00\x00\x00',
       b'\x018821F6201300\x00\x00\x00\x00',
     ],
     (Ecu.fwdCamera, 0x750, 0x6d): [
+      b'\x028646F5303300\x00\x00\x00\x008646G5301200\x00\x00\x00\x00',
       b'\x028646F5303400\x00\x00\x00\x008646G3304000\x00\x00\x00\x00',
     ],
   },

selfdrive/debug/profiling/simpleperf/eon_perf.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-set -e
-
-cd "$( dirname "${BASH_SOURCE[0]}" )"
-
-ssh "$1" '$HOME/one/external/simpleperf/bin/android/arm64/simpleperf record --call-graph fp -a --duration 10 -o /tmp/perf.data'
-scp "$1":/tmp/perf.data /tmp/perf.data
-python2 report_html.py -i /tmp/perf.data -o /tmp/report.html

selfdrive/debug/profiling/simpleperf/get.txt

@@ -1,3 +0,0 @@
-git clone https://android.googlesource.com/platform/prebuilts/simpleperf
-git reset --hard 311a9d2cd27841498fc90a0b26a755deb47e7ebd
-cp -r report_html.* simpleperf_report_lib.py utils.py inferno lib ~/one/external/simpleperf/

tools/replay/replay.cc

@@ -321,13 +321,12 @@ void Replay::mergeSegments(const SegmentMap::iterator &begin, const SegmentMap::
 void Replay::startStream(const Segment *cur_segment) {
   const auto &events = cur_segment->log->events;
 
-  // get route start time from initData
+  // each segment has an INIT_DATA
-  auto it = std::find_if(events.begin(), events.end(), [](auto e) { return e->which == cereal::Event::Which::INIT_DATA; });
+  route_start_ts_ = events.front()->mono_time;
-  route_start_ts_ = it != events.end() ? (*it)->mono_time : events[0]->mono_time;
+  cur_mono_time_ += route_start_ts_ - 1;
-  cur_mono_time_ += route_start_ts_;
 
   // write CarParams
-  it = std::find_if(events.begin(), events.end(), [](auto e) { return e->which == cereal::Event::Which::CAR_PARAMS; });
+  auto it = std::find_if(events.begin(), events.end(), [](auto e) { return e->which == cereal::Event::Which::CAR_PARAMS; });
   if (it != events.end()) {
     car_fingerprint_ = (*it)->event.getCarParams().getCarFingerprint();
     capnp::MallocMessageBuilder builder;

tools/replay/tests/test_replay.cc

@@ -126,7 +126,6 @@ std::string download_demo_route() {
       std::string log_path = util::string_format("%s/%s--%d/", data_dir.c_str(), route_name.c_str(), i);
       util::create_directories(log_path, 0755);
       REQUIRE(download_to_file(remote_route.at(i).rlog.toStdString(), log_path + "rlog.bz2"));
-      REQUIRE(download_to_file(remote_route.at(i).road_cam.toStdString(), log_path + "fcamera.hevc"));
       REQUIRE(download_to_file(remote_route.at(i).driver_cam.toStdString(), log_path + "dcamera.hevc"));
       REQUIRE(download_to_file(remote_route.at(i).wide_road_cam.toStdString(), log_path + "ecamera.hevc"));
       REQUIRE(download_to_file(remote_route.at(i).qcamera.toStdString(), log_path + "qcamera.ts"));
@@ -137,10 +136,9 @@ std::string download_demo_route() {
 }
 
 
-TEST_CASE("Route") {
+TEST_CASE("Local route") {
   std::string data_dir = download_demo_route();
 
-  SECTION("Local route") {
   auto flags = GENERATE(REPLAY_FLAG_DCAM | REPLAY_FLAG_ECAM, REPLAY_FLAG_QCAMERA);
   Route route(DEMO_ROUTE, QString::fromStdString(data_dir));
   REQUIRE(route.load());
@@ -148,8 +146,9 @@ TEST_CASE("Route") {
   for (int i = 0; i < route.segments().size(); ++i) {
     read_segment(i, route.at(i), flags);
   }
-  };
+}
-  SECTION("Remote route") {
+
+TEST_CASE("Remote route") {
   auto flags = GENERATE(REPLAY_FLAG_DCAM | REPLAY_FLAG_ECAM, REPLAY_FLAG_QCAMERA);
   Route route(DEMO_ROUTE);
   REQUIRE(route.load());
@@ -157,7 +156,6 @@ TEST_CASE("Route") {
   for (int i = 0; i < 2; ++i) {
     read_segment(i, route.at(i), flags);
   }
-  };
 }
 
 // helper class for unit tests

tools/sim/bridge/common.py

@@ -32,7 +32,7 @@ class SimulatorBridge(ABC):
     set_params_enabled()
     self.params = Params()
 
-    self.rk = Ratekeeper(100)
+    self.rk = Ratekeeper(100, None)
 
     msg = messaging.new_message('liveCalibration')
     msg.liveCalibration.validBlocks = 20
@@ -90,8 +90,6 @@ class SimulatorBridge(ABC):
                                                                         100, self._exit_event))
     self.simulated_car_thread.start()
 
-    rk = Ratekeeper(100, print_delay_threshold=None)
-
     # Simulation tends to be slow in the initial steps. This prevents lagging later
     for _ in range(20):
       self.world.tick()
@@ -162,4 +160,4 @@ class SimulatorBridge(ABC):
 
       self.started = True
 
-      rk.keep_time()
+      self.rk.keep_time()

tools/sim/launch_openpilot.sh

@@ -6,7 +6,7 @@ export SIMULATION="1"
 export SKIP_FW_QUERY="1"
 export FINGERPRINT="HONDA CIVIC 2016"
 
-export BLOCK="camerad,loggerd,encoderd,micd,logmessaged"
+export BLOCK="${BLOCK},camerad,loggerd,encoderd,micd,logmessaged"
 if [[ "$CI" ]]; then
   # TODO: offscreen UI should work
   export BLOCK="${BLOCK},ui"