Merge branch 'incognitojam/ford-mach-e-fw' into ford-platform-codes

2 years ago · 772e31208e
parent a5553de1df 491bcec108
commit 772e31208e
12 changed files with 102 additions and 58 deletions
--- a/docs/CARS.md
+++ b/docs/CARS.md
@ -28,7 +28,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Chevrolet|Volt 2017-18[<sup>4</sup>](#footnotes)|Adaptive Cruise Control (ACC)|openpilot|0 mph|7 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 OBD-II connector<br>- 1 comma 3X<br>- 2 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chevrolet&model=Volt 2017-18">Buy Here</a></sub></details>|<a href="https://youtu.be/QeMCN_4TFfQ" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Chrysler|Pacifica 2017-18|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica 2017-18">Buy Here</a></sub></details>||
 |Chrysler|Pacifica 2019-20|Adaptive Cruise Control (ACC)|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica 2019-20">Buy Here</a></sub></details>||
-|Chrysler|Pacifica 2021|All|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica 2021">Buy Here</a></sub></details>||
+|Chrysler|Pacifica 2021-23|All|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica 2021-23">Buy Here</a></sub></details>||
 |Chrysler|Pacifica Hybrid 2017-18|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica Hybrid 2017-18">Buy Here</a></sub></details>||
 |Chrysler|Pacifica Hybrid 2019-23|Adaptive Cruise Control (ACC)|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica Hybrid 2019-23">Buy Here</a></sub></details>||
 |comma|body|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|None||
--- a/selfdrive/car/chrysler/fingerprints.py
+++ b/selfdrive/car/chrysler/fingerprints.py
@ -62,6 +62,29 @@ FINGERPRINTS = {
 }
 FW_VERSIONS = {
  CAR.PACIFICA_2020: {
    (Ecu.combinationMeter, 0x742, None): [
      b'68594993AB',
    ],
    (Ecu.srs, 0x744, None): [
      b'68526663AB',
    ],
    (Ecu.abs, 0x747, None): [
      b'68593395AA',
    ],
    (Ecu.fwdRadar, 0x753, None): [
      b'68598670AB',
    ],
    (Ecu.eps, 0x75a, None): [
      b'68594340AB',
    ],
    (Ecu.engine, 0x7e0, None): [
      b'68700306AB ',
    ],
    (Ecu.transmission, 0x7e1, None): [
      b'68586231AD',
    ],
  },
  CAR.JEEP_GRAND_CHEROKEE_2019: {
    (Ecu.combinationMeter, 0x742, None): [
      b'68402971AD',
--- a/selfdrive/car/chrysler/values.py
+++ b/selfdrive/car/chrysler/values.py
@ -70,7 +70,7 @@ CAR_INFO: Dict[str, Optional[Union[ChryslerCarInfo, List[ChryslerCarInfo]]]] = {
  CAR.PACIFICA_2018: ChryslerCarInfo("Chrysler Pacifica 2017-18"),
  CAR.PACIFICA_2020: [
    ChryslerCarInfo("Chrysler Pacifica 2019-20"),
-    ChryslerCarInfo("Chrysler Pacifica 2021", package="All"),
+    ChryslerCarInfo("Chrysler Pacifica 2021-23", package="All"),
  ],
  CAR.JEEP_GRAND_CHEROKEE: ChryslerCarInfo("Jeep Grand Cherokee 2016-18", video_link="https://www.youtube.com/watch?v=eLR9o2JkuRk"),
  CAR.JEEP_GRAND_CHEROKEE_2019: ChryslerCarInfo("Jeep Grand Cherokee 2019-21", video_link="https://www.youtube.com/watch?v=jBe4lWnRSu4"),
--- a/selfdrive/car/ford/fingerprints.py
+++ b/selfdrive/car/ford/fingerprints.py
@ -145,6 +145,7 @@ FW_VERSIONS = {
      b'ML3T-14H102-ABS\x00\x00\x00\x00\x00\x00\x00\x00\x00',
    ],
    (Ecu.engine, 0x7e0, None): [
      b'MJ98-14C204-BBP\x00\x00\x00\x00\x00\x00\x00\x00\x00',
      b'MJ98-14C204-BBS\x00\x00\x00\x00\x00\x00\x00\x00\x00',
      b'NJ98-14C204-VH\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00',
    ],
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@ -105,11 +105,14 @@ def match_fw_to_car_fuzzy(live_fw_versions, match_brand=None, log=True, exclude=
    return set()
-def match_fw_to_car_exact(live_fw_versions, match_brand=None, log=True) -> Set[str]:
+def match_fw_to_car_exact(live_fw_versions, match_brand=None, log=True, extra_fw_versions=None) -> Set[str]:
  """Do an exact FW match. Returns all cars that match the given
  FW versions for a list of "essential" ECUs. If an ECU is not considered
  essential the FW version can be missing to get a fingerprint, but if it's present it
  needs to match the database."""
  if extra_fw_versions is None:
    extra_fw_versions = {}
  invalid = set()
  candidates = {c: f for c, f in FW_VERSIONS.items() if
                is_brand(MODEL_TO_BRAND[c], match_brand)}
@ -117,6 +120,7 @@ def match_fw_to_car_exact(live_fw_versions, match_brand=None, log=True) -> Set[s
  for candidate, fws in candidates.items():
    config = FW_QUERY_CONFIGS[MODEL_TO_BRAND[candidate]]
    for ecu, expected_versions in fws.items():
      expected_versions = expected_versions + extra_fw_versions.get(candidate, {}).get(ecu, [])
      ecu_type = ecu[0]
      addr = ecu[1:]
--- a/selfdrive/car/nissan/values.py
+++ b/selfdrive/car/nissan/values.py
@ -59,27 +59,35 @@ NISSAN_VERSION_RESPONSE_KWP = b'\x61\x83'
 NISSAN_RX_OFFSET = 0x20
 FW_QUERY_CONFIG = FwQueryConfig(
-  requests=[
+  requests=[request for bus, logging in ((0, True), (1, False)) for request in [
    Request(
      [NISSAN_DIAGNOSTIC_REQUEST_KWP, NISSAN_VERSION_REQUEST_KWP],
      [NISSAN_DIAGNOSTIC_RESPONSE_KWP, NISSAN_VERSION_RESPONSE_KWP],
      bus=bus,
      logging=logging,
    ),
    Request(
      [NISSAN_DIAGNOSTIC_REQUEST_KWP, NISSAN_VERSION_REQUEST_KWP],
      [NISSAN_DIAGNOSTIC_RESPONSE_KWP, NISSAN_VERSION_RESPONSE_KWP],
      rx_offset=NISSAN_RX_OFFSET,
      bus=bus,
      logging=logging,
    ),
    # Rogue's engine solely responds to this
    Request(
      [NISSAN_DIAGNOSTIC_REQUEST_KWP_2, NISSAN_VERSION_REQUEST_KWP],
      [NISSAN_DIAGNOSTIC_RESPONSE_KWP_2, NISSAN_VERSION_RESPONSE_KWP],
      bus=bus,
      logging=logging,
    ),
    Request(
      [StdQueries.MANUFACTURER_SOFTWARE_VERSION_REQUEST],
      [StdQueries.MANUFACTURER_SOFTWARE_VERSION_RESPONSE],
      rx_offset=NISSAN_RX_OFFSET,
      bus=bus,
      logging=logging,
    ),
-  ],
+  ]],
 )
 DBC = {
--- a/selfdrive/car/tests/test_fw_fingerprint.py
+++ b/selfdrive/car/tests/test_fw_fingerprint.py
@ -238,7 +238,7 @@ class TestFwFingerprintTiming(unittest.TestCase):
  @pytest.mark.timeout(60)
  def test_fw_query_timing(self):
-    total_ref_time = 6.1
+    total_ref_time = 6.5
    brand_ref_times = {
      1: {
        'body': 0.1,
@ -247,7 +247,7 @@ class TestFwFingerprintTiming(unittest.TestCase):
        'honda': 0.45,
        'hyundai': 0.65,
        'mazda': 0.2,
-        'nissan': 0.4,
+        'nissan': 0.8,
        'subaru': 0.45,
        'tesla': 0.2,
        'toyota': 1.6,
--- a/selfdrive/debug/test_fw_query_on_routes.py
+++ b/selfdrive/debug/test_fw_query_on_routes.py
@ -6,8 +6,8 @@ import argparse
 import os
 import traceback
 from tqdm import tqdm
-from openpilot.tools.lib.logreader import LogReader
+from openpilot.tools.lib.logreader import LogReader, ReadMode
-from openpilot.tools.lib.route import Route
+from openpilot.tools.lib.route import SegmentRange
 from openpilot.selfdrive.car.car_helpers import interface_names
 from openpilot.selfdrive.car.fw_versions import VERSIONS, match_fw_to_car
@ -44,23 +44,14 @@ if __name__ == "__main__":
  dongles = []
  for route in tqdm(routes):
-    route = route.rstrip()
+    dongle_id = SegmentRange(route).dongle_id
    dongle_id, time = route.split('|')
    if dongle_id in dongles:
      continue
-    if NO_API:
+    lr = LogReader(route, default_mode=ReadMode.QLOG)
      qlog_path = f"cd:/{dongle_id}/{time}/0/qlog.bz2"
    else:
      route = Route(route)
      qlog_path = next((p for p in route.qlog_paths() if p is not None), None)
    if qlog_path is None:
      continue
    try:
      lr = LogReader(qlog_path)
      dongles.append(dongle_id)
      CP = None
@ -98,13 +89,11 @@ if __name__ == "__main__":
            if len(fuzzy_matches) == 1:
              if list(fuzzy_matches)[0] != live_fingerprint:
                wrong_fuzzy += 1
                print(f"{dongle_id}|{time}")
                print("Fuzzy match wrong! Fuzzy:", fuzzy_matches, "Live:", live_fingerprint)
              else:
                good_fuzzy += 1
            break
          print(f"{dongle_id}|{time}")
          print("Old style:", live_fingerprint, "Vin", CP.carVin)
          print("New style (exact):", exact_matches)
          print("New style (fuzzy):", fuzzy_matches)
--- a/tools/car_porting/auto_fingerprint.py
+++ b/tools/car_porting/auto_fingerprint.py
@ -7,7 +7,7 @@ from openpilot.selfdrive.debug.format_fingerprints import format_brand_fw_versio
 from openpilot.selfdrive.car.fw_versions import match_fw_to_car
 from openpilot.selfdrive.car.interfaces import get_interface_attr
-from openpilot.tools.lib.logreader import LogReader, ReadMode
+from openpilot.tools.lib.logreader import LogReader, ReadMode, get_first_message
 ALL_FW_VERSIONS = get_interface_attr("FW_VERSIONS")
@ -32,16 +32,15 @@ if __name__ == "__main__":
  platform: Optional[str] = None
-  for msg in lr:
+  CP = get_first_message(lr, "carParams")
    if msg.which() == "carParams":
      carFw = msg.carParams.carFw
      carVin = msg.carParams.carVin
      carPlatform = msg.carParams.carFingerprint
      break
-  if carFw is None:
+  if CP is None:
    raise Exception("No fw versions in the provided route...")
  carFw = CP.carParams.carFw
  carVin = CP.carParams.carVin
  carPlatform = CP.carParams.carFingerprint
  if args.platform is None: # attempt to auto-determine platform with other fuzzy fingerprints
    _, possible_platforms = match_fw_to_car(carFw, log=False)
--- a/tools/car_porting/examples/ford_vin_fingerprint.ipynb
+++ b/tools/car_porting/examples/ford_vin_fingerprint.ipynb
@ -2,9 +2,17 @@
 "cells": [
  {
   "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
   "metadata": {},
-   "outputs": [],
+   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /mnt/c/Users/jnewb/AppData/Local/Programs/Microsoft VS Code\n"
     ]
    }
   ],
   "source": [
    "\"\"\"In this example, we use the public comma car segments database to check if vin fingerprinting is feasible for ford.\"\"\"\n",
    "\n",
@ -19,7 +27,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
@ -54,7 +62,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
@ -66,7 +74,7 @@
    }
   ],
   "source": [
-    "from openpilot.tools.lib.logreader import comma_car_segments_source, get_first_message\n",
+    "from openpilot.tools.lib.logreader import get_first_message\n",
    "\n",
    "\n",
    "VINS_TO_CHECK = set()\n",
@ -77,35 +85,35 @@
    "    continue\n",
    "\n",
    "  for segment in database[platform]:\n",
-    "    lr = LogReader(segment, default_source=comma_car_segments_source)\n",
+    "    lr = LogReader(segment)\n",
    "    CP = get_first_message(lr, \"carParams\").carParams\n",
    "    VINS_TO_CHECK.add((CP.carVin, CP.carFingerprint))"
   ]
  },
  {
   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vin: 3FTTW8E31PRXXXXXX real platform: FORD MAVERICK 1ST GEN              determined platform: mock                              correct: False\n",
      "vin: 1FM5K8GC7LGXXXXXX real platform: FORD EXPLORER 6TH GEN              determined platform: mock                              correct: False\n",
-      "vin: 00000000000XXXXXX real platform: FORD ESCAPE 4TH GEN                determined platform: mock                              correct: False\n",
+      "vin: 3FTTW8E99NRXXXXXX real platform: FORD MAVERICK 1ST GEN              determined platform: mock                              correct: False\n",
-      "vin: 3FTTW8F98NRXXXXXX real platform: FORD MAVERICK 1ST GEN              determined platform: mock                              correct: False\n",
+      "vin: 1FM5K8GC7NGXXXXXX real platform: FORD EXPLORER 6TH GEN              determined platform: mock                              correct: False\n",
      "vin: 1FTVW1EL4NWXXXXXX real platform: FORD F-150 LIGHTNING 1ST GEN       determined platform: FORD F-150 LIGHTNING 1ST GEN      correct: True\n",
      "vin: 1FM5K7LC0MGXXXXXX real platform: FORD EXPLORER 6TH GEN              determined platform: mock                              correct: False\n",
      "vin: 1FM5K8HC7MGXXXXXX real platform: FORD EXPLORER 6TH GEN              determined platform: mock                              correct: False\n",
      "vin: 3FMTK3SU0MMXXXXXX real platform: FORD MUSTANG MACH-E 1ST GEN        determined platform: FORD MUSTANG MACH-E 1ST GEN       correct: True\n",
      "vin: WF0NXXGCHNJXXXXXX real platform: FORD FOCUS 4TH GEN                 determined platform: mock                              correct: False\n",
-      "vin: 1FMCU9J94MUXXXXXX real platform: FORD ESCAPE 4TH GEN                determined platform: mock                              correct: False\n",
+      "vin: 1FTVW1EL4NWXXXXXX real platform: FORD F-150 LIGHTNING 1ST GEN       determined platform: FORD F-150 LIGHTNING 1ST GEN      correct: True\n",
      "vin: 00000000000XXXXXX real platform: FORD ESCAPE 4TH GEN                determined platform: mock                              correct: False\n",
      "vin: 3FTTW8F98NRXXXXXX real platform: FORD MAVERICK 1ST GEN              determined platform: mock                              correct: False\n",
      "vin: 5LM5J7XC9LGXXXXXX real platform: FORD EXPLORER 6TH GEN              determined platform: mock                              correct: False\n",
      "vin: 3FMCR9B69NRXXXXXX real platform: FORD BRONCO SPORT 1ST GEN          determined platform: mock                              correct: False\n",
      "vin: 3FMTK3SU0MMXXXXXX real platform: FORD MUSTANG MACH-E 1ST GEN        determined platform: FORD MUSTANG MACH-E 1ST GEN       correct: True\n",
      "vin: 1FM5K8HC7MGXXXXXX real platform: FORD EXPLORER 6TH GEN              determined platform: mock                              correct: False\n",
      "vin: 1FM5K8GC7NGXXXXXX real platform: FORD EXPLORER 6TH GEN              determined platform: mock                              correct: False\n",
      "vin: 5LM5J7XC8MGXXXXXX real platform: FORD EXPLORER 6TH GEN              determined platform: mock                              correct: False\n",
-      "vin: 3FTTW8E31PRXXXXXX real platform: FORD MAVERICK 1ST GEN              determined platform: mock                              correct: False\n",
+      "vin: 3FMCR9B69NRXXXXXX real platform: FORD BRONCO SPORT 1ST GEN          determined platform: mock                              correct: False\n",
-      "vin: 3FTTW8E99NRXXXXXX real platform: FORD MAVERICK 1ST GEN              determined platform: mock                              correct: False\n"
+      "vin: 1FMCU9J94MUXXXXXX real platform: FORD ESCAPE 4TH GEN                determined platform: mock                              correct: False\n"
     ]
    }
   ],
--- a/tools/lib/logreader.py
+++ b/tools/lib/logreader.py
@ -12,7 +12,7 @@ import sys
 import urllib.parse
 import warnings
-from typing import Iterable, Iterator, List, Type
+from typing import Dict, Iterable, Iterator, List, Type
 from urllib.parse import parse_qs, urlparse
 from cereal import log as capnp_log
@ -129,7 +129,7 @@ def comma_api_source(sr: SegmentRange, mode: ReadMode):
  route = Route(sr.route_name)
  rlog_paths = [route.log_paths()[seg] for seg in segs]
-  qlog_paths = [route.log_paths()[seg] for seg in segs]
+  qlog_paths = [route.qlog_paths()[seg] for seg in segs]
  return apply_strategy(mode, rlog_paths, qlog_paths)
@ -173,7 +173,7 @@ def check_source(source, *args):
 def auto_source(*args):
  # Automatically determine viable source
-  for source in [internal_source, openpilotci_source]:
+  for source in [comma_car_segments_source, internal_source, openpilotci_source]:
    valid, ret = check_source(source, *args)
    if valid:
      return ret
@ -232,20 +232,25 @@ class LogReader:
    self.sort_by_time = sort_by_time
    self.only_union_types = only_union_types
    self.__lrs: Dict[int, _LogFileReader] = {}
    self.reset()
  def _get_lr(self, i):
    if i not in self.__lrs:
      self.__lrs[i] = _LogFileReader(self.logreader_identifiers[i])
    return self.__lrs[i]
  def __iter__(self):
-    for identifier in self.logreader_identifiers:
+    for i in range(len(self.logreader_identifiers)):
-      yield from _LogFileReader(identifier)
+      yield from self._get_lr(i)
-  def _run_on_segment(self, func, identifier):
+  def _run_on_segment(self, func, i):
-    lr = _LogFileReader(identifier)
+    return func(self._get_lr(i))
    return func(lr)
  def run_across_segments(self, num_processes, func):
    with multiprocessing.Pool(num_processes) as pool:
      ret = []
-      for p in pool.map(partial(self._run_on_segment, func), self.logreader_identifiers):
+      for p in pool.map(partial(self._run_on_segment, func), range(len(self.logreader_identifiers))):
        ret.extend(p)
      return ret
--- a/tools/lib/tests/test_logreader.py
+++ b/tools/lib/tests/test_logreader.py
@ -3,8 +3,11 @@ import tempfile
 import numpy as np
 import unittest
 import pytest
 from parameterized import parameterized
 import requests
 from parameterized import parameterized
 from unittest import mock
 from openpilot.tools.lib.logreader import LogReader, parse_indirect, parse_slice, ReadMode
 from openpilot.tools.lib.route import SegmentRange
@ -104,11 +107,15 @@ class TestLogReader(unittest.TestCase):
    self.assertEqual(qlog_len*2, qlog_len_2)
  @pytest.mark.slow
-  def test_multiple_iterations(self):
+  @mock.patch("openpilot.tools.lib.logreader._LogFileReader")
  def test_multiple_iterations(self, init_mock):
    lr = LogReader(f"{TEST_ROUTE}/0/q")
    qlog_len1 = len(list(lr))
    qlog_len2 = len(list(lr))
    # ensure we don't create multiple instances of _LogFileReader, which means downloading the files twice
    self.assertEqual(init_mock.call_count, 1)
    self.assertEqual(qlog_len1, qlog_len2)