From 2cf913c407c452a37e3b997aa578cfc545b9faaf Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 7 Oct 2023 18:09:52 -0700 Subject: [PATCH] Toyota: parse FW versions for more reliable fuzzy fingerprinting (#28641) * add test from hyundai * found tss2 * abs and engine are very rarely shared (avalonh tss2 and camryh tss2 is one example) * some bad regex * some pattern work * some work * . * some conceptual clean up * fix short fw pattern * hyundai test * clean up values.py a bit * print platform codes * hyundai fuzzy * pass test * move around constants * clean up * clean up hyundai * fix print * more clean up, fix med pattern * add documentation * use major version * some clean up from merge * some clean up from merge 2 * spot check * clean up imports * missing FW_QUERY_CONFIG * short version: always prefixed with 3, get real platform code * limit to max chunks seen (3) * rm engine * fix that * get_platform_codes returns dict * tests * comments * fix test * enable a test * fix script * print ecu parts * enable old test * clean up some tests * clean up * more clean up * static * this is all it took? * add note * ... * use less ECUs * bump * todo * clean up fuzzy fp function * make deterministic in a feat of engineering * add temp exclude_fw argument for testing * fix logic * add blacklist * add platform and its matches * fix for nb * remove fw exclusion * Revert "remove fw exclusion" This reverts commit 0e3b47c5a30ebdb647caf5d792b10f517cbdf48f. * clean up * Revert "Revert "remove fw exclusion"" This reverts commit 42c55f006a4a0596ff404c67db1a88eb64b45d9e. * these two have similar chassis according to wikipedia (but mass is relatively different) * Revert "Revert "Revert "remove fw exclusion""" This reverts commit 0f874233422d2007cc87bfb1c714cabed6e80049. * Revert "Revert "Revert "Revert "remove fw exclusion"""" This reverts commit 2411967f5a2b0712773d614ee589321cca4d52a5. * oof * shadows global variable * rm comment old-commit-hash: c14b76562384e7e9b2aa129ef124f2825db082e4 --- selfdrive/car/hyundai/values.py | 2 +- selfdrive/car/toyota/tests/test_toyota.py | 25 ++++++++++++++- selfdrive/car/toyota/values.py | 37 +++++++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/selfdrive/car/hyundai/values.py b/selfdrive/car/hyundai/values.py index 5489d8cffa..110f0c540a 100644 --- a/selfdrive/car/hyundai/values.py +++ b/selfdrive/car/hyundai/values.py @@ -400,7 +400,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: # to distinguish between hybrid and ICE. All EVs so far are either exclusively # electric or specify electric in the platform code. # TODO: whitelist platforms that we've seen hybrid and ICE versions of that have these specifiers - fuzzy_platform_blacklist = {str(car) for car in set(CANFD_CAR - EV_CAR)} + fuzzy_platform_blacklist = {str(c) for c in set(CANFD_CAR - EV_CAR)} candidates: Set[str] = set() for candidate, fws in FW_VERSIONS.items(): diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index cdaef55269..2a82ca5ec0 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -3,8 +3,10 @@ from hypothesis import given, settings, strategies as st import unittest from cereal import car +from openpilot.selfdrive.car.fw_versions import build_fw_dict from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS, \ - PLATFORM_CODE_ECUS, get_platform_codes + FW_QUERY_CONFIG, PLATFORM_CODE_ECUS, FUZZY_EXCLUDED_PLATFORMS, \ + get_platform_codes Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} @@ -120,6 +122,27 @@ class TestToyotaFingerprint(unittest.TestCase): ]) self.assertEqual(results, {b"F1526-07-1": {b"10", b"40"}, b"8646F-41-04": {b"100"}, b"58-79": {b"000"}}) + def test_fuzzy_excluded_platforms(self): + # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared. + platforms_with_shared_codes = set() + for platform, fw_by_addr in FW_VERSIONS.items(): + car_fw = [] + for ecu, fw_versions in fw_by_addr.items(): + ecu_name, addr, sub_addr = ecu + for fw in fw_versions: + car_fw.append({"ecu": ecu_name, "fwVersion": fw, "address": addr, + "subAddress": 0 if sub_addr is None else sub_addr}) + + CP = car.CarParams.new_message(carFw=car_fw) + matches = FW_QUERY_CONFIG.match_fw_to_car_fuzzy(build_fw_dict(CP.carFw)) + if len(matches) == 1: + self.assertEqual(list(matches)[0], platform) + else: + # If a platform has multiple matches, add it and its matches + platforms_with_shared_codes |= {platform, *matches} + + self.assertEqual(platforms_with_shared_codes, FUZZY_EXCLUDED_PLATFORMS, (len(platforms_with_shared_codes), len(FW_VERSIONS))) + if __name__ == "__main__": unittest.main() diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index dfa3101b48..7d26ffd2e7 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -284,6 +284,40 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: return dict(codes) +def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]: + candidates = set() + + for candidate, fws in FW_VERSIONS.items(): + # Keep track of ECUs which pass all checks (platform codes, within sub-version range) + valid_found_ecus = set() + valid_expected_ecus = {ecu[1:] for ecu in fws if ecu[0] in PLATFORM_CODE_ECUS} + for ecu, expected_versions in fws.items(): + addr = ecu[1:] + # Only check ECUs expected to have platform codes + if ecu[0] not in PLATFORM_CODE_ECUS: + continue + + # Expected platform codes & versions + expected_platform_codes = get_platform_codes(expected_versions) + + # Found platform codes & versions + found_platform_codes = get_platform_codes(live_fw_versions.get(addr, set())) + + # Check part number + platform code + major version matches for any found versions + # Platform codes and major versions change for different physical parts, generation, API, etc. + # Sub-versions are incremented for minor recalls, do not need to be checked. + if not any(found_platform_code in expected_platform_codes for found_platform_code in found_platform_codes): + break + + valid_found_ecus.add(addr) + + # If all live ECUs pass all checks for candidate, add it as a match + if valid_expected_ecus.issubset(valid_found_ecus): + candidates.add(candidate) + + return {str(c) for c in (candidates - FUZZY_EXCLUDED_PLATFORMS)} + + # Regex patterns for parsing more general platform-specific identifiers from FW versions. # - Part number: Toyota part number (usually last character needs to be ignored to find a match). # Each ECU address has just one part number. @@ -311,6 +345,8 @@ FW_CHUNK_LEN = 16 # - eps: describes lateral API changes for the EPS, such as using LTA for lane keeping and rejecting LKA messages PLATFORM_CODE_ECUS = [Ecu.fwdCamera, Ecu.abs, Ecu.eps] +# These platforms have at least one platform code for all ECUs shared with another platform. +FUZZY_EXCLUDED_PLATFORMS = {CAR.LEXUS_ES_TSS2, CAR.LEXUS_RX_TSS2} # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. # Toyota diagnostic software first gets the supported data ids, then queries them one by one. @@ -382,6 +418,7 @@ FW_QUERY_CONFIG = FwQueryConfig( (Ecu.combinationMeter, 0x7c0, None), (Ecu.hvac, 0x7c4, None), ], + match_fw_to_car_fuzzy=match_fw_to_car_fuzzy, ) FW_VERSIONS = {