From c3d0bf7b5e09de43484613e98a02f2606642bbd0 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 23 Sep 2023 02:13:58 -0700 Subject: [PATCH] Toyota: improve platform code understanding (#30015) * return a dict where minor version is not in keys * limit valid chunks to 3 (max seen) * First short version character is always 3 (we were using wrong platform code) * docs updates * not here * fixes for printing new platform code format * ecu notes * notes * platform code tests * no tuple * can visualize the whole thing now * make it clear there's no major versions make it clear there's no major versions * static analysis * two minor versions * fix * not using dsu * comment * comment * comment * forgot this one --- .../car/toyota/tests/print_platform_codes.py | 21 ++++- selfdrive/car/toyota/tests/test_toyota.py | 78 +++++++++++++++++-- selfdrive/car/toyota/values.py | 44 ++++++----- 3 files changed, 113 insertions(+), 30 deletions(-) diff --git a/selfdrive/car/toyota/tests/print_platform_codes.py b/selfdrive/car/toyota/tests/print_platform_codes.py index 94badc5cde..636a013242 100755 --- a/selfdrive/car/toyota/tests/print_platform_codes.py +++ b/selfdrive/car/toyota/tests/print_platform_codes.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +from collections import defaultdict from cereal import car from openpilot.selfdrive.car.toyota.values import FW_VERSIONS, PLATFORM_CODE_ECUS, get_platform_codes @@ -6,6 +7,8 @@ Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} if __name__ == "__main__": + parts_for_ecu: dict = defaultdict(set) + cars_for_code: dict = defaultdict(lambda: defaultdict(set)) for car_model, ecus in FW_VERSIONS.items(): print() print(car_model) @@ -14,8 +17,18 @@ if __name__ == "__main__": continue platform_codes = get_platform_codes(ecus[ecu]) - codes = {code for code, _ in platform_codes} - dates = {date for _, date in platform_codes if date is not None} + parts_for_ecu[ecu] |= {code.split(b'-')[0] for code in platform_codes if code.count(b'-') > 1} + for code in platform_codes: + cars_for_code[ecu][b'-'.join(code.split(b'-')[:2])] |= {car_model} print(f' (Ecu.{ECU_NAME[ecu[0]]}, {hex(ecu[1])}, {ecu[2]}):') - print(f' Codes: {codes}') - print(f' Versions: {dates}') + print(f' Codes: {platform_codes}') + + print('\nECU parts:') + for ecu, parts in parts_for_ecu.items(): + print(f' (Ecu.{ECU_NAME[ecu[0]]}, {hex(ecu[1])}, {ecu[2]}): {parts}') + + print('\nCar models vs. platform codes (no major versions):') + for ecu, codes in cars_for_code.items(): + print(f' (Ecu.{ECU_NAME[ecu[0]]}, {hex(ecu[1])}, {ecu[2]}):') + for code, cars in codes.items(): + print(f' {code!r}: {sorted(cars)}') diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index 0c503ebbf0..5241a95a99 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -4,7 +4,7 @@ import unittest from cereal import car from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS, \ - get_platform_codes + PLATFORM_CODE_ECUS, get_platform_codes Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} @@ -43,6 +43,8 @@ class TestToyotaInterfaces(unittest.TestCase): class TestToyotaFingerprint(unittest.TestCase): + # Tests for part numbers, platform codes, and sub-versions which Toyota will use to fuzzy + # fingerprint in the absence of full FW matches: @settings(max_examples=100) @given(data=st.data()) def test_platform_codes_fuzzy_fw(self, data): @@ -50,13 +52,73 @@ class TestToyotaFingerprint(unittest.TestCase): fws = data.draw(fw_strategy) get_platform_codes(fws) - def test_fw_pattern(self): - """Asserts all ECUs can be parsed""" - for ecus in FW_VERSIONS.values(): - for fws in ecus.values(): - for fw in fws: - ret = get_platform_codes([fw]) - self.assertTrue(len(ret)) + def test_platform_code_ecus_available(self): + # Asserts ECU keys essential for fuzzy fingerprinting are available on all platforms + for car_model, ecus in FW_VERSIONS.items(): + with self.subTest(car_model=car_model): + for platform_code_ecu in PLATFORM_CODE_ECUS: + if platform_code_ecu == Ecu.eps and car_model in (CAR.PRIUS_V, CAR.LEXUS_CTH,): + continue + if platform_code_ecu == Ecu.abs and car_model in (CAR.ALPHARD_TSS2,): + continue + self.assertIn(platform_code_ecu, [e[0] for e in ecus]) + + def test_fw_format(self): + # Asserts: + # - every supported ECU FW version returns one platform code + # - every supported ECU FW version has a part number + # - expected parsing of ECU sub-versions + + for car_model, ecus in FW_VERSIONS.items(): + with self.subTest(car_model=car_model): + for ecu, fws in ecus.items(): + if ecu[0] not in PLATFORM_CODE_ECUS: + continue + + codes = dict() + for fw in fws: + result = get_platform_codes([fw]) + # Check only one platform code and sub-version + self.assertEqual(1, len(result), f"Unable to parse FW: {fw}") + self.assertEqual(1, len(list(result.values())[0]), f"Unable to parse FW: {fw}") + codes |= result + + # Toyota places the ECU part number in their FW versions, assert all parsable + # Note that there is only one unique part number per ECU across the fleet, so this + # is not important for identification, just a sanity check. + self.assertTrue(all(code.count(b"-") > 1 for code in codes), + f"FW does not have part number: {fw} {codes}") + + def test_platform_codes_spot_check(self): + # Asserts basic platform code parsing behavior for a few cases + results = get_platform_codes([ + b"F152607140\x00\x00\x00\x00\x00\x00", + b"F152607171\x00\x00\x00\x00\x00\x00", + b"F152607110\x00\x00\x00\x00\x00\x00", + b"F152607180\x00\x00\x00\x00\x00\x00", + ]) + self.assertEqual(results, {b"F1526-07-1": {b"10", b"40", b"71", b"80"}}) + + results = get_platform_codes([ + b"\x028646F4104100\x00\x00\x00\x008646G5301200\x00\x00\x00\x00", + b"\x028646F4104100\x00\x00\x00\x008646G3304000\x00\x00\x00\x00", + ]) + self.assertEqual(results, {b"8646F-41-04": {b"100"}}) + + # Short version has no part number + results = get_platform_codes([ + b"\x0235870000\x00\x00\x00\x00\x00\x00\x00\x00A0202000\x00\x00\x00\x00\x00\x00\x00\x00", + b"\x0235883000\x00\x00\x00\x00\x00\x00\x00\x00A0202000\x00\x00\x00\x00\x00\x00\x00\x00", + ]) + self.assertEqual(results, {b"58-70": {b"000"}, b"58-83": {b"000"}}) + + results = get_platform_codes([ + b"F152607110\x00\x00\x00\x00\x00\x00", + b"F152607140\x00\x00\x00\x00\x00\x00", + b"\x028646F4104100\x00\x00\x00\x008646G5301200\x00\x00\x00\x00", + b"\x0235879000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00", + ]) + self.assertEqual(results, {b"F1526-07-1": {b"10", b"40"}, b"8646F-41-04": {b"100"}, b"58-79": {b"000"}}) if __name__ == "__main__": diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index bb9672fb0e..cf7f329dd0 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -2,7 +2,7 @@ import re from collections import defaultdict from dataclasses import dataclass, field from enum import Enum, IntFlag -from typing import Dict, List, Set, Tuple, Union +from typing import Dict, List, Set, Union from cereal import car from openpilot.common.conversions import Conversions as CV @@ -236,8 +236,9 @@ STATIC_DSU_MSGS = [ ] -def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, bytes]]: - codes = set() # (Optional[part]-platform-major_version, minor_version) +def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]: + # Returns sub versions in a dict so comparisons can be made within part-platform-major_version combos + codes = defaultdict(set) # Optional[part]-platform-major_version: set of sub_version for fw in fw_versions: # FW versions returned from UDS queries can return multiple fields/chunks of data (different ECU calibrations, different data?) # and are prefixed with a byte that describes how many chunks of data there are. @@ -262,42 +263,49 @@ def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, bytes]]: fw_match = SHORT_FW_PATTERN.search(first_chunk) if fw_match is not None: platform, major_version, sub_version = fw_match.groups() - # codes.add((platform + b'-' + major_version, sub_version)) - codes.add((b'-'.join((platform, major_version)), sub_version)) + codes[b'-'.join((platform, major_version))].add(sub_version) elif len(first_chunk) == 10: fw_match = MEDIUM_FW_PATTERN.search(first_chunk) if fw_match is not None: part, platform, major_version, sub_version = fw_match.groups() - codes.add((b'-'.join((part, platform, major_version)), sub_version)) + codes[b'-'.join((part, platform, major_version))].add(sub_version) elif len(first_chunk) == 12: fw_match = LONG_FW_PATTERN.search(first_chunk) if fw_match is not None: part, platform, major_version, sub_version = fw_match.groups() - codes.add((b'-'.join((part, platform, major_version)), sub_version)) + codes[b'-'.join((part, platform, major_version))].add(sub_version) - return codes + return dict(codes) # Regex patterns for parsing more general platform-specific identifiers from FW versions. # - Part number: Toyota part number (usually last character needs to be ignored to find a match). -# - Platform: usually multiple codes per an openpilot platform, however this has the less variability and +# Each ECU address has just one part number. +# - Platform: usually multiple codes per an openpilot platform, however this is the least variable and # is usually shared across ECUs and model years signifying this describes something about the specific platform. -# - Major version: second least variable part of the FW version. Seen splitting cars by model year such as RAV4 2022/2023 and Prius. +# This describes more generational changes (TSS-P vs TSS2), or manufacture region. +# - Major version: second least variable part of the FW version. Seen splitting cars by model year/API such as +# RAV4 2022/2023 and Avalon. Used to differentiate cars where API has changed slightly, but is not a generational change. # It is important to note that these aren't always consecutive, for example: -# Prius TSS-P has these major versions over 16 FW: 2, 3, 4, 6, 8 while Prius TSS2 has: 5 -# - Sub version: exclusive to major version, but shared with other cars. Should only be used for further filtering, -# more exploration is needed. -SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{4})') +# Avalon 2016-18's fwdCamera has these major versions: 01, 03 while 2019 has: 02 +# - Sub version: exclusive to major version, but shared with other cars. Should only be used for further filtering. +# Seen bumped in TSB FW updates, and describes other minor differences. +SHORT_FW_PATTERN = re.compile(b'[A-Z0-9](?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{1})(?P[A-Z0-9]{2})') LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') -FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. highest seen is 3 chunks, 16 bytes each +FW_LEN_CODE = re.compile(b'^[\x01-\x03]') # highest seen is 3 chunks, 16 bytes each FW_CHUNK_LEN = 16 -# List of ECUs expected to have platform codes -# TODO: use hybrid ECU, splits many similar ICE and hybrid variants -PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.engine, Ecu.eps, Ecu.dsu, Ecu.fwdCamera, Ecu.fwdRadar] +# List of ECUs that are most unique across openpilot platforms +# TODO: use hybrid ECU, splits similar ICE and hybrid variants +# - fwdCamera: describes actual features related to ADAS. For example, on the Avalon it describes +# when TSS-P became standard, whether the car supports stop and go, and whether it's TSS2. +# On the RAV4, it describes the move to the radar doing ACC, and the use of LTA for lane keeping. +# - abs: differentiates hybrid/ICE on most cars (Corolla TSS2 is an exception) +# - eps: describes lateral API changes for the EPS, such as using LTA for lane keeping and rejecting LKA messages +PLATFORM_CODE_ECUS = [Ecu.fwdCamera, Ecu.abs, Ecu.eps] # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers.