diff --git a/cereal b/cereal index df08568318..cda60ec965 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit df08568318da97ed6f87747caee0a5b2c30086c4 +Subproject commit cda60ec9652c05de4ccfcad1fae7936e708434a3 diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py index 758485c393..b79f61d94d 100755 --- a/selfdrive/car/fw_versions.py +++ b/selfdrive/car/fw_versions.py @@ -194,12 +194,13 @@ def chunks(l, n=128): yield l[i:i + n] -def build_fw_dict(fw_versions): +def build_fw_dict(fw_versions, filter_brand=None): fw_versions_dict = {} for fw in fw_versions: - addr = fw.address - sub_addr = fw.subAddress if fw.subAddress != 0 else None - fw_versions_dict[(addr, sub_addr)] = fw.fwVersion + if filter_brand is None or fw.brand == filter_brand: + addr = fw.address + sub_addr = fw.subAddress if fw.subAddress != 0 else None + fw_versions_dict[(addr, sub_addr)] = fw.fwVersion return fw_versions_dict @@ -284,18 +285,27 @@ def match_fw_to_car_exact(fw_versions_dict): def match_fw_to_car(fw_versions, allow_fuzzy=True): - fw_versions_dict = build_fw_dict(fw_versions) - matches = match_fw_to_car_exact(fw_versions_dict) + versions = get_interface_attr('FW_VERSIONS', ignore_none=True) + + # Try exact matching first + exact_matches = [True] + if allow_fuzzy: + exact_matches.append(False) + + for exact_match in exact_matches: + # For each brand, attempt to fingerprint using FW returned from its queries + for brand in versions.keys(): + fw_versions_dict = build_fw_dict(fw_versions, filter_brand=brand) - exact_match = True - if allow_fuzzy and len(matches) == 0: - matches = match_fw_to_car_fuzzy(fw_versions_dict) + if exact_match: + matches = match_fw_to_car_exact(fw_versions_dict) + else: + matches = match_fw_to_car_fuzzy(fw_versions_dict) - # Fuzzy match found - if len(matches) == 1: - exact_match = False + if len(matches) == 1: + return exact_match, matches - return exact_match, matches + return True, [] def get_present_ecus(logcan, sendcan): @@ -372,20 +382,21 @@ def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progr if addrs: query = IsoTpParallelQuery(sendcan, logcan, r.bus, addrs, r.request, r.response, r.rx_offset, debug=debug) t = 2 * timeout if i == 0 else timeout - fw_versions.update({addr: (version, r.request, r.rx_offset) for addr, version in query.get_data(t).items()}) + fw_versions.update({(r.brand, addr): (version, r) for addr, version in query.get_data(t).items()}) except Exception: cloudlog.warning(f"FW query exception: {traceback.format_exc()}") # Build capnp list to put into CarParams car_fw = [] - for addr, (version, request, rx_offset) in fw_versions.items(): + for (brand, addr), (version, request) in fw_versions.items(): f = car.CarParams.CarFw.new_message() f.ecu = ecu_types[addr] f.fwVersion = version f.address = addr[0] - f.responseAddress = uds.get_rx_addr_for_tx_addr(addr[0], rx_offset) - f.request = request + f.responseAddress = uds.get_rx_addr_for_tx_addr(addr[0], request.rx_offset) + f.request = request.request + f.brand = brand if addr[1] is not None: f.subAddress = addr[1] diff --git a/selfdrive/car/tests/test_fw_fingerprint.py b/selfdrive/car/tests/test_fw_fingerprint.py index ed7e420e1a..49fa66d36d 100755 --- a/selfdrive/car/tests/test_fw_fingerprint.py +++ b/selfdrive/car/tests/test_fw_fingerprint.py @@ -12,6 +12,7 @@ CarFw = car.CarParams.CarFw Ecu = car.CarParams.Ecu ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} +VERSIONS = get_interface_attr("FW_VERSIONS", ignore_none=True) class TestFwFingerprint(unittest.TestCase): @@ -20,14 +21,14 @@ class TestFwFingerprint(unittest.TestCase): self.assertEqual(len(candidates), 1, f"got more than one candidate: {candidates}") self.assertEqual(candidates[0], expected) - @parameterized.expand([(k, v) for k, v in FW_VERSIONS.items()]) - def test_fw_fingerprint(self, car_model, ecus): + @parameterized.expand([(b, c, e[c]) for b, e in VERSIONS.items() for c in e]) + def test_fw_fingerprint(self, brand, car_model, ecus): CP = car.CarParams.new_message() for _ in range(200): fw = [] for ecu, fw_versions in ecus.items(): ecu_name, addr, sub_addr = ecu - fw.append({"ecu": ecu_name, "fwVersion": random.choice(fw_versions), + fw.append({"ecu": ecu_name, "fwVersion": random.choice(fw_versions), 'brand': brand, "address": addr, "subAddress": 0 if sub_addr is None else sub_addr}) CP.carFw = fw _, matches = match_fw_to_car(CP.carFw) @@ -60,10 +61,9 @@ class TestFwFingerprint(unittest.TestCase): def test_fw_request_ecu_whitelist(self): passed = True brands = set(r.brand for r in REQUESTS) - versions = get_interface_attr('FW_VERSIONS') for brand in brands: whitelisted_ecus = [ecu for r in REQUESTS for ecu in r.whitelist_ecus if r.brand == brand] - brand_ecus = set([fw[0] for car_fw in versions[brand].values() for fw in car_fw]) + brand_ecus = set([fw[0] for car_fw in VERSIONS[brand].values() for fw in car_fw]) # each ecu in brand's fw versions needs to be whitelisted at least once ecus_not_whitelisted = set(brand_ecus) - set(whitelisted_ecus) diff --git a/selfdrive/controls/tests/test_startup.py b/selfdrive/controls/tests/test_startup.py index 9d13453045..a94311c8c7 100755 --- a/selfdrive/controls/tests/test_startup.py +++ b/selfdrive/controls/tests/test_startup.py @@ -42,27 +42,27 @@ class TestStartup(unittest.TestCase): # TODO: test EventName.startup for release branches # officially supported car - (EventName.startupMaster, TOYOTA.COROLLA, COROLLA_FW_VERSIONS), - (EventName.startupMaster, TOYOTA.COROLLA, COROLLA_FW_VERSIONS), + (EventName.startupMaster, TOYOTA.COROLLA, COROLLA_FW_VERSIONS, "toyota"), + (EventName.startupMaster, TOYOTA.COROLLA, COROLLA_FW_VERSIONS, "toyota"), # dashcamOnly car - (EventName.startupNoControl, MAZDA.CX5, CX5_FW_VERSIONS), - (EventName.startupNoControl, MAZDA.CX5, CX5_FW_VERSIONS), + (EventName.startupNoControl, MAZDA.CX5, CX5_FW_VERSIONS, "mazda"), + (EventName.startupNoControl, MAZDA.CX5, CX5_FW_VERSIONS, "mazda"), # unrecognized car with no fw - (EventName.startupNoFw, None, None), - (EventName.startupNoFw, None, None), + (EventName.startupNoFw, None, None, ""), + (EventName.startupNoFw, None, None, ""), # unrecognized car - (EventName.startupNoCar, None, COROLLA_FW_VERSIONS[:1]), - (EventName.startupNoCar, None, COROLLA_FW_VERSIONS[:1]), + (EventName.startupNoCar, None, COROLLA_FW_VERSIONS[:1], "toyota"), + (EventName.startupNoCar, None, COROLLA_FW_VERSIONS[:1], "toyota"), # fuzzy match - (EventName.startupMaster, TOYOTA.COROLLA, COROLLA_FW_VERSIONS_FUZZY), - (EventName.startupMaster, TOYOTA.COROLLA, COROLLA_FW_VERSIONS_FUZZY), + (EventName.startupMaster, TOYOTA.COROLLA, COROLLA_FW_VERSIONS_FUZZY, "toyota"), + (EventName.startupMaster, TOYOTA.COROLLA, COROLLA_FW_VERSIONS_FUZZY, "toyota"), ]) @with_processes(['controlsd']) - def test_startup_alert(self, expected_event, car_model, fw_versions): + def test_startup_alert(self, expected_event, car_model, fw_versions, brand): # TODO: this should be done without any real sockets controls_sock = messaging.sub_sock("controlsState") @@ -82,6 +82,7 @@ class TestStartup(unittest.TestCase): f.ecu = ecu f.address = addr f.fwVersion = version + f.brand = brand if subaddress is not None: f.subAddress = subaddress diff --git a/selfdrive/debug/test_fw_query_on_routes.py b/selfdrive/debug/test_fw_query_on_routes.py index 011dd6c9a3..789baeca4b 100755 --- a/selfdrive/debug/test_fw_query_on_routes.py +++ b/selfdrive/debug/test_fw_query_on_routes.py @@ -8,24 +8,15 @@ import traceback from tqdm import tqdm from tools.lib.logreader import LogReader from tools.lib.route import Route +from selfdrive.car.interfaces import get_interface_attr from selfdrive.car.car_helpers import interface_names from selfdrive.car.fw_versions import match_fw_to_car_exact, match_fw_to_car_fuzzy, build_fw_dict -from selfdrive.car.toyota.values import FW_VERSIONS as TOYOTA_FW_VERSIONS -from selfdrive.car.honda.values import FW_VERSIONS as HONDA_FW_VERSIONS -from selfdrive.car.hyundai.values import FW_VERSIONS as HYUNDAI_FW_VERSIONS -from selfdrive.car.volkswagen.values import FW_VERSIONS as VW_FW_VERSIONS -from selfdrive.car.mazda.values import FW_VERSIONS as MAZDA_FW_VERSIONS -from selfdrive.car.subaru.values import FW_VERSIONS as SUBARU_FW_VERSIONS NO_API = "NO_API" in os.environ -SUPPORTED_CARS = set(interface_names['toyota']) -SUPPORTED_CARS |= set(interface_names['honda']) -SUPPORTED_CARS |= set(interface_names['hyundai']) -SUPPORTED_CARS |= set(interface_names['volkswagen']) -SUPPORTED_CARS |= set(interface_names['mazda']) -SUPPORTED_CARS |= set(interface_names['subaru']) -SUPPORTED_CARS |= set(interface_names['nissan']) +VERSIONS = get_interface_attr('FW_VERSIONS', ignore_none=True) +SUPPORTED_BRANDS = VERSIONS.keys() +SUPPORTED_CARS = [brand for brand in SUPPORTED_BRANDS for brand in interface_names[brand]] try: from xx.pipeline.c.CarState import migration @@ -97,9 +88,24 @@ if __name__ == "__main__": print("not in supported cars") break - fw_versions_dict = build_fw_dict(car_fw) - exact_matches = match_fw_to_car_exact(fw_versions_dict) - fuzzy_matches = match_fw_to_car_fuzzy(fw_versions_dict) + # Older routes only have carFw from their brand + old_route = not any([len(fw.brand) for fw in car_fw]) + brands = SUPPORTED_BRANDS if not old_route else [None] + + # Exact match + exact_matches, fuzzy_matches = [], [] + for brand in brands: + fw_versions_dict = build_fw_dict(car_fw, filter_brand=brand) + exact_matches = match_fw_to_car_exact(fw_versions_dict) + if len(exact_matches) == 1: + break + + # Fuzzy match + for brand in brands: + fw_versions_dict = build_fw_dict(car_fw, filter_brand=brand) + fuzzy_matches = match_fw_to_car_fuzzy(fw_versions_dict) + if len(fuzzy_matches) == 1: + break if (len(exact_matches) == 1) and (list(exact_matches)[0] == live_fingerprint): good_exact += 1 @@ -126,12 +132,15 @@ if __name__ == "__main__": print("Mismatches") found = False - for car_fws in [TOYOTA_FW_VERSIONS, HONDA_FW_VERSIONS, HYUNDAI_FW_VERSIONS, VW_FW_VERSIONS, MAZDA_FW_VERSIONS, SUBARU_FW_VERSIONS]: + for brand in SUPPORTED_BRANDS: + car_fws = VERSIONS[brand] if live_fingerprint in car_fws: found = True expected = car_fws[live_fingerprint] for (_, expected_addr, expected_sub_addr), v in expected.items(): for version in car_fw: + if version.brand != brand and len(version.brand): + continue sub_addr = None if version.subAddress == 0 else version.subAddress addr = version.address diff --git a/selfdrive/test/process_replay/ref_commit b/selfdrive/test/process_replay/ref_commit index 999081b4df..0eeae1e3e3 100644 --- a/selfdrive/test/process_replay/ref_commit +++ b/selfdrive/test/process_replay/ref_commit @@ -1 +1 @@ -a57bbbffbee434e59e08b98b667dc13b6b505f08 \ No newline at end of file +b904e52e9de4ff7b2bd7f6af8b19abaf4957e6cc \ No newline at end of file diff --git a/selfdrive/test/process_replay/regen.py b/selfdrive/test/process_replay/regen.py index 793e548705..1a2d436f1a 100755 --- a/selfdrive/test/process_replay/regen.py +++ b/selfdrive/test/process_replay/regen.py @@ -179,8 +179,22 @@ def replay_cameras(lr, frs, disable_tqdm=False): return vs, p +def migrate_carparams(lr): + all_msgs = [] + for msg in lr: + if msg.which() == 'carParams': + CP = messaging.new_message('carParams') + CP.carParams = msg.carParams.as_builder() + for car_fw in CP.carParams.carFw: + car_fw.brand = CP.carParams.carName + msg = CP.as_reader() + all_msgs.append(msg) + + return all_msgs + + def regen_segment(lr, frs=None, outdir=FAKEDATA, disable_tqdm=False): - lr = list(lr) + lr = migrate_carparams(list(lr)) if frs is None: frs = dict() diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 4e7ba4a6dd..9cbf4439ac 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -35,19 +35,19 @@ original_segments = [ ] segments = [ - ("BODY", "bd6a637565e91581|2022-04-04--22-05-08--0"), - ("HYUNDAI", "fakedata|2022-01-20--17-49-04--0"), - ("TOYOTA", "fakedata|2022-04-29--15-57-12--0"), - ("TOYOTA2", "fakedata|2022-04-29--16-08-01--0"), - ("TOYOTA3", "fakedata|2022-04-29--16-17-39--0"), - ("HONDA", "fakedata|2022-01-20--17-56-40--0"), - ("HONDA2", "fakedata|2022-04-29--16-31-55--0"), - ("CHRYSLER", "fakedata|2022-01-20--18-00-11--0"), - ("SUBARU", "fakedata|2022-01-20--18-01-57--0"), - ("GM", "fakedata|2022-01-20--18-03-41--0"), - ("NISSAN", "fakedata|2022-01-20--18-05-29--0"), - ("VOLKSWAGEN", "fakedata|2022-01-20--18-07-15--0"), - ("MAZDA", "fakedata|2022-01-20--18-09-32--0"), + ("BODY", "regen660D86654BA|2022-07-06--14-27-15--0"), + ("HYUNDAI", "regen657E25856BB|2022-07-06--14-26-51--0"), + ("TOYOTA", "regenBA97410FBEC|2022-07-06--14-26-49--0"), + ("TOYOTA2", "regenDEDB1D9C991|2022-07-06--14-54-08--0"), + ("TOYOTA3", "regenDDC1FE60734|2022-07-06--14-32-06--0"), + ("HONDA", "regen17B09D158B8|2022-07-06--14-31-46--0"), + ("HONDA2", "regen041739C3E9A|2022-07-06--15-08-02--0"), + ("CHRYSLER", "regenBB2F9C1425C|2022-07-06--14-31-41--0"), + ("SUBARU", "regen732B69F33B1|2022-07-06--14-36-18--0"), + ("GM", "regen01D09D915B5|2022-07-06--14-36-20--0"), + ("NISSAN", "regenEA6FB2773F5|2022-07-06--14-58-23--0"), + ("VOLKSWAGEN", "regen007098CA0EF|2022-07-06--15-01-26--0"), + ("MAZDA", "regen61BA413D53B|2022-07-06--14-39-42--0"), ] # dashcamOnly makes don't need to be tested until a full port is done