From baa77ced19add6241cdc634840205b7115c584ab Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 15 Feb 2024 02:35:21 -0600 Subject: [PATCH 01/16] TestFwFingerprint: test to prevent mismatches when utilizing non-essential ECUs (#31478) * test * same speed --- selfdrive/car/tests/test_fw_fingerprint.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/selfdrive/car/tests/test_fw_fingerprint.py b/selfdrive/car/tests/test_fw_fingerprint.py index 17eba80d2a..493efc1bab 100755 --- a/selfdrive/car/tests/test_fw_fingerprint.py +++ b/selfdrive/car/tests/test_fw_fingerprint.py @@ -33,18 +33,29 @@ class TestFwFingerprint(unittest.TestCase): self.assertEqual(len(candidates), 1, f"got more than one candidate: {candidates}") self.assertEqual(candidates[0], expected) - @parameterized.expand([(b, c, e[c]) for b, e in VERSIONS.items() for c in e]) - def test_exact_match(self, brand, car_model, ecus): + @parameterized.expand([(b, c, e[c], n) for b, e in VERSIONS.items() for c in e for n in (True, False)]) + def test_exact_match(self, brand, car_model, ecus, test_non_essential): + config = FW_QUERY_CONFIGS[brand] CP = car.CarParams.new_message() - for _ in range(200): + for _ in range(100): fw = [] for ecu, fw_versions in ecus.items(): + # Assume non-essential ECUs apply to all cars, so we catch cases where Car A with + # missing ECUs won't match to Car B where only Car B has labeled non-essential ECUs + if ecu[0] in config.non_essential_ecus and test_non_essential: + continue + ecu_name, addr, sub_addr = ecu fw.append({"ecu": ecu_name, "fwVersion": random.choice(fw_versions), 'brand': brand, "address": addr, "subAddress": 0 if sub_addr is None else sub_addr}) CP.carFw = fw _, matches = match_fw_to_car(CP.carFw, allow_fuzzy=False) - self.assertFingerprints(matches, car_model) + if not test_non_essential: + self.assertFingerprints(matches, car_model) + else: + # if we're removing ECUs we expect some match loss, but it shouldn't mismatch + if len(matches) != 0: + self.assertFingerprints(matches, car_model) @parameterized.expand([(b, c, e[c]) for b, e in VERSIONS.items() for c in e]) def test_custom_fuzzy_match(self, brand, car_model, ecus): From 8aee0d2af2c9e9c1755afa2b46f68b199958ccdb Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 15 Feb 2024 14:24:27 -0500 Subject: [PATCH 02/16] test_logreader: test interactive mode + fix typo in AUTO_INTERACTIVE (#31481) * fix spelling * test interactive * remove that * test taht * move that --- tools/lib/logreader.py | 4 ++-- tools/lib/tests/test_logreader.py | 19 ++++++++++++++++--- tools/plotjuggler/juggle.py | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index af2c23ef48..2269430792 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -73,7 +73,7 @@ class ReadMode(enum.StrEnum): QLOG = "q" # only read qlogs SANITIZED = "s" # read from the commaCarSegments database AUTO = "a" # default to rlogs, fallback to qlogs - AUTO_INTERACIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user + AUTO_INTERACTIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user LogPath = Optional[str] @@ -106,7 +106,7 @@ def apply_strategy(mode: ReadMode, rlog_paths: LogPaths, qlog_paths: LogPaths, v return qlog_paths elif mode == ReadMode.AUTO: return auto_strategy(rlog_paths, qlog_paths, False, valid_file) - elif mode == ReadMode.AUTO_INTERACIVE: + elif mode == ReadMode.AUTO_INTERACTIVE: return auto_strategy(rlog_paths, qlog_paths, True, valid_file) raise Exception(f"invalid mode: {mode}") diff --git a/tools/lib/tests/test_logreader.py b/tools/lib/tests/test_logreader.py index 53b78064ab..974182d638 100755 --- a/tools/lib/tests/test_logreader.py +++ b/tools/lib/tests/test_logreader.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import io import shutil import tempfile import os @@ -168,10 +169,22 @@ class TestLogReader(unittest.TestCase): with mock.patch("openpilot.tools.lib.route.Route.log_paths") as log_paths_mock: log_paths_mock.return_value = [None] * NUM_SEGS # Should fall back to qlogs since rlogs are not available - lr = LogReader(f"{TEST_ROUTE}/0/a", default_source=comma_api_source) - log_len = len(list(lr)) - self.assertEqual(qlog_len, log_len) + with self.subTest("interactive_yes"): + with mock.patch("sys.stdin", new=io.StringIO("y\n")): + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, default_source=comma_api_source) + log_len = len(list(lr)) + self.assertEqual(qlog_len, log_len) + + with self.subTest("interactive_no"): + with mock.patch("sys.stdin", new=io.StringIO("n\n")): + with self.assertRaises(AssertionError): + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, default_source=comma_api_source) + + with self.subTest("non_interactive"): + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, default_source=comma_api_source) + log_len = len(list(lr)) + self.assertEqual(qlog_len, log_len) if __name__ == "__main__": diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index cc21095414..dc94062801 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -73,7 +73,7 @@ def process(can, lr): return [d for d in lr if can or d.which() not in ['can', 'sendcan']] def juggle_route(route_or_segment_name, can, layout, dbc=None): - sr = LogReader(route_or_segment_name, default_mode=ReadMode.AUTO_INTERACIVE) + sr = LogReader(route_or_segment_name, default_mode=ReadMode.AUTO_INTERACTIVE) all_data = sr.run_across_segments(24, partial(process, can)) From d31269f6639b86405708f05c098ac1fd665c3107 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 15 Feb 2024 11:52:57 -0800 Subject: [PATCH 03/16] fix no GPS alert when driving slowly in a tunnel (#31483) --- selfdrive/controls/controlsd.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/selfdrive/controls/controlsd.py b/selfdrive/controls/controlsd.py index 3fb19c7da7..adc680ed2a 100755 --- a/selfdrive/controls/controlsd.py +++ b/selfdrive/controls/controlsd.py @@ -411,9 +411,12 @@ class Controls: # TODO: fix simulator if not SIMULATION or REPLAY: + # Not show in first 1 km to allow for driving out of garage. This event shows after 5 minutes if not self.sm['liveLocationKalman'].gpsOK and self.sm['liveLocationKalman'].inputsOK and (self.distance_traveled > 1000): - # Not show in first 1 km to allow for driving out of garage. This event shows after 5 minutes self.events.add(EventName.noGps) + if self.sm['liveLocationKalman'].gpsOK: + self.distance_traveled = 0 + self.distance_traveled += CS.vEgo * DT_CTRL if self.sm['modelV2'].frameDropPerc > 20: self.events.add(EventName.modeldLagging) @@ -476,8 +479,6 @@ class Controls: if ps.safetyModel not in IGNORED_SAFETY_MODES): self.mismatch_counter += 1 - self.distance_traveled += CS.vEgo * DT_CTRL - return CS def state_transition(self, CS): From 16d13395536e84ddf492b2352d827d33befd601e Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 15 Feb 2024 15:10:56 -0500 Subject: [PATCH 04/16] test_logreader: test internal scenarios (#31484) * test source scenario * test source scenario * fix --- tools/lib/logreader.py | 6 ++++-- tools/lib/tests/test_logreader.py | 32 ++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 2269430792..7957712aff 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -81,6 +81,8 @@ LogPaths = List[LogPath] ValidFileCallable = Callable[[LogPath], bool] Source = Callable[[SegmentRange, ReadMode], LogPaths] +InternalUnavailableException = Exception("Internal source not available") + def default_valid_file(fn: LogPath) -> bool: return fn is not None and file_exists(fn) @@ -126,7 +128,7 @@ def comma_api_source(sr: SegmentRange, mode: ReadMode) -> LogPaths: def internal_source(sr: SegmentRange, mode: ReadMode) -> LogPaths: if not internal_source_available(): - raise Exception("Internal source not available") + raise InternalUnavailableException def get_internal_url(sr: SegmentRange, seg, file): return f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{file}.bz2" @@ -160,7 +162,7 @@ def get_invalid_files(files): def check_source(source: Source, *args) -> LogPaths: files = source(*args) - assert next(get_invalid_files(files), None) is None + assert next(get_invalid_files(files), False) is False return files diff --git a/tools/lib/tests/test_logreader.py b/tools/lib/tests/test_logreader.py index 974182d638..2141915b87 100755 --- a/tools/lib/tests/test_logreader.py +++ b/tools/lib/tests/test_logreader.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import contextlib import io import shutil import tempfile @@ -10,7 +11,7 @@ import requests from parameterized import parameterized from unittest import mock -from openpilot.tools.lib.logreader import LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode +from openpilot.tools.lib.logreader import LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException from openpilot.tools.lib.route import SegmentRange from openpilot.tools.lib.url_file import URLFileException @@ -24,6 +25,24 @@ def noop(segment: LogIterable): return segment +@contextlib.contextmanager +def setup_source_scenario(is_internal=False): + with ( + mock.patch("openpilot.tools.lib.logreader.internal_source") as internal_source_mock, + mock.patch("openpilot.tools.lib.logreader.openpilotci_source") as openpilotci_source_mock, + mock.patch("openpilot.tools.lib.logreader.comma_api_source") as comma_api_source_mock, + ): + if is_internal: + internal_source_mock.return_value = [QLOG_FILE] + else: + internal_source_mock.side_effect = InternalUnavailableException + + openpilotci_source_mock.return_value = [None] + comma_api_source_mock.return_value = [QLOG_FILE] + + yield + + class TestLogReader(unittest.TestCase): @parameterized.expand([ (f"{TEST_ROUTE}", ALL_SEGS), @@ -186,6 +205,17 @@ class TestLogReader(unittest.TestCase): log_len = len(list(lr)) self.assertEqual(qlog_len, log_len) + @parameterized.expand([(True,), (False,)]) + @pytest.mark.slow + def test_auto_source_scenarios(self, is_internal): + lr = LogReader(QLOG_FILE) + qlog_len = len(list(lr)) + + with setup_source_scenario(is_internal=is_internal): + lr = LogReader(f"{TEST_ROUTE}/0/q") + log_len = len(list(lr)) + self.assertEqual(qlog_len, log_len) + if __name__ == "__main__": unittest.main() From c2b9f7163ace207038d4d17096f056b61c7644ba Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 15 Feb 2024 16:12:11 -0500 Subject: [PATCH 05/16] test_qcomgpsd: subtest the gps_runtime scenarios (#31485) * subtest this * runtime --------- Co-authored-by: Comma Device --- system/qcomgpsd/tests/test_qcomgpsd.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/system/qcomgpsd/tests/test_qcomgpsd.py b/system/qcomgpsd/tests/test_qcomgpsd.py index 8291f2cc32..6c93f7dd93 100755 --- a/system/qcomgpsd/tests/test_qcomgpsd.py +++ b/system/qcomgpsd/tests/test_qcomgpsd.py @@ -68,13 +68,14 @@ class TestRawgpsd(unittest.TestCase): def test_turns_off_gnss(self): for s in (0.1, 1, 5): - managed_processes['qcomgpsd'].start() - time.sleep(s) - managed_processes['qcomgpsd'].stop() + with self.subTest(runtime=s): + managed_processes['qcomgpsd'].start() + time.sleep(s) + managed_processes['qcomgpsd'].stop() - ls = subprocess.check_output("mmcli -m any --location-status --output-json", shell=True, encoding='utf-8') - loc_status = json.loads(ls) - assert set(loc_status['modem']['location']['enabled']) <= {'3gpp-lac-ci'} + ls = subprocess.check_output("mmcli -m any --location-status --output-json", shell=True, encoding='utf-8') + loc_status = json.loads(ls) + assert set(loc_status['modem']['location']['enabled']) <= {'3gpp-lac-ci'} def check_assistance(self, should_be_loaded): From 64851baea8e61d9e83b7935f3b4727a6409d62ce Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 15 Feb 2024 13:10:33 -0800 Subject: [PATCH 06/16] debug/count_events.py improvements --- selfdrive/debug/count_events.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/selfdrive/debug/count_events.py b/selfdrive/debug/count_events.py index 0d545b3153..b40e195939 100755 --- a/selfdrive/debug/count_events.py +++ b/selfdrive/debug/count_events.py @@ -16,28 +16,35 @@ if __name__ == "__main__": cams = [s for s in SERVICE_LIST if s.endswith('CameraState')] cnt_cameras = dict.fromkeys(cams, 0) + events = [] alerts: List[Tuple[float, str]] = [] start_time = math.inf end_time = -math.inf ignition_off = None for msg in LogReader(sys.argv[1], ReadMode.QLOG): + t = (msg.logMonoTime - start_time) / 1e9 end_time = max(end_time, msg.logMonoTime) start_time = min(start_time, msg.logMonoTime) if msg.which() == 'onroadEvents': for e in msg.onroadEvents: cnt_events[e.name] += 1 + + ae = {str(e.name) for e in msg.onroadEvents if e.name not in ('pedalPressed', 'steerOverride', 'gasPressedOverride')} + if len(events) == 0 or ae != events[-1][1]: + events.append((t, ae)) + elif msg.which() == 'controlsState': at = msg.controlsState.alertType if "/override" not in at or "lanechange" in at.lower(): if len(alerts) == 0 or alerts[-1][1] != at: - t = (msg.logMonoTime - start_time) / 1e9 alerts.append((t, at)) elif msg.which() == 'pandaStates': if ignition_off is None: ign = any(ps.ignitionLine or ps.ignitionCan for ps in msg.pandaStates) if not ign: ignition_off = msg.logMonoTime + break elif msg.which() in cams: cnt_cameras[msg.which()] += 1 @@ -64,9 +71,14 @@ if __name__ == "__main__": print("Alerts") for t, a in alerts: print(f"{t:8.2f} {a}") + + print("\n") + print("Events") + for t, a in events: + print(f"{t:8.2f} {a}") + + print("\n") if ignition_off is not None: ignition_off = round((ignition_off - start_time) / 1e9, 2) print("Ignition off at", ignition_off) - - print("\n") print("Route duration", datetime.timedelta(seconds=duration)) From eb5a847dc87fdbf2118aadd670487a5abd950f6c Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 15 Feb 2024 13:18:19 -0800 Subject: [PATCH 07/16] Update RELEASES.md --- RELEASES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASES.md b/RELEASES.md index 375a33b867..b3c5b1d6da 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -11,7 +11,7 @@ Version 0.9.6 (2024-02-22) * Improved fuzzy fingerprinting for many makes and models * Alpha longitudinal support for new Toyota models * Chevrolet Equinox 2019-22 support thanks to JasonJShuler and nworb-cire! -* Dodge Duranago 2020-21 support +* Dodge Durango 2020-21 support * Hyundai Staria 2023 support thanks to sunnyhaibin! * Kia Niro Plug-in Hybrid 2022 support thanks to sunnyhaibin! * Lexus LC 2024 support thanks to nelsonjchen! From 663f7017f2476f3f50b23bfcbaa89947e4e99fbd Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 15 Feb 2024 13:35:33 -0800 Subject: [PATCH 08/16] fix linter --- selfdrive/debug/count_events.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/selfdrive/debug/count_events.py b/selfdrive/debug/count_events.py index b40e195939..567d1a9e07 100755 --- a/selfdrive/debug/count_events.py +++ b/selfdrive/debug/count_events.py @@ -16,7 +16,7 @@ if __name__ == "__main__": cams = [s for s in SERVICE_LIST if s.endswith('CameraState')] cnt_cameras = dict.fromkeys(cams, 0) - events = [] + events: List[Tuple[float, set[str]]] = [] alerts: List[Tuple[float, str]] = [] start_time = math.inf end_time = -math.inf @@ -74,8 +74,8 @@ if __name__ == "__main__": print("\n") print("Events") - for t, a in events: - print(f"{t:8.2f} {a}") + for t, evt in events: + print(f"{t:8.2f} {evt}") print("\n") if ignition_off is not None: From 0cb206cb95ac3aa674d79ce0121893fbf7c68c88 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 15 Feb 2024 16:38:42 -0500 Subject: [PATCH 09/16] conftest: cleanup environment cleaner (#31486) * clean env * no self --- conftest.py | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/conftest.py b/conftest.py index e4dc6640c0..0d2a4a8fc4 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,4 @@ +import contextlib import gc import os import pytest @@ -25,42 +26,42 @@ def pytest_runtest_call(item): yield -@pytest.fixture(scope="function", autouse=True) -def openpilot_function_fixture(request): +@contextlib.contextmanager +def clean_env(): starting_env = dict(os.environ) + yield + os.environ.clear() + os.environ.update(starting_env) - random.seed(0) - # setup a clean environment for each test - with OpenpilotPrefix(shared_download_cache=request.node.get_closest_marker("shared_download_cache") is not None) as prefix: - prefix = os.environ["OPENPILOT_PREFIX"] +@pytest.fixture(scope="function", autouse=True) +def openpilot_function_fixture(request): + random.seed(0) - yield + with clean_env(): + # setup a clean environment for each test + with OpenpilotPrefix(shared_download_cache=request.node.get_closest_marker("shared_download_cache") is not None) as prefix: + prefix = os.environ["OPENPILOT_PREFIX"] - # ensure the test doesn't change the prefix - assert "OPENPILOT_PREFIX" in os.environ and prefix == os.environ["OPENPILOT_PREFIX"] + yield - os.environ.clear() - os.environ.update(starting_env) + # ensure the test doesn't change the prefix + assert "OPENPILOT_PREFIX" in os.environ and prefix == os.environ["OPENPILOT_PREFIX"] - # cleanup any started processes - manager.manager_cleanup() + # cleanup any started processes + manager.manager_cleanup() - # some processes disable gc for performance, re-enable here - if not gc.isenabled(): - gc.enable() - gc.collect() + # some processes disable gc for performance, re-enable here + if not gc.isenabled(): + gc.enable() + gc.collect() # If you use setUpClass, the environment variables won't be cleared properly, # so we need to hook both the function and class pytest fixtures @pytest.fixture(scope="class", autouse=True) def openpilot_class_fixture(): - starting_env = dict(os.environ) - - yield - - os.environ.clear() - os.environ.update(starting_env) + with clean_env(): + yield @pytest.fixture(scope="function") From 0a5e994947170dde632c20069b80470e8c8df31e Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 15 Feb 2024 23:44:02 -0600 Subject: [PATCH 10/16] [bot] Fingerprints: add missing FW versions from new users (#31489) Export fingerprints --- selfdrive/car/honda/fingerprints.py | 1 + selfdrive/car/hyundai/fingerprints.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/selfdrive/car/honda/fingerprints.py b/selfdrive/car/honda/fingerprints.py index d24838ffbc..fbb1714ea1 100644 --- a/selfdrive/car/honda/fingerprints.py +++ b/selfdrive/car/honda/fingerprints.py @@ -299,6 +299,7 @@ FW_VERSIONS = { (Ecu.srs, 0x18da53f1, None): [ b'77959-TBA-A030\x00\x00', b'77959-TBA-A040\x00\x00', + b'77959-TBG-A020\x00\x00', b'77959-TBG-A030\x00\x00', b'77959-TEA-Q820\x00\x00', ], diff --git a/selfdrive/car/hyundai/fingerprints.py b/selfdrive/car/hyundai/fingerprints.py index d3b282422c..d1fc1faabb 100644 --- a/selfdrive/car/hyundai/fingerprints.py +++ b/selfdrive/car/hyundai/fingerprints.py @@ -1549,6 +1549,7 @@ FW_VERSIONS = { b'\xf1\x00NE1 MFC AT EUR LHD 1.00 1.06 99211-GI000 210813', b'\xf1\x00NE1 MFC AT EUR RHD 1.00 1.01 99211-GI010 211007', b'\xf1\x00NE1 MFC AT EUR RHD 1.00 1.02 99211-GI010 211206', + b'\xf1\x00NE1 MFC AT KOR LHD 1.00 1.00 99211-GI020 230719', b'\xf1\x00NE1 MFC AT KOR LHD 1.00 1.05 99211-GI010 220614', b'\xf1\x00NE1 MFC AT USA LHD 1.00 1.00 99211-GI020 230719', b'\xf1\x00NE1 MFC AT USA LHD 1.00 1.01 99211-GI010 211007', @@ -1556,7 +1557,6 @@ FW_VERSIONS = { b'\xf1\x00NE1 MFC AT USA LHD 1.00 1.03 99211-GI010 220401', b'\xf1\x00NE1 MFC AT USA LHD 1.00 1.05 99211-GI010 220614', b'\xf1\x00NE1 MFC AT USA LHD 1.00 1.06 99211-GI010 230110', - b'\xf1\x00NE1 MFC AT KOR LHD 1.00 1.00 99211-GI020 230719', ], }, CAR.IONIQ_6: { From 5a441ec0c4bf39a5e066e307ee6c9a8b55f8d8b5 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 15 Feb 2024 22:20:00 -0800 Subject: [PATCH 11/16] modeld: fix and cleanup getting carParams (#31488) --- selfdrive/modeld/modeld.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 5b227d08e9..01acca7bcb 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -152,12 +152,8 @@ def main(demo=False): pm = PubMaster(["modelV2", "cameraOdometry"]) sm = SubMaster(["carState", "roadCameraState", "liveCalibration", "driverMonitoringState", "navModel", "navInstruction", "carControl"]) - publish_state = PublishState() params = Params() - with car.CarParams.from_bytes(params.get("CarParams", block=True)) as msg: - steer_delay = msg.steerActuatorDelay + .2 - #steer_delay = 0.4 # setup filter to track dropped frames frame_dropped_filter = FirstOrderFilter(0., 10., 1. / ModelConstants.MODEL_FREQ) @@ -177,13 +173,15 @@ def main(demo=False): if demo: CP = get_demo_car_params() - with car.CarParams.from_bytes(params.get("CarParams", block=True)) as msg: - CP = msg - cloudlog.info("plannerd got CarParams: %s", CP.carName) + else: + with car.CarParams.from_bytes(params.get("CarParams", block=True)) as msg: + CP = msg + cloudlog.info("modeld got CarParams: %s", CP.carName) + # TODO this needs more thought, use .2s extra for now to estimate other delays steer_delay = CP.steerActuatorDelay + .2 - DH = DesireHelper() + DH = DesireHelper() while True: # Keep receiving frames until we are at least 1 frame ahead of previous extra frame From 0723c2bc5f22a079685c1643fa2d72f70eeff087 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 16 Feb 2024 13:19:10 -0800 Subject: [PATCH 12/16] log git commit date (#31490) * log git commit date * unix ts * fix * bump cereal * cleanup --- cereal | 2 +- common/params.cc | 1 + selfdrive/manager/manager.py | 3 ++- system/loggerd/logger.cc | 1 + system/loggerd/tests/test_loggerd.py | 1 + system/version.py | 18 ++++++++++++------ 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/cereal b/cereal index bd31b25aac..7de3c7111e 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit bd31b25aacc5b39f36cedcb0dabd05db471da59f +Subproject commit 7de3c7111e78d87f9c43e2861a3e18aa59fde956 diff --git a/common/params.cc b/common/params.cc index 3ce2505243..eb75705ca3 100644 --- a/common/params.cc +++ b/common/params.cc @@ -125,6 +125,7 @@ std::unordered_map keys = { {"ForcePowerDown", PERSISTENT}, {"GitBranch", PERSISTENT}, {"GitCommit", PERSISTENT}, + {"GitCommitDate", PERSISTENT}, {"GitDiff", PERSISTENT}, {"GithubSshKeys", PERSISTENT}, {"GithubUsername", PERSISTENT}, diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 50ba73f18c..bf1eeb8fd0 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -19,7 +19,7 @@ from openpilot.selfdrive.athena.registration import register, UNREGISTERED_DONGL from openpilot.common.swaglog import cloudlog, add_file_handler from openpilot.system.version import is_dirty, get_commit, get_version, get_origin, get_short_branch, \ get_normalized_origin, terms_version, training_version, \ - is_tested_branch, is_release_branch + is_tested_branch, is_release_branch, get_commit_date @@ -66,6 +66,7 @@ def manager_init() -> None: params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) params.put("GitCommit", get_commit()) + params.put("GitCommitDate", get_commit_date()) params.put("GitBranch", get_short_branch()) params.put("GitRemote", get_origin()) params.put_bool("IsTestedBranch", is_tested_branch()) diff --git a/system/loggerd/logger.cc b/system/loggerd/logger.cc index 2fc6492ad4..7a829a2f1f 100644 --- a/system/loggerd/logger.cc +++ b/system/loggerd/logger.cc @@ -44,6 +44,7 @@ kj::Array logger_build_init_data() { std::map params_map = params.readAll(); init.setGitCommit(params_map["GitCommit"]); + init.setGitCommitDate(params_map["GitCommitDate"]); init.setGitBranch(params_map["GitBranch"]); init.setGitRemote(params_map["GitRemote"]); init.setPassive(false); diff --git a/system/loggerd/tests/test_loggerd.py b/system/loggerd/tests/test_loggerd.py index 0cd8548809..963978926d 100755 --- a/system/loggerd/tests/test_loggerd.py +++ b/system/loggerd/tests/test_loggerd.py @@ -107,6 +107,7 @@ class TestLoggerd: # param, initData field, value ("DongleId", "dongleId", dongle), ("GitCommit", "gitCommit", "commit"), + ("GitCommitDate", "gitCommitDate", "date"), ("GitBranch", "gitBranch", "branch"), ("GitRemote", "gitRemote", "remote"), ] diff --git a/system/version.py b/system/version.py index 980a4fcc7c..e34458f16f 100755 --- a/system/version.py +++ b/system/version.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import os import subprocess -from typing import List, Optional, Callable, TypeVar +from typing import List, Callable, TypeVar from functools import lru_cache from openpilot.common.basedir import BASEDIR @@ -22,7 +22,7 @@ def run_cmd(cmd: List[str]) -> str: return subprocess.check_output(cmd, encoding='utf8').strip() -def run_cmd_default(cmd: List[str], default: Optional[str] = None) -> Optional[str]: +def run_cmd_default(cmd: List[str], default: str = "") -> str: try: return run_cmd(cmd) except subprocess.CalledProcessError: @@ -31,17 +31,22 @@ def run_cmd_default(cmd: List[str], default: Optional[str] = None) -> Optional[s @cache def get_commit(branch: str = "HEAD") -> str: - return run_cmd_default(["git", "rev-parse", branch]) or "" + return run_cmd_default(["git", "rev-parse", branch]) + + +@cache +def get_commit_date(commit: str = "HEAD") -> str: + return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit]) @cache def get_short_branch() -> str: - return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"]) or "" + return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"]) @cache def get_branch() -> str: - return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]) or "" + return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]) @cache @@ -51,7 +56,7 @@ def get_origin() -> str: tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"]) return run_cmd(["git", "config", "remote." + tracking_remote + ".url"]) except subprocess.CalledProcessError: # Not on a branch, fallback - return run_cmd_default(["git", "config", "--get", "remote.origin.url"]) or "" + return run_cmd_default(["git", "config", "--get", "remote.origin.url"]) @cache @@ -132,3 +137,4 @@ if __name__ == "__main__": print(f"Branch: {get_branch()}") print(f"Short branch: {get_short_branch()}") print(f"Prebuilt: {is_prebuilt()}") + print(f"Commit date: {get_commit_date()}") From b218abcaa3015e6c9202550eddbbc0eff7482729 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 16 Feb 2024 13:35:45 -0800 Subject: [PATCH 13/16] controlsd: increase initializing timeout --- selfdrive/controls/controlsd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/controls/controlsd.py b/selfdrive/controls/controlsd.py index adc680ed2a..cf978ada32 100755 --- a/selfdrive/controls/controlsd.py +++ b/selfdrive/controls/controlsd.py @@ -434,7 +434,7 @@ class Controls: if not self.initialized: all_valid = CS.canValid and self.sm.all_checks() - timed_out = self.sm.frame * DT_CTRL > (6. if REPLAY else 3.5) + timed_out = self.sm.frame * DT_CTRL > (6. if REPLAY else 4.0) if all_valid or timed_out or (SIMULATION and not REPLAY): available_streams = VisionIpcClient.available_streams("camerad", block=False) if VisionStreamType.VISION_STREAM_ROAD not in available_streams: From 86410a0ef0607ff78ba21ab2f4dc6b5135496e06 Mon Sep 17 00:00:00 2001 From: Mitchell Goff Date: Fri, 16 Feb 2024 22:32:43 +0000 Subject: [PATCH 14/16] Bumped model replay ref for new map tiles (#31493) --- selfdrive/test/process_replay/model_replay_ref_commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/test/process_replay/model_replay_ref_commit b/selfdrive/test/process_replay/model_replay_ref_commit index a2f6896307..984690e291 100644 --- a/selfdrive/test/process_replay/model_replay_ref_commit +++ b/selfdrive/test/process_replay/model_replay_ref_commit @@ -1 +1 @@ -fee90bcee1e545c7ec9a39d3c7d4e42cfefb9955 +73fe68ba29ad4bbfc9627622f29ac41ead75bc53 From 900300a928bf2029c74a2949e0778c037b355ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Fri, 16 Feb 2024 15:18:26 -0800 Subject: [PATCH 15/16] Calibration Transform: border pad (#31495) --- selfdrive/modeld/transforms/transform.cl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/selfdrive/modeld/transforms/transform.cl b/selfdrive/modeld/transforms/transform.cl index 357ef87321..2ca25920cd 100644 --- a/selfdrive/modeld/transforms/transform.cl +++ b/selfdrive/modeld/transforms/transform.cl @@ -22,20 +22,20 @@ __kernel void warpPerspective(__global const uchar * src, W = W != 0.0f ? INTER_TAB_SIZE / W : 0.0f; int X = rint(X0 * W), Y = rint(Y0 * W); - short sx = convert_short_sat(X >> INTER_BITS); - short sy = convert_short_sat(Y >> INTER_BITS); + int sx = convert_short_sat(X >> INTER_BITS); + int sy = convert_short_sat(Y >> INTER_BITS); + + short sx_clamp = clamp(sx, 0, src_cols - 1); + short sx_p1_clamp = clamp(sx + 1, 0, src_cols - 1); + short sy_clamp = clamp(sy, 0, src_rows - 1); + short sy_p1_clamp = clamp(sy + 1, 0, src_rows - 1); + int v0 = convert_int(src[mad24(sy_clamp, src_row_stride, src_offset + sx_clamp*src_px_stride)]); + int v1 = convert_int(src[mad24(sy_clamp, src_row_stride, src_offset + sx_p1_clamp*src_px_stride)]); + int v2 = convert_int(src[mad24(sy_p1_clamp, src_row_stride, src_offset + sx_clamp*src_px_stride)]); + int v3 = convert_int(src[mad24(sy_p1_clamp, src_row_stride, src_offset + sx_p1_clamp*src_px_stride)]); + short ay = (short)(Y & (INTER_TAB_SIZE - 1)); short ax = (short)(X & (INTER_TAB_SIZE - 1)); - - int v0 = (sx >= 0 && sx < src_cols && sy >= 0 && sy < src_rows) ? - convert_int(src[mad24(sy, src_row_stride, src_offset + sx*src_px_stride)]) : 0; - int v1 = (sx+1 >= 0 && sx+1 < src_cols && sy >= 0 && sy < src_rows) ? - convert_int(src[mad24(sy, src_row_stride, src_offset + (sx+1)*src_px_stride)]) : 0; - int v2 = (sx >= 0 && sx < src_cols && sy+1 >= 0 && sy+1 < src_rows) ? - convert_int(src[mad24(sy+1, src_row_stride, src_offset + sx*src_px_stride)]) : 0; - int v3 = (sx+1 >= 0 && sx+1 < src_cols && sy+1 >= 0 && sy+1 < src_rows) ? - convert_int(src[mad24(sy+1, src_row_stride, src_offset + (sx+1)*src_px_stride)]) : 0; - float taby = 1.f/INTER_TAB_SIZE*ay; float tabx = 1.f/INTER_TAB_SIZE*ax; From c5f1f4c67663161d335048238e6dd5506bde3555 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 16 Feb 2024 19:55:15 -0600 Subject: [PATCH 16/16] test_fw_query_on_routes: get first qlog (#31496) * fast * this isn't internal * see --- selfdrive/debug/test_fw_query_on_routes.py | 6 +++++- tools/lib/route.py | 10 +++++----- tools/lib/tests/test_comma_car_segments.py | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/selfdrive/debug/test_fw_query_on_routes.py b/selfdrive/debug/test_fw_query_on_routes.py index dd6243a44c..cc6fc2ae17 100755 --- a/selfdrive/debug/test_fw_query_on_routes.py +++ b/selfdrive/debug/test_fw_query_on_routes.py @@ -44,11 +44,15 @@ if __name__ == "__main__": dongles = [] for route in tqdm(routes): - dongle_id = SegmentRange(route).dongle_id + sr = SegmentRange(route) + dongle_id = sr.dongle_id if dongle_id in dongles: continue + if sr.slice == '' and sr.selector is None: + route += '/0' + lr = LogReader(route, default_mode=ReadMode.QLOG) try: diff --git a/tools/lib/route.py b/tools/lib/route.py index aba95718d5..47ebdc7a51 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -264,7 +264,7 @@ class SegmentRange: return self.m.group("timestamp") @property - def _slice(self) -> str: + def slice(self) -> str: return self.m.group("slice") or "" @property @@ -273,12 +273,12 @@ class SegmentRange: @property def seg_idxs(self) -> list[int]: - m = re.fullmatch(RE.SLICE, self._slice) - assert m is not None, f"Invalid slice: {self._slice}" + m = re.fullmatch(RE.SLICE, self.slice) + assert m is not None, f"Invalid slice: {self.slice}" start, end, step = (None if s is None else int(s) for s in m.groups()) # one segment specified - if start is not None and end is None and ':' not in self._slice: + if start is not None and end is None and ':' not in self.slice: if start < 0: start += get_max_seg_number_cached(self) + 1 return [start] @@ -291,7 +291,7 @@ class SegmentRange: return list(range(end + 1))[s] def __str__(self) -> str: - return f"{self.dongle_id}/{self.timestamp}" + (f"/{self._slice}" if self._slice else "") + (f"/{self.selector}" if self.selector else "") + return f"{self.dongle_id}/{self.timestamp}" + (f"/{self.slice}" if self.slice else "") + (f"/{self.selector}" if self.selector else "") def __repr__(self) -> str: return self.__str__() diff --git a/tools/lib/tests/test_comma_car_segments.py b/tools/lib/tests/test_comma_car_segments.py index 484a4aae08..b355b0fe60 100644 --- a/tools/lib/tests/test_comma_car_segments.py +++ b/tools/lib/tests/test_comma_car_segments.py @@ -25,7 +25,7 @@ class TestCommaCarSegments(unittest.TestCase): sr = SegmentRange(segment) - url = get_url(sr.route_name, sr._slice) + url = get_url(sr.route_name, sr.slice) resp = requests.get(url) self.assertEqual(resp.status_code, 200)