diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index 5cecfb7e4e..e141071d09 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -88,4 +88,22 @@ jobs: devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/ devcontainer exec --workspace-folder . pip install pip-install-test devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json - devcontainer exec --workspace-folder . sudo touch /root/test.txt \ No newline at end of file + devcontainer exec --workspace-folder . sudo touch /root/test.txt + + notebooks: + name: notebooks + runs-on: ubuntu-20.04 + if: github.repository == 'commaai/openpilot' + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + - name: Build openpilot + timeout-minutes: 5 + run: ${{ env.RUN }} "scons -j$(nproc)" + - name: Test notebooks + timeout-minutes: 2 + run: | + ${{ env.RUN }} "pip install nbmake && pytest --nbmake tools/car_porting/examples/" \ No newline at end of file diff --git a/docs/CARS.md b/docs/CARS.md index b27fb1b943..55098a39a3 100644 --- a/docs/CARS.md +++ b/docs/CARS.md @@ -181,7 +181,7 @@ A supported vehicle is one that just works when you install a comma device. All |Nissan|Leaf 2018-23|ProPILOT Assist|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|
Parts- 1 Nissan A connector
- 1 RJ45 cable (7 ft)
- 1 USB-C coupler
- 1 comma 3X
- 1 harness box
- 1 long OBD-C cable
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Nissan|Rogue 2018-20|ProPILOT Assist|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|
Parts- 1 Nissan A connector
- 1 RJ45 cable (7 ft)
- 1 USB-C coupler
- 1 comma 3X
- 1 harness box
- 1 long OBD-C cable
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Nissan|X-Trail 2017|ProPILOT Assist|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|
Parts- 1 Nissan A connector
- 1 RJ45 cable (7 ft)
- 1 USB-C coupler
- 1 comma 3X
- 1 harness box
- 1 long OBD-C cable
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| -|Ram|1500 2019-23|Adaptive Cruise Control (ACC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 RJ45 cable (7 ft)
- 1 Ram connector
- 1 comma 3X
- 1 comma power v2
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| +|Ram|1500 2019-24|Adaptive Cruise Control (ACC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 RJ45 cable (7 ft)
- 1 Ram connector
- 1 comma 3X
- 1 comma power v2
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |SEAT|Ateca 2018|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[1,13](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 J533 connector
- 1 USB-C coupler
- 1 comma 3X
- 1 harness box
- 1 long OBD-C cable
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |SEAT|Leon 2014-20|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[1,13](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 J533 connector
- 1 USB-C coupler
- 1 comma 3X
- 1 harness box
- 1 long OBD-C cable
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Subaru|Ascent 2019-21|All[7](#footnotes)|openpilot available[1,8](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|
Parts- 1 RJ45 cable (7 ft)
- 1 Subaru A connector
- 1 comma 3X
- 1 comma power v2
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
Tools- 1 Pry Tool
- 1 Socket Wrench 8mm or 5/16" (deep)
|| diff --git a/selfdrive/car/chrysler/fingerprints.py b/selfdrive/car/chrysler/fingerprints.py index 1a04e79c7d..9ef991e8ed 100644 --- a/selfdrive/car/chrysler/fingerprints.py +++ b/selfdrive/car/chrysler/fingerprints.py @@ -66,6 +66,7 @@ FW_VERSIONS = { (Ecu.combinationMeter, 0x742, None): [ b'68402971AD', b'68454144AD', + b'68454152AB', ], (Ecu.srs, 0x744, None): [ b'68355363AB', @@ -87,6 +88,7 @@ FW_VERSIONS = { (Ecu.transmission, 0x7e1, None): [ b'05035707AA', b'68495807AA', + b'68495807AB', ], }, CAR.RAM_1500: { @@ -98,6 +100,7 @@ FW_VERSIONS = { b'68294063AH', b'68294063AI', b'68434846AC', + b'68434849AC', b'68434858AC', b'68434859AC', b'68434860AC', @@ -112,7 +115,10 @@ FW_VERSIONS = { b'68453513AC', b'68453513AD', b'68453514AD', + b'68505633AB', + b'68510277AG', b'68510280AG', + b'68510282AG', b'68510282AH', b'68510283AG', b'68527346AE', @@ -120,6 +126,7 @@ FW_VERSIONS = { b'68527375AD', b'68527382AE', b'68527387AE', + b'68631942AA', ], (Ecu.srs, 0x744, None): [ b'68428609AB', @@ -192,15 +199,19 @@ FW_VERSIONS = { b'05149591AD ', b'05149591AE ', b'05149592AE ', + b'05149600AD ', b'05149846AA ', b'05149848AA ', + b'05190341AD', b'68378695AJ ', b'68378696AJ ', b'68378701AI ', b'68378748AL ', b'68378758AM ', b'68448163AJ', + b'68448163AK', b'68448163AL', + b'68448165AG', b'68448165AK', b'68455119AC ', b'68455145AC ', @@ -218,11 +229,13 @@ FW_VERSIONS = { b'68539650AF', b'68586101AA ', b'68586105AB ', + b'68629926AC ', ], (Ecu.transmission, 0x7e1, None): [ b'05035706AD', b'05036069AA', b'05149536AC', + b'05149537AC', b'68360078AL', b'68360080AM', b'68360081AM', @@ -240,6 +253,7 @@ FW_VERSIONS = { b'68520867AE', b'68520867AF', b'68540431AB', + b'68629936AC', ], }, CAR.RAM_HD: { diff --git a/selfdrive/car/chrysler/values.py b/selfdrive/car/chrysler/values.py index 055efba829..2a3bceec96 100644 --- a/selfdrive/car/chrysler/values.py +++ b/selfdrive/car/chrysler/values.py @@ -74,7 +74,7 @@ CAR_INFO: Dict[str, Optional[Union[ChryslerCarInfo, List[ChryslerCarInfo]]]] = { ], CAR.JEEP_GRAND_CHEROKEE: ChryslerCarInfo("Jeep Grand Cherokee 2016-18", video_link="https://www.youtube.com/watch?v=eLR9o2JkuRk"), CAR.JEEP_GRAND_CHEROKEE_2019: ChryslerCarInfo("Jeep Grand Cherokee 2019-21", video_link="https://www.youtube.com/watch?v=jBe4lWnRSu4"), - CAR.RAM_1500: ChryslerCarInfo("Ram 1500 2019-23", car_parts=CarParts.common([CarHarness.ram])), + CAR.RAM_1500: ChryslerCarInfo("Ram 1500 2019-24", car_parts=CarParts.common([CarHarness.ram])), CAR.RAM_HD: [ ChryslerCarInfo("Ram 2500 2020-24", car_parts=CarParts.common([CarHarness.ram])), ChryslerCarInfo("Ram 3500 2019-22", car_parts=CarParts.common([CarHarness.ram])), diff --git a/selfdrive/car/honda/fingerprints.py b/selfdrive/car/honda/fingerprints.py index 0d174f91fb..25fb88aa39 100644 --- a/selfdrive/car/honda/fingerprints.py +++ b/selfdrive/car/honda/fingerprints.py @@ -928,6 +928,8 @@ FW_VERSIONS = { (Ecu.transmission, 0x18da1ef1, None): [ b'28101-5EY-A050\x00\x00', b'28101-5EY-A100\x00\x00', + b'28101-5EY-A430\x00\x00', + b'28101-5EY-A500\x00\x00', b'28101-5EZ-A050\x00\x00', b'28101-5EZ-A060\x00\x00', b'28101-5EZ-A100\x00\x00', @@ -947,6 +949,7 @@ FW_VERSIONS = { b'37805-RLV-B220\x00\x00', b'37805-RLV-B420\x00\x00', b'37805-RLV-B430\x00\x00', + b'37805-RLV-B620\x00\x00', b'37805-RLV-B710\x00\x00', b'37805-RLV-B720\x00\x00', b'37805-RLV-C430\x00\x00', @@ -958,6 +961,7 @@ FW_VERSIONS = { b'37805-RLV-L090\x00\x00', b'37805-RLV-L160\x00\x00', b'37805-RLV-L180\x00\x00', + b'37805-RLV-L350\x00\x00', b'37805-RLV-L410\x00\x00', b'37805-RLV-L430\x00\x00', b'37805-RLV-L850\x00\x00', @@ -987,6 +991,7 @@ FW_VERSIONS = { b'36161-TG7-D520\x00\x00', b'36161-TG7-D630\x00\x00', b'36161-TG7-Y630\x00\x00', + b'36161-TG8-A410\x00\x00', b'36161-TG8-A520\x00\x00', b'36161-TG8-A630\x00\x00', b'36161-TG8-A720\x00\x00', @@ -1031,6 +1036,7 @@ FW_VERSIONS = { b'78109-TG8-AJ10\x00\x00', b'78109-TG8-AJ20\x00\x00', b'78109-TG8-AK20\x00\x00', + b'78109-TG8-AS20\x00\x00', b'78109-TGS-AB10\x00\x00', b'78109-TGS-AC10\x00\x00', b'78109-TGS-AD10\x00\x00', diff --git a/tools/lib/helpers.py b/tools/lib/helpers.py index c184efe6a2..e2c8d07dd8 100644 --- a/tools/lib/helpers.py +++ b/tools/lib/helpers.py @@ -11,7 +11,7 @@ class RE: SEGMENT_NAME = r'{}(?:--|/)(?P[0-9]+)'.format(ROUTE_NAME) INDEX = r'-?[0-9]+' SLICE = r'(?P{})?:?(?P{})?:?(?P{})?'.format(INDEX, INDEX, INDEX) - SEGMENT_RANGE = r'{}(?:--|/)?(?P({}))?/?(?P([qr]))?'.format(ROUTE_NAME, SLICE) + SEGMENT_RANGE = r'{}(?:--|/)?(?P({}))?/?(?P([qra]))?'.format(ROUTE_NAME, SLICE) BOOTLOG_NAME = ROUTE_NAME EXPLORER_FILE = r'^(?P{})--(?P[a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index c1f4f0f4a9..c29662dfbc 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -16,6 +16,7 @@ from typing import Iterable, Iterator, List, Type from urllib.parse import parse_qs, urlparse from cereal import log as capnp_log +from openpilot.common.swaglog import cloudlog from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url from openpilot.tools.lib.openpilotci import get_url from openpilot.tools.lib.filereader import FileReader, file_exists @@ -71,8 +72,8 @@ class _LogFileReader: class ReadMode(enum.StrEnum): RLOG = "r" # only read rlogs QLOG = "q" # only read qlogs - #AUTO = "a" # default to rlogs, fallback to qlogs, not supported yet - + AUTO = "a" # default to rlogs, fallback to qlogs + AUTO_INTERACIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user def create_slice_from_string(s: str): m = re.fullmatch(RE.SLICE, s) @@ -86,44 +87,86 @@ def create_slice_from_string(s: str): return start return slice(start, end, step) -def parse_slice(sr: SegmentRange, route: Route): - segs = np.arange(route.max_seg_number+1) +def auto_strategy(rlog_paths, qlog_paths, interactive): + # auto select logs based on availability + if any(rlog is None or not file_exists(rlog) for rlog in rlog_paths): + if interactive: + if input("Some rlogs were not found, would you like to fallback to qlogs for those segments? (y/n) ").lower() != "y": + return rlog_paths + else: + cloudlog.warning("Some rlogs were not found, falling back to qlogs for those segments...") + + return [rlog if (rlog is not None and file_exists(rlog)) else (qlog if (qlog is not None and file_exists(qlog)) else None) + for (rlog, qlog) in zip(rlog_paths, qlog_paths, strict=True)] + return rlog_paths + +def apply_strategy(mode: ReadMode, rlog_paths, qlog_paths): + if mode == ReadMode.RLOG: + return rlog_paths + elif mode == ReadMode.QLOG: + return qlog_paths + elif mode == ReadMode.AUTO: + return auto_strategy(rlog_paths, qlog_paths, False) + elif mode == ReadMode.AUTO_INTERACIVE: + return auto_strategy(rlog_paths, qlog_paths, True) + +def parse_slice(sr: SegmentRange): s = create_slice_from_string(sr._slice) - return segs[s] if isinstance(s, slice) else [segs[s]] + if isinstance(s, slice): + if s.stop is None or s.stop < 0 or (s.start is not None and s.start < 0): # we need the number of segments in order to parse this slice + segs = np.arange(sr.get_max_seg_number()+1) + else: + segs = np.arange(s.stop + 1) + return segs[s] + else: + if s < 0: + s = sr.get_max_seg_number() + s + 1 + return [s] + +def comma_api_source(sr: SegmentRange, mode: ReadMode): + segs = parse_slice(sr) -def comma_api_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): - segs = parse_slice(sr, route) + route = Route(sr.route_name) - log_paths = route.log_paths() if mode == ReadMode.RLOG else route.qlog_paths() + rlog_paths = [route.log_paths()[seg] for seg in segs] + qlog_paths = [route.log_paths()[seg] for seg in segs] - invalid_segs = [seg for seg in segs if log_paths[seg] is None] + return apply_strategy(mode, rlog_paths, qlog_paths) - assert not len(invalid_segs), f"Some of the requested segments are not available: {invalid_segs}" +def internal_source(sr: SegmentRange, mode: ReadMode): + segs = parse_slice(sr) - return [(log_paths[seg]) for seg in segs] + def get_internal_url(sr: SegmentRange, seg, file): + return f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{file}.bz2" -def internal_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): - segs = parse_slice(sr, route) + rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in segs] + qlog_paths = [get_internal_url(sr, seg, "qlog") for seg in segs] - return [f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{'rlog' if mode == ReadMode.RLOG else 'qlog'}.bz2" for seg in segs] + return apply_strategy(mode, rlog_paths, qlog_paths) -def openpilotci_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): - segs = parse_slice(sr, route) +def openpilotci_source(sr: SegmentRange, mode: ReadMode): + segs = parse_slice(sr) - return [get_url(sr.route_name, seg, 'rlog' if mode == ReadMode.RLOG else 'qlog') for seg in segs] + rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in segs] + qlog_paths = [get_url(sr.route_name, seg, "qlog") for seg in segs] -def comma_car_segments_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): - segs = parse_slice(sr, route) + return apply_strategy(mode, rlog_paths, qlog_paths) + +def comma_car_segments_source(sr: SegmentRange, mode=ReadMode.RLOG): + segs = parse_slice(sr) return [get_comma_segments_url(sr.route_name, seg) for seg in segs] def direct_source(file_or_url): return [file_or_url] +def get_invalid_files(files): + return [f for f in files if f is None or not file_exists(f)] + def check_source(source, *args): try: files = source(*args) - assert all(file_exists(f) for f in files) + assert len(get_invalid_files(files)) == 0 return True, files except Exception: return False, None @@ -176,11 +219,10 @@ class LogReader: return direct_source(identifier) sr = SegmentRange(parsed) - route = Route(sr.route_name) mode = self.default_mode if sr.selector is None else ReadMode(sr.selector) source = self.default_source if source is None else source - return source(sr, route, mode) + return source(sr, mode) def __init__(self, identifier: str | List[str], default_mode=ReadMode.RLOG, default_source=auto_source, sort_by_time=False, only_union_types=False): self.default_mode = default_mode @@ -209,6 +251,9 @@ class LogReader: def reset(self): self.logreader_identifiers = self._parse_identifiers(self.identifier) + invalid_count = len(get_invalid_files(self.logreader_identifiers)) + assert invalid_count == 0, f"{invalid_count}/{len(self.logreader_identifiers)} invalid log(s) found, please ensure all logs \ +are uploaded or auto fallback to qlogs with '/a' selector at the end of the route name." @staticmethod def from_bytes(dat): diff --git a/tools/lib/route.py b/tools/lib/route.py index d3df93ccda..f7c3c432c8 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -1,5 +1,6 @@ import os import re +from functools import cache from urllib.parse import urlparse from collections import defaultdict from itertools import chain @@ -231,11 +232,23 @@ class SegmentName: def __str__(self) -> str: return self._canonical_name +@cache +def get_max_seg_number_cached(sr: 'SegmentRange'): + try: + api = CommaApi(get_token()) + return api.get("/v1/route/" + sr.route_name.replace("/", "|"))["segment_numbers"][-1] + except Exception as e: + raise Exception("unable to get max_segment_number. ensure you have access to this route or the route is public.") from e + + class SegmentRange: def __init__(self, segment_range: str): self.m = re.fullmatch(RE.SEGMENT_RANGE, segment_range) assert self.m, f"Segment range is not valid {segment_range}" + def get_max_seg_number(self): + return get_max_seg_number_cached(self) + @property def route_name(self): return self.m.group("route_name") @@ -255,3 +268,6 @@ class SegmentRange: @property def selector(self): return self.m.group("selector") + + def __str__(self): + return f"{self.dongle_id}/{self.timestamp}" + (f"/{self._slice}" if self._slice else "") + (f"/{self.selector}" if self.selector else "") diff --git a/tools/lib/tests/test_logreader.py b/tools/lib/tests/test_logreader.py index 74ee82d152..f7874a3fb3 100644 --- a/tools/lib/tests/test_logreader.py +++ b/tools/lib/tests/test_logreader.py @@ -6,7 +6,7 @@ import pytest from parameterized import parameterized import requests from openpilot.tools.lib.logreader import LogReader, parse_indirect, parse_slice, ReadMode -from openpilot.tools.lib.route import Route, SegmentRange +from openpilot.tools.lib.route import SegmentRange NUM_SEGS = 17 # number of segments in the test route ALL_SEGS = list(np.arange(NUM_SEGS)) @@ -42,10 +42,21 @@ class TestLogReader(unittest.TestCase): def test_indirect_parsing(self, identifier, expected): parsed, _, _ = parse_indirect(identifier) sr = SegmentRange(parsed) - route = Route(sr.route_name) - segs = parse_slice(sr, route) + segs = parse_slice(sr) self.assertListEqual(list(segs), expected) + @parameterized.expand([ + (f"{TEST_ROUTE}", f"{TEST_ROUTE}"), + (f"{TEST_ROUTE.replace('/', '|')}", f"{TEST_ROUTE}"), + (f"{TEST_ROUTE}--5", f"{TEST_ROUTE}/5"), + (f"{TEST_ROUTE}/0/q", f"{TEST_ROUTE}/0/q"), + (f"{TEST_ROUTE}/5:6/r", f"{TEST_ROUTE}/5:6/r"), + (f"{TEST_ROUTE}/5", f"{TEST_ROUTE}/5"), + ]) + def test_canonical_name(self, identifier, expected): + sr = SegmentRange(identifier) + self.assertEqual(str(sr), expected) + def test_direct_parsing(self): qlog = tempfile.NamedTemporaryFile(mode='wb', delete=False) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 97c0a639a7..5c6f187eee 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -1,3 +1,4 @@ +import logging import os import time import threading @@ -12,6 +13,7 @@ from openpilot.system.hardware.hw import Paths K = 1000 CHUNK_SIZE = 1000 * K +logging.getLogger("urllib3").setLevel(logging.WARNING) def hash_256(link): hsh = str(sha256((link.split("?")[0]).encode('utf-8')).hexdigest()) diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 292db5a50a..cc21095414 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -13,7 +13,7 @@ from functools import partial from openpilot.common.basedir import BASEDIR from openpilot.tools.lib.helpers import save_log -from openpilot.tools.lib.logreader import LogReader +from openpilot.tools.lib.logreader import LogReader, ReadMode juggle_dir = os.path.dirname(os.path.realpath(__file__)) @@ -73,7 +73,7 @@ def process(can, lr): return [d for d in lr if can or d.which() not in ['can', 'sendcan']] def juggle_route(route_or_segment_name, can, layout, dbc=None): - sr = LogReader(route_or_segment_name) + sr = LogReader(route_or_segment_name, default_mode=ReadMode.AUTO_INTERACIVE) all_data = sr.run_across_segments(24, partial(process, can))