diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index 973101d4ca..537214d14c 100755 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -23,9 +23,8 @@ from openpilot.selfdrive.car.tests.routes import non_tested_cars, routes, CarTes from openpilot.selfdrive.controls.controlsd import Controls from openpilot.selfdrive.test.helpers import read_segment_list from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT -from openpilot.tools.lib.openpilotci import get_url +from openpilot.tools.lib.comma_car_segments import get_url from openpilot.tools.lib.logreader import LogReader -from openpilot.tools.lib.sanitizer import sanitize from openpilot.tools.lib.route import Route, SegmentName, RouteName from panda.tests.libpanda import libpanda_py @@ -86,7 +85,6 @@ class TestCarModelBase(unittest.TestCase): @classmethod def get_testing_data_from_logreader(cls, lr): - lr = sanitize(lr) car_fw = [] can_msgs = [] cls.elm_frame = None diff --git a/tools/lib/comma_car_segments.py b/tools/lib/comma_car_segments.py new file mode 100644 index 0000000000..995b26d8e6 --- /dev/null +++ b/tools/lib/comma_car_segments.py @@ -0,0 +1,82 @@ +import os +import requests + +# Forks with additional car support can fork the commaCarSegments repo on huggingface or host the LFS files themselves +COMMA_CAR_SEGMENTS_REPO = os.environ.get("COMMA_CAR_SEGMENTS_REPO", "https://huggingface.co/datasets/commaai/commaCarSegments") +COMMA_CAR_SEGMENTS_BRANCH = os.environ.get("COMMA_CAR_SEGMENTS_BRANCH", "main") + +COMMA_CAR_SEGMENTS_LFS_INSTANCE = os.environ.get("COMMA_CAR_SEGMENTS_LFS_INSTANCE", "https://huggingface.co/datasets/commaai/commaCarSegments") + +def get_comma_car_segments_database(): + return requests.get(get_repo_raw_url("database.json")).json() + + +# Helpers related to interfacing with the openpilot-data repository, which contains a collection of public segments for users to perform validation on. + +def parse_lfs_pointer(text): + header, lfs_version = text.splitlines()[0].split(" ") + assert header == "version" + assert lfs_version == "https://git-lfs.github.com/spec/v1" + + header, oid_raw = text.splitlines()[1].split(" ") + assert header == "oid" + header, oid = oid_raw.split(":") + assert header == "sha256" + + header, size = text.splitlines()[2].split(" ") + assert header == "size" + + return oid, size + +def get_lfs_file_url(oid, size): + data = { + "operation": "download", + "transfers": [ "basic" ], + "objects": [ + { + "oid": oid, + "size": int(size) + } + ], + "hash_algo": "sha256" + } + + headers = { + "Accept": "application/vnd.git-lfs+json", + "Content-Type": "application/vnd.git-lfs+json" + } + + response = requests.post(f"{COMMA_CAR_SEGMENTS_LFS_INSTANCE}.git/info/lfs/objects/batch", json=data, headers=headers) + + assert response.ok + + obj = response.json()["objects"][0] + + assert "error" not in obj, obj + + return obj["actions"]["download"]["href"] + +def get_repo_raw_url(path): + if "huggingface" in COMMA_CAR_SEGMENTS_REPO: + return f"{COMMA_CAR_SEGMENTS_REPO}/raw/{COMMA_CAR_SEGMENTS_BRANCH}/{path}" + +def get_repo_url(path): + # Automatically switch to LFS if we are requesting a file that is stored in LFS + + response = requests.head(get_repo_raw_url(path)) + + if "text/plain" in response.headers.get("content-type"): + # This is an LFS pointer, so download the raw data from lfs + response = requests.get(get_repo_raw_url(path)) + assert response.status_code == 200 + oid, size = parse_lfs_pointer(response.text) + + return get_lfs_file_url(oid, size) + else: + # File has not been uploaded to LFS yet + # (either we are on a fork where the data hasn't been pushed to LFS yet, or the CI job to push hasn't finished) + return get_repo_raw_url(path) + + +def get_url(route, segment, file="rlog.bz2"): + return get_repo_url(f"segments/{route.replace('|', '/')}/{segment}/{file}") diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index c7f9f9fae0..9334f542f6 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -209,6 +209,10 @@ class LogReader: return _LogFileReader("", dat=dat) +def get_first_message(lr: LogIterable, msg_type): + return next(filter(lambda m: m.which() == msg_type, lr), None) + + if __name__ == "__main__": import codecs # capnproto <= 0.8.0 throws errors converting byte data to string diff --git a/tools/lib/tests/test_comma_car_segments.py b/tools/lib/tests/test_comma_car_segments.py new file mode 100644 index 0000000000..1d6088d821 --- /dev/null +++ b/tools/lib/tests/test_comma_car_segments.py @@ -0,0 +1,41 @@ + + +import unittest + +import requests +from openpilot.tools.lib.comma_car_segments import get_comma_car_segments_database, get_url +from openpilot.tools.lib.logreader import LogReader, get_first_message +from openpilot.tools.lib.route import SegmentRange + + +class TestCommaCarSegments(unittest.TestCase): + def test_database(self): + database = get_comma_car_segments_database() + + platforms = database.keys() + + assert len(platforms) > 100 + + def test_download_segment(self): + database = get_comma_car_segments_database() + + fp = "SUBARU FORESTER 2019" + + segment = database[fp][0] + + sr = SegmentRange(segment) + + url = get_url(sr.route_name, sr._slice) + + resp = requests.get(url) + self.assertEqual(resp.status_code, 200) + + lr = LogReader(url) + + CP = get_first_message(lr, "carParams").carParams + + self.assertEqual(CP.carFingerprint, fp) + + +if __name__ == "__main__": + unittest.main()