From abfa8fcda9c43350cad7628d5443a0effbb03b2f Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Sat, 27 Jan 2024 01:07:29 -0800 Subject: [PATCH] CommaCarSegments: BIG Database! (#31169) * sanetized selector * god damn * add to re * slog branch * slog * source * big * not slog * cleanup * no space * works! * trigger * main! old-commit-hash: 824ddffec8c67758f3ea080c75e805d4b27f38ef --- .../examples/ford_vin_fingerprint.ipynb | 65 +++++++++++-------- tools/lib/helpers.py | 2 +- tools/lib/logreader.py | 24 ++++--- 3 files changed, 55 insertions(+), 36 deletions(-) diff --git a/tools/car_porting/examples/ford_vin_fingerprint.ipynb b/tools/car_porting/examples/ford_vin_fingerprint.ipynb index b020b1100b..21d1cb62c0 100644 --- a/tools/car_porting/examples/ford_vin_fingerprint.ipynb +++ b/tools/car_porting/examples/ford_vin_fingerprint.ipynb @@ -2,17 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 5, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /home/batman\n" - ] - } - ], + "outputs": [], "source": [ "\"\"\"In this example, we use the public comma car segments database to check if vin fingerprinting is feasible for ford.\"\"\"\n", "\n", @@ -27,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -62,55 +54,74 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Skipping platform: FORD F-150 14TH GEN, no data available\n" + "Got 287 segments for platform FORD BRONCO SPORT 1ST GEN, sampling 5 segments\n", + "Got 137 segments for platform FORD ESCAPE 4TH GEN, sampling 5 segments\n", + "Got 1041 segments for platform FORD EXPLORER 6TH GEN, sampling 5 segments\n", + "Got 5 segments for platform FORD F-150 14TH GEN, sampling 5 segments\n", + "Got 56 segments for platform FORD FOCUS 4TH GEN, sampling 5 segments\n", + "Got 637 segments for platform FORD MAVERICK 1ST GEN, sampling 5 segments\n", + "Got 3 segments for platform FORD F-150 LIGHTNING 1ST GEN, sampling 3 segments\n", + "Got 3 segments for platform FORD MUSTANG MACH-E 1ST GEN, sampling 3 segments\n" ] } ], "source": [ + "import random\n", + "\n", + "MAX_SEGS_PER_PLATFORM = 5\n", + "\n", "VINS_TO_CHECK = set()\n", "\n", "for platform in platforms:\n", " if platform not in database:\n", " print(f\"Skipping platform: {platform}, no data available\")\n", " continue\n", + " \n", + " all_segments = database[platform]\n", "\n", - " for segment in database[platform]:\n", + " NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n", + "\n", + " print(f\"Got {len(all_segments)} segments for platform {platform}, sampling {NUM_SEGMENTS} segments\")\n", + "\n", + " segments = random.sample(all_segments, NUM_SEGMENTS)\n", + "\n", + " for segment in segments:\n", " lr = LogReader(segment)\n", " CP = lr.first(\"carParams\")\n", + " if \"FORD\" not in CP.carFingerprint:\n", + " print(segment, CP.carFingerprint)\n", " VINS_TO_CHECK.add((CP.carVin, CP.carFingerprint))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "vin: 3FTTW8E31PRXXXXXX real platform: FORD MAVERICK 1ST GEN determined platform: mock correct: False\n", - "vin: 1FM5K8GC7LGXXXXXX real platform: FORD EXPLORER 6TH GEN determined platform: mock correct: False\n", - "vin: 3FTTW8E99NRXXXXXX real platform: FORD MAVERICK 1ST GEN determined platform: mock correct: False\n", - "vin: 1FM5K8GC7NGXXXXXX real platform: FORD EXPLORER 6TH GEN determined platform: mock correct: False\n", - "vin: 1FM5K7LC0MGXXXXXX real platform: FORD EXPLORER 6TH GEN determined platform: mock correct: False\n", - "vin: 1FM5K8HC7MGXXXXXX real platform: FORD EXPLORER 6TH GEN determined platform: mock correct: False\n", + "vin: 3FTTW8E34PRXXXXXX real platform: FORD MAVERICK 1ST GEN determined platform: mock correct: False\n", + "vin: 00000000000XXXXXX real platform: FORD F-150 14TH GEN determined platform: mock correct: False\n", "vin: 3FMTK3SU0MMXXXXXX real platform: FORD MUSTANG MACH-E 1ST GEN determined platform: FORD MUSTANG MACH-E 1ST GEN correct: True\n", - "vin: WF0NXXGCHNJXXXXXX real platform: FORD FOCUS 4TH GEN determined platform: mock correct: False\n", "vin: 1FTVW1EL4NWXXXXXX real platform: FORD F-150 LIGHTNING 1ST GEN determined platform: FORD F-150 LIGHTNING 1ST GEN correct: True\n", - "vin: 00000000000XXXXXX real platform: FORD ESCAPE 4TH GEN determined platform: mock correct: False\n", - "vin: 3FTTW8F98NRXXXXXX real platform: FORD MAVERICK 1ST GEN determined platform: mock correct: False\n", - "vin: 5LM5J7XC9LGXXXXXX real platform: FORD EXPLORER 6TH GEN determined platform: mock correct: False\n", - "vin: 5LM5J7XC8MGXXXXXX real platform: FORD EXPLORER 6TH GEN determined platform: mock correct: False\n", + "vin: WF0NXXGCHNJXXXXXX real platform: FORD FOCUS 4TH GEN determined platform: mock correct: False\n", + "vin: 1FMCU9J94MUXXXXXX real platform: FORD ESCAPE 4TH GEN determined platform: mock correct: False\n", + "vin: 3FTTW8E33NRXXXXXX real platform: FORD MAVERICK 1ST GEN determined platform: mock correct: False\n", "vin: 3FMCR9B69NRXXXXXX real platform: FORD BRONCO SPORT 1ST GEN determined platform: mock correct: False\n", - "vin: 1FMCU9J94MUXXXXXX real platform: FORD ESCAPE 4TH GEN determined platform: mock correct: False\n" + "vin: 1FM5K8GC7LGXXXXXX real platform: FORD EXPLORER 6TH GEN determined platform: mock correct: False\n", + "vin: 5LM5J7XC9LGXXXXXX real platform: FORD EXPLORER 6TH GEN determined platform: mock correct: False\n", + "vin: 5LM5J7XC1LGXXXXXX real platform: FORD EXPLORER 6TH GEN determined platform: mock correct: False\n", + "vin: 3FTTW8F97NRXXXXXX real platform: FORD MAVERICK 1ST GEN determined platform: mock correct: False\n", + "vin: 3FTTW8E99NRXXXXXX real platform: FORD MAVERICK 1ST GEN determined platform: mock correct: False\n" ] } ], diff --git a/tools/lib/helpers.py b/tools/lib/helpers.py index 37bbaf0c4a..423f207b4d 100644 --- a/tools/lib/helpers.py +++ b/tools/lib/helpers.py @@ -14,7 +14,7 @@ class RE: INDEX = r'-?[0-9]+' SLICE = r'(?P{})?:?(?P{})?:?(?P{})?'.format(INDEX, INDEX, INDEX) - SEGMENT_RANGE = r'{}(?:--|/)?(?P({}))?/?(?P([qra]))?'.format(ROUTE_NAME, SLICE) + SEGMENT_RANGE = r'{}(?:--|/)?(?P({}))?/?(?P([qras]))?'.format(ROUTE_NAME, SLICE) BOOTLOG_NAME = ROUTE_NAME diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index d6c1cc90c2..e86690f3c3 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -72,6 +72,7 @@ class _LogFileReader: class ReadMode(enum.StrEnum): RLOG = "r" # only read rlogs QLOG = "q" # only read qlogs + SANITIZED = "s" # read from the commaCarSegments database AUTO = "a" # default to rlogs, fallback to qlogs AUTO_INTERACIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user @@ -169,17 +170,24 @@ def check_source(source, *args): try: files = source(*args) assert next(get_invalid_files(files), None) is None - return True, files - except Exception: - return False, None + return None, files + except Exception as e: + return e, None -def auto_source(*args): +def auto_source(sr: SegmentRange, mode=ReadMode.RLOG): + if mode == ReadMode.SANITIZED: + return comma_car_segments_source(sr, mode) + + exceptions = [] # Automatically determine viable source - for source in [comma_car_segments_source, internal_source, openpilotci_source]: - valid, ret = check_source(source, *args) - if valid: + for source in [internal_source, openpilotci_source, comma_api_source, comma_car_segments_source]: + exception, ret = check_source(source, sr, mode) + if exception is None: return ret - return comma_api_source(*args) + else: + exceptions.append(exception) + + raise Exception(f"auto_source could not find any valid source, exceptions for sources: {exceptions}") def parse_useradmin(identifier): if "useradmin.comma.ai" in identifier: