From ed0f50635907ea31b5cb1acc265c32c57e82272e Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 5 Oct 2023 01:24:05 -0700 Subject: [PATCH 1/8] Toyota: add missing engine FW for TSS-P Camry Hybrid (#30177) * missing FW for 2ed432126627c66a * dups * dups --- selfdrive/car/toyota/values.py | 1 + 1 file changed, 1 insertion(+) diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index 38ce674fe3..0e6fb9caa0 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -590,6 +590,7 @@ FW_VERSIONS = { b'\x018966333X0000\x00\x00\x00\x00', b'\x018966333X4000\x00\x00\x00\x00', b'\x01896633T16000\x00\x00\x00\x00', + b'\x018966306L9000\x00\x00\x00\x00', b'\x028966306B2100\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00', b'\x028966306B2300\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00', b'\x028966306B2500\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00', From 29f10fd2aa0730681986631838f866462eb9af55 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Thu, 5 Oct 2023 16:48:56 +0800 Subject: [PATCH 2/8] cabana: fix parsing multiple line comment (#30178) fix parse error --- tools/cabana/dbc/dbcfile.cc | 7 ++++--- tools/cabana/tests/test_cabana.cc | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tools/cabana/dbc/dbcfile.cc b/tools/cabana/dbc/dbcfile.cc index 063f516ead..c923e4d5b2 100644 --- a/tools/cabana/dbc/dbcfile.cc +++ b/tools/cabana/dbc/dbcfile.cc @@ -107,7 +107,8 @@ void DBCFile::parse(const QString &content) { int multiplexor_cnt = 0; while (!stream.atEnd()) { ++line_num; - line = stream.readLine().trimmed(); + QString raw_line = stream.readLine(); + line = raw_line.trimmed(); if (line.startsWith("BO_ ")) { multiplexor_cnt = 0; auto match = bo_regexp.match(line); @@ -170,7 +171,7 @@ void DBCFile::parse(const QString &content) { } } else if (line.startsWith("CM_ BO_")) { if (!line.endsWith("\";")) { - int pos = stream.pos() - line.length() - 1; + int pos = stream.pos() - raw_line.length() - 1; line = content.mid(pos, content.indexOf("\";", pos)); } auto match = msg_comment_regexp.match(line); @@ -180,7 +181,7 @@ void DBCFile::parse(const QString &content) { } } else if (line.startsWith("CM_ SG_ ")) { if (!line.endsWith("\";")) { - int pos = stream.pos() - line.length() - 1; + int pos = stream.pos() - raw_line.length() - 1; line = content.mid(pos, content.indexOf("\";", pos)); } auto match = sg_comment_regexp.match(line); diff --git a/tools/cabana/tests/test_cabana.cc b/tools/cabana/tests/test_cabana.cc index d114f72ea5..791dfd1329 100644 --- a/tools/cabana/tests/test_cabana.cc +++ b/tools/cabana/tests/test_cabana.cc @@ -69,7 +69,7 @@ TEST_CASE("Parse can messages") { } } -TEST_CASE("Parse dbc") { +TEST_CASE("parse_dbc") { QString content = R"( BO_ 160 message_1: 8 EON SG_ signal_1 : 0|12@1+ (1,0) [0|4095] "unit" XXX @@ -83,7 +83,7 @@ VAL_ 160 signal_1 0 "disabled" 1.2 "initializing" 2 "fault"; CM_ BO_ 160 "message comment" ; CM_ SG_ 160 signal_1 "signal comment"; -CM_ SG_ 160 signal_2 "multiple line comment +CM_ SG_ 160 signal_2 "multiple line comment 1 2 ";)"; @@ -113,7 +113,7 @@ CM_ SG_ 160 signal_2 "multiple line comment REQUIRE(sig_1->val_desc[2] == std::pair{2, "fault"}); auto &sig_2 = msg->sigs[1]; - REQUIRE(sig_2->comment == "multiple line comment\n1\n2"); + REQUIRE(sig_2->comment == "multiple line comment \n1\n2"); // multiplexed signals msg = file.msg(162); From 3aa74c28fcd426f73da2201b2fdd7ac2be21ba71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20R=C4=85czy?= Date: Thu, 5 Oct 2023 13:59:00 -0700 Subject: [PATCH 3/8] CI: add regen job (#30157) * Ability to whitelist/blacklist cars in regen_all * Add CI job for regen, running on 2 segments * Run regen_all, not regen * Use coverage run * Add test_regen * Use test_regen in ci test * Add test case names * ONNXCPU = 1 * Add mazda segment * Use RUN_CL * build cl image before running * unset PYTHONWARNINGS * Create regen cache * Replace daemons with processes * Skip ford * Skip mazda * Add comment about commented segments * Update selfdrive/test/process_replay/test_regen.py Co-authored-by: Adeeb Shihadeh * Remove unset pythonwarnings --------- Co-authored-by: Adeeb Shihadeh --- .github/workflows/selfdrive_tests.yaml | 27 ++++++++++++ selfdrive/test/process_replay/regen_all.py | 14 +++++- selfdrive/test/process_replay/test_regen.py | 48 +++++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) create mode 100755 selfdrive/test/process_replay/test_regen.py diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index d2174b409f..c5820a5f5a 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -228,6 +228,33 @@ jobs: - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v3 + regen: + name: regen + runs-on: 'ubuntu-20.04' + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + - name: Cache test routes + id: dependency-cache + uses: actions/cache@v3 + with: + path: .ci_cache/comma_download_cache + key: regen-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_regen.py') }} + - name: Build base Docker image + run: eval "$BUILD" + - name: Build Docker image + run: eval "$BUILD_CL" + - name: Build openpilot + run: | + ${{ env.RUN }} "scons -j$(nproc)" + - name: Run regen + timeout-minutes: 30 + run: | + ${{ env.RUN_CL }} "ONNXCPU=1 $PYTEST -n auto --dist=loadscope selfdrive/test/process_replay/test_regen.py && \ + chmod -R 777 /tmp/comma_download_cache" + test_modeld: name: model tests runs-on: ubuntu-20.04 diff --git a/selfdrive/test/process_replay/regen_all.py b/selfdrive/test/process_replay/regen_all.py index df7c76a14d..b797b9b0da 100755 --- a/selfdrive/test/process_replay/regen_all.py +++ b/selfdrive/test/process_replay/regen_all.py @@ -29,15 +29,25 @@ def regen_job(segment, upload, disable_tqdm): if __name__ == "__main__": + all_cars = {car for car, _ in segments} + parser = argparse.ArgumentParser(description="Generate new segments from old ones") parser.add_argument("-j", "--jobs", type=int, default=1) parser.add_argument("--no-upload", action="store_true") + parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars, + help="Whitelist given cars from the test (e.g. HONDA)") + parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[], + help="Blacklist given cars from the test (e.g. HONDA)") args = parser.parse_args() + tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars) + tested_cars = {c.upper() for c in tested_cars} + tested_segments = [(car, segment) for car, segment in segments if car in tested_cars] + with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool: - p = pool.map(regen_job, segments, [not args.no_upload] * len(segments), [args.jobs > 1] * len(segments)) + p = pool.map(regen_job, tested_segments, [not args.no_upload] * len(tested_segments), [args.jobs > 1] * len(tested_segments)) msg = "Copy these new segments into test_processes.py:" - for seg in tqdm(p, desc="Generating segments", total=len(segments)): + for seg in tqdm(p, desc="Generating segments", total=len(tested_segments)): msg += "\n" + str(seg) print() print() diff --git a/selfdrive/test/process_replay/test_regen.py b/selfdrive/test/process_replay/test_regen.py new file mode 100755 index 0000000000..ec1277a76c --- /dev/null +++ b/selfdrive/test/process_replay/test_regen.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import unittest + +from parameterized import parameterized + +from openpilot.selfdrive.test.process_replay.regen import regen_segment +from openpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled, CONFIGS +from openpilot.selfdrive.test.openpilotci import get_url +from openpilot.tools.lib.logreader import LogReader +from openpilot.tools.lib.framereader import FrameReader + +EXCLUDED_PROCESSES = {"dmonitoringd", "dmonitoringmodeld"} +TESTED_SEGMENTS = [ + ("PRIUS_C2", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA PRIUS 2017: NEO, pandaStateDEPRECATED, no peripheralState, sensorEventsDEPRECATED + # Enable these once regen on CI becomes faster or use them for different tests running controlsd in isolation + # ("MAZDA_C3", "bd6a637565e91581|2021-10-30--15-14-53--4"), # MAZDA.CX9_2021: TICI, incomplete managerState + # ("FORD_C3", "54827bf84c38b14f|2023-01-26--21-59-07--4"), # FORD.BRONCO_SPORT_MK1: TICI +] + + +def ci_setup_data_readers(route, sidx): + lr = LogReader(get_url(route, sidx, "rlog")) + # dm disabled + frs = { + 'roadCameraState': FrameReader(get_url(route, sidx, "fcamera")), + } + if next((True for m in lr if m.which() == "wideRoadCameraState"), False): + frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera")) + + return lr, frs + + +class TestRegen(unittest.TestCase): + @parameterized.expand(TESTED_SEGMENTS) + def test_engaged(self, case_name, segment): + tested_procs = [p for p in CONFIGS if p.proc_name not in EXCLUDED_PROCESSES] + + route, sidx = segment.rsplit("--", 1) + lr, frs = ci_setup_data_readers(route, sidx) + output_logs = regen_segment(lr, frs, processes=tested_procs, disable_tqdm=True) + + engaged = check_openpilot_enabled(output_logs) + self.assertTrue(engaged, f"openpilot not engaged in {case_name}") + + +if __name__=='__main__': + unittest.main() From f80db10720099d34fd19e3aa0be36d05647ea897 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 5 Oct 2023 14:53:49 -0700 Subject: [PATCH 4/8] UI: remove drive stats (#30183) * UI: remove drive stats * little more * vanish --- common/params.cc | 1 - selfdrive/ui/SConscript | 2 +- selfdrive/ui/qt/home.cc | 4 +- selfdrive/ui/qt/widgets/drive_stats.cc | 97 ------------------------ selfdrive/ui/qt/widgets/drive_stats.h | 25 ------ selfdrive/ui/translations/main_ar.ts | 27 ------- selfdrive/ui/translations/main_de.ts | 27 ------- selfdrive/ui/translations/main_fr.ts | 27 ------- selfdrive/ui/translations/main_ja.ts | 27 ------- selfdrive/ui/translations/main_ko.ts | 27 ------- selfdrive/ui/translations/main_pt-BR.ts | 27 ------- selfdrive/ui/translations/main_th.ts | 27 ------- selfdrive/ui/translations/main_tr.ts | 27 ------- selfdrive/ui/translations/main_zh-CHS.ts | 27 ------- selfdrive/ui/translations/main_zh-CHT.ts | 27 ------- 15 files changed, 2 insertions(+), 397 deletions(-) delete mode 100644 selfdrive/ui/qt/widgets/drive_stats.cc delete mode 100644 selfdrive/ui/qt/widgets/drive_stats.h diff --git a/common/params.cc b/common/params.cc index 9212a6edfc..3f6cb7044c 100644 --- a/common/params.cc +++ b/common/params.cc @@ -88,7 +88,6 @@ private: std::unordered_map keys = { {"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG}, {"ApiCache_Device", PERSISTENT}, - {"ApiCache_DriveStats", PERSISTENT}, {"ApiCache_NavDestinations", PERSISTENT}, {"AssistNowToken", PERSISTENT}, {"AthenadPid", PERSISTENT}, diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index 017ce66793..036164d6cc 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -20,7 +20,7 @@ if arch == "Darwin": qt_env['FRAMEWORKS'] += ['OpenCL'] qt_util = qt_env.Library("qt_util", ["#selfdrive/ui/qt/api.cc", "#selfdrive/ui/qt/util.cc"], LIBS=base_libs) -widgets_src = ["ui.cc", "qt/widgets/input.cc", "qt/widgets/drive_stats.cc", "qt/widgets/wifi.cc", +widgets_src = ["ui.cc", "qt/widgets/input.cc", "qt/widgets/wifi.cc", "qt/widgets/ssh_keys.cc", "qt/widgets/toggle.cc", "qt/widgets/controls.cc", "qt/widgets/offroad_alerts.cc", "qt/widgets/prime.cc", "qt/widgets/keyboard.cc", "qt/widgets/scrollview.cc", "qt/widgets/cameraview.cc", "#third_party/qrcode/QrCode.cc", diff --git a/selfdrive/ui/qt/home.cc b/selfdrive/ui/qt/home.cc index f93b590007..9dbe7cbae3 100644 --- a/selfdrive/ui/qt/home.cc +++ b/selfdrive/ui/qt/home.cc @@ -11,8 +11,6 @@ #ifdef ENABLE_MAPS #include "selfdrive/ui/qt/maps/map_settings.h" -#else -#include "selfdrive/ui/qt/widgets/drive_stats.h" #endif // HomeWindow: the container for the offroad and onroad UIs @@ -152,7 +150,7 @@ OffroadHome::OffroadHome(QWidget* parent) : QFrame(parent) { #ifdef ENABLE_MAPS left_widget->addWidget(new MapSettings); #else - left_widget->addWidget(new DriveStats); + left_widget->addWidget(new QWidget); #endif left_widget->addWidget(new PrimeAdWidget); left_widget->setStyleSheet("border-radius: 10px;"); diff --git a/selfdrive/ui/qt/widgets/drive_stats.cc b/selfdrive/ui/qt/widgets/drive_stats.cc deleted file mode 100644 index 31009f03ca..0000000000 --- a/selfdrive/ui/qt/widgets/drive_stats.cc +++ /dev/null @@ -1,97 +0,0 @@ -#include "selfdrive/ui/qt/widgets/drive_stats.h" - -#include -#include -#include -#include - -#include "common/params.h" -#include "selfdrive/ui/qt/request_repeater.h" -#include "selfdrive/ui/qt/util.h" - -static QLabel* newLabel(const QString& text, const QString &type) { - QLabel* label = new QLabel(text); - label->setProperty("type", type); - return label; -} - -DriveStats::DriveStats(QWidget* parent) : QFrame(parent) { - metric_ = Params().getBool("IsMetric"); - - QVBoxLayout* main_layout = new QVBoxLayout(this); - main_layout->setContentsMargins(50, 50, 50, 60); - - auto add_stats_layouts = [=](const QString &title, StatsLabels& labels) { - QGridLayout* grid_layout = new QGridLayout; - grid_layout->setVerticalSpacing(10); - grid_layout->setContentsMargins(0, 10, 0, 10); - - int row = 0; - grid_layout->addWidget(newLabel(title, "title"), row++, 0, 1, 3); - grid_layout->addItem(new QSpacerItem(0, 50), row++, 0, 1, 1); - - grid_layout->addWidget(labels.routes = newLabel("0", "number"), row, 0, Qt::AlignLeft); - grid_layout->addWidget(labels.distance = newLabel("0", "number"), row, 1, Qt::AlignLeft); - grid_layout->addWidget(labels.hours = newLabel("0", "number"), row, 2, Qt::AlignLeft); - - grid_layout->addWidget(newLabel((tr("Drives")), "unit"), row + 1, 0, Qt::AlignLeft); - grid_layout->addWidget(labels.distance_unit = newLabel(getDistanceUnit(), "unit"), row + 1, 1, Qt::AlignLeft); - grid_layout->addWidget(newLabel(tr("Hours"), "unit"), row + 1, 2, Qt::AlignLeft); - - main_layout->addLayout(grid_layout); - }; - - add_stats_layouts(tr("ALL TIME"), all_); - main_layout->addStretch(); - add_stats_layouts(tr("PAST WEEK"), week_); - - if (auto dongleId = getDongleId()) { - QString url = CommaApi::BASE_URL + "/v1.1/devices/" + *dongleId + "/stats"; - RequestRepeater* repeater = new RequestRepeater(this, url, "ApiCache_DriveStats", 30); - QObject::connect(repeater, &RequestRepeater::requestDone, this, &DriveStats::parseResponse); - } - - setStyleSheet(R"( - DriveStats { - background-color: #333333; - border-radius: 10px; - } - - QLabel[type="title"] { font-size: 51px; font-weight: 500; } - QLabel[type="number"] { font-size: 78px; font-weight: 500; } - QLabel[type="unit"] { font-size: 51px; font-weight: 300; color: #A0A0A0; } - )"); -} - -void DriveStats::updateStats() { - auto update = [=](const QJsonObject& obj, StatsLabels& labels) { - labels.routes->setText(QString::number((int)obj["routes"].toDouble())); - labels.distance->setText(QString::number(int(obj["distance"].toDouble() * (metric_ ? MILE_TO_KM : 1)))); - labels.distance_unit->setText(getDistanceUnit()); - labels.hours->setText(QString::number((int)(obj["minutes"].toDouble() / 60))); - }; - - QJsonObject json = stats_.object(); - update(json["all"].toObject(), all_); - update(json["week"].toObject(), week_); -} - -void DriveStats::parseResponse(const QString& response, bool success) { - if (!success) return; - - QJsonDocument doc = QJsonDocument::fromJson(response.trimmed().toUtf8()); - if (doc.isNull()) { - qDebug() << "JSON Parse failed on getting past drives statistics"; - return; - } - stats_ = doc; - updateStats(); -} - -void DriveStats::showEvent(QShowEvent* event) { - bool metric = Params().getBool("IsMetric"); - if (metric_ != metric) { - metric_ = metric; - updateStats(); - } -} diff --git a/selfdrive/ui/qt/widgets/drive_stats.h b/selfdrive/ui/qt/widgets/drive_stats.h deleted file mode 100644 index 5e2d96b240..0000000000 --- a/selfdrive/ui/qt/widgets/drive_stats.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include -#include - -class DriveStats : public QFrame { - Q_OBJECT - -public: - explicit DriveStats(QWidget* parent = 0); - -private: - void showEvent(QShowEvent *event) override; - void updateStats(); - inline QString getDistanceUnit() const { return metric_ ? tr("KM") : tr("Miles"); } - - bool metric_; - QJsonDocument stats_; - struct StatsLabels { - QLabel *routes, *distance, *distance_unit, *hours; - } all_, week_; - -private slots: - void parseResponse(const QString &response, bool success); -}; diff --git a/selfdrive/ui/translations/main_ar.ts b/selfdrive/ui/translations/main_ar.ts index de5a04e760..4159f28c31 100644 --- a/selfdrive/ui/translations/main_ar.ts +++ b/selfdrive/ui/translations/main_ar.ts @@ -274,33 +274,6 @@ مراجعة - - DriveStats - - Drives - القيادة - - - Hours - ساعات - - - ALL TIME - كامل الوقت - - - PAST WEEK - الأسبوع الماضي - - - KM - كم - - - Miles - ميل - - DriverViewScene diff --git a/selfdrive/ui/translations/main_de.ts b/selfdrive/ui/translations/main_de.ts index c53134c5b3..e2e5771905 100644 --- a/selfdrive/ui/translations/main_de.ts +++ b/selfdrive/ui/translations/main_de.ts @@ -274,33 +274,6 @@ Überprüfen - - DriveStats - - Drives - Fahrten - - - Hours - Stunden - - - ALL TIME - Gesamtzeit - - - PAST WEEK - Letzte Woche - - - KM - KM - - - Miles - Meilen - - DriverViewScene diff --git a/selfdrive/ui/translations/main_fr.ts b/selfdrive/ui/translations/main_fr.ts index 95afab46a7..456efe6366 100644 --- a/selfdrive/ui/translations/main_fr.ts +++ b/selfdrive/ui/translations/main_fr.ts @@ -274,33 +274,6 @@ Désengager pour éteindre - - DriveStats - - Drives - Trajets - - - Hours - Heures - - - ALL TIME - DEPUIS TOUJOURS - - - PAST WEEK - CETTE SEMAINE - - - KM - KM - - - Miles - Miles - - DriverViewScene diff --git a/selfdrive/ui/translations/main_ja.ts b/selfdrive/ui/translations/main_ja.ts index 8abd794c60..63de8f0471 100644 --- a/selfdrive/ui/translations/main_ja.ts +++ b/selfdrive/ui/translations/main_ja.ts @@ -274,33 +274,6 @@ 確認 - - DriveStats - - Drives - 運転履歴 - - - Hours - 時間 - - - ALL TIME - 累計 - - - PAST WEEK - 先週 - - - KM - km - - - Miles - マイル - - DriverViewScene diff --git a/selfdrive/ui/translations/main_ko.ts b/selfdrive/ui/translations/main_ko.ts index 33ff74c938..bc0c5f2b0e 100644 --- a/selfdrive/ui/translations/main_ko.ts +++ b/selfdrive/ui/translations/main_ko.ts @@ -274,33 +274,6 @@ 다시보기 - - DriveStats - - Drives - 주행 - - - Hours - 시간 - - - ALL TIME - 전체 - - - PAST WEEK - 지난 주 - - - KM - km - - - Miles - 마일 - - DriverViewScene diff --git a/selfdrive/ui/translations/main_pt-BR.ts b/selfdrive/ui/translations/main_pt-BR.ts index 3f429c2acf..4cf37cc585 100644 --- a/selfdrive/ui/translations/main_pt-BR.ts +++ b/selfdrive/ui/translations/main_pt-BR.ts @@ -274,33 +274,6 @@ Revisar - - DriveStats - - Drives - Dirigidas - - - Hours - Horas - - - ALL TIME - TOTAL - - - PAST WEEK - SEMANA PASSADA - - - KM - KM - - - Miles - Milhas - - DriverViewScene diff --git a/selfdrive/ui/translations/main_th.ts b/selfdrive/ui/translations/main_th.ts index 5b6ecea49d..a1748a6951 100644 --- a/selfdrive/ui/translations/main_th.ts +++ b/selfdrive/ui/translations/main_th.ts @@ -274,33 +274,6 @@ ทบทวน - - DriveStats - - Drives - การขับขี่ - - - Hours - ชั่วโมง - - - ALL TIME - ทั้งหมด - - - PAST WEEK - สัปดาห์ที่ผ่านมา - - - KM - กิโลเมตร - - - Miles - ไมล์ - - DriverViewScene diff --git a/selfdrive/ui/translations/main_tr.ts b/selfdrive/ui/translations/main_tr.ts index 97e1282c68..51fdddd5ce 100644 --- a/selfdrive/ui/translations/main_tr.ts +++ b/selfdrive/ui/translations/main_tr.ts @@ -274,33 +274,6 @@ - - DriveStats - - Drives - Sürücüler - - - Hours - Saat - - - ALL TIME - TÜM ZAMANLAR - - - PAST WEEK - GEÇEN HAFTA - - - KM - KM - - - Miles - Mil - - DriverViewScene diff --git a/selfdrive/ui/translations/main_zh-CHS.ts b/selfdrive/ui/translations/main_zh-CHS.ts index 9253d922f5..7b682535ce 100644 --- a/selfdrive/ui/translations/main_zh-CHS.ts +++ b/selfdrive/ui/translations/main_zh-CHS.ts @@ -274,33 +274,6 @@ 预览 - - DriveStats - - Drives - 旅程数 - - - Hours - 小时 - - - ALL TIME - 全部 - - - PAST WEEK - 过去一周 - - - KM - 公里 - - - Miles - 英里 - - DriverViewScene diff --git a/selfdrive/ui/translations/main_zh-CHT.ts b/selfdrive/ui/translations/main_zh-CHT.ts index 3a2040bc3b..1794cc78ac 100644 --- a/selfdrive/ui/translations/main_zh-CHT.ts +++ b/selfdrive/ui/translations/main_zh-CHT.ts @@ -274,33 +274,6 @@ 回顧 - - DriveStats - - Drives - 旅程 - - - Hours - 小時 - - - ALL TIME - 總共 - - - PAST WEEK - 上週 - - - KM - 公里 - - - Miles - 英里 - - DriverViewScene From d731c85b2edd5f568390f293a0f60f03f1f1c6b3 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 5 Oct 2023 15:53:07 -0700 Subject: [PATCH 5/8] Docker: add batman to sudo group (#30188) add batman to sudo group --- .github/workflows/tools_tests.yaml | 3 ++- Dockerfile.openpilot_base | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index 218d2c802d..b7dc19f6be 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -105,4 +105,5 @@ jobs: run: | devcontainer exec --workspace-folder . scons -j$(nproc) devcontainer exec --workspace-folder . pip install pip-install-test - devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json \ No newline at end of file + devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json + devcontainer exec --workspace-folder . sudo touch /root/test.txt \ No newline at end of file diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 8b434536df..98d07ea39e 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -25,6 +25,8 @@ RUN cd /tmp && \ ARG USER=batman ARG USER_UID=1000 RUN useradd -m -s /bin/bash -u $USER_UID $USER +RUN usermod -aG sudo $USER +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER $USER ENV POETRY_VIRTUALENVS_CREATE=false From 7f945f10c65b445f630b970ddf4b245d917d5448 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 5 Oct 2023 17:15:26 -0700 Subject: [PATCH 6/8] Subaru: Outback 2023 fingerprint (#30190) fingerprint --- selfdrive/car/subaru/values.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/selfdrive/car/subaru/values.py b/selfdrive/car/subaru/values.py index 4c57b626bf..787d7fdcea 100644 --- a/selfdrive/car/subaru/values.py +++ b/selfdrive/car/subaru/values.py @@ -668,19 +668,23 @@ FW_VERSIONS = { }, CAR.OUTBACK_2023: { (Ecu.abs, 0x7b0, None): [ + b'\xa1 #\x14\x00', b'\xa1 #\x17\x00', ], (Ecu.eps, 0x746, None): [ + b'+\xc0\x10\x11\x00', b'+\xc0\x12\x11\x00', ], (Ecu.fwdCamera, 0x787, None): [ b'\t!\x08\x046\x05!\x08\x01/', ], (Ecu.engine, 0x7a2, None): [ + b'\xed,\xa0q\x07', b'\xed,\xa2q\x07', ], (Ecu.transmission, 0x7a3, None): [ b'\xa8\x8e\xf41\x00', + b'\xa8\xfe\xf41\x00', ] } } From f8e488f881cec5c654282b44910b07a541832fa8 Mon Sep 17 00:00:00 2001 From: Greg Hogan Date: Thu, 5 Oct 2023 18:40:24 -0700 Subject: [PATCH 7/8] convert vidindex to python (#30176) * convert vidindex to python * fix whitespace * corrupt file option * fix up typings * fix return type * update framereader * change length delimiter to uint32 value * change length to uint32 value * move url_file changes to separate PR * cleanup caching * revert whitespace change * fix frame type param type --- .dockerignore | 2 - tools/lib/framereader.py | 75 +------- tools/lib/vidindex.py | 301 ++++++++++++++++++++++++++++++++ tools/lib/vidindex/.gitignore | 1 - tools/lib/vidindex/Makefile | 6 - tools/lib/vidindex/bitstream.c | 118 ------------- tools/lib/vidindex/bitstream.h | 26 --- tools/lib/vidindex/vidindex.c | 307 --------------------------------- 8 files changed, 309 insertions(+), 527 deletions(-) create mode 100755 tools/lib/vidindex.py delete mode 100644 tools/lib/vidindex/.gitignore delete mode 100644 tools/lib/vidindex/Makefile delete mode 100644 tools/lib/vidindex/bitstream.c delete mode 100644 tools/lib/vidindex/bitstream.h delete mode 100644 tools/lib/vidindex/vidindex.c diff --git a/.dockerignore b/.dockerignore index 1f261adf30..cfc34e5848 100644 --- a/.dockerignore +++ b/.dockerignore @@ -28,10 +28,8 @@ chffr/app2 chffr/backend/env selfdrive/nav selfdrive/baseui -chffr/lib/vidindex/vidindex selfdrive/test/simulator2 **/cache_data -xx/chffr/lib/vidindex/vidindex xx/plus xx/community xx/projects diff --git a/tools/lib/framereader.py b/tools/lib/framereader.py index 06fee6857d..e0b24963ca 100644 --- a/tools/lib/framereader.py +++ b/tools/lib/framereader.py @@ -14,6 +14,7 @@ from lru import LRU import _io from openpilot.tools.lib.cache import cache_path_for_file_path, DEFAULT_CACHE_DIR from openpilot.tools.lib.exceptions import DataUnreadableError +from openpilot.tools.lib.vidindex import hevc_index from openpilot.common.file_helpers import atomic_write_in_dir from openpilot.tools.lib.filereader import FileReader @@ -75,31 +76,6 @@ def ffprobe(fn, fmt=None): return json.loads(ffprobe_output) -def vidindex(fn, typ): - vidindex_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "vidindex") - vidindex = os.path.join(vidindex_dir, "vidindex") - - subprocess.check_call(["make"], cwd=vidindex_dir, stdout=subprocess.DEVNULL) - - with tempfile.NamedTemporaryFile() as prefix_f, \ - tempfile.NamedTemporaryFile() as index_f: - try: - subprocess.check_call([vidindex, typ, fn, prefix_f.name, index_f.name]) - except subprocess.CalledProcessError as e: - raise DataUnreadableError(f"vidindex failed on file {fn}") from e - with open(index_f.name, "rb") as f: - index = f.read() - with open(prefix_f.name, "rb") as f: - prefix = f.read() - - index = np.frombuffer(index, np.uint32).reshape(-1, 2) - - assert index[-1, 0] == 0xFFFFFFFF - assert index[-1, 1] == os.path.getsize(fn) - - return index, prefix - - def cache_fn(func): @wraps(func) def cache_inner(fn, *args, **kwargs): @@ -114,7 +90,6 @@ def cache_fn(func): cache_value = pickle.load(cache_file) else: cache_value = func(fn, *args, **kwargs) - if cache_path: with atomic_write_in_dir(cache_path, mode="wb", overwrite=True) as cache_file: pickle.dump(cache_value, cache_file, -1) @@ -125,13 +100,13 @@ def cache_fn(func): @cache_fn -def index_stream(fn, typ): - assert typ in ("hevc", ) +def index_stream(fn, ft): + if ft != FrameType.h265_stream: + raise NotImplementedError("Only h265 supported") - with FileReader(fn) as f: - assert os.path.exists(f.name), fn - index, prefix = vidindex(f.name, typ) - probe = ffprobe(f.name, typ) + frame_types, dat_len, prefix = hevc_index(fn) + index = np.array(frame_types + [(0xFFFFFFFF, dat_len)], dtype=np.uint32) + probe = ffprobe(fn, "hevc") return { 'index': index, @@ -140,42 +115,8 @@ def index_stream(fn, typ): } -def index_videos(camera_paths, cache_dir=DEFAULT_CACHE_DIR): - """Requires that paths in camera_paths are contiguous and of the same type.""" - if len(camera_paths) < 1: - raise ValueError("must provide at least one video to index") - - frame_type = fingerprint_video(camera_paths[0]) - for fn in camera_paths: - index_video(fn, frame_type, cache_dir) - - -def index_video(fn, frame_type=None, cache_dir=DEFAULT_CACHE_DIR): - cache_path = cache_path_for_file_path(fn, cache_dir) - - if os.path.exists(cache_path): - return - - if frame_type is None: - frame_type = fingerprint_video(fn[0]) - - if frame_type == FrameType.h265_stream: - index_stream(fn, "hevc", cache_dir=cache_dir) - else: - raise NotImplementedError("Only h265 supported") - - def get_video_index(fn, frame_type, cache_dir=DEFAULT_CACHE_DIR): - cache_path = cache_path_for_file_path(fn, cache_dir) - - if not os.path.exists(cache_path): - index_video(fn, frame_type, cache_dir) - - if not os.path.exists(cache_path): - return None - with open(cache_path, "rb") as cache_file: - return pickle.load(cache_file) - + return index_stream(fn, frame_type, cache_dir=cache_dir) def read_file_check_size(f, sz, cookie): buff = bytearray(sz) diff --git a/tools/lib/vidindex.py b/tools/lib/vidindex.py new file mode 100755 index 0000000000..9404d035ba --- /dev/null +++ b/tools/lib/vidindex.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +import argparse +import struct +from enum import IntEnum +from typing import Tuple + +from openpilot.tools.lib.filereader import FileReader + +# H.265 specification +# https://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-H.265-201802-S!!PDF-E&type=items + +NAL_UNIT_START_CODE = b"\x00\x00\x01" +NAL_UNIT_START_CODE_SIZE = len(NAL_UNIT_START_CODE) +NAL_UNIT_HEADER_SIZE = 2 + +class HevcNalUnitType(IntEnum): + TRAIL_N = 0 # RBSP structure: slice_segment_layer_rbsp( ) + TRAIL_R = 1 # RBSP structure: slice_segment_layer_rbsp( ) + TSA_N = 2 # RBSP structure: slice_segment_layer_rbsp( ) + TSA_R = 3 # RBSP structure: slice_segment_layer_rbsp( ) + STSA_N = 4 # RBSP structure: slice_segment_layer_rbsp( ) + STSA_R = 5 # RBSP structure: slice_segment_layer_rbsp( ) + RADL_N = 6 # RBSP structure: slice_segment_layer_rbsp( ) + RADL_R = 7 # RBSP structure: slice_segment_layer_rbsp( ) + RASL_N = 8 # RBSP structure: slice_segment_layer_rbsp( ) + RASL_R = 9 # RBSP structure: slice_segment_layer_rbsp( ) + RSV_VCL_N10 = 10 + RSV_VCL_R11 = 11 + RSV_VCL_N12 = 12 + RSV_VCL_R13 = 13 + RSV_VCL_N14 = 14 + RSV_VCL_R15 = 15 + BLA_W_LP = 16 # RBSP structure: slice_segment_layer_rbsp( ) + BLA_W_RADL = 17 # RBSP structure: slice_segment_layer_rbsp( ) + BLA_N_LP = 18 # RBSP structure: slice_segment_layer_rbsp( ) + IDR_W_RADL = 19 # RBSP structure: slice_segment_layer_rbsp( ) + IDR_N_LP = 20 # RBSP structure: slice_segment_layer_rbsp( ) + CRA_NUT = 21 # RBSP structure: slice_segment_layer_rbsp( ) + RSV_IRAP_VCL22 = 22 + RSV_IRAP_VCL23 = 23 + RSV_VCL24 = 24 + RSV_VCL25 = 25 + RSV_VCL26 = 26 + RSV_VCL27 = 27 + RSV_VCL28 = 28 + RSV_VCL29 = 29 + RSV_VCL30 = 30 + RSV_VCL31 = 31 + VPS_NUT = 32 # RBSP structure: video_parameter_set_rbsp( ) + SPS_NUT = 33 # RBSP structure: seq_parameter_set_rbsp( ) + PPS_NUT = 34 # RBSP structure: pic_parameter_set_rbsp( ) + AUD_NUT = 35 + EOS_NUT = 36 + EOB_NUT = 37 + FD_NUT = 38 + PREFIX_SEI_NUT = 39 + SUFFIX_SEI_NUT = 40 + RSV_NVCL41 = 41 + RSV_NVCL42 = 42 + RSV_NVCL43 = 43 + RSV_NVCL44 = 44 + RSV_NVCL45 = 45 + RSV_NVCL46 = 46 + RSV_NVCL47 = 47 + UNSPEC48 = 48 + UNSPEC49 = 49 + UNSPEC50 = 50 + UNSPEC51 = 51 + UNSPEC52 = 52 + UNSPEC53 = 53 + UNSPEC54 = 54 + UNSPEC55 = 55 + UNSPEC56 = 56 + UNSPEC57 = 57 + UNSPEC58 = 58 + UNSPEC59 = 59 + UNSPEC60 = 60 + UNSPEC61 = 61 + UNSPEC62 = 62 + UNSPEC63 = 63 + +# B.2.2 Byte stream NAL unit semantics +# - The nal_unit_type within the nal_unit( ) syntax structure is equal to VPS_NUT, SPS_NUT or PPS_NUT. +# - The byte stream NAL unit syntax structure contains the first NAL unit of an access unit in decoding +# order, as specified in clause 7.4.2.4.4. +HEVC_PARAMETER_SET_NAL_UNITS = ( + HevcNalUnitType.VPS_NUT, + HevcNalUnitType.SPS_NUT, + HevcNalUnitType.PPS_NUT, +) + +# 3.29 coded slice segment NAL unit: A NAL unit that has nal_unit_type in the range of TRAIL_N to RASL_R, +# inclusive, or in the range of BLA_W_LP to RSV_IRAP_VCL23, inclusive, which indicates that the NAL unit +# contains a coded slice segment +HEVC_CODED_SLICE_SEGMENT_NAL_UNITS = ( + HevcNalUnitType.TRAIL_N, + HevcNalUnitType.TRAIL_R, + HevcNalUnitType.TSA_N, + HevcNalUnitType.TSA_R, + HevcNalUnitType.STSA_N, + HevcNalUnitType.STSA_R, + HevcNalUnitType.RADL_N, + HevcNalUnitType.RADL_R, + HevcNalUnitType.RASL_N, + HevcNalUnitType.RASL_R, + HevcNalUnitType.BLA_W_LP, + HevcNalUnitType.BLA_W_RADL, + HevcNalUnitType.BLA_N_LP, + HevcNalUnitType.IDR_W_RADL, + HevcNalUnitType.IDR_N_LP, + HevcNalUnitType.CRA_NUT, +) + +class VideoFileInvalid(Exception): + pass + +def get_ue(dat: bytes, start_idx: int, skip_bits: int) -> Tuple[int, int]: + prefix_val = 0 + prefix_len = 0 + suffix_val = 0 + suffix_len = 0 + + i = start_idx + while i < len(dat): + j = 7 + while j >= 0: + if skip_bits > 0: + skip_bits -= 1 + elif prefix_val == 0: + prefix_val = (dat[i] >> j) & 1 + prefix_len += 1 + else: + suffix_val = (suffix_val << 1) | ((dat[i] >> j) & 1) + suffix_len += 1 + j -= 1 + + if prefix_val == 1 and prefix_len - 1 == suffix_len: + val = 2**(prefix_len-1) - 1 + suffix_val + size = prefix_len + suffix_len + return val, size + i += 1 + + raise VideoFileInvalid("invalid exponential-golomb code") + +def require_nal_unit_start(dat: bytes, nal_unit_start: int) -> None: + if nal_unit_start >= len(dat): + raise ValueError("start index must be less than data length") + + if nal_unit_start < 1: + raise ValueError("start index must be greater than zero") + + if dat[nal_unit_start-1] != 0x00: + raise VideoFileInvalid("start code must be preceded by 0x00") + + if dat[nal_unit_start:nal_unit_start + NAL_UNIT_START_CODE_SIZE] != NAL_UNIT_START_CODE: + raise VideoFileInvalid("data must begin with start code") + +def get_hevc_nal_unit_length(dat: bytes, nal_unit_start: int) -> int: + try: + pos = dat.index(NAL_UNIT_START_CODE, nal_unit_start + NAL_UNIT_START_CODE_SIZE) + except ValueError: + pos = -1 + + # length of NAL unit is byte count up to next NAL unit start index + nal_unit_len = (pos if pos != -1 else len(dat)) - nal_unit_start + return nal_unit_len + +def get_hevc_nal_unit_type(dat: bytes, nal_unit_start: int) -> HevcNalUnitType: + # 7.3.1.2 NAL unit header syntax + # nal_unit_header( ) { // descriptor + # forbidden_zero_bit f(1) + # nal_unit_type u(6) + # nuh_layer_id u(6) + # nuh_temporal_id_plus1 u(3) + # } + header_start = nal_unit_start + NAL_UNIT_START_CODE_SIZE + nal_unit_header = dat[header_start:header_start + NAL_UNIT_HEADER_SIZE] + if len(nal_unit_header) != 2: + raise VideoFileInvalid("data to short to contain nal unit header") + return HevcNalUnitType((nal_unit_header[0] >> 1) & 0x3F) + +def get_hevc_slice_type(dat: bytes, nal_unit_start: int, nal_unit_type: HevcNalUnitType) -> Tuple[int, bool]: + # 7.3.2.9 Slice segment layer RBSP syntax + # slice_segment_layer_rbsp( ) { + # slice_segment_header( ) + # slice_segment_data( ) + # rbsp_slice_segment_trailing_bits( ) + # } + # ... + # 7.3.6.1 General slice segment header syntax + # slice_segment_header( ) { // descriptor + # first_slice_segment_in_pic_flag u(1) + # if( nal_unit_type >= BLA_W_LP && nal_unit_type <= RSV_IRAP_VCL23 ) + # no_output_of_prior_pics_flag u(1) + # slice_pic_parameter_set_id ue(v) + # if( !first_slice_segment_in_pic_flag ) { + # if( dependent_slice_segments_enabled_flag ) + # dependent_slice_segment_flag u(1) + # slice_segment_address u(v) + # } + # if( !dependent_slice_segment_flag ) { + # for( i = 0; i < num_extra_slice_header_bits; i++ ) + # slice_reserved_flag[ i ] u(1) + # slice_type ue(v) + # ... + + rbsp_start = nal_unit_start + NAL_UNIT_START_CODE_SIZE + NAL_UNIT_HEADER_SIZE + skip_bits = 0 + + # 7.4.7.1 General slice segment header semantics + # first_slice_segment_in_pic_flag equal to 1 specifies that the slice segment is the first slice segment of the picture in + # decoding order. first_slice_segment_in_pic_flag equal to 0 specifies that the slice segment is not the first slice segment + # of the picture in decoding order. + is_first_slice = dat[rbsp_start] >> 7 & 1 == 1 + if not is_first_slice: + # TODO: parse dependent_slice_segment_flag and slice_segment_address and get real slice_type + # for now since we don't use it return -1 for slice_type + return (-1, is_first_slice) + skip_bits += 1 # skip past first_slice_segment_in_pic_flag + + if nal_unit_type >= HevcNalUnitType.BLA_W_LP and nal_unit_type <= HevcNalUnitType.RSV_IRAP_VCL23: + # 7.4.7.1 General slice segment header semantics + # no_output_of_prior_pics_flag affects the output of previously-decoded pictures in the decoded picture buffer after the + # decoding of an IDR or a BLA picture that is not the first picture in the bitstream as specified in Annex C. + skip_bits += 1 # skip past no_output_of_prior_pics_flag + + # 7.4.7.1 General slice segment header semantics + # slice_pic_parameter_set_id specifies the value of pps_pic_parameter_set_id for the PPS in use. + # The value of slice_pic_parameter_set_id shall be in the range of 0 to 63, inclusive. + _, size = get_ue(dat, rbsp_start, skip_bits) + skip_bits += size # skip past slice_pic_parameter_set_id + + # 7.4.3.3.1 General picture parameter set RBSP semanal_unit_lenntics + # num_extra_slice_header_bits specifies the number of extra slice header bits that are present in the slice header RBSP + # for coded pictures referring to the PPS. The value of num_extra_slice_header_bits shall be in the range of 0 to 2, inclusive, + # in bitstreams conforming to this version of this Specification. Other values for num_extra_slice_header_bits are reserved + # for future use by ITU-T | ISO/IEC. However, decoders shall allow num_extra_slice_header_bits to have any value. + # TODO: get from PPS_NUT pic_parameter_set_rbsp( ) for corresponding slice_pic_parameter_set_id + num_extra_slice_header_bits = 0 + skip_bits += num_extra_slice_header_bits + + # 7.4.7.1 General slice segment header semantics + # slice_type specifies the coding type of the slice according to Table 7-7. + # Table 7-7 - Name association to slice_type + # slice_type | Name of slice_type + # 0 | B (B slice) + # 1 | P (P slice) + # 2 | I (I slice) + # unsigned integer 0-th order Exp-Golomb-coded syntax element with the left bit first + slice_type, _ = get_ue(dat, rbsp_start, skip_bits) + if slice_type > 2: + raise VideoFileInvalid("slice_type must be 0, 1, or 2") + return slice_type, is_first_slice + +def hevc_index(hevc_file_name: str, allow_corrupt: bool=False) -> Tuple[list, int, bytes]: + with FileReader(hevc_file_name) as f: + dat = f.read() + + if len(dat) < NAL_UNIT_START_CODE_SIZE + 1: + raise VideoFileInvalid("data is too short") + + prefix_dat = b"" + frame_types = list() + + try: + i = 1 # skip past first byte 0x00 (verified by get_hevc_nal_info) + while i < len(dat): + require_nal_unit_start(dat, i) + nal_unit_len = get_hevc_nal_unit_length(dat, i) + nal_unit_type = get_hevc_nal_unit_type(dat, i) + if nal_unit_type in HEVC_PARAMETER_SET_NAL_UNITS: + prefix_dat += dat[i:i+nal_unit_len] + elif nal_unit_type in HEVC_CODED_SLICE_SEGMENT_NAL_UNITS: + slice_type, is_first_slice = get_hevc_slice_type(dat, i, nal_unit_type) + if is_first_slice: + frame_types.append((slice_type, i)) + i += nal_unit_len + except Exception: + if not allow_corrupt: + raise + + return frame_types, len(dat), prefix_dat + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("input_file", type=str) + parser.add_argument("output_prefix_file", type=str) + parser.add_argument("output_index_file", type=str) + args = parser.parse_args() + + frame_types, dat_len, prefix_dat = hevc_index(args.input_file) + with open(args.output_prefix_file, "wb") as f: + f.write(prefix_dat) + + with open(args.output_index_file, "wb") as f: + for ft, fp in frame_types: + f.write(struct.pack(" -#include - -static const uint32_t BS_MASKS[33] = { - 0, 0x1L, 0x3L, 0x7L, 0xFL, 0x1FL, - 0x3FL, 0x7FL, 0xFFL, 0x1FFL, 0x3FFL, 0x7FFL, - 0xFFFL, 0x1FFFL, 0x3FFFL, 0x7FFFL, 0xFFFFL, 0x1FFFFL, - 0x3FFFFL, 0x7FFFFL, 0xFFFFFL, 0x1FFFFFL, 0x3FFFFFL, 0x7FFFFFL, - 0xFFFFFFL, 0x1FFFFFFL, 0x3FFFFFFL, 0x7FFFFFFL, 0xFFFFFFFL, 0x1FFFFFFFL, - 0x3FFFFFFFL, 0x7FFFFFFFL, 0xFFFFFFFFL}; - -void bs_init(struct bitstream* bs, const uint8_t* buffer, size_t input_size) { - bs->buffer_ptr = buffer; - bs->buffer_end = buffer + input_size; - bs->value = 0; - bs->pos = 0; - bs->shift = 8; - bs->size = input_size * 8; -} - -uint32_t bs_get(struct bitstream* bs, int n) { - if (n > 32) - return 0; - - bs->pos += n; - bs->shift += n; - while (bs->shift > 8) { - if (bs->buffer_ptr < bs->buffer_end) { - bs->value <<= 8; - bs->value |= *bs->buffer_ptr++; - bs->shift -= 8; - } else { - bs_seek(bs, bs->pos - n); - return 0; - // bs->value <<= 8; - // bs->shift -= 8; - } - } - return (bs->value >> (8 - bs->shift)) & BS_MASKS[n]; -} - -void bs_seek(struct bitstream* bs, size_t new_pos) { - bs->pos = (new_pos / 32) * 32; - bs->shift = 8; - bs->value = 0; - bs_get(bs, new_pos % 32); -} - -uint32_t bs_peek(struct bitstream* bs, int n) { - struct bitstream bak = *bs; - return bs_get(&bak, n); -} - -size_t bs_remain(struct bitstream* bs) { - return bs->size - bs->pos; -} - -int bs_eof(struct bitstream* bs) { - return bs_remain(bs) == 0; -} - -uint32_t bs_ue(struct bitstream* bs) { - static const uint8_t exp_golomb_bits[256] = { - 8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - }; - uint32_t bits, read = 0; - int bits_left; - uint8_t coded; - int done = 0; - bits = 0; - // we want to read 8 bits at a time - if we don't have 8 bits, - // read what's left, and shift. The exp_golomb_bits calc remains the - // same. - while (!done) { - bits_left = bs_remain(bs); - if (bits_left < 8) { - read = bs_peek(bs, bits_left) << (8 - bits_left); - done = 1; - } else { - read = bs_peek(bs, 8); - if (read == 0) { - bs_get(bs, 8); - bits += 8; - } else { - done = 1; - } - } - } - coded = exp_golomb_bits[read]; - bs_get(bs, coded); - bits += coded; - - // printf("ue - bits %d\n", bits); - return bs_get(bs, bits + 1) - 1; -} - -int32_t bs_se(struct bitstream* bs) { - uint32_t ret; - ret = bs_ue(bs); - if ((ret & 0x1) == 0) { - ret >>= 1; - int32_t temp = 0 - ret; - return temp; - } - return (ret + 1) >> 1; -} diff --git a/tools/lib/vidindex/bitstream.h b/tools/lib/vidindex/bitstream.h deleted file mode 100644 index 0f538a59ab..0000000000 --- a/tools/lib/vidindex/bitstream.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef bitstream_H -#define bitstream_H - - -#include -#include - -struct bitstream { - const uint8_t *buffer_ptr; - const uint8_t *buffer_end; - uint64_t value; - uint32_t pos; - uint32_t shift; - size_t size; -}; - -void bs_init(struct bitstream *bs, const uint8_t *buffer, size_t input_size); -void bs_seek(struct bitstream *bs, size_t new_pos); -uint32_t bs_get(struct bitstream *bs, int n); -uint32_t bs_peek(struct bitstream *bs, int n); -size_t bs_remain(struct bitstream *bs); -int bs_eof(struct bitstream *bs); -uint32_t bs_ue(struct bitstream *bs); -int32_t bs_se(struct bitstream *bs); - -#endif diff --git a/tools/lib/vidindex/vidindex.c b/tools/lib/vidindex/vidindex.c deleted file mode 100644 index 1d7da2e193..0000000000 --- a/tools/lib/vidindex/vidindex.c +++ /dev/null @@ -1,307 +0,0 @@ -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "./bitstream.h" - -#define START_CODE 0x000001 - -static uint32_t read24be(const uint8_t* ptr) { - return (ptr[0] << 16) | (ptr[1] << 8) | ptr[2]; -} -static void write32le(FILE *of, uint32_t v) { - uint8_t va[4] = { - v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, (v >> 24) & 0xff - }; - fwrite(va, 1, sizeof(va), of); -} - -// Table 7-1 -enum hevc_nal_type { - HEVC_NAL_TYPE_TRAIL_N = 0, - HEVC_NAL_TYPE_TRAIL_R = 1, - HEVC_NAL_TYPE_TSA_N = 2, - HEVC_NAL_TYPE_TSA_R = 3, - HEVC_NAL_TYPE_STSA_N = 4, - HEVC_NAL_TYPE_STSA_R = 5, - HEVC_NAL_TYPE_RADL_N = 6, - HEVC_NAL_TYPE_RADL_R = 7, - HEVC_NAL_TYPE_RASL_N = 8, - HEVC_NAL_TYPE_RASL_R = 9, - HEVC_NAL_TYPE_BLA_W_LP = 16, - HEVC_NAL_TYPE_BLA_W_RADL = 17, - HEVC_NAL_TYPE_BLA_N_LP = 18, - HEVC_NAL_TYPE_IDR_W_RADL = 19, - HEVC_NAL_TYPE_IDR_N_LP = 20, - HEVC_NAL_TYPE_CRA_NUT = 21, - HEVC_NAL_TYPE_RSV_IRAP_VCL23 = 23, - HEVC_NAL_TYPE_VPS_NUT = 32, - HEVC_NAL_TYPE_SPS_NUT = 33, - HEVC_NAL_TYPE_PPS_NUT = 34, - HEVC_NAL_TYPE_AUD_NUT = 35, - HEVC_NAL_TYPE_EOS_NUT = 36, - HEVC_NAL_TYPE_EOB_NUT = 37, - HEVC_NAL_TYPE_FD_NUT = 38, - HEVC_NAL_TYPE_PREFIX_SEI_NUT = 39, - HEVC_NAL_TYPE_SUFFIX_SEI_NUT = 40, -}; - -// Table 7-7 -enum hevc_slice_type { - HEVC_SLICE_B = 0, - HEVC_SLICE_P = 1, - HEVC_SLICE_I = 2, -}; - -static void hevc_index(const uint8_t *data, size_t file_size, FILE *of_prefix, FILE *of_index) { - const uint8_t* ptr = data; - const uint8_t* ptr_end = data + file_size; - - assert(ptr[0] == 0); - ptr++; - assert(read24be(ptr) == START_CODE); - - // pps. ignore for now - uint32_t num_extra_slice_header_bits = 0; - uint32_t dependent_slice_segments_enabled_flag = 0; - - while (ptr < ptr_end) { - const uint8_t* next = ptr+1; - for (; next < ptr_end-4; next++) { - if (read24be(next) == START_CODE) break; - } - size_t nal_size = next - ptr; - if (nal_size < 6) { - break; - } - - { - struct bitstream bs = {0}; - bs_init(&bs, ptr, nal_size); - - uint32_t start_code = bs_get(&bs, 24); - assert(start_code == 0x000001); - - // nal_unit_header - uint32_t forbidden_zero_bit = bs_get(&bs, 1); - uint32_t nal_unit_type = bs_get(&bs, 6); - uint32_t nuh_layer_id = bs_get(&bs, 6); - uint32_t nuh_temporal_id_plus1 = bs_get(&bs, 3); - - // if (nal_unit_type != 1) printf("%3d -- %3d %10d %lu\n", nal_unit_type, frame_num, (uint32_t)(ptr-data), nal_size); - - switch (nal_unit_type) { - case HEVC_NAL_TYPE_VPS_NUT: - case HEVC_NAL_TYPE_SPS_NUT: - case HEVC_NAL_TYPE_PPS_NUT: - fwrite(ptr, 1, nal_size, of_prefix); - break; - case HEVC_NAL_TYPE_TRAIL_N: - case HEVC_NAL_TYPE_TRAIL_R: - case HEVC_NAL_TYPE_TSA_N: - case HEVC_NAL_TYPE_TSA_R: - case HEVC_NAL_TYPE_STSA_N: - case HEVC_NAL_TYPE_STSA_R: - case HEVC_NAL_TYPE_RADL_N: - case HEVC_NAL_TYPE_RADL_R: - case HEVC_NAL_TYPE_RASL_N: - case HEVC_NAL_TYPE_RASL_R: - case HEVC_NAL_TYPE_BLA_W_LP: - case HEVC_NAL_TYPE_BLA_W_RADL: - case HEVC_NAL_TYPE_BLA_N_LP: - case HEVC_NAL_TYPE_IDR_W_RADL: - case HEVC_NAL_TYPE_IDR_N_LP: - case HEVC_NAL_TYPE_CRA_NUT: { - // slice_segment_header - uint32_t first_slice_segment_in_pic_flag = bs_get(&bs, 1); - if (nal_unit_type >= HEVC_NAL_TYPE_BLA_W_LP && nal_unit_type <= HEVC_NAL_TYPE_RSV_IRAP_VCL23) { - uint32_t no_output_of_prior_pics_flag = bs_get(&bs, 1); - } - uint32_t slice_pic_parameter_set_id = bs_get(&bs, 1); - if (!first_slice_segment_in_pic_flag) { - // ... - break; - } - - if (!dependent_slice_segments_enabled_flag) { - for (int i=0; i 4); - - const uint8_t* data = (const uint8_t*)mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0); - assert(data != MAP_FAILED); - - if (strcmp(file_type, "hevc") == 0) { - hevc_index(data, file_size, of_prefix, of_index); - } else if (strcmp(file_type, "h264") == 0) { - h264_index(data, file_size, of_prefix, of_index); - } else { - assert(false); - } - - munmap((void*)data, file_size); - close(fd); - - return 0; -} From 86b90a8ba263df067406e0089eb507c3a397640e Mon Sep 17 00:00:00 2001 From: Greg Hogan Date: Thu, 5 Oct 2023 19:45:46 -0700 Subject: [PATCH 8/8] filereader cleanup (#30191) * filereader cleanup * make name return url * remove unused imports --- tools/lib/filereader.py | 8 ++++++-- tools/lib/framereader.py | 3 ++- tools/lib/url_file.py | 36 ++++++++---------------------------- 3 files changed, 16 insertions(+), 31 deletions(-) diff --git a/tools/lib/filereader.py b/tools/lib/filereader.py index 5ac23d57ec..4aec965f1a 100644 --- a/tools/lib/filereader.py +++ b/tools/lib/filereader.py @@ -3,9 +3,13 @@ from openpilot.tools.lib.url_file import URLFile DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/") -def FileReader(fn, debug=False): +def resolve_name(fn): if fn.startswith("cd:/"): - fn = fn.replace("cd:/", DATA_ENDPOINT) + return fn.replace("cd:/", DATA_ENDPOINT) + return fn + +def FileReader(fn, debug=False): + fn = resolve_name(fn) if fn.startswith(("http://", "https://")): return URLFile(fn, debug=debug) return open(fn, "rb") diff --git a/tools/lib/framereader.py b/tools/lib/framereader.py index e0b24963ca..8ab9e10edc 100644 --- a/tools/lib/framereader.py +++ b/tools/lib/framereader.py @@ -17,7 +17,7 @@ from openpilot.tools.lib.exceptions import DataUnreadableError from openpilot.tools.lib.vidindex import hevc_index from openpilot.common.file_helpers import atomic_write_in_dir -from openpilot.tools.lib.filereader import FileReader +from openpilot.tools.lib.filereader import FileReader, resolve_name HEVC_SLICE_B = 0 HEVC_SLICE_P = 1 @@ -60,6 +60,7 @@ def fingerprint_video(fn): def ffprobe(fn, fmt=None): + fn = resolve_name(fn) cmd = ["ffprobe", "-v", "quiet", "-print_format", "json", diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 7612996223..2f933e3b7f 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -1,8 +1,6 @@ import os import time -import tempfile import threading -import urllib.parse import pycurl from hashlib import sha256 from io import BytesIO @@ -37,7 +35,8 @@ class URLFile: self._curl = self._tlocal.curl except AttributeError: self._curl = self._tlocal.curl = pycurl.Curl() - mkdirs_exists_ok(Paths.download_cache_root()) + if not self._force_download: + mkdirs_exists_ok(Paths.download_cache_root()) def __enter__(self): return self @@ -65,12 +64,13 @@ class URLFile: def get_length(self): if self._length is not None: return self._length + file_length_path = os.path.join(Paths.download_cache_root(), hash_256(self._url) + "_length") - if os.path.exists(file_length_path) and not self._force_download: + if not self._force_download and os.path.exists(file_length_path): with open(file_length_path) as file_length: - content = file_length.read() - self._length = int(content) - return self._length + content = file_length.read() + self._length = int(content) + return self._length self._length = self.get_length_online() if not self._force_download and self._length != -1: @@ -173,24 +173,4 @@ class URLFile: @property def name(self): - """Returns a local path to file with the URLFile's contents. - - This can be used to interface with modules that require local files. - """ - if self._local_file is None: - _, ext = os.path.splitext(urllib.parse.urlparse(self._url).path) - local_fd, local_path = tempfile.mkstemp(suffix=ext) - try: - os.write(local_fd, self.read()) - local_file = open(local_path, "rb") - except Exception: - os.remove(local_path) - raise - finally: - os.close(local_fd) - - self._local_file = local_file - self.read = self._local_file.read - self.seek = self._local_file.seek - - return self._local_file.name + return self._url