Merge remote-tracking branch 'upstream/master' into toyota-fuzzy-v2

pull/28641/head
Shane Smiskol 2 years ago
commit 8d604ddde9
  1. 2
      .dockerignore
  2. 27
      .github/workflows/selfdrive_tests.yaml
  3. 3
      .github/workflows/tools_tests.yaml
  4. 2
      Dockerfile.openpilot_base
  5. 1
      common/params.cc
  6. 4
      selfdrive/car/subaru/values.py
  7. 1
      selfdrive/car/toyota/values.py
  8. 14
      selfdrive/test/process_replay/regen_all.py
  9. 48
      selfdrive/test/process_replay/test_regen.py
  10. 2
      selfdrive/ui/SConscript
  11. 4
      selfdrive/ui/qt/home.cc
  12. 97
      selfdrive/ui/qt/widgets/drive_stats.cc
  13. 25
      selfdrive/ui/qt/widgets/drive_stats.h
  14. 27
      selfdrive/ui/translations/main_ar.ts
  15. 27
      selfdrive/ui/translations/main_de.ts
  16. 27
      selfdrive/ui/translations/main_fr.ts
  17. 27
      selfdrive/ui/translations/main_ja.ts
  18. 27
      selfdrive/ui/translations/main_ko.ts
  19. 27
      selfdrive/ui/translations/main_pt-BR.ts
  20. 27
      selfdrive/ui/translations/main_th.ts
  21. 27
      selfdrive/ui/translations/main_tr.ts
  22. 27
      selfdrive/ui/translations/main_zh-CHS.ts
  23. 27
      selfdrive/ui/translations/main_zh-CHT.ts
  24. 7
      tools/cabana/dbc/dbcfile.cc
  25. 6
      tools/cabana/tests/test_cabana.cc
  26. 8
      tools/lib/filereader.py
  27. 78
      tools/lib/framereader.py
  28. 36
      tools/lib/url_file.py
  29. 301
      tools/lib/vidindex.py
  30. 1
      tools/lib/vidindex/.gitignore
  31. 6
      tools/lib/vidindex/Makefile
  32. 118
      tools/lib/vidindex/bitstream.c
  33. 26
      tools/lib/vidindex/bitstream.h
  34. 307
      tools/lib/vidindex/vidindex.c

@ -28,10 +28,8 @@ chffr/app2
chffr/backend/env
selfdrive/nav
selfdrive/baseui
chffr/lib/vidindex/vidindex
selfdrive/test/simulator2
**/cache_data
xx/chffr/lib/vidindex/vidindex
xx/plus
xx/community
xx/projects

@ -228,6 +228,33 @@ jobs:
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v3
regen:
name: regen
runs-on: 'ubuntu-20.04'
steps:
- uses: actions/checkout@v3
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Cache test routes
id: dependency-cache
uses: actions/cache@v3
with:
path: .ci_cache/comma_download_cache
key: regen-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_regen.py') }}
- name: Build base Docker image
run: eval "$BUILD"
- name: Build Docker image
run: eval "$BUILD_CL"
- name: Build openpilot
run: |
${{ env.RUN }} "scons -j$(nproc)"
- name: Run regen
timeout-minutes: 30
run: |
${{ env.RUN_CL }} "ONNXCPU=1 $PYTEST -n auto --dist=loadscope selfdrive/test/process_replay/test_regen.py && \
chmod -R 777 /tmp/comma_download_cache"
test_modeld:
name: model tests
runs-on: ubuntu-20.04

@ -105,4 +105,5 @@ jobs:
run: |
devcontainer exec --workspace-folder . scons -j$(nproc)
devcontainer exec --workspace-folder . pip install pip-install-test
devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
devcontainer exec --workspace-folder . sudo touch /root/test.txt

@ -25,6 +25,8 @@ RUN cd /tmp && \
ARG USER=batman
ARG USER_UID=1000
RUN useradd -m -s /bin/bash -u $USER_UID $USER
RUN usermod -aG sudo $USER
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER $USER
ENV POETRY_VIRTUALENVS_CREATE=false

@ -88,7 +88,6 @@ private:
std::unordered_map<std::string, uint32_t> keys = {
{"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
{"ApiCache_Device", PERSISTENT},
{"ApiCache_DriveStats", PERSISTENT},
{"ApiCache_NavDestinations", PERSISTENT},
{"AssistNowToken", PERSISTENT},
{"AthenadPid", PERSISTENT},

@ -668,19 +668,23 @@ FW_VERSIONS = {
},
CAR.OUTBACK_2023: {
(Ecu.abs, 0x7b0, None): [
b'\xa1 #\x14\x00',
b'\xa1 #\x17\x00',
],
(Ecu.eps, 0x746, None): [
b'+\xc0\x10\x11\x00',
b'+\xc0\x12\x11\x00',
],
(Ecu.fwdCamera, 0x787, None): [
b'\t!\x08\x046\x05!\x08\x01/',
],
(Ecu.engine, 0x7a2, None): [
b'\xed,\xa0q\x07',
b'\xed,\xa2q\x07',
],
(Ecu.transmission, 0x7a3, None): [
b'\xa8\x8e\xf41\x00',
b'\xa8\xfe\xf41\x00',
]
}
}

@ -631,6 +631,7 @@ FW_VERSIONS = {
b'\x018966333X0000\x00\x00\x00\x00',
b'\x018966333X4000\x00\x00\x00\x00',
b'\x01896633T16000\x00\x00\x00\x00',
b'\x018966306L9000\x00\x00\x00\x00',
b'\x028966306B2100\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00',
b'\x028966306B2300\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00',
b'\x028966306B2500\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00',

@ -29,15 +29,25 @@ def regen_job(segment, upload, disable_tqdm):
if __name__ == "__main__":
all_cars = {car for car, _ in segments}
parser = argparse.ArgumentParser(description="Generate new segments from old ones")
parser.add_argument("-j", "--jobs", type=int, default=1)
parser.add_argument("--no-upload", action="store_true")
parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars,
help="Whitelist given cars from the test (e.g. HONDA)")
parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[],
help="Blacklist given cars from the test (e.g. HONDA)")
args = parser.parse_args()
tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars)
tested_cars = {c.upper() for c in tested_cars}
tested_segments = [(car, segment) for car, segment in segments if car in tested_cars]
with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool:
p = pool.map(regen_job, segments, [not args.no_upload] * len(segments), [args.jobs > 1] * len(segments))
p = pool.map(regen_job, tested_segments, [not args.no_upload] * len(tested_segments), [args.jobs > 1] * len(tested_segments))
msg = "Copy these new segments into test_processes.py:"
for seg in tqdm(p, desc="Generating segments", total=len(segments)):
for seg in tqdm(p, desc="Generating segments", total=len(tested_segments)):
msg += "\n" + str(seg)
print()
print()

@ -0,0 +1,48 @@
#!/usr/bin/env python3
import unittest
from parameterized import parameterized
from openpilot.selfdrive.test.process_replay.regen import regen_segment
from openpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled, CONFIGS
from openpilot.selfdrive.test.openpilotci import get_url
from openpilot.tools.lib.logreader import LogReader
from openpilot.tools.lib.framereader import FrameReader
EXCLUDED_PROCESSES = {"dmonitoringd", "dmonitoringmodeld"}
TESTED_SEGMENTS = [
("PRIUS_C2", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA PRIUS 2017: NEO, pandaStateDEPRECATED, no peripheralState, sensorEventsDEPRECATED
# Enable these once regen on CI becomes faster or use them for different tests running controlsd in isolation
# ("MAZDA_C3", "bd6a637565e91581|2021-10-30--15-14-53--4"), # MAZDA.CX9_2021: TICI, incomplete managerState
# ("FORD_C3", "54827bf84c38b14f|2023-01-26--21-59-07--4"), # FORD.BRONCO_SPORT_MK1: TICI
]
def ci_setup_data_readers(route, sidx):
lr = LogReader(get_url(route, sidx, "rlog"))
# dm disabled
frs = {
'roadCameraState': FrameReader(get_url(route, sidx, "fcamera")),
}
if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera"))
return lr, frs
class TestRegen(unittest.TestCase):
@parameterized.expand(TESTED_SEGMENTS)
def test_engaged(self, case_name, segment):
tested_procs = [p for p in CONFIGS if p.proc_name not in EXCLUDED_PROCESSES]
route, sidx = segment.rsplit("--", 1)
lr, frs = ci_setup_data_readers(route, sidx)
output_logs = regen_segment(lr, frs, processes=tested_procs, disable_tqdm=True)
engaged = check_openpilot_enabled(output_logs)
self.assertTrue(engaged, f"openpilot not engaged in {case_name}")
if __name__=='__main__':
unittest.main()

@ -20,7 +20,7 @@ if arch == "Darwin":
qt_env['FRAMEWORKS'] += ['OpenCL']
qt_util = qt_env.Library("qt_util", ["#selfdrive/ui/qt/api.cc", "#selfdrive/ui/qt/util.cc"], LIBS=base_libs)
widgets_src = ["ui.cc", "qt/widgets/input.cc", "qt/widgets/drive_stats.cc", "qt/widgets/wifi.cc",
widgets_src = ["ui.cc", "qt/widgets/input.cc", "qt/widgets/wifi.cc",
"qt/widgets/ssh_keys.cc", "qt/widgets/toggle.cc", "qt/widgets/controls.cc",
"qt/widgets/offroad_alerts.cc", "qt/widgets/prime.cc", "qt/widgets/keyboard.cc",
"qt/widgets/scrollview.cc", "qt/widgets/cameraview.cc", "#third_party/qrcode/QrCode.cc",

@ -11,8 +11,6 @@
#ifdef ENABLE_MAPS
#include "selfdrive/ui/qt/maps/map_settings.h"
#else
#include "selfdrive/ui/qt/widgets/drive_stats.h"
#endif
// HomeWindow: the container for the offroad and onroad UIs
@ -152,7 +150,7 @@ OffroadHome::OffroadHome(QWidget* parent) : QFrame(parent) {
#ifdef ENABLE_MAPS
left_widget->addWidget(new MapSettings);
#else
left_widget->addWidget(new DriveStats);
left_widget->addWidget(new QWidget);
#endif
left_widget->addWidget(new PrimeAdWidget);
left_widget->setStyleSheet("border-radius: 10px;");

@ -1,97 +0,0 @@
#include "selfdrive/ui/qt/widgets/drive_stats.h"
#include <QDebug>
#include <QGridLayout>
#include <QJsonObject>
#include <QVBoxLayout>
#include "common/params.h"
#include "selfdrive/ui/qt/request_repeater.h"
#include "selfdrive/ui/qt/util.h"
static QLabel* newLabel(const QString& text, const QString &type) {
QLabel* label = new QLabel(text);
label->setProperty("type", type);
return label;
}
DriveStats::DriveStats(QWidget* parent) : QFrame(parent) {
metric_ = Params().getBool("IsMetric");
QVBoxLayout* main_layout = new QVBoxLayout(this);
main_layout->setContentsMargins(50, 50, 50, 60);
auto add_stats_layouts = [=](const QString &title, StatsLabels& labels) {
QGridLayout* grid_layout = new QGridLayout;
grid_layout->setVerticalSpacing(10);
grid_layout->setContentsMargins(0, 10, 0, 10);
int row = 0;
grid_layout->addWidget(newLabel(title, "title"), row++, 0, 1, 3);
grid_layout->addItem(new QSpacerItem(0, 50), row++, 0, 1, 1);
grid_layout->addWidget(labels.routes = newLabel("0", "number"), row, 0, Qt::AlignLeft);
grid_layout->addWidget(labels.distance = newLabel("0", "number"), row, 1, Qt::AlignLeft);
grid_layout->addWidget(labels.hours = newLabel("0", "number"), row, 2, Qt::AlignLeft);
grid_layout->addWidget(newLabel((tr("Drives")), "unit"), row + 1, 0, Qt::AlignLeft);
grid_layout->addWidget(labels.distance_unit = newLabel(getDistanceUnit(), "unit"), row + 1, 1, Qt::AlignLeft);
grid_layout->addWidget(newLabel(tr("Hours"), "unit"), row + 1, 2, Qt::AlignLeft);
main_layout->addLayout(grid_layout);
};
add_stats_layouts(tr("ALL TIME"), all_);
main_layout->addStretch();
add_stats_layouts(tr("PAST WEEK"), week_);
if (auto dongleId = getDongleId()) {
QString url = CommaApi::BASE_URL + "/v1.1/devices/" + *dongleId + "/stats";
RequestRepeater* repeater = new RequestRepeater(this, url, "ApiCache_DriveStats", 30);
QObject::connect(repeater, &RequestRepeater::requestDone, this, &DriveStats::parseResponse);
}
setStyleSheet(R"(
DriveStats {
background-color: #333333;
border-radius: 10px;
}
QLabel[type="title"] { font-size: 51px; font-weight: 500; }
QLabel[type="number"] { font-size: 78px; font-weight: 500; }
QLabel[type="unit"] { font-size: 51px; font-weight: 300; color: #A0A0A0; }
)");
}
void DriveStats::updateStats() {
auto update = [=](const QJsonObject& obj, StatsLabels& labels) {
labels.routes->setText(QString::number((int)obj["routes"].toDouble()));
labels.distance->setText(QString::number(int(obj["distance"].toDouble() * (metric_ ? MILE_TO_KM : 1))));
labels.distance_unit->setText(getDistanceUnit());
labels.hours->setText(QString::number((int)(obj["minutes"].toDouble() / 60)));
};
QJsonObject json = stats_.object();
update(json["all"].toObject(), all_);
update(json["week"].toObject(), week_);
}
void DriveStats::parseResponse(const QString& response, bool success) {
if (!success) return;
QJsonDocument doc = QJsonDocument::fromJson(response.trimmed().toUtf8());
if (doc.isNull()) {
qDebug() << "JSON Parse failed on getting past drives statistics";
return;
}
stats_ = doc;
updateStats();
}
void DriveStats::showEvent(QShowEvent* event) {
bool metric = Params().getBool("IsMetric");
if (metric_ != metric) {
metric_ = metric;
updateStats();
}
}

@ -1,25 +0,0 @@
#pragma once
#include <QJsonDocument>
#include <QLabel>
class DriveStats : public QFrame {
Q_OBJECT
public:
explicit DriveStats(QWidget* parent = 0);
private:
void showEvent(QShowEvent *event) override;
void updateStats();
inline QString getDistanceUnit() const { return metric_ ? tr("KM") : tr("Miles"); }
bool metric_;
QJsonDocument stats_;
struct StatsLabels {
QLabel *routes, *distance, *distance_unit, *hours;
} all_, week_;
private slots:
void parseResponse(const QString &response, bool success);
};

@ -274,33 +274,6 @@
<translation>مراجعة</translation>
</message>
</context>
<context>
<name>DriveStats</name>
<message>
<source>Drives</source>
<translation>القيادة</translation>
</message>
<message>
<source>Hours</source>
<translation>ساعات</translation>
</message>
<message>
<source>ALL TIME</source>
<translation>كامل الوقت</translation>
</message>
<message>
<source>PAST WEEK</source>
<translation>الأسبوع الماضي</translation>
</message>
<message>
<source>KM</source>
<translation>كم</translation>
</message>
<message>
<source>Miles</source>
<translation>ميل</translation>
</message>
</context>
<context>
<name>DriverViewScene</name>
<message>

@ -274,33 +274,6 @@
<translation>Überprüfen</translation>
</message>
</context>
<context>
<name>DriveStats</name>
<message>
<source>Drives</source>
<translation>Fahrten</translation>
</message>
<message>
<source>Hours</source>
<translation>Stunden</translation>
</message>
<message>
<source>ALL TIME</source>
<translation>Gesamtzeit</translation>
</message>
<message>
<source>PAST WEEK</source>
<translation>Letzte Woche</translation>
</message>
<message>
<source>KM</source>
<translation>KM</translation>
</message>
<message>
<source>Miles</source>
<translation>Meilen</translation>
</message>
</context>
<context>
<name>DriverViewScene</name>
<message>

@ -274,33 +274,6 @@
<translation>Désengager pour éteindre</translation>
</message>
</context>
<context>
<name>DriveStats</name>
<message>
<source>Drives</source>
<translation>Trajets</translation>
</message>
<message>
<source>Hours</source>
<translation>Heures</translation>
</message>
<message>
<source>ALL TIME</source>
<translation>DEPUIS TOUJOURS</translation>
</message>
<message>
<source>PAST WEEK</source>
<translation>CETTE SEMAINE</translation>
</message>
<message>
<source>KM</source>
<translation>KM</translation>
</message>
<message>
<source>Miles</source>
<translation>Miles</translation>
</message>
</context>
<context>
<name>DriverViewScene</name>
<message>

@ -274,33 +274,6 @@
<translation></translation>
</message>
</context>
<context>
<name>DriveStats</name>
<message>
<source>Drives</source>
<translation></translation>
</message>
<message>
<source>Hours</source>
<translation></translation>
</message>
<message>
<source>ALL TIME</source>
<translation></translation>
</message>
<message>
<source>PAST WEEK</source>
<translation></translation>
</message>
<message>
<source>KM</source>
<translation>km</translation>
</message>
<message>
<source>Miles</source>
<translation></translation>
</message>
</context>
<context>
<name>DriverViewScene</name>
<message>

@ -274,33 +274,6 @@
<translation></translation>
</message>
</context>
<context>
<name>DriveStats</name>
<message>
<source>Drives</source>
<translation></translation>
</message>
<message>
<source>Hours</source>
<translation></translation>
</message>
<message>
<source>ALL TIME</source>
<translation></translation>
</message>
<message>
<source>PAST WEEK</source>
<translation> </translation>
</message>
<message>
<source>KM</source>
<translation>km</translation>
</message>
<message>
<source>Miles</source>
<translation></translation>
</message>
</context>
<context>
<name>DriverViewScene</name>
<message>

@ -274,33 +274,6 @@
<translation>Revisar</translation>
</message>
</context>
<context>
<name>DriveStats</name>
<message>
<source>Drives</source>
<translation>Dirigidas</translation>
</message>
<message>
<source>Hours</source>
<translation>Horas</translation>
</message>
<message>
<source>ALL TIME</source>
<translation>TOTAL</translation>
</message>
<message>
<source>PAST WEEK</source>
<translation>SEMANA PASSADA</translation>
</message>
<message>
<source>KM</source>
<translation>KM</translation>
</message>
<message>
<source>Miles</source>
<translation>Milhas</translation>
</message>
</context>
<context>
<name>DriverViewScene</name>
<message>

@ -274,33 +274,6 @@
<translation></translation>
</message>
</context>
<context>
<name>DriveStats</name>
<message>
<source>Drives</source>
<translation></translation>
</message>
<message>
<source>Hours</source>
<translation></translation>
</message>
<message>
<source>ALL TIME</source>
<translation></translation>
</message>
<message>
<source>PAST WEEK</source>
<translation></translation>
</message>
<message>
<source>KM</source>
<translation></translation>
</message>
<message>
<source>Miles</source>
<translation></translation>
</message>
</context>
<context>
<name>DriverViewScene</name>
<message>

@ -274,33 +274,6 @@
<translation type="unfinished"></translation>
</message>
</context>
<context>
<name>DriveStats</name>
<message>
<source>Drives</source>
<translation>Sürücüler</translation>
</message>
<message>
<source>Hours</source>
<translation>Saat</translation>
</message>
<message>
<source>ALL TIME</source>
<translation>TÜM ZAMANLAR</translation>
</message>
<message>
<source>PAST WEEK</source>
<translation>GEÇEN HAFTA</translation>
</message>
<message>
<source>KM</source>
<translation>KM</translation>
</message>
<message>
<source>Miles</source>
<translation>Mil</translation>
</message>
</context>
<context>
<name>DriverViewScene</name>
<message>

@ -274,33 +274,6 @@
<translation></translation>
</message>
</context>
<context>
<name>DriveStats</name>
<message>
<source>Drives</source>
<translation></translation>
</message>
<message>
<source>Hours</source>
<translation></translation>
</message>
<message>
<source>ALL TIME</source>
<translation></translation>
</message>
<message>
<source>PAST WEEK</source>
<translation></translation>
</message>
<message>
<source>KM</source>
<translation></translation>
</message>
<message>
<source>Miles</source>
<translation></translation>
</message>
</context>
<context>
<name>DriverViewScene</name>
<message>

@ -274,33 +274,6 @@
<translation></translation>
</message>
</context>
<context>
<name>DriveStats</name>
<message>
<source>Drives</source>
<translation></translation>
</message>
<message>
<source>Hours</source>
<translation></translation>
</message>
<message>
<source>ALL TIME</source>
<translation></translation>
</message>
<message>
<source>PAST WEEK</source>
<translation></translation>
</message>
<message>
<source>KM</source>
<translation></translation>
</message>
<message>
<source>Miles</source>
<translation></translation>
</message>
</context>
<context>
<name>DriverViewScene</name>
<message>

@ -107,7 +107,8 @@ void DBCFile::parse(const QString &content) {
int multiplexor_cnt = 0;
while (!stream.atEnd()) {
++line_num;
line = stream.readLine().trimmed();
QString raw_line = stream.readLine();
line = raw_line.trimmed();
if (line.startsWith("BO_ ")) {
multiplexor_cnt = 0;
auto match = bo_regexp.match(line);
@ -170,7 +171,7 @@ void DBCFile::parse(const QString &content) {
}
} else if (line.startsWith("CM_ BO_")) {
if (!line.endsWith("\";")) {
int pos = stream.pos() - line.length() - 1;
int pos = stream.pos() - raw_line.length() - 1;
line = content.mid(pos, content.indexOf("\";", pos));
}
auto match = msg_comment_regexp.match(line);
@ -180,7 +181,7 @@ void DBCFile::parse(const QString &content) {
}
} else if (line.startsWith("CM_ SG_ ")) {
if (!line.endsWith("\";")) {
int pos = stream.pos() - line.length() - 1;
int pos = stream.pos() - raw_line.length() - 1;
line = content.mid(pos, content.indexOf("\";", pos));
}
auto match = sg_comment_regexp.match(line);

@ -69,7 +69,7 @@ TEST_CASE("Parse can messages") {
}
}
TEST_CASE("Parse dbc") {
TEST_CASE("parse_dbc") {
QString content = R"(
BO_ 160 message_1: 8 EON
SG_ signal_1 : 0|12@1+ (1,0) [0|4095] "unit" XXX
@ -83,7 +83,7 @@ VAL_ 160 signal_1 0 "disabled" 1.2 "initializing" 2 "fault";
CM_ BO_ 160 "message comment" ;
CM_ SG_ 160 signal_1 "signal comment";
CM_ SG_ 160 signal_2 "multiple line comment
CM_ SG_ 160 signal_2 "multiple line comment
1
2
";)";
@ -113,7 +113,7 @@ CM_ SG_ 160 signal_2 "multiple line comment
REQUIRE(sig_1->val_desc[2] == std::pair<double, QString>{2, "fault"});
auto &sig_2 = msg->sigs[1];
REQUIRE(sig_2->comment == "multiple line comment\n1\n2");
REQUIRE(sig_2->comment == "multiple line comment \n1\n2");
// multiplexed signals
msg = file.msg(162);

@ -3,9 +3,13 @@ from openpilot.tools.lib.url_file import URLFile
DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/")
def FileReader(fn, debug=False):
def resolve_name(fn):
if fn.startswith("cd:/"):
fn = fn.replace("cd:/", DATA_ENDPOINT)
return fn.replace("cd:/", DATA_ENDPOINT)
return fn
def FileReader(fn, debug=False):
fn = resolve_name(fn)
if fn.startswith(("http://", "https://")):
return URLFile(fn, debug=debug)
return open(fn, "rb")

@ -14,9 +14,10 @@ from lru import LRU
import _io
from openpilot.tools.lib.cache import cache_path_for_file_path, DEFAULT_CACHE_DIR
from openpilot.tools.lib.exceptions import DataUnreadableError
from openpilot.tools.lib.vidindex import hevc_index
from openpilot.common.file_helpers import atomic_write_in_dir
from openpilot.tools.lib.filereader import FileReader
from openpilot.tools.lib.filereader import FileReader, resolve_name
HEVC_SLICE_B = 0
HEVC_SLICE_P = 1
@ -59,6 +60,7 @@ def fingerprint_video(fn):
def ffprobe(fn, fmt=None):
fn = resolve_name(fn)
cmd = ["ffprobe",
"-v", "quiet",
"-print_format", "json",
@ -75,31 +77,6 @@ def ffprobe(fn, fmt=None):
return json.loads(ffprobe_output)
def vidindex(fn, typ):
vidindex_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "vidindex")
vidindex = os.path.join(vidindex_dir, "vidindex")
subprocess.check_call(["make"], cwd=vidindex_dir, stdout=subprocess.DEVNULL)
with tempfile.NamedTemporaryFile() as prefix_f, \
tempfile.NamedTemporaryFile() as index_f:
try:
subprocess.check_call([vidindex, typ, fn, prefix_f.name, index_f.name])
except subprocess.CalledProcessError as e:
raise DataUnreadableError(f"vidindex failed on file {fn}") from e
with open(index_f.name, "rb") as f:
index = f.read()
with open(prefix_f.name, "rb") as f:
prefix = f.read()
index = np.frombuffer(index, np.uint32).reshape(-1, 2)
assert index[-1, 0] == 0xFFFFFFFF
assert index[-1, 1] == os.path.getsize(fn)
return index, prefix
def cache_fn(func):
@wraps(func)
def cache_inner(fn, *args, **kwargs):
@ -114,7 +91,6 @@ def cache_fn(func):
cache_value = pickle.load(cache_file)
else:
cache_value = func(fn, *args, **kwargs)
if cache_path:
with atomic_write_in_dir(cache_path, mode="wb", overwrite=True) as cache_file:
pickle.dump(cache_value, cache_file, -1)
@ -125,13 +101,13 @@ def cache_fn(func):
@cache_fn
def index_stream(fn, typ):
assert typ in ("hevc", )
def index_stream(fn, ft):
if ft != FrameType.h265_stream:
raise NotImplementedError("Only h265 supported")
with FileReader(fn) as f:
assert os.path.exists(f.name), fn
index, prefix = vidindex(f.name, typ)
probe = ffprobe(f.name, typ)
frame_types, dat_len, prefix = hevc_index(fn)
index = np.array(frame_types + [(0xFFFFFFFF, dat_len)], dtype=np.uint32)
probe = ffprobe(fn, "hevc")
return {
'index': index,
@ -140,42 +116,8 @@ def index_stream(fn, typ):
}
def index_videos(camera_paths, cache_dir=DEFAULT_CACHE_DIR):
"""Requires that paths in camera_paths are contiguous and of the same type."""
if len(camera_paths) < 1:
raise ValueError("must provide at least one video to index")
frame_type = fingerprint_video(camera_paths[0])
for fn in camera_paths:
index_video(fn, frame_type, cache_dir)
def index_video(fn, frame_type=None, cache_dir=DEFAULT_CACHE_DIR):
cache_path = cache_path_for_file_path(fn, cache_dir)
if os.path.exists(cache_path):
return
if frame_type is None:
frame_type = fingerprint_video(fn[0])
if frame_type == FrameType.h265_stream:
index_stream(fn, "hevc", cache_dir=cache_dir)
else:
raise NotImplementedError("Only h265 supported")
def get_video_index(fn, frame_type, cache_dir=DEFAULT_CACHE_DIR):
cache_path = cache_path_for_file_path(fn, cache_dir)
if not os.path.exists(cache_path):
index_video(fn, frame_type, cache_dir)
if not os.path.exists(cache_path):
return None
with open(cache_path, "rb") as cache_file:
return pickle.load(cache_file)
return index_stream(fn, frame_type, cache_dir=cache_dir)
def read_file_check_size(f, sz, cookie):
buff = bytearray(sz)

@ -1,8 +1,6 @@
import os
import time
import tempfile
import threading
import urllib.parse
import pycurl
from hashlib import sha256
from io import BytesIO
@ -37,7 +35,8 @@ class URLFile:
self._curl = self._tlocal.curl
except AttributeError:
self._curl = self._tlocal.curl = pycurl.Curl()
mkdirs_exists_ok(Paths.download_cache_root())
if not self._force_download:
mkdirs_exists_ok(Paths.download_cache_root())
def __enter__(self):
return self
@ -65,12 +64,13 @@ class URLFile:
def get_length(self):
if self._length is not None:
return self._length
file_length_path = os.path.join(Paths.download_cache_root(), hash_256(self._url) + "_length")
if os.path.exists(file_length_path) and not self._force_download:
if not self._force_download and os.path.exists(file_length_path):
with open(file_length_path) as file_length:
content = file_length.read()
self._length = int(content)
return self._length
content = file_length.read()
self._length = int(content)
return self._length
self._length = self.get_length_online()
if not self._force_download and self._length != -1:
@ -173,24 +173,4 @@ class URLFile:
@property
def name(self):
"""Returns a local path to file with the URLFile's contents.
This can be used to interface with modules that require local files.
"""
if self._local_file is None:
_, ext = os.path.splitext(urllib.parse.urlparse(self._url).path)
local_fd, local_path = tempfile.mkstemp(suffix=ext)
try:
os.write(local_fd, self.read())
local_file = open(local_path, "rb")
except Exception:
os.remove(local_path)
raise
finally:
os.close(local_fd)
self._local_file = local_file
self.read = self._local_file.read
self.seek = self._local_file.seek
return self._local_file.name
return self._url

@ -0,0 +1,301 @@
#!/usr/bin/env python3
import argparse
import struct
from enum import IntEnum
from typing import Tuple
from openpilot.tools.lib.filereader import FileReader
# H.265 specification
# https://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-H.265-201802-S!!PDF-E&type=items
NAL_UNIT_START_CODE = b"\x00\x00\x01"
NAL_UNIT_START_CODE_SIZE = len(NAL_UNIT_START_CODE)
NAL_UNIT_HEADER_SIZE = 2
class HevcNalUnitType(IntEnum):
TRAIL_N = 0 # RBSP structure: slice_segment_layer_rbsp( )
TRAIL_R = 1 # RBSP structure: slice_segment_layer_rbsp( )
TSA_N = 2 # RBSP structure: slice_segment_layer_rbsp( )
TSA_R = 3 # RBSP structure: slice_segment_layer_rbsp( )
STSA_N = 4 # RBSP structure: slice_segment_layer_rbsp( )
STSA_R = 5 # RBSP structure: slice_segment_layer_rbsp( )
RADL_N = 6 # RBSP structure: slice_segment_layer_rbsp( )
RADL_R = 7 # RBSP structure: slice_segment_layer_rbsp( )
RASL_N = 8 # RBSP structure: slice_segment_layer_rbsp( )
RASL_R = 9 # RBSP structure: slice_segment_layer_rbsp( )
RSV_VCL_N10 = 10
RSV_VCL_R11 = 11
RSV_VCL_N12 = 12
RSV_VCL_R13 = 13
RSV_VCL_N14 = 14
RSV_VCL_R15 = 15
BLA_W_LP = 16 # RBSP structure: slice_segment_layer_rbsp( )
BLA_W_RADL = 17 # RBSP structure: slice_segment_layer_rbsp( )
BLA_N_LP = 18 # RBSP structure: slice_segment_layer_rbsp( )
IDR_W_RADL = 19 # RBSP structure: slice_segment_layer_rbsp( )
IDR_N_LP = 20 # RBSP structure: slice_segment_layer_rbsp( )
CRA_NUT = 21 # RBSP structure: slice_segment_layer_rbsp( )
RSV_IRAP_VCL22 = 22
RSV_IRAP_VCL23 = 23
RSV_VCL24 = 24
RSV_VCL25 = 25
RSV_VCL26 = 26
RSV_VCL27 = 27
RSV_VCL28 = 28
RSV_VCL29 = 29
RSV_VCL30 = 30
RSV_VCL31 = 31
VPS_NUT = 32 # RBSP structure: video_parameter_set_rbsp( )
SPS_NUT = 33 # RBSP structure: seq_parameter_set_rbsp( )
PPS_NUT = 34 # RBSP structure: pic_parameter_set_rbsp( )
AUD_NUT = 35
EOS_NUT = 36
EOB_NUT = 37
FD_NUT = 38
PREFIX_SEI_NUT = 39
SUFFIX_SEI_NUT = 40
RSV_NVCL41 = 41
RSV_NVCL42 = 42
RSV_NVCL43 = 43
RSV_NVCL44 = 44
RSV_NVCL45 = 45
RSV_NVCL46 = 46
RSV_NVCL47 = 47
UNSPEC48 = 48
UNSPEC49 = 49
UNSPEC50 = 50
UNSPEC51 = 51
UNSPEC52 = 52
UNSPEC53 = 53
UNSPEC54 = 54
UNSPEC55 = 55
UNSPEC56 = 56
UNSPEC57 = 57
UNSPEC58 = 58
UNSPEC59 = 59
UNSPEC60 = 60
UNSPEC61 = 61
UNSPEC62 = 62
UNSPEC63 = 63
# B.2.2 Byte stream NAL unit semantics
# - The nal_unit_type within the nal_unit( ) syntax structure is equal to VPS_NUT, SPS_NUT or PPS_NUT.
# - The byte stream NAL unit syntax structure contains the first NAL unit of an access unit in decoding
# order, as specified in clause 7.4.2.4.4.
HEVC_PARAMETER_SET_NAL_UNITS = (
HevcNalUnitType.VPS_NUT,
HevcNalUnitType.SPS_NUT,
HevcNalUnitType.PPS_NUT,
)
# 3.29 coded slice segment NAL unit: A NAL unit that has nal_unit_type in the range of TRAIL_N to RASL_R,
# inclusive, or in the range of BLA_W_LP to RSV_IRAP_VCL23, inclusive, which indicates that the NAL unit
# contains a coded slice segment
HEVC_CODED_SLICE_SEGMENT_NAL_UNITS = (
HevcNalUnitType.TRAIL_N,
HevcNalUnitType.TRAIL_R,
HevcNalUnitType.TSA_N,
HevcNalUnitType.TSA_R,
HevcNalUnitType.STSA_N,
HevcNalUnitType.STSA_R,
HevcNalUnitType.RADL_N,
HevcNalUnitType.RADL_R,
HevcNalUnitType.RASL_N,
HevcNalUnitType.RASL_R,
HevcNalUnitType.BLA_W_LP,
HevcNalUnitType.BLA_W_RADL,
HevcNalUnitType.BLA_N_LP,
HevcNalUnitType.IDR_W_RADL,
HevcNalUnitType.IDR_N_LP,
HevcNalUnitType.CRA_NUT,
)
class VideoFileInvalid(Exception):
pass
def get_ue(dat: bytes, start_idx: int, skip_bits: int) -> Tuple[int, int]:
prefix_val = 0
prefix_len = 0
suffix_val = 0
suffix_len = 0
i = start_idx
while i < len(dat):
j = 7
while j >= 0:
if skip_bits > 0:
skip_bits -= 1
elif prefix_val == 0:
prefix_val = (dat[i] >> j) & 1
prefix_len += 1
else:
suffix_val = (suffix_val << 1) | ((dat[i] >> j) & 1)
suffix_len += 1
j -= 1
if prefix_val == 1 and prefix_len - 1 == suffix_len:
val = 2**(prefix_len-1) - 1 + suffix_val
size = prefix_len + suffix_len
return val, size
i += 1
raise VideoFileInvalid("invalid exponential-golomb code")
def require_nal_unit_start(dat: bytes, nal_unit_start: int) -> None:
if nal_unit_start >= len(dat):
raise ValueError("start index must be less than data length")
if nal_unit_start < 1:
raise ValueError("start index must be greater than zero")
if dat[nal_unit_start-1] != 0x00:
raise VideoFileInvalid("start code must be preceded by 0x00")
if dat[nal_unit_start:nal_unit_start + NAL_UNIT_START_CODE_SIZE] != NAL_UNIT_START_CODE:
raise VideoFileInvalid("data must begin with start code")
def get_hevc_nal_unit_length(dat: bytes, nal_unit_start: int) -> int:
try:
pos = dat.index(NAL_UNIT_START_CODE, nal_unit_start + NAL_UNIT_START_CODE_SIZE)
except ValueError:
pos = -1
# length of NAL unit is byte count up to next NAL unit start index
nal_unit_len = (pos if pos != -1 else len(dat)) - nal_unit_start
return nal_unit_len
def get_hevc_nal_unit_type(dat: bytes, nal_unit_start: int) -> HevcNalUnitType:
# 7.3.1.2 NAL unit header syntax
# nal_unit_header( ) { // descriptor
# forbidden_zero_bit f(1)
# nal_unit_type u(6)
# nuh_layer_id u(6)
# nuh_temporal_id_plus1 u(3)
# }
header_start = nal_unit_start + NAL_UNIT_START_CODE_SIZE
nal_unit_header = dat[header_start:header_start + NAL_UNIT_HEADER_SIZE]
if len(nal_unit_header) != 2:
raise VideoFileInvalid("data to short to contain nal unit header")
return HevcNalUnitType((nal_unit_header[0] >> 1) & 0x3F)
def get_hevc_slice_type(dat: bytes, nal_unit_start: int, nal_unit_type: HevcNalUnitType) -> Tuple[int, bool]:
# 7.3.2.9 Slice segment layer RBSP syntax
# slice_segment_layer_rbsp( ) {
# slice_segment_header( )
# slice_segment_data( )
# rbsp_slice_segment_trailing_bits( )
# }
# ...
# 7.3.6.1 General slice segment header syntax
# slice_segment_header( ) { // descriptor
# first_slice_segment_in_pic_flag u(1)
# if( nal_unit_type >= BLA_W_LP && nal_unit_type <= RSV_IRAP_VCL23 )
# no_output_of_prior_pics_flag u(1)
# slice_pic_parameter_set_id ue(v)
# if( !first_slice_segment_in_pic_flag ) {
# if( dependent_slice_segments_enabled_flag )
# dependent_slice_segment_flag u(1)
# slice_segment_address u(v)
# }
# if( !dependent_slice_segment_flag ) {
# for( i = 0; i < num_extra_slice_header_bits; i++ )
# slice_reserved_flag[ i ] u(1)
# slice_type ue(v)
# ...
rbsp_start = nal_unit_start + NAL_UNIT_START_CODE_SIZE + NAL_UNIT_HEADER_SIZE
skip_bits = 0
# 7.4.7.1 General slice segment header semantics
# first_slice_segment_in_pic_flag equal to 1 specifies that the slice segment is the first slice segment of the picture in
# decoding order. first_slice_segment_in_pic_flag equal to 0 specifies that the slice segment is not the first slice segment
# of the picture in decoding order.
is_first_slice = dat[rbsp_start] >> 7 & 1 == 1
if not is_first_slice:
# TODO: parse dependent_slice_segment_flag and slice_segment_address and get real slice_type
# for now since we don't use it return -1 for slice_type
return (-1, is_first_slice)
skip_bits += 1 # skip past first_slice_segment_in_pic_flag
if nal_unit_type >= HevcNalUnitType.BLA_W_LP and nal_unit_type <= HevcNalUnitType.RSV_IRAP_VCL23:
# 7.4.7.1 General slice segment header semantics
# no_output_of_prior_pics_flag affects the output of previously-decoded pictures in the decoded picture buffer after the
# decoding of an IDR or a BLA picture that is not the first picture in the bitstream as specified in Annex C.
skip_bits += 1 # skip past no_output_of_prior_pics_flag
# 7.4.7.1 General slice segment header semantics
# slice_pic_parameter_set_id specifies the value of pps_pic_parameter_set_id for the PPS in use.
# The value of slice_pic_parameter_set_id shall be in the range of 0 to 63, inclusive.
_, size = get_ue(dat, rbsp_start, skip_bits)
skip_bits += size # skip past slice_pic_parameter_set_id
# 7.4.3.3.1 General picture parameter set RBSP semanal_unit_lenntics
# num_extra_slice_header_bits specifies the number of extra slice header bits that are present in the slice header RBSP
# for coded pictures referring to the PPS. The value of num_extra_slice_header_bits shall be in the range of 0 to 2, inclusive,
# in bitstreams conforming to this version of this Specification. Other values for num_extra_slice_header_bits are reserved
# for future use by ITU-T | ISO/IEC. However, decoders shall allow num_extra_slice_header_bits to have any value.
# TODO: get from PPS_NUT pic_parameter_set_rbsp( ) for corresponding slice_pic_parameter_set_id
num_extra_slice_header_bits = 0
skip_bits += num_extra_slice_header_bits
# 7.4.7.1 General slice segment header semantics
# slice_type specifies the coding type of the slice according to Table 7-7.
# Table 7-7 - Name association to slice_type
# slice_type | Name of slice_type
# 0 | B (B slice)
# 1 | P (P slice)
# 2 | I (I slice)
# unsigned integer 0-th order Exp-Golomb-coded syntax element with the left bit first
slice_type, _ = get_ue(dat, rbsp_start, skip_bits)
if slice_type > 2:
raise VideoFileInvalid("slice_type must be 0, 1, or 2")
return slice_type, is_first_slice
def hevc_index(hevc_file_name: str, allow_corrupt: bool=False) -> Tuple[list, int, bytes]:
with FileReader(hevc_file_name) as f:
dat = f.read()
if len(dat) < NAL_UNIT_START_CODE_SIZE + 1:
raise VideoFileInvalid("data is too short")
prefix_dat = b""
frame_types = list()
try:
i = 1 # skip past first byte 0x00 (verified by get_hevc_nal_info)
while i < len(dat):
require_nal_unit_start(dat, i)
nal_unit_len = get_hevc_nal_unit_length(dat, i)
nal_unit_type = get_hevc_nal_unit_type(dat, i)
if nal_unit_type in HEVC_PARAMETER_SET_NAL_UNITS:
prefix_dat += dat[i:i+nal_unit_len]
elif nal_unit_type in HEVC_CODED_SLICE_SEGMENT_NAL_UNITS:
slice_type, is_first_slice = get_hevc_slice_type(dat, i, nal_unit_type)
if is_first_slice:
frame_types.append((slice_type, i))
i += nal_unit_len
except Exception:
if not allow_corrupt:
raise
return frame_types, len(dat), prefix_dat
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("input_file", type=str)
parser.add_argument("output_prefix_file", type=str)
parser.add_argument("output_index_file", type=str)
args = parser.parse_args()
frame_types, dat_len, prefix_dat = hevc_index(args.input_file)
with open(args.output_prefix_file, "wb") as f:
f.write(prefix_dat)
with open(args.output_index_file, "wb") as f:
for ft, fp in frame_types:
f.write(struct.pack("<II", ft, fp))
f.write(struct.pack("<II", 0xFFFFFFFF, dat_len))
if __name__ == "__main__":
main()

@ -1 +0,0 @@
vidindex

@ -1,6 +0,0 @@
CC := gcc
vidindex: bitstream.c bitstream.h vidindex.c
$(eval $@_TMP := $(shell mktemp))
$(CC) -std=c99 bitstream.c vidindex.c -o $($@_TMP)
mv $($@_TMP) $@

@ -1,118 +0,0 @@
#include "./bitstream.h"
#include <stdbool.h>
#include <assert.h>
static const uint32_t BS_MASKS[33] = {
0, 0x1L, 0x3L, 0x7L, 0xFL, 0x1FL,
0x3FL, 0x7FL, 0xFFL, 0x1FFL, 0x3FFL, 0x7FFL,
0xFFFL, 0x1FFFL, 0x3FFFL, 0x7FFFL, 0xFFFFL, 0x1FFFFL,
0x3FFFFL, 0x7FFFFL, 0xFFFFFL, 0x1FFFFFL, 0x3FFFFFL, 0x7FFFFFL,
0xFFFFFFL, 0x1FFFFFFL, 0x3FFFFFFL, 0x7FFFFFFL, 0xFFFFFFFL, 0x1FFFFFFFL,
0x3FFFFFFFL, 0x7FFFFFFFL, 0xFFFFFFFFL};
void bs_init(struct bitstream* bs, const uint8_t* buffer, size_t input_size) {
bs->buffer_ptr = buffer;
bs->buffer_end = buffer + input_size;
bs->value = 0;
bs->pos = 0;
bs->shift = 8;
bs->size = input_size * 8;
}
uint32_t bs_get(struct bitstream* bs, int n) {
if (n > 32)
return 0;
bs->pos += n;
bs->shift += n;
while (bs->shift > 8) {
if (bs->buffer_ptr < bs->buffer_end) {
bs->value <<= 8;
bs->value |= *bs->buffer_ptr++;
bs->shift -= 8;
} else {
bs_seek(bs, bs->pos - n);
return 0;
// bs->value <<= 8;
// bs->shift -= 8;
}
}
return (bs->value >> (8 - bs->shift)) & BS_MASKS[n];
}
void bs_seek(struct bitstream* bs, size_t new_pos) {
bs->pos = (new_pos / 32) * 32;
bs->shift = 8;
bs->value = 0;
bs_get(bs, new_pos % 32);
}
uint32_t bs_peek(struct bitstream* bs, int n) {
struct bitstream bak = *bs;
return bs_get(&bak, n);
}
size_t bs_remain(struct bitstream* bs) {
return bs->size - bs->pos;
}
int bs_eof(struct bitstream* bs) {
return bs_remain(bs) == 0;
}
uint32_t bs_ue(struct bitstream* bs) {
static const uint8_t exp_golomb_bits[256] = {
8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
uint32_t bits, read = 0;
int bits_left;
uint8_t coded;
int done = 0;
bits = 0;
// we want to read 8 bits at a time - if we don't have 8 bits,
// read what's left, and shift. The exp_golomb_bits calc remains the
// same.
while (!done) {
bits_left = bs_remain(bs);
if (bits_left < 8) {
read = bs_peek(bs, bits_left) << (8 - bits_left);
done = 1;
} else {
read = bs_peek(bs, 8);
if (read == 0) {
bs_get(bs, 8);
bits += 8;
} else {
done = 1;
}
}
}
coded = exp_golomb_bits[read];
bs_get(bs, coded);
bits += coded;
// printf("ue - bits %d\n", bits);
return bs_get(bs, bits + 1) - 1;
}
int32_t bs_se(struct bitstream* bs) {
uint32_t ret;
ret = bs_ue(bs);
if ((ret & 0x1) == 0) {
ret >>= 1;
int32_t temp = 0 - ret;
return temp;
}
return (ret + 1) >> 1;
}

@ -1,26 +0,0 @@
#ifndef bitstream_H
#define bitstream_H
#include <stddef.h>
#include <stdint.h>
struct bitstream {
const uint8_t *buffer_ptr;
const uint8_t *buffer_end;
uint64_t value;
uint32_t pos;
uint32_t shift;
size_t size;
};
void bs_init(struct bitstream *bs, const uint8_t *buffer, size_t input_size);
void bs_seek(struct bitstream *bs, size_t new_pos);
uint32_t bs_get(struct bitstream *bs, int n);
uint32_t bs_peek(struct bitstream *bs, int n);
size_t bs_remain(struct bitstream *bs);
int bs_eof(struct bitstream *bs);
uint32_t bs_ue(struct bitstream *bs);
int32_t bs_se(struct bitstream *bs);
#endif

@ -1,307 +0,0 @@
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <assert.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include "./bitstream.h"
#define START_CODE 0x000001
static uint32_t read24be(const uint8_t* ptr) {
return (ptr[0] << 16) | (ptr[1] << 8) | ptr[2];
}
static void write32le(FILE *of, uint32_t v) {
uint8_t va[4] = {
v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, (v >> 24) & 0xff
};
fwrite(va, 1, sizeof(va), of);
}
// Table 7-1
enum hevc_nal_type {
HEVC_NAL_TYPE_TRAIL_N = 0,
HEVC_NAL_TYPE_TRAIL_R = 1,
HEVC_NAL_TYPE_TSA_N = 2,
HEVC_NAL_TYPE_TSA_R = 3,
HEVC_NAL_TYPE_STSA_N = 4,
HEVC_NAL_TYPE_STSA_R = 5,
HEVC_NAL_TYPE_RADL_N = 6,
HEVC_NAL_TYPE_RADL_R = 7,
HEVC_NAL_TYPE_RASL_N = 8,
HEVC_NAL_TYPE_RASL_R = 9,
HEVC_NAL_TYPE_BLA_W_LP = 16,
HEVC_NAL_TYPE_BLA_W_RADL = 17,
HEVC_NAL_TYPE_BLA_N_LP = 18,
HEVC_NAL_TYPE_IDR_W_RADL = 19,
HEVC_NAL_TYPE_IDR_N_LP = 20,
HEVC_NAL_TYPE_CRA_NUT = 21,
HEVC_NAL_TYPE_RSV_IRAP_VCL23 = 23,
HEVC_NAL_TYPE_VPS_NUT = 32,
HEVC_NAL_TYPE_SPS_NUT = 33,
HEVC_NAL_TYPE_PPS_NUT = 34,
HEVC_NAL_TYPE_AUD_NUT = 35,
HEVC_NAL_TYPE_EOS_NUT = 36,
HEVC_NAL_TYPE_EOB_NUT = 37,
HEVC_NAL_TYPE_FD_NUT = 38,
HEVC_NAL_TYPE_PREFIX_SEI_NUT = 39,
HEVC_NAL_TYPE_SUFFIX_SEI_NUT = 40,
};
// Table 7-7
enum hevc_slice_type {
HEVC_SLICE_B = 0,
HEVC_SLICE_P = 1,
HEVC_SLICE_I = 2,
};
static void hevc_index(const uint8_t *data, size_t file_size, FILE *of_prefix, FILE *of_index) {
const uint8_t* ptr = data;
const uint8_t* ptr_end = data + file_size;
assert(ptr[0] == 0);
ptr++;
assert(read24be(ptr) == START_CODE);
// pps. ignore for now
uint32_t num_extra_slice_header_bits = 0;
uint32_t dependent_slice_segments_enabled_flag = 0;
while (ptr < ptr_end) {
const uint8_t* next = ptr+1;
for (; next < ptr_end-4; next++) {
if (read24be(next) == START_CODE) break;
}
size_t nal_size = next - ptr;
if (nal_size < 6) {
break;
}
{
struct bitstream bs = {0};
bs_init(&bs, ptr, nal_size);
uint32_t start_code = bs_get(&bs, 24);
assert(start_code == 0x000001);
// nal_unit_header
uint32_t forbidden_zero_bit = bs_get(&bs, 1);
uint32_t nal_unit_type = bs_get(&bs, 6);
uint32_t nuh_layer_id = bs_get(&bs, 6);
uint32_t nuh_temporal_id_plus1 = bs_get(&bs, 3);
// if (nal_unit_type != 1) printf("%3d -- %3d %10d %lu\n", nal_unit_type, frame_num, (uint32_t)(ptr-data), nal_size);
switch (nal_unit_type) {
case HEVC_NAL_TYPE_VPS_NUT:
case HEVC_NAL_TYPE_SPS_NUT:
case HEVC_NAL_TYPE_PPS_NUT:
fwrite(ptr, 1, nal_size, of_prefix);
break;
case HEVC_NAL_TYPE_TRAIL_N:
case HEVC_NAL_TYPE_TRAIL_R:
case HEVC_NAL_TYPE_TSA_N:
case HEVC_NAL_TYPE_TSA_R:
case HEVC_NAL_TYPE_STSA_N:
case HEVC_NAL_TYPE_STSA_R:
case HEVC_NAL_TYPE_RADL_N:
case HEVC_NAL_TYPE_RADL_R:
case HEVC_NAL_TYPE_RASL_N:
case HEVC_NAL_TYPE_RASL_R:
case HEVC_NAL_TYPE_BLA_W_LP:
case HEVC_NAL_TYPE_BLA_W_RADL:
case HEVC_NAL_TYPE_BLA_N_LP:
case HEVC_NAL_TYPE_IDR_W_RADL:
case HEVC_NAL_TYPE_IDR_N_LP:
case HEVC_NAL_TYPE_CRA_NUT: {
// slice_segment_header
uint32_t first_slice_segment_in_pic_flag = bs_get(&bs, 1);
if (nal_unit_type >= HEVC_NAL_TYPE_BLA_W_LP && nal_unit_type <= HEVC_NAL_TYPE_RSV_IRAP_VCL23) {
uint32_t no_output_of_prior_pics_flag = bs_get(&bs, 1);
}
uint32_t slice_pic_parameter_set_id = bs_get(&bs, 1);
if (!first_slice_segment_in_pic_flag) {
// ...
break;
}
if (!dependent_slice_segments_enabled_flag) {
for (int i=0; i<num_extra_slice_header_bits; i++) {
bs_get(&bs, 1);
}
uint32_t slice_type = bs_ue(&bs);
// write the index
write32le(of_index, slice_type);
write32le(of_index, ptr - data);
// ...
}
break;
}
}
//...
// emulation_prevention_three_byte
}
ptr = next;
}
write32le(of_index, -1);
write32le(of_index, file_size);
}
// Table 7-1
enum h264_nal_type {
H264_NAL_SLICE = 1,
H264_NAL_DPA = 2,
H264_NAL_DPB = 3,
H264_NAL_DPC = 4,
H264_NAL_IDR_SLICE = 5,
H264_NAL_SEI = 6,
H264_NAL_SPS = 7,
H264_NAL_PPS = 8,
H264_NAL_AUD = 9,
H264_NAL_END_SEQUENCE = 10,
H264_NAL_END_STREAM = 11,
H264_NAL_FILLER_DATA = 12,
H264_NAL_SPS_EXT = 13,
H264_NAL_AUXILIARY_SLICE = 19,
};
enum h264_slice_type {
H264_SLICE_P = 0,
H264_SLICE_B = 1,
H264_SLICE_I = 2,
// ...
};
static void h264_index(const uint8_t *data, size_t file_size, FILE *of_prefix, FILE *of_index) {
const uint8_t* ptr = data;
const uint8_t* ptr_end = data + file_size;
assert(ptr[0] == 0);
ptr++;
assert(read24be(ptr) == START_CODE);
uint32_t sps_log2_max_frame_num_minus4;
int last_frame_num = -1;
while (ptr < ptr_end) {
const uint8_t* next = ptr+1;
for (; next < ptr_end-4; next++) {
if (read24be(next) == START_CODE) break;
}
size_t nal_size = next - ptr;
if (nal_size < 5) {
break;
}
{
struct bitstream bs = {0};
bs_init(&bs, ptr, nal_size);
uint32_t start_code = bs_get(&bs, 24);
assert(start_code == 0x000001);
// nal_unit_header
uint32_t forbidden_zero_bit = bs_get(&bs, 1);
uint32_t nal_ref_idx = bs_get(&bs, 2);
uint32_t nal_unit_type = bs_get(&bs, 5);
switch (nal_unit_type) {
case H264_NAL_SPS:
{
uint32_t profile_idx = bs_get(&bs, 8);
uint32_t constraint_sets = bs_get(&bs, 4);
uint32_t reserved = bs_get(&bs, 5);
uint32_t level_idc = bs_get(&bs, 5);
uint32_t seq_parameter_set_id = bs_ue(&bs);
sps_log2_max_frame_num_minus4 = bs_ue(&bs);
}
// fallthrough
case H264_NAL_PPS:
fwrite(ptr, 1, nal_size, of_prefix);
break;
case H264_NAL_SLICE:
case H264_NAL_IDR_SLICE: {
// slice header
uint32_t first_mb_in_slice = bs_ue(&bs);
uint32_t slice_type = bs_ue(&bs);
uint32_t pic_parameter_set_id = bs_ue(&bs);
uint32_t frame_num = bs_get(&bs, sps_log2_max_frame_num_minus4+4);
if (first_mb_in_slice == 0) {
write32le(of_index, slice_type);
write32le(of_index, ptr - data);
}
break;
}
}
}
ptr = next;
}
write32le(of_index, -1);
write32le(of_index, file_size);
}
int main(int argc, char** argv) {
if (argc != 5) {
fprintf(stderr, "usage: %s h264|hevc file_path out_prefix out_index\n", argv[0]);
exit(1);
}
const char* file_type = argv[1];
const char* file_path = argv[2];
int fd = open(file_path, O_RDONLY, 0);
if (fd < 0) {
fprintf(stderr, "error: couldn't open %s\n", file_path);
exit(1);
}
FILE *of_prefix = fopen(argv[3], "wb");
assert(of_prefix);
FILE *of_index = fopen(argv[4], "wb");
assert(of_index);
off_t file_size = lseek(fd, 0, SEEK_END);
lseek(fd, 0, SEEK_SET);
assert(file_size > 4);
const uint8_t* data = (const uint8_t*)mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
assert(data != MAP_FAILED);
if (strcmp(file_type, "hevc") == 0) {
hevc_index(data, file_size, of_prefix, of_index);
} else if (strcmp(file_type, "h264") == 0) {
h264_index(data, file_size, of_prefix, of_index);
} else {
assert(false);
}
munmap((void*)data, file_size);
close(fd);
return 0;
}
Loading…
Cancel
Save