loggerd: move to system/ (#27534)

.github/workflows/selfdrive_tests.yaml

@@ -204,7 +204,7 @@ jobs:
                         $UNIT_TEST selfdrive/boardd && \
                         $UNIT_TEST selfdrive/controls && \
                         $UNIT_TEST selfdrive/monitoring && \
-                        $UNIT_TEST selfdrive/loggerd && \
+                        $UNIT_TEST system/loggerd && \
                         $UNIT_TEST selfdrive/car && \
                         $UNIT_TEST selfdrive/locationd && \
                         selfdrive/locationd/test/_test_locationd_lib.py && \
@@ -219,7 +219,7 @@ jobs:
                         ./common/tests/test_util && \
                         ./common/tests/test_swaglog && \
                         ./selfdrive/boardd/tests/test_boardd_usbprotocol && \
-                        ./selfdrive/loggerd/tests/test_logger &&\
+                        ./system/loggerd/tests/test_logger &&\
                         ./system/proclogd/tests/test_proclog && \
                         ./tools/replay/tests/test_replay && \
                         ./tools/cabana/tests/test_cabana && \

Jenkinsfile

@@ -157,8 +157,8 @@ pipeline {
             phone_steps("tici-common", [
               ["build", "cd selfdrive/manager && ./build.py"],
               ["test power draw", "python system/hardware/tici/test_power_draw.py"],
-              ["test loggerd", "python selfdrive/loggerd/tests/test_loggerd.py"],
+              ["test loggerd", "python system/loggerd/tests/test_loggerd.py"],
-              ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib python selfdrive/loggerd/tests/test_encoder.py"],
+              ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib python system/loggerd/tests/test_encoder.py"],
               ["test pigeond", "python system/sensord/tests/test_pigeond.py"],
               ["test manager", "python selfdrive/manager/test/test_manager.py"],
               ["test pandad", "python selfdrive/boardd/tests/test_pandad.py"],

SConstruct

@@ -424,7 +424,7 @@ SConscript(['selfdrive/controls/lib/longitudinal_mpc_lib/SConscript'])
 
 SConscript(['selfdrive/boardd/SConscript'])
 
-SConscript(['selfdrive/loggerd/SConscript'])
+SConscript(['system/loggerd/SConscript'])
 
 SConscript(['selfdrive/locationd/SConscript'])
 SConscript(['system/sensord/SConscript'])

docs/overview.rst

@@ -5,7 +5,7 @@ openpilot
    :maxdepth: 4
 
    Debugging <selfdrive/debug/README.md>
-   selfdrive/loggerd/README.md
+   system/loggerd/README.md
    Driver Monitoring <selfdrive/monitoring/README.md>
    Process Replay <selfdrive/test/process_replay/README.md>
 

release/files_common

@@ -253,28 +253,28 @@ system/proclogd/main.cc
 system/proclogd/proclog.cc
 system/proclogd/proclog.h
 
-selfdrive/loggerd/.gitignore
+system/loggerd/.gitignore
-selfdrive/loggerd/SConscript
+system/loggerd/SConscript
-selfdrive/loggerd/encoder/encoder.cc
+system/loggerd/encoder/encoder.cc
-selfdrive/loggerd/encoder/encoder.h
+system/loggerd/encoder/encoder.h
-selfdrive/loggerd/encoder/v4l_encoder.cc
+system/loggerd/encoder/v4l_encoder.cc
-selfdrive/loggerd/encoder/v4l_encoder.h
+system/loggerd/encoder/v4l_encoder.h
-selfdrive/loggerd/video_writer.cc
+system/loggerd/video_writer.cc
-selfdrive/loggerd/video_writer.h
+system/loggerd/video_writer.h
-selfdrive/loggerd/logger.cc
+system/loggerd/logger.cc
-selfdrive/loggerd/logger.h
+system/loggerd/logger.h
-selfdrive/loggerd/loggerd.cc
+system/loggerd/loggerd.cc
-selfdrive/loggerd/loggerd.h
+system/loggerd/loggerd.h
-selfdrive/loggerd/encoderd.cc
+system/loggerd/encoderd.cc
-selfdrive/loggerd/bootlog.cc
+system/loggerd/bootlog.cc
-selfdrive/loggerd/encoder/ffmpeg_encoder.cc
+system/loggerd/encoder/ffmpeg_encoder.cc
-selfdrive/loggerd/encoder/ffmpeg_encoder.h
+system/loggerd/encoder/ffmpeg_encoder.h
-
+
-selfdrive/loggerd/__init__.py
+system/loggerd/__init__.py
-selfdrive/loggerd/config.py
+system/loggerd/config.py
-selfdrive/loggerd/uploader.py
+system/loggerd/uploader.py
-selfdrive/loggerd/deleter.py
+system/loggerd/deleter.py
-selfdrive/loggerd/xattr_cache.py
+system/loggerd/xattr_cache.py
 
 system/sensord/.gitignore
 system/sensord/SConscript

selfdrive/athena/athenad.py

@@ -36,8 +36,8 @@ from common.file_helpers import CallbackReader
 from common.params import Params
 from common.realtime import sec_since_boot, set_core_affinity
 from system.hardware import HARDWARE, PC, AGNOS
-from selfdrive.loggerd.config import ROOT
+from system.loggerd.config import ROOT
-from selfdrive.loggerd.xattr_cache import getxattr, setxattr
+from system.loggerd.xattr_cache import getxattr, setxattr
 from selfdrive.statsd import STATS_DIR
 from system.swaglog import SWAGLOG_DIR, cloudlog
 from system.version import get_commit, get_origin, get_short_branch, get_version

selfdrive/manager/manager.py

@@ -29,7 +29,7 @@ def manager_init() -> None:
   set_time(cloudlog)
 
   # save boot log
-  subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"))
+  subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "system/loggerd"))
 
   params = Params()
   params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)

selfdrive/manager/process_config.py

@@ -34,8 +34,8 @@ procs = [
 
   DaemonProcess("manage_athenad", "selfdrive.athena.manage_athenad", "AthenadPid"),
   NativeProcess("dmonitoringmodeld", "selfdrive/modeld", ["./dmonitoringmodeld"], enabled=(not PC or WEBCAM), callback=driverview),
-  NativeProcess("encoderd", "selfdrive/loggerd", ["./encoderd"]),
+  NativeProcess("encoderd", "system/loggerd", ["./encoderd"]),
-  NativeProcess("loggerd", "selfdrive/loggerd", ["./loggerd"], onroad=False, callback=logging),
+  NativeProcess("loggerd", "system/loggerd", ["./loggerd"], onroad=False, callback=logging),
   NativeProcess("modeld", "selfdrive/modeld", ["./modeld"]),
   NativeProcess("mapsd", "selfdrive/navd", ["./map_renderer"], enabled=False),
   NativeProcess("navmodeld", "selfdrive/modeld", ["./navmodeld"], enabled=False),
@@ -47,7 +47,7 @@ procs = [
   PythonProcess("calibrationd", "selfdrive.locationd.calibrationd"),
   PythonProcess("torqued", "selfdrive.locationd.torqued"),
   PythonProcess("controlsd", "selfdrive.controls.controlsd"),
-  PythonProcess("deleter", "selfdrive.loggerd.deleter", offroad=True),
+  PythonProcess("deleter", "system.loggerd.deleter", offroad=True),
   PythonProcess("dmonitoringd", "selfdrive.monitoring.dmonitoringd", enabled=(not PC or WEBCAM), callback=driverview),
   PythonProcess("laikad", "selfdrive.locationd.laikad"),
   PythonProcess("rawgpsd", "system.sensord.rawgps.rawgpsd", enabled=TICI),
@@ -61,7 +61,7 @@ procs = [
   PythonProcess("thermald", "selfdrive.thermald.thermald", offroad=True),
   PythonProcess("tombstoned", "selfdrive.tombstoned", enabled=not PC, offroad=True),
   PythonProcess("updated", "selfdrive.updated", enabled=not PC, onroad=False, offroad=True),
-  PythonProcess("uploader", "selfdrive.loggerd.uploader", offroad=True),
+  PythonProcess("uploader", "system.loggerd.uploader", offroad=True),
   PythonProcess("statsd", "selfdrive.statsd", offroad=True),
 
   # debug procs

selfdrive/statsd.py

@@ -13,7 +13,7 @@ from system.swaglog import cloudlog
 from system.hardware import HARDWARE
 from common.file_helpers import atomic_write_in_dir
 from system.version import get_normalized_origin, get_short_branch, get_short_version, is_dirty
-from selfdrive.loggerd.config import STATS_DIR, STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S
+from system.loggerd.config import STATS_DIR, STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S
 
 
 class METRIC_TYPE:

selfdrive/test/test_onroad.py

@@ -15,7 +15,7 @@ from common.basedir import BASEDIR
 from common.timeout import Timeout
 from common.params import Params
 from selfdrive.controls.lib.events import EVENTS, ET
-from selfdrive.loggerd.config import ROOT
+from system.loggerd.config import ROOT
 from selfdrive.test.helpers import set_params_enabled, release_only
 from tools.lib.logreader import LogReader
 

selfdrive/thermald/thermald.py

@@ -18,7 +18,7 @@ from common.params import Params
 from common.realtime import DT_TRML, sec_since_boot
 from selfdrive.controls.lib.alertmanager import set_offroad_alert
 from system.hardware import HARDWARE, TICI, AGNOS
-from selfdrive.loggerd.config import get_available_percent
+from system.loggerd.config import get_available_percent
 from selfdrive.statsd import statlog
 from system.swaglog import cloudlog
 from selfdrive.thermald.power_monitoring import PowerMonitoring

selfdrive/tombstoned.py

@@ -10,7 +10,7 @@ import glob
 from typing import NoReturn
 
 from common.file_helpers import mkdirs_exists_ok
-from selfdrive.loggerd.config import ROOT
+from system.loggerd.config import ROOT
 import selfdrive.sentry as sentry
 from system.swaglog import cloudlog
 from system.version import get_commit

system/loggerd/bootlog.cc

@@ -3,7 +3,7 @@
 
 #include "cereal/messaging/messaging.h"
 #include "common/swaglog.h"
-#include "selfdrive/loggerd/logger.h"
+#include "system/loggerd/logger.h"
 
 
 static kj::Array<capnp::word> build_boot_log() {

system/loggerd/deleter.py

@@ -3,8 +3,8 @@ import os
 import shutil
 import threading
 from system.swaglog import cloudlog
-from selfdrive.loggerd.config import ROOT, get_available_bytes, get_available_percent
+from system.loggerd.config import ROOT, get_available_bytes, get_available_percent
-from selfdrive.loggerd.uploader import listdir_by_creation
+from system.loggerd.uploader import listdir_by_creation
 
 MIN_BYTES = 5 * 1024 * 1024 * 1024
 MIN_PERCENT = 10

system/loggerd/encoder/encoder.cc

@@ -1,5 +1,5 @@
 #include <cassert>
-#include "selfdrive/loggerd/encoder/encoder.h"
+#include "system/loggerd/encoder/encoder.h"
 
 VideoEncoder::~VideoEncoder() {}
 

system/loggerd/encoder/encoder.h

@@ -7,7 +7,7 @@
 #include "cereal/messaging/messaging.h"
 #include "cereal/visionipc/visionipc.h"
 #include "common/queue.h"
-#include "selfdrive/loggerd/video_writer.h"
+#include "system/loggerd/video_writer.h"
 #include "system/camerad/cameras/camera_common.h"
 
 #define V4L2_BUF_FLAG_KEYFRAME 8

system/loggerd/encoder/ffmpeg_encoder.cc

@@ -1,6 +1,6 @@
 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
 
-#include "selfdrive/loggerd/encoder/ffmpeg_encoder.h"
+#include "system/loggerd/encoder/ffmpeg_encoder.h"
 
 #include <fcntl.h>
 #include <unistd.h>

system/loggerd/encoder/ffmpeg_encoder.h

@@ -11,8 +11,8 @@ extern "C" {
 #include <libavutil/imgutils.h>
 }
 
-#include "selfdrive/loggerd/encoder/encoder.h"
+#include "system/loggerd/encoder/encoder.h"
-#include "selfdrive/loggerd/loggerd.h"
+#include "system/loggerd/loggerd.h"
 
 class FfmpegEncoder : public VideoEncoder {
  public:

system/loggerd/encoder/v4l_encoder.cc

@@ -2,7 +2,7 @@
 #include <sys/ioctl.h>
 #include <poll.h>
 
-#include "selfdrive/loggerd/encoder/v4l_encoder.h"
+#include "system/loggerd/encoder/v4l_encoder.h"
 #include "common/util.h"
 #include "common/timing.h"
 

system/loggerd/encoder/v4l_encoder.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "common/queue.h"
-#include "selfdrive/loggerd/encoder/encoder.h"
+#include "system/loggerd/encoder/encoder.h"
 
 #define BUF_IN_COUNT 7
 #define BUF_OUT_COUNT 6

system/loggerd/encoderd.cc

@@ -1,4 +1,4 @@
-#include "selfdrive/loggerd/loggerd.h"
+#include "system/loggerd/loggerd.h"
 
 ExitHandler do_exit;
 

system/loggerd/logger.cc

@@ -1,4 +1,4 @@
-#include "selfdrive/loggerd/logger.h"
+#include "system/loggerd/logger.h"
 
 #include <sys/stat.h>
 #include <unistd.h>

system/loggerd/loggerd.cc

@@ -1,5 +1,5 @@
-#include "selfdrive/loggerd/loggerd.h"
+#include "system/loggerd/loggerd.h"
-#include "selfdrive/loggerd/video_writer.h"
+#include "system/loggerd/video_writer.h"
 
 ExitHandler do_exit;
 

system/loggerd/loggerd.h

@@ -23,13 +23,13 @@
 #include "common/util.h"
 #include "system/hardware/hw.h"
 
-#include "selfdrive/loggerd/encoder/encoder.h"
+#include "system/loggerd/encoder/encoder.h"
-#include "selfdrive/loggerd/logger.h"
+#include "system/loggerd/logger.h"
 #ifdef QCOM2
-#include "selfdrive/loggerd/encoder/v4l_encoder.h"
+#include "system/loggerd/encoder/v4l_encoder.h"
 #define Encoder V4LEncoder
 #else
-#include "selfdrive/loggerd/encoder/ffmpeg_encoder.h"
+#include "system/loggerd/encoder/ffmpeg_encoder.h"
 #define Encoder FfmpegEncoder
 #endif
 

system/loggerd/tests/fill_eon.py

@@ -3,8 +3,8 @@
 
 import os
 
-from selfdrive.loggerd.config import ROOT, get_available_percent
+from system.loggerd.config import ROOT, get_available_percent
-from selfdrive.loggerd.tests.loggerd_tests_common import create_random_file
+from system.loggerd.tests.loggerd_tests_common import create_random_file
 
 
 if __name__ == "__main__":

system/loggerd/tests/loggerd_tests_common.py

@@ -5,7 +5,7 @@ import random
 import tempfile
 import unittest
 
-import selfdrive.loggerd.uploader as uploader
+import system.loggerd.uploader as uploader
 
 def create_random_file(file_path, size_mb, lock=False):
   try:

system/loggerd/tests/test_deleter.py

@@ -6,8 +6,8 @@ import unittest
 from collections import namedtuple
 
 from common.timeout import Timeout, TimeoutException
-import selfdrive.loggerd.deleter as deleter
+import system.loggerd.deleter as deleter
-from selfdrive.loggerd.tests.loggerd_tests_common import UploaderTestCase
+from system.loggerd.tests.loggerd_tests_common import UploaderTestCase
 
 Stats = namedtuple("Stats", ['f_bavail', 'f_blocks', 'f_frsize'])
 

system/loggerd/tests/test_encoder.py

@@ -14,7 +14,7 @@ from tqdm import trange
 from common.params import Params
 from common.timeout import Timeout
 from system.hardware import TICI
-from selfdrive.loggerd.config import ROOT
+from system.loggerd.config import ROOT
 from selfdrive.manager.process_config import managed_processes
 from tools.lib.logreader import LogReader
 

system/loggerd/tests/test_logger.cc

@@ -9,7 +9,7 @@
 #include "catch2/catch.hpp"
 #include "cereal/messaging/messaging.h"
 #include "common/util.h"
-#include "selfdrive/loggerd/logger.h"
+#include "system/loggerd/logger.h"
 #include "tools/replay/util.h"
 
 typedef cereal::Sentinel::SentinelType SentinelType;

system/loggerd/tests/test_loggerd.py

@@ -15,7 +15,7 @@ from cereal.services import service_list
 from common.basedir import BASEDIR
 from common.params import Params
 from common.timeout import Timeout
-from selfdrive.loggerd.config import ROOT
+from system.loggerd.config import ROOT
 from selfdrive.manager.process_config import managed_processes
 from system.version import get_version
 from tools.lib.logreader import LogReader
@@ -51,7 +51,7 @@ class TestLoggerd(unittest.TestCase):
 
   def _gen_bootlog(self):
     with Timeout(5):
-      out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8')
+      out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "system/loggerd"), encoding='utf-8')
 
     log_fn = self._get_log_fn(out)
 

system/loggerd/tests/test_uploader.py

@@ -7,9 +7,9 @@ import logging
 import json
 
 from system.swaglog import cloudlog
-import selfdrive.loggerd.uploader as uploader
+import system.loggerd.uploader as uploader
 
-from selfdrive.loggerd.tests.loggerd_tests_common import UploaderTestCase
+from system.loggerd.tests.loggerd_tests_common import UploaderTestCase
 
 
 class TestLogHandler(logging.Handler):

system/loggerd/tools/mark_all_uploaded.py

@@ -1,7 +1,7 @@
 import os
-from selfdrive.loggerd.uploader import UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE
+from system.loggerd.uploader import UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE
 
-from selfdrive.loggerd.config import ROOT
+from system.loggerd.config import ROOT
 for folder in os.walk(ROOT):
   for file1 in folder[2]:
     full_path = os.path.join(folder[0], file1)

system/loggerd/tools/mark_unuploaded.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import os
 import sys
-from selfdrive.loggerd.uploader import UPLOAD_ATTR_NAME
+from system.loggerd.uploader import UPLOAD_ATTR_NAME
 
 for fn in sys.argv[1:]:
   print(f"unmarking {fn}")

system/loggerd/uploader.py

@@ -16,8 +16,8 @@ from common.api import Api
 from common.params import Params
 from common.realtime import set_core_affinity
 from system.hardware import TICI
-from selfdrive.loggerd.xattr_cache import getxattr, setxattr
+from system.loggerd.xattr_cache import getxattr, setxattr
-from selfdrive.loggerd.config import ROOT
+from system.loggerd.config import ROOT
 from system.swaglog import cloudlog
 
 NetworkType = log.DeviceState.NetworkType

system/loggerd/video_writer.cc

@@ -2,7 +2,7 @@
 #include <cassert>
 #include <cstdlib>
 
-#include "selfdrive/loggerd/video_writer.h"
+#include "system/loggerd/video_writer.h"
 #include "common/swaglog.h"
 #include "common/util.h"
 

tools/lib/README.md

@@ -1,6 +1,6 @@
 ## LogReader
 
-Route is a class for conveniently accessing all the [logs](/selfdrive/loggerd/) from your routes. The LogReader class reads the non-video logs, i.e. rlog.bz2 and qlog.bz2. There's also a matching FrameReader class for reading the videos.
+Route is a class for conveniently accessing all the [logs](/system/loggerd/) from your routes. The LogReader class reads the non-video logs, i.e. rlog.bz2 and qlog.bz2. There's also a matching FrameReader class for reading the videos.
 
 ```python
 from tools.lib.route import Route