Put bootlog in own folder (#19939)

* put bootlog in own folder

* fix uploader

* remove print statements

* update test

* remove comment

* also write initddata

* remove sentinels from test

selfdrive/loggerd/bootlog.cc

@@ -1,37 +1,60 @@
 #include <assert.h>
 #include <string>
+#include <cstdio>
+
+#include <bzlib.h>
 
 #include "common/swaglog.h"
 #include "common/util.h"
-#include "logger.h"
 #include "messaging.hpp"
+#include "logger.h"
 
 int main(int argc, char** argv) {
-  LoggerState logger;
+  char filename[64] = {'\0'};
-  logger_init(&logger, "bootlog", false);
+
+  time_t rawtime = time(NULL);
+  struct tm timeinfo;
+
+  localtime_r(&rawtime, &timeinfo);
+  strftime(filename, sizeof(filename),
+           "%Y-%m-%d--%H-%M-%S.bz2", &timeinfo);
+
+  std::string path = LOG_ROOT + "/boot/" + std::string(filename);
+  LOGW("bootlog to %s", path.c_str());
+
+  MessageBuilder boot_msg;
+  logger_build_boot(boot_msg);
+
+  MessageBuilder init_msg;
+  logger_build_init_data(init_msg);
 
-  char segment_path[4096];
-  int err = logger_next(&logger, LOG_ROOT.c_str(), segment_path, sizeof(segment_path), nullptr);
-  assert(err == 0);
-  LOGW("bootlog to %s", segment_path);
 
-  MessageBuilder msg;
+  // Open bootlog
-  auto boot = msg.initEvent().initBoot();
+  int r = logger_mkpath((char*)path.c_str());
+  assert(r == 0);
 
-  boot.setWallTimeNanos(nanos_since_epoch());
+  FILE * file = fopen(path.c_str(), "wb");
+  assert(file != nullptr);
 
-  std::string lastKmsg = util::read_file("/sys/fs/pstore/console-ramoops");
+  // Open as bz2
-  boot.setLastKmsg(capnp::Data::Reader((const kj::byte*)lastKmsg.data(), lastKmsg.size()));
+  int bzerror;
+  BZFILE* bz_file = BZ2_bzWriteOpen(&bzerror, file, 9, 0, 30);
+  assert(bzerror == BZ_OK);
 
-  std::string lastPmsg = util::read_file("/sys/fs/pstore/pmsg-ramoops-0");
+  // Write initdata
-  boot.setLastPmsg(capnp::Data::Reader((const kj::byte*)lastPmsg.data(), lastPmsg.size()));
+  auto bytes = init_msg.toBytes();
+  BZ2_bzWrite(&bzerror, bz_file, bytes.begin(), bytes.size());
+  assert(bzerror == BZ_OK);
 
-  std::string launchLog = util::read_file("/tmp/launch_log");
+  // Write bootlog
-  boot.setLaunchLog(capnp::Text::Reader(launchLog.data(), launchLog.size()));
+  bytes = boot_msg.toBytes();
+  BZ2_bzWrite(&bzerror, bz_file, bytes.begin(), bytes.size());
+  assert(bzerror == BZ_OK);
 
-  auto bytes = msg.toBytes();
+  // Close bz2 and file
-  logger_log(&logger, bytes.begin(), bytes.size(), false);
+  BZ2_bzWriteClose(&bzerror, bz_file, 0, NULL, NULL);
+  assert(bzerror == BZ_OK);
 
-  logger_close(&logger);
+  fclose(file);
   return 0;
 }

selfdrive/loggerd/config.py

@@ -1,8 +1,11 @@
 import os
+from pathlib import Path
+from selfdrive.hardware import PC
 
 if os.environ.get('LOGGERD_ROOT', False):
   ROOT = os.environ['LOGGERD_ROOT']
-  print("Custom loggerd root: ", ROOT)
+elif PC:
+  ROOT = os.path.join(str(Path.home()), ".comma", "media", "0", "realdata")
 else:
   ROOT = '/data/media/0/realdata/'
 

selfdrive/loggerd/logger.cc

@@ -35,7 +35,7 @@ void append_property(const char* key, const char* value, void *cookie) {
   properties->push_back(std::make_pair(std::string(key), std::string(value)));
 }
 
-static int mkpath(char* file_path) {
+int logger_mkpath(char* file_path) {
   assert(file_path && *file_path);
   char* p;
   for (p=strchr(file_path+1, '/'); p; p=strchr(p+1, '/')) {
@@ -52,9 +52,22 @@ static int mkpath(char* file_path) {
 }
 
 // ***** log metadata *****
+void logger_build_boot(MessageBuilder &msg) {
+  auto boot = msg.initEvent().initBoot();
 
-void log_init_data(LoggerState *s) {
+  boot.setWallTimeNanos(nanos_since_epoch());
-  MessageBuilder msg;
+
+  std::string lastKmsg = util::read_file("/sys/fs/pstore/console-ramoops");
+  boot.setLastKmsg(capnp::Data::Reader((const kj::byte*)lastKmsg.data(), lastKmsg.size()));
+
+  std::string lastPmsg = util::read_file("/sys/fs/pstore/pmsg-ramoops-0");
+  boot.setLastPmsg(capnp::Data::Reader((const kj::byte*)lastPmsg.data(), lastPmsg.size()));
+
+  std::string launchLog = util::read_file("/tmp/launch_log");
+  boot.setLaunchLog(capnp::Text::Reader(launchLog.data(), launchLog.size()));
+}
+
+void logger_build_init_data(MessageBuilder &msg) {
   auto init = msg.initEvent().initInitData();
 
   if (util::file_exists("/EON")) {
@@ -119,7 +132,11 @@ void log_init_data(LoggerState *s) {
       i++;
     }
   }
+}
 
+void log_init_data(LoggerState *s) {
+  MessageBuilder msg;
+  logger_build_init_data(msg);
   auto bytes = msg.toBytes();
   logger_log(s, bytes.begin(), bytes.size(), s->has_qlog);
 }
@@ -174,7 +191,7 @@ static LoggerHandle* logger_open(LoggerState *s, const char* root_path) {
   snprintf(h->qlog_path, sizeof(h->qlog_path), "%s/qlog.bz2", h->segment_path);
   snprintf(h->lock_path, sizeof(h->lock_path), "%s.lock", h->log_path);
 
-  err = mkpath(h->log_path);
+  err = logger_mkpath(h->log_path);
   if (err) return NULL;
 
   FILE* lock_file = fopen(h->lock_path, "wb");

selfdrive/loggerd/logger.h

@@ -40,6 +40,9 @@ typedef struct LoggerState {
   LoggerHandle* cur_handle;
 } LoggerState;
 
+int logger_mkpath(char* file_path);
+void logger_build_boot(MessageBuilder &msg);
+void logger_build_init_data(MessageBuilder &msg);
 void logger_init(LoggerState *s, const char* log_name, bool has_qlog);
 int logger_next(LoggerState *s, const char* root_path,
                             char* out_segment_path, size_t out_segment_path_len,

selfdrive/loggerd/tests/test_loggerd.py

@@ -45,15 +45,23 @@ class TestLoggerd(unittest.TestCase):
         return path
     return None
 
+  def _get_log_fn(self, x):
+    for p in x.split(' '):
+      path = Path(p.strip())
+      if path.is_file():
+        return path
+    return None
+
   def _gen_bootlog(self):
     with Timeout(5):
       out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8')
 
+    log_fn = self._get_log_fn(out)
+
     # check existence
-    d = self._get_log_dir(out)
+    assert log_fn is not None
-    path = Path(os.path.join(d, "bootlog.bz2"))
+
-    assert path.is_file(), "failed to create bootlog file"
+    return log_fn
-    return path
 
   def _check_init_data(self, msgs):
     msg = msgs[0]
@@ -131,11 +139,9 @@ class TestLoggerd(unittest.TestCase):
     lr = list(LogReader(str(bootlog_path)))
 
     # check length
-    assert len(lr) == 4 # boot + initData + 2x sentinel
+    assert len(lr) == 2  # boot + initData
 
-    # check initData and sentinel
     self._check_init_data(lr)
-    self._check_sentinel(lr, True)
 
     # check msgs
     bootlog_msgs = [m for m in lr if m.which() == 'boot']

selfdrive/loggerd/uploader.py

@@ -58,7 +58,7 @@ class Uploader():
     self.last_resp = None
     self.last_exc = None
 
-    self.immediate_folders = ["crash/"]
+    self.immediate_folders = ["crash/", "boot/"]
     self.immediate_priority = {"qlog.bz2": 0, "qcamera.ts": 1}
     self.high_priority = {"rlog.bz2": 0, "fcamera.hevc": 1, "dcamera.hevc": 2, "ecamera.hevc": 3}