Systemd logcatd (#2085)

* Systemd logcatd infrastructure

* This should work

* Cleanup

* Split

* More compact and cleanup

* Add to ubuntu setup package list

* Run logcatd on all non pc platforms

Dockerfile.openpilot_base

@@ -27,6 +27,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libopencv-dev \
     libssl-dev \
     libsqlite3-dev \
+    libsystemd-dev \
     libusb-1.0-0-dev \
     libczmq-dev \
     libzmq3-dev \

SConstruct

@@ -311,7 +311,7 @@ SConscript(['selfdrive/locationd/SConscript'])
 SConscript(['selfdrive/locationd/models/SConscript'])
 SConscript(['selfdrive/sensord/SConscript'])
 
-if arch == "aarch64":
+if arch != "Darwin":
   SConscript(['selfdrive/logcatd/SConscript'])
 
 if arch != "larch64":

release/files_common

@@ -294,7 +294,8 @@ selfdrive/locationd/calibrationd.py
 selfdrive/locationd/calibration_helpers.py
 
 selfdrive/logcatd/SConscript
-selfdrive/logcatd/logcatd.cc
+selfdrive/logcatd/logcatd_android.cc
+selfdrive/logcatd/logcatd_systemd.cc
 
 selfdrive/proclogd/SConscript
 selfdrive/proclogd/proclogd.cc

selfdrive/logcatd/SConscript

@@ -1,2 +1,6 @@
-Import('env', 'cereal', 'messaging')
+Import('env', 'cereal', 'messaging', 'arch')
-env.Program('logcatd.cc', LIBS=[cereal, messaging, 'cutils', 'zmq', 'czmq', 'capnp', 'kj'])
+
+if arch == "aarch64":
+  env.Program('logcatd', 'logcatd_android.cc', LIBS=[cereal, messaging, 'cutils', 'zmq', 'czmq', 'capnp', 'kj'])
+else:
+  env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, messaging, 'zmq', 'capnp', 'kj', 'systemd', 'json11'])

selfdrive/logcatd/logcatd_systemd.cc

@@ -0,0 +1,73 @@
+#include <iostream>
+#include <cassert>
+#include <string>
+#include <map>
+
+#include "json11.hpp"
+#include <systemd/sd-journal.h>
+
+#include "common/timing.h"
+#include "messaging.hpp"
+
+int main(int argc, char *argv[]) {
+  PubMaster pm({"androidLog"});
+
+  sd_journal *journal;
+  assert(sd_journal_open(&journal, 0) >= 0);
+  assert(sd_journal_get_fd(journal) >= 0); // needed so sd_journal_wait() works properly if files rotate
+  assert(sd_journal_seek_tail(journal) >= 0);
+
+  int r;
+  while (true) {
+    r = sd_journal_next(journal);
+    assert(r >= 0);
+
+    // Wait for new message if we didn't receive anything
+    if (r == 0){
+      do {
+        r = sd_journal_wait(journal, (uint64_t)-1);
+        assert(r >= 0);
+      } while (r == SD_JOURNAL_NOP);
+
+      r = sd_journal_next(journal);
+      assert(r >= 0);
+
+      if (r == 0) continue; // Try again if we still didn't get anything
+    }
+
+    uint64_t timestamp = 0;
+    r = sd_journal_get_realtime_usec(journal, &timestamp);
+    assert(r >= 0);
+
+    const void *data;
+    size_t length;
+    std::map<std::string, std::string> kv;
+
+    SD_JOURNAL_FOREACH_DATA(journal, data, length){
+      std::string str((char*)data, length);
+
+      // Split "KEY=VALUE"" on "=" and put in map
+      std::size_t found = str.find("=");
+      if (found != std::string::npos){
+        kv[str.substr(0, found)] = str.substr(found + 1, std::string::npos);
+      }
+    }
+
+    capnp::MallocMessageBuilder msg;
+    cereal::Event::Builder event = msg.initRoot<cereal::Event>();
+    event.setLogMonoTime(nanos_since_boot());
+
+    // Build message
+    auto androidEntry = event.initAndroidLog();
+    androidEntry.setTs(timestamp);
+    androidEntry.setMessage(json11::Json(kv).dump());
+    if (kv.count("_PID")) androidEntry.setPid(std::atoi(kv["_PID"].c_str()));
+    if (kv.count("PRIORITY")) androidEntry.setPriority(std::atoi(kv["PRIORITY"].c_str()));
+    if (kv.count("SYSLOG_IDENTIFIER")) androidEntry.setTag(kv["SYSLOG_IDENTIFIER"]);
+
+    pm.send("androidLog", msg);
+  }
+
+  sd_journal_close(journal);
+  return 0;
+}

selfdrive/manager.py

@@ -217,9 +217,13 @@ persistent_processes = [
   'uploader',
 ]
 
-if ANDROID:
+if not PC:
   persistent_processes += [
     'logcatd',
+  ]
+
+if ANDROID:
+  persistent_processes += [
     'tombstoned',
     'updated',
     'deleter',

tools/ubuntu_setup.sh

@@ -35,6 +35,7 @@ sudo apt-get update && sudo apt-get install -y \
     libczmq-dev \
     libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-ttf2.0-dev libsmpeg-dev \
     libsdl1.2-dev  libportmidi-dev libswscale-dev libavformat-dev libavcodec-dev libfreetype6-dev \
+    libsystemd-dev \
     locales \
     ocl-icd-libopencl1 \
     ocl-icd-opencl-dev \