Add sentinels to logs and qlogs (#1205)

* sentinel WIP

* logger is now a cc, not a c

* simpler code doesn't touch loggerd.cc

* oops, double lock. and logreader is nicer

cereal

@@ -1 +1 @@
-Subproject commit bb2cc7572de99becce1bfbae63f3b38d5464e162
+Subproject commit 4589107594e2dc9666e9676fba3b8248f1b5da3b

release/files_common

@@ -273,7 +273,8 @@ selfdrive/proclogd/proclogd.cc
 selfdrive/loggerd/SConscript
 selfdrive/loggerd/encoder.[c,h]
 selfdrive/loggerd/frame_logger.h
-selfdrive/loggerd/logger.[c,h]
+selfdrive/loggerd/logger.cc
+selfdrive/loggerd/logger.h
 selfdrive/loggerd/loggerd.cc
 selfdrive/loggerd/raw_logger.cc
 selfdrive/loggerd/raw_logger.h

selfdrive/loggerd/SConscript

@@ -1,6 +1,6 @@
 Import('env', 'arch', 'messaging', 'common', 'visionipc')
 
-src = ['loggerd.cc', 'logger.c']
+src = ['loggerd.cc', 'logger.cc']
 libs = ['zmq', 'czmq', 'capnp', 'kj', 'z',
   'avformat', 'avcodec', 'swscale', 'avutil',
   'yuv', 'bz2', common, 'json', messaging, visionipc]

selfdrive/loggerd/logger.cc

@@ -17,6 +17,21 @@
 
 #include "logger.h"
 
+#include <capnp/serialize.h>
+#include "cereal/gen/cpp/log.capnp.h"
+
+static void log_sentinel(LoggerState *s, cereal::Sentinel::SentinelType type) {
+  capnp::MallocMessageBuilder msg;
+  auto event = msg.initRoot<cereal::Event>();
+  event.setLogMonoTime(nanos_since_boot());
+  auto sen = event.initSentinel();
+  sen.setType(type);
+  auto words = capnp::messageToFlatArray(msg);
+  auto bytes = words.asBytes();
+
+  logger_log(s, bytes.begin(), bytes.size(), true);
+}
+
 static int mkpath(char* file_path) {
   assert(file_path && *file_path);
   char* p;
@@ -125,6 +140,9 @@ fail:
 int logger_next(LoggerState *s, const char* root_path,
                             char* out_segment_path, size_t out_segment_path_len,
                             int* out_part) {
+  bool is_start_of_route = !s->cur_handle;
+  if (!is_start_of_route) log_sentinel(s, cereal::Sentinel::SentinelType::END_OF_SEGMENT);
+
   pthread_mutex_lock(&s->lock);
   s->part++;
 
@@ -147,6 +165,8 @@ int logger_next(LoggerState *s, const char* root_path,
   }
 
   pthread_mutex_unlock(&s->lock);
+
+  log_sentinel(s, is_start_of_route ? cereal::Sentinel::SentinelType::START_OF_ROUTE : cereal::Sentinel::SentinelType::START_OF_SEGMENT);
   return 0;
 }
 
@@ -171,6 +191,8 @@ void logger_log(LoggerState *s, uint8_t* data, size_t data_size, bool in_qlog) {
 }
 
 void logger_close(LoggerState *s) {
+  log_sentinel(s, cereal::Sentinel::SentinelType::END_OF_ROUTE);
+
   pthread_mutex_lock(&s->lock);
   free(s->init_data);
   if (s->cur_handle) {

tools/lib/logreader.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import os
 import sys
 import json