sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

loggerd: split bootlog to a separate program (#19831)

Browse Source 
* bootlog

* blank lines

* move file_exists to util.h

* src = ['loggerd.cc']

* log_init_data

* Revert "src = ['loggerd.cc']"

This reverts commit 18a587023d.

* add bootlog to .gitignore

* use s->has_qlog instead of true

* add blank lines & remove extern C
pull/19841/head
Dean Lee 4 years ago committed by GitHub
parent 978d0c8964
commit f973c56a36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 154 additions and 180 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      .gitignore
  2. 1
      release/files_common
  3. 6
      selfdrive/common/util.h
  4. 7
      selfdrive/loggerd/SConscript
  5. 37
      selfdrive/loggerd/bootlog.cc
  6. 114
      selfdrive/loggerd/logger.cc
  7. 23
      selfdrive/loggerd/logger.h
  8. 141
      selfdrive/loggerd/loggerd.cc
  9. 2
      selfdrive/loggerd/tests/test_loggerd.py
  10. 2
      selfdrive/manager.py

1
.gitignore vendored
Unescape View File

@ -44,6 +44,7 @@ selfdrive/ui/_ui
selfdrive/test/longitudinal_maneuvers/out
selfdrive/visiond/visiond
selfdrive/loggerd/loggerd
selfdrive/loggerd/bootlog
selfdrive/sensord/_gpsd
selfdrive/sensord/_sensord
selfdrive/camerad/camerad

1
release/files_common
Unescape View File

@ -307,6 +307,7 @@ selfdrive/loggerd/omx_encoder.h
selfdrive/loggerd/logger.cc
selfdrive/loggerd/logger.h
selfdrive/loggerd/loggerd.cc
selfdrive/loggerd/bootlog.cc
selfdrive/loggerd/raw_logger.cc
selfdrive/loggerd/raw_logger.h
selfdrive/loggerd/include/msm_media_info.h

6
selfdrive/common/util.h
Unescape View File

@ -96,6 +96,12 @@ inline std::string getenv_default(const char* env_var, const char * suffix, cons
inline void sleep_for(const int milliseconds) {
std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));
}
inline bool file_exists(const std::string& fn) {
std::ifstream f(fn);
return f.good();
}
}
class ExitHandler {

7
selfdrive/loggerd/SConscript
Unescape View File

@ -1,7 +1,9 @@
Import('env', 'arch', 'cereal', 'messaging', 'common', 'visionipc', 'gpucommon')
src = ['loggerd.cc', 'logger.cc']
libs = ['zmq', 'capnp', 'kj', 'z',
logger_lib = env.Library('logger', ["logger.cc"])
src = ['loggerd.cc']
libs = [logger_lib, 'zmq', 'capnp', 'kj', 'z',
'avformat', 'avcodec', 'swscale', 'avutil',
'yuv', 'bz2', 'OpenCL', common, cereal, messaging, visionipc]
@ -22,3 +24,4 @@ if arch == "Darwin":
env['FRAMEWORKS'] = ['OpenCL']
env.Program(src, LIBS=libs)
env.Program('bootlog.cc', LIBS=libs)

37
selfdrive/loggerd/bootlog.cc
Unescape View File

@ -0,0 +1,37 @@
#include <assert.h>
#include <string>
#include "common/swaglog.h"
#include "common/util.h"
#include "logger.h"
#include "messaging.hpp"
int main(int argc, char** argv) {
LoggerState logger;
logger_init(&logger, "bootlog", false);
char segment_path[4096];
int err = logger_next(&logger, LOG_ROOT.c_str(), segment_path, sizeof(segment_path), nullptr);
assert(err == 0);
LOGW("bootlog to %s", segment_path);
MessageBuilder msg;
auto boot = msg.initEvent().initBoot();
boot.setWallTimeNanos(nanos_since_epoch());
std::string lastKmsg = util::read_file("/sys/fs/pstore/console-ramoops");
boot.setLastKmsg(capnp::Data::Reader((const kj::byte*)lastKmsg.data(), lastKmsg.size()));
std::string lastPmsg = util::read_file("/sys/fs/pstore/pmsg-ramoops-0");
boot.setLastPmsg(capnp::Data::Reader((const kj::byte*)lastPmsg.data(), lastPmsg.size()));
std::string launchLog = util::read_file("/tmp/launch_log");
boot.setLaunchLog(capnp::Text::Reader(launchLog.data(), launchLog.size()));
auto bytes = msg.toBytes();
logger_log(&logger, bytes.begin(), bytes.size(), false);
logger_close(&logger);
return 0;
}

114
selfdrive/loggerd/logger.cc
Unescape View File

@ -12,12 +12,99 @@
#include <pthread.h>
#include <bzlib.h>
#include <iostream>
#include <fstream>
#include <streambuf>
#ifdef QCOM
#include <cutils/properties.h>
#endif
#include "messaging.hpp"
#include "common/swaglog.h"
#include "common/params.h"
#include "common/util.h"
#include "common/version.h"
#include "logger.h"
void append_property(const char* key, const char* value, void *cookie) {
std::vector<std::pair<std::string, std::string> > *properties =
(std::vector<std::pair<std::string, std::string> > *)cookie;
properties->push_back(std::make_pair(std::string(key), std::string(value)));
}
void log_init_data(LoggerState *s) {
MessageBuilder msg;
auto init = msg.initEvent().initInitData();
if (util::file_exists("/EON")) {
init.setDeviceType(cereal::InitData::DeviceType::NEO);
} else if (util::file_exists("/TICI")) {
init.setDeviceType(cereal::InitData::DeviceType::TICI);
} else {
init.setDeviceType(cereal::InitData::DeviceType::PC);
}
init.setVersion(capnp::Text::Reader(COMMA_VERSION));
std::ifstream cmdline_stream("/proc/cmdline");
std::vector<std::string> kernel_args;
std::string buf;
while (cmdline_stream >> buf) {
kernel_args.push_back(buf);
}
auto lkernel_args = init.initKernelArgs(kernel_args.size());
for (int i=0; i<kernel_args.size(); i++) {
lkernel_args.set(i, kernel_args[i]);
}
init.setKernelVersion(util::read_file("/proc/version"));
#ifdef QCOM
{
std::vector<std::pair<std::string, std::string> > properties;
property_list(append_property, (void*)&properties);
auto lentries = init.initAndroidProperties().initEntries(properties.size());
for (int i=0; i<properties.size(); i++) {
auto lentry = lentries[i];
lentry.setKey(properties[i].first);
lentry.setValue(properties[i].second);
}
}
#endif
const char* dongle_id = getenv("DONGLE_ID");
if (dongle_id) {
init.setDongleId(std::string(dongle_id));
}
init.setDirty(!getenv("CLEAN"));
// log params
Params params = Params();
init.setGitCommit(params.get("GitCommit"));
init.setGitBranch(params.get("GitBranch"));
init.setGitRemote(params.get("GitRemote"));
init.setPassive(params.read_db_bool("Passive"));
{
std::map<std::string, std::string> params_map;
params.read_db_all(&params_map);
auto lparams = init.initParams().initEntries(params_map.size());
int i = 0;
for (auto& kv : params_map) {
auto lentry = lparams[i];
lentry.setKey(kv.first);
lentry.setValue(kv.second);
i++;
}
}
auto bytes = msg.toBytes();
logger_log(s, bytes.begin(), bytes.size(), s->has_qlog);
}
static void log_sentinel(LoggerState *s, cereal::Sentinel::SentinelType type) {
MessageBuilder msg;
auto sen = msg.initEvent().initSentinel();
@ -43,15 +130,9 @@ static int mkpath(char* file_path) {
return 0;
}
void logger_init(LoggerState *s, const char* log_name, const uint8_t* init_data, size_t init_data_len, bool has_qlog) {
void logger_init(LoggerState *s, const char* log_name, bool has_qlog) {
memset(s, 0, sizeof(*s));
if (init_data) {
s->init_data = (uint8_t*)malloc(init_data_len);
assert(s->init_data);
memcpy(s->init_data, init_data, init_data_len);
s->init_data_len = init_data_len;
}
umask(0);
pthread_mutex_init(&s->lock, NULL);
@ -110,20 +191,10 @@ static LoggerHandle* logger_open(LoggerState *s, const char* root_path) {
h->bz_qlog = BZ2_bzWriteOpen(&bzerror, h->qlog_file, 9, 0, 30);
if (bzerror != BZ_OK) goto fail;
}
if (s->init_data) {
BZ2_bzWrite(&bzerror, h->bz_file, s->init_data, s->init_data_len);
if (bzerror != BZ_OK) goto fail;
if (s->has_qlog) {
// init data goes in the qlog too
BZ2_bzWrite(&bzerror, h->bz_qlog, s->init_data, s->init_data_len);
if (bzerror != BZ_OK) goto fail;
}
}
pthread_mutex_init(&h->lock, NULL);
h->refcnt++;
log_init_data(s);
return h;
fail:
LOGE("logger failed to open files");
@ -203,7 +274,6 @@ void logger_close(LoggerState *s) {
log_sentinel(s, cereal::Sentinel::SentinelType::END_OF_ROUTE);
pthread_mutex_lock(&s->lock);
free(s->init_data);
if (s->cur_handle) {
lh_close(s->cur_handle);
}

23
selfdrive/loggerd/logger.h
Unescape View File

@ -1,13 +1,16 @@
#ifndef LOGGER_H
#define LOGGER_H
#pragma once
#include <stdio.h>
#include <stdint.h>
#include <pthread.h>
#include <bzlib.h>
#include <kj/array.h>
#include <capnp/serialize.h>
#ifdef __cplusplus
extern "C" {
#if defined(QCOM) || defined(QCOM2)
const std::string LOG_ROOT = "/data/media/0/realdata";
#else
const std::string LOG_ROOT = util::getenv_default("HOME", "/.comma/media/0/realdata", "/data/media/0/realdata");
#endif
#define LOGGER_MAX_HANDLES 16
@ -28,10 +31,6 @@ typedef struct LoggerHandle {
typedef struct LoggerState {
pthread_mutex_t lock;
uint8_t* init_data;
size_t init_data_len;
int part;
char route_name[64];
char log_name[64];
@ -41,7 +40,7 @@ typedef struct LoggerState {
LoggerHandle* cur_handle;
} LoggerState;
void logger_init(LoggerState *s, const char* log_name, const uint8_t* init_data, size_t init_data_len, bool has_qlog);
void logger_init(LoggerState *s, const char* log_name, bool has_qlog);
int logger_next(LoggerState *s, const char* root_path,
char* out_segment_path, size_t out_segment_path_len,
int* out_part);
@ -51,9 +50,3 @@ void logger_log(LoggerState *s, uint8_t* data, size_t data_size, bool in_qlog);
void lh_log(LoggerHandle* h, uint8_t* data, size_t data_size, bool in_qlog);
void lh_close(LoggerHandle* h);
#ifdef __cplusplus
}
#endif
#endif

141
selfdrive/loggerd/loggerd.cc
Unescape View File

@ -10,9 +10,7 @@
#include <sys/resource.h>
#include <string>
#include <iostream>
#include <fstream>
#include <streambuf>
#include <thread>
#include <mutex>
#include <condition_variable>
@ -20,11 +18,6 @@
#include <random>
#include <ftw.h>
#ifdef QCOM
#include <cutils/properties.h>
#endif
#include "common/version.h"
#include "common/timing.h"
#include "common/params.h"
#include "common/swaglog.h"
@ -107,11 +100,6 @@ LogCameraInfo cameras_logged[LOG_CAMERA_ID_MAX] = {
namespace {
constexpr int SEGMENT_LENGTH = 60;
#if defined(QCOM) || defined(QCOM2)
std::string LOG_ROOT = "/data/media/0/realdata";
#else
std::string LOG_ROOT = util::getenv_default("HOME", "/.comma/media/0/realdata", "/data/media/0/realdata");
#endif
double randrange(double a, double b) __attribute__((unused));
double randrange(double a, double b) {
@ -123,11 +111,6 @@ double randrange(double a, double b) {
ExitHandler do_exit;
static bool file_exists(const std::string& fn) {
std::ifstream f(fn);
return f.good();
}
class RotateState {
public:
SubSocket* fpkt_sock;
@ -328,82 +311,6 @@ void encoder_thread(int cam_idx) {
}
void append_property(const char* key, const char* value, void *cookie) {
std::vector<std::pair<std::string, std::string> > *properties =
(std::vector<std::pair<std::string, std::string> > *)cookie;
properties->push_back(std::make_pair(std::string(key), std::string(value)));
}
kj::Array<capnp::word> gen_init_data() {
MessageBuilder msg;
auto init = msg.initEvent().initInitData();
if (file_exists("/EON")) {
init.setDeviceType(cereal::InitData::DeviceType::NEO);
} else if (file_exists("/TICI")) {
init.setDeviceType(cereal::InitData::DeviceType::TICI);
} else {
init.setDeviceType(cereal::InitData::DeviceType::PC);
}
init.setVersion(capnp::Text::Reader(COMMA_VERSION));
std::ifstream cmdline_stream("/proc/cmdline");
std::vector<std::string> kernel_args;
std::string buf;
while (cmdline_stream >> buf) {
kernel_args.push_back(buf);
}
auto lkernel_args = init.initKernelArgs(kernel_args.size());
for (int i=0; i<kernel_args.size(); i++) {
lkernel_args.set(i, kernel_args[i]);
}
init.setKernelVersion(util::read_file("/proc/version"));
#ifdef QCOM
{
std::vector<std::pair<std::string, std::string> > properties;
property_list(append_property, (void*)&properties);
auto lentries = init.initAndroidProperties().initEntries(properties.size());
for (int i=0; i<properties.size(); i++) {
auto lentry = lentries[i];
lentry.setKey(properties[i].first);
lentry.setValue(properties[i].second);
}
}
#endif
const char* dongle_id = getenv("DONGLE_ID");
if (dongle_id) {
init.setDongleId(std::string(dongle_id));
}
init.setDirty(!getenv("CLEAN"));
// log params
Params params = Params();
init.setGitCommit(params.get("GitCommit"));
init.setGitBranch(params.get("GitBranch"));
init.setGitRemote(params.get("GitRemote"));
init.setPassive(params.read_db_bool("Passive"));
{
std::map<std::string, std::string> params_map;
params.read_db_all(&params_map);
auto lparams = init.initParams().initEntries(params_map.size());
int i = 0;
for (auto& kv : params_map) {
auto lentry = lparams[i];
lentry.setKey(kv.first);
lentry.setValue(kv.second);
i++;
}
}
return capnp::messageToFlatArray(msg);
}
static int clear_locks_fn(const char* fpath, const struct stat *sb, int tyupeflag) {
const char* dot = strrchr(fpath, '.');
if (dot && strcmp(dot, ".lock") == 0) {
@ -416,50 +323,10 @@ static void clear_locks() {
ftw(LOG_ROOT.c_str(), clear_locks_fn, 16);
}
static void bootlog() {
int err;
{
auto words = gen_init_data();
auto bytes = words.asBytes();
logger_init(&s.logger, "bootlog", bytes.begin(), bytes.size(), false);
}
err = logger_next(&s.logger, LOG_ROOT.c_str(), s.segment_path, sizeof(s.segment_path), &s.rotate_segment);
assert(err == 0);
LOGW("bootlog to %s", s.segment_path);
{
MessageBuilder msg;
auto boot = msg.initEvent().initBoot();
boot.setWallTimeNanos(nanos_since_epoch());
std::string lastKmsg = util::read_file("/sys/fs/pstore/console-ramoops");
boot.setLastKmsg(capnp::Data::Reader((const kj::byte*)lastKmsg.data(), lastKmsg.size()));
std::string lastPmsg = util::read_file("/sys/fs/pstore/pmsg-ramoops-0");
boot.setLastPmsg(capnp::Data::Reader((const kj::byte*)lastPmsg.data(), lastPmsg.size()));
std::string launchLog = util::read_file("/tmp/launch_log");
boot.setLaunchLog(capnp::Text::Reader(launchLog.data(), launchLog.size()));
auto bytes = msg.toBytes();
logger_log(&s.logger, bytes.begin(), bytes.size(), false);
}
logger_close(&s.logger);
}
int main(int argc, char** argv) {
setpriority(PRIO_PROCESS, 0, -12);
if (argc > 1 && strcmp(argv[1], "--bootlog") == 0) {
bootlog();
return 0;
}
int segment_length = SEGMENT_LENGTH;
if (getenv("LOGGERD_TEST")) {
segment_length = atoi(getenv("LOGGERD_SEGMENT_LENGTH"));
@ -495,11 +362,7 @@ int main(int argc, char** argv) {
}
// init logger
{
auto words = gen_init_data();
auto bytes = words.asBytes();
logger_init(&s.logger, "rlog", bytes.begin(), bytes.size(), true);
}
logger_init(&s.logger, "rlog", true);
// init encoders
pthread_mutex_init(&s.rotate_lock, NULL);

2
selfdrive/loggerd/tests/test_loggerd.py
Unescape View File

@ -47,7 +47,7 @@ class TestLoggerd(unittest.TestCase):
def _gen_bootlog(self):
with Timeout(5):
out = subprocess.check_output(["./loggerd", "--bootlog"], cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8')
out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8')
# check existence
d = self._get_log_dir(out)

2
selfdrive/manager.py
Unescape View File

@ -440,7 +440,7 @@ def manager_thread():
cloudlog.info({"environ": os.environ})
# save boot log
subprocess.call(["./loggerd", "--bootlog"], cwd=os.path.join(BASEDIR, "selfdrive/loggerd"))
subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"))
# start daemon processes
for p in daemon_processes:

Write Preview
Loading…
Cancel
Save