From f973c56a3695525442ac6c3e89c644b27cbeb384 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Wed, 20 Jan 2021 04:16:12 +0800 Subject: [PATCH] loggerd: split bootlog to a separate program (#19831) * bootlog * blank lines * move file_exists to util.h * src = ['loggerd.cc'] * log_init_data * Revert "src = ['loggerd.cc']" This reverts commit 18a587023d75b3a3a54f1ceaf9cb31a51640a067. * add bootlog to .gitignore * use s->has_qlog instead of true * add blank lines & remove extern C --- .gitignore | 1 + release/files_common | 1 + selfdrive/common/util.h | 6 + selfdrive/loggerd/SConscript | 7 +- selfdrive/loggerd/bootlog.cc | 37 +++++++ selfdrive/loggerd/logger.cc | 114 +++++++++++++++---- selfdrive/loggerd/logger.h | 23 ++-- selfdrive/loggerd/loggerd.cc | 141 +----------------------- selfdrive/loggerd/tests/test_loggerd.py | 2 +- selfdrive/manager.py | 2 +- 10 files changed, 154 insertions(+), 180 deletions(-) create mode 100644 selfdrive/loggerd/bootlog.cc diff --git a/.gitignore b/.gitignore index 2510f33b20..ec33cd9f75 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ selfdrive/ui/_ui selfdrive/test/longitudinal_maneuvers/out selfdrive/visiond/visiond selfdrive/loggerd/loggerd +selfdrive/loggerd/bootlog selfdrive/sensord/_gpsd selfdrive/sensord/_sensord selfdrive/camerad/camerad diff --git a/release/files_common b/release/files_common index 4e39fbe26b..76058366df 100644 --- a/release/files_common +++ b/release/files_common @@ -307,6 +307,7 @@ selfdrive/loggerd/omx_encoder.h selfdrive/loggerd/logger.cc selfdrive/loggerd/logger.h selfdrive/loggerd/loggerd.cc +selfdrive/loggerd/bootlog.cc selfdrive/loggerd/raw_logger.cc selfdrive/loggerd/raw_logger.h selfdrive/loggerd/include/msm_media_info.h diff --git a/selfdrive/common/util.h b/selfdrive/common/util.h index 96b3d6d5e1..2797cbd44a 100644 --- a/selfdrive/common/util.h +++ b/selfdrive/common/util.h @@ -96,6 +96,12 @@ inline std::string getenv_default(const char* env_var, const char * suffix, cons inline void sleep_for(const int milliseconds) { std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds)); } + +inline bool file_exists(const std::string& fn) { + std::ifstream f(fn); + return f.good(); +} + } class ExitHandler { diff --git a/selfdrive/loggerd/SConscript b/selfdrive/loggerd/SConscript index 9e6bf72ab1..b597406bac 100644 --- a/selfdrive/loggerd/SConscript +++ b/selfdrive/loggerd/SConscript @@ -1,7 +1,9 @@ Import('env', 'arch', 'cereal', 'messaging', 'common', 'visionipc', 'gpucommon') -src = ['loggerd.cc', 'logger.cc'] -libs = ['zmq', 'capnp', 'kj', 'z', + +logger_lib = env.Library('logger', ["logger.cc"]) +src = ['loggerd.cc'] +libs = [logger_lib, 'zmq', 'capnp', 'kj', 'z', 'avformat', 'avcodec', 'swscale', 'avutil', 'yuv', 'bz2', 'OpenCL', common, cereal, messaging, visionipc] @@ -22,3 +24,4 @@ if arch == "Darwin": env['FRAMEWORKS'] = ['OpenCL'] env.Program(src, LIBS=libs) +env.Program('bootlog.cc', LIBS=libs) diff --git a/selfdrive/loggerd/bootlog.cc b/selfdrive/loggerd/bootlog.cc new file mode 100644 index 0000000000..0da3872848 --- /dev/null +++ b/selfdrive/loggerd/bootlog.cc @@ -0,0 +1,37 @@ +#include +#include + +#include "common/swaglog.h" +#include "common/util.h" +#include "logger.h" +#include "messaging.hpp" + +int main(int argc, char** argv) { + LoggerState logger; + logger_init(&logger, "bootlog", false); + + char segment_path[4096]; + int err = logger_next(&logger, LOG_ROOT.c_str(), segment_path, sizeof(segment_path), nullptr); + assert(err == 0); + LOGW("bootlog to %s", segment_path); + + MessageBuilder msg; + auto boot = msg.initEvent().initBoot(); + + boot.setWallTimeNanos(nanos_since_epoch()); + + std::string lastKmsg = util::read_file("/sys/fs/pstore/console-ramoops"); + boot.setLastKmsg(capnp::Data::Reader((const kj::byte*)lastKmsg.data(), lastKmsg.size())); + + std::string lastPmsg = util::read_file("/sys/fs/pstore/pmsg-ramoops-0"); + boot.setLastPmsg(capnp::Data::Reader((const kj::byte*)lastPmsg.data(), lastPmsg.size())); + + std::string launchLog = util::read_file("/tmp/launch_log"); + boot.setLaunchLog(capnp::Text::Reader(launchLog.data(), launchLog.size())); + + auto bytes = msg.toBytes(); + logger_log(&logger, bytes.begin(), bytes.size(), false); + + logger_close(&logger); + return 0; +} diff --git a/selfdrive/loggerd/logger.cc b/selfdrive/loggerd/logger.cc index 0cec20f680..4ac496d2d3 100644 --- a/selfdrive/loggerd/logger.cc +++ b/selfdrive/loggerd/logger.cc @@ -12,12 +12,99 @@ #include #include +#include +#include +#include +#ifdef QCOM +#include +#endif #include "messaging.hpp" #include "common/swaglog.h" - +#include "common/params.h" +#include "common/util.h" +#include "common/version.h" #include "logger.h" +void append_property(const char* key, const char* value, void *cookie) { + std::vector > *properties = + (std::vector > *)cookie; + + properties->push_back(std::make_pair(std::string(key), std::string(value))); +} + +void log_init_data(LoggerState *s) { + MessageBuilder msg; + auto init = msg.initEvent().initInitData(); + + if (util::file_exists("/EON")) { + init.setDeviceType(cereal::InitData::DeviceType::NEO); + } else if (util::file_exists("/TICI")) { + init.setDeviceType(cereal::InitData::DeviceType::TICI); + } else { + init.setDeviceType(cereal::InitData::DeviceType::PC); + } + + init.setVersion(capnp::Text::Reader(COMMA_VERSION)); + + std::ifstream cmdline_stream("/proc/cmdline"); + std::vector kernel_args; + std::string buf; + while (cmdline_stream >> buf) { + kernel_args.push_back(buf); + } + + auto lkernel_args = init.initKernelArgs(kernel_args.size()); + for (int i=0; i > properties; + property_list(append_property, (void*)&properties); + + auto lentries = init.initAndroidProperties().initEntries(properties.size()); + for (int i=0; i params_map; + params.read_db_all(¶ms_map); + auto lparams = init.initParams().initEntries(params_map.size()); + int i = 0; + for (auto& kv : params_map) { + auto lentry = lparams[i]; + lentry.setKey(kv.first); + lentry.setValue(kv.second); + i++; + } + } + + auto bytes = msg.toBytes(); + logger_log(s, bytes.begin(), bytes.size(), s->has_qlog); +} + + static void log_sentinel(LoggerState *s, cereal::Sentinel::SentinelType type) { MessageBuilder msg; auto sen = msg.initEvent().initSentinel(); @@ -43,15 +130,9 @@ static int mkpath(char* file_path) { return 0; } -void logger_init(LoggerState *s, const char* log_name, const uint8_t* init_data, size_t init_data_len, bool has_qlog) { +void logger_init(LoggerState *s, const char* log_name, bool has_qlog) { memset(s, 0, sizeof(*s)); - if (init_data) { - s->init_data = (uint8_t*)malloc(init_data_len); - assert(s->init_data); - memcpy(s->init_data, init_data, init_data_len); - s->init_data_len = init_data_len; - } - + umask(0); pthread_mutex_init(&s->lock, NULL); @@ -110,20 +191,10 @@ static LoggerHandle* logger_open(LoggerState *s, const char* root_path) { h->bz_qlog = BZ2_bzWriteOpen(&bzerror, h->qlog_file, 9, 0, 30); if (bzerror != BZ_OK) goto fail; } - - if (s->init_data) { - BZ2_bzWrite(&bzerror, h->bz_file, s->init_data, s->init_data_len); - if (bzerror != BZ_OK) goto fail; - - if (s->has_qlog) { - // init data goes in the qlog too - BZ2_bzWrite(&bzerror, h->bz_qlog, s->init_data, s->init_data_len); - if (bzerror != BZ_OK) goto fail; - } - } - + pthread_mutex_init(&h->lock, NULL); h->refcnt++; + log_init_data(s); return h; fail: LOGE("logger failed to open files"); @@ -203,7 +274,6 @@ void logger_close(LoggerState *s) { log_sentinel(s, cereal::Sentinel::SentinelType::END_OF_ROUTE); pthread_mutex_lock(&s->lock); - free(s->init_data); if (s->cur_handle) { lh_close(s->cur_handle); } diff --git a/selfdrive/loggerd/logger.h b/selfdrive/loggerd/logger.h index ab8798481c..d845d19742 100644 --- a/selfdrive/loggerd/logger.h +++ b/selfdrive/loggerd/logger.h @@ -1,13 +1,16 @@ -#ifndef LOGGER_H -#define LOGGER_H +#pragma once #include #include #include #include +#include +#include -#ifdef __cplusplus -extern "C" { +#if defined(QCOM) || defined(QCOM2) +const std::string LOG_ROOT = "/data/media/0/realdata"; +#else +const std::string LOG_ROOT = util::getenv_default("HOME", "/.comma/media/0/realdata", "/data/media/0/realdata"); #endif #define LOGGER_MAX_HANDLES 16 @@ -28,10 +31,6 @@ typedef struct LoggerHandle { typedef struct LoggerState { pthread_mutex_t lock; - - uint8_t* init_data; - size_t init_data_len; - int part; char route_name[64]; char log_name[64]; @@ -41,7 +40,7 @@ typedef struct LoggerState { LoggerHandle* cur_handle; } LoggerState; -void logger_init(LoggerState *s, const char* log_name, const uint8_t* init_data, size_t init_data_len, bool has_qlog); +void logger_init(LoggerState *s, const char* log_name, bool has_qlog); int logger_next(LoggerState *s, const char* root_path, char* out_segment_path, size_t out_segment_path_len, int* out_part); @@ -51,9 +50,3 @@ void logger_log(LoggerState *s, uint8_t* data, size_t data_size, bool in_qlog); void lh_log(LoggerHandle* h, uint8_t* data, size_t data_size, bool in_qlog); void lh_close(LoggerHandle* h); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/selfdrive/loggerd/loggerd.cc b/selfdrive/loggerd/loggerd.cc index ff888e38cf..c8348e3e2a 100644 --- a/selfdrive/loggerd/loggerd.cc +++ b/selfdrive/loggerd/loggerd.cc @@ -10,9 +10,7 @@ #include #include -#include -#include -#include + #include #include #include @@ -20,11 +18,6 @@ #include #include -#ifdef QCOM -#include -#endif - -#include "common/version.h" #include "common/timing.h" #include "common/params.h" #include "common/swaglog.h" @@ -107,11 +100,6 @@ LogCameraInfo cameras_logged[LOG_CAMERA_ID_MAX] = { namespace { constexpr int SEGMENT_LENGTH = 60; -#if defined(QCOM) || defined(QCOM2) -std::string LOG_ROOT = "/data/media/0/realdata"; -#else -std::string LOG_ROOT = util::getenv_default("HOME", "/.comma/media/0/realdata", "/data/media/0/realdata"); -#endif double randrange(double a, double b) __attribute__((unused)); double randrange(double a, double b) { @@ -123,11 +111,6 @@ double randrange(double a, double b) { ExitHandler do_exit; -static bool file_exists(const std::string& fn) { - std::ifstream f(fn); - return f.good(); -} - class RotateState { public: SubSocket* fpkt_sock; @@ -328,82 +311,6 @@ void encoder_thread(int cam_idx) { } -void append_property(const char* key, const char* value, void *cookie) { - std::vector > *properties = - (std::vector > *)cookie; - - properties->push_back(std::make_pair(std::string(key), std::string(value))); -} - -kj::Array gen_init_data() { - MessageBuilder msg; - auto init = msg.initEvent().initInitData(); - - if (file_exists("/EON")) { - init.setDeviceType(cereal::InitData::DeviceType::NEO); - } else if (file_exists("/TICI")) { - init.setDeviceType(cereal::InitData::DeviceType::TICI); - } else { - init.setDeviceType(cereal::InitData::DeviceType::PC); - } - - init.setVersion(capnp::Text::Reader(COMMA_VERSION)); - - std::ifstream cmdline_stream("/proc/cmdline"); - std::vector kernel_args; - std::string buf; - while (cmdline_stream >> buf) { - kernel_args.push_back(buf); - } - - auto lkernel_args = init.initKernelArgs(kernel_args.size()); - for (int i=0; i > properties; - property_list(append_property, (void*)&properties); - - auto lentries = init.initAndroidProperties().initEntries(properties.size()); - for (int i=0; i params_map; - params.read_db_all(¶ms_map); - auto lparams = init.initParams().initEntries(params_map.size()); - int i = 0; - for (auto& kv : params_map) { - auto lentry = lparams[i]; - lentry.setKey(kv.first); - lentry.setValue(kv.second); - i++; - } - } - return capnp::messageToFlatArray(msg); -} - static int clear_locks_fn(const char* fpath, const struct stat *sb, int tyupeflag) { const char* dot = strrchr(fpath, '.'); if (dot && strcmp(dot, ".lock") == 0) { @@ -416,50 +323,10 @@ static void clear_locks() { ftw(LOG_ROOT.c_str(), clear_locks_fn, 16); } -static void bootlog() { - int err; - - { - auto words = gen_init_data(); - auto bytes = words.asBytes(); - logger_init(&s.logger, "bootlog", bytes.begin(), bytes.size(), false); - } - - err = logger_next(&s.logger, LOG_ROOT.c_str(), s.segment_path, sizeof(s.segment_path), &s.rotate_segment); - assert(err == 0); - LOGW("bootlog to %s", s.segment_path); - - { - MessageBuilder msg; - auto boot = msg.initEvent().initBoot(); - - boot.setWallTimeNanos(nanos_since_epoch()); - - std::string lastKmsg = util::read_file("/sys/fs/pstore/console-ramoops"); - boot.setLastKmsg(capnp::Data::Reader((const kj::byte*)lastKmsg.data(), lastKmsg.size())); - - std::string lastPmsg = util::read_file("/sys/fs/pstore/pmsg-ramoops-0"); - boot.setLastPmsg(capnp::Data::Reader((const kj::byte*)lastPmsg.data(), lastPmsg.size())); - - std::string launchLog = util::read_file("/tmp/launch_log"); - boot.setLaunchLog(capnp::Text::Reader(launchLog.data(), launchLog.size())); - - auto bytes = msg.toBytes(); - logger_log(&s.logger, bytes.begin(), bytes.size(), false); - } - - logger_close(&s.logger); -} - int main(int argc, char** argv) { setpriority(PRIO_PROCESS, 0, -12); - if (argc > 1 && strcmp(argv[1], "--bootlog") == 0) { - bootlog(); - return 0; - } - int segment_length = SEGMENT_LENGTH; if (getenv("LOGGERD_TEST")) { segment_length = atoi(getenv("LOGGERD_SEGMENT_LENGTH")); @@ -495,11 +362,7 @@ int main(int argc, char** argv) { } // init logger - { - auto words = gen_init_data(); - auto bytes = words.asBytes(); - logger_init(&s.logger, "rlog", bytes.begin(), bytes.size(), true); - } + logger_init(&s.logger, "rlog", true); // init encoders pthread_mutex_init(&s.rotate_lock, NULL); diff --git a/selfdrive/loggerd/tests/test_loggerd.py b/selfdrive/loggerd/tests/test_loggerd.py index 870d9c3a72..84cf97270d 100755 --- a/selfdrive/loggerd/tests/test_loggerd.py +++ b/selfdrive/loggerd/tests/test_loggerd.py @@ -47,7 +47,7 @@ class TestLoggerd(unittest.TestCase): def _gen_bootlog(self): with Timeout(5): - out = subprocess.check_output(["./loggerd", "--bootlog"], cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8') + out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd"), encoding='utf-8') # check existence d = self._get_log_dir(out) diff --git a/selfdrive/manager.py b/selfdrive/manager.py index f576f2bbc2..80d7c388a9 100755 --- a/selfdrive/manager.py +++ b/selfdrive/manager.py @@ -440,7 +440,7 @@ def manager_thread(): cloudlog.info({"environ": os.environ}) # save boot log - subprocess.call(["./loggerd", "--bootlog"], cwd=os.path.join(BASEDIR, "selfdrive/loggerd")) + subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "selfdrive/loggerd")) # start daemon processes for p in daemon_processes: