From 52673f0532ce13ac29ef1f90cbcfe20b9c8bd762 Mon Sep 17 00:00:00 2001 From: Comma Device Date: Fri, 20 Nov 2020 16:37:32 +0000 Subject: [PATCH] reproduce dmon lag --- selfdrive/modeld/test/dmon_lag/fuzz.py | 54 ++++++++++++ selfdrive/modeld/test/dmon_lag/repro.cc | 110 ++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100755 selfdrive/modeld/test/dmon_lag/fuzz.py create mode 100644 selfdrive/modeld/test/dmon_lag/repro.cc diff --git a/selfdrive/modeld/test/dmon_lag/fuzz.py b/selfdrive/modeld/test/dmon_lag/fuzz.py new file mode 100755 index 0000000000..e4a298bde9 --- /dev/null +++ b/selfdrive/modeld/test/dmon_lag/fuzz.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +from selfdrive.manager import start_managed_process, kill_managed_process +import random +import os +import time +import cereal.messaging as messaging + +if __name__ == "__main__": + logmessage = messaging.sub_sock('logMessage') + hitcount = 0 + hits = [] + ln = 0 + while 1: + print("\n***** loop %d with hit count %d %r\n" % (ln, hitcount, hits)) + start_managed_process("camerad") + #time.sleep(random.random()) + os.environ['LOGPRINT'] = "debug" + start_managed_process("dmonitoringmodeld") + os.environ['LOGPRINT'] = "" + + # drain all old messages + messaging.drain_sock(logmessage, False) + + done = False + cnt = 0 + best = 100 + for i in range(100): + ret = messaging.drain_sock(logmessage, True) + for r in ret: + if 'dmonitoring process' in r.logMessage: + cnt += 1 + done = r.logMessage + ms = float(done.split('dmonitoring process: ')[1].split("ms")[0]) + best = min(ms, best) + if cnt >= 5: + break + + print(ln, best, done) + #if best > 16: + if best > 4: + print("HIT HIT HIT") + hitcount += 1 + hits.append(best) + + + #start_managed_process("modeld") + kill_managed_process("dmonitoringmodeld") + #kill_managed_process("modeld") + kill_managed_process("camerad") + ln += 1 + + if hitcount >= 1: + break + diff --git a/selfdrive/modeld/test/dmon_lag/repro.cc b/selfdrive/modeld/test/dmon_lag/repro.cc new file mode 100644 index 0000000000..60c9aee3e6 --- /dev/null +++ b/selfdrive/modeld/test/dmon_lag/repro.cc @@ -0,0 +1,110 @@ +// clang++ -mcpu=cortex-a57 -O2 repro.cc + +#include +#include +#include + +static inline double millis_since_boot() { + struct timespec t; + clock_gettime(CLOCK_BOOTTIME, &t); + return t.tv_sec * 1000.0 + t.tv_nsec * 1e-6; +} + +int set_realtime_priority(int level) { + long tid = getpid(); + + // should match python using chrt + struct sched_param sa; + memset(&sa, 0, sizeof(sa)); + sa.sched_priority = level; + return sched_setscheduler(tid, SCHED_FIFO, &sa); +} + +#define MODEL_WIDTH 320 +#define MODEL_HEIGHT 640 +#define input_lambda(x) (x - 128.f) * 0.0078125f + +template +static inline T *get_buffer(std::vector &buf, const size_t size) { + if (buf.size() < size) { + buf.resize(size); + } + return buf.data(); +} + +void inner(uint8_t *resized_buf, float *net_input_buf) { + int resized_width = MODEL_WIDTH; + int resized_height = MODEL_HEIGHT; + + // one shot conversion, O(n) anyway + // yuvframe2tensor, normalize + for (int r = 0; r < MODEL_HEIGHT/2; r++) { + for (int c = 0; c < MODEL_WIDTH/2; c++) { + // Y_ul + net_input_buf[(c*MODEL_HEIGHT/2) + r] = input_lambda(resized_buf[(2*r*resized_width) + (2*c)]); + // Y_ur + net_input_buf[(c*MODEL_HEIGHT/2) + r + (2*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(2*r*resized_width) + (2*c+1)]); + // Y_dl + net_input_buf[(c*MODEL_HEIGHT/2) + r + ((MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(2*r*resized_width+1) + (2*c)]); + // Y_dr + net_input_buf[(c*MODEL_HEIGHT/2) + r + (3*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(2*r*resized_width+1) + (2*c+1)]); + // U + net_input_buf[(c*MODEL_HEIGHT/2) + r + (4*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(resized_width*resized_height) + (r*resized_width/2) + c]); + // V + net_input_buf[(c*MODEL_HEIGHT/2) + r + (5*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(resized_width*resized_height) + ((resized_width/2)*(resized_height/2)) + (r*resized_width/2) + c]); + } + } +} + +float trial() { + std::vector vec_resized_buf; + std::vector vec_net_input_buf; + + int resized_width = MODEL_WIDTH; + int resized_height = MODEL_HEIGHT; + uint8_t *resized_buf = get_buffer(vec_resized_buf, resized_width*resized_height*3/2); + + int yuv_buf_len = (MODEL_WIDTH/2) * (MODEL_HEIGHT/2) * 6; // Y|u|v -> y|y|y|y|u|v + float *net_input_buf = get_buffer(vec_net_input_buf, yuv_buf_len); + + float avg = 0.0; + for (int i = 0; i < 20; i++) { + //__builtin___clear_cache((char*)resized_buf, (char*)resized_buf + (resized_width*resized_height*3/2)); + //__builtin___clear_cache((char*)net_input_buf, (char*)net_input_buf + yuv_buf_len); + + double s4 = millis_since_boot(); + inner(resized_buf, net_input_buf); + double s5 = millis_since_boot(); + avg += s5-s4; + } + + avg /= 20; + if (avg > 5) { + printf("HIT %f\n", avg); + printf("BAD\n"); + + for (int i = 0; i < 200; i++) { + //__builtin___clear_cache((char*)resized_buf, (char*)resized_buf + (resized_width*resized_height*3/2)); + //__builtin___clear_cache((char*)net_input_buf, (char*)net_input_buf + yuv_buf_len); + + double s4 = millis_since_boot(); + inner(resized_buf, net_input_buf); + double s5 = millis_since_boot(); + printf("%.2f ", s5-s4); + } + printf("\n"); + + exit(0); + } + return avg; +} + +int main() { + set_realtime_priority(51); + + while (1) { + float ret = trial(); + printf("got %f\n", ret); + } +} +