From 7c8ea3053239a06de80e735c8a077627d434229b Mon Sep 17 00:00:00 2001 From: HaraldSchafer Date: Thu, 19 Nov 2020 21:05:29 -0800 Subject: [PATCH] fix frame drop percentage (#2578) * correct, I think * add model execution time * remove unused * token is a file * this is gonna change Co-authored-by: Adeeb Shihadeh old-commit-hash: 7378bc42845a094bbb22fb76154cb40ecb098aaf --- selfdrive/modeld/modeld.cc | 11 ++++++----- selfdrive/modeld/models/driving.cc | 14 ++++++++++---- selfdrive/modeld/models/driving.h | 9 ++++++--- selfdrive/test/openpilotci.py | 8 +++++++- selfdrive/test/process_replay/camera_replay.py | 2 +- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/selfdrive/modeld/modeld.cc b/selfdrive/modeld/modeld.cc index 7250fd38d5..c528ba98d0 100644 --- a/selfdrive/modeld/modeld.cc +++ b/selfdrive/modeld/modeld.cc @@ -193,17 +193,18 @@ int main(int argc, char **argv) { model_eval_frame(&model, q, yuv_ion.buf_cl, buf_info.width, buf_info.height, model_transform, NULL, vec_desire); mt2 = millis_since_boot(); + float model_execution_time = (mt2 - mt1) / 1000.0; // tracked dropped frames uint32_t vipc_dropped_frames = extra.frame_id - last_vipc_frame_id - 1; frames_dropped = (1. - frame_filter_k) * frames_dropped + frame_filter_k * (float)std::min(vipc_dropped_frames, 10U); - float frame_drop_perc = frames_dropped / MODEL_FREQ; + float frame_drop_ratio = frames_dropped / (1 + frames_dropped); - model_publish(pm, extra.frame_id, frame_id, vipc_dropped_frames, frame_drop_perc, model_buf, extra.timestamp_eof); - model_publish_v2(pm, extra.frame_id, frame_id, vipc_dropped_frames, frame_drop_perc, model_buf, extra.timestamp_eof); - posenet_publish(pm, extra.frame_id, frame_id, vipc_dropped_frames, frame_drop_perc, model_buf, extra.timestamp_eof); + model_publish(pm, extra.frame_id, frame_id, vipc_dropped_frames, frame_drop_ratio, model_buf, extra.timestamp_eof, model_execution_time); + model_publish_v2(pm, extra.frame_id, frame_id, vipc_dropped_frames, frame_drop_ratio, model_buf, extra.timestamp_eof, model_execution_time); + posenet_publish(pm, extra.frame_id, frame_id, vipc_dropped_frames, frame_drop_ratio, model_buf, extra.timestamp_eof); - LOGD("model process: %.2fms, from last %.2fms, vipc_frame_id %zu, frame_id, %zu, frame_drop %.3f", mt2-mt1, mt1-last, extra.frame_id, frame_id, frame_drop_perc); + LOGD("model process: %.2fms, from last %.2fms, vipc_frame_id %zu, frame_id, %zu, frame_drop %.3f", mt2-mt1, mt1-last, extra.frame_id, frame_id, frame_drop_ratio); last = mt1; last_vipc_frame_id = extra.frame_id; } diff --git a/selfdrive/modeld/models/driving.cc b/selfdrive/modeld/models/driving.cc index 301e7fafb3..418060ffe3 100644 --- a/selfdrive/modeld/models/driving.cc +++ b/selfdrive/modeld/models/driving.cc @@ -307,7 +307,8 @@ void fill_xyzt(cereal::ModelDataV2::XYZTData::Builder xyzt, const float * data, void model_publish_v2(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id, uint32_t vipc_dropped_frames, float frame_drop, - const ModelDataRaw &net_outputs, uint64_t timestamp_eof) { + const ModelDataRaw &net_outputs, uint64_t timestamp_eof, + float model_execution_time) { // make msg MessageBuilder msg; auto framed = msg.initEvent(frame_drop < MAX_FRAME_DROP).initModelV2(); @@ -316,6 +317,7 @@ void model_publish_v2(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id, framed.setFrameAge(frame_age); framed.setFrameDropPerc(frame_drop * 100); framed.setTimestampEof(timestamp_eof); + framed.setModelExecutionTime(model_execution_time); // plan int plan_mhp_max_idx = 0; @@ -385,15 +387,18 @@ void model_publish_v2(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id, } void model_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id, - uint32_t vipc_dropped_frames, float frame_drop, const ModelDataRaw &net_outputs, uint64_t timestamp_eof) { - uint32_t frame_age = (frame_id > vipc_frame_id) ? (frame_id - vipc_frame_id) : 0; + uint32_t vipc_dropped_frames, float frame_drop, + const ModelDataRaw &net_outputs, uint64_t timestamp_eof, + float model_execution_time) { + uint32_t frame_age = (frame_id > vipc_frame_id) ? (frame_id - vipc_frame_id) : 0; MessageBuilder msg; auto framed = msg.initEvent(frame_drop < MAX_FRAME_DROP).initModel(); framed.setFrameId(vipc_frame_id); framed.setFrameAge(frame_age); framed.setFrameDropPerc(frame_drop * 100); framed.setTimestampEof(timestamp_eof); + framed.setModelExecutionTime(model_execution_time); // Find the distribution that corresponds to the most probable plan int plan_mhp_max_idx = 0; @@ -461,7 +466,8 @@ void model_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id, } void posenet_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id, - uint32_t vipc_dropped_frames, float frame_drop, const ModelDataRaw &net_outputs, uint64_t timestamp_eof) { + uint32_t vipc_dropped_frames, float frame_drop, + const ModelDataRaw &net_outputs, uint64_t timestamp_eof) { float trans_arr[3]; float trans_std_arr[3]; float rot_arr[3]; diff --git a/selfdrive/modeld/models/driving.h b/selfdrive/modeld/models/driving.h index 613ba139cb..346e6e8760 100644 --- a/selfdrive/modeld/models/driving.h +++ b/selfdrive/modeld/models/driving.h @@ -78,9 +78,12 @@ void model_free(ModelState* s); void poly_fit(float *in_pts, float *in_stds, float *out); void model_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id, - uint32_t vipc_dropped_frames, float frame_drop, const ModelDataRaw &data, uint64_t timestamp_eof); + uint32_t vipc_dropped_frames, float frame_drop, const ModelDataRaw &data, + uint64_t timestamp_eof, float model_execution_time); void model_publish_v2(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id, - uint32_t vipc_dropped_frames, float frame_drop, const ModelDataRaw &data, uint64_t timestamp_eof); + uint32_t vipc_dropped_frames, float frame_drop, const ModelDataRaw &data, + uint64_t timestamp_eof, float model_execution_time); void posenet_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id, - uint32_t vipc_dropped_frames, float frame_drop, const ModelDataRaw &data, uint64_t timestamp_eof); + uint32_t vipc_dropped_frames, float frame_drop, const ModelDataRaw &data, + uint64_t timestamp_eof); #endif diff --git a/selfdrive/test/openpilotci.py b/selfdrive/test/openpilotci.py index be8aefac78..6cda6e04e1 100644 --- a/selfdrive/test/openpilotci.py +++ b/selfdrive/test/openpilotci.py @@ -5,13 +5,19 @@ import subprocess BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" +TOKEN_PATH = "/data/azure_token" + def get_url(route_name, segment_num, log_type="rlog"): ext = "hevc" if log_type in ["fcamera", "dcamera"] else "bz2" return BASE_URL + "%s/%s/%s.%s" % (route_name.replace("|", "/"), segment_num, log_type, ext) def upload_file(path, name): from azure.storage.blob import BlockBlobService - sas_token = os.getenv("TOKEN", None) + + sas_token = None + if os.path.isfile(TOKEN_PATH): + sas_token = open(TOKEN_PATH).read().strip + if sas_token is None: sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw \ --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv", shell=True).decode().strip("\n") diff --git a/selfdrive/test/process_replay/camera_replay.py b/selfdrive/test/process_replay/camera_replay.py index a580313eea..0a50be404d 100755 --- a/selfdrive/test/process_replay/camera_replay.py +++ b/selfdrive/test/process_replay/camera_replay.py @@ -90,7 +90,7 @@ if __name__ == "__main__": log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit) cmp_log = LogReader(BASE_URL + log_fn) results: Any = {TEST_ROUTE: {}} - results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=['logMonoTime', 'valid', 'model.frameDropPerc']) + results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=['logMonoTime', 'valid', 'model.frameDropPerc', 'model.modelExecutionTime']) diff1, diff2, failed = format_diff(results, ref_commit) print(diff1)