diff --git a/selfdrive/modeld/constants.py b/selfdrive/modeld/constants.py index 8be12b2205..26ea8e86b3 100644 --- a/selfdrive/modeld/constants.py +++ b/selfdrive/modeld/constants.py @@ -28,6 +28,7 @@ FCW_THRESHOLDS_3MS2 = np.array([.7, .7], dtype=np.float32) DISENGAGE_WIDTH = 5 POSE_WIDTH = 6 +WIDE_FROM_DEVICE_WIDTH = 3 SIM_POSE_WIDTH = 6 LEAD_WIDTH = 4 LANE_LINES_WIDTH = 2 diff --git a/selfdrive/modeld/models/driving.cc b/selfdrive/modeld/models/driving.cc deleted file mode 100644 index 0a7f0c949d..0000000000 --- a/selfdrive/modeld/models/driving.cc +++ /dev/null @@ -1,330 +0,0 @@ -#include "selfdrive/modeld/models/driving.h" - -#include - - -void fill_lead(cereal::ModelDataV2::LeadDataV3::Builder lead, const ModelOutputLeads &leads, int t_idx, float prob_t) { - std::array lead_t = {0.0, 2.0, 4.0, 6.0, 8.0, 10.0}; - const auto &best_prediction = leads.get_best_prediction(t_idx); - lead.setProb(sigmoid(leads.prob[t_idx])); - lead.setProbTime(prob_t); - std::array lead_x, lead_y, lead_v, lead_a; - std::array lead_x_std, lead_y_std, lead_v_std, lead_a_std; - for (int i=0; i desire_state_softmax; - softmax(meta_data.desire_state_prob.array.data(), desire_state_softmax.data(), DESIRE_LEN); - - std::array desire_pred_softmax; - for (int i=0; i lat_long_t = {2, 4, 6, 8, 10}; - std::array gas_disengage_sigmoid, brake_disengage_sigmoid, steer_override_sigmoid, - brake_3ms2_sigmoid, brake_4ms2_sigmoid, brake_5ms2_sigmoid; - for (int i=0; i threshold; - } - for (int i=0; i FCW_THRESHOLD_3MS2; - } - - auto disengage = meta.initDisengagePredictions(); - disengage.setT(to_kj_array_ptr(lat_long_t)); - disengage.setGasDisengageProbs(to_kj_array_ptr(gas_disengage_sigmoid)); - disengage.setBrakeDisengageProbs(to_kj_array_ptr(brake_disengage_sigmoid)); - disengage.setSteerOverrideProbs(to_kj_array_ptr(steer_override_sigmoid)); - disengage.setBrake3MetersPerSecondSquaredProbs(to_kj_array_ptr(brake_3ms2_sigmoid)); - disengage.setBrake4MetersPerSecondSquaredProbs(to_kj_array_ptr(brake_4ms2_sigmoid)); - disengage.setBrake5MetersPerSecondSquaredProbs(to_kj_array_ptr(brake_5ms2_sigmoid)); - - meta.setEngagedProb(sigmoid(meta_data.engaged_prob)); - meta.setDesirePrediction(to_kj_array_ptr(desire_pred_softmax)); - meta.setDesireState(to_kj_array_ptr(desire_state_softmax)); - meta.setHardBrakePredicted(above_fcw_threshold); -} - -void fill_confidence(cereal::ModelDataV2::Builder &framed, PublishState &ps) { - if (framed.getFrameId() % (2*MODEL_FREQ) == 0) { - // update every 2s to match predictions interval - auto dbps = framed.getMeta().getDisengagePredictions().getBrakeDisengageProbs(); - auto dgps = framed.getMeta().getDisengagePredictions().getGasDisengageProbs(); - auto dsps = framed.getMeta().getDisengagePredictions().getSteerOverrideProbs(); - - float any_dp[DISENGAGE_LEN]; - float dp_ind[DISENGAGE_LEN]; - - for (int i = 0; i < DISENGAGE_LEN; i++) { - any_dp[i] = 1 - ((1-dbps[i])*(1-dgps[i])*(1-dsps[i])); // any disengage prob - } - - dp_ind[0] = any_dp[0]; - for (int i = 0; i < DISENGAGE_LEN-1; i++) { - dp_ind[i+1] = (any_dp[i+1] - any_dp[i]) / (1 - any_dp[i]); // independent disengage prob for each 2s slice - } - - // rolling buf for 2, 4, 6, 8, 10s - std::memmove(&ps.disengage_buffer[0], &ps.disengage_buffer[DISENGAGE_LEN], sizeof(float) * DISENGAGE_LEN * (DISENGAGE_LEN-1)); - std::memcpy(&ps.disengage_buffer[DISENGAGE_LEN * (DISENGAGE_LEN-1)], &dp_ind[0], sizeof(float) * DISENGAGE_LEN); - } - - float score = 0; - for (int i = 0; i < DISENGAGE_LEN; i++) { - score += ps.disengage_buffer[i*DISENGAGE_LEN+DISENGAGE_LEN-1-i] / DISENGAGE_LEN; - } - - if (score < RYG_GREEN) { - framed.setConfidence(cereal::ModelDataV2::ConfidenceClass::GREEN); - } else if (score < RYG_YELLOW) { - framed.setConfidence(cereal::ModelDataV2::ConfidenceClass::YELLOW); - } else { - framed.setConfidence(cereal::ModelDataV2::ConfidenceClass::RED); - } -} - -template -void fill_xyzt(cereal::XYZTData::Builder xyzt, const std::array &t, - const std::array &x, const std::array &y, const std::array &z) { - xyzt.setT(to_kj_array_ptr(t)); - xyzt.setX(to_kj_array_ptr(x)); - xyzt.setY(to_kj_array_ptr(y)); - xyzt.setZ(to_kj_array_ptr(z)); -} - -template -void fill_xyzt(cereal::XYZTData::Builder xyzt, const std::array &t, - const std::array &x, const std::array &y, const std::array &z, - const std::array &x_std, const std::array &y_std, const std::array &z_std) { - fill_xyzt(xyzt, t, x, y, z); - xyzt.setXStd(to_kj_array_ptr(x_std)); - xyzt.setYStd(to_kj_array_ptr(y_std)); - xyzt.setZStd(to_kj_array_ptr(z_std)); -} - -void fill_plan(cereal::ModelDataV2::Builder &framed, const ModelOutputPlanPrediction &plan) { - std::array pos_x, pos_y, pos_z; - std::array pos_x_std, pos_y_std, pos_z_std; - std::array vel_x, vel_y, vel_z; - std::array rot_x, rot_y, rot_z; - std::array acc_x, acc_y, acc_z; - std::array rot_rate_x, rot_rate_y, rot_rate_z; - - for (int i=0; i &plan_t, - const ModelOutputLaneLines &lanes) { - std::array left_far_y, left_far_z; - std::array left_near_y, left_near_z; - std::array right_near_y, right_near_z; - std::array right_far_y, right_far_z; - for (int j=0; j &plan_t, - const ModelOutputRoadEdges &edges) { - std::array left_y, left_z; - std::array right_y, right_z; - for (int j=0; j plan_t; - std::fill_n(plan_t.data(), plan_t.size(), NAN); - plan_t[0] = 0.0; - for (int xidx=1, tidx=0; xidx t_offsets = {0.0, 2.0, 4.0}; - for (int i=0; i vipc_frame_id) ? (frame_id - vipc_frame_id) : 0; - auto framed = msg.initEvent(valid).initModelV2(); - framed.setFrameId(vipc_frame_id); - framed.setFrameIdExtra(vipc_frame_id_extra); - framed.setFrameAge(frame_age); - framed.setFrameDropPerc(frame_drop * 100); - framed.setTimestampEof(timestamp_eof); - framed.setLocationMonoTime(timestamp_llk); - framed.setModelExecutionTime(model_execution_time); - framed.setNavEnabled(nav_enabled); - if (send_raw_pred) { - framed.setRawPredictions(kj::ArrayPtr(net_output_data, NET_OUTPUT_SIZE).asBytes()); - } - fill_model(framed, *((ModelOutput*) net_output_data), ps); -} - -void fill_pose_msg(MessageBuilder &msg, float *net_output_data, uint32_t vipc_frame_id, uint32_t vipc_dropped_frames, uint64_t timestamp_eof, const bool valid) { - const ModelOutput &net_outputs = *((ModelOutput*) net_output_data); - const auto &v_mean = net_outputs.pose.velocity_mean; - const auto &r_mean = net_outputs.pose.rotation_mean; - const auto &t_mean = net_outputs.wide_from_device_euler.mean; - const auto &v_std = net_outputs.pose.velocity_std; - const auto &r_std = net_outputs.pose.rotation_std; - const auto &t_std = net_outputs.wide_from_device_euler.std; - const auto &road_transform_trans_mean = net_outputs.road_transform.position_mean; - const auto &road_transform_trans_std = net_outputs.road_transform.position_std; - - auto posenetd = msg.initEvent(valid && (vipc_dropped_frames < 1)).initCameraOdometry(); - posenetd.setTrans({v_mean.x, v_mean.y, v_mean.z}); - posenetd.setRot({r_mean.x, r_mean.y, r_mean.z}); - posenetd.setWideFromDeviceEuler({t_mean.x, t_mean.y, t_mean.z}); - posenetd.setRoadTransformTrans({road_transform_trans_mean.x, road_transform_trans_mean.y, road_transform_trans_mean.z}); - posenetd.setTransStd({exp(v_std.x), exp(v_std.y), exp(v_std.z)}); - posenetd.setRotStd({exp(r_std.x), exp(r_std.y), exp(r_std.z)}); - posenetd.setWideFromDeviceEulerStd({exp(t_std.x), exp(t_std.y), exp(t_std.z)}); - posenetd.setRoadTransformTransStd({exp(road_transform_trans_std.x), exp(road_transform_trans_std.y), exp(road_transform_trans_std.z)}); - - posenetd.setTimestampEof(timestamp_eof); - posenetd.setFrameId(vipc_frame_id); -} diff --git a/selfdrive/modeld/models/driving.h b/selfdrive/modeld/models/driving.h deleted file mode 100644 index 5df172dad6..0000000000 --- a/selfdrive/modeld/models/driving.h +++ /dev/null @@ -1,257 +0,0 @@ -#pragma once - -#include -#include - -#include "cereal/messaging/messaging.h" -#include "common/modeldata.h" -#include "common/util.h" -#include "selfdrive/modeld/models/commonmodel.h" -#include "selfdrive/modeld/runners/run.h" - -constexpr int FEATURE_LEN = 512; -constexpr int HISTORY_BUFFER_LEN = 99; -constexpr int DESIRE_LEN = 8; -constexpr int DESIRE_PRED_LEN = 4; -constexpr int TRAFFIC_CONVENTION_LEN = 2; -constexpr int NAV_FEATURE_LEN = 256; -constexpr int NAV_INSTRUCTION_LEN = 150; -constexpr int DRIVING_STYLE_LEN = 12; -constexpr int MODEL_FREQ = 20; - -constexpr int DISENGAGE_LEN = 5; -constexpr int BLINKER_LEN = 6; -constexpr int META_STRIDE = 7; - -constexpr int PLAN_MHP_N = 5; -constexpr int LEAD_MHP_N = 2; -constexpr int LEAD_TRAJ_LEN = 6; -constexpr int LEAD_MHP_SELECTION = 3; -// Padding to get output shape as multiple of 4 -constexpr int PAD_SIZE = 2; - -constexpr float FCW_THRESHOLD_5MS2_HIGH = 0.15; -constexpr float FCW_THRESHOLD_5MS2_LOW = 0.05; -constexpr float FCW_THRESHOLD_3MS2 = 0.7; - -struct ModelOutputXYZ { - float x; - float y; - float z; -}; -static_assert(sizeof(ModelOutputXYZ) == sizeof(float)*3); - -struct ModelOutputYZ { - float y; - float z; -}; -static_assert(sizeof(ModelOutputYZ) == sizeof(float)*2); - -struct ModelOutputPlanElement { - ModelOutputXYZ position; - ModelOutputXYZ velocity; - ModelOutputXYZ acceleration; - ModelOutputXYZ rotation; - ModelOutputXYZ rotation_rate; -}; -static_assert(sizeof(ModelOutputPlanElement) == sizeof(ModelOutputXYZ)*5); - -struct ModelOutputPlanPrediction { - std::array mean; - std::array std; - float prob; -}; -static_assert(sizeof(ModelOutputPlanPrediction) == (sizeof(ModelOutputPlanElement)*TRAJECTORY_SIZE*2) + sizeof(float)); - -struct ModelOutputPlans { - std::array prediction; - - constexpr const ModelOutputPlanPrediction &get_best_prediction() const { - int max_idx = 0; - for (int i = 1; i < prediction.size(); i++) { - if (prediction[i].prob > prediction[max_idx].prob) { - max_idx = i; - } - } - return prediction[max_idx]; - } -}; -static_assert(sizeof(ModelOutputPlans) == sizeof(ModelOutputPlanPrediction)*PLAN_MHP_N); - -struct ModelOutputLinesXY { - std::array left_far; - std::array left_near; - std::array right_near; - std::array right_far; -}; -static_assert(sizeof(ModelOutputLinesXY) == sizeof(ModelOutputYZ)*TRAJECTORY_SIZE*4); - -struct ModelOutputLineProbVal { - float val_deprecated; - float val; -}; -static_assert(sizeof(ModelOutputLineProbVal) == sizeof(float)*2); - -struct ModelOutputLinesProb { - ModelOutputLineProbVal left_far; - ModelOutputLineProbVal left_near; - ModelOutputLineProbVal right_near; - ModelOutputLineProbVal right_far; -}; -static_assert(sizeof(ModelOutputLinesProb) == sizeof(ModelOutputLineProbVal)*4); - -struct ModelOutputLaneLines { - ModelOutputLinesXY mean; - ModelOutputLinesXY std; - ModelOutputLinesProb prob; -}; -static_assert(sizeof(ModelOutputLaneLines) == (sizeof(ModelOutputLinesXY)*2) + sizeof(ModelOutputLinesProb)); - -struct ModelOutputEdgessXY { - std::array left; - std::array right; -}; -static_assert(sizeof(ModelOutputEdgessXY) == sizeof(ModelOutputYZ)*TRAJECTORY_SIZE*2); - -struct ModelOutputRoadEdges { - ModelOutputEdgessXY mean; - ModelOutputEdgessXY std; -}; -static_assert(sizeof(ModelOutputRoadEdges) == (sizeof(ModelOutputEdgessXY)*2)); - -struct ModelOutputLeadElement { - float x; - float y; - float velocity; - float acceleration; -}; -static_assert(sizeof(ModelOutputLeadElement) == sizeof(float)*4); - -struct ModelOutputLeadPrediction { - std::array mean; - std::array std; - std::array prob; -}; -static_assert(sizeof(ModelOutputLeadPrediction) == (sizeof(ModelOutputLeadElement)*LEAD_TRAJ_LEN*2) + (sizeof(float)*LEAD_MHP_SELECTION)); - -struct ModelOutputLeads { - std::array prediction; - std::array prob; - - constexpr const ModelOutputLeadPrediction &get_best_prediction(int t_idx) const { - int max_idx = 0; - for (int i = 1; i < prediction.size(); i++) { - if (prediction[i].prob[t_idx] > prediction[max_idx].prob[t_idx]) { - max_idx = i; - } - } - return prediction[max_idx]; - } -}; -static_assert(sizeof(ModelOutputLeads) == (sizeof(ModelOutputLeadPrediction)*LEAD_MHP_N) + (sizeof(float)*LEAD_MHP_SELECTION)); - - -struct ModelOutputPose { - ModelOutputXYZ velocity_mean; - ModelOutputXYZ rotation_mean; - ModelOutputXYZ velocity_std; - ModelOutputXYZ rotation_std; -}; -static_assert(sizeof(ModelOutputPose) == sizeof(ModelOutputXYZ)*4); - -struct ModelOutputWideFromDeviceEuler { - ModelOutputXYZ mean; - ModelOutputXYZ std; -}; -static_assert(sizeof(ModelOutputWideFromDeviceEuler) == sizeof(ModelOutputXYZ)*2); - -struct ModelOutputTemporalPose { - ModelOutputXYZ velocity_mean; - ModelOutputXYZ rotation_mean; - ModelOutputXYZ velocity_std; - ModelOutputXYZ rotation_std; -}; -static_assert(sizeof(ModelOutputTemporalPose) == sizeof(ModelOutputXYZ)*4); - -struct ModelOutputRoadTransform { - ModelOutputXYZ position_mean; - ModelOutputXYZ rotation_mean; - ModelOutputXYZ position_std; - ModelOutputXYZ rotation_std; -}; -static_assert(sizeof(ModelOutputRoadTransform) == sizeof(ModelOutputXYZ)*4); - -struct ModelOutputDisengageProb { - float gas_disengage; - float brake_disengage; - float steer_override; - float brake_3ms2; - float brake_4ms2; - float brake_5ms2; - float gas_pressed; -}; -static_assert(sizeof(ModelOutputDisengageProb) == sizeof(float)*7); - -struct ModelOutputBlinkerProb { - float left; - float right; -}; -static_assert(sizeof(ModelOutputBlinkerProb) == sizeof(float)*2); - -struct ModelOutputDesireProb { - union { - struct { - float none; - float turn_left; - float turn_right; - float lane_change_left; - float lane_change_right; - float keep_left; - float keep_right; - float null; - }; - struct { - std::array array; - }; - }; -}; -static_assert(sizeof(ModelOutputDesireProb) == sizeof(float)*DESIRE_LEN); - -struct ModelOutputMeta { - ModelOutputDesireProb desire_state_prob; - float engaged_prob; - std::array disengage_prob; - std::array blinker_prob; - std::array desire_pred_prob; -}; -static_assert(sizeof(ModelOutputMeta) == sizeof(ModelOutputDesireProb) + sizeof(float) + (sizeof(ModelOutputDisengageProb)*DISENGAGE_LEN) + (sizeof(ModelOutputBlinkerProb)*BLINKER_LEN) + (sizeof(ModelOutputDesireProb)*DESIRE_PRED_LEN)); - -struct ModelOutputFeatures { - std::array feature; -}; -static_assert(sizeof(ModelOutputFeatures) == (sizeof(float)*FEATURE_LEN)); - -struct ModelOutput { - const ModelOutputPlans plans; - const ModelOutputLaneLines lane_lines; - const ModelOutputRoadEdges road_edges; - const ModelOutputLeads leads; - const ModelOutputMeta meta; - const ModelOutputPose pose; - const ModelOutputWideFromDeviceEuler wide_from_device_euler; - const ModelOutputTemporalPose temporal_pose; - const ModelOutputRoadTransform road_transform; -}; - -constexpr int OUTPUT_SIZE = sizeof(ModelOutput) / sizeof(float); -constexpr int NET_OUTPUT_SIZE = OUTPUT_SIZE + FEATURE_LEN + PAD_SIZE; - -struct PublishState { - std::array disengage_buffer = {}; - std::array prev_brake_5ms2_probs = {}; - std::array prev_brake_3ms2_probs = {}; -}; - -void fill_model_msg(MessageBuilder &msg, float *net_output_data, PublishState &ps, uint32_t vipc_frame_id, uint32_t vipc_frame_id_extra, uint32_t frame_id, float frame_drop, - uint64_t timestamp_eof, uint64_t timestamp_llk, float model_execution_time, const bool nav_enabled, const bool valid); -void fill_pose_msg(MessageBuilder &msg, float *net_outputs, uint32_t vipc_frame_id, uint32_t vipc_dropped_frames, uint64_t timestamp_eof, const bool valid); diff --git a/selfdrive/modeld/parse_model_outputs.py b/selfdrive/modeld/parse_model_outputs.py index faeb12883c..4f84292cb7 100755 --- a/selfdrive/modeld/parse_model_outputs.py +++ b/selfdrive/modeld/parse_model_outputs.py @@ -80,7 +80,7 @@ def parse_outputs(outs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: parse_mdn('pose', outs, in_N=0, out_N=0, out_shape=(POSE_WIDTH,)) parse_mdn('road_transform', outs, in_N=0, out_N=0, out_shape=(POSE_WIDTH,)) parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(POSE_WIDTH,)) - parse_mdn('wide_from_device_euler', outs, in_N=0, out_N=0, out_shape=(POSE_WIDTH // 2,)) + parse_mdn('wide_from_device_euler', outs, in_N=0, out_N=0, out_shape=(WIDE_FROM_DEVICE_WIDTH,)) parse_mdn('lead', outs, in_N=LEAD_MHP_N, out_N=LEAD_MHP_SELECTION, out_shape=(LEAD_TRAJ_LEN,LEAD_WIDTH)) for k in ['lead_prob', 'lane_lines_prob', 'meta']: parse_binary_crossentropy(k, outs)