#include "selfdrive/modeld/models/driving.h" #include #include #include #include #include #include "common/clutil.h" #include "common/params.h" #include "common/timing.h" #include "common/swaglog.h" #include "common/transformations/orientation.hpp" mat3 update_calibration(float *device_from_calib_euler, bool wide_camera, bool bigmodel_frame) { /* import numpy as np from common.transformations.model import medmodel_frame_from_calib_frame medmodel_frame_from_calib_frame = medmodel_frame_from_calib_frame[:, :3] calib_from_smedmodel_frame = np.linalg.inv(medmodel_frame_from_calib_frame) */ static const auto calib_from_medmodel = (Eigen::Matrix() << 0.00000000e+00, 0.00000000e+00, 1.00000000e+00, 1.09890110e-03, 0.00000000e+00, -2.81318681e-01, -2.25466395e-20, 1.09890110e-03, -5.23076923e-02).finished(); static const auto calib_from_sbigmodel = (Eigen::Matrix() << 0.00000000e+00, 7.31372216e-19, 1.00000000e+00, 2.19780220e-03, 4.11497335e-19, -5.62637363e-01, -6.66298828e-20, 2.19780220e-03, -3.33626374e-01).finished(); static const auto view_from_device = (Eigen::Matrix() << 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0).finished(); Eigen::Vector3d device_from_calib_euler_vec; for (int i=0; i<3; i++) { device_from_calib_euler_vec(i) = device_from_calib_euler[i]; } const auto cam_intrinsics = Eigen::Matrix(wide_camera ? ECAM_INTRINSIC_MATRIX.v : FCAM_INTRINSIC_MATRIX.v); Eigen::Matrix device_from_calib = euler2rot(device_from_calib_euler_vec).cast (); auto calib_from_model = bigmodel_frame ? calib_from_sbigmodel : calib_from_medmodel; auto camera_from_calib = cam_intrinsics * view_from_device * device_from_calib; auto warp_matrix = camera_from_calib * calib_from_model; mat3 transform = {}; for (int i=0; i<3*3; i++) { transform.v[i] = warp_matrix(i / 3, i % 3); } return transform; } void fill_lead(cereal::ModelDataV2::LeadDataV3::Builder lead, const ModelOutputLeads &leads, int t_idx, float prob_t) { std::array lead_t = {0.0, 2.0, 4.0, 6.0, 8.0, 10.0}; const auto &best_prediction = leads.get_best_prediction(t_idx); lead.setProb(sigmoid(leads.prob[t_idx])); lead.setProbTime(prob_t); std::array lead_x, lead_y, lead_v, lead_a; std::array lead_x_std, lead_y_std, lead_v_std, lead_a_std; for (int i=0; i desire_state_softmax; softmax(meta_data.desire_state_prob.array.data(), desire_state_softmax.data(), DESIRE_LEN); std::array desire_pred_softmax; for (int i=0; i lat_long_t = {2, 4, 6, 8, 10}; std::array gas_disengage_sigmoid, brake_disengage_sigmoid, steer_override_sigmoid, brake_3ms2_sigmoid, brake_4ms2_sigmoid, brake_5ms2_sigmoid; for (int i=0; i threshold; } for (int i=0; i FCW_THRESHOLD_3MS2; } auto disengage = meta.initDisengagePredictions(); disengage.setT(to_kj_array_ptr(lat_long_t)); disengage.setGasDisengageProbs(to_kj_array_ptr(gas_disengage_sigmoid)); disengage.setBrakeDisengageProbs(to_kj_array_ptr(brake_disengage_sigmoid)); disengage.setSteerOverrideProbs(to_kj_array_ptr(steer_override_sigmoid)); disengage.setBrake3MetersPerSecondSquaredProbs(to_kj_array_ptr(brake_3ms2_sigmoid)); disengage.setBrake4MetersPerSecondSquaredProbs(to_kj_array_ptr(brake_4ms2_sigmoid)); disengage.setBrake5MetersPerSecondSquaredProbs(to_kj_array_ptr(brake_5ms2_sigmoid)); meta.setEngagedProb(sigmoid(meta_data.engaged_prob)); meta.setDesirePrediction(to_kj_array_ptr(desire_pred_softmax)); meta.setDesireState(to_kj_array_ptr(desire_state_softmax)); meta.setHardBrakePredicted(above_fcw_threshold); } void fill_confidence(cereal::ModelDataV2::Builder &framed, PublishState &ps) { if (framed.getFrameId() % (2*MODEL_FREQ) == 0) { // update every 2s to match predictions interval auto dbps = framed.getMeta().getDisengagePredictions().getBrakeDisengageProbs(); auto dgps = framed.getMeta().getDisengagePredictions().getGasDisengageProbs(); auto dsps = framed.getMeta().getDisengagePredictions().getSteerOverrideProbs(); float any_dp[DISENGAGE_LEN]; float dp_ind[DISENGAGE_LEN]; for (int i = 0; i < DISENGAGE_LEN; i++) { any_dp[i] = 1 - ((1-dbps[i])*(1-dgps[i])*(1-dsps[i])); // any disengage prob } dp_ind[0] = any_dp[0]; for (int i = 0; i < DISENGAGE_LEN-1; i++) { dp_ind[i+1] = (any_dp[i+1] - any_dp[i]) / (1 - any_dp[i]); // independent disengage prob for each 2s slice } // rolling buf for 2, 4, 6, 8, 10s std::memmove(&ps.disengage_buffer[0], &ps.disengage_buffer[DISENGAGE_LEN], sizeof(float) * DISENGAGE_LEN * (DISENGAGE_LEN-1)); std::memcpy(&ps.disengage_buffer[DISENGAGE_LEN * (DISENGAGE_LEN-1)], &dp_ind[0], sizeof(float) * DISENGAGE_LEN); } float score = 0; for (int i = 0; i < DISENGAGE_LEN; i++) { score += ps.disengage_buffer[i*DISENGAGE_LEN+DISENGAGE_LEN-1-i] / DISENGAGE_LEN; } if (score < RYG_GREEN) { framed.setConfidence(cereal::ModelDataV2::ConfidenceClass::GREEN); } else if (score < RYG_YELLOW) { framed.setConfidence(cereal::ModelDataV2::ConfidenceClass::YELLOW); } else { framed.setConfidence(cereal::ModelDataV2::ConfidenceClass::RED); } } template void fill_xyzt(cereal::XYZTData::Builder xyzt, const std::array &t, const std::array &x, const std::array &y, const std::array &z) { xyzt.setT(to_kj_array_ptr(t)); xyzt.setX(to_kj_array_ptr(x)); xyzt.setY(to_kj_array_ptr(y)); xyzt.setZ(to_kj_array_ptr(z)); } template void fill_xyzt(cereal::XYZTData::Builder xyzt, const std::array &t, const std::array &x, const std::array &y, const std::array &z, const std::array &x_std, const std::array &y_std, const std::array &z_std) { fill_xyzt(xyzt, t, x, y, z); xyzt.setXStd(to_kj_array_ptr(x_std)); xyzt.setYStd(to_kj_array_ptr(y_std)); xyzt.setZStd(to_kj_array_ptr(z_std)); } void fill_plan(cereal::ModelDataV2::Builder &framed, const ModelOutputPlanPrediction &plan) { std::array pos_x, pos_y, pos_z; std::array pos_x_std, pos_y_std, pos_z_std; std::array vel_x, vel_y, vel_z; std::array rot_x, rot_y, rot_z; std::array acc_x, acc_y, acc_z; std::array rot_rate_x, rot_rate_y, rot_rate_z; for (int i=0; i &plan_t, const ModelOutputLaneLines &lanes) { std::array left_far_y, left_far_z; std::array left_near_y, left_near_z; std::array right_near_y, right_near_z; std::array right_far_y, right_far_z; for (int j=0; j &plan_t, const ModelOutputRoadEdges &edges) { std::array left_y, left_z; std::array right_y, right_z; for (int j=0; j plan_t; std::fill_n(plan_t.data(), plan_t.size(), NAN); plan_t[0] = 0.0; for (int xidx=1, tidx=0; xidx t_offsets = {0.0, 2.0, 4.0}; for (int i=0; i vipc_frame_id) ? (frame_id - vipc_frame_id) : 0; auto framed = msg.initEvent(valid).initModelV2(); framed.setFrameId(vipc_frame_id); framed.setFrameIdExtra(vipc_frame_id_extra); framed.setFrameAge(frame_age); framed.setFrameDropPerc(frame_drop * 100); framed.setTimestampEof(timestamp_eof); framed.setLocationMonoTime(timestamp_llk); framed.setModelExecutionTime(model_execution_time); framed.setNavEnabled(nav_enabled); if (send_raw_pred) { framed.setRawPredictions(kj::ArrayPtr(net_output_data, NET_OUTPUT_SIZE).asBytes()); } fill_model(framed, *((ModelOutput*) net_output_data), ps); } void fill_pose_msg(MessageBuilder &msg, float *net_output_data, uint32_t vipc_frame_id, uint32_t vipc_dropped_frames, uint64_t timestamp_eof, const bool valid) { const ModelOutput &net_outputs = *((ModelOutput*) net_output_data); const auto &v_mean = net_outputs.pose.velocity_mean; const auto &r_mean = net_outputs.pose.rotation_mean; const auto &t_mean = net_outputs.wide_from_device_euler.mean; const auto &v_std = net_outputs.pose.velocity_std; const auto &r_std = net_outputs.pose.rotation_std; const auto &t_std = net_outputs.wide_from_device_euler.std; const auto &road_transform_trans_mean = net_outputs.road_transform.position_mean; const auto &road_transform_trans_std = net_outputs.road_transform.position_std; auto posenetd = msg.initEvent(valid && (vipc_dropped_frames < 1)).initCameraOdometry(); posenetd.setTrans({v_mean.x, v_mean.y, v_mean.z}); posenetd.setRot({r_mean.x, r_mean.y, r_mean.z}); posenetd.setWideFromDeviceEuler({t_mean.x, t_mean.y, t_mean.z}); posenetd.setRoadTransformTrans({road_transform_trans_mean.x, road_transform_trans_mean.y, road_transform_trans_mean.z}); posenetd.setTransStd({exp(v_std.x), exp(v_std.y), exp(v_std.z)}); posenetd.setRotStd({exp(r_std.x), exp(r_std.y), exp(r_std.z)}); posenetd.setWideFromDeviceEulerStd({exp(t_std.x), exp(t_std.y), exp(t_std.z)}); posenetd.setRoadTransformTransStd({exp(road_transform_trans_std.x), exp(road_transform_trans_std.y), exp(road_transform_trans_std.z)}); posenetd.setTimestampEof(timestamp_eof); posenetd.setFrameId(vipc_frame_id); }