#include "selfdrive/modeld/models/driving.h" #include #include #include #include #include #include "selfdrive/common/clutil.h" #include "selfdrive/common/params.h" #include "selfdrive/common/timing.h" constexpr int DESIRE_PRED_SIZE = 32; constexpr int OTHER_META_SIZE = 48; constexpr int NUM_META_INTERVALS = 5; constexpr int META_STRIDE = 7; constexpr int PLAN_MHP_VALS = 15*33; constexpr int PLAN_MHP_SELECTION = 1; constexpr int PLAN_MHP_GROUP_SIZE = (2*PLAN_MHP_VALS + PLAN_MHP_SELECTION); constexpr int LEAD_MHP_N = 2; constexpr int LEAD_TRAJ_LEN = 6; constexpr int LEAD_PRED_DIM = 4; constexpr int LEAD_MHP_VALS = LEAD_PRED_DIM*LEAD_TRAJ_LEN; constexpr int LEAD_MHP_SELECTION = 3; constexpr int LEAD_MHP_GROUP_SIZE = (2*LEAD_MHP_VALS + LEAD_MHP_SELECTION); constexpr int POSE_SIZE = 12; constexpr int PLAN_IDX = 0; constexpr int LL_IDX = PLAN_IDX + PLAN_MHP_N*PLAN_MHP_GROUP_SIZE; constexpr int LL_PROB_IDX = LL_IDX + 4*2*2*33; constexpr int RE_IDX = LL_PROB_IDX + 8; constexpr int LEAD_IDX = RE_IDX + 2*2*2*33; constexpr int LEAD_PROB_IDX = LEAD_IDX + LEAD_MHP_N*(LEAD_MHP_GROUP_SIZE); constexpr int DESIRE_STATE_IDX = LEAD_PROB_IDX + 3; constexpr int META_IDX = DESIRE_STATE_IDX + DESIRE_LEN; constexpr int POSE_IDX = META_IDX + OTHER_META_SIZE + DESIRE_PRED_SIZE; constexpr int OUTPUT_SIZE = POSE_IDX + POSE_SIZE; #ifdef TEMPORAL constexpr int TEMPORAL_SIZE = 512; #else constexpr int TEMPORAL_SIZE = 0; #endif constexpr float FCW_THRESHOLD_5MS2_HIGH = 0.15; constexpr float FCW_THRESHOLD_5MS2_LOW = 0.05; constexpr float FCW_THRESHOLD_3MS2 = 0.7; float prev_brake_5ms2_probs[5] = {0,0,0,0,0}; float prev_brake_3ms2_probs[3] = {0,0,0}; // #define DUMP_YUV template constexpr const kj::ArrayPtr to_kj_array_ptr(const std::array &arr) { return kj::ArrayPtr(arr.data(), arr.size()); } void model_init(ModelState* s, cl_device_id device_id, cl_context context) { s->frame = new ModelFrame(device_id, context); constexpr int output_size = OUTPUT_SIZE + TEMPORAL_SIZE; s->output.resize(output_size); #ifdef USE_THNEED s->m = std::make_unique("../../models/supercombo.thneed", &s->output[0], output_size, USE_GPU_RUNTIME); #elif USE_ONNX_MODEL s->m = std::make_unique("../../models/supercombo.onnx", &s->output[0], output_size, USE_GPU_RUNTIME); #else s->m = std::make_unique("../../models/supercombo.dlc", &s->output[0], output_size, USE_GPU_RUNTIME); #endif #ifdef TEMPORAL s->m->addRecurrent(&s->output[OUTPUT_SIZE], TEMPORAL_SIZE); #endif #ifdef DESIRE s->m->addDesire(s->pulse_desire, DESIRE_LEN); #endif #ifdef TRAFFIC_CONVENTION const int idx = Params().getBool("IsRHD") ? 1 : 0; s->traffic_convention[idx] = 1.0; s->m->addTrafficConvention(s->traffic_convention, TRAFFIC_CONVENTION_LEN); #endif } ModelDataRaw model_eval_frame(ModelState* s, cl_mem yuv_cl, int width, int height, const mat3 &transform, float *desire_in) { #ifdef DESIRE if (desire_in != NULL) { for (int i = 1; i < DESIRE_LEN; i++) { // Model decides when action is completed // so desire input is just a pulse triggered on rising edge if (desire_in[i] - s->prev_desire[i] > .99) { s->pulse_desire[i] = desire_in[i]; } else { s->pulse_desire[i] = 0.0; } s->prev_desire[i] = desire_in[i]; } } #endif //for (int i = 0; i < OUTPUT_SIZE + TEMPORAL_SIZE; i++) { printf("%f ", s->output[i]); } printf("\n"); // if getInputBuf is not NULL, net_input_buf will be auto net_input_buf = s->frame->prepare(yuv_cl, width, height, transform, static_cast(s->m->getInputBuf())); s->m->execute(net_input_buf, s->frame->buf_size); // net outputs ModelDataRaw net_outputs; net_outputs.plan = (ModelDataRawPlan*)&s->output[PLAN_IDX]; net_outputs.lane_lines = &s->output[LL_IDX]; net_outputs.lane_lines_prob = &s->output[LL_PROB_IDX]; net_outputs.road_edges = &s->output[RE_IDX]; net_outputs.lead = &s->output[LEAD_IDX]; net_outputs.lead_prob = &s->output[LEAD_PROB_IDX]; net_outputs.meta = &s->output[DESIRE_STATE_IDX]; net_outputs.pose = (ModelDataRawPose*)&s->output[POSE_IDX]; return net_outputs; } void model_free(ModelState* s) { delete s->frame; } static const float *get_best_data(const float *data, int size, int group_size, int weight_idx) { int max_idx = 0; for (int i = 1; i < size; i++) { if (data[i * group_size + weight_idx] > data[max_idx * group_size + weight_idx]) { max_idx = i; } } return &data[max_idx * group_size]; } static const float *get_lead_data(const float *lead, int t_offset) { return get_best_data(lead, LEAD_MHP_N, LEAD_MHP_GROUP_SIZE, LEAD_MHP_GROUP_SIZE - LEAD_MHP_SELECTION + t_offset); } void fill_sigmoid(const float *input, float *output, int len, int stride) { for (int i=0; i threshold; } for (int i=0; i<3; i++) { above_fcw_threshold = above_fcw_threshold && prev_brake_3ms2_probs[i] > FCW_THRESHOLD_3MS2; } auto disengage = meta.initDisengagePredictions(); disengage.setT({2,4,6,8,10}); disengage.setGasDisengageProbs(gas_disengage_sigmoid); disengage.setBrakeDisengageProbs(brake_disengage_sigmoid); disengage.setSteerOverrideProbs(steer_override_sigmoid); disengage.setBrake3MetersPerSecondSquaredProbs(brake_3ms2_sigmoid); disengage.setBrake4MetersPerSecondSquaredProbs(brake_4ms2_sigmoid); disengage.setBrake5MetersPerSecondSquaredProbs(brake_5ms2_sigmoid); meta.setEngagedProb(sigmoid(meta_data[DESIRE_LEN])); meta.setDesirePrediction(desire_pred_softmax); meta.setDesireState(desire_state_softmax); meta.setHardBrakePredicted(above_fcw_threshold); } template void fill_xyzt(cereal::ModelDataV2::XYZTData::Builder xyzt, const std::array &t, const std::array &x, const std::array &y, const std::array &z) { xyzt.setT(to_kj_array_ptr(t)); xyzt.setX(to_kj_array_ptr(x)); xyzt.setY(to_kj_array_ptr(y)); xyzt.setZ(to_kj_array_ptr(z)); } template void fill_xyzt(cereal::ModelDataV2::XYZTData::Builder xyzt, const std::array &t, const std::array &x, const std::array &y, const std::array &z, const std::array &x_std, const std::array &y_std, const std::array &z_std) { fill_xyzt(xyzt, t, x, y, z); xyzt.setXStd(to_kj_array_ptr(x_std)); xyzt.setYStd(to_kj_array_ptr(y_std)); xyzt.setZStd(to_kj_array_ptr(z_std)); } void fill_xyzt(cereal::ModelDataV2::XYZTData::Builder xyzt, const float *data, int columns, float *plan_t_arr = nullptr, bool fill_std = false) { float x_arr[TRAJECTORY_SIZE] = {}; float y_arr[TRAJECTORY_SIZE] = {}; float z_arr[TRAJECTORY_SIZE] = {}; float x_std_arr[TRAJECTORY_SIZE]; float y_std_arr[TRAJECTORY_SIZE]; float z_std_arr[TRAJECTORY_SIZE]; float t_arr[TRAJECTORY_SIZE]; for (int i=0; i pos_x, pos_y, pos_z; std::array pos_x_std, pos_y_std, pos_z_std; std::array vel_x, vel_y, vel_z; std::array rot_x, rot_y, rot_z; std::array rot_rate_x, rot_rate_y, rot_rate_z; for(int i=0; ibest_plan(); float plan_t_arr[TRAJECTORY_SIZE]; std::fill_n(plan_t_arr, TRAJECTORY_SIZE, NAN); plan_t_arr[0] = 0.0; for (int xidx=1, tidx=0; xidx raw_pred) { const uint32_t frame_age = (frame_id > vipc_frame_id) ? (frame_id - vipc_frame_id) : 0; MessageBuilder msg; auto framed = msg.initEvent().initModelV2(); framed.setFrameId(vipc_frame_id); framed.setFrameAge(frame_age); framed.setFrameDropPerc(frame_drop * 100); framed.setTimestampEof(timestamp_eof); framed.setModelExecutionTime(model_execution_time); if (send_raw_pred) { framed.setRawPredictions(raw_pred.asBytes()); } fill_model(framed, net_outputs); pm.send("modelV2", msg); } void posenet_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t vipc_dropped_frames, const ModelDataRaw &net_outputs, uint64_t timestamp_eof) { MessageBuilder msg; auto v_mean = net_outputs.pose->velocity_mean; auto r_mean = net_outputs.pose->rotation_mean; auto v_std = net_outputs.pose->velocity_std; auto r_std = net_outputs.pose->rotation_std; auto posenetd = msg.initEvent(vipc_dropped_frames < 1).initCameraOdometry(); posenetd.setTrans({v_mean.x, v_mean.y, v_mean.z}); posenetd.setRot({r_mean.x, r_mean.y, r_mean.z}); posenetd.setTransStd({exp(v_std.x), exp(v_std.y), exp(v_std.z)}); posenetd.setRotStd({exp(r_std.x), exp(r_std.y), exp(r_std.z)}); posenetd.setTimestampEof(timestamp_eof); posenetd.setFrameId(vipc_frame_id); pm.send("cameraOdometry", msg); }