diff --git a/selfdrive/common/modeldata.h b/selfdrive/common/modeldata.h index 7bc0647147..7aec15e7be 100644 --- a/selfdrive/common/modeldata.h +++ b/selfdrive/common/modeldata.h @@ -34,6 +34,7 @@ const std::array X_IDXS = { 60.75 , 67.6875, 75. , 82.6875, 90.75 , 99.1875, 108. , 117.1875, 126.75 , 136.6875, 147. , 157.6875, 168.75 , 180.1875, 192.}; +const auto X_IDXS_FLOAT = convert_array_to_type(X_IDXS); #ifdef __cplusplus diff --git a/selfdrive/modeld/models/driving.cc b/selfdrive/modeld/models/driving.cc index bab7038b32..a6877a1c4d 100644 --- a/selfdrive/modeld/models/driving.cc +++ b/selfdrive/modeld/models/driving.cc @@ -12,40 +12,6 @@ #include "selfdrive/common/params.h" #include "selfdrive/common/timing.h" -constexpr int DESIRE_PRED_SIZE = 32; -constexpr int OTHER_META_SIZE = 48; -constexpr int NUM_META_INTERVALS = 5; -constexpr int META_STRIDE = 7; - -constexpr int PLAN_MHP_VALS = 15*33; -constexpr int PLAN_MHP_SELECTION = 1; -constexpr int PLAN_MHP_GROUP_SIZE = (2*PLAN_MHP_VALS + PLAN_MHP_SELECTION); - -constexpr int LEAD_MHP_N = 2; -constexpr int LEAD_TRAJ_LEN = 6; -constexpr int LEAD_PRED_DIM = 4; -constexpr int LEAD_MHP_VALS = LEAD_PRED_DIM*LEAD_TRAJ_LEN; -constexpr int LEAD_MHP_SELECTION = 3; -constexpr int LEAD_MHP_GROUP_SIZE = (2*LEAD_MHP_VALS + LEAD_MHP_SELECTION); - -constexpr int POSE_SIZE = 12; - -constexpr int PLAN_IDX = 0; -constexpr int LL_IDX = PLAN_IDX + PLAN_MHP_N*PLAN_MHP_GROUP_SIZE; -constexpr int LL_PROB_IDX = LL_IDX + 4*2*2*33; -constexpr int RE_IDX = LL_PROB_IDX + 8; -constexpr int LEAD_IDX = RE_IDX + 2*2*2*33; -constexpr int LEAD_PROB_IDX = LEAD_IDX + LEAD_MHP_N*(LEAD_MHP_GROUP_SIZE); -constexpr int DESIRE_STATE_IDX = LEAD_PROB_IDX + 3; -constexpr int META_IDX = DESIRE_STATE_IDX + DESIRE_LEN; -constexpr int POSE_IDX = META_IDX + OTHER_META_SIZE + DESIRE_PRED_SIZE; -constexpr int OUTPUT_SIZE = POSE_IDX + POSE_SIZE; -#ifdef TEMPORAL - constexpr int TEMPORAL_SIZE = 512; -#else - constexpr int TEMPORAL_SIZE = 0; -#endif - constexpr float FCW_THRESHOLD_5MS2_HIGH = 0.15; constexpr float FCW_THRESHOLD_5MS2_LOW = 0.05; constexpr float FCW_THRESHOLD_3MS2 = 0.7; @@ -63,15 +29,12 @@ constexpr const kj::ArrayPtr to_kj_array_ptr(const std::array void model_init(ModelState* s, cl_device_id device_id, cl_context context) { s->frame = new ModelFrame(device_id, context); - constexpr int output_size = OUTPUT_SIZE + TEMPORAL_SIZE; - s->output.resize(output_size); - #ifdef USE_THNEED - s->m = std::make_unique("../../models/supercombo.thneed", &s->output[0], output_size, USE_GPU_RUNTIME); + s->m = std::make_unique("../../models/supercombo.thneed", &s->output[0], NET_OUTPUT_SIZE, USE_GPU_RUNTIME); #elif USE_ONNX_MODEL - s->m = std::make_unique("../../models/supercombo.onnx", &s->output[0], output_size, USE_GPU_RUNTIME); + s->m = std::make_unique("../../models/supercombo.onnx", &s->output[0], NET_OUTPUT_SIZE, USE_GPU_RUNTIME); #else - s->m = std::make_unique("../../models/supercombo.dlc", &s->output[0], output_size, USE_GPU_RUNTIME); + s->m = std::make_unique("../../models/supercombo.dlc", &s->output[0], NET_OUTPUT_SIZE, USE_GPU_RUNTIME); #endif #ifdef TEMPORAL @@ -106,22 +69,22 @@ ModelDataRaw model_eval_frame(ModelState* s, cl_mem yuv_cl, int width, int heigh } #endif - //for (int i = 0; i < OUTPUT_SIZE + TEMPORAL_SIZE; i++) { printf("%f ", s->output[i]); } printf("\n"); + //for (int i = 0; i < NET_OUTPUT_SIZE; i++) { printf("%f ", s->output[i]); } printf("\n"); // if getInputBuf is not NULL, net_input_buf will be auto net_input_buf = s->frame->prepare(yuv_cl, width, height, transform, static_cast(s->m->getInputBuf())); s->m->execute(net_input_buf, s->frame->buf_size); // net outputs - ModelDataRaw net_outputs; - net_outputs.plan = (ModelDataRawPlan*)&s->output[PLAN_IDX]; - net_outputs.lane_lines = &s->output[LL_IDX]; - net_outputs.lane_lines_prob = &s->output[LL_PROB_IDX]; - net_outputs.road_edges = &s->output[RE_IDX]; - net_outputs.lead = &s->output[LEAD_IDX]; - net_outputs.lead_prob = &s->output[LEAD_PROB_IDX]; - net_outputs.meta = &s->output[DESIRE_STATE_IDX]; - net_outputs.pose = (ModelDataRawPose*)&s->output[POSE_IDX]; + ModelDataRaw net_outputs { + .plan = (ModelDataRawPlans*)&s->output[PLAN_IDX], + .lane_lines = (ModelDataRawLaneLines*)&s->output[LL_IDX], + .road_edges = (ModelDataRawRoadEdges*)&s->output[RE_IDX], + .lead = &s->output[LEAD_IDX], + .lead_prob = &s->output[LEAD_PROB_IDX], + .meta = &s->output[DESIRE_STATE_IDX], + .pose = (ModelDataRawPose*)&s->output[POSE_IDX], + }; return net_outputs; } @@ -144,7 +107,6 @@ static const float *get_lead_data(const float *lead, int t_offset) { return get_best_data(lead, LEAD_MHP_N, LEAD_MHP_GROUP_SIZE, LEAD_MHP_GROUP_SIZE - LEAD_MHP_SELECTION + t_offset); } - void fill_sigmoid(const float *input, float *output, int len, int stride) { for (int i=0; i pos_x, pos_y, pos_z; std::array pos_x_std, pos_y_std, pos_z_std; @@ -324,10 +250,69 @@ void fill_plan(cereal::ModelDataV2::Builder &framed, const ModelDataRawPlanPath fill_xyzt(framed.initOrientationRate(), T_IDXS_FLOAT, rot_rate_x, rot_rate_y, rot_rate_z); } +void fill_lane_lines(cereal::ModelDataV2::Builder &framed, const std::array &plan_t_arr, + const ModelDataRawLaneLines &lanes) { + std::array left_far_y, left_far_z; + std::array left_near_y, left_near_z; + std::array right_near_y, right_near_z; + std::array right_far_y, right_far_z; + for (int j=0; j &plan_t_arr, + const ModelDataRawRoadEdges &edges) { + std::array left_y, left_z; + std::array right_y, right_z; + for (int j=0; jbest_plan(); - float plan_t_arr[TRAJECTORY_SIZE]; - std::fill_n(plan_t_arr, TRAJECTORY_SIZE, NAN); + auto best_plan = net_outputs.plan->get_best_plan(); + std::array plan_t_arr; + std::fill_n(plan_t_arr.data(), plan_t_arr.size(), NAN); plan_t_arr[0] = 0.0; for (int xidx=1, tidx=0; xidx path; - constexpr const ModelDataRawPlanPath &best_plan() const { + constexpr const ModelDataRawPlanPath &get_best_plan() const { int max_idx = 0; for (int i = 1; i < path.size(); i++) { if (path[i].prob > path[max_idx].prob) { @@ -57,7 +97,48 @@ struct ModelDataRawPlan { return path[max_idx]; } }; -static_assert(sizeof(ModelDataRawPlan) == sizeof(ModelDataRawPlanPath)*PLAN_MHP_N); +static_assert(sizeof(ModelDataRawPlans) == sizeof(ModelDataRawPlanPath)*PLAN_MHP_N); + +struct ModelDataRawLinesXY { + std::array left_far; + std::array left_near; + std::array right_near; + std::array right_far; +}; +static_assert(sizeof(ModelDataRawLinesXY) == sizeof(ModelDataRawYZ)*TRAJECTORY_SIZE*4); + +struct ModelDataRawLineProbVal { + float val_deprecated; + float val; +}; +static_assert(sizeof(ModelDataRawLineProbVal) == sizeof(float)*2); + +struct ModelDataRawLinesProb { + ModelDataRawLineProbVal left_far; + ModelDataRawLineProbVal left_near; + ModelDataRawLineProbVal right_near; + ModelDataRawLineProbVal right_far; +}; +static_assert(sizeof(ModelDataRawLinesProb) == sizeof(ModelDataRawLineProbVal)*4); + +struct ModelDataRawLaneLines { + ModelDataRawLinesXY mean; + ModelDataRawLinesXY std; + ModelDataRawLinesProb prob; +}; +static_assert(sizeof(ModelDataRawLaneLines) == (sizeof(ModelDataRawLinesXY)*2) + sizeof(ModelDataRawLinesProb)); + +struct ModelDataRawEdgessXY { + std::array left; + std::array right; +}; +static_assert(sizeof(ModelDataRawEdgessXY) == sizeof(ModelDataRawYZ)*TRAJECTORY_SIZE*2); + +struct ModelDataRawRoadEdges { + ModelDataRawEdgessXY mean; + ModelDataRawEdgessXY std; +}; +static_assert(sizeof(ModelDataRawRoadEdges) == (sizeof(ModelDataRawEdgessXY)*2)); struct ModelDataRawPose { ModelDataRawXYZ velocity_mean; @@ -68,21 +149,21 @@ struct ModelDataRawPose { static_assert(sizeof(ModelDataRawPose) == sizeof(ModelDataRawXYZ)*4); struct ModelDataRaw { - ModelDataRawPlan *plan; - float *lane_lines; - float *lane_lines_prob; - float *road_edges; - float *lead; - float *lead_prob; - float *desire_state; - float *meta; - float *desire_pred; - ModelDataRawPose *pose; + const ModelDataRawPlans *const plan; + const ModelDataRawLaneLines *const lane_lines; + const ModelDataRawRoadEdges *const road_edges; + const float *const lead; + const float *const lead_prob; + const float *const desire_state; + const float *const meta; + const float *const desire_pred; + const ModelDataRawPose *const pose; }; -typedef struct ModelState { +// TODO: convert remaining arrays to std::array and update model runners +struct ModelState { ModelFrame *frame; - std::vector output; + std::array output = {}; std::unique_ptr m; #ifdef DESIRE float prev_desire[DESIRE_LEN] = {}; @@ -91,7 +172,7 @@ typedef struct ModelState { #ifdef TRAFFIC_CONVENTION float traffic_convention[TRAFFIC_CONVENTION_LEN] = {}; #endif -} ModelState; +}; void model_init(ModelState* s, cl_device_id device_id, cl_context context); ModelDataRaw model_eval_frame(ModelState* s, cl_mem yuv_cl, int width, int height,