152 const std::map<
double, std::vector<std::shared_ptr<ov_core::Feature>>> &used_features_map)
155 static bool wait_for_steady_init =
true;
156 vio_packet.quality = -1;
160 wait_for_steady_init =
true;
166 fprintf(stderr,
"[PUBLISH] VIO in FAILED state - REQUESTING RESET\n");
170 memset(&vio_packet, 0,
sizeof(vio_data_t));
171 vio_packet.magic_number = VIO_MAGIC_NUMBER;
172 vio_packet.timestamp_ns = state->_timestamp * 1e9;
173 vio_packet.quality = -1;
174 vio_packet.state = VIO_STATE_FAILED;
177 pipe_server_write(
SIMPLE_CH, (
char *)&vio_packet,
sizeof(vio_data_t));
181 static int64_t last_failed_msg = 0;
182 int64_t now = _apps_time_monotonic_ns();
183 if (now - last_failed_msg > 1000000000)
185 printf(
"[PUBLISH] Published FAILED packet, reset_requested=%d\n",
187 last_failed_msg = now;
193 vio_packet.magic_number = VIO_MAGIC_NUMBER;
194 vio_packet.timestamp_ns = state->_timestamp * 1e9;
202 Eigen::Matrix<double, 4, 1> q_I_G = state->_imu->quat_fej();
206 Eigen::Matrix3d R_I_G = ov_core::quat_2_Rot(q_I_G);
207 auto RPY = ov_core::rot2rpy(R_I_G);
220 RPY(1) = M_PI - RPY(1);
221 RPY(2) = -M_PI + RPY(2);
222 R_I_G = ov_core::rot_x(RPY(0)) * ov_core::rot_y(RPY(1)) * ov_core::rot_z(RPY(2));
226 grav_vec[2] =
static_cast<float>(-9.81);
232 RPY(1) = -M_PI + RPY(1);
233 RPY(2) = M_PI - RPY(2);
234 R_I_G = ov_core::rot_x(RPY(0)) * ov_core::rot_y(RPY(1)) * ov_core::rot_z(RPY(2));
238 grav_vec[2] =
static_cast<float>(9.81);
244 printf(
"Frame transform is not initialized. Not publishing packet.\n");
248 memcpy(vio_packet.gravity_vector, grav_vec,
sizeof(
float) * 3);
253 Eigen::Matrix<double, 3, 1> v_I_G = ov2frd * state->_imu->vel();
255 Eigen::Matrix<double, 3, 1> p_I_G = ov2frd * (state->_imu->pos());
266 if (!prev_initialized)
268 double yaw = ov_core::rot2rpy(R_I_G)(2);
274 ned_rot_zero = ov_core::rot_z(-yaw);
285 R_I_G = ned_rot_zero * R_I_G;
286 v_I_G = ned_rot_zero * v_I_G;
287 p_I_G = ned_rot_zero * p_I_G;
290 if (
vio_state.load() == VIO_STATE_INITIALIZING)
295 prev_initialized = now_init;
297 for (
int i = 0; i < 3; i++)
299 vio_packet.T_imu_wrt_vio[i] =
static_cast<float>(p_I_G(i));
300 vio_packet.vel_imu_wrt_vio[i] =
static_cast<float>(v_I_G(i));
302 alt_z.store(
static_cast<float>(p_I_G(2)), std::memory_order_release);
303 for (
int i = 0; i < 3; i++)
305 for (
int j = 0; j < 3; j++)
307 vio_packet.R_imu_to_vio[i][j] =
static_cast<float>(R_I_G(i, j));
310 q_I_G = ov_core::rot_2_quat(R_I_G);
312 static int64_t prev_timestamp_ns = 0;
318 for (
int i = 0; i < 3; i++)
320 vio_packet.imu_angular_vel[i] = 0.0f;
323 prev_timestamp_ns = vio_packet.timestamp_ns;
327 double dt = (vio_packet.timestamp_ns - prev_timestamp_ns);
329 Eigen::Matrix<double, 3, 1> ang_vel_imu =
dirtyOmega(past_q_I_G, q_I_G, dt);
331 for (
int i = 0; i < 3; i++)
333 vio_packet.imu_angular_vel[i] =
static_cast<float>(ang_vel_imu(i));
337 prev_timestamp_ns = vio_packet.timestamp_ns;
341 std::vector<std::shared_ptr<ov_type::Type>> statevars;
342 statevars.push_back(state->_imu->p());
343 statevars.push_back(state->_imu->q());
344 statevars.push_back(state->_imu->v());
345 Eigen::Matrix<double, 9, 9> covariance_posori =
346 ov_msckf::StateHelper::get_marginal_covariance(state,
350 vio_packet.pose_covariance[0] =
static_cast<float>(covariance_posori(0, 0));
351 vio_packet.pose_covariance[6] =
static_cast<float>(covariance_posori(1, 1));
352 vio_packet.pose_covariance[11] =
static_cast<float>(covariance_posori(2, 2));
353 vio_packet.pose_covariance[15] =
static_cast<float>(covariance_posori(3, 3));
354 vio_packet.pose_covariance[18] =
static_cast<float>(covariance_posori(4, 4));
355 vio_packet.pose_covariance[20] =
static_cast<float>(covariance_posori(5, 5));
356 vio_packet.velocity_covariance[0] =
static_cast<float>(covariance_posori(6, 6));
357 vio_packet.velocity_covariance[6] =
static_cast<float>(covariance_posori(7, 7));
358 vio_packet.velocity_covariance[11] =
static_cast<float>(covariance_posori(8, 8));
361 Eigen::Matrix3d cam_out = ov_core::quat_2_Rot(state->_calib_IMUtoCAM[0]->quat()).transpose();
362 for (
int i = 0; i < 3; i++)
364 for (
int j = 0; j < 3; j++)
366 vio_packet.R_cam_to_imu[i][j] =
static_cast<float>(cam_out(i, j));
370 Eigen::Vector3d t_cam_wrt_imu = -(ov_core::quat_2_Rot(state->_calib_IMUtoCAM[0]->quat()).transpose() * state->_calib_IMUtoCAM[0]->pos());
371 for (
int i = 0; i < 3; i++)
373 vio_packet.T_cam_wrt_imu[i] =
static_cast<float>(t_cam_wrt_imu(i));
378 if (covariance_posori(3, 3) < 0.0 || covariance_posori(4, 4) < 0.0 || covariance_posori(5, 5) < 0.0)
380 fprintf(stderr,
"ERROR: covariance diagonal went negative\n");
385 static int64_t last_sent_timestamp_ns = 0;
386 if (vio_packet.timestamp_ns < last_sent_timestamp_ns)
391 first_packet =
false;
396 int64_t time_diff = last_sent_timestamp_ns - vio_packet.timestamp_ns;
397 if (time_diff > 1000000)
399 fprintf(stderr,
"WARNING: skipping pose data from the past %ld %ld (diff: %ld ns)\n",
400 vio_packet.timestamp_ns, last_sent_timestamp_ns, time_diff);
405 last_sent_timestamp_ns = vio_packet.timestamp_ns;
408 double V_uncertainty = 0.0;
409 V_uncertainty += covariance_posori(6, 6) * covariance_posori(6, 6);
410 V_uncertainty += covariance_posori(7, 7) * covariance_posori(7, 7);
411 V_uncertainty += covariance_posori(8, 8) * covariance_posori(8, 8);
412 V_uncertainty = sqrt(V_uncertainty);
416 double current_velocity = state->_imu->vel().norm();
419 fprintf(stderr,
"ERROR: exceeded maximum velocity %f vs %f\n",
424 std::unordered_map<size_t, std::shared_ptr<ov_type::Landmark>> SLAM_FEATS = state->_features_SLAM;
428 vio_packet.n_feature_points =
static_cast<uint16_t
>(SLAM_FEATS.size());
431 static int64_t last_good_feat_ts = 0;
432 static int64_t last_good_qual_ts = 0;
433 static bool wait_for_features =
true;
435 static int64_t last_good_state_ns = 0;
438 const uint32_t current_reset_count =
reset_num_counter.load(std::memory_order_acquire);
439 if (last_reset_count != current_reset_count)
443 last_good_state_ns = 0;
444 wait_for_steady_init =
true;
445 last_reset_count = current_reset_count;
446 wait_for_features =
true;
447 last_good_feat_ts = vio_packet.timestamp_ns;
448 last_good_qual_ts = vio_packet.timestamp_ns;
452 printf(
"[QUALITY] VIO reset detected (reset_count=%u), feature/quality tracking reset\n",
453 current_reset_count);
458 if (wait_for_features)
460 vio_error_codes.fetch_and(~ERROR_CODE_NO_FEATURES, std::memory_order_relaxed);
463 last_good_feat_ts = vio_packet.timestamp_ns;
464 wait_for_features =
false;
471 last_good_feat_ts = vio_packet.timestamp_ns;
474 double ts = (vio_packet.timestamp_ns - last_good_feat_ts) * 1e-9;
477 fprintf(stderr,
"ERROR: insufficient features for too long! cur: %d, min_req: %d\n",
480 wait_for_features =
true;
485 static int64_t start_spin_time = 0;
486 static bool spinning_detected =
false;
488 double yawrate = vio_packet.imu_angular_vel[2];
490 fabs(vio_packet.vel_imu_wrt_vio[0]) <= 1.0 &&
491 fabs(vio_packet.vel_imu_wrt_vio[1]) <= 1.0);
493 if (!spinning_in_place)
495 start_spin_time = vio_packet.timestamp_ns;
496 spinning_detected =
false;
498 else if (!spinning_detected)
500 double spin_duration = (vio_packet.timestamp_ns - start_spin_time) * 1e-9;
503 fprintf(stderr,
"ERROR: exceeded spin rate over time threshold %f!\n",
fast_yaw_timeout_s);
505 spinning_detected =
true;
514 double T_uncertainty = 0.0;
515 T_uncertainty += covariance_posori(0, 0) * covariance_posori(0, 0);
516 T_uncertainty += covariance_posori(1, 1) * covariance_posori(1, 1);
517 T_uncertainty += covariance_posori(2, 2) * covariance_posori(2, 2);
518 T_uncertainty = sqrt(T_uncertainty);
520 double R_uncertainty = 0.0;
521 R_uncertainty += covariance_posori(3, 3) * covariance_posori(3, 3);
522 R_uncertainty += covariance_posori(4, 4) * covariance_posori(4, 4);
523 R_uncertainty += covariance_posori(5, 5) * covariance_posori(5, 5);
524 R_uncertainty = sqrt(R_uncertainty);
527 constexpr double max_allowable_cep = 0.15;
528 auto calculate_cep_quality = [max_allowable_cep](
double t_uncertainty) ->
double
530 return std::max(0.0, 100.0 * (1.0 - t_uncertainty / max_allowable_cep));
534 double calculated_quality;
536 const bool throttling_active =
538 vio_state.load(std::memory_order_acquire) == VIO_STATE_OK &&
541 if (!is_during_initialization && !throttling_active)
544 calculated_quality =
calcQuality(used_features_map, SLAM_FEATS, state);
553 calculated_quality = calculate_cep_quality(T_uncertainty);
555 if (
en_debug && is_during_initialization)
557 printf(
"[QUALITY_INIT] VIO initializing - CEP quality: %.1f (features=%d, T_unc=%.4f m)\n",
558 calculated_quality, vio_packet.n_feature_points, T_uncertainty);
560 else if (
en_debug && throttling_active)
562 printf(
"[QUALITY_THROTTLE] Using CEP quality during throttling: %.1f (T_unc=%.4f m, static=%d, acc_no_jerk=%d)\n",
563 calculated_quality, T_uncertainty,
570 calculated_quality = std::max(0.0, std::min(100.0, calculated_quality));
578 static bool qual_check_armed =
false;
581 if (!qual_check_armed)
583 last_good_qual_ts = vio_packet.timestamp_ns;
584 qual_check_armed =
true;
587 if (calculated_quality >= 1)
589 last_good_qual_ts = vio_packet.timestamp_ns;
592 double qual_ts = (vio_packet.timestamp_ns - last_good_qual_ts) * 1e-9;
593 if (qual_ts > ts_threshold)
595 fprintf(stderr,
"ERROR: actual quality was bad for too long!\n");
601 qual_check_armed =
false;
625 static QualityState quality_state = INITIAL;
626 static int consecutive_above_40 = 0;
627 static int consecutive_below_20 = 0;
631 bool should_reset_quality_state = (last_quality_reset_count != current_reset_count) ||
634 if (should_reset_quality_state)
636 quality_state = INITIAL;
637 consecutive_above_40 = 0;
638 consecutive_below_20 = 0;
639 last_quality_reset_count = current_reset_count;
647 qual_src_map_size = 0;
651 printf(
"[QUALITY] RESET to INITIAL state - will report quality as-is (reset_count=%u)\n",
652 current_reset_count);
662 switch (quality_state)
668 consecutive_below_20++;
672 consecutive_below_20 = 0;
673 consecutive_above_40 = 0;
675 printf(
"[QUALITY] Transition: INITIAL → BAD (quality degraded)\n");
680 consecutive_below_20 = 0;
684 consecutive_above_40++;
687 quality_state = GOOD;
688 consecutive_above_40 = 0;
689 consecutive_below_20 = 0;
695 consecutive_above_40 = 0;
703 consecutive_above_40++;
706 quality_state = GOOD;
707 consecutive_above_40 = 0;
708 consecutive_below_20 = 0;
714 consecutive_above_40 = 0;
722 consecutive_below_20++;
726 consecutive_below_20 = 0;
727 consecutive_above_40 = 0;
733 consecutive_below_20 = 0;
740 int reported_quality;
741 switch (quality_state)
749 const double cep_quality = calculate_cep_quality(T_uncertainty);
750 reported_quality =
static_cast<int>(cep_quality);
754 printf(
"[QUALITY_INIT] INITIAL state - CEP quality: %.1f (T_unc=%.4f m) [ZUPT done]\n",
755 cep_quality, T_uncertainty);
760 reported_quality = 0;
764 printf(
"[QUALITY_INIT] INITIAL state - Quality: 0 [waiting for ZUPT]\n");
771 reported_quality = 0;
775 reported_quality =
static_cast<int>(calculated_quality);
780 reported_quality = std::max(0, std::min(100, reported_quality));
783 if (
en_debug && (quality_state != GOOD || reported_quality < 10))
785 const char *state_str = (quality_state == INITIAL) ?
"INITIAL" : (quality_state == BAD) ?
"BAD"
787 printf(
"[QUALITY] State: %s, Reported: %d, Calculated: %.1f, Consec(>40): %d, Consec(≤20): %d\n",
788 state_str, reported_quality, calculated_quality,
789 consecutive_above_40, consecutive_below_20);
794 int quality_for_error_detection =
static_cast<int32_t
>(calculated_quality);
795 if (quality_for_error_detection > 100)
796 quality_for_error_detection = 100;
797 if (quality_for_error_detection < 0)
798 quality_for_error_detection = 0;
801 bool in_grace_period =
false;
804 const int64_t now_ns = _apps_time_monotonic_ns();
807 if ((last_good_state_ns == 0 || vio_packet.n_feature_points <
auto_reset_min_features) && wait_for_steady_init)
809 last_good_state_ns = now_ns;
813 const int64_t dt_ok_ns = now_ns - last_good_state_ns;
816 in_grace_period = (dt_ok_ns < grace_ns && wait_for_steady_init);
834 vio_state.store(VIO_STATE_INITIALIZING, std::memory_order_release);
837 vio_packet.state = VIO_STATE_OK;
841 vio_packet.state = VIO_STATE_INITIALIZING;
848 vio_packet.quality = std::min(reported_quality, 15);
852 printf(
"[STATE] VIO_STATE_INITIALIZING - vio_manager: %s, initialized: %s, quality: %d\n",
862 vio_packet.state = VIO_STATE_INITIALIZING;
863 vio_state.store(VIO_STATE_INITIALIZING, std::memory_order_release);
864 vio_packet.quality = 0;
868 printf(
"[STATE] VIO_STATE_INITIALIZING (resetting) - quality: %d\n", vio_packet.quality);
872 else if (!in_grace_period &&
873 should_auto_reset(state, quality_for_error_detection, vio_packet.n_feature_points,
874 yawrate, current_velocity,
875 vio_packet.vel_imu_wrt_vio[0], vio_packet.vel_imu_wrt_vio[1]))
878 fprintf(stderr,
"WARNING: Auto-reset conditions detected! Quality: %d, Features: %d, V_uncertainty: %f\n",
879 quality_for_error_detection, vio_packet.n_feature_points, V_uncertainty);
881 vio_packet.quality = -1;
882 vio_packet.state = VIO_STATE_FAILED;
883 vio_state.store(VIO_STATE_FAILED, std::memory_order_release);
889 fprintf(stderr,
"[AUTO_RESET] REQUESTING RESET due to auto-reset conditions\n");
895 vio_packet.state = VIO_STATE_OK;
896 vio_state.store(VIO_STATE_OK, std::memory_order_release);
901 if (reported_quality == 0)
903 vio_packet.quality = 0;
908 vio_packet.quality = std::min(reported_quality, 15);
913 printf(
"[GRACE_PERIOD] In grace period: reported_quality=%d, vio_packet.quality=%d\n",
914 reported_quality, vio_packet.quality);
919 wait_for_steady_init =
false;
921 vio_packet.quality = reported_quality;
926 vio_packet.frame = 0;
932 vio_packet.quality = 0;
933 vio_packet.state = VIO_STATE_INITIALIZING;
937 pipe_server_write(
SIMPLE_CH, (
char *)&vio_packet,
sizeof(vio_data_t));
941 ext_vio_data_t ext_vio_packet;
942 ext_vio_packet.v = vio_packet;
943 ext_vio_packet.n_total_features = 0;
945 double current_timestamp = state->_timestamp;
946 auto timestamp_iter = used_features_map.find(current_timestamp);
951 if (timestamp_iter == used_features_map.end() && !used_features_map.empty())
953 timestamp_iter = std::prev(used_features_map.end());
954 current_timestamp = timestamp_iter->first;
957 if (timestamp_iter != used_features_map.end())
959 const auto &features = timestamp_iter->second;
962 std::map<int, std::vector<std::shared_ptr<ov_core::Feature>>> features_by_camera;
963 for (
const auto &feature : features)
966 for (
const auto &cam_meas : feature->timestamps)
968 int cam_id =
static_cast<int>(cam_meas.first);
969 features_by_camera[cam_id].push_back(feature);
973 ov_type::LandmarkRepresentation::Representation msckf_ref = state->_options.feat_rep_msckf;
976 for (
const auto &camera_pair : features_by_camera)
978 int cam_id = camera_pair.first;
979 const auto &cam_features = camera_pair.second;
981 for (
const auto &feature : cam_features)
984 if (feature->timestamps.find(cam_id) == feature->timestamps.end() ||
985 feature->uvs.find(cam_id) == feature->uvs.end())
991 const auto ×tamps = feature->timestamps.at(cam_id);
992 auto time_iter = std::find(timestamps.begin(), timestamps.end(), current_timestamp);
993 if (time_iter == timestamps.end())
998 size_t meas_idx = std::distance(timestamps.begin(), time_iter);
999 const auto &uv_meas = feature->uvs.at(cam_id)[meas_idx];
1002 if (SLAM_FEATS.find(feature->featid) != SLAM_FEATS.end())
1004 if (ext_vio_packet.n_total_features >= VIO_MAX_REPORTED_FEATURES)
1007 int i_global = ext_vio_packet.n_total_features++;
1009 ext_vio_packet.features[i_global].id = feature->featid;
1010 ext_vio_packet.features[i_global].cam_id = cam_id;
1013 ext_vio_packet.features[i_global].pix_loc[0] = uv_meas(0);
1014 ext_vio_packet.features[i_global].pix_loc[1] = uv_meas(1);
1016 Eigen::Vector3d p_FinG;
1017 if (SLAM_FEATS[feature->featid]->_feat_representation == ov_type::LandmarkRepresentation::Representation::GLOBAL_3D)
1019 p_FinG = SLAM_FEATS[feature->featid]->get_xyz(
true);
1020 p_FinG = ov2frd * p_FinG;
1022 else if (SLAM_FEATS[feature->featid]->_feat_representation == ov_type::LandmarkRepresentation::Representation::ANCHORED_MSCKF_INVERSE_DEPTH)
1026 auto &slam_feat = SLAM_FEATS[feature->featid];
1028 Eigen::Vector3d p_FinA = slam_feat->get_xyz(
false);
1031 auto imu_clone = state->_clones_IMU.at(slam_feat->_anchor_clone_timestamp);
1032 auto calib = state->_calib_IMUtoCAM.at(slam_feat->_anchor_cam_id);
1034 Eigen::Matrix3d R_GtoI = imu_clone->Rot();
1035 Eigen::Vector3d p_IinG = imu_clone->pos();
1037 Eigen::Matrix3d R_ItoC = calib->Rot();
1038 Eigen::Vector3d p_IinC = calib->pos();
1041 Eigen::Matrix3d R_GtoC = R_ItoC * R_GtoI;
1042 Eigen::Vector3d p_CinG = p_IinG - R_GtoC.transpose() * p_IinC;
1045 Eigen::Vector3d p_FinG_ov = R_GtoC.transpose() * p_FinA + p_CinG;
1048 p_FinG = ov2frd * p_FinG_ov;
1050 catch (
const std::exception &e)
1052 std::cerr << e.what() <<
'\n';
1057 printf(
"[WARNING] SLAM feat representation: %d not recognized, 3D point locations are likely invalid\n", SLAM_FEATS[feature->featid]->_feat_representation);
1060 ext_vio_packet.features[i_global].tsf[0] = p_FinG[0];
1061 ext_vio_packet.features[i_global].tsf[1] = p_FinG[1];
1062 ext_vio_packet.features[i_global].tsf[2] = p_FinG[2];
1064 ext_vio_packet.features[i_global].point_quality = HIGH;
1069 if (ext_vio_packet.n_total_features >= VIO_MAX_REPORTED_FEATURES)
1072 int i_global = ext_vio_packet.n_total_features++;
1074 ext_vio_packet.features[i_global].id = feature->featid;
1075 ext_vio_packet.features[i_global].cam_id = cam_id;
1078 ext_vio_packet.features[i_global].pix_loc[0] = uv_meas(0);
1079 ext_vio_packet.features[i_global].pix_loc[1] = uv_meas(1);
1081 Eigen::Vector3d p_FinA = feature->p_FinA;
1082 Eigen::Vector3d p_FinG = feature->p_FinG;
1084 if (msckf_ref == ov_type::LandmarkRepresentation::Representation::GLOBAL_3D)
1086 p_FinG = ov2frd * p_FinG;
1088 else if (msckf_ref == ov_type::LandmarkRepresentation::Representation::ANCHORED_MSCKF_INVERSE_DEPTH)
1093 auto imu_clone = state->_clones_IMU.at(feature->anchor_clone_timestamp);
1094 auto calib = state->_calib_IMUtoCAM.at(feature->anchor_cam_id);
1096 Eigen::Matrix3d R_GtoI = imu_clone->Rot();
1097 Eigen::Vector3d p_IinG = imu_clone->pos();
1099 Eigen::Matrix3d R_ItoC = calib->Rot();
1100 Eigen::Vector3d p_IinC = calib->pos();
1103 Eigen::Matrix3d R_GtoC = R_ItoC * R_GtoI;
1104 Eigen::Vector3d p_CinG = p_IinG - R_GtoC.transpose() * p_IinC;
1107 Eigen::Vector3d p_FinG_ov = R_GtoC.transpose() * p_FinA + p_CinG;
1110 p_FinG = ov2frd * p_FinG_ov;
1112 catch (
const std::exception &e)
1114 std::cerr << e.what() <<
'\n';
1119 printf(
"[WARNING] MSCKF feat representation: %s not recognized, 3D point locations are likely invalid\n", ov_type::LandmarkRepresentation::as_string(msckf_ref).c_str());
1121 ext_vio_packet.features[i_global].tsf[0] = p_FinG[0];
1122 ext_vio_packet.features[i_global].tsf[1] = p_FinG[1];
1123 ext_vio_packet.features[i_global].tsf[2] = p_FinG[2];
1125 ext_vio_packet.features[i_global].point_quality = MEDIUM;
1131 pipe_server_write(
EXTENDED_CH, (
char *)&ext_vio_packet,
sizeof(ext_vio_data_t));