expose aspect_ratio_sd parameter

Summary:
It is questionable to assign the same std for focal length and aspect ratio, because aspect ratio, being the ratio of fu and fv, should be one order of magnitude smaller than fu and fv themselves.

In practice we have noticed problematic maps built from opensfm BA that have very wrong aspect ratio, causing the camera poses to be inaccurate.

Reviewed By: paulinus

Differential Revision: D78990090

fbshipit-source-id: 65334567a693ab8872e91746942ef011f2e62de6

Author: RuibinMa

annotation_gui_gcp/run_ba.py

@@ -301,6 +301,7 @@ def bundle_with_fixed_images(
     )
     ba.set_internal_parameters_prior_sd(
         config["exif_focal_sd"],
+        config["aspect_ratio_sd"],
         config["principal_point_sd"],
         config["radial_distortion_k1_sd"],
         config["radial_distortion_k2_sd"],

opensfm/config.py

@@ -239,6 +239,8 @@ class OpenSfMConfig:
     reprojection_error_sd: float = 0.004
     # The standard deviation of the exif focal length in log-scale
     exif_focal_sd: float = 0.01
+    # The standard deviation of aspect ratio, i.e. fu/fv, in log-scale
+    aspect_ratio_sd: float = 0.01
     # The standard deviation of the principal point coordinates
     principal_point_sd: float = 0.01
     # The standard deviation of the first radial distortion parameter

opensfm/src/bundle/bundle_adjuster.h

@@ -252,9 +252,10 @@ class BundleAdjuster {
   void SetLinearSolverType(std::string t);
   void SetCovarianceAlgorithmType(std::string t);
 
-  void SetInternalParametersPriorSD(double focal_sd, double c_sd, double k1_sd,
-                                    double k2_sd, double p1_sd, double p2_sd,
-                                    double k3_sd, double k4_sd);
+  void SetInternalParametersPriorSD(double focal_sd, double aspect_ratio_sd,
+                                    double c_sd, double k1_sd, double k2_sd,
+                                    double p1_sd, double p2_sd, double k3_sd,
+                                    double k4_sd);
   void SetRigParametersPriorSD(double rig_translation_sd,
                                double rig_rotation_sd);
 
@@ -324,6 +325,7 @@ class BundleAdjuster {
 
   // Camera parameters prior
   double focal_prior_sd_;
+  double aspect_ratio_prior_sd_;
   double c_prior_sd_;
   double k1_sd_;
   double k2_sd_;

opensfm/src/bundle/pybundle.pyi

@@ -25,6 +25,7 @@ __all__ = [
     "ReconstructionAlignment",
     "RelativeMotion",
     "RelativeRotation",
+    "StaticExtensionLoader",
 ]
 
 class BundleAdjuster:
@@ -36,7 +37,7 @@ class BundleAdjuster:
     def add_absolute_roll(self, arg0: str, arg1: float, arg2: float) -> None: ...
     def add_absolute_tilt(self, arg0: str, arg1: float, arg2: float) -> None: ...
     def add_absolute_up_vector(
-        self, arg0: str, arg1: numpy.ndarray, arg2: float
+        self, arg0: str, arg1: numpy.typing.NDArray, arg2: float
     ) -> None: ...
     def add_camera(
         self,
@@ -49,20 +50,24 @@ class BundleAdjuster:
         self, arg0: str, arg1: str, arg2: float, arg3: float
     ) -> None: ...
     def add_heatmap(
-        self, arg0: str, arg1: List[float], arg2: int, arg3: float
+        self, arg0: str, arg1: list[float], arg2: int, arg3: float
     ) -> None: ...
     def add_linear_motion(
         self, arg0: str, arg1: str, arg2: str, arg3: float, arg4: float, arg5: float
     ) -> None: ...
-    def add_point(self, arg0: str, arg1: numpy.ndarray, arg2: bool) -> None: ...
+    def add_point(self, arg0: str, arg1: numpy.typing.NDArray, arg2: bool) -> None: ...
     def add_point_prior(
-        self, arg0: str, arg1: numpy.ndarray, arg2: numpy.ndarray, arg3: bool
+        self,
+        arg0: str,
+        arg1: numpy.typing.NDArray,
+        arg2: numpy.typing.NDArray,
+        arg3: bool,
     ) -> None: ...
     def add_point_projection_observation(
         self,
         shot: str,
         point: str,
-        observation: numpy.ndarray,
+        observation: numpy.typing.NDArray,
         std_deviation: float,
         depth_prior: Optional[opensfm.pymap.Depth] = None,
     ) -> None: ...
@@ -83,12 +88,16 @@ class BundleAdjuster:
         self,
         arg0: str,
         arg1: opensfm.pygeometry.Pose,
-        arg2: Dict[str, str],
-        arg3: Dict[str, str],
+        arg2: dict[str, str],
+        arg3: dict[str, str],
         arg4: bool,
     ) -> None: ...
     def add_rig_instance_position_prior(
-        self, arg0: str, arg1: numpy.ndarray, arg2: numpy.ndarray, arg3: str
+        self,
+        arg0: str,
+        arg1: numpy.typing.NDArray,
+        arg2: numpy.typing.NDArray,
+        arg3: str,
     ) -> None: ...
     def brief_report(self) -> str: ...
     def full_report(self) -> str: ...
@@ -106,14 +115,15 @@ class BundleAdjuster:
     def set_gauge_fix_shots(self, arg0: str, arg1: str) -> None: ...
     def set_internal_parameters_prior_sd(
         self,
-        arg0: float,
-        arg1: float,
-        arg2: float,
-        arg3: float,
-        arg4: float,
-        arg5: float,
-        arg6: float,
-        arg7: float,
+        focal_sd: float,
+        aspect_ratio_sd: float,
+        c_sd: float,
+        k1_sd: float,
+        k2_sd: float,
+        p1_sd: float,
+        p2_sd: float,
+        k3_sd: float,
+        k4_sd: float,
     ) -> None: ...
     def set_linear_solver_type(self, arg0: str) -> None: ...
     def set_max_num_iterations(self, arg0: int) -> None: ...
@@ -127,11 +137,11 @@ class Point:
     @property
     def id(self) -> str: ...
     @property
-    def p(self) -> numpy.ndarray: ...
+    def p(self) -> numpy.typing.NDArray: ...
     @property
-    def reprojection_errors(self) -> Dict[str, numpy.ndarray]: ...
+    def reprojection_errors(self) -> dict[str, numpy.typing.NDArray]: ...
     @reprojection_errors.setter
-    def reprojection_errors(self, arg0: Dict[str, numpy.ndarray]) -> None: ...
+    def reprojection_errors(self, arg0: dict[str, numpy.typing.NDArray]) -> None: ...
 
 class RAReconstruction:
     def __init__(self) -> None: ...
@@ -314,13 +324,13 @@ class RelativeMotion:
         self,
         arg0: str,
         arg1: str,
-        arg2: numpy.ndarray,
-        arg3: numpy.ndarray,
+        arg2: numpy.typing.NDArray,
+        arg3: numpy.typing.NDArray,
         arg4: float,
         arg5: float,
         arg6: bool,
     ) -> None: ...
-    def set_scale_matrix(self, arg0: numpy.ndarray) -> None: ...
+    def set_scale_matrix(self, arg0: numpy.typing.NDArray) -> None: ...
     @property
     def rig_instance_i(self) -> str: ...
     @rig_instance_i.setter
@@ -331,12 +341,12 @@ class RelativeMotion:
     def rig_instance_j(self, arg0: str) -> None: ...
 
 class RelativeRotation:
-    def __init__(self, arg0: str, arg1: str, arg2: numpy.ndarray) -> None: ...
-    def set_scale_matrix(self, arg0: numpy.ndarray) -> None: ...
+    def __init__(self, arg0: str, arg1: str, arg2: numpy.typing.NDArray) -> None: ...
+    def set_scale_matrix(self, arg0: numpy.typing.NDArray) -> None: ...
     @property
-    def r(self) -> numpy.ndarray: ...
+    def r(self) -> numpy.typing.NDArray: ...
     @r.setter
-    def r(self, arg1: numpy.ndarray) -> None: ...
+    def r(self, arg1: numpy.typing.NDArray) -> None: ...
     @property
     def shot_i(self) -> str: ...
     @shot_i.setter
@@ -345,3 +355,6 @@ class RelativeRotation:
     def shot_j(self) -> str: ...
     @shot_j.setter
     def shot_j(self, arg0: str) -> None: ...
+
+class StaticExtensionLoader:
+    pass

opensfm/src/bundle/python/pybind.cc

@@ -94,7 +94,10 @@ PYBIND11_MODULE(pybundle, m) {
       .def("add_linear_motion", &bundle::BundleAdjuster::AddLinearMotion)
       .def("set_gauge_fix_shots", &bundle::BundleAdjuster::SetGaugeFixShots)
       .def("set_internal_parameters_prior_sd",
-           &bundle::BundleAdjuster::SetInternalParametersPriorSD)
+           &bundle::BundleAdjuster::SetInternalParametersPriorSD,
+           py::arg("focal_sd"), py::arg("aspect_ratio_sd"), py::arg("c_sd"),
+           py::arg("k1_sd"), py::arg("k2_sd"), py::arg("p1_sd"),
+           py::arg("p2_sd"), py::arg("k3_sd"), py::arg("k4_sd"))
       .def("set_compute_covariances",
            &bundle::BundleAdjuster::SetComputeCovariances)
       .def("get_covariance_estimation_valid",

opensfm/src/bundle/src/bundle_adjuster.cc

@@ -25,6 +25,7 @@ BundleAdjuster::BundleAdjuster() {
   SetPointProjectionLossFunction("CauchyLoss", 1.0);
   SetRelativeMotionLossFunction("CauchyLoss", 1.0);
   focal_prior_sd_ = 1;
+  aspect_ratio_prior_sd_ = 1;
   c_prior_sd_ = 1;
   k1_sd_ = 1;
   k2_sd_ = 1;
@@ -48,7 +49,7 @@ geometry::Camera BundleAdjuster::GetDefaultCameraSigma(
   std_dev_map[static_cast<int>(geometry::Camera::Parameters::Focal)] =
       focal_prior_sd_;
   std_dev_map[static_cast<int>(geometry::Camera::Parameters::AspectRatio)] =
-      focal_prior_sd_;
+      aspect_ratio_prior_sd_;
   std_dev_map[static_cast<int>(geometry::Camera::Parameters::Cx)] = c_prior_sd_;
   std_dev_map[static_cast<int>(geometry::Camera::Parameters::Cy)] = c_prior_sd_;
   std_dev_map[static_cast<int>(geometry::Camera::Parameters::K1)] = k1_sd_;
@@ -376,11 +377,11 @@ void BundleAdjuster::SetCovarianceAlgorithmType(std::string t) {
   covariance_algorithm_type_ = t;
 }
 
-void BundleAdjuster::SetInternalParametersPriorSD(double focal_sd, double c_sd,
-                                                  double k1_sd, double k2_sd,
-                                                  double p1_sd, double p2_sd,
-                                                  double k3_sd, double k4_sd) {
+void BundleAdjuster::SetInternalParametersPriorSD(
+    double focal_sd, double aspect_ratio_sd, double c_sd, double k1_sd,
+    double k2_sd, double p1_sd, double p2_sd, double k3_sd, double k4_sd) {
   focal_prior_sd_ = focal_sd;
+  aspect_ratio_prior_sd_ = aspect_ratio_sd;
   c_prior_sd_ = c_sd;
   k1_sd_ = k1_sd;
   k2_sd_ = k2_sd;

opensfm/src/sfm/src/ba_helpers.cc

@@ -255,6 +255,7 @@ py::tuple BAHelpers::BundleLocal(
       config["loss_function_threshold"].cast<double>());
   ba.SetInternalParametersPriorSD(
       config["exif_focal_sd"].cast<double>(),
+      config["aspect_ratio_sd"].cast<double>(),
       config["principal_point_sd"].cast<double>(),
       config["radial_distortion_k1_sd"].cast<double>(),
       config["radial_distortion_k2_sd"].cast<double>(),
@@ -536,6 +537,7 @@ py::dict BAHelpers::BundleShotPoses(
       config["loss_function_threshold"].cast<double>());
   ba.SetInternalParametersPriorSD(
       config["exif_focal_sd"].cast<double>(),
+      config["aspect_ratio_sd"].cast<double>(),
       config["principal_point_sd"].cast<double>(),
       config["radial_distortion_k1_sd"].cast<double>(),
       config["radial_distortion_k2_sd"].cast<double>(),
@@ -728,6 +730,7 @@ py::dict BAHelpers::Bundle(
       config["loss_function_threshold"].cast<double>());
   ba.SetInternalParametersPriorSD(
       config["exif_focal_sd"].cast<double>(),
+      config["aspect_ratio_sd"].cast<double>(),
       config["principal_point_sd"].cast<double>(),
       config["radial_distortion_k1_sd"].cast<double>(),
       config["radial_distortion_k2_sd"].cast<double>(),