Solve the bug for camera calibration

lantern_scripts/mp4_videos_to_png_frames.py

@@ -25,12 +25,20 @@ if __name__ == '__main__':
         os.makedirs(output_folder, exist_ok=True)
 
         # Run FFmpeg command to convert video to PNG frames
-        command = [
-            "ffmpeg",
-            "-i", video,
-            "-vf", "fps=29.97",
-            f"{output_folder}/output_%04d.png"
-        ]
+        if video_name == 'left_sfm' or video_name == 'right_sfm':
+            command = [
+                "ffmpeg",
+                "-i", video,
+                "-vf", "fps=4.0",
+                f"{output_folder}/output_%04d.png"
+            ]
+        else:
+            command = [
+                "ffmpeg",
+                "-i", video,
+                "-vf", "fps=15.0",
+                f"{output_folder}/output_%04d.png"
+            ]
         subprocess.run(command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
         # only keep frames where mod 10 = 0

nerfstudio/data/dataparsers/nerfstudio_dataparser.py

@@ -61,7 +61,7 @@ class NerfstudioDataParserConfig(DataParserConfig):
     """The method to use for orientation."""
     center_method: Literal["poses", "focus", "none"] = "poses"
     """The method to use to center the poses."""
-    auto_scale_poses: bool = False
+    auto_scale_poses: bool = True
     """Whether to automatically scale the poses to fit in +/- 1 bounding box."""
     train_split_fraction: float = 0.9
     """The fraction of images to use for training. The remaining images are for eval."""

nerfstudio/process_data/openSFM_to_nerfstudio.py

@@ -105,7 +105,7 @@ class OpenSFMToNeRFStudioDataset(BaseConverterToNerfstudioDataset):
     """If True, skips copying and downscaling of images and only runs COLMAP if possible and enabled"""
     is_HDR: bool = False
     """If True, process the .exr files as HDR images."""
-    use_mask: bool = False
+    use_mask: bool = True
     """If True, process the .exr files with mask."""
     use_exposure: bool = False
     """If True, using different exposures."""
@@ -148,6 +148,9 @@ class OpenSFMToNeRFStudioDataset(BaseConverterToNerfstudioDataset):
             frames_names = openSFM_reconstruction[0]['shots'].keys()
             frames_names = sorted(frames_names)
 
+            # Remove GT files from the training sets
+            frames_names = list(set(frames_names) - set( [x for x in frames_names if x.startswith('GT')]))
+            
             # Computing the basis transform matrix
             rot_list = []
             trn_list = []
@@ -209,26 +212,25 @@ class OpenSFMToNeRFStudioDataset(BaseConverterToNerfstudioDataset):
             # Convert OpenSFM coordinates to the NeRFStudio
             camera_dict = {}
             camera_dict["frames"] = []
-            for idx, fname in enumerate(frames_names):
+            for idx, fname in enumerate([x for x in frames_names if x.startswith('left_e1')]):
                 # if fname.startswith("left_sfm"):
-                if fname.startswith("left_e1"):
-                    right_fname = fname.replace("left_e1", "right_e2")
-                    # right_fname = fname.replace("left_sfm", "right_sfm")
-                    shot_data = openSFM_reconstruction[0]['shots'][fname]
-                    left_C2W, left_rvec, left_tvec = opensfm_to_opengl(shot_data)
-                    if valid_idx[idx]:
-                        camera_dict["frames"].append(
-                        {
-                            "file_path": fname.split('.')[0],
-                            "transform_matrix": left_C2W.tolist()
-                        })
-                        
-                        right_C2W = left_C2W @ basis_change
-                        camera_dict["frames"].append(
-                        {
-                            "file_path": right_fname.split('.')[0],
-                            "transform_matrix": right_C2W.tolist()
-                        })
+                right_fname = fname.replace("left_e1", "right_e2")
+                # right_fname = fname.replace("left_sfm", "right_sfm")
+                shot_data = openSFM_reconstruction[0]['shots'][fname]
+                left_C2W, left_rvec, left_tvec = opensfm_to_opengl(shot_data)
+                if valid_idx[idx]:
+                    camera_dict["frames"].append(
+                    {
+                        "file_path": fname.split('.')[0],
+                        "transform_matrix": left_C2W.tolist()
+                    })
+                    
+                    right_C2W = left_C2W @ basis_change
+                    camera_dict["frames"].append(
+                    {
+                        "file_path": right_fname.split('.')[0],
+                        "transform_matrix": right_C2W.tolist()
+                    })
                         
             with open(self.output_dir / 'panoramic_transforms.json', 'w') as f:
                 json.dump(camera_dict, f, indent=4)