![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
+ ""
]
},
{
@@ -63,31 +63,31 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "RGr33zHaHak0",
- "outputId": "14f33da4-1e52-4784-e4e5-79e5d884fb8f"
+ "outputId": "2641dce6-cd57-4d31-d8b1-860bc1adf17f",
+ "cellView": "form"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
- "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
- "\u001b[0m⏬ Downloading https://github.com/jaimergp/miniforge/releases/latest/download/Mambaforge-colab-Linux-x86_64.sh...\n",
+ "⏬ Downloading https://github.com/jaimergp/miniforge/releases/latest/download/Mambaforge-colab-Linux-x86_64.sh...\n",
"📦 Installing...\n",
"📌 Adjusting configuration...\n",
"🩹 Patching environment...\n",
- "⏲ Done in 0:00:18\n",
+ "⏲ Done in 0:00:27\n",
"🔁 Restarting kernel...\n"
]
}
],
"source": [
- "#@title # Install Conda (requires runtime restart) { vertical-output: true, display-mode: \"form\" }\n",
+ "#@markdown '))\n", " display(HTML('
Note: This may take time, especially on hires inputs, so we recommend to download dataset after creation.'))\n",
@@ -199,19 +192,23 @@
" !ns-process-data video --data $video_path --output-dir /content/data/nerfstudio/custom_data/\n",
"\n",
" scene = \"custom_data\"\n",
+ "else:\n",
+ " %cd /content/\n",
+ " !ns-download-data --dataset=nerfstudio --capture=$scene\n",
+ "\n",
"print(\"Data Processing Succeeded!\")"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 928
},
"id": "VoKDxqEcjmfC",
- "outputId": "d6c97ce9-f77a-4026-d6f3-328918435bb0",
+ "outputId": "d2919aa4-96dd-4e50-829f-289e4208882b",
"cellView": "form"
},
"outputs": [
@@ -221,9 +218,9 @@
"text": [
"/content\n",
"\u001b[K\u001b[?25h/tools/node/bin/lt -> /tools/node/lib/node_modules/localtunnel/bin/lt.js\n",
- "+ localtunnel@2.0.2\n",
- "updated 1 package in 1.523s\n",
- "https://viewer.nerf.studio/?websocket_url=wss://fifty-jars-return-34-124-199-158.loca.lt\n",
+ "\u001b[K\u001b[?25h+ localtunnel@2.0.2\n",
+ "added 22 packages from 22 contributors in 2.07s\n",
+ "https://viewer.nerf.studio/?websocket_url=wss://cyan-facts-matter-34-91-1-218.loca.lt\n",
"You may need to click Refresh Page after you start training!\n"
]
},
@@ -231,14 +228,14 @@
"output_type": "execute_result",
"data": {
"text/plain": [
- "Set up and Start Viewer
\n",
"\n",
"%cd /content\n",
"\n",
@@ -289,7 +286,7 @@
},
"outputs": [],
"source": [
- "#@title # Start Training { vertical-output: true }\n",
+ "#@markdown Start Training
\n",
"\n",
"%cd /content\n",
"if os.path.exists(f\"data/nerfstudio/{scene}/transforms.json\"):\n",
@@ -386,4 +383,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
-}
\ No newline at end of file
+}
diff --git a/docs/quickstart/custom_dataset.md b/docs/quickstart/custom_dataset.md
index 903c8df31..1c4f05722 100644
--- a/docs/quickstart/custom_dataset.md
+++ b/docs/quickstart/custom_dataset.md
@@ -1,31 +1,46 @@
# Using custom data
-Training model on existing datasets is only so fun. If you would like to train on self captured data you will need to process the data into an existing format. Specifically we need to know the camera poses for each image. [COLMAP](https://github.com/colmap/colmap) is a standard tool for extracting poses. It is possible to use other methods like [SLAM](https://en.wikipedia.org/wiki/Simultaneous_localization_and_mapping) or hardware recorded poses. We intend to add documentation for these other methods in the future.
+Training model on existing datasets is only so fun. If you would like to train on self captured data you will need to process the data into the nerfstudio format. Specifically we need to know the camera poses for each image.
-## Nerfstudio dataset
-
-To assist running on custom data we have a script that will process a video or folder of images into a format that is compatible with nerfstudio. We use [COLMAP](https://colmap.github.io) and [FFmpeg](https://ffmpeg.org/download.html) in our data processing script, please have these installed. We have provided a quickstart to installing COLMAP below, FFmpeg can be downloaded from [here](https://ffmpeg.org/download.html)
-
-To process your data run:
+To process your own data run:
```bash
-ns-process-data {video,images,insta360,record3d} --data {DATA_PATH} --output-dir {PROCESSED_DATA_DIR}
+ns-process-data {video,images,polycam,insta360,record3d} --data {DATA_PATH} --output-dir {PROCESSED_DATA_DIR}
```
A full set of arguments can be found {doc}`here`.
+We Currently support the following custom data types:
+| Data | Requirements | Preprocessing Speed |
+| -------- | ----------------------- | ------------------- |
+| [Images](images_and_video) | COLMAP | 🐢 |
+| [Video](images_and_video) | COLMAP | 🐢 |
+| [Polycam](polycam) | LiDAR iOS Device | 🐇 |
+| [Record3D](record3d) | LiDAR iOS Device | 🐇 |
+| Insta360 | 2 Sensor camera, COLMAP | 🐢 |
+
+(images_and_video)=
+
+## Images or Video
+
+To assist running on custom data we have a script that will process a video or folder of images into a format that is compatible with nerfstudio. We use [COLMAP](https://colmap.github.io) and [FFmpeg](https://ffmpeg.org/download.html) in our data processing script, please have these installed. We have provided a quickstart to installing COLMAP below, FFmpeg can be downloaded from [here](https://ffmpeg.org/download.html)
+
:::{admonition} Tip
:class: info
- COLMAP can be finicky. Try your best to capture overlapping, non-blurry images.
:::
-### Training on your data
-
-Simply specify that you are using the `nerfstudio` dataparser and point the data directory to your processed data.
+### Processing Data
```bash
-ns-train nerfacto --data {PROCESSED_DATA_DIR} nerfstudio-data
+ns-process-data {images, video} --data {DATA_PATH} --output-dir {PROCESSED_DATA_DIR}
+```
+
+### Training on your data
+
+```bash
+ns-train nerfacto --data {PROCESSED_DATA_DIR}
```
### Installing COLMAP
@@ -108,6 +123,59 @@ cd vcpkg
:::::
::::::
+(polycam)=
+
+## Polycam Capture
+
+Nerfstudio can also be trained directly from captures from the [Polycam app](https://poly.cam//). This avoids the need to use COLMAP. Polycam's poses are globally optimized which make them more robust to drift (an issue with ARKit or SLAM methods).
+
+To get the best results, try to reduce motion blur as much as possible and try to view the target from as many viewpoinrts as possible. Polycam recommends having good lighting and moving the camera slowly if using auto mode. Or, even better, use the manual shutter mode to capture less blurry images.
+
+:::{admonition} Note
+:class: info
+A LiDAR enabled iPhone or iPad is necessary.
+:::
+
+### Setting up Polycam
+
+```{image} imgs/polycam_settings.png
+:width: 200
+:align: center
+:alt: polycam settings
+```
+
+Devoloper settings must be enabled in Polycam. To do this, navigate to the settings screen and select `Developer mode`. Note that this will only apply for future captures, you will not be able to process existing captures with nerfstudio.
+
+### Process data
+
+```{image} imgs/polycam_export.png
+:width: 400
+:align: center
+:alt: polycam export options
+```
+
+0. Capture data in LiDAR or Room mode.
+
+1. Tap `Process` to process the data in the Polycam app.
+
+2. Navigate to the export app pane.
+
+3. Select `raw data` to export a `.zip` file.
+
+4. Convert the Polycam data into the nerfstudio format using the following command:
+
+```bash
+ns-process-data polycam --data {OUTPUT_FILE.zip} --output-dir {output directory}
+```
+
+5. Train the model using the following command:
+
+```bash
+ns-train nerfacto --data {output directory}
+```
+
+(record3d)=
+
## Record3D Capture
Nerfstudio can also be trained directly from >=iPhone 12 Pro captures from the [Record3D app](https://record3d.app/). This uses the iPhone's LiDAR sensors to calculate camera poses, so COLMAP is not needed.
@@ -125,20 +193,14 @@ At a high level, you can follow these 3 steps:
2. Then, move the exported capture folder from your iPhone to your computer.
+3. Convert the data to the nerfstudio format.
+
+```bash
+ns-process-data record3d --data {data directory} --output-dir {output directory}
+```
+
3. Train with nerfstudio!
```
-ns-train nerfacto --data {RECORD3D_CAPTURE_DIR/EXR_RGBD} record3d-data
-```
-
-We provide some example recordings for you to try out and to see the correct formatting.
-
-```shell
-ns-download-data --dataset record3d
-```
-
-This will download data to `data/record3d/bear`. Then you can train, and don't forget to open up the viewer.
-
-```shell
-ns-train nerfacto record3d-data --data data/record3d/bear
+ns-train nerfacto --data {output directory}
```
diff --git a/docs/quickstart/imgs/polycam_export.png b/docs/quickstart/imgs/polycam_export.png
new file mode 100644
index 000000000..4d4997169
Binary files /dev/null and b/docs/quickstart/imgs/polycam_export.png differ
diff --git a/docs/quickstart/imgs/polycam_settings.png b/docs/quickstart/imgs/polycam_settings.png
new file mode 100644
index 000000000..04702c144
Binary files /dev/null and b/docs/quickstart/imgs/polycam_settings.png differ
diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py
index afb66c657..c1eb62715 100644
--- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py
+++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py
@@ -113,9 +113,17 @@ class Nerfstudio(DataParser):
else:
raise ValueError(f"Unknown dataparser split {split}")
+ if "orientation_override" in meta:
+ orientation_method = meta["orientation_override"]
+ CONSOLE.log(f"[yellow] Dataset is overriding orientation method to {orientation_method}")
+ else:
+ orientation_method = self.config.orientation_method
+
poses = torch.from_numpy(np.array(poses).astype(np.float32))
poses = camera_utils.auto_orient_and_center_poses(
- poses, method=self.config.orientation_method, center_poses=self.config.center_poses
+ poses,
+ method=orientation_method,
+ center_poses=self.config.center_poses,
)
# Scale poses
diff --git a/scripts/process_data.py b/scripts/process_data.py
index d6f7e0961..9655d1d35 100755
--- a/scripts/process_data.py
+++ b/scripts/process_data.py
@@ -1,10 +1,12 @@
#!/usr/bin/env python
+# pylint: disable=too-many-lines
"""Processes a video or image sequence to a nerfstudio compatible dataset."""
import json
import shutil
import subprocess
import sys
+import zipfile
from contextlib import nullcontext
from dataclasses import dataclass
from enum import Enum
@@ -251,11 +253,14 @@ def convert_insta360_to_images(
return summary_log, num_final_frames
-def copy_images_list(image_paths: List[Path], image_dir: Path, verbose) -> List[Path]:
+def copy_images_list(
+ image_paths: List[Path], image_dir: Path, crop_border_pixels: Optional[int] = None, verbose: bool = False
+) -> List[Path]:
"""Copy all images in a list of Paths. Useful for filtering from a directory.
Args:
image_paths: List of Paths of images to copy to a new directory.
image_dir: Path to the output directory.
+ crop_border_pixels: If not None, crops each edge by the specified number of pixels.
verbose: If True, print extra logging.
Returns:
A list of the copied image Paths.
@@ -276,6 +281,13 @@ def copy_images_list(image_paths: List[Path], image_dir: Path, verbose) -> List[
shutil.copy(image_path, copied_image_path)
copied_image_paths.append(copied_image_path)
+ if crop_border_pixels is not None:
+ file_type = image_paths[0].suffix
+ filename = f"frame_%05d{file_type}"
+ crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}"
+ ffmpeg_cmd = f"ffmpeg -y -i {image_dir / filename} -q:v 2 -vf {crop} {image_dir / filename}"
+ run_command(ffmpeg_cmd, verbose=verbose)
+
num_frames = len(image_paths)
if num_frames == 0:
@@ -630,6 +642,76 @@ def get_matching_summary(num_intial_frames: int, num_matched_frames: int) -> str
return f"[bold green]COLMAP found poses for {num_matched_frames / num_intial_frames * 100:.2f}% of the images."
+def polycam_to_json(
+ image_filenames: List[Path],
+ cameras_dir: Path,
+ output_dir: Path,
+ min_blur_score: float = 0.0,
+ crop_border_pixels: int = 0,
+) -> List[str]:
+ """Convert Polycam data into a nerfstudio dataset.
+
+ Args:
+ image_filenames: List of paths to the original images.
+ cameras_dir: Path to the polycam cameras directory.
+ output_dir: Path to the output directory.
+ min_blur_score: Minimum blur score to use an image. Images below this value will be skipped.
+ crop_border_pixels: Number of pixels to crop from each border of the image.
+
+ Returns:
+ Summary of the conversion.
+ """
+
+ frame_json = io.load_from_json(cameras_dir / f"{image_filenames[0].stem}.json")
+
+ # TODO Add per-frame intrinsics
+
+ data = {}
+ data["fl_x"] = frame_json["fx"]
+ data["fl_y"] = frame_json["fy"]
+ data["cx"] = frame_json["cx"] - crop_border_pixels
+ data["cy"] = frame_json["cy"] - crop_border_pixels
+ data["w"] = frame_json["width"] - crop_border_pixels * 2
+ data["h"] = frame_json["height"] - crop_border_pixels * 2
+ data["camera_model"] = CAMERA_MODELS["perspective"].value
+ # Needs to be a string for camera_utils.auto_orient_and_center_poses
+ data["orientation_override"] = "none"
+
+ frames = []
+ skipped_frames = 0
+ for i, image_filename in enumerate(image_filenames):
+ json_filename = cameras_dir / f"{image_filename.stem}.json"
+ frame_json = io.load_from_json(json_filename)
+ if "blur_score" in frame_json and frame_json["blur_score"] < min_blur_score:
+ skipped_frames += 1
+ continue
+ frame = {}
+ frame["file_path"] = f"./images/frame_{i+1:05d}{image_filename.suffix}"
+ # Transform matrix to nerfstudio format. Please refer to the documentation for coordinate system conventions.
+ frame["transform_matrix"] = [
+ [frame_json["t_20"], frame_json["t_21"], frame_json["t_22"], frame_json["t_23"]],
+ [frame_json["t_00"], frame_json["t_01"], frame_json["t_02"], frame_json["t_03"]],
+ [frame_json["t_10"], frame_json["t_11"], frame_json["t_12"], frame_json["t_13"]],
+ [0.0, 0.0, 0.0, 1.0],
+ ]
+ frames.append(frame)
+ data["frames"] = frames
+
+ with open(output_dir / "transforms.json", "w", encoding="utf-8") as f:
+ json.dump(data, f, indent=4)
+
+ summary = []
+ if skipped_frames > 0:
+ summary.append(f"Skipped {skipped_frames} frames due to low blur score.")
+ summary.append(f"Final dataset is {len(image_filenames) - skipped_frames} frames.")
+
+ if len(image_filenames) - skipped_frames == 0:
+ CONSOLE.print("[bold red]No images remain after filtering, exiting")
+ sys.exit(1)
+
+ return summary
+
+
@dataclass
class ProcessImages:
"""Process images into a nerfstudio dataset.
@@ -904,11 +986,11 @@ class ProcessRecord3D:
This script does the following:
1. Scales images to a specified size.
- 2. Converts Record3D poses into the nerfacto format.
+ 2. Converts Record3D poses into the nerfstudio format.
"""
data: Path
- """Path the data, either a video file or a directory of images."""
+ """Path to the record3D data."""
output_dir: Path
"""Path to the output directory."""
num_downscales: int = 3
@@ -971,9 +1053,128 @@ class ProcessRecord3D:
CONSOLE.rule()
+@dataclass
+class ProcessPolycam:
+ """Process Polycam data into a nerfstudio dataset.
+
+ To capture data, use the Polycam app on an iPhone or iPad with LiDAR. The capture must be in LiDAR or ROOM mode.
+ Developer mode must be enabled in the app settings, this will enable a raw data export option in the export menus.
+ The exported data folder is used as the input to this script.
+
+ This script does the following:
+
+ 1. Scales images to a specified size.
+ 2. Converts Polycam poses into the nerfstudio format.
+ """
+
+ data: Path
+ """Path the polycam export data folder. Can be .zip file or folder."""
+ output_dir: Path
+ """Path to the output directory."""
+ num_downscales: int = 3
+ """Number of times to downscale the images. Downscales by 2 each time. For example a value of 3
+ will downscale the images by 2x, 4x, and 8x."""
+ use_uncorrected_images: bool = False
+ """If True, use the raw images from the polycam export. If False, use the corrected images."""
+ max_dataset_size: int = 600
+ """Max number of images to train on. If the dataset has more, images will be sampled approximately evenly. If -1,
+ use all images."""
+ min_blur_score: float = 25
+ """Minimum blur score to use an image. If the blur score is below this value, the image will be skipped."""
+ crop_border_pixels: int = 15
+ """Number of pixels to crop from each border of the image. Useful as borders may be black due to undistortion."""
+
+ verbose: bool = False
+ """If True, print extra logging."""
+
+ def main(self) -> None:
+ """Process images into a nerfstudio dataset."""
+
+ self.output_dir.mkdir(parents=True, exist_ok=True)
+ image_dir = self.output_dir / "images"
+ image_dir.mkdir(parents=True, exist_ok=True)
+
+ summary_log = []
+
+ if self.data.suffix == ".zip":
+ with zipfile.ZipFile(self.data, "r") as zip_ref:
+ zip_ref.extractall(self.output_dir)
+ extracted_folder = zip_ref.namelist()[0].split("/")[0]
+ self.data = self.output_dir / extracted_folder
+
+ if (self.data / "keyframes" / "corrected_images").exists() and not self.use_uncorrected_images:
+ polycam_image_dir = self.data / "keyframes" / "corrected_images"
+ polycam_cameras_dir = self.data / "keyframes" / "corrected_cameras"
+ else:
+ polycam_image_dir = self.data / "keyframes" / "images"
+ polycam_cameras_dir = self.data / "keyframes" / "cameras"
+ self.crop_border_pixels = 0
+ if not self.use_uncorrected_images:
+ CONSOLE.print("[bold yellow]Corrected images not found, using raw images.")
+
+ if not polycam_image_dir.exists():
+ raise ValueError(f"Image directory {polycam_image_dir} doesn't exist")
+
+ # Copy images to output directory
+
+ polycam_image_filenames = []
+ for f in polycam_image_dir.iterdir():
+ polycam_image_filenames.append(f)
+ polycam_image_filenames = sorted(polycam_image_filenames, key=lambda fn: int(fn.stem))
+ num_images = len(polycam_image_filenames)
+ idx = np.arange(num_images)
+ if self.max_dataset_size != -1 and num_images > self.max_dataset_size:
+ idx = np.round(np.linspace(0, num_images - 1, self.max_dataset_size)).astype(int)
+
+ polycam_image_filenames = list(np.array(polycam_image_filenames)[idx])
+ # Copy images to output directory
+ copied_image_paths = copy_images_list(
+ polycam_image_filenames,
+ image_dir=image_dir,
+ crop_border_pixels=self.crop_border_pixels,
+ verbose=self.verbose,
+ )
+ num_frames = len(copied_image_paths)
+
+ copied_image_paths = [Path("images/" + copied_image_path.name) for copied_image_path in copied_image_paths]
+
+ if self.max_dataset_size > 0 and num_frames != num_images:
+ summary_log.append(f"Started with {num_frames} images out of {num_images} total")
+ summary_log.append(
+ "To change the size of the dataset add the argument --max_dataset_size to larger than the "
+ f"current value ({self.max_dataset_size}), or -1 to use all images."
+ )
+ else:
+ summary_log.append(f"Started with {num_frames} images")
+
+ # Downscale images
+ summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
+
+ # Save json
+ if num_frames == 0:
+ CONSOLE.print("[bold red]No images found, exiting")
+ sys.exit(1)
+ summary_log.extend(
+ polycam_to_json(
+ image_filenames=polycam_image_filenames,
+ cameras_dir=polycam_cameras_dir,
+ output_dir=self.output_dir,
+ min_blur_score=self.min_blur_score,
+ crop_border_pixels=self.crop_border_pixels,
+ )
+ )
+
+ CONSOLE.rule("[bold green]:tada: :tada: :tada: All DONE :tada: :tada: :tada:")
+
+ for summary in summary_log:
+ CONSOLE.print(summary, justify="center")
+ CONSOLE.rule()
+
+
Commands = Union[
Annotated[ProcessImages, tyro.conf.subcommand(name="images")],
Annotated[ProcessVideo, tyro.conf.subcommand(name="video")],
+ Annotated[ProcessPolycam, tyro.conf.subcommand(name="polycam")],
Annotated[ProcessInsta360, tyro.conf.subcommand(name="insta360")],
Annotated[ProcessRecord3D, tyro.conf.subcommand(name="record3d")],
]