diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bdb8ff4f4..a93cfbd1f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.1.13 + rev: v0.6.2 hooks: - id: ruff types_or: [ python, pyi, jupyter ] diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index 5b3af8b55..5cf8b5771 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -211,10 +211,15 @@ class NerfactoField(Field): # Make sure the tcnn gets inputs between 0 and 1. selector = ((positions > 0.0) & (positions < 1.0)).all(dim=-1) positions = positions * selector[..., None] + + assert positions.numel() > 0, "positions is empty." + self._sample_locations = positions if not self._sample_locations.requires_grad: self._sample_locations.requires_grad = True positions_flat = positions.view(-1, 3) + + assert positions_flat.numel() > 0, "positions_flat is empty." h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1) density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1) self._density_before_activation = density_before_activation diff --git a/nerfstudio/models/splatfacto.py b/nerfstudio/models/splatfacto.py index f934b1263..61d9eda19 100644 --- a/nerfstudio/models/splatfacto.py +++ b/nerfstudio/models/splatfacto.py @@ -314,7 +314,10 @@ class SplatfactoModel(Model): @property def shs_0(self): - return self.features_dc + if self.config.sh_degree > 0: + return self.features_dc + else: + return RGB2SH(torch.sigmoid(self.features_dc)) @property def shs_rest(self): diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py index c359efcaf..8e105328f 100644 --- a/nerfstudio/scripts/exporter.py +++ b/nerfstudio/scripts/exporter.py @@ -513,6 +513,9 @@ class ExportGaussianSplat(Exporter): """Rotation of the oriented bounding box. Expressed as RPY Euler angles in radians""" obb_scale: Optional[Tuple[float, float, float]] = None """Scale of the oriented bounding box along each axis.""" + ply_color_mode: Literal["sh_coeffs", "rgb"] = "sh_coeffs" + """If "rgb", export colors as red/green/blue fields. Otherwise, export colors as + spherical harmonics coefficients.""" @staticmethod def write_ply( @@ -532,7 +535,7 @@ class ExportGaussianSplat(Exporter): """ # Ensure count matches the length of all tensors - if not all(len(tensor) == count for tensor in map_to_tensors.values()): + if not all(tensor.size == count for tensor in map_to_tensors.values()): raise ValueError("Count does not match the length of all tensors") # Type check for numpy arrays of type float or uint8 and non-empty @@ -580,7 +583,6 @@ class ExportGaussianSplat(Exporter): filename = self.output_dir / "splat.ply" - count = 0 map_to_tensors = OrderedDict() with torch.no_grad(): @@ -594,19 +596,28 @@ class ExportGaussianSplat(Exporter): map_to_tensors["ny"] = np.zeros(n, dtype=np.float32) map_to_tensors["nz"] = np.zeros(n, dtype=np.float32) - if model.config.sh_degree > 0: + if self.ply_color_mode == "rgb": + colors = torch.clamp(model.colors.clone(), 0.0, 1.0).data.cpu().numpy() + colors = (colors * 255).astype(np.uint8) + map_to_tensors["red"] = colors[:, 0] + map_to_tensors["green"] = colors[:, 1] + map_to_tensors["blue"] = colors[:, 2] + elif self.ply_color_mode == "sh_coeffs": shs_0 = model.shs_0.contiguous().cpu().numpy() for i in range(shs_0.shape[1]): map_to_tensors[f"f_dc_{i}"] = shs_0[:, i, None] - # transpose(1, 2) was needed to match the sh order in Inria version - shs_rest = model.shs_rest.transpose(1, 2).contiguous().cpu().numpy() - shs_rest = shs_rest.reshape((n, -1)) - for i in range(shs_rest.shape[-1]): - map_to_tensors[f"f_rest_{i}"] = shs_rest[:, i, None] - else: - colors = torch.clamp(model.colors.clone(), 0.0, 1.0).data.cpu().numpy() - map_to_tensors["colors"] = (colors * 255).astype(np.uint8) + if model.config.sh_degree > 0: + if self.ply_color_mode == "rgb": + CONSOLE.print( + "Warning: model has higher level of spherical harmonics, ignoring them and only export rgb." + ) + elif self.ply_color_mode == "sh_coeffs": + # transpose(1, 2) was needed to match the sh order in Inria version + shs_rest = model.shs_rest.transpose(1, 2).contiguous().cpu().numpy() + shs_rest = shs_rest.reshape((n, -1)) + for i in range(shs_rest.shape[-1]): + map_to_tensors[f"f_rest_{i}"] = shs_rest[:, i, None] map_to_tensors["opacity"] = model.opacities.data.cpu().numpy() diff --git a/nerfstudio/utils/writer.py b/nerfstudio/utils/writer.py index a986618ca..45b23c8fc 100644 --- a/nerfstudio/utils/writer.py +++ b/nerfstudio/utils/writer.py @@ -334,7 +334,7 @@ class WandbWriter(Writer): """ import wandb # wandb is slow to import, so we only import it if we need it. - wandb.config.update(config_dict, allow_val_change=True) + wandb.config.update(config_dict, allow_val_change=True) # type: ignore @decorate_all([check_main_thread]) diff --git a/pyproject.toml b/pyproject.toml index 753c12762..027ef1374 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "nerfstudio" -version = "1.1.3" +version = "1.1.4" description = "All-in-one repository for state-of-the-art NeRFs" readme = "README.md" license = { text="Apache 2.0"} @@ -56,7 +56,7 @@ dependencies = [ "torchvision>=0.14.1", "torchmetrics[image]>=1.0.1", "typing_extensions>=4.4.0", - "viser==0.2.3", + "viser==0.2.7", "nuscenes-devkit>=1.1.1", "wandb>=0.13.3", "xatlas",