semantic_nerfw_configurability (#1467)

* fix semseg vis

* semantic configs

* change defaults

* Fix type

* format

* fix imports

* suggested fix for issue 1465

* fix docstring placement; avoid log spam and runtime errors

`nerfstudio\models\semantic_nerfw.py:81: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  self.colormap = torch.tensor(self.semantics.colors).to(self.device)`

* hide semantics output choice, since it kills the viewer

* revert some changes

* caps

* move both image and mask to GPU to avoid runtime error

as suggested in https://github.com/nerfstudio-project/nerfstudio/issues/1465#issuecomment-1441117562

---------

Co-authored-by: machenmusik <machenmusik@users.noreply.github.com>
This commit is contained in:
Nikhil Mohanandmachenmusik authored and GitHub committed 2023-02-22 22:21:04 -05:00
1 parent 69fdc7db9c
commit a91b92f89d
5 files changed
+25 -14

No files matched your search

+13 -4
View File
@@ -61,6 +61,8 @@ class SemanticNerfWModelConfig(NerfactoModelConfig):
_target: Type = field(default_factory=lambda: SemanticNerfWModel)
use_transient_embedding: bool = False
"""Whether to use transient embedding."""
semantic_loss_weight: float = 1.0
pass_semantic_gradients: bool = False
class SemanticNerfWModel(Model):
@@ -76,6 +78,7 @@ class SemanticNerfWModel(Model):
assert "semantics" in metadata.keys() and isinstance(metadata["semantics"], Semantics)
self.semantics = metadata["semantics"]
super().__init__(config=config, **kwargs)
self.colormap = self.semantics.colors.clone().detach().to(self.device)
def populate_modules(self):
"""Set the fields and modules."""
@@ -98,6 +101,7 @@ class SemanticNerfWModel(Model):
use_transient_embedding=self.config.use_transient_embedding,
use_semantics=True,
num_semantic_classes=len(self.semantics.classes),
pass_semantic_gradients=self.config.pass_semantic_gradients,
)
# Build the proposal network(s)
@@ -207,13 +211,16 @@ class SemanticNerfWModel(Model):
outputs["density_transient"] = field_outputs[FieldHeadNames.TRANSIENT_DENSITY]
# semantics
semantic_weights = weights_static
if not self.config.pass_semantic_gradients:
semantic_weights = semantic_weights.detach()
outputs["semantics"] = self.renderer_semantics(
field_outputs[FieldHeadNames.SEMANTICS], weights=weights_static.detach()
field_outputs[FieldHeadNames.SEMANTICS], weights=semantic_weights
)
# semantics colormaps
semantic_labels = torch.argmax(torch.nn.functional.softmax(outputs["semantics"], dim=-1), dim=-1)
outputs["semantics_colormap"] = self.semantics.colors[semantic_labels]
outputs["semantics_colormap"] = self.colormap.to(self.device)[semantic_labels]
return outputs
@@ -243,7 +250,9 @@ class SemanticNerfWModel(Model):
loss_dict["rgb_loss"] = self.rgb_loss(image, outputs["rgb"])
# semantic loss
loss_dict["semantics_loss"] = self.cross_entropy_loss(outputs["semantics"], batch["semantics"][..., 0].long())
loss_dict["semantics_loss"] = self.config.semantic_loss_weight * self.cross_entropy_loss(
outputs["semantics"], batch["semantics"][..., 0].long()
)
return loss_dict
def get_image_metrics_and_images(
@@ -287,7 +296,7 @@ class SemanticNerfWModel(Model):
# semantics
semantic_labels = torch.argmax(torch.nn.functional.softmax(outputs["semantics"], dim=-1), dim=-1)
images_dict["semantics_colormap"] = self.semantics.colors[semantic_labels]
images_dict["semantics_colormap"] = self.colormap.to(self.device)[semantic_labels]
# valid mask
images_dict["mask"] = batch["mask"].repeat(1, 1, 3)