mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 06:35:36 +08:00
semantic_nerfw_configurability (#1467)
* fix semseg vis * semantic configs * change defaults * Fix type * format * fix imports * suggested fix for issue 1465 * fix docstring placement; avoid log spam and runtime errors `nerfstudio\models\semantic_nerfw.py:81: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor). self.colormap = torch.tensor(self.semantics.colors).to(self.device)` * hide semantics output choice, since it kills the viewer * revert some changes * caps * move both image and mask to GPU to avoid runtime error as suggested in https://github.com/nerfstudio-project/nerfstudio/issues/1465#issuecomment-1441117562 --------- Co-authored-by: machenmusik <machenmusik@users.noreply.github.com>
This commit is contained in:
1 parent
69fdc7db9c
commit
a91b92f89d
5 files changed
+25
-14
No files matched your search
@@ -61,6 +61,8 @@ class SemanticNerfWModelConfig(NerfactoModelConfig):
|
||||
_target: Type = field(default_factory=lambda: SemanticNerfWModel)
|
||||
use_transient_embedding: bool = False
|
||||
"""Whether to use transient embedding."""
|
||||
semantic_loss_weight: float = 1.0
|
||||
pass_semantic_gradients: bool = False
|
||||
|
||||
|
||||
class SemanticNerfWModel(Model):
|
||||
@@ -76,6 +78,7 @@ class SemanticNerfWModel(Model):
|
||||
assert "semantics" in metadata.keys() and isinstance(metadata["semantics"], Semantics)
|
||||
self.semantics = metadata["semantics"]
|
||||
super().__init__(config=config, **kwargs)
|
||||
self.colormap = self.semantics.colors.clone().detach().to(self.device)
|
||||
|
||||
def populate_modules(self):
|
||||
"""Set the fields and modules."""
|
||||
@@ -98,6 +101,7 @@ class SemanticNerfWModel(Model):
|
||||
use_transient_embedding=self.config.use_transient_embedding,
|
||||
use_semantics=True,
|
||||
num_semantic_classes=len(self.semantics.classes),
|
||||
pass_semantic_gradients=self.config.pass_semantic_gradients,
|
||||
)
|
||||
|
||||
# Build the proposal network(s)
|
||||
@@ -207,13 +211,16 @@ class SemanticNerfWModel(Model):
|
||||
outputs["density_transient"] = field_outputs[FieldHeadNames.TRANSIENT_DENSITY]
|
||||
|
||||
# semantics
|
||||
semantic_weights = weights_static
|
||||
if not self.config.pass_semantic_gradients:
|
||||
semantic_weights = semantic_weights.detach()
|
||||
outputs["semantics"] = self.renderer_semantics(
|
||||
field_outputs[FieldHeadNames.SEMANTICS], weights=weights_static.detach()
|
||||
field_outputs[FieldHeadNames.SEMANTICS], weights=semantic_weights
|
||||
)
|
||||
|
||||
# semantics colormaps
|
||||
semantic_labels = torch.argmax(torch.nn.functional.softmax(outputs["semantics"], dim=-1), dim=-1)
|
||||
outputs["semantics_colormap"] = self.semantics.colors[semantic_labels]
|
||||
outputs["semantics_colormap"] = self.colormap.to(self.device)[semantic_labels]
|
||||
|
||||
return outputs
|
||||
|
||||
@@ -243,7 +250,9 @@ class SemanticNerfWModel(Model):
|
||||
loss_dict["rgb_loss"] = self.rgb_loss(image, outputs["rgb"])
|
||||
|
||||
# semantic loss
|
||||
loss_dict["semantics_loss"] = self.cross_entropy_loss(outputs["semantics"], batch["semantics"][..., 0].long())
|
||||
loss_dict["semantics_loss"] = self.config.semantic_loss_weight * self.cross_entropy_loss(
|
||||
outputs["semantics"], batch["semantics"][..., 0].long()
|
||||
)
|
||||
return loss_dict
|
||||
|
||||
def get_image_metrics_and_images(
|
||||
@@ -287,7 +296,7 @@ class SemanticNerfWModel(Model):
|
||||
|
||||
# semantics
|
||||
semantic_labels = torch.argmax(torch.nn.functional.softmax(outputs["semantics"], dim=-1), dim=-1)
|
||||
images_dict["semantics_colormap"] = self.semantics.colors[semantic_labels]
|
||||
images_dict["semantics_colormap"] = self.colormap.to(self.device)[semantic_labels]
|
||||
|
||||
# valid mask
|
||||
images_dict["mask"] = batch["mask"].repeat(1, 1, 3)
|
||||
|
||||
Reference in new issue
Block a user