mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-19 22:25:44 +08:00
Mess around with normals
This commit is contained in:
1 parent
b3dca69ddb
commit
7c40594c37
5 files changed
+32
-25
No files matched your search
@@ -360,6 +360,8 @@ method_configs["dreamfusion"] = DreamfusionTrainerConfig(
|
||||
proposal_warmup=500,
|
||||
proposal_update_every=5,
|
||||
proposal_weights_anneal_max_num_iters=100,
|
||||
start_lambertian_training=1000,
|
||||
start_normals_training=500,
|
||||
),
|
||||
alphas_loss_mult=1.0,
|
||||
opacity_loss_mult=1.0,
|
||||
|
||||
@@ -46,7 +46,7 @@ class NerfstudioDataParserConfig(DataParserConfig):
|
||||
|
||||
_target: Type = field(default_factory=lambda: Nerfstudio)
|
||||
"""target class to instantiate"""
|
||||
data: Path = Path("data/nerfstudio/poster")
|
||||
data: Path = Path("data/nerfstudio")
|
||||
"""Directory or explicit json file path specifying location of data."""
|
||||
scale_factor: float = 1.0
|
||||
"""How much to scale the camera origins by."""
|
||||
|
||||
@@ -194,7 +194,7 @@ class DreamFusionField(Field):
|
||||
pred_normals_inp = torch.cat([positions_flat, density_embedding.view(-1, self.geo_feat_dim)], dim=-1)
|
||||
|
||||
x = self.mlp_pred_normals(pred_normals_inp).view(*outputs_shape, -1).to(directions)
|
||||
outputs[FieldHeadNames.PRED_NORMALS] = math.safe_normalize(self.field_head_pred_normals(x))
|
||||
outputs[FieldHeadNames.PRED_NORMALS] = self.field_head_pred_normals(x)
|
||||
|
||||
h = density_embedding.view(-1, self.geo_feat_dim)
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ class DreamFusionModelConfig(ModelConfig):
|
||||
_target: Type = field(default_factory=lambda: DreamFusionModel)
|
||||
"""target class to instantiate"""
|
||||
|
||||
orientation_loss_mult: float = 0.01
|
||||
orientation_loss_mult: float = 0.0001
|
||||
"""Orientation loss multipier on computed normals."""
|
||||
pred_normal_loss_mult: float = 0.001
|
||||
"""Predicted normal loss multiplier."""
|
||||
@@ -321,33 +321,34 @@ class DreamFusionModel(Model):
|
||||
outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
|
||||
|
||||
normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
|
||||
pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) # .detach()
|
||||
pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
|
||||
|
||||
# lambertian shading
|
||||
if self.config.random_light_source and self.training: # and if training
|
||||
if self.config.random_light_source: # and self.training:
|
||||
light_d = ray_bundle.origins[0] + torch.randn(3, dtype=torch.float).to(normals)
|
||||
else:
|
||||
light_d = ray_bundle.origins[0]
|
||||
light_d = math.safe_normalize(light_d)
|
||||
|
||||
if self.train_shaded and np.random.random_sample() > 0.75:
|
||||
if (self.train_shaded and np.random.random_sample() > 0.75) or not self.training:
|
||||
shading_weight = 0.9
|
||||
else:
|
||||
shading_weight = 0.0
|
||||
|
||||
shaded, shaded_albedo = self.shader_lambertian(
|
||||
rgb=rgb, normals=pred_normals, light_direction=light_d, shading_weight=shading_weight, detach_normals=True
|
||||
rgb=rgb, normals=pred_normals, light_direction=light_d, shading_weight=shading_weight, detach_normals=False
|
||||
)
|
||||
|
||||
outputs["normals"] = self.shader_normals(normals, weights=accum_mask)
|
||||
outputs["pred_normals"] = self.shader_normals(pred_normals, weights=accum_mask)
|
||||
outputs["shaded"] = accum_mask * shaded + torch.ones_like(shaded) * accum_mask_inv
|
||||
outputs["shaded_albedo"] = shaded_albedo
|
||||
outputs["shaded"] = accum_mask * shaded
|
||||
outputs["other_train_output"] = accum_mask * shaded_albedo + background
|
||||
outputs["shaded_albedo"] = accum_mask * shaded_albedo
|
||||
outputs["rgb"] = accum_mask * rgb + background
|
||||
|
||||
if shading_weight < 0:
|
||||
if np.random.random_sample() > 0.5:
|
||||
outputs["train_output"] = shaded
|
||||
if shading_weight > 0:
|
||||
if np.random.random_sample() > 0.5 and not self.training:
|
||||
outputs["train_output"] = outputs["shaded"]
|
||||
else:
|
||||
outputs["train_output"] = accum_mask * shaded_albedo + background
|
||||
else:
|
||||
@@ -355,7 +356,7 @@ class DreamFusionModel(Model):
|
||||
|
||||
if self.training or True:
|
||||
outputs["rendered_orientation_loss"] = orientation_loss(
|
||||
weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
|
||||
weights, field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
|
||||
)
|
||||
|
||||
outputs["rendered_pred_normal_loss"] = pred_normal_loss(
|
||||
@@ -374,16 +375,18 @@ class DreamFusionModel(Model):
|
||||
|
||||
loss_dict = {}
|
||||
loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients)
|
||||
# if self.train_normals:
|
||||
# orientation loss for computed normals
|
||||
loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean(
|
||||
outputs["rendered_orientation_loss"]
|
||||
)
|
||||
|
||||
# ground truth supervision for normals
|
||||
loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean(
|
||||
outputs["rendered_pred_normal_loss"]
|
||||
)
|
||||
if self.train_normals:
|
||||
# orientation loss for computed normals
|
||||
loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean(
|
||||
outputs["rendered_orientation_loss"]
|
||||
)
|
||||
# ground truth supervision for normals
|
||||
loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean(
|
||||
outputs["rendered_pred_normal_loss"]
|
||||
)
|
||||
else:
|
||||
loss_dict["orientation_loss"] = 0
|
||||
loss_dict["pred_normal_loss"] = 0
|
||||
|
||||
loss_dict["alphas_loss"] = outputs["alphas_loss"].mean()
|
||||
|
||||
@@ -423,5 +426,7 @@ class DreamFusionModel(Model):
|
||||
"depth": depth,
|
||||
"prop_depth_0": prop_depth_0,
|
||||
"prop_depth_1": prop_depth_1,
|
||||
"normals": outputs["normals"],
|
||||
"pred_normals": outputs["pred_normals"],
|
||||
}
|
||||
return metrics_dict, images_dict
|
||||
@@ -51,7 +51,7 @@ class DreamfusionPipelineConfig(VanillaPipelineConfig):
|
||||
front_prompt: str = ", front view"
|
||||
"""appended to prompt for front view"""
|
||||
back_prompt: str = ", back view"
|
||||
"""appended to prompt for back view"""
|
||||
"""appended to prompt for back view"""
|
||||
alphas_penalty: bool = True
|
||||
"""enables penalty to encourage sparse weights (penalizing for uniform density along ray)"""
|
||||
alphas_loss_mult: float = 1
|
||||
@@ -137,7 +137,7 @@ class DreamfusionPipeline(VanillaPipeline):
|
||||
text_embedding = (vert * text_embedding + (90 - vert) * self.top_text_embedding) / 90.0
|
||||
|
||||
else:
|
||||
if batch["vertical"] < 30:
|
||||
if batch["vertical"] < 40:
|
||||
text_embedding = self.top_text_embedding
|
||||
elif batch["central"] > 315 or batch["central"] <= 45:
|
||||
text_embedding = self.front_text_embedding
|
||||
|
||||
Reference in new issue
Block a user