mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 22:04:02 +08:00
RGB + Validity mask
This commit is contained in:
1 parent
836ac8b223
commit
6688f39275
5 files changed
+160
-41
No files matched your search
@@ -122,10 +122,12 @@ if __name__ == "__main__":
|
||||
parser.add_argument('--has_exposure', action='store_true')
|
||||
parser.add_argument('--exposure1', type=float, default=1.0)
|
||||
parser.add_argument('--exposure2', type=float, default=0.009)
|
||||
parser.add_argument('--skip_the_json', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
assert os.path.isfile(args.left_dir + '/transforms.json'), 'The JSON files for left set is missing!'
|
||||
assert os.path.isfile(args.right_dir + '/transforms.json'), 'The JSON files for right set is missing!'
|
||||
if not args.skip_the_json:
|
||||
assert os.path.isfile(args.left_dir + '/transforms.json'), 'The JSON files for left set is missing!'
|
||||
assert os.path.isfile(args.right_dir + '/transforms.json'), 'The JSON files for right set is missing!'
|
||||
assert os.path.isdir(args.left_dir + '/HDR_Normal/'), 'The data folder for left set is missing!'
|
||||
assert os.path.isdir(args.right_dir + '/HDR_Normal/'), 'The data folder for right set is missing!'
|
||||
|
||||
@@ -133,32 +135,33 @@ if __name__ == "__main__":
|
||||
os.mkdir(args.out_dir)
|
||||
os.mkdir(args.out_dir + 'HDR_Normal/')
|
||||
|
||||
cam_params_lhs = load_from_json( Path(args.left_dir + '/transforms.json') )
|
||||
cam_params_combine = {}
|
||||
cam_params_combine["camera_angle_x"] = cam_params_lhs["camera_angle_x"]
|
||||
cam_params_combine["frames"] = []
|
||||
for frame in tqdm(cam_params_lhs["frames"]):
|
||||
file_path = 'lhs_' + frame["file_path"].split('/')[-1]
|
||||
transform_matrix = frame["transform_matrix"]
|
||||
cam_params_combine["frames"].append(
|
||||
{
|
||||
"file_path": file_path,
|
||||
"transform_matrix": transform_matrix
|
||||
}
|
||||
)
|
||||
# TODO adding exposures
|
||||
cam_params_rhs = load_from_json( Path(args.right_dir + '/transforms.json') )
|
||||
for frame in tqdm(cam_params_rhs["frames"]):
|
||||
file_path = 'rhs_' + frame["file_path"].split('/')[-1]
|
||||
transform_matrix = frame["transform_matrix"]
|
||||
cam_params_combine["frames"].append(
|
||||
{
|
||||
"file_path": file_path,
|
||||
"transform_matrix": transform_matrix
|
||||
}
|
||||
)
|
||||
# TODO adding exposures
|
||||
write_to_json(Path(args.out_dir + '/transforms_all.json'), cam_params_combine)
|
||||
if not args.skip_the_json:
|
||||
cam_params_lhs = load_from_json( Path(args.left_dir + '/transforms.json') )
|
||||
cam_params_combine = {}
|
||||
cam_params_combine["camera_angle_x"] = cam_params_lhs["camera_angle_x"]
|
||||
cam_params_combine["frames"] = []
|
||||
for frame in tqdm(cam_params_lhs["frames"]):
|
||||
file_path = 'lhs_' + frame["file_path"].split('/')[-1]
|
||||
transform_matrix = frame["transform_matrix"]
|
||||
cam_params_combine["frames"].append(
|
||||
{
|
||||
"file_path": file_path,
|
||||
"transform_matrix": transform_matrix
|
||||
}
|
||||
)
|
||||
# TODO adding exposures
|
||||
cam_params_rhs = load_from_json( Path(args.right_dir + '/transforms.json') )
|
||||
for frame in tqdm(cam_params_rhs["frames"]):
|
||||
file_path = 'rhs_' + frame["file_path"].split('/')[-1]
|
||||
transform_matrix = frame["transform_matrix"]
|
||||
cam_params_combine["frames"].append(
|
||||
{
|
||||
"file_path": file_path,
|
||||
"transform_matrix": transform_matrix
|
||||
}
|
||||
)
|
||||
# TODO adding exposures
|
||||
write_to_json(Path(args.out_dir + '/transforms_all.json'), cam_params_combine)
|
||||
|
||||
process_all_images(args.left_dir + '/HDR_Normal/', args.out_dir + '/HDR_Normal/', 'lhs_', args.has_exposure, args.exposure1, 0.0, 0.95)
|
||||
process_all_images(args.right_dir + '/HDR_Normal/', args.out_dir + '/HDR_Normal/', 'rhs_', args.has_exposure, args.exposure2, 0.2, 1.0)
|
||||
|
||||
@@ -97,7 +97,8 @@ class PixelSampler:
|
||||
num_images: number of images to sample over
|
||||
mask: mask of possible pixels in an image to sample from.
|
||||
"""
|
||||
if isinstance(mask, torch.Tensor):
|
||||
# TODO Defining a flag for sampling even when there is a mask
|
||||
if isinstance(mask, torch.Tensor) and False:
|
||||
# mask size: batch * h * w * 1
|
||||
# Init state, cache masks indices.
|
||||
if self.states_of_mask == 0:
|
||||
|
||||
@@ -59,12 +59,12 @@ class LanternNerfactoField(NerfactoField):
|
||||
use_transient_embedding, use_semantics, num_semantic_classes,
|
||||
pass_semantic_gradients, use_pred_normals, use_average_appearance_embedding,
|
||||
spatial_distortion, implementation)
|
||||
# Define the last part MLP for RGB outputs
|
||||
# Define the last part MLP for RGB outputs and Masks
|
||||
self.mlp_head = MLP(
|
||||
in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim + self.appearance_embedding_dim,
|
||||
num_layers=num_layers_color,
|
||||
layer_width=hidden_dim_color,
|
||||
out_dim=3,
|
||||
out_dim= 3 + 2,
|
||||
activation=nn.ReLU(),
|
||||
out_activation=nn.ReLU(),
|
||||
implementation=implementation,
|
||||
|
||||
+108
-10
@@ -12,9 +12,11 @@ from torchmetrics.functional import structural_similarity_index_measure
|
||||
from torchmetrics.image import PeakSignalNoiseRatio
|
||||
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
|
||||
|
||||
from nerfstudio.cameras.rays import RayBundle, RaySamples
|
||||
from nerfstudio.lantern.field import LanternNerfactoField
|
||||
from nerfstudio.lantern.renderer import RGBRenderer_HDR
|
||||
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames
|
||||
from nerfstudio.field_components.spatial_distortions import SceneContraction
|
||||
from nerfstudio.fields.density_fields import HashMLPDensityField
|
||||
from nerfstudio.model_components.losses import (
|
||||
@@ -144,6 +146,57 @@ class LanternModel(NerfactoModel):
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
|
||||
|
||||
|
||||
def get_outputs(self, ray_bundle: RayBundle):
|
||||
ray_samples: RaySamples
|
||||
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
|
||||
field_outputs = self.field.forward(ray_samples, compute_normals=self.config.predict_normals)
|
||||
if self.config.use_gradient_scaling:
|
||||
field_outputs = scale_gradients_by_distance_squared(field_outputs, ray_samples)
|
||||
|
||||
weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY])
|
||||
weights_list.append(weights)
|
||||
ray_samples_list.append(ray_samples)
|
||||
|
||||
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
|
||||
depth = self.renderer_depth(weights=weights, ray_samples=ray_samples)
|
||||
accumulation = self.renderer_accumulation(weights=weights)
|
||||
|
||||
outputs = {
|
||||
"rgb": rgb[:, 0:3],
|
||||
"accumulation": accumulation,
|
||||
"depth": depth,
|
||||
"validity_w": rgb[:, 4],
|
||||
"validity_f": rgb[:, 3],
|
||||
}
|
||||
|
||||
if self.config.predict_normals:
|
||||
normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
|
||||
pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
|
||||
outputs["normals"] = self.normals_shader(normals)
|
||||
outputs["pred_normals"] = self.normals_shader(pred_normals)
|
||||
# These use a lot of GPU memory, so we avoid storing them for eval.
|
||||
if self.training:
|
||||
outputs["weights_list"] = weights_list
|
||||
outputs["ray_samples_list"] = ray_samples_list
|
||||
|
||||
if self.training and self.config.predict_normals:
|
||||
outputs["rendered_orientation_loss"] = orientation_loss(
|
||||
weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
|
||||
)
|
||||
|
||||
outputs["rendered_pred_normal_loss"] = pred_normal_loss(
|
||||
weights.detach(),
|
||||
field_outputs[FieldHeadNames.NORMALS].detach(),
|
||||
field_outputs[FieldHeadNames.PRED_NORMALS],
|
||||
)
|
||||
|
||||
for i in range(self.config.num_proposal_iterations):
|
||||
outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
|
||||
|
||||
return outputs
|
||||
|
||||
|
||||
def get_image_metrics_and_images(
|
||||
self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
|
||||
@@ -192,7 +245,7 @@ class LanternModel(NerfactoModel):
|
||||
return metrics_dict, images_dict
|
||||
|
||||
|
||||
def weighting_function(self, pixels, zmin, zmax):
|
||||
def hat_weighting_function(self, pixels, zmin, zmax):
|
||||
threshold = 0.5 * (zmin + zmax)
|
||||
weights = torch.zeros(pixels.shape, dtype = pixels.dtype).to(self.device)
|
||||
weights = weights + torch.finfo(torch.float32).eps
|
||||
@@ -206,35 +259,80 @@ class LanternModel(NerfactoModel):
|
||||
weights[mask_upperbound] = 2.0 * (zmax - pixels[mask_upperbound])
|
||||
|
||||
return weights
|
||||
|
||||
|
||||
def cut_weighting_function(self, pixels, exposures):
|
||||
# TODO make sure the values for cuts are not fixed
|
||||
|
||||
well_exposure = torch.max(exposures)
|
||||
fast_exposure = torch.min(exposures)
|
||||
|
||||
weights = torch.ones(pixels.shape, dtype = pixels.dtype).to(self.device)
|
||||
|
||||
fast_expo_valids = (exposures == fast_exposure).repeat(1,pixels.shape[-1])
|
||||
weights[fast_expo_valids] = torch.clip(10.0 * (pixels[fast_expo_valids] - 0.1), 0.0, 1.0)
|
||||
|
||||
well_expo_valids = ~fast_expo_valids
|
||||
weights[well_expo_valids] = torch.clip(-20.0 * (pixels[well_expo_valids] - 0.9) + 1, 0.0, 1.0)
|
||||
|
||||
return weights
|
||||
|
||||
|
||||
def cut_weighting_for_mask_function(self, pixels, exposures):
|
||||
well_exposure = torch.max(exposures)
|
||||
fast_exposure = torch.min(exposures)
|
||||
|
||||
weights1 = torch.zeros(exposures.shape, dtype = exposures.dtype).to(self.device)
|
||||
weights2 = torch.zeros(exposures.shape, dtype = exposures.dtype).to(self.device)
|
||||
|
||||
fast_expo_valids = (exposures == fast_exposure).repeat(1, 1)
|
||||
weights1[fast_expo_valids] = 1.0
|
||||
|
||||
well_expo_valids = ~fast_expo_valids
|
||||
weights2[well_expo_valids] = 1.0
|
||||
|
||||
return weights1, weights2
|
||||
|
||||
|
||||
def get_loss_dict(self, outputs, batch, metrics_dict=None):
|
||||
loss_dict = {}
|
||||
weights_for_loss = None
|
||||
weights_for_RGB_loss = None
|
||||
|
||||
image = batch["image"].to(self.device)
|
||||
|
||||
|
||||
if "exposure" in batch:
|
||||
u = 5000.
|
||||
img_uncompress = torch.exp(image * torch.log(torch.tensor(u+1.))) - 1.
|
||||
img_uncompress /= u
|
||||
|
||||
exposures_resized = batch["exposure"].view(batch["exposure"].shape[0], 1)
|
||||
|
||||
img_uncompress_re_exposed = exposures_resized * img_uncompress
|
||||
weights_for_loss = self.weighting_function(torch.clip(img_uncompress_re_exposed, 0, 1), 0.0, 1.0)
|
||||
|
||||
# Debevec Hat weights for Loss
|
||||
# weights_for_loss = self.hat_weighting_function(torch.clip(img_uncompress_re_exposed, 0, 1), 0.0, 1.0)
|
||||
|
||||
# Clear cut weights for Loss
|
||||
weights_for_RGB_loss = self.cut_weighting_function(torch.clip(img_uncompress_re_exposed, 0, 1), exposures_resized)
|
||||
mask1_w, mask2_w = self.cut_weighting_for_mask_function(img_uncompress_re_exposed, exposures_resized)
|
||||
|
||||
pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation(
|
||||
pred_image=outputs["rgb"],
|
||||
pred_accumulation=outputs["accumulation"],
|
||||
gt_image=image,
|
||||
)
|
||||
|
||||
if weights_for_loss is not None:
|
||||
loss_dict["rgb_loss"] = (weights_for_loss * ((gt_rgb - pred_rgb) ** 2)).mean()
|
||||
|
||||
# MSE loss for mask with weights
|
||||
mask_in_float = batch['mask'].type(torch.float).to(self.device)
|
||||
negative_mask_in_float = torch.ones(mask_in_float.shape).to(self.device) - mask_in_float
|
||||
loss_mask_fast_expo = (mask1_w * (( negative_mask_in_float - torch.unsqueeze(outputs["validity_f"], 1)) ** 2)).mean()
|
||||
loss_mask_well_expo = (mask2_w * (( mask_in_float - torch.unsqueeze(outputs["validity_w"], 1)) ** 2)).mean()
|
||||
loss_dict["validity_loss_well_exposed"] = loss_mask_well_expo
|
||||
loss_dict["validity_loss_fast_exposure"] = loss_mask_fast_expo
|
||||
|
||||
if weights_for_RGB_loss is not None:
|
||||
loss_dict["rgb_loss"] = (weights_for_RGB_loss * ((gt_rgb - pred_rgb) ** 2)).mean()
|
||||
else:
|
||||
loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb)
|
||||
|
||||
|
||||
if self.training:
|
||||
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
|
||||
outputs["weights_list"], outputs["ray_samples_list"]
|
||||
|
||||
@@ -156,6 +156,7 @@ def _render_trajectory_video(
|
||||
sys.exit(1)
|
||||
output_image = outputs[rendered_output_name]
|
||||
is_depth = rendered_output_name.find("depth") != -1
|
||||
is_validity = rendered_output_name.find("validity_w") != -1 or rendered_output_name.find("validity_f") != -1
|
||||
if is_depth:
|
||||
output_image = (
|
||||
colormaps.apply_depth_colormap(
|
||||
@@ -168,6 +169,22 @@ def _render_trajectory_video(
|
||||
.cpu()
|
||||
.numpy()
|
||||
)
|
||||
elif is_validity:
|
||||
mask_f = torch.clip(outputs["validity_f"], 0, 1)
|
||||
mask_f = torch.ones(mask_f.shape).to(mask_f.get_device()) - mask_f
|
||||
|
||||
final_mask = torch.clip(outputs["validity_w"], 0, 1) + mask_f
|
||||
final_mask[final_mask > 0.5] = 1
|
||||
final_mask[final_mask <= 0.5] = 0
|
||||
|
||||
output_image = (
|
||||
colormaps.apply_colormap(
|
||||
final_mask,
|
||||
colormap_options=colormap_options,
|
||||
)
|
||||
.cpu()
|
||||
.numpy()
|
||||
)
|
||||
elif image_format == "exr":
|
||||
u = 5000.
|
||||
output_image = torch.exp(output_image * torch.log(torch.tensor(u+1.))) - 1.
|
||||
|
||||
Reference in new issue
Block a user