RGB + Validity mask

This commit is contained in:
mokad6 committed 2023-12-21 10:48:40 -05:00
1 parent 836ac8b223
commit 6688f39275
5 files changed
+160 -41

No files matched your search

+31 -28
View File
@@ -122,10 +122,12 @@ if __name__ == "__main__":
parser.add_argument('--has_exposure', action='store_true')
parser.add_argument('--exposure1', type=float, default=1.0)
parser.add_argument('--exposure2', type=float, default=0.009)
parser.add_argument('--skip_the_json', action='store_true')
args = parser.parse_args()
assert os.path.isfile(args.left_dir + '/transforms.json'), 'The JSON files for left set is missing!'
assert os.path.isfile(args.right_dir + '/transforms.json'), 'The JSON files for right set is missing!'
if not args.skip_the_json:
assert os.path.isfile(args.left_dir + '/transforms.json'), 'The JSON files for left set is missing!'
assert os.path.isfile(args.right_dir + '/transforms.json'), 'The JSON files for right set is missing!'
assert os.path.isdir(args.left_dir + '/HDR_Normal/'), 'The data folder for left set is missing!'
assert os.path.isdir(args.right_dir + '/HDR_Normal/'), 'The data folder for right set is missing!'
@@ -133,32 +135,33 @@ if __name__ == "__main__":
os.mkdir(args.out_dir)
os.mkdir(args.out_dir + 'HDR_Normal/')
cam_params_lhs = load_from_json( Path(args.left_dir + '/transforms.json') )
cam_params_combine = {}
cam_params_combine["camera_angle_x"] = cam_params_lhs["camera_angle_x"]
cam_params_combine["frames"] = []
for frame in tqdm(cam_params_lhs["frames"]):
file_path = 'lhs_' + frame["file_path"].split('/')[-1]
transform_matrix = frame["transform_matrix"]
cam_params_combine["frames"].append(
{
"file_path": file_path,
"transform_matrix": transform_matrix
}
)
# TODO adding exposures
cam_params_rhs = load_from_json( Path(args.right_dir + '/transforms.json') )
for frame in tqdm(cam_params_rhs["frames"]):
file_path = 'rhs_' + frame["file_path"].split('/')[-1]
transform_matrix = frame["transform_matrix"]
cam_params_combine["frames"].append(
{
"file_path": file_path,
"transform_matrix": transform_matrix
}
)
# TODO adding exposures
write_to_json(Path(args.out_dir + '/transforms_all.json'), cam_params_combine)
if not args.skip_the_json:
cam_params_lhs = load_from_json( Path(args.left_dir + '/transforms.json') )
cam_params_combine = {}
cam_params_combine["camera_angle_x"] = cam_params_lhs["camera_angle_x"]
cam_params_combine["frames"] = []
for frame in tqdm(cam_params_lhs["frames"]):
file_path = 'lhs_' + frame["file_path"].split('/')[-1]
transform_matrix = frame["transform_matrix"]
cam_params_combine["frames"].append(
{
"file_path": file_path,
"transform_matrix": transform_matrix
}
)
# TODO adding exposures
cam_params_rhs = load_from_json( Path(args.right_dir + '/transforms.json') )
for frame in tqdm(cam_params_rhs["frames"]):
file_path = 'rhs_' + frame["file_path"].split('/')[-1]
transform_matrix = frame["transform_matrix"]
cam_params_combine["frames"].append(
{
"file_path": file_path,
"transform_matrix": transform_matrix
}
)
# TODO adding exposures
write_to_json(Path(args.out_dir + '/transforms_all.json'), cam_params_combine)
process_all_images(args.left_dir + '/HDR_Normal/', args.out_dir + '/HDR_Normal/', 'lhs_', args.has_exposure, args.exposure1, 0.0, 0.95)
process_all_images(args.right_dir + '/HDR_Normal/', args.out_dir + '/HDR_Normal/', 'rhs_', args.has_exposure, args.exposure2, 0.2, 1.0)
+2 -1
View File
@@ -97,7 +97,8 @@ class PixelSampler:
num_images: number of images to sample over
mask: mask of possible pixels in an image to sample from.
"""
if isinstance(mask, torch.Tensor):
# TODO Defining a flag for sampling even when there is a mask
if isinstance(mask, torch.Tensor) and False:
# mask size: batch * h * w * 1
# Init state, cache masks indices.
if self.states_of_mask == 0:
+2 -2
View File
@@ -59,12 +59,12 @@ class LanternNerfactoField(NerfactoField):
use_transient_embedding, use_semantics, num_semantic_classes,
pass_semantic_gradients, use_pred_normals, use_average_appearance_embedding,
spatial_distortion, implementation)
# Define the last part MLP for RGB outputs
# Define the last part MLP for RGB outputs and Masks
self.mlp_head = MLP(
in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim + self.appearance_embedding_dim,
num_layers=num_layers_color,
layer_width=hidden_dim_color,
out_dim=3,
out_dim= 3 + 2,
activation=nn.ReLU(),
out_activation=nn.ReLU(),
implementation=implementation,
+108 -10
View File
@@ -12,9 +12,11 @@ from torchmetrics.functional import structural_similarity_index_measure
from torchmetrics.image import PeakSignalNoiseRatio
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
from nerfstudio.cameras.rays import RayBundle, RaySamples
from nerfstudio.lantern.field import LanternNerfactoField
from nerfstudio.lantern.renderer import RGBRenderer_HDR
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.field_components.spatial_distortions import SceneContraction
from nerfstudio.fields.density_fields import HashMLPDensityField
from nerfstudio.model_components.losses import (
@@ -144,6 +146,57 @@ class LanternModel(NerfactoModel):
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True)
def get_outputs(self, ray_bundle: RayBundle):
ray_samples: RaySamples
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
field_outputs = self.field.forward(ray_samples, compute_normals=self.config.predict_normals)
if self.config.use_gradient_scaling:
field_outputs = scale_gradients_by_distance_squared(field_outputs, ray_samples)
weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY])
weights_list.append(weights)
ray_samples_list.append(ray_samples)
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
depth = self.renderer_depth(weights=weights, ray_samples=ray_samples)
accumulation = self.renderer_accumulation(weights=weights)
outputs = {
"rgb": rgb[:, 0:3],
"accumulation": accumulation,
"depth": depth,
"validity_w": rgb[:, 4],
"validity_f": rgb[:, 3],
}
if self.config.predict_normals:
normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
outputs["normals"] = self.normals_shader(normals)
outputs["pred_normals"] = self.normals_shader(pred_normals)
# These use a lot of GPU memory, so we avoid storing them for eval.
if self.training:
outputs["weights_list"] = weights_list
outputs["ray_samples_list"] = ray_samples_list
if self.training and self.config.predict_normals:
outputs["rendered_orientation_loss"] = orientation_loss(
weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
)
outputs["rendered_pred_normal_loss"] = pred_normal_loss(
weights.detach(),
field_outputs[FieldHeadNames.NORMALS].detach(),
field_outputs[FieldHeadNames.PRED_NORMALS],
)
for i in range(self.config.num_proposal_iterations):
outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
return outputs
def get_image_metrics_and_images(
self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
@@ -192,7 +245,7 @@ class LanternModel(NerfactoModel):
return metrics_dict, images_dict
def weighting_function(self, pixels, zmin, zmax):
def hat_weighting_function(self, pixels, zmin, zmax):
threshold = 0.5 * (zmin + zmax)
weights = torch.zeros(pixels.shape, dtype = pixels.dtype).to(self.device)
weights = weights + torch.finfo(torch.float32).eps
@@ -206,35 +259,80 @@ class LanternModel(NerfactoModel):
weights[mask_upperbound] = 2.0 * (zmax - pixels[mask_upperbound])
return weights
def cut_weighting_function(self, pixels, exposures):
# TODO make sure the values for cuts are not fixed
well_exposure = torch.max(exposures)
fast_exposure = torch.min(exposures)
weights = torch.ones(pixels.shape, dtype = pixels.dtype).to(self.device)
fast_expo_valids = (exposures == fast_exposure).repeat(1,pixels.shape[-1])
weights[fast_expo_valids] = torch.clip(10.0 * (pixels[fast_expo_valids] - 0.1), 0.0, 1.0)
well_expo_valids = ~fast_expo_valids
weights[well_expo_valids] = torch.clip(-20.0 * (pixels[well_expo_valids] - 0.9) + 1, 0.0, 1.0)
return weights
def cut_weighting_for_mask_function(self, pixels, exposures):
well_exposure = torch.max(exposures)
fast_exposure = torch.min(exposures)
weights1 = torch.zeros(exposures.shape, dtype = exposures.dtype).to(self.device)
weights2 = torch.zeros(exposures.shape, dtype = exposures.dtype).to(self.device)
fast_expo_valids = (exposures == fast_exposure).repeat(1, 1)
weights1[fast_expo_valids] = 1.0
well_expo_valids = ~fast_expo_valids
weights2[well_expo_valids] = 1.0
return weights1, weights2
def get_loss_dict(self, outputs, batch, metrics_dict=None):
loss_dict = {}
weights_for_loss = None
weights_for_RGB_loss = None
image = batch["image"].to(self.device)
if "exposure" in batch:
u = 5000.
img_uncompress = torch.exp(image * torch.log(torch.tensor(u+1.))) - 1.
img_uncompress /= u
exposures_resized = batch["exposure"].view(batch["exposure"].shape[0], 1)
img_uncompress_re_exposed = exposures_resized * img_uncompress
weights_for_loss = self.weighting_function(torch.clip(img_uncompress_re_exposed, 0, 1), 0.0, 1.0)
# Debevec Hat weights for Loss
# weights_for_loss = self.hat_weighting_function(torch.clip(img_uncompress_re_exposed, 0, 1), 0.0, 1.0)
# Clear cut weights for Loss
weights_for_RGB_loss = self.cut_weighting_function(torch.clip(img_uncompress_re_exposed, 0, 1), exposures_resized)
mask1_w, mask2_w = self.cut_weighting_for_mask_function(img_uncompress_re_exposed, exposures_resized)
pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation(
pred_image=outputs["rgb"],
pred_accumulation=outputs["accumulation"],
gt_image=image,
)
if weights_for_loss is not None:
loss_dict["rgb_loss"] = (weights_for_loss * ((gt_rgb - pred_rgb) ** 2)).mean()
# MSE loss for mask with weights
mask_in_float = batch['mask'].type(torch.float).to(self.device)
negative_mask_in_float = torch.ones(mask_in_float.shape).to(self.device) - mask_in_float
loss_mask_fast_expo = (mask1_w * (( negative_mask_in_float - torch.unsqueeze(outputs["validity_f"], 1)) ** 2)).mean()
loss_mask_well_expo = (mask2_w * (( mask_in_float - torch.unsqueeze(outputs["validity_w"], 1)) ** 2)).mean()
loss_dict["validity_loss_well_exposed"] = loss_mask_well_expo
loss_dict["validity_loss_fast_exposure"] = loss_mask_fast_expo
if weights_for_RGB_loss is not None:
loss_dict["rgb_loss"] = (weights_for_RGB_loss * ((gt_rgb - pred_rgb) ** 2)).mean()
else:
loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb)
if self.training:
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
outputs["weights_list"], outputs["ray_samples_list"]
+17
View File
@@ -156,6 +156,7 @@ def _render_trajectory_video(
sys.exit(1)
output_image = outputs[rendered_output_name]
is_depth = rendered_output_name.find("depth") != -1
is_validity = rendered_output_name.find("validity_w") != -1 or rendered_output_name.find("validity_f") != -1
if is_depth:
output_image = (
colormaps.apply_depth_colormap(
@@ -168,6 +169,22 @@ def _render_trajectory_video(
.cpu()
.numpy()
)
elif is_validity:
mask_f = torch.clip(outputs["validity_f"], 0, 1)
mask_f = torch.ones(mask_f.shape).to(mask_f.get_device()) - mask_f
final_mask = torch.clip(outputs["validity_w"], 0, 1) + mask_f
final_mask[final_mask > 0.5] = 1
final_mask[final_mask <= 0.5] = 0
output_image = (
colormaps.apply_colormap(
final_mask,
colormap_options=colormap_options,
)
.cpu()
.numpy()
)
elif image_format == "exr":
u = 5000.
output_image = torch.exp(output_image * torch.log(torch.tensor(u+1.))) - 1.