From 54d5bff3515dd8f23a3dfdea98f16316c631f9c0 Mon Sep 17 00:00:00 2001 From: mokad6 Date: Thu, 18 Jan 2024 09:36:32 -0500 Subject: [PATCH] Stage 3 of Pandora --- .../convert_pandora_poses_to_full_HDR.py | 47 +++++++++++++++++++ nerfstudio/lantern/model.py | 15 +++--- nerfstudio/scripts/render.py | 10 ++-- 3 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 lantern_scripts/convert_pandora_poses_to_full_HDR.py diff --git a/lantern_scripts/convert_pandora_poses_to_full_HDR.py b/lantern_scripts/convert_pandora_poses_to_full_HDR.py new file mode 100644 index 000000000..4cff241a9 --- /dev/null +++ b/lantern_scripts/convert_pandora_poses_to_full_HDR.py @@ -0,0 +1,47 @@ +import argparse +import os +from tqdm import tqdm +from pathlib import Path +import json + + +def load_from_json(filename: Path): + """Load a dictionary from a JSON filename. + + Args: + filename: The filename to load from. + """ + assert filename.suffix == ".json" + with open(filename, encoding="UTF-8") as file: + return json.load(file) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--input_json', type=str) + parser.add_argument('--output_json', type=str) + parser.add_argument('--img_addr', type=str) + args = parser.parse_args() + + assert os.path.isfile(args.input_json), 'The input JSON file is missing!' + + old_json_data = load_from_json( Path(args.input_json) ) + new_json_data = {} + all_keys = old_json_data.keys() - {'frames'} + for key in all_keys: + new_json_data[key] = old_json_data[key] + frames = [] + for frame in tqdm(old_json_data['frames']): + base = str(Path(frame["file_path"]).stem) + if Path(args.img_addr + '/' + base + '.exr').is_file(): + new_frame = { + "file_path": frame["file_path"], + "transform_matrix": frame["transform_matrix"], + } + frames.append(new_frame) + new_json_data['frames'] = frames + + with open(args.output_json, "w") as outfile: + json.dump(new_json_data, outfile) + + print("All done!") \ No newline at end of file diff --git a/nerfstudio/lantern/model.py b/nerfstudio/lantern/model.py index 2d642dc1a..b5ff7dc13 100644 --- a/nerfstudio/lantern/model.py +++ b/nerfstudio/lantern/model.py @@ -320,13 +320,14 @@ class LanternModel(NerfactoModel): gt_image=image, ) - # MSE loss for mask with weights - mask_in_float = batch['mask'].type(torch.float).to(self.device) - negative_mask_in_float = torch.ones(mask_in_float.shape).to(self.device) - mask_in_float - loss_mask_fast_expo = (mask1_w * (( negative_mask_in_float - torch.unsqueeze(outputs["validity_f"], 1)) ** 2)).mean() - loss_mask_well_expo = (mask2_w * (( mask_in_float - torch.unsqueeze(outputs["validity_w"], 1)) ** 2)).mean() - loss_dict["validity_loss_well_exposed"] = loss_mask_well_expo - loss_dict["validity_loss_fast_exposure"] = loss_mask_fast_expo + if "mask" in batch: + # MSE loss for mask with weights + mask_in_float = batch['mask'].type(torch.float).to(self.device) + negative_mask_in_float = torch.ones(mask_in_float.shape).to(self.device) - mask_in_float + loss_mask_fast_expo = (mask1_w * (( negative_mask_in_float - torch.unsqueeze(outputs["validity_f"], 1)) ** 2)).mean() + loss_mask_well_expo = (mask2_w * (( mask_in_float - torch.unsqueeze(outputs["validity_w"], 1)) ** 2)).mean() + loss_dict["validity_loss_well_exposed"] = loss_mask_well_expo + loss_dict["validity_loss_fast_exposure"] = loss_mask_fast_expo if weights_for_RGB_loss is not None: loss_dict["rgb_loss"] = (weights_for_RGB_loss * ((gt_rgb - pred_rgb) ** 2)).mean() diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index 83edc8f45..e0322723b 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -549,16 +549,20 @@ class RenderInterpolated(BaseRender): install_checks.check_ffmpeg_installed() + output_names = [] if self.pose_source == "eval": assert pipeline.datamanager.eval_dataset is not None cameras = pipeline.datamanager.eval_dataset.cameras + for file_name in pipeline.datamanager.eval_dataset.image_filenames: + output_names.append(file_name.stem) else: assert pipeline.datamanager.train_dataset is not None cameras = pipeline.datamanager.train_dataset.cameras + for file_name in pipeline.datamanager.train_dataset.image_filenames: + output_names.append(file_name.stem) - output_names = [] - for file_name in pipeline.datamanager.train_dataset.image_filenames: - output_names.append(file_name.stem) + + seconds = self.interpolation_steps * len(cameras) / self.frame_rate camera_path = get_interpolated_camera_path( cameras=cameras,