From be2b4360248ea5450fc9c5d5565d0144c9262921 Mon Sep 17 00:00:00 2001 From: lefreud Date: Fri, 10 May 2024 01:08:43 +0000 Subject: [PATCH] WIP hdr-nerfacto --- .gitignore | 1 + .../generate_small_subset_hdr_nerf.py | 4 ++-- nerfstudio/hdr_nerfacto/hdr_nerf_field.py | 2 +- nerfstudio/hdr_nerfacto/hdr_nerf_model.py | 2 ++ nerfstudio/lantern/dataset.py | 14 +++++++------- nerfstudio/models/nerfacto.py | 18 +++++++++++++++--- 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 661bf0136..6ec98f95b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Ours Dual_Cameras gt_test +hdr-nerfacto-subset-out # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/lantern_scripts/generate_small_subset_hdr_nerf.py b/lantern_scripts/generate_small_subset_hdr_nerf.py index e9ab61010..45133ed6d 100644 --- a/lantern_scripts/generate_small_subset_hdr_nerf.py +++ b/lantern_scripts/generate_small_subset_hdr_nerf.py @@ -29,8 +29,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--input_json', type=str, required=True) parser.add_argument('--output_dir', type=str, required=True) - parser.add_argument('--train_split_fraction', type=float, default= 0.8) - parser.add_argument('--eval_split_fraction', type=float, default= 0.1) + parser.add_argument('--train_split_fraction', type=float, default= 0.60) + parser.add_argument('--eval_split_fraction', type=float, default= 0.01) parser.add_argument('--min_frame_number', type=int, default=100) parser.add_argument('--frames_per_direction', type=int, default=100) args = parser.parse_args() diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py index 3f726d115..362d5d2c5 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py @@ -247,7 +247,7 @@ class HdrNerfactoField(Field): exposures = torch.ones(ray_samples.frustums.directions.shape[:-1], device=ray_samples.frustums.directions.device)[..., None] else: exposures = ray_samples.metadata['exposures'].to(ray_samples.frustums.directions.device) - exposures = exposures.reshape(-1, 1).clone().detach() / 200 + exposures = exposures.reshape(-1, 1).clone().detach() assert density_embedding is not None outputs = {} diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py index f10b1e005..662ab9f80 100644 --- a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -358,6 +358,7 @@ class HdrNerfactoModel(Model): def get_loss_dict(self, outputs, batch, metrics_dict=None): loss_dict = {} image = batch["image"].to(self.device) + pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( pred_image=outputs["rgb"], pred_accumulation=outputs["accumulation"], @@ -369,6 +370,7 @@ class HdrNerfactoModel(Model): ) loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb) + # TODO: check that # if self.training: # loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( # outputs["weights_list"], outputs["ray_samples_list"] diff --git a/nerfstudio/lantern/dataset.py b/nerfstudio/lantern/dataset.py index b3d493941..a0dc0061a 100644 --- a/nerfstudio/lantern/dataset.py +++ b/nerfstudio/lantern/dataset.py @@ -58,9 +58,9 @@ class HDRNerfactoInputDataset(InputDataset): Args: image_idx: The image index in the dataset. """ - image_filename = self._dataparser_outputs.image_filenames[image_idx] + image_filename = str(self._dataparser_outputs.image_filenames[image_idx]).replace('_linear.exr', '.png') #print('image_filename', image_filename) - hdr_image = cv2.imread(str(image_filename), cv2.IMREAD_UNCHANGED) + hdr_image = cv2.imread(str(image_filename)) # print('hdr_image', hdr_image.shape, hdr_image.dtype) # print('hdr_image', hdr_image.max(), hdr_image.min()) # print('hdr_image', hdr_image[0,0,:]) @@ -70,8 +70,8 @@ class HDRNerfactoInputDataset(InputDataset): height, width = hdr_image.shape[:2] newsize = (int(width * self.scale_factor), int(height * self.scale_factor)) hdr_image = cv2.resize(hdr_image, newsize, interpolation = cv2.INTER_LINEAR) - hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB) - hdr_image = hdr_image.astype("float32") # shape is (h, w) or (h, w, 3 or 4) + hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB) + hdr_image = hdr_image.astype("float32") / 255 # shape is (h, w) or (h, w, 3 or 4) if len(hdr_image.shape) == 2: hdr_image = hdr_image[:, :, None].repeat(3, axis=2) assert len(hdr_image.shape) == 3 @@ -87,11 +87,11 @@ class HDRNerfactoInputDataset(InputDataset): """ image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32")) #print('a', image.max()) - image = image * self._dataparser_outputs.exposures[image_idx] + # image = image * self._dataparser_outputs.exposures[image_idx] # print('b', image.max()) - if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: - image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) + # if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: + # image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) return image def get_data(self, image_idx: int) -> Dict: diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 1d243632b..c09b58c93 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -29,7 +29,11 @@ from torchmetrics.image import PeakSignalNoiseRatio from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle, RaySamples -from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction from nerfstudio.fields.density_fields import HashMLPDensityField @@ -42,8 +46,16 @@ from nerfstudio.model_components.losses import ( pred_normal_loss, scale_gradients_by_distance_squared, ) -from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler, UniformSampler -from nerfstudio.model_components.renderers import AccumulationRenderer, DepthRenderer, NormalsRenderer, RGBRenderer +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + NormalsRenderer, + RGBRenderer, +) from nerfstudio.model_components.scene_colliders import NearFarCollider from nerfstudio.model_components.shaders import NormalsShader from nerfstudio.models.base_model import Model, ModelConfig