WIP hdr-nerfacto

This commit is contained in:
lefreud committed 2024-05-10 01:08:43 +00:00
1 parent faa8a7dfb6
commit be2b436024
6 files changed
+28 -13

No files matched your search

+1
View File
@@ -1,6 +1,7 @@
# Ours
Dual_Cameras
gt_test
hdr-nerfacto-subset-out
# Byte-compiled / optimized / DLL files
__pycache__/
@@ -29,8 +29,8 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--input_json', type=str, required=True)
parser.add_argument('--output_dir', type=str, required=True)
parser.add_argument('--train_split_fraction', type=float, default= 0.8)
parser.add_argument('--eval_split_fraction', type=float, default= 0.1)
parser.add_argument('--train_split_fraction', type=float, default= 0.60)
parser.add_argument('--eval_split_fraction', type=float, default= 0.01)
parser.add_argument('--min_frame_number', type=int, default=100)
parser.add_argument('--frames_per_direction', type=int, default=100)
args = parser.parse_args()
+1 -1
View File
@@ -247,7 +247,7 @@ class HdrNerfactoField(Field):
exposures = torch.ones(ray_samples.frustums.directions.shape[:-1], device=ray_samples.frustums.directions.device)[..., None]
else:
exposures = ray_samples.metadata['exposures'].to(ray_samples.frustums.directions.device)
exposures = exposures.reshape(-1, 1).clone().detach() / 200
exposures = exposures.reshape(-1, 1).clone().detach()
assert density_embedding is not None
outputs = {}
@@ -358,6 +358,7 @@ class HdrNerfactoModel(Model):
def get_loss_dict(self, outputs, batch, metrics_dict=None):
loss_dict = {}
image = batch["image"].to(self.device)
pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation(
pred_image=outputs["rgb"],
pred_accumulation=outputs["accumulation"],
@@ -369,6 +370,7 @@ class HdrNerfactoModel(Model):
)
loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb)
# TODO: check that
# if self.training:
# loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
# outputs["weights_list"], outputs["ray_samples_list"]
+7 -7
View File
@@ -58,9 +58,9 @@ class HDRNerfactoInputDataset(InputDataset):
Args:
image_idx: The image index in the dataset.
"""
image_filename = self._dataparser_outputs.image_filenames[image_idx]
image_filename = str(self._dataparser_outputs.image_filenames[image_idx]).replace('_linear.exr', '.png')
#print('image_filename', image_filename)
hdr_image = cv2.imread(str(image_filename), cv2.IMREAD_UNCHANGED)
hdr_image = cv2.imread(str(image_filename))
# print('hdr_image', hdr_image.shape, hdr_image.dtype)
# print('hdr_image', hdr_image.max(), hdr_image.min())
# print('hdr_image', hdr_image[0,0,:])
@@ -70,8 +70,8 @@ class HDRNerfactoInputDataset(InputDataset):
height, width = hdr_image.shape[:2]
newsize = (int(width * self.scale_factor), int(height * self.scale_factor))
hdr_image = cv2.resize(hdr_image, newsize, interpolation = cv2.INTER_LINEAR)
hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB)
hdr_image = hdr_image.astype("float32") # shape is (h, w) or (h, w, 3 or 4)
hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB)
hdr_image = hdr_image.astype("float32") / 255 # shape is (h, w) or (h, w, 3 or 4)
if len(hdr_image.shape) == 2:
hdr_image = hdr_image[:, :, None].repeat(3, axis=2)
assert len(hdr_image.shape) == 3
@@ -87,11 +87,11 @@ class HDRNerfactoInputDataset(InputDataset):
"""
image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32"))
#print('a', image.max())
image = image * self._dataparser_outputs.exposures[image_idx]
# image = image * self._dataparser_outputs.exposures[image_idx]
# print('b', image.max())
if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4:
image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:])
# if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4:
# image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:])
return image
def get_data(self, image_idx: int) -> Dict:
+15 -3
View File
@@ -29,7 +29,11 @@ from torchmetrics.image import PeakSignalNoiseRatio
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
from nerfstudio.cameras.rays import RayBundle, RaySamples
from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation
from nerfstudio.engine.callbacks import (
TrainingCallback,
TrainingCallbackAttributes,
TrainingCallbackLocation,
)
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.field_components.spatial_distortions import SceneContraction
from nerfstudio.fields.density_fields import HashMLPDensityField
@@ -42,8 +46,16 @@ from nerfstudio.model_components.losses import (
pred_normal_loss,
scale_gradients_by_distance_squared,
)
from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler, UniformSampler
from nerfstudio.model_components.renderers import AccumulationRenderer, DepthRenderer, NormalsRenderer, RGBRenderer
from nerfstudio.model_components.ray_samplers import (
ProposalNetworkSampler,
UniformSampler,
)
from nerfstudio.model_components.renderers import (
AccumulationRenderer,
DepthRenderer,
NormalsRenderer,
RGBRenderer,
)
from nerfstudio.model_components.scene_colliders import NearFarCollider
from nerfstudio.model_components.shaders import NormalsShader
from nerfstudio.models.base_model import Model, ModelConfig