This commit is contained in:
lefreud committed 2024-05-13 00:05:46 +00:00
1 parent 745bfa56e7
commit 3ba1d6b767
7 files changed
+144 -61

No files matched your search

+18 -2
View File
@@ -115,7 +115,7 @@ After convergence, go to the next step.
ns-train lantern-nerfacto --help
```
If you need to use less frames, add this argument: `--pipeline.datamanager.train-num-images-to-sample-from 200 --pipeline.datamanager.eval-num-images-to-sample-from 30`
If you need to use less frames, add this argument: `--pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30`
> Thing to keep in mind: the way Nerfstudio splits train and test images may not be optimal: left might be in one dataset and right in another... that's a bit of a data leak.
@@ -134,6 +134,22 @@ TODO: validate this part
## 4. Evaluation
```
ns-train hdr-nerfacto --data data/lab_downstairs_ns --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30
```
```
ns-eval --load-config outputs/LVSN_LAB_OpenSFM/hdr-nerfacto/2024-05-10_032723/config.yml --output-path ablations_out/hdr_nerfacto/output.json --render-output-path ablations_out/hdr-nerfacto
```
```
```
ns-render camera-path --load-config outputs/lab_downstairs_ns/hdr-nerfacto/2024-05-10_211317/config.yml --camera-path-filename data/lab_downstairs_ns/GT_transforms.json --output-path renders/lab_downstairs_ns/2024-05-10_211317/hdr-nerfacto --output-format images --image-format exr
```
```
docker run -it --rm -v /gel/usr/$USER:/mnt/workspace/ -v /home-local2/$USER.extra.nobkp/:/mnt/data/ --gpus '"device=1"' -p 7007:7007 lantern_docker:latest /bin/bash
export PYTHONPATH=$PYTHONPATH:$(pwd)
```
docker exec -it 57d2bf549c9a /bin/bash
ns-train hdr-nerfacto --data data/lab_downstairs_ns/ --pipeline.datamanager.train-num-images-to-sample-from 1800 --viewer.websocket-port 9009
+20 -3
View File
@@ -12,7 +12,7 @@ Train:
```
export SCENE=lab_downstairs_ns
ns-train --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30
ns-train hdr-nerfacto --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30
```
Eval (GT+normal):
@@ -21,8 +21,25 @@ export SCENE=lab_downstairs_ns
export MODEL_NAME=hdr-nerfacto/2024-05-11_141957
ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \
ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format exr --rendered-output-names rgb ; \
ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format exr --rendered-output-names rgb_fast ; \
ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \
ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \
ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr
```
Move (outside docker)
```
mkdir -p /gel/usr/mokad6/ml_class/fred-out/ ; \
cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/ablations_out /gel/usr/mokad6/ml_class/fred-out/ns-eval ; \
cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/renders /gel/usr/mokad6/ml_class/fred-out/ns-render
```
1800 samples:
export SCENE=lab_downstairs_ns
export MODEL_NAME=hdr-nerfacto/2024-05-12_115558
Test wo CRF
export SCENE=lab_downstairs_ns
export MODEL_NAME=hdr-nerfacto/2024-05-12_115558
+3 -1
View File
@@ -78,6 +78,7 @@ descriptions = {
"nerfacto-HDR": "Nerfacto that take HDR as input, and output HDR field.",
"lantern-nerfacto": "Nerfacto that take HDR as input, and output HDR field.",
"hdr-nerfacto": "HDR-Nerfacto, adapted to the Nerfacto model.",
"hdr-nerfacto-wo-crf": "HDR-Nerfacto without CRF, adapted to the Nerfacto model.",
"depth-nerfacto": "Nerfacto with depth supervision.",
"volinga": "Real-time rendering model from Volinga. Directly exportable to NVOL format at https://volinga.ai/",
"instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for unbounded scenes.",
@@ -128,7 +129,8 @@ method_configs["nerfacto"] = TrainerConfig(
method_configs["lantern-nerfacto"] = Get_lantern_config()
method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config()
method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config(use_crf=True)
method_configs["hdr-nerfacto-wo-crf"] = get_hdr_nerfacto_config(use_crf=False)
method_configs["nerfacto-big"] = TrainerConfig(
method_name="nerfacto",
+14 -5
View File
@@ -6,19 +6,27 @@ from nerfstudio.engine.optimizers import AdamOptimizerConfig
from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig
from nerfstudio.engine.trainer import TrainerConfig
from nerfstudio.hdr_nerfacto.hdr_nerf_model import HdrNerfactoModelConfig
from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig
from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig, HDRNerfactoWoCrfDataManagerConfig
from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig
def get_hdr_nerfacto_config():
def get_hdr_nerfacto_config(*, use_crf=True):
if use_crf:
method_name = "hdr-nerfacto"
else:
method_name = "hdr-nerfacto-wo-crf"
datamanager_config_class = HDRNerfactoDataManagerConfig if use_crf else HDRNerfactoWoCrfDataManagerConfig
return TrainerConfig(
method_name="hdr-nerfacto",
method_name=method_name,
steps_per_eval_batch=500,
steps_per_save=2000,
# max_num_iterations=1000,
max_num_iterations=30000,
mixed_precision=True,
pipeline=VanillaPipelineConfig(
datamanager=HDRNerfactoDataManagerConfig(
datamanager=datamanager_config_class(
dataparser=NerfstudioDataParserConfig(),
train_num_rays_per_batch=4096,
eval_num_rays_per_batch=4096,
@@ -27,7 +35,8 @@ def get_hdr_nerfacto_config():
),
),
model=HdrNerfactoModelConfig(
eval_num_rays_per_chunk=1 << 15
eval_num_rays_per_chunk=1 << 15,
use_crf=use_crf,
),
),
optimizers={
+41 -30
View File
@@ -96,6 +96,7 @@ class HdrNerfactoField(Field):
use_average_appearance_embedding: bool = False,
spatial_distortion: Optional[SpatialDistortion] = None,
implementation: Literal["tcnn", "torch"] = "tcnn",
use_crf: bool = True,
) -> None:
super().__init__()
@@ -200,17 +201,19 @@ class HdrNerfactoField(Field):
implementation=implementation,
)
# HDR NeRF, 1 MLP per channel
self.crf_mlps = nn.ModuleList([
MLP(in_dim=1,
num_layers=2,
layer_width=128,
out_dim=1,
activation=nn.ReLU(),
out_activation=nn.Sigmoid(),
implementation=implementation,
) for _ in range(3)
])
self.use_crf = use_crf
if use_crf:
# HDR NeRF, 1 MLP per channel
self.crf_mlps = nn.ModuleList([
MLP(in_dim=1,
num_layers=2,
layer_width=128,
out_dim=1,
activation=nn.ReLU(),
out_activation=nn.Sigmoid(),
implementation=implementation,
) for _ in range(3)
])
def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]:
@@ -316,29 +319,37 @@ class HdrNerfactoField(Field):
dim=-1,
)
log_rgb_hdr = self.mlp_head(h) # .view(*outputs_shape, -1).to(directions)
rgb_hdr = torch.exp(log_rgb_hdr).view(*outputs_shape, -1).to(directions)
log_rgb_hdr_exposed = log_rgb_hdr + torch.log(exposures)
log_rgb_hdr_exposed_fast = log_rgb_hdr + torch.log(exposures * 0.004) # TODO: adaptive fast exposure
#print('exposure_min', exposures.min(), 'exposure_max', exposures.max())
# TODO: validate this
rgb_hdr = torch.exp(log_rgb_hdr_exposed).view(*outputs_shape, -1).to(directions)
#print('rgb_hdr_min', rgb_hdr.min(), 'rgb_hdr_max', rgb_hdr.max())
pred_rgb_fast = torch.cat([
self.crf_mlps[channel](log_rgb_hdr_exposed_fast[..., channel:channel+1]) for channel in range(3)
], dim=-1).view(*outputs_shape, -1).to(directions)
# outputs[FieldHeadNames.RGB_HDR] = torch.clamp(rgb_hdr, 0, 1)
outputs[FieldHeadNames.RGB_HDR] = rgb_hdr
pred_rgb_ldr = torch.cat([
self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3)
], dim=-1)
pred_rgb_ldr = pred_rgb_ldr.view(*outputs_shape, -1).to(directions)
if self.use_crf:
log_rgb_hdr_exposed_fast = log_rgb_hdr + torch.log(exposures * 0.004) # TODO: adaptive fast exposure
pred_rgb_fast = torch.cat([
self.crf_mlps[channel](log_rgb_hdr_exposed_fast[..., channel:channel+1]) for channel in range(3)
], dim=-1).view(*outputs_shape, -1).to(directions)
zero_radiance_crf = torch.cat([
self.crf_mlps[channel](torch.zeros((1, 1), requires_grad=False, device=log_rgb_hdr_exposed.device)) for channel in range(3)
], dim=-1)
zero_radiance_crf = zero_radiance_crf.expand(pred_rgb_ldr.shape).to(directions)
pred_rgb_ldr = torch.cat([
self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3)
], dim=-1)
pred_rgb_ldr = pred_rgb_ldr.view(*outputs_shape, -1).to(directions)
outputs.update({
FieldHeadNames.RGB: pred_rgb_ldr,
FieldHeadNames.RGB_FAST: pred_rgb_fast,
FieldHeadNames.RGB_HDR: rgb_hdr,
FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf,
})
zero_radiance_crf = torch.cat([
self.crf_mlps[channel](torch.zeros((1, 1), requires_grad=False, device=log_rgb_hdr_exposed.device)) for channel in range(3)
], dim=-1)
zero_radiance_crf = zero_radiance_crf.expand(pred_rgb_ldr.shape).to(directions)
outputs.update({
FieldHeadNames.RGB: pred_rgb_ldr,
FieldHeadNames.RGB_FAST: pred_rgb_fast,
FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf,
})
return outputs
+36 -19
View File
@@ -142,6 +142,8 @@ class HdrNerfactoModelConfig(ModelConfig):
unit_exposure_loss_mult: float = 0.5
unit_exposure_expected_value: float = 0.5
use_crf: bool = True
class HdrNerfactoModel(Model):
"""Nerfacto model
@@ -178,6 +180,7 @@ class HdrNerfactoModel(Model):
use_average_appearance_embedding=self.config.use_average_appearance_embedding,
appearance_embedding_dim=self.config.appearance_embed_dim,
implementation=self.config.implementation,
use_crf=self.config.use_crf,
)
self.density_fns = []
@@ -301,24 +304,25 @@ class HdrNerfactoModel(Model):
weights_list.append(weights)
ray_samples_list.append(ray_samples)
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights)
depth = self.renderer_depth(weights=weights, ray_samples=ray_samples)
accumulation = self.renderer_accumulation(weights=weights)
# HDR-Nerfacto
rgb_hdr = self.renderer_rgb(
rgb=field_outputs[FieldHeadNames.RGB_HDR], weights=weights
)
depth = self.renderer_depth(weights=weights, ray_samples=ray_samples)
accumulation = self.renderer_accumulation(weights=weights)
outputs = {
"rgb": rgb,
"rgb_fast": rgb_fast,
"accumulation": accumulation,
"depth": depth,
"rgb_hdr": rgb_hdr,
"zero_radiance_crf": field_outputs[FieldHeadNames.ZERO_RADIANCE_CRF],
}
if self.config.use_crf:
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights)
outputs["rgb"] = rgb
outputs["rgb_fast"] = rgb_fast
outputs['zero_radiance_crf'] = field_outputs[FieldHeadNames.ZERO_RADIANCE_CRF]
if self.config.predict_normals:
normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
@@ -350,7 +354,10 @@ class HdrNerfactoModel(Model):
metrics_dict = {}
gt_rgb = batch["image"].to(self.device) # RGB or RGBA image
gt_rgb = self.renderer_rgb.blend_background(gt_rgb) # Blend if RGBA
predicted_rgb = outputs["rgb"]
if self.config.use_crf:
predicted_rgb = outputs["rgb"]
else:
predicted_rgb = outputs["rgb_hdr"]
metrics_dict["psnr"] = self.psnr(predicted_rgb, gt_rgb)
if self.training:
@@ -361,17 +368,24 @@ class HdrNerfactoModel(Model):
loss_dict = {}
image = batch["image"].to(self.device)
pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation(
pred_image=outputs["rgb"],
pred_accumulation=outputs["accumulation"],
gt_image=image,
)
loss_dict["unit_exposure_loss"] = self.config.unit_exposure_loss_mult * self.unit_exposure_loss(
outputs["zero_radiance_crf"], torch.ones_like(outputs["zero_radiance_crf"], requires_grad=False) * self.config.unit_exposure_expected_value
)
if self.config.use_crf:
pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation(
pred_image=outputs["rgb"],
pred_accumulation=outputs["accumulation"],
gt_image=image,
)
loss_dict["unit_exposure_loss"] = self.config.unit_exposure_loss_mult * self.unit_exposure_loss(
outputs["zero_radiance_crf"], torch.ones_like(outputs["zero_radiance_crf"], requires_grad=False) * self.config.unit_exposure_expected_value
)
else:
pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation(
pred_image=outputs["rgb_hdr"],
pred_accumulation=outputs["accumulation"],
gt_image=image,
)
loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb)
# TODO: check that
# if self.training:
# loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
@@ -395,7 +409,10 @@ class HdrNerfactoModel(Model):
self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]:
gt_rgb = batch["image"].to(self.device)
predicted_rgb = outputs["rgb"] # Blended with background (black if random background)
if self.config.use_crf:
predicted_rgb = outputs["rgb"]
else:
predicted_rgb = outputs["rgb_hdr"]
gt_rgb = self.renderer_rgb.blend_background(gt_rgb)
acc = colormaps.apply_colormap(outputs["accumulation"])
depth = colormaps.apply_depth_colormap(
+12 -1
View File
@@ -9,7 +9,11 @@ from nerfstudio.data.datamanagers.base_datamanager import (
VanillaDataManager,
VanillaDataManagerConfig,
)
from nerfstudio.lantern.dataset import HDRInputDataset, HDRNerfactoInputDataset
from nerfstudio.lantern.dataset import (
HDRInputDataset,
HDRNerfactoInputDataset,
HDRNerfactoWoCrfInputDataset,
)
@dataclass
@@ -19,6 +23,13 @@ class HDRNerfactoDataManagerConfig(VanillaDataManagerConfig):
_target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoInputDataset])
"""Target class to instantiate."""
@dataclass
class HDRNerfactoWoCrfDataManagerConfig(VanillaDataManagerConfig):
"""A HDR data manager, based on VanillaDataManager"""
_target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoWoCrfInputDataset])
"""Target class to instantiate."""
@dataclass
class HDRVanillaDataManagerConfig(VanillaDataManagerConfig):