mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 13:53:44 +08:00
wo_CRF
This commit is contained in:
1 parent
745bfa56e7
commit
3ba1d6b767
7 files changed
+144
-61
No files matched your search
+18
-2
@@ -115,7 +115,7 @@ After convergence, go to the next step.
|
||||
ns-train lantern-nerfacto --help
|
||||
```
|
||||
|
||||
If you need to use less frames, add this argument: `--pipeline.datamanager.train-num-images-to-sample-from 200 --pipeline.datamanager.eval-num-images-to-sample-from 30`
|
||||
If you need to use less frames, add this argument: `--pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30`
|
||||
|
||||
> Thing to keep in mind: the way Nerfstudio splits train and test images may not be optimal: left might be in one dataset and right in another... that's a bit of a data leak.
|
||||
|
||||
@@ -134,6 +134,22 @@ TODO: validate this part
|
||||
|
||||
## 4. Evaluation
|
||||
|
||||
```
|
||||
ns-train hdr-nerfacto --data data/lab_downstairs_ns --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30
|
||||
```
|
||||
|
||||
```
|
||||
ns-eval --load-config outputs/LVSN_LAB_OpenSFM/hdr-nerfacto/2024-05-10_032723/config.yml --output-path ablations_out/hdr_nerfacto/output.json --render-output-path ablations_out/hdr-nerfacto
|
||||
```
|
||||
```
|
||||
|
||||
```
|
||||
ns-render camera-path --load-config outputs/lab_downstairs_ns/hdr-nerfacto/2024-05-10_211317/config.yml --camera-path-filename data/lab_downstairs_ns/GT_transforms.json --output-path renders/lab_downstairs_ns/2024-05-10_211317/hdr-nerfacto --output-format images --image-format exr
|
||||
```
|
||||
|
||||
```
|
||||
docker run -it --rm -v /gel/usr/$USER:/mnt/workspace/ -v /home-local2/$USER.extra.nobkp/:/mnt/data/ --gpus '"device=1"' -p 7007:7007 lantern_docker:latest /bin/bash
|
||||
export PYTHONPATH=$PYTHONPATH:$(pwd)
|
||||
```
|
||||
|
||||
docker exec -it 57d2bf549c9a /bin/bash
|
||||
ns-train hdr-nerfacto --data data/lab_downstairs_ns/ --pipeline.datamanager.train-num-images-to-sample-from 1800 --viewer.websocket-port 9009
|
||||
+20
-3
@@ -12,7 +12,7 @@ Train:
|
||||
```
|
||||
export SCENE=lab_downstairs_ns
|
||||
|
||||
ns-train --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30
|
||||
ns-train hdr-nerfacto --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 400 --pipeline.datamanager.eval-num-images-to-sample-from 30
|
||||
```
|
||||
|
||||
Eval (GT+normal):
|
||||
@@ -21,8 +21,25 @@ export SCENE=lab_downstairs_ns
|
||||
export MODEL_NAME=hdr-nerfacto/2024-05-11_141957
|
||||
|
||||
ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \
|
||||
ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format exr --rendered-output-names rgb ; \
|
||||
ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format exr --rendered-output-names rgb_fast ; \
|
||||
ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \
|
||||
ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \
|
||||
ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr
|
||||
```
|
||||
|
||||
Move (outside docker)
|
||||
```
|
||||
mkdir -p /gel/usr/mokad6/ml_class/fred-out/ ; \
|
||||
cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/ablations_out /gel/usr/mokad6/ml_class/fred-out/ns-eval ; \
|
||||
cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/renders /gel/usr/mokad6/ml_class/fred-out/ns-render
|
||||
```
|
||||
|
||||
1800 samples:
|
||||
export SCENE=lab_downstairs_ns
|
||||
export MODEL_NAME=hdr-nerfacto/2024-05-12_115558
|
||||
|
||||
|
||||
|
||||
Test wo CRF
|
||||
|
||||
export SCENE=lab_downstairs_ns
|
||||
export MODEL_NAME=hdr-nerfacto/2024-05-12_115558
|
||||
@@ -78,6 +78,7 @@ descriptions = {
|
||||
"nerfacto-HDR": "Nerfacto that take HDR as input, and output HDR field.",
|
||||
"lantern-nerfacto": "Nerfacto that take HDR as input, and output HDR field.",
|
||||
"hdr-nerfacto": "HDR-Nerfacto, adapted to the Nerfacto model.",
|
||||
"hdr-nerfacto-wo-crf": "HDR-Nerfacto without CRF, adapted to the Nerfacto model.",
|
||||
"depth-nerfacto": "Nerfacto with depth supervision.",
|
||||
"volinga": "Real-time rendering model from Volinga. Directly exportable to NVOL format at https://volinga.ai/",
|
||||
"instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for unbounded scenes.",
|
||||
@@ -128,7 +129,8 @@ method_configs["nerfacto"] = TrainerConfig(
|
||||
|
||||
method_configs["lantern-nerfacto"] = Get_lantern_config()
|
||||
|
||||
method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config()
|
||||
method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config(use_crf=True)
|
||||
method_configs["hdr-nerfacto-wo-crf"] = get_hdr_nerfacto_config(use_crf=False)
|
||||
|
||||
method_configs["nerfacto-big"] = TrainerConfig(
|
||||
method_name="nerfacto",
|
||||
|
||||
@@ -6,19 +6,27 @@ from nerfstudio.engine.optimizers import AdamOptimizerConfig
|
||||
from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig
|
||||
from nerfstudio.engine.trainer import TrainerConfig
|
||||
from nerfstudio.hdr_nerfacto.hdr_nerf_model import HdrNerfactoModelConfig
|
||||
from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig
|
||||
from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig, HDRNerfactoWoCrfDataManagerConfig
|
||||
from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig
|
||||
|
||||
|
||||
def get_hdr_nerfacto_config():
|
||||
def get_hdr_nerfacto_config(*, use_crf=True):
|
||||
if use_crf:
|
||||
method_name = "hdr-nerfacto"
|
||||
else:
|
||||
method_name = "hdr-nerfacto-wo-crf"
|
||||
|
||||
datamanager_config_class = HDRNerfactoDataManagerConfig if use_crf else HDRNerfactoWoCrfDataManagerConfig
|
||||
|
||||
return TrainerConfig(
|
||||
method_name="hdr-nerfacto",
|
||||
method_name=method_name,
|
||||
steps_per_eval_batch=500,
|
||||
steps_per_save=2000,
|
||||
# max_num_iterations=1000,
|
||||
max_num_iterations=30000,
|
||||
mixed_precision=True,
|
||||
pipeline=VanillaPipelineConfig(
|
||||
datamanager=HDRNerfactoDataManagerConfig(
|
||||
datamanager=datamanager_config_class(
|
||||
dataparser=NerfstudioDataParserConfig(),
|
||||
train_num_rays_per_batch=4096,
|
||||
eval_num_rays_per_batch=4096,
|
||||
@@ -27,7 +35,8 @@ def get_hdr_nerfacto_config():
|
||||
),
|
||||
),
|
||||
model=HdrNerfactoModelConfig(
|
||||
eval_num_rays_per_chunk=1 << 15
|
||||
eval_num_rays_per_chunk=1 << 15,
|
||||
use_crf=use_crf,
|
||||
),
|
||||
),
|
||||
optimizers={
|
||||
|
||||
@@ -96,6 +96,7 @@ class HdrNerfactoField(Field):
|
||||
use_average_appearance_embedding: bool = False,
|
||||
spatial_distortion: Optional[SpatialDistortion] = None,
|
||||
implementation: Literal["tcnn", "torch"] = "tcnn",
|
||||
use_crf: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -200,17 +201,19 @@ class HdrNerfactoField(Field):
|
||||
implementation=implementation,
|
||||
)
|
||||
|
||||
# HDR NeRF, 1 MLP per channel
|
||||
self.crf_mlps = nn.ModuleList([
|
||||
MLP(in_dim=1,
|
||||
num_layers=2,
|
||||
layer_width=128,
|
||||
out_dim=1,
|
||||
activation=nn.ReLU(),
|
||||
out_activation=nn.Sigmoid(),
|
||||
implementation=implementation,
|
||||
) for _ in range(3)
|
||||
])
|
||||
self.use_crf = use_crf
|
||||
if use_crf:
|
||||
# HDR NeRF, 1 MLP per channel
|
||||
self.crf_mlps = nn.ModuleList([
|
||||
MLP(in_dim=1,
|
||||
num_layers=2,
|
||||
layer_width=128,
|
||||
out_dim=1,
|
||||
activation=nn.ReLU(),
|
||||
out_activation=nn.Sigmoid(),
|
||||
implementation=implementation,
|
||||
) for _ in range(3)
|
||||
])
|
||||
|
||||
|
||||
def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]:
|
||||
@@ -316,29 +319,37 @@ class HdrNerfactoField(Field):
|
||||
dim=-1,
|
||||
)
|
||||
log_rgb_hdr = self.mlp_head(h) # .view(*outputs_shape, -1).to(directions)
|
||||
rgb_hdr = torch.exp(log_rgb_hdr).view(*outputs_shape, -1).to(directions)
|
||||
|
||||
log_rgb_hdr_exposed = log_rgb_hdr + torch.log(exposures)
|
||||
log_rgb_hdr_exposed_fast = log_rgb_hdr + torch.log(exposures * 0.004) # TODO: adaptive fast exposure
|
||||
#print('exposure_min', exposures.min(), 'exposure_max', exposures.max())
|
||||
# TODO: validate this
|
||||
rgb_hdr = torch.exp(log_rgb_hdr_exposed).view(*outputs_shape, -1).to(directions)
|
||||
#print('rgb_hdr_min', rgb_hdr.min(), 'rgb_hdr_max', rgb_hdr.max())
|
||||
|
||||
pred_rgb_fast = torch.cat([
|
||||
self.crf_mlps[channel](log_rgb_hdr_exposed_fast[..., channel:channel+1]) for channel in range(3)
|
||||
], dim=-1).view(*outputs_shape, -1).to(directions)
|
||||
# outputs[FieldHeadNames.RGB_HDR] = torch.clamp(rgb_hdr, 0, 1)
|
||||
outputs[FieldHeadNames.RGB_HDR] = rgb_hdr
|
||||
|
||||
pred_rgb_ldr = torch.cat([
|
||||
self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3)
|
||||
], dim=-1)
|
||||
pred_rgb_ldr = pred_rgb_ldr.view(*outputs_shape, -1).to(directions)
|
||||
if self.use_crf:
|
||||
log_rgb_hdr_exposed_fast = log_rgb_hdr + torch.log(exposures * 0.004) # TODO: adaptive fast exposure
|
||||
pred_rgb_fast = torch.cat([
|
||||
self.crf_mlps[channel](log_rgb_hdr_exposed_fast[..., channel:channel+1]) for channel in range(3)
|
||||
], dim=-1).view(*outputs_shape, -1).to(directions)
|
||||
|
||||
zero_radiance_crf = torch.cat([
|
||||
self.crf_mlps[channel](torch.zeros((1, 1), requires_grad=False, device=log_rgb_hdr_exposed.device)) for channel in range(3)
|
||||
], dim=-1)
|
||||
zero_radiance_crf = zero_radiance_crf.expand(pred_rgb_ldr.shape).to(directions)
|
||||
pred_rgb_ldr = torch.cat([
|
||||
self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3)
|
||||
], dim=-1)
|
||||
pred_rgb_ldr = pred_rgb_ldr.view(*outputs_shape, -1).to(directions)
|
||||
|
||||
outputs.update({
|
||||
FieldHeadNames.RGB: pred_rgb_ldr,
|
||||
FieldHeadNames.RGB_FAST: pred_rgb_fast,
|
||||
FieldHeadNames.RGB_HDR: rgb_hdr,
|
||||
FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf,
|
||||
})
|
||||
zero_radiance_crf = torch.cat([
|
||||
self.crf_mlps[channel](torch.zeros((1, 1), requires_grad=False, device=log_rgb_hdr_exposed.device)) for channel in range(3)
|
||||
], dim=-1)
|
||||
zero_radiance_crf = zero_radiance_crf.expand(pred_rgb_ldr.shape).to(directions)
|
||||
|
||||
outputs.update({
|
||||
FieldHeadNames.RGB: pred_rgb_ldr,
|
||||
FieldHeadNames.RGB_FAST: pred_rgb_fast,
|
||||
FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf,
|
||||
})
|
||||
|
||||
return outputs
|
||||
|
||||
@@ -142,6 +142,8 @@ class HdrNerfactoModelConfig(ModelConfig):
|
||||
unit_exposure_loss_mult: float = 0.5
|
||||
unit_exposure_expected_value: float = 0.5
|
||||
|
||||
use_crf: bool = True
|
||||
|
||||
|
||||
class HdrNerfactoModel(Model):
|
||||
"""Nerfacto model
|
||||
@@ -178,6 +180,7 @@ class HdrNerfactoModel(Model):
|
||||
use_average_appearance_embedding=self.config.use_average_appearance_embedding,
|
||||
appearance_embedding_dim=self.config.appearance_embed_dim,
|
||||
implementation=self.config.implementation,
|
||||
use_crf=self.config.use_crf,
|
||||
)
|
||||
|
||||
self.density_fns = []
|
||||
@@ -301,24 +304,25 @@ class HdrNerfactoModel(Model):
|
||||
weights_list.append(weights)
|
||||
ray_samples_list.append(ray_samples)
|
||||
|
||||
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
|
||||
rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights)
|
||||
depth = self.renderer_depth(weights=weights, ray_samples=ray_samples)
|
||||
accumulation = self.renderer_accumulation(weights=weights)
|
||||
|
||||
# HDR-Nerfacto
|
||||
rgb_hdr = self.renderer_rgb(
|
||||
rgb=field_outputs[FieldHeadNames.RGB_HDR], weights=weights
|
||||
)
|
||||
|
||||
depth = self.renderer_depth(weights=weights, ray_samples=ray_samples)
|
||||
accumulation = self.renderer_accumulation(weights=weights)
|
||||
|
||||
outputs = {
|
||||
"rgb": rgb,
|
||||
"rgb_fast": rgb_fast,
|
||||
"accumulation": accumulation,
|
||||
"depth": depth,
|
||||
"rgb_hdr": rgb_hdr,
|
||||
"zero_radiance_crf": field_outputs[FieldHeadNames.ZERO_RADIANCE_CRF],
|
||||
}
|
||||
if self.config.use_crf:
|
||||
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
|
||||
rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights)
|
||||
outputs["rgb"] = rgb
|
||||
outputs["rgb_fast"] = rgb_fast
|
||||
outputs['zero_radiance_crf'] = field_outputs[FieldHeadNames.ZERO_RADIANCE_CRF]
|
||||
|
||||
if self.config.predict_normals:
|
||||
normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
|
||||
@@ -350,7 +354,10 @@ class HdrNerfactoModel(Model):
|
||||
metrics_dict = {}
|
||||
gt_rgb = batch["image"].to(self.device) # RGB or RGBA image
|
||||
gt_rgb = self.renderer_rgb.blend_background(gt_rgb) # Blend if RGBA
|
||||
predicted_rgb = outputs["rgb"]
|
||||
if self.config.use_crf:
|
||||
predicted_rgb = outputs["rgb"]
|
||||
else:
|
||||
predicted_rgb = outputs["rgb_hdr"]
|
||||
metrics_dict["psnr"] = self.psnr(predicted_rgb, gt_rgb)
|
||||
|
||||
if self.training:
|
||||
@@ -361,17 +368,24 @@ class HdrNerfactoModel(Model):
|
||||
loss_dict = {}
|
||||
image = batch["image"].to(self.device)
|
||||
|
||||
pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation(
|
||||
pred_image=outputs["rgb"],
|
||||
pred_accumulation=outputs["accumulation"],
|
||||
gt_image=image,
|
||||
)
|
||||
|
||||
loss_dict["unit_exposure_loss"] = self.config.unit_exposure_loss_mult * self.unit_exposure_loss(
|
||||
outputs["zero_radiance_crf"], torch.ones_like(outputs["zero_radiance_crf"], requires_grad=False) * self.config.unit_exposure_expected_value
|
||||
)
|
||||
if self.config.use_crf:
|
||||
pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation(
|
||||
pred_image=outputs["rgb"],
|
||||
pred_accumulation=outputs["accumulation"],
|
||||
gt_image=image,
|
||||
)
|
||||
loss_dict["unit_exposure_loss"] = self.config.unit_exposure_loss_mult * self.unit_exposure_loss(
|
||||
outputs["zero_radiance_crf"], torch.ones_like(outputs["zero_radiance_crf"], requires_grad=False) * self.config.unit_exposure_expected_value
|
||||
)
|
||||
else:
|
||||
pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation(
|
||||
pred_image=outputs["rgb_hdr"],
|
||||
pred_accumulation=outputs["accumulation"],
|
||||
gt_image=image,
|
||||
)
|
||||
|
||||
loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb)
|
||||
|
||||
# TODO: check that
|
||||
# if self.training:
|
||||
# loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
|
||||
@@ -395,7 +409,10 @@ class HdrNerfactoModel(Model):
|
||||
self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
|
||||
) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]:
|
||||
gt_rgb = batch["image"].to(self.device)
|
||||
predicted_rgb = outputs["rgb"] # Blended with background (black if random background)
|
||||
if self.config.use_crf:
|
||||
predicted_rgb = outputs["rgb"]
|
||||
else:
|
||||
predicted_rgb = outputs["rgb_hdr"]
|
||||
gt_rgb = self.renderer_rgb.blend_background(gt_rgb)
|
||||
acc = colormaps.apply_colormap(outputs["accumulation"])
|
||||
depth = colormaps.apply_depth_colormap(
|
||||
|
||||
@@ -9,7 +9,11 @@ from nerfstudio.data.datamanagers.base_datamanager import (
|
||||
VanillaDataManager,
|
||||
VanillaDataManagerConfig,
|
||||
)
|
||||
from nerfstudio.lantern.dataset import HDRInputDataset, HDRNerfactoInputDataset
|
||||
from nerfstudio.lantern.dataset import (
|
||||
HDRInputDataset,
|
||||
HDRNerfactoInputDataset,
|
||||
HDRNerfactoWoCrfInputDataset,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -19,6 +23,13 @@ class HDRNerfactoDataManagerConfig(VanillaDataManagerConfig):
|
||||
_target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoInputDataset])
|
||||
"""Target class to instantiate."""
|
||||
|
||||
@dataclass
|
||||
class HDRNerfactoWoCrfDataManagerConfig(VanillaDataManagerConfig):
|
||||
"""A HDR data manager, based on VanillaDataManager"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoWoCrfInputDataset])
|
||||
"""Target class to instantiate."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class HDRVanillaDataManagerConfig(VanillaDataManagerConfig):
|
||||
|
||||
Reference in new issue
Block a user