From 22340b37f3e3da08aff45e9a0506fa038ae0e274 Mon Sep 17 00:00:00 2001 From: jake-austin Date: Wed, 28 Dec 2022 07:06:15 +0000 Subject: [PATCH 1/8] Updating dreamfusion to use custom trainer, fixed bug, multi-gpu support (put diffusion on one specific device), merged with main --- nerfstudio/configs/method_configs.py | 10 ++- .../datamanagers/dreamfusion_datamanager.py | 15 ++-- nerfstudio/engine/dreamfusion_trainer.py | 69 +++++++++++++++++ nerfstudio/engine/trainer.py | 19 ++--- nerfstudio/fields/dreamfusion_field.py | 19 +---- nerfstudio/generative/stable_diffusion.py | 14 ++++ .../model_components/scene_colliders.py | 11 ++- nerfstudio/models/dreamfusion.py | 2 +- nerfstudio/pipelines/dreamfusion_pipeline.py | 74 +++++++++++-------- nerfstudio/utils/math.py | 1 + tests/test_train.py | 2 +- 11 files changed, 160 insertions(+), 76 deletions(-) create mode 100644 nerfstudio/engine/dreamfusion_trainer.py diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 09c1cea01..e028443f7 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -43,6 +43,7 @@ from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataPars from nerfstudio.data.dataparsers.phototourism_dataparser import ( PhototourismDataParserConfig, ) +from nerfstudio.engine.dreamfusion_trainer import DreamfusionTrainerConfig from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConfig from nerfstudio.engine.schedulers import SchedulerConfig from nerfstudio.engine.trainer import TrainerConfig @@ -292,11 +293,12 @@ method_configs["phototourism"] = TrainerConfig( vis="viewer", ) -method_configs["dreamfusion"] = ExperimentConfig( +method_configs["dreamfusion"] = DreamfusionTrainerConfig( method_name="dreamfusion", - trainer=TrainerConfig( - steps_per_eval_batch=500, steps_per_save=2000, max_num_iterations=30000, mixed_precision=True - ), + steps_per_eval_batch=500, + steps_per_save=2000, + max_num_iterations=30000, + mixed_precision=True, pipeline=DreamfusionPipelineConfig( datamanager=DreamFusionDataManagerConfig( dataparser=NerfstudioDataParserConfig(), diff --git a/nerfstudio/data/datamanagers/dreamfusion_datamanager.py b/nerfstudio/data/datamanagers/dreamfusion_datamanager.py index 7f34e5aea..e7f978ebe 100644 --- a/nerfstudio/data/datamanagers/dreamfusion_datamanager.py +++ b/nerfstudio/data/datamanagers/dreamfusion_datamanager.py @@ -43,6 +43,7 @@ CONSOLE = Console(width=120) class TrivialDataset(InputDataset): """A trivial dataset with blank images for the viewer""" + # pylint: disable=super-init-not-called def __init__(self, cameras: Cameras): self.size = cameras.size self.cameras = cameras @@ -176,6 +177,7 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me config: DreamFusionDataManagerConfig + # pylint: disable=super-init-not-called def __init__( self, config: DreamFusionDataManagerConfig, @@ -208,6 +210,7 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me self.train_dataset = TrivialDataset(self.eval_cameras) self.eval_dataset = TrivialDataset(self.eval_cameras) + # pylint: disable=non-parent-init-called DataManager.__init__(self) def next_train(self, step: int) -> Tuple[RayBundle, Dict]: @@ -215,13 +218,15 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me self.train_count += 1 - # TODO Reimplement when cameras are fully working + # # TODO Reimplement when cameras are fully working # if step > 2000: # cameras, _, _ = random_train_pose( # self.config.train_images_per_batch, self.config.train_resolution, device=self.device # ) - # ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten() + # ray_bundle = cameras.generate_rays( + # torch.tensor(list(range(self.config.train_images_per_batch))) + # ).flatten() # return ray_bundle, {"initialization": False} # TODO below @@ -235,10 +240,10 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me # vertical_rotation_range=[-180, 180], # jitter_std=0, ) - # ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten() + ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten() - camera_idx = torch.randint(0, self.eval_cameras.shape[0], [1], dtype=torch.long, device=self.device) - ray_bundle = self.eval_cameras.generate_rays(camera_idx).flatten() + # camera_idx = torch.randint(0, self.eval_cameras.shape[0], [1], dtype=torch.long, device=self.device) + # ray_bundle = self.eval_cameras.generate_rays(camera_idx).flatten() return ray_bundle, {"vertical": vertical_rotation, "central": central_rotation, "initialization": True} diff --git a/nerfstudio/engine/dreamfusion_trainer.py b/nerfstudio/engine/dreamfusion_trainer.py new file mode 100644 index 000000000..1f062318d --- /dev/null +++ b/nerfstudio/engine/dreamfusion_trainer.py @@ -0,0 +1,69 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dreamfusion trainer""" + +from dataclasses import dataclass, field +from typing import Dict, Tuple, Type + +import torch + +from nerfstudio.engine.trainer import Trainer, TrainerConfig +from nerfstudio.pipelines.dreamfusion_pipeline import ( + DreamfusionPipeline, + DreamfusionPipelineConfig, +) +from nerfstudio.utils import profiler + + +@dataclass +class DreamfusionTrainerConfig(TrainerConfig): + """Configuration for trainer instantiation""" + + _target: Type = field(default_factory=lambda: DreamfusionTrainer) + """target class to instantiate""" + pipeline: DreamfusionPipelineConfig = DreamfusionPipelineConfig() + + +class DreamfusionTrainer(Trainer): + """Dreamfusion trainer""" + + pipeline: DreamfusionPipeline + + def __init__(self, config: DreamfusionTrainerConfig, local_rank: int = 0, world_size: int = 1): + assert isinstance(config, DreamfusionTrainerConfig) + Trainer.__init__(self, config=config, local_rank=local_rank, world_size=world_size) + + @profiler.time_function + def train_iteration(self, step: int) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], Dict[str, torch.Tensor]]: + """This function gets your training loss dict. This will be responsible for + getting the next batch of data from the DataManager and interfacing with the + Model class, feeding the data to the model's forward function. + + Args: + step: current iteration step to update sampler if using DDP (distributed) + """ + self.optimizers.zero_grad_all() + model_outputs, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step) + latents = model_outputs["latents"] + grad = model_outputs["grad"] + self.grad_scaler.scale(latents).backward(gradient=grad, retain_graph=True) + normals_loss = loss_dict["orientation_loss"] + loss_dict["pred_normal_loss"] + self.grad_scaler.scale(normals_loss).backward() # type: ignore + self.optimizers.optimizer_scaler_step_all(self.grad_scaler) + + self.grad_scaler.update() + self.optimizers.scheduler_step_all(step) + + return torch.tensor(0.0), loss_dict, metrics_dict diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 48e6415e6..63d7cea05 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -360,20 +360,13 @@ class Trainer: """ self.optimizers.zero_grad_all() cpu_or_cuda_str = self.device.split(":")[0] - - if hasattr(self.pipeline, "custom_step"): - _, loss_dict, metrics_dict = self.pipeline.custom_step( - step=step, grad_scaler=self.grad_scaler, optimizers=self.optimizers - ) + with torch.autocast(device_type=cpu_or_cuda_str, enabled=self.mixed_precision): + _, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step=step) loss = functools.reduce(torch.add, loss_dict.values()) - else: - with torch.autocast(device_type=cpu_or_cuda_str, enabled=self.mixed_precision): - _, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step=step) - loss = functools.reduce(torch.add, loss_dict.values()) - self.grad_scaler.scale(loss).backward() # type: ignore - self.optimizers.optimizer_scaler_step_all(self.grad_scaler) - self.grad_scaler.update() - self.optimizers.scheduler_step_all(step) + self.grad_scaler.scale(loss).backward() # type: ignore + self.optimizers.optimizer_scaler_step_all(self.grad_scaler) + self.grad_scaler.update() + self.optimizers.scheduler_step_all(step) # Merging loss and metrics dict into a single output. return loss, loss_dict, metrics_dict diff --git a/nerfstudio/fields/dreamfusion_field.py b/nerfstudio/fields/dreamfusion_field.py index a89d33a2c..d2662ae85 100644 --- a/nerfstudio/fields/dreamfusion_field.py +++ b/nerfstudio/fields/dreamfusion_field.py @@ -17,31 +17,18 @@ Field for dreamfusion model """ -from typing import Dict, Optional, Tuple +from typing import Optional import numpy as np import torch -from torch import nn from torch.nn.parameter import Parameter from torchtyping import TensorType from nerfstudio.cameras.rays import RayBundle, RaySamples from nerfstudio.data.scene_box import SceneBox from nerfstudio.field_components.activations import trunc_exp -from nerfstudio.field_components.embedding import Embedding -from nerfstudio.field_components.encodings import Encoding, HashEncoding, SHEncoding -from nerfstudio.field_components.field_heads import ( - DensityFieldHead, - FieldHead, - FieldHeadNames, - PredNormalsFieldHead, - RGBFieldHead, -) -from nerfstudio.field_components.mlp import MLP -from nerfstudio.field_components.spatial_distortions import ( - SceneContraction, - SpatialDistortion, -) +from nerfstudio.field_components.field_heads import FieldHeadNames, PredNormalsFieldHead +from nerfstudio.field_components.spatial_distortions import SpatialDistortion from nerfstudio.fields.base_field import Field from nerfstudio.utils import math diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py index f5e83d7ca..5f12704a8 100644 --- a/nerfstudio/generative/stable_diffusion.py +++ b/nerfstudio/generative/stable_diffusion.py @@ -1,3 +1,17 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # From https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py import matplotlib.pyplot as plt diff --git a/nerfstudio/model_components/scene_colliders.py b/nerfstudio/model_components/scene_colliders.py index fe27cdb8e..b28b60b90 100644 --- a/nerfstudio/model_components/scene_colliders.py +++ b/nerfstudio/model_components/scene_colliders.py @@ -107,6 +107,7 @@ class AABBBoxCollider(SceneCollider): ray_bundle.fars = fars[..., None] return ray_bundle + class SphereCollider(SceneCollider): """Module for colliding rays with the scene box to compute near and far values. @@ -121,9 +122,7 @@ class SphereCollider(SceneCollider): self.radius = radius self.near_plane = near_plane - def _intersect_with_sphere( - self, rays_o: TensorType["num_rays":..., 3], rays_d: TensorType["num_rays":..., 3] - ): + def _intersect_with_sphere(self, rays_o: TensorType["num_rays":..., 3], rays_d: TensorType["num_rays":..., 3]): """Returns collection of valid rays within a specified near/far bounding box along with a mask specifying which rays are valid @@ -132,12 +131,12 @@ class SphereCollider(SceneCollider): rays_d: (num_rays, 3) ray directions """ - a = (rays_d * rays_d).sum(dim = -1, keepdim = True) + a = (rays_d * rays_d).sum(dim=-1, keepdim=True) self.center = self.center.to(rays_o) b = 2 * (rays_o - self.center) * rays_d - b = b.sum(dim = -1, keepdim = True) + b = b.sum(dim=-1, keepdim=True) c = (rays_o - self.center) * (rays_o - self.center) - c = c.sum(dim = -1, keepdim = True) - self.radius ** 2 + c = c.sum(dim=-1, keepdim=True) - self.radius**2 # clamp to near plane near_plane = self.near_plane if self.training else 0 diff --git a/nerfstudio/models/dreamfusion.py b/nerfstudio/models/dreamfusion.py index 19e7e5a55..6ef860aac 100644 --- a/nerfstudio/models/dreamfusion.py +++ b/nerfstudio/models/dreamfusion.py @@ -43,7 +43,7 @@ from nerfstudio.model_components.losses import ( orientation_loss, pred_normal_loss, ) -from nerfstudio.model_components.ray_samplers import PDFSampler, UniformSampler +from nerfstudio.model_components.ray_samplers import UniformSampler from nerfstudio.model_components.renderers import ( AccumulationRenderer, DepthRenderer, diff --git a/nerfstudio/pipelines/dreamfusion_pipeline.py b/nerfstudio/pipelines/dreamfusion_pipeline.py index 950eedfbc..84030af28 100644 --- a/nerfstudio/pipelines/dreamfusion_pipeline.py +++ b/nerfstudio/pipelines/dreamfusion_pipeline.py @@ -1,18 +1,32 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dreamfusion Pipeline and trainer""" + from dataclasses import dataclass, field -from typing import Type +from typing import Optional, Type import matplotlib.pyplot as plt import numpy as np -from torch.cuda.amp.grad_scaler import GradScaler +import torch from typing_extensions import Literal from nerfstudio.data.datamanagers.dreamfusion_datamanager import ( DreamFusionDataManagerConfig, ) -from nerfstudio.engine.optimizers import Optimizers from nerfstudio.generative.stable_diffusion import StableDiffusion from nerfstudio.pipelines.base_pipeline import VanillaPipeline, VanillaPipelineConfig -from nerfstudio.utils import profiler @dataclass @@ -43,8 +57,14 @@ class DreamfusionPipelineConfig(VanillaPipelineConfig): guidance_scale: float = 100 """guidance scale for sds loss""" + stablediffusion_device: Optional[str] = None + class DreamfusionPipeline(VanillaPipeline): + """Dreamfusion pipeline""" + + config: DreamfusionPipelineConfig + def __init__( self, config: DreamfusionPipelineConfig, @@ -55,7 +75,12 @@ class DreamfusionPipeline(VanillaPipeline): ): super().__init__(config, device, test_mode, world_size, local_rank) self.generative = True - self.sd = StableDiffusion(device) + self.sd_device = ( + torch.device(device) + if self.config.stablediffusion_device is None + else torch.device(self.config.stablediffusion_device) + ) + self.sd = StableDiffusion(self.sd_device) if config.location_based_prompting: self.front_text_embedding = self.sd.get_text_embeds(f"{config.prompt}{config.front_prompt}", "") self.side_text_embedding = self.sd.get_text_embeds(f"{config.prompt}{config.side_prompt}", "") @@ -63,20 +88,15 @@ class DreamfusionPipeline(VanillaPipeline): else: self.base_text_embedding = self.sd.get_text_embeds(config.prompt, "") - @profiler.time_function - def custom_step(self, step: int, grad_scaler: GradScaler, optimizers: Optimizers): - """This function gets your training loss dict. This will be responsible for - getting the next batch of data from the DataManager and interfacing with the - Model class, feeding the data to the model's forward function. + def get_train_loss_dict(self, step: int): - Args: - step: current iteration step to update sampler if using DDP (distributed) - """ if self.world_size > 1 and step: assert self.datamanager.train_sampler is not None self.datamanager.train_sampler.set_epoch(step) ray_bundle, batch = self.datamanager.next_train(step) model_outputs = self.model(ray_bundle) + metrics_dict = self.model.get_metrics_dict(model_outputs, batch) + loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict) train_output = model_outputs["train_output"].view(1, 64, 64, 3).permute(0, 3, 1, 2) @@ -99,33 +119,27 @@ class DreamfusionPipeline(VanillaPipeline): text_embedding = self.side_text_embedding elif batch["central"] > 135 and batch["central"] <= 225: text_embedding = self.back_text_embedding - elif batch["central"] > 225 and batch["central"] <= 315: + else: # batch["central"] > 225 and batch["central"] <= 315: text_embedding = self.side_text_embedding else: text_embedding = self.base_text_embedding sds_loss, latents, grad = self.sd.sds_loss( - text_embedding, train_output, guidance_scale=self.config.guidance_scale + text_embedding.to(self.sd_device), + train_output.to(self.sd_device), + guidance_scale=int(self.config.guidance_scale), ) + loss_dict["sds_loss"] = sds_loss.to(self.device) # TODO: opacity penalty using transmittance, not accumultation if self.config.opacity_penalty: accum_mean = np.mean(1.0 - accumulation) sds_loss *= np.min((0.5, accum_mean)) - grad_scaler.scale(latents).backward(gradient=grad, retain_graph=True) - # optimizers.scheduler_step_all(step) - # optimizers.optimizer_scaler_step_all(grad_scaler) - # grad_scaler.update() - - metrics_dict = self.model.get_metrics_dict(model_outputs, batch) - loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict) - loss_dict["sds_loss"] = sds_loss - - normals_loss = loss_dict["orientation_loss"] + loss_dict["pred_normal_loss"] - grad_scaler.scale(normals_loss).backward() # type: ignore - optimizers.optimizer_scaler_step_all(grad_scaler) - - grad_scaler.update() - optimizers.scheduler_step_all(step) + model_outputs["latents"] = latents + model_outputs["grad"] = grad return model_outputs, loss_dict, metrics_dict + + def forward(self): + """Not implemented since we only want the parameter saving of the nn module, but not forward()""" + raise NotImplementedError diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index f5a0f8219..cf93874bf 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -195,4 +195,5 @@ def expected_sin(x_means: torch.Tensor, x_vars: torch.Tensor) -> torch.Tensor: # From https://github.com/ashawkey/stable-dreamfusion def safe_normalize(x, eps=1e-20): + """Safe normalization w/o divide by zero issues""" return x / torch.sqrt(torch.clamp(torch.sum(x * x, -1, keepdim=True), min=eps)) diff --git a/tests/test_train.py b/tests/test_train.py index 63a8099e9..ac0aa7bc3 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -15,7 +15,7 @@ from nerfstudio.data.dataparsers.minimal_dataparser import MinimalDataParserConf from nerfstudio.engine.trainer import TrainerConfig from scripts.train import train_loop -BLACKLIST = ["base", "semantic-nerfw", "instant-ngp", "instant-ngp-bounded", "nerfacto", "phototourism"] +BLACKLIST = ["base", "semantic-nerfw", "instant-ngp", "instant-ngp-bounded", "nerfacto", "phototourism", "dreamfusion"] def set_reduced_config(config: TrainerConfig): From e2b345ed8c837551de185856d1608f3e4c39c6c4 Mon Sep 17 00:00:00 2001 From: Jake Austin Date: Sat, 31 Dec 2022 03:26:08 +0000 Subject: [PATCH 2/8] Hard coding less, offloading to config --- .../datamanagers/dreamfusion_datamanager.py | 46 +++++++++++-------- nerfstudio/generative/stable_diffusion.py | 5 +- nerfstudio/pipelines/dreamfusion_pipeline.py | 18 ++++++-- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/nerfstudio/data/datamanagers/dreamfusion_datamanager.py b/nerfstudio/data/datamanagers/dreamfusion_datamanager.py index e7f978ebe..454c5880d 100644 --- a/nerfstudio/data/datamanagers/dreamfusion_datamanager.py +++ b/nerfstudio/data/datamanagers/dreamfusion_datamanager.py @@ -160,6 +160,16 @@ class DreamFusionDataManagerConfig(VanillaDataManagerConfig): """Number of images per batch for training""" eval_images_per_batch: int = 1 """Number of images per batch for evaluation""" + radius_mean: float = 1.6 + """Mean radius of camera orbit""" + radius_std: float = 0.2 + """Std of radius of camera orbit""" + focal_range: Tuple[float, float] = (0.75, 1.35) + """Range of focal length""" + vertical_rotation_range: Tuple[float, float] = (-90, 30) + """Range of vertical rotation""" + jitter_std: float = 0.05 + """Std of camera direction jitter, so we don't just point the cameras towards the center every time""" class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-method @@ -196,19 +206,19 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me self.test_split = "test" if test_mode in ["test", "inference"] else "val" self.dataparser = self.config.dataparser.setup() - self.eval_cameras, _, _ = random_train_pose( + cameras, vertical_rotation, central_rotation = random_train_pose( self.config.num_eval_angles, self.config.eval_resolution, device=self.device, - radius_mean=1.5, - radius_std=0, - focal_range=[1, 1], - vertical_rotation_range=[-30, 30], - # jitter_std=0, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=self.config.vertical_rotation_range, + jitter_std=self.config.jitter_std, ) - self.train_dataset = TrivialDataset(self.eval_cameras) - self.eval_dataset = TrivialDataset(self.eval_cameras) + self.train_dataset = TrivialDataset(cameras) + self.eval_dataset = TrivialDataset(cameras) # pylint: disable=non-parent-init-called DataManager.__init__(self) @@ -234,11 +244,11 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me self.config.train_images_per_batch, self.config.train_resolution, device=self.device, - radius_mean=1.5, - radius_std=0, - focal_range=[1, 1], - # vertical_rotation_range=[-180, 180], - # jitter_std=0, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=self.config.vertical_rotation_range, + jitter_std=self.config.jitter_std, ) ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten() @@ -263,11 +273,11 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me self.config.eval_images_per_batch, self.config.eval_resolution, device=self.device, - radius_mean=1.5, - radius_std=0, - focal_range=[1, 1], - vertical_rotation_range=[-180, 180], - jitter_std=0, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=self.config.vertical_rotation_range, + jitter_std=self.config.jitter_std, ) ray_bundle = cameras.generate_rays(torch.tensor([[i] for i in range(self.config.train_images_per_batch)])) diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py index 5f12704a8..5dee76ceb 100644 --- a/nerfstudio/generative/stable_diffusion.py +++ b/nerfstudio/generative/stable_diffusion.py @@ -105,8 +105,7 @@ class StableDiffusion(nn.Module): return text_embeddings def sds_loss(self, text_embeddings, nerf_output, guidance_scale=100): - - nerf_output_np = nerf_output.view(3, 64, 64).permute(1, 2, 0).detach().cpu().numpy() + nerf_output_np = nerf_output.squeeze(0).permute(1, 2, 0).detach().cpu().numpy() nerf_output_np = np.clip(nerf_output_np, 0.0, 1.0) plt.imsave("nerf_output.png", nerf_output_np) nerf_output = F.interpolate(nerf_output, (IMG_DIM, IMG_DIM), mode="bilinear") @@ -215,7 +214,7 @@ class StableDiffusion(nn.Module): latents=latents, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale, - ) # [1, 4, 64, 64] + ) # [1, 4, resolution, resolution] diffused_img = self.latents_to_img(latents) diffused_img = diffused_img.detach().cpu().permute(0, 2, 3, 1).numpy() diff --git a/nerfstudio/pipelines/dreamfusion_pipeline.py b/nerfstudio/pipelines/dreamfusion_pipeline.py index 84030af28..667ac7072 100644 --- a/nerfstudio/pipelines/dreamfusion_pipeline.py +++ b/nerfstudio/pipelines/dreamfusion_pipeline.py @@ -98,17 +98,27 @@ class DreamfusionPipeline(VanillaPipeline): metrics_dict = self.model.get_metrics_dict(model_outputs, batch) loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict) - train_output = model_outputs["train_output"].view(1, 64, 64, 3).permute(0, 3, 1, 2) + train_output = ( + model_outputs["train_output"] + .view(1, self.config.datamanager.train_resolution, self.config.datamanager.train_resolution, 3) + .permute(0, 3, 1, 2) + ) - accumulation = model_outputs["accumulation"].view(64, 64).detach().cpu().numpy() + accumulation = ( + model_outputs["accumulation"] + .view(self.config.datamanager.train_resolution, self.config.datamanager.train_resolution) + .detach() + .cpu() + .numpy() + ) accumulation = np.clip(accumulation, 0.0, 1.0) plt.imsave("nerf_accumulation.jpg", accumulation) - # background = model_outputs["background"].view(64, 64, 3).detach().cpu().numpy() + # background = model_outputs["background"].view(res, res, 3).detach().cpu().numpy() # background = np.clip(background, 0.0, 1.0) # plt.imsave("nerf_background.jpg", background) - # shaded = model_outputs["shaded"].view(64, 64, 3).detach().cpu().numpy() + # shaded = model_outputs["shaded"].view(res, res, 3).detach().cpu().numpy() # shaded = np.clip(shaded, 0.0, 1.0) # plt.imsave("nerf_textureless.jpg", shaded) From b9e1410c086f69c3c965fce48005b133b0e0dde8 Mon Sep 17 00:00:00 2001 From: Jake Austin Date: Sat, 31 Dec 2022 03:27:57 +0000 Subject: [PATCH 3/8] Appeasing the linter --- nerfstudio/data/datamanagers/dreamfusion_datamanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/data/datamanagers/dreamfusion_datamanager.py b/nerfstudio/data/datamanagers/dreamfusion_datamanager.py index 454c5880d..9d768e989 100644 --- a/nerfstudio/data/datamanagers/dreamfusion_datamanager.py +++ b/nerfstudio/data/datamanagers/dreamfusion_datamanager.py @@ -206,7 +206,7 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me self.test_split = "test" if test_mode in ["test", "inference"] else "val" self.dataparser = self.config.dataparser.setup() - cameras, vertical_rotation, central_rotation = random_train_pose( + cameras, _, _ = random_train_pose( self.config.num_eval_angles, self.config.eval_resolution, device=self.device, From 18df29fe1ad95863e5b73848f648edc665a47db1 Mon Sep 17 00:00:00 2001 From: Jake Austin Date: Tue, 3 Jan 2023 22:56:17 +0000 Subject: [PATCH 4/8] First pass opacity / sparsity losses --- nerfstudio/engine/dreamfusion_trainer.py | 4 ++++ nerfstudio/models/dreamfusion.py | 6 ++++++ nerfstudio/pipelines/dreamfusion_pipeline.py | 10 +++++++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/nerfstudio/engine/dreamfusion_trainer.py b/nerfstudio/engine/dreamfusion_trainer.py index 1f062318d..de456152d 100644 --- a/nerfstudio/engine/dreamfusion_trainer.py +++ b/nerfstudio/engine/dreamfusion_trainer.py @@ -60,6 +60,10 @@ class DreamfusionTrainer(Trainer): grad = model_outputs["grad"] self.grad_scaler.scale(latents).backward(gradient=grad, retain_graph=True) normals_loss = loss_dict["orientation_loss"] + loss_dict["pred_normal_loss"] + if self.pipeline.config.alphas_penalty: + normals_loss += loss_dict["alphas_loss"] + if self.pipeline.config.opacity_penalty: + normals_loss += loss_dict["opacity_loss"] self.grad_scaler.scale(normals_loss).backward() # type: ignore self.optimizers.optimizer_scaler_step_all(self.grad_scaler) diff --git a/nerfstudio/models/dreamfusion.py b/nerfstudio/models/dreamfusion.py index 6ef860aac..38211bacd 100644 --- a/nerfstudio/models/dreamfusion.py +++ b/nerfstudio/models/dreamfusion.py @@ -259,6 +259,9 @@ class DreamFusionModel(Model): field_outputs[FieldHeadNames.PRED_NORMALS], ) + assert weights.shape[-1] == 1 + outputs["alphas_loss"] = torch.sqrt(torch.sum(weights, dim=-2) ** 2 + 0.01).mean() + return outputs def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]: @@ -276,6 +279,9 @@ class DreamFusionModel(Model): loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( outputs["rendered_pred_normal_loss"] ) + + loss_dict["alphas_loss"] = outputs["alphas_loss"] + return loss_dict def get_image_metrics_and_images( diff --git a/nerfstudio/pipelines/dreamfusion_pipeline.py b/nerfstudio/pipelines/dreamfusion_pipeline.py index 667ac7072..b8477ab3c 100644 --- a/nerfstudio/pipelines/dreamfusion_pipeline.py +++ b/nerfstudio/pipelines/dreamfusion_pipeline.py @@ -48,10 +48,11 @@ class DreamfusionPipelineConfig(VanillaPipelineConfig): """appended to prompt for front view""" back_prompt: str = ", back view" """appended to prompt for back view""" - + alphas_penalty: bool = True + """enables penalty to encourage sparse weights (penalizing for uniform density along ray)""" opacity_penalty: bool = True - """enables penalty to encourage transparent scenes""" - target_transmittance: float = 0.5 + """enables penalty to encourage transparent scenes, as in "dreamfields" paper""" + target_transmittance: float = 0.8 """target transmittance for opacity penalty""" guidance_scale: float = 100 @@ -144,6 +145,9 @@ class DreamfusionPipeline(VanillaPipeline): if self.config.opacity_penalty: accum_mean = np.mean(1.0 - accumulation) sds_loss *= np.min((0.5, accum_mean)) + loss_dict["opacity_loss"] = -torch.minimum( + (1 - model_outputs["accumulation"]).mean(), torch.tensor(self.config.target_transmittance) + ) model_outputs["latents"] = latents model_outputs["grad"] = grad From 295dc00c22e2495af873469749dbe5dc234c3f1a Mon Sep 17 00:00:00 2001 From: Jake Austin Date: Fri, 6 Jan 2023 02:01:21 +0000 Subject: [PATCH 5/8] Small patches and configs --- HF_TOKEN | 1 + nerf_accumulation.jpg | Bin 0 -> 1705 bytes nerf_output.png | Bin 0 -> 8374 bytes nerfstudio/engine/dreamfusion_trainer.py | 35 +++++++++++++------ nerfstudio/models/dreamfusion.py | 34 ++++++++++++++++-- nerfstudio/pipelines/dreamfusion_pipeline.py | 10 +++--- 6 files changed, 61 insertions(+), 19 deletions(-) create mode 100644 HF_TOKEN create mode 100644 nerf_accumulation.jpg create mode 100644 nerf_output.png diff --git a/HF_TOKEN b/HF_TOKEN new file mode 100644 index 000000000..82d1555a8 --- /dev/null +++ b/HF_TOKEN @@ -0,0 +1 @@ +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 \ No newline at end of file diff --git a/nerf_accumulation.jpg b/nerf_accumulation.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d4f10cb84a6ed1bc75c4ba4fe40d9f8aa26facdf GIT binary patch literal 1705 zcmbW!do!nvqayR9dr1aw!^G zxyI&BkxP*#mpCpVgfd-dQ!b;Z%#PhX?K%6`?mo|X|9YPDobTtn;$iV5u-@L5W(zx|%8mi`CFJ)YZUk)52o6 zn&@r&j$mwTtgcJ7G$mRXl8lLKM<7Tf5+#dLmXlK^YGO5s|GC8X0JJo)1K0sVaDWsV z0-_<}2Y@;NKwy%zYk~hNh!hBg!4c9(8Cl6f&3Zrz0)kRd5C(%nCEF(@a{!8lDQFU` z;2Y?^2wb=#k(-<&tz}(tU&*=mgEonIG!7}Vag*|9jLsI_t$KK4GR4G{YG$)@m+fvl zn*BZ(S2u>cho|4cL;eAQhlALhh+~n*qoU88Jr{rerv%>Rl+?8JD_1izZ{+6X7ZeuV zEUv7ouBp9!r>?%G^+DT1!K3z$zW#wHgG0klN1nf!625%(`pxvr-2B3#XzAmp<&`xT z1OWeJeUtseg_gLapimHsSaU(7A|($*Lt&Z(xPlcO;TygIN90N?S|{gJ+(&AWoIfZr zkM_!J)HZ&uGq*)J+U^_53@}!?beb2l5%G7CHO!7NM zgHev)lxh84@6lY}y^HKsr8j3!O7l`AueG zvOS+QDMWke=iBCZRXKsTloJka6P&57m)#A7K+~b=xDrQ=x;tnAAB-C>?7Q9K z7p_=l)jn^eg>}OuI=DHE$NG~;Z9+i{`j6!n6f=g=t>cHwI2n2Kt#?w>WY`M9d$4Q$Ng@!f=<1u;og+l*Yht5X}pCZgh z-4#p=^v<+`eRoe1KSz`=%(?|2%&%Mu?(Cae?MSM!_*i^f4c8Pm+>&yn{aM8x&lT@h z-HSwuo73mv28zo1!db8PmOR?6)Tn|FHYumEQ`duXizcpD6}O(?dT})tMP=IM1G6lz zBPj%C7(Ci7X(W%gX{77eVtXmxn=EtGnca9dFF2uaze(YZ(3&2{>c-0xd+@BOOOSYf z&89SKxmfvH(=a>my>MM@+P&7B?TV;?ZuJY%N%l8JQXgv)%8}EVd5L(myyg|KjnQ?h!hCw)BsM>hEG(9BH z=E#t|mK@KCH0{ouHw=w1Z(CU;@PnW0ssv5Ugbgw}Et}d^Uen{u)uWR-;_b3``A34y zK?0PE5Q|Ay$9ffGnDGLG@qfsPAum)eJ|%!PumLaLfJUf4Suc2 zf#uW^MO|U0&&-t`QXmQIg!JHMt@u(*zN0n!6FvOp<#$8Jlt`JyLM!Tp3wl%~U-yT( z1!E$z(yJ$_wEOU)!A#nG{x9}AhS?tYXPGH31H;1x=~k)yN0Up7#f(Tv7FKo^qOVAS U)=u40VEds}&$J^rgf4#iJB|+V#sB~S literal 0 HcmV?d00001 diff --git a/nerf_output.png b/nerf_output.png new file mode 100644 index 0000000000000000000000000000000000000000..494994a2f036f896be911bbdad1a8974b7ee4799 GIT binary patch literal 8374 zcmV;nAW7eeP)V>bE;TMQEFfrfbZ~PzFE4FjbZ~5MbZlv2E^l&Y zFU>2t(f|Me32;bRa{vGjVE_ORVF9Q=r)dBHAR|dcK~#90ZM#X6WJz-0`BPio;#+K) zm08u*g>G~c=$Rp9gd&s?itxe%k8_yIT=_}(^|%Omo8b@ufo`Ces>(GscYm9ism{S( zL}nHcAs+7j{KfO9YN~(T{wDwDfB3&mc=`p_XIWo;dTr0oVC#EBOxM>6rU>C}*{=kwa%oM{=tH;d@HN&vy_hwe_ z&I~iw&-io<;ORYp8NtY~3jRraK0CJlbziO{(zf@;4}oS>bW~f!pJop9s7v;vWfL1^lFd&m?*J z90-tB!JY*FTmjFj;?uv_^W#bIpY49Zd|J5w;f(h}+Xv<$(X@eiOx(UX0?K#4zDBK} zxsY;3i*bH5wg*pW16Djpt`7{*^cE&#!*L``Oyu2U;k7gxCVS9)?+?_Z+ z4s7-T6D6lY&KVJC7C{S4W1;DV(0a@i)p|>YU?zZ~1@aR4#m&EFkq7?p?!SL&?)K#> z+p2+2R`I1&s|CnsV#;R%^DOY^tN-Hgiq7|xSxG7J;q6D}Y2mBaH-s1vV7K3)Iq}=S z{v$UpFF9Ow#Kw_xW|=bMFkxzJwjF7P*)nEEDS}fF5llfSSrxE6*%|h%geR5MS7OGi z%byb^;?zL_QzI{l+jkGV{oy0aGGPYS*B5AU{NZ%)kXhM43lLi=#}h`14nn{PxX% z`*P(!7x*(pJiQKPX%+gD1;}T<1MuDTU(|4BU`a6rD~5P@`@=iV4 zd7xMUjjaqxfqyWjA4fBk!yg|uYyqD;ekE-v%&bNVKzX)#Y^J)Q4R**3=-);W|(|KVyjWt$dl#&q#B8KyjQ{nr6c+Zc2 zxaIBlx4iuN0_WZH<#Wmm=aF|m-ZBmos)eU`EE4e2<7qK-oR2fplt@WQb3t{D>!P?| z#MU7mv>3>m{;D8OD8*K#!8~S~jYquUti@SF7xvs7UavUt^!rH*0m>ID`NieGWH~BY z3eE{7E8+w*BSc4$LP?pn3!J@h_x_P(P7LRXzHd1l2LN&|e0+CL-#3(O>@Pd|{f^i; z#A5wQbu4Gk!|i}LxO%az0E^fa8WcjVc?fjEOKAfE)+qNG$9P7AT|XlgJD5*#MV zl9w<1|MP?(VCDJE&#rz=KEZUF$xFruAON&vLkdI&MQ`gyZ00K9|pQjL)+!5Ag{#eF|-05 zLQb`K5)UGdoD?n9Ri+`)be`CFN-;`S5byyIVbkte(!?~Mf9eSYp*$b)^MPe4Xfc{D zFrO{>?`_q@IPdxrV+xh;IMfL~nwE7Hj*xzP3wLD1wFPKv3K=1f`&P?*Pz zbHaAt(r+XC%a(rI@Njn|=R%C3KDU81XF?2k@0iD#ACDuy{j2X$W6k%CrirxOE5r$H zA1m}(1zXLIQVQOC)RbjXa>=BrAnr-{HDgpk1;*1%Yy)lQX|@f)N1U6v+P^|gnU>)* z1)#jX{Bz3SSjLQI!!;FPn$FYpk$Ftyl+gn7ysYmF^E7kw)d3eAUwyl0vkxqD=JEbS zN}0ZIIiDxyX+eu}Iu1BjL)qi~z%gE?j-J#|XsE;oF97bB!Dve|rD7pSge;&HR$neP$jLAyyL? zY68-2V$GD*YABoofB4&X7!?$!Dd7VwlhCxD7q1W8e6>ThMvI5Lf#3h#TU3pCUdXB7 zoUcl;dhq98a7A!ns& z8^$3sjhWbZ9zIMQZhC@`s2Ia3p(@N%VwnLxW7FUqQ}&Dyssd)+4ba97gMuQ zq#%o68qi{>x=$7mB;pz)L?K3x7R7rHra12iQIM{h;Zsb$xZZL;U2r~*EKBBms^&MI zX4H)FJR?pRhk&Zkv<)FRCNo3?=N*zWh)`5%+JLGu4m0z-us?J-5AhTbS3rf+rO-~Jo^=7;}CFck5L zwu?-yC(VTr9bkAb&*4P`l!DJNofk~40^76@qv!N6;k;XGdSw|hkIO(Q#&DXLr-XB^ zHcHf3mRu7N?`hh|JTBao`>GrOlND?|4o@xx)y%se?@-NLT<@8tg?`f!+rZuXN8Ea? z5Ioy`RifsnzM~^eM$Se~g>>wm!l2<0oD-TR(6*j_=huQn!NB{q_^}#WLv+OWWKzmJ zCB!?1(@bnVP^}>)WtlSlrseo}Mpf~?y4G&LVVM&Bwj)H(I85ArnCLehXFp>Z#0kq( zz?5dw5*tsyX*pb7)tWH zJS}B$)${{6=g6s6TFwdcT8f4!h=;t?P}y%gR7Xlt%oHb%qRRQ>#Q9iPToB4)JWOY_ zz%nP6IgxXz#rGNtLlAD?pIGJ$>Y#JHe|w9XqFLDNI@A?JT0-;{jL!>g-_Y+`9zLF! z&I|Ffmf&WFZ)%+oW8ic*b3RR&!gk+craXR}`1a<1B{*>uW6ikW@phyXW3!EjbC^N1 zmZHm9DMcxzu*`|gu4OpQjOPV4Bh872yEAD?bbU)+49KJ40s>d9RjKa=K?+i`o6`7 zKuHxl<}t6>1l~JLjk_Obf-{eb;6w;f*lZheE|g--r=^ymeIPXTpO!h(Z6a+SYnfVV zd682gUi3B0`7N=DY&IR!n0Wu=9ghzM@c|5)Ht_gxrt2D-uA%EYni#oxagFz$$NNXp zQV5~taMiKd^k^}Z7VM&=>& z@%;n8`d_{wpdK+nMDQLqdykrD9t$CQN-}cJ3=a#T33wMkAeYRtWV{!4hYiR3fo|K- zZ#ukpIPYlNmQo74{f^)N_6HvCkCbe97uak%+OFm5`hX8G%(YrXgw1x#IjpRiM(KAgG(>Mb?+g%4{Jlq+_^l1|)DcA0gI+i67VqLxAm>8#oDKxP`)i_*kaKSN;iBf7wd;k89-~QG2 zqzQ1<^h4`emV_C^HH|eh=Bajzoi`2_J5o9md?e?>o9{o6m%{#HN2i&bYQC||3u#ua zZZ1hlX_}5DEj&J)S>}b^VM|_=H-CJ`JT7Y+t&mfuX*zZnJ;Aw}cc)o7-cQ8Q*Z3(l zH6Dk=G8b|#IB}%KR-}jVyzudKq~A7lo0dL$+CH$%g@@aLWhp=nr59H_PRA4TGP5iT zF-CU#EzSu!RSYsU@CqnmN=^&Ys1!?xKx`{;IuWLE;ltbenzNcCFAK+qf$Nu-#OT>{ zTc(H1hmUuBe1FH)^?}`?XBuZNF1K7OTjseoH4V@rbX|*(`0iJ)NU6~Ek)nmH>AA;;6H2C5cuS$( zM51rdBIJ_s;_>3}As`Mmo1S(PdAvVjX5^*zC6{?2hKTcy@jNn=>M}Pk4@|?v<<*X+ z^9<*a*aSjT^8_;^SaH>lN=+K_0;k8JhPEIzTPua*m5y7i=Udd&!nZ6>nA&N_2PoCR*Xdp7dJg~T6pp5g2UAYQ{!|TnWu$$T=@9z zf#YK&eE4zV&7Y2>+1Tz|#`D6%!@&7CvE6lS_Z>OaS>5%`p000*O-(wS3mhNMoX->Q ze!N3OxV+xczS@&g=6F9+a$%Y#Qd+pXd*IK0`87GeM2m8KI8#dE>gJODVPC_ePK@UP z=Y-e_u?c+h=a+1EE$7pSnbD>WD^M~Fr;$<$AKu-eC9~absyl8PL>xXCT^~6;PHP>Y z48wx+Qo(R{Xz)(BKi3-he4IJnP2Al*a({c``sJ3*wq>{P5fO5#@zlFOE``%^L_~P; z>Vn;V%QR2ieK@kr8PgiF%#3A8OvB9Gha)itx;r_(y zZ(p$6cH}aXi{s;)N4EQ(zHgzx-TM>HdoHgJH6;3)_=ppVWp+*_xCd& z-`(@!-97t5Pw0B#+h1N1Vn9np`Q5(d;&Q|7$0Nged9J-5@9PTec02a_mSvo2+m@z@ z_~Xb0}{q2dS4K!`FnA_WXmSthH-Oz2f zYl-XCe0+OP$%)5D<9wV5 z&J+BCIN^Mph+&U!9GM!wRh4ejaCyC@luQ&yE(I3^r4T~xDSGF4yd628X1@RZdoHea zwSG{e-!ueqD@nmTmp2{rn7R47!3V)qxV&kYC-~#vd;Et}ZEBQSDmP7Jo@T~jr0;9( zpR)mu-M&Sfu)FX~!^HVCvfXy%lDW9(xwzS|eIZ~AFJ5oJG9pHd4LK(^n~u%4r`tA! z=r}!0q?CxQ$1jEZ?~i=<>sM?K9m~A1jER(M4T4eU1q8OchP)I$o{t1d#Wc({XV`n337QHy z71}1Szw8)J6EA+YV}H>xJf;;>7N&7wf7o*KqBbq&dEw$_%Q7X7w-fuT9_JkQAI`k{ z;gO~f+}|F#xY^)iP1B{w%1Zz(@bT>78(voQqEk{1rwmmqNY@|ssaKk)tUKQaz8 zn9;T#&4ufiTeK*TcOyW)TT;qgUhSCYnQqe%((KVvCqA)pxKI_M zYAl2B|NNIXwFGTyclP?lj*<-L9Yu{09BtRs?C`RJ)w!! z=goNhI5C`NF0S^x{ox+x>)>-77GmpBt7UV)3;gOYzh-+~&3YM)oD#3Uy+XCn^?_wc zHL+1xrox~8;R9DUwKv(cfn{0fHja5#wp~L?3q>8N1SzschS!qN-6v5$w@%49y zn%1Vo<@KH>M22C;N$nJ=R`!-it+hoC+*&6KGSLS3Oh{PqL=Hqvi9F0Nag3uK+x9QM5a z_5cRcxN!Gj;NfoI@~ftX%f^9IT&&Gk4En8Scj?KCJ-6n>95}1ftdx2Evr8WD=Gvq^ zFC6a&V(0Oprt43XtFZ>XMhQ4qV(s;wB?LgPbvQ`2D;#9$K)rVul-8G(ixWU5695 z;tMpLtK>=nq_V|)W0a}moA367AUN}gFPMPJh~S7_BuFJviq(8Yf?{H*C4BE_x(&@3 zQEhMtr6}Vm@#6Ka;>4V5U7<=aHzG8+;0Wq*Zq;^ixB#&&IA3Tr0G`XQGKCX!e#DBQ z7~}i^2yJt~yM^dME#ss?#Sn{_n=v8~E55{p$^xbsO45nOUt$Hiosnj$fCZe-6k8~k zQOl?nyac>!F)OI0T3lhM71~f}cRQ{_!+0|8KNMcP+Tk3`OQPF2QZl;U5!5n4TEx{s znG05FGNBNfg`_7^$#_@W$ZDB!z9;1w5l=1)Su@^6w9Et-$$G}&5hC6;K%r0&>WWro zs8Q@p(I<+I;V|MRA#uW|4Vn$#WeP`%juc%;WyVbL;)$WhiC>jcA0m#TBOx?2U0@j< z*DqQ`U>*v&DBeMA9GjgdWF6NUU-7X)TtRKFEDp8G-PM&%YKoIeZmP%Y6xfMJFuboE zL5Bx%^?YkkKUeM_E2r3L6~0X9${+T=#bice#u1Qy#$-WlCgn4^Oe@zo6I@dpD3(zQ z4u?@2DNY<}GiEc;(zFh6V8)V^AP^#K_JL-T2)Rsn2{>s$1f#4MvsD;}trvLuapG1c znd0P$TV|jclSHwEb1hyX_=E#&RT6*|hq7RnG0lj0gt7_{Fq=>^xs0f0K=ICFCb)H6 z=ftnyA1k&XRhbk7N754^HgsEGhb)t^+0|*Y3l)e}9tdWYk7}hwRR}R6lJTysayhBy zZJAtVmU5(M!MlKIex}#C)jdQj$2|Bgo)%n26;$d37j+OD9KIOkDYRs~mAYkZ3rhd9HuR9C^+9^A@68TiwO|Ci8& zO+`|tEBy_`8kChyVTs^&Ea}L!JYrf8(}Jq6v|J z6G7cfumyc>qkwhoC>(JKTK_Z6KHQaQ1d4*+;yIlH;c4AU(sk2rC79|$hu z#6RPNcY)|zoH&AyMBf66>hy_)o_j1u#AQ5j#gP@b2w)lKGpbXaOasw3G$GWw3o$f6wYGW>%d-+v9$C_aWJf6@zV|p<(iF(WIOhL> zTWOI2;N9Kd@bdCk6}8=ps#Yg8P3SRKBSiGIFX$y=nkmfdx%#Z2Y8|rTMCIzKI-C=n zGmuQtp#mI+$&7amsxu(f%%xgYa2^5L*t4WFC69O?@y=J;jEfjr!?7+n7x7L|SNj84 zJw7cT*>xSU+0sVkcz5E7A`zZKThWCO+I72%u6`GB67X@e#`=m0tbioqaAqK$CUl56 zf^RCpii9U(2;<4)Jv1GND5?vl3q==-$g?{cWM{Nc?S?YJ2f{7u7=eUm) zlu|2GO?8$KYvP7fyEdDNZOf0h|LJ-4pA{g$-Qyp5aq-L5w^`$?v7{s3MVhchR=jWvHTB=n#k4Q_-<8wk5L$^k8zutFYE`NL${&~NEd?wDvhkxL5|4scT-%-j) zDXE4c%`{09uf%@ASa7%+x3$J?=PIyPD~LGf zR~#xBGrsb339e4D)e4wV@=&u!TZmz^Hbi?;840es2|(~$Fk@QocsTvIV#X(aeE1}O z{TZe^`eu(4Ps&fB&&Z~jX8P_5C!tz|K`sk|L$ww*!SA75V!$%O{oTYj-?fM&RAjr-?(e4dZbs~{q{FaogM zZrJX?xieX(Y9;IH2z9@acNNTP*sEk{-uPs%8oF{!J<%Iy~cEtz>aVp`S})8`B4JYiu(BvN#u z=_>dXovD=E8sAqM*mF3m2P!CvREi#^IM?B%q1eDvLG+Ykh)_eJ->{^+8d5ERN|CI2o7cn!mq?$sqlA5IM9hl0ICRV?z$&VxFiD|hf`VKIPJb;zw zlG;&QTD1aPX*n*|DpV7uGvXRd7sLgGf=I#RF&D{}YjRvU_2e?FWX?*utazj>c|bM) zBgB7B#6DBVCk%Mjiu~mHNkN~8R8L*xoR_uoi)*+MN}jQj38AUo8eN{#<+|O)k$FdSzw)Em{YTEg0Cu5B|}!=y*7&I+m$$}{+S*C3%KjYqIjE#)(lhfikzB$pWkkg#KB#I0X0PZ{N!N{c)t=Czc? zc>*oDR2R~<0(DXgvRuy0^CPP1%TQ;}#`{U|&#d7y4EU#om!JHu9`fnD&)r3H6?83@ zYy~{6lNU;!uyyOq=OM6GY_+zpJI4z-pX>G<>DMUHJk#Qze7#y%$Fdwrc|@&z;q&&H z)K6`jXNy>s5`Jp#&-UKuh(d{@IXtR?!VdBu|@*ypm@m#*;t19DP;dwa_8>Hq)$ M07*qoM6N<$f<+x&WB>pF literal 0 HcmV?d00001 diff --git a/nerfstudio/engine/dreamfusion_trainer.py b/nerfstudio/engine/dreamfusion_trainer.py index de456152d..4aa9f1644 100644 --- a/nerfstudio/engine/dreamfusion_trainer.py +++ b/nerfstudio/engine/dreamfusion_trainer.py @@ -34,12 +34,16 @@ class DreamfusionTrainerConfig(TrainerConfig): _target: Type = field(default_factory=lambda: DreamfusionTrainer) """target class to instantiate""" pipeline: DreamfusionPipelineConfig = DreamfusionPipelineConfig() + """specifies the pipeline config""" + gradient_accumulation_steps: int = 4 + """number of gradient accumulation steps before optimizer step""" class DreamfusionTrainer(Trainer): """Dreamfusion trainer""" pipeline: DreamfusionPipeline + config: DreamfusionTrainerConfig def __init__(self, config: DreamfusionTrainerConfig, local_rank: int = 0, world_size: int = 1): assert isinstance(config, DreamfusionTrainerConfig) @@ -54,17 +58,26 @@ class DreamfusionTrainer(Trainer): Args: step: current iteration step to update sampler if using DDP (distributed) """ - self.optimizers.zero_grad_all() - model_outputs, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step) - latents = model_outputs["latents"] - grad = model_outputs["grad"] - self.grad_scaler.scale(latents).backward(gradient=grad, retain_graph=True) - normals_loss = loss_dict["orientation_loss"] + loss_dict["pred_normal_loss"] - if self.pipeline.config.alphas_penalty: - normals_loss += loss_dict["alphas_loss"] - if self.pipeline.config.opacity_penalty: - normals_loss += loss_dict["opacity_loss"] - self.grad_scaler.scale(normals_loss).backward() # type: ignore + for i in range(self.config.gradient_accumulation_steps): + self.optimizers.zero_grad_all() + model_outputs, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step) + latents = model_outputs["latents"] + grad = model_outputs["grad"] + self.grad_scaler.scale(latents).backward(gradient=grad, retain_graph=True) + normals_loss = loss_dict["orientation_loss"] + loss_dict["pred_normal_loss"] + if self.pipeline.config.alphas_penalty: + normals_loss += loss_dict["alphas_loss"] * self.config.pipeline.alphas_scale + if self.pipeline.config.opacity_penalty: + normals_loss += loss_dict["opacity_loss"] * self.config.pipeline.opacity_scale + self.grad_scaler.scale(normals_loss).backward() # type: ignore + # I noticed that without these del statements and the empty cache, I was getting OOM errors on the device + # running stable diffusion, since these variables aren't deleted (normally they would be freed when we + # exit the frame since they no longer are being referenced, but now we loop back instead of exiting the + # frame) + if i != self.config.gradient_accumulation_steps - 1: + del normals_loss, latents, grad, model_outputs, loss_dict, metrics_dict + torch.cuda.empty_cache() + self.optimizers.optimizer_scaler_step_all(self.grad_scaler) self.grad_scaler.update() diff --git a/nerfstudio/models/dreamfusion.py b/nerfstudio/models/dreamfusion.py index 38211bacd..e5f7cd341 100644 --- a/nerfstudio/models/dreamfusion.py +++ b/nerfstudio/models/dreamfusion.py @@ -76,6 +76,14 @@ class DreamFusionModelConfig(ModelConfig): """Initial strength of center density""" sphere_collider: bool = True """Use spherical collider instead of box""" + target_transmittance_start: float = 0.8 + """target transmittance for opacity penalty. This is the percent of the scene that is + background when rendered at the start of training""" + target_transmittance_end: float = 0.4 + """target transmittance for opacity penalty. This is the percent of the scene that is + background when rendered at the end of training""" + transmittance_end_schedule: int = 1000 + """number of iterations to reach target_transmittance_end""" class DreamFusionModel(Model): @@ -97,6 +105,7 @@ class DreamFusionModel(Model): self.train_normals = False self.train_shaded = False self.density_strength = 1.0 + self.target_transmittance = config.target_transmittance_start super().__init__(config=config, **kwargs) def get_training_callbacks( @@ -119,6 +128,13 @@ class DreamFusionModel(Model): ): self.train_shaded = True + def update_target_transmittance( + self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument + ): + self.target_transmittance -= (1 / self.config.transmittance_end_schedule) * ( + self.config.target_transmittance_start - self.config.target_transmittance_end + ) + callbacks = [ TrainingCallback( where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], @@ -138,6 +154,12 @@ class DreamFusionModel(Model): func=start_shaded_training, args=[self, training_callback_attributes], ), + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + iters=(self.config.transmittance_end_schedule,), + func=update_target_transmittance, + args=[self, training_callback_attributes], + ), ] return callbacks @@ -260,7 +282,7 @@ class DreamFusionModel(Model): ) assert weights.shape[-1] == 1 - outputs["alphas_loss"] = torch.sqrt(torch.sum(weights, dim=-2) ** 2 + 0.01).mean() + outputs["alphas_loss"] = torch.sqrt(torch.sum(weights, dim=-2) ** 2 + 0.01) return outputs @@ -280,7 +302,15 @@ class DreamFusionModel(Model): outputs["rendered_pred_normal_loss"] ) - loss_dict["alphas_loss"] = outputs["alphas_loss"] + loss_dict["alphas_loss"] = outputs["alphas_loss"].mean() + + assert ( + self.target_transmittance >= self.config.target_transmittance_end + and self.target_transmittance <= self.config.target_transmittance_start + ) + loss_dict["opacity_loss"] = -torch.minimum( + (1 - outputs["accumulation"]).mean(), torch.tensor(self.target_transmittance) + ) return loss_dict diff --git a/nerfstudio/pipelines/dreamfusion_pipeline.py b/nerfstudio/pipelines/dreamfusion_pipeline.py index b8477ab3c..288dbc6fc 100644 --- a/nerfstudio/pipelines/dreamfusion_pipeline.py +++ b/nerfstudio/pipelines/dreamfusion_pipeline.py @@ -50,11 +50,12 @@ class DreamfusionPipelineConfig(VanillaPipelineConfig): """appended to prompt for back view""" alphas_penalty: bool = True """enables penalty to encourage sparse weights (penalizing for uniform density along ray)""" + alphas_scale: float = 1 + """scale for alphas penalty""" opacity_penalty: bool = True """enables penalty to encourage transparent scenes, as in "dreamfields" paper""" - target_transmittance: float = 0.8 - """target transmittance for opacity penalty""" - + opacity_scale: float = 1 + """scale for opacity penalty""" guidance_scale: float = 100 """guidance scale for sds loss""" @@ -145,9 +146,6 @@ class DreamfusionPipeline(VanillaPipeline): if self.config.opacity_penalty: accum_mean = np.mean(1.0 - accumulation) sds_loss *= np.min((0.5, accum_mean)) - loss_dict["opacity_loss"] = -torch.minimum( - (1 - model_outputs["accumulation"]).mean(), torch.tensor(self.config.target_transmittance) - ) model_outputs["latents"] = latents model_outputs["grad"] = grad From 71e860b29cf6bb096c2d4e62d17918d1e4825e0e Mon Sep 17 00:00:00 2001 From: Jake Austin Date: Fri, 6 Jan 2023 02:03:25 +0000 Subject: [PATCH 6/8] Accidentally pushed logs --- HF_TOKEN | 1 - nerf_accumulation.jpg | Bin 1705 -> 1371 bytes nerf_output.png | Bin 8374 -> 6923 bytes 3 files changed, 1 deletion(-) delete mode 100644 HF_TOKEN diff --git a/HF_TOKEN b/HF_TOKEN deleted file mode 100644 index 82d1555a8..000000000 --- a/HF_TOKEN +++ /dev/null @@ -1 +0,0 @@ -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 \ No newline at end of file diff --git a/nerf_accumulation.jpg b/nerf_accumulation.jpg index d4f10cb84a6ed1bc75c4ba4fe40d9f8aa26facdf..4ebe6790c538da138173232bc5c4879ea06d1526 100644 GIT binary patch delta 761 zcmV}{gE(g*|)7lU4Ptw6BeJYbEW@m)oo?6x+nR2_h48Llr} zx=Ah`P<~Q#iu1BOEm~Nrt-HbOcoL}!{>m{{l3I+7VhBkhp|pY}xi20unxbx^WKgU* zHFHkR{lIZwQsUHGHK|5)smZqWHW~*m7=w<~+kjcsM^4pnNM3&t*CUKkX=?JjN_a)& zlU{eVB2VLtCL-n z>nOtSbJW6P)*hSJR&4(OXcV+4a7h^Exot}lbxhQabj)_(@lTC|ADMDHR8YgJ4YkzH zj3+`$a@NL0wwQksR3V83@(Af&m6nKng_r!Ar2@s*W)H3_X4dm$1rBgKis#2ytIKL+ z;&HOCD^FoTQ4a6|mW_pc|OZmLgK za`LWAqb^9zyl0LZ)}e~)VBmdtqT=YyAYb98tV3xy<2`??)r@`L-fqmSmgF}P!U0f# zlisVU73A_N=zz0jj}>kUXbB4^UrN!}RO7Cuv8!cbgmQx(#a?Tm(=^E>h?hbE^{Mpj zI?zkKw&il3kT-&2zQ1o;D73gD6 zjLnZDK~idbO<_J%DtEu4D)0EIVE(bG@4 zL}gV%qb<)rop)L>iL`-cXY&FnP|S0I*nSnpc$Y%eZf=ocXPSRC;B7e~zDq96Yu3fe zjv}f_UwtmR9(FSg&rUT{Z<}{}+qr>xrYf>ALQI^GrDojtX3}V~(X33vVwfE|0oYg5(Zo5$H7l;?nG98WFO^BF9&@Sa_gY+lOu3DQ zRmVa-{VP7bR{np?Tv>T;JkL;9I2Go)jg#43G88I=<#Xv>bZUD@-1@w$GWy(kXYnt@ z^qnyyi%W_%hZ_I`13Y`y&XIR9y(u95-rOAXOw)8bsbYGB0>0LFPEgGi(0GxsMtDhEg zt1rxaw*C;`H{)AT%(2I_VL%t;%NEb#YvXfC%BxLE=f9RlG_TDY8T!7Ex}!Xb6tVe1 z?kgR1XssBDWGu&nwz>;P0Br=81ePMJOdz?q5yzYZg5Ianx58s_6t6+a@-HJCRvL{< zR!iP~W;K7?LvI5-QDCqF#Wu!?k*?)7hPyyS)&q$=~LX0}Vq$n*GhJKIvpn~zw;m$Cg4ln;lWfV|u}@n1T*D`hGB#FD``ElT z8xW`NTkk!JNv)K5rtm|PfE?qWYNM#%pR=mNe1ThmAoTUGB(T)(Zd+`VF=B8AdRC9x zq>2~X%y#t2t}0wL3UX3&y}P7qiKkvSr=oAQ&rGzvlHX4%ZsU*vhGrqYl|l`E54JdP zNCSWGNZzNa?Uafy7euBYW0QcTEfvwZ?(?hydS;!SCZQ5S215M9ldz2mThQ5Y}XGX4T?@P z=~!a&E^l=^#rpIyoNM7H8*1O`aMoALJ-&a*8|P7iJ0E(F%7#duMP_#cziP(}ROB2Q zmK&KNVo|Z42=%Y0x%H0^%G-lBwChP-*J#TebnGizQj}Uov4Qii-UL<#o2W}}PQrOP z`eK&q@vdbOGvvACQER^=hOQc=OhR+`QE}yx?bmh{Gn%U8K{@Bb1HlmiQ3nJ!WlmuX$IQ9kh1;#8dVkzY%IxN)fbxyMJMJT_a;UZjMc7T`KpxQmf+T z83@NM4t`Ehfl{CpA>tJWl>Y+?NI_7{3!v2ZyCWiqNJhNVTH*HV;Pvc)@VZs*kA_)6 z;yfeeR1qTGAtG-EeN4s_Y88$i*`THff|+2`Giq^dMTJ(CIbn=5dyFwAW*!G3PALU< za0FEWkAIs~agJ$po7~oN-F(DeGyMzZ3nKA)?r|VBB0g!Y@O-Ykw@&Yqzy5l2yEjCI z_q##X*18sOXG}*mYSl5^BBkea@Hn7#DHM&m6pJ9t3B5bhZh@;}n3}#)|%Fmx0pPxJXKIq55 zJuc$cuZ?xBjIrYG0UkK!@zoevhncR^JkvYg)Xd{A476tSGfUj!#^;O-5;1TvGm2{5 zsCfb+2%6ZW*fe@~j_!;epxOJR6uGW_T3~~#FlI&M#4o`-O2em6KfbN(`{4aL__Kdr z@P9VOq#uLcCr6*``{ePgnW9l=o-aC$3m}(;C@EmQ#lPWiY3EU^s?M?~HYcMyug?xZ zDT=ONI8aK|eIJv>0`cAB`+gX`50EH{)(Y!dC?&2zB}%|7Fw4I1JHQj6e*3nvZG+oi z3-`yu`+YD5hzhF8AAdaf{(Zgbe7z~PI)9}^>r-_E;V6-M;1?LxYP1@Nk2}Yq*pxte zhcbmRg#DNaW_CqbGK1nUg<6DtpMX<~vLD9#J#lkt71p(~G+|worAeT^oChh0k{Jdz z6$kbms)%#HH@^RNi_-+Qtuy+hHARbXyEh(>cA;yn5p7k=^b`jdeu zPm4IBM$Pisx40k?p*F?b(F$$R_MZF*}sbl{g{+nxqAMXMk|7Ay?98iO7FoXYJVws75WH< zs1jLt^dMLJ9vNHvF{w>imI4lAOvW(2f4_4agHn_^;#u7glJA)+I$U7=>EF%^4l~Vf0}U`5tU;Es-@rs-oYTIsFK%iMR-3 zl3fl&G;nHJ3g$r>Y89T(g@0`ue0(h2ZcpZPmbFA{8P06biX{lfe{+J(=)$UzZlckS z^wbG?P;sG!IRj&YH5&UN>|3WcWnBxk7R(GY<2WLUxH0DB^Rv@?@PuV)*o-zATqUky zx=_^V1ICD6JQGw^qg?i*hZY$7OoR=jthKN#m5+}l{-8jsC3^bRo`39$ascAVxxX?X z>NiWoa&lmhEoXsxjC z#+=5wDn%W4r4KmvnY;p83&v|^m^&3<-zRfM#PF=e%!tRcaJ!YDtt8)09xh_k8gJwl ztwqIAHQe>W1~NKxhJRY@Jxj+d(OxkTZnwr5abaUdDz{p23yS>LuY;dI4_==GC%V@^ z{SsXFILu?Fow~`L^N)`*7$yF@wJE{ zGEohnL`!gQm-rcuqqvo1ej25bbau|D%V2JCAVmw^EhChY*ncL=Qs^B%KM#KX+L^<6 z+?Qy}MOYSPjNk=h7|T*oz=U9mNyNsCOrK`iQjIDiK|SY}4xiJqwJh{*f&8l6YTUe* zl(rHIPjFBW7aWax6v3=u7UXA62^A*=!2;Bh&)Za%7JT1k@Pj2eLuc}SPmbN#w#mAz z$m;y|ec^F$xPKXD(FewKjy@4l+~RO*QOsdKCSwfdoXmOl4X563E&2IDb)k=m` zjFinF6mDR%26vZ<6orr{(FQqtZedPkbfNbUM*&nBjgdXQ8$~3)H#vHzXt^k_ji}%E zJN%0Kec}7}g{4Ku8PnkK3uDaU0LO5i&&D*Pk5k78M}N1t7X&RrEd_3C0dH*ExI|1T zC%6U66OA@fCb%nFh)&=oq@G8uOETYyM)WOee&3>O%$zyk{hIWVhXV9InbTNmWz0~Q zq3VJZMT=EFo=aj-;6m_-_&W4|Nb6c^Y5@WeDWNk|nZwxA^Qi6?H|<8P!tEBwf4fzd z7WLnnhJR~g#vFn!mOg+0Es;7DS1AUr!M80m{;?bT=DfG#lFVa7PgK$133~boQ5JIq z&oB>Nva~{58tYnVO>m#l%67Ya1rgzKZwP@^%ktH6j$@!D@M&F@QiJ!@^;W5=!C#*p z)o3dp&xQ4-lo}-EOJEp99W9{)+?3JlRafg0Cx32J+1}v_eSS`U{TgiBNZqc~R-?3R zpaRcli-WUBQ4wL=2e-Acv`VevNwyZGE6T(;j)A#xzqKpPwM53Aer2setHVDCPe4X6 z)avP{-EWOr^y2hL)J;jRt*9Vs=*a|w-lQ)WKvzoCwriyRdmH@qS7+ah&sXRD>b&0v zMSm;z`%0;W(T#84)*vxs258kv@8Rbi`#>d#l^Bc}I23)|k}4~&_dzXT*q)^;0!u52 zOVP#59GiwN(Uoejw7A)2i8^t2?)O5i(L2q<1S!euMFRN3R#I?HHD)1eT@aL_l{p6Q z*W~rt`T28WjG0k9^}28z(JhXBpc;Js^?yDB2o3t2Jgm}cWqTjgq7>CjKOB3<+?iu0 z@|VjPIaQ(=7ZVf&(cmU^StO_;U#;QssMH$$_0;#49YIyF88;mVry{@;LCh!D&w{&T1*8Z6Rr>k@9j?Ghh3q=beq3D81hHaY9MNy-dQKLY9jjS%727<3rQRH_pt|(2l|G%P>p_nY#~YC`T2Hl^gel& z=qjO#;I_6fLy7OH!u{TG7oN|RTCyaGD|VY4-4Wtmk|6jxte!IUI^zbXfSh z`Ea00)RC$%W>{UdqtC!!QdEGqK&wE=4+XdQ~Ic zPm@9}0_3S=rp3VG3nL_nNvOR!!WRtU5>BrruRnWpt#H3Z7WCdJ2|#{*L{v*H)TU{d zD`O0L_ba{KjXBaYP(kb2dVdS|TV+||&_z78OjzI`NxkC$B3O|T^|pqNk4!i%B~Kd* zG$VVOANv;lzm}6R#>vkiyf>pYNn@p_BP6U#%^Pt3`f1l|m8{v;%2jsZ132&ZVBZJ( zJ`pLjrpX5i>r%hCK?#!9^6Pbt`N6cD0j5F?%3kPWUN;X;4P}>3kADV983SC-@zXhh zi34d1l%$_?oVcNMfu{=lt{g`wt}#z>B?!*9j~Hr3cQ`$Ryoux;`w@z*_n@o$KDphL zr3J9HT6o+S)+JhDDAg0UoT8e8QYa!E>3NP}96hk0)@-8+)ht}zqCrRJ%yLK`9TYes zX45HESgOVlblSz$Nq;f2EP`r9G)7xOILZ5A)LJiOKlw%|(3nS#?(D~4rrkAq3^V7P zF$rE8_j_Y$%DPl;Yr#BpNNdSaaK_9Qo5sv>O!gz@8&T#VqDnA9jk*%31b06FE19b1 zUzU~)CENWq$>|;(VLzOHI4U7++Zxn%KW2&-yQaPOqKMcv6MqEdz3uUHPf%X2?~` zixxxzc@ggtGK7RDZAGuY|B$7h&6>RWLc3DcshYSsgyX7=!wHIZG2ZwabJoMY!JsP%bdk zFBeD9><(di%yUq2cn(P&S))%+vvhvIQAn@hvyV&Qs3<>w9sK*xH$VQ|V$!r2trc2r zxigR`zAAmhMb)C1rY139dIEW5M6ED@b0$`6bRUUNaeok_JNq^eLK26N2(nN#=&#8}(P>voTWrwp=<&eo3Jv~U}^*T6?P<8ue>D^CCsr=dw z`q;VO7M9ld{!#LJChi6ENlV^8Ml5oSIfGxM^NR%PBTlO^TTIaymi@lR2ir z6=7YMI81}zf4}p;{`KT>FKBVrMOhcc381rSthZ2!%Be@r`9bvlQ+Smce#FdSP#rUb zLm|K=JikE7Ipdf1V_eLx6fW;jo3ga1=Z7ill7EAcs@$3|i()p@UW>he_xs3>624qCY#^aEvek=U``^vHguYU-AAAmQ_V(|eBiHu<$U8#?l2)7#Gyt^}d zlxtBgJ06^MjqbDUM!nxcP?$xdH%t9*e;<7rI?qk90FzcjVn&Auk6o^5fUs@OaXA0= z-!Fdrc=LXDetjP7+sFg0IFs925m8zr-5k4!S*=k{(tZNV)a129B2~SV-a&$`W476IV zjunC~2$1S&u?H83?XjTV)8IJ1GJorSuT-~?i|32MDc1$JsFmL5^|_9y%b8mo=Ws)_ z^_aour}6K9Z~XVaKY6`3ER1+m;PzN}JQ}xk;oG-0TpU+g74G+vtt(n^WI_xxfBbb| z79{K2vmqiJeSj;ihDERHv}V|hvb{^R>@h=l_05uig>1elF&PpK;&Yyq&41kDinL(P zaA@Z+M8-9I2&7FCk_RTD{P=P3>oaKWNW1E`R?6dR==!)Xd^{Iwb3`f-(d%hhoojTb zx_sONTmEj#JlEc?BXcg8qXaAg@fhB04=1XjZj1lv}Qj~j%lbKVHE>w3ox+jQz zEjy0ssDyCY_h?n-eoc~>rKUEJ6h)KISG2oQ!qn(Ttb~3%SDqgWOMfer8Z&@Xf-g(# z2cIi$`yO-rb%~pAHKJaN+GRwS+|FYlCawf>iWA=|@a!UhhGVT^R27N;x;b7QKwkbE_*`H1}JL z^%*8J_X|bONp#%kT7M$iEw@l=F+?0|e^aX#H-BP@7HE*BZM@eQU?ZupO=F0%A3=!D zS)FJ&x}wg$#SUK#B-d+>x9`TUUkB?_xZNu5&bCE*&Z#Upa1n{2$M5UPx)z?#hDy}8 zO{XoD`%UTHSz5_!$$FMR_=r4-^!Xt(Csn6p*Z=QEV|4em-7IOJ` zp0ZdHj0B;!*u}%)sG7S(o$udQS}m+g_=&M!bfdmHr6}J%R`x!5-eb|a*09sgZlfhz zU+R2!&2li7mgp|&BMCXto#`US#oK9%%S0u{;pgF{PR`j!a0~ZS7zy>ZmdnmA3hSyD z7ufcZi&E#hx_>aHF-J^!?{{T2Nn10J!%%_KkZtYJU-KR#M@G{}<~llXZz`-CC;9 z(}f1KOlmvVUS_gBzcA_ih{FiU>#n$$%Z&ANTrZxQA<(~E@<5^lZtK?uXl=0vaG8fC zETr?90^C_|=fKfeS`ee#T2oZPeO0!@{>Bz9X>>jhFrzfx^XZZ)c!|0GnI*Ai&t;av zNo`{d-hW+*&AM=b+|y3apP9!npq3o5i*x#)LEXbms^;P2x9pE$F$D|jBj$QG?Q(LT zk4h=Rwnb@(5+>y;tgEopSVjyNwp=FUaX3GJb@~XRkL+uh{(AHee zJ{PmDR6lo<6_wmS;=~rYFvhdZMB6=goF`+3Uw>K8?XYRNik>7Q1`_e6X0eD+N-R{` zjG0?)!u=KqUaN4w1u$ueT6;VK8<04FWm)<48Ou!DK5-9)IA@7jAuS!Bm>k#Yw34fI zeqK*J0CAc&hT^m3ft)w5nK5$V%lUj05&-Jt6C?h&CpV_S+&!I{FhHJr*+sN4YYDgbU|1PU7GUrD&|-)MW|RwfC@=ME|Y|g~wfaKH&W(2MZt?fcCg6 zk4LbwnRB=kbhMpd6R%Z7&<6TZW4`CkTYsW$o&ylS`xUiySyFU4)0EMJ*_9%U8Q>SD ztR+?i7M&(;Ns|Ycw9PFGC=Y9ZJ9XewMr+*L$xgB=2 zv+Z^bI?PWZ72Kj##YHY#ZjKnW^=9A+He}D1log?>+g1)aj?jd&0S>fXzkLsPEJm)E~1w<}Qao$fo zUEbrE+;5fd-x-O@Zwt}(er<)ov$_sJ?LT2!vu#LtzM zxQ^d{tBfgpzB)%Y-tQo6fl;{?GK}q*X@BKjU=(v=*jd(^{-IONo!`DK{O3QH?Dt>$ z`zk?qHMIc*TPz-|ocw%RXzuZdO7wNl@pwdd*!ggC?D~+{b3;b1R%NYe!|OTSi6}=_ zNAA!URj zlo5*X!UKMwwrhgAxL>(fAYR0Udqngp8 zh=Y;~J{p7xwM5YcwTz*d86O}1z>+7-6vI%f$IT2i!?5S~W>)Xc3^Ufx_;d{5={k=@MFxtV;Oi>RY5ZG#QeOT&gmhPz6$@bW~f!pJop9s7v;vWfL1^lFd&m?*J90-tB!JY*FTmjFj z;?uv_^W#bIpY49Zd|J5w;f(h}+Xv<$(X@eiOx(UX0?K#4zDBK}xsY;3i*bH5wg*pW z16Djpt`7{WOk<(xh0uD;6xDi5hF~Uuq6P92`Nhq@W|0T}@b15VY3}yrD%+}oPge1zRI3HZ zXJX1{0)O)?@aL=l;_!;j_mo*lDe>X$N9JkatJgP#7!Y8$-=R72+rR!JH!m+aTy@08 zk#lC5GUG5|YHYS0X@=P{W=1K3QxFkMK~*pjFv0MME0_tIJ;7b`%UA!3?{EJH<9zyJ z<^Snb|A|;n3V0IyS#+yl_fsqA+YQ&7mpF5vhJUEx#j(tZu;~w9lmB6GUAQcdQSHfU1t=@48x2MLc3`IC|bY26a$2LR7x_$388^B z3(gCn-SExD|IR7D;fD`@_r>b-pA#kG)IkAL zBY!W6+jkGV{oy0aGGPYS*B5AU{NZ%)kXhM43lLi=#}h`14nn{PxX%`*P(!7x*(p zJiQKPX%+gD1;}T<1MuDTU(|4BU`a6rD}RP~c>BXU&JQEwe5Pq5@88{Vae1I=18K?J ze!NG`c>BXWvGKh6*(G3f+W;8Dap8QLnZ`uPpc&NQ{Fn&Q;XF)3W|<0p>v3KXryxct zMp~59!%S#A!zY2C!0-zt$WJWb`2=}B(|}*>|2aB>m4c6gm4E6i zW(9HZfBd`O@OOXxdzgi^Wb&d+!+S0+_H6gpTwEU59$Fsn&oo`+@FL=4LE?g$u@TQW zcw&sS8;`1ylTotayrEhjKDZi!ogzU=vmpZFzVIbC4$EK(EKwsu!bljI*YX$KzkfM%`ZzEi7xJj&xiFs-(~wxkOw$Yh*T4U_Y;u8-$D;D6Z))-%6$ z|AO<~$ULSBJi#*$bF}~w=6Pnb?I_6uaJblUKAqP%ti(om^TT`o;(z-VC!VVpTXt6t z+(6cur%cbt_{MYjRaKzX)#Y^J)Q4R)3H>&|g1=D$9IE^({W0aB+2O@^^kW=CNe|XQ2f4JrC z_qV+K`U2R#bk!4N{=ZU^=IUNT8axQ#)cTe9plx*xTJNo^O*f_*u z{YrH#XV1g!fH=5%v8@1$;==0R4xD%7*}#D~gDfDQ1lOXZR2WVRvGHhXFbWbJCd!hR zFZ}=Wgdkw$`OVL+et%6q!E~C*OU4Hw&4p!N2*I(;3o!=F3nd%7L(k*gnQ@#6!E^oM zf>H|OFk@y6=ZQc5!#iHS%ETs6igER_BXk|kD^`TN_Y?OY2D(i{+vTbtuf*svv;rMM zPPKRv4VF9Yp*$b)^MPe4Xfc{D zFrO*TV3Q(BG zjB~bji!mT-7CZi zZ67Q2Sp{3oj#3KVd(@O=QgX?psUYr2_%&lxK?TOsOn+L z;WGuGyuSQ%%HUYWjAp|%6=0gq)Af;gOyrc&0`t7A?+f!ZbMw^!7aU)GyJxcxEOX}Z z{zOWdzHd37C+2BEi*h;+I9Eg2tHBPB8o@PoeTz$322*l3e+lFm(jT{_*djGedxqunX{F2jsW*!qE zRudO$0@7__&6L$@D4YX-_}h0F6%?l_;R7s_(6pWxuMgaOwL`T=i-)^`-~ZiPRE>FF z$f@9*uS&6c^EnlkB{MuOXg1PZxcPcRj2?q98a7 zA%AD3X&c5NGmV+pcpg4X9Bz7okEj^KDWNLNQ(~DCZQF3ZU%0eY3sJc%U5G3LnBSaxaj~2yy52iTp2vLx(n&DGSzPR3UK3#A=jx0;& ze5&R*o@UgH@jN3=7>9tW(6kL9I3_bh1b^opk~4@bT|+36cATH zh127V7OOdh6-qY7VdVbq$i>wTfaAl!?$D#EY!4lqT|~UCJp>4?pb4f^<~QH|8~)~p z|41+t@rkyJOsyx)g%BNJcrVZ4MFfgmbPoO4L}EToV!RY1+sL2YPc8-3%)1}&P|aLi@0q5Be$x@# zz}@>t+Z#0kq(z?5dw5*tsyX*pb7)tWHJS}B$)${{6=g6s6TFwdcT8f4!h=;t?P}y%gR7Xlt%oHb%qJPTy^jsHL|Q`h6^zdd zZQs!ES{^>0n9d9FvXr1rIw+6AT;%# zmO0aHB5faQnObUjky9aF^fk=+EwPDgHXYNLc>m)aj}Ha$0SuZp@c3}1>l&J_q3b)E z7`b_IjrX3%`$y7J2%+V0)v?+1Xfe9J0~2B!(NdFI7ab)PLgNWxZK<2_@L^<{W|}ro zv>*bZaUAbP<{|U({eJ_$`d_{wpdK+nMDQLqdykrD9t$CQN-}cJ3=a#T33wMkAeYRt zWV{!4hYiR3fo|K-Z#ukpIPYlNmQo74{f^)N_6HvCkCbe97uak%+OFm5`hX8G%(YrX zgw1x#IjpRiM(SI{d6Vo^YJ=JM;b{9Rt zxte#USvlTM#L?IIDK#}7hr}`$axOS=q{UXGhw;4d@pPo$HguboK6=_du*`*r+ks^% zKnP2{}~^GBxlDC}K)Z3xCt76ibLeY%6d&5vFnB!`u6s zvzj9>3&)3n>z9|r=-G5zriaXjk9T~0f5+AJf!(2J8fPvpw_IFrxwzgCtknKHCUkwn zJS9$d3BwppGw0(>&NYpFKRx1{!#OxUj_me51{@As=D9XC4bUQVU5k+T?pLo!snGS2 zqJ^yKxqrup6H2C5cuS$(M51rdBIJ_s;_>3}As`Mmo1S(PdAvVjX5^*zC6{?2hKTcy z@jNn=>M}Pk4@|?v<<*X+^9<*a*aSjT^8_;^SaH>lN=+K_0;k8JhPEIzTPua*O zwZg^4o_Q{`ZNv5D71Oj38zC>s_3Ir?TZ^BXw$G%cmg^@wbM@kauvUyk3l}#%b6R-u z>Vm`722jGnkG_OxVwAc&wlweIln}U za(p;bO5*C~lKo*{!=p}&=K<%0*b1=;eDmj*YBQXZLLqqHW zIb|*`c9c}Dp>10}zQ5z){>1BVU$EPDvGZ=t~5`xDN4F0T(YB!Bvv z_=ppVWp+*_xCd&-`(@!-97t5Pw0B#+h1N1Vn9np`Q5(d;&Q|7$0Nge zd9J-5@9PTec02a_mSvo2+m@z@_~XbWp!yZJ@6XKh z!u{=urVTW0wV2!6dzNKkv)#~bw`_ME5BHDkFMD#%%u{Wu4(Hl0==z2b0)L0A4PrtF zfxrCcKd0*(EG~R}dr!%U$4BFQoCwYn{DL^)e4L13k8d2A8oyPQZqsmiy`_{)6h|%v z7X+mcLhUJf=XkswIiF^}|NVO|u6DJ4P@~^81aT`#!915Y9rKvE`MSXe!Bn`sX_zPY zk9_y*S8NX*%e=6RiIi&% zf>GxM1h%_|yc9m3j~pLr`Rqh!R@V$zrLZ)eae31K4Y4cW3YHr}1ApgJ^(T=!vU=*G z8W2O|@or!|C0>1d#Wc({XV`n337QHy71}1Szw8)J6EA+YV}H>xJf;;>7N&7wf7o*K zqBbq&dEw$_%Q7X7w-fuT9_JkQAI`k{;gO~f+}|F#xY^)iP1B{w%1Zz(@bT>9}TaK_Z=HVvo8GtLJ}(h8tXxcl%(zik;F6U&^LPMMOGo0pe3Mjmbl`o7_Cx#i(* zVzV&>&yp7o*Owr0`SO}qKR@vO?>{mQGnmn~9?ga8ms_+bkAHU~PCPz3=4s~YXP2Z| z*=$=<%3NOUnCF>p(-6|^(NZTqv2nOi6{2b^gYf_Smp8QpZEAP+`o)ft4Cft1jSw7d z*VOFq?!?{gBc{-{k<;lB?;Is(_Ln`OiPh)Lc>FjqoMtYr_PqV!9_Q=ea~u|8>rty^ zbH5Ax>My@$dw*TcdKrzJ60g6#LbcHKfn`ZGu~As2!k_-(16Mb-H`%m-Wm)Jpj(JwL zT|-I>MIGCHOE6toU?QYhX+%I2!Qq1O^>>Gw)~3Ye^`0g~hGE7@?G&k2_LfMkwM7ox ze09NY*ORkWARI=-*QzlsiNnQ)*my3lcRbt;e0Y0Dzkh8gs%XiKgX7iL*SKKp4;|Y} z2U5TrzA^fJU_58q-qUV8u@$CsK|*C){pvTbdGq}pzx(SGS2sJZU+(F*5s^C5-(U7L zou}<$tp&u@xm(GOywufoG1GML)wc~duOrI@@86s$8Tvkc!N+mFk{)u3{OUKaxp}>* z&6KGSLVsPMKm7Is{Wj8XJ1(wUoC{=~*c|q}{`LR{)3|W=Vc_9z;PR`chReo*Q(Uag zR}A{CXLsqzi#@mI#2h%Q)2x(v{j*CR@8;U1Jue*Z2V&>(p{DZvCNhnwhC$R(vVauC zS)IS3n0b6KO0ILdWm4`w47~d0f{W|6#@g$q3V%Bqp>>3&_VDupv3G!F{YrFlHM@&#oj@aoW?_3#`7C*H zY_>a=N$I+_Mgu3P>N+>bv~6IT5;6MP6U5bCooZ!cnHiQn^VtzvXxa*p-~H;6m#+?7 z+!F)OI0T3lhM71~f} zcRQ{_!+0|8KNMcP+Tk3`OQPF2QZl;U5!5n4TEx{snG05FGNBNfg`_7^$#_@W$bV{? zalR+z84*t|3t2PXMYPNW7s-0Y;SnOiAONJuN*;#2XXa$YfwK|?jI|s*l88M zOz6rV_P)hrMqBJFc=~bTRwkL^jjHqTn@y=r=xOH6T#IN5UE4Cn2nG^&^(i0&z zbX#ACER(U>)oHT}6^KOD9KIOkDYRs~mAYkZ3rhd9HuR9C^+9^A@68TiwO|Ci8&O+`|tEBy_`8kChyVTs^& zEa}L!JYrf8(}Jq6v|J6G7cfuzv>cJR%+^zPgqM z3P;RRbwMkXl2Dm3YB$-$@X}yb2QrAOpemoNqlrC@->mjpa1v-cB{qih1*DK|sH1>& z?I;{^K7utAPUJj1gPv+$eX!J4^g2g?mB%ZBNR8!A1QA%u9p3c_9w!aaIjqDr>nsFm zK)~4+lMy2!Rv8}H6MuzYSG=hW7%y(UhlF>ITxLW9kcmEE8pvg?*y0+zYZ0myI19)+ z{Sj9fhxc{hJ1zGWoe@$wv6K%0cwafYxy%gHEh&#Uad;mHF5<*Li#^oK^?t!#+ogvQ^&rZX;>Vl`?1Dp1yvZ|h7%8cMT1YyZ%MCuqS_y+GH z-Z#}E^15L{3%a6#lWLA;igN)cuJZniJk32YNr)*X87CF|aygMWlFLLTBNx0xnr4q# z?Ma#~h&W6OF@H2bwYGW>%d-+v9$C_aWJf6@zV|p<(iF(WIOhL>TWOI2;N9Kd@bdCk z6}8=ps#Yg8P3SRKBSiGIFX$y=nkmfdx%#Z2Y8|rTMCIzKI-C=nGmuQtp#mI+$&7am zsxu(f%%xgYa2^5L*t4WFC69O?@y=J;jEfjr!?7+n7k}|iP*?i{SUo;1AK7&svDwl_ z<#>1Ei6Rl6LR-;=5ZZOSimrYaaT4%xv&Q<039Nu5;&5glo+fmNID&5~!HR?@VhH2O z<2^JTh$yNHrVB+EipaA&JVm57w1@=SxJPXUJXz{@n}UfUgy*=A6qHgcQ%!Z25NqOw zRJ%5tiGOX&kGKEndG((aAi&+@A9-={%hk78sK}(?DR%WM<6UFA0mt5mJYT&{e=fSzS;z~o1mP{hJK=hGh87S)nDWY1c zRXUGIOU~nSLKZ`}MsdI1cVRAnd=~zBzkqxu&VR>;f8cWeP5md|QOZausfHrWG+|SD zg5o||0YcmC@vdDn#zGqpIQERES~h#H#D2k8aJU+`wZ?7dDzH{7h&bn094Z(yzVdSk zu1>Ml3YbyyP_su{h+(rfM0-*h39h;cK=4~IV_NQbIQ_U{#wUJ!_#}S)8Kyh>W{(q3 z%70Ix&&Z~jX8P_5C!tz|K`sk|L$ww*!SA75V!$%O{oTYj-?fM&RAi~vpC;ibi-mefALZXapdj8Q51 zr3^e3KCI#G2?M-~&)^$eSPgZ-iPf7o#fyM=ob!lO#j8$u-%)g`RwYpKfKVu9VI02{ z{%J3W2!wyUUrg$}Zo02HfYqHMD5V(Z@jYF$r;YpPe0-jd&#NFJxG(~+-EP?Kz<;?j zS*L0x>*@$~zmazp%xc)HWMH)gw>C+HiaRQmP^D_P6@X*d0#Y|zmJ{H}`NY$H!Ze*p zX?QmKX9?GW_t^#HS@ciJD4H>;tHR3d7XdAqc{yTQ))mv|3+FsxVMHWSbfW1h_!OO~ zl-wHMR~p!JII9OLD2h~y9;P_g;eVu|*uYak^pshQZ27J3#2?%_mnU8 z!?12c_=Exfl<=$n97kRR==ysk0a-aX}Kr*4ls&5fR*Qx+EH6t zwE|peIWE>JR1>B%;u=gB#07+cNWtSV7s-}ua$Gs}dSzw z)Em{YTEg0Cu5B|}!=y*7&I+m$$}{+S*C3%KjYqIjE#)(lhfikzB$pWkk9Wdno5g2CFZr1#d!iP zxl|X@wE}fg3$k3!%=06v>B~@O&&K;n@XxH_GYt5rg_ocFt{(E~ywBZ5a}{(gmTUz) zt&66(LB@QpM1SqSI4p(NqKoht$gA0_L zmfi_lx6KLxW3CvnhPmn-g<*VG!!)cV`lortlO@>ave}oe@c#pHQh Date: Fri, 6 Jan 2023 02:08:54 +0000 Subject: [PATCH 7/8] Removing logs for real this time --- nerf_accumulation.jpg | Bin 1371 -> 0 bytes nerf_output.png | Bin 6923 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 nerf_accumulation.jpg delete mode 100644 nerf_output.png diff --git a/nerf_accumulation.jpg b/nerf_accumulation.jpg deleted file mode 100644 index 4ebe6790c538da138173232bc5c4879ea06d1526..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1371 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<ECr+Na zbot8FYu9hwy!G(W<0ns_J%91?)yGetzkL1n{m0K=Ab&A3Fhjfr_ZgbM1cClyVqsxs zVF&q(k*OSrnFU!`6%E;h90S=C3x$=88aYIqCNA7~kW<+>=!0ld(M2vX6_bamA3AtVfV!aKuFsonbfY3xt=)a_O@+`(u79Tt!zM-b^d@n5DlOd7 zq`FgI&T!VEyhSMiYb;HR{m%V8P#7Qd=t{KdO1F}gC%1hylVaDCXzsgmyCL$byWiCU zk2t~W$~}RZS)X=$Rk%Kx99O>eY0RY;6%!}#oLDt4cH7+>Ne0=otrDgcD(9)&BwUZK z7T7Ui)2ErQ18=`PWOirut+UFEjz5?s6=f}fWpUz5oRJduqwtr&onj2mxx6G^6{O+w0uNt7$x5GB` zgRe^O?>(*dM>6wOqYIq>9QnRpeO}%w|FEJ@>%151d7Ryo-Y0!+Vb|+~2elQ89=$lL z5dU!58s|)l$F{4l_SD|Lmv?5(v?u0*hZqAICf{4VO7zJmtrrcOQ~O2pxHCAp)#JS` zUJrS^ZrR4wsYxwG&2mTMvKL%7IjPby-HGw*$}exNE_m6NeH+v4?5fnvT}P;Nwdt|Z3KJ|OOmA{+ocY?>ptn48@&B6u#$rbk diff --git a/nerf_output.png b/nerf_output.png deleted file mode 100644 index a360f56f314c5d7164a6d6ebb68169d89e7a0ce7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6923 zcmV+m8}#IfP)V>bE;TMQEFfrfbZ~PzFE4FjbZ~5MbZlv2E^l&Y zFU>2t(f|Me32;bRa{vGjVE_ORVF9Q=r)dBH8hA-WK~#90Rhw&)0;)5dl#L1UF?)VGPI2x!;A`t)qJ2 zIBtc}8)GWJ{(|>5nceyP9DMt}a9a!RhPxvoyk3*hVGQFqjMm`e+4%l#0h~UJrN#Tr z>8J?i(7W*;-wPj4r?rmCL@K&5ICz&Vc27{;6j?oO?R-Y45ODMcwo0nqpa zIF4ZEhzodp-8+~$r3k%4DV5ehWCD3uLqK2-*!GDk*mMAD1v6nzs72|0f(Ywc5OLPE zfH=Jyr3fNm2C9OJfWfzCWnGzY%IxN)fbxyJ2=+BV#&l zj!kD>D)+lmtK#Mv2*)iBeojzfFvhFL-4JR{{)5hC3oB5wwLOvV&y6^u=9tR>$DFt_M1XTf#n^bX*X>^<1)^go^#9lM~3+4+V@p|rYAT=UBX|3>luDrKS z?~}j&dULxsM1}XeLDtr~7I0@wM>T5IG29}h=XCHmpmiw}jk*+zAj}E9JJaOKqCSjT z<22@Q+@0ev=5WR^RHFn%1b4>(&!4qgIC_+Xkb%f@G*eVO-q$Mbiio4)%nW8-*UHbI z8=s#$`#$K$z&$SF*RPFrt&Fka?g1V+=JC}SS%;af(>&8V-qg(FFATJ1^fOD`;>PEU z3=%PLFf)p3+^Bg1A_$t;q}ViicaH9i9-!I#q!hWXeOh3Ht1xCo0`cAB`+gX`50EH{)(Y!dC?&2zB}%|7Fw4I1JHQj6 ze*3nvZG+oi3-`yu`+YD5hzhF8AAdaf{(Zgbe7z~PI;BMGQ*{L4D3N;L7Z}xQv>J$y zJIA5elt6liGKDdO{g?@6c12h+gW@oST7-R{fK!XIAIAGVadT=F*0r)UVO^A^Nua)* z2Pue>83r~L2lgGRh;zR;zW;WM(*(AyGy0@8MT>B|Hy)36p=+%XZB@(k6cA^QNUc%> zAMnVgq7nVp;$n^?&gAG5Gr$UC#Lb8pBU^1P5m`|vHKS;-Ph;PWV>jFr zQUd);0bDNdn0t9On9*>3o{f4S=?1RtGo!TmMSTv9#v?fV9bFZIHE?)^4YhzAQ7Q9#oW;fZPECf8C>M-^~<6Zh2Ehb(HhO;a~}gU&DV=e z^a<4??Q3z|C4kz5{fI7e98kAyTwwNiESNdVQdw_}IcBsIj|*59LFFVBk`X^6o$c10 zl7}RcpR;xP)F>~BJ|3VJ;20pK;Eu$bAlbi*3jLUrTe*7vm_{ptYrS|#txE5~C2A>n z75WHimI4lAOvW(2f4_4agHn_^;#u7glJA)+I$U7=>EF%^4l~Vf0}U`5tU;Es-@rs-oYTIsFK% ziMR-3l3fl&G;nHJ3g$r>Y89T(g>4&rd@S5ij5Zlu zC9Yw*P}J!I#)w`#6I4~BT=t`f78v_Xgbk#uwXiIekB=q(pg^l7divF#?22*#;>o$c zG9cF=M?6&%o?E-$4q!W8gs@q6j54Lh6UU7+uL}(I@^2je6ExN>sn~7 zuRM z^)1zEYV&u5pK7}7;#}^Mk=>ja0`n3*RO-0KM!7?11GxI zKmNEUjSdYW0@7phQb>ZsLNn(aUewt-7O=OlGrB8Qs^B%KM#KX+L^<6 z+?Qy}MOYSPjNk=h7|T*oz=U9mNyNsCOrK`iQjIDiK|SY}4xiJqwJh{*f&8l6YTUe* zl(rHIPjFBW7aWax6v3=u7UXA62^A*=!2;Bh&)Za%7JT1k@Pj2eLuc}SPmbN#w#mAz z$m;y|ec^F$xEW^A2gY=cJ`qve;&5tF%waz!V+`h;%z5?=r`~TZ`T0S0p^u6Cm#npn zl+7R%ZeX(pcbAD2g^(xF2043fVNPXqq4y9+0aO`{kv+W|MI^sBIeMpPxhSuVsNeTH z{EGX1;rsW6rA5aX)8Oz6W6a_J$8es{#x$dkQ^yELx40JsEkZ2?ZfgN=Y}>d*OerV0 z1qHN5ZIpF=8^pS@G^gfx>SZZa=P?w?V zf)quIRX(0eVo=~h@QC<2^nXa}T54(m0uU*oGgO(w*wgc+HX-#mS(aLtad<7BVac>BLRm<|#agJl4CGcrol~RNE)b&=Wsli{L9o1+n zAJ2vLrj!~as43l5+`m`+1}v_eSS`U{TgiBNZqc~R-?3RpaRcl zi-WUBQ4wL=2e-Acv`VevNwyZGE6T(;j)A#xzqKpPwM53Aer2setHVDCPe4X6)avP{ z-EWOr^y2hL)J;jRt*9Vs=*a|w-lQ)WKvzoCwriyRdmH@qS7+ah&sXRD>b&0vMJxCF zN~wj>jc?!9ATeVGXw^#Z;pZLuKqZKk7>pS>6n)*2Dl4z|K`mj}o~0`SODl;>(Z$Rh zn}#mYm1?lGxY=ciI&pXI_d>1FJI%ucDaq?a0{FsKQgBT*W+7``5R{^oIR@|7-WCUsdPs3Kpj;qj={8vXUu_m&+&Rj?U19S5hE=^5n583xCm@-y z1!^E_oZeX{wQ3^z>B@w93rQRH_pt|(2l|G%P>p_nY#~YC`T2Hl^gel&=qjO#;I_6f zLy7OH!u{TG7oN|RTCyaGD|VY4-4Wtmk|6jxte!IUI^zbXfSh`Ea00)RC$% zW>{Udqb0{j;z{SmJrO-!| z#nOcP?UqpsD}3K4_{7s;J`YGs+6n(|Jhe<%;2=r8;{YO9krDN_hK`R+I4vbl8w)fedzv5n7X81L zlQG80&mp`wqcurmrKck#tV_)saQ^yf*K3um+1AQccHsj!@AqKe2m3w|DYT}^2MX&_ zzqmmOlGgI;b&dJKw44E^LJi7Z=wn_t4^9nbmrjocNf`rN&hgVZfr$fY3zVdvbDX%L zbb+S|`>q^ED6TP2aU}@Ow~rWVMt3+pgS?649s3cAt@og-`#!ndl%)l*wOV-G7uF?O zVJOuTx16GygHk9W9O-$EVH`cMpw?`o3e_xJ-l9QA=ge|Q9vu`oB4*PmRamOV5Omtb z)k!h3EP`r9G)7xOILZ5A)LJiOKlw%|(3nS#?(D~4rrkAq3^V7PF$rE8_j_Y$%DPl; zYr#BpNNdSaaK_9Qo5sv>O!gz@8&T#VqDnA9jk*%31b06FE19b1UzU~)CENWq$>|;( zVLzOHI4U7++Zxn%KW2&-yQaPOqKMcv69nbG?eTL@P;3=dM-%CTH;{lAoomkY5G4jS zDjwPbc<-Jw2%#;(BhHD`(qg1mU$3x<+?35dS)NDRv#()h$W_aW7DNK%T8)knZmsd& zChJl!rnaIVrk?vh*-iui>wR zuv-^l+o4r3IhiTk)|y!zKEW7+`g%D_6E?NWge*n4-vdxCFw`#>N6_pJVS3DSP;qz; zNgY|EPfxRSe!x*kui>+gOW>#|KYtzk`_DH&{@h~Hv>2@wT5Y*AkSM+?eZ)o8qL`*8 zF<^QEd1OSbFo1I=R%>)0iBEA5qdWUH5JD1%kO;C+H0ZG8jg+GFo}M1gv43>}++x^R z3+t*m$WV?RAZ3TMROOJ#7(G2m<@Gu^j!<>`X6fBeN~!$X4*J-+-xikE`2JDydM54# z^GQqIKSnHaj5&i}r1Oge>LX68F2ir6=7YMI81}zf4}p;{`KT> zFKBVrMOhcc381rSthZ2!%Be@r`9bvlQ+Smce#FdSP#rUbLm|K=JikE7Ipdf1V_eLx z6fW;jo3ga1=Z7ill7o<{+?p_pVm8xWi@kvN`^b(GzGFWoz0VxVh5`ER`QVTLxbx3{ z-Y7Ms`14UXt+z2j=a&tCEj{G2#`)pI^+y0Q?5@1tGiD(I)ht?r6Q)7X$y0cQzlO_2 zWs%_fOV!K&Ukj9Sy1!w9giU3*g|dbM(F+gE5?MANQQ%82|d$2mk!% z%HyGDwlZx-O%IC&hnamGu6=+xL+PCd;qELg*&R8+N!NR(r%=P&FF`hzaDHYAATz%- zK+XCI?=3Fswj^c7X*IgX965X(#u&!ur}5*jgZ(gGuY>oiv+d)uy=U$Lt@p;qbK!m~ z{QmpOvIegReII}~&0_HZ3yF+j9$l%Amg}+Lt;jU2#;N^X@Ib8&T%;Z_1`ak{CM+zcYb{y?Ayo#tvHk0 zS`krNBi$Ukh*_;sPSYlqoWrJ}MHn-jq98WO?y~=n8GfLMM&_R5`EyIfg*JtKH|o*~ zV_FUnJzbt~PfnGn5@}sDjMLcRnW3t3HKu5|LvG6ZZT$Sz`SZul|NE~W{Q2iA4Uuzn zRVfa?{r2Rx#?6!>tZNV)a129B2~SV-a&$`W476IVjunC~2$1S&u?H83?XjTV)8IJ1 zGV6Y?RJV|e=ZnE9*9EtzmEPy|xsIsInOhv^a6_~8n8D|#@$Y|c{P({m48n<=f+qX4b99LQu?)Q?dD_U@5LJTv1{B>X!Bb6$O<7?>pxG#J>7ix1vDiG1@X<40Xbf>y}+yh(wZqF^w^c7C3eGg6I&fPA3|xcL_wMvDQH@oyxRJ?m81A97U#|lo!6^O)gl(Jr__6bP zADG8{@xE5Rf4kFK;g8?%k+MzscqZ3ZPmeItHTD+6@K_gROygM!%aX&_GH4anHD~{v zo`EH3pXN(!gp3jM!BUiaiIbUAkS8ONo+4pEw=6+3*m!+mQkQ7Ce z&sVg&Qo_{eN34W?JXf9{3rj1M8Z&@Xf-g(#2cIi$`yO-rb%~pAHKJaN+GRwS+|FYlCawf>iWA z=|@a!UhhGVT^R27N;x;b7QKwkbE_*`H1}JL^%*8J_X|bONp#%kS|Zvlw@_*^L>y~> zQ>zv?e`1LiXpp9Dyw@0DBdM@WV~DaJL5R*-ooG0^qRzg>4qpr;*K3Zq@5ZlR2kTO} z-74CPRiidZNj>Q!lp;8{wWd=WWmrnc^Hv+n5`T-tQnpHhpg2gRKjIV*jd0DS zJtj?9EJ+k~yqu&ifYCg%qP7|*^zo=M6sewT>R~L1)YuRcq4${FltOBTdRfg8y#ToP zJNAuxH);`XR#M@G{}<~llXZz`-CC;9(}f1KOlmvVUS_gBzcA_ih{FiU>#n$$%Z&AN zTrZxQA<(~E@<5^lZtK?uXl=0vaG8fCETr?90^C_|=fKfeS`ee#T2oZPeO0!@{>Bz9 zX>>jhFrzfx^XZZ)c!|0GnI*Ai&t;avNo`{d-d%~!x^RKq(@xKyna41omK?E*bNZh_ z-NQ|)=HcVF?2lnF1qf{q6 z{1V8xvk3cp{(n<5E+cCwJ51rqltKhV2PV6akuot zlKg}}`1u;Rmmo)c{oXW7yI$}DP4rTAC+kx=y+^q+goF#{G*05+45etS;nZab*R}Vs zmPG%q3WdjAc|PF%CI<^38i4k=E00I8vYB(Z6LhqlU=y!ZM9>EMQDeU6&Re2wo&ylS z`xUiySyFU4)0EMJ*_9%U8Q>SDtR+?i7M&(;Ns|Ycw9PFGCtZ78H`eii6n>l?5 zOM_*J8{PJJvy#&Fd@M#X{bS9=5sBTu_hy;GA$`xRSz1A!r>cxa%GZ0Oi7|5+6GloO zr{7nS1{-qRa|R3(kaG)Bepv6fhPoKx6( zMCiUoZJr!Fc!AUN5RN{t2Oml+g1)aj?jd&0S>fXz zkLsPEJm)E~1w<}Qao$foUEbrE+;5fd-SR(CYx~)VWxT-7Dj6d`QGuIk+R+*f~v8)H+rF`C+Qx}v3q3Iwh7CcqdNccEkb|f zfTTtVxHxs+Cu17lKN|O2?B<*i2G4&-ot@Dxr=-op7#1cpQ@AZ&qAxfL>u?U-qQ Date: Sat, 7 Jan 2023 10:06:52 +0000 Subject: [PATCH 8/8] Transmittance loss bug fix --- nerfstudio/models/dreamfusion.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nerfstudio/models/dreamfusion.py b/nerfstudio/models/dreamfusion.py index e5f7cd341..78bf96f43 100644 --- a/nerfstudio/models/dreamfusion.py +++ b/nerfstudio/models/dreamfusion.py @@ -131,9 +131,10 @@ class DreamFusionModel(Model): def update_target_transmittance( self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument ): - self.target_transmittance -= (1 / self.config.transmittance_end_schedule) * ( - self.config.target_transmittance_start - self.config.target_transmittance_end - ) + if step < self.config.transmittance_end_schedule: + self.target_transmittance -= (1 / self.config.transmittance_end_schedule) * ( + self.config.target_transmittance_start - self.config.target_transmittance_end + ) callbacks = [ TrainingCallback( @@ -156,7 +157,7 @@ class DreamFusionModel(Model): ), TrainingCallback( where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], - iters=(self.config.transmittance_end_schedule,), + update_every_num_iters=1, func=update_target_transmittance, args=[self, training_callback_attributes], ),