diff --git a/mattport/nerf/graph/nerfw.py b/mattport/nerf/graph/nerfw.py index 80fba0402..06ae5d81f 100644 --- a/mattport/nerf/graph/nerfw.py +++ b/mattport/nerf/graph/nerfw.py @@ -212,6 +212,4 @@ class NerfWGraph(NeRFGraph): row2 = torch.cat([depth_fine, depth_fine_static, depth_coarse], dim=-2) combined_image = torch.cat([row0, row1, row2], dim=-3) - writer.write_event( - {"name": f"image_idx_{image_idx}-nerfw", "x": combined_image, "step": step, "group": "val_img"} - ) + writer.write_image(name="image_idx_{image_idx}-nerfw", image=combined_image, step=step, group="img") diff --git a/mattport/nerf/graph/semantic_nerf.py b/mattport/nerf/graph/semantic_nerf.py index 9b32c6376..02357caf6 100644 --- a/mattport/nerf/graph/semantic_nerf.py +++ b/mattport/nerf/graph/semantic_nerf.py @@ -155,6 +155,4 @@ class SemanticNerfGraph(NeRFGraph): self.stuff_classes ) semantic_labels_image = visualization.apply_colormap(semantic_labels[..., None]) - writer.write_event( - {"name": f"semantic_image_idx_{image_idx}", "x": semantic_labels_image, "step": step, "group": "val_sem"} - ) + writer.write_image(name="image_idx_{image_idx}", image=semantic_labels_image, step=step, group="semantics") diff --git a/mattport/nerf/graph/vanilla_nerf.py b/mattport/nerf/graph/vanilla_nerf.py index 3332bfc43..931995969 100644 --- a/mattport/nerf/graph/vanilla_nerf.py +++ b/mattport/nerf/graph/vanilla_nerf.py @@ -206,19 +206,8 @@ class NeRFGraph(Graph): def log_test_image_outputs(self, image_idx, step, image, outputs): rgb_coarse = outputs["rgb_coarse"] rgb_fine = outputs["rgb_fine"] - - combined_image = torch.cat([image, rgb_coarse, rgb_fine], dim=1) - writer.write_event( - {"name": f"image_idx_{image_idx}-rgb_coarse_fine", "x": combined_image, "step": step, "group": "val_img"} - ) - - accumulation_coarse = visualization.apply_colormap(outputs["accumulation_coarse"]) - accumulation_fine = visualization.apply_colormap(outputs["accumulation_fine"]) - combined_image = torch.cat([accumulation_coarse, accumulation_fine], dim=1) - writer.write_event( - {"name": f"image_idx_{image_idx}", "x": combined_image, "step": step, "group": "val_accumulation"} - ) - + acc_coarse = visualization.apply_colormap(outputs["accumulation_coarse"]) + acc_fine = visualization.apply_colormap(outputs["accumulation_fine"]) depth_coarse = visualization.apply_depth_colormap( outputs["depth_coarse"], accumulation=outputs["accumulation_coarse"], @@ -231,8 +220,14 @@ class NeRFGraph(Graph): near_plane=self.near_plane, far_plane=self.far_plane, ) - combined_image = torch.cat([depth_coarse, depth_fine], dim=1) - writer.write_event({"name": f"image_idx_{image_idx}", "x": combined_image, "step": step, "group": "val_depth"}) + + combined_rgb = torch.cat([image, rgb_coarse, rgb_fine], dim=1) + combined_acc = torch.cat([acc_coarse, acc_fine], dim=1) + combined_depth = torch.cat([depth_coarse, depth_fine], dim=1) + + writer.write_image(name=f"image_idx_{image_idx}", image=combined_rgb, step=step, group="img") + writer.write_image(name=f"image_idx_{image_idx}", image=combined_acc, step=step, group="accumulation") + writer.write_image(name=f"image_idx_{image_idx}", image=combined_depth, step=step, group="depth") # Switch images from [H, W, C] to [1, C, H, W] for metrics computations image = torch.moveaxis(image, -1, 0)[None, ...] @@ -240,23 +235,17 @@ class NeRFGraph(Graph): rgb_fine = torch.moveaxis(rgb_fine, -1, 0)[None, ...] coarse_psnr = self.psnr(image, rgb_coarse) - writer.write_event( - {"name": f"val_{image_idx}-coarse", "scalar": float(coarse_psnr), "step": step, "group": "psnr"} - ) - fine_psnr = self.psnr(image, rgb_fine) + fine_ssim = self.ssim(image, rgb_fine) + fine_lpips = self.lpips(image, rgb_fine) + + writer.write_scalar(name=f"val_{image_idx}-coarse", scalar=float(coarse_psnr), step=step, group="psnr") + writer.write_scalar(name=f"val_{image_idx}-fine", scalar=float(fine_psnr), step=step, group="psnr") + writer.write_scalar(name=f"val_{image_idx}", scalar=float(fine_ssim), step=step, group="ssim") + writer.write_scalar(name=f"val_{image_idx}", scalar=float(fine_lpips), step=step, group="lpips") + stats_tracker.update_stats( {"name": stats_tracker.Stats.CURR_TEST_PSNR, "value": float(fine_psnr), "step": step} ) - writer.write_event( - {"name": f"val_idx_{image_idx}-fine", "scalar": float(fine_psnr), "step": step, "group": "psnr"} - ) - fine_ssim = self.ssim(image, rgb_fine) - writer.write_event({"name": f"val_idx_{image_idx}", "scalar": float(fine_ssim), "step": step, "group": "ssim"}) - - fine_lpips = self.lpips(image, rgb_fine) - writer.write_event( - {"name": f"val_idx_{image_idx}", "scalar": float(fine_lpips), "step": step, "group": "lpips"} - ) return fine_psnr.item() diff --git a/mattport/nerf/optimizers.py b/mattport/nerf/optimizers.py index 480358f64..917eeed0d 100644 --- a/mattport/nerf/optimizers.py +++ b/mattport/nerf/optimizers.py @@ -83,4 +83,4 @@ class Optimizers: scheduler.step() # TODO(ethan): clean this up. why is there indexing into a list? lr = scheduler.get_last_lr()[0] - writer.write_event({"name": f"{param_group_name}", "scalar": lr, "step": step, "group": "learning_rate"}) + writer.write_scalar(name=f"{param_group_name}", scalar=lr, step=step, group="learning_rate") diff --git a/mattport/nerf/trainer.py b/mattport/nerf/trainer.py index 6a04b91fa..a612609c4 100644 --- a/mattport/nerf/trainer.py +++ b/mattport/nerf/trainer.py @@ -227,7 +227,7 @@ class Trainer: ) if step != 0 and step % self.config.logging.steps_per_log == 0: - writer.write_event({"scalar_dict": loss_dict, "step": step, "group": "Loss", "prefix": "train-"}) + writer.write_scalar_dict(scalar_dict=loss_dict, step=step, group="Loss", prefix="train-") # TODO: add the learning rates to tensorboard/logging if step != 0 and self.config.graph.steps_per_save and step % self.config.graph.steps_per_save == 0: self.save_checkpoint(self.config.graph.model_dir, step) diff --git a/mattport/utils/writer.py b/mattport/utils/writer.py index 94a6a167d..735f2687d 100644 --- a/mattport/utils/writer.py +++ b/mattport/utils/writer.py @@ -5,7 +5,7 @@ Generic Writer class import os from abc import abstractmethod -from typing import Any, Dict +from typing import Dict import imageio import numpy as np @@ -22,18 +22,46 @@ to8b = lambda x: (255 * torch.clamp(x, min=0, max=1)).to(torch.uint8) EVENT_WRITERS = [] -def write_event(args: Dict[str, Any]): - """write out values for all the event writers""" +def write_image(name: str, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None) -> None: + """Write image to all writers + Args: + name (str): data identifier + image (TensorType["H", "W", "C"]): rendered image to write + step (int): the time step to log + group (str): the group e.g., "Loss", "Accuracy", "Time" + prefix (str): the prefix e.g., "train-", "test-" + """ assert comms.is_main_process(), "Writing out with process other than main" for writer in EVENT_WRITERS: - if "x" in args: - writer.write_image(**args) - elif "scalar" in args: - writer.write_scalar(**args) - elif "scalar_dict" in args: - writer.write_scalar_dict(**args) - else: - raise NotImplementedError + writer.write_image(name, image, step, group, prefix) + + +def write_scalar(name: str, scalar: float, step: int, group: str = None, prefix: str = None) -> None: + """Write scalar to all writers + + Args: + name (str): data identifier + step (int): the time step to log + group (str): the group e.g., "Loss", "Accuracy", "Time" + prefix (str): the prefix e.g., "train-", "test-" + """ + assert comms.is_main_process(), "Writing out with process other than main" + for writer in EVENT_WRITERS: + writer.write_scalar(name, scalar, step, group, prefix) + + +def write_scalar_dict(scalar_dict: Dict[str, float], step: int, group: str = None, prefix: str = None) -> None: + """Write dictionary of scalars to all writers + + Args: + scalar_dict (dict): dictionary containing all scalar values with key names and quantities + step (int): the time step to log + group (str): the group e.g., "Loss", "Accuracy", "Time" + prefix (str): the prefix e.g., "train-", "test-" + """ + assert comms.is_main_process(), "Writing out with process other than main" + for writer in EVENT_WRITERS: + writer.write_scalar_dict(scalar_dict, step, group, prefix) def setup_event_writers(config: DictConfig) -> None: @@ -62,13 +90,13 @@ class Writer: @abstractmethod def write_image( - self, name: str, x: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None + self, name: str, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None ) -> None: """_summary_ Args: name (str): data identifier - x (TensorType["H", "W", "C"]): rendered image to write + image (TensorType["H", "W", "C"]): rendered image to write step (int): the time step to log group (str): the group e.g., "Loss", "Accuracy", "Time" prefix (str): the prefix e.g., "train-", "test-" @@ -112,17 +140,17 @@ class TensorboardWriter(Writer): self.tb_writer = SummaryWriter(log_dir=self.save_dir) def write_image( - self, name: str, x: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None + self, name: str, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None ) -> None: """_summary_ Args: name (str): data identifier - x (TensorType["H", "W", "C"]): rendered image to write + image (TensorType["H", "W", "C"]): rendered image to write """ - x = to8b(x) + image = to8b(image) tensorboard_name = get_tensorboard_name(name, group=group, prefix=prefix) - self.tb_writer.add_image(tensorboard_name, x, step, dataformats="HWC") + self.tb_writer.add_image(tensorboard_name, image, step, dataformats="HWC") def write_scalar(self, name: str, scalar: float, step: int, group: str = None, prefix: str = None) -> None: """Tensorboard method to write a single scalar value to the logger @@ -142,11 +170,11 @@ class LocalWriter(Writer): """Local Writer Class""" def write_image( - self, name: str, x: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None + self, name: str, image: TensorType["H", "W", "C"], step: int, group: str = None, prefix: str = None ) -> None: - x = to8b(x) + image = to8b(image) image_path = os.path.join(self.save_dir, f"{name}.jpg") - imageio.imwrite(image_path, np.uint8(x.cpu().numpy() * 255.0)) + imageio.imwrite(image_path, np.uint8(image.cpu().numpy() * 255.0)) def write_scalar(self, name: str, scalar: float, step: int, group: str = None, prefix: str = None) -> None: raise NotImplementedError