From 29ba6d8abcc1b6b815059091f1b3d87746c51890 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Tue, 10 May 2022 11:52:32 -0700 Subject: [PATCH] Add ssim and update typechecks (#66) * Add ssim and update typechecks * update requirements * Fix type error --- .vscode/settings.json | 6 ++++ environment/requirements.txt | 1 + mattport/nerf/graph/vanilla_nerf.py | 16 +++++++-- mattport/nerf/metrics.py | 54 +++++++++++++++++++++++++++-- 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 2196d04e5..73037029d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,4 +21,10 @@ ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, + "python.linting.ignorePatterns": [ + "**/site-packages/**/*.py", + ".vscode/*.py", + "external/**/*.py", + ], + "python.analysis.typeCheckingMode": "basic", } \ No newline at end of file diff --git a/environment/requirements.txt b/environment/requirements.txt index 56624bc05..a09a8a05e 100644 --- a/environment/requirements.txt +++ b/environment/requirements.txt @@ -10,6 +10,7 @@ mediapy plotly==5.7.0 pylint==2.13.4 pytest +pytorch-msssim>=0.2.1 tensorboard==2.8.0 torch==1.11.0 torchtyping diff --git a/mattport/nerf/graph/vanilla_nerf.py b/mattport/nerf/graph/vanilla_nerf.py index 8d5a62d0d..4d5acc1b7 100644 --- a/mattport/nerf/graph/vanilla_nerf.py +++ b/mattport/nerf/graph/vanilla_nerf.py @@ -9,12 +9,12 @@ import torch from torch import nn from torch.nn import Parameter +from mattport.nerf import metrics from mattport.nerf.field_modules.encoding import NeRFEncoding from mattport.nerf.field_modules.field_heads import DensityFieldHead, FieldHeadNames, RGBFieldHead from mattport.nerf.field_modules.mlp import MLP from mattport.nerf.graph.base import Graph from mattport.nerf.loss import MSELoss -from mattport.nerf.metrics import get_psnr from mattport.nerf.renderers import AccumulationRenderer, DepthRenderer, RGBRenderer from mattport.nerf.sampler import PDFSampler, UniformSampler from mattport.structures import colors @@ -221,15 +221,25 @@ class NeRFGraph(Graph): combined_image = torch.cat([depth_coarse, depth_fine], dim=1) writer.write_event({"name": f"image_idx_{image_idx}", "x": combined_image, "step": step, "group": "val_depth"}) - coarse_psnr = get_psnr(image, rgb_coarse) + coarse_psnr = metrics.get_psnr(image, rgb_coarse) writer.write_event( {"name": f"image_idx_{image_idx}", "scalar": float(coarse_psnr), "step": step, "group": "val"} ) - fine_psnr = get_psnr(image, rgb_fine) + coarse_ssim = metrics.get_ssim(image, rgb_coarse) + writer.write_event( + {"name": f"image_idx_{image_idx}", "scalar": float(coarse_ssim), "step": step, "group": "ssim"} + ) + + fine_psnr = metrics.get_psnr(image, rgb_fine) stats_tracker.update_stats( {"name": stats_tracker.Stats.CURR_TEST_PSNR, "value": float(fine_psnr), "step": step} ) writer.write_event( {"name": f"image_idx_{image_idx}-fine_psnr", "scalar": float(fine_psnr), "step": step, "group": "val"} ) + + fine_ssim = metrics.get_ssim(image, rgb_fine) + writer.write_event( + {"name": f"image_idx_{image_idx}-fine_ssim", "scalar": float(fine_ssim), "step": step, "group": "ssim"} + ) diff --git a/mattport/nerf/metrics.py b/mattport/nerf/metrics.py index 230b66327..d712e770f 100644 --- a/mattport/nerf/metrics.py +++ b/mattport/nerf/metrics.py @@ -3,11 +3,61 @@ Collection of metrics. """ import torch +from pytorch_msssim import ssim, ms_ssim +from torchtyping import TensorType -def get_psnr(image_gt, image_pred): +def get_psnr(image_gt: TensorType, image_pred: TensorType) -> TensorType: """ - Returns the psnr for an image. TODO: add masking + Returns the PSNR (peak signal-to-noise ratio) for an image. TODO: add masking + + Args: + image_gt (TensorType): Ground truth image + image_pred (TensorType): Predicted image + Returns: + float: PSNR """ mean = torch.mean((image_gt - image_pred) ** 2) return -10 * torch.log10(mean) + + +def get_ssim(image_gt: TensorType, image_pred: TensorType) -> TensorType: + """ + Returns the SSIM (structural similarty) for an image. + + Args: + image_gt (TensorType): Ground truth image + image_pred (TensorType): Predicted image + Returns: + float: SSIM + """ + assert image_gt.shape[-1] in [1, 3] + assert image_pred.shape[-1] in [1, 3] + assert image_gt.shape == image_pred.shape + if image_gt.dim() == 3: + image_gt = image_gt[None, ...] + if image_pred.dim() == 3: + image_pred = image_pred[None, ...] + + return ssim(image_gt, image_pred, data_range=1.0) + + +def get_ms_ssim(image_gt: TensorType, image_pred: TensorType) -> TensorType: + """ + Returns the MS_SSIM (multi-scale structural similarty) for an image. + + Args: + image_gt (TensorType): Ground truth image + image_pred (TensorType): Predicted image + Returns: + float: MS-SSIM + """ + assert image_gt.shape[-1] in [1, 3] + assert image_pred.shape[-1] in [1, 3] + assert image_gt.shape == image_pred.shape + if image_gt.dim() == 3: + image_gt = image_gt[None, ...] + if image_pred.dim() == 3: + image_pred = image_pred[None, ...] + + return ms_ssim(image_gt, image_pred, data_range=1.0)