mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 22:04:02 +08:00
Update Instant-NGP & Density Grid with CUDA supports. Much faster and better PSNR. (#193)
This commit is contained in:
1 parent
31b52ae186
commit
12cd321e1a
39 files changed
+1721
-650
No files matched your search
@@ -94,6 +94,7 @@ class RaySamples(TensorDataclass):
|
||||
camera_indices (TensorType[..., 1]): Camera index.
|
||||
valid_mask (TensorType[..., 1]): Rays that are valid.
|
||||
deltas (TensorType[..., 1]): "width" of each sample.
|
||||
ts (TensorType[..., 1]): sample-camera distance of each sample.
|
||||
"""
|
||||
|
||||
frustums: Frustums
|
||||
|
||||
@@ -11,3 +11,69 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""CUDA Functions"""
|
||||
import torch
|
||||
from torch.cuda.amp import custom_bwd, custom_fwd
|
||||
|
||||
from pyrad.cuda.backend import _C
|
||||
|
||||
packbits = _C.packbits
|
||||
ray_aabb_intersect = _C.ray_aabb_intersect
|
||||
morton3D = _C.morton3D
|
||||
morton3D_invert = _C.morton3D_invert
|
||||
raymarching = _C.raymarching
|
||||
volumetric_rendering_forward = _C.volumetric_rendering_forward
|
||||
volumetric_rendering_backward = _C.volumetric_rendering_backward
|
||||
|
||||
# pylint: disable=abstract-method,arguments-differ
|
||||
class VolumeRenderer(torch.autograd.Function):
|
||||
"""CUDA Volumetirc Renderer"""
|
||||
|
||||
@staticmethod
|
||||
@custom_fwd(cast_inputs=torch.float32)
|
||||
def forward(ctx, packed_info, starts, ends, sigmas, rgbs):
|
||||
accumulated_weight, accumulated_depth, accumulated_color, mask = volumetric_rendering_forward(
|
||||
packed_info, starts, ends, sigmas, rgbs
|
||||
)
|
||||
# TODO(ruilongli): accelerate for torch.no_grad()?
|
||||
ctx.save_for_backward(
|
||||
accumulated_weight,
|
||||
accumulated_depth,
|
||||
accumulated_color,
|
||||
packed_info,
|
||||
starts,
|
||||
ends,
|
||||
sigmas,
|
||||
rgbs,
|
||||
)
|
||||
return accumulated_weight, accumulated_depth, accumulated_color, mask
|
||||
|
||||
@staticmethod
|
||||
@custom_bwd
|
||||
def backward(ctx, grad_weight, grad_depth, grad_color, _grad_mask):
|
||||
(
|
||||
accumulated_weight,
|
||||
accumulated_depth,
|
||||
accumulated_color,
|
||||
packed_info,
|
||||
starts,
|
||||
ends,
|
||||
sigmas,
|
||||
rgbs,
|
||||
) = ctx.saved_tensors
|
||||
grad_sigmas, grad_rgbs = volumetric_rendering_backward(
|
||||
accumulated_weight,
|
||||
accumulated_depth,
|
||||
accumulated_color,
|
||||
grad_weight,
|
||||
grad_depth,
|
||||
grad_color,
|
||||
packed_info,
|
||||
starts,
|
||||
ends,
|
||||
sigmas,
|
||||
rgbs,
|
||||
)
|
||||
# corresponds to the input argument list of forward()
|
||||
return None, None, None, grad_sigmas, grad_rgbs
|
||||
@@ -0,0 +1,47 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import glob
|
||||
import os
|
||||
from subprocess import DEVNULL, call
|
||||
|
||||
from torch.utils.cpp_extension import load
|
||||
|
||||
PATH = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def cuda_toolkit_available():
|
||||
"""Check if the nvcc is avaiable on the machine."""
|
||||
# https://github.com/idiap/fast-transformers/blob/master/setup.py
|
||||
try:
|
||||
call(["nvcc"], stdout=DEVNULL, stderr=DEVNULL)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
if cuda_toolkit_available():
|
||||
sources = glob.glob(os.path.join(PATH, "csrc/*.cu"))
|
||||
else:
|
||||
sources = glob.glob(os.path.join(PATH, "csrc/*.cpp"))
|
||||
|
||||
extra_cflags = ["-O2"]
|
||||
extra_cuda_cflags = ["-O2"]
|
||||
_C = load(
|
||||
name="pyrad_extension",
|
||||
sources=sources,
|
||||
extra_cflags=extra_cflags,
|
||||
extra_cuda_cflags=extra_cuda_cflags,
|
||||
)
|
||||
|
||||
__all__ = ["_C"]
|
||||
@@ -0,0 +1,53 @@
|
||||
#pragma once
|
||||
|
||||
#ifdef __CUDACC__
|
||||
#define CUDA_HOSTDEV __host__ __device__
|
||||
#else
|
||||
#define CUDA_HOSTDEV
|
||||
#endif
|
||||
|
||||
#include <torch/extension.h>
|
||||
|
||||
|
||||
inline constexpr CUDA_HOSTDEV float __SQRT3() { return 1.73205080757f; }
|
||||
|
||||
template <typename scalar_t>
|
||||
inline CUDA_HOSTDEV void __swap(scalar_t &a, scalar_t &b)
|
||||
{
|
||||
scalar_t c = a;
|
||||
a = b;
|
||||
b = c;
|
||||
}
|
||||
|
||||
inline CUDA_HOSTDEV float __clamp(float f, float a, float b) { return fmaxf(a, fminf(f, b)); }
|
||||
|
||||
inline CUDA_HOSTDEV int __clamp(int f, int a, int b) { return std::max(a, std::min(f, b)); }
|
||||
|
||||
inline CUDA_HOSTDEV float __sign(float x) { return copysignf(1.0, x); }
|
||||
|
||||
inline CUDA_HOSTDEV uint32_t __expand_bits(uint32_t v)
|
||||
{
|
||||
v = (v * 0x00010001u) & 0xFF0000FFu;
|
||||
v = (v * 0x00000101u) & 0x0F00F00Fu;
|
||||
v = (v * 0x00000011u) & 0xC30C30C3u;
|
||||
v = (v * 0x00000005u) & 0x49249249u;
|
||||
return v;
|
||||
}
|
||||
|
||||
inline CUDA_HOSTDEV uint32_t __morton3D(uint32_t x, uint32_t y, uint32_t z)
|
||||
{
|
||||
uint32_t xx = __expand_bits(x);
|
||||
uint32_t yy = __expand_bits(y);
|
||||
uint32_t zz = __expand_bits(z);
|
||||
return xx | (yy << 1) | (zz << 2);
|
||||
}
|
||||
|
||||
inline CUDA_HOSTDEV uint32_t __morton3D_invert(uint32_t x)
|
||||
{
|
||||
x = x & 0x49249249;
|
||||
x = (x | (x >> 2)) & 0xc30c30c3;
|
||||
x = (x | (x >> 4)) & 0x0f00f00f;
|
||||
x = (x | (x >> 8)) & 0xff0000ff;
|
||||
x = (x | (x >> 16)) & 0x0000ffff;
|
||||
return x;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include "helpers.h"
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
|
||||
#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
|
||||
#define CHECK_CONTIGUOUS(x) \
|
||||
TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
|
||||
#define CHECK_INPUT(x) \
|
||||
CHECK_CUDA(x); \
|
||||
CHECK_CONTIGUOUS(x)
|
||||
#define CUDA_GET_THREAD_ID(tid, Q) \
|
||||
const int tid = blockIdx.x * blockDim.x + threadIdx.x; \
|
||||
if (tid >= Q) \
|
||||
return
|
||||
#define CUDA_N_BLOCKS_NEEDED(Q, CUDA_N_THREADS) ((Q - 1) / CUDA_N_THREADS + 1)
|
||||
#define DEVICE_GUARD(_ten) \
|
||||
const at::cuda::OptionalCUDAGuard device_guard(device_of(_ten));
|
||||
@@ -0,0 +1,118 @@
|
||||
#include "include/helpers_cuda.h"
|
||||
|
||||
|
||||
template <typename scalar_t>
|
||||
inline __host__ __device__ void _ray_aabb_intersect(
|
||||
const scalar_t* rays_o,
|
||||
const scalar_t* rays_d,
|
||||
const scalar_t* aabb,
|
||||
scalar_t* near,
|
||||
scalar_t* far
|
||||
) {
|
||||
// aabb is [xmin, ymin, zmin, xmax, ymax, zmax]
|
||||
scalar_t tmin = (aabb[0] - rays_o[0]) / rays_d[0];
|
||||
scalar_t tmax = (aabb[3] - rays_o[0]) / rays_d[0];
|
||||
if (tmin > tmax) __swap(tmin, tmax);
|
||||
|
||||
scalar_t tymin = (aabb[1] - rays_o[1]) / rays_d[1];
|
||||
scalar_t tymax = (aabb[4] - rays_o[1]) / rays_d[1];
|
||||
if (tymin > tymax) __swap(tymin, tymax);
|
||||
|
||||
if (tmin > tymax || tymin > tmax){
|
||||
*near = std::numeric_limits<scalar_t>::max();
|
||||
*far = std::numeric_limits<scalar_t>::max();
|
||||
return;
|
||||
}
|
||||
|
||||
if (tymin > tmin) tmin = tymin;
|
||||
if (tymax < tmax) tmax = tymax;
|
||||
|
||||
scalar_t tzmin = (aabb[2] - rays_o[2]) / rays_d[2];
|
||||
scalar_t tzmax = (aabb[5] - rays_o[2]) / rays_d[2];
|
||||
if (tzmin > tzmax) __swap(tzmin, tzmax);
|
||||
|
||||
if (tmin > tzmax || tzmin > tmax){
|
||||
*near = std::numeric_limits<scalar_t>::max();
|
||||
*far = std::numeric_limits<scalar_t>::max();
|
||||
return;
|
||||
}
|
||||
|
||||
if (tzmin > tmin) tmin = tzmin;
|
||||
if (tzmax < tmax) tmax = tzmax;
|
||||
|
||||
*near = tmin;
|
||||
*far = tmax;
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void kernel_ray_aabb_intersect(
|
||||
const int N,
|
||||
const scalar_t* rays_o,
|
||||
const scalar_t* rays_d,
|
||||
const scalar_t* aabb,
|
||||
scalar_t* t_min,
|
||||
scalar_t* t_max
|
||||
){
|
||||
// aabb is [xmin, ymin, zmin, xmax, ymax, zmax]
|
||||
CUDA_GET_THREAD_ID(thread_id, N);
|
||||
|
||||
// locate
|
||||
rays_o += thread_id * 3;
|
||||
rays_d += thread_id * 3;
|
||||
t_min += thread_id;
|
||||
t_max += thread_id;
|
||||
|
||||
_ray_aabb_intersect<scalar_t>(rays_o, rays_d, aabb, t_min, t_max);
|
||||
|
||||
scalar_t zero = static_cast<scalar_t>(0.f);
|
||||
*t_min = *t_min > zero ? *t_min : zero;
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Ray AABB Test
|
||||
*
|
||||
* @param rays_o Ray origins. Tensor with shape [N, 3].
|
||||
* @param rays_d Normalized ray directions. Tensor with shape [N, 3].
|
||||
* @param aabb Scene AABB [xmin, ymin, zmin, xmax, ymax, zmax]. Tensor with shape [6].
|
||||
* @return std::vector<torch::Tensor>
|
||||
* Ray AABB intersection {t_min, t_max} with shape [N] respectively. Note the t_min is
|
||||
* clipped to minimum zero.
|
||||
*/
|
||||
std::vector<torch::Tensor> ray_aabb_intersect(
|
||||
const torch::Tensor rays_o, const torch::Tensor rays_d, const torch::Tensor aabb
|
||||
) {
|
||||
DEVICE_GUARD(rays_o);
|
||||
CHECK_INPUT(rays_o);
|
||||
CHECK_INPUT(rays_d);
|
||||
CHECK_INPUT(aabb);
|
||||
TORCH_CHECK(rays_o.ndimension() == 2 & rays_o.size(1) == 3)
|
||||
TORCH_CHECK(rays_d.ndimension() == 2 & rays_d.size(1) == 3)
|
||||
TORCH_CHECK(aabb.ndimension() == 1 & aabb.size(0) == 6)
|
||||
|
||||
const int N = rays_o.size(0);
|
||||
|
||||
const int threads = 256;
|
||||
const int blocks = CUDA_N_BLOCKS_NEEDED(N, threads);
|
||||
|
||||
torch::Tensor t_min = torch::empty({N}, rays_o.options());
|
||||
torch::Tensor t_max = torch::empty({N}, rays_o.options());
|
||||
|
||||
AT_DISPATCH_FLOATING_TYPES_AND_HALF(
|
||||
rays_o.scalar_type(), "ray_aabb_intersect",
|
||||
([&] {
|
||||
kernel_ray_aabb_intersect<scalar_t><<<blocks, threads>>>(
|
||||
N,
|
||||
rays_o.data_ptr<scalar_t>(),
|
||||
rays_d.data_ptr<scalar_t>(),
|
||||
aabb.data_ptr<scalar_t>(),
|
||||
t_min.data_ptr<scalar_t>(),
|
||||
t_max.data_ptr<scalar_t>()
|
||||
);
|
||||
})
|
||||
);
|
||||
|
||||
return {t_min, t_max};
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
#include "include/helpers_cuda.h"
|
||||
|
||||
|
||||
template <typename index_t>
|
||||
__global__ void kernel_morton3D(
|
||||
const int N,
|
||||
const index_t* coords,
|
||||
index_t* indices
|
||||
){
|
||||
CUDA_GET_THREAD_ID(thread_id, N);
|
||||
|
||||
// locate
|
||||
coords += thread_id * 3;
|
||||
indices += thread_id;
|
||||
|
||||
uint32_t index = __morton3D(
|
||||
static_cast<uint32_t>(coords[0]),
|
||||
static_cast<uint32_t>(coords[1]),
|
||||
static_cast<uint32_t>(coords[2])
|
||||
);
|
||||
*indices = static_cast<index_t>(index);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
template <typename index_t>
|
||||
__global__ void kernel_morton3D_invert(
|
||||
const int N,
|
||||
const index_t* indices,
|
||||
index_t* coords
|
||||
){
|
||||
CUDA_GET_THREAD_ID(thread_id, N);
|
||||
|
||||
// locate
|
||||
coords += thread_id * 3;
|
||||
indices += thread_id;
|
||||
|
||||
uint32_t index = static_cast<uint32_t>(indices[0]);
|
||||
coords[0] = static_cast<index_t>(__morton3D_invert(index >> 0));
|
||||
coords[1] = static_cast<index_t>(__morton3D_invert(index >> 1));
|
||||
coords[2] = static_cast<index_t>(__morton3D_invert(index >> 2));
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Convert 3D coords to indices in z-order
|
||||
*
|
||||
* @param coords Integer coords with shape [N, 3]
|
||||
* @return torch::Tensor Integer indices with shape [N]
|
||||
*/
|
||||
torch::Tensor morton3D(const torch::Tensor coords){
|
||||
DEVICE_GUARD(coords);
|
||||
CHECK_INPUT(coords);
|
||||
TORCH_CHECK(coords.ndimension() == 2 & coords.size(1) == 3)
|
||||
|
||||
const int N = coords.size(0);
|
||||
|
||||
const int threads = 256;
|
||||
const int blocks = CUDA_N_BLOCKS_NEEDED(N, threads);
|
||||
|
||||
torch::Tensor indices = torch::empty({N}, coords.options());
|
||||
|
||||
AT_DISPATCH_INDEX_TYPES(
|
||||
coords.scalar_type(), "morton3D",
|
||||
([&] {
|
||||
kernel_morton3D<index_t><<<blocks, threads>>>(
|
||||
N,
|
||||
coords.data_ptr<index_t>(),
|
||||
indices.data_ptr<index_t>()
|
||||
);
|
||||
})
|
||||
);
|
||||
|
||||
return indices;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Convert 3D indices to coords in z-order
|
||||
*
|
||||
* @param indices Integer indices with shape [N]
|
||||
* @return torch::Tensor Integer coords with shape [N, 3]
|
||||
*/
|
||||
torch::Tensor morton3D_invert(const torch::Tensor indices){
|
||||
DEVICE_GUARD(indices);
|
||||
CHECK_INPUT(indices);
|
||||
TORCH_CHECK(indices.ndimension() == 1)
|
||||
|
||||
const int N = indices.size(0);
|
||||
|
||||
const int threads = 256;
|
||||
const int blocks = CUDA_N_BLOCKS_NEEDED(N, threads);
|
||||
|
||||
torch::Tensor coords = torch::empty({N, 3}, indices.options());
|
||||
|
||||
AT_DISPATCH_INDEX_TYPES(
|
||||
indices.scalar_type(), "morton3D_invert",
|
||||
([&] {
|
||||
kernel_morton3D_invert<index_t><<<blocks, threads>>>(
|
||||
N,
|
||||
indices.data_ptr<index_t>(),
|
||||
coords.data_ptr<index_t>()
|
||||
);
|
||||
})
|
||||
);
|
||||
|
||||
return coords;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
#include "include/helpers_cuda.h"
|
||||
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void kernel_packbits(
|
||||
const int N,
|
||||
const float threshold,
|
||||
const scalar_t* data,
|
||||
uint8_t* bitfield
|
||||
){
|
||||
CUDA_GET_THREAD_ID(thread_id, N);
|
||||
|
||||
data += thread_id * 8;
|
||||
bitfield += thread_id;
|
||||
|
||||
uint8_t bits = 0;
|
||||
#pragma unroll 8
|
||||
for (uint8_t i = 0; i < 8; i++) {
|
||||
bits |= (data[i] > threshold) ? ((uint8_t)1 << i) : 0;
|
||||
}
|
||||
bitfield[0] = bits;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Pack data into uint8 bits based on threshold.
|
||||
*
|
||||
* @param data: Tensor with any shape. Total elements must be N * 8.
|
||||
* @param threshold:
|
||||
* @return Tensor: bitfield that has shape of [N]
|
||||
*/
|
||||
torch::Tensor packbits(
|
||||
const torch::Tensor data, const float threshold
|
||||
) {
|
||||
DEVICE_GUARD(data);
|
||||
CHECK_INPUT(data);
|
||||
TORCH_CHECK(data.numel() % 8 == 0)
|
||||
|
||||
const int N = data.numel() / 8;
|
||||
|
||||
const int threads = 256;
|
||||
const int blocks = CUDA_N_BLOCKS_NEEDED(N, threads);
|
||||
|
||||
torch::Tensor bitfield = torch::empty(
|
||||
{N,}, data.options().dtype(torch::kUInt8));
|
||||
|
||||
AT_DISPATCH_FLOATING_TYPES_AND_HALF(
|
||||
data.scalar_type(), "packbits",
|
||||
([&] {
|
||||
kernel_packbits<scalar_t><<<blocks, threads>>>(
|
||||
N,
|
||||
threshold,
|
||||
data.data_ptr<scalar_t>(),
|
||||
bitfield.data_ptr<uint8_t>()
|
||||
);
|
||||
})
|
||||
);
|
||||
|
||||
return bitfield;
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
#include "include/helpers.h"
|
||||
|
||||
|
||||
torch::Tensor packbits(
|
||||
const torch::Tensor data, const float threshold
|
||||
) {};
|
||||
|
||||
std::vector<torch::Tensor> ray_aabb_intersect(
|
||||
const torch::Tensor rays_o,
|
||||
const torch::Tensor rays_d,
|
||||
const torch::Tensor aabb
|
||||
) {};
|
||||
|
||||
torch::Tensor morton3D(const torch::Tensor coords) {};
|
||||
|
||||
torch::Tensor morton3D_invert(const torch::Tensor indices) {};
|
||||
|
||||
std::vector<torch::Tensor> raymarching(
|
||||
// rays
|
||||
const torch::Tensor rays_o,
|
||||
const torch::Tensor rays_d,
|
||||
const torch::Tensor t_min,
|
||||
const torch::Tensor t_max,
|
||||
// density grid
|
||||
const float grid_center,
|
||||
const float grid_scale,
|
||||
const int grid_cascades,
|
||||
const int grid_size,
|
||||
const torch::Tensor grid_bitfield,
|
||||
// sampling args
|
||||
const int max_total_samples,
|
||||
const int num_steps,
|
||||
const float cone_angle,
|
||||
const float step_scale
|
||||
) {};
|
||||
|
||||
std::vector<torch::Tensor> volumetric_rendering_forward(
|
||||
torch::Tensor packed_info,
|
||||
torch::Tensor starts,
|
||||
torch::Tensor ends,
|
||||
torch::Tensor sigmas,
|
||||
torch::Tensor rgbs
|
||||
) {};
|
||||
|
||||
std::vector<torch::Tensor> volumetric_rendering_backward(
|
||||
torch::Tensor accumulated_weight,
|
||||
torch::Tensor accumulated_depth,
|
||||
torch::Tensor accumulated_color,
|
||||
torch::Tensor grad_weight,
|
||||
torch::Tensor grad_depth,
|
||||
torch::Tensor grad_color,
|
||||
torch::Tensor packed_info,
|
||||
torch::Tensor starts,
|
||||
torch::Tensor ends,
|
||||
torch::Tensor sigmas,
|
||||
torch::Tensor rgbs
|
||||
) {};
|
||||
|
||||
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)
|
||||
{
|
||||
m.def("packbits", &packbits);
|
||||
m.def("ray_aabb_intersect", &ray_aabb_intersect);
|
||||
m.def("morton3D", &morton3D);
|
||||
m.def("morton3D_invert", &morton3D_invert);
|
||||
m.def("raymarching", &raymarching);
|
||||
m.def("volumetric_rendering_forward", &volumetric_rendering_forward);
|
||||
m.def("volumetric_rendering_backward", &volumetric_rendering_backward);
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
#include "include/helpers_cuda.h"
|
||||
|
||||
|
||||
torch::Tensor packbits(
|
||||
const torch::Tensor data, const float threshold
|
||||
);
|
||||
|
||||
std::vector<torch::Tensor> ray_aabb_intersect(
|
||||
const torch::Tensor rays_o,
|
||||
const torch::Tensor rays_d,
|
||||
const torch::Tensor aabb
|
||||
);
|
||||
|
||||
torch::Tensor morton3D(const torch::Tensor coords);
|
||||
|
||||
torch::Tensor morton3D_invert(const torch::Tensor indices);
|
||||
|
||||
std::vector<torch::Tensor> raymarching(
|
||||
// rays
|
||||
const torch::Tensor rays_o,
|
||||
const torch::Tensor rays_d,
|
||||
const torch::Tensor t_min,
|
||||
const torch::Tensor t_max,
|
||||
// density grid
|
||||
const float grid_center,
|
||||
const float grid_scale,
|
||||
const int grid_cascades,
|
||||
const int grid_size,
|
||||
const torch::Tensor grid_bitfield,
|
||||
// sampling args
|
||||
const int max_total_samples,
|
||||
const int num_steps,
|
||||
const float cone_angle,
|
||||
const float step_scale
|
||||
);
|
||||
|
||||
std::vector<torch::Tensor> volumetric_rendering_forward(
|
||||
torch::Tensor packed_info,
|
||||
torch::Tensor starts,
|
||||
torch::Tensor ends,
|
||||
torch::Tensor sigmas,
|
||||
torch::Tensor rgbs
|
||||
);
|
||||
|
||||
std::vector<torch::Tensor> volumetric_rendering_backward(
|
||||
torch::Tensor accumulated_weight,
|
||||
torch::Tensor accumulated_depth,
|
||||
torch::Tensor accumulated_color,
|
||||
torch::Tensor grad_weight,
|
||||
torch::Tensor grad_depth,
|
||||
torch::Tensor grad_color,
|
||||
torch::Tensor packed_info,
|
||||
torch::Tensor starts,
|
||||
torch::Tensor ends,
|
||||
torch::Tensor sigmas,
|
||||
torch::Tensor rgbs
|
||||
);
|
||||
|
||||
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)
|
||||
{
|
||||
m.def("packbits", &packbits);
|
||||
m.def("ray_aabb_intersect", &ray_aabb_intersect);
|
||||
m.def("morton3D", &morton3D);
|
||||
m.def("morton3D_invert", &morton3D_invert);
|
||||
m.def("raymarching", &raymarching);
|
||||
m.def("volumetric_rendering_forward", &volumetric_rendering_forward);
|
||||
m.def("volumetric_rendering_backward", &volumetric_rendering_backward);
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
#include "include/helpers_cuda.h"
|
||||
|
||||
|
||||
inline __device__ float min_step_size(uint32_t num_steps) {
|
||||
return __SQRT3() / num_steps;
|
||||
}
|
||||
|
||||
// Maximum step size is the width of the coarsest gridsize cell.
|
||||
inline __device__ float max_step_size(uint32_t grid_cascades, uint32_t grid_size) {
|
||||
// Note(ruilongli): use mip=0 step size?
|
||||
return __SQRT3() * (1 << (grid_cascades-1)) / grid_size;
|
||||
}
|
||||
|
||||
// Perform fixed-size stepping in unit-cube scenes (like original NeRF) and exponential
|
||||
// stepping in larger scenes.
|
||||
inline __device__ float calc_dt(float t, float cone_angle, float dt_min, float dt_max) {
|
||||
// TODO(ruilongli): scene_scale related to cone_angle?
|
||||
return __clamp(t * cone_angle, dt_min, dt_max);
|
||||
}
|
||||
|
||||
inline __device__ int mip_from_pos(float x, float y, float z, uint32_t grid_cascades, float grid_center, float grid_scale) {
|
||||
float maxval = fmaxf(fmaxf(fabsf(x - grid_center), fabsf(y - grid_center)), fabsf(z - grid_center)) / grid_scale;
|
||||
int exponent; frexpf(maxval, &exponent);
|
||||
return min(grid_cascades-1, max(0, exponent+1));
|
||||
}
|
||||
|
||||
inline __device__ int mip_from_dt(
|
||||
float x, float y, float z, uint32_t grid_cascades, float dt, int grid_size, float grid_center, float grid_scale
|
||||
) {
|
||||
int mip = mip_from_pos(x, y, z, grid_cascades, grid_center, grid_scale);
|
||||
dt *= 2 * grid_size;
|
||||
if (dt<1.f) return mip; // exponent would be zero
|
||||
int exponent; frexpf(dt, &exponent);
|
||||
return min(grid_cascades-1, max(exponent, mip));
|
||||
}
|
||||
|
||||
inline __device__ uint32_t grid_mip_offset(uint32_t mip, int grid_size) {
|
||||
return (grid_size * grid_size * grid_size) * mip;
|
||||
}
|
||||
|
||||
inline __device__ uint32_t cascaded_grid_idx_at(
|
||||
float x, float y, float z, uint32_t mip, int grid_size, float grid_center, float grid_scale
|
||||
) {
|
||||
float mip_scale = scalbnf(1.0f, -mip) / grid_scale;
|
||||
int ix = (int)((mip_scale * (x - grid_center) + 0.5f) * grid_size);
|
||||
int iy = (int)((mip_scale * (y - grid_center) + 0.5f) * grid_size);
|
||||
int iz = (int)((mip_scale * (z - grid_center) + 0.5f) * grid_size);
|
||||
// printf("[input] x %f, y %f, z %f, mip %d, grid_size %d, grid_center %f\n", x, y, z, mip, grid_size, grid_center);
|
||||
// printf("[output] mip_scale %f, ix %d iy %d, iz %d\n", mip_scale, ix, iy, iz);
|
||||
uint32_t idx = __morton3D(
|
||||
__clamp(ix, 0, grid_size-1),
|
||||
__clamp(iy, 0, grid_size-1),
|
||||
__clamp(iz, 0, grid_size-1)
|
||||
);
|
||||
return idx;
|
||||
}
|
||||
|
||||
inline __device__ bool grid_occupied_at(
|
||||
float x, float y, float z,
|
||||
const uint8_t* grid_bitfield,
|
||||
uint32_t mip, int grid_size, float grid_center, float grid_scale
|
||||
) {
|
||||
uint32_t idx = (
|
||||
cascaded_grid_idx_at(x, y, z, mip, grid_size, grid_center, grid_scale)
|
||||
+ grid_mip_offset(mip, grid_size)
|
||||
);
|
||||
return grid_bitfield[idx/8] & (1<<(idx%8));
|
||||
}
|
||||
|
||||
inline __device__ float distance_to_next_voxel(
|
||||
float x, float y, float z,
|
||||
float dir_x, float dir_y, float dir_z,
|
||||
float idir_x, float idir_y, float idir_z,
|
||||
float res
|
||||
) { // dda like step
|
||||
// TODO: warning: expression has no effect?
|
||||
x, y, z = res * x, res * y, res * z;
|
||||
float tx = (floorf(x + 0.5f + 0.5f * __sign(dir_x)) - x) * idir_x;
|
||||
float ty = (floorf(y + 0.5f + 0.5f * __sign(dir_y)) - y) * idir_y;
|
||||
float tz = (floorf(z + 0.5f + 0.5f * __sign(dir_z)) - z) * idir_z;
|
||||
float t = min(min(tx, ty), tz);
|
||||
|
||||
return fmaxf(t / res, 0.0f);
|
||||
}
|
||||
|
||||
inline __device__ float advance_to_next_voxel(
|
||||
float t,
|
||||
float x, float y, float z,
|
||||
float dir_x, float dir_y, float dir_z,
|
||||
float idir_x, float idir_y, float idir_z,
|
||||
float res, float dt_min) {
|
||||
// Regular stepping (may be slower but matches non-empty space)
|
||||
float t_target = t + distance_to_next_voxel(
|
||||
x, y, z, dir_x, dir_y, dir_z, idir_x, idir_y, idir_z, res
|
||||
);
|
||||
do {
|
||||
t += dt_min;
|
||||
} while (t < t_target);
|
||||
return t;
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void kernel_raymarching(
|
||||
// rays info
|
||||
const uint32_t n_rays,
|
||||
const scalar_t* rays_o,
|
||||
const scalar_t* rays_d,
|
||||
const scalar_t* t_min,
|
||||
const scalar_t* t_max,
|
||||
// density grid
|
||||
const float grid_center,
|
||||
const float grid_scale,
|
||||
const int grid_cascades,
|
||||
const int grid_size,
|
||||
const uint8_t* grid_bitfield,
|
||||
// sampling
|
||||
const int max_total_samples,
|
||||
const int num_steps, // default 1024
|
||||
const float cone_angle, // default 0. for nerf-syn and 1/256 for large scene
|
||||
const float step_scale,
|
||||
int* numsteps_counter,
|
||||
int* rays_counter,
|
||||
int* packed_info_out,
|
||||
scalar_t* origins_out,
|
||||
scalar_t* dirs_out,
|
||||
scalar_t* starts_out,
|
||||
scalar_t* ends_out
|
||||
) {
|
||||
CUDA_GET_THREAD_ID(i, n_rays);
|
||||
|
||||
// locate
|
||||
rays_o += i * 3;
|
||||
rays_d += i * 3;
|
||||
t_min += i;
|
||||
t_max += i;
|
||||
|
||||
const float ox = rays_o[0], oy = rays_o[1], oz = rays_o[2];
|
||||
const float dx = rays_d[0], dy = rays_d[1], dz = rays_d[2];
|
||||
const float rdx = 1 / dx, rdy = 1 / dy, rdz = 1 / dz;
|
||||
const float near = t_min[0], far = t_max[0];
|
||||
|
||||
// TODO(ruilongli): perturb `near` as in ngp_pl?
|
||||
// TODO(ruilongli): pre-compute `dt_min`, `dt_max` as it is a constant?
|
||||
float dt_min = min_step_size(num_steps) * step_scale;
|
||||
float dt_max = max_step_size(grid_cascades, grid_size) * grid_scale;
|
||||
|
||||
// first pass to compute an accurate number of steps
|
||||
uint32_t j = 0;
|
||||
float t0 = near;
|
||||
float dt = calc_dt(t0, cone_angle, dt_min, dt_max);
|
||||
float t1 = t0 + dt;
|
||||
float t_mid = (t0 + t1) * 0.5f;
|
||||
|
||||
while (t_mid < far && j < num_steps) {
|
||||
// current center
|
||||
const float x = ox + t_mid * dx;
|
||||
const float y = oy + t_mid * dy;
|
||||
const float z = oz + t_mid * dz;
|
||||
uint32_t mip = mip_from_dt(x, y, z, grid_cascades, dt, grid_size, grid_center, grid_scale);
|
||||
|
||||
if (grid_occupied_at(x, y, z, grid_bitfield, mip, grid_size, grid_center, grid_scale)) {
|
||||
++j;
|
||||
// march to next sample
|
||||
t0 = t1;
|
||||
dt = calc_dt(t0, cone_angle, dt_min, dt_max);
|
||||
t1 = t0 + dt;
|
||||
t_mid = (t0 + t1) * 0.5f;
|
||||
}
|
||||
else {
|
||||
// march to next sample
|
||||
float res = (grid_size >> mip) * grid_scale;
|
||||
t_mid = advance_to_next_voxel(
|
||||
t_mid, x, y, z, dx, dy, dz, rdx, rdy, rdz, res, dt_min
|
||||
);
|
||||
dt = calc_dt(t_mid, cone_angle, dt_min, dt_max);
|
||||
t0 = t_mid - dt * 0.5f;
|
||||
t1 = t_mid + dt * 0.5f;
|
||||
}
|
||||
}
|
||||
if (j == 0) return;
|
||||
|
||||
uint32_t numsteps = j;
|
||||
uint32_t base = atomicAdd(numsteps_counter, numsteps);
|
||||
if (base + numsteps > max_total_samples) return;
|
||||
|
||||
// locate
|
||||
origins_out += base * 3;
|
||||
dirs_out += base * 3;
|
||||
starts_out += base;
|
||||
ends_out += base;
|
||||
|
||||
uint32_t ray_idx = atomicAdd(rays_counter, 1);
|
||||
|
||||
packed_info_out[ray_idx * 3 + 0] = i; // ray idx in {rays_o, rays_d}
|
||||
packed_info_out[ray_idx * 3 + 1] = base; // point idx start.
|
||||
packed_info_out[ray_idx * 3 + 2] = numsteps; // point idx shift.
|
||||
|
||||
// Second round
|
||||
j = 0;
|
||||
t0 = near;
|
||||
dt = calc_dt(t0, cone_angle, dt_min, dt_max);
|
||||
t1 = t0 + dt;
|
||||
t_mid = (t0 + t1) / 2.;
|
||||
|
||||
while (t_mid < far && j < num_steps) {
|
||||
// current center
|
||||
const float x = ox + t_mid * dx;
|
||||
const float y = oy + t_mid * dy;
|
||||
const float z = oz + t_mid * dz;
|
||||
uint32_t mip = mip_from_dt(x, y, z, grid_cascades, dt, grid_size, grid_center, grid_scale);
|
||||
|
||||
if (grid_occupied_at(x, y, z, grid_bitfield, mip, grid_size, grid_center, grid_scale)) {
|
||||
origins_out[j * 3 + 0] = ox;
|
||||
origins_out[j * 3 + 1] = oy;
|
||||
origins_out[j * 3 + 2] = oz;
|
||||
dirs_out[j * 3 + 0] = dx;
|
||||
dirs_out[j * 3 + 1] = dy;
|
||||
dirs_out[j * 3 + 2] = dz;
|
||||
starts_out[j] = t0;
|
||||
ends_out[j] = t1;
|
||||
++j;
|
||||
// march to next sample
|
||||
t0 = t1;
|
||||
dt = calc_dt(t0, cone_angle, dt_min, dt_max);
|
||||
t1 = t0 + dt;
|
||||
t_mid = (t0 + t1) * 0.5f;
|
||||
}
|
||||
else {
|
||||
// march to next sample
|
||||
float res = (grid_size >> mip) * grid_scale;
|
||||
t_mid = advance_to_next_voxel(
|
||||
t_mid, x, y, z, dx, dy, dz, rdx, rdy, rdz, res, dt_min
|
||||
);
|
||||
dt = calc_dt(t_mid, cone_angle, dt_min, dt_max);
|
||||
t0 = t_mid - dt * 0.5f;
|
||||
t1 = t_mid + dt * 0.5f;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Sample points by ray marching.
|
||||
*
|
||||
* @param rays_o Ray origins Shape of [n_rays, 3].
|
||||
* @param rays_d Normalized ray directions. Shape of [n_rays, 3].
|
||||
* @param t_min Near planes of rays. Shape of [n_rays].
|
||||
* @param t_max Far planes of rays. Shape of [n_rays].
|
||||
* @param grid_center Density grid center. TODO: support 3-dims.
|
||||
* @param grid_scale Density grid base level scale. TODO: support 3-dims.
|
||||
* @param grid_cascades Density grid levels.
|
||||
* @param grid_size Density grid resolution.
|
||||
* @param grid_bitfield Density grid uint8 bit field.
|
||||
* @param max_total_samples Maximum total number of samples in this batch.
|
||||
* @param num_steps Used to define the minimal step size: SQRT3() / num_steps.
|
||||
* @param cone_angle 0. for nerf-synthetic and 1./256 for real scenes.
|
||||
* @param step_scale Scale up the step size by this much. Usually equals to scene scale.
|
||||
* @return std::vector<torch::Tensor>
|
||||
* - packed_info: Stores how to index the ray samples from the returned values.
|
||||
* Shape of [n_rays, 3]. First value is the ray index. Second value is the sample
|
||||
* start index in the results for this ray. Third value is the number of samples for
|
||||
* this ray. Note for rays that have zero samples, we simply skip them so the `packed_info`
|
||||
* has some zero padding in the end.
|
||||
* - origins: Ray origins for those samples. [max_total_samples, 3]
|
||||
* - dirs: Ray directions for those samples. [max_total_samples, 3]
|
||||
* - starts: Where the frustum-shape sample starts along a ray. [max_total_samples, 1]
|
||||
* - ends: Where the frustum-shape sample ends along a ray. [max_total_samples, 1]
|
||||
*/
|
||||
std::vector<torch::Tensor> raymarching(
|
||||
// rays
|
||||
const torch::Tensor rays_o,
|
||||
const torch::Tensor rays_d,
|
||||
const torch::Tensor t_min,
|
||||
const torch::Tensor t_max,
|
||||
// density grid
|
||||
const float grid_center,
|
||||
const float grid_scale,
|
||||
const int grid_cascades,
|
||||
const int grid_size,
|
||||
const torch::Tensor grid_bitfield,
|
||||
// sampling args
|
||||
const int max_total_samples,
|
||||
const int num_steps,
|
||||
const float cone_angle,
|
||||
const float step_scale
|
||||
) {
|
||||
DEVICE_GUARD(rays_o);
|
||||
|
||||
CHECK_INPUT(rays_o);
|
||||
CHECK_INPUT(rays_d);
|
||||
CHECK_INPUT(t_min);
|
||||
CHECK_INPUT(t_max);
|
||||
CHECK_INPUT(grid_bitfield);
|
||||
|
||||
const int n_rays = rays_o.size(0);
|
||||
|
||||
const int threads = 256;
|
||||
const int blocks = CUDA_N_BLOCKS_NEEDED(n_rays, threads);
|
||||
|
||||
// helper counter
|
||||
torch::Tensor numsteps_counter = torch::zeros(
|
||||
{1}, rays_o.options().dtype(torch::kInt32));
|
||||
torch::Tensor rays_counter = torch::zeros(
|
||||
{1}, rays_o.options().dtype(torch::kInt32));
|
||||
|
||||
// output samples
|
||||
torch::Tensor packed_info = torch::zeros(
|
||||
{n_rays, 3}, rays_o.options().dtype(torch::kInt32)); // ray_id, sample_id, num_samples
|
||||
torch::Tensor origins = torch::zeros({max_total_samples, 3}, rays_o.options());
|
||||
torch::Tensor dirs = torch::zeros({max_total_samples, 3}, rays_o.options());
|
||||
torch::Tensor starts = torch::zeros({max_total_samples, 1}, rays_o.options());
|
||||
torch::Tensor ends = torch::zeros({max_total_samples, 1}, rays_o.options());
|
||||
|
||||
AT_DISPATCH_FLOATING_TYPES_AND_HALF(
|
||||
rays_o.scalar_type(),
|
||||
"raymarching_train",
|
||||
([&]
|
||||
{ kernel_raymarching<scalar_t><<<blocks, threads>>>(
|
||||
// rays
|
||||
n_rays,
|
||||
rays_o.data_ptr<scalar_t>(),
|
||||
rays_d.data_ptr<scalar_t>(),
|
||||
t_min.data_ptr<scalar_t>(),
|
||||
t_max.data_ptr<scalar_t>(),
|
||||
// density grid
|
||||
grid_center,
|
||||
grid_scale,
|
||||
grid_cascades,
|
||||
grid_size,
|
||||
grid_bitfield.data_ptr<uint8_t>(),
|
||||
// sampling args
|
||||
max_total_samples,
|
||||
num_steps,
|
||||
cone_angle,
|
||||
step_scale,
|
||||
numsteps_counter.data_ptr<int>(), // total samples.
|
||||
rays_counter.data_ptr<int>(), // total rays.
|
||||
packed_info.data_ptr<int>(),
|
||||
origins.data_ptr<scalar_t>(),
|
||||
dirs.data_ptr<scalar_t>(),
|
||||
starts.data_ptr<scalar_t>(),
|
||||
ends.data_ptr<scalar_t>()
|
||||
);
|
||||
}));
|
||||
|
||||
return {packed_info, origins, dirs, starts, ends};
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
#include "include/helpers_cuda.h"
|
||||
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void volumetric_rendering_forward_kernel(
|
||||
const uint32_t n_rays,
|
||||
const int* packed_info, // input ray & point indices.
|
||||
const scalar_t* starts, // input start t
|
||||
const scalar_t* ends, // input end t
|
||||
const scalar_t* sigmas, // input density after activation
|
||||
const scalar_t* rgbs, // input rgb after activation
|
||||
// should be all-zero initialized
|
||||
scalar_t* accumulated_weight, // output
|
||||
scalar_t* accumulated_depth, // output
|
||||
scalar_t* accumulated_color, // output
|
||||
bool* mask // output
|
||||
) {
|
||||
CUDA_GET_THREAD_ID(thread_id, n_rays);
|
||||
|
||||
// locate
|
||||
const int i = packed_info[thread_id * 3 + 0]; // ray idx in {rays_o, rays_d}
|
||||
const int base = packed_info[thread_id * 3 + 1]; // point idx start.
|
||||
const int numsteps = packed_info[thread_id * 3 + 2]; // point idx shift.
|
||||
if (numsteps == 0) return;
|
||||
|
||||
starts += base;
|
||||
ends += base;
|
||||
sigmas += base;
|
||||
rgbs += base * 3;
|
||||
|
||||
accumulated_weight += i;
|
||||
accumulated_depth += i;
|
||||
accumulated_color += i * 3;
|
||||
mask += i;
|
||||
|
||||
// accumulated rendering
|
||||
scalar_t T = 1.f;
|
||||
scalar_t EPSILON = 1e-4f;
|
||||
int j = 0;
|
||||
for (; j < numsteps; ++j) {
|
||||
if (T < EPSILON) {
|
||||
break;
|
||||
}
|
||||
const scalar_t delta = ends[j] - starts[j];
|
||||
const scalar_t t = (ends[j] + starts[j]) * 0.5f;
|
||||
|
||||
const scalar_t alpha = 1.f - __expf(-sigmas[j] * delta);
|
||||
const scalar_t weight = alpha * T;
|
||||
accumulated_weight[0] += weight;
|
||||
accumulated_depth[0] += weight * t;
|
||||
accumulated_color[0] += weight * rgbs[j * 3 + 0];
|
||||
accumulated_color[1] += weight * rgbs[j * 3 + 1];
|
||||
accumulated_color[2] += weight * rgbs[j * 3 + 2];
|
||||
T *= (1.f - alpha);
|
||||
}
|
||||
mask[0] = true;
|
||||
}
|
||||
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void volumetric_rendering_backward_kernel(
|
||||
const uint32_t n_rays,
|
||||
const int* packed_info, // input ray & point indices.
|
||||
const scalar_t* starts, // input start t
|
||||
const scalar_t* ends, // input end t
|
||||
const scalar_t* sigmas, // input density after activation
|
||||
const scalar_t* rgbs, // input rgb after activation
|
||||
const scalar_t* accumulated_weight, // forward output
|
||||
const scalar_t* accumulated_depth, // forward output
|
||||
const scalar_t* accumulated_color, // forward output
|
||||
const scalar_t* grad_weight, // input
|
||||
const scalar_t* grad_depth, // input
|
||||
const scalar_t* grad_color, // input
|
||||
scalar_t* grad_sigmas, // output
|
||||
scalar_t* grad_rgbs // output
|
||||
) {
|
||||
CUDA_GET_THREAD_ID(thread_id, n_rays);
|
||||
|
||||
// locate
|
||||
const int i = packed_info[thread_id * 3 + 0]; // ray idx in {rays_o, rays_d}
|
||||
const int base = packed_info[thread_id * 3 + 1]; // point idx start.
|
||||
const int numsteps = packed_info[thread_id * 3 + 2]; // point idx shift.
|
||||
if (numsteps == 0) return;
|
||||
|
||||
starts += base;
|
||||
ends += base;
|
||||
sigmas += base;
|
||||
rgbs += base * 3;
|
||||
|
||||
grad_sigmas += base;
|
||||
grad_rgbs += base * 3;
|
||||
|
||||
accumulated_weight += i;
|
||||
accumulated_depth += i;
|
||||
accumulated_color += i * 3;
|
||||
|
||||
grad_weight += i;
|
||||
grad_depth += i;
|
||||
grad_color += i * 3;
|
||||
|
||||
// backward of accumulated rendering
|
||||
scalar_t T = 1.f;
|
||||
scalar_t EPSILON = 1e-4f;
|
||||
int j = 0;
|
||||
scalar_t r = 0, g = 0, b = 0, d = 0;
|
||||
for (; j < numsteps; ++j) {
|
||||
if (T < EPSILON) {
|
||||
break;
|
||||
}
|
||||
const scalar_t delta = ends[j] - starts[j];
|
||||
const scalar_t t = (ends[j] + starts[j]) * 0.5f;
|
||||
|
||||
const scalar_t alpha = 1.f - __expf(-sigmas[j] * delta);
|
||||
const scalar_t weight = alpha * T;
|
||||
|
||||
r += weight * rgbs[j * 3 + 0];
|
||||
g += weight * rgbs[j * 3 + 1];
|
||||
b += weight * rgbs[j * 3 + 2];
|
||||
d += weight * t;
|
||||
|
||||
T *= (1.f - alpha);
|
||||
|
||||
grad_rgbs[j * 3 + 0] = grad_color[0] * weight;
|
||||
grad_rgbs[j * 3 + 1] = grad_color[1] * weight;
|
||||
grad_rgbs[j * 3 + 2] = grad_color[2] * weight;
|
||||
|
||||
grad_sigmas[j] = delta * (
|
||||
grad_color[0] * (T * rgbs[j * 3 + 0] - (accumulated_color[0] - r)) +
|
||||
grad_color[1] * (T * rgbs[j * 3 + 1] - (accumulated_color[1] - g)) +
|
||||
grad_color[2] * (T * rgbs[j * 3 + 2] - (accumulated_color[2] - b)) +
|
||||
grad_weight[0] * (1.f - accumulated_weight[0]) +
|
||||
grad_depth[0] * (t * T - (accumulated_depth[0] - d))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Volumetric Rendering: Accumulating samples in the forward pass.
|
||||
* The inputs, excepct for `sigmas` and `rgbs`, are the outputs of our
|
||||
* cuda ray marching function in `raymarching.cu`
|
||||
*
|
||||
* @param packed_info Stores how to index the ray samples from the returned values.
|
||||
* Shape of [n_rays, 3]. First value is the ray index. Second value is the sample
|
||||
* start index in the results for this ray. Third value is the number of samples for
|
||||
* this ray. Note for rays that have zero samples, we simply skip them so the `packed_info`
|
||||
* has some zero padding in the end.
|
||||
* @param starts: Where the frustum-shape sample starts along a ray. [total_samples, 1]
|
||||
* @param ends: Where the frustum-shape sample ends along a ray. [total_samples, 1]
|
||||
* @param sigmas Densities at those samples. [total_samples, 1]
|
||||
* @param rgbs RGBs at those samples. [total_samples, 3]
|
||||
* @return std::vector<torch::Tensor>
|
||||
* - accumulated_weight: Ray opacity. [n_rays, 1]
|
||||
* - accumulated_depth: Ray depth. [n_rays, 1]
|
||||
* - accumulated_color: Ray color. [n_rays, 3]
|
||||
* - mask: Boolen value store if this ray has valid samples from packed_info. [n_rays]
|
||||
*/
|
||||
std::vector<torch::Tensor> volumetric_rendering_forward(
|
||||
torch::Tensor packed_info,
|
||||
torch::Tensor starts,
|
||||
torch::Tensor ends,
|
||||
torch::Tensor sigmas,
|
||||
torch::Tensor rgbs
|
||||
) {
|
||||
DEVICE_GUARD(packed_info);
|
||||
CHECK_INPUT(packed_info);
|
||||
CHECK_INPUT(starts);
|
||||
CHECK_INPUT(ends);
|
||||
CHECK_INPUT(sigmas);
|
||||
CHECK_INPUT(rgbs);
|
||||
TORCH_CHECK(packed_info.ndimension() == 2 & packed_info.size(1) == 3);
|
||||
TORCH_CHECK(starts.ndimension() == 2 & starts.size(1) == 1);
|
||||
TORCH_CHECK(ends.ndimension() == 2 & ends.size(1) == 1);
|
||||
TORCH_CHECK(sigmas.ndimension() == 2 & sigmas.size(1) == 1);
|
||||
TORCH_CHECK(rgbs.ndimension() == 2 & rgbs.size(1) == 3);
|
||||
|
||||
const uint32_t n_rays = packed_info.size(0);
|
||||
|
||||
const int threads = 256;
|
||||
const int blocks = CUDA_N_BLOCKS_NEEDED(n_rays, threads);
|
||||
|
||||
// outputs
|
||||
torch::Tensor accumulated_weight = torch::zeros({n_rays, 1}, sigmas.options());
|
||||
torch::Tensor accumulated_depth = torch::zeros({n_rays, 1}, sigmas.options());
|
||||
torch::Tensor accumulated_color = torch::zeros({n_rays, 3}, sigmas.options());
|
||||
// The rays that are not skipped during sampling.
|
||||
torch::Tensor mask = torch::zeros({n_rays}, sigmas.options().dtype(torch::kBool));
|
||||
|
||||
AT_DISPATCH_FLOATING_TYPES_AND_HALF(
|
||||
sigmas.scalar_type(),
|
||||
"volumetric_rendering_forward",
|
||||
([&]
|
||||
{ volumetric_rendering_forward_kernel<scalar_t><<<blocks, threads>>>(
|
||||
n_rays,
|
||||
packed_info.data_ptr<int>(),
|
||||
starts.data_ptr<scalar_t>(),
|
||||
ends.data_ptr<scalar_t>(),
|
||||
sigmas.data_ptr<scalar_t>(),
|
||||
rgbs.data_ptr<scalar_t>(),
|
||||
accumulated_weight.data_ptr<scalar_t>(),
|
||||
accumulated_depth.data_ptr<scalar_t>(),
|
||||
accumulated_color.data_ptr<scalar_t>(),
|
||||
mask.data_ptr<bool>()
|
||||
);
|
||||
}));
|
||||
|
||||
return {accumulated_weight, accumulated_depth, accumulated_color, mask};
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @brief Volumetric Rendering: Accumulating samples in the backward pass.
|
||||
*/
|
||||
std::vector<torch::Tensor> volumetric_rendering_backward(
|
||||
torch::Tensor accumulated_weight,
|
||||
torch::Tensor accumulated_depth,
|
||||
torch::Tensor accumulated_color,
|
||||
torch::Tensor grad_weight,
|
||||
torch::Tensor grad_depth,
|
||||
torch::Tensor grad_color,
|
||||
torch::Tensor packed_info,
|
||||
torch::Tensor starts,
|
||||
torch::Tensor ends,
|
||||
torch::Tensor sigmas,
|
||||
torch::Tensor rgbs
|
||||
) {
|
||||
DEVICE_GUARD(packed_info);
|
||||
const uint32_t n_rays = packed_info.size(0);
|
||||
|
||||
const int threads = 256;
|
||||
const int blocks = CUDA_N_BLOCKS_NEEDED(n_rays, threads);
|
||||
|
||||
// outputs
|
||||
torch::Tensor grad_sigmas = torch::zeros(sigmas.sizes(), sigmas.options());
|
||||
torch::Tensor grad_rgbs = torch::zeros(rgbs.sizes(), rgbs.options());
|
||||
|
||||
AT_DISPATCH_FLOATING_TYPES_AND_HALF(
|
||||
sigmas.scalar_type(),
|
||||
"volumetric_rendering_backward",
|
||||
([&]
|
||||
{ volumetric_rendering_backward_kernel<scalar_t><<<blocks, threads>>>(
|
||||
n_rays,
|
||||
packed_info.data_ptr<int>(),
|
||||
starts.data_ptr<scalar_t>(),
|
||||
ends.data_ptr<scalar_t>(),
|
||||
sigmas.data_ptr<scalar_t>(),
|
||||
rgbs.data_ptr<scalar_t>(),
|
||||
accumulated_weight.data_ptr<scalar_t>(),
|
||||
accumulated_depth.data_ptr<scalar_t>(),
|
||||
accumulated_color.data_ptr<scalar_t>(),
|
||||
grad_weight.data_ptr<scalar_t>(),
|
||||
grad_depth.data_ptr<scalar_t>(),
|
||||
grad_color.data_ptr<scalar_t>(),
|
||||
grad_sigmas.data_ptr<scalar_t>(),
|
||||
grad_rgbs.data_ptr<scalar_t>()
|
||||
);
|
||||
}));
|
||||
|
||||
return {grad_sigmas, grad_rgbs};
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
/*
|
||||
* Helper code.
|
||||
*/
|
||||
|
||||
#define CUDA_NUM_BLOCKS_NEEDED(Q, CUDA_N_THREADS) ((Q - 1) / CUDA_N_THREADS + 1)
|
||||
@@ -1,30 +0,0 @@
|
||||
/*
|
||||
This file uses pybind to make CUDA calls.
|
||||
*/
|
||||
|
||||
#include <torch/extension.h>
|
||||
|
||||
#include <vector>
|
||||
|
||||
// CUDA forward declarations
|
||||
|
||||
std::vector<torch::Tensor> sample_uniformly_along_ray_bundle(
|
||||
torch::Tensor origins,
|
||||
torch::Tensor directions,
|
||||
torch::Tensor nears,
|
||||
torch::Tensor fars,
|
||||
torch::Tensor offsets,
|
||||
int max_num_samples);
|
||||
|
||||
// C++ interface
|
||||
|
||||
#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor")
|
||||
#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous")
|
||||
#define CHECK_INPUT(x) \
|
||||
CHECK_CUDA(x); \
|
||||
CHECK_CONTIGUOUS(x)
|
||||
|
||||
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)
|
||||
{
|
||||
m.def("sample_uniformly_along_ray_bundle", &sample_uniformly_along_ray_bundle);
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
/*
|
||||
Note that this file has the _kernel.cu extension because
|
||||
setuptools cannot handle files with the same name but different extensions.
|
||||
*/
|
||||
|
||||
#include <torch/extension.h>
|
||||
|
||||
#include <cuda.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <vector>
|
||||
|
||||
#define CUDA_N_BLOCKS_NEEDED(Q, CUDA_N_THREADS) ((Q - 1) / CUDA_N_THREADS + 1)
|
||||
|
||||
namespace
|
||||
{
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ scalar_t sigmoid(scalar_t z)
|
||||
{
|
||||
return 1.0 / (1.0 + exp(-z));
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ scalar_t d_sigmoid(scalar_t z)
|
||||
{
|
||||
const auto s = sigmoid(z);
|
||||
return (1.0 - s) * s;
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ scalar_t d_tanh(scalar_t z)
|
||||
{
|
||||
const auto t = tanh(z);
|
||||
return 1 - (t * t);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ scalar_t elu(scalar_t z, scalar_t alpha = 1.0)
|
||||
{
|
||||
return fmaxf(0.0, z) + fminf(0.0, alpha * (exp(z) - 1.0));
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__device__ __forceinline__ scalar_t d_elu(scalar_t z, scalar_t alpha = 1.0)
|
||||
{
|
||||
const auto e = exp(z);
|
||||
const auto d_relu = z < 0.0 ? 0.0 : 1.0;
|
||||
return d_relu + (((alpha * (e - 1.0)) < 0.0) ? (alpha * e) : 0.0);
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void sample_uniformly_along_ray_bundle_kernel(
|
||||
const torch::PackedTensorAccessor32<scalar_t, 2, torch::RestrictPtrTraits> origins,
|
||||
const torch::PackedTensorAccessor32<scalar_t, 2, torch::RestrictPtrTraits> directions,
|
||||
const torch::PackedTensorAccessor32<scalar_t, 1, torch::RestrictPtrTraits> nears,
|
||||
const torch::PackedTensorAccessor32<scalar_t, 1, torch::RestrictPtrTraits> fars,
|
||||
const torch::PackedTensorAccessor32<scalar_t, 1, torch::RestrictPtrTraits> offsets,
|
||||
torch::PackedTensorAccessor32<scalar_t, 2, torch::RestrictPtrTraits> output_time_steps,
|
||||
torch::PackedTensorAccessor32<scalar_t, 3, torch::RestrictPtrTraits> output_samples,
|
||||
torch::PackedTensorAccessor32<scalar_t, 2, torch::RestrictPtrTraits> output_time_steps_mask)
|
||||
{
|
||||
|
||||
const int thread_id = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
const int max_num_samples = output_samples.size(1);
|
||||
|
||||
scalar_t t_min = nears[thread_id];
|
||||
scalar_t t_max = fars[thread_id];
|
||||
scalar_t offset = offsets[thread_id];
|
||||
int sample_index = 0;
|
||||
|
||||
// #pragma unroll
|
||||
for (scalar_t t = t_min; t <= t_max; t += offset)
|
||||
{
|
||||
// #pragma unroll
|
||||
for (int i = 0; i < 3; ++i)
|
||||
{
|
||||
if (sample_index < max_num_samples)
|
||||
{
|
||||
scalar_t pos_i = origins[thread_id][i] + t * directions[thread_id][i];
|
||||
output_samples[thread_id][sample_index][i] = pos_i;
|
||||
}
|
||||
}
|
||||
if (sample_index < max_num_samples)
|
||||
{
|
||||
output_time_steps[thread_id][sample_index] = t;
|
||||
output_time_steps_mask[thread_id][sample_index] = 1.0;
|
||||
}
|
||||
sample_index += 1;
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::vector<torch::Tensor> sample_uniformly_along_ray_bundle(
|
||||
torch::Tensor origins,
|
||||
torch::Tensor directions,
|
||||
torch::Tensor nears,
|
||||
torch::Tensor fars,
|
||||
torch::Tensor offsets,
|
||||
int max_num_samples)
|
||||
{
|
||||
std::cout << "sample_uniformly_along_ray_bundle -- start" << std::endl;
|
||||
|
||||
TORCH_CHECK(origins.is_floating_point()); // should be float32
|
||||
TORCH_CHECK(origins.ndimension() == 2); // and check that shape is (num_rays, 3)
|
||||
TORCH_CHECK(origins.size(1) == 3);
|
||||
const auto num_rays = origins.size(0);
|
||||
|
||||
// TODO: change the number of threads based on GPU
|
||||
const int threads = 1024; // number of threads in a thread block
|
||||
// const int blocks = num_rays / threads;
|
||||
const int blocks = CUDA_N_BLOCKS_NEEDED(num_rays, threads);
|
||||
|
||||
// std::cout << origins.options() << std::endl;
|
||||
torch::Tensor output_time_steps = torch::zeros({num_rays, max_num_samples}, origins.options());
|
||||
torch::Tensor output_samples = torch::zeros({num_rays, max_num_samples, 3}, origins.options());
|
||||
torch::Tensor output_time_steps_mask = torch::zeros({num_rays, max_num_samples}, origins.options());
|
||||
|
||||
// The second argument is used for error messages.
|
||||
// __FUNCTION__ macro == "sample_uniformly_along_ray_bundle" in this case.
|
||||
AT_DISPATCH_FLOATING_TYPES(origins.type(), __FUNCTION__, [&]
|
||||
{ sample_uniformly_along_ray_bundle_kernel<scalar_t><<<blocks, threads>>>(
|
||||
origins.packed_accessor32<scalar_t, 2, torch::RestrictPtrTraits>(),
|
||||
directions.packed_accessor32<scalar_t, 2, torch::RestrictPtrTraits>(),
|
||||
nears.packed_accessor32<scalar_t, 1, torch::RestrictPtrTraits>(),
|
||||
fars.packed_accessor32<scalar_t, 1, torch::RestrictPtrTraits>(),
|
||||
offsets.packed_accessor32<scalar_t, 1, torch::RestrictPtrTraits>(),
|
||||
output_time_steps.packed_accessor32<scalar_t, 2, torch::RestrictPtrTraits>(),
|
||||
output_samples.packed_accessor32<scalar_t, 3, torch::RestrictPtrTraits>(),
|
||||
output_time_steps_mask.packed_accessor32<scalar_t, 2, torch::RestrictPtrTraits>()); });
|
||||
|
||||
std::cout << "sample_uniformly_along_ray_bundle -- end" << std::endl;
|
||||
return {output_time_steps, output_samples, output_time_steps_mask};
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Sampler.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import pyrad_cuda
|
||||
|
||||
|
||||
class LLTMFunction(torch.autograd.Function):
|
||||
@staticmethod
|
||||
def forward(ctx, input, weights, bias, old_h, old_cell):
|
||||
outputs = pyrad_cuda.forward(input, weights, bias, old_h, old_cell)
|
||||
new_h, new_cell = outputs[:2]
|
||||
variables = outputs[1:] + [weights]
|
||||
ctx.save_for_backward(*variables)
|
||||
|
||||
return new_h, new_cell
|
||||
|
||||
@staticmethod
|
||||
def backward(ctx, grad_h, grad_cell):
|
||||
outputs = pyrad_cuda.backward(grad_h.contiguous(), grad_cell.contiguous(), *ctx.saved_variables)
|
||||
d_old_h, d_input, d_weights, d_bias, d_old_cell, d_gates = outputs
|
||||
return d_input, d_weights, d_bias, d_old_h, d_old_cell
|
||||
|
||||
|
||||
class Sampler(nn.Module):
|
||||
def __init__(self, input_features, state_size):
|
||||
super().__init__()
|
||||
self.input_features = input_features
|
||||
self.state_size = state_size
|
||||
self.weights = nn.Parameter(torch.Tensor(3 * state_size, input_features + state_size))
|
||||
self.bias = nn.Parameter(torch.Tensor(1, 3 * state_size))
|
||||
self.reset_parameters()
|
||||
|
||||
def reset_parameters(self):
|
||||
stdv = 1.0 / math.sqrt(self.state_size)
|
||||
for weight in self.parameters():
|
||||
weight.data.uniform_(-stdv, +stdv)
|
||||
|
||||
def forward(self, input, state):
|
||||
return LLTMFunction.apply(input, self.weights, self.bias, *state)
|
||||
@@ -19,12 +19,12 @@ Base class for the graphs.
|
||||
from abc import abstractmethod
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
from torch import nn
|
||||
import torch
|
||||
from torch import nn
|
||||
from torchtyping import TensorType
|
||||
from pyrad.fields.modules.field_heads import FieldHeadNames
|
||||
|
||||
from pyrad.cameras.rays import Frustums, RaySamples
|
||||
from pyrad.fields.modules.field_heads import FieldHeadNames
|
||||
from pyrad.utils.misc import is_not_none
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ class Field(nn.Module):
|
||||
"""Base class for fields."""
|
||||
|
||||
def density_fn(self, positions):
|
||||
"""Returns only the density. Used primarily with the occupancy grid."""
|
||||
"""Returns only the density. Used primarily with the density grid."""
|
||||
# Need to figure out a better way to descibe positions with a ray.
|
||||
ray_samples = RaySamples(
|
||||
frustums=Frustums(
|
||||
|
||||
File renamed without changes.
@@ -0,0 +1,194 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Code to implement the density grid.
|
||||
"""
|
||||
from typing import Callable, List, NoReturn, Tuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torchtyping import TensorType
|
||||
|
||||
import pyrad.cuda as pyrad_cuda
|
||||
|
||||
|
||||
def _create_grid_coords(resolution: int, device: torch.device = "cpu") -> TensorType["n_coords", 3]:
|
||||
"""Create 3D grid coordinates
|
||||
|
||||
Args:
|
||||
resolution (int): The 3D resolution of the grid.
|
||||
device (torch.device): Which device you want the returned tensor to live int.
|
||||
|
||||
Returns:
|
||||
TensorType["n_coords", 3]: All grid coordinates with shape [res * res * res, 3]
|
||||
"""
|
||||
arrange = torch.arange(resolution, device=device)
|
||||
grid_x, grid_y, grid_z = torch.meshgrid([arrange, arrange, arrange], indexing="ij")
|
||||
coords = torch.stack([grid_x, grid_y, grid_z]) # [3, res, res, res]
|
||||
coords = coords.reshape(3, -1).t().contiguous() # [N, 3]
|
||||
return coords
|
||||
|
||||
|
||||
class DensityGrid(nn.Module):
|
||||
"""Cascaded multi-res density grids.
|
||||
|
||||
The multi-res grids are all centerd at [center, center, center] in the world space.
|
||||
The first grid covers regions (center - base_scale / 2, center + base_scale / 2)
|
||||
and the followings grows by 2x scale each.
|
||||
|
||||
TODO(ruilongli): support `center` and `base_scale` with 3-dim.
|
||||
|
||||
Args:
|
||||
center (float, optional): Center of all the grids in the world space. Defaults to 0.
|
||||
base_scale (float, optional): Center of the scale of the base level grid. Defaults to 1.
|
||||
num_cascades (int, optional): Number of the cascaded multi-res levels. Defaults to 1.
|
||||
resolution (int, optional): Resolution of the grid. Defaults to 128.
|
||||
update_every_num_iters (int, optional): How frequently to update the grid values. Defaults to 16.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
center: float = 0.0,
|
||||
base_scale: float = 1.0,
|
||||
num_cascades: int = 1,
|
||||
resolution: int = 128,
|
||||
update_every_num_iters: int = 16,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.center = center
|
||||
self.base_scale = base_scale
|
||||
self.num_cascades = num_cascades # the number of levels (i.e, cascades)
|
||||
self.resolution = resolution
|
||||
self.update_every_num_iters = update_every_num_iters
|
||||
|
||||
density_grid = torch.zeros([self.num_cascades] + [self.resolution**3])
|
||||
self.register_buffer("density_grid", density_grid)
|
||||
|
||||
density_bitfield = torch.zeros([self.num_cascades * self.resolution**3 // 8], dtype=torch.uint8)
|
||||
self.register_buffer("density_bitfield", density_bitfield)
|
||||
|
||||
# Integer grid coords / indices that do not related to cascades
|
||||
grid_coords = _create_grid_coords(resolution)
|
||||
# TODO(ruilongli): hacky way for now. support cpu version of morton3D?
|
||||
grid_indices = pyrad_cuda.morton3D(grid_coords.to("cuda:0")).to("cpu")
|
||||
self.register_buffer("grid_coords", grid_coords)
|
||||
self.register_buffer("grid_indices", grid_indices)
|
||||
|
||||
@torch.no_grad()
|
||||
def get_all_cells(self) -> List[Tuple[TensorType["n_cells"], TensorType["n_cells", 3]]]:
|
||||
"""Returns all cells of the grid.
|
||||
|
||||
Returns:
|
||||
List[Tuple[TensorType["n_cells"], TensorType["n_cells", 3]]]: All grid indices and grid
|
||||
coordinates of all cascade levels.
|
||||
"""
|
||||
return [(self.grid_indices, self.grid_coords)] * self.num_cascades
|
||||
|
||||
@torch.no_grad()
|
||||
def sample_uniform_and_occupied_cells(self, n: int) -> List[Tuple[TensorType["n"], TensorType["n", 3]]]:
|
||||
"""Samples both n uniform and occupied cells (per cascade)
|
||||
|
||||
Args:
|
||||
n: Number of cells to be sampled.
|
||||
|
||||
Returns:
|
||||
List[Tuple[TensorType["n"], TensorType["n", 3]]]: Sampled grid indices and grid
|
||||
coordinates of all cascade levels.
|
||||
"""
|
||||
device = self.density_grid.device
|
||||
|
||||
cells = []
|
||||
for c in range(self.num_cascades):
|
||||
uniform_coords = torch.randint(self.resolution, (n, 3), device=device)
|
||||
uniform_indices = pyrad_cuda.morton3D(uniform_coords)
|
||||
|
||||
occupied_indices = torch.nonzero(self.density_grid[c] > 0)[:, 0]
|
||||
if n < len(occupied_indices):
|
||||
selector = torch.randint(len(occupied_indices), (n,), device=device)
|
||||
occupied_indices = occupied_indices[selector]
|
||||
occupied_coords = pyrad_cuda.morton3D_invert(occupied_indices)
|
||||
|
||||
# concatenate
|
||||
coords = torch.cat([uniform_coords, occupied_coords], dim=0)
|
||||
indices = torch.cat([uniform_indices, occupied_indices], dim=0)
|
||||
cells.append((indices, coords))
|
||||
return cells
|
||||
|
||||
@torch.no_grad()
|
||||
def update_density_grid(
|
||||
self,
|
||||
density_eval_func: Callable,
|
||||
step: int,
|
||||
density_threshold: float = 2, # 0.01 / (SQRT3 / 1024 * 3)
|
||||
decay: float = 0.95,
|
||||
) -> NoReturn:
|
||||
"""Update the density grid in EMA way.
|
||||
|
||||
Args:
|
||||
density_eval_func (Callable): A Callable function that takes in sample positions (N, 3) and
|
||||
returns densities (N, 1).
|
||||
step (int): The current training step. Instant-NGP has a warmup stage where all cells are updated.
|
||||
After certain amount of steps (256), it speeds up by only update sampled cells.
|
||||
density_threshold (float): The threshold to prune cells. Recommand to calculate it using this rule:
|
||||
`weight_threshold / dt`. For example for Instant-NGP on Lego, the minimum step size `dt` is
|
||||
`3` (scale of the scene) * `sqrt(3)` / `1024` (number of samples). And if we want to the weight
|
||||
threshold to be `0.01`, we will get `0.01 / (3 * sqrt(3) / 1024) ~= 2.` for density threshold.
|
||||
decay (float): EMA decay for density updating.
|
||||
"""
|
||||
# create temporary grid
|
||||
tmp_grid = -torch.ones_like(self.density_grid)
|
||||
if step < 256:
|
||||
cells = self.get_all_cells()
|
||||
else:
|
||||
N = self.resolution**3 // 4
|
||||
cells = self.sample_uniform_and_occupied_cells(N)
|
||||
|
||||
# infer sigmas
|
||||
for mip_level in range(self.num_cascades):
|
||||
mip_scale = 2 ** (-mip_level) / self.base_scale
|
||||
indices, coords = cells[mip_level]
|
||||
# `coords` denotes the i-th cell. It's a poor naming here.
|
||||
# the actually coordinates in world space x has mapping to
|
||||
# the cell like this:
|
||||
# x \in [0, 1/res] maps to 0-th cell.
|
||||
# x \in [1 - 1/res, 1] maps to (res-1)-th cell.
|
||||
x = (coords + torch.rand_like(coords.float())) / self.resolution
|
||||
x = (x - 0.5) / mip_scale + self.center
|
||||
tmp_grid[mip_level, indices] = density_eval_func(x).squeeze(-1)
|
||||
|
||||
# ema update
|
||||
valid_mask = (self.density_grid >= 0) & (tmp_grid >= 0)
|
||||
self.density_grid[valid_mask] = torch.maximum(self.density_grid[valid_mask] * decay, tmp_grid[valid_mask])
|
||||
mean_density = self.density_grid.mean().item()
|
||||
|
||||
# pack to bitfield
|
||||
self.density_bitfield.data = pyrad_cuda.packbits(self.density_grid, min(mean_density, density_threshold))
|
||||
|
||||
# TODO(ruilongli): max pooling? https://github.com/NVlabs/instant-ngp/blob/master/src/testbed_nerf.cu#L578
|
||||
|
||||
@torch.no_grad()
|
||||
def get_densities(self, xyzs: TensorType[..., 3]) -> TensorType[..., 1]:
|
||||
"""Trilinear interpolation to get the density values.
|
||||
Args:
|
||||
xyzs (TensorType[..., 3]): 3D querry coordinate
|
||||
Returns:
|
||||
TensorType[..., 1]: Density values
|
||||
"""
|
||||
# TODO(ruilongli) pass in RaySamples because we need dt to decide mip level.
|
||||
raise NotImplementedError("We haven't implement the logic of querying densities from cascaded grid.")
|
||||
|
||||
def forward(self, x):
|
||||
"""Not implemented."""
|
||||
raise RuntimeError("Shouldn't be called!")
|
||||
@@ -22,11 +22,11 @@ from typing import Tuple
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from pyrad.cameras.rays import RaySamples
|
||||
from pyrad.fields.base import Field
|
||||
from pyrad.fields.modules.encoding import Encoding, HashEncoding, SHEncoding
|
||||
from pyrad.fields.modules.field_heads import FieldHeadNames
|
||||
from pyrad.fields.base import Field
|
||||
from pyrad.fields.nerf_field import NeRFField
|
||||
from pyrad.cameras.rays import RaySamples
|
||||
from pyrad.utils.activations import trunc_exp
|
||||
|
||||
try:
|
||||
@@ -113,7 +113,7 @@ class TCNNInstantNGPField(Field):
|
||||
# Rectifying the density with an exponential is much more stable than a ReLU or
|
||||
# softplus, because it enables high post-activation (float32) density outputs
|
||||
# from smaller internal (float16) parameters.
|
||||
density = trunc_exp(density_before_activation)
|
||||
density = trunc_exp(density_before_activation.to(positions))
|
||||
return density, base_mlp_out
|
||||
|
||||
def get_outputs(self, ray_samples: RaySamples, density_embedding=None):
|
||||
@@ -123,7 +123,7 @@ class TCNNInstantNGPField(Field):
|
||||
directions_flat = directions.view(-1, 3)
|
||||
d = self.direction_encoding(directions_flat)
|
||||
h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1)
|
||||
rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1)
|
||||
rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions)
|
||||
return {FieldHeadNames.RGB: rgb}
|
||||
|
||||
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Code to implement the density grid.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from torchtyping import TensorType
|
||||
|
||||
from pyrad.utils.misc import is_not_none
|
||||
|
||||
|
||||
class OccupancyGrid(nn.Module):
|
||||
"""Module to keep track of the density and occupancy."""
|
||||
|
||||
def __init__(
|
||||
self, num_cascades: int = 1, resolution: int = 128, aabb=None, update_every_num_iters: int = 16
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert is_not_none(aabb), "The axis-aligned bounding box aabb is not defined!"
|
||||
self.num_cascades = num_cascades # the number of levels (i.e, cascades)
|
||||
self.resolution = resolution
|
||||
self.register_buffer("aabb", aabb) # axis-aligned bounding box
|
||||
occupancy_grid = torch.ones(
|
||||
[self.num_cascades] + [self.resolution] * 3
|
||||
) # (num_cascades, x_resolution, y_resolution, z_resolution)
|
||||
# not a module parameter, but it still should be saved and part of `state_dict`
|
||||
self.register_buffer("occupancy_grid", occupancy_grid)
|
||||
self.mean_density = 0.0
|
||||
|
||||
self.update_every_num_iters = update_every_num_iters
|
||||
|
||||
def reset(self):
|
||||
"""Zero out the occupancy grid."""
|
||||
self.occupancy_grid.zero_()
|
||||
|
||||
@torch.no_grad()
|
||||
def mark_untrained_grid(self):
|
||||
"""Marks the locations in the density grid that are never seen by training views."""
|
||||
# TODO: only query points that are covered by the training cameras viewing frustrum
|
||||
# We'll need to pass in the camera poses and intrinsics to this function
|
||||
raise NotImplementedError()
|
||||
|
||||
@torch.no_grad()
|
||||
def update_occupancy_grid(self, density_fn, decay=0.95, split_size=64):
|
||||
"""Update the occupancy grid with density values.
|
||||
|
||||
Args:
|
||||
density_fn: Function from takes points in the shape (..., 3) and
|
||||
returns a tensor of densities of shape (..., 1).
|
||||
"""
|
||||
# TODO(ethan): remove this pylint disable
|
||||
# pylint: disable=invalid-name
|
||||
device = self.occupancy_grid.device
|
||||
|
||||
assert self.num_cascades == 1, "Still need to implement code for multiple cascades."
|
||||
temp_occupancy_grid = torch.zeros_like(self.occupancy_grid)
|
||||
|
||||
X_length, Y_length, Z_length = self.aabb[1] - self.aabb[0]
|
||||
X_half_voxel_length = 0.5 * X_length / self.resolution
|
||||
Y_half_voxel_length = 0.5 * Y_length / self.resolution
|
||||
Z_half_voxel_length = 0.5 * Z_length / self.resolution
|
||||
half_voxel_lengths = torch.tensor(
|
||||
[X_half_voxel_length, Y_half_voxel_length, Z_half_voxel_length], device=device
|
||||
)
|
||||
X = torch.linspace(
|
||||
self.aabb[0][0] + X_half_voxel_length, self.aabb[1][0] - X_half_voxel_length, self.resolution, device=device
|
||||
).split(split_size)
|
||||
Y = torch.linspace(
|
||||
self.aabb[0][1] + Y_half_voxel_length, self.aabb[1][1] - Y_half_voxel_length, self.resolution, device=device
|
||||
).split(split_size)
|
||||
Z = torch.linspace(
|
||||
self.aabb[0][2] + Z_half_voxel_length, self.aabb[1][2] - Z_half_voxel_length, self.resolution, device=device
|
||||
).split(split_size)
|
||||
for x_index, xs in enumerate(X):
|
||||
for y_index, ys in enumerate(Y):
|
||||
for z_index, zs in enumerate(Z):
|
||||
x_len, y_len, z_len = len(xs), len(ys), len(zs)
|
||||
grid_x, grid_y, grid_z = torch.meshgrid(xs, ys, zs, indexing="ij")
|
||||
xyzs = torch.stack([grid_x, grid_y, grid_z], dim=-1) # (resolution, resolution,, resolution, 3)
|
||||
# add noise to the points
|
||||
xyzs_rand = (
|
||||
torch.rand_like(xyzs) * half_voxel_lengths
|
||||
) # range [-half_voxel_length, half_voxel_length] for each dimension
|
||||
xyzs += xyzs_rand
|
||||
densities = density_fn(xyzs)
|
||||
assert densities.shape[-1] == 1
|
||||
temp_occupancy_grid[
|
||||
0,
|
||||
x_index * split_size : x_index * split_size + x_len,
|
||||
y_index * split_size : y_index * split_size + y_len,
|
||||
z_index * split_size : z_index * split_size + z_len,
|
||||
] = densities[..., 0]
|
||||
|
||||
# exponential moving average update
|
||||
valid_mask = self.occupancy_grid >= 0.0
|
||||
self.occupancy_grid[valid_mask] = torch.maximum(
|
||||
self.occupancy_grid[valid_mask] * decay, temp_occupancy_grid[valid_mask]
|
||||
)
|
||||
self.mean_density = torch.mean(self.occupancy_grid[valid_mask]).item()
|
||||
|
||||
@torch.no_grad()
|
||||
def get_densities(self, xyzs: TensorType[..., 3]) -> TensorType[..., 1]:
|
||||
"""Trilinear interpolation to get the density values.
|
||||
|
||||
Args:
|
||||
xyzs (TensorType[..., 3]): 3D querry coordinate
|
||||
|
||||
Returns:
|
||||
TensorType[..., 1]: Density values
|
||||
"""
|
||||
occupancy_grid = self.occupancy_grid[None, ...] # shape (1, num_cascades, X_res, Y_res, Z_res)
|
||||
xyzs_shape = xyzs.shape
|
||||
xyzs_reshaped = xyzs.view(1, -1, 1, 1, 3)
|
||||
voxel_lengths = self.aabb[1] - self.aabb[0]
|
||||
xyzs_reshaped_normalized = ((xyzs_reshaped - self.aabb[0]) / voxel_lengths) * 2.0 - 1.0
|
||||
densities = F.grid_sample(
|
||||
occupancy_grid.permute(
|
||||
0, 1, 4, 3, 2
|
||||
), # (xyz to zyx) for grid sample useage because the xyzs have xyz ordering
|
||||
xyzs_reshaped_normalized,
|
||||
align_corners=True,
|
||||
padding_mode="zeros",
|
||||
)
|
||||
densities = densities.view(*xyzs_shape[:-1], 1)
|
||||
return densities
|
||||
|
||||
def forward(self, x):
|
||||
"""Not implemented."""
|
||||
raise NotImplementedError()
|
||||
@@ -76,7 +76,7 @@ class Graph(AbstractGraph):
|
||||
intrinsics (torch.Tensor): Camera intrinsics.
|
||||
camera_to_world (torch.Tensor): Camera to world transformation.
|
||||
loss_coefficients (DictConfig): Loss specific weights.
|
||||
steps_per_occupancy_grid_update (int): How often to update occupancy grid.
|
||||
steps_per_density_grid_update (int): How often to update density grid.
|
||||
scene_bounds (SceneBounds): Bounds of target scene.
|
||||
collider_config (DictConfig): Configuration of scene collider.
|
||||
"""
|
||||
@@ -86,19 +86,19 @@ class Graph(AbstractGraph):
|
||||
intrinsics: torch.Tensor = None,
|
||||
camera_to_world: torch.Tensor = None,
|
||||
loss_coefficients: DictConfig = None,
|
||||
steps_per_occupancy_grid_update: int = 16,
|
||||
steps_per_density_grid_update: int = 16,
|
||||
scene_bounds: SceneBounds = None,
|
||||
collider_config: DictConfig = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert is_not_none(scene_bounds), "scene_bounds is needed to use the occupancy grid"
|
||||
assert is_not_none(scene_bounds), "scene_bounds is needed to use the density grid"
|
||||
self.intrinsics = intrinsics
|
||||
self.camera_to_world = camera_to_world
|
||||
self.scene_bounds = scene_bounds
|
||||
self.collider_config = collider_config
|
||||
self.loss_coefficients = loss_coefficients
|
||||
self.steps_per_occupancy_grid_update = steps_per_occupancy_grid_update
|
||||
self.steps_per_density_grid_update = steps_per_density_grid_update
|
||||
self.kwargs = kwargs
|
||||
self.collider = None
|
||||
self.ray_generator = RayGenerator(self.intrinsics, self.camera_to_world)
|
||||
|
||||
+35
-47
@@ -21,19 +21,18 @@ from typing import Dict, List
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
from torchmetrics import PeakSignalNoiseRatio
|
||||
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
|
||||
from torchmetrics.functional import structural_similarity_index_measure
|
||||
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
|
||||
|
||||
from pyrad.fields.modules.field_heads import FieldHeadNames
|
||||
from pyrad.fields.instant_ngp_field import field_implementation_to_class
|
||||
from pyrad.graphs.base import Graph
|
||||
from pyrad.optimizers.loss import MSELoss
|
||||
from pyrad.fields.occupancy_fields.occupancy_grid import OccupancyGrid
|
||||
from pyrad.renderers.renderers import AccumulationRenderer, DepthRenderer, RGBRenderer
|
||||
from pyrad.graphs.modules.ray_sampler import UniformSampler
|
||||
from pyrad.utils import colors
|
||||
import pyrad.cuda as pyrad_cuda
|
||||
from pyrad.cameras.rays import RayBundle
|
||||
from pyrad.utils import visualization, writer
|
||||
from pyrad.fields.density_fields.density_grid import DensityGrid
|
||||
from pyrad.fields.instant_ngp_field import field_implementation_to_class
|
||||
from pyrad.fields.modules.field_heads import FieldHeadNames
|
||||
from pyrad.graphs.base import Graph
|
||||
from pyrad.graphs.modules.ray_sampler import NGPSpacedSampler
|
||||
from pyrad.optimizers.loss import MSELoss
|
||||
from pyrad.utils import colors, visualization, writer
|
||||
from pyrad.utils.callbacks import Callback
|
||||
|
||||
|
||||
@@ -51,9 +50,9 @@ class NGPGraph(Graph):
|
||||
"""defining callbacks to run after every training iteration"""
|
||||
self.callbacks = [
|
||||
Callback(
|
||||
self.occupancy_grid.update_every_num_iters,
|
||||
self.occupancy_grid.update_occupancy_grid,
|
||||
density_fn=self.field.density_fn,
|
||||
self.density_grid.update_every_num_iters,
|
||||
self.density_grid.update_density_grid,
|
||||
density_eval_func=self.field.density_fn,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -63,19 +62,11 @@ class NGPGraph(Graph):
|
||||
self.field = field_implementation_to_class[self.field_implementation](self.scene_bounds.aabb)
|
||||
|
||||
def populate_misc_modules(self):
|
||||
# occupancy grid
|
||||
self.occupancy_grid = OccupancyGrid(aabb=self.scene_bounds.aabb)
|
||||
# density grid
|
||||
self.density_grid = DensityGrid(center=0.0, base_scale=3, num_cascades=1)
|
||||
|
||||
# samplers
|
||||
self.sampler_occupancy_grid = UniformSampler(num_samples=128)
|
||||
|
||||
# TODO stabalize occupancy grid.
|
||||
# self.sampler_occupancy_grid = UniformSampler(num_samples=128, occupancy_field=self.occupancy_grid)
|
||||
|
||||
# renderers
|
||||
self.renderer_rgb = RGBRenderer(background_color=colors.WHITE)
|
||||
self.renderer_accumulation = AccumulationRenderer()
|
||||
self.renderer_depth = DepthRenderer()
|
||||
self.sampler = NGPSpacedSampler(num_samples=1024, density_field=self.density_grid)
|
||||
|
||||
# losses
|
||||
self.rgb_loss = MSELoss()
|
||||
@@ -90,37 +81,39 @@ class NGPGraph(Graph):
|
||||
param_groups["fields"] = list(self.field.parameters())
|
||||
return param_groups
|
||||
|
||||
@torch.cuda.amp.autocast()
|
||||
def get_outputs(self, ray_bundle: RayBundle):
|
||||
# TODO(ruilongli)
|
||||
# - train test difference
|
||||
# - visualize "depth_density_grid"
|
||||
ray_samples, packed_info, t_min, t_max = self.sampler(ray_bundle, self.field.aabb)
|
||||
|
||||
# uniform sampling
|
||||
ray_samples = self.sampler_occupancy_grid(ray_bundle)
|
||||
field_outputs = self.field.forward(ray_samples)
|
||||
weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY])
|
||||
rgbs = field_outputs[FieldHeadNames.RGB]
|
||||
sigmas = field_outputs[FieldHeadNames.DENSITY]
|
||||
|
||||
weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY])
|
||||
rgb = self.renderer_rgb(
|
||||
rgb=field_outputs[FieldHeadNames.RGB],
|
||||
weights=weights,
|
||||
accumulated_weight, accumulated_depth, accumulated_color, mask = pyrad_cuda.VolumeRenderer.apply(
|
||||
packed_info, ray_samples.frustums.starts, ray_samples.frustums.ends, sigmas, rgbs
|
||||
)
|
||||
accumulation = self.renderer_accumulation(weights)
|
||||
depth = self.renderer_depth(weights, ray_samples)
|
||||
|
||||
densities_occupancy_grid = self.occupancy_grid.get_densities(ray_samples.frustums.get_positions())
|
||||
weights_occupancy_grid = ray_samples.get_weights(densities_occupancy_grid)
|
||||
depth_occupancy_grid = self.renderer_depth(weights_occupancy_grid, ray_samples)
|
||||
accumulated_depth = torch.clip(accumulated_depth, t_min[:, None], t_max[:, None])
|
||||
accumulated_color = accumulated_color + colors.WHITE.to(accumulated_color) * (1.0 - accumulated_weight)
|
||||
|
||||
outputs = {
|
||||
"rgb": rgb,
|
||||
"accumulation": accumulation,
|
||||
"depth": depth,
|
||||
"depth_occupancy_grid": depth_occupancy_grid,
|
||||
"rgb": accumulated_color,
|
||||
"accumulation": accumulated_weight,
|
||||
"depth": accumulated_depth,
|
||||
"mask": mask, # the rays we skipped from sampler
|
||||
}
|
||||
return outputs
|
||||
|
||||
def get_loss_dict(self, outputs, batch):
|
||||
device = self.get_device()
|
||||
image = batch["image"].to(device)
|
||||
rgb_loss = self.rgb_loss(image, outputs["rgb"])
|
||||
if "mask" in outputs:
|
||||
mask = outputs["mask"]
|
||||
rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask])
|
||||
else:
|
||||
rgb_loss = self.rgb_loss(image, outputs["rgb"])
|
||||
loss_dict = {"rgb_loss": rgb_loss}
|
||||
return loss_dict
|
||||
|
||||
@@ -137,8 +130,6 @@ class NGPGraph(Graph):
|
||||
outputs["accumulation"] = combined_acc
|
||||
combined_depth = torch.cat([depth], dim=1)
|
||||
outputs["depth"] = combined_depth
|
||||
depth = visualization.apply_depth_colormap(outputs["depth_occupancy_grid"])
|
||||
outputs["depth_occupancy_grid"] = combined_depth
|
||||
|
||||
def log_test_image_outputs(self, image_idx, step, batch, outputs):
|
||||
image = batch["image"]
|
||||
@@ -157,9 +148,6 @@ class NGPGraph(Graph):
|
||||
writer.put_image(name=f"accumulation/image_idx_{image_idx}", image=combined_acc, step=step)
|
||||
writer.put_image(name=f"depth/image_idx_{image_idx}", image=combined_depth, step=step)
|
||||
|
||||
occupancy_depth = visualization.apply_depth_colormap(outputs["depth_occupancy_grid"])
|
||||
writer.put_image(name=f"depth_occupancy_grid/image_idx_{image_idx}", image=occupancy_depth, step=step)
|
||||
|
||||
# Switch images from [H, W, C] to [1, C, H, W] for metrics computations
|
||||
image = torch.moveaxis(image, -1, 0)[None, ...]
|
||||
rgb = torch.moveaxis(rgb, -1, 0)[None, ...]
|
||||
|
||||
@@ -17,25 +17,26 @@ Collection of sampling strategies
|
||||
"""
|
||||
|
||||
from abc import abstractmethod
|
||||
from typing import Callable, Optional
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torchtyping import TensorType
|
||||
|
||||
from pyrad.cameras.rays import RayBundle, RaySamples
|
||||
from pyrad.fields.occupancy_fields.occupancy_grid import OccupancyGrid
|
||||
import pyrad.cuda as pyrad_cuda
|
||||
from pyrad.cameras.rays import Frustums, RayBundle, RaySamples
|
||||
from pyrad.fields.density_fields.density_grid import DensityGrid
|
||||
|
||||
|
||||
class Sampler(nn.Module):
|
||||
"""Generate Samples"""
|
||||
|
||||
def __init__(
|
||||
self, num_samples: int, occupancy_field: Optional[OccupancyGrid] = None, weight_threshold: float = 1e-4
|
||||
self, num_samples: int, density_field: Optional[DensityGrid] = None, weight_threshold: float = 1e-4
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.num_samples = num_samples
|
||||
self.occupancy_field = occupancy_field
|
||||
self.density_field = density_field
|
||||
self.weight_threshold = weight_threshold
|
||||
|
||||
@abstractmethod
|
||||
@@ -43,10 +44,10 @@ class Sampler(nn.Module):
|
||||
"""Generate Ray Samples"""
|
||||
|
||||
def forward(self, *args, **kwargs) -> RaySamples:
|
||||
"""Generate ray samples with optional occupancy filtering"""
|
||||
"""Generate ray samples with optional density filtering"""
|
||||
ray_samples = self.generate_ray_samples(*args, **kwargs)
|
||||
if self.occupancy_field is not None:
|
||||
densities = self.occupancy_field.get_densities(ray_samples.frustums.get_positions())
|
||||
if self.density_field is not None:
|
||||
densities = self.density_field.get_densities(ray_samples.frustums.get_positions())
|
||||
weights = ray_samples.get_weights(densities)
|
||||
|
||||
valid_mask = weights >= self.weight_threshold
|
||||
@@ -63,7 +64,7 @@ class SpacedSampler(Sampler):
|
||||
spacing_fn: Callable,
|
||||
spacing_fn_inv: Callable,
|
||||
train_stratified=True,
|
||||
occupancy_field: Optional[OccupancyGrid] = None,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-4,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -72,11 +73,11 @@ class SpacedSampler(Sampler):
|
||||
spacing_fn (Callable): Function that dictates sample spacing (ie `lambda x : x` is uniform).
|
||||
spacing_fn_inv (Callable): The inverse of spacing_fn.
|
||||
train_stratified (bool): Use stratified sampling during training. Defults to True
|
||||
occupancy_field (OccupancyGrid, optional): Occupancy grid. If provides,
|
||||
density_field (DensityGrid, optional): Density grid. If provides,
|
||||
samples below weight_threshold as set as invalid.
|
||||
weight_thershold (float): Removes samples below threshold weight. Only used if occupancy field is provided.
|
||||
weight_threshold (float): Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
super().__init__(num_samples=num_samples, occupancy_field=occupancy_field, weight_threshold=weight_threshold)
|
||||
super().__init__(num_samples=num_samples, density_field=density_field, weight_threshold=weight_threshold)
|
||||
self.train_stratified = train_stratified
|
||||
self.spacing_fn = spacing_fn
|
||||
self.spacing_fn_inv = spacing_fn_inv
|
||||
@@ -127,23 +128,23 @@ class UniformSampler(SpacedSampler):
|
||||
self,
|
||||
num_samples: int,
|
||||
train_stratified=True,
|
||||
occupancy_field: Optional[OccupancyGrid] = None,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-4,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
num_samples (int): Number of samples per ray
|
||||
train_stratified (bool): Use stratified sampling during training. Defults to True
|
||||
occupancy_field (OccupancyGrid, optional): Occupancy grid. If provides,
|
||||
density_field (DensityGrid, optional): Density grid. If provides,
|
||||
samples below weight_threshold as set as invalid.
|
||||
weight_thershold (float): Removes samples below threshold weight. Only used if occupancy field is provided.
|
||||
weight_threshold (float): Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
super().__init__(
|
||||
num_samples=num_samples,
|
||||
spacing_fn=lambda x: x,
|
||||
spacing_fn_inv=lambda x: x,
|
||||
train_stratified=train_stratified,
|
||||
occupancy_field=occupancy_field,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
|
||||
@@ -155,23 +156,23 @@ class LinearDisparitySampler(SpacedSampler):
|
||||
self,
|
||||
num_samples: int,
|
||||
train_stratified=True,
|
||||
occupancy_field: Optional[OccupancyGrid] = None,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-4,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
num_samples (int): Number of samples per ray
|
||||
train_stratified (bool): Use stratified sampling during training. Defults to True
|
||||
occupancy_field (OccupancyGrid, optional): Occupancy grid. If provides,
|
||||
density_field (DensityGrid, optional): Density grid. If provides,
|
||||
samples below weight_threshold as set as invalid.
|
||||
weight_thershold (float): Removes samples below threshold weight. Only used if occupancy field is provided.
|
||||
weight_threshold (float): Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
super().__init__(
|
||||
num_samples=num_samples,
|
||||
spacing_fn=lambda x: 1 / x,
|
||||
spacing_fn_inv=lambda x: 1 / x,
|
||||
train_stratified=train_stratified,
|
||||
occupancy_field=occupancy_field,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
|
||||
@@ -183,23 +184,23 @@ class SqrtSampler(SpacedSampler):
|
||||
self,
|
||||
num_samples: int,
|
||||
train_stratified=True,
|
||||
occupancy_field: Optional[OccupancyGrid] = None,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-4,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
num_samples (int): Number of samples per ray
|
||||
train_stratified (bool): Use stratified sampling during training. Defults to True
|
||||
occupancy_field (OccupancyGrid, optional): Occupancy grid. If provides,
|
||||
density_field (DensityGrid, optional): Density grid. If provides,
|
||||
samples below weight_threshold as set as invalid.
|
||||
weight_thershold (float): Removes samples below threshold weight. Only used if occupancy field is provided.
|
||||
weight_threshold (float): Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
super().__init__(
|
||||
num_samples=num_samples,
|
||||
spacing_fn=torch.sqrt,
|
||||
spacing_fn_inv=lambda x: x**2,
|
||||
train_stratified=train_stratified,
|
||||
occupancy_field=occupancy_field,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
|
||||
@@ -211,23 +212,23 @@ class LogSampler(SpacedSampler):
|
||||
self,
|
||||
num_samples: int,
|
||||
train_stratified=True,
|
||||
occupancy_field: Optional[OccupancyGrid] = None,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-4,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
num_samples (int): Number of samples per ray
|
||||
train_stratified (bool): Use stratified sampling during training. Defults to True
|
||||
occupancy_field (OccupancyGrid, optional): Occupancy grid. If provides,
|
||||
density_field (DensityGrid, optional): Density grid. If provides,
|
||||
samples below weight_threshold as set as invalid.
|
||||
weight_thershold (float): Removes samples below threshold weight. Only used if occupancy field is provided.
|
||||
weight_threshold (float): Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
super().__init__(
|
||||
num_samples=num_samples,
|
||||
spacing_fn=torch.log,
|
||||
spacing_fn_inv=torch.exp,
|
||||
train_stratified=train_stratified,
|
||||
occupancy_field=occupancy_field,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
|
||||
@@ -240,7 +241,7 @@ class PDFSampler(Sampler):
|
||||
num_samples: int,
|
||||
train_stratified: bool = True,
|
||||
include_original: bool = True,
|
||||
occupancy_field: OccupancyGrid = None,
|
||||
density_field: DensityGrid = None,
|
||||
weight_threshold: float = 1e-4,
|
||||
histogram_padding: float = 0.01,
|
||||
) -> None:
|
||||
@@ -249,12 +250,12 @@ class PDFSampler(Sampler):
|
||||
num_samples (int): Number of samples per ray
|
||||
train_stratified: boolean: Randomize location within each bin during training. Defaults to True
|
||||
include_original: Add original samples to ray. Defaults to True
|
||||
occupancy_field (OccupancyGrid, optional): Occupancy grid. If provides,
|
||||
density_field (DensityGrid, optional): Density grid. If provides,
|
||||
samples below weight_threshold as set as invalid.
|
||||
weight_thershold (float): Removes samples below threshold weight. Only used if occupancy field is provided.
|
||||
weight_threshold (float): Removes samples below threshold weight. Only used if density field is provided.
|
||||
histogram_padding (float): Amount to weights prior to computing PDF. Defaults to 0.01.
|
||||
"""
|
||||
super().__init__(num_samples=num_samples, occupancy_field=occupancy_field, weight_threshold=weight_threshold)
|
||||
super().__init__(num_samples=num_samples, density_field=density_field, weight_threshold=weight_threshold)
|
||||
self.train_stratified = train_stratified
|
||||
self.include_original = include_original
|
||||
self.histogram_padding = histogram_padding
|
||||
@@ -336,3 +337,83 @@ class PDFSampler(Sampler):
|
||||
ray_samples = ray_bundle.get_ray_samples(bin_starts=bins[..., :-1, None], bin_ends=bins[..., 1:, None])
|
||||
|
||||
return ray_samples
|
||||
|
||||
|
||||
class NGPSpacedSampler(Sampler):
|
||||
"""Sampler that matches Instant-NGP paper.
|
||||
|
||||
This sampler does ray-box AABB test, ray samples generation and density check, all together.
|
||||
|
||||
TODO(ruilongli): check whether fuse AABB test together with `raymarching` can speed things up.
|
||||
Otherwise we can seperate it out and use collision detecoter that already exists in the repo.
|
||||
"""
|
||||
|
||||
def generate_ray_samples(self) -> RaySamples:
|
||||
raise RuntimeError("For NGP we fused ray samples and density check together. Please call forward() directly.")
|
||||
|
||||
# pylint: disable=arguments-differ
|
||||
def forward(
|
||||
self,
|
||||
ray_bundle: RayBundle,
|
||||
aabb: TensorType[2, 3],
|
||||
num_samples: Optional[int] = None,
|
||||
) -> Tuple[RaySamples, TensorType["total_samples", 3], TensorType["total_samples"], TensorType["total_samples"]]:
|
||||
"""Generate ray samples in a bounding box.
|
||||
|
||||
TODO(ruilongli): write a Packed[Ray_samples] class to ray_samples with packed_info.
|
||||
TODO(ruilongli): maybe move aabb test to the collision detector?
|
||||
|
||||
Args:
|
||||
ray_bundle (RayBundle): Rays to generate samples for
|
||||
aabb (TensorType[2, 3]): Bounding box of the scene.
|
||||
num_samples (Optional[int]): Number of samples per ray
|
||||
|
||||
Returns:
|
||||
Tuple[RaySamples, TensorType["total_samples", 3], TensorType["total_samples"], TensorType["total_samples"]]:
|
||||
First return is ray samples in a packed way where only the valid samples are kept.
|
||||
Second return contains all the information to recover packed samples into unpacked mode for rendering.
|
||||
The last two returns are t_min and t_max from ray-aabb test.
|
||||
"""
|
||||
num_samples = num_samples or self.num_samples
|
||||
|
||||
aabb = aabb.flatten()
|
||||
rays_o = ray_bundle.origins.contiguous()
|
||||
rays_d = ray_bundle.directions.contiguous()
|
||||
t_min, t_max = pyrad_cuda.ray_aabb_intersect(rays_o, rays_d, aabb)
|
||||
scene_scale = (aabb[3:6] - aabb[0:3]).max()
|
||||
|
||||
# TODO(ruilongli): * 16 is for original impl for training. Need to run
|
||||
# some profiling test with this choice.
|
||||
max_samples_per_batch = len(rays_o) * num_samples
|
||||
|
||||
packed_info, origins, dirs, starts, ends = pyrad_cuda.raymarching(
|
||||
# rays
|
||||
rays_o,
|
||||
rays_d,
|
||||
t_min,
|
||||
t_max,
|
||||
# density grid
|
||||
self.density_field.center,
|
||||
self.density_field.base_scale,
|
||||
self.density_field.num_cascades,
|
||||
self.density_field.resolution,
|
||||
self.density_field.density_bitfield,
|
||||
# sampling args
|
||||
max_samples_per_batch,
|
||||
num_samples,
|
||||
0.0,
|
||||
scene_scale,
|
||||
)
|
||||
total_samples = max(packed_info[:, -1].sum(), 1)
|
||||
packed_info = packed_info[:total_samples]
|
||||
origins = origins[:total_samples]
|
||||
dirs = dirs[:total_samples]
|
||||
starts = starts[:total_samples]
|
||||
ends = ends[:total_samples]
|
||||
|
||||
zeros = torch.zeros_like(origins[:, :1])
|
||||
|
||||
ray_samples = RaySamples(
|
||||
frustums=Frustums(origins=origins, directions=dirs, starts=starts, ends=ends, pixel_area=zeros),
|
||||
)
|
||||
return ray_samples, packed_info, t_min, t_max
|
||||
@@ -43,4 +43,4 @@ class Callback:
|
||||
step (int): current iteration step
|
||||
"""
|
||||
if step % self.update_every_num_iters == 0:
|
||||
self.func(*self.args, **self.kwargs)
|
||||
self.func(*self.args, **self.kwargs, step=step)
|
||||
@@ -15,6 +15,7 @@
|
||||
""" Helper functions for visualizing outputs """
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
from matplotlib import cm
|
||||
from torchtyping import TensorType
|
||||
|
||||
Reference in new issue
Block a user