mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-11 02:24:38 +08:00
9eb880aa91
* Blender addon script and documentation First version of blender addon python script located in scripts/blender and new documentation for the addon. The Blender addon is for importing or exporting a camera path for visual effects and compositing. * Updated formatting of Blender script Updated the formatting based on the script tests for the blender addon script. * Updated Blender addon to fix FOV and alignment Updated how FOV is calculated for both blender cam to json and json to blender cam. The FOV is calculated based on the aspect ratio, angle of view, and sensor fit and size. * Updated Blender addon documentation with examples Updated the Blender add-on documentation with a few example gif composite animations * Updated Blender addon to match frame rate and fixed formatting Updated the Blender add-on script and docs to match the Blender frame rate. Also the formatting should be fixed. There are also now error message popups instead of console prints for invalid input in the add-on. * Updated the blender addon for formatting Updated pylint formatting and comments
447 lines
16 KiB
Python
447 lines
16 KiB
Python
"""
|
|
nerfstudio_blender.py
|
|
"""
|
|
|
|
bl_info = {
|
|
"name": "Nerfstudio Add-On",
|
|
"description": "Create a Nerfstudio JSON camera path from the Blender camera path \
|
|
or import a Nerfstudio camera path as a Blender camera to composite Blender renders \
|
|
over a NeRF background render for VFX",
|
|
"author": "Cyrus Vachha",
|
|
"version": (1, 0),
|
|
"blender": (3, 0, 0),
|
|
"category": "Nerfstudio",
|
|
}
|
|
|
|
# pylint: disable=wrong-import-position
|
|
import json
|
|
from math import atan, degrees, radians, tan
|
|
|
|
# pylint: disable=import-error
|
|
import bpy
|
|
from mathutils import Matrix
|
|
|
|
|
|
class CreateJSONCameraPath(bpy.types.Operator):
|
|
"""Create a JSON camera path from the Blender camera animation."""
|
|
|
|
bl_idname = "opr.create_json_camera_path"
|
|
bl_label = "Nerfstudio Camera Path Generator"
|
|
|
|
cam_obj = None # the render camera is the active camera
|
|
nerf_bg_mesh = None # the background NeRF as a mesh
|
|
|
|
fov_list = [] # list of FOV at each frame
|
|
transformed_camera_path_mat = [] # final transformed world matrix of the camera at each frame
|
|
|
|
complete_json_obj = {} # full Nerfstudio input json object
|
|
|
|
file_path_json = "" # file path input
|
|
|
|
def get_camera_coordinates(self):
|
|
"""Create a list of transformed Blender camera coordinates and converted FOV."""
|
|
|
|
org_camera_path_mat = [] # list of world matrix of the active camera at each frame
|
|
nerf_mesh_mat_list = [] # list of world matrix of the NeRF mesh at each frame
|
|
|
|
curr_frame = bpy.context.scene.frame_start
|
|
|
|
while curr_frame <= bpy.context.scene.frame_end:
|
|
bpy.context.scene.frame_set(curr_frame)
|
|
org_camera_path_mat += [self.cam_obj.matrix_world.copy()]
|
|
|
|
if bpy.context.scene.render.resolution_y >= bpy.context.scene.render.resolution_x:
|
|
# portrait orientation
|
|
|
|
if self.cam_obj.data.sensor_fit == "HORIZONTAL":
|
|
# convert horizontal fov to vertical fov with aspect ratio
|
|
cam_aspect_ratio = bpy.context.scene.render.resolution_y / bpy.context.scene.render.resolution_x
|
|
nerfstudio_fov = 2 * atan(tan(self.cam_obj.data.angle / 2.0) * cam_aspect_ratio)
|
|
else:
|
|
# sensor fit is either vertical or auto
|
|
nerfstudio_fov = self.cam_obj.data.angle
|
|
|
|
else:
|
|
# landscape orientation
|
|
|
|
if self.cam_obj.data.sensor_fit == "VERTICAL":
|
|
nerfstudio_fov = self.cam_obj.data.angle
|
|
else:
|
|
# sensor fit is either horizontal or auto
|
|
# convert horizontal fov to vertical fov with aspect ratio
|
|
cam_aspect_ratio = bpy.context.scene.render.resolution_y / bpy.context.scene.render.resolution_x
|
|
nerfstudio_fov = 2 * atan(tan(self.cam_obj.data.angle / 2.0) * cam_aspect_ratio)
|
|
|
|
self.fov_list += [degrees(nerfstudio_fov)]
|
|
curr_frame += bpy.context.scene.frame_step
|
|
nerf_mesh_mat_list += [self.nerf_bg_mesh.matrix_world.copy()]
|
|
|
|
# transform the camera world matrix based on the NeRF mesh transformation
|
|
for i, org_cam_path_mat_val in enumerate(org_camera_path_mat):
|
|
self.transformed_camera_path_mat += [nerf_mesh_mat_list[i].inverted() @ org_cam_path_mat_val]
|
|
|
|
def get_list_from_matrix_path(self, input_mat): # pylint: disable=no-self-use
|
|
"""Flatten matrix to list for camera path."""
|
|
full_arr = list(input_mat.row[0]) + list(input_mat.row[1]) + list(input_mat.row[2]) + list(input_mat.row[3])
|
|
return full_arr
|
|
|
|
def get_list_from_matrix_keyframe(self, input_mat): # pylint: disable=no-self-use
|
|
"""Flatten matrix to list for keyframes."""
|
|
full_arr = list(input_mat.col[0]) + list(input_mat.col[1]) + list(input_mat.col[2]) + list(input_mat.col[3])
|
|
return full_arr
|
|
|
|
def construct_json_obj(self):
|
|
"""Get fields for JSON camera path."""
|
|
cam_type = self.cam_obj.data.type
|
|
if cam_type == "PERSP":
|
|
cam_type = "perspective"
|
|
elif cam_type == "PANO" and self.cam_obj.data.cycles.panorama_type == "EQUIRECTANGULAR":
|
|
cam_type = "equirectangular"
|
|
else:
|
|
self.report(
|
|
{"WARNING"}, "Nerfstudio Add-on Warning: Only perspective and equirectangular cameras are supported"
|
|
)
|
|
cam_type = "perspective"
|
|
|
|
render_height = int(
|
|
bpy.context.scene.render.resolution_y * (bpy.context.scene.render.resolution_percentage * 0.01)
|
|
)
|
|
render_width = int(
|
|
bpy.context.scene.render.resolution_x * (bpy.context.scene.render.resolution_percentage * 0.01)
|
|
)
|
|
render_fps = bpy.context.scene.render.fps
|
|
render_seconds = (
|
|
(bpy.context.scene.frame_end - bpy.context.scene.frame_start + 1) // (bpy.context.scene.frame_step)
|
|
) / render_fps
|
|
smoothness_value = 0
|
|
is_cycle = False
|
|
|
|
# construct camera path
|
|
final_camera_path = []
|
|
|
|
for i, transformed_camera_path_mat_val in enumerate(self.transformed_camera_path_mat):
|
|
camera_path_elem = {
|
|
"camera_to_world": self.get_list_from_matrix_path(transformed_camera_path_mat_val),
|
|
"fov": self.fov_list[i],
|
|
"aspect": 1,
|
|
}
|
|
final_camera_path += [camera_path_elem]
|
|
|
|
# construct keyframes
|
|
keyframe_list = []
|
|
|
|
for i, transformed_camera_path_mat_val in enumerate(self.transformed_camera_path_mat):
|
|
|
|
curr_properties = (
|
|
'[["FOV",'
|
|
+ str(self.fov_list[i])
|
|
+ '],["NAME","Camera '
|
|
+ str(i)
|
|
+ '"],["TIME",'
|
|
+ str(i / render_fps)
|
|
+ "]]"
|
|
)
|
|
|
|
keyframe_elem = {
|
|
"matrix": str(self.get_list_from_matrix_keyframe(self.transformed_camera_path_mat[i])),
|
|
"fov": self.fov_list[i],
|
|
"aspect": 1,
|
|
"properties": curr_properties,
|
|
}
|
|
keyframe_list += [keyframe_elem]
|
|
|
|
overall_json = {
|
|
"keyframes": keyframe_list,
|
|
"camera_type": cam_type,
|
|
"render_height": render_height,
|
|
"render_width": render_width,
|
|
"camera_path": final_camera_path,
|
|
"fps": render_fps,
|
|
"seconds": render_seconds,
|
|
"smoothness_value": smoothness_value,
|
|
"is_cycle": is_cycle,
|
|
}
|
|
|
|
self.complete_json_obj = json.dumps(overall_json, indent=2)
|
|
|
|
def write_json_to_file(self):
|
|
"""Write the JSON object to a new file."""
|
|
|
|
full_abs_file_path = bpy.path.abspath(self.file_path_json + "camera_path_blender.json")
|
|
with open(full_abs_file_path, "w", encoding="utf8") as output_json_camera_path:
|
|
output_json_camera_path.truncate(0)
|
|
output_json_camera_path.write(self.complete_json_obj)
|
|
|
|
self.complete_json_obj = {}
|
|
print("\nFinished creating camera path json file at " + full_abs_file_path + "\n")
|
|
|
|
def execute(self, context):
|
|
"""Execute the camera path creation process."""
|
|
# get user specified values from UI
|
|
|
|
self.cam_obj = bpy.context.scene.camera
|
|
self.nerf_bg_mesh = context.scene.NeRF
|
|
self.file_path_json = context.scene.JSONInputFilePath
|
|
|
|
# check input
|
|
if self.nerf_bg_mesh is None:
|
|
self.report(
|
|
{"ERROR"}, "Nerfstudio add-on Error! - Please input NeRF representation (as mesh or point cloud)"
|
|
)
|
|
return {"FINISHED"}
|
|
|
|
if self.file_path_json == "":
|
|
self.report({"ERROR"}, "Nerfstudio add-on Error! - Please input a file path for the output JSON")
|
|
return {"FINISHED"}
|
|
|
|
# reset lists before running
|
|
self.fov_list = []
|
|
self.transformed_camera_path_mat = []
|
|
self.complete_json_obj = {}
|
|
|
|
# create the path
|
|
self.get_camera_coordinates()
|
|
self.construct_json_obj()
|
|
self.write_json_to_file()
|
|
|
|
return {"FINISHED"}
|
|
|
|
|
|
class ReadJSONinputCameraPath(bpy.types.Operator):
|
|
"""Create a camera with an animation path based on an input Nerfstudio JSON."""
|
|
|
|
bl_idname = "opr.read_json_camera_path"
|
|
bl_label = "Blender Camera Generator from JSON"
|
|
|
|
# cam_obj = None # the render camera is the active camera
|
|
nerf_bg_mesh = None # the background NeRF as a mesh
|
|
|
|
fov_list = [] # list of FOV at each frame
|
|
transformed_camera_path_mat = [] # final transformed world matrix of the camera at each frame
|
|
input_json = None
|
|
|
|
def read_camera_coodinates(self):
|
|
"""Read the camera coordinates (world matrix and fov) from the json camera path."""
|
|
|
|
json_cam_path = self.input_json["camera_path"]
|
|
self.fov_list = []
|
|
self.transformed_camera_path_mat = []
|
|
|
|
keyframe_counter = 0
|
|
for cam_keyframe in json_cam_path:
|
|
cam_to_world = cam_keyframe["camera_to_world"]
|
|
|
|
# convert cam_to_world to 4x4 matrix
|
|
orig_cam_mat = Matrix([cam_to_world[0:4], cam_to_world[4:8], cam_to_world[8:12], cam_to_world[12:]])
|
|
|
|
# matrix transformation based on the nerf mesh to find relative camera positions
|
|
self.transformed_camera_path_mat += [self.nerf_bg_mesh.matrix_world.copy() @ orig_cam_mat]
|
|
|
|
# record fov
|
|
self.fov_list += [cam_keyframe["fov"]]
|
|
|
|
keyframe_counter += 1
|
|
|
|
def generate_camera(self):
|
|
"""Create a new camera with the animation (position and fov) and the corresponding type."""
|
|
|
|
json_cam_path = self.input_json["camera_path"]
|
|
|
|
camera_data = bpy.data.cameras.new(name="NerfstudioCamera")
|
|
camera_data = bpy.data.cameras.new(name="NerfstudioCamera")
|
|
nerfstudio_camera_object = bpy.data.objects.new("NerfstudioCamera", camera_data)
|
|
bpy.context.scene.collection.objects.link(nerfstudio_camera_object)
|
|
|
|
curr_frame = 0
|
|
while curr_frame < len(json_cam_path):
|
|
actual_frame = curr_frame + 1
|
|
# animate camera transform
|
|
nerfstudio_camera_object.matrix_world = self.transformed_camera_path_mat[curr_frame]
|
|
nerfstudio_camera_object.keyframe_insert("location", frame=actual_frame)
|
|
nerfstudio_camera_object.keyframe_insert("rotation_euler", frame=actual_frame)
|
|
|
|
# set scale to 1,1,1 (scale is not keyframed)
|
|
nerfstudio_camera_object.scale = (1, 1, 1)
|
|
|
|
# animate fov
|
|
nerfstudio_camera_object.data.sensor_fit = "VERTICAL"
|
|
nerfstudio_camera_object.data.lens_unit = "FOV"
|
|
nerfstudio_camera_object.data.angle = radians(self.fov_list[curr_frame])
|
|
|
|
# set keyframe for focal length
|
|
nerfstudio_camera_object.data.keyframe_insert(data_path="lens", frame=actual_frame)
|
|
|
|
curr_frame += 1
|
|
|
|
# set camera attributes
|
|
input_cam_type = self.input_json["camera_type"]
|
|
if input_cam_type == "perspective":
|
|
nerfstudio_camera_object.data.type = "PERSP"
|
|
if input_cam_type == "equirectangular":
|
|
nerfstudio_camera_object.data.type = "PANO"
|
|
bpy.context.scene.render.engine = "CYCLES"
|
|
nerfstudio_camera_object.data.cycles.panorama_type = "EQUIRECTANGULAR"
|
|
if input_cam_type == "fisheye":
|
|
nerfstudio_camera_object.data.type = "PERSP"
|
|
self.report({"WARNING"}, "Nerfstudio Add-on Warning: Fisheye cameras are not supported")
|
|
|
|
def execute(self, context):
|
|
"""Execute Blender camera creation process."""
|
|
|
|
# initializat variables
|
|
self.nerf_bg_mesh = context.scene.NeRF
|
|
file_path_ns_json = context.scene.NS_input_jsonFilePath # input file path for the input json file
|
|
|
|
# check input
|
|
if self.nerf_bg_mesh is None:
|
|
self.report(
|
|
{"ERROR"}, "Nerfstudio add-on Error! - Please input NeRF representation (as mesh or point cloud)"
|
|
)
|
|
return {"FINISHED"}
|
|
|
|
if file_path_ns_json == "":
|
|
self.report({"ERROR"}, "Nerfstudio add-on Error! - Please input a Nerfstudio JSON camera path")
|
|
return {"FINISHED"}
|
|
|
|
# open the json file
|
|
full_abs_file_path = bpy.path.abspath(file_path_ns_json)
|
|
with open(full_abs_file_path, encoding="utf8") as json_ns_file:
|
|
self.input_json = json.load(json_ns_file)
|
|
|
|
# call methods to read cam path and create camera
|
|
self.read_camera_coodinates()
|
|
self.generate_camera()
|
|
|
|
return {"FINISHED"}
|
|
|
|
|
|
# --- Blender UI Panel --- #
|
|
|
|
|
|
class NerfstudioMainPanel(bpy.types.Panel): # pylint: disable=too-few-public-methods
|
|
"""Blender UI main panel for the add-on."""
|
|
|
|
bl_idname = "NERFSTUDIO_PT_NerfstudioMainPanel"
|
|
bl_label = "Nerfstudio Add-on"
|
|
bl_space_type = "PROPERTIES"
|
|
bl_region_type = "WINDOW"
|
|
bl_context = "render"
|
|
|
|
def draw(self, context):
|
|
"""Main panel UI components."""
|
|
# NeRF representation object input box
|
|
self.layout.label(text="NeRF Representation (mesh or point cloud)")
|
|
self.layout.prop_search(context.scene, "NeRF", context.scene, "objects")
|
|
_ = self.layout.column()
|
|
|
|
|
|
class NerfstudioBgPanel(bpy.types.Panel): # pylint: disable=too-few-public-methods
|
|
"""Blender UI sub-panel for the camera path creation."""
|
|
|
|
bl_idname = "NERFSTUDIO_PT_NerfstudioBgPanel"
|
|
bl_label = "Nerfstudio Path Generator"
|
|
bl_parent_id = "NERFSTUDIO_PT_NerfstudioMainPanel"
|
|
bl_space_type = "PROPERTIES"
|
|
bl_region_type = "WINDOW"
|
|
bl_context = "render"
|
|
|
|
def draw(self, context):
|
|
"""Sub-panel UI components."""
|
|
|
|
self.layout.label(text="Camera path for Nerfstudio")
|
|
|
|
col = self.layout.column()
|
|
for (prop_name, _) in INPUT_PROPERTIES:
|
|
row = col.row()
|
|
row.prop(context.scene, prop_name)
|
|
|
|
col.operator("opr.create_json_camera_path", text="Generate JSON File")
|
|
|
|
|
|
class NerfstudioInputPanel(bpy.types.Panel): # pylint: disable=too-few-public-methods
|
|
"""Blender UI sub-panel for the Blender camera creation."""
|
|
|
|
bl_idname = "NERFSTUDIO_PT_NerfstudioInputPanel"
|
|
bl_label = "Nerfstudio Camera Generator"
|
|
bl_parent_id = "NERFSTUDIO_PT_NerfstudioMainPanel"
|
|
bl_space_type = "PROPERTIES"
|
|
bl_region_type = "WINDOW"
|
|
bl_context = "render"
|
|
|
|
def draw(self, context):
|
|
"""Sub-panel UI components."""
|
|
|
|
col = self.layout.column()
|
|
self.layout.label(text="Create Blender Camera From Nerfstudio JSON")
|
|
col = self.layout.column()
|
|
|
|
for (prop_name, _) in INPUT_PROPERTIES_NS_CAMERA:
|
|
row = col.row()
|
|
row.prop(context.scene, prop_name)
|
|
|
|
col.operator("opr.read_json_camera_path", text="Create Camera from JSON")
|
|
|
|
|
|
CLASSES = [
|
|
NerfstudioMainPanel,
|
|
NerfstudioBgPanel,
|
|
NerfstudioInputPanel,
|
|
CreateJSONCameraPath,
|
|
ReadJSONinputCameraPath,
|
|
]
|
|
|
|
INPUT_PROPERTIES = [
|
|
(
|
|
"JSONInputFilePath",
|
|
bpy.props.StringProperty(name="JSON File Path", default="//", description="Path for JSON", subtype="DIR_PATH"),
|
|
)
|
|
]
|
|
|
|
INPUT_PROPERTIES_NS_CAMERA = [
|
|
(
|
|
"NS_input_jsonFilePath",
|
|
bpy.props.StringProperty(
|
|
name="JSON Nerfstudio File",
|
|
default="",
|
|
description="Path for JSON from Nerfstudio editor",
|
|
subtype="FILE_PATH",
|
|
),
|
|
)
|
|
]
|
|
|
|
OBJ_PROPERTIES = ["NeRF", "RenderCamera"]
|
|
|
|
|
|
def register():
|
|
"""Register classes for UI panel."""
|
|
|
|
for (prop_name, prop_value) in INPUT_PROPERTIES:
|
|
setattr(bpy.types.Scene, prop_name, prop_value)
|
|
|
|
for (prop_name, prop_value) in INPUT_PROPERTIES_NS_CAMERA:
|
|
setattr(bpy.types.Scene, prop_name, prop_value)
|
|
|
|
bpy.types.Scene.NeRF = bpy.props.PointerProperty(type=bpy.types.Object)
|
|
|
|
for curr_class in CLASSES:
|
|
bpy.utils.register_class(curr_class)
|
|
|
|
|
|
def unregister():
|
|
"""Unregister classes for UI panel."""
|
|
|
|
for (prop_name, _) in INPUT_PROPERTIES:
|
|
delattr(bpy.types.Scene, prop_name)
|
|
|
|
for (prop_name, _) in INPUT_PROPERTIES_NS_CAMERA:
|
|
delattr(bpy.types.Scene, prop_name)
|
|
|
|
del bpy.types.Scene.NeRF
|
|
|
|
for curr_class in CLASSES:
|
|
bpy.utils.unregister_class(curr_class)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
register()
|