mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 22:04:02 +08:00
Attempt at making viewer robust to random errors (#429)
* catch more specific errors * reduce complication; simple handle of oom error * lints
This commit is contained in:
1 parent
65ab15b5ce
commit
5469f2e6b8
5 files changed
+125
-8
No files matched your search
@@ -20,6 +20,7 @@ from __future__ import annotations
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
@@ -120,7 +121,6 @@ class Trainer:
|
||||
with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME):
|
||||
num_iterations = self.config.trainer.max_num_iterations
|
||||
for step in range(self.start_step, self.start_step + num_iterations):
|
||||
|
||||
# if the visualizer used, the rendering of the visualizer will be included in the iteration train time
|
||||
with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t:
|
||||
|
||||
@@ -138,7 +138,14 @@ class Trainer:
|
||||
|
||||
with TimeWriter(writer, EventName.ITER_VIS_TIME, step=step) as vis_t:
|
||||
num_rays_per_batch = self.config.pipeline.datamanager.train_num_rays_per_batch
|
||||
self.visualizer_state.update_scene(step, self.pipeline.model, num_rays_per_batch)
|
||||
try:
|
||||
self.visualizer_state.update_scene(step, self.pipeline.model, num_rays_per_batch)
|
||||
except RuntimeError:
|
||||
time.sleep(0.03) # sleep to allow buffer to reset
|
||||
assert self.visualizer_state.vis is not None
|
||||
self.visualizer_state.vis["renderingState/log_errors"].write(
|
||||
"Error: GPU out of memory. Reduce resolution to prevent viewer from crashing."
|
||||
)
|
||||
|
||||
if step != 0 and step % self.config.logging.steps_per_log == 0:
|
||||
writer.put_dict(name="Loss/train-loss_metrics_dict", scalar_dict=loss_metric_dict, step=step)
|
||||
|
||||
Reference in new issue
Block a user