Fix bug related to distributed data parallel training (#1856)

* Fix bug related to distributed data parallel training

* fix format by black

* Add nerfacto-big multi-gpu commands

* cleaned up docs

* reverted default dataset to poster

---------

Co-authored-by: Ethan Weber <eweb0124@gmail.com>
This commit is contained in:
J.YandEthan Weber authored and GitHub committed 2023-05-15 14:23:29 -07:00
1 parent f1b60d4641
commit 7851dfa9db
3 files changed
+59 -17

No files matched your search

+4 -2
View File
@@ -251,7 +251,9 @@ class Trainer:
if step > 1:
writer.put_time(
name=EventName.TRAIN_RAYS_PER_SEC,
duration=self.pipeline.datamanager.get_train_rays_per_batch() / train_t.duration,
duration=self.world_size
* self.pipeline.datamanager.get_train_rays_per_batch()
/ train_t.duration,
step=step,
avg_over_steps=True,
)
@@ -365,7 +367,7 @@ class Trainer:
train_num_rays_per_batch: int = self.pipeline.datamanager.get_train_rays_per_batch()
writer.put_time(
name=EventName.TRAIN_RAYS_PER_SEC,
duration=train_num_rays_per_batch / (train_t.duration - vis_t.duration),
duration=self.world_size * train_num_rays_per_batch / (train_t.duration - vis_t.duration),
step=step,
avg_over_steps=True,
)