Experimental autocompletion setup scripts (#378)

* Working completion scripts

* generate.py => install.py

* Uninstall for completions, docs/README updates

* Tune emojis, improve concurrency for completion install

* wip

* Cleanup

* Permissions, shebang checks
This commit is contained in:
Brent Yi authored and GitHub committed 2022-09-13 12:44:27 -07:00
1 parent 61022f7dbe
commit 86a805c88c
17 files changed
+386 -32

No files matched your search

+5
View File
@@ -104,6 +104,11 @@ pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.py
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
```
Experimental: install tab completion for nerfactory (bash & zsh)
```
python scripts/completions/configure.py install
```
Run the test cases
```
pytest tests
+8 -1
View File
@@ -46,6 +46,13 @@ pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.py
pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch
```
Experimental: install tab completion for nerfactory (bash and zsh)
```bash
# If the CLI changes, we can also re-install to synchronize completions
python scripts/completion/configure.py install
```
# Downloading data
Download the original NeRF Blender dataset. We support the major datasets and allow users to create their own dataset, described in detail [here](https://plenoptix-nerfactory.readthedocs-hosted.com/en/latest/tutorials/data/index.html).
@@ -137,4 +144,4 @@ When installing dependencies and nerfactory with `pip install -e .`, you run int
This can be fixed by upgrading pip to the latest version:
```
python -m pip install --upgrade pip
```
```
+29 -18
View File
@@ -104,8 +104,8 @@ class TensorboardWriterConfig(InstantiateConfig):
"""if True enables tensorboard logging, else disables"""
relative_log_dir: Path = Path("./")
"""relative path to save all tensorboard events"""
log_dir: Optional[Path] = None # full log dir path to be dynamically set
"""auto populated absolute path to saved tensorboard events [Do not set!]"""
log_dir: dcargs.conf.Fixed[Path] = dcargs.MISSING
"""auto-populated absolute path to saved tensorboard events"""
@dataclass
@@ -118,8 +118,8 @@ class WandbWriterConfig(InstantiateConfig):
"""if True enables wandb logging, else disables"""
relative_log_dir: Path = Path("./")
"""relative path to save all wandb events"""
log_dir: Optional[Path] = None # full log dir path to be dynamically set
"""auto populated absolute path to saved wandb events [Do not set!]"""
log_dir: dcargs.conf.Fixed[Path] = dcargs.MISSING
"""auto-populated absolute path to saved wandb events"""
@dataclass
@@ -142,8 +142,8 @@ class LocalWriterConfig(InstantiateConfig):
"""maximum number of rows to print before wrapping. if 0, will print everything."""
relative_log_dir: Path = Path("./")
"""relative local path to save all events"""
log_dir: Optional[Path] = None # full log dir path to be dynamically set
"""auto populated absolute local path to saved events [Do not set!]"""
log_dir: dcargs.conf.Fixed[Path] = dcargs.MISSING
"""auto-populated absolute local path to saved events"""
def setup(self, banner_messages: Optional[List[str]] = None, **kwargs) -> Any:
"""Instantiate local writer
@@ -189,8 +189,8 @@ class TrainerConfig(PrintableConfig):
"""whether or not to use mixed precision for training"""
relative_model_dir: Path = Path("nerfactory_models/")
"""relative path to save all checkpoints"""
model_dir: Optional[Path] = None # full model dir path to be dynamically set
"""auto populated absolute path to saved checkpoints [Do not set!]"""
model_dir: dcargs.conf.Fixed[Path] = dcargs.MISSING
"""auto-populated absolute path to saved checkpoints"""
# optional parameters if we want to resume training
load_dir: Optional[Path] = None
"""optionally specify a pre-trained model directory to load from"""
@@ -484,8 +484,10 @@ class PipelineConfig(InstantiateConfig):
class ViewerConfig(PrintableConfig):
"""Configuration for viewer instantiation"""
log_filename: Optional[Path] = None
"""Filename to use for the log file. Defaults to None. If None, no log file is created."""
relative_log_filename: str = "viewer_log_filename.txt"
"""Filename to use for the log file."""
log_filename: dcargs.conf.Fixed[Path] = dcargs.MISSING
"""Absolute path to the log file. Set automatically."""
enable: bool = False
"""whether to enable viewer"""
start_train: bool = True
@@ -538,7 +540,8 @@ class Config(PrintableConfig):
experiment_name: str = "blender_lego"
method_name: str = "base_method"
base_dir: Optional[Path] = None # base dir path to be dynamically set
base_dir: dcargs.conf.Fixed[Path] = dcargs.MISSING
"""Experiment base directory. Set automatically."""
machine: MachineConfig = MachineConfig()
logging: LoggingConfig = LoggingConfig()
trainer: TrainerConfig = TrainerConfig()
@@ -554,12 +557,20 @@ class Config(PrintableConfig):
viewer: ViewerConfig = ViewerConfig()
def __post_init__(self):
"""Convert logging directories to more specific filepaths"""
dt_str = datetime.now().strftime("%Y-%m-%d_%H%M%S")
self.base_dir = Path(f"outputs/{self.experiment_name}/{self.method_name}/{dt_str}")
if self.trainer.model_dir is None:
self.trainer.model_dir = self.base_dir / self.trainer.relative_model_dir
"""Make paths more specific using the current timestamp."""
self.set_timestamp()
def set_timestamp(self, timestamp: Optional[str] = None) -> None:
"""Make paths in our config more specific using a timestamp.
Args:
timestamp: Timestamp to use, as a string. If None, defaults to the current
time in the form YYYY-MM-DD_HHMMMSS.
"""
if timestamp is None:
timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S")
self.base_dir = Path(f"outputs/{self.experiment_name}/{self.method_name}/{timestamp}")
self.trainer.model_dir = self.base_dir / self.trainer.relative_model_dir
for curr_writer in self.logging.writer:
curr_writer.log_dir = self.base_dir / curr_writer.relative_log_dir
if self.viewer.log_filename is None:
self.viewer.log_filename = self.base_dir / "viewer_log_filename.txt"
self.viewer.log_filename = self.base_dir / self.viewer.relative_log_filename
+33 -1
View File
@@ -16,7 +16,11 @@
Put all the method implementations in one location.
"""
from typing import Dict
import copy
from typing import Dict, Type
import dcargs
from typeguard import typeguard_ignore
from nerfactory.configs.base import (
BlenderDataParserConfig,
@@ -169,3 +173,31 @@ base_configs["compound"] = Config(
}
},
)
@typeguard_ignore # TypeGuard doesn't understand the generic alias that's returned here.
def _make_base_config_subcommand_type() -> Type[Config]:
"""Generate a Union[] type over the possible base config types, with runtime
annotations containing default values. Used to generate CLIs.
Returns:
An annotated Union type, which can be used to pick a base configuration.
"""
# When a base config is used to generate a CLI: replace the auto-generated timestamp
# with {timestamp}. This makes the CLI helptext (and, for zsh, autocomplete
# generation) consistent everytime you run a script with --help.
#
# Note that when a config is instantiated with dcargs.cli(), the __post_init__
# called when the config is instantiated will set the correct timestamp.
base_configs_placeholder_timestamp = {}
for name, config in base_configs.items():
base_configs_placeholder_timestamp[name] = copy.deepcopy(config)
base_configs_placeholder_timestamp[name].set_timestamp("{timestamp}")
return dcargs.extras.subcommand_type_from_defaults(base_configs_placeholder_timestamp)
AnnotatedBaseConfigUnion = _make_base_config_subcommand_type()
"""Union[] type over config types, annotated with default instances for use with
dcargs.cli(). Allows the user to pick between one of several base configurations, and
then override values in it."""
+2
View File
@@ -0,0 +1,2 @@
bash/
zsh/
+249
View File
@@ -0,0 +1,249 @@
#!/usr/bin/env python
"""Configuration script for setting up tab completion for nerfactory in bash and zsh."""
import concurrent.futures
import itertools
import os
import pathlib
import random
import shutil
import stat
import subprocess
import sys
from typing import List, Literal
import dcargs
from rich.console import Console
from rich.prompt import Confirm
from typing_extensions import assert_never
ConfigureMode = Literal["install", "uninstall"]
ShellType = Literal["zsh", "bash"]
CONSOLE = Console(width=120)
def _check_dcargs_cli(script_path: pathlib.Path) -> bool:
"""Check if a path points to a script containing a dcargs.cli() call. Also checks
for any permissions/shebang issues.
Args:
script_path: Path to prospective CLI.
Returns:
True if a completion is can be generated.
"""
assert script_path.suffix == ".py"
script_src = script_path.read_text()
if '\nif __name__ == "__main__":\n' in script_src:
# Check script for execute permissions. For consistency, note that we apply this
# and the shebang check even if dcargs isn't used.
execute_flags = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
if not script_path.stat().st_mode & execute_flags and Confirm.ask(
f"[yellow]:warning: {script_path} is not marked as executable. Fix?[/yellow]",
default=True,
):
script_path.chmod(script_path.stat().st_mode | execute_flags)
# Check that script has a shebang.
if not script_src.startswith("#!/") and Confirm.ask(
f"[yellow]:warning: {script_path} is missing a shebang. Fix?[/yellow]",
default=True,
):
script_path.write_text("#!/usr/bin/env python\n" + script_src)
# Return True only if compatible with dcargs.
return "import dcargs" in script_src and "dcargs.cli" in script_src
return False
def _generate_completion(script_path: pathlib.Path, shell: ShellType, completions_dir: pathlib.Path) -> pathlib.Path:
"""Given a path to a dcargs CLI, write a completion script to a target directory.
Args:
script_path: Path to Python CLI to generate completion script for.
shell: Shell to generate completion script for.
completions_dir: Directory to write completion script to.
Returns:
Success flag.
"""
# Use zsh standard for naming completion scripts.
target_path = completions_dir / shell / ("_" + script_path.name.replace(".", "_"))
# Generate and write the new completion.
new = subprocess.run(
args=[sys.executable, str(script_path), "--dcargs-print-completion", shell],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf8",
check=True,
).stdout
target_path.parent.mkdir(parents=True, exist_ok=True)
if not target_path.exists():
target_path.write_text(new)
CONSOLE.log(f":heavy_check_mark: Wrote new completion to {target_path}!")
elif target_path.read_text().strip() != new.strip():
target_path.write_text(new)
CONSOLE.log(f":heavy_check_mark: Updated completion at {target_path}!")
else:
CONSOLE.log(f"[dim]:heavy_check_mark: Nothing to do for {target_path}[/dim].")
return target_path
def _exclamation() -> str:
return random.choice(["Cool", "Nice", "Neat", "Great", "Exciting", "Excellent", "Ok"]) + "!"
def _update_rc(
completions_dir: pathlib.Path,
mode: ConfigureMode,
shell: ShellType,
) -> None:
"""Try to add a `source /.../completions/setup.{shell}` line automatically to a user's zshrc or bashrc.
Args:
completions_dir: Path to location of this script.
shell: Shell to install completion scripts for.
mode: Install or uninstall completions.
"""
# Install or uninstall `source_line`.
source_lines = "\n".join(
[
"",
"# Source nerfactory autocompletions.",
f"source {completions_dir / 'setup'}.{shell}",
]
)
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
if mode == "install":
if source_lines in rc_path.read_text():
CONSOLE.log(f":call_me_hand: Completions are already installed in {rc_path}. {_exclamation()}")
return
if not Confirm.ask(f"[bold yellow]Install to {rc_path}?", default=True):
CONSOLE.log(f"[bold red]Skipping install for {rc_path.name}.")
return
rc_path.write_text(rc_path.read_text() + source_lines)
CONSOLE.log(
f":person_gesturing_ok: Completions installed to {rc_path}. {_exclamation()} Open a new shell to try them"
" out."
)
elif mode == "uninstall":
if source_lines not in rc_path.read_text():
CONSOLE.log(f":heavy_check_mark: No completions to uninstall from {rc_path.name}.")
return
if not Confirm.ask(f"[bold yellow]Uninstall from {rc_path}?", default=True):
CONSOLE.log(f"[dim red]Skipping uninstall for {rc_path.name}.")
return
rc_path.write_text(rc_path.read_text().replace(source_lines, ""))
CONSOLE.log(f":broom: Completions uninstalled from {rc_path}.")
else:
assert_never(mode)
def main(
mode: ConfigureMode,
/,
) -> None:
"""Main script.
Args:
mode: Choose between installing or uninstalling completions.
shells: Shell(s) to install or uninstall.
"""
if "HOME" not in os.environ:
CONSOLE.log("[bold red]$HOME is not set. Exiting.")
return
# Try to locate the user's bashrc or zshrc.
shells_supported: List[ShellType] = ["zsh", "bash"]
shells_found: List[ShellType] = []
for shell in shells_supported:
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
if not rc_path.exists():
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
else:
CONSOLE.log(f":mag: Found {rc_path.name}!")
shells_found.append(shell)
# Get scripts/ directory.
completions_dir = pathlib.Path(__file__).absolute().parent
scripts_dir = completions_dir.parent
assert completions_dir.name == "completions"
assert scripts_dir.name == "scripts"
# Install mode: Generate completion for each dcargs script.
if mode == "uninstall":
for shell in shells_supported:
# Reset target directory for each shell type.
target_dir = completions_dir / shell
if target_dir.exists():
assert target_dir.is_dir()
shutil.rmtree(target_dir, ignore_errors=True)
CONSOLE.log(f":broom: Deleted existing completion directory: {target_dir}.")
else:
CONSOLE.log(f":heavy_check_mark: No existing completions at: {target_dir}.")
elif mode == "install":
# Find dcargs CLIs.
script_paths = list(filter(_check_dcargs_cli, scripts_dir.glob("**/*.py")))
script_names = tuple(p.name for p in script_paths)
assert len(set(script_names)) == len(script_names)
# Get existing completion files.
existing_completions = set()
for shell in shells_supported:
target_dir = completions_dir / shell
if target_dir.exists():
existing_completions |= set(target_dir.glob("*"))
# Run generation jobs.
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
completion_paths = list(
concurrent_executor.map(
lambda path_and_shell: _generate_completion(path_and_shell[0], path_and_shell[1], completions_dir),
itertools.product(script_paths, shells_found),
)
)
# Delete obsolete completion files.
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
p.absolute() for p in completion_paths
):
if unexpected_path.is_dir():
shutil.rmtree(unexpected_path)
elif unexpected_path.exists():
unexpected_path.unlink()
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
else:
assert_never(mode)
# Install or uninstall from bashrc/zshrc.
for shell in shells_found:
_update_rc(completions_dir, mode, shell)
CONSOLE.print("[bold]All done![/bold]")
if mode == "install":
CONSOLE.print()
CONSOLE.print("Notes:")
CONSOLE.print(
":warning: In bash, completions will trigger via [dim white]./scripts/run_train.py <TAB>[/dim white], but"
" not [dim white]python ./scripts/run_train.py <TAB>[/dim white]."
)
CONSOLE.print(
":warning: Completions are in an experimental state. If you run into any issues, please file an issue!"
)
if __name__ == "__main__":
dcargs.cli(main, description=__doc__)
+17
View File
@@ -0,0 +1,17 @@
# Nerfactory completions for bash.
#
# This should generally be installed automatically by `configure.py`.
completions_dir="$(dirname "$BASH_SOURCE")"/bash
if [ ! -d "${completions_dir}" ]; then
echo "$0: Completions are missing!"
echo "Please generate them with nerfactory/scripts/completions/generate.py!"
return 1
fi
# Source each completion script.
for completion_path in ${completions_dir}/*
do
source $completion_path
done
+29
View File
@@ -0,0 +1,29 @@
# Nerfactory completions for zsh.
#
# This should generally be installed automatically by `configure.py`.
completions_dir="${0:a:h}"/zsh
if [ ! -d "${completions_dir}" ]; then
echo "$0: Completions are missing!"
echo "Please generate them with nerfactory/scripts/completions/generate.py!"
return 1
fi
# Manually load and define each completion.
#
# Adding the completions directory to our fpath and re-initializing would work
# as well:
# fpath+=${completions_dir}
# autoload -Uz compinit; compinit
# But would be several orders of magnitude slower.
for completion_path in ${completions_dir}/*
do
# /some/path/to/_our_completion_py => _our_completion_py
completion_name=${completion_path##*/}
# _our_completion_py => our_completion.py
script_name="${completion_name:1:-3}.py"
autoload -Uz $completion_path
compdef $completion_name $script_name
done
Regular → Executable
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""Simple yaml debugger"""
import subprocess
import sys
Regular → Executable
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""Helper that add tags to notebooks based on cell comments."""
import sys
Regular → Executable
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""Simple yaml debugger"""
import subprocess
import sys
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""Download datasets"""
import zipfile
from pathlib import Path
Regular → Executable
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""Processes a video or image sequence to a nerfactory compatible dataset."""
import json
Regular → Executable
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""
run_eval.py
"""
Regular → Executable
+4 -8
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""
run_train_nerf.py
"""
@@ -18,6 +19,7 @@ import yaml
from rich.console import Console
from nerfactory.configs import base as cfg
from nerfactory.configs.base_configs import AnnotatedBaseConfigUnion
from nerfactory.engine.trainer import train_loop
from nerfactory.utils import comms, profiler
@@ -209,11 +211,5 @@ def main(config: cfg.Config) -> None:
if __name__ == "__main__":
from nerfactory.configs.base_configs import base_configs
# Create a subcommand for each base config.
SubcommandType = dcargs.extras.subcommand_type_from_defaults(base_configs)
instantiated_config = dcargs.cli(SubcommandType)
main(config=instantiated_config)
# Choose a base configuration and override values.
main(dcargs.cli(AnnotatedBaseConfigUnion))
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""View Bridge Server"""
from nerfactory.viewer.server.server import run_viewer_bridge_server
Regular → Executable
+3 -4
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""
view_dataset.py
"""
@@ -11,6 +12,7 @@ import yaml
from rich.console import Console
from nerfactory.configs import base as cfg
from nerfactory.configs.base_configs import AnnotatedBaseConfigUnion
from nerfactory.viewer.server import viewer_utils
logging.basicConfig(format="[%(filename)s:%(lineno)d] %(message)s", level=logging.DEBUG)
@@ -34,10 +36,7 @@ def main(config: cfg.Config) -> None:
if __name__ == "__main__":
console = Console(width=120)
from nerfactory.configs.base_configs import base_configs
SubcommandType = dcargs.extras.subcommand_type_from_defaults(base_configs)
instantiated_config = dcargs.cli(SubcommandType)
instantiated_config = dcargs.cli(AnnotatedBaseConfigUnion)
if instantiated_config.trainer.load_config:
logging.info(f"Loading pre-set config from: {instantiated_config.trainer.load_config}")
instantiated_config = yaml.load(instantiated_config.trainer.load_config.read_text(), Loader=yaml.Loader)