Experimental autocompletion setup scripts (#378)

* Working completion scripts

* generate.py => install.py

* Uninstall for completions, docs/README updates

* Tune emojis, improve concurrency for completion install

* wip

* Cleanup

* Permissions, shebang checks
This commit is contained in:
Brent Yi authored and GitHub committed 2022-09-13 12:44:27 -07:00
1 parent 61022f7dbe
commit 86a805c88c
17 files changed
+386 -32

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
bash/
zsh/
+249
View File
@@ -0,0 +1,249 @@
#!/usr/bin/env python
"""Configuration script for setting up tab completion for nerfactory in bash and zsh."""
import concurrent.futures
import itertools
import os
import pathlib
import random
import shutil
import stat
import subprocess
import sys
from typing import List, Literal
import dcargs
from rich.console import Console
from rich.prompt import Confirm
from typing_extensions import assert_never
ConfigureMode = Literal["install", "uninstall"]
ShellType = Literal["zsh", "bash"]
CONSOLE = Console(width=120)
def _check_dcargs_cli(script_path: pathlib.Path) -> bool:
"""Check if a path points to a script containing a dcargs.cli() call. Also checks
for any permissions/shebang issues.
Args:
script_path: Path to prospective CLI.
Returns:
True if a completion is can be generated.
"""
assert script_path.suffix == ".py"
script_src = script_path.read_text()
if '\nif __name__ == "__main__":\n' in script_src:
# Check script for execute permissions. For consistency, note that we apply this
# and the shebang check even if dcargs isn't used.
execute_flags = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
if not script_path.stat().st_mode & execute_flags and Confirm.ask(
f"[yellow]:warning: {script_path} is not marked as executable. Fix?[/yellow]",
default=True,
):
script_path.chmod(script_path.stat().st_mode | execute_flags)
# Check that script has a shebang.
if not script_src.startswith("#!/") and Confirm.ask(
f"[yellow]:warning: {script_path} is missing a shebang. Fix?[/yellow]",
default=True,
):
script_path.write_text("#!/usr/bin/env python\n" + script_src)
# Return True only if compatible with dcargs.
return "import dcargs" in script_src and "dcargs.cli" in script_src
return False
def _generate_completion(script_path: pathlib.Path, shell: ShellType, completions_dir: pathlib.Path) -> pathlib.Path:
"""Given a path to a dcargs CLI, write a completion script to a target directory.
Args:
script_path: Path to Python CLI to generate completion script for.
shell: Shell to generate completion script for.
completions_dir: Directory to write completion script to.
Returns:
Success flag.
"""
# Use zsh standard for naming completion scripts.
target_path = completions_dir / shell / ("_" + script_path.name.replace(".", "_"))
# Generate and write the new completion.
new = subprocess.run(
args=[sys.executable, str(script_path), "--dcargs-print-completion", shell],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf8",
check=True,
).stdout
target_path.parent.mkdir(parents=True, exist_ok=True)
if not target_path.exists():
target_path.write_text(new)
CONSOLE.log(f":heavy_check_mark: Wrote new completion to {target_path}!")
elif target_path.read_text().strip() != new.strip():
target_path.write_text(new)
CONSOLE.log(f":heavy_check_mark: Updated completion at {target_path}!")
else:
CONSOLE.log(f"[dim]:heavy_check_mark: Nothing to do for {target_path}[/dim].")
return target_path
def _exclamation() -> str:
return random.choice(["Cool", "Nice", "Neat", "Great", "Exciting", "Excellent", "Ok"]) + "!"
def _update_rc(
completions_dir: pathlib.Path,
mode: ConfigureMode,
shell: ShellType,
) -> None:
"""Try to add a `source /.../completions/setup.{shell}` line automatically to a user's zshrc or bashrc.
Args:
completions_dir: Path to location of this script.
shell: Shell to install completion scripts for.
mode: Install or uninstall completions.
"""
# Install or uninstall `source_line`.
source_lines = "\n".join(
[
"",
"# Source nerfactory autocompletions.",
f"source {completions_dir / 'setup'}.{shell}",
]
)
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
if mode == "install":
if source_lines in rc_path.read_text():
CONSOLE.log(f":call_me_hand: Completions are already installed in {rc_path}. {_exclamation()}")
return
if not Confirm.ask(f"[bold yellow]Install to {rc_path}?", default=True):
CONSOLE.log(f"[bold red]Skipping install for {rc_path.name}.")
return
rc_path.write_text(rc_path.read_text() + source_lines)
CONSOLE.log(
f":person_gesturing_ok: Completions installed to {rc_path}. {_exclamation()} Open a new shell to try them"
" out."
)
elif mode == "uninstall":
if source_lines not in rc_path.read_text():
CONSOLE.log(f":heavy_check_mark: No completions to uninstall from {rc_path.name}.")
return
if not Confirm.ask(f"[bold yellow]Uninstall from {rc_path}?", default=True):
CONSOLE.log(f"[dim red]Skipping uninstall for {rc_path.name}.")
return
rc_path.write_text(rc_path.read_text().replace(source_lines, ""))
CONSOLE.log(f":broom: Completions uninstalled from {rc_path}.")
else:
assert_never(mode)
def main(
mode: ConfigureMode,
/,
) -> None:
"""Main script.
Args:
mode: Choose between installing or uninstalling completions.
shells: Shell(s) to install or uninstall.
"""
if "HOME" not in os.environ:
CONSOLE.log("[bold red]$HOME is not set. Exiting.")
return
# Try to locate the user's bashrc or zshrc.
shells_supported: List[ShellType] = ["zsh", "bash"]
shells_found: List[ShellType] = []
for shell in shells_supported:
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
if not rc_path.exists():
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
else:
CONSOLE.log(f":mag: Found {rc_path.name}!")
shells_found.append(shell)
# Get scripts/ directory.
completions_dir = pathlib.Path(__file__).absolute().parent
scripts_dir = completions_dir.parent
assert completions_dir.name == "completions"
assert scripts_dir.name == "scripts"
# Install mode: Generate completion for each dcargs script.
if mode == "uninstall":
for shell in shells_supported:
# Reset target directory for each shell type.
target_dir = completions_dir / shell
if target_dir.exists():
assert target_dir.is_dir()
shutil.rmtree(target_dir, ignore_errors=True)
CONSOLE.log(f":broom: Deleted existing completion directory: {target_dir}.")
else:
CONSOLE.log(f":heavy_check_mark: No existing completions at: {target_dir}.")
elif mode == "install":
# Find dcargs CLIs.
script_paths = list(filter(_check_dcargs_cli, scripts_dir.glob("**/*.py")))
script_names = tuple(p.name for p in script_paths)
assert len(set(script_names)) == len(script_names)
# Get existing completion files.
existing_completions = set()
for shell in shells_supported:
target_dir = completions_dir / shell
if target_dir.exists():
existing_completions |= set(target_dir.glob("*"))
# Run generation jobs.
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
completion_paths = list(
concurrent_executor.map(
lambda path_and_shell: _generate_completion(path_and_shell[0], path_and_shell[1], completions_dir),
itertools.product(script_paths, shells_found),
)
)
# Delete obsolete completion files.
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
p.absolute() for p in completion_paths
):
if unexpected_path.is_dir():
shutil.rmtree(unexpected_path)
elif unexpected_path.exists():
unexpected_path.unlink()
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
else:
assert_never(mode)
# Install or uninstall from bashrc/zshrc.
for shell in shells_found:
_update_rc(completions_dir, mode, shell)
CONSOLE.print("[bold]All done![/bold]")
if mode == "install":
CONSOLE.print()
CONSOLE.print("Notes:")
CONSOLE.print(
":warning: In bash, completions will trigger via [dim white]./scripts/run_train.py <TAB>[/dim white], but"
" not [dim white]python ./scripts/run_train.py <TAB>[/dim white]."
)
CONSOLE.print(
":warning: Completions are in an experimental state. If you run into any issues, please file an issue!"
)
if __name__ == "__main__":
dcargs.cli(main, description=__doc__)
+17
View File
@@ -0,0 +1,17 @@
# Nerfactory completions for bash.
#
# This should generally be installed automatically by `configure.py`.
completions_dir="$(dirname "$BASH_SOURCE")"/bash
if [ ! -d "${completions_dir}" ]; then
echo "$0: Completions are missing!"
echo "Please generate them with nerfactory/scripts/completions/generate.py!"
return 1
fi
# Source each completion script.
for completion_path in ${completions_dir}/*
do
source $completion_path
done
+29
View File
@@ -0,0 +1,29 @@
# Nerfactory completions for zsh.
#
# This should generally be installed automatically by `configure.py`.
completions_dir="${0:a:h}"/zsh
if [ ! -d "${completions_dir}" ]; then
echo "$0: Completions are missing!"
echo "Please generate them with nerfactory/scripts/completions/generate.py!"
return 1
fi
# Manually load and define each completion.
#
# Adding the completions directory to our fpath and re-initializing would work
# as well:
# fpath+=${completions_dir}
# autoload -Uz compinit; compinit
# But would be several orders of magnitude slower.
for completion_path in ${completions_dir}/*
do
# /some/path/to/_our_completion_py => _our_completion_py
completion_name=${completion_path##*/}
# _our_completion_py => our_completion.py
script_name="${completion_name:1:-3}.py"
autoload -Uz $completion_path
compdef $completion_name $script_name
done
Regular → Executable
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""Simple yaml debugger"""
import subprocess
import sys
Regular → Executable
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""Helper that add tags to notebooks based on cell comments."""
import sys
Regular → Executable
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""Simple yaml debugger"""
import subprocess
import sys
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""Download datasets"""
import zipfile
from pathlib import Path
Regular → Executable
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""Processes a video or image sequence to a nerfactory compatible dataset."""
import json
Regular → Executable
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""
run_eval.py
"""
Regular → Executable
+4 -8
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""
run_train_nerf.py
"""
@@ -18,6 +19,7 @@ import yaml
from rich.console import Console
from nerfactory.configs import base as cfg
from nerfactory.configs.base_configs import AnnotatedBaseConfigUnion
from nerfactory.engine.trainer import train_loop
from nerfactory.utils import comms, profiler
@@ -209,11 +211,5 @@ def main(config: cfg.Config) -> None:
if __name__ == "__main__":
from nerfactory.configs.base_configs import base_configs
# Create a subcommand for each base config.
SubcommandType = dcargs.extras.subcommand_type_from_defaults(base_configs)
instantiated_config = dcargs.cli(SubcommandType)
main(config=instantiated_config)
# Choose a base configuration and override values.
main(dcargs.cli(AnnotatedBaseConfigUnion))
+1
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""View Bridge Server"""
from nerfactory.viewer.server.server import run_viewer_bridge_server
Regular → Executable
+3 -4
View File
@@ -1,3 +1,4 @@
#!/usr/bin/env python
"""
view_dataset.py
"""
@@ -11,6 +12,7 @@ import yaml
from rich.console import Console
from nerfactory.configs import base as cfg
from nerfactory.configs.base_configs import AnnotatedBaseConfigUnion
from nerfactory.viewer.server import viewer_utils
logging.basicConfig(format="[%(filename)s:%(lineno)d] %(message)s", level=logging.DEBUG)
@@ -34,10 +36,7 @@ def main(config: cfg.Config) -> None:
if __name__ == "__main__":
console = Console(width=120)
from nerfactory.configs.base_configs import base_configs
SubcommandType = dcargs.extras.subcommand_type_from_defaults(base_configs)
instantiated_config = dcargs.cli(SubcommandType)
instantiated_config = dcargs.cli(AnnotatedBaseConfigUnion)
if instantiated_config.trainer.load_config:
logging.info(f"Loading pre-set config from: {instantiated_config.trainer.load_config}")
instantiated_config = yaml.load(instantiated_config.trainer.load_config.read_text(), Loader=yaml.Loader)