From 416669d8083024b8f8b86b8af4dc1ff21a200cca Mon Sep 17 00:00:00 2001 From: evonneng Date: Mon, 12 Sep 2022 16:11:14 -0700 Subject: [PATCH] added basic download script (#410) * basic script to download dataset * update docs --- .gitignore | 3 +- README.md | 8 +++- docs/tutorials/quickstart.md | 8 +++- pyproject.toml | 1 + scripts/downloads/download_data.py | 77 ++++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 scripts/downloads/download_data.py diff --git a/.gitignore b/.gitignore index 31dafbea1..f3126029d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ build/ develop-eggs/ dist/ downloads/ +!scripts/downloads/ eggs/ .eggs/ lib/ @@ -185,4 +186,4 @@ node_modules/ scripts/colmap2nerf.py bash/ nerfactory/ngp_pl/ -cache/ \ No newline at end of file +cache/ diff --git a/README.md b/README.md index 32e14349d..f13931bb7 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,13 @@ pytest tests #### 2. Getting the data -Download the original [NeRF dataset](https://drive.google.com/drive/folders/128yBriW1IG_3NJ5Rp7APSTZsJqdJdfc1) and unfold it in the following format. This is for the blender dataset type. We support the major datasets and allow users to create their own dataset, described in detail [here](https://plenoptix-nerfactory.readthedocs-hosted.com/en/latest/tutorials/data/index.html). +Download the original NeRF Blender dataset. We support the major datasets and allow users to create their own dataset, described in detail [here](https://plenoptix-nerfactory.readthedocs-hosted.com/en/latest/tutorials/data/index.html). + +``` +python scripts/downloads/download_data.py --dataset=blender +``` + +Use `--help` to view all currently available datasets. The resulting script should download and unpack the dataset as follows: ``` |─ nerfactory/ diff --git a/docs/tutorials/quickstart.md b/docs/tutorials/quickstart.md index 5f60bc2f2..69b9fc18b 100644 --- a/docs/tutorials/quickstart.md +++ b/docs/tutorials/quickstart.md @@ -48,7 +48,13 @@ pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/to # Downloading data -Download the original [NeRF dataset](https://drive.google.com/drive/folders/128yBriW1IG_3NJ5Rp7APSTZsJqdJdfc1) and unfold it in the following format. This is for the blender dataset type. We support the major datasets and allow users to create their own dataset, described in detail [here](data/creating_dataset.md). +Download the original NeRF Blender dataset. We support the major datasets and allow users to create their own dataset, described in detail [here](https://plenoptix-nerfactory.readthedocs-hosted.com/en/latest/tutorials/data/index.html). + +``` +python scripts/downloads/download_data.py --dataset=blender +``` + +Use `--help` to view all currently available datasets. The resulting script should download and unpack the dataset as follows: ``` |─ nerfactory/ diff --git a/pyproject.toml b/pyproject.toml index 9e88b30b1..26ea8dd3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "av==9.2.0", "black[jupyter]==22.3.0", "dcargs>=0.3.1", + "gdown==4.5.1", "ninja==1.10.2.3", "functorch==0.2.1", "h5py>=2.9.0", diff --git a/scripts/downloads/download_data.py b/scripts/downloads/download_data.py new file mode 100644 index 000000000..918d9521b --- /dev/null +++ b/scripts/downloads/download_data.py @@ -0,0 +1,77 @@ +"""Download datasets""" +import zipfile +from pathlib import Path +from typing import Literal + +import dcargs +import gdown + + +def download_friends(): + """Script ported from: https://github.com/ethanweber/sitcoms3D/blob/master/download_data.py""" + data_filename_to_file_id = [ + ("./data/human_data.zip", "1zk12qxnZbReKqdvy-JYp1jQn2QA3oJzz"), + ("./data/human_pairs.zip", "1927rnmJ3mWcjsj0afqawSto2IK2lLwmA"), + ("./data/sparse_reconstruction_and_nerf_data.zip", "1RmwDUAp1T4RkwZg1S_7L9JAViWtBZfAG"), + ] + + disk_data_filename_to_file_id = [ # pylint: disable=unused-variable + ("./data/sparse_reconstruction_and_nerf_data/ELR-apartment-disk.zip", "1YcTwMv5PP0uqXYdN-Lp8k0WnXUdHBXrY"), + ("./data/sparse_reconstruction_and_nerf_data/Frasier-apartment-disk.zip", "1vPqplDv5rrFu5LrxTHbET0FiePTGi8zE"), + ( + "./data/sparse_reconstruction_and_nerf_data/Friends-monica_apartment-disk.zip", + "1yNpU4M44gIuWEaC5tSAOKy-A6KX8wTm8", + ), + ( + "./data/sparse_reconstruction_and_nerf_data/HIMYM-red_apartment-disk.zip", + "1m76i46YYlpDKvlx-Jn7qDDAvkOVVVmqF", + ), + ( + "./data/sparse_reconstruction_and_nerf_data/Seinfeld-jerry_living_room-disk.zip", + "1YFoV3Gd9asKsRarZvzRTYtri1755rBPx", + ), + ("./data/sparse_reconstruction_and_nerf_data/TAAHM-kitchen-disk.zip", "1QpF8kVGfgkqm_cDY5sY_E82BT8VOWL38"), + ( + "./data/sparse_reconstruction_and_nerf_data/TBBT-big_living_room-disk.zip", + "175BjkpxMAOt75ZVIjYpArxWpO8uRexgJ", + ), + ] + + # !!! Large files !!! Only uncomment if you need the disk correspondences. + # data_filename_to_file_id += disk_data_filename_to_file_id + + for data_filename, file_id in data_filename_to_file_id: + # Download the files + url = f"https://drive.google.com/uc?id={file_id}" + gdown.download(url, output=data_filename, quiet=False) + with zipfile.ZipFile(data_filename, "r") as zip_ref: + zip_ref.extractall(data_filename.replace(".zip", "").replace("-disk", "")) + Path(data_filename).unlink(missing_ok=True) + + +def download_blender(): + """Download blender dataset and format accordingly""" + url = "https://drive.google.com/uc?id=18JxhpWD-4ZmuFKLzKlAw-w5PpzZxXOcG" + download_path = Path("data/blender_data.zip") + gdown.download(url, output=str(download_path), quiet=False) + with zipfile.ZipFile(download_path, "r") as zip_ref: + zip_ref.extractall("data/") + unzip_path = Path("data/nerf_synthetic") + final_path = Path("data/blender") + unzip_path.rename(final_path) + download_path.unlink(missing_ok=True) + + +def main(dataset: Literal["blender", "friends"]): + """Main download script to download all data""" + save_dir = Path("data/") + save_dir.mkdir(parents=True, exist_ok=True) + + if dataset == "blender": + download_blender() + if dataset == "friends": + download_friends() + + +if __name__ == "__main__": + dcargs.cli(main)