SG-Nav
[NeurIPS 2024] SG-Nav: Online 3D Scene Graph Prompting for LLM-based Zero-shot Object Navigation
File Explorer
Download Latest Version (.zip)- demo1.gif
- demo2.gif
- pipeline.png
- challenge_objectnav2021.local.rgbd.yaml
- challenge_objectnav2022.local.rgbd.yaml
- challenge_pointnav2021.local.rgbd.yaml
- challenge_pointnav2021.local.rgbd_test_scene.yaml
- ddppo_objectnav.yaml
- ddppo_pointnav.yaml
- ddppo_pointnav_.yaml
- test.yaml
- val.yaml
- minival.yaml
- val.yaml
- Aquarium_Aquarium_Combined.v2-raw-1024.coco.yaml
- glip_Swin_L.yaml
- glip_Swin_T_O365_GoldG.yaml
- main_model.pdf
- main_model.png
- __init__.py
- defaults.py
- paths_catalog.py
- nms_cpu.cpp
- ROIAlign_cpu.cpp
- soft_nms.cpp
- vision.h
- deform_conv_cuda.cu
- deform_conv_kernel_cuda.cu
- deform_pool_cuda.cu
- deform_pool_kernel_cuda.cu
- ml_nms.cu
- nms.cu
- ROIAlign_cuda.cu
- ROIPool_cuda.cu
- SigmoidFocalLoss_cuda.cu
- vision.h
- deform_conv.h
- deform_pool.h
- ml_nms.h
- nms.h
- ROIAlign.h
- ROIPool.h
- SigmoidFocalLoss.h
- vision.cpp
- __init__.py
- coco_eval.py
- __init__.py
- flickr_eval.py
- _change_lvis_annotation.py
- lvis.py
- lvis_eval.py
- __init__.py
- od_eval.py
- __init__.py
- vg_eval.py
- __init__.py
- voc_eval.py
- __init__.py
- box_aug.py
- od_eval.py
- __init__.py
- background.py
- box_label_loader.py
- caption.py
- coco.py
- coco_dt.py
- concat_dataset.py
- custom_distributed_sampler.py
- duplicate_dataset.py
- flickr.py
- gqa.py
- imagenet.py
- list_dataset.py
- lvis.py
- mixed.py
- mixup.py
- modulated_coco.py
- object365.py
- od_to_grounding.py
- phrasecut.py
- pseudo_data.py
- refexp.py
- tsv.py
- vg.py
- voc.py
- __init__.py
- distributed.py
- grouped_batch_sampler.py
- iteration_based_batch_sampler.py
- __init__.py
- build.py
- transforms.py
- __init__.py
- build.py
- collate_batch.py
- __init__.py
- alter_trainer.py
- evolution.py
- inference.py
- predictor.py
- predictor_glip.py
- singlepath_trainer.py
- stage_trainer.py
- trainer.py
- __init__.py
- batch_norm.py
- deform_conv.py
- deform_pool.py
- dropblock.py
- dyhead.py
- dyrelu.py
- evonorm.py
- iou_loss.py
- misc.py
- nms.py
- roi_align.py
- roi_pool.py
- se.py
- set_loss.py
- sigmoid_focal_loss.py
- smooth_l1_loss.py
- __init__.py
- bifpn.py
- blocks.py
- efficientdet.py
- efficientnet.py
- fbnet.py
- fpn.py
- mixer.py
- ops.py
- resnet.py
- swint.py
- swint_v2.py
- swint_v2_vl.py
- swint_vl.py
- __init__.py
- generalized_rcnn.py
- generalized_vl_rcnn.py
- __init__.py
- backbone.py
- bert_model.py
- bpe_simple_vocab_16e6.txt.gz
- build.py
- clip_model.py
- hfpt_tokenizer.py
- rnn_model.py
- simple_tokenizer.py
- test_clip_tokenizer.py
- word_utils.py
- __init__.py
- box_head.py
- inference.py
- loss.py
- roi_box_feature_extractors.py
- roi_box_predictors.py
- inference.py
- keypoint_head.py
- loss.py
- roi_keypoint_feature_extractors.py
- roi_keypoint_predictors.py
- __init__.py
- hourglass.py
- inference.py
- loss.py
- mask_head.py
- roi_mask_feature_extractors.py
- roi_mask_predictors.py
- __init__.py
- __init__.py
- anchor_generator.py
- atss.py
- dyhead.py
- fcos.py
- inference.py
- loss.py
- modeling_bert.py
- retina.py
- rpn.py
- transformer.py
- vldyhead.py
- .DS_Store
- __init__.py
- balanced_positive_negative_sampler.py
- box_coder.py
- make_layers.py
- matcher.py
- poolers.py
- registry.py
- utils.py
- __init__.py
- build.py
- lr_scheduler.py
- __init__.py
- bounding_box.py
- boxlist_ops.py
- image_list.py
- keypoint.py
- segmentation_mask.py
- __init__.py
- amp.py
- big_model_loading.py
- c2_model_loading.py
- checkpoint.py
- collect_env.py
- comm.py
- cv2_util.py
- dist.py
- ema.py
- env.py
- flops.py
- fuse_helper.py
- imports.py
- logger.py
- mdetr_dist.py
- metric_logger.py
- miscellaneous.py
- model_serialization.py
- model_zoo.py
- pretrain_model_loading.py
- README.md
- registry.py
- shallow_contrastive_loss_helper.py
- stats.py
- __init__.py
- _C.cpython-37m-x86_64-linux-gnu.so
- _C.cpython-39-x86_64-linux-gnu.so
- convert_cityscapes_to_coco.py
- instances2dict_with_polygons.py
- eval_all.py
- finetune.py
- test_grounding_net.py
- test_net.py
- train_net.py
- CODE_OF_CONDUCT.md
- DATA.md
- LICENSE
- README.md
- SECURITY.md
- setup.py
- SUPPORT.md
- arch.png
- cats.png
- COCO.png
- GD_GLIGEN.png
- GD_SD.png
- hero_figure.png
- ODinW.png
- gradio_app.py
- inference_on_a_image.py
- GroundingDINO_SwinB.py
- GroundingDINO_SwinT_OGC.py
- __init__.py
- transforms.py
- __init__.py
- backbone.py
- position_encoding.py
- swin_transformer.py
- ms_deform_attn.h
- ms_deform_attn_cpu.cpp
- ms_deform_attn_cpu.h
- ms_deform_attn_cuda.cu
- ms_deform_attn_cuda.h
- ms_deform_im2col_cuda.cuh
- cuda_version.cu
- vision.cpp
- __init__.py
- bertwarper.py
- fuse_modules.py
- groundingdino.py
- ms_deform_attn.py
- transformer.py
- transformer_vanilla.py
- utils.py
- __init__.py
- registry.py
- __init__.py
- box_ops.py
- get_tokenlizer.py
- inference.py
- logger.py
- misc.py
- slconfig.py
- slio.py
- time_counter.py
- utils.py
- visualizer.py
- vl_utils.py
- __init__.py
- version.py
- LICENSE
- pyproject.toml
- README.md
- requirements.txt
- setup.py
- config.yml
- bug-report.md
- feature-request.md
- questions-help-support.md
- PULL_REQUEST_TEMPLATE.md
- ppo.yaml
- mp3d.yaml
- gibson.yaml
- mp3d.yaml
- hm3d.yaml
- mp3d.yaml
- gibson.yaml
- gibson_0_plus.yaml
- gibson_v2.yaml
- habitat_test.yaml
- hm3d.yaml
- mp3d.yaml
- replica_cad.yaml
- mp3d_r2r.yaml
- single_episode.yaml
- pick.yaml
- pick_spa.yaml
- pick_state.yaml
- play.yaml
- eqa_mp3d.yaml
- imagenav.yaml
- imagenav_gibson.yaml
- objectnav_hm3d.yaml
- objectnav_mp3d.yaml
- pointnav.yaml
- pointnav_gibson.yaml
- pointnav_hm3d.yaml
- pointnav_mp3d.yaml
- pointnav_rgbd.yaml
- vln_r2r.yaml
- habitat_all_sensors_test.yaml
- habitat_mp3d_eqa_test.yaml
- habitat_mp3d_object_nav_test.yaml
- habitat_r2r_vln_test.yaml
- new_keys_test.yaml
- habitat-lab-demo.png
- habitat-lab-demo.rst
- habitat-sim-demo.png
- habitat-sim-demo.rst
- index.rst
- quickstart.png
- quickstart.rst
- view-transform-warp.png
- view-transform-warp.rst
- .gitignore
- build-public.sh
- build.sh
- conf-public.py
- conf.py
- docs.rst
- Habitat_Lab.ipynb
- Habitat_Lab.py
- __init__.py
- benchmark.py
- example.py
- example_pointnav.py
- interactive_play.py
- new_actions.py
- register_new_sensors_and_measures.py
- shortest_path_follower_example.py
- visualization_examples.py
- vln_benchmark.py
- vln_reference_path_follower_example.py
- __init__.py
- default.py
- __init__.py
- agent.py
- benchmark.py
- challenge.py
- dataset.py
- embodied_task.py
- env.py
- environments.py
- logging.py
- registry.py
- simulator.py
- spaces.py
- utils.py
- vector_env.py
- __init__.py
- mp3d_eqa_dataset.py
- __init__.py
- object_nav_dataset.py
- __init__.py
- pointnav_dataset.py
- pointnav_generator.py
- counter.yaml
- test_config.yaml
- __init__.py
- generate_episode_inits.py
- rearrange_dataset.py
- rearrange_generator.py
- receptacle.py
- samplers.py
- __init__.py
- r2r_vln_dataset.py
- __init__.py
- registration.py
- utils.py
- __init__.py
- actions.py
- debug_visualizer.py
- habitat_simulator.py
- sim_utilities.py
- __init__.py
- pyrobot.py
- __init__.py
- registration.py
- __init__.py
- eqa.py
- __init__.py
- nav.py
- object_nav_task.py
- shortest_path_follower.py
- pick_sensors.py
- pick_task.py
- __init__.py
- actions.py
- grip_actions.py
- marker_info.py
- policy_modules.py
- rearrange_grasp_manager.py
- rearrange_sensors.py
- rearrange_sim.py
- rearrange_task.py
- utils.py
- __init__.py
- vln.py
- __init__.py
- registration.py
- utils.py
- 100x100.png
- __init__.py
- fog_of_war.py
- maps.py
- utils.py
- __init__.py
- common.py
- geometry_utils.py
- pickle5_multiprocessing.py
- profiling_wrapper.py
- test_utils.py
- __init__.py
- py.typed
- version.py
- __init__.py
- benchmark_gym.py
- mp_agents.py
- ppo_agents.py
- simple_agents.py
- slam_agents.py
- __init__.py
- base_il_trainer.py
- base_trainer.py
- baseline_registry.py
- environments.py
- obs_transformers.py
- rollout_storage.py
- tensor_dict.py
- tensorboard_utils.py
- il_eqa_cnn_pretrain.yaml
- il_pacman_nav.yaml
- il_vqa.yaml
- ddppo_imagenav_example.yaml
- ddppo_imagenav_gibson.yaml
- ppo_imagenav_example.yaml
- ddppo_objectnav.yaml
- ddppo_objectnav_hm3d.yaml
- ddppo_pointnav.yaml
- ppo_pointnav.yaml
- ppo_pointnav_example.yaml
- ppo_pointnav_habitat_iccv19.yaml
- rl_pick.yaml
- rl_pick_state.yaml
- spap_pick.yaml
- ddppo_imagenav_test.yaml
- ddppo_pointnav_test.yaml
- ppo_imagenav_test.yaml
- ppo_pointnav_test.yaml
- __init__.py
- default.py
- __init__.py
- models.py
- __init__.py
- eqa_cnn_pretrain_trainer.py
- pacman_trainer.py
- vqa_trainer.py
- __init__.py
- metrics.py
- README.md
- requirements.txt
- __init__.py
- grasp_generator.py
- motion_plan.py
- mp_sim.py
- mp_spaces.py
- robot_target.py
- __init__.py
- ddppo.py
- create_gibson_large_dataset.py
- gibson_dset_with_qual.json
- __init__.py
- resnet.py
- resnet_policy.py
- running_mean_and_var.py
- __init__.py
- ddp_utils.py
- multi_node_slurm.sh
- README.md
- requirements.txt
- single_node.sh
- __init__.py
- rnn_state_encoder.py
- simple_cnn.py
- __init__.py
- policy.py
- ppo.py
- ppo_trainer.py
- __init__.py
- requirements.txt
- __init__.py
- install_deps.sh
- mappers.py
- monodepth.py
- path_planners.py
- README.md
- reprojection.py
- requirements.txt
- utils.py
- __init__.py
- utils.py
- __init__.py
- common.py
- env_utils.py
- gym_adapter.py
- gym_definitions.py
- render_wrapper.py
- __init__.py
- py.typed
- README.md
- run.py
- generate_profile_shell_scripts.py
- habitat-sim_trajectory_data.json
- test-sim-geodesic-distance-test-golden.json
- test_baseline_agents.py
- test_baseline_trainers.py
- test_config.py
- test_dataset.py
- test_ddppo_reduce.py
- test_demo_notebook.py
- test_examples.py
- test_gym_wrapper.py
- test_habitat_env.py
- test_habitat_example.py
- test_habitat_sim.py
- test_habitat_task.py
- test_install.py
- test_motion_plan.py
- test_mp3d_eqa.py
- test_object_nav_task.py
- test_pointnav_dataset.py
- test_pyrobot.py
- test_r2r_vln.py
- test_rearrange_task.py
- test_rnn_state_encoder.py
- test_sensors.py
- test_spaces.py
- test_tensor_dict.py
- test_visual_utils.py
- .editorconfig
- .gitignore
- .pre-commit-config.yaml
- CODE_OF_CONDUCT.md
- CONTRIBUTING.md
- Dockerfile
- LICENSE
- MANIFEST.in
- mypy.ini
- pyproject.toml
- README.md
- requirements.txt
- setup.cfg
- setup.py
- masks1.png
- masks2.jpg
- model_diagram.png
- notebook1.png
- notebook2.png
- dog.jpg
- groceries.jpg
- truck.jpg
- automatic_mask_generator_example.ipynb
- onnx_model_example.ipynb
- predictor_example.ipynb
- amg.py
- export_onnx_model.py
- __init__.py
- common.py
- image_encoder.py
- mask_decoder.py
- mask_decoder_hq.py
- prompt_encoder.py
- sam.py
- transformer.py
- __init__.py
- amg.py
- onnx.py
- transforms.py
- __init__.py
- automatic_mask_generator.py
- build_sam.py
- build_sam_hq.py
- predictor.py
- .flake8
- CODE_OF_CONDUCT.md
- CONTRIBUTING.md
- LICENSE
- linter.sh
- README.md
- setup.cfg
- setup.py
- agent.py
- arial.ttf
- download_mp.py
- matterport_category_mappings.tsv
- obj.npy
- replica.yaml
- room.npy
- val.json.gz
- __init__.py
- control_helper.py
- depth_utils.py
- fmm_planner.py
- mapping.py
- model.py
- pose_utils.py
- rotation_utils.py
- __init__.py
- grounded_sam_demo.py
- iou.py
- mapping.py
- slam_classes.py
- utils.py
- __init__.py
- image_process.py
- utils_glip.py
- .gitignore
- LICENSE
- README.md
- requirements.txt
- scenegraph.py
- SG_Nav.py
# Installation Guide
1. Get the code
git clone https://github.com/bagh2178/SG-Nav
Downloads the entire project code from GitHub to your computer.
cd SG-Nav
Moves into the project folder you just downloaded.
2. Official Install Script
Easy RecommendedPrerequisites
- Python 3 Python is required to use pip.
pip install torch==1.9.1+cu111 torchvision==0.10.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html
Installs the package published on PyPI directly โ no need to clone the source.
pip install "git+https://github.com/facebookresearch/pytorch3d.git"
Installs the package published on PyPI directly โ no need to clone the source.
pip install --no-build-isolation -e GroundingDINO
Installs the package published on PyPI directly โ no need to clone the source.
curl -fsSL https://ollama.com/install.sh | sh
Downloads and runs the official install script in one line โ this handles the full setup automatically.
After installing, open a new terminal and run the program's version command (e.g. --version) to confirm it worked.
Pulled directly from this repo's README.
3. Docker
EasyPrerequisites
- Git Needed to download the project code from GitHub.
- Docker Desktop Needed to build and run containers. Install it and keep it running in the background.
docker build -f habitat-lab/Dockerfile -t sg-nav .
Builds a runnable image based on the Dockerfile.
docker run -p 8080:80 sg-nav
Runs the built image as an actual container.
Run docker compose ps to check the containers are Up. If the README mentions a port, open http://localhost:PORT in your browser.
4. Python
EasyPrerequisites
pip install -e habitat-lab
Installs the Python libraries listed in requirements.txt (or similar).
HABITAT_SIM_PATH=$(pip show habitat_sim | grep 'Location:' | awk '{print $2}')
Type this command into your terminal and run it.
pip install torch==1.9.1+cu111 torchvision==0.10.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html
Installs the package published on PyPI directly โ no need to clone the source.
pip install -r requirements.txt
Installs the Python libraries listed in requirements.txt (or similar).
pip install "git+https://github.com/facebookresearch/pytorch3d.git"
Installs the package published on PyPI directly โ no need to clone the source.
If it runs without errors and prints output in the terminal, it worked.
Pulled directly from this repo's README.
// repository documentation
Was this content helpful?
(0 ratings)