SoFar
[NeurIPS 2025 Spotlight] SoFar: Language-Grounded Orientation Bridges Spatial Reasoning and Object Manipulation
File Explorer
Download Latest Version (.zip)- .DS_Store
- box.jpg
- data_construction.jpg
- drawer.npy
- drawer.png
- navigation.npy
- navigation.png
- open6dor.npy
- open6dor.png
- orientext300k.jpg
- simpler_env.npy
- simpler_env.png
- table.jpg
- teaser.jpg
- spatial_data.json
- DATASET.md
- orentext300k_render_views.py
- __init__.py
- attention.py
- block.py
- drop_path.py
- layer_scale.py
- mlp.py
- patch_embed.py
- swiglu_ffn.py
- blocks.py
- transform.py
- __init__.py
- dinov2.py
- dpt.py
- __init__.py
- _data_base_.py
- __init__.py
- convnext_large.py
- dino_vit_large_reg.py
- dino_vit_small_reg.py
- __init__.py
- convnext_large.hourglassdecoder.py
- dino_vit_large_reg.dpt_raft.py
- dino_vit_small_reg.dpt_raft.py
- __init__.py
- __init__.py
- _data_base_.py
- default_runtime.py
- __init__.py
- convlarge.0.3_150.py
- test_kitti_convlarge.0.3_150.py
- test_nyu_convlarge.0.3_150.py
- vit.raft5.large.py
- vit.raft5.small.py
- __init__.py
- __init__.py
- ConvNeXt.py
- ViT_DINO.py
- ViT_DINO_reg.py
- __init__.py
- HourGlassDecoder.py
- RAFTDepthNormalDPTDecoder5.py
- __base_model__.py
- __init__.py
- dense_pipeline.py
- __init__.py
- monodepth_model.py
- __init__.py
- test_scale_cano.py
- __init__.py
- avg_meter.py
- comm.py
- custom_data.py
- do_test.py
- logger.py
- mldb.py
- running.py
- transform.py
- unproj_pcd.py
- visualization.py
- __init__.py
- __init__.py
- backbones.py
- utils.py
- __init__.py
- attention.py
- block.py
- dino_head.py
- drop_path.py
- layer_scale.py
- mlp.py
- patch_embed.py
- swiglu_ffn.py
- __init__.py
- vision_transformer.py
- __init__.py
- cluster.py
- config.py
- dtype.py
- param_groups.py
- utils.py
- __init__.py
- __init__.py
- moge_model.py
- utils.py
- __init__.py
- download.py
- geometry_numpy.py
- geometry_torch.py
- io.py
- pipeline.py
- tools.py
- vis.py
- webfile.py
- webzipfile.py
- __init__.py
- __init__.pyi
- __init__.py
- colmap.py
- obj.py
- ply.py
- texture.fsh
- texture.vsh
- vertex_attribute.fsh
- vertex_attribute.vsh
- __init__.py
- _helpers.py
- mesh.py
- quadmesh.py
- rasterization.py
- spline.py
- transforms.py
- utils.py
- __init__.py
- _helpers.py
- mesh.py
- nerf.py
- rasterization.py
- transforms.py
- utils.py
- __init__.py
- _helpers.py
- README.md
- __init__.py
- __init__.py
- depthanythingv2.py
- metric3dv2.py
- moge.py
- utils.py
- __init__.py
- eval_open6dor.py
- evaluator.py
- open6dor_perception.py
- open6dor_perception_llava.py
- utils.py
- SD_FT.yaml
- SO.yaml
- base.yaml
- base_ft.yaml
- large.yaml
- small.yaml
- __init__.py
- bpe_simple_vocab_16e6.txt.gz
- clip.py
- model.py
- simple_tokenizer.py
- __init__.py
- build.py
- SemanticOrientation.py
- utils.py
- __init__.py
- build.py
- CrossModal.py
- PointSO.py
- transformer.py
- __init__.py
- builder.py
- runner.py
- AverageMeter.py
- checkpoint.py
- config.py
- dist_utils.py
- knn.py
- logger.py
- misc.py
- parser.py
- registry.py
- __init__.py
- LICENSE
- test.sh
- train.py
- train.sh
- train_ddp.sh
- train_ddp_ft.sh
- get_mask.py
- gradio_demo.py
- long_horizon_planning.py
- manipulation_demo.py
- navigation_demo.py
- open6dor_demo.py
- qwen_demo.py
- video_predict_orientation.py
- vqa_demo.py
- arch.png
- cats.png
- COCO.png
- GD_GLIGEN.png
- GD_SD.png
- hero_figure.png
- ODinW.png
- gradio_app.py
- inference_on_a_image.py
- __init__.py
- GroundingDINO_SwinB.py
- GroundingDINO_SwinT_OGC.py
- __init__.py
- transforms.py
- __init__.py
- backbone.py
- position_encoding.py
- swin_transformer.py
- ms_deform_attn.h
- ms_deform_attn_cpu.cpp
- ms_deform_attn_cpu.h
- ms_deform_attn_cuda.cu
- ms_deform_attn_cuda.h
- ms_deform_im2col_cuda.cuh
- cuda_version.cu
- vision.cpp
- __init__.py
- bertwarper.py
- fuse_modules.py
- groundingdino.py
- ms_deform_attn.py
- transformer.py
- transformer_vanilla.py
- utils.py
- __init__.py
- registry.py
- __init__.py
- box_ops.py
- get_tokenlizer.py
- inference.py
- logger.py
- misc.py
- slconfig.py
- slio.py
- time_counter.py
- utils.py
- visualizer.py
- vl_utils.py
- __init__.py
- version.py
- __init__.py
- LICENSE
- pyproject.toml
- README.md
- requirements.txt
- setup.py
- __init__.py
- amg.py
- export_onnx_model.py
- __init__.py
- common.py
- image_encoder.py
- mask_decoder.py
- mask_decoder_hq.py
- prompt_encoder.py
- sam.py
- transformer.py
- __init__.py
- amg.py
- onnx.py
- transforms.py
- __init__.py
- automatic_mask_generator.py
- build_sam.py
- build_sam_hq.py
- predictor.py
- .flake8
- __init__.py
- CODE_OF_CONDUCT.md
- CONTRIBUTING.md
- LICENSE
- linter.sh
- README.md
- setup.cfg
- setup.py
- .DS_Store
- __init__.py
- auto_sam.py
- florence.py
- grounding_dino.py
- sam.py
- yoloworld.py
- chatgpt.py
- pointso.py
- qwen_inference.py
- scene_graph.py
- sofar_llava_inference.py
- system_prompts.py
- utils.py
- visualization.py
- llava_llama.py
- llava_mistral.py
- llava_mpt.py
- builder.py
- clip_encoder.py
- builder.py
- __init__.py
- apply_delta.py
- builder.py
- consolidate.py
- llava_arch.py
- make_delta.py
- utils.py
- __init__.py
- llama_flash_attn_monkey_patch.py
- llama_xformers_attn_monkey_patch.py
- llava_trainer.py
- train.py
- train_mem.py
- train_xformers.py
- __init__.py
- constants.py
- conversation.py
- mm_utils.py
- utils.py
- coco2014_val_gpt4_qa_30x3.jsonl
- extract_mm_projector.py
- finetune_lora_obj_parsing.sh
- finetune_lora_vlm.sh
- finetune_ori.sh
- merge_lora_weights.py
- pretrain_ori.sh
- zero2.json
- zero3.json
- zero3_offload.json
- __init__.py
- cog.yaml
- predict.py
- pyproject.toml
- baseline.py
- eval_embspatial_bench.py
- eval_spatialbench.py
- eval_spatialbot.py
- INSTALL.md
- pyproject.toml
- README.md
# Installation Guide
1. Get the code
git clone https://github.com/qizekun/SoFar
Downloads the entire project code from GitHub to your computer.
cd SoFar
Moves into the project folder you just downloaded.
2. Official Install Script
Easy RecommendedPrerequisites
- Python 3 Python is required to use pip.
pip install qwen-vl-utils[decord]==0.0.8 triton
Installs the package published on PyPI directly β no need to clone the source.
pip install flash-attn --no-build-isolation
Installs the package published on PyPI directly β no need to clone the source.
After installing, open a new terminal and run the program's version command (e.g. --version) to confirm it worked.
Pulled directly from this repo's README.
3. Python
EasyPrerequisites
pip install -e .
Installs the Python libraries listed in requirements.txt (or similar).
pip install -e segmentation/SAM
Installs the Python libraries listed in requirements.txt (or similar).
pip install qwen-vl-utils[decord]==0.0.8 triton
Installs the package published on PyPI directly β no need to clone the source.
pip install flash-attn --no-build-isolation
Installs the package published on PyPI directly β no need to clone the source.
If it runs without errors and prints output in the terminal, it worked.
Pulled directly from this repo's README.
// repository documentation
Was this content helpful?
(0 ratings)