RL-Factory
Train your Agent model via our easy and efficient framework
File Explorer
Download Latest Version (.zip)- rl-factory.png
- we_group3.jpg
- Apptainerfile.rocm
- Dockerfile.megatron
- Dockerfile.ngc.vllm
- Dockerfile.ngc.vllm0.8
- Dockerfile.ngc.vllm0.8.sagemaker
- Dockerfile.rocm
- Dockerfile.sglang
- Dockerfile.vemlp.vllm.te
- Dockerfile.vllm.sglang.megatron
- Dockfile.ngc.vllm0.8
- runllm-widget.js
- logo.png
- checkpoint.rst
- dpo_extension.rst
- fsdp_extension.rst
- megatron_extension.rst
- placement.rst
- rope.rst
- amd_build_dockerfile.md
- amd_build_dockerfile_page.rst
- amd_existing_docker.md
- amd_vllm_page.rst
- trainer.rst
- utils.rst
- config.rst
- gsm8k_example.rst
- multi_modal_example.rst
- ppo_code_architecture.rst
- sandbox_fusion_example.rst
- ppo.rst
- faq.rst
- device_tuning.rst
- perf_tuning.rst
- prepare_data.rst
- reward_function.rst
- centralized_tool_manager.md
- framework_design.md
- main_tutorial.md
- rewards.md
- tools.md
- main_tutorial.md
- README.md
- rewards.md
- tools.md
- main_tutorial.md
- main_tutorial_zh.md
- multiturn.rst
- install.rst
- multinode.rst
- quickstart.rst
- ray_debug_tutorial.rst
- fsdp_workers.rst
- megatron_workers.rst
- ray_trainer.rst
- sglang_worker.rst
- conf.py
- data.rst
- hybrid_flow.rst
- index.rst
- Makefile
- MCP_CACHE_README.md
- README.md
- README_android.md
- README_vllm0.7.md
- README_vllm0.8.md
- requirements-docs.txt
- __init__.py
- android_utils.py
- context_utils.py
- logging_utils.py
- parse_utils.py
- env_config.py
- __init__.py
- android_env.py
- base_env.py
- base_env_config.py
- env_config.py
- envs.py
- reward_evaluator.py
- __init__.py
- android.py
- adv.sh
- android_control.py
- base.py
- env_manager.py
- pool_api.py
- README_android.md
- redis_port.py
- start_emulators.sh
- calculator.json
- chat_template.jinja
- mcp_tools.pydata
- mcp_vision_tools.pydata
- sse_mcp_tools.pydata
- cache_base.py
- cachebox_cache.py
- storage_manager.py
- disk_persist.py
- persist_base.py
- storage_test.py
- __init__.py
- CacheMe.md
- distributed_cache_system.png
- centralized_qwen3_manager.py
- __init__.py
- base_manager.py
- config_manager.py
- llama3_manager.py
- mm_base_manager.py
- qwen2_5_manager.py
- qwen2_5_vl_manager.py
- qwen3_manager.py
- rotate.py
- search.py
- async_mcp_manager.py
- concurrency_limiter.py
- get_prompt.py
- mcp_manager.py
- mm_tool_utils.py
- redis_cache_manager.py
- schema.py
- suppress_mcp_warnings.py
- tool_utils.py
- util.py
- __init__.py
- base.py
- mmbase.py
- reward_rollout_example.py
- search.py
- vision.py
- run_deepseek_megatron_ckpt.sh
- run_qwen_megatron_ckpt.sh
- full_hh_rlhf.py
- geo3k.py
- gsm8k.py
- gsm8k_multiturn_w_tool.py
- hellaswag.py
- math_dataset.py
- multiturn.py
- process_vl.py
- run_deepseek7b_mutli_node.sh
- run_deepseek_v2_lite_math.sh
- run_deepseek7b_llm.sh
- run_deepseek7b_llm_math.sh
- run_deepseek7b_llm_math_megatron.sh
- run_deepseek7b_llm_megatron.sh
- run_deepseek7b_llm_seq_balance.sh
- run_qwen2-7b.sh
- run_qwen2-7b_math.sh
- run_qwen2-7b_math_megatron.sh
- run_qwen2-7b_megatron.sh
- run_qwen2-7b_seq_balance.sh
- run_qwen2-7b_sgl_megatron.sh
- run_qwen2_5-7b_math_megatron_diff_tp.sh
- run_qwen2_5_vl-7b.sh
- run_qwen3-8b.sh
- naive_chat_scheduler.py
- run_deepseek7b_llm.sh
- run_deepseek7b_llm_modelscope.sh
- run_deepseek7b_llm_sandbox_fusion.sh
- run_deepseek7b_llm_sp2.sh
- run_deepseek_full_hh_rlhf.sh
- run_deepseek_math_gsm8k_megatron.sh
- run_deepseek_megatron.sh
- run_gemma.sh
- run_qwen1.5_moe_a2.7b-gsm8k_megatron.sh
- run_qwen2-7b_math_gsm8k_megatron.sh
- run_qwen2-7b_megatron.sh
- run_qwen2-7b_rm.sh
- run_qwen2-7b_rm_seq_balance.sh
- run_qwen2-7b_seq_balance.sh
- run_qwen2-7b_sglang_seq_balance.sh
- run_qwen2.5-32b.sh
- verl_getting_started.ipynb
- tutorial.ipynb
- run_qwen2-7b_math_rf.sh
- run_qwen2-7b_math_rf_baseline.sh
- run_qwen2.5-3b_seq_balance.sh
- run_qwen2.5-7b_seq_balance.sh
- centralized_tool_manager.sh
- qwen25vl_tool.sh
- reward_rollout_test.sh
- search_ppo.sh
- run_qwen2-7b.sh
- run_deepseek_6b7.sh
- run_gemma_2b.sh
- run_gemma_7b.sh
- run_qwen_05_peft.sh
- run_qwen_05_sp2.sh
- run_qwen_05_sp2_liger.sh
- run_qwen_05_sp2.sh
- gsm8k_tool_config.yaml
- gsm8k_multiturn_grpo.yaml
- README.md
- run_qwen2.5-3b_gsm8k_multiturn.sh
- run_qwen2.5-3b_gsm8k_multiturn_4xgpu.sh
- ray_on_slurm.slurm
- ppo_trainer_split.yaml
- main_ppo_split.py
- README.md
- run_deepseek7b_llm.sh
- split_monkey_patch.py
- qwen2_14b_grpo_4_h800_fsdp_vllm.sh
- qwen2_32B_grpo_8_h20_megatron_vllm.sh
- qwen2-70b_grpo_32_h20_fsdp_vllm.sh
- qwen2-70b_grpo_32_h800_fsdp_vllm.sh
- qwen2-7b_grpo_2_h800_fsdp_vllm.sh
- __init__.py
- api_generator.py
- base_generator.py
- test_questions.json
- nq_search.py
- download.py
- launch.sh
- README.md
- retrieval_server.py
- dapo_trainer.yaml
- dapo_ray_trainer.py
- main_dapo.py
- prepare_dapo_data.sh
- README.md
- run_dapo_early_qwen2.5_32b.sh
- run_dapo_qwen2.5_0.5b.sh
- run_dapo_qwen2.5_32b.sh
- run_dapo_wo_ds_qwen2.5_32b.sh
- test_dapo_7b.sh
- README.md
- prime_trainer.yaml
- __init__.py
- main_prime.py
- prime_core_algos.py
- prime_dp_rm.py
- prime_fsdp_workers.py
- prime_ray_trainer.py
- run_prime_qwen.sh
- evaluation.yaml
- __init__.py
- gpqa.py
- livecodebench.py
- math.py
- __init__.py
- data_process.py
- main_eval.py
- README.md
- reward_score.py
- run_r1_distill_qwen.sh
- sppo_trainer.yaml
- __init__.py
- dp_actor.py
- main_sppo.py
- README.md
- run_qwen2.5-7b_rm.sh
- sppo_ray_trainer.py
- sppo_worker.py
- travelplanner.py
- clean_accommodations_2022.csv
- attractions.csv
- citySet.txt
- citySet_with_states.txt
- stateSet.txt
- distance.csv
- clean_restaurant_2022.csv
- __init__.py
- commonsense_constraint.py
- custom_eval.py
- eval.py
- hard_constraint.py
- __init__.py
- apis.py
- apis.py
- apis.py
- __init__.py
- apis.py
- apis.py
- apis.py
- test.py
- apis.py
- env.py
- sole_planning.py
- test.py
- __init__.py
- apis.py
- __init__.py
- __init__.py
- hard_constraints.py
- reward_test.py
- travel.py
- __init__.py
- accommodation_server.py
- attraction_server.py
- base_server.py
- distance_server.py
- flight_server.py
- launch.sh
- restaurant_server.py
- accommodation.py
- attraction.py
- distance.py
- flight.py
- restaurant.py
- test.py
- main_grpo.sh
- README_TravelPlanner.md
- travelplanner_preprocess.py
- client.py
- README.md
- start_redis.sh
- converter_hf_to_mcore.py
- diagnose.py
- format.sh
- install_nginx.sh
- install_vllm_sglang_mcore.sh
- model_merger.py
- nq_search.py
- run_vllm_with_nginx.sh
- vllm_server.sh
- r1_dataset.py
- r1_reward.py
- rlfactory_env.json
- rlfactory_reward.py
- rlfactory_toolcall.py
- run_deepseek_megatron_ckpt.sh
- run_qwen_megatron_ckpt.sh
- test_fsdp_ckpt.py
- run_all.sh
- test_tensor_dict.py
- requirements.py
- create_dataset.py
- config.json
- create_model_tokenizer.py
- generation_config.json
- model.safetensors
- tokenizer_config.json
- main_trainer.py
- README.md
- __init__.py
- task.py
- tokenizer.py
- __init__.py
- run_gen_qwen05.sh
- run_function_reward.sh
- run_model_reward.sh
- run_sft.sh
- test_sp_loss_match.py
- __init__.py
- check_custom_rwd_fn.py
- check_results.py
- run_dapo.sh
- run_deepseek_grpo.sh
- run_deepseek_grpo_megatron.sh
- run_deepseek_megatron.sh
- run_deepseek_megatron_parallelism.sh
- run_gsm8k_fsdp_sgl_multiturn_w_tool.sh
- run_ppo_trainer_megatron.sh
- run_prime.sh
- run_qwen2vl_geo3k_function_rm.sh
- run_qwen_grpo.sh
- run_qwen_grpo_megatron.sh
- run_qwen_gsm8k_custom_function_rm.sh
- run_qwen_gsm8k_function_rm.sh
- run_qwen_gsm8k_function_rm_grpo.sh
- run_qwen_gsm8k_function_rm_no_rmpad.sh
- run_qwen_gsm8k_function_rm_remax.sh
- run_qwen_gsm8k_model_rm.sh
- run_qwen_gsm8k_model_rm_liger_kernel.sh
- run_qwen_gsm8k_model_rm_no_rmpad.sh
- run_qwen_gsm8k_model_rm_seq_balance.sh
- run_qwen_gsm8k_model_rm_ulysses.sh
- run_qwen_gsm8k_prime.sh
- run_qwen_megatron.sh
- run_qwen_megatron_parallelism.sh
- run_r1_distill_qwen_aime24_eval.sh
- run_ray_trainer.sh
- run_ray_trainer_fire_sampling.sh
- run_ray_trainer_rmpad.sh
- run_sppo.sh
- run_test.sh
- run_gen_qwen05.sh
- test_memory_buffers.py
- test_ops.py
- test_torch_functional.py
- test_linear_cross_entropy.py
- test_transformer.py
- test_transformers_ulysses.py
- test_transformer.py
- test_transformers_ulysses.py
- main.py
- client.py
- README.md
- run.sh
- server.py
- test_check_worker_alive.py
- test_colocated_workers.py
- test_data_transfer.py
- test_driverfunc_to_worker.py
- test_high_level_scheduling_api.py
- test_ray_local_envs.py
- test_rvdz.py
- test_worker_group_basics.py
- test_worker_group_torch.py
- main.py
- test_auto_padding.py
- test_check_worker_alive.py
- test_decorator.py
- test_fused_workers.py
- test_ray_local_envs.py
- client.py
- README.md
- run.sh
- server.py
- test_colocated_workers.py
- test_colocated_workers_fused.py
- test_data_transfer.py
- test_driverfunc_to_worker.py
- test_high_level_scheduling_api.py
- test_rvdz.py
- test_worker_group_basics.py
- test_worker_group_torch.py
- test_sandbox_fusion.py
- test_tool_use.py
- async_generator_test.py
- async_results.csv
- test.sh
- test_parallel.py
- test_async_tool_manager.py
- test_llama3_manager.py
- test_qwen3_manager.py
- run_fsdp_vllm.py
- test_sglang_spmd.py
- test_vllm_hf_loader.py
- test_vllm_spmd.py
- test_sandbox.py
- check_license.py
- test_import.py
- run_sft.sh
- run_sft_qwen05_peft.sh
- run_sft_qwen05_sp2_liger.sh
- run_sft_sp_loss_match.sh
- test_sp_loss_match.py
- test_decorator.py
- __init__.py
- test_metric_utils.py
- __init__.py
- test_tensor_dict_utilities.py
- test_fs.py
- test_import_utils.py
- test_model.py
- test_module.py
- test_timeout_decorator.py
- test_fsdp_ckpt.py
- test_multiturn_sft_dataset.py
- test_rl_dataset.py
- test_rm_dataset.py
- test_sft_dataset.py
- test_flops_counter.py
- test_seqlen_balancing.py
- test_torch_functional.py
- test_rl_dataset.py
- test_rm_dataset.py
- test_sft_dataset.py
- async_rollout_utils.py
- run_fsdp_vllm.py
- test_hf_rollout.py
- test_sglang_async_rollout_w_tools.py
- test_sglang_async_spmd.py
- test_sglang_spmd.py
- test_vllm_hf_loader.py
- test_vllm_multi_turn.py
- test_vllm_spmd.py
- test_vllm_tool_calling.py
- utils_sglang.py
- __init__.py
- kill_github_tests.sh
- test_mcp_cache_real.py
- test_protocol.py
- __init__.py
- agent_loop.py
- single_turn_agent_loop.py
- tool_agent_loop.py
- tool_parser.py
- __init__.py
- sampler.py
- __init__.py
- dynamicgen_dataset.py
- __init__.py
- __init__.py
- interaction_registry.py
- __init__.py
- base.py
- gsm8k_interaction.py
- __init__.py
- __main__.py
- base_model_merger.py
- fsdp_model_merger.py
- megatron_model_merger.py
- __init__.py
- llama_loader.py
- llama_loader_depracated.py
- llama_saver.py
- __init__.py
- parallel_attention.py
- parallel_decoder.py
- parallel_linear.py
- parallel_mlp.py
- parallel_rmsnorm.py
- __init__.py
- modeling_llama_megatron.py
- __init__.py
- __init__.py
- attention.py
- model.py
- rope_utils.py
- vision_config.py
- vision_model.py
- vision_transformer_block.py
- __init__.py
- config_converter.py
- loader.py
- mbridge.py
- model_forward.py
- model_forward_fused.py
- model_initializer.py
- patch_v012.py
- readme.md
- registry.py
- saver.py
- util.py
- weight_converter.py
- __init__.py
- qwen2_loader.py
- qwen2_loader_depracated.py
- qwen2_saver.py
- __init__.py
- parallel_attention.py
- parallel_decoder.py
- parallel_linear.py
- parallel_mlp.py
- parallel_rmsnorm.py
- __init__.py
- modeling_qwen2_megatron.py
- __init__.py
- __init__.py
- dense_common.py
- kimi_vl.py
- llama.py
- monkey_patch.py
- npu_patch.py
- qwen2.py
- qwen2_5_vl.py
- qwen2_vl.py
- __init__.py
- README.md
- registry.py
- weight_loader_registry.py
- __init__.py
- worker.py
- worker_group.py
- __init__.py
- ray.py
- __init__.py
- decorator.py
- worker.py
- worker_group.py
- __init__.py
- base.py
- megatron.py
- __init__.py
- __init__.py
- parallel_state.py
- __init__.py
- state_dict.py
- __init__.py
- _state_dict_utils.py
- __init__.py
- __init__.py
- __init__.py
- McpClientManager.py
- utils.py
- __init__.py
- search_r1_like_utils.py
- tool_registry.py
- __init__.py
- base_tool.py
- geo3k_tool.py
- gsm8k_tool.py
- mcp_base_tool.py
- mcp_search_tool.py
- sandbox_fusion_tools.py
- schemas.py
- search_tool.py
- actor.yaml
- dp_actor.yaml
- megatron_actor.yaml
- critic.yaml
- dp_critic.yaml
- megatron_critic.yaml
- legacy_data.yaml
- npu_profile.yaml
- dp_ref.yaml
- megatron_ref.yaml
- ref.yaml
- dp_reward_model.yaml
- megatron_reward_model.yaml
- reward_model.yaml
- reward_rollout.yaml
- rollout.yaml
- __init__.py
- _generated_ppo_megatron_trainer.yaml
- _generated_ppo_trainer.yaml
- algorithm.py
- config.py
- evaluation.yaml
- generation.yaml
- ppo_megatron_trainer.yaml
- ppo_trainer.yaml
- rl_factory_ppo_megatron_trainer.yaml
- rl_factory_ppo_trainer.yaml
- sft_trainer.yaml
- __init__.py
- core_algos.py
- metric_utils.py
- ray_trainer.py
- reward.py
- __init__.py
- constants_ppo.py
- fsdp_sft_trainer.py
- main_eval.py
- main_evaluate.py
- main_generation.py
- main_ppo.py
- runtime_env.yaml
- __init__.py
- checkpoint_manager.py
- fsdp_checkpoint_manager.py
- megatron_checkpoint_manager.py
- __init__.py
- multiturn_sft_dataset.py
- README.md
- rl_dataset.py
- rm_dataset.py
- sft_dataset.py
- vision_utils.py
- __init__.py
- empty_annotations.py
- nvtx_profile.py
- performance.py
- trajectory_tracker.py
- __init__.py
- torch_functional.py
- __init__.py
- kernels.py
- linear_cross_entropy.py
- __init__.py
- aggregate_logger.py
- __init__.py
- dist_checkpointing.py
- memory.py
- optimizer.py
- pipeline_parallel.py
- sequence_parallel.py
- tensor_parallel.py
- __init__.py
- utils.py
- __init__.py
- config.py
- empty_annotations.py
- mstx_profile.py
- nvtx_profile.py
- performance.py
- profile.py
- __init__.py
- ray_backend.py
- __init__.py
- README.md
- testing_util.py
- utils.py
- __init__.py
- grader.py
- math_normalize.py
- __init__.py
- utils.py
- __init__.py
- geo3k.py
- gsm8k.py
- math.py
- math_batch.py
- math_dapo.py
- math_verify.py
- search_r1_like_qa_em.py
- __init__.py
- activation_offload.py
- config.py
- device.py
- distributed.py
- flops_counter.py
- fs.py
- fsdp_utils.py
- hdfs_io.py
- import_utils.py
- logging_utils.py
- megatron_utils.py
- memory_buffer.py
- model.py
- net_utils.py
- py_functional.py
- ray_utils.py
- rollout_trace.py
- seqlen_balancing.py
- tokenizer.py
- torch_dtypes.py
- torch_functional.py
- tracking.py
- ulysses.py
- vllm_utils.py
- version
- __init__.py
- base.py
- dp_actor.py
- megatron_actor.py
- __init__.py
- actor.py
- engine.py
- optimizer.py
- __init__.py
- base.py
- dp_critic.py
- megatron_critic.py
- __init__.py
- engine_impl.py
- utils.py
- __init__.py
- engine_impl.py
- __init__.py
- base.py
- __init__.py
- batch.py
- dapo.py
- naive.py
- parallel.py
- prime.py
- registry.py
- __init__.py
- reward_model.py
- __init__.py
- base.py
- __init__.py
- actor.py
- critic.py
- __init__.py
- naive_rollout.py
- __init__.py
- async_sglang_server.py
- sglang_rollout.py
- utils.py
- __init__.py
- vllm_async_server.py
- vllm_rollout_spmd.py
- __init__.py
- async_server.py
- base.py
- chat_scheduler.py
- hf_rollout.py
- schemas.py
- tokenizer.py
- __init__.py
- base.py
- fsdp_sglang.py
- fsdp_ulysses.py
- fsdp_vllm.py
- megatron_sglang.py
- megatron_vllm.py
- __init__.py
- fsdp_workers.py
- megatron_workers.py
- __init__.py
- base_config.py
- protocol.py
- py.typed
- __init__.py
- base.py
- graders.py
- qwen_math.py
- flow_editor.py
- flow_editor.pyi
- reward_config.json
- get_env_codes.py
- __init__.py
- data_processing.py
- project_management.py
- reward_definition.py
- tool_definition.py
- training_deployment.py
- app.py
- README.md
- requirements.txt
- run_webui.sh
- .gitignore
- install.sh
- LICENSE
- main_eval.sh
- main_grpo.sh
- main_grpo_megatron.sh
- main_gspo_megatron.sh
- main_ppo.sh
- pyproject.toml
- README.md
- requirements.txt
- requirements_sglang.txt
- setup.py
- swift_grpo.sh
# Installation Guide
1. Get the code
git clone https://github.com/Simple-Efficient/RL-Factory
Downloads the entire project code from GitHub to your computer.
cd RL-Factory
Moves into the project folder you just downloaded.
2. Official Install Script
Easy RecommendedPrerequisites
- Python 3 Python is required to use pip.
pip3 install accelerate bitsandbytes datasets deepspeed==0.16.4 einops flash-attn==2.7.0.post2 isort jsonlines loralib optimum packaging peft pynvml>=12.0.0 ray[default]==2.46.0 tensorboard torch==2.6.0 torchmetrics tqdm transformers==4.51.3 transformers_stream_generator wandb wheel
Installs the package published on PyPI directly β no need to clone the source.
pip3 install vllm==0.8.5 # Mainly for Qwen3 model support
Installs the package published on PyPI directly β no need to clone the source.
pip3 install "qwen-agent[code_interpreter]"
Installs the package published on PyPI directly β no need to clone the source.
pip3 install llama_index bs4 pymilvus infinity_client codetiming tensordict==0.6 omegaconf torchdata==0.10.0 hydra-core easydict dill python-multipart mcp==1.9.3
Installs the package published on PyPI directly β no need to clone the source.
After installing, open a new terminal and run the program's version command (e.g. --version) to confirm it worked.
Pulled directly from this repo's README.
3. Docker
EasyPrerequisites
- Git Needed to download the project code from GitHub.
- Docker Desktop Needed to build and run containers. Install it and keep it running in the background.
docker build -f docker/Dockerfile.megatron -t rl-factory .
Builds a runnable image based on the Dockerfile.
docker run -p 8080:80 rl-factory
Runs the built image as an actual container.
Run docker compose ps to check the containers are Up. If the README mentions a port, open http://localhost:PORT in your browser.
4. Python
EasyPrerequisites
pip3 install accelerate bitsandbytes datasets deepspeed==0.16.4 einops flash-attn==2.7.0.post2 isort jsonlines loralib optimum packaging peft pynvml>=12.0.0 ray[default]==2.46.0 tensorboard torch==2.6.0 torchmetrics tqdm transformers==4.51.3 transformers_stream_generator wandb wheel
Installs the package published on PyPI directly β no need to clone the source.
pip3 install vllm==0.8.5 # Mainly for Qwen3 model support
Installs the package published on PyPI directly β no need to clone the source.
pip3 install "qwen-agent[code_interpreter]"
Installs the package published on PyPI directly β no need to clone the source.
pip3 install llama_index bs4 pymilvus infinity_client codetiming tensordict==0.6 omegaconf torchdata==0.10.0 hydra-core easydict dill python-multipart mcp==1.9.3
Installs the package published on PyPI directly β no need to clone the source.
pip3 install -e . --no-deps
Installs the Python libraries listed in requirements.txt (or similar).
If it runs without errors and prints output in the terminal, it worked.
Pulled directly from this repo's README.
// repository documentation
Was this content helpful?
(0 ratings)