DataDreamer
DataDreamer: Prompt. Generate Synthetic Data. Train & Align Models. β π€π€
νμΌ νμκΈ°
μ΅μ’ λ²μ λ€μ΄λ‘λ (.zip)- cla.yml
- release.yml
- tests.yml
- custom.css
- cover.png
- demo.svg
- demo_code.png
- favicon.svg
- logo.svg
- custom.js
- module.rst_t
- package.rst_t
- brand.html
- base.html
- metatags.html
- page.html
- index.rst
- llm.rst
- other.rst
- step.rst
- trainer.rst
- index.rst
- running_models_on_multiple_gpus.rst
- running_steps_in_parallel.rst
- training_models_on_multiple_gpus.rst
- caching_and_saved_outputs.rst
- index.rst
- parameter_efficient_training.rst
- quantization.rst
- abstract_to_tweet.rst
- aligning.rst
- attributed_prompts.rst
- bootstrapping_machine_translation.rst
- dataset_augmentation.rst
- dataset_cleaning.rst
- index.rst
- instruction_tuning.rst
- openai_distillation.rst
- self_rewarding.rst
- installation.rst
- motivation_and_design.rst
- overview_guide.rst
- contributing.rst
- __init__.py
- conf.py
- docstrings.py
- index.rst
- .gitignore
- make.bat
- Makefile
- .gitignore
- _direct_config.sh
- _direct_env.sh
- cancel.sh
- cancel_all.sh
- interactive_submit.sh
- reset_venv.sh
- status.sh
- status_all.sh
- submit.sh
- .gitignore
- _qsub_config.sh
- cancel.sh
- cancel_all.sh
- htop.sh
- interactive_submit.sh
- reset_venv.sh
- shell.sh
- status.sh
- status_all.sh
- submit.sh
- .gitignore
- _sbatch_config.sh
- cancel.sh
- cancel_all.sh
- htop.sh
- interactive_submit.sh
- nvidia-smi.sh
- reset_venv.sh
- shell.sh
- status.sh
- status_all.sh
- submit.sh
- .gitignore
- _boot.sh
- _command.sh
- _lock.sh
- _setup.sh
- args_log.sh
- config_log.sh
- full_reset.sh
- install_log.sh
- reset.sh
- reset_venv.sh
- series_log.sh
- stderr_log.sh
- stdout_log.sh
- submission_log.sh
- tag.sh
- post-checkout
- pre-commit
- extensions.json
- settings.json
- tasks.json
- .python-version
- docs.sh
- format.sh
- lint.sh
- package.sh
- package_publish.sh
- project.env
- run.sh
- __init__.py
- _cachable.py
- _parallel_cachable.py
- __init__.py
- datasets_reset_state_hack.py
- setfit_import_hack.py
- __init__.pyi
- .gitkeep
- __init__.py
- datasets.py
- utils.py
- __init__.py
- embedder.py
- openai_embedder.py
- parallel_embedder.py
- sentence_transformers_embedder.py
- together_embedder.py
- __init__.py
- step.py
- __init__.py
- __init__.py
- _chat_prompt_templates.py
- _litellm.py
- _llm_api.py
- _tokenizers.py
- ai21.py
- anthropic.py
- bedrock.py
- cohere.py
- ctransformers.py
- google_ai_studio.py
- hf_api_endpoint.py
- hf_transformers.py
- llm.py
- mistral_ai.py
- openai.py
- openai_assistant.py
- parallel_llm.py
- petals.py
- together.py
- vertex_ai.py
- vllm.py
- __init__.py
- logger.py
- __init__.py
- pickle.py
- __init__.py
- builtin_tasks.py
- debug.py
- devices.py
- environment.py
- pennnlp.py
- persistent_storage.py
- report.py
- serve.py
- __init__.py
- embedding_retriever.py
- parallel_retriever.py
- retriever.py
- csv_data_source.py
- data_source.py
- hf_dataset_data_source.py
- hf_hub_data_source.py
- json_data_source.py
- text_data_source.py
- _prompt_base.py
- data_from_attributed_prompt.py
- data_from_prompt.py
- few_shot_prompt.py
- few_shot_prompt_with_retrieval.py
- filter_with_prompt.py
- judge_generation_pairs_with_prompt.py
- judge_pairs_with_prompt.py
- process_with_prompt.py
- prompt.py
- rag_prompt.py
- rank_with_prompt.py
- cosine_similarity.py
- embed.py
- retrieve.py
- run_task_model.py
- __init__.py
- data_card.py
- step.py
- step_background.py
- step_export.py
- step_operations.py
- step_output.py
- __init__.py
- hf_classification_task_model.py
- parallel_task_model.py
- task_model.py
- __init__.py
- test_datasets.py
- test_utils.py
- __init__.py
- test_embedders.py
- __init__.py
- test_llms.py
- __init__.py
- test_retrievers.py
- __init__.py
- test_prompt.py
- __init__.py
- test_tasks.py
- __init__.py
- test_data_sources.py
- test_step.py
- test_step_background.py
- test_step_export.py
- test_step_operations.py
- test_step_output.py
- __init__.py
- test_task_models.py
- __init__.py
- bitsandbytes_fixture.py
- clear_space.py
- cli_runner.py
- create_datadreamer.py
- create_test_step.py
- mock_llm.py
- restore_os_environ.py
- __init__.py
- config.py
- __init__.py
- test_distributed.py
- test_trainers.py
- __init__.py
- test_device_utils.py
- __init__.py
- conftest.py
- test_cli.py
- test_datadreamer.py
- test_package.py
- __init__.py
- _dpo_helper.py
- _sentence_transformer_helper.py
- _setfit_helper.py
- dpo_trainer.py
- __init__.py
- _train_hf_base.py
- train_hf_classifier.py
- train_hf_dpo.py
- train_hf_finetune.py
- train_hf_ppo.py
- train_hf_reward_model.py
- train_openai_finetune.py
- train_sentence_transformer.py
- train_setfit_classifier.py
- trainer.py
- __init__.py
- arg_utils.py
- background_utils.py
- collection_utils.py
- device_utils.py
- distributed_utils.py
- fingerprint_utils.py
- fs_utils.py
- hf_chat_prompt_templates.py
- hf_hub_utils.py
- hf_model_utils.py
- hf_structured_decoding_utils.py
- hf_training_utils.py
- import_utils.py
- ring_utils.py
- str_utils.py
- time_utils.py
- .env
- .gitignore
- .secrets.template.env
- __cli__.py
- __init__.py
- __main__.py
- datadreamer.py
- py.typed
- requirements-accelerator-device.txt
- requirements-cpu.txt
- requirements-dev.txt
- requirements-test.txt
- requirements.txt
- .cirun.yml
- .gitignore
- LICENSE.txt
- pyproject.toml
- README.md
π μ€μΉ κ°μ΄λ
1. μ½λ λ΄λ €λ°κΈ°
git clone https://github.com/datadreamer-dev/DataDreamer
κΉνλΈμμ νλ‘μ νΈ μ½λ μ 체λ₯Ό λ΄ μ»΄ν¨ν°λ‘ λ΄λ €λ°μ΅λλ€.
cd DataDreamer
λ°©κΈ λ΄λ €λ°μ νλ‘μ νΈ ν΄λ μμΌλ‘ μ΄λν©λλ€.
2. Python
μ¬μ μΆμ²μ¬μ μ€λΉλ¬Ό
<p align="center"><b>Installation</b></p> <pre lang="bash">pip3 install datadreamer.dev</pre>
requirements.txt λ±μ λͺ μλ νμ΄μ¬ λΌμ΄λΈλ¬λ¦¬λ₯Ό μ€μΉν©λλ€.
μλ¬ λ©μμ§ μμ΄ μ€νλκ³ ν°λ―Έλμ μλ΄ λ¬Έκ΅¬κ° μΆλ ₯λλ©΄ μ μμ
λλ€.
μ΄ λ ν¬μ READMEμ μ ν μ€μ λͺ λ Ήμ΄λ₯Ό κ·Έλλ‘ κ°μ Έμμ΅λλ€.
3. Make
보ν΅μ¬μ μ€λΉλ¬Ό
- Git GitHubμμ νλ‘μ νΈ μ½λλ₯Ό λ΄λ €λ°μΌλ €λ©΄ νμν©λλ€.
- Make Linux/macOSλ λ³΄ν΅ κΈ°λ³Έ μ€μΉλμ΄ μμ΅λλ€. Windowsλ λ³λ μ€μΉ(μ: MSYS2, WSL)κ° νμν©λλ€.
cd docs
μ΄ νλ‘μ νΈμ κ΄λ ¨ νμΌμ΄ νμ ν΄λ μμ μμ΄μ, λ¨Όμ κ·Έ ν΄λλ‘ μ΄λν©λλ€.
make
μμ±λ λΉλ μ€μ μ λ°νμΌλ‘ μ€μ μ»΄νμΌμ μ§νν΄ μ€ν νμΌμ λ§λλλ€.
μλ¬ μμ΄ λλλ©΄ μ±κ³΅μ
λλ€. μμ±λ μ€ν νμΌμ μ§μ μ€νν΄λ³΄μΈμ.
// repository documentation
Was this content helpful?
(0 ratings)