whirl
Fast iterative local development and testing of Apache Airflow workflows
File Explorer
Download Latest Version (.zip)- ci-verifier.md
- plugin.json
- create-environment
- create-example
- verify-ci-impact
- version-bumper
- SKILL.md
- SKILL.md
- ci_impact.sh
- SKILL.md
- SKILL.md
- whirl-ci.yml
- whirl-release.yml
- date_replacement.sh
- auth.json
- delete_all_airflow_connections.py
- Dockerfile
- entrypoint.sh
- harden.sh
- nginx-ssl.conf
- Dockerfile
- core-site.xml
- Dockerfile
- hive-site.xml
- init-hive.sh
- 01_add_connection_s3.sh
- 02_configure_logging_to_s3.sh
- .gitkeep
- .whirl.env
- docker-compose.yml
- .gitkeep
- .whirl.env
- docker-compose.yml
- 01_add_connection_s3server.sh
- .gitkeep
- .whirl.env
- docker-compose.yml
- pod_template.yaml
- .gitkeep
- registries.yaml
- 01_push_airflow_worker_image_to_local_registry.sh
- 02_clean_local_data_share.sh
- Dockerfile.worker
- 01_add_connection_s3server.sh
- 02_add_kubectl.sh
- 03_k8s_persistant_volume.sh
- .gitignore
- .whirl.env
- docker-compose.yml
- entrypoint.sh
- README.md
- 01_add_connection_api.sh
- .gitkeep
- .whirl.env
- docker-compose.yml
- 01_clean_pg_data_dir.sh
- 01_add_connection_s3.sh
- 02_add_connection_postgres.sh
- 03_add_spark_config.sh
- 04_install_dbt.sh
- .gitignore
- .gitkeep
- .whirl.env
- docker-compose.yml
- .whirl.env
- docker-compose.yml
- .gitkeep
- .whirl.env
- docker-compose.yml
- .gitkeep
- .whirl.env
- docker-compose.yml
- .gitkeep
- .whirl.env
- docker-compose.yml
- 01_enable_local_ssh.sh
- .whirl.env
- docker-compose.yml
- 01_add_connection_s3.sh
- 02_add_connection_postgres.sh
- 03_add_spark_config.sh
- .gitkeep
- .whirl.env
- docker-compose.yml
- 01_add_connection_s3.sh
- 02_add_connection_postgres.sh
- 03_add_spark_config.sh
- .whirl.env
- docker-compose.yml
- 01_add_connection_s3.sh
- 02_add_connection_postgres.sh
- 03_add_spark_config.sh
- .gitkeep
- .whirl.env
- docker-compose.yml
- 01_check_available_memory.sh
- 02_clean_s3_mount_dir.sh
- 01_add_connection_s3.sh
- 03_add_spark_config.sh
- .gitignore
- .gitkeep
- .whirl.env
- docker-compose.yml
- 01_check_available_memory.sh
- 02_clean_s3_mount_dir.sh
- core-site.xml
- delta-sharing.yml
- log4j.properties
- 01_add_connection_s3.sh
- 02_add_spark_config.sh
- 03_add_delta_config.sh
- .gitignore
- .gitkeep
- .whirl.env
- docker-compose.yml
- 01_check_available_memory.sh
- 02_clean_s3_mount_dir.sh
- core-site.xml
- delta-sharing.yml
- log4j.properties
- 01_add_connection_s3.sh
- 02_add_spark_config.sh
- 03_add_delta_config.sh
- .gitignore
- .gitkeep
- .whirl.env
- docker-compose.yml
- 01_check_available_memory.sh
- 02_clean_s3_mount_dir.sh
- 03_clean_pg_data_dir.sh
- core-site.xml
- delta-sharing.yml
- log4j.properties
- dev.sql
- 01_add_connection_s3.sh
- 02_add_spark_config.sh
- 03_add_delta_config.sh
- .gitignore
- .gitkeep
- .whirl.env
- docker-compose.yml
- 01_prepare_sftp.sh
- 02_prepare_mysql.sh
- .whirl.env
- docker-compose.yml
- mysql.env
- sftp.env
- dag_policy.py
- .gitkeep
- 01_configure_cluster_policies.sh
- .airflowignore
- .whirl.env
- dag.py
- README.md
- datasets.py
- 01_add_connection_api_and_mockdata.sh
- .airflowignore
- .whirl.env
- dataset-A-producer.py
- dataset-ABC-consumer.py
- dataset-ABCD-consumer.py
- dataset-B-producer.py
- dataset-C-producer.py
- dataset-D-producer.py
- dataset-E-producer.py
- README.md
- single-dataset-consumer.py
- single-dataset-producer.py
- .gitkeep
- .airflowignore
- .whirl.env
- dag.py
- README.md
- __init__.py
- api_check_operator.py
- __init__.py
- api_check_trigger.py
- __init__.py
- .gitkeep
- 01_add_connection_api_and_mockdata.sh
- .airflowignore
- .whirl.env
- dag.py
- README.md
- setup.py
- __init__.py
- fullmoon.py
- __init__.py
- setup.cfg
- setup.py
- .gitkeep
- 01_install_custom_timetable.sh
- .airflowignore
- .whirl.env
- dag.py
- README.md
- 01_add_connection_api_and_mockdata.sh
- .airflowignore
- .whirl.env
- dag.py
- README.md
- 02_clean_dbt_log.sh
- 01_show_dbt_log.sh
- .gitkeep
- airports_dbt.sql
- carriers_dbt.sql
- flights_dbt.sql
- schema.yml
- enriched_flights.sql
- flights_per_carrier.sql
- schema.yml
- .gitignore
- dbt_project.yml
- profiles.yml
- flights_data.zip
- s3topostgres.py
- .gitkeep
- 01_add_mockdata_to_s3.sh
- 02_install_postgres_dbt.sh
- .airflowignore
- .whirl.env
- dag.py
- README.md
- 02_clean_dbt_log.sh
- 01_show_dbt_log.sh
- airports_dbt.sql
- carriers_dbt.sql
- flights_dbt.sql
- schema.yml
- enriched_flights.sql
- flights_per_carrier.sql
- schema.yml
- .gitignore
- dbt_project.yml
- profiles.yml
- flights_data.zip
- s3tohive.py
- .gitkeep
- 01_add_mockdata_to_s3.sh
- 02_add_dbt.sh
- .airflowignore
- .whirl.env
- dag.py
- README.md
- .gitkeep
- .airflowignore
- .whirl.env
- dag.py
- README.md
- .gitkeep
- .airflowignore
- .whirl.env
- dag.py
- README.md
- .airflowignore
- .whirl.env
- dag.py
- README.md
- .gitkeep
- .airflowignore
- .whirl.env
- dag.py
- README.md
- mocked-data-#ds_nodash#.csv
- 00_install_airflow_providers.sh
- 01_cp_mock_data_to_sftp.sh
- .airflowignore
- .whirl.env
- dag.py
- README.md
- input.json
- readdeltasharing.py
- s3todelta-nonpartitioned.py
- s3todelta.py
- 01_add_mockdata_to_s3.sh
- .airflowignore
- .whirl.env
- dag.py
- README.md
- datasources.yml
- prometheus.yml
- tempo.yml
- input.json
- otel-collector-config.yaml
- s3topostgres.py
- 01_add_mockdata_to_s3.sh
- 03_install_airflow_providers.sh
- 04_install_spark_opentelemetry.sh
- .airflowignore
- .gitignore
- .whirl.env
- dag.py
- README.md
- input.json
- s3tohive.py
- 01_add_mockdata_to_s3.sh
- .airflowignore
- .whirl.env
- dag.py
- README.md
- input.json
- s3topostgres.py
- 01_add_mockdata_to_s3.sh
- 03_install_airflow_providers.sh
- .airflowignore
- .whirl.env
- dag.py
- README.md
- logo.py
- logo.svg
- README.md
- requirements.txt
- conftest.py
- test_dag_validation.py
- test_foundation.py
- test_setup_scripts.py
- .editorconfig
- .gitattributes
- .gitignore
- .whirl.env
- CLAUDE.md
- CONTRIBUTING.md
- LICENSE
- logo.png
- pyproject.toml
- README.md
- SECURITY.md
- uv.lock
- whirl
- whirl-youtube.png
# Installation Guide
1. Get the code
git clone https://github.com/godatadriven/whirl
Downloads the entire project code from GitHub to your computer.
cd whirl
Moves into the project folder you just downloaded.
2. Official Install Script
Easy RecommendedPrerequisites
- Homebrew A package manager for macOS/Linux.
brew install jq
Installs the pre-built package via Homebrew โ no source build required.
After installing, open a new terminal and run the program's version command (e.g. --version) to confirm it worked.
Pulled directly from this repo's README.
3. Docker
EasyPrerequisites
- Git Needed to download the project code from GitHub.
- Docker Desktop Needed to build and run containers. Install it and keep it running in the background.
โ ๏ธ This is a large repository, so this method may point to an internal sub-package rather than the actual core product. Check the full README as well.
docker compose -f envs/airflow-s3-logging/docker-compose.yml up -d --build
Runs the command against the services defined in the compose file.
Run docker compose ps to check the containers are Up. If the README mentions a port, open http://localhost:PORT in your browser.
4. Python
EasyPrerequisites
pip install -r logo/requirements.txt
Installs the Python libraries listed in requirements.txt (or similar).
python <์คํํ ํ์ผ๋ช
>.py # README์์ ์ ํํ ์คํ ํ์ผ๋ช
์ ํ์ธํ์ธ์
Runs the Python script (or module).
If it runs without errors and prints output in the terminal, it worked.
// repository documentation
Was this content helpful?
(0 ratings)