datafusion-distributed

(★ 127)

Library for bringing distributed capabilities to Apache DataFusion

File Explorer

Download Latest Version (.zip)

- - ci.yml
  - docs.yml
- - broadcast_cache_scenarios.rs
  - local_repartition.rs
  - shuffle.rs
  - transport.rs
- - compare.rs
  - lib.rs
  - main.rs
  - prepare_clickbench.rs
  - prepare_tpcds.rs
  - prepare_tpch.rs
  - results.rs
  - run.rs
  - stats.rs
- build.rs
- Cargo.toml
- gen-clickbench.sh
- gen-tpcds.sh
- gen-tpch.sh
- README.md
- run.sh
- - main.rs
- Cargo.toml
- README.md
- - main.rs
- Cargo.toml
- regen.sh
- - cluster.md
  - cluster.rs
  - console.md
  - console_run.rs
  - console_worker.rs
  - tpcds_runner.md
  - tpcds_runner.rs
- - app.rs
  - input.rs
  - main.rs
  - state.rs
  - worker.rs
- .gitignore
- Cargo.toml
- README.md
- - conf.py
  - index.rst
- - 3.0.0.md
  - 4.0.0.md
- .gitignore
- build.sh
- Makefile
- README.md
- requirements.txt
- custom_distributed_partial_reduction_tree.md
- custom_distributed_partial_reduction_tree.rs
- custom_execution_plan.md
- custom_execution_plan.rs
- custom_worker_url_routing.md
- custom_worker_url_routing.rs
- in_memory.md
- in_memory_cluster.rs
- localhost.md
- localhost_run.rs
- localhost_versioned_run.rs
- localhost_versioned_worker.rs
- localhost_worker.rs
- work_unit_feed.md
- work_unit_feed.rs
- - distributed_codec.rs
  - mod.rs
  - physical_plan.rs
  - user_codec.rs
- - children_helpers.rs
  - maybe_encoded.rs
  - mod.rs
  - once_lock.rs
  - recursion.rs
  - task_context_helpers.rs
  - time.rs
  - uuid.rs
  - vec.rs
- - distributed.rs
  - latency_metric.rs
  - metrics_store.rs
  - mod.rs
  - prepare_dynamic_plan.rs
  - prepare_static_plan.rs
  - query_coordinator.rs
- - AGENTS.md
  - CLAUDE.md
  - distributed_config.rs
  - distributed_query_planner.rs
  - inject_network_boundaries.rs
  - insert_broadcast.rs
  - insert_children_isolator_union.rs
  - mod.rs
  - network_boundary.rs
  - normalize_collect_joins.rs
  - partial_reduce_below_network_shuffles.rs
  - prepare_network_boundaries.rs
  - push_fetch_into_network_coalesce.rs
  - session_state_builder_ext.rs
- - AGENTS.md
  - CLAUDE.md
  - common.rs
  - desired_task_count.rs
  - mod.rs
  - route_tasks.rs
  - scale_up_leaf_node.rs
  - worker_plan_rewrite.rs
- - broadcast.rs
  - children_isolator_union.rs
  - common.rs
  - distributed_leaf.rs
  - metrics.rs
  - mod.rs
  - network_broadcast.rs
  - network_coalesce.rs
  - network_shuffle.rs
  - sampler.rs
- - bytes_metric.rs
  - latency_metric.rs
  - max_gauge_metric.rs
  - mod.rs
  - task_metrics_collector.rs
  - task_metrics_rewriter.rs
- - channel_resolver.rs
  - mod.rs
  - worker_channel.rs
- - in_memory_channel_resolver.rs
  - insta.rs
  - localhost.rs
  - metrics.rs
  - mock_exec.rs
  - mod.rs
  - parquet.rs
  - plans.rs
  - property_based.rs
  - routing.rs
  - session_context.rs
  - test_work_unit_feed.rs
  - work_unit_file_scan.rs
- - mod.rs
  - remote_work_unit_feed.rs
  - work_unit.rs
  - work_unit_feed.rs
  - work_unit_feed_provider.rs
  - work_unit_feed_registry.rs
- - impl_coordinator_channel.rs
  - impl_execute_task.rs
  - mod.rs
  - session_builder.rs
  - single_write_multi_read.rs
  - task_data.rs
  - worker_connection_pool.rs
  - worker_service.rs
- config_extension_ext.rs
- distributed_ext.rs
- explain_analyze.rs
- lib.rs
- passthrough_headers.rs
- stage.rs
- worker_resolver.rs
- - generate_parquet_from_csv.sql
- - README.md
- - result-000000.parquet
  - result-000001.parquet
  - result-000002.parquet
- flights-1m.parquet
- clickbench_correctness_test.rs
- clickbench_plans_test.rs
- custom_config_extension.rs
- custom_extension_codec.rs
- distributed_aggregation.rs
- distributed_unions.rs
- empty_columns_between_workers.rs
- error_propagation.rs
- explain_analyze.rs
- introspection.rs
- join.rs
- local_connections.rs
- metrics_collection.rs
- multi_task_collect_join_repros.rs
- passthrough_headers.rs
- stateful_data_cleanup.rs
- stateful_execution_plan.rs
- task_estimator_test.rs
- tpcds_correctness_test.rs
- tpcds_plans_test.rs
- tpch_correctness_test.rs
- tpch_plans_test.rs
- udfs.rs
- work_unit_feed.rs
- worker_plan_hook.rs
.gitattributes
.gitignore
AGENTS.md
Cargo.lock
Cargo.toml
CLAUDE.md
clippy.toml
LICENSE.txt
README.md
rust-toolchain.toml

# Installation Guide

1. Get the code

git clone https://github.com/datafusion-contrib/datafusion-distributed

Downloads the entire project code from GitHub to your computer.

cd datafusion-distributed

Moves into the project folder you just downloaded.

2. Rust

Medium Recommended

Prerequisites

Git Needed to download the project code from GitHub.
Rust (rustup) Installing via rustup also installs cargo.

cargo build --release

Compiles the Rust project.

cargo run

Builds and then immediately runs the program.

✅ If cargo build finishes without errors, it worked. The executable is created under target/.