hail

(★ 1,069)

Cloud-native genomic dataframes and batch computing

File Explorer

Download Latest Version (.zip)

Showing a partial file list — download the zip above to see everything.

- - bug-report.yml
  - change-request.yml
  - config.yml
  - new-developer.yml
  - new-feature-request.yml
- - actionlint.yml
  - check-worker-image.yml
  - create-patch-release.yml
  - pip-npm-bump.yml
  - static-and-unit-tests.yml
  - trivy-scan.yml
  - validate-build-yaml.yml
- pull_request_template.md
- .gitignore
- admin-pod.yaml
- Dockerfile
- Makefile
- - __init__.py
  - __main__.py
  - auth.py
  - auth_utils.py
  - exceptions.py
- - 000-initial.sql
  - 001-copy-paste-tokens.sql
  - 002-drop-bucket.sql
  - 003-add-trial-billing-project.sql
  - 004-add-roles.sql
  - 005-rename-gsa-identity.sql
  - 006-support-azure-oauth.sql
  - 007-change-username-collation.sql
  - 008-add-hail-identity-uid.sql
  - 009-add-users-last-active.sql
  - 010-refactor-roles.sql
  - 011-refactor-system-permissions.sql
  - estimated-current.sql
- - __init__.py
  - pytest.ini
  - test_auth_utils.py
  - test_system_permissions.py
- .gitignore
- auth-driver-service-account.yaml
- deployment.yaml
- Dockerfile
- Makefile
- pyproject.toml
- - __init__.py
  - batch.py
  - batch_configuration.py
  - batch_format_version.py
  - exceptions.py
  - file_store.py
  - globals.py
  - inst_coll_config.py
  - instance_config.py
  - publicly_available_images.py
  - resource_usage.py
  - resources.py
  - semaphore.py
  - spec_writer.py
  - utils.py
- - .gitignore
  - .scalafix.conf
  - .scalafmt.conf
  - build.mill
  - mill
- - test-max-scheduling-rate.py
- - 000-initial.sql
  - 001-insert_globals.py
  - 002-create-billing-projects.sql
  - 003-increase-spec-size.sql
  - 004-remove-fk-batches-user-resources.sql
  - 005-change-ready-cores-bigint.sql
  - 006-add-user-resources-ready-cores-token.sql
  - 007-add-lock-in-share-mode.sql
  - 008-add-batch-create-token.sql
  - 009-improve-cancel.sql
  - 010-store-specs-in-gcs.sql
  - 011-add-batches-time-closed.sql
  - 012-add-instance-zone.sql
  - 013-add-batches-time-created-index.sql
  - 014-change_test_worker_disk_size.py
  - 015-add-resources.sql
  - 016-insert_resources.py
  - 017-insert_standing_worker_globals.py
  - 018-insert_local_ssd_resource.py
  - 019-fix-mark-job-complete-on-error.sql
  - 020-add-worker-pd-ssd-data-disk.sql
  - 021-add-aggregated-batch-resources-tokens.sql
  - 022-change_test_worker_pool_size.py
  - 023-add-test-dev-to-billing-project-users.sql
  - 024-add-status-flag-billing-projects.sql
  - 025-add-aggregated-billing-project-resources.sql
  - 026-add-enable-standing-worker.py
  - 027-add-inst-colls.sql
  - 028-add-creating-job-state.sql
  - 029-add-job-private-inst-coll.sql
  - 030-insert_nonpreemptible_resources.py
  - 031-fix-schedule-job.sql
  - 032-increase-test-and-dev-pool-sizes.py
  - 033-fix-provisioning-bug.sql
  - 034-do-not-lock-entire-batch.sql
  - 035-add-fail-fast.sql
  - 036-add-frozen-mode.sql
  - 037-add-instance-config.sql
  - 038-cloud-agnostic.sql
  - 039-support-azure.sql
  - 040-add_azure_tables.py
  - 041-change_azure_test_highcpu_pool.py
  - 042-add-latest-product-versions.sql
  - 043-add_products_and_versions.py
  - 044-kill-more-deadlocks.sql
  - 045-kill-more-deadlocks2.sql
  - 046-no-locks-add-attempt.sql
  - 047-fix-n-cancelled-creating-jobs.sql
  - 048-big-test-instances.py
  - 049-kill-mjc-deadlocks.sql
  - 050-add-nonpreemptible-pools.sql
  - 051-set-test-and-dev-pools-to-8-core-max-2.py
  - 052-set-test-and-dev-jpim-to-max-5.py
  - 053-minimize-deadlock-errors.sql
  - 054-add-resource-ids.sql
  - 055-add_resource_ids.py
  - 056-modify-resource-id-trigger.sql
  - 057-rm-resource-names-agg-resources.sql
  - 058-rm-resource-foreign-keys.py
  - 059-rm-resource-names-agg-resources-pt-2.sql
  - 060-add-att-resources-format-version-lt-3.sql
  - 061-insert_attempt_resources_format_version_lt_3.py
  - 062-revert-attempt-resources-trigger-back-compat.sql
  - 063-add-agg-billing-by-date.sql
  - 064-populate_agg_billing_by_date.py
  - 065-revert-temp-agg-by-date-infra.sql
  - 066-rename-timestamp-to-date.sql
  - 067-add-real-time-billing.sql
  - 068-add-batch-updates.sql
  - 069-populate_batch_updates.py
  - 070-cleanup-add-batch-updates.sql
  - 071-add-jobs-update-id-index.sql
  - 072-no_dev_standing_workers_by_default.py
  - 073-add-commit-updates.sql
  - 074-add-job-regions.sql
  - 075-faster-regions-pool-scheduler.sql
  - 076-add-batch-state-index.sql
  - 077-case-sensitive-billing-project.sql
  - 078-driver_behavior.py
  - 079-cleanup-old-billing-tables.sql
  - 080-add-deduped-resource-ids-to-resources.sql
  - 081-create_resource_dedup_mapping.py
  - 082-config-pool-min-instances.sql
  - 083-add-deduped-resource-ids-to-attempt-resources.sql
  - 084-setup-deduped-resources-migration.sql
  - 085-dedup_attempt_resources.py
  - 086-mitigate-bad-attempt-resources-trigger.sql
  - 087-set-test-and-dev-pools-to-16-core-max-3.py
  - 088-fix-billing-triggers-deduped-resource-id.sql
  - 089-set-test-and-dev-pools-to-max-16-standing-16.py
  - 090-set-test-min-pool-size-to-1.py
  - 091-dedup_billing_project_users_v2.py
  - 092-list-jobs-extra-indices.sql
  - 093-dedup_billing_project_users_by_date.py
  - 094-add-jobs-ready-time.sql
  - 095-add-list-batches-index.sql
  - 096-add-feature-flags.sql
  - 097-add-oms-agent-flag.sql
  - 098-increase_test_max_idle_time.py
  - 099-turn_off_oms_agent_test_dev.py
  - 100-add-billing-index-token.sql
  - 101-jobs-after-update-simple.sql
  - 102-turn_on_billing_compaction.py
  - 103-cleanup-deprecated-functions.sql
  - 104-set-test-pools-to-known-parameters.py
  - 105-add-gcp-support-logs-specs-and-firewall-fees.py
  - 106-add-product-sku.sql
  - 107-dedup_agg_batch_resources.py
  - 108-setup-job-groups.sql
  - 109-dedup_job_resources.py
  - 110-fix-job-groups-state-enum.sql
  - 111-populate_job_groups.py
  - 112-rename-job-groups-tables.sql
  - 113-remove-v2-billing-writes.sql
  - 114-update-ip-fee-resource.py
  - 115-remove-v2-billing-tables.sql
  - 116-finalize-job-groups.sql
  - 117-fix-mark-job-complete-deadlocks.sql
  - 118-jobs-add-n-max-attempts.sql
  - 119-is-job-cancelled.sql
  - 120-add-dockerhub-proxy-flag.sql
  - dedup_billing_project_users.py
  - estimated-current.sql
- - .gitignore
  - .helmignore
  - Chart.yaml.tmpl
  - Dockerfile.batch
  - Dockerfile.worker
  - Makefile
  - values.yaml.tmpl
- - __init__.py
  - billing_projects.py
  - conftest.py
  - failure_injecting_client_session.py
  - pytest.ini
  - test_accounts.py
  - test_aioclient.py
  - test_batch.py
  - test_batch_rare_instances.py
  - test_dag.py
  - test_invariants.py
  - test_scale.py
  - test_time_limited_max_size_cache.py
  - test_utils.py
  - test_worker_image_security.py
  - utils.py
- - stress.py
- .dockerignore
- .gitignore
- az-create-worker-image.sh
- build-batch-worker-image-startup-azure.sh
- build-batch-worker-image-startup-gcp.sh
- deployment.yaml
- Dockerfile
- Dockerfile.worker
- gcp-create-worker-image.sh
- Makefile
- pinned-requirements.txt
- pyproject.toml
- requirements.txt
- service-account.yaml
- test-sa.yaml
- .gitignore
- deployment.yaml
- Dockerfile
- gateway.nginx.conf
- gzip.conf
- Makefile
- nginx.conf
- service.yaml
- - convert-cloud-credentials-to-docker-auth-config
  - Dockerfile
- - __init__.py
  - __main__.py
  - build.py
  - build_selection.py
  - ci.py
  - constants.py
  - environment.py
  - envoy.py
  - github.py
  - globals.py
  - utils.py
- - 000-initial.sql
  - 001-invalidated-batches.sql
  - 002-add-frozen-mode.sql
  - 003-active-namespaces.sql
  - 004-fix-default-namespace.py
  - 005-record-failure-notifications.sql
  - 006-add-service-rate-limits.sql
  - 007-retried-tests.sql
  - estimated-current.sql
- - pytest.ini
  - test_ci.py
  - test_envoy.py
- - conftest.py
  - pytest.ini
  - test_build_selection.py
  - test_github_unit.py
  - test_utils.py
- .dockerignore
- .gitignore
- bootstrap.py
- bootstrap.yaml
- bootstrap_create_accounts.py
- ci-agent.yaml
- create_database.py
- create_local_database.py
- delete_dev_database.py
- deployment.yaml
- Dockerfile
- Dockerfile.ci-utils
- jinja2_render.py
- Makefile
- pinned-requirements.txt
- pyproject.toml
- requirements.txt
- wait-for.py
- - extract.1000_Genomes_phase3_GRCh37.sh
  - extract.1000_Genomes_phase3_GRCh38.sh
  - extract.1000_Genomes_phase3_samples.sh
  - extract.CADD_v1.4_GRCh37.sh
  - extract.CADD_v1.4_GRCh38.sh
  - extract.DANN_GRCh37.sh
  - extract.Ensembl_homo_sapiens_features.GRCh37.sh
  - extract.Ensembl_homo_sapiens_features.GRCh38.sh
  - extract.Ensembl_homo_sapiens_low_complexity_regions.GRCh37.sh
  - extract.Ensembl_homo_sapiens_low_complexity_regions.GRCh38.sh
  - extract.Ensembl_homo_sapiens_reference_genome.GRCh37.sh
  - extract.Ensembl_homo_sapiens_reference_genome.GRCh38.sh
  - extract.gencode.v30.annotation.gff3.sh
  - extract.gnomad_v2.1.1_lof_metrics_by_gene.sh
  - extract.GTEx_v7_annotations.sh
  - extract.GTEx_v7_eQTL.sh
  - extract.GTEx_v7_RNA_seq.sh
  - extract.Human_Cell_Atlas_bone_marrow_immunocytes.py
  - extract.Human_Cell_Atlas_cord_blood_immunocytes.py
  - extract.LDSC_baseline_v1.1_bed_files_GRCh37.sh
  - extract.LDSC_baseline_v1.1_ld_scores.GRCh37.sh
  - extract.LDSC_baselineLD_v2.2_bed_files_GRCh37.sh
  - extract.LDSC_baselineLD_v2.2_ld_scores.GRCh37.sh
  - extract_1000_Genomes_30x_GRCh38_samples.sh
  - extract_1000_Genomes_NYGC_30x_GRCh38.py
  - extract_CADD.py
  - extract_dbSNP.py
  - extract_gencode_v35_annotation_gtf.sh
- - load.1000_Genomes_phase3_autosomes.GRCh37.py
  - load.1000_Genomes_phase3_autosomes.GRCh38.py
  - load.1000_Genomes_phase3_chrMT.GRCh37.py
  - load.1000_Genomes_phase3_chrX.GRCh37.py
  - load.1000_Genomes_phase3_chrX.GRCh38.py
  - load.1000_Genomes_phase3_chrY.GRCh37.py
  - load.1000_Genomes_phase3_chrY.GRCh38.py
  - load.1000_Genomes_phase3_European_autosomes_maf_gt_001_block_matrix.py
  - load.1000_Genomes_phase3_European_autosomes_maf_gt_001_block_matrix_standardized.py
  - load.1000_Genomes_phase3_relationships.py
  - load.1000_Genomes_phase3_samples.py
  - load.1000_Genomes_phase3_sites.GRCh37.py
  - load.CADD.py
  - load.DANN.py
  - load.dbSNP.py
  - load.Ensembl.py
  - load.Ensembl_homo_sapiens_features.py
  - load.Ensembl_homo_sapiens_low_complexity_regions.py
  - load.Ensembl_homo_sapiens_reference_genome.py
  - load.gencode_v30_annotation_gff3.py
  - load.GERP++.py
  - load.gnomad_v2.1.1_lof_metrics_by_gene.py
  - load.GTEx.py
  - load.GTEx_v7.py
  - load.GTEx_v7_RNA_seq.py
  - load.LDSC_baseline_v1.1_bed_files.py
  - load.LDSC_baseline_v1.1_ld_scores.GRCh37.py
  - load.LDSC_baselineLD_v2.2_annotations.GRCh37.py
  - load.LDSC_baselineLD_v2.2_ld_scores.GRCh37.py
  - load.LDSC_baselineLD_v2.2_ld_scores.py
  - load.UK_Biobank_Rapid_GWAS.py
- - 1kg_NYGC_30x_datasets.ipynb
  - CADD_datasets.ipynb
  - dbSNP_datasets.ipynb
  - GENCODE.ipynb
  - giant_datasets.ipynb
  - GTEx_MatrixTables.ipynb
  - GTEx_Tables.ipynb
  - reformat_buckets.ipynb
  - reformat_buckets.txt
  - reformat_buckets_mappings.json
- annotation_ui.json
- README.md
- - development_tools.md
- - fast-restarts.rst
  - hail-query-lifecycle.md
  - query-on-batch.md
  - randomness.md
  - testing.md
- - connecting_docker_to_container_registry_creds.md
  - creating-a-developer-account.md
  - docker-hub-cache.md
  - flaky_tests.md
  - gateways.md
  - Hail Batch Architectural Diagram.png
  - key-rotation.md
  - kubernetes-operations.md
  - letsencrypt.md
  - rotating-github-oauth-token.md
  - services-development-faq.md
  - setting_the_kubectl_context.md
  - tls-cookbook.md
  - tls.md
  - updating-third-party-images.md
- algolia.md
- development-process.md
- dismiss-review.png
- git-practices.md
- google-cloud-cookbook.md
- gvs.md
- hail-for-new-engineers.md
- hail-overview.md
- hail-production-issues.md
- pip-dependencies.md
- python-version-bump.md
- refreshing-trivy-scanner-github-secret.md
- releasing.md
- check_build_yaml_order.py
- check_public_ip_no_ingress.py
- dev_proxy.py
- functions.sh
- generate_gcp_ar_cleanup_policy.py
- hail-build-llvm
- hail-build-native-compiler
- krsync.sh
- migrate-gsa-permissions.sh
- prune_acr.sh
- rotate_keys.py
- sync.py
- trivy-scan-trigger.py
- - controller.sh
  - curlrc
  - Dockerfile
  - hail-apt-get-install
  - hail-pip-install
  - pip.conf
  - retry
  - uv.toml
- - Makefile
  - mirror_images.sh
- - compose.yaml
  - db.yaml
- - copy_images.sh
  - images.txt
  - Makefile
- - vep.py
- .gitignore
- copy_image.sh
- core-site.xml
- Dockerfile.base
- Dockerfile.hail-dev
- .gitignore
- deployment.yaml
- envoy.yaml
- Makefile
- service.yaml
- - __init__.py
  - auth.py
  - auth_utils.py
  - clients.py
  - cloud_config.py
  - csrf.py
  - database.py
  - http_server_utils.py
  - k8s_cache.py
  - metrics.py
  - profiling.py
  - py.typed
  - session.py
  - system_permissions.py
  - time_limited_max_size_cache.py
- pinned-requirements.txt
- pyproject.toml
- requirements.txt
- deployment.yaml
- Makefile
- - auth-b.png
  - auth-w.png
  - batch-b.png
  - batch-w.png
  - benchmark-b.png
  - benchmark-w.png
  - ci-b.png
  - ci-w.png
  - disks-b.png
  - disks-w.png
  - helix-b.png
  - helix-w.png
  - log-b.png
  - log-w.png
  - notebook-b.png
  - notebook-w.png
  - notebook2-b.png
  - notebook2-w.png
  - query-b.png
  - query-w.png
- - logo1.png
  - logo2.png
- - logo1.svg
  - logo2.svg
- - ApproximateQuantiles_test.cpp
  - cache-tests.cpp
  - Decoder.cpp
  - Encoder.cpp
  - FS.cpp
  - ibs.cpp
  - ibs.h
  - Logging.cpp
  - Makefile
  - NativeBoot.cpp
  - NativeCodeSuite.cpp
  - NativeLongFunc.cpp
  - NativeModule.cpp
  - NativePtr.cpp
  - NativeStatus.cpp
  - ObjectArray.cpp
  - PartitionIterators.cpp
  - README.md
  - Region.cpp
  - Region_test.cpp
  - test.cpp
  - Upcalls.cpp
- - package.mill
- .gitignore
- .mill-jvm-opts
- .scalafix.conf
- .scalafmt.conf
- AGENTS.md
- build.mill
- CLAUDE.md
- code_style.xml
- Dockerfile.hail-run
- env_var.mk
- generate_splits.py
- Makefile
.codacy.yaml
.dir-locals.el
.dockerignore
.git-blame-ignore-revs
.gitattributes
.gitignore
.pre-commit-config.yaml
.zenodo.json
AUTHORS
build.yaml
check-sql.sh
check_pip_requirements.sh
CITATION.cff
CODEOWNERS
config.mk
docker-build.sh
generate-pip-lockfile.sh
LICENSE
Makefile
README.md

// repository documentation

Was this content helpful?

★ 0 (0 ratings)

hail

File Explorer

Today's picks

Was this content helpful?

Recent Feedback

hail

File Explorer

Today's picks

Was this content helpful?

Recent Feedback

Download README