lakehouse-engine
The Lakehouse Engine is a configuration driven Spark framework, written in Python, serving as a scalable and distributed engine for several lakehouse algorithms, data flows and utilities for Data Products.
File Explorer
Download Latest Version (.zip)- bug_report.md
- feature_request.md
- create_release.yaml
- pull_request_template.md
- gab_advantages.png
- gab_build_insert_sql_instruction.png
- gab_dimensions.png
- gab_extended_window_calculator.png
- gab_insert_use_case.png
- gab_lkp_query_builder.png
- gab_metrics.png
- gab_metrics_calculations.png
- gab_metrics_configuration.png
- gab_overview.png
- gab_query_metrics.png
- gab_recon.png
- gab_sample_templated_query.png
- gab_stages_configuration.png
- gab_use_case.png
- gab_view_name_and_filters.png
- 1_article_category.sql
- 2_f_agg_dummy_sales_kpi.sql
- dim_calendar.sql
- dummy_sales_kpi.sql
- gab_log_events.sql
- gab_use_case_results.sql
- lkp_query_builder.sql
- gab.py
- gab_dim_calendar.py
- gab_job_manager.py
- query_builder_helper.py
- databricks_job_utils.py
- query_builder_utils.py
- dq_dashboard.png
- fixed_window.png
- heartbeat_dependency_flag.png
- heartbeat_sensor_os.png
- lakehouse_dp_usage.drawio.png
- lakehouse_engine_logo.png
- lakehouse_engine_logo_symbol_large.png
- lakehouse_engine_logo_symbol_small.png
- row_tagging.png
- sensor_os.png
- watermarking.png
- dq_checks_table_w_no_rules.png
- missing_expectation.png
- missing_expectation_parameters.png
- examples.json
- gen_ref_nav.py
- mkdocs.yml
- mkdocs_macros.py
- render_docs.py
- .bumpversion.cfg
- bandit.yaml
- docker-compose-spark-connect.yml
- Dockerfile
- flake8.conf
- Jenkinsfile
- Jenkinsfile_deploy
- meta.yaml
- requirements.lock
- requirements.txt
- requirements_azure.lock
- requirements_azure.txt
- requirements_cicd.txt
- requirements_dq.lock
- requirements_dq.txt
- requirements_full.lock
- requirements_os.lock
- requirements_os.txt
- requirements_sftp.lock
- requirements_sftp.txt
- requirements_sharepoint.lock
- requirements_sharepoint.txt
- __init__.py
- heartbeat.py
- sensor.py
- __init__.py
- algorithm.py
- data_loader.py
- dq_validator.py
- exceptions.py
- gab.py
- reconciliator.py
- __init__.py
- engine.yaml
- __init__.py
- dbfs_file_manager.py
- definitions.py
- exec_env.py
- executable.py
- file_manager.py
- gab_manager.py
- gab_sql_generator.py
- s3_file_manager.py
- sensor_manager.py
- table_manager.py
- __init__.py
- expect_column_pair_a_to_be_not_equal_to_b.py
- expect_column_pair_a_to_be_smaller_or_equal_than_b.py
- expect_column_pair_date_a_to_be_greater_than_or_equal_to_date_b.py
- expect_column_values_to_be_date_not_older_than.py
- expect_column_values_to_not_be_null_or_empty_string.py
- expect_multicolumn_column_a_must_equal_b_or_c.py
- expect_queried_column_agg_value_to_be.py
- __init__.py
- dq_factory.py
- exceptions.py
- validator.py
- __init__.py
- dataframe_reader.py
- file_reader.py
- jdbc_reader.py
- kafka_reader.py
- query_reader.py
- sap_b4_reader.py
- sap_bw_reader.py
- sftp_reader.py
- sharepoint_reader.py
- table_reader.py
- __init__.py
- console_writer.py
- dataframe_writer.py
- delta_merge_writer.py
- file_writer.py
- jdbc_writer.py
- kafka_writer.py
- rest_api_writer.py
- sharepoint_writer.py
- table_writer.py
- __init__.py
- exceptions.py
- reader.py
- reader_factory.py
- writer.py
- writer_factory.py
- __init__.py
- email_notifier.py
- exceptions.py
- notification_templates.py
- __init__.py
- cdf_processor.py
- dataset_optimizer.py
- notifier.py
- notifier_factory.py
- sensor_terminator.py
- spark_terminator.py
- terminator_factory.py
- __init__.py
- aggregators.py
- column_creators.py
- column_reshapers.py
- condensers.py
- custom_transformers.py
- data_maskers.py
- date_transformers.py
- exceptions.py
- filters.py
- joiners.py
- null_handlers.py
- optimizers.py
- regex_transformers.py
- repartitioners.py
- transformer_factory.py
- unions.py
- watermarker.py
- __init__.py
- config_utils.py
- __init__.py
- jdbc_extraction_utils.py
- sap_b4_extraction_utils.py
- sap_bw_extraction_utils.py
- sftp_extraction_utils.py
- __init__.py
- dbfs_storage.py
- file_storage.py
- file_storage_functions.py
- local_fs_storage.py
- s3_storage.py
- __init__.py
- acon_utils.py
- databricks_utils.py
- dq_utils.py
- engine_usage_stats.py
- expectations_utils.py
- file_utils.py
- gab_utils.py
- logging_handler.py
- paimon_utils.py
- rest_api.py
- schema_utils.py
- sharepoint_utils.py
- spark_utils.py
- sql_parser_utils.py
- __init__.py
- engine.py
- __init__.py
- append_load_from_jdbc_with_permissive_mode.md
- __init__.py
- append_load_with_failfast.md
- __init__.py
- batch_delta_load_init_delta_backfill_with_merge.md
- __init__.py
- custom_transformer.md
- __init__.py
- custom_transformer_sql.md
- __init__.py
- extract_from_sap_b4_adso.md
- __init__.py
- extract_from_sap_bw_dso.md
- __init__.py
- extract_from_sftp.md
- __init__.py
- extract_using_jdbc_connection.md
- __init__.py
- filtered_full_load.md
- __init__.py
- filtered_full_load_with_selective_replace.md
- __init__.py
- flatten_schema_and_explode_columns.md
- __init__.py
- full_load.md
- __init__.py
- read_from_dataframe.md
- __init__.py
- read_from_sharepoint.md
- __init__.py
- streaming_append_load_with_malformed.md
- __init__.py
- streaming_append_load_with_terminator.md
- __init__.py
- streaming_delta_load_with_group_and_rank_condensation.md
- __init__.py
- streaming_delta_with_late_arriving_and_out_of_order_events.md
- __init__.py
- write_and_read_dataframe.md
- __init__.py
- write_to_console.md
- __init__.py
- write_to_rest_api.md
- __init__.py
- write_to_sharepoint.md
- __init__.py
- writing_paimon_tables.md
- __init__.py
- data_loader.md
- __init__.py
- custom_expectations.md
- __init__.py
- data_quality_validator.md
- __init__.py
- minimal_example.md
- __init__.py
- prisma.md
- __init__.py
- result_sink.md
- __init__.py
- row_tagging.md
- __init__.py
- validations_failing.md
- __init__.py
- data_quality.md
- __init__.py
- step_by_step.md
- __init__.py
- gab.md
- managerhelper.md
- operations-script.js
- operations-styles-mkdocs.css
- styles-mkdocs.css
- __init__.py
- reconciliator.md
- __init__.py
- delta_table.md
- __init__.py
- heartbeat_sensor_data_feed.md
- __init__.py
- kafka.md
- __init__.py
- manual_table.md
- __init__.py
- sap_bw_b4.md
- __init__.py
- trigger_file.md
- __init__.py
- update_heartbeat_sensor_status.md
- __init__.py
- heartbeat.md
- __init__.py
- delta_table.md
- __init__.py
- delta_upstream_sensor_table.md
- __init__.py
- file.md
- __init__.py
- jdbc_table.md
- __init__.py
- kafka.md
- __init__.py
- sap_bw_b4.md
- __init__.py
- update_sensor_status.md
- __init__.py
- sensor.md
- __init__.py
- sensors.md
- __init__.py
- lakehouse_engine_usage.md
- cricket_dq_tutorial.py
- tpch_load_and_analysis_tutorial.py
- __init__.py
- engine.yaml
- __init__.py
- test_custom_expectations.py
- test_expectation_validity.py
- __init__.py
- test_data_loader_custom_transformer_calculate_kpi.py
- test_data_loader_custom_transformer_delta_load.py
- test_data_loader_custom_transformer_sql_transformation.py
- __init__.py
- test_delta_load_group_and_rank.py
- test_delta_load_merge_options.py
- test_delta_load_record_mode_cdc.py
- __init__.py
- test_chain_transformations.py
- test_column_creators.py
- test_column_reshapers.py
- test_data_maskers.py
- test_date_transformers.py
- test_drop_duplicate_rows.py
- test_joiners.py
- test_multiple_transformations.py
- test_null_handlers.py
- test_optimizers.py
- test_regex_transformers.py
- test_unions.py
- test_watermarker.py
- __init__.py
- test_append_load.py
- test_data_quality.py
- test_dq_validator.py
- test_engine_usage_stats.py
- test_extract_from_sap_b4.py
- test_extract_from_sap_bw.py
- test_file_manager_dbfs.py
- test_file_manager_s3.py
- test_full_load.py
- test_gab.py
- test_heartbeat.py
- test_jdbc_reader.py
- test_materialize_cdf.py
- test_notification.py
- test_reconciliation.py
- test_schema_evolution.py
- test_sensors.py
- test_sftp_reader.py
- test_sharepoint_reader.py
- test_sharepoint_writer.py
- test_table_manager.py
- test_writers.py
- part-01.csv
- part-02.csv
- part-03.csv
- batch.json
- batch_init.json
- part-01.csv
- part-01.csv
- part-02.csv
- part-03.csv
- batch.json
- batch_init.json
- part-01.csv
- part-01.csv
- part-02.csv
- part-03.csv
- streaming.json
- part-01.csv
- part-01.csv
- streaming.json
- dq_control_success.csv
- part-01.csv
- part-02.csv
- batch.json
- dq_sales_schema.json
- streaming.json
- dq_control_success.csv
- part-01.csv
- part-02.csv
- batch.json
- dq_sales_schema.json
- streaming.json
- dq_control_success.csv
- part-01.csv
- part-02.csv
- batch.json
- dq_sales_schema.json
- streaming.json
- dq_control_success.csv
- part-01.csv
- part-02.csv
- batch.json
- dq_sales_schema.json
- streaming.json
- dq_control_success.csv
- part-01.csv
- part-02.csv
- batch.json
- dq_sales_schema.json
- streaming.json
- dq_control_success.csv
- part-01.csv
- part-02.csv
- batch.json
- dq_sales_schema.json
- streaming.json
- dq_control_success.csv
- part-01.csv
- part-02.csv
- batch.json
- dq_sales_schema.json
- streaming.json
- part-01.csv
- part-01.csv
- control_schema.json
- source_schema.json
- part-01.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- part-01.csv
- part-01.csv
- control_schema.json
- source_schema.json
- data_validator.json
- data_validator_schema.json
- sales.json
- sales_schema.json
- test_db.dq_functions_source_load_with_dq_table_delta_with_dupl_tag_gen_fail_init.csv
- test_db.dq_functions_source_load_with_dq_table_delta_with_dupl_tag_gen_fail_new.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- streaming_init.json
- streaming_new.json
- data_validator.json
- data_validator_schema.json
- sales.json
- sales_schema.json
- test_db.dq_functions_source_load_with_dq_table_delta_with_duplicates_tag_init.csv
- test_db.dq_functions_source_load_with_dq_table_delta_with_duplicates_tag_new.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- streaming_init.json
- streaming_new.json
- data_validator.json
- data_validator_schema.json
- sales.json
- sales_schema.json
- test_db.dq_functions_source_load_with_dq_table_full_overwrite_tag_init.csv
- test_db.dq_functions_source_load_with_dq_table_full_overwrite_tag_new.csv
- part-01.csv
- part-02.csv
- batch_init.json
- batch_new.json
- data_validator.json
- data_validator_schema.json
- sales.json
- sales_schema.json
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- streaming_init.json
- streaming_new.json
- data_validator.json
- data_validator_schema.json
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- streaming_init.json
- streaming_new.json
- data_validator.json
- data_validator_schema.json
- sales.json
- sales_schema.json
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- streaming_init.json
- streaming_new.json
- data_validator.json
- data_validator_schema.json
- part-01.csv
- part-02.csv
- batch_init.json
- batch_new.json
- data_validator.json
- data_validator_schema.json
- sales.json
- sales_schema.json
- part-01.csv
- part-02.csv
- batch_init.json
- batch_new.json
- data_validator.json
- data_validator_schema.json
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- streaming_init.json
- streaming_new.json
- data_validator.csv
- test_db.dq_functions_source_dq_failure.csv
- test_db.dq_functions_source_dq_failure_error_disabled.csv
- test_db.dq_functions_source_dq_failure_max_percentage.csv
- test_db.dq_functions_source_dq_success.csv
- part-01.csv
- batch.csv
- streaming.csv
- WE_SO_SCL_202108111400000000.csv
- WE_SO_SCL_202108111500000000.csv
- WE_SO_SCL_202108111600000000.csv
- batch_delta.json
- batch_init.json
- control_batch_schema.json
- control_streaming_schema.json
- source_schema.json
- streaming_delta.json
- batch.csv
- streaming.csv
- WE_SO_SCL_202108111400000000.csv
- WE_SO_SCL_202108111500000000.csv
- WE_SO_SCL_202108111600000000.csv
- batch_delta.json
- batch_init.json
- control_batch_schema.json
- control_streaming_schema.json
- source_schema.json
- streaming_delta.json
- batch.csv
- WE_SO_SCL_202108111400000000.csv
- WE_SO_SCL_202108111500000000.csv
- batch_delta.json
- batch_init.json
- batch.csv
- WE_SO_SCL_202108111400000000.csv
- WE_SO_SCL_202108111500000000.csv
- batch_delta.json
- batch_init.json
- batch.csv
- WE_SO_SCL_202108111400000000.csv
- WE_SO_SCL_202108111500000000.csv
- batch_delta.json
- batch_init.json
- control_batch_schema.json
- source_schema.json
- part-01.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- part-05.csv
- batch_backfill.json
- batch_delta.json
- batch_init.json
- part-01.csv
- part-02.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- batch_delta.json
- batch_init.json
- part-01.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- batch_delta.json
- batch_init.json
- streaming_delta.json
- part-01.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- batch_delta.json
- batch_init.json
- streaming_delta.json
- part-01.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- batch_delta.json
- batch_init.json
- part-01.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- batch_delta.json
- batch_init.json
- part-01.csv
- part-01.json
- part-02.json
- part-03.json
- part-04.json
- batch_delta.json
- batch_init.json
- data_restore_control.csv
- dq_control_failure.csv
- dq_control_failure_disabled.csv
- dq_control_success.csv
- dq_control_success_explode.csv
- dq_control_success_explode_disabled.csv
- test_db.dq_functions_source_table_failure.csv
- test_db.dq_functions_source_table_success.csv
- part-01.csv
- part-02.csv
- test_db.dq_functions_streaming_dataframe_two_runs_first_run.csv
- test_db.dq_functions_streaming_dataframe_two_runs_second_run.csv
- test_db.dq_functions_source_table_failure.csv
- test_db.dq_functions_source_table_success.csv
- test_db.dq_functions_source_table_failure.csv
- test_db.dq_functions_source_table_success.csv
- test_db.dq_table_rule_id_failure.csv
- test_db.dq_table_rule_id_success.csv
- test_db.dq_functions_source_table_failure.csv
- test_db.dq_functions_source_table_success.csv
- test_db.dq_functions_source_table_failure.csv
- test_db.dq_functions_source_table_success.csv
- test_db.dq_table_rule_id_failure.csv
- test_db.dq_table_rule_id_success.csv
- test_db.dq_functions_source_table_failure.csv
- test_db.dq_functions_source_table_success.csv
- test_db.dq_functions_source_table_failure.csv
- test_db.dq_functions_source_table_success.csv
- batch.json
- dq_sales_schema.json
- streaming.json
- control.json
- source.csv
- control.json
- source.csv
- control.json
- source.csv
- control.json
- dummy_table.csv
- dummy_table_join_condition.csv
- dummy_table_schema.json
- dummy_table.csv
- dummy_table_1.csv
- dummy_table_2.csv
- rspmrequest.csv
- dummy_table_schema.json
- rspmrequest_schema.json
- dummy_table.csv
- dummy_table_join_condition.csv
- dummy_table_schema.json
- dummy_table.csv
- dummy_table_cl_1.csv
- dummy_table_cl_2.csv
- rspmrequest.csv
- dummy_table_cl_schema.json
- dummy_table_schema.json
- rspmrequest_schema.json
- RSBASIDOC.csv
- RSTSODS.csv
- RSBASIDOC_schema.json
- RSTSODS_schema.json
- dummy_table.csv
- dummy_table_join_condition.csv
- dummy_table_schema.json
- dummy_table.csv
- dummy_table_cl_1.csv
- dummy_table_cl_2.csv
- rsodsactreq.csv
- dummy_table_cl_schema.json
- dummy_table_schema.json
- rsodsactreq_schema.json
- dummy_table.csv
- dummy_table_actreq_timestamp.csv
- dummy_table_join_condition.csv
- dummy_table_schema.json
- dummy_table.csv
- dummy_table_1.csv
- dummy_table_2.csv
- rsodsactreq.csv
- dummy_table_schema.json
- rsodsactreq_schema.json
- acon_copy_directory.json
- acon_copy_directory_dry_run.json
- acon_copy_single_object.json
- acon_delete_objects.json
- acon_delete_objects_dry_run.json
- acon_move_objects.json
- acon_move_objects_dry_run.json
- acon_check_restore_status_directory.json
- acon_check_restore_status_single_object.json
- acon_copy_directory.json
- acon_copy_directory_dry_run.json
- acon_copy_single_object.json
- acon_copy_single_object_dry_run.json
- acon_delete_objects.json
- acon_delete_objects_dry_run.json
- acon_request_restore_directory.json
- acon_request_restore_single_object.json
- acon_request_restore_to_destination_and_wait_directory.json
- acon_request_restore_to_destination_and_wait_single_object.json
- acon_request_restore_to_destination_and_wait_single_object_raise_error.json
- part-01.csv
- part-01.csv
- part-02.csv
- batch.json
- batch_init.json
- part-01.csv
- part-01.csv
- part-02.csv
- batch.json
- batch_init.json
- part-01.csv
- part-01.csv
- part-02.csv
- batch.json
- batch_init.json
- vw_dummy_sales_kpi.csv
- vw_nam_orders_all_snapshot.csv
- vw_nam_orders_filtered_snapshot.csv
- vw_negative_offset_orders_all.csv
- vw_negative_offset_orders_filtered.csv
- vw_orders_all.csv
- vw_orders_all_snapshot.csv
- vw_orders_filtered.csv
- vw_orders_filtered_snapshot.csv
- vw_dummy_sales_kpi.json
- vw_orders.json
- calendar.json
- dummy_sales_kpi.json
- gab_log_events.json
- gab_use_case_results.json
- lkp_query_builder.json
- order_events.json
- dummy_sales_kpi.csv
- lkp_query_builder.csv
- order_events.csv
- dummy_sales_kpi.json
- lkp_query_builder.json
- order_events.json
- dummy_sales_kpi.json
- 1_article_category.sql
- 2_dummy_sales_kpi.sql
- order_events.json
- order_events_nam.json
- order_events_negative_timezone_offset.json
- order_events_snapshot.json
- skip_use_case_by_empty_reconciliation.json
- skip_use_case_by_empty_requested_cadence.json
- skip_use_case_by_not_configured_cadence.json
- skip_use_case_by_unexisting_cadence.json
- 1_order_events.sql
- ctr_heart_tbl_heartb_feed.csv
- ctrl_heart_tbl_exec_sensor.csv
- ctrl_heart_tbl_trigger_job.csv
- ctrl_heart_tbl_updated.csv
- ctrl_sensor_tbl_upd_status.json
- ctrl_heart_tbl_schema.json
- ctrl_heart_tbl_trig_schema.json
- ctr_heart_tbl_heartb_feed.csv
- ctrl_heart_tbl_exec_sensor.csv
- ctrl_heart_tbl_trigger_job.csv
- ctrl_heart_tbl_updated.csv
- ctrl_sensor_tbl_upd_status.json
- ctrl_heart_tbl_schema.json
- heartbeat_sensor_control_table.json
- sensor_table.json
- setup_heartbeat_data.csv
- setup_sensor_data.json
- schema_sensor_df.json
- heartbeat_sensor_control_table.json
- sensor_table.json
- setup_heartbeat_data.csv
- setup_sensor_data.json
- schema_sensor_df.json
- part-01.csv
- part-01.csv
- batch_init.json
- batch_init.json
- batch_init.json
- part-01.csv
- part-01.csv
- batch_init.json
- batch_init.json
- part-01_cdf.csv
- part-01.csv
- part-02.csv
- streaming_with_cdf.sql
- acon_create_table.json
- control_schema.json
- streaming_with_clean_and_vacuum.json
- streaming_without_clean_cdf.json
- test_attachement.txt
- test_image.png
- current.json
- current_different_rows.json
- current_fail.json
- current_nulls_and_zeros.json
- current_nulls_and_zeros_fail.json
- truth.json
- truth_different_rows.json
- truth_empty.json
- truth_nulls_and_zeros.json
- truth_nulls_and_zeros_fail.json
- part-02.csv
- part-03.csv
- part-05.csv
- part-06.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- part-05.csv
- part-06.csv
- control_schema.json
- control_schema_add_column.json
- control_schema_rename.json
- source_part-01_schema.json
- source_part-02_schema.json
- source_part-03_schema.json
- source_part-04_schema.json
- source_part-05_schema.json
- source_part-06_schema.json
- batch_append_disabled.json
- batch_append_disabled_cast.json
- batch_append_enabled.json
- batch_append_enabled_cast.json
- batch_init_disabled.json
- batch_init_enabled.json
- part-02.csv
- part-03.csv
- part-04.csv
- part-05.csv
- part-06.csv
- part-01.csv
- part-02.csv
- part-03.csv
- part-04.csv
- part-05.csv
- part-06.csv
- control_schema.json
- control_schema_add_column.json
- control_schema_rename.json
- source_part-01_schema.json
- source_part-02_schema.json
- source_part-03_schema.json
- source_part-04_schema.json
- source_part-05_schema.json
- source_part-06_schema.json
- batch_delta_disabled.json
- batch_delta_disabled_rename.json
- batch_delta_enabled.json
- batch_init_disabled.json
- batch_init_enabled.json
- part-02.csv
- part-01.csv
- part-02.csv
- control_schema_merge_enabled.json
- control_schema_overwrite.json
- source_part-01_schema.json
- source_part-02_schema.json
- batch_init.json
- batch_merge_disabled.json
- batch_merge_enabled.json
- batch_overwrite.json
- file.csv
- file1.csv
- file2.csv
- file3.json
- file4.xml
- file5.txt
- file6.compress.gz
- file7.zip
- drive_exception.json
- endpoint_exception.json
- local_path_exception.json
- site_exception.json
- streaming_exception.json
- schema.json
- read_file_name_and_file_pattern_conflict_should_fail.json
- read_file_name_unsupported_extension_should_fail.json
- read_folder_csv_archive_enabled_success.json
- read_folder_csv_archive_success_subfolder_override_success.json
- read_folder_csv_no_csv_files_should_fail.json
- read_folder_csv_one_file_schema_mismatch_custom_error_subfolder_should_archive_error.json
- read_folder_csv_one_file_schema_mismatch_should_archive_error.json
- read_folder_csv_pattern_matches_no_files_should_fail.json
- read_folder_csv_pattern_success.json
- read_folder_csv_success.json
- read_folder_path_does_not_exist_should_fail.json
- read_folder_relative_path_looks_like_file_unsupported_extension_should_fail.json
- read_single_csv_archive_default_enabled_success.json
- read_single_csv_archive_enabled_success.json
- read_single_csv_archive_success_subfolder_override_success.json
- read_single_csv_download_error_should_archive_error.json
- read_single_csv_empty_file_should_archive_error.json
- read_single_csv_full_path_success.json
- read_single_csv_full_path_with_file_name_should_fail.json
- read_single_csv_full_path_with_file_pattern_should_fail.json
- read_single_csv_full_path_with_file_type_should_fail.json
- read_single_csv_spark_load_fails_should_archive_error.json
- read_single_csv_success.json
- read_unsupported_file_type_should_fail.json
- bad_schema.csv
- other.csv
- sample_1.csv
- sample_2.csv
- get_drive_id.json
- get_file_metadata.json
- get_site_id.json
- rename_file.json
- write_to_local_success.json
- file_control.csv
- file_source.csv
- create_upload_session.json
- get_drive_id.json
- get_site_id.json
- schema.json
- table_stats_complex_default_scenario1.json
- table_stats_complex_default_scenario2.json
- table_stats_complex_different_delimiter_scenario1.json
- table_stats_complex_different_delimiter_scenario2.json
- table_stats_simple_split_scenario.json
- test_table_complex_default_scenario.sql
- test_table_complex_different_delimiter_scenario.sql
- test_table_simple_split_scenario.sql
- test_view_complex_default_scenario.sql
- test_view_complex_different_delimiter_scenario.sql
- test_view_simple_split_scenario.sql
- acon_create_table_complex_default_scenario.json
- acon_create_table_complex_different_delimiter_scenario.json
- acon_create_table_simple_split_scenario.json
- acon_create_view_complex_default_scenario.json
- acon_create_view_complex_different_delimiter_scenario.json
- acon_create_view_simple_split_scenario.json
- acon_delete_where_table_simple_split_scenario.json
- acon_describe_simple_split_scenario.json
- acon_drop_table_simple_split_scenario.json
- acon_drop_view_simple_split_scenario.json
- acon_execute_sql_complex_default_scenario.json
- acon_execute_sql_complex_different_delimiter_scenario.json
- acon_execute_sql_simple_split_scenario.json
- get_tbl_pk_simple_split_scenario.json
- optimize_location_simple_split_scenario.json
- optimize_table_simple_split_scenario.json
- show_tbl_properties_simple_split_scenario.json
- acon_vacuum_location_simple_split_scenario.json
- acon_vacuum_table_simple_split_scenario.json
- batch.json
- streaming.json
- streaming_batch.json
- write_streaming_struct_data.json
- write_streaming_struct_data_fail.json
- chain_control.csv
- struct_data.json
- customer_schema.json
- sales_schema.json
- struct_data_schema.json
- customers.csv
- sales_historical.csv
- sales_new.csv
- struct_data.csv
- part-01.json
- part-01.csv
- batch.json
- source_schema.json
- streaming.json
- part-01.csv
- part-01.json
- batch.json
- source_schema.json
- streaming.json
- part-01.csv
- part-01.json
- batch.json
- source_schema.json
- streaming.json
- part-01.csv
- part-01.json
- batch.json
- source_schema.json
- streaming.json
- drop_columns.csv
- hash_masking.csv
- part-01.csv
- drop_columns.json
- drop_columns_control_schema.json
- hash_masking.json
- hash_masking_control_schema.json
- source_schema.json
- part-01.csv
- part-01.csv
- control_schema.json
- source_schema.json
- streaming.json
- batch_distinct.json
- batch_drop_duplicates.json
- streaming_distinct.json
- streaming_drop_duplicates.json
- part-01.csv
- part-02.csv
- batch.json
- source_schema.json
- streaming.json
- control_scenario_1_and_2.csv
- control_scenario_3.csv
- customer-part-01.csv
- sales-part-01.csv
- sales-part-02.csv
- batch.json
- control_scenario_1_and_2_schema.json
- control_scenario_3_schema.json
- customer_schema.json
- sales_schema.json
- streaming.json
- streaming_foreachBatch.json
- streaming_without_broadcast.json
- streaming_without_column_rename.json
- part-01.json
- part-01.csv
- batch.json
- source_schema.json
- replace_nulls.csv
- replace_nulls_col_subset.csv
- part-01.csv
- control_schema.json
- replace_nulls.json
- replace_nulls_col_subset.json
- source_schema.json
- part-01.csv
- part-01.csv
- WE_SO_SCL_202108111400000029.csv
- batch.json
- control_schema.json
- source_schema.json
- control_sales.csv
- control_sales_shipment.csv
- control_sales_shipment_streaming.csv
- control_sales_shipment_streaming_foreachBatch.csv
- control_sales_streaming.csv
- control_sales_streaming_foreachBatch.csv
- sales-historical-part-01.csv
- sales-historical-part-02.csv
- sales-new-part-01.csv
- sales-new-part-02.csv
- sales-shipment-part-01.csv
- sales-shipment-part-02.csv
- batch_union.json
- batch_union_diff_schema.json
- batch_unionByName.json
- batch_unionByName_diff_schema.json
- batch_unionByName_diff_schema_error.json
- sales_schema.json
- sales_shipment_schema.json
- streaming_union.json
- streaming_union_foreachBatch.json
- streaming_unionByName_diff_schema.json
- streaming_unionByName_diff_schema_foreachBatch.json
- streaming_drop_duplicates.csv
- part-01.csv
- part-02.csv
- source_schema.json
- streaming_drop_duplicates.json
- streaming_drop_duplicates_overall_watermark.csv
- part-01.csv
- part-02.csv
- source_schema.json
- streaming_drop_duplicates_overall_watermark.json
- streaming_inner_join.csv
- customer-part-01.csv
- sales-part-01.csv
- sales-part-02.csv
- customer_schema.json
- sales_schema.json
- streaming_inner_join.json
- streaming_inner_join_control_schema.json
- streaming_left_outer_join.csv
- customer-part-01.csv
- customer-part-02.csv
- customer-part-03.csv
- customer-part-04.csv
- customer-part-05.csv
- sales-part-01.csv
- sales-part-02.csv
- sales-part-03.csv
- sales-part-04.csv
- sales-part-05.csv
- customer_schema.json
- sales_schema.json
- streaming_left_outer_join.json
- streaming_left_outer_join_control_schema.json
- streaming_right_outer_join.csv
- customer-part-01.csv
- sales-part-01.csv
- sales-part-02.csv
- customer_schema.json
- sales_schema.json
- streaming_right_outer_join.json
- streaming_right_outer_join_control_schema.json
- write_batch_console.json
- write_batch_dataframe.json
- write_batch_files.json
- write_batch_jdbc.json
- write_batch_rest_api.json
- write_batch_table.json
- write_streaming_console.json
- write_streaming_dataframe.json
- write_streaming_df_with_checkpoint.json
- write_streaming_files.json
- write_streaming_foreachBatch_console.json
- write_streaming_foreachBatch_dataframe.json
- write_streaming_foreachBatch_df_with_checkpoint.json
- write_streaming_foreachBatch_files.json
- write_streaming_foreachBatch_jdbc.json
- write_streaming_foreachBatch_table.json
- write_streaming_multiple_dfs.json
- write_streaming_rest_api.json
- write_streaming_table.json
- writers_control.csv
- writers_control_streaming_dataframe_1.csv
- writers_control_streaming_dataframe_2.csv
- writers_control_streaming_dataframe_foreachBatch_1.csv
- writers_control_streaming_dataframe_foreachBatch_2.csv
- sales_schema.json
- sales_historical_1.csv
- sales_historical_2.csv
- sales_new_1.csv
- sales_new_2.csv
- custom_engine_config.yaml
- heartbeat_sensor_control_table.json
- sensor_table.json
- heartbeat_sensor_control_table.json
- sensor_table.json
- sample_ok.csv
- sample_other_delim.csv
- __init__.py
- test_acon_validation.py
- test_custom_configs.py
- test_databricks_utils.py
- test_failure_notification_creation.py
- test_heartbeat_acon_creation.py
- test_heartbeat_anchor_job.py
- test_log_filter_sensitive_data.py
- test_notification_creation.py
- test_notification_factory.py
- test_paimon_utils.py
- test_prisma_dq_rule_id.py
- test_prisma_function_definition.py
- test_rest_api_functions.py
- test_sensor.py
- test_sensor_manager.py
- test_sharepoint_csv_reader.py
- test_sharepoint_options.py
- test_spark_session.py
- test_version.py
- __init__.py
- dataframe_helpers.py
- dq_rules_table_utils.py
- exec_env_helpers.py
- jdbc_helpers.py
- local_storage.py
- mocks.py
- smtp_server.py
- __init__.py
- conftest.py
- .coveragerc
- .gitignore
- AGENTS.md
- CONTRIBUTING.md
- LICENSE.txt
- Makefile
- pyproject.toml
- README.md
// repository documentation
Was this content helpful?
(0 ratings)