qbeast-spark
Qbeast-spark: DataSource enabling multi-dimensional indexing and efficient data sampling. Big Data, free from the unnecessary!
File Explorer
Download Latest Version (.zip)- bug_report.md
- test-artifact.yml
- pull_request_template.md
- Block.scala
- BroadcastTableChanges.scala
- ColumnsToIndexSelector.scala
- ColumnToIndex.scala
- ColumnToIndexUtils.scala
- CubeDomainsBuilder.scala
- CubeId.scala
- DataWriter.scala
- DenormalizedBlock.scala
- IndexFileBuilder.scala
- IndexManager.scala
- MetadataManager.scala
- NormalizedWeight.scala
- package.scala
- Point.scala
- PointWeightIndexer.scala
- PreCommitHook.scala
- QbeastColumnStats.scala
- QbeastCoreContext.scala
- QbeastFiles.scala
- QbeastOptions.scala
- QbeastSnapshot.scala
- QDataType.scala
- QuerySpace.scala
- RevisionClasses.scala
- StagingUtils.scala
- Weight.scala
- WeightRange.scala
- WriteMode.scala
- CDFNumericQuantilesTransformation.scala
- CDFNumericQuantilesTransformer.scala
- CDFQuantilesTransformation.scala
- CDFQuantilesTransformer.scala
- CDFStringQuantilesTransformation.scala
- CDFStringQuantilesTransformer.scala
- EmptyTransformation.scala
- EmptyTransformer.scala
- HashTransformation.scala
- HashTransformer.scala
- HistogramTransformation.scala
- HistogramTransformer.scala
- IdentityTransformation.scala
- LinearTransformation.scala
- LinearTransformer.scala
- StringHistogramTransformation.scala
- StringHistogramTransformer.scala
- Transformation.scala
- Transformer.scala
- package.scala
- QbeastFilters.scala
- QueryExecutor.scala
- QueryFiltersUtils.scala
- QuerySpec.scala
- QuerySpecBuilder.scala
- DefaultListFilesStrategy.scala
- ListFilesStrategy.scala
- SamplingListFilesStrategy.scala
- ColumnsToIndex.scala
- DefaultFileIndex.scala
- EmptyFileIndex.scala
- IndexStatusBuilder.scala
- OTreeDataAnalyzer.scala
- package.scala
- QbeastColumns.scala
- RowUtils.scala
- SparkColumnsToIndexSelector.scala
- SparkOTreeManager.scala
- SparkPointWeightIndexer.scala
- SparkRevisionFactory.scala
- QbeastMurmur3Hash.scala
- QbeastFunctions.scala
- MetadataOperation.scala
- IndexMetrics.scala
- Params.scala
- QbeastExceptionMessages.scala
- SparkToQTypesUtils.scala
- IndexFileWriter.scala
- IndexFileWriterFactory.scala
- Rollup.scala
- RollupDataWriter.scala
- StatsTracker.scala
- package.scala
- config.scala
- SparkCatalogV2Util.scala
- AnalysisExceptionFactory.scala
- DataFrameUtils.scala
- DatasetFactory.scala
- SchemaUtils.scala
- SparkCatalogUtils.scala
- V2AndV1Traits.scala
- CubeKeyFactoryJava.java
- CubeKeyFactoryJava.java
- BlockTest.scala
- ColumnToIndexTest.scala
- ColumnToIndexUtilsTest.scala
- CreateCubeKeySpeedTest.scala
- CreateCubeKeyTest.scala
- CubeDomainsBuilderTest.scala
- CubeIdSpeedTest.scala
- CubeIdTest.scala
- CubeKeyFactory.scala
- EstimateGroupCubeSizeTest.scala
- IndexFileTest.scala
- IndexStatusTest.scala
- JSONSerializationTests.scala
- PointTest.scala
- PointWeightIndexerTest.scala
- QbeastHookLoaderTest.scala
- QTableIDTest.scala
- QuerySpaceFromToTest.scala
- RevisionTest.scala
- WeightAndCountTest.scala
- CDFNumericQuantilesTransformationTest.scala
- CDFQuantilesTransformerTest.scala
- CDFStringQuantilesTransformationTest.scala
- EmptyTransformationTest.scala
- HashTransformationTest.scala
- IdentityTransformationTest.scala
- LinearTransformationTest.scala
- StringHistogramTransformationTest.scala
- TransformerTest.scala
- DeltaMetadataManager.scala
- DeltaMetadataOperation.scala
- DeltaMetadataWriter.scala
- DeltaQbeastFileUtils.scala
- DeltaQbeastSnapshot.scala
- DeltaRollupDataWriter.scala
- DeltaStagingUtils.scala
- DeltaStatsCollectionUtils.scala
- DeltaQbeastFileUtilsTest.scala
- DeltaQbeastStatsTest.scala
- delta.png
- g1.png
- g2.png
- proper-cube.png
- Qbeast-spark.png
- rsd-all.png
- rsd-tree.png
- spark_delta_demo.gif
- spark_qbeast_demo.gif
- states-and-transitions.png
- AdvancedConfiguration.md
- CloudStorages.md
- ColumnsToIndexSelector.md
- FAQ.md
- OTreeAlgorithm.md
- QbeastFormat.md
- QbeastFormatChanges.md
- QbeastTable.md
- Quickstart.md
- sample_pushdown_demo.ipynb
- build.properties
- Dependencies.scala
- plugins.sbt
- org.apache.spark.sql.sources.DataSourceRegister
- reference.conf
- DefaultStagedTable.scala
- QbeastCatalog.scala
- QbeastCatalogUtils.scala
- TableCreationMode.scala
- QbeastContext.scala
- AlterQbeastTableCommands.scala
- ConvertToQbeastCommand.scala
- OptimizeTableCommand.scala
- QbeastAnalysis.scala
- QbeastAnalysisUtils.scala
- SampleRule.scala
- SaveAsTableRule.scala
- QbeastStagedTableImpl.scala
- QbeastTableImpl.scala
- QbeastWriteBuilder.scala
- QbeastBaseRelation.scala
- QbeastDataSource.scala
- QbeastSparkSessionExtension.scala
- IndexedTable.scala
- QbeastTable.scala
- QbeastUtils.scala
- ecommerce100K_2019_Oct.csv
- ecommerce300k_2019_Nov.csv
- CatalogTestSuite.scala
- DefaultStagedTableTest.scala
- QbeastCatalogIntegrationTest.scala
- QbeastCatalogTest.scala
- QbeastConfigTest.scala
- QbeastContextTest.scala
- DenormalizedBlockTest.scala
- NormalizedWeightTest.scala
- QbeastColumnStatsTest.scala
- WeightTest.scala
- QbeastBaseRelationTest.scala
- QbeastDataSourceTest.scala
- QbeastOptionsTest.scala
- QbeastStagedTableTest.scala
- QbeastTableImplTest.scala
- ConvertToQbeastDeltaTest.scala
- DefaultFileIndexTest.scala
- DeltaQbeastStatsIntegrationTest.scala
- DeltaRollupDataWriterTest.scala
- PreCommitHookDeltaTest.scala
- QbeastDeltaIntegrationTest.scala
- QbeastOptimizeDeltaTest.scala
- QbeastOptionsDeltaTest.scala
- QbeastSchemaDeltaTest.scala
- QbeastSnapshotDeltaTest.scala
- QbeastSparkTxnTest.scala
- DocumentationTests.scala
- CDFNumericQuantilesIndexingTest.scala
- CDFQuantilesTestUtils.scala
- CDFStringQuantilesIndexingTest.scala
- HashTransformerIndexingTest.scala
- TransformerIndexingTest.scala
- DisjunctiveQuerySpecTest.scala
- QueryExecutorTest.scala
- QuerySpecBuilderTest.scala
- QueryTestSpec.scala
- TimeSeriesQueryTest.scala
- ColumnsToIndexTest.scala
- CubeDomainsIntegrationTest.scala
- DoublePassOTreeDataAnalyzerTest.scala
- IndexStatusBuilderTest.scala
- IndexTest.scala
- IndexTestChecks.scala
- NewRevisionTest.scala
- NormalizedWeightIntegrationTest.scala
- OTreeAlgorithmTest.scala
- QbeastColumnsTest.scala
- RevisionTest.scala
- SparkColumnsToIndexSelectorTest.scala
- SparkPointWeightIndexerTest.scala
- SparkRevisionChangesUtilsTest.scala
- SparkRevisionFactoryTest.scala
- IndexMetricsTest.scala
- MetadataManagerTest.scala
- QbeastCreateTableSQLTest.scala
- QbeastFilterPushdownTest.scala
- QbeastInsertToTest.scala
- QbeastOptimizeIntegrationTest.scala
- QbeastSamplingTest.scala
- QbeastSnapshotTest.scala
- QbeastSparkCorrectnessTest.scala
- QbeastSparkIntegrationTest.scala
- QbeastSQLIntegrationTest.scala
- QbeastStagingTest.scala
- SizeStatsTest.scala
- RollupTest.scala
- WriteTestSpec.scala
- IndexedTableTest.scala
- QbeastTableTest.scala
- QbeastUtilsTest.scala
- TransformationsUtilsTest.scala
- QbeastIntegrationTestSpec.scala
- TestClasses.scala
- TestUtils.scala
- sampling-fraction.png
- __init__.py
- block.py
- content_loader.py
- cube.py
- qviz.py
- sampling_info.py
- 00000000000000000000.json
- 00000000000000000001.json
- block_test.py
- cube_test.py
- sampling_info_test.py
- poetry.lock
- pyproject.toml
- README.md
- setup.py
- .gitignore
- .scalafix.conf
- .scalafmt.conf
- build.sbt
- CODE_OF_CONDUCT.md
- CONTRIBUTING.md
- LICENSE
- NOTICE
- README.md
# Installation Guide
1. Get the code
git clone https://github.com/Qbeast-io/qbeast-spark
Downloads the entire project code from GitHub to your computer.
cd qbeast-spark
Moves into the project folder you just downloaded.
2. Python
Easy RecommendedPrerequisites
β οΈ This is a large repository, so this method may point to an internal sub-package rather than the actual core product. Check the full README as well.
pip install .
Installs the package published on PyPI directly β no need to clone the source.
jupyter notebook
Launches Jupyter in your browser so you can open and run the notebook (.ipynb) files.
If it runs without errors and prints output in the terminal, it worked.
// repository documentation
Was this content helpful?
(0 ratings)