tispark
TiSpark is built for running Apache Spark on top of TiDB/TiKV
File Explorer
Download Latest Version (.zip)- bug_report.md
- enhancement.md
- feature_request.md
- general-question.md
- proposal.md
- alter-primary-key-false-test.yml
- changelog.yml
- codeql-analysis.yml
- followerread.yml
- issue-check.yaml
- license-check.yml
- TLS-test.yml
- verify.yml
- license-checker-config.yml
- pull_request_template.md
- assembly.xml
- pom.xml
- client-keystore
- server-cert-store
- client-pkcs8.key
- client.key
- client.pem
- openssl.cnf
- pd.key
- pd.pem
- root.key
- root.pem
- root.srl
- tidb.key
- tidb.pem
- tikv.key
- tikv.pem
- pd.toml
- tidb.toml
- tikv.toml
- hive-site.xml.template
- pd-4.0.toml
- pd-TLS.toml
- pd.toml
- tidb-4.0.toml
- tidb-TLS.toml
- tidb.toml
- tiflash-daily-test.toml
- tiflash-learner-daily-test.toml
- tiflash-learner.toml
- tiflash.toml
- tikv-4.0.toml
- tikv-TLS.toml
- tikv.toml
- DumpHiveMetastore.sql
- fetch-test-data.sh
- version.sh
- TiColumnarBatchHelper.java
- TiColumnVectorAdapter.java
- TypeMapping.java
- UriHostMapping.java
- org.apache.spark.sql.sources.DataSourceRegister
- log4j.properties.template
- CacheInvalidateAccumulator.scala
- MySQLPriv.scala
- TiAuthorization.scala
- CacheInvalidateEventHandler.scala
- CacheInvalidateListener.scala
- PDCacheInvalidateListener.scala
- ServiceSafePoint.scala
- TableSizeEstimator.scala
- StatisticsHelper.scala
- StatisticsManager.scala
- Telemetry.scala
- TelemetryRule.scala
- TeleMsg.scala
- TiSparkTeleConf.scala
- TiSparkTeleInfo.scala
- HttpClientUtil.scala
- ReflectionUtil.scala
- SystemInfoUtil.scala
- TiUtil.scala
- TwoPhaseCommitHepler.scala
- WriteUtil.scala
- TiDBBatchWrite.scala
- TiDBDataWrite.scala
- TiDBDataWriterFactory.scala
- TiDBTable.scala
- TiDBTableProvider.scala
- TiDBTableScan.scala
- TiDBTableScanBuilder.scala
- DBTable.scala
- EncodedKVPair.scala
- SerializableKey.scala
- TiBatchWrite.scala
- TiBatchWriteTable.scala
- TiDBDelete.scala
- TiDBOptions.scala
- TiDBWriter.scala
- TiReginSplitPartitioner.scala
- WrappedEncodedRow.scala
- WrappedRow.scala
- DefaultSource.scala
- MetaManager.scala
- TiConfigConst.scala
- TiDBRelation.scala
- TiDBUtils.scala
- TiPartition.scala
- TiSparkInfo.scala
- TiTableReference.scala
- TiCatalog.scala
- SpecialSum.scala
- BasicExpression.scala
- TiExprUtils.scala
- TiParser.scala
- TiParserFactory.scala
- TiAggregation.scala
- TiAggregationImpl.scala
- TiStrategyFactory.scala
- BasicLogicalPlan.scala
- TiAuthorizationRule.scala
- TiAuthRuleFactory.scala
- TiStatisticsRuleFactory.scala
- CoprocessorRDD.scala
- TiConverter.scala
- TiHandleRDD.scala
- TiRDD.scala
- TiRowRDD.scala
- TiContext.scala
- TiExtensions.scala
- LargeColumn.sql
- PrefixTest.sql
- UTF8Test.sql
- 1_account.sql
- 1_give.sql
- 2_get.sql
- ddl.sql
- q1.sql
- q2.sql
- sum_account.sql
- q1.sql
- q10.sql
- q11.sql
- q12.sql
- q13.sql
- q14a.sql
- q14b.sql
- q15.sql
- q16.sql
- q17.sql
- q18.sql
- q19.sql
- q2.sql
- q20.sql
- q21.sql
- q22.sql
- q23a.sql
- q23b.sql
- q24a.sql
- q24b.sql
- q25.sql
- q26.sql
- q27.sql
- q28.sql
- q29.sql
- q3.sql
- q30.sql
- q31.sql
- q32.sql
- q33.sql
- q34.sql
- q35.sql
- q36.sql
- q37.sql
- q38.sql
- q39a.sql
- q39b.sql
- q4.sql
- q40.sql
- q41.sql
- q42.sql
- q43.sql
- q44.sql
- q45.sql
- q46.sql
- q47.sql
- q48.sql
- q49.sql
- q5.sql
- q50.sql
- q51.sql
- q52.sql
- q53.sql
- q54.sql
- q55.sql
- q56.sql
- q57.sql
- q58.sql
- q59.sql
- q6.sql
- q60.sql
- q61.sql
- q62.sql
- q63.sql
- q64.sql
- q65.sql
- q66.sql
- q67.sql
- q68.sql
- q69.sql
- q7.sql
- q70.sql
- q71.sql
- q72.sql
- q73.sql
- q74.sql
- q75.sql
- q76.sql
- q77.sql
- q78.sql
- q79.sql
- q8.sql
- q80.sql
- q81.sql
- q82.sql
- q83.sql
- q84.sql
- q85.sql
- q86.sql
- q87.sql
- q88.sql
- q89.sql
- q9.sql
- q90.sql
- q91.sql
- q92.sql
- q93.sql
- q94.sql
- q95.sql
- q96.sql
- q97.sql
- q98.sql
- q99.sql
- ss_max.sql
- ss_maxb.sql
- q1.sql
- q10.sql
- q11.sql
- q12.sql
- q13.sql
- q14.sql
- q15.sql
- q16.sql
- q17.sql
- q18.sql
- q19.sql
- q2.sql
- q20.sql
- q21.sql
- q22.sql
- q3.sql
- q4.sql
- q5.sql
- q6.sql
- q7.sql
- q8.sql
- q9.sql
- tidb_config.properties.template
- tidb_config.properties.TLS.template
- tidb_config.properties.withoutdataload.template
- tispark-test
- TiAuthIntegrationSuite.scala
- TiAuthSuite.scala
- CollationSuite.scala
- ConcurrencyTest.scala
- WriteDDLConflictSuite.scala
- WriteDDLNotConflictSuite.scala
- WriteReadSuite.scala
- WriteWriteConflictSuite.scala
- ToBitSuite.scala
- ToBytesSuite.scala
- ToDateSuite.scala
- ToDateTimeSuite.scala
- ToDecimalSuite.scala
- ToEnumSuite.scala
- ToRealSuite.scala
- ToSignedSuite.scala
- ToStringSuite.scala
- ToTimestampSuite.scala
- ToUnsignedSuite.scala
- AddingIndexInsertSuite.scala
- AddingIndexReplaceSuite.scala
- AutoIncrementSuite.scala
- AutoRandomSuite.scala
- BaseBatchWriteTest.scala
- BaseBatchWriteWithoutDropTableTest.scala
- BaseDataSourceTest.scala
- BasicBatchWriteSuite.scala
- BasicSQLSuite.scala
- CheckUnsupportedSuite.scala
- ColumnMappingSuite.scala
- EdgeConditionSuite.scala
- ExceptionTestSuite.scala
- FilterPushdownSuite.scala
- InsertSuite.scala
- MissingParameterSuite.scala
- OnlyOnePkSuite.scala
- RowIDAllocatorSuite.scala
- ShardRowIDBitsSuite.scala
- TiSparkTypeSuite.scala
- UpperCaseColumnNameSuite.scala
- BatchWriteDataTypeSuite.scala
- DecimalTypeSuite.scala
- DeleteCompatibilitySuite.scala
- DeleteConcurrencyTestSuite.scala
- DeleteNotSupportSuite.scala
- DeleteWhereClauseSuite.scala
- HostMappingSuite.scala
- LineItemSuite.scala
- InsertSuite.scala
- MultiTableWriteSuite.scala
- BitOverflowSuite.scala
- BytesOverflowSuite.scala
- DateOverflowSuite.scala
- DateTimeOverflowSuite.scala
- DecimalOverflowSuite.scala
- EnumOverflowSuite.scala
- SignedOverflowSuite.scala
- StringOverflowSuite.scala
- UnsignedOverflowSuite.scala
- PartitionBaseSuite.scala
- PartitionNewCollationSuite.scala
- PartitionWriteSuite.scala
- TelemetrySuite.scala
- TiFlashSuite.scala
- CheckTLSEnable.scala
- JDBCTLSSuite.scala
- TiKVClientTLSSuite.scala
- TiSparkTLSSuite.scala
- InitialTTLExpiredSuite.scala
- LockTimeoutSuite.scala
- BatchWriteIssueSuite.scala
- TiBatchWriteSuite.scala
- UnitSuite.scala
- TPCDSQuerySuite.scala
- TPCHQuerySuite.scala
- CatalogTestSuite.scala
- LogicalPlanTestSuite.scala
- StatisticsTestSuite.scala
- BasePlanTest.scala
- ClusteredIndexTest.scala
- IndexScan0Suite.scala
- IndexScan1Suite.scala
- TableScan0Suite.scala
- TableScan1Suite.scala
- Between0Suite.scala
- ComprehensiveSuite.scala
- CoveringIndex0Suite.scala
- InTest0Suite.scala
- Join0Suite.scala
- PrefixIndexTestSuite.scala
- Special0Suite.scala
- UnsignedTestSuite.scala
- Aggregate0Suite.scala
- ArithmeticAgg0Suite.scala
- ArithmeticTest0Suite.scala
- ArithmeticTest1Suite.scala
- ArithmeticTest2Suite.scala
- Between0Suite.scala
- ComplexAggregateSuite.scala
- ComplexGroupBySuite.scala
- Count0Suite.scala
- Distinct0Suite.scala
- FirstLast0Suite.scala
- Having0Suite.scala
- InTest0Suite.scala
- LikeTestSuite.scala
- OtherTestSuite.scala
- Union0Suite.scala
- FollowerReadSuite.scala
- BatchWritePKAndIndexSuite.scala
- BatchWritePkSuite.scala
- BatchWriteUniqueIndexSuite.scala
- AvgPushDownSuite.scala
- BasePushDownSuite.scala
- CountPushDownSuite.scala
- SumPushDownSuite.scala
- StaleReadSuite.scala
- StatisticsManagerSuite.scala
- SharedSQLContext.scala
- TestConstants.scala
- TestSparkSession.scala
- Utils.scala
- TxnTestSuite.scala
- ClusterIndexScanSuites.scala
- DataTypePKSuite.scala
- MultiColumnClusterIndexDataSuite00.scala
- MultiColumnPKDataTypeSuite00.scala
- MultiColumnPKDataTypeSuite01.scala
- MultiColumnPKDataTypeSuite02.scala
- MultiColumnPKDataTypeSuite03.scala
- MultiColumnPKDataTypeSuite04.scala
- MultiColumnPKDataTypeSuite05.scala
- MultiColumnPKDataTypeSuite06.scala
- MultiColumnPKDataTypeSuite07.scala
- MultiColumnPKDataTypeSuite08.scala
- MultiColumnPKDataTypeSuite09.scala
- MultiColumnPKDataTypeSuite10.scala
- MultiColumnPKDataTypeSuite11.scala
- MultiColumnPKDataTypeSuite12.scala
- MultiColumnPKDataTypeSuite13.scala
- MultiColumnPKDataTypeSuite14.scala
- MultiColumnPKDataTypeSuite15.scala
- MultiColumnPKDataTypeSuite16.scala
- MultiColumnPKDataTypeSuite17.scala
- MultiColumnPKDataTypeSuite18.scala
- MultiColumnPKDataTypeSuite19.scala
- MultiColumnPKDataTypeSuite20.scala
- MultiColumnPKDataTypeSuite21.scala
- MultiColumnPKDataTypeSuite22.scala
- MultiColumnPKDataTypeSuite23.scala
- MultiColumnPKDataTypeSuite24.scala
- MultiColumnPKDataTypeSuite25.scala
- MultiColumnPKDataTypeSuite26.scala
- MultiColumnPKDataTypeSuite27.scala
- MultiColumnPKDataTypeSuite28.scala
- MultiColumnPKDataTypeSuite29.scala
- MultiColumnPKDataTypeSuite30.scala
- MultiColumnPKDataTypeSuite31.scala
- MultiColumnPKDataTypeSuite32.scala
- MultiColumnPKDataTypeSuite33.scala
- MultiColumnPKDataTypeSuite34.scala
- MultiColumnPKDataTypeSuite35.scala
- MultiColumnPKDataTypeSuites.scala
- BaseRandomDataTypeTest.scala
- DataTypeNormalSuite.scala
- MultiColumnDataTypeSuite.scala
- SpecialTiDBTypeTestSuite.scala
- AlterTableTestSuite.scala
- BaseInitialOnceTest.scala
- BaseTestGenerationSpec.scala
- BaseTiSparkEnableBatchWriteTest.scala
- BaseTiSparkTest.scala
- ExpressionIndexSuite.scala
- InvisibleIndexSuite.scala
- IssueTestSuite.scala
- MultipleSparkSessionTestSuite.scala
- MysqlDataTypeSuite.scala
- OutputOffsetsSuite.scala
- PartitionTableSuite.scala
- QueryTest.scala
- SequenceTestSuite.scala
- SparkDataTypeTestSuite.scala
- TiDBMapDatabaseSuite.scala
- TimezoneTestSuite.scala
- TiSparkTestSpec.scala
- ViewTestSuite.scala
- SharedSparkContext.scala
- SparkFunSuite.scala
- Readme.md
- pom.xml
- KEEPME
- pom.xml
- ColumnValueGenerator.scala
- DataGenerator.scala
- DataType.scala
- GeneratorUtils.scala
- Index.scala
- IndexColumn.scala
- NullableType.scala
- Schema.scala
- SchemaAndData.scala
- RandomTest.scala
- DataGeneratorSuite.scala
- RandomTestSuite.scala
- pom.xml
- .scalafmt.conf
- intellij-java-google-style.xml
- javafmt
- README.md
- scalafmt
- delete_step
- delete_step.png
- rowid
- rowid.png
- dsv2.png
- new_write
- new_write.png
- write.png
- code_flow_chart.png
- meta
- meta.png
- 2022-02-28-dsv2-support.md
- 2022-03-21-delete-support.md
- 2022-05-09-staleread-support.md
- 2022-06-15-phyical_plan_explain.md
- 2022-06-30-partition-table-batchwrite-suppport.md
- 2022-07-20-insert-support.md
- 2022-08-11-normalize-client.md
- 2022-09-16-follower-read.md
- TEMPLATE.md
- get_the_code_build_and_test.md
- install_scala_java_spark_maven.md
- start_tidb_cluster.md
- code_style.md
- commit_the_code_and_submit_a_pull_request.md
- report_an_issue.md
- conflict.png
- fork.png
- issue_choose.png
- issue_template.png
- java_format.png
- overview.png
- pr.png
- pr_templete.png
- scala_format.png
- overview.md
- write.md
- tispark_development_guide.md
- authorization_userguide.md
- datasource_api_userguide.md
- delete_userguide.md
- follower_read.md
- insert_sql_userguide.md
- partition_table.md
- push_down.md
- query_execution_plan_in_TiSpark.md
- service_safe_point.md
- stale_read.md
- telemetry.md
- architecture.png
- how_to_use_tidb_as_metastore_db.md
- userguide_2.0.md
- userguide_3.0.md
- SparkWrapper.scala
- TiBasicExpression.scala
- TiBasicLogicalPlan.scala
- TiDBWriteBuilder.scala
- TiAggregationProjection.scala
- TiStrategy.scala
- pom.xml
- SparkWrapper.scala
- TiBasicExpression.scala
- TiBasicLogicalPlan.scala
- TiDBWriteBuilder.scala
- TiAggregationProjectionV2.scala
- TiStrategy.scala
- pom.xml
- SparkWrapper.scala
- TiBasicExpression.scala
- TiBasicLogicalPlan.scala
- TiDBWriteBuilder.scala
- TiAggregationProjectionV2.scala
- TiStrategy.scala
- pom.xml
- SparkWrapper.scala
- TiBasicExpression.scala
- TiBasicLogicalPlan.scala
- TiDBWriteBuilder.scala
- TiAggregationProjectionV2.scala
- TiStrategy.scala
- pom.xml
- SparkWrapper.scala
- TiBasicExpression.scala
- TiBasicLogicalPlan.scala
- TiDBWriteBuilder.scala
- TiAggregationProjectionV2.scala
- TiStrategy.scala
- pom.xml
- proto.sh
- RowIDAllocator.java
- Catalog.java
- CatalogTransaction.java
- Codec.java
- CodecDataInput.java
- CodecDataInputLittleEndian.java
- CodecDataOutput.java
- CodecDataOutputLittleEndian.java
- ExtendedDateTime.java
- KeyUtils.java
- MetaCodec.java
- MyDecimal.java
- RowDecoderV2.java
- RowEncoderV2.java
- RowV2.java
- TableCodec.java
- TableCodecV1.java
- TableCodecV2.java
- AutoGrowByteBuffer.java
- CHType.java
- CHTypeDate.java
- CHTypeDateTime.java
- CHTypeDecimal.java
- CHTypeFixedString.java
- CHTypeMyDate.java
- CHTypeMyDateTime.java
- CHTypeNumber.java
- CHTypeString.java
- BatchedTiChunkColumnVector.java
- TiBlockColumnVector.java
- TiChunk.java
- TiChunkColumnVector.java
- TiColumnVector.java
- TiCoreTime.java
- TiRowColumnVector.java
- ColumnMatcher.java
- DefaultVisitor.java
- IndexMatcher.java
- IndexRangeSetBuilder.java
- MetaResolver.java
- PartAndFilterExprRewriter.java
- ProtoConverter.java
- PrunedPartitionBuilder.java
- PseudoCostCalculator.java
- RangePartitionLocator.java
- RangeSetBuilder.java
- SupportedExpressionValidator.java
- AggregateFunction.java
- ArithmeticBinaryExpression.java
- Blocklist.java
- ByItem.java
- ColumnRef.java
- ComparisonBinaryExpression.java
- Constant.java
- Expression.java
- ExpressionBlocklist.java
- FuncCallExpr.java
- FuncCallExprEval.java
- IsNull.java
- LogicalBinaryExpression.java
- Not.java
- PartitionPruner.java
- RangeColumnPartitionPruner.java
- RangePartitionPruner.java
- StringRegExpression.java
- TypeBlocklist.java
- Visitor.java
- CommonHandle.java
- Handle.java
- IntHandle.java
- JDBCClient.java
- RowMapper.java
- CommonHandleScanKeyRangeBuilder.java
- CompoundKey.java
- IndexKey.java
- IndexScanKeyRangeBuilder.java
- Key.java
- KeyRangeBuilder.java
- RowKey.java
- StatisticsKeyRangeBuilder.java
- TypedKey.java
- BinPaddingCollator.java
- GeneralCICollator.java
- UnicodeCICollator.java
- CIStr.java
- Collation.java
- IndexType.java
- SchemaState.java
- TiColumnInfo.java
- TiDAGRequest.java
- TiDBInfo.java
- TiFlashReplicaInfo.java
- TiIndexColumn.java
- TiIndexInfo.java
- TiPartitionDef.java
- TiPartitionInfo.java
- TiSequenceInfo.java
- TiTableInfo.java
- TiUserIdentity.java
- TiViewInfo.java
- ChunkIterator.java
- ConcreteScanIterator.java
- CoprocessorIterator.java
- DAGIterator.java
- IndexScanIterator.java
- ScanIterator.java
- SchemaInfer.java
- AstBuilder.java
- CaseChangingCharStream.java
- MySqlLexer.g4
- MySqlParser.g4
- PartitionWriteAstBuilder.java
- TiParser.java
- PartitionedTable.java
- PartitionExpression.java
- TableCommon.java
- IndexRange.java
- PredicateUtils.java
- ScanSpec.java
- SelectivityCalculator.java
- TiKVScanAnalyzer.java
- DefaultRowReader.java
- ObjectRowImpl.java
- Row.java
- RowReader.java
- RowReaderFactory.java
- Bucket.java
- CMSketch.java
- ColumnStatistics.java
- Histogram.java
- IndexStatistics.java
- TableStatistics.java
- AbstractDateTimeType.java
- BitType.java
- BytesType.java
- Charset.java
- Converter.java
- DataType.java
- DataTypeFactory.java
- DateTimeType.java
- DateType.java
- DecimalType.java
- EnumType.java
- IntegerType.java
- JsonType.java
- MySQLType.java
- RealType.java
- SetType.java
- StringType.java
- TimestampType.java
- TimeType.java
- UninitializedType.java
- CHTypeMapping.java
- ConvertUpstreamUtils.java
- LogDesensitization.java
- MemoryUtil.java
- RangeSplitter.java
- RegionUtils.java
- BUILD
- ClientSession.java
- Main.java
- ReplicaReadPolicy.java
- Snapshot.java
- TiConfiguration.java
- TiDBJDBCClient.java
- TiFlashClient.java
- unicode_map_table
- RowIDAllocatorTest.java
- CodecTest.java
- KeyUtilsTest.java
- MyDecimalTest.java
- TableCodecTest.java
- TableCodecV1Test.java
- TableCodecV2Test.java
- CollationTest.java
- PartAndFilterExprRewriterTest.java
- PartitionLocatorTest.java
- ComparisonBinaryExpressionTest.java
- CompoundKeyTest.java
- IndexKeyTest.java
- KeyTest.java
- DBInfoTest.java
- MetaUtils.java
- TiDAGRequestTest.java
- TiTableInfoTest.java
- ChunkIteratorTest.java
- SchemaInferTest.java
- TiParserTest.java
- IndexMatcherTest.java
- PredicateUtilsTest.java
- TiKVScanAnalyzerTest.java
- ConverterTest.java
- DataTypeFactoryTest.java
- DecimalTypeTest.java
- IntegerTypeTest.java
- RealTypeTest.java
- ConcreteBackOffTest.java
- BUILD
- rule.bzl
- TiConfigurationTest.java
- log4j.properties
- BUILD
- LICENSE
- Makefile
- pom.xml
- README.md
- shading_rule
- WORKSPACE
- .gitignore
- .gitmodules
- .lift.toml
- .travis.yml
- CHANGELOG.md
- docker-compose-4.0.yaml
- docker-compose-TiDB-TLS.yaml
- docker-compose.yaml
- LICENSE
- OWNERS
- pom.xml
- README.md
π Installation Guide
1. Get the code
git clone https://github.com/pingcap/tispark
Downloads the entire project code from GitHub to your computer.
cd tispark
Moves into the project folder you just downloaded.
2. Docker
Easy RecommendedPrerequisites
- Git Needed to download the project code from GitHub.
- Docker Desktop Needed to build and run containers. Install it and keep it running in the background.
docker compose up -d --build
Builds and starts all defined containers (server, database, etc.) at once, in the background.
Run docker compose ps to check the containers are Up. If the README mentions a port, open http://localhost:PORT in your browser.
3. Maven (Java)
MediumPrerequisites
- Git Needed to download the project code from GitHub.
- JDK (Java) Required to build and run Java projects.
- Maven The build tool used for the mvn command.
cd assembly
This project's files live in a subfolder, so move into it first.
mvn clean install
Installs dependencies and builds the project using Maven.
A BUILD SUCCESS message means it worked. The output is created under target/.
4. Make
MediumPrerequisites
- Git Needed to download the project code from GitHub.
- Make Usually pre-installed on Linux/macOS. On Windows, install separately (e.g. via MSYS2 or WSL).
cd tikv-client
This project's files live in a subfolder, so move into it first.
make
Compiles the code based on the generated build configuration to produce an executable.
If it finishes without errors, it worked. Try running the generated executable directly.
// repository documentation
Was this content helpful?
(0 ratings)