iceberg-cxx

KO | EN

gitlite — search

#javascript #ai #llm #go #golang #android #machine-learning #docker #awesome #cli #kubernetes #awesome-list #mcp #ios

(★ 93)

Apache Iceberg C++ library

File Explorer

Download Latest Version (.zip)

- - build-n-test.yml
- CODEOWNERS
- - batch.h
  - CMakeLists.txt
  - compression.h
  - equality_delete.h
  - fields.h
  - generators.cpp
  - generators.h
  - list.h
  - log.h
  - positional_delete.h
  - processor.h
  - program.h
  - table.h
  - writer.h
- - CMakeLists.txt
  - dataset.cpp
  - dataset.h
  - generators.h
  - list.cpp
  - list.h
  - main.cpp
  - tables.cpp
  - tables.h
  - text.cpp
  - text.h
- CMakeLists.txt
- - batch.h
  - CMakeLists.txt
  - defer.h
  - error.h
  - json_parse.cpp
  - json_parse.h
  - logger.h
  - rg_metadata.cpp
  - rg_metadata.h
  - selection_vector.h
  - threadpool.h
- - CMakeLists.txt
  - common.cpp
  - common.h
  - delete.cpp
  - delete.h
  - generic_delete.cpp
  - generic_delete.h
  - handler.cpp
  - handler.h
  - specialized_multiple_column_delete.h
  - specialized_one_column_delete.h
  - stats.h
  - utils.h
- - CMakeLists.txt
- - manifest_file.hh
  - manifest_file_schema.h
  - manifest_file_schema.json
- - io.cpp
  - io.h
- - CMakeLists.txt
  - positional_delete.cpp
  - positional_delete.h
  - stats.h
- - CMakeLists.txt
  - main.cpp
- - arrow_array.cpp
  - arrow_array.h
  - assertions.h
  - CMakeLists.txt
  - column.cpp
  - column.h
  - common.h
  - metadata_writer.h
  - optional_vector.h
  - scoped_temp_dir.cpp
  - scoped_temp_dir.h
  - write.cpp
  - write.h
- - manifest_entry_stats_getter_test.cpp
  - puffin_test.cpp
- avro_schema.cpp
- avro_schema.h
- catalog.h
- CMakeLists.txt
- CPPLINT.cfg
- deletion_vector.cpp
- deletion_vector.h
- experimental_representations.cpp
- experimental_representations.h
- hash_utils.h
- literals.cpp
- literals.h
- manifest_entry.cpp
- manifest_entry.h
- manifest_entry_stats_getter.cpp
- manifest_entry_stats_getter.h
- manifest_file.h
- nested_field.h
- puffin.cpp
- puffin.h
- result.h
- schema.cpp
- schema.h
- snapshot.h
- sql_catalog.cpp
- sql_catalog.h
- table.h
- table_metadata.cpp
- table_metadata.h
- tea_column_stats.h
- tea_hive_catalog.cpp
- tea_hive_catalog.h
- tea_manifest_metadata.cpp
- tea_nessie_catalog.cpp
- tea_nessie_catalog.h
- tea_remote_catalog.cpp
- tea_remote_catalog.h
- tea_rest_catalog.cpp
- tea_rest_catalog.h
- tea_scan.cpp
- tea_scan.h
- tea_scan_hashers.cpp
- tea_scan_hashers.h
- transaction.cpp
- transaction.h
- transforms.cpp
- transforms.h
- type.cpp
- type.h
- uuid.cpp
- uuid.h
- write.cpp
- write.h
- - dictionary_serializer.h
  - distinct.h
  - distinct_hll.h
  - distinct_naive.h
  - distinct_theta.h
  - frequent_items.h
  - quantiles.h
  - string_view_serializer.h
- - sketch_updater.h
  - values_provider.h
- - prepare.sh
  - run.sh
- - analyzer_test.cpp
  - sketch_test.cpp
- analyzer.cpp
- analyzer.h
- CMakeLists.txt
- data.h
- main.cpp
- measure.h
- puffin.cpp
- puffin.h
- types.h
- - partitions_test.cpp
  - transforms_test.cpp
- - 00000-6-d4e36f4d-a2c0-467d-90e7-0ef1a54e2724-0-00001.parquet
  - 00000-7-d4e36f4d-a2c0-467d-90e7-0ef1a54e2724-0-00002.parquet
- - CMakeLists.txt
  - docker-compose.yml
  - gen_manifest_files.cpp
  - gen_performance.sh
  - README.md
  - run_performance_tests.cpp
  - s3.env
- - 00000-800cc6aa-5051-47d5-9579-46aafcba1de6.metadata.json
  - 00001-6d216ef0-8d58-4f27-a1d9-1cb22c1f3415.metadata.json
  - 00002-37c508a5-8a06-4823-845e-889dff066f72.metadata.json
  - 00003-ca406d8e-6c7b-4672-87ff-bfd76f84f949.metadata.json
  - 00004-f28a5f22-3e23-4bab-b753-6c96553ea45d.metadata.json
  - 00005-176a8e24-c5e2-48a8-884b-b0186a1c86fe.metadata.json
  - 00006-8caf3988-3dcc-4ca2-a472-0d96a273eaeb.metadata.json
  - 00007-2ed28880-9928-4670-934a-bda11c8130c9.metadata.json
  - 02ce7cec-31fd-4de7-8811-02ce7cec44a9-m0.avro
  - 3ccdc97b-a744-4930-98c7-4abc66c26625-m0.avro
  - 41f34bc8-eedf-4573-96b0-10c04e7c84c4-m0.avro
  - 66ee5e5e-f6c6-47a5-a609-1a439232d1ea-m0.avro
  - 7e6e13cb-31fd-4de7-8811-02ce7cec44a9-m0.avro
  - broken1.avro
  - broken1.metadata.json
  - broken2.avro
  - broken2.metadata.json
  - empty.avro
  - empty.metadata.json
  - snap-154324552338474173-1-66ee5e5e-f6c6-47a5-a609-1a439232d1ea.avro
  - snap-1638951453256129678-1-eea762e4-1b7a-4717-b361-eae34da54fd4.avro
  - snap-293191470325800019-1-3ccdc97b-a744-4930-98c7-4abc66c26625.avro
  - snap-2dfe8bdb-200b-1006-9bb5-0242ac110002.avro
  - snap-5231658854638766100-1-7e6e13cb-31fd-4de7-8811-02ce7cec44a9.avro
  - snap-7558608030923099867-1-41f34bc8-eedf-4573-96b0-10c04e7c84c4.avro
- - test_nessie_catalog.cpp
- - setup_lakekeeper.sql
  - test_rest_catalog.cpp
- - MockMetadataWithFixed.json
  - MockMetadataWithListOfLists.json
  - restrictions.json
  - restrictionsBad.json
  - restrictionsNotFull.json
  - SchemaWithDefaults.json
- CMakeLists.txt
- deletion_vector_test.cpp
- manifest_entry_test.cpp
- manifest_metadata_test.cpp
- puffin_footer_test.cpp
- referenced_data_file_test.cpp
- run_tests.cpp
- schema_validation_test.cpp
- snapshot_test.cpp
- streams_test.cpp
- table_metadata_test.cpp
- tea_scan_test.cpp
- validation_test.cpp
- CMakeLists.txt
- common.cpp
- common.h
- explorer.cpp
- fix_location.cpp
- hive_metastore_client.cpp
- hive_metastore_client.h
- hive_metastore_server.cpp
- hive_metastore_server.h
- ls.cpp
- make_metadata.cpp
- metadata_tree.cpp
- metadata_tree.h
- metastore_client.h
- s3client.h
- validation.cpp
- validation.h
- validator_tool.cpp
- - arrow-fix-snappy-empty-column.patch
  - arrow_ensure_xsimd.patch
  - fix_c-ares_url.patch
  - fix_zlib_url.patch
- - avro_default_value.patch
  - avro_format.patch
  - avro_metadata.patch
  - avro_missing_cstring.patch
  - CMakeLists.txt
- - CMakeLists.txt
- - CMakeLists.txt
- - CMakeLists.txt
- - CMakeLists.txt
- - count.patch
- - CMakeLists.txt
- - CMakeLists.txt
- - CMakeLists.txt
- - CMakeLists.txt
- - CMakeLists.txt
- CMakeLists.txt
.clang-format
.editorconfig
.gitattributes
.gitignore
CMakeLists.txt
CODE_OF_CONDUCT.md
CONTRIBUTING.md
CPPLINT.cfg
LICENSE
README.md
test-validator.sh

// repository documentation

Was this content helpful?

★ 0 (0 ratings)

Apache Iceberg C++ library

# Apache Iceberg™ C++

You do not need Java to use [Apache Iceberg™](https://iceberg.apache.org/).

It's an alternative for [iceberg-cpp](https://github.com/apache/iceberg-cpp).
The library is used by [Tea](https://github.com/lithium-tech/tea), an open-source extension for Greenplum that allows it to read Iceberg data from S3 compatible storage using HMS and Nessie catalogs.

## Supported features

Source https://iceberg.apache.org/status/

### Data types

| Data type  | Iceberg version | Cxx | Java |
|--------------------------|---|-----|-----|
| boolean                  | 2 | +   | +   |
| int                      | 2 | +   | +   |
| float                    | 2 | +   | +   |
| double                   | 2 | +   | +   |
| decimal                  | 2 | +   | +   |
| date                     | 2 | +   | +   |
| time                     | 2 | +   | +   |
| timestamp                | 2 | +   | +   |
| timestamptz              | 2 | +   | +   |
| timestamp_ns             | 3 | +   | +   |
| timestamptz_ns           | 3 | +   | +   |
| string                   | 2 | +   | +   |
| uuid                     | 2 | +   | +   |
| fixed                    | 2 | +   | +   |
| binary                   | 2 | +   | +   |
| variant                  | 3 | -   | +   |
| list                     | 2 | +   | +   |
| map                      | 2 | -   | +   |
| struct                   | 2 | -   | +   |
| unknown                  | 3 | +   | ?   |

#### Datetime types restrictions

Datetime restrictions are defined in [Iceberg spec](https://iceberg.apache.org/spec/?h=timestamp_ns#parquet).
For `date` underlying type is `int32`. For `time*` it's `int64`.
`timestamp` and `timestamptz` store microseconds from `1970-01-01 00:00:00.000000`.
`timestamp_ns` and `timestamptz_ns` store nanoseconds from `1970-01-01 00:00:00.000000000`.
`tz` suffix means the time is adjusted to UTC.

### Data files format

| File format   | Cxx | Java |
|---------------|-----|-----|
| Parquet       | +   | +   |
| ORC           | -   | +   |
| Puffin        | +   | +   |
| Avro          | -   | +   |

### File IO

| File IO            | Cxx | Java |
|--------------------|-----|-----|
| Local Filesystem   | +   | +   |
| Hadoop Filesystem  | -   | +   |
| S3 Compatible      | +   | +   |
| GCS Compatible     | -   | +   |
| ADLS Compatible    | -   | +   |

### Table Maintenance Operations

Not implemented

### Table Update Operations

Not implemented

### Table read operations

| Operation        | Iceberg version | Cxx | Java |
|------------------------------|-----|-----|-----|
| Plan with data file          | 1,2 | +   | +   |
| Plan with position deletes   |   2 | +   | +   |
| Plan with equality deletes   |   2 | +   | +   |
| Plan with puffin statistics  | 1,2 | -   | +   |
| Read data file               | 1,2 | +   | +   |
| Read with position deletes   |   2 | +   | +   |
| Read with equality deletes   |   2 | +   | +   |

### Table write operations

| Operation     | Iceberg version | Cxx | Java |
|---------------------------|-----|-----|-----|
| Append data               | 1,2 | +   | +   |
| Write position deletes    |   2 | -   | +   |
| Write equality deletes    |   2 | -   | +   |
| Write deletion vectors    |   3 | +   | +   |

### Catalog

| Table Operation           | Nessie | Glue | HMS | REST |
|----------------------------|-----|-----|-----|-----|
| listTable                  | -   | -   | -   | -   |
| createTable                | -   | -   | -   | -   |
| dropTable                  | -   | -   | -   | -   |
| loadTable                  | +-  | -   | +-  | +-  |
| updateTable                | -   | -   | -   | -   |
| renameTable                | -   | -   | -   | -   |
| tableExists                | -   | -   | -   | -   |
| createView                 | -   | -   | -   | -   |
| dropView                   | -   | -   | -   | -   |
| listView                   | -   | -   | -   | -   |
| viewExists                 | -   | -   | -   | -   |
| replaceView                | -   | -   | -   | -   |
| renameView                 | -   | -   | -   | -   |
| listNamespaces             | -   | -   | -   | -   |
| createNamespace            | -   | -   | -   | -   |
| dropNamespace	             | -   | -   | -   | -   |
| namespaceExists            | -   | -   | -   | -   |
| updateNamespaceProperties  | -   | -   | -   | -   |
| loadNamespaceMetadata      | -   | -   | -   | -   |

## Requirements

- C++20 compliant compiler
- CMake 3.20 or higher
- OpenSSL

### Build, Run Test

You have to [download Apache Arrow dependencies](https://arrow.apache.org/docs/15.0/developers/cpp/building.html#offline-builds) first.
```bash
mkdir _deps && cd _deps
git clone --single-branch -b maint-15.0.2 https://github.com/apache/arrow.git
cd arrow && git apply ../../vendor/arrow/fix_c-ares_url.patch && cd ..
./arrow/cpp/thirdparty/download_dependencies.sh ./arrow-thirdparty
```

```bash
mkdir _build
cd _build
ln -s ../_deps/arrow-thirdparty arrow-thirdparty
cmake -GNinja ../
ninja
cd tests/
../iceberg/iceberg-cpp-test
../iceberg/common/fs/iceberg_common_fs_test
./iceberg_local_test
```

iceberg-cxx

File Explorer

Today's picks

Was this content helpful?

Recent Feedback

iceberg-cxx

File Explorer

Today's picks

Was this content helpful?

Recent Feedback

Download README