Enabling Iceberg Catalogs

Connecting to Iceberg catalogs

An Iceberg catalog is a registry of Apache Iceberg tables and metadata that makes it easy to work with datasets stored in object storage. The ACM makes it easy to work with an Iceberg catalog inside SaaS and Bring Your Own Cloud (BYOC) environments. For Bring Your Own Kubernetes (BYOK) environments, Altinity doesn’t have access to create S3 buckets in your account. Instructions for enabling Iceberg catalogs in BYOK environments are at the bottom of this page.

Enabling Iceberg catalogs in SaaS and BYOC environments

The first time you click the Iceberg Catalog menu item, you’ll be told that Iceberg catalogs are not enabled:

Figure 1 - Iceberg catalogs are not enabled

Click the button to enable catalogs. You’ll see this dialog:

Figure 2 - Iceberg catalog enablement in progress

Click the button. At some point you’ll see the details of your catalog:

Figure 3 - Iceberg catalog details

You can see these details at any time by clicking on the Enabled link next to Iceberg Catalogs on the Environment summary panel:

Figure 4 - Iceberg catalog status

To disable the Iceberg catalog, you can click the button. You’ll be asked to confirm your choice before the catalog is disabled.

Enabling Iceberg catalogs for BYOK environments

NOTE: Currently we only support BYOK Iceberg catalogs in AWS environments.

Altinity doesn’t have permissions to create S3 buckets within your Kubernetes environment, so you’ll need to set those up and give us the details. Fortunately, we provide a Terraform script that you can use with your AWS credentials. Save the following text in a file named main.tf:

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 6.0"
    }
  }
}

provider "aws" {}

variable "eks_cluster_name" {
  type        = string
  description = "EKS cluster name"
}

variable "catalog_name" {
  type        = string
  description = "Iceberg Catalog name (empty = default)"
  default     = ""
  nullable    = false
}

variable "s3_bucket_prefix" {
  type    = string
  default = "iceberg"
}

variable "s3_bucket_name" {
  type    = string
  default = null
}

variable "s3_bucket_new" {
  type    = bool
  default = true
}

locals {
  catalog_qualifier = var.catalog_name != "" ? "-${var.catalog_name}" : ""
  tags = {}
}

data "aws_s3_bucket" "this" {
  count  = var.s3_bucket_new ? 0 : 1
  bucket = var.s3_bucket_name
}

resource "aws_s3_bucket" "this" {
  count         = var.s3_bucket_new ? 1 : 0
  bucket        = var.s3_bucket_name
  bucket_prefix = var.s3_bucket_name == null ? var.s3_bucket_prefix : null
  tags          = local.tags
  force_destroy = true
}

locals {
  s3_bucket_name = var.s3_bucket_new ? aws_s3_bucket.this[0].id : data.aws_s3_bucket.this[0].id
  s3_bucket_arn  = var.s3_bucket_new ? aws_s3_bucket.this[0].arn : data.aws_s3_bucket.this[0].arn
}

data "aws_eks_cluster" "current" {
  name = var.eks_cluster_name
}

data "aws_caller_identity" "current" {}

locals {
  oidc_provider = replace(data.aws_eks_cluster.current.identity.0.oidc.0.issuer, "https://", "")
  oidc_provider_id = split("/id/", local.oidc_provider)[1]
}

resource "aws_iam_role" "this" {
  name               = "ice-rest-catalog-${local.oidc_provider_id}${local.catalog_qualifier}"
  assume_role_policy = <<-EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Federated": "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_provider}"
      },
      "Action": "sts:AssumeRoleWithWebIdentity",
      "Condition": {
        "StringEquals": {
          "${local.oidc_provider}:sub": "system:serviceaccount:altinity-cloud-managed-clickhouse:iceberg${local.catalog_qualifier}"
        }
      }
    }
  ]
}
  EOF
  tags               = local.tags
}

resource "aws_iam_role_policy" "this" {
  name   = "ice-rest-catalog-${local.oidc_provider_id}${local.catalog_qualifier}"
  role   = aws_iam_role.this.id
  policy = <<-EOF
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "s3:*"
            ],
            "Resource": [
                "${local.s3_bucket_arn}",
                "${local.s3_bucket_arn}/*"
            ],
            "Effect": "Allow"
        },
        {
            "Action": "sts:AssumeRole",
            "Resource": ["${aws_iam_role.rw.arn}", "${aws_iam_role.ro.arn}"],
            "Effect": "Allow"
        }
    ]
}
  EOF
}


resource "aws_iam_role" "rw" {
  name               = "ice-rest-catalog-rw-${local.oidc_provider_id}${local.catalog_qualifier}"
  assume_role_policy = <<-EOF
{
  "Version": "2012-10-17",
  "Statement": [{
    "Effect": "Allow",
    "Principal": {
      "AWS": "${aws_iam_role.this.arn}"
    },
    "Action": "sts:AssumeRole"
  }]
}
  EOF
  tags               = local.tags
}

resource "aws_iam_role_policy" "rw" {
  name   = "ice-rest-catalog-rw-${local.oidc_provider_id}${local.catalog_qualifier}"
  role   = aws_iam_role.rw.id
  policy = <<-EOF
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "s3:Get*",
                "s3:List*",
                "s3:Describe*",
                "s3:PutObject",
                "s3:PutObject*",
                "s3:DeleteObject",
                "s3:DeleteObject*",
                "s3:AbortMultipartUpload"
            ],
            "Resource": [
                "${local.s3_bucket_arn}",
                "${local.s3_bucket_arn}/*"
            ],
            "Effect": "Allow"
        }
    ]
}
  EOF
}

resource "aws_iam_role" "ro" {
  name               = "ice-rest-catalog-ro-${local.oidc_provider_id}${local.catalog_qualifier}"
  assume_role_policy = <<-EOF
{
  "Version": "2012-10-17",
  "Statement": [{
    "Effect": "Allow",
    "Principal": {
      "AWS": "${aws_iam_role.this.arn}"
    },
    "Action": "sts:AssumeRole"
  }]
}
  EOF
  tags               = local.tags
}

resource "aws_iam_role_policy" "ro" {
  name   = "ice-rest-catalog-ro-${local.oidc_provider_id}${local.catalog_qualifier}"
  role   = aws_iam_role.ro.id
  policy = <<-EOF
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "s3:Get*",
                "s3:List*",
                "s3:Describe*"
            ],
            "Resource": [
                "${local.s3_bucket_arn}",
                "${local.s3_bucket_arn}/*"
            ],
            "Effect": "Allow"
        }
    ]
}
  EOF
}

output "s3_bucket_name" {
  value = local.s3_bucket_name
}

output "iam_role_arn" {
  value = aws_iam_role.this.arn
}

output "iam_role_rw_arn" {
  value = aws_iam_role.rw.arn
}

output "iam_role_ro_arn" {
  value = aws_iam_role.ro.arn
}

output "domain" {
  value = "iceberg-catalog${local.catalog_qualifier}"
}

Running the script is simple:

Define your AWS credentials in the environment variables AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_SESSION_TOKEN.
Terraform will create the S3 buckets in your default AWS region. Define the variables AWS_DEFAULT_REGION and AWS_REGION with the correct region (us-east-1, for example) to make sure your buckets are created where you want them.
Run terraform init to download the dependencies for your Terraform script.
Run terraform apply -var eks_cluster_name=my-cluster to specify the name of your EKS cluster and create the S3 resources you’ll use for your Iceberg catalog.

When the Terraform script is done, it will output five variables:

Apply complete! Resources: 7 added, 0 changed, 0 destroyed.

Outputs:

domain = "iceberg-catalog"
iam_role_arn = "arn:aws:iam::123456789012:role/ice-rest-catalog-1234567890ABCDEF1234567890ABCDEF"
iam_role_ro_arn = "arn:aws:iam::123456789012:role/ice-rest-catalog-ro-1234567890ABCDEF1234567890ABCDEF"
iam_role_rw_arn = "arn:aws:iam::123456789012:role/ice-rest-catalog-rw-1234567890ABCDEF1234567890ABCDEF"
s3_bucket_name = "iceberg20251030123456789012345678"

Now you’ll need to contact Altinity support with the S3 bucket name. They’ll complete the setup for you. When that’s done, you can use the Enabled link in the Environment summary panel (see Figure 4 above) to get the access token and endpoint details as shown above in Figure 3.