make_v1_table.py

"""
Reproduce the DuckDB iceberg_scan error on a format-version 1 Iceberg table:

    Invalid Configuration Error:
    'manifest_entry.sequence_number' is only allowed to be NULL for ADDED entries

Strategy: create a *format-version 1* table, append data, then delete a subset of
rows. The delete is a copy-on-write operation that rewrites manifests and produces
EXISTING / DELETED manifest entries (status != ADDED). In a v1 table sequence_number
is absent (NULL) for every entry, which trips DuckDB's v2 validation rule.
"""

import os
import shutil

import pyarrow as pa
from pyiceberg.catalog.sql import SqlCatalog

WAREHOUSE = "/tmp/iceberg_v1_repro/warehouse"
NS = "repro"
TBL = "merch_v1"


def main() -> None:
    # Clean slate
    if os.path.exists(WAREHOUSE):
        shutil.rmtree(WAREHOUSE)
    os.makedirs(WAREHOUSE, exist_ok=True)

    catalog = SqlCatalog(
        "repro",
        uri=f"sqlite:///{WAREHOUSE}/catalog.db",
        warehouse=f"file://{WAREHOUSE}",
    )
    catalog.create_namespace_if_not_exists(NS)

    schema = pa.schema(
        [
            ("id", pa.int64()),
            ("league", pa.string()),
            ("ats_qty", pa.int64()),
        ]
    )

    # format-version = 1 is the whole point of this repro.
    table = catalog.create_table(
        f"{NS}.{TBL}",
        schema=schema,
        properties={"format-version": "1"},
    )

    # Append two batches (each append => new data files / ADDED entries).
    batch1 = pa.table(
        {"id": [1, 2, 3], "league": ["nfl", "nba", "mlb"], "ats_qty": [10, 20, 30]},
        schema=schema,
    )
    batch2 = pa.table(
        {"id": [4, 5, 6], "league": ["nhl", "nfl", "nba"], "ats_qty": [40, 50, 60]},
        schema=schema,
    )
    table.append(batch1)
    table.append(batch2)

    # Delete a subset -> copy-on-write rewrite -> DELETED + EXISTING manifest entries.
    table.delete(delete_filter="league == 'nfl'")

    table.refresh()
    fmt = table.metadata.format_version
    print(f"format-version          : {fmt}")
    print(f"table location          : {table.location()}")
    print(f"current metadata file   : {table.metadata_location}")
    print(f"current snapshot op     : {table.current_snapshot().summary.operation.value}")


if __name__ == "__main__":
    main()

patch_to_existing.py

"""
Turn the pyiceberg-generated v1 table into a faithful reproduction of the Spark
v1 'overwrite' shape: a *live* manifest whose entries are EXISTING (status=0) and
whose avro has no sequence_number field (=> NULL on read).

This is exactly what DuckDB's v1.5.3 iceberg build (4008894c) rejects with:
    'manifest_entry.sequence_number' is only allowed to be NULL for ADDED entries
"""

import glob
import os

import fastavro

META = "/tmp/iceberg_v1_repro/warehouse/repro/merch_v1/metadata"


def _read(path):
    with open(path, "rb") as f:
        rd = fastavro.reader(f)
        schema = rd.writer_schema
        codec = rd.codec
        # keep only non-avro.* header metadata (iceberg keys: schema, partition-spec, ...)
        meta = {k: v for k, v in rd.metadata.items() if not k.startswith("avro.")}
        recs = list(rd)
    return schema, codec, meta, recs


def _write(path, schema, codec, meta, recs):
    with open(path, "wb") as f:
        fastavro.writer(f, schema, recs, codec=codec, metadata=meta)


def main() -> None:
    # m0 holds the live (ADDED) data files after the delete.
    m0 = sorted(glob.glob(f"{META}/*-m0.avro"))[-1]
    schema, codec, meta, recs = _read(m0)
    flipped = 0
    for r in recs:
        if r["status"] == 1:  # ADDED -> EXISTING
            r["status"] = 0
            flipped += 1
    _write(m0, schema, codec, meta, recs)
    new_len = os.path.getsize(m0)
    print(f"m0: flipped {flipped} ADDED entries -> EXISTING ({os.path.basename(m0)})")

    # Fix the manifest list so the m0 manifest is reported as existing (so DuckDB reads it).
    ml = sorted(glob.glob(f"{META}/snap-*.avro"))[-1]
    schema, codec, meta, recs = _read(ml)
    for r in recs:
        if r["manifest_path"].endswith(os.path.basename(m0)):
            r["added_files_count"] = 0
            r["existing_files_count"] = flipped
            r["manifest_length"] = new_len
    _write(ml, schema, codec, meta, recs)
    print(f"manifest-list patched: {os.path.basename(ml)}")


if __name__ == "__main__":
    main()

patch_v2.py

"""
Faithful repro of the Spark v1 'overwrite' manifest shape that DuckDB 1.5.3 rejects.

Key realization: the error names `manifest_entry.sequence_number`, which means the
manifest avro SCHEMA contains that (optional) field, set to NULL, on a non-ADDED
(EXISTING/DELETED) entry. A pure-v1 manifest with the field *absent* is read fine
(DuckDB infers seq=0). So we:

  1. inject an optional `sequence_number` field (["null","long"]) into the manifest_entry
     avro schema,
  2. set its value to NULL on every entry,
  3. mark the live entries EXISTING (status=0),

then point the manifest list at it. This is what trips:
    'manifest_entry.sequence_number' is only allowed to be NULL for ADDED entries
"""

import glob
import os

import fastavro

META = "/tmp/iceberg_v1_repro/warehouse/repro/merch_v1/metadata"


def _read(path):
    with open(path, "rb") as f:
        rd = fastavro.reader(f)
        return rd.writer_schema, rd.codec, {k: v for k, v in rd.metadata.items() if not k.startswith("avro.")}, list(rd)


def _write(path, schema, codec, meta, recs):
    with open(path, "wb") as f:
        fastavro.writer(f, schema, recs, codec=codec, metadata=meta)


def main() -> None:
    m0 = sorted(glob.glob(f"{META}/*-m0.avro"))[-1]
    schema, codec, meta, recs = _read(m0)

    field_names = [f["name"] for f in schema["fields"]]
    if "sequence_number" not in field_names:
        # insert optional sequence_number (iceberg field-id 3) after snapshot_id
        seq_field = {"name": "sequence_number", "type": ["null", "long"], "default": None, "field-id": 3}
        idx = field_names.index("snapshot_id") + 1 if "snapshot_id" in field_names else len(schema["fields"])
        schema["fields"].insert(idx, seq_field)

    for r in recs:
        r["status"] = 0          # EXISTING
        r["sequence_number"] = None

    _write(m0, schema, codec, meta, recs)
    new_len = os.path.getsize(m0)
    print(f"m0 patched: {len(recs)} EXISTING entries, sequence_number field present & NULL ({os.path.basename(m0)})")

    ml = sorted(glob.glob(f"{META}/snap-*.avro"))[-1]
    schema, codec, meta, recs = _read(ml)
    for r in recs:
        if r["manifest_path"].endswith(os.path.basename(m0)):
            r["added_files_count"] = 0
            r["existing_files_count"] = 1
            r["manifest_length"] = new_len
    _write(ml, schema, codec, meta, recs)
    print(f"manifest-list patched: {os.path.basename(ml)}")


if __name__ == "__main__":
    main()

patch_v3.py

"""
Faithful repro of the Spark v1 'overwrite' manifest shape DuckDB 1.5.3 rejects.

Follows the *current* snapshot's manifest chain (metadata.json -> manifest list ->
data manifest) so we patch exactly the files the scan will read:

  - inject an optional `sequence_number` field (["null","long"], field-id 3) into the
    data manifest_entry schema, value NULL
  - mark its live entries EXISTING (status=0)
  - update that manifest's row in the manifest list (added->0, existing->n, length)

This triggers IcebergManifestEntry::GetSequenceNumber():
    'manifest_entry.sequence_number' is only allowed to be NULL for ADDED entries
"""

import glob
import json
import os

import fastavro

META = "/tmp/iceberg_v1_repro/warehouse/repro/merch_v1/metadata"


def _read(path):
    with open(path, "rb") as f:
        rd = fastavro.reader(f)
        return rd.writer_schema, rd.codec, {k: v for k, v in rd.metadata.items() if not k.startswith("avro.")}, list(rd)


def _write(path, schema, codec, meta, recs):
    with open(path, "wb") as f:
        fastavro.writer(f, schema, recs, codec=codec, metadata=meta)


def _local(path: str) -> str:
    return path.replace("file://", "")


def main() -> None:
    # 1. current metadata json -> current snapshot -> manifest list
    meta_json = sorted(glob.glob(f"{META}/*.metadata.json"))[-1]
    md = json.load(open(meta_json))
    cur_id = md["current-snapshot-id"]
    snap = next(s for s in md["snapshots"] if s["snapshot-id"] == cur_id)
    manifest_list_path = _local(snap["manifest-list"])
    print(f"current snapshot {cur_id} op={snap['summary']['operation']}")
    print(f"manifest list: {os.path.basename(manifest_list_path)}")

    # 2. read manifest list, find the data manifest that has live (added) files
    ml_schema, ml_codec, ml_meta, ml_recs = _read(manifest_list_path)
    target = next(r for r in ml_recs if r.get("added_files_count", 0) > 0)
    data_manifest = _local(target["manifest_path"])
    print(f"data manifest: {os.path.basename(data_manifest)}  added={target['added_files_count']}")

    # 3. patch the data manifest: inject null sequence_number, set EXISTING
    schema, codec, meta, recs = _read(data_manifest)
    names = [f["name"] for f in schema["fields"]]
    if "sequence_number" not in names:
        idx = names.index("snapshot_id") + 1
        schema["fields"].insert(idx, {"name": "sequence_number", "type": ["null", "long"], "default": None, "field-id": 3})
    n = 0
    for r in recs:
        r["status"] = 0           # EXISTING
        r["sequence_number"] = None
        n += 1
    _write(data_manifest, schema, codec, meta, recs)
    new_len = os.path.getsize(data_manifest)
    print(f"patched {n} entries -> EXISTING, sequence_number present & NULL")

    # 4. fix the manifest-list row for this manifest
    for r in ml_recs:
        if _local(r["manifest_path"]) == data_manifest:
            r["added_files_count"] = 0
            r["existing_files_count"] = n
            r["manifest_length"] = new_len
    _write(manifest_list_path, ml_schema, ml_codec, ml_meta, ml_recs)
    print("manifest list updated")


if __name__ == "__main__":
    main()

reproduce.sh

#!/usr/bin/env bash
#
# Reproduce: DuckDB iceberg_scan fails on a format-version 1 table whose manifest_entry
# has a NULL sequence_number on a non-ADDED (EXISTING) entry.
#
#   Invalid Configuration Error:
#   'manifest_entry.sequence_number' is only allowed to be NULL for ADDED entries
#
# Requirements: python3, duckdb CLI (>= 1.5.x, iceberg ext 4008894c reproduces).
set -euo pipefail
cd "$(dirname "$0")"

# Isolated venvs: pyiceberg writes the v1 table; fastavro patches the manifest.
python3 -m venv .pyvenv   && .pyvenv/bin/pip -q install "pyiceberg[sql-sqlite]" pyarrow
python3 -m venv .avrovenv && .avrovenv/bin/pip -q install fastavro

.pyvenv/bin/python  make_v1_table.py
.avrovenv/bin/python patch_v3.py

echo "### iceberg_scan (expect the error) ###"
duckdb -c "
  LOAD iceberg;
  SET unsafe_enable_version_guessing = true;
  SELECT * FROM iceberg_scan('$(pwd)/warehouse/repro/merch_v1', allow_moved_paths=true);
"