diff --git a/core/src/main/java/org/apache/iceberg/ColumnFile.java b/core/src/main/java/org/apache/iceberg/ColumnFile.java new file mode 100644 index 000000000000..dde090666a91 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/ColumnFile.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.List; +import org.apache.iceberg.types.Types; + +interface ColumnFile { + Types.NestedField LOCATION = + Types.NestedField.required( + 160, "location", Types.StringType.get(), "Location of the column file"); + Types.NestedField FIELD_IDS = + Types.NestedField.required( + 161, + "field_ids", + Types.ListType.ofRequired(162, Types.IntegerType.get()), + "Live field IDs in this column file"); + Types.NestedField FILE_FORMAT = + Types.NestedField.required( + 163, + "file_format", + Types.StringType.get(), + "String file format name for this column file"); + Types.NestedField FILE_SIZE_IN_BYTES = + Types.NestedField.required( + 164, "file_size_in_bytes", Types.LongType.get(), "Total column file size in bytes"); + Types.NestedField KEY_METADATA = + Types.NestedField.optional( + 165, + "key_metadata", + Types.BinaryType.get(), + "Key metadata for encryption; specific to the encryption scheme."); + + static Types.StructType schema() { + return Types.StructType.of(LOCATION, FIELD_IDS, FILE_FORMAT, FILE_SIZE_IN_BYTES, KEY_METADATA); + } + + /** Returns the location of this column file. */ + String location(); + + /** Returns the field IDs contained in this column file. */ + List fieldIds(); + + /** Returns the format of this column file. */ + FileFormat fileFormat(); + + /** Returns the total size of this column file in bytes. */ + long fileSizeInBytes(); + + /** Returns encryption key metadata, or null if this column file is not encrypted. */ + ByteBuffer keyMetadata(); + + /** Copies this column file. */ + ColumnFile copy(); +} diff --git a/core/src/main/java/org/apache/iceberg/ColumnFileStruct.java b/core/src/main/java/org/apache/iceberg/ColumnFileStruct.java new file mode 100644 index 000000000000..6213cde35ea6 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/ColumnFileStruct.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import org.apache.iceberg.avro.SupportsIndexProjection; +import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ArrayUtil; +import org.apache.iceberg.util.ByteBuffers; + +/** Mutable {@link StructLike} implementation of {@link ColumnFile}. */ +class ColumnFileStruct extends SupportsIndexProjection implements ColumnFile, Serializable { + private static final Types.StructType BASE_TYPE = + Types.StructType.of( + ColumnFile.LOCATION, + ColumnFile.FIELD_IDS, + ColumnFile.FILE_FORMAT, + ColumnFile.FILE_SIZE_IN_BYTES, + ColumnFile.KEY_METADATA); + + private String location = null; + private int[] fieldIds = null; + private FileFormat fileFormat = null; + private long fileSizeInBytes = -1L; + private byte[] keyMetadata = null; + + /** Used by internal readers to instantiate this class with a projection schema. */ + ColumnFileStruct(Types.StructType projection) { + super(BASE_TYPE, projection); + } + + ColumnFileStruct( + String location, + List fieldIds, + FileFormat fileFormat, + long fileSizeInBytes, + ByteBuffer keyMetadata) { + super(BASE_TYPE.fields().size()); + this.location = location; + this.fieldIds = ArrayUtil.toIntArray(fieldIds); + this.fileFormat = fileFormat; + this.fileSizeInBytes = fileSizeInBytes; + this.keyMetadata = ByteBuffers.toByteArray(keyMetadata); + } + + /** Copy constructor. */ + private ColumnFileStruct(ColumnFileStruct toCopy) { + super(toCopy); + this.location = toCopy.location; + this.fieldIds = + toCopy.fieldIds != null ? Arrays.copyOf(toCopy.fieldIds, toCopy.fieldIds.length) : null; + this.fileFormat = toCopy.fileFormat; + this.fileSizeInBytes = toCopy.fileSizeInBytes; + this.keyMetadata = + toCopy.keyMetadata != null + ? Arrays.copyOf(toCopy.keyMetadata, toCopy.keyMetadata.length) + : null; + } + + /** Constructor for Java serialization. */ + ColumnFileStruct() { + super(BASE_TYPE.fields().size()); + } + + @Override + public String location() { + return location; + } + + @Override + public List fieldIds() { + return fieldIds != null ? ArrayUtil.toUnmodifiableIntList(fieldIds) : null; + } + + @Override + public FileFormat fileFormat() { + return fileFormat; + } + + @Override + public long fileSizeInBytes() { + return fileSizeInBytes; + } + + @Override + public ByteBuffer keyMetadata() { + return keyMetadata != null ? ByteBuffer.wrap(keyMetadata) : null; + } + + @Override + public ColumnFile copy() { + return new ColumnFileStruct(this); + } + + @Override + protected T internalGet(int pos, Class javaClass) { + return javaClass.cast(getByPos(pos)); + } + + private Object getByPos(int pos) { + return switch (pos) { + case 0 -> location; + case 1 -> fieldIds(); + case 2 -> fileFormat != null ? fileFormat.toString() : null; + case 3 -> fileSizeInBytes; + case 4 -> keyMetadata(); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; + } + + @Override + @SuppressWarnings("unchecked") + protected void internalSet(int pos, T value) { + switch (pos) { + // always coerce to String for Serializable + case 0 -> this.location = value.toString(); + case 1 -> this.fieldIds = ArrayUtil.toIntArray((List) value); + case 2 -> this.fileFormat = FileFormat.fromString(value.toString()); + case 3 -> this.fileSizeInBytes = (long) value; + case 4 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); + default -> { + // ignore the object, it must be from a newer version of the format + } + } + } + + static Builder builder() { + return new Builder(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("location", location) + .add("field_ids", fieldIds) + .add("file_format", fileFormat) + .add("file_size_in_bytes", fileSizeInBytes) + .add("key_metadata", keyMetadata == null ? "null" : "(redacted)") + .toString(); + } + + static class Builder { + private String location = null; + private List fieldIds = null; + private FileFormat fileFormat = null; + private Long fileSizeInBytes = null; + private ByteBuffer keyMetadata = null; + + Builder location(String newLocation) { + Preconditions.checkArgument(newLocation != null, "Invalid location: null"); + Preconditions.checkArgument(!newLocation.isEmpty(), "Invalid location: empty"); + this.location = newLocation; + return this; + } + + Builder fieldIds(List newFieldIds) { + Preconditions.checkArgument(newFieldIds != null, "Invalid field IDs: null"); + Preconditions.checkArgument(!newFieldIds.isEmpty(), "Invalid field IDs: empty"); + Preconditions.checkArgument( + Sets.newHashSet(newFieldIds).size() == newFieldIds.size(), + "Invalid field IDs: duplicated IDs found in: %s", + newFieldIds); + List sortedFieldIds = Lists.newArrayList(newFieldIds); + sortedFieldIds.sort(Comparator.naturalOrder()); + this.fieldIds = sortedFieldIds; + return this; + } + + Builder fileFormat(FileFormat newFileFormat) { + Preconditions.checkArgument(newFileFormat != null, "Invalid file format: null"); + this.fileFormat = newFileFormat; + return this; + } + + Builder fileSizeInBytes(long newFileSizeInBytes) { + Preconditions.checkArgument( + newFileSizeInBytes >= 0, + "Invalid file size in bytes: %s (must be >= 0)", + newFileSizeInBytes); + this.fileSizeInBytes = newFileSizeInBytes; + return this; + } + + Builder keyMetadata(ByteBuffer newKeyMetadata) { + this.keyMetadata = newKeyMetadata; + return this; + } + + ColumnFile build() { + Preconditions.checkArgument(location != null, "Missing required value: location"); + Preconditions.checkArgument(fieldIds != null, "Missing required value: field IDs"); + Preconditions.checkArgument(fileFormat != null, "Missing required value: file format"); + Preconditions.checkArgument( + fileSizeInBytes != null, "Missing required value: file size in bytes"); + return new ColumnFileStruct(location, fieldIds, fileFormat, fileSizeInBytes, keyMetadata); + } + } +} diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index 32757d235bcd..3581a22ec3c8 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -87,6 +87,9 @@ interface TrackedFile { "split_offsets", Types.ListType.ofRequired(133, Types.LongType.get()), "Split offsets for the data file"); + Types.NestedField COLUMN_FILES = + Types.NestedField.optional( + 158, "column_files", Types.ListType.ofRequired(159, ColumnFile.schema()), "Column files"); private static List fields( Types.StructType partitionType, Types.StructType contentStatsType) { @@ -109,7 +112,8 @@ PARTITION_ID, PARTITION_NAME, typeOrUnknown(partitionType), PARTITION_DOC), DELETION_VECTOR, MANIFEST_INFO, KEY_METADATA, - SPLIT_OFFSETS); + SPLIT_OFFSETS, + COLUMN_FILES); } private static Type typeOrUnknown(Types.StructType structType) { @@ -178,6 +182,9 @@ static Schema readSchema(Types.StructType partitionType, Types.StructType conten /** Returns the list of recommended split locations, or null. */ List splitOffsets(); + /** Returns the column files for this file. */ + List columnFiles(); + /** Copies this tracked file. */ TrackedFile copy(); diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index e19ab7ab0ef3..f6a96edeb55f 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -21,11 +21,13 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Set; import org.apache.iceberg.avro.SupportsIndexProjection; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ArrayUtil; import org.apache.iceberg.util.ByteBuffers; @@ -59,7 +61,8 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile, TrackedFile.DELETION_VECTOR, TrackedFile.MANIFEST_INFO, TrackedFile.KEY_METADATA, - TrackedFile.SPLIT_OFFSETS); + TrackedFile.SPLIT_OFFSETS, + TrackedFile.COLUMN_FILES); private FileContent contentType = null; private String location = null; @@ -77,6 +80,7 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile, private ManifestInfo manifestInfo = null; private byte[] keyMetadata = null; private long[] splitOffsets = null; + private List columnFiles = null; private transient StructProjection partitionProjection = null; @@ -104,7 +108,8 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile, DeletionVector deletionVector, ManifestInfo manifestInfo, ByteBuffer keyMetadata, - List splitOffsets) { + List splitOffsets, + List columnFiles) { super(BASE_TYPE.fields().size()); this.tracking = tracking; this.contentType = contentType; @@ -120,9 +125,11 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile, this.manifestInfo = manifestInfo; this.keyMetadata = ByteBuffers.toByteArray(keyMetadata); this.splitOffsets = ArrayUtil.toLongArray(splitOffsets); + this.columnFiles = columnFiles != null ? Lists.newArrayList(columnFiles) : null; } /** Copy constructor. */ + @SuppressWarnings("CyclomaticComplexity") private TrackedFileStruct(TrackedFileStruct toCopy, Set statsIds) { super(toCopy); this.contentType = toCopy.contentType; @@ -152,6 +159,14 @@ private TrackedFileStruct(TrackedFileStruct toCopy, Set statsIds) { toCopy.splitOffsets != null ? Arrays.copyOf(toCopy.splitOffsets, toCopy.splitOffsets.length) : null; + if (toCopy.columnFiles != null) { + this.columnFiles = Lists.newArrayListWithCapacity(toCopy.columnFiles.size()); + for (ColumnFile columnFile : toCopy.columnFiles) { + this.columnFiles.add(columnFile.copy()); + } + } else { + this.columnFiles = null; + } } @Override @@ -244,6 +259,11 @@ public List splitOffsets() { return splitOffsets != null ? ArrayUtil.toUnmodifiableLongList(splitOffsets) : null; } + @Override + public List columnFiles() { + return columnFiles != null ? Collections.unmodifiableList(columnFiles) : null; + } + @Override public TrackedFile copy() { return new TrackedFileStruct(this, null); @@ -275,6 +295,7 @@ private Object getByPos(int pos) { case 11 -> manifestInfo; case 12 -> keyMetadata(); case 13 -> splitOffsets(); + case 14 -> columnFiles(); default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); }; } @@ -298,6 +319,7 @@ protected void internalSet(int pos, T value) { case 11 -> this.manifestInfo = (ManifestInfo) value; case 12 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); case 13 -> this.splitOffsets = ArrayUtil.toLongArray((List) value); + case 14 -> this.columnFiles = (List) value; default -> { // ignore the object, it must be from a newer version of the format } @@ -321,6 +343,7 @@ public String toString() { .add("manifest_info", manifestInfo) .add("key_metadata", keyMetadata == null ? "null" : "(redacted)") .add("split_offsets", splitOffsets == null ? "null" : splitOffsets()) + .add("column_files", columnFiles) .toString(); } } diff --git a/core/src/main/java/org/apache/iceberg/TrackingBuilder.java b/core/src/main/java/org/apache/iceberg/TrackingBuilder.java index c076531fcb45..64e7cae043b6 100644 --- a/core/src/main/java/org/apache/iceberg/TrackingBuilder.java +++ b/core/src/main/java/org/apache/iceberg/TrackingBuilder.java @@ -25,10 +25,10 @@ class TrackingBuilder { private final long newSnapshotId; private final Long snapshotId; - private final Long dataSequenceNumber; private final Long fileSequenceNumber; private final Long firstRowId; private EntryStatus status; + private Long dataSequenceNumber; private Long modifiedSnapshotId; private byte[] deletedPositions; private byte[] replacedPositions; @@ -115,6 +115,19 @@ TrackingBuilder dvUpdated() { return this; } + /** Indicates that the column files list has been updated for the new Tracking. */ + TrackingBuilder columnFilesUpdated() { + this.modifiedSnapshotId = newSnapshotId; + if (status == EntryStatus.EXISTING) { + this.status = EntryStatus.MODIFIED; + } + // Reset to null to inherit from the new snapshot sequence number. It is safe to bump up the + // dataSequenceNumber as writers are required to rewrite v2 equality and position deletes to DVs + // when applying column update. + this.dataSequenceNumber = null; + return this; + } + /** Sets the positions deleted by this commit for a manifest entry. */ TrackingBuilder deletedPositions(ByteBuffer positions) { Preconditions.checkState( diff --git a/core/src/main/java/org/apache/iceberg/TrackingStruct.java b/core/src/main/java/org/apache/iceberg/TrackingStruct.java index c804497d829e..d06eae71c58d 100644 --- a/core/src/main/java/org/apache/iceberg/TrackingStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackingStruct.java @@ -141,8 +141,9 @@ void inherit(long manifestSnapshotId, long manifestSeqNumber) { } boolean isAdded = status == EntryStatus.ADDED; + boolean isModified = status == EntryStatus.MODIFIED; - if (null == dataSequenceNumber && (isAdded || manifestSeqNumber == 0)) { + if (null == dataSequenceNumber && (isAdded || isModified || manifestSeqNumber == 0)) { this.dataSequenceNumber = manifestSeqNumber; } diff --git a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java index 2e0307fe9f65..9f06e17c4f06 100644 --- a/core/src/test/java/org/apache/iceberg/StatsTestUtil.java +++ b/core/src/test/java/org/apache/iceberg/StatsTestUtil.java @@ -40,6 +40,7 @@ static TrackedFile trackedFile(String location, long recordCount, ContentStats s null, null, null, + null, null); } diff --git a/core/src/test/java/org/apache/iceberg/TestColumnFileStruct.java b/core/src/test/java/org/apache/iceberg/TestColumnFileStruct.java new file mode 100644 index 000000000000..19c4f1754697 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestColumnFileStruct.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Comparators; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +class TestColumnFileStruct { + + private static final String LOCATION = "s3://bucket/data/column.parquet"; + private static final List FIELD_IDS = Lists.newArrayList(1, 2, 3); + private static final FileFormat FILE_FORMAT = FileFormat.PARQUET; + private static final long FILE_SIZE_IN_BYTES = 1024L; + private static final ByteBuffer KEY_METADATA = ByteBuffer.wrap(new byte[] {1, 2, 3}); + private static final Comparator COLUMN_FILE_COMPARATOR = + Comparators.forType(ColumnFile.schema()); + + @Test + void fieldAccess() { + ColumnFile columnFile = + new ColumnFileStruct(LOCATION, FIELD_IDS, FILE_FORMAT, FILE_SIZE_IN_BYTES, KEY_METADATA); + + assertThat(columnFile.location()).isEqualTo(LOCATION); + assertThat(columnFile.fieldIds()).containsExactlyElementsOf(FIELD_IDS); + assertThat(columnFile.fileFormat()).isEqualTo(FILE_FORMAT); + assertThat(columnFile.fileSizeInBytes()).isEqualTo(FILE_SIZE_IN_BYTES); + assertThat(columnFile.keyMetadata()).isEqualTo(KEY_METADATA); + } + + @Test + void copy() { + ColumnFile columnFile = + new ColumnFileStruct(LOCATION, FIELD_IDS, FILE_FORMAT, FILE_SIZE_IN_BYTES, KEY_METADATA); + + ColumnFile copy = columnFile.copy(); + + assertThat((StructLike) copy).usingComparator(COLUMN_FILE_COMPARATOR).isEqualTo(columnFile); + } + + @Test + void structLikeSize() { + ColumnFileStruct columnFile = new ColumnFileStruct(); + assertThat(columnFile.size()).isEqualTo(5); + } + + @Test + void setFieldsByOrdinals() { + ColumnFileStruct columnFile = new ColumnFileStruct(); + + columnFile.set(0, LOCATION); + columnFile.set(1, FIELD_IDS); + columnFile.set(2, FILE_FORMAT.toString()); + columnFile.set(3, FILE_SIZE_IN_BYTES); + columnFile.set(4, KEY_METADATA); + + assertThat(columnFile.location()).isEqualTo(LOCATION); + assertThat(columnFile.fieldIds()).containsExactlyElementsOf(FIELD_IDS); + assertThat(columnFile.fileFormat()).isEqualTo(FILE_FORMAT); + assertThat(columnFile.fileSizeInBytes()).isEqualTo(FILE_SIZE_IN_BYTES); + assertThat(columnFile.keyMetadata()).isEqualTo(KEY_METADATA); + } + + @Test + void getFieldsByOrdinals() { + ColumnFileStruct columnFile = + new ColumnFileStruct(LOCATION, FIELD_IDS, FILE_FORMAT, FILE_SIZE_IN_BYTES, KEY_METADATA); + + assertThat(columnFile.get(0, String.class)).isEqualTo(LOCATION); + assertThat(columnFile.get(1, List.class)).containsExactlyElementsOf(FIELD_IDS); + assertThat(columnFile.get(2, String.class)).isEqualTo(FILE_FORMAT.toString()); + assertThat(columnFile.get(3, Long.class)).isEqualTo(FILE_SIZE_IN_BYTES); + assertThat(columnFile.get(4, ByteBuffer.class)).isEqualTo(KEY_METADATA); + } + + @Test + void projectedStructLike() { + Types.StructType projection = + Types.StructType.of(ColumnFile.LOCATION, ColumnFile.FILE_SIZE_IN_BYTES); + + ColumnFileStruct columnFile = new ColumnFileStruct(projection); + assertThat(columnFile.size()).isEqualTo(2); + + // projected position 0 maps to internal position of location + // projected position 1 maps to internal position of file_size_in_bytes + columnFile.set(0, LOCATION); + columnFile.set(1, FILE_SIZE_IN_BYTES); + + assertThat(columnFile.location()).isEqualTo(LOCATION); + assertThat(columnFile.fileSizeInBytes()).isEqualTo(FILE_SIZE_IN_BYTES); + assertThat(columnFile.get(0, String.class)).isEqualTo(LOCATION); + assertThat(columnFile.get(1, Long.class)).isEqualTo(FILE_SIZE_IN_BYTES); + } + + @ParameterizedTest + @MethodSource("org.apache.iceberg.TestHelpers#serializers") + void serializationRoundTrip(TestHelpers.RoundTripSerializer roundTripSerializer) + throws IOException, ClassNotFoundException { + ColumnFile columnFile = + new ColumnFileStruct(LOCATION, FIELD_IDS, FILE_FORMAT, FILE_SIZE_IN_BYTES, KEY_METADATA); + + ColumnFile deserialized = roundTripSerializer.apply(columnFile); + + assertThat((StructLike) deserialized) + .usingComparator(COLUMN_FILE_COMPARATOR) + .isEqualTo(columnFile); + } + + @Test + void invalidBuilderValues() { + assertThatThrownBy(() -> ColumnFileStruct.builder().location(null).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid location: null"); + + assertThatThrownBy(() -> ColumnFileStruct.builder().location("").build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid location: empty"); + + assertThatThrownBy(() -> ColumnFileStruct.builder().fieldIds(null).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid field IDs: null"); + + assertThatThrownBy(() -> ColumnFileStruct.builder().fieldIds(Lists.newArrayList()).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid field IDs: empty"); + + assertThatThrownBy( + () -> ColumnFileStruct.builder().fieldIds(Lists.newArrayList(1, 2, 1)).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid field IDs: duplicated IDs found in: [1, 2, 1]"); + + assertThatThrownBy(() -> ColumnFileStruct.builder().fileFormat(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file format: null"); + + assertThatThrownBy(() -> ColumnFileStruct.builder().fileSizeInBytes(-1).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file size in bytes: -1 (must be >= 0)"); + } + + @Test + void missingBuilderValues() { + assertThatThrownBy( + () -> + ColumnFileStruct.builder() + .fieldIds(FIELD_IDS) + .fileFormat(FILE_FORMAT) + .fileSizeInBytes(FILE_SIZE_IN_BYTES) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required value: location"); + + assertThatThrownBy( + () -> + ColumnFileStruct.builder() + .location(LOCATION) + .fileFormat(FILE_FORMAT) + .fileSizeInBytes(FILE_SIZE_IN_BYTES) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required value: field IDs"); + + assertThatThrownBy( + () -> + ColumnFileStruct.builder() + .location(LOCATION) + .fieldIds(FIELD_IDS) + .fileSizeInBytes(FILE_SIZE_IN_BYTES) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required value: file format"); + + assertThatThrownBy( + () -> + ColumnFileStruct.builder() + .location(LOCATION) + .fieldIds(FIELD_IDS) + .fileFormat(FILE_FORMAT) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required value: file size in bytes"); + } + + @Test + void builderSortsFieldIds() { + List unsortedFieldIds = Lists.newArrayList(3, 1, 2); + ColumnFile columnFile = + ColumnFileStruct.builder() + .location(LOCATION) + .fieldIds(unsortedFieldIds) + .fileFormat(FILE_FORMAT) + .fileSizeInBytes(FILE_SIZE_IN_BYTES) + .build(); + + assertThat(columnFile.fieldIds()).containsExactly(1, 2, 3); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java index dbb589ee3682..d4c4e467e8d6 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java @@ -59,7 +59,8 @@ public void schemaFieldOrder() { "deletion_vector", "manifest_info", "key_metadata", - "split_offsets"); + "split_offsets", + "column_files"); } @Test @@ -69,7 +70,7 @@ public void schemaFieldIds() { assertThat(fields) .extracting(Types.NestedField::fieldId) - .containsExactly(147, 134, 100, 101, 103, 104, 141, 102, 146, 140, 148, 150, 131, 132); + .containsExactly(147, 134, 100, 101, 103, 104, 141, 102, 146, 140, 148, 150, 131, 132, 158); } @Test diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index 22f980d93bf1..c225550d92bb 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -157,7 +157,8 @@ void dataFileAdapterDelegation() { dv, null, ByteBuffer.wrap(new byte[] {1, 2, 3}), - ImmutableList.of(50L, 100L)); + ImmutableList.of(50L, 100L), + null); DataFile dataFile = TrackedFileAdapters.asDataFile(file, specsById(PARTITIONED_SPEC)); @@ -244,6 +245,7 @@ void dvDeleteFileAdapterDelegation() { dv, null, null, + null, null); DeleteFile dvFile = TrackedFileAdapters.asDVDeleteFile(file, specsById(PARTITIONED_SPEC)); @@ -374,7 +376,8 @@ private static TrackedFile serializableDataEntry() { dv, // deletionVector null, // manifestInfo KEY_METADATA, - ImmutableList.of(50L, 100L)); // splitOffsets + ImmutableList.of(50L, 100L), // splitOffsets + null); // columnFiles } @ParameterizedTest @@ -416,7 +419,8 @@ void manifestFileAdapterDelegation(FileContent contentType) { null, // deletionVector MANIFEST_INFO, MANIFEST_KEY_METADATA, - null); // splitOffsets + null, // splitOffsets + null); // columnFiles ManifestFile manifest = TrackedFileAdapters.asManifestFile(file); @@ -518,6 +522,7 @@ void nullTrackingReturnsNullTrackingFields() { deletionVector(), null, null, + null, null); assertNullTrackingFields(TrackedFileAdapters.asDVDeleteFile(fileWithDV, UNPARTITIONED)); } @@ -570,6 +575,7 @@ void unknownSpecIdThrows() { null, null, null, + null, null); assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, ImmutableMap.of())) @@ -612,6 +618,7 @@ private static TrackedFileStruct dummyTrackedFile(FileContent contentType) { null, null, null, + null, null); } diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index 91919ca24866..f65ce4941e15 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -57,11 +57,18 @@ class TestTrackedFileStruct { private static final ManifestInfo MANIFEST_INFO = Mockito.mock(ManifestInfo.class); private static final ManifestInfo MANIFEST_INFO_COPY = Mockito.mock(ManifestInfo.class); + private static final ColumnFile COLUMN_FILE_1 = Mockito.mock(ColumnFile.class); + private static final ColumnFile COLUMN_FILE_2 = Mockito.mock(ColumnFile.class); + private static final ColumnFile COLUMN_FILE_1_COPY = Mockito.mock(ColumnFile.class); + private static final ColumnFile COLUMN_FILE_2_COPY = Mockito.mock(ColumnFile.class); + static { Mockito.when(TRACKING.copy()).thenReturn(TRACKING_COPY); Mockito.when(CONTENT_STATS.copy()).thenReturn(CONTENT_STATS_COPY); Mockito.when(DELETION_VECTOR.copy()).thenReturn(DELETION_VECTOR_COPY); Mockito.when(MANIFEST_INFO.copy()).thenReturn(MANIFEST_INFO_COPY); + Mockito.when(COLUMN_FILE_1.copy()).thenReturn(COLUMN_FILE_1_COPY); + Mockito.when(COLUMN_FILE_2.copy()).thenReturn(COLUMN_FILE_2_COPY); } @Test @@ -81,7 +88,8 @@ void fieldAccess() { DELETION_VECTOR, MANIFEST_INFO, ByteBuffer.wrap(new byte[] {1, 2, 3}), - ImmutableList.of(100L, 200L)); + ImmutableList.of(100L, 200L), + ImmutableList.of(COLUMN_FILE_1, COLUMN_FILE_2)); assertThat(file.tracking()).isSameAs(TRACKING); assertThat(file.contentType()).isEqualTo(FileContent.DATA); @@ -97,6 +105,7 @@ void fieldAccess() { assertThat(file.manifestInfo()).isSameAs(MANIFEST_INFO); assertThat(file.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); assertThat(file.splitOffsets()).containsExactly(100L, 200L); + assertThat(file.columnFiles()).containsExactly(COLUMN_FILE_1, COLUMN_FILE_2); } @Test @@ -150,7 +159,8 @@ void getByPosition() { DELETION_VECTOR, MANIFEST_INFO, ByteBuffer.wrap(new byte[] {1, 2, 3}), - ImmutableList.of(100L, 200L)); + ImmutableList.of(100L, 200L), + ImmutableList.of(COLUMN_FILE_1, COLUMN_FILE_2)); assertThat(file.get(pos("tracking"), Tracking.class)).isSameAs(TRACKING); assertThat(file.get(pos("content_type"), Integer.class)).isEqualTo(FileContent.DATA.id()); @@ -187,7 +197,8 @@ void copy() { DELETION_VECTOR, MANIFEST_INFO, ByteBuffer.wrap(new byte[] {1, 2, 3}), - ImmutableList.of(100L, 200L)); + ImmutableList.of(100L, 200L), + ImmutableList.of(COLUMN_FILE_1, COLUMN_FILE_2)); TrackedFile copy = file.copy(); @@ -206,6 +217,7 @@ void copy() { assertThat(copy.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); assertThat(copy.splitOffsets()).containsExactly(100L, 200L); assertThat(copy.partition()).isNotSameAs(PARTITION); + assertThat(copy.columnFiles()).containsExactly(COLUMN_FILE_1_COPY, COLUMN_FILE_2_COPY); // mutable fields are deep-copied, not shared with the original assertThat(copy.keyMetadata()).isNotSameAs(file.keyMetadata()); @@ -232,7 +244,8 @@ void copyWithStats() { DELETION_VECTOR, MANIFEST_INFO, ByteBuffer.wrap(new byte[] {1, 2, 3}), - ImmutableList.of(100L, 200L)); + ImmutableList.of(100L, 200L), + ImmutableList.of(COLUMN_FILE_1, COLUMN_FILE_2)); TrackedFile copy = file.copyWithStats(ImmutableSet.of(1)); @@ -251,6 +264,7 @@ void copyWithStats() { assertThat(copy.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); assertThat(copy.splitOffsets()).containsExactly(100L, 200L); assertThat(copy.partition()).isNotSameAs(PARTITION); + assertThat(copy.columnFiles()).containsExactly(COLUMN_FILE_1_COPY, COLUMN_FILE_2_COPY); // mutable fields are deep-copied, not shared with the original assertThat(copy.keyMetadata()).isNotSameAs(file.keyMetadata()); @@ -275,7 +289,8 @@ void copyWithoutStats() { DELETION_VECTOR, MANIFEST_INFO, ByteBuffer.wrap(new byte[] {1, 2, 3}), - ImmutableList.of(100L, 200L)); + ImmutableList.of(100L, 200L), + ImmutableList.of(COLUMN_FILE_1, COLUMN_FILE_2)); TrackedFile copy = file.copyWithoutStats(); @@ -297,9 +312,11 @@ void copyWithoutStats() { assertThat(copy.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); assertThat(copy.splitOffsets()).containsExactly(100L, 200L); assertThat(copy.partition()).isNotSameAs(PARTITION); + assertThat(copy.columnFiles()).containsExactly(COLUMN_FILE_1_COPY, COLUMN_FILE_2_COPY); // mutable fields are deep-copied, not shared with the original assertThat(copy.keyMetadata()).isNotSameAs(file.keyMetadata()); + assertThat(copy.columnFiles()).isNotSameAs(file.columnFiles()); } @Test @@ -421,7 +438,8 @@ void serializationRoundTrip(RoundTripSerializer serializer) t null, // DeletionVector has its own serialization tests null, // ManifestInfo has its own serialization tests ByteBuffer.wrap(new byte[] {1, 2, 3}), - ImmutableList.of(50L)); + ImmutableList.of(50L), + null); // ColumnFile has its own serialization tests TrackedFileStruct deserialized = serializer.apply(file); @@ -440,6 +458,7 @@ void serializationRoundTrip(RoundTripSerializer serializer) t assertThat(deserialized.manifestInfo()).isNull(); assertThat(deserialized.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); assertThat(deserialized.splitOffsets()).containsExactly(50L); + assertThat(deserialized.columnFiles()).isNull(); } @ParameterizedTest @@ -494,7 +513,8 @@ private static TrackedFileStruct trackedFile(Integer specId, PartitionData parti null, // deletionVector null, // manifestInfo null, // keyMetadata - null); // splitOffsets + null, // splitOffsets + null); // column files } private static int pos(String fieldName) { diff --git a/core/src/test/java/org/apache/iceberg/TestTrackingBuilder.java b/core/src/test/java/org/apache/iceberg/TestTrackingBuilder.java index 6fac40ecc91e..0813e5561ae6 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackingBuilder.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackingBuilder.java @@ -280,4 +280,31 @@ void manifestDVPositionsProduceModified() { assertThat(modified.modifiedSnapshotId()).isEqualTo(999L); assertThat(modified.deletedPositions()).isEqualTo(deletedBytes); } + + @Test + void manifestPositionsWithColumnFilesUpdated() { + ByteBuffer deletedBytes = ByteBuffer.wrap(new byte[] {1}); + Tracking withDeletedPositions = + TrackingBuilder.from(SOURCE_TRACKING_ADDED, 999L) + .columnFilesUpdated() + .deletedPositions(deletedBytes) + .build(); + + assertThat(withDeletedPositions.status()).isEqualTo(EntryStatus.MODIFIED); + assertThat(withDeletedPositions.modifiedSnapshotId()).isEqualTo(999L); + assertThat(withDeletedPositions.deletedPositions()).isEqualTo(deletedBytes); + assertThat(withDeletedPositions.dataSequenceNumber()).isNull(); + + ByteBuffer replacedBytes = ByteBuffer.wrap(new byte[] {2}); + Tracking withReplacedPositions = + TrackingBuilder.from(SOURCE_TRACKING_ADDED, 999L) + .columnFilesUpdated() + .replacedPositions(replacedBytes) + .build(); + + assertThat(withReplacedPositions.status()).isEqualTo(EntryStatus.MODIFIED); + assertThat(withReplacedPositions.modifiedSnapshotId()).isEqualTo(999L); + assertThat(withReplacedPositions.replacedPositions()).isEqualTo(replacedBytes); + assertThat(withReplacedPositions.dataSequenceNumber()).isNull(); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java index 3975706e5ca8..eea6e4e0a78d 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackingStruct.java @@ -189,9 +189,19 @@ void inheritanceAddedEntriesInheritSequenceNumber() { assertThat(tracking.fileSequenceNumber()).isEqualTo(60L); } + @Test + void inheritanceModifiedEntriesInheritDataSequenceNumber() { + TrackingStruct tracking = + new TrackingStruct(EntryStatus.MODIFIED, 42L, null, null, null, null, null, null); + + tracking.inherit(100L, 60L); + + assertThat(tracking.dataSequenceNumber()).isEqualTo(60L); + assertThat(tracking.fileSequenceNumber()).isNull(); + } + private static final List NON_INHERITING_STATUSES = - List.of( - EntryStatus.EXISTING, EntryStatus.MODIFIED, EntryStatus.DELETED, EntryStatus.REPLACED); + List.of(EntryStatus.EXISTING, EntryStatus.DELETED, EntryStatus.REPLACED); @ParameterizedTest @FieldSource("NON_INHERITING_STATUSES") diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 39ec97c195a5..df05f48c6ba7 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -172,7 +172,8 @@ public void readDataFile(FileFormat format) throws IOException { DV, null, // manifest info ByteBuffer.wrap(new byte[] {1, 2, 3}), // key metadata - ImmutableList.of(50L, 100L)); + ImmutableList.of(50L, 100L), + null); // column files ManifestFile manifest = writeManifest(format, ID_PARTITIONED_TYPE, file); @@ -202,7 +203,8 @@ public void readForScanPlanningDoesNotCopyStats(FileFormat format) throws IOExce DV, null, // manifest info ByteBuffer.wrap(new byte[] {1, 2, 3}), // key metadata - ImmutableList.of(50L, 100L)); + ImmutableList.of(50L, 100L), + null); // column files ManifestFile manifest = writeManifest(format, ID_PARTITIONED_TYPE, file); @@ -234,7 +236,8 @@ public void readForScanPlanningCopiesRequestedStats(FileFormat format) throws IO DV, null, // manifest info ByteBuffer.wrap(new byte[] {1, 2, 3}), // key metadata - ImmutableList.of(50L, 100L)); + ImmutableList.of(50L, 100L), + null); // column files ManifestFile manifest = writeManifest(format, ID_PARTITIONED_TYPE, file); @@ -268,7 +271,8 @@ public void readManifestFile(FileFormat format) throws IOException { null, // dv MANIFEST_INFO, ByteBuffer.wrap(new byte[] {1, 2, 3}), // key metadata - null); // split offsets + null, // split offsets + null); // column files ManifestFile manifest = writeManifest(format, ID_PARTITIONED_TYPE, manifestRef); @@ -1127,7 +1131,8 @@ public void statsFilterRecordCountFiltering(FileFormat format) throws IOExceptio null, null, null, - List.of(4L)); + List.of(4L), + null); // column files ManifestFile manifest = writeManifest(format, UNPARTITIONED_TYPE, ImmutableList.of(emptyTrackedFile, FILE_D)); @@ -1162,7 +1167,8 @@ public void statsFilterInvalidRecordCountNotFiltered(FileFormat format) throws I null, null, null, - List.of(4L)); + List.of(4L), + null); // column files ManifestFile manifest = writeManifest(format, UNPARTITIONED_TYPE, ImmutableList.of(invalidRecordCountFile, FILE_D)); @@ -1345,7 +1351,8 @@ public void partitionFilterBucketPartitionID(FileFormat format) throws IOExcepti null, // dv null, // manifest info null, // key metadata - List.of(4L)); + List.of(4L), + null); // column files ManifestFile manifest = writeManifest( @@ -1774,7 +1781,7 @@ private static TrackedFile unpartitionedFileWithoutStats(String location) { SNAPSHOT_ID, 3L, // data sequence number 3L, // file sequence number - null, // dv snapshot id + null, // modified snapshot id null, // first row id null, // deleted positions null); // replaced positions diff --git a/core/src/test/java/org/apache/iceberg/V4TestHelpers.java b/core/src/test/java/org/apache/iceberg/V4TestHelpers.java index 2e803be312d2..23106d38c710 100644 --- a/core/src/test/java/org/apache/iceberg/V4TestHelpers.java +++ b/core/src/test/java/org/apache/iceberg/V4TestHelpers.java @@ -108,7 +108,8 @@ private static TrackedFile trackedFile( dv, manifestInfo, null, // keyMetadata - null); // splitOffsets + null, // splitOffsets + null); // columnFiles } static DeletionVector deletionVector(String location) {