Skip to content

Commit

Permalink
Default values and benchmark enhancement
Browse files Browse the repository at this point in the history
  • Loading branch information
@ajantha-bhat
ajantha-bhat committed Aug 23, 2024
1 parent d3345b5 commit 161394f
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 76 deletions.
13 changes: 13 additions & 0 deletions core/src/main/java/org/apache/iceberg/PartitionStatsUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,27 +157,40 @@ private static Record fromManifestEntry(
record.set(Column.DATA_RECORD_COUNT.ordinal(), entry.file().recordCount());
record.set(Column.DATA_FILE_COUNT.ordinal(), 1);
record.set(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.ordinal(), entry.file().fileSizeInBytes());
// default values
record.set(Column.POSITION_DELETE_RECORD_COUNT.ordinal(), 0L);
record.set(Column.POSITION_DELETE_FILE_COUNT.ordinal(), 0);
record.set(Column.EQUALITY_DELETE_RECORD_COUNT.ordinal(), 0L);
record.set(Column.EQUALITY_DELETE_FILE_COUNT.ordinal(), 0);
break;
case POSITION_DELETES:
record.set(Column.POSITION_DELETE_RECORD_COUNT.ordinal(), entry.file().recordCount());
record.set(Column.POSITION_DELETE_FILE_COUNT.ordinal(), 1);
// default values
record.set(Column.DATA_RECORD_COUNT.ordinal(), 0L);
record.set(Column.DATA_FILE_COUNT.ordinal(), 0);
record.set(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.ordinal(), 0L);
record.set(Column.EQUALITY_DELETE_RECORD_COUNT.ordinal(), 0L);
record.set(Column.EQUALITY_DELETE_FILE_COUNT.ordinal(), 0);
break;
case EQUALITY_DELETES:
record.set(Column.EQUALITY_DELETE_RECORD_COUNT.ordinal(), entry.file().recordCount());
record.set(Column.EQUALITY_DELETE_FILE_COUNT.ordinal(), 1);
// default values
record.set(Column.DATA_RECORD_COUNT.ordinal(), 0L);
record.set(Column.DATA_FILE_COUNT.ordinal(), 0);
record.set(Column.TOTAL_DATA_FILE_SIZE_IN_BYTES.ordinal(), 0L);
record.set(Column.POSITION_DELETE_RECORD_COUNT.ordinal(), 0L);
record.set(Column.POSITION_DELETE_FILE_COUNT.ordinal(), 0);
break;
default:
throw new UnsupportedOperationException(
"Unsupported file content type: " + entry.file().content());
}

// Note: Not computing the `TOTAL_RECORD_COUNT` for now as it needs scanning the data.
record.set(Column.TOTAL_RECORD_COUNT.ordinal(), 0L);

return record;
}

Expand Down
152 changes: 76 additions & 76 deletions data/src/test/java/org/apache/iceberg/data/TestPartitionStatsGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,11 +182,11 @@ public void testPartitionStats() throws Exception {
9L,
3,
3 * dataFile1.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot1.timestampMillis(),
snapshot1.snapshotId()),
Tuple.tuple(
Expand All @@ -195,11 +195,11 @@ public void testPartitionStats() throws Exception {
3L,
3,
3 * dataFile2.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot1.timestampMillis(),
snapshot1.snapshotId()),
Tuple.tuple(
Expand All @@ -208,11 +208,11 @@ public void testPartitionStats() throws Exception {
3L,
3,
3 * dataFile3.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot1.timestampMillis(),
snapshot1.snapshotId()),
Tuple.tuple(
Expand All @@ -221,11 +221,11 @@ public void testPartitionStats() throws Exception {
6L,
3,
3 * dataFile4.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot1.timestampMillis(),
snapshot1.snapshotId()));

Expand All @@ -247,11 +247,11 @@ public void testPartitionStats() throws Exception {
9L,
3,
3 * dataFile1.fileSizeInBytes(),
null,
null,
0L,
0,
eqDeletes.recordCount(),
1,
null,
0L,
snapshot3.timestampMillis(),
snapshot3.snapshotId()),
Tuple.tuple(
Expand All @@ -260,11 +260,11 @@ public void testPartitionStats() throws Exception {
3L,
3,
3 * dataFile2.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot1.timestampMillis(),
snapshot1.snapshotId()),
Tuple.tuple(
Expand All @@ -275,9 +275,9 @@ public void testPartitionStats() throws Exception {
3 * dataFile3.fileSizeInBytes(),
posDeletes.recordCount(),
1,
null,
null,
null,
0L,
0,
0L,
snapshot2.timestampMillis(),
snapshot2.snapshotId()),
Tuple.tuple(
Expand All @@ -286,11 +286,11 @@ public void testPartitionStats() throws Exception {
6L,
3,
3 * dataFile4.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot1.timestampMillis(),
snapshot1.snapshotId()));
}
Expand Down Expand Up @@ -336,11 +336,11 @@ public void testPartitionStatsWithSchemaEvolution() throws Exception {
8L,
2,
2 * dataFile1.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot1.timestampMillis(),
snapshot1.snapshotId()),
Tuple.tuple(
Expand All @@ -349,11 +349,11 @@ public void testPartitionStatsWithSchemaEvolution() throws Exception {
6L,
2,
2 * dataFile2.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot1.timestampMillis(),
snapshot1.snapshotId()));

Expand Down Expand Up @@ -402,11 +402,11 @@ public void testPartitionStatsWithSchemaEvolution() throws Exception {
8L,
2,
2 * dataFile1.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot1.timestampMillis(),
snapshot1.snapshotId()),
Tuple.tuple(
Expand All @@ -415,11 +415,11 @@ public void testPartitionStatsWithSchemaEvolution() throws Exception {
7L,
3,
2 * dataFile2.fileSizeInBytes() + dataFile7.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot2.timestampMillis(),
snapshot2.snapshotId()),
Tuple.tuple(
Expand All @@ -428,11 +428,11 @@ public void testPartitionStatsWithSchemaEvolution() throws Exception {
3L,
1,
dataFile3.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot2.timestampMillis(),
snapshot2.snapshotId()),
Tuple.tuple(
Expand All @@ -441,11 +441,11 @@ public void testPartitionStatsWithSchemaEvolution() throws Exception {
1L,
1,
dataFile4.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot2.timestampMillis(),
snapshot2.snapshotId()),
Tuple.tuple(
Expand All @@ -454,11 +454,11 @@ public void testPartitionStatsWithSchemaEvolution() throws Exception {
1L,
1,
dataFile5.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot2.timestampMillis(),
snapshot2.snapshotId()),
Tuple.tuple(
Expand All @@ -467,11 +467,11 @@ public void testPartitionStatsWithSchemaEvolution() throws Exception {
2L,
1,
dataFile6.fileSizeInBytes(),
null,
null,
null,
null,
null,
0L,
0,
0L,
0,
0L,
snapshot2.timestampMillis(),
snapshot2.snapshotId()));
}
Expand Down

0 comments on commit 161394f

Please sign in to comment.