Implement FlatMapValueWriterFactory

presto-orc/src/main/java/com/facebook/presto/orc/OrcWriter.java

@@ -480,7 +480,7 @@ private List<DataOutput> bufferStripeData(long stripeStartOffset, FlushReason fl
         long offset = 0;
         int previousEncryptionGroup = -1;
         for (ColumnWriter columnWriter : columnWriters) {
-            for (StreamDataOutput indexStream : columnWriter.getIndexStreams()) {
+            for (StreamDataOutput indexStream : columnWriter.getIndexStreams(Optional.empty())) {
                 // The ordering is critical because the stream only contain a length with no offset.
                 // if the previous stream was part of a different encryption group, need to specify an offset so we know the column order
                 outputData.add(indexStream);

presto-orc/src/main/java/com/facebook/presto/orc/writer/BooleanColumnWriter.java

@@ -149,8 +149,7 @@ public Map<Integer, ColumnStatistics> getColumnStripeStatistics()
         return ImmutableMap.of(column, ColumnStatistics.mergeColumnStatistics(rowGroupColumnStatistics));
     }
 
-    @Override
-    public List<StreamDataOutput> getIndexStreams()
+    public List<StreamDataOutput> getIndexStreams(Optional<List<BooleanStreamCheckpoint>> dwrfFlatMapStreamCheckpoints)
             throws IOException
     {
         checkState(closed);
@@ -164,7 +163,8 @@ public List<StreamDataOutput> getIndexStreams()
             ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId);
             BooleanStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId);
             Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId));
-            List<Integer> positions = createBooleanColumnPositionList(compressed, dataCheckpoint, presentCheckpoint);
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint = dwrfFlatMapStreamCheckpoints.map(checkpoints -> checkpoints.get(groupId));
+            List<Integer> positions = createBooleanColumnPositionList(compressed, dataCheckpoint, presentCheckpoint, inMapCheckpoint);
             rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics));
         }
 
@@ -176,9 +176,11 @@ public List<StreamDataOutput> getIndexStreams()
     private static List<Integer> createBooleanColumnPositionList(
             boolean compressed,
             BooleanStreamCheckpoint dataCheckpoint,
-            Optional<BooleanStreamCheckpoint> presentCheckpoint)
+            Optional<BooleanStreamCheckpoint> presentCheckpoint,
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint)
     {
         ImmutableList.Builder<Integer> positionList = ImmutableList.builder();
+        inMapCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         presentCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         positionList.addAll(dataCheckpoint.toPositionList(compressed));
         return positionList.build();

presto-orc/src/main/java/com/facebook/presto/orc/writer/ByteColumnWriter.java

@@ -143,7 +143,7 @@ public Map<Integer, ColumnStatistics> getColumnStripeStatistics()
     }
 
     @Override
-    public List<StreamDataOutput> getIndexStreams()
+    public List<StreamDataOutput> getIndexStreams(Optional<List<BooleanStreamCheckpoint>> dwrfFlatMapStreamCheckpoints)
             throws IOException
     {
         checkState(closed);
@@ -157,7 +157,8 @@ public List<StreamDataOutput> getIndexStreams()
             ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId);
             ByteStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId);
             Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId));
-            List<Integer> positions = createByteColumnPositionList(compressed, dataCheckpoint, presentCheckpoint);
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint = dwrfFlatMapStreamCheckpoints.map(checkpoints -> checkpoints.get(groupId));
+            List<Integer> positions = createByteColumnPositionList(compressed, dataCheckpoint, presentCheckpoint, inMapCheckpoint);
             rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics));
         }
 
@@ -169,9 +170,11 @@ public List<StreamDataOutput> getIndexStreams()
     private static List<Integer> createByteColumnPositionList(
             boolean compressed,
             ByteStreamCheckpoint dataCheckpoint,
-            Optional<BooleanStreamCheckpoint> presentCheckpoint)
+            Optional<BooleanStreamCheckpoint> presentCheckpoint,
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint)
     {
         ImmutableList.Builder<Integer> positionList = ImmutableList.builder();
+        inMapCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         presentCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         positionList.addAll(dataCheckpoint.toPositionList(compressed));
         return positionList.build();

presto-orc/src/main/java/com/facebook/presto/orc/writer/ColumnWriter.java

@@ -14,6 +14,7 @@
 package com.facebook.presto.orc.writer;
 
 import com.facebook.presto.common.block.Block;
+import com.facebook.presto.orc.checkpoint.BooleanStreamCheckpoint;
 import com.facebook.presto.orc.metadata.ColumnEncoding;
 import com.facebook.presto.orc.metadata.statistics.ColumnStatistics;
 import com.facebook.presto.orc.stream.StreamDataOutput;
@@ -22,6 +23,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 public interface ColumnWriter
 {
@@ -46,8 +48,12 @@ default List<ColumnWriter> getNestedColumnWriters()
      * Write index streams to the output and return the streams in the
      * order in which they were written.  The ordering is critical because
      * the stream only contain a length with no offset.
+     *
+     * @param dwrfFlatMapStreamCheckpoints optional parameter when ColumnWriter is used inside a FlatMap
+     *        FlatMap encodes the values in the map as a sequence of ColumnWriters
+     *        Each column writer will need the InMap stream checkpoints to generate index streams.
      */
-    List<StreamDataOutput> getIndexStreams()
+    List<StreamDataOutput> getIndexStreams(Optional<List<BooleanStreamCheckpoint>> dwrfFlatMapStreamCheckpoints)
             throws IOException;
 
     /**

presto-orc/src/main/java/com/facebook/presto/orc/writer/DecimalColumnWriter.java

@@ -190,7 +190,7 @@ public Map<Integer, ColumnStatistics> getColumnStripeStatistics()
     }
 
     @Override
-    public List<StreamDataOutput> getIndexStreams()
+    public List<StreamDataOutput> getIndexStreams(Optional<List<BooleanStreamCheckpoint>> dwrfFlatMapStreamCheckpoints)
             throws IOException
     {
         checkState(closed);
@@ -205,8 +205,9 @@ public List<StreamDataOutput> getIndexStreams()
             ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId);
             DecimalStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId);
             LongStreamCheckpoint scaleCheckpoint = scaleCheckpoints.get(groupId);
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint = dwrfFlatMapStreamCheckpoints.map(checkpoints -> checkpoints.get(groupId));
             Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId));
-            List<Integer> positions = createDecimalColumnPositionList(compressed, dataCheckpoint, scaleCheckpoint, presentCheckpoint);
+            List<Integer> positions = createDecimalColumnPositionList(compressed, dataCheckpoint, scaleCheckpoint, presentCheckpoint, inMapCheckpoint);
             rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics));
         }
 
@@ -219,9 +220,11 @@ private static List<Integer> createDecimalColumnPositionList(
             boolean compressed,
             DecimalStreamCheckpoint dataCheckpoint,
             LongStreamCheckpoint scaleCheckpoint,
-            Optional<BooleanStreamCheckpoint> presentCheckpoint)
+            Optional<BooleanStreamCheckpoint> presentCheckpoint,
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint)
     {
         ImmutableList.Builder<Integer> positionList = ImmutableList.builder();
+        inMapCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         presentCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         positionList.addAll(dataCheckpoint.toPositionList(compressed));
         positionList.addAll(scaleCheckpoint.toPositionList(compressed));

presto-orc/src/main/java/com/facebook/presto/orc/writer/DictionaryColumnWriter.java

@@ -347,13 +347,13 @@ private void bufferOutputData()
     }
 
     @Override
-    public List<StreamDataOutput> getIndexStreams()
+    public List<StreamDataOutput> getIndexStreams(Optional<List<BooleanStreamCheckpoint>> dwrfFlatMapStreamCheckpoints)
             throws IOException
     {
         checkState(closed);
 
         if (directEncoded) {
-            return getDirectColumnWriter().getIndexStreams();
+            return getDirectColumnWriter().getIndexStreams(dwrfFlatMapStreamCheckpoints);
         }
 
         ImmutableList.Builder<RowGroupIndex> rowGroupIndexes = ImmutableList.builder();
@@ -364,7 +364,8 @@ public List<StreamDataOutput> getIndexStreams()
             ColumnStatistics columnStatistics = rowGroups.get(groupId).getColumnStatistics();
             LongStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId);
             Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId));
-            List<Integer> positions = createSliceColumnPositionList(columnWriterOptions.getCompressionKind() != NONE, dataCheckpoint, presentCheckpoint);
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint = dwrfFlatMapStreamCheckpoints.map(checkpoints -> checkpoints.get(groupId));
+            List<Integer> positions = createSliceColumnPositionList(columnWriterOptions.getCompressionKind() != NONE, dataCheckpoint, presentCheckpoint, inMapCheckpoint);
             rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics));
         }
 
@@ -376,9 +377,11 @@ public List<StreamDataOutput> getIndexStreams()
     private static List<Integer> createSliceColumnPositionList(
             boolean compressed,
             LongStreamCheckpoint dataCheckpoint,
-            Optional<BooleanStreamCheckpoint> presentCheckpoint)
+            Optional<BooleanStreamCheckpoint> presentCheckpoint,
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint)
     {
         ImmutableList.Builder<Integer> positionList = ImmutableList.builder();
+        inMapCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         presentCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         positionList.addAll(dataCheckpoint.toPositionList(compressed));
         return positionList.build();

presto-orc/src/main/java/com/facebook/presto/orc/writer/DoubleColumnWriter.java

@@ -145,7 +145,7 @@ public Map<Integer, ColumnStatistics> getColumnStripeStatistics()
     }
 
     @Override
-    public List<StreamDataOutput> getIndexStreams()
+    public List<StreamDataOutput> getIndexStreams(Optional<List<BooleanStreamCheckpoint>> dwrfFlatMapStreamCheckpoints)
             throws IOException
     {
         checkState(closed);
@@ -159,7 +159,8 @@ public List<StreamDataOutput> getIndexStreams()
             ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId);
             DoubleStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId);
             Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId));
-            List<Integer> positions = createDoubleColumnPositionList(compressed, dataCheckpoint, presentCheckpoint);
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint = dwrfFlatMapStreamCheckpoints.map(checkpoints -> checkpoints.get(groupId));
+            List<Integer> positions = createDoubleColumnPositionList(compressed, dataCheckpoint, presentCheckpoint, inMapCheckpoint);
             rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics));
         }
 
@@ -171,9 +172,11 @@ public List<StreamDataOutput> getIndexStreams()
     private static List<Integer> createDoubleColumnPositionList(
             boolean compressed,
             DoubleStreamCheckpoint dataCheckpoint,
-            Optional<BooleanStreamCheckpoint> presentCheckpoint)
+            Optional<BooleanStreamCheckpoint> presentCheckpoint,
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint)
     {
         ImmutableList.Builder<Integer> positionList = ImmutableList.builder();
+        inMapCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         presentCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         positionList.addAll(dataCheckpoint.toPositionList(compressed));
         return positionList.build();

presto-orc/src/main/java/com/facebook/presto/orc/writer/FlatMapValueWriterFactory.java

@@ -0,0 +1,90 @@
+
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.orc.writer;
+
+import com.facebook.presto.common.type.Type;
+import com.facebook.presto.orc.ColumnWriterOptions;
+import com.facebook.presto.orc.DwrfEncryptionInfo;
+import com.facebook.presto.orc.OrcEncoding;
+import com.facebook.presto.orc.metadata.MetadataWriter;
+import com.facebook.presto.orc.metadata.OrcType;
+import io.airlift.units.DataSize;
+import org.joda.time.DateTimeZone;
+
+import java.util.List;
+
+import static com.facebook.presto.orc.writer.ColumnWriters.createColumnWriter;
+import static com.google.common.base.Preconditions.checkState;
+import static java.util.Objects.requireNonNull;
+
+public class FlatMapValueWriterFactory
+{
+    private final int columnIndex;
+    private final List<OrcType> orcTypes;
+    private final Type type;
+    private final ColumnWriterOptions columnWriterOptions;
+    private final OrcEncoding orcEncoding;
+    private final DateTimeZone hiveStorageTimeZone;
+    private final DwrfEncryptionInfo dwrfEncryptors;
+    private final MetadataWriter metadataWriter;
+
+    public FlatMapValueWriterFactory(
+            final int columnIndex,
+            final List<OrcType> orcTypes,
+            final Type type,
+            final ColumnWriterOptions columnWriterOptions,
+            final OrcEncoding orcEncoding,
+            final DateTimeZone hiveStorageTimeZone,
+            final DwrfEncryptionInfo dwrfEncryptors,
+            final MetadataWriter metadataWriter)
+    {
+        checkState(columnIndex >= 0, "column index is negative");
+        requireNonNull(orcTypes, "orcTypes is null");
+        requireNonNull(type, "type is null");
+        requireNonNull(columnWriterOptions, "columnWriterOptions is null");
+        requireNonNull(metadataWriter, "metadataWriter is null");
+
+        this.columnIndex = columnIndex;
+        this.orcTypes = orcTypes;
+        this.type = type;
+        // disable dictionary encoding
+        this.columnWriterOptions =
+                new ColumnWriterOptions(columnWriterOptions.getCompressionKind(),
+                        columnWriterOptions.getCompressionLevel(),
+                        new DataSize(columnWriterOptions.getCompressionMaxBufferSize(), DataSize.Unit.BYTE),
+                        columnWriterOptions.getStringStatisticsLimit(),
+                        false);
+        this.orcEncoding = orcEncoding;
+        this.hiveStorageTimeZone = hiveStorageTimeZone;
+        this.dwrfEncryptors = dwrfEncryptors;
+        this.metadataWriter = metadataWriter;
+    }
+
+    ColumnWriter getColumnWriter(int dwrfSequence)
+    {
+        checkState(dwrfSequence >= 1, "sequence should be positive");
+        ColumnWriter columnWriter = createColumnWriter(
+                columnIndex,
+                dwrfSequence,
+                orcTypes,
+                type,
+                columnWriterOptions,
+                orcEncoding,
+                hiveStorageTimeZone,
+                dwrfEncryptors,
+                metadataWriter);
+        return columnWriter;
+    }
+}

presto-orc/src/main/java/com/facebook/presto/orc/writer/FloatColumnWriter.java

@@ -147,7 +147,7 @@ public Map<Integer, ColumnStatistics> getColumnStripeStatistics()
     }
 
     @Override
-    public List<StreamDataOutput> getIndexStreams()
+    public List<StreamDataOutput> getIndexStreams(Optional<List<BooleanStreamCheckpoint>> dwrfFlatMapStreamCheckpoints)
             throws IOException
     {
         checkState(closed);
@@ -161,7 +161,8 @@ public List<StreamDataOutput> getIndexStreams()
             ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId);
             FloatStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId);
             Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId));
-            List<Integer> positions = createFloatColumnPositionList(compressed, dataCheckpoint, presentCheckpoint);
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint = dwrfFlatMapStreamCheckpoints.map(checkpoints -> checkpoints.get(groupId));
+            List<Integer> positions = createFloatColumnPositionList(compressed, dataCheckpoint, presentCheckpoint, inMapCheckpoint);
             rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics));
         }
 
@@ -173,9 +174,11 @@ public List<StreamDataOutput> getIndexStreams()
     private static List<Integer> createFloatColumnPositionList(
             boolean compressed,
             FloatStreamCheckpoint dataCheckpoint,
-            Optional<BooleanStreamCheckpoint> presentCheckpoint)
+            Optional<BooleanStreamCheckpoint> presentCheckpoint,
+            Optional<BooleanStreamCheckpoint> inMapCheckpoint)
     {
         ImmutableList.Builder<Integer> positionList = ImmutableList.builder();
+        inMapCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         presentCheckpoint.ifPresent(booleanStreamCheckpoint -> positionList.addAll(booleanStreamCheckpoint.toPositionList(compressed)));
         positionList.addAll(dataCheckpoint.toPositionList(compressed));
         return positionList.build();