Merge pull request #43 from mukunku/v2.3.3

v2.3.3

Author: mukunku

README.md

@@ -13,6 +13,7 @@ If you'd like to add any new features feel free to send a pull request.
 Some Key Features:
 * Run simple sql-like queries on chunks of the file
 * Generate ansi sql schema for opened files
+* View parquet file metadata
 
 # Limitations
 This application can only open Parquet files located on the Windows machine the app is running on. It cannot connect to HDFS to read parquet data. 

src/ParquetFileViewer/Helpers/ExtensionMethods.cs

@@ -1,6 +1,9 @@
-using System.Collections.Generic;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
 using System.Data;
-using System.Windows.Forms;
+using System.IO;
 
 namespace ParquetFileViewer.Helpers
 {
@@ -20,5 +23,33 @@ public static IList<string> GetColumnNames(this DataTable datatable)
             }
             return columns;
         }
+
+        public static string FormatJSON(this string input)
+        {
+            if (input == null)
+                return null;
+
+            try
+            {
+                return JValue.Parse(input).ToString(Formatting.Indented);
+            }
+            catch (Exception)
+            {
+                //malformed json detected
+                return input;
+            }
+        }
+
+        public static string Base64Encode(this string plainText)
+        {
+            var plainTextBytes = System.Text.Encoding.UTF8.GetBytes(plainText);
+            return Convert.ToBase64String(plainTextBytes);
+        }
+
+        public static string Base64Decode(this string base64EncodedData)
+        {
+            var base64EncodedBytes = Convert.FromBase64String(base64EncodedData);
+            return System.Text.Encoding.UTF8.GetString(base64EncodedBytes);
+        }
     }
 }

src/ParquetFileViewer/Helpers/UtilityMethods.cs

@@ -108,7 +108,7 @@ private static void ProcessRowGroup(DataTable dataTable, ParquetRowGroupReader g
                     else if (field.DataType == Parquet.Data.DataType.DateTimeOffset)
                         dataTable.Rows[rowIndex][field.Name] = ((DateTimeOffset)value).DateTime; 
                     else if (field.DataType == Parquet.Data.DataType.Int64
-                        && logicalType.TIMESTAMP != null)
+                        && logicalType?.TIMESTAMP != null)
                     {
                         int divideBy = 0;
                         if (logicalType.TIMESTAMP.Unit.NANOS != null)
@@ -168,7 +168,7 @@ public static Type ParquetNetTypeToCSharpType(Parquet.Thrift.SchemaElement thrif
                     columnType = typeof(int);
                     break;
                 case Parquet.Data.DataType.Int64:
-                    columnType = thriftSchema.LogicalType.TIMESTAMP != null ? typeof(DateTime) : typeof(long);
+                    columnType = thriftSchema.LogicalType?.TIMESTAMP != null ? typeof(DateTime) : typeof(long);
                     break;
                 case Parquet.Data.DataType.UnsignedByte:
                     columnType = typeof(byte);

src/ParquetFileViewer/MainForm.Designer.cs

@@ -44,6 +44,8 @@ private string OpenFilePath
                 this.openFilePath = value;
                 this.changeFieldsMenuStripButton.Enabled = false;
                 this.getSQLCreateTableScriptToolStripMenuItem.Enabled = false;
+                this.saveAsToolStripMenuItem.Enabled = false;
+                this.metadataViewerToolStripMenuItem.Enabled = false;
                 this.recordCountStatusBarLabel.Text = "0";
                 this.totalRowCountStatusBarLabel.Text = "0";
                 this.MainDataSource.Clear();
@@ -57,7 +59,9 @@ private string OpenFilePath
                 {
                     this.Text = string.Concat("Open File: ", value);
                     this.changeFieldsMenuStripButton.Enabled = true;
+                    this.saveAsToolStripMenuItem.Enabled = true;
                     this.getSQLCreateTableScriptToolStripMenuItem.Enabled = true;
+                    this.metadataViewerToolStripMenuItem.Enabled = true;
                 }
             }
         }
@@ -953,5 +957,22 @@ private void MultithreadedParquetEngineToolStripMenuItem_Click(object sender, Ev
                 this.ShowError(ex);
             }
         }
+
+        private void MetadataViewerToolStripMenuItem_Click(object sender, EventArgs e)
+        {
+            try
+            {
+                using (Stream parquetStream = new FileStream(this.OpenFilePath, FileMode.Open, FileAccess.Read))
+                using (var parquetReader = new ParquetReader(parquetStream, new ParquetOptions() { TreatByteArrayAsString = true }))
+                using (var metadataViewer = new MetadataViewer(parquetReader))
+                {
+                    metadataViewer.ShowDialog(this);
+                }
+            }
+            catch(Exception ex)
+            {
+                this.ShowError(ex);
+            }
+        }
     }
 }

src/ParquetFileViewer/MainForm.cs

@@ -165,6 +165,21 @@
         r00Cgk/n38S7wBwC04u4UGxHrMHF4CbEJtyDLj5fCDIzhljfSxzeavRgyw4Zj9t64GvvQ0d3P3pfD2Kv
         2QqNvgFxDN6urYdWmyMElJMnevh60obRktA701PRtGlg1DOdSkXwzrisaMG/RZLWAE60OMW5fNhvAAAA
         AElFTkSuQmCC
+</value>
+  </data>
+  <data name="saveAsToolStripMenuItem.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+    <value>
+        iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
+        wwAADsMBx2+oZAAAAjJJREFUOE+tkt9Lk1EYx99/oftugi66rakxnIvV4C3XUJYNw4qwH2+kWZou00TF
+        yjRq/oJshUsnrVaTcFKRP8AUKRFlKlmauaIlDjXFJoLB1/OcvXlcRRfRF56bl/fzOed5ziP9lzx/FgaV
+        r3UJLd5FPHTPo6kxhLuOadTVfoH9VgDXyydRWjKOwoK3yMsdwflzfqi4JBHY1QnU1qzgb1leWYUptQuH
+        Up/yUnFJ8j5Z4ILKim/qr7+H4ND8MnZZXkCjUWA96BEC94M5Ligt+ar+Hp2f8MfgAmLMPhRcmsIBS7MQ
+        OBtmuOCi7YOKiGyEhydC2CZ7kZM9huQkpxDU3w5yQdbZERWL5Fe4zx/EZoMbijIAs9khBDXVn7lAOTWg
+        on+GO/s/YVN8Iw4feYXExDohuFE5hY52sKcEjh1/A+vRXj5tGhj1TNemkyNwN1KsLyHLdiG4UjaBNh/Y
+        DgDNrh/s3b+j/NosHyoNjHrOzPCvw/uTW2E0VgjB5cIxeB4B952rbHHCMKVlR5Ul3Ya0M0UROInB+zww
+        GMqEwJY3ymCguirMTyaIfUaMTsbg+xmMTs5Cyb2KPaYWDut2u6DXFwkBraXjDtavTQKCT+ffxLvAHALT
+        i7hQbEeswcXgJsQm3IMuPl8IMjOGWN9LHN5q9GDLDhmP23rga+9DR3c/el8PYq/ZCo2+AXEM3q6th1ab
+        IwSUkyd6+HrShtGS0DvTU9G0aWDUM51KRfDOuKxowb9FktYATrQ4xbl82G8AAAAASUVORK5CYII=
 </value>
   </data>
   <metadata name="FileSchemaBackgroundWorker.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">