Skip to content

Commit 91eb3ee

Browse files
author
Roberto Congiu
committed
Merge branch 'release/1.3'
2 parents 34cb964 + ed28e4a commit 91eb3ee

File tree

17 files changed

+335
-32
lines changed

17 files changed

+335
-32
lines changed

‎.gitignore

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
1-
**/target/**
2-
*.class
3-
4-
# Package Files #
5-
*.jar
6-
*.war
7-
*.ear
1+
target/
2+
nb-configuration.xml
3+
.idea/
4+
*.iml

‎README.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,10 @@ To build for CDH5:
3131
mvn -Pcdh5 clean package
3232
```
3333

34-
34+
the serde will be in
35+
```
36+
json-serde/target/json-serde-VERSION-jar-with-dependencies.jar
37+
```
3538

3639

3740

‎json-serde-cdh4-shim/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<groupId>org.openx.data</groupId>
66
<artifactId>json-serde-parent</artifactId>
77
<relativePath>../pom.xml</relativePath>
8-
<version>1.2</version>
8+
<version>1.3</version>
99
</parent>
1010
<modelVersion>4.0.0</modelVersion>
1111

‎json-serde-cdh5-shim/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<groupId>org.openx.data</groupId>
66
<artifactId>json-serde-parent</artifactId>
77
<relativePath>../pom.xml</relativePath>
8-
<version>1.2</version>
8+
<version>1.3</version>
99
</parent>
1010
<modelVersion>4.0.0</modelVersion>
1111

‎json-serde/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<groupId>org.openx.data</groupId>
66
<artifactId>json-serde-parent</artifactId>
77
<relativePath>../pom.xml</relativePath>
8-
<version>1.2</version>
8+
<version>1.3</version>
99
</parent>
1010
<modelVersion>4.0.0</modelVersion>
1111

‎json-serde/src/main/java/org/openx/data/jsonserde/JsonSerDe.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -425,7 +425,7 @@ private Map<String, String> getMappings(Properties tbl) {
425425
String s = (String) o;
426426

427427
if(s.startsWith(PFX) ) {
428-
mps.put(s.substring(n), tbl.getProperty(s));
428+
mps.put(s.substring(n), tbl.getProperty(s).toLowerCase());
429429
}
430430
}
431431
return mps;

‎json-serde/src/main/java/org/openx/data/jsonserde/objectinspector/JsonMapObjectInspector.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import java.util.Map;
1616
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
1717
import org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector;
18+
import org.apache.hadoop.io.Text;
1819
import org.openx.data.jsonserde.json.JSONException;
1920
import org.openx.data.jsonserde.json.JSONObject;
2021

@@ -32,7 +33,7 @@ public JsonMapObjectInspector(ObjectInspector mapKeyObjectInspector,
3233

3334
@Override
3435
public Map<?, ?> getMap(Object data) {
35-
if (data == null) {
36+
if (JsonObjectInspectorUtils.checkObject(data) == null) {
3637
return null;
3738
}
3839

@@ -41,9 +42,11 @@ public JsonMapObjectInspector(ObjectInspector mapKeyObjectInspector,
4142
return new JSONObjectMapAdapter(jObj);
4243
}
4344

45+
46+
4447
@Override
4548
public int getMapSize(Object data) {
46-
if (data == null) {
49+
if (JsonObjectInspectorUtils.checkObject(data) == null) {
4750
return -1;
4851
}
4952
JSONObject jObj = (JSONObject) data;
@@ -52,7 +55,7 @@ public int getMapSize(Object data) {
5255

5356
@Override
5457
public Object getMapValueElement(Object data, Object key) {
55-
if (data == null) {
58+
if (JsonObjectInspectorUtils.checkObject(data) == null) {
5659
return -1;
5760
}
5861

‎json-serde/src/main/java/org/openx/data/jsonserde/objectinspector/JsonObjectInspectorUtils.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright 2014 rcongiu.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.openx.data.jsonserde.objectinspector;
18+
19+
import org.apache.hadoop.io.Text;
20+
21+
/**
22+
*
23+
* @author rcongiu
24+
*/
25+
public class JsonObjectInspectorUtils {
26+
public static Object checkObject(Object data) {
27+
// just check for null first thing
28+
if(data == null) return data;
29+
30+
if(data instanceof String ||
31+
data instanceof Text) {
32+
String str = (data instanceof String ? (String)data : ((Text)data).toString() );
33+
if(str.trim().isEmpty())
34+
return null;
35+
}
36+
return data;
37+
}
38+
39+
}

‎json-serde/src/main/java/org/openx/data/jsonserde/objectinspector/JsonStructObjectInspector.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public JsonStructObjectInspector(List<String> structFieldNames,
5151
*/
5252
@Override
5353
public Object getStructFieldData(Object data, StructField fieldRef) {
54-
if (data == null) {
54+
if (JsonObjectInspectorUtils.checkObject(data) == null) {
5555
return null;
5656
}
5757

@@ -86,7 +86,7 @@ public Object getStructFieldDataFromList(List data, StructField fieldRef ) {
8686

8787

8888
public Object getStructFieldDataFromJsonObject(JSONObject data, StructField fieldRef ) {
89-
if (data == null) {
89+
if (JsonObjectInspectorUtils.checkObject(data) == null) {
9090
return null;
9191
}
9292

@@ -127,7 +127,9 @@ protected String getJsonField(StructField fr) {
127127
List<Object> values = new ArrayList<Object>();
128128
@Override
129129
public List<Object> getStructFieldsDataAsList(Object o) {
130-
if(o == null) return null;
130+
if (JsonObjectInspectorUtils.checkObject(o) == null) {
131+
return null;
132+
}
131133
JSONObject jObj = (JSONObject) o;
132134
values.clear();
133135

‎json-serde/src/main/java/org/openx/data/jsonserde/objectinspector/primitive/JavaStringLongObjectInspector.java

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,24 +31,30 @@ public JavaStringLongObjectInspector() {
3131
@Override
3232
public Object getPrimitiveWritableObject(Object o) {
3333
if(o == null) return null;
34-
34+
3535
if(o instanceof String) {
36-
return new LongWritable(ParsePrimitiveUtils.parseLong((String)o));
36+
return new LongWritable(ParsePrimitiveUtils.parseLong((String)o));
3737
} else {
3838
return new LongWritable(((Long) o).longValue());
3939
}
4040
}
4141

4242
@Override
4343
public long get(Object o) {
44-
44+
4545
if(o instanceof String) {
46-
return ParsePrimitiveUtils.parseLong((String)o);
46+
return ParsePrimitiveUtils.parseLong((String)o);
4747
} else {
4848
return ((Long) o);
4949
}
5050
}
5151

52+
@Override
53+
public Object getPrimitiveJavaObject(Object o)
54+
{
55+
return get(o);
56+
}
57+
5258
@Override
5359
public Object create(long value) {
5460
return value;

‎json-serde/src/test/java/org/openx/data/jsonserde/JsonMapTest.java

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*======================================================================*
2+
* Copyright (c) 2011, OpenX Technologies, Inc. All rights reserved. *
3+
* *
4+
* Licensed under the New BSD License (the "License"); you may not use *
5+
* this file except in compliance with the License. Unless required *
6+
* by applicable law or agreed to in writing, software distributed *
7+
* under the License is distributed on an "AS IS" BASIS, WITHOUT *
8+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
9+
* See the License for the specific language governing permissions and *
10+
* limitations under the License. See accompanying LICENSE file. *
11+
*======================================================================*/
12+
13+
14+
package org.openx.data.jsonserde;
15+
16+
import java.util.Properties;
17+
import org.apache.hadoop.conf.Configuration;
18+
import org.apache.hadoop.hive.serde.Constants;
19+
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
20+
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
21+
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
22+
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
23+
import org.apache.hadoop.io.Text;
24+
import org.apache.hadoop.io.Writable;
25+
26+
import static org.junit.Assert.*;
27+
import org.junit.Before;
28+
import org.junit.Test;
29+
import static org.openx.data.jsonserde.NestedStructureTest.instance;
30+
import org.openx.data.jsonserde.json.JSONObject;
31+
32+
/**
33+
*
34+
* @author rcongiu
35+
*/
36+
public class JsonMapTest {
37+
static JsonSerDe instance;
38+
39+
@Before
40+
public void setUp() throws Exception {
41+
initialize();
42+
}
43+
44+
static public void initialize() throws Exception {
45+
instance = new JsonSerDe();
46+
Configuration conf = null;
47+
Properties tbl = new Properties();
48+
// from google video API
49+
tbl.setProperty(Constants.LIST_COLUMNS, "country,languages,religions");
50+
tbl.setProperty(Constants.LIST_COLUMN_TYPES, "string,string,map<string,string>".toLowerCase());
51+
52+
instance.initialize(conf, tbl);
53+
}
54+
55+
@Test
56+
public void testDeSerialize() throws Exception {
57+
// Test that timestamp object can be deserialized
58+
Writable w = new Text("{\"country\":\"Switzerland\",\"languages\":\"Italian\",\"religions\":\"\"}");
59+
60+
JSONObject result = (JSONObject) instance.deserialize(w);
61+
62+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
63+
64+
StructField sfr = soi.getStructFieldRef("religions");
65+
66+
assertEquals(sfr.getFieldObjectInspector().getCategory(),ObjectInspector.Category.MAP);
67+
68+
MapObjectInspector moi = (MapObjectInspector) sfr.getFieldObjectInspector();
69+
70+
Object val = soi.getStructFieldData(result, sfr) ;
71+
72+
assertEquals(-1, moi.getMapSize(val));
73+
74+
}
75+
}

‎json-serde/src/test/java/org/openx/data/jsonserde/NestedStructureTest.java

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,10 @@
1515
import java.util.Properties;
1616
import org.apache.hadoop.conf.Configuration;
1717
import org.apache.hadoop.hive.serde.Constants;
18-
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
1918
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
2019
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
2120
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
2221
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
23-
import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaIntObjectInspector;
24-
import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaLongObjectInspector;
2522
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector;
2623
import org.apache.hadoop.io.Text;
2724
import org.apache.hadoop.io.Writable;
@@ -31,11 +28,7 @@
3128
import org.junit.Test;
3229
import org.openx.data.jsonserde.json.JSONObject;
3330

34-
/*
35-
* To change this license header, choose License Headers in Project Properties.
36-
* To change this template file, choose Tools | Templates
37-
* and open the template in the editor.
38-
*/
31+
3932

4033
/**
4134
*

0 commit comments

Comments
 (0)