Skip to content

Commit f5d416a

Browse files
rcongiurcongiu
committed
Merge pull request #90 from wmoss/fix-issue-70
Fix for #issue-70 related to get_json_object
2 parents 8859c6d + 6770f89 commit f5d416a

File tree

3 files changed

+205
-2
lines changed

3 files changed

+205
-2
lines changed

‎json-serde/src/main/java/org/openx/data/jsonserde/objectinspector/JsonObjectInspectorFactory.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import java.util.HashMap;
1717
import java.util.List;
1818
import java.util.Map;
19+
1920
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
2021
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
2122
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
@@ -29,6 +30,7 @@
2930
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringDoubleObjectInspector;
3031
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringFloatObjectInspector;
3132
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringIntObjectInspector;
33+
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringJsonObjectInspector;
3234
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringLongObjectInspector;
3335
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringShortObjectInspector;
3436
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringTimestampObjectInspector;
@@ -171,6 +173,7 @@ public static JsonMapObjectInspector getJsonMapObjectInspector(
171173
= new EnumMap<PrimitiveCategory, AbstractPrimitiveJavaObjectInspector>(PrimitiveCategory.class);
172174

173175
static {
176+
primitiveOICache.put(PrimitiveCategory.STRING, new JavaStringJsonObjectInspector());
174177
primitiveOICache.put(PrimitiveCategory.BYTE, new JavaStringByteObjectInspector());
175178
primitiveOICache.put(PrimitiveCategory.SHORT, new JavaStringShortObjectInspector());
176179
primitiveOICache.put(PrimitiveCategory.INT, new JavaStringIntObjectInspector());
@@ -179,7 +182,7 @@ public static JsonMapObjectInspector getJsonMapObjectInspector(
179182
primitiveOICache.put(PrimitiveCategory.DOUBLE, new JavaStringDoubleObjectInspector());
180183
primitiveOICache.put(PrimitiveCategory.TIMESTAMP, new JavaStringTimestampObjectInspector());
181184
}
182-
185+
183186
/**
184187
* gets the appropriate adapter wrapper around the object inspector if
185188
* necessary, that is, if we're dealing with numbers. The JSON parser won't
@@ -190,7 +193,7 @@ public static JsonMapObjectInspector getJsonMapObjectInspector(
190193
*/
191194
public static AbstractPrimitiveJavaObjectInspector getPrimitiveJavaObjectInspector(
192195
PrimitiveCategory primitiveCategory) {
193-
196+
194197
if(! primitiveOICache.containsKey(primitiveCategory)) {
195198
primitiveOICache.put(primitiveCategory, PrimitiveObjectInspectorFactory.
196199
getPrimitiveJavaObjectInspector(primitiveCategory));

‎json-serde/src/main/java/org/openx/data/jsonserde/objectinspector/primitive/JavaStringJsonObjectInspector.java

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package org.openx.data.jsonserde.objectinspector.primitive;
2+
3+
4+
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
5+
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
6+
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector;
7+
import org.apache.hadoop.io.Text;
8+
9+
public class JavaStringJsonObjectInspector
10+
extends AbstractPrimitiveJavaObjectInspector
11+
implements SettableStringObjectInspector {
12+
13+
public JavaStringJsonObjectInspector() {
14+
super(TypeEntryShim.stringType);
15+
}
16+
17+
@Override
18+
public Text getPrimitiveWritableObject(Object o) {
19+
return o == null ? null : new Text((String) o.toString());
20+
}
21+
22+
@Override
23+
public String getPrimitiveJavaObject(Object o) {
24+
return o == null ? null : o.toString();
25+
}
26+
27+
@Override
28+
public Object create(Text value) {
29+
return value == null ? null : value.toString();
30+
}
31+
32+
@Override
33+
public Object set(Object o, Text value) {
34+
return value == null ? null : value.toString();
35+
}
36+
37+
@Override
38+
public Object create(String value) {
39+
return value;
40+
}
41+
42+
@Override
43+
public Object set(Object o, String value) {
44+
return value;
45+
}
46+
}

‎json-serde/src/test/java/org/openx/data/jsonserde/GetJsonObjectTest.java

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
package org.openx.data.jsonserde;
2+
3+
import static org.junit.Assert.*;
4+
5+
import java.util.Properties;
6+
7+
import org.apache.hadoop.conf.Configuration;
8+
import org.apache.hadoop.hive.ql.udf.UDFJson;
9+
import org.apache.hadoop.hive.serde.Constants;
10+
import org.apache.hadoop.hive.serde2.SerDeException;
11+
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
12+
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
13+
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
14+
import org.apache.hadoop.io.Text;
15+
import org.apache.hadoop.io.Writable;
16+
import org.junit.Before;
17+
import org.junit.Test;
18+
import org.openx.data.jsonserde.json.JSONException;
19+
import org.openx.data.jsonserde.json.JSONObject;
20+
21+
/**
22+
* Tests getJson Object
23+
*
24+
* @author snagmote
25+
*
26+
*/
27+
public class GetJsonObjectTest {
28+
29+
static JsonSerDe instance;
30+
31+
@Before
32+
public void setUp() throws Exception {
33+
initialize();
34+
}
35+
36+
static public void initialize() throws Exception {
37+
instance = new JsonSerDe();
38+
Configuration conf = null;
39+
Properties tbl = new Properties();
40+
// from google video API
41+
tbl.setProperty(Constants.LIST_COLUMNS, "kind,etag,pageInfo,v_items");
42+
tbl.setProperty(
43+
Constants.LIST_COLUMN_TYPES,
44+
("string,string," + "string,"
45+
+ "ARRAY<STRUCT<kind:STRING,"
46+
+ "etag:STRING,"
47+
+ "id:STRING,"
48+
+ "v_statistics:STRUCT<viewCount:INT,likeCount:INT,dislikeCount:INT,favoriteCount:INT,commentCount:INT>,"
49+
+ "topicDetails:STRUCT<topicIds:ARRAY<STRING>,relevantTopicIds:ARRAY<STRING>>"
50+
+ ">>").toLowerCase());
51+
tbl.setProperty("mapping.v_items", "items");
52+
tbl.setProperty("mapping.v_statistics", "statistics");
53+
54+
instance.initialize(conf, tbl);
55+
tbl.setProperty("mapping.v_items", "items");
56+
tbl.setProperty("mapping.v_statistics", "statistics");
57+
58+
instance.initialize(conf, tbl);
59+
}
60+
61+
@Test
62+
public void testGetJsonObject() throws SerDeException, JSONException {
63+
Writable w = new Text(
64+
"{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }");
65+
66+
JSONObject result = (JSONObject) instance.deserialize(w);
67+
68+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
69+
70+
Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo"));
71+
72+
StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo")
73+
.getFieldObjectInspector();
74+
75+
UDFJson udfJson = new UDFJson();
76+
Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.totalresults");
77+
assertEquals("1", output.toString());
78+
79+
}
80+
81+
@Test
82+
public void testNestedGetJsonObject() throws SerDeException, JSONException {
83+
Writable w = new Text(
84+
"{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"pagehit\":{ \"kind\": \"youtube#video\" } ,\"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }");
85+
86+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
87+
JSONObject result = (JSONObject) instance.deserialize(w);
88+
89+
Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo"));
90+
91+
StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo")
92+
.getFieldObjectInspector();
93+
94+
UDFJson udfJson = new UDFJson();
95+
Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.pagehit");
96+
assertEquals("{\"kind\":\"youtube#video\"}", output.toString());
97+
}
98+
99+
@Test
100+
public void testStringWhenNotJson() throws SerDeException, JSONException {
101+
Writable w = new Text(
102+
"{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": \"page\", \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }");
103+
104+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
105+
JSONObject result = (JSONObject) instance.deserialize(w);
106+
107+
Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo"));
108+
109+
StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo")
110+
.getFieldObjectInspector();
111+
112+
UDFJson udfJson = new UDFJson();
113+
Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.test_field");
114+
assertNull(output);
115+
}
116+
117+
@Test
118+
public void testStringWhenFieldIsNotInJson() throws SerDeException, JSONException {
119+
Writable w = new Text(
120+
"{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }");
121+
122+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
123+
JSONObject result = (JSONObject) instance.deserialize(w);
124+
125+
Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo"));
126+
127+
StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo")
128+
.getFieldObjectInspector();
129+
130+
UDFJson udfJson = new UDFJson();
131+
Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.test_field");
132+
assertNull(output);
133+
}
134+
135+
136+
@Test
137+
public void testStringWhenJson() throws SerDeException, JSONException {
138+
Writable w = new Text(
139+
"{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": \"page\", \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }");
140+
141+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
142+
JSONObject result = (JSONObject) instance.deserialize(w);
143+
144+
Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo"));
145+
146+
StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo")
147+
.getFieldObjectInspector();
148+
149+
String sres = loi.getPrimitiveJavaObject(res);
150+
assertEquals("page", sres);
151+
152+
}
153+
154+
}

0 commit comments

Comments
 (0)