Skip to content

Commit 813aa07

Browse files
Will Mossyjin
Will Moss
authored and
yjin
committed
r - Fix for #issue-70 related to get_json_object
This is a reworking of rcongiu#71 to update it to the new directory structure in 1.3
1 parent 50f60be commit 813aa07

File tree

3 files changed

+211
-3
lines changed

3 files changed

+211
-3
lines changed

‎json-serde/src/main/java/org/openx/data/jsonserde/objectinspector/JsonObjectInspectorFactory.java

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,11 @@
1111
*======================================================================*/
1212
package org.openx.data.jsonserde.objectinspector;
1313

14-
import java.util.*;
14+
import java.util.ArrayList;
15+
import java.util.EnumMap;
16+
import java.util.HashMap;
17+
import java.util.List;
18+
import java.util.Map;
1519

1620
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
1721
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -24,6 +28,7 @@
2428
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringDoubleObjectInspector;
2529
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringFloatObjectInspector;
2630
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringIntObjectInspector;
31+
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringJsonObjectInspector;
2732
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringLongObjectInspector;
2833
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringShortObjectInspector;
2934
import org.openx.data.jsonserde.objectinspector.primitive.JavaStringTimestampObjectInspector;
@@ -201,14 +206,17 @@ public static JsonMapObjectInspector getJsonMapObjectInspector(
201206
static {
202207
primitiveOICache.put(PrimitiveCategory.BYTE, new JavaStringByteObjectInspector());
203208
primitiveOICache.put(PrimitiveCategory.SHORT, new JavaStringShortObjectInspector());
209+
primitiveOICache.put(PrimitiveCategory.STRING, new JavaStringJsonObjectInspector());
210+
primitiveOICache.put(PrimitiveCategory.BYTE, new JavaStringByteObjectInspector());
211+
primitiveOICache.put(PrimitiveCategory.SHORT, new JavaStringShortObjectInspector());
204212
primitiveOICache.put(PrimitiveCategory.INT, new JavaStringIntObjectInspector());
205213
primitiveOICache.put(PrimitiveCategory.LONG, new JavaStringLongObjectInspector());
206214
primitiveOICache.put(PrimitiveCategory.FLOAT, new JavaStringFloatObjectInspector());
207215
primitiveOICache.put(PrimitiveCategory.DOUBLE, new JavaStringDoubleObjectInspector());
208216
primitiveOICache.put(PrimitiveCategory.TIMESTAMP, new JavaStringTimestampObjectInspector());
209217
primitiveOICache.put(PrimitiveCategory.BOOLEAN, new JavaStringBooleanObjectInspector());
210218
}
211-
219+
212220
/**
213221
* gets the appropriate adapter wrapper around the object inspector if
214222
* necessary, that is, if we're dealing with numbers. The JSON parser won't
@@ -219,7 +227,7 @@ public static JsonMapObjectInspector getJsonMapObjectInspector(
219227
*/
220228
public static AbstractPrimitiveJavaObjectInspector getPrimitiveJavaObjectInspector(
221229
PrimitiveCategory primitiveCategory) {
222-
230+
223231
if(! primitiveOICache.containsKey(primitiveCategory)) {
224232
primitiveOICache.put(primitiveCategory, PrimitiveObjectInspectorFactory.
225233
getPrimitiveJavaObjectInspector(primitiveCategory));

‎json-serde/src/main/java/org/openx/data/jsonserde/objectinspector/primitive/JavaStringJsonObjectInspector.java

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package org.openx.data.jsonserde.objectinspector.primitive;
2+
3+
4+
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
5+
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
6+
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector;
7+
import org.apache.hadoop.io.Text;
8+
9+
public class JavaStringJsonObjectInspector
10+
extends AbstractPrimitiveJavaObjectInspector
11+
implements SettableStringObjectInspector {
12+
13+
public JavaStringJsonObjectInspector() {
14+
super(TypeEntryShim.stringType);
15+
}
16+
17+
@Override
18+
public Text getPrimitiveWritableObject(Object o) {
19+
return o == null ? null : new Text((String) o.toString());
20+
}
21+
22+
@Override
23+
public String getPrimitiveJavaObject(Object o) {
24+
return o == null ? null : o.toString();
25+
}
26+
27+
@Override
28+
public Object create(Text value) {
29+
return value == null ? null : value.toString();
30+
}
31+
32+
@Override
33+
public Object set(Object o, Text value) {
34+
return value == null ? null : value.toString();
35+
}
36+
37+
@Override
38+
public Object create(String value) {
39+
return value;
40+
}
41+
42+
@Override
43+
public Object set(Object o, String value) {
44+
return value;
45+
}
46+
}

‎json-serde/src/test/java/org/openx/data/jsonserde/GetJsonObjectTest.java

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
package org.openx.data.jsonserde;
2+
3+
import static org.junit.Assert.*;
4+
5+
import java.util.Properties;
6+
7+
import org.apache.hadoop.conf.Configuration;
8+
import org.apache.hadoop.hive.ql.udf.UDFJson;
9+
import org.apache.hadoop.hive.serde.Constants;
10+
import org.apache.hadoop.hive.serde2.SerDeException;
11+
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
12+
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
13+
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
14+
import org.apache.hadoop.io.Text;
15+
import org.apache.hadoop.io.Writable;
16+
import org.junit.Before;
17+
import org.junit.Test;
18+
import org.openx.data.jsonserde.json.JSONException;
19+
import org.openx.data.jsonserde.json.JSONObject;
20+
21+
/**
22+
* Tests getJson Object
23+
*
24+
* @author snagmote
25+
*
26+
*/
27+
public class GetJsonObjectTest {
28+
29+
static JsonSerDe instance;
30+
31+
@Before
32+
public void setUp() throws Exception {
33+
initialize();
34+
}
35+
36+
static public void initialize() throws Exception {
37+
instance = new JsonSerDe();
38+
Configuration conf = null;
39+
Properties tbl = new Properties();
40+
// from google video API
41+
tbl.setProperty(Constants.LIST_COLUMNS, "kind,etag,pageInfo,v_items");
42+
tbl.setProperty(
43+
Constants.LIST_COLUMN_TYPES,
44+
("string,string," + "string,"
45+
+ "ARRAY<STRUCT<kind:STRING,"
46+
+ "etag:STRING,"
47+
+ "id:STRING,"
48+
+ "v_statistics:STRUCT<viewCount:INT,likeCount:INT,dislikeCount:INT,favoriteCount:INT,commentCount:INT>,"
49+
+ "topicDetails:STRUCT<topicIds:ARRAY<STRING>,relevantTopicIds:ARRAY<STRING>>"
50+
+ ">>").toLowerCase());
51+
tbl.setProperty("mapping.v_items", "items");
52+
tbl.setProperty("mapping.v_statistics", "statistics");
53+
54+
instance.initialize(conf, tbl);
55+
tbl.setProperty("mapping.v_items", "items");
56+
tbl.setProperty("mapping.v_statistics", "statistics");
57+
58+
instance.initialize(conf, tbl);
59+
}
60+
61+
@Test
62+
public void testGetJsonObject() throws SerDeException, JSONException {
63+
Writable w = new Text(
64+
"{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }");
65+
66+
JSONObject result = (JSONObject) instance.deserialize(w);
67+
68+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
69+
70+
Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo"));
71+
72+
StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo")
73+
.getFieldObjectInspector();
74+
75+
UDFJson udfJson = new UDFJson();
76+
Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.totalresults");
77+
assertEquals("1", output.toString());
78+
79+
}
80+
81+
@Test
82+
public void testNestedGetJsonObject() throws SerDeException, JSONException {
83+
Writable w = new Text(
84+
"{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"pagehit\":{ \"kind\": \"youtube#video\" } ,\"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }");
85+
86+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
87+
JSONObject result = (JSONObject) instance.deserialize(w);
88+
89+
Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo"));
90+
91+
StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo")
92+
.getFieldObjectInspector();
93+
94+
UDFJson udfJson = new UDFJson();
95+
Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.pagehit");
96+
assertEquals("{\"kind\":\"youtube#video\"}", output.toString());
97+
}
98+
99+
@Test
100+
public void testStringWhenNotJson() throws SerDeException, JSONException {
101+
Writable w = new Text(
102+
"{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": \"page\", \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }");
103+
104+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
105+
JSONObject result = (JSONObject) instance.deserialize(w);
106+
107+
Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo"));
108+
109+
StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo")
110+
.getFieldObjectInspector();
111+
112+
UDFJson udfJson = new UDFJson();
113+
Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.test_field");
114+
assertNull(output);
115+
}
116+
117+
@Test
118+
public void testStringWhenFieldIsNotInJson() throws SerDeException, JSONException {
119+
Writable w = new Text(
120+
"{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }");
121+
122+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
123+
JSONObject result = (JSONObject) instance.deserialize(w);
124+
125+
Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo"));
126+
127+
StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo")
128+
.getFieldObjectInspector();
129+
130+
UDFJson udfJson = new UDFJson();
131+
Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.test_field");
132+
assertNull(output);
133+
}
134+
135+
136+
@Test
137+
public void testStringWhenJson() throws SerDeException, JSONException {
138+
Writable w = new Text(
139+
"{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": \"page\", \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }");
140+
141+
StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector();
142+
JSONObject result = (JSONObject) instance.deserialize(w);
143+
144+
Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo"));
145+
146+
StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo")
147+
.getFieldObjectInspector();
148+
149+
String sres = loi.getPrimitiveJavaObject(res);
150+
assertEquals("page", sres);
151+
152+
}
153+
154+
}

0 commit comments

Comments
 (0)