Skip to content

fixed #69 & #72 #73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

fixed #69 & #72 #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a1935e4
fixed #69 & #72
Feb 23, 2021
d0145ce
fixed #69 & #72
Feb 24, 2021
63ce9d3
fixed #69 & #72
Feb 24, 2021
d80a94f
fixed #69 & #72
Feb 24, 2021
7f434c4
fixed #69 & #72
Feb 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 33 additions & 18 deletions flatten_json/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,11 @@ def _flatten(object_, key):
else:
flattened_dict[key] = object_

def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner):
def _flatten_low_entropy(object_,
key,
cur_depth,
cur_list_depth,
max_depth_inner):
"""
For dict, list and set objects_ calls itself on the elements and for
other types assigns the object_ to
Expand All @@ -217,15 +221,15 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner):
# write latest child as value if max_depth exceeded
if cur_depth > max_depth_inner:
global_max_record = int(max(list(
list_prebuilt_flattened_dict.keys())))
list_prebuilt_flattened_dict.keys()), key=int))
for d in list_prebuilt_flattened_dict[str(global_max_record)]:
d[key] = object_

else:
# Empty object can't be iterated, take as is
if not object_:
global_max_record = int(max(list(
list_prebuilt_flattened_dict.keys())))
list_prebuilt_flattened_dict.keys()), key=int))
for d in list_prebuilt_flattened_dict[str(global_max_record)]:
d[key] = object_

Expand All @@ -239,7 +243,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner):
and not (isinstance(object_[first_key], dict)
or isinstance(object_[first_key], list)):
global_max_record = int(max(list(
list_prebuilt_flattened_dict.keys())))
list_prebuilt_flattened_dict.keys()), key=int))

for d in list_prebuilt_flattened_dict[
str(global_max_record)
Expand All @@ -261,10 +265,13 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner):
object_key,
replace_separators=replace_separators),
cur_depth,
cur_list_depth,
max_depth_inner)

# lists could go into rows, like in a relational database
elif isinstance(object_, list) or isinstance(object_, set):
cur_list_depth = cur_list_depth + 1

if debug:
print("\nparent key of list:",
key, "| length: ",
Expand All @@ -273,7 +280,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner):
# need to remember global list state when we entered
# this recursion
global_max_record_start = int(max(list(
list_prebuilt_flattened_dict.keys())))
list_prebuilt_flattened_dict.keys()), key=int))
entry = copy.deepcopy(list_prebuilt_flattened_dict[
str(global_max_record_start)
])
Expand All @@ -294,35 +301,40 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner):
# start from second element, 1st element is like column
if index > 0:
global_max_record = int(max(list(
list_prebuilt_flattened_dict.keys())))
list_prebuilt_flattened_dict.keys()), key=int))

list_prebuilt_flattened_dict[
str(global_max_record + 1)
] = copy.deepcopy(entry)

_flatten_low_entropy(item, key, cur_depth,
_flatten_low_entropy(item,
key,
cur_depth,
cur_list_depth,
max_depth_inner)
else:
pass

list_prebuilt_flattened_dict['0'] = \
[subel for k, v in
sorted(list_prebuilt_flattened_dict.items())
for idx, subel in enumerate(v)]
cur_list_depth = cur_list_depth - 1
if cur_list_depth == 0:
list_prebuilt_flattened_dict['0'] = \
[subel for k, v in
sorted(list_prebuilt_flattened_dict.items())
for idx, subel in enumerate(v)]

for key in list(sorted(list_prebuilt_flattened_dict.keys())):
if key != '0':
del list_prebuilt_flattened_dict[key]
if debug:
print("collapsed global list")
for key in list(sorted(list_prebuilt_flattened_dict.keys())):
if key != '0':
del list_prebuilt_flattened_dict[key]
if debug:
print("collapsed global list")

# Anything left take as is, assuming you hit the end of the line.
else:
# in this case, there may be
# a list of prebuilt_flattened_dict by now
# so need to update them all.
global_max_record = int(max(list(
list_prebuilt_flattened_dict.keys())))
list_prebuilt_flattened_dict.keys()), key=int))

for d in list_prebuilt_flattened_dict[str(global_max_record)]:
d[key] = object_
Expand All @@ -348,7 +360,10 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner):
# initialize global record list
list_prebuilt_flattened_dict = {'0': [prebuilt_flattened_dict]}

_flatten_low_entropy(nested_dict, None, cur_depth=0,
_flatten_low_entropy(nested_dict,
None,
cur_depth=0,
cur_list_depth=0,
max_depth_inner=max_depth)

return list_prebuilt_flattened_dict['0']
Expand Down
134 changes: 134 additions & 0 deletions test_flatten.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2170,6 +2170,140 @@ def test_flatten_preserve_lists_issue43(self):
actual = flatten_preserve_lists(dic, max_list_index=50, max_depth=10)
self.assertEqual(expected, actual)

def test_flatten_preserve_lists_issue69(self):
"""https://github.com/amirziai/flatten/issues/69"""
dic = {
'a': 'a',
'b': [
{'b': 1, 'c': 1},
{'b': 2, 'c': 2},
{'b': 3, 'c': 3},
{'b': 4, 'c': 4},
{'b': 5, 'c': 5},
{'b': 6, 'c': 6},
{'b': 7, 'c': 7},
{'b': 8, 'c': 8},
{'b': 9, 'c': 9},
{'b': 10, 'c': 10},
{'b': 11, 'c': 11},
{'b': 12, 'c': 12},
{'b': 13, 'c': 13},
{'b': 14, 'c': 14},
{'b': 15, 'c': 15},
{'b': 16, 'c': 16},
{'b': 17, 'c': 17},
{'b': 15, 'c': 18},
{'b': 16, 'c': 19},
{'b': 17, 'c': 20}
]
}

expected = [
{'b.c': 1,
'b.b': 1,
'a': 'a'},
{'b.c': 2,
'b.b': 2,
'a': 'a'},
{'b.c': 11,
'b.b': 11,
'a': 'a'},
{'b.c': 12,
'b.b': 12,
'a': 'a'},
{'b.c': 13,
'b.b': 13,
'a': 'a'},
{'b.c': 14,
'b.b': 14,
'a': 'a'},
{'b.c': 15,
'b.b': 15,
'a': 'a'},
{'b.c': 16,
'b.b': 16,
'a': 'a'},
{'b.c': 17,
'b.b': 17,
'a': 'a'},
{'b.c': 18,
'b.b': 15,
'a': 'a'},
{'b.c': 19,
'b.b': 16,
'a': 'a'},
{'b.c': 20,
'b.b': 17,
'a': 'a'},
{'b.c': 3,
'b.b': 3,
'a': 'a'},
{'b.c': 4,
'b.b': 4,
'a': 'a'},
{'b.c': 5,
'b.b': 5,
'a': 'a'},
{'b.c': 6,
'b.b': 6,
'a': 'a'},
{'b.c': 7,
'b.b': 7,
'a': 'a'},
{'b.c': 8,
'b.b': 8,
'a': 'a'},
{'b.c': 9,
'b.b': 9,
'a': 'a'},
{'b.c': 10,
'b.b': 10,
'a': 'a'}]

actual = flatten_preserve_lists(dic,
separator='.',
max_list_index=100,
max_depth=5)
self.assertEqual(expected, actual)

def test_flatten_preserve_lists_issue72(self):
"""https://github.com/amirziai/flatten/issues/72"""
dic = {'a': 0,
'b': [
{'c': 1,
'd': [{'e': 1}]},
{'c': 2,
'd': [{'e': 2}]},
{'c': 3,
'd': [{'e': 3}]},
{'c': 4,
'd': [{'e': 4}]},
{'c': 5,
'd': [{'e': 5}]}]}

expected = [
{'b.c': 1,
'a': 0,
'b.d': 1},
{'b.c': 2,
'a': 0,
'b.d': 2},
{'b.c': 3,
'a': 0,
'b.d': 3},
{'b.c': 4,
'a': 0,
'b.d': 4},
{'b.c': 5,
'a': 0,
'b.d': 5}]

actual = flatten_preserve_lists(dic,
separator='.',
max_list_index=10,
max_depth=5)
self.assertEqual(expected, actual)

def test_unflatten_with_list_deep(self):
dic = {'a': [
{'b': [{'c': [{'a': 5, 'b': {'a': [1, 2, 3]}, 'c': {'x': 3}}]}]}]}
Expand Down