Merge pull request #39 from alingse/fix-expand-restore

Fix expand restore && enhance is_array

Author: alingse

README.rst

@@ -24,8 +24,8 @@ cat the raw.json to csv/xls use command line tool
 
 .. code-block:: bash
 
-    cat raw.json| jsoncsv | mkexcel > output.csv
-    cat raw.json| jsoncsv | mkexcel -t xls > output.xls
+    cat raw.json | jsoncsv | mkexcel > output.csv
+    cat raw.json | jsoncsv | mkexcel -t xls > output.xls
 
 make sure each line of raw json text file is a json object
 
@@ -87,7 +87,7 @@ just expand/restore the json, the expand json is one layer json.
 
     jsoncsv raw.json expand.json
     jsoncsv -r expand.json raw.json
-    cat raw.json|jsoncsv |jsoncsv -r > raw2.json
+    cat raw.json | jsoncsv | jsoncsv -r > raw2.json
 
 mkexcel the expanded json (one layer)
 
@@ -104,17 +104,17 @@ expand json, 展开 json
 
 .. code-block:: bash
 
-    jsoncsv -e raw.json expand.json
-    cat raw.json expand.json
+    $jsoncsv -e raw.json expand.json
+    $cat raw.json expand.json
     {"s":[1,2,{"w":1}]}
     {"s.2.w": 1,"s.0": 1,"s.1": 2}
 
 
-{"s":[1,2,{"w":1}]} transformed to {"s.2.w": 1,"s.0": 1,"s.1": 2}
+{"s":[1,2,{"w":1}]} will transformed to {"s.2.w": 1,"s.0": 1,"s.1": 2}
 
-expand.json is only one layer json, it can be easy change to csv or xlsx
+the output "expand.json" is only one layer json, it can be easy change to csv or xlsx (with `mkexcel`)
 
--r,--restore
+-r, --restore
 ---------------
 restore the expanded json 重构被展开的json
 
@@ -127,7 +127,7 @@ restore the expanded json 重构被展开的json
 
 {"s.2.w": 1,"s.0": 1,"s.1": 2} change to {"s":[1,2,{"w":1}]}
 
--s,--separator
+-s, --separator
 ---------------
 
 separator used for combine the keys in the tree
@@ -156,7 +156,7 @@ dump expanded (by **jsoncsv**) json file to csv or xls file
 
     mkexcel expand.json output.csv
 
--t,--type
+-t, --type
 --------------
 
 chose dump type in ['csv', 'xls'] default is 'csv'
@@ -191,3 +191,9 @@ wait next next version
 -----------------------------------------
 
 but better than python strand library csv.
+
+4. Windows is poor support
+-----------------------------------------
+see https://github.com/alingse/jsoncsv/issues/37
+
+try use https://jsoncsv.jsonutil.online/ instead

fixture/files/raw.2.json

@@ -0,0 +1,2 @@
+["a", "ab", "b", "a", "ab", "b", "a", "ab", "b", "a", "ab", "b"]
+["c", "ab", "b", "a", "ab", "b", "a", "ab", "b", "a", "ab", "c"]

jsoncsv/jsontool.py

@@ -5,7 +5,6 @@
 from __future__ import unicode_literals
 
 import json
-
 from copy import deepcopy
 from itertools import groupby
 from operator import itemgetter
@@ -26,7 +25,6 @@ def gen_leaf(root, path=None):
     if path is None:
         path = []
 
-    # the leaf
     if not isinstance(root, (dict, list)) or not root:
         leaf = (path, root)
         yield leaf
@@ -36,7 +34,6 @@ def gen_leaf(root, path=None):
                 items = root.iteritems()
             else:
                 items = root.items()
-
         else:
             items = enumerate(root)
 
@@ -47,14 +44,17 @@ def gen_leaf(root, path=None):
                 yield leaf
 
 
-def is_array(keys, ensure_str=True):
-    copy_keys = list(deepcopy(keys))
-    int_keys = list(range(len(copy_keys)))
-    if copy_keys == int_keys:
+def is_array_index(keys, enable_str=True):
+    keys = list(deepcopy(keys))
+    # 不强调有序
+    key_map = {key: True for key in keys}
+    int_keys = range(len(keys))
+
+    if all(key in key_map for key in int_keys):
         return True
-    if ensure_str:
-        str_keys = list(map(str, int_keys))
-        if copy_keys == str_keys:
+
+    if enable_str:
+        if all(str(key) in key_map for key in int_keys):
             return True
     return False
 
@@ -84,7 +84,7 @@ def from_leaf(leafs):
         child.append((head, _child))
 
     child_keys = map(_get_head, child)
-    if is_array(child_keys):
+    if is_array_index(child_keys):
         child.sort(key=lambda x: int(x[0]))
         return list(map(_get_leaf, child))
 
@@ -161,6 +161,7 @@ def gen_objs_from_array():
         content = json.dumps(new, ensure_ascii=False)
         if PY2:
             fout.write(content.encode('utf-8'))
+            fout.write('\n'.encode('utf-8'))
         else:
             fout.write(content)
-        fout.write(str('\n'))
+            fout.write('\n')

jsoncsv/main.py

@@ -7,7 +7,15 @@
 from jsoncsv import dumptool
 from jsoncsv.dumptool import dump_excel
 from jsoncsv.jsontool import convert_json
-from jsoncsv.utils import separator_type
+from jsoncsv.utils import unit_char
+
+
+def separator_type(sep):
+    if len(sep) != 1:
+        raise click.BadOptionUsage('separator can only be a char')
+    if sep == unit_char:
+        raise click.BadOptionUsage('separator can not be `\\` ')
+    return sep
 
 
 @click.command()

jsoncsv/utils.py

@@ -2,21 +2,9 @@
 # author@alingse
 # 2016.11.20
 
-import click
-
-__all__ = []
-
 unit_char = '\\'
 
 
-def separator_type(sep):
-    if len(sep) > 1:
-        raise click.BadOptionUsage('separator can only be a char')
-    if sep == unit_char:
-        raise click.BadOptionUsage('separator can not be `\\` ')
-    return sep
-
-
 def encode_safe_key(path, separator):
     path = [p.replace(unit_char, unit_char * 2) for p in path]
     separator = unit_char + separator

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name='jsoncsv',
-    version='2.2.2',
+    version='2.2.3',
     url='https://github.com/alingse/jsoncsv',
     description='A command tool easily convert json file to csv or xlsx.',
     long_description=readme,

tests/test_jsoncsv.py

@@ -1,7 +1,8 @@
 # coding=utf-8
 # author@alingse
 # 2016.08.09
-
+import io
+import json
 import unittest
 from click.testing import CliRunner
 
@@ -21,3 +22,18 @@ def test_jsoncsv_expand_with_json_array(self):
         args = ['-e', 'fixture/files/raw.1.json', 'fixture/files/tmp.expand.1.json', '-A']
         result = runner.invoke(jsoncsv, args=args)
         assert result.exit_code == 0
+
+    def test_jsoncsv_expand_restore(self):
+        runner = CliRunner(echo_stdin=True)
+        result = runner.invoke(jsoncsv, args=['-e', 'fixture/files/raw.2.json', 'fixture/files/tmp.expand.2.json'])
+        assert result.exit_code == 0
+        result = runner.invoke(jsoncsv, args=['-r', 'fixture/files/tmp.expand.2.json', 'fixture/files/tmp.restore.2.json'])
+        assert result.exit_code == 0
+
+        with io.open('fixture/files/raw.2.json', 'r') as f:
+            input_data = [json.loads(line) for line in f]
+
+        with io.open('fixture/files/tmp.restore.2.json', 'r') as f:
+            resotre_data = [json.loads(line) for line in f]
+
+        self.assertEqual(input_data, resotre_data)

tests/test_jsontool.py

@@ -8,7 +8,7 @@
 
 from jsoncsv import PY2
 from jsoncsv.jsontool import expand, restore
-from jsoncsv.jsontool import is_array
+from jsoncsv.jsontool import is_array_index
 from jsoncsv.jsontool import convert_json
 
 
@@ -63,11 +63,13 @@ def test_complex(self):
 
         self.assertListEqual(s[4], _s[4])
 
-    def test_is_array(self):
-        assert is_array([0, 1, 2, 3])
-        assert is_array(['0', '1', '2', '3'])
-        assert not is_array([1, 2, 3])
-        assert not is_array(['0', 1, 2])
+    def test_is_array_index(self):
+        self.assertTrue(is_array_index([0, 1, 2, 3]))
+        self.assertTrue(is_array_index(['0', '1', '2', '3']))
+        # string order
+        self.assertTrue(is_array_index(['0', '1', '10', '2', '3', '4', '5', '6', '7', '8', '9']))
+        self.assertFalse(is_array_index([1, 2, 3]))
+        self.assertFalse(is_array_index(['0', 1, 2]))
 
     def test_unicode(self):
         data = [
@@ -90,6 +92,15 @@ def test_expand_with_safe(self):
         origin = restore(expobj, safe=True)
         self.assertEqual(origin, data)
 
+    def test_expand_and_restore(self):
+        data = ["a", "ab", "b"] * 4
+        expobj = expand(data)
+        self.assertEqual(expobj["0"], "a")
+        self.assertEqual(expobj["1"], "ab")
+
+        origin = restore(expobj)
+        self.assertEqual(data, origin)
+
 
 class TestConvertJSON(unittest.TestCase):