Creating plugin for yaml

Makefile

@@ -4,6 +4,9 @@ envtest: clean
 test:
 	tox
 
+tests_nose: 
+	nosetests -dsv --with-yanc --with-coverage --cover-package rows tests/*.py
+
 clean:
 	find -regex '.*\.pyc' -exec rm {} \;
 	find -regex '.*~' -exec rm {} \;
@@ -31,7 +34,11 @@ man:
 	head -1 rows.1.txt > rows.1
 	txt2man rows.1.txt | egrep -v '^\.TH' >> rows.1
 
+dev-setup:
+	pip install --editable .[all]
+	pip install	-r requirements-development.txt
+
 release:
 	python setup.py bdist bdist_wheel bdist_egg upload
 
-.PHONY:	test clean lint lint-tests install uninstall man release
+.PHONY:	test clean lint lint-tests install uninstall man release

rows/__init__.py

@@ -27,6 +27,9 @@
 
 
 # Don't have dependencies or dependencies installed on `install_requires`
+import_from_yaml = plugins.yaml.import_from_yaml
+export_to_yaml = plugins.yaml.export_to_yaml
+
 
 import_from_json = plugins.json.import_from_json
 export_to_json = plugins.json.export_to_json

rows/plugins/__init__.py

@@ -16,6 +16,7 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from . import plugin_json as json
+from . import plugin_yaml as yaml
 from . import dicts as dicts
 from . import plugin_csv as csv
 from . import txt as txt

rows/plugins/plugin_json.py

@@ -17,10 +17,7 @@
 
 from __future__ import unicode_literals
 
-import datetime
-import decimal
 import json
-
 import six
 
 from rows import fields

rows/plugins/plugin_yaml.py

@@ -0,0 +1,85 @@
+# coding: utf-8
+
+# Copyright 2014-2016 Álvaro Justen <https://github.com/turicas/rows/>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+import six
+import yaml
+
+from rows import fields
+from rows.plugins.utils import (create_table, export_data,
+                                get_filename_and_fobj, prepare_to_export)
+
+
+def import_from_yaml(filename_or_fobj, encoding='utf-8', *args, **kwargs):
+    '''Import a YAML file or file-like object into a `rows.Table`
+
+    If a file-like object is provided it MUST be open in text (non-binary) mode
+    on Python 3 and could be open in both binary or text mode on Python 2.
+    '''
+
+    filename, fobj = get_filename_and_fobj(filename_or_fobj)
+
+    yaml_obj = yaml.load(fobj)
+    field_names = list(yaml_obj[0].keys())
+    table_rows = [[item[key] for key in field_names] for item in yaml_obj]
+
+    meta = {
+        'imported_from': 'yaml',
+        'filename': filename,
+        'encoding': encoding
+    }
+    return create_table([field_names] + table_rows, meta=meta, *args, **kwargs)
+
+
+def _convert(value, field_type, *args, **kwargs):
+    if value is None or field_type in (
+            fields.BinaryField,
+            fields.BoolField,
+            fields.FloatField,
+            fields.IntegerField,
+            fields.JSONField,
+            fields.TextField,):
+        return value
+    else:
+        return field_type.serialize(value, *args, **kwargs)
+
+
+def export_to_yaml(table, filename_or_fobj=None, encoding='utf-8', indent=None,
+                   *args, **kwargs):
+    '''Export a `rows.Table` to a YAML file or file-like object
+    '''
+
+    all_fields = table.fields
+    prepared_table = prepare_to_export(table, *args, **kwargs)
+    field_names = next(prepared_table)
+    data = [{field_name: _convert(value,
+                                  all_fields[field_name],
+                                  *args,
+                                  **kwargs)
+             for field_name, value in zip(field_names, row)}
+            for row in prepared_table]
+
+    result = yaml.dump(data, indent=indent)
+    if type(result) is six.text_type:
+        result = result.encode(encoding)
+
+    if indent is not None:
+        # clean up empty spaces at the end of lines
+        result = b'\n'.join(line.rstrip() for line in result.splitlines())
+
+    return export_data(filename_or_fobj, result, mode='wb')

setup.py

@@ -22,6 +22,7 @@
 
 EXTRA_REQUIREMENTS = {
     'csv': ['unicodecsv'],
+    'yaml': ['pyyaml'],
     'cli': ['click', 'requests'],
     'html': ['lxml'],  # apt: libxslt-dev libxml2-dev
     'ods': ['lxml'],

tests/data/all-field-types.yaml

@@ -0,0 +1,57 @@
+---
+- float_column: 3.141592
+  decimal_column: 3.141592
+  bool_column: 'True'
+  integer_column: 1
+  date_column: '2015-01-01'
+  datetime_column: '2015-08-18T15:10:00'
+  percent_column: 1%
+  unicode_column: Álvaro
+- float_column: 1.234
+  decimal_column: 1.234
+  bool_column: 'False'
+  integer_column: 2
+  date_column: '1999-02-03'
+  datetime_column: '1999-02-03T00:01:02'
+  percent_column: 11.69%
+  unicode_column: àáãâä¹²³
+- float_column: 4.56
+  decimal_column: 4.56
+  bool_column: true
+  integer_column: 3
+  date_column: '2050-01-02'
+  datetime_column: '2050-01-02T23:45:31'
+  percent_column: 12%
+  unicode_column: éèẽêë
+- float_column: 7.89
+  decimal_column: 7.89
+  bool_column: false
+  integer_column: 4
+  date_column: '2015-08-18'
+  datetime_column: '2015-08-18T22:21:33'
+  percent_column: 13.64%
+  unicode_column: "~~~~"
+- float_column: 9.87
+  decimal_column: 9.87
+  bool_column: 'yes'
+  integer_column: 5
+  date_column: '2015-03-04'
+  datetime_column: '2015-03-04T16:00:01'
+  percent_column: 13.14%
+  unicode_column: álvaro
+- float_column: 1.2345
+  decimal_column: 1.2345
+  bool_column: 'no'
+  integer_column: 6
+  date_column: '2015-05-06'
+  datetime_column: '2015-05-06T12:01:02'
+  percent_column: 2%
+  unicode_column: test
+- float_column: ''
+  decimal_column: "-"
+  bool_column: 'null'
+  integer_column: nil
+  date_column: none
+  datetime_column: n/a
+  percent_column: 'null'
+  unicode_column: ''

tests/tests_plugin_yaml.py

@@ -0,0 +1,181 @@
+# coding: utf-8
+
+# Copyright 2014-2016 Álvaro Justen <https://github.com/turicas/rows/>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import unicode_literals
+
+import unittest
+import tempfile
+import yaml
+
+from collections import Counter
+from collections import OrderedDict
+from collections import defaultdict
+
+import six
+import mock
+
+import rows
+import tests.utils as utils
+
+
+class PluginYamlTestCase(utils.RowsTestMixIn, unittest.TestCase):
+
+    plugin_name = 'yaml'
+    file_extension = 'yaml'
+    filename = 'tests/data/all-field-types.yaml'
+    encoding = 'utf-8'
+    assert_meta_encoding = True
+
+    def test_imports(self):
+        self.assertIs(rows.import_from_yaml,
+                      rows.plugins.plugin_yaml.import_from_yaml)
+        self.assertIs(rows.export_to_yaml,
+                      rows.plugins.plugin_yaml.export_to_yaml)
+
+    @mock.patch('rows.plugins.plugin_yaml.create_table')
+    def test_import_from_yaml_uses_create_table(self, mocked_create_table):
+        mocked_create_table.return_value = 42
+        kwargs = {'some_key': 123, 'other': 456, }
+        result = rows.import_from_yaml(self.filename, encoding=self.encoding,
+                                       **kwargs)
+        self.assertTrue(mocked_create_table.called)
+        self.assertEqual(mocked_create_table.call_count, 1)
+        self.assertEqual(result, 42)
+
+        call = mocked_create_table.call_args
+        kwargs['meta'] = {'imported_from': 'yaml',
+                          'filename': self.filename,
+                          'encoding': self.encoding,}
+        self.assertEqual(call[1], kwargs)
+
+    @mock.patch('rows.plugins.plugin_yaml.create_table')
+    def test_import_from_yaml_retrieve_desired_data(self, mocked_create_table):
+        mocked_create_table.return_value = 42
+
+        # import using filename
+        table_1 = rows.import_from_yaml(self.filename)
+        call_args = mocked_create_table.call_args_list[0]
+        self.assert_create_table_data(call_args, field_ordering=False)
+
+        # import using fobj
+        with open(self.filename) as fobj:
+            table_2 = rows.import_from_yaml(fobj)
+            call_args = mocked_create_table.call_args_list[1]
+            self.assert_create_table_data(call_args, field_ordering=False)
+
+    @mock.patch('rows.plugins.plugin_yaml.create_table')
+    def test_import_from_yaml_uses_create_table(self, mocked_create_table):
+        mocked_create_table.return_value = 42
+        kwargs = {'some_key': 123, 'other': 456, }
+        encoding = 'iso-8859-15'
+        result = rows.import_from_yaml(self.filename, encoding=encoding,
+                                       **kwargs)
+        self.assertTrue(mocked_create_table.called)
+        self.assertEqual(mocked_create_table.call_count, 1)
+        self.assertEqual(result, 42)
+
+        call = mocked_create_table.call_args
+        kwargs['meta'] = {'imported_from': 'yaml',
+                          'filename': self.filename,
+                          'encoding': encoding,}
+        self.assertEqual(call[1], kwargs)
+
+    @mock.patch('rows.plugins.plugin_yaml.prepare_to_export')
+    def test_export_to_yaml_uses_prepare_to_export(self,
+            mocked_prepare_to_export):
+        temp = tempfile.NamedTemporaryFile(delete=False, mode='wb')
+        self.files_to_delete.append(temp.name)
+        kwargs = {'test': 123, 'parameter': 3.14, }
+        mocked_prepare_to_export.return_value = \
+                iter([utils.table.fields.keys()])
+
+        rows.export_to_yaml(utils.table, temp.name, **kwargs)
+        self.assertTrue(mocked_prepare_to_export.called)
+        self.assertEqual(mocked_prepare_to_export.call_count, 1)
+
+        call = mocked_prepare_to_export.call_args
+        self.assertEqual(call[0], (utils.table, ))
+        self.assertEqual(call[1], kwargs)
+
+    @mock.patch('rows.plugins.plugin_yaml.export_data')
+    def test_export_to_yaml_uses_export_data(self, mocked_export_data):
+        temp = tempfile.NamedTemporaryFile(delete=False, mode='wb')
+        self.files_to_delete.append(temp.name)
+        kwargs = {'test': 123, 'parameter': 3.14, }
+        mocked_export_data.return_value = 42
+
+        result = rows.export_to_yaml(utils.table, temp.name, **kwargs)
+        self.assertTrue(mocked_export_data.called)
+        self.assertEqual(mocked_export_data.call_count, 1)
+        self.assertEqual(result, 42)
+
+        call = mocked_export_data.call_args
+        self.assertEqual(call[0][0], temp.name)
+        self.assertEqual(call[1], {'mode': 'wb'})
+
+    def test_export_to_yaml_filename(self):
+        # TODO: may test file contents
+        temp = tempfile.NamedTemporaryFile(delete=False, mode='wb')
+        self.files_to_delete.append(temp.name)
+        rows.export_to_yaml(utils.table, temp.name)
+        table = rows.import_from_yaml(temp.name)
+        self.assert_table_equal(table, utils.table)
+
+    def test_export_to_yaml_fobj(self):
+        # TODO: may test with codecs.open passing an encoding
+        # TODO: may test file contents
+        temp = tempfile.NamedTemporaryFile(delete=False, mode='wb')
+        self.files_to_delete.append(temp.name)
+        rows.export_to_yaml(utils.table, temp.file)
+
+        table = rows.import_from_yaml(temp.name)
+        self.assert_table_equal(table, utils.table)
+
+    def test_export_to_yaml_filename_save_data_in_correct_format(self):
+        temp = tempfile.NamedTemporaryFile(delete=False, mode='wb')
+        self.files_to_delete.append(temp.name)
+
+        rows.export_to_yaml(utils.table, temp.name)
+
+        with open(temp.name) as fobj:
+            imported_yaml = yaml.load(fobj)
+
+        COLUMN_TYPE = {
+                'float_column': float,
+                'decimal_column': float,
+                'bool_column': bool,
+                'integer_column': int,
+                'date_column': six.text_type,
+                'datetime_column': six.text_type,
+                'percent_column': six.text_type,
+                'unicode_column': six.text_type,
+        }
+        field_types = defaultdict(list)
+        for row in imported_yaml:
+            for field_name, value in row.items():
+                field_types[field_name].append(type(value))
+        # We test if the JSON was created serializing all the fields correctly
+        # (some as native JSON values, like int and float) and others needed to
+        # be serialized, like date, datetime etc.
+        for field_name, value_types in field_types.items():
+            if field_name != 'unicode_column':
+                self.assertEqual(Counter(value_types),
+                                 Counter({type(None): 1,
+                                          COLUMN_TYPE[field_name]: 6}))
+            else:
+                self.assertEqual(Counter(value_types),
+                                 Counter({COLUMN_TYPE[field_name]: 7}))