diff --git a/README.rst b/README.rst index 94c04e2..e06873f 100644 --- a/README.rst +++ b/README.rst @@ -24,6 +24,7 @@ Siilo supports for the following file storages: - Local Filesystem - Apache Libcloud - Amazon S3 +- CMIS Siilo has the following goals: diff --git a/docs/index.rst b/docs/index.rst index dab1304..0250847 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,6 +13,7 @@ Siilo supports for the following file storages: - :ref:`local-filesystem` - :ref:`apache-libcloud` - :ref:`amazon-s3` + - :ref:`cmis` Siilo has the following goals: @@ -34,6 +35,7 @@ Contents storages/amazon_s3 storages/apache_libcloud storages/filesystem + storages/cmis api changelog license diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 505ebb4..c517b01 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -3,8 +3,8 @@ Quickstart This page gives you an introduction to Siilo. It shows you an example of using Siilo in the local filesystem. You can read how you can use Siilo for the -:ref:`local-filesystem`, :ref:`amazon-s3` and :ref:`apache-libcloud` in their -respective chapters. +:ref:`local-filesystem`, :ref:`amazon-s3`, :ref:`apache-libcloud` and +:ref:`cmis` in their respective chapters. If you do not already have Siilo installed, run this in your terminal:: diff --git a/docs/storages/cmis.rst b/docs/storages/cmis.rst new file mode 100644 index 0000000..6b0212d --- /dev/null +++ b/docs/storages/cmis.rst @@ -0,0 +1,14 @@ +.. _cmis: + +CMIS +==== + +.. module:: siilo.storages.cmis + +.. autoclass:: CmisStorage + :members: + :show-inheritance: + +.. autoclass:: CmisFile + :members: + :show-inheritance: diff --git a/siilo/storages/cmis.py b/siilo/storages/cmis.py new file mode 100644 index 0000000..cde39d2 --- /dev/null +++ b/siilo/storages/cmis.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- +""" + siilo.storages.cmis + ~~~~~~~~~~~~~~~~~~~ + + :copyright: (c) 2014 by Janne Vanhala. + :license: MIT, see LICENSE for more details. +""" +import io +import os +import shutil +import tempfile + +from cmislib.exceptions import ObjectNotFoundException, RuntimeException + +from ..exceptions import FileNotFoundError +from .base import Storage + + +class CmisStorage(Storage): + """ + A storage for a `Content Management Interoperability Services`_ (CMIS) + compatible CMS. + + .. _Content Management Interoperability Services: + http://chemistry.apache.org/project/cmis.html + + In order to use this storage driver you need to have Apache Chemistry + CmisLib installed. You can install it using pip:: + + pip install cmislib + + **Note:** The current version of cmislib 0.5.1 is not compatible with + Python 3. + + Example:: + + import cmislib + from siilo.storages.cmis import CmisStorage + + client = CmisClient( + 'http://cmis.alfresco.com/s/cmis', 'admin', 'admin') + repository = client.defaultRepository + + storage = CmisStorage(repository) + + with storage.open('hello.txt', 'w') as f: + f.write('Hello World!') + + :param repository: + the :class:`cmislib.Repository` used by this storage for file + operations. + """ + def __init__(self, repository): + self.repository = repository + + def _get_object(self, name): + try: + return self.repository.getObjectByPath(name) + except ObjectNotFoundException: + raise FileNotFoundError(name) + + def delete(self, name, all_versions=False): + obj = self._get_object(name) + try: + obj.delete(allVersions=all_versions) + # If we try to delete a document which is already removed on the + # server side it will return a RunTimeException with HTTP Error + # Code 500 + except RuntimeException: + raise FileNotFoundError(name) + + def exists(self, name): + try: + self._get_object(name) + except FileNotFoundError: + return False + return True + + def open(self, name, mode='r', encoding=None): + return CmisFile( + storage=self, + name=name, + mode=mode, + encoding=encoding + ) + + def size(self, name): + obj = self._get_object(name) + return obj.properties.get('cmis:contentStreamLength') + + def __repr__(self): + return ''.format(self.repository) + + +class CmisFile(object): + """ + A file like object for abstracting operations with the + :class:`CmisStorage` class. + + Example:: + + with storage.open('/folder/subfolder/file.txt') as f: + content = f.readlines() + title = f.cmis_object.title + properties = f.cmis_object.properties + + :param storage: + the :class:`CmisStorage` instance. + + :param name: + name of the remote file, can include directories. Adds a leading slash + if missing. + + :param mode: + file mode. + + :param encoding: + file encoding. + """ + def __init__(self, storage, name, mode='r', encoding=None): + self.storage = storage + #: :class:`cmislib.Document` object. Can be used for accessing + #: properties of the document. See example above. + self.cmis_object = None + if not name.startswith('/'): + name = '/' + name + self._name = name + + self._should_download = 'r' in mode or 'a' in mode + self._has_changed = 'w' in mode + + self._open(mode, encoding) + + def _open(self, mode, encoding): + self._make_temporary_directory() + + if self._should_download: + self._download_or_mark_changed(mode) + + self._stream = io.open( + self._temporary_filename, + mode=mode, + encoding=encoding + ) + + def close(self): + if not self.closed: + self._stream.close() + if self._has_changed: + self._upload() + self._remove_temporary_directory() + + @property + def name(self): + return self._name + + def read(self): + return self._stream.read() + + def write(self, data): + self._has_changed = True + self._stream.write(data) + + def writelines(self, lines): + self._has_changed = True + self._stream.writelines(lines) + + closed = property(lambda self: self._stream.closed) + encoding = property(lambda self: self._stream.encoding) + fileno = property(lambda self: self._stream.fileno) + flush = property(lambda self: self._stream.flush) + isatty = property(lambda self: self._stream.isatty) + mode = property(lambda self: self._stream.mode) + readable = property(lambda self: self._stream.readable) + readall = property(lambda self: self._stream.readall) + readinto = property(lambda self: self._stream.readinto) + readline = property(lambda self: self._stream.readline) + readlines = property(lambda self: self._stream.readlines) + seekable = property(lambda self: self._stream.seekable) + tell = property(lambda self: self._stream.tell) + writable = property(lambda self: self._stream.writable) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def __iter__(self): + return iter(self._stream) + + def __repr__(self): + args = [ + ('storage', self.storage), + ('name', self.name), + ('mode', self.mode), + ] + if hasattr(self, 'encoding'): + args.append(('encoding', self.encoding)) + args = ', '.join( + '{key}={value!r}'.format(key=key, value=value) + for key, value in args + ) + return ''.format(args=args) + + def _make_temporary_directory(self): + self._temporary_directory = tempfile.mkdtemp() + + def _remove_temporary_directory(self): + shutil.rmtree(self._temporary_directory) + + @property + def _temporary_filename(self): + return os.path.join( + self._temporary_directory, + os.path.basename(self.name) + ) + + def _download_or_mark_changed(self, mode): + try: + self._download() + except FileNotFoundError: + if 'a' in mode: + self._has_changed = True + else: + raise + + def _download(self): + with io.open(self._temporary_filename, mode='wb') as f: + self.cmis_obj = self.storage._get_object(self.name) + for data in self.cmis_obj.getContentStream(): + f.write(data) + + def _upload(self): + with io.open(self._temporary_filename, mode='rb') as f: + try: + self.cmis_obj = self.storage._get_object(self.name) + self.cmis_obj.setContentStream(f) + except FileNotFoundError: + obj = self.storage.repository.rootFolder + dirpath, filename = os.path.split(self.name) + for dirname in dirpath.split('/'): + if dirname: + folder_obj = None + rs = obj.getTree() + for d in rs.getResults(): + if d.getName() == dirname: + folder_obj = d + break + if not folder_obj: + obj = obj.createFolder(dirname) + else: + obj = folder_obj + self.cmis_obj = obj.createDocument(filename, contentFile=f) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3100ded --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,5 @@ +import sys + +collect_ignore = ["setup.py"] +if sys.version_info[0] > 2: + collect_ignore.append("storages/test_cmis.py") diff --git a/tests/storages/test_cmis.py b/tests/storages/test_cmis.py new file mode 100644 index 0000000..d5a2d09 --- /dev/null +++ b/tests/storages/test_cmis.py @@ -0,0 +1,351 @@ +# -*- coding: utf-8 -*- +import locale +import os +try: + from unittest import mock +except ImportError: + import mock + +from cmislib import Repository +from cmislib.exceptions import ObjectNotFoundException, RuntimeException +import pytest + +from siilo.exceptions import FileNotFoundError + + +@pytest.fixture +def repository(): + return mock.MagicMock(name='repository', spec=Repository) + + +@pytest.fixture +def storage(repository): + from siilo.storages.cmis import CmisStorage + return CmisStorage(repository=repository) + + +@pytest.fixture +def object_does_not_exist(): + return ObjectNotFoundException( + status=mock.sentinel.status, + url=mock.sentinel.url, + ) + + +@pytest.fixture +def object_does_not_exist_on_delete(): + return RuntimeException( + status=mock.sentinel.status, + url=mock.sentinel.url, + ) + + +def test_storage_repr(storage, repository): + expected = ''.format(repository) + assert repr(storage) == expected + + +def test_delete_removes_the_file(storage, repository): + storage.delete('some_file.txt') + repository.getObjectByPath.assert_called_with('some_file.txt') + obj = repository.getObjectByPath('some_file.txt') + assert obj.delete.called + + +def test_delete_raises_error_if_file_doesnt_exist_on_get_object( + storage, repository, object_does_not_exist +): + repository.getObjectByPath.side_effect = object_does_not_exist + with pytest.raises(FileNotFoundError) as excinfo: + storage.delete('some_file.txt') + assert excinfo.value.name == 'some_file.txt' + + +def test_delete_raises_error_if_file_doesnt_exist_on_delete( + storage, repository, object_does_not_exist_on_delete +): + obj = repository.getObjectByPath('some_file.txt') + obj.delete.side_effect = object_does_not_exist_on_delete + with pytest.raises(FileNotFoundError) as excinfo: + storage.delete('some_file.txt') + assert excinfo.value.name == 'some_file.txt' + + +def test_exists_returns_true_if_file_exists(storage, repository): + assert storage.exists('some_file.txt') is True + repository.getObjectByPath.assert_called_with('some_file.txt') + + +def test_size_returns_file_size_in_bytes(storage, repository): + expected_size = 4 + obj = repository.getObjectByPath('some_file.txt') + obj.properties.get.return_value = expected_size + assert storage.size('some_file.txt') == expected_size + + +def test_size_raises_error_if_file_doesnt_exist( + storage, repository, object_does_not_exist +): + repository.getObjectByPath.side_effect = object_does_not_exist + with pytest.raises(FileNotFoundError) as excinfo: + storage.size('some_file.txt') + assert excinfo.value.name == 'some_file.txt' + + +def test_url_raises_not_implemented(storage, repository): + with pytest.raises(NotImplementedError) as excinfo: + storage.url('some_file.txt') + assert excinfo.type == NotImplementedError + + +def test_open_returns_cmis_file_with_default_mode_and_encoding(storage): + with mock.patch( + 'siilo.storages.cmis.CmisFile' + ) as MockFile: + MockFile.return_value = mock.sentinel.file + file_ = storage.open('some_file.txt') + MockFile.assert_called_with( + storage=storage, + name='some_file.txt', + mode='r', + encoding=None + ) + assert file_ is mock.sentinel.file + + +def test_open_returns_cmis_file_with_given_mode_and_encoding(storage): + with mock.patch( + 'siilo.storages.cmis.CmisFile' + ) as MockFile: + MockFile.return_value = mock.sentinel.file + file_ = storage.open('some_file.txt', 'w', 'utf-8') + MockFile.assert_called_with( + storage=storage, + name='some_file.txt', + mode='w', + encoding='utf-8' + ) + assert file_ is mock.sentinel.file + + +@pytest.mark.parametrize( + 'mode', ['a', 'a+', 'a+b', 'ab', 'r', 'r+', 'r+b', 'rb'] +) +def test_downloads_file_for_read_and_append_modes(storage, repository, mode): + contents = b'Quick brown fox jumps over lazy dog' + + obj = repository.getObjectByPath('/some_file.txt') + obj.getContentStream.return_value = iter([contents]) + + with storage.open('some_file.txt', mode) as file_: + with open(file_._temporary_filename, 'rb') as tempfile: + assert tempfile.read() == contents + + +@pytest.mark.parametrize('mode', ['w', 'wb', 'w+', 'w+b']) +def test_doesnt_download_file_for_write_modes(storage, repository, mode): + with storage.open('some_file.txt', mode): + assert not repository.getObjectByPath.called + # when closing it gets downloaded because it has to check if the file + # exists in order to change the contents, creating a new version + assert repository.getObjectByPath.called + + +@pytest.mark.parametrize( + ('mode', 'encoding'), + [ + ('r', None), + ('w', 'UTF-8'), + ] +) +def test_opens_temporary_file_with_given_mode_and_encoding( + storage, mode, encoding +): + with storage.open('some_file.txt', mode, encoding) as file_: + assert file_._stream.name == file_._temporary_filename + assert file_._stream.mode == mode + assert file_._stream.encoding == encoding or locale.getdefaultlocale() + + +@pytest.mark.parametrize( + 'filename', + [ + '/etc/passwd', + '../etc/passwd', + ] +) +def test_always_opens_temporary_file_within_the_temporary_directory( + storage, filename +): + with storage.open(filename) as file_: + assert file_._temporary_filename == os.path.abspath( + file_._temporary_filename + ) + assert file_._temporary_filename.startswith(file_._temporary_directory) + + +def test_removes_temporary_directory_after_file_is_closed(storage, repository): + with storage.open('some_file.txt', 'r') as file_: + pass + assert not os.path.exists(file_._temporary_directory) + + +@pytest.mark.parametrize( + ('method_name', 'method_args', 'method_returns'), + [ + ('fileno', [], True), + ('flush', [], False), + ('isatty', [], True), + ('read', [], True), + ('readable', [], True), + ('readall', [], True), + ('readinto', [], True), + ('readline', [], True), + ('readlines', [], True), + ('seekable', [], True), + ('tell', [], False), + ('writable', [], True), + ('write', ['foo'], False), + ('writelines', [['foo', 'bar']], False), + ] +) +def test_delegates_file_api_methods_to_underlying_temporary_file( + storage, method_name, method_args, method_returns +): + with storage.open('some_file.txt', 'r') as file_: + file_._stream = mock.MagicMock(name='stream') + + method = getattr(file_, method_name) + rv = method(*method_args) + + stream_method = getattr(file_._stream, method_name) + stream_method.assert_called_with(*method_args) + if method_returns: + assert rv == stream_method(*method_args) + + +@pytest.mark.parametrize( + 'property_name', + [ + 'closed', + 'encoding', + 'mode', + ] +) +def test_delegates_file_api_properties_to_underlying_temporary_file( + storage, property_name +): + with storage.open('some_file.txt', 'r') as file_: + file_._stream = mock.MagicMock(name='stream') + + actual_value = getattr(file_, property_name) + expected_value = getattr(file_._stream, property_name) + + assert actual_value == expected_value + + +def test_reclosing_file_is_noop(storage): + with storage.open('some_file.txt', 'r') as file_: + pass + file_.close() + + +def test_can_iterate_over_file(storage, repository): + contents = b'Quick brown fox\njumps over lazy dog' + + obj = repository.getObjectByPath('some_file.txt') + obj.getContentStream.return_value = iter([contents]) + + with storage.open('some_file.txt', 'r') as file_: + lines = list(line for line in file_) + + assert lines == [ + u'Quick brown fox\n', + u'jumps over lazy dog', + ] + + +@pytest.mark.parametrize( + ('mode', 'encoding', 'expected_format'), + [ + ( + 'r', + 'UTF-8', + ( + '' + ) + ), + ( + 'rb', + None, + '' + ), + ] +) +def test_cmisfile_repr(storage, mode, encoding, expected_format): + with storage.open('some_file.txt', mode, encoding) as file_: + assert repr(file_) == expected_format.format( + storage=storage, + name=file_.name, + mode=file_.mode, + encoding=encoding + ) + + +@pytest.mark.parametrize( + 'mode', + ['a', 'a+', 'a+b', 'ab', 'w', 'wb', 'w+', 'w+b'] +) +def test_uploads_file_opened_in_write_mode_but_new_for_storage( + storage, repository, mode, object_does_not_exist +): + repository.getObjectByPath.side_effect = object_does_not_exist + with mock.patch('siilo.storages.cmis.io.open') as mock_open: + mock_open.return_value = mock.MagicMock(closed=False) + with storage.open('some_file.txt', mode) as file_: + pass + with mock_open(file_._temporary_filename, mode='rb') as temp_file: + repository.rootFolder.createDocument.assert_called_with( + 'some_file.txt', contentFile=temp_file) + + +@pytest.mark.parametrize( + 'mode', + ['a', 'a+', 'a+b', 'ab', 'w', 'wb', 'w+', 'w+b'] +) +def test_uploads_file_with_subfolders_opened_in_write_mode_but_new_for_storage( + storage, repository, mode, object_does_not_exist +): + repository.getObjectByPath.side_effect = object_does_not_exist + with mock.patch('siilo.storages.cmis.io.open') as mock_open: + mock_open.return_value = mock.MagicMock(closed=False) + with storage.open('/folder/some_file.txt', mode) as file_: + pass + with mock_open(file_._temporary_filename, mode='rb') as temp_file: + repository.rootFolder.createFolder.assert_called_with('folder') + repository.rootFolder.createFolder.return_value.createDocument \ + .assert_called_with('some_file.txt', contentFile=temp_file) + + +@pytest.mark.parametrize( + 'mode', + ['a', 'a+', 'a+b', 'ab', 'r+', 'r+b', 'w', 'wb', 'w+', 'w+b'] +) +@pytest.mark.parametrize( + ('method_name', 'method_args'), + [ + ('write', 'foo'), + ('writelines', ['foo', 'bar']), + ] +) +def test_uploads_file_already_on_storage( + storage, repository, mode, method_name, method_args +): + with mock.patch('siilo.storages.cmis.io.open') as mock_open: + mock_open.return_value = mock.MagicMock(closed=False) + with storage.open('some_file.txt', mode) as file_: + method = getattr(file_, method_name) + method(method_args) + with mock_open(file_._temporary_filename, mode='rb') as temp_file: + file_.cmis_obj.setContentStream.assert_called_with(temp_file) diff --git a/tox.ini b/tox.ini index ee59c96..e996c49 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ deps = mock pytest pytest-cov + cmislib commands = py.test {posargs} [testenv:py33] @@ -16,6 +17,7 @@ deps = freezegun>=0.1.12 pytest pytest-cov + cmislib [testenv:lint] deps = flake8 @@ -25,5 +27,6 @@ commands = flake8 siilo/ tests/ basepython = python deps = Sphinx + cmislib changedir = docs commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html