Merge pull request #506 from Sage-Bionetworks/develop

1.7.4 Release to master

Author: zimingd

.synapseConfig

@@ -0,0 +1,62 @@
+
+###########################
+# Login Credentials       #
+###########################
+
+## Used for logging in to Synapse
+## you may also specify an apikey instead of password. If both password and apikey are specified, the apikey is ignored
+## Alternatively you can use rememberMe=True in synapseclient.login or login subcommand of the commandline client to 
+## cache your API key elsewhere.
+#[authentication]
+#username = <username>
+#password = <password>
+#apikey = <apikey>
+
+
+
+## If you have projects with file stored on SFTP servers, you can specify your credentials here
+## You can specify multiple sftp credentials
+#[sftp://some.sftp.url.com]
+#username= <sftpuser>
+#password= <sftppwd>
+#[sftp://a.different.sftp.url.com]
+#username= <sftpuser>
+#password= <sftppwd>
+
+
+## If you have projects that need to be stored in an S3-like (e.g. AWS S3, Openstack) storage but cannot allow Synapse
+## to manage access your storage you may put your credentials here.
+## To avoid duplicating credentials with that used by the AWS Command Line Client,
+## simply put the profile name form your ~/.aws/credentials file
+## more information about aws credentials can be found here http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html
+#[https://s3.amazonaws.com/bucket_name] # this is the bucket's endpoint
+#profile_name=local_credential_profile_name
+
+
+###########################
+# Caching                 #
+###########################
+## your downloaded files are cached to avoid repeat downloads of the same file. change 'location' to use a different folder on your computer as the cache location
+#[cache]
+#location = ~/.synapseCache
+
+
+###########################
+# Advanced Configurations #
+###########################
+
+## If this section is specified, then the synapseclient will print out debug information
+#[debug]
+
+## Some integration tests require a second Synapse user. This should only be necessary for developers
+#[test-authentication]
+#username = <username>
+#password = <password>
+#principalid = <userId>
+
+## Configuring these will cause the Python client to use these as Synapse service endpoints instead of the default prod endpoints.
+#[endpoints]
+#repoEndpoint=<repoEndpoint>
+#authEndpoint=<authEndpoint>
+#fileHandleEndpoint=<fileHandleEndpoint>
+#portalEndpoint=<portalEndpoint>

.travis.yml

@@ -1,17 +1,49 @@
 sudo: false
 language: python
-python:
-- '2.7'
-- '3.4'
-- '3.5'
+
 script:
 - python setup.py install
 - nosetests -vs tests/unit
 - 'if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then nosetests -vs tests/integration; else echo "Skipping integration tests"; fi'
-before_install:
-- 'if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then openssl aes-256-cbc -K $encrypted_d17283647768_key -iv $encrypted_d17283647768_iv
-  -in test.synapseConfig.enc -out test.synapseConfig -d; mv test.synapseConfig ~/.synapseConfig; openssl aes-256-cbc -K $encrypted_d17283647768_key -iv $encrypted_d17283647768_iv
-  -in test.awscredentials.enc -out test.awscredentials -d; mkdir -p ~/.aws && mv test.awscredentials ~/.aws/credentials;fi'
-- travis_wait pip install pysftp cython pandas boto3
+
+before_install: |
+  if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+    brew update
+    brew outdated pyenv || brew upgrade pyenv
+    brew install pyenv-virtualenv
+    pyenv install $PY_VERSION
+    export PYENV_VERSION=$PY_VERSION
+    export PATH="/Users/travis/.pyenv/shims:${PATH}"
+    pyenv-virtualenv venv
+    source venv/bin/activate
+    pip install nose mock
+  fi
+
+  if [[ "${TRAVIS_PULL_REQUEST}" = "false" ]]; then
+    openssl aes-256-cbc -K $encrypted_d17283647768_key -iv $encrypted_d17283647768_iv -in test.synapseConfig.enc -out test.synapseConfig -d
+    mv test.synapseConfig ~/.synapseConfig
+    openssl aes-256-cbc -K $encrypted_d17283647768_key -iv $encrypted_d17283647768_iv -in test.awscredentials.enc -out test.awscredentials -d
+    mkdir -p ~/.aws && mv test.awscredentials ~/.aws/credentials
+  fi
+  travis_wait pip install pysftp cython pandas boto3
+
 addons:
   ssh_known_hosts: ec2-54-212-85-156.us-west-2.compute.amazonaws.com
+
+matrix:
+  include:
+    - os: linux
+      python: 2.7.14
+    - os: linux
+      python: 3.5.3
+    - os: linux
+      python: 3.6.2
+    - os: osx
+      language: generic
+      env: PY_VERSION=2.7.14
+    - os: osx
+      language: generic
+      env: PY_VERSION=3.5.3
+    - os: osx
+      language: generic
+      env: PY_VERSION=3.6.2

appveyor.yml

@@ -0,0 +1,26 @@
+build: off
+
+environment:
+    matrix:
+        - PYTHON: "C:\\Python27-x64"
+        - PYTHON: "C:\\Python36-x64"
+
+install:
+    - "%PYTHON%\\python.exe -m pip install nose mock pysftp cython pandas boto3"
+    - "cd %APPVEYOR_BUILD_FOLDER%"
+    - "%PYTHON%\\python.exe setup.py install"
+
+before_test:
+    - IF NOT DEFINED APPVEYOR_PULL_REQUEST_NUMBER ( ssh-keyscan -t ecdsa ec2-54-212-85-156.us-west-2.compute.amazonaws.com >> %userprofile%\\.ssh\\known_hosts )
+
+test_script:
+    - cmd: "%PYTHON%\\python.exe -m nose -vs tests\\unit"
+    - ps: |
+        if ( -Not $env:APPVEYOR_PULL_REQUEST_NUMBER ){
+          openssl aes-256-cbc -K $env:encrypted_d17283647768_key -iv $env:encrypted_d17283647768_iv -in test.synapseConfig.enc -out test.synapseConfig -d
+          move test.synapseConfig ${env:userprofile}\\.synapseConfig
+          openssl aes-256-cbc -K $env:encrypted_d17283647768_key -iv $env:encrypted_d17283647768_iv -in test.awscredentials.enc -out test.awscredentials -d
+          mkdir ${env:userprofile}\\.aws
+          move test.awscredentials ${env:userprofile}\\.aws\\credentials
+        }
+    - cmd: "IF NOT DEFINED APPVEYOR_PULL_REQUEST_NUMBER ( %PYTHON%\\python.exe -m nose -vs tests\\integration ) ELSE ( ECHO Skipping integration tests )"

doc/Entity.rst

@@ -1 +1 @@
-.. automodule:: synapseclient.entity
+.. automodule:: synapseclient.entity

doc/Team.rst

@@ -0,0 +1,3 @@
+.. automodule:: synapseclient.team
+   :members:
+   

doc/synapseutils.rst

@@ -18,9 +18,9 @@ sync
 .. automodule:: synapseutils.sync
    :members:
 
-****
+*******
 monitor
-****
+*******
 .. automodule:: synapseutils.monitor
    :members:
 

release_notes/release-1.7.md

@@ -1,12 +1,72 @@
 Release Notes - Synapse Python Client - Version py-1.7
 
+**Install Instructions:** `pip install --upgrade synapseclient` or see [http://docs.synapse.org/python/#installation](http://docs.synapse.org/python/#installation)  
+**Documentation:** [http://docs.synapse.org/python/](http://docs.synapse.org/python/)
+
+# Release 1.7.4
+
+**Release Date:** 29-January-2018
+
+Release 1.7.4 
+
+This release mostly includes bugfixes and improvements for various Table classes:
+ * Fixed bug where you couldn't store a table converted to a `pandas.Dataframe` if it had a INTEGER column with some missing values.
+ * `EntityViewSchema` can now automatically add all annotations within your defined `scopes` as columns. Just set the view's `addAnnotationColumns=True` before calling `syn.store()`. This attribute defaults to `True` for all newly created `EntityViewSchemas`. Setting `addAnnotationColumns=True` on existing tables will only add annotation columns that are not already a part of your schema.
+ * You can now use `synapseutils.notifyMe` as a decorator to notify you by email when your function has completed. You will also be notified of any Errors if they are thrown while your function runs.
+
+We also added some new features:
+ * `syn.findEntityId()` function that allows you to find an Entity by its name and parentId, set parentId to `None` to search for Projects by name.
+ * The bulk upload functionality of `synapseutils.syncToSynapse` is avaliable from the command line using: `synapse sync`.
+
+Below are the full list of issues addressed by this release:
+
+
+## New Feature
+
+*   [[SYNPY-506](https://sagebionetworks.jira.com/browse/SYNPY-506)] - need convenience function for /entity/child
+*   [[SYNPY-517](https://sagebionetworks.jira.com/browse/SYNPY-517)] - sync command line
 
-# Release 1.7.3
 
+## Improvement
+
+*   [[SYNPY-267](https://sagebionetworks.jira.com/browse/SYNPY-267)] - Update Synapse tables for integer types
+*   [[SYNPY-304](https://sagebionetworks.jira.com/browse/SYNPY-304)] - Table objects should implement len()
+*   [[SYNPY-416](https://sagebionetworks.jira.com/browse/SYNPY-416)] - warning message for recursive get when a non-Project of Folder entity is passed
+*   [[SYNPY-482](https://sagebionetworks.jira.com/browse/SYNPY-482)] - Create a sample synapseConfig if none is present
+*   [[SYNPY-489](https://sagebionetworks.jira.com/browse/SYNPY-489)] - Add a boolean paramter in EntityViewSchema that will indicate whether the client should create columns based on annotations in the specified scopes
+*   [[SYNPY-494](https://sagebionetworks.jira.com/browse/SYNPY-494)] - Link should be able to take an entity object as the parameter and derive its id
+*   [[SYNPY-511](https://sagebionetworks.jira.com/browse/SYNPY-511)] - improve exception handling
+*   [[SYNPY-512](https://sagebionetworks.jira.com/browse/SYNPY-512)] - Remove the use of PaginatedResult's totalNumberOfResult
+*   [[SYNPY-539](https://sagebionetworks.jira.com/browse/SYNPY-539)] - When creating table Schemas, enforce a limit on the number of columns that can be created.
+
+
+## Bug
+
+*   [[SYNPY-235](https://sagebionetworks.jira.com/browse/SYNPY-235)] - can't print Row objects with dates in them
+*   [[SYNPY-272](https://sagebionetworks.jira.com/browse/SYNPY-272)] - bug syn.storing rowsets containing Python datetime objects
+*   [[SYNPY-297](https://sagebionetworks.jira.com/browse/SYNPY-297)] - as_table_columns shouldn't give fractional max size
+*   [[SYNPY-404](https://sagebionetworks.jira.com/browse/SYNPY-404)] - when we get a SynapseMd5MismatchError we should delete the downloaded file
+*   [[SYNPY-425](https://sagebionetworks.jira.com/browse/SYNPY-425)] - onweb doesn't work for tables
+*   [[SYNPY-438](https://sagebionetworks.jira.com/browse/SYNPY-438)] - Need to change 'submit' not to use evaluation/id/accessRequirementUnfulfilled
+*   [[SYNPY-496](https://sagebionetworks.jira.com/browse/SYNPY-496)] - monitor.NotifyMe can not be used as an annotation decorator
+*   [[SYNPY-521](https://sagebionetworks.jira.com/browse/SYNPY-521)] - inconsistent error message when username/password is wrong on login
+*   [[SYNPY-536](https://sagebionetworks.jira.com/browse/SYNPY-536)] - presigned upload URL expired warnings using Python client sync function
+*   [[SYNPY-555](https://sagebionetworks.jira.com/browse/SYNPY-555)] - EntityViewSchema is missing from sphinx documentation
+*   [[SYNPY-558](https://sagebionetworks.jira.com/browse/SYNPY-558)] - synapseutils.sync.syncFromSynapse throws error when syncing a Table object
+*   [[SYNPY-595](https://sagebionetworks.jira.com/browse/SYNPY-595)] - Get recursive folders filled with Links fails
+*   [[SYNPY-605](https://sagebionetworks.jira.com/browse/SYNPY-605)] - Update documentation for getUserProfile to include information about refreshing and memoization
+
+
+## Task
+
+*   [[SYNPY-451](https://sagebionetworks.jira.com/browse/SYNPY-451)] - Add limit and offset for accessApproval and accessRequirement API calls and remove 0x400 flag default when calling GET /entity/{id}/bundle
+*   [[SYNPY-546](https://sagebionetworks.jira.com/browse/SYNPY-546)] - Change warning message when user does not DOWNLOAD permissions.
+
+
+
+# Release 1.7.3
 
 **Release Date:** 08-December-2017  
-**Install Instructions:** `pip install --upgrade synapseclient` or see [http://docs.synapse.org/python/#installation](http://docs.synapse.org/python/#installation)  
-**Documentation:** [http://docs.synapse.org/python/](http://docs.synapse.org/python/)
 
 Release 1.7.3 introduces fixes and quality of life changes to Tables and synapseutils:
 * Changes to Tables:
@@ -52,9 +112,7 @@ Below are the full list of issues addressed by this release:
 
 # Release 1.7.1
 
-**Release Date:** 17-June-2017  
-**Install Instructions:** `pip install --upgrade synapseclient` or see [http://docs.synapse.org/python/#installation](http://docs.synapse.org/python/#installation)  
-**Documentation:** [http://docs.synapse.org/python/](http://docs.synapse.org/python/)
+**Release Date:** 17-June-2017
 
 Release 1.7 is a large bugfix release with several new features. The main ones include:
 * We have expanded the [syanpaseutils packages](docs.synapse.org/python/synapseutils.html#module-synapseutils) to add the  abilitity to:

setup.py

@@ -1,14 +1,15 @@
 ## Installation script for Synapse Client for Python
 ############################################################
 import sys
+from os.path import expanduser, exists
 
 ## check Python version, before we do anything
 if sys.version_info < (2, 7, 0):
     sys.stderr.write("The Synapse Client for Python requires Python 2.7 or 3.4 or higher.\n")
     sys.stderr.write("Your Python appears to be version %d.%d.%d\n" % sys.version_info[:3])
     sys.exit(-1)
 
-from setuptools import setup
+from setuptools import setup, find_packages
 import json
 
 description = """A client for Synapse, a collaborative compute space 
@@ -24,6 +25,9 @@
 
 __version__=json.loads(open('synapseclient/synapsePythonClient').read())['latestVersion']
 
+#make sure not to overwrite existing .synapseConfig with our example one
+data_files = [(expanduser('~'), ['.synapseConfig'])] if not exists(expanduser('~/.synapseConfig')) else []
+
 setup(name='synapseclient',
     version=__version__,
     description=description,
@@ -33,7 +37,7 @@
     author='Synapse Team',
     author_email='[email protected]',
     license='Apache',
-    packages=['synapseclient','synapseutils'],
+    packages=find_packages(),
     install_requires=[
         'requests>=1.2',
         'six',
@@ -52,12 +56,13 @@
     },
     zip_safe=False,
     package_data={'synapseclient': ['synapsePythonClient']},
+    data_files=data_files,
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Intended Audience :: Developers',
         'Intended Audience :: Science/Research',
         'License :: OSI Approved :: Apache Software License',

synapseclient/__init__.py

@@ -320,6 +320,7 @@
 from .version_check import release_notes
 
 import json
+from . import custom_json
 import pkg_resources
 __version__ = json.loads(pkg_resources.resource_string('synapseclient', 'synapsePythonClient').decode())['latestVersion']
 

synapseclient/__main__.py

@@ -167,6 +167,11 @@ def get(args, syn):
 
         print('Creating %s' % entity.path)
 
+def sync(args, syn):
+    synapseutils.syncToSynapse(syn, manifestFile=args.manifestFile,
+                               dryRun=args.dryRun, sendMessages=args.sendMessages,
+                               retries=args.retries)
+
 def store(args, syn):
     #If we are storing a fileEntity we need to have id or parentId
     if args.parentid is None and args.id is None and args.file is not None:
@@ -508,6 +513,17 @@ def build_parser():
             help='Synapse ID of form syn123 of desired data object.')
     parser_get.set_defaults(func=get)
 
+    parser_sync = subparsers.add_parser('sync',
+                                        help='Synchronize files described in a manifest to Synapse')
+    parser_sync.add_argument('--dryRun', action='store_true', default=False,
+                             help='Perform validation without uploading.')
+    parser_sync.add_argument('--sendMessages', action='store_true', default=False,
+                             help='Send notifications via Synapse messaging (email) at specific intervals, on errors and on completion.')
+    parser_sync.add_argument('--retries', metavar='INT', type=int, default=4)
+    parser_sync.add_argument('manifestFile',  metavar='FILE', type=str,
+                             help='A tsv file with file locations and metadata to be pushed to Synapse.')
+    parser_sync.set_defaults(func=sync)
+
     parser_store = subparsers.add_parser('store', #Python 3.2+ would support alias=['store']
             help='uploads and adds a file to Synapse')
     parent_id_group = parser_store.add_mutually_exclusive_group(required=True)