Merge pull request #668 from jupyter-incubator/master

Release from master

Author: itamarst

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.16.0
+
+### Bug fixes
+
+* Fix ContextualVersionConflict in Dockerfile.spark. Thanks Linan Zheng, @LinanZheng
+* Fix Info Subcommand in RemoteSparkMagic. Thanks Linan Zheng, @LinanZheng
+* Fix to ignore unsupported spark session types whilst fetching the session list. Thanks, Murat Burak Migdisoglu, @mmigdiso
+
 ## 0.15.0
 
 ### Features

Dockerfile.spark

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -yq --no-install-recommends --force-yes \
     r-base-core && \
     rm -rf /var/lib/apt/lists/*
 
+# Install pip for Python3
 RUN easy_install3 pip py4j
 RUN pip install --upgrade setuptools
 
@@ -30,9 +31,19 @@ RUN mkdir -p /apps/build && \
   cp -r /apps/build/spark/dist $SPARK_HOME && \
   rm -rf $SPARK_BUILD_PATH
 
+# ----------
+# Build Livy
+# ----------
 ENV LIVY_BUILD_VERSION 0.6.0-incubating
 ENV LIVY_APP_PATH /apps/apache-livy-$LIVY_BUILD_VERSION-bin
 ENV LIVY_BUILD_PATH /apps/build/livy
+
+# Install setuptools for Python 2.7 for Livy
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python get-pip.py --user && \
+    rm get-pip.py && \
+    python -m pip install --upgrade setuptools
+
 RUN cd /apps/build && \
 	git clone https://github.com/apache/incubator-livy.git livy && \
 	cd $LIVY_BUILD_PATH && \

README.md

@@ -37,9 +37,9 @@ The sparkmagic library provides a %%spark magic that you can use to easily run c
 The sparkmagic library also provides a set of Scala and Python kernels that allow you to automatically connect to a remote Spark cluster, run code and SQL queries, manage your Livy server and Spark job configuration, and generate automatic visualizations.
 See [Pyspark](examples/Pyspark%20Kernel.ipynb) and [Spark](examples/Spark%20Kernel.ipynb) sample notebooks.
 
-### 3. Sending data to Spark%20Kernel
+### 3. Sending local data to Spark Kernel
 
-See the [sending data to Spark notebook](examples/Send local data to Spark.ipynb).
+See the [Sending Local Data to Spark notebook](examples/Send%20local%20data%20to%20Spark.ipynb).
 
 ## Installation
 
@@ -53,7 +53,7 @@ See the [sending data to Spark notebook](examples/Send local data to Spark.ipynb
 
 3. If you're using JupyterLab, you'll need to run another command:
 
-        jupyter labextension install @jupyter-widgets/jupyterlab-manager
+        jupyter labextension install "@jupyter-widgets/jupyterlab-manager"
 
 4. (Optional) Install the wrapper kernels. Do `pip show sparkmagic` and it will show the path where `sparkmagic` is installed at. `cd` to that location and do:
 

autovizwidget/autovizwidget/__init__.py

@@ -1 +1 @@
-__version__ = '0.15.0'
+__version__ = '0.16.0'

hdijupyterutils/hdijupyterutils/__init__.py

@@ -1 +1 @@
-__version__ = '0.15.0'
+__version__ = '0.16.0'

hdijupyterutils/requirements.txt

@@ -7,3 +7,5 @@ jupyter>=1
 pandas>=0.17.1
 numpy
 notebook>=4.2
+# Work around broken-on-Python-2 pyrsistent release:
+pyrsistent < 0.17 ; python_version < '3.0'

hdijupyterutils/setup.py

@@ -64,5 +64,7 @@ def version(path):
           'jupyter>=1',
           'pandas>=0.17.1',
           'numpy',
-          'notebook>=4.2'
+          'notebook>=4.2',
+          # Work around broken-on-Python-2 pyrsistent release:
+          "pyrsistent < 0.17 ; python_version < '3.0'",
       ])

sparkmagic/sparkmagic/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.15.0'
+__version__ = '0.16.0'
 
 from sparkmagic.serverextension.handlers import load_jupyter_server_extension
 

sparkmagic/sparkmagic/kernels/kernelmagics.py

@@ -138,7 +138,7 @@ def help(self, line, cell="", local_ns=None):
   </tr>
   <tr>
     <td>send_to_spark</td>
-    <td>%%send_to_spark -o variable -t str -n var</td>
+    <td>%%send_to_spark -i variable -t str -n var</td>
     <td>Sends a variable from local output to spark cluster.
     <br/>
     Parameters:

sparkmagic/sparkmagic/livyclientlib/sparkcontroller.py

@@ -46,9 +46,10 @@ def run_sqlquery(self, sqlquery, client_name=None):
     def get_all_sessions_endpoint(self, endpoint):
         http_client = self._http_client(endpoint)
         sessions = http_client.get_sessions()[u"sessions"]
+        supported_sessions = filter(lambda s: (s[constants.LIVY_KIND_PARAM] in constants.SESSION_KINDS_SUPPORTED), sessions)
         session_list = [self._livy_session(http_client, {constants.LIVY_KIND_PARAM: s[constants.LIVY_KIND_PARAM]},
                                            self.ipython_display, s[u"id"])
-                        for s in sessions]
+                        for s in supported_sessions]
         for s in session_list:
             s.refresh_status_and_info()
         return session_list

sparkmagic/sparkmagic/magics/remotesparkmagics.py

@@ -119,10 +119,10 @@ def spark(self, line, cell="", local_ns=None):
 
         # info
         if subcommand == "info":
-            if args.url is not None:
+            if args.url is not None and args.id is not None:
                 endpoint = Endpoint(args.url, args.auth, args.user, args.password)
                 info_sessions = self.spark_controller.get_all_sessions_endpoint_info(endpoint)
-                self._print_endpoint_info(info_sessions)
+                self._print_endpoint_info(info_sessions, args.id)
             else:
                 self._print_local_info()
         # config

sparkmagic/sparkmagic/tests/test_remotesparkmagics.py

@@ -45,12 +45,12 @@ def test_info_command_parses():
 def test_info_endpoint_command_parses():
     print_info_mock = MagicMock()
     magic._print_endpoint_info = print_info_mock
-    command = "info -u http://microsoft.com"
+    command = "info -u http://microsoft.com -i 1234"
     spark_controller.get_all_sessions_endpoint_info = MagicMock(return_value=None)
 
     magic.spark(command)
 
-    print_info_mock.assert_called_once_with(None)
+    print_info_mock.assert_called_once_with(None,1234)
 
 
 @with_setup(_setup, _teardown)

sparkmagic/sparkmagic/tests/test_sparkcontroller.py

@@ -132,8 +132,9 @@ def test_get_client_keys():
 @with_setup(_setup, _teardown)
 def test_get_all_sessions():
     http_client = MagicMock()
-    http_client.get_sessions.return_value = json.loads('{"from":0,"total":2,"sessions":[{"id":0,"state":"idle","kind":'
+    http_client.get_sessions.return_value = json.loads('{"from":0,"total":3,"sessions":[{"id":0,"state":"idle","kind":'
                                                        '"spark","log":[""]}, {"id":1,"state":"busy","kind":"spark","log"'
+                                                       ':[""]},{"id":2,"state":"busy","kind":"sql","log"'
                                                        ':[""]}]}')
     controller._http_client = MagicMock(return_value=http_client)
     controller._livy_session = MagicMock()