Merge branch 'develop'

Author: ktbyers

Diff for: Dockerfile

@@ -0,0 +1,15 @@
+FROM debian:stretch
+
+## Install min deps
+RUN apt-get update
+
+COPY ./ /var/cache/napalm/
+
+## Install NAPALM & underlying libraries dependencies
+RUN apt-get install -y python-cffi python-dev libxslt1-dev libssl-dev libffi-dev \
+    && apt-get install -y python-pip \
+    && pip install -U cffi \
+    && pip install -U cryptography \
+    && pip install /var/cache/napalm/
+
+RUN rm -rf /var/lib/apt/lists/*

Diff for: docs/index.rst

@@ -22,9 +22,7 @@ Supported Network Operating Systems:
 extras
 ______
 
-In addition to the core drivers napalm also supports community driven drivers. You can find more information about them here:
-
-TBD Link to extras/core drivers' documentation
+In addition to the core drivers napalm also supports community driven drivers. You can find more information about them here: :ref:`contributing-drivers`
 
 Selecting the right driver
 --------------------------

Diff for: docs/support/index.rst

@@ -109,24 +109,24 @@ NAPALM supports passing certain optional arguments to some drivers. To do that y
 List of supported optional arguments
 ____________________________________
 
-* :code:`allow_agent` (ios, iosxr) - Paramiko argument, enable connecting to the SSH agent (default: ``False``).
-* :code:`alt_host_keys` (ios, iosxr) - If ``True``, host keys will be loaded from the file specified in ``alt_key_file``.
-* :code:`alt_key_file` (ios, iosxr) - SSH host key file to use (if ``alt_host_keys`` is ``True``).
+* :code:`allow_agent` (ios, iosxr, nxos_ssh) - Paramiko argument, enable connecting to the SSH agent (default: ``False``).
+* :code:`alt_host_keys` (ios, iosxr, nxos_ssh) - If ``True``, host keys will be loaded from the file specified in ``alt_key_file``.
+* :code:`alt_key_file` (ios, iosxr, nxos_ssh) - SSH host key file to use (if ``alt_host_keys`` is ``True``).
 * :code:`auto_rollback_on_error` (ios) - Disable automatic rollback (certain versions of IOS support configure replace, but not rollback on error) (default: ``True``).
 * :code:`config_lock` (iosxr, junos) - Lock the config during open() (default: ``False``).
 * :code:`canonical_int` (ios) - Convert operational interface's returned name to canonical name (fully expanded name) (default: ``False``).
 * :code:`dest_file_system` (ios) - Destination file system for SCP transfers (default: ``flash:``).
 * :code:`enable_password` (eos) - Password required to enter privileged exec (enable) (default: ``''``).
-* :code:`global_delay_factor` (ios) - Allow for additional delay in command execution (default: ``1``).
+* :code:`global_delay_factor` (ios, nxos_ssh) - Allow for additional delay in command execution (default: ``1``).
 * :code:`ignore_warning` (junos) - Allows to set `ignore_warning` when loading configuration to avoid exceptions via junos-pyez. (default: ``False``).
-* :code:`keepalive` (junos, iosxr) - SSH keepalive interval, in seconds (default: ``30`` seconds).
-* :code:`key_file` (junos, ios, iosxr) - Path to a private key file. (default: ``False``).
-* :code:`port` (eos, iosxr, junos, ios, nxos) - Allows you to specify a port other than the default.
-* :code:`secret` (ios) - Password required to enter privileged exec (enable) (default: ``''``).
-* :code:`ssh_config_file` (junos, ios, iosxr) - File name of OpenSSH configuration file.
-* :code:`ssh_strict` (iosxr, ios) - Automatically reject unknown SSH host keys (default: ``False``, which means unknown SSH host keys will be accepted).
+* :code:`keepalive` (iosxr, junos) - SSH keepalive interval, in seconds (default: ``30`` seconds).
+* :code:`key_file` (ios, iosxr, junos, nxos_ssh) - Path to a private key file. (default: ``False``).
+* :code:`port` (eos, ios, iosxr, junos, nxos, nxos_ssh) - Allows you to specify a port other than the default.
+* :code:`secret` (ios, nxos_ssh) - Password required to enter privileged exec (enable) (default: ``''``).
+* :code:`ssh_config_file` (ios, iosxr, junos, nxos_ssh) - File name of OpenSSH configuration file.
+* :code:`ssh_strict` (ios, iosxr, nxos_ssh) - Automatically reject unknown SSH host keys (default: ``False``, which means unknown SSH host keys will be accepted).
 * :code:`transport` (eos, ios, nxos) - Protocol to connect with (see `The transport argument`_ for more information).
-* :code:`use_keys` (iosxr, ios, panos) - Paramiko argument, enable searching for discoverable private key files in ``~/.ssh/`` (default: ``False``).
+* :code:`use_keys` (ios, iosxr, nxos_ssh) - Paramiko argument, enable searching for discoverable private key files in ``~/.ssh/`` (default: ``False``).
 * :code:`eos_autoComplete` (eos) - Allows to set `autoComplete` when running commands. (default: ``None`` equivalent to ``False``)
 
 The transport argument

Diff for: docs/tutorials/index.rst

@@ -13,3 +13,4 @@ Tutorials
    extend_driver
    wrapup
    ansible-napalm
+   mock_driver

Diff for: docs/tutorials/mock_driver.rst

@@ -0,0 +1,178 @@
+Unit tests: Mock driver
+=======================
+
+A mock driver is a software that imitates the response pattern of another
+system. It is meant to do nothing but returns the same predictable result,
+usually of the cases in a testing environment.
+
+A driver `mock` can mock all actions done by a common napalm driver. It can be
+used for unit tests, either to test napalm itself or inside external projects
+making use of napalm.
+
+
+Overview
+--------
+
+For any action, the ``mock`` driver will use a file matching a specific pattern
+to return its content as a result.
+
+Each of these files will be located inside a directory specified at the driver
+initialization. Their names depend on the entire call name made to the
+driver, and about their order in the call stack.
+
+
+Replacing a standard driver by a ``mock``
+-----------------------------------------
+
+Get the driver in napalm::
+
+    >>> import napalm
+    >>> driver = napalm.get_network_driver('mock')
+
+And instantiate it with any host and credentials::
+
+    device = driver(
+        hostname='foo', username='user', password='pass',
+        optional_args={'path': path_to_results}
+    )
+
+Like other drivers, ``mock`` takes optional arguments:
+
+- ``path`` - Required. Directory where results files are located
+
+Open the driver::
+
+    >>> device.open()
+
+A user should now be able to call any function of a standard driver::
+
+    >>> device.get_network_instances()
+
+But should get an error because no mocked data is yet written::
+
+    NotImplementedError: You can provide mocked data in get_network_instances.1
+
+
+Mocked data
+-----------
+
+We will use ``/tmp/mock`` as an example of a directory that will contain
+our mocked data. Define a device using this path::
+
+    >>> with driver('foo', 'user', 'pass', optional_args={'path': '/tmp/mock'}) as device:
+
+Mock a single call
+~~~~~~~~~~~~~~~~~~
+
+In order to be able to call, for example, ``device.get_interfaces()``, a mocked
+data is needed.
+
+To build the file name that the driver will look for, take the function name
+(``get_interfaces``) and suffix it with the place of this call in the device
+call stack.
+
+.. note::
+    ``device.open()`` counts as a command. Each following order of call will
+    start at 1.
+
+Here, ``get_interfaces`` is the first call made to ``device`` after ``open()``,
+so the mocked data need to be put in ``/tmp/mock/get_interfaces.1``::
+
+
+    {
+        "Ethernet1/1": {
+            "is_up": true, "is_enabled": true, "description": "",
+            "last_flapped": 1478175306.5162635, "speed": 10000,
+            "mac_address": "FF:FF:FF:FF:FF:FF"
+        },
+        "Ethernet1/2": {
+            "is_up": true, "is_enabled": true, "description": "",
+            "last_flapped": 1492172106.5163276, "speed": 10000,
+            "mac_address": "FF:FF:FF:FF:FF:FF"
+        }
+    }
+
+The content is the wanted result of ``get_interfaces`` in JSON, exactly as
+another driver would return it.
+
+Mock multiple iterative calls
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If ``/tmp/mock/get_interfaces.1`` was defined and used, for any other call on
+the same device, the number of calls needs to be incremented.
+
+For example, to call ``device.get_interfaces_ip()`` after
+``device.get_interfaces()``, the file ``/tmp/mock/get_interfaces_ip.2`` needs
+to be defined::
+
+    {
+        "Ethernet1/1": {
+            "ipv6": {"2001:DB8::": {"prefix_length": 64}}
+        }
+    }
+
+Mock a CLI call
+~~~~~~~~~~~~~~~
+
+``device.cli(commands)`` calls are a bit different to mock, as a suffix
+corresponding to the command applied to the device needs to be added. As
+before, the data mocked file will start by ``cli`` and the number of calls done
+before (here, ``cli.1``). Then, the same process needs to be applied to each
+command.
+
+Each command needs to be sanitized: any special character (`` -,./\``, etc.)
+needs to be replaced by ``_``. Add the index of this command as it is sent to
+``device.cli()``. Each file then will contain the raw wanted output of its
+associated command.
+
+Example
+^^^^^^^
+
+Example with 2 commands, ``show interface Ethernet 1/1`` and ``show interface
+Ethernet 1/2``.
+
+To define the mocked data, create a file ``/tmp/mock/cli.1.show_interface_Ethernet_1_1.0``::
+
+    Ethernet1/1 is up
+    admin state is up, Dedicated Interface
+
+And a file ``/tmp/mock/cli.1.show_interface_Ethernet_1_2.1``::
+
+    Ethernet1/2 is up
+    admin state is up, Dedicated Interface
+
+And now they can be called::
+
+    >>> device.cli(["show interface Ethernet 1/1", "show interface Ethernet 1/2"])
+
+
+Mock an error
+~~~~~~~~~~~~~
+
+The `mock` driver can raise an exception during a call, to simulate an error.
+An error definition is  actually a json composed of 3 keys:
+
+    * `exception`: the exception type that will be raised
+    * `args` and `kwargs`: parameters sent to the exception constructor
+
+For example, to raise the exception `ConnectionClosedException` when calling
+``device.get_interfaces()``, the file ``/tmp/mock/get_interfaces.1`` needs to
+be defined::
+
+    {
+        "exception": "napalm.base.exceptions.ConnectionClosedException",
+        "args": [
+            "Connection closed."
+        ],
+        "kwargs": {}
+    }
+
+Now calling `get_interfaces()` for the 1st time will raise an exception::
+
+    >>> device.get_interfaces()
+    ConnectionClosedException: Connection closed
+
+As before, mock will depend on the number of calls. If a second file
+``/tmp/mock/get_interfaces.2`` was defined and filled with some expected data
+(not an exception), retrying `get_interfaces()` will run correctly if the first
+exception was caught.

Diff for: napalm/base/helpers.py

@@ -102,44 +102,46 @@ def textfsm_extractor(cls, template_name, raw_text):
     :return: table-like list of entries
     """
     textfsm_data = list()
-    cls.__class__.__name__.replace('Driver', '')
-    current_dir = os.path.dirname(os.path.abspath(sys.modules[cls.__module__].__file__))
-    template_dir_path = '{current_dir}/utils/textfsm_templates'.format(
-        current_dir=current_dir
-    )
-    template_path = '{template_dir_path}/{template_name}.tpl'.format(
-        template_dir_path=template_dir_path,
-        template_name=template_name
-    )
-
-    try:
-        fsm_handler = textfsm.TextFSM(open(template_path))
-    except IOError:
-        raise napalm.base.exceptions.TemplateNotImplemented(
-            "TextFSM template {template_name}.tpl is not defined under {path}".format(
-                template_name=template_name,
-                path=template_dir_path
-            )
+    fsm_handler = None
+    for c in cls.__class__.mro():
+        if c is object:
+            continue
+        current_dir = os.path.dirname(os.path.abspath(sys.modules[c.__module__].__file__))
+        template_dir_path = '{current_dir}/utils/textfsm_templates'.format(
+            current_dir=current_dir
         )
-    except textfsm.TextFSMTemplateError as tfte:
-        raise napalm.base.exceptions.TemplateRenderException(
-            "Wrong format of TextFSM template {template_name}: {error}".format(
-                template_name=template_name,
-                error=py23_compat.text_type(tfte)
-            )
+        template_path = '{template_dir_path}/{template_name}.tpl'.format(
+            template_dir_path=template_dir_path,
+            template_name=template_name
         )
 
-    objects = fsm_handler.ParseText(raw_text)
-
-    for obj in objects:
-        index = 0
-        entry = {}
-        for entry_value in obj:
-            entry[fsm_handler.header[index].lower()] = entry_value
-            index += 1
-        textfsm_data.append(entry)
+        try:
+            with open(template_path) as f:
+                fsm_handler = textfsm.TextFSM(f)
+
+                for obj in fsm_handler.ParseText(raw_text):
+                    entry = {}
+                    for index, entry_value in enumerate(obj):
+                        entry[fsm_handler.header[index].lower()] = entry_value
+                    textfsm_data.append(entry)
+
+                return textfsm_data
+        except IOError:  # Template not present in this class
+            continue  # Continue up the MRO
+        except textfsm.TextFSMTemplateError as tfte:
+            raise napalm.base.exceptions.TemplateRenderException(
+                "Wrong format of TextFSM template {template_name}: {error}".format(
+                    template_name=template_name,
+                    error=py23_compat.text_type(tfte)
+                )
+            )
 
-    return textfsm_data
+    raise napalm.base.exceptions.TemplateNotImplemented(
+        "TextFSM template {template_name}.tpl is not defined under {path}".format(
+            template_name=template_name,
+            path=template_dir_path
+        )
+    )
 
 
 def find_txt(xml_tree, path, default=''):

Diff for: napalm/base/mock.py

@@ -17,9 +17,9 @@
 from __future__ import unicode_literals
 
 from napalm.base.base import NetworkDriver
+from napalm.base.utils import py23_compat
 import napalm.base.exceptions
 
-import inspect
 import json
 import os
 import re
@@ -28,10 +28,6 @@
 from pydoc import locate
 
 
-# inspect.getargspec deprecated in Python 3.5, use getfullargspec if available
-inspect_getargspec = getattr(inspect, "getfullargspec", inspect.getargspec)
-
-
 def raise_exception(result):
     exc = locate(result["exception"])
     if exc:
@@ -49,7 +45,7 @@ def is_mocked_method(method):
 
 def mocked_method(path, name, count):
     parent_method = getattr(NetworkDriver, name)
-    parent_method_args = inspect_getargspec(parent_method)
+    parent_method_args = py23_compat.argspec(parent_method)
     modifier = 0 if 'self' not in parent_method_args.args else 1
 
     def _mocked_method(*args, **kwargs):

Diff for: napalm/base/netmiko_helpers.py

@@ -10,7 +10,7 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 from __future__ import unicode_literals
-import inspect
+from napalm.base.utils import py23_compat
 from netmiko import BaseConnection
 
 
@@ -20,13 +20,15 @@ def netmiko_args(optional_args):
     Return a dictionary of these optional args  that will be passed into the Netmiko
     ConnectHandler call.
     """
-    netmiko_args, _, _, netmiko_defaults = inspect.getargspec(BaseConnection.__init__)
+    fields = py23_compat.argspec(BaseConnection.__init__)
+    args = fields[0]
+    defaults = fields[3]
 
-    check_self = netmiko_args.pop(0)
+    check_self = args.pop(0)
     if check_self != 'self':
         raise ValueError("Error processing Netmiko arguments")
 
-    netmiko_argument_map = dict(zip(netmiko_args, netmiko_defaults))
+    netmiko_argument_map = dict(zip(args, defaults))
 
     # Netmiko arguments that are integrated into NAPALM already
     netmiko_filter = ['ip', 'host', 'username', 'password', 'device_type', 'timeout']