[AAM-65858] Add ops-cli Teleport support

README.md

@@ -762,7 +762,7 @@ env LDFLAGS="-L$(brew --prefix openssl)/lib" CFLAGS="-I$(brew --prefix openssl)/
 
 ## Running tests
 
-- on your machine: `py.test tests`
+- on your machine: `python -m pytest tests` or `build_scripts/run_tests.sh`
 
 # Troubleshooting
 

src/ops/cli/parser.py

@@ -112,5 +112,7 @@ def configure_common_ansible_args(parser):
     parser.add_argument('--noscb', action='store_false', dest='use_scb',
                         help='Disable use of Shell Control Box (SCB) even if '
                              'it is enabled in the cluster config')
+    parser.add_argument('--teleport', action='store_false', dest='use_teleport',
+                        help='Use Teleport for SSH')
 
     return parser

src/ops/cli/playbook.py

@@ -73,9 +73,11 @@ def __init__(self, ops_config, root_dir, inventory_generator,
     def run(self, args, extra_args):
         logger.info("Found extra_args %s", extra_args)
         inventory_path, ssh_config_paths = self.inventory_generator.generate()
-        ssh_config_path = SshConfigGenerator.get_ssh_config_path(self.cluster_config,
+        ssh_config_generator = SshConfigGenerator(self.ops_config.package_dir)
+        ssh_config_path = ssh_config_generator.get_ssh_config_path(self.cluster_config,
                                                                  ssh_config_paths,
-                                                                 args.use_scb)
+                                                                 args)
+
         ssh_config = f"ANSIBLE_SSH_ARGS='-F {ssh_config_path}'"
 
         ansible_config = "ANSIBLE_CONFIG=%s" % self.ops_config.ansible_config_path

src/ops/cli/run.py

@@ -68,9 +68,10 @@ def run(self, args, extra_args):
         logger.info("Found extra_args %s", extra_args)
         inventory_path, ssh_config_paths = self.inventory_generator.generate()
         limit = args.host_pattern
-        ssh_config_path = SshConfigGenerator.get_ssh_config_path(self.cluster_config,
+        ssh_config_generator = SshConfigGenerator(self.ops_config.package_dir)
+        ssh_config_path = ssh_config_generator.get_ssh_config_path(self.cluster_config,
                                                                  ssh_config_paths,
-                                                                 args.use_scb)
+                                                                 args)
         extra_args = ' '.join(args.extra_args)
         command = """cd {root_dir}
 ANSIBLE_SSH_ARGS='-F {ssh_config}' ANSIBLE_CONFIG={ansible_config_path} ansible -i {inventory_path} '{limit}' \\

src/ops/cli/sync.py

@@ -31,11 +31,15 @@ def configure(self, parser):
         parser.add_argument('--noscb', action='store_false', dest='use_scb',
                             help='Disable use of Shell Control Box (SCB) '
                                  'even if it is enabled in the cluster config')
+        parser.add_argument(
+            '--teleport',
+            action='store_false',
+            dest='use_teleport',
+            help='Use Teleport for SSH')
         parser.add_argument(
             'opts',
-            default=['-va --progress'],
             nargs='*',
-            help='Rsync opts')
+            help='Sync opts')
 
     def get_help(self):
         return 'Sync files from/to a cluster'
@@ -54,6 +58,9 @@ def get_epilog(self):
 
         # extra rsync options
         ops cluster.yml sync 'dcs[0]:/usr/local/demdex/conf' /tmp/configurator-data -l remote_user -- --progress
+
+        # extra sync option for Teleport (recursive download, quiet, port)
+        ops cluster.yml sync 'dcs[0]:/usr/local/demdex/conf' /tmp/configurator-data -- --recursive/port/quiet
         """
 
 
@@ -73,29 +80,36 @@ def __init__(self, cluster_config, root_dir,
 
     def run(self, args, extra_args):
         logger.info("Found extra_args %s", extra_args)
-        inventory_path, ssh_config_paths = self.inventory_generator.generate()
+
         src = PathExpr(args.src)
         dest = PathExpr(args.dest)
+        remote = self.get_remote(dest, src)
 
-        ssh_config_path = SshConfigGenerator.get_ssh_config_path(self.cluster_config,
-                                                                 ssh_config_paths,
-                                                                 args.use_scb)
         if src.is_remote and dest.is_remote:
             display(
-                'Too remote expressions are not allowed',
+                'Two remote expressions are not allowed',
                 stderr=True,
                 color='red')
             return
 
-        if src.is_remote:
-            remote = src
-        else:
-            remote = dest
-
         display(
             "Looking for hosts for pattern '%s'" %
             remote.pattern, stderr=True)
 
+        if self.is_teleport_enabled(args):
+            command = self.execute_teleport_scp(args, src, dest)
+        else:
+            ssh_user = self.cluster_config.get('ssh_user') or self.ops_config.get('ssh.user') or getpass.getuser()
+            if remote.remote_user:
+                ssh_user = remote.remote_user
+            elif args.user:
+                ssh_user = args.user
+            ssh_host = self.populate_remote_hosts(remote)[0]
+            command = self.execute_rsync_scp(args, src, dest, ssh_user, ssh_host, self.get_ssh_config_path(args))
+
+        return dict(command=command)
+
+    def populate_remote_hosts(self, remote):
         remote_hosts = []
         hosts = self.ansible_inventory.get_hosts(remote.pattern)
         if not hosts:
@@ -106,28 +120,43 @@ def run(self, args, extra_args):
             for host in hosts:
                 ssh_host = host.get_vars().get('ansible_ssh_host') or host
                 remote_hosts.append(ssh_host)
+        return remote_hosts
 
-        for ssh_host in remote_hosts:
-            ssh_user = self.cluster_config.get('ssh_user') or self.ops_config.get(
-                'ssh.user') or getpass.getuser()
-            if remote.remote_user:
-                ssh_user = remote.remote_user
-            elif args.user:
-                ssh_user = args.user
-
-            from_path = src.with_user_and_path(ssh_user, ssh_host)
-            to_path = dest.with_user_and_path(ssh_user, ssh_host)
-
-            command = 'rsync {opts} {from_path} {to_path} -e "ssh -F {ssh_config}"'.format(
-                opts=" ".join(args.opts),
-                from_path=from_path,
-                to_path=to_path,
-                ssh_config=ssh_config_path
-
-            )
-
-            return dict(command=command)
-
+    def get_remote(self, dest, src):
+        if src.is_remote:
+            remote = src
+        else:
+            remote = dest
+        return remote
+
+    def get_ssh_config_path(self, args):
+        ssh_config_generator = SshConfigGenerator(self.ops_config.package_dir)
+        _, ssh_config_paths = self.inventory_generator.generate()
+        return ssh_config_generator.get_ssh_config_path(self.cluster_config,
+                                                ssh_config_paths,
+
+                                                args)
+
+    def execute_teleport_scp(self, args, src, dest):
+        return 'tsh scp {opts} {from_path} {to_path}'.format(
+                    from_path=src,
+                    to_path=dest,
+                    opts=" ".join(args.opts)
+                )
+
+    def execute_rsync_scp(self, args, src, dest, ssh_user, ssh_host, ssh_config_path):
+        from_path = src.with_user_and_path(ssh_user, ssh_host)
+        to_path = dest.with_user_and_path(ssh_user, ssh_host)
+        return 'rsync {opts} {from_path} {to_path} -e "ssh -F {ssh_config}"'.format(
+                    opts=" ".join(args.opts),
+                    from_path=from_path,
+                    to_path=to_path,
+                    ssh_config=ssh_config_path
+                )
+
+
+    def is_teleport_enabled(self, args):
+         return True if self.cluster_config.get('teleport', {}).get('enabled') and args.use_teleport else False
 
 class PathExpr(object):
 

src/ops/data/ssh/ssh.teleport.config.tpl

@@ -0,0 +1,9 @@
+# teleport.cfg
+Host *
+    ProxyCommand "/usr/local/bin/tsh" proxy ssh --cluster=teleport-prod --proxy=teleport.adobe.net:443 %r@%n:%p
+    UserKnownHostsFile /dev/null
+    StrictHostKeyChecking no
+    ControlMaster auto
+    ControlPersist 600s
+    User {ssh_username}
+    Port 22

src/ops/inventory/ec2inventory.py

@@ -17,18 +17,21 @@
 
 
 class Ec2Inventory(object):
+
+
     @staticmethod
     def _empty_inventory():
         return {"_meta": {"hostvars": {}}}
 
-    def __init__(self, boto_profile, regions, filters=None, bastion_filters=None):
+    def __init__(self, boto_profile, regions, filters=None, bastion_filters=None, teleport_enabled=False):
 
         self.filters = filters or []
         self.regions = regions.split(',')
         self.boto_profile = boto_profile
         self.bastion_filters = bastion_filters or []
         self.group_callbacks = []
         self.boto3_session = self.create_boto3_session(boto_profile)
+        self.teleport_enabled = teleport_enabled
 
         # Inventory grouped by instance IDs, tags, security groups, regions,
         # and availability zones
@@ -145,7 +148,11 @@ def add_instance(self, bastion_ip, instance, region):
         if not dest:
             return
 
-        if bastion_ip and bastion_ip != instance.get('PublicIpAddress'):
+        if self.teleport_enabled:
+            ansible_ssh_host = self.get_tag_value(instance, ['hostname','CMDB_hostname','Adobe:FQDN'])
+            if not ansible_ssh_host:
+                ansible_ssh_host = instance.get('PrivateIpAddress')
+        elif bastion_ip and bastion_ip != instance.get('PublicIpAddress'):
             ansible_ssh_host = bastion_ip + "--" + instance.get('PrivateIpAddress')
         elif instance.get('PublicIpAddress'):
             ansible_ssh_host = instance.get('PublicIpAddress')
@@ -183,6 +190,13 @@ def add_instance(self, bastion_ip, instance, region):
         self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance)
         self.inventory["_meta"]["hostvars"][dest]['ansible_ssh_host'] = ansible_ssh_host
 
+    def get_tag_value(self, instance, tag_keys):
+        for tag_key in tag_keys:
+            for tag in instance.get('Tags', []):
+                if tag['Key'] == tag_key:
+                    return tag['Value']
+        return None
+
     def get_host_info_dict_from_instance(self, instance):
         instance_vars = {}
         for key, value in instance.items():

src/ops/inventory/plugin/cns.py

@@ -26,6 +26,7 @@ def cns(args):
             jsn = ec2(dict(
                 region=region,
                 boto_profile=profile,
+                teleport_enabled=args.get('teleport_enabled', False),
                 cache=args.get('cache', 3600 * 24),
                 filters=[
                     {'Name': 'tag:cluster', 'Values': [cns_cluster]}

src/ops/inventory/plugin/ec2.py

@@ -14,6 +14,7 @@
 def ec2(args):
     filters = args.get('filters', [])
     bastion_filters = args.get('bastion', [])
+    teleport_enabled = args.get('teleport_enabled')
 
     if args.get('cluster') and not args.get('filters'):
         filters = [{'Name': 'tag:cluster', 'Values': [args.get('cluster')]}]
@@ -27,4 +28,6 @@ def ec2(args):
     return Ec2Inventory(boto_profile=args['boto_profile'],
                         regions=args['region'],
                         filters=filters,
-                        bastion_filters=bastion_filters).get_as_json()
+                        bastion_filters=bastion_filters,
+                        teleport_enabled=teleport_enabled
+                        ).get_as_json()

src/ops/inventory/sshconfig.py

@@ -18,11 +18,12 @@
 class SshConfigGenerator(object):
     SSH_CONFIG_FILE = "ssh.config"
     SSH_SCB_PROXY_TPL_FILE = "ssh.scb.proxy.config.tpl"
+    SSH_TELEPORT_PROXY_TPL_FILE = "ssh.teleport.config.tpl"
 
     def __init__(self, package_dir):
         self.package_dir = package_dir
         self.ssh_data_dir = self.package_dir + '/data/ssh'
-        self.ssh_config_files = [self.SSH_CONFIG_FILE, self.SSH_SCB_PROXY_TPL_FILE]
+        self.ssh_config_files = [self.SSH_CONFIG_FILE, self.SSH_SCB_PROXY_TPL_FILE, self.SSH_TELEPORT_PROXY_TPL_FILE]
 
     def generate(self, directory):
         dest_ssh_config = {}
@@ -36,63 +37,71 @@ def _get_ssh_config(self):
         return [f"{self.ssh_data_dir}/{ssh_config_file}"
                 for ssh_config_file in self.ssh_config_files]
 
-    @staticmethod
-    def get_ssh_config_path(cluster_config, ssh_config_paths, use_scb):
-        scb_settings = cluster_config.get('scb', {})
-        scb_enabled = scb_settings.get('enabled') and use_scb
+    def get_ssh_config_path(self, cluster_config, ssh_config_paths, args):
+        scb_enabled, teleport_enabled = self.get_proxy_type_information(cluster_config, args)
         if scb_enabled:
             ssh_config_tpl_path = ssh_config_paths.get(SshConfigGenerator.SSH_SCB_PROXY_TPL_FILE)
-            scb_proxy_port = SshConfigGenerator.get_ssh_scb_proxy_port(ssh_config_tpl_path)
-            ssh_config_path = SshConfigGenerator.generate_ssh_scb_config(ssh_config_tpl_path,
-                                                                         scb_proxy_port)
-            display.display(f"Connecting via scb proxy at 127.0.0.1:{scb_proxy_port}.\n"
-                            f"This proxy should have already been started and running "
-                            f"in a different terminal window.\n"
-                            f"If there are connection issues double check that "
-                            f"the proxy is running.",
-                            color='blue',
-                            stderr=True)
+            ssh_proxy_port = self.get_ssh_proxy_port(ssh_config_tpl_path)
+            display.display(f"Connecting via scb proxy at 127.0.0.1:{ssh_proxy_port}.\n"
+                          f"This proxy should have already been started and running "
+                          f"in a different terminal window.\n"
+                          f"If there are connection issues double check that "
+                          f"the proxy is running.",
+                          color='blue',
+                          stderr=True)
+            return self.generate_ssh_config_from_template(ssh_config_tpl_path, ssh_proxy_port)
+        elif teleport_enabled:
+            display.display(f"Using Teleport for SSH connections.\n"
+                          f"Make sure you are logged in with 'tsh login'.",
+                          color='blue',
+                          stderr=True)
+            ssh_config_tpl_path = ssh_config_paths.get(SshConfigGenerator.SSH_TELEPORT_PROXY_TPL_FILE)
+            return self.generate_ssh_config_from_template(ssh_config_tpl_path, ssh_username=os.getlogin())
         else:
-            ssh_config_path = ssh_config_paths.get(SshConfigGenerator.SSH_CONFIG_FILE)
-        return ssh_config_path
+            return ssh_config_paths.get(SshConfigGenerator.SSH_CONFIG_FILE)
 
-    @staticmethod
-    def generate_ssh_scb_proxy_port(ssh_config_path, auto_scb_port, scb_config_port):
+    def generate_ssh_scb_proxy_port(self, ssh_config_path, auto_scb_port, scb_config_port):
         ssh_config_port_path = f"{ssh_config_path}/ssh_scb_proxy_config_port"
         if auto_scb_port:
-            with socketserver.TCPServer(("localhost", 0), None) as s:
-                generated_port = s.server_address[1]
-            display.display(f"Using auto generated port {generated_port} for scb proxy port",
-                            color='blue',
-                            stderr=True)
+            generated_port = self.get_random_generated_port()
         else:
             generated_port = scb_config_port
             display.display(f"Using port {generated_port} from cluster config for scb proxy port",
-                            color='blue',
-                            stderr=True)
+                          color='blue',
+                          stderr=True)
 
         with open(ssh_config_port_path, 'w') as f:
             f.write(str(generated_port))
             os.fchmod(f.fileno(), 0o644)
 
         return generated_port
 
-
-    @staticmethod
-    def get_ssh_scb_proxy_port(ssh_config_path):
+    def get_ssh_proxy_port(self, ssh_config_path):
         ssh_port_path = ssh_config_path.replace("_tpl", "_port")
-        ssh_scb_proxy_port = Path(ssh_port_path).read_text()
-        return ssh_scb_proxy_port
-
-    @staticmethod
-    def generate_ssh_scb_config(ssh_config_tpl_path, scb_proxy_port):
-        ssh_config_template = Path(ssh_config_tpl_path).read_text()
-        ssh_config_content = ssh_config_template.format(
-            scb_proxy_port=scb_proxy_port
-        )
+        return  Path(ssh_port_path).read_text()
+
+    def generate_ssh_config_from_template(self, ssh_config_tpl_path, **template_vars):
+        ssh_config_content = Path(ssh_config_tpl_path).read_text().format(**template_vars)
+        return self.generate_file_from_template(ssh_config_tpl_path, ssh_config_content)
+
+    def generate_file_from_template(self, ssh_config_tpl_path, ssh_config_content):
         ssh_config_path = ssh_config_tpl_path.removesuffix("_tpl")
         with open(ssh_config_path, 'w') as f:
             f.write(ssh_config_content)
             os.fchmod(f.fileno(), 0o644)
-
         return ssh_config_path
+
+    def  get_proxy_type_information(self, cluster_config, args):
+            scb_settings = cluster_config.get('scb', {})
+            scb_enabled = scb_settings.get('enabled') and args.use_scb
+            teleport_settings = cluster_config.get('teleport', {})
+            teleport_enabled = teleport_settings.get('enabled') and args.use_teleport
+            return scb_enabled, teleport_enabled
+
+    def get_random_generated_port(self):
+        with socketserver.TCPServer(("localhost", 0), None) as s:
+            generated_port = s.server_address[1]
+            display.display(f"Using auto generated port {generated_port} for scb proxy port",
+                        color='blue',
+                        stderr=True)
+            return generated_port