Add explores parser

Author: paco-valdez

README.md

@@ -15,11 +15,17 @@ pip install lkml2cube
 ### Convert LookML views into Cube YAML definition.
 
 ```sh
-lkml2cube cubes path/to/file.view.lkml
+lkml2cube cubes path/to/file.view.lkml --outputdir examples/
 ```
 
 ### Show Python dict representation of the LookerML object
 
 ```sh
 lkml2cube cubes --parseonly path/to/file.view.lkml
-```
+```
+
+### Convert LookML Explroes into Cube's views YAML definition.
+
+```sh
+lkml2cube views path/to/file.explore.lkml --outputdir examples/
+```

lkml2cube/main.py

@@ -2,7 +2,7 @@
 import typer
 import yaml
 
-from lkml2cube.parser.explores import parse_explores
+from lkml2cube.parser.explores import parse_explores, generate_cube_joins
 from lkml2cube.parser.loader import file_loader, write_files
 from lkml2cube.parser.views import parse_view
 from typing_extensions import Annotated
@@ -36,6 +36,7 @@ def cubes(
         return
 
     cube_def = parse_view(lookml_model)
+    cube_def = generate_cube_joins(cube_def, lookml_model)
 
     if printonly:
         typer.echo(yaml.dump(cube_def))

lkml2cube/parser/explores.py

@@ -1,8 +1,157 @@
 
-import yaml
+import re
+import traceback
 import typer
-from pprint import pprint
+
+from pprint import pformat
+
+from lkml2cube.parser.views import parse_view
+
+snake_case = r'\{([a-zA-Z]+(?:_[a-zA-Z]+)*\.[a-zA-Z]+(?:_[a-zA-Z]+)*)\}'
+
+def snakify(s):
+  return '_'.join(
+        re.sub('([A-Z][a-z]+)', r' \1',
+        re.sub('([A-Z]+)', r' \1',
+        s.replace('-', ' '))).split()
+    ).lower()
+
+def build_cube_name_look_up(cube_def):
+    if 'cube_name_look_up' in cube_def:
+        return
+    cube_name_look_up = {}
+    for cube_element in cube_def['cubes']:
+        cube_name_look_up[cube_element['name']] = cube_element
+    cube_def['cube_name_look_up'] = cube_name_look_up
+
+def get_cube_from_cube_def(cube_def, cube_name):
+    if 'cube_name_look_up' not in cube_def:
+        build_cube_name_look_up(cube_def)
+    if cube_name in cube_def['cube_name_look_up']:
+        return cube_def['cube_name_look_up'][cube_name]
+    return None
+
+def get_cube_names_from_join_condition(join_condition):
+    return [cube.split('.')[0] for cube in re.findall(snake_case, join_condition)]
+
+def traverse_graph(join_paths, cube_left, cube_right):
+    # Create a queue for BFS
+    queue = []
+    queue.append([cube_left])
+
+    while queue:
+        #Dequeue a vertex from queue
+        tmp_path = queue.pop(0)
+        # If this adjacent node is the destination node,
+        # then return true
+        last_node = tmp_path[len(tmp_path)-1]
+        if last_node == cube_right:
+            return '.'.join(tmp_path)
+        #  Else, continue to do BFS
+        if last_node in join_paths:
+            for cube in join_paths[last_node]:
+                if cube not in tmp_path:
+                    new_path = []
+                    new_path = tmp_path + [cube]
+                    queue.append(new_path)
+
+    typer.echo(f'Cubes are not reachable: {cube_left}, {cube_right}')
+    return '.'.join(cube_left, cube_right)
+
+
+def generate_cube_joins(cube_def, lookml_model):
+    for explore in lookml_model['explores']:
+        if 'joins' not in explore:
+            continue
+
+        for join_element in explore['joins']:
+            try:
+                cube_right = join_element['name']
+            
+                joined_cubes = [cube for cube in get_cube_names_from_join_condition(join_element['sql_on']) if cube != cube_right]
+                if joined_cubes:
+                    if 'from' in join_element:
+                        cube = {
+                            'name': cube_right,
+                            'extends': join_element['from'],
+                            'shown': False,
+                        }
+                        cube_def['cubes'].append(cube)
+                    else:
+                        cube = get_cube_from_cube_def(cube_def, cube_right)
+
+                    join_condition = join_element['sql_on']
+
+                    if 'joins' not in cube:
+                        cube['joins'] = []
+
+                    cube['joins'].append({
+                        'name': joined_cubes[0],
+                        'sql': join_condition,
+                        'relationship': join_element['relationship']
+                    })
+            except Exception:
+                typer.echo(f'Error while parsing explore: {pformat(explore)}')
+                typer.echo(traceback.format_exc())
+
+    return cube_def
+
+def generate_cube_views(cube_def, lookml_model):
+    if 'views' not in cube_def:
+        cube_def['views'] = []
+    for explore in lookml_model['explores']:
+        try:
+            central_cube = explore['name']
+            view_name = snakify(explore['label'])
+            view = {
+                'name': view_name,
+                'description': explore['label'],
+                'cubes': [{
+                    'join_path': central_cube,
+                    'includes': "*",
+                    'alias': view_name
+                }]
+            }
+
+            if 'joins' not in explore:
+                cube_def['views'].append(view)
+                continue
+            # Create Graph
+            join_paths = {}
+            for join_element in explore['joins']:
+                cube_right = join_element['name']
+                cube_left = [cube for cube in get_cube_names_from_join_condition(join_element['sql_on']) if cube != cube_right][0]
+
+                if cube_left in join_paths:
+                    join_paths[cube_left].append(cube_right)
+                else:
+                    join_paths[cube_left] = [cube_right]
+            # traverse graph
+            for join_element in explore['joins']:
+                cube_right = join_element['name']
+                join_path = {
+                    'join_path': traverse_graph(join_paths, central_cube, cube_right),
+                    'includes': "*",
+                    'alias': cube_right
+                }
+                view['cubes'].append(join_path)
+                
+            # End
+            cube_def['views'].append(view)
+
+        except Exception:
+            typer.echo(f'Error while parsing explore: {pformat(explore)}')
+            typer.echo(traceback.format_exc())
+    return cube_def
 
 
 def parse_explores(lookml_model):
-    return pprint(lookml_model)
+    # First we read all possible lookml views.
+    cube_def = parse_view(lookml_model, raise_when_views_not_present=False)
+    if 'explores' not in lookml_model:
+        raise Exception('LookML explores are needed to generate Cube Views, no explore found in path.')
+    cube_def = generate_cube_joins(cube_def, lookml_model)
+
+    cube_def = generate_cube_views(cube_def, lookml_model)
+
+    return cube_def

lkml2cube/parser/loader.py

@@ -9,12 +9,15 @@
 visited_path = {}
 
 def update_namespace(namespace, new_file):
+
     if namespace is None:
         return new_file
     for key, value in new_file.items():
         if key in namespace and key in ('views', 'explores'):
             namespace[key] = namespace[key] + new_file[key]
-        elif key in ('views', 'explores'):
+        elif key in namespace and key in ('includes'): # remove duplicates
+            namespace[key] = list(set(namespace[key] + new_file[key]))
+        elif key in ('views', 'explores', 'includes'):
             namespace[key] = new_file[key]
         elif key in ('connection'):
             pass # ignored keys
@@ -23,6 +26,7 @@ def update_namespace(namespace, new_file):
     return namespace
 
 def file_loader(file_path_input, namespace=None):
+
     file_paths = glob.glob(file_path_input)
     for file_path in file_paths:
         if file_path in visited_path:
@@ -31,6 +35,8 @@ def file_loader(file_path_input, namespace=None):
         lookml_model = lkml.load(open(file_path, 'r'))
         if 'includes' in lookml_model:
             for included_path in lookml_model['includes']:
+                if namespace and 'includes' in namespace and included_path in namespace['includes']:
+                    continue
                 if included_path.startswith('/'):
                     included_path = included_path[1:]
                 root_dir = dirname(abspath(file_path))
@@ -39,10 +45,41 @@ def file_loader(file_path_input, namespace=None):
     return namespace
 
 
-def write_files(cube_def, outputdir, file_name = 'my_cubes.yml'):
-    if 'cubes' in cube_def and len(cube_def['cubes']) == 1:
-        file_name = cube_def['cubes'][0]['name'] + '.yml'
-    Path(join(outputdir, 'cubes')).mkdir(parents=True, exist_ok=True)
-    f = open(join(outputdir, 'cubes', file_name), 'w')
+def write_single_file(cube_def: dict, outputdir: str, subdir: str = 'cubes', file_name: str = 'my_cubes.yml'):
+
+    f = open(join(outputdir, subdir, file_name), 'w')
     f.write(yaml.dump(cube_def))
     f.close()
+
+
+def write_files(cube_def, outputdir):
+    
+    if not cube_def:
+        raise Exception('No cube definition available')
+    
+    for cube_root_element in ('cubes', 'views'):
+
+        if cube_root_element in cube_def:
+
+            Path(join(outputdir, cube_root_element)).mkdir(parents=True, exist_ok=True)
+
+            if len(cube_def[cube_root_element]) == 1:
+                write_single_file(cube_def=cube_def, 
+                                outputdir=outputdir,
+                                subdir=cube_root_element,
+                                file_name=cube_def[cube_root_element][0]['name'] + '.yml')
+
+            elif len(cube_def[cube_root_element]) > 1:
+                for cube_element in cube_def[cube_root_element]:
+                    new_def = {
+                        cube_root_element: [cube_element]
+                    }
+                    write_single_file(cube_def=new_def,
+                                    outputdir=outputdir,
+                                    subdir=cube_root_element,
+                                    file_name=cube_element['name'] + '.yml')
+            else:
+                # Empty 'cubes' definition
+                # not expected but not invalid
+                pass
+