Merge pull request #5 from cube-js/parse_explores

Parse explores

Author: paco-valdez

.gitignore

@@ -159,3 +159,9 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 .pdm-python
+
+
+# default location to write files
+cubes/
+views/
+examples/

README.md

@@ -15,11 +15,17 @@ pip install lkml2cube
 ### Convert LookML views into Cube YAML definition.
 
 ```sh
-lkml2cube cubes path/to/file.view.lkml
+lkml2cube cubes path/to/file.view.lkml --outputdir examples/
 ```
 
 ### Show Python dict representation of the LookerML object
 
 ```sh
 lkml2cube cubes --parseonly path/to/file.view.lkml
-```
+```
+
+### Convert LookML Explroes into Cube's views YAML definition.
+
+```sh
+lkml2cube views path/to/file.explore.lkml --outputdir examples/
+```

lkml2cube/main.py

@@ -1,7 +1,9 @@
 import pprint
 import typer
+import yaml
 
-from lkml2cube.parser.loader import file_loader
+from lkml2cube.parser.explores import parse_explores, generate_cube_joins
+from lkml2cube.parser.loader import file_loader, write_files
 from lkml2cube.parser.views import parse_view
 from typing_extensions import Annotated
 
@@ -12,14 +14,17 @@ def callback():
     """
     lkml2cube is a tool to convert LookML models into Cube data models.
     """
-    typer.echo(("lkml2cube is a tool to convert LookML models into Cube data models.\n"
-                "Use lkml2cube --help to see usage."))
+    # typer.echo(("lkml2cube is a tool to convert LookML models into Cube data models.\n"
+    #             "Use lkml2cube --help to see usage."))
+    pass
 
 @app.command()
 def cubes(
-    file_path: Annotated[str, typer.Argument(help="The path for the file to read")],
-    parseonly: Annotated[bool, typer.Option(help=("When present it will only show the python"
-                                                  " dict read from the lookml file"))] = False
+        file_path: Annotated[str, typer.Argument(help="The path for the file to read")],
+        parseonly: Annotated[bool, typer.Option(help=("When present it will only show the python"
+                                                      " dict read from the lookml file"))] = False,
+        outputdir: Annotated[str, typer.Option(help="The path for the output files to be generated")] = '.',
+        printonly: Annotated[bool, typer.Option(help="Print to stdout the parsed files")] = False,
     ):
     """
     Generate cubes-only given a LookML file that contains LookML Views.
@@ -30,7 +35,41 @@ def cubes(
         typer.echo(pprint.pformat(lookml_model))
         return
 
-    typer.echo(parse_view(lookml_model))
+    cube_def = parse_view(lookml_model)
+    cube_def = generate_cube_joins(cube_def, lookml_model)
+    
+    if printonly:
+        typer.echo(yaml.dump(cube_def))
+        return
+    
+    write_files(cube_def, outputdir=outputdir)
+    
+
+
+@app.command()
+def views(
+        file_path: Annotated[str, typer.Argument(help="The path for the explore to read")],
+        parseonly: Annotated[bool, typer.Option(help=("When present it will only show the python"
+                                                      " dict read from the lookml file"))] = False,
+        outputdir: Annotated[str, typer.Option(help="The path for the output files to be generated")] = '.',
+        printonly: Annotated[bool, typer.Option(help="Print to stdout the parsed files")] = False,
+    ):
+    """
+    Generate cubes-only given a LookML file that contains LookML Views.
+    """
+    
+    lookml_model = file_loader(file_path)
+    if parseonly:
+        typer.echo(pprint.pformat(lookml_model))
+        return
+    
+    cube_def = parse_explores(lookml_model)
+
+    if printonly:
+        typer.echo(yaml.dump(cube_def))
+        return
+    
+    write_files(cube_def, outputdir=outputdir)
 
 if __name__ == "__main__":
     app()

lkml2cube/parser/explores.py

@@ -0,0 +1,157 @@
+
+import re
+import traceback
+import typer
+
+from pprint import pformat
+
+from lkml2cube.parser.views import parse_view
+
+snake_case = r'\{([a-zA-Z]+(?:_[a-zA-Z]+)*\.[a-zA-Z]+(?:_[a-zA-Z]+)*)\}'
+
+def snakify(s):
+  return '_'.join(
+        re.sub('([A-Z][a-z]+)', r' \1',
+        re.sub('([A-Z]+)', r' \1',
+        s.replace('-', ' '))).split()
+    ).lower()
+
+def build_cube_name_look_up(cube_def):
+    if 'cube_name_look_up' in cube_def:
+        return
+    cube_name_look_up = {}
+    for cube_element in cube_def['cubes']:
+        cube_name_look_up[cube_element['name']] = cube_element
+    cube_def['cube_name_look_up'] = cube_name_look_up
+
+def get_cube_from_cube_def(cube_def, cube_name):
+    if 'cube_name_look_up' not in cube_def:
+        build_cube_name_look_up(cube_def)
+    if cube_name in cube_def['cube_name_look_up']:
+        return cube_def['cube_name_look_up'][cube_name]
+    return None
+
+def get_cube_names_from_join_condition(join_condition):
+    return [cube.split('.')[0] for cube in re.findall(snake_case, join_condition)]
+
+def traverse_graph(join_paths, cube_left, cube_right):
+    # Create a queue for BFS
+    queue = []
+    queue.append([cube_left])
+
+    while queue:
+        #Dequeue a vertex from queue
+        tmp_path = queue.pop(0)
+        # If this adjacent node is the destination node,
+        # then return true
+        last_node = tmp_path[len(tmp_path)-1]
+        if last_node == cube_right:
+            return '.'.join(tmp_path)
+        #  Else, continue to do BFS
+        if last_node in join_paths:
+            for cube in join_paths[last_node]:
+                if cube not in tmp_path:
+                    new_path = []
+                    new_path = tmp_path + [cube]
+                    queue.append(new_path)
+
+    typer.echo(f'Cubes are not reachable: {cube_left}, {cube_right}')
+    return '.'.join(cube_left, cube_right)
+
+
+def generate_cube_joins(cube_def, lookml_model):
+    for explore in lookml_model['explores']:
+        if 'joins' not in explore:
+            continue
+
+        for join_element in explore['joins']:
+            try:
+                cube_right = join_element['name']
+            
+                joined_cubes = [cube for cube in get_cube_names_from_join_condition(join_element['sql_on']) if cube != cube_right]
+                if joined_cubes:
+                    if 'from' in join_element:
+                        cube = {
+                            'name': cube_right,
+                            'extends': join_element['from'],
+                            'shown': False,
+                        }
+                        cube_def['cubes'].append(cube)
+                    else:
+                        cube = get_cube_from_cube_def(cube_def, cube_right)
+
+                    join_condition = join_element['sql_on']
+
+                    if 'joins' not in cube:
+                        cube['joins'] = []
+
+                    cube['joins'].append({
+                        'name': joined_cubes[0],
+                        'sql': join_condition,
+                        'relationship': join_element['relationship']
+                    })
+            except Exception:
+                typer.echo(f'Error while parsing explore: {pformat(explore)}')
+                typer.echo(traceback.format_exc())
+
+    return cube_def
+
+def generate_cube_views(cube_def, lookml_model):
+    if 'views' not in cube_def:
+        cube_def['views'] = []
+    for explore in lookml_model['explores']:
+        try:
+            central_cube = explore['name']
+            view_name = snakify(explore['label'])
+            view = {
+                'name': view_name,
+                'description': explore['label'],
+                'cubes': [{
+                    'join_path': central_cube,
+                    'includes': "*",
+                    'alias': view_name
+                }]
+            }
+
+            if 'joins' not in explore:
+                cube_def['views'].append(view)
+                continue
+            # Create Graph
+            join_paths = {}
+            for join_element in explore['joins']:
+                cube_right = join_element['name']
+                cube_left = [cube for cube in get_cube_names_from_join_condition(join_element['sql_on']) if cube != cube_right][0]
+
+                if cube_left in join_paths:
+                    join_paths[cube_left].append(cube_right)
+                else:
+                    join_paths[cube_left] = [cube_right]
+            # traverse graph
+            for join_element in explore['joins']:
+                cube_right = join_element['name']
+                join_path = {
+                    'join_path': traverse_graph(join_paths, central_cube, cube_right),
+                    'includes': "*",
+                    'alias': cube_right
+                }
+                view['cubes'].append(join_path)
+                
+            # End
+            cube_def['views'].append(view)
+
+        except Exception:
+            typer.echo(f'Error while parsing explore: {pformat(explore)}')
+            typer.echo(traceback.format_exc())
+    return cube_def
+
+
+def parse_explores(lookml_model):
+    # First we read all possible lookml views.
+    cube_def = parse_view(lookml_model, raise_when_views_not_present=False)
+    if 'explores' not in lookml_model:
+        raise Exception('LookML explores are needed to generate Cube Views, no explore found in path.')
+    cube_def = generate_cube_joins(cube_def, lookml_model)
+
+    cube_def = generate_cube_views(cube_def, lookml_model)
+
+    return cube_def

lkml2cube/parser/loader.py

@@ -1,5 +1,85 @@
+import glob
 import lkml
+import typer
+import yaml
 
-def file_loader(file_path):
-    lookml_model = lkml.load(open(file_path, 'r'))
-    return lookml_model
+from os.path import abspath, dirname, join
+from pathlib import Path
+
+visited_path = {}
+
+def update_namespace(namespace, new_file):
+
+    if namespace is None:
+        return new_file
+    for key, value in new_file.items():
+        if key in namespace and key in ('views', 'explores'):
+            namespace[key] = namespace[key] + new_file[key]
+        elif key in namespace and key in ('includes'): # remove duplicates
+            namespace[key] = list(set(namespace[key] + new_file[key]))
+        elif key in ('views', 'explores', 'includes'):
+            namespace[key] = new_file[key]
+        elif key in ('connection'):
+            pass # ignored keys
+        else:
+            typer.echo(f'Key not supported yet: {key}')
+    return namespace
+
+def file_loader(file_path_input, namespace=None):
+
+    file_paths = glob.glob(file_path_input)
+    for file_path in file_paths:
+        if file_path in visited_path:
+            continue
+        visited_path[file_path] = True
+        lookml_model = lkml.load(open(file_path, 'r'))
+        if 'includes' in lookml_model:
+            for included_path in lookml_model['includes']:
+                if namespace and 'includes' in namespace and included_path in namespace['includes']:
+                    continue
+                if included_path.startswith('/'):
+                    included_path = included_path[1:]
+                root_dir = dirname(abspath(file_path))
+                namespace = file_loader(join(root_dir, included_path), namespace=namespace)
+        namespace = update_namespace(namespace, lookml_model)
+    return namespace
+
+
+def write_single_file(cube_def: dict, outputdir: str, subdir: str = 'cubes', file_name: str = 'my_cubes.yml'):
+
+    f = open(join(outputdir, subdir, file_name), 'w')
+    f.write(yaml.dump(cube_def))
+    f.close()
+
+
+def write_files(cube_def, outputdir):
+    
+    if not cube_def:
+        raise Exception('No cube definition available')
+    
+    for cube_root_element in ('cubes', 'views'):
+
+        if cube_root_element in cube_def:
+
+            Path(join(outputdir, cube_root_element)).mkdir(parents=True, exist_ok=True)
+
+            if len(cube_def[cube_root_element]) == 1:
+                write_single_file(cube_def=cube_def, 
+                                outputdir=outputdir,
+                                subdir=cube_root_element,
+                                file_name=cube_def[cube_root_element][0]['name'] + '.yml')
+
+            elif len(cube_def[cube_root_element]) > 1:
+                for cube_element in cube_def[cube_root_element]:
+                    new_def = {
+                        cube_root_element: [cube_element]
+                    }
+                    write_single_file(cube_def=new_def,
+                                    outputdir=outputdir,
+                                    subdir=cube_root_element,
+                                    file_name=cube_element['name'] + '.yml')
+            else:
+                # Empty 'cubes' definition
+                # not expected but not invalid
+                pass
+