Merge branch 'master' into daniel.zhou/CWS-3394-tf-provider

Author: QuentinGuillard

.generator/README.md

@@ -0,0 +1,79 @@
+# Terraform Generation
+
+The goal of this sub-project is to generate the scaffolding to create a Terraform resource.
+
+> [!CAUTION]
+> This code is HIGHLY experimental and should stabilize over the next weeks/months. As such this code is NOT intended for production uses.
+
+## How to use
+
+### Requirements
+
+- This project
+- Poetry
+- An OpenApi 3.0.x specification (Datadog's OpenApi spec can be found [here](https://github.com/DataDog/datadog-api-client-go/tree/master/.generator/schemas))
+- Go
+
+### Install dependencies
+
+Install the necessary dependencies by running `poetry install`
+
+Install go as we use the `go fmt` command on the generated files to format them.
+
+### Marking the resources to be generated
+
+The generator reads a configuration file in order to generate the appropriate resources.
+The configuration file should look like the following:
+
+```yaml
+resources:
+  { resource_name }:
+    read:
+      method: { read_method }
+      path: { read_path }
+    create:
+      method: { create_method }
+      path: { create_path }
+    update:
+      method: { update_method }
+      path: { update_path }
+    delete:
+      method: { delete_method }
+      path: { delete_path }
+  ...
+```
+
+- `resource_name` is the name of the resource to be generated.
+- `xxx_method` should be the HTTP method used by the relevant route
+- `xxx_path` should be the HTTP route of the resource's CRUD operation
+
+> [!NOTE]
+> An example using the `team` resource would look like this:
+>
+> ```yaml
+> resources:
+>   team:
+>     read:
+>       method: get
+>       path: /api/v2/team/{team_id}
+>     create:
+>       method: post
+>       path: /api/v2/team
+>     update:
+>       method: patch
+>       path: /api/v2/team/{team_id}
+>     delete:
+>       method: delete
+>       path: /api/v2/team/{team_id}
+> ```
+
+### Running the generator
+
+Once the configuration file is written, you can run the following command to generate the Terraform resources:
+
+```sh
+  $ poetry run python -m generator <openapi_spec_path> <configuration_path>
+```
+
+> [!NOTE]
+> The generated resources will be placed in `datadog/fwprovider/`

.generator/poetry.lock

@@ -0,0 +1,19 @@
+[project]
+name = "generator"
+version = "0.1.0"
+description = ""
+authors = [{ name = "Datadog", email = "[email protected]"}]
+license = "Apache-2.0"
+requires-python = ">=3.10"
+dependencies = [
+    "click (>=8.1.8,<9.0.0)",
+    "PyYaml (>=6.0,<7.0)",
+    "jsonref (>=1.1.0,<2.0.0)",
+    "jinja2 (>=3.1.5,<4.0.0)",
+    "pytest (>=8.3.4,<9.0.0)",
+    "pytest-bdd (>=8.1.0,<9.0.0)",
+]   
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

.generator/pyproject.toml

@@ -0,0 +1,3 @@
+from .cli import cli
+
+cli()

.generator/src/generator/__init__.py

@@ -0,0 +1,84 @@
+import os
+import pathlib
+import shlex
+import click
+
+from jinja2 import Template
+
+from . import setup
+from . import openapi
+
+
+@click.command()
+@click.argument(
+    "spec_path",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, path_type=pathlib.Path
+    ),
+)
+@click.argument(
+    "config_path",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, path_type=pathlib.Path
+    ),
+)
+@click.option(
+    "-o",
+    "--output",
+    default="../datadog/",
+    type=click.Path(path_type=pathlib.Path),
+)
+def cli(spec_path, config_path, output):
+    """
+    Generate a terraform code snippet from OpenAPI specification.
+    """
+    env = setup.load_environment(version=spec_path.parent.name)
+
+    templates = setup.load_templates(env=env)
+
+    spec = setup.load(spec_path)
+    config = setup.load(config_path)
+
+    resources_to_generate = openapi.get_resources(spec, config)
+
+    for name, resource in resources_to_generate.items():
+        generate_resource(
+            name=name,
+            resource=resource,
+            output=output,
+            templates=templates,
+        )
+
+
+def generate_resource(
+    name: str, resource: dict, output: pathlib.Path, templates: dict[str, Template]
+) -> None:
+    """
+    Generates files related to a resource.
+
+    :param name: The name of the resource.
+    :param operation: The intermediate representation of the resource.
+    :param output: The root where the files will be generated.
+    :param templates: The templates of the generated files.
+    """
+    # TF resource file
+    filename = output / f"fwprovider/resource_datadog_{name}.go"
+    with filename.open("w") as fp:
+        fp.write(templates["base"].render(name=name, operations=resource))
+    os.system(shlex.quote(f"go fmt {filename}"))
+
+    # TF test file
+    filename = output / "tests" / f"resource_datadog_{name}_test.go"
+    with filename.open("w") as fp:
+        fp.write(templates["test"].render(name=name, operations=resource))
+    os.system(shlex.quote(f"go fmt {filename}"))
+
+    # TF resource example
+    filename = output.parent / f"examples/resources/datadog_{name}/resource.tf"
+    with filename.open("w") as fp:
+        fp.write(templates["example"].render(name=name, operations=resource))
+
+    # TF import example
+    filename = output.parent / f"examples/resources/datadog_{name}/import.sh"
+    with filename.open("w") as fp:
+        fp.write(templates["import"].render(name=name, operations=resource))

.generator/src/generator/__main__.py

@@ -0,0 +1,141 @@
+"""Data formatter."""
+
+from .utils import snake_case, camel_case, untitle_case, schema_name
+
+PRIMITIVE_TYPES = ["string", "number", "boolean", "integer"]
+
+KEYWORDS = {
+    "break",
+    "case",
+    "chan",
+    "const",
+    "continue",
+    "default",
+    "defer",
+    "else",
+    "fallthrough",
+    "for",
+    "func",
+    "go",
+    "goto",
+    "if",
+    "import",
+    "interface",
+    "map",
+    "meta",
+    "package",
+    "range",
+    "return",
+    "select",
+    "struct",
+    "switch",
+    "type",
+    "var",
+}
+
+SUFFIXES = {
+    # Test
+    "test",
+    # $GOOS
+    "aix",
+    "android",
+    "darwin",
+    "dragonfly",
+    "freebsd",
+    "illumos",
+    "js",
+    "linux",
+    "netbsd",
+    "openbsd",
+    "plan9",
+    "solaris",
+    "windows",
+    # $GOARCH
+    "386",
+    "amd64",
+    "arm",
+    "arm64",
+    "mips",
+    "mips64",
+    "mips64le",
+    "mipsle",
+    "ppc64",
+    "ppc64le",
+    "s390x",
+    "wasm",
+}
+
+
+def sanitize_description(description):
+    escaped_description = description.replace('"', '\\"')
+    return " ".join(escaped_description.splitlines())
+
+
+def escape_reserved_keyword(word):
+    """
+    Escape reserved language keywords like openapi generator does it
+    :param word: Word to escape
+    :return: The escaped word if it was a reserved keyword, the word unchanged otherwise
+    """
+    if word in KEYWORDS:
+        return f"{word}Var"
+    return word
+
+
+def attribute_name(attribute):
+    return escape_reserved_keyword(camel_case(attribute))
+
+
+def variable_name(attribute):
+    return escape_reserved_keyword(untitle_case(camel_case(attribute)))
+
+
+def simple_type(schema, render_nullable=False, render_new=False):
+    """Return the simple type of a schema.
+
+    :param schema: The schema to extract the type from
+    :return: The simple type name
+    """
+    type_name = schema.get("type")
+    type_format = schema.get("format")
+    nullable = render_nullable and schema.get("nullable", False)
+
+    nullable_prefix = "datadog.NewNullable" if render_new else "datadog.Nullable"
+
+    if type_name == "integer":
+        return {
+            "int32": "int32" if not nullable else f"{nullable_prefix}Int32",
+            "int64": "int64" if not nullable else f"{nullable_prefix}Int64",
+            None: "int32" if not nullable else f"{nullable_prefix}Int32",
+        }[type_format]
+
+    if type_name == "number":
+        return {
+            "double": "float64" if not nullable else f"{nullable_prefix}Float64",
+            None: "float" if not nullable else f"{nullable_prefix}Float",
+        }[type_format]
+
+    if type_name == "string":
+        return {
+            "date": "time.Time" if not nullable else f"{nullable_prefix}Time",
+            "date-time": "time.Time" if not nullable else f"{nullable_prefix}Time",
+            "email": "string" if not nullable else f"{nullable_prefix}String",
+            "binary": "*os.File",
+            None: "string" if not nullable else f"{nullable_prefix}String",
+        }[type_format]
+    if type_name == "boolean":
+        return "bool" if not nullable else f"{nullable_prefix}Bool"
+
+    return None
+
+
+def get_terraform_schema_type(schema):
+    return {
+        "string": "String",
+        "boolean": "Bool",
+        "integer": "Int64",
+        "number": "Int64",
+        "array": "List",
+        "object": "Block",
+        None: "String",
+    }[schema.get("type")]