Merge pull request #67 from infragov-project/rego_integration

Rego integration on security smells

Author: Nfsaavedra

.github/workflows/rego_python.yml

@@ -0,0 +1,180 @@
+name: rego-python
+
+on:
+  push:
+    paths:
+      - ".github/workflows/rego-python.yml"
+      - "glitch/rego/rego_python/**"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  build-binaries:
+    name: Build Rego Python binaries
+    runs-on: ${{ matrix.os_runner }}
+    strategy:
+      matrix:
+        include:
+          # Linux
+          - os: linux
+            arch: amd64
+            os_runner: ubuntu-latest
+          - os: linux
+            arch: arm64
+            os_runner: ubuntu-24.04-arm
+
+          # macOS
+          - os: darwin
+            arch: amd64
+            os_runner: macos-15-intel
+          - os: darwin
+            arch: arm64
+            os_runner: macos-latest
+
+          # Windows
+          - os: windows
+            arch: amd64
+            os_runner: windows-latest
+          #- os: windows
+          #  arch: arm64
+          #  os_runner: windows-11-arm
+          # This fails the build because the runner is still limited
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 'stable'
+          cache-dependency-path: glitch/rego/rego_python/src/rego_python/go/go.sum
+
+      - name: Build binary
+        shell: bash
+        run: |
+          OS=${{ matrix.os }}
+          ARCH=${{ matrix.arch }}
+
+          # Determine file extension
+          if [ "$OS" = "windows" ]; then
+            EXT="dll"
+          elif [ "$OS" = "darwin" ]; then
+            EXT="dylib"
+          else
+            EXT="so"
+          fi
+
+          OUTPUT="../bin/librego-$OS-$ARCH.$EXT"
+          echo "Building $OUTPUT ..."
+
+          cd glitch/rego/rego_python/src/rego_python/go
+          GOOS=$OS GOARCH=$ARCH go build -o "$OUTPUT" -buildmode=c-shared regolib.go
+          rm -f ../bin/librego-*.h
+
+      - name: List bin folder
+        run: ls -l glitch/rego/rego_python/src/rego_python/bin/
+      
+      - name: Upload binary artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: librego-${{ matrix.os }}-${{ matrix.arch }}
+          path: glitch/rego/rego_python/src/rego_python/bin/librego-${{ matrix.os }}-${{ matrix.arch }}.*
+          retention-days: 1
+
+  build-package:
+    name: Build Python package
+    runs-on: ubuntu-latest
+    needs: build-binaries
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # Download all binary artifacts into bin/
+      - uses: actions/download-artifact@v4
+        with:
+          path: glitch/rego/rego_python/src/rego_python/bin
+          merge-multiple: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Install build tools
+        run: |
+          python -m pip install --upgrade build twine
+
+      - name: Build wheel and sdist
+        working-directory: glitch/rego/rego_python
+        run: python -m build
+
+      - name: Upload package artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package
+          path: glitch/rego/rego_python/dist/*
+          retention-days: 1
+
+  release:
+    name: Create GitHub Release
+    runs-on: ubuntu-latest
+    needs: build-package 
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Extract version from pyproject.toml
+        id: get_version
+        run: |
+          VERSION=$(grep '^version = ' glitch/rego/rego_python/pyproject.toml | sed -E 's/version = "(.*)"/\1/')
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Version extracted: $VERSION"
+
+      - name: Create tag for release
+        run: |
+          git config user.name "rego_python-release-bot"
+          git config user.email "rego_python@users.noreply.github.com"
+          TAG=rego_python-v${{ steps.get_version.outputs.version }}
+          git tag $TAG
+          git push origin $TAG
+
+          NON_PREFIXED_TAG=v${{ steps.get_version.outputs.version }}
+          if git rev-parse "$NON_PREFIXED_TAG" >/dev/null 2>&1; then
+            echo "WARNING: Non-prefixed tag $NON_PREFIXED_TAG exists. Consider deleting it to avoid confusion."
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download built package and binaries
+        uses: actions/download-artifact@v4
+        with:
+          path: release_assets
+          merge-multiple: true
+
+      - name: List downloaded assets
+        run: |
+          echo "Listing contents of release_assets directory:"
+          ls -lR release_assets
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: "rego_python-v${{ steps.get_version.outputs.version }}"
+          name: "Rego Python v${{ steps.get_version.outputs.version }}"
+          files: |
+            release_assets/*.whl
+            release_assets/*.tar.gz
+            release_assets/*.so
+            release_assets/*.dylib
+            release_assets/*.dll
+          generate_release_notes: false
+          draft: false
+          prerelease: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 
 # C extensions
 *.so
+glitch/rego/go/library/librego.h
 
 # Distribution / packaging
 .Python

README.md

@@ -58,6 +58,32 @@ To install GLITCH using Poetry, run:
 poetry install
 ```
 
+### Rego
+
+To do all checks, you need to use Rego. For that, you need the appropriate binary for your environment and architecture.
+
+You can find these binaries in the Rego Python release.
+
+Or, you can compile your own binary, but this requires to have ```go``` installed. To do that, run the following commands on the root folder:
+
+```
+OS={Operating System}
+ARCH=${{ Architecture }}
+# Determine file extension
+if [ "$OS" = "windows" ]; then
+  EXT="dll"
+elif [ "$OS" = "darwin" ]; then
+  EXT="dylib"
+else
+  EXT="so"
+fi
+
+OUTPUT="../bin/librego-$OS-$ARCH.$EXT"
+
+cd glitch/rego/rego_python/src/rego_python/go
+GOOS=$OS GOARCH=$ARCH go build -o "$OUTPUT" -buildmode=c-shared regolib.go
+```
+
 **WARNING**: _For now, the GLITCH VSCode extension does not function if GLITCH 
 is installed via Poetry. Since Poetry uses virtual environments it does not 
 create a binary for GLITCH available in the user's PATH, which is required for 
@@ -92,9 +118,9 @@ glitch lint --help
 
 ## Tests
 
-To run the tests for GLITCH go to the folder ```glitch``` and run:
+To run the tests for GLITCH run the following command:
 ```
-python -m unittest discover tests
+poetry run pytest -s
 ```
 
 ## Configs

glitch/__main__.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 from typing import Tuple, List, Set, Optional, TextIO, Dict, Any
 from glitch.analysis.rules import Error, RuleVisitor
-from glitch.helpers import get_smell_types, get_smells
+from glitch.helpers import get_smell_types, get_smells, ini_to_json_dict
 from glitch.parsers.docker import DockerParser
 from glitch.stats.print import print_stats
 from glitch.stats.stats import FileStats
@@ -23,6 +23,7 @@
 from pkg_resources import resource_filename
 from copy import deepcopy
 from concurrent.futures import ThreadPoolExecutor, Future, as_completed
+from glitch.rego.engine import load_rego_from_path, run_analyses
 
 
 # NOTE: These are necessary in order for python to load the visitors.
@@ -38,20 +39,62 @@ def __parse_and_check(
     parser: Parser,
     analyses: List[RuleVisitor],
     stats: FileStats,
+    config_rego: str,
+    rego_modules: Dict[str,str]
 ) -> Set[Error]:
     errors: Set[Error] = set()
     inter = parser.parse(path, type, module)
     # Avoids problems with multiple threads (and possibly multiple files)
     # sharing the same object
+    
     analyses = deepcopy(analyses)
-
     if inter != None:
         for analysis in analyses:
             errors.update(analysis.check(inter))
+    
+        inputRego = json.dumps(inter.as_dict(), indent=2)
+        
+        errors.update(run_analyses(inputRego, config_rego, rego_modules))
+        
         stats.compute(inter)
 
     return errors
 
+def __filter_analysis(
+    smell_types: Tuple[str, ...],
+    config: str,
+    tech: Tech
+) -> Tuple[Dict[str,str], List[RuleVisitor]]:
+    rego_modules: Dict[str,str] = {}
+    python_analyses: List[RuleVisitor] = []
+
+    if not os.path.exists("./glitch/rego/queries/library/glitch_lib.rego"):
+        raise FileNotFoundError("The rego query library does not exist.")
+    load_rego_from_path("./glitch/rego/queries/library/glitch_lib.rego", rego_modules)
+
+    for smell_type in smell_types:
+        smells: List[str] = get_smells([smell_type], tech)
+        fallback: Set[str] = set()
+
+        for smell in smells:
+            if os.path.exists(f"./glitch/rego/queries/{smell_type}/{smell}.rego"):
+                load_rego_from_path(f"./glitch/rego/queries/{smell_type}/{smell}.rego", rego_modules)
+            else:
+                fallback.add(smell)
+    
+        if len(fallback) > 0:
+            match smell_type:
+                case "design":
+                    visitor = DesignVisitor(tech, fallback)
+                case "security":
+                    visitor = SecurityVisitor(tech, fallback)
+                case _:
+                    raise ValueError(f"Invalid smell type: {smell_type}")
+
+            visitor.config(config)
+            python_analyses.append(visitor)
+
+    return rego_modules, python_analyses
 
 def __get_tech(tech: str) -> Tech:
     for t in Tech:
@@ -217,7 +260,7 @@ def lint(
     output: Optional[str],
     table_format: str,
     linter: bool,
-    n_workers: int,
+    n_workers: int
 ):
     tech: Tech = __get_tech(tech)
     type = UnitBlockType(type)
@@ -238,17 +281,15 @@ def lint(
     if tech == Tech.terraform:
         config = resource_filename("glitch", "configs/terraform.ini")
     file_stats = FileStats()
-
+    
     if smell_types == ():
         smell_types = get_smell_types()
 
-    analyses: List[RuleVisitor] = []
-    rules = RuleVisitor.__subclasses__()
-    for r in rules:
-        if smell_types == () or r.get_name() in smell_types:
-            analysis = r(tech)
-            analysis.config(config)
-            analyses.append(analysis)
+    config_rego = ini_to_json_dict(config)
+    
+    temp = __filter_analysis(smell_types, config, tech)
+    rego_modules: Dict[str,str] = temp[0] # type: ignore
+    analyses: List[RuleVisitor] = temp[1]
 
     errors: List[Error] = []
     paths: Set[str]
@@ -261,12 +302,14 @@ def lint(
     for p in paths:
         futures.append(
             executor.submit(
-                __parse_and_check, type, p, module, parser, analyses, file_stats
+                __parse_and_check, type, p, module, parser, analyses, file_stats, config_rego, rego_modules
             )
         )
         future_to_path[futures[-1]] = p
 
     f = sys.stdout if output is None else open(output, "w")
+    if csv:
+        print("PATH,LINE,ERROR,CODE,DESCRIPTION", file=f)
     for future in tqdm.tqdm(as_completed(futures), total=len(futures), desc=title):
         try:
             new_errors = future.result()

glitch/analysis/design/duplicate_block.py

@@ -4,7 +4,7 @@
 from glitch.repr.inter import *
 
 
-class UnguardedVariable(DesignSmellChecker):
+class DuplicateBlock(DesignSmellChecker):
     def __get_line(self, i: int, lines: List[Tuple[int, int]]):
         for j, line in lines:
             if i < j: