Merge pull request #247 from GlodoUK/cloc-19.0

[19.0][FEAT] cloc Implementation

Author: GarethGloSystems

glodo_client/__manifest__.py

@@ -7,7 +7,7 @@
     "version": "19.0.1.0.0",
     "depends": ["base"],
     "external_dependencies": {
-        "python": ["cryptography"],
+        "python": ["cryptography", "manifestoo_core"],
     },
     "license": "Other proprietary",
 }

glodo_client/controllers/main.py

@@ -17,8 +17,8 @@
 from odoo.http import Controller, request, route
 from odoo.modules.registry import Registry
 from odoo.service.db import list_dbs
-from odoo.tools import cloc
 
+from ..utils.cloc import CustomCloc
 from ..utils.crypto import (
     get_client_config,
     glodo_authenticated,
@@ -78,7 +78,7 @@ def info(self, **kwargs):
 
         Response includes:
         - Instance-level information (Odoo version, etc.)
-        - List of databases with CLOC and module info
+        - List of databases with module info and CLOC breakdown
         """
         # payload available via request.glodo_payload if needed
 
@@ -139,27 +139,22 @@ def _get_database_info(self, db_name: str) -> dict:
                     for m in modules
                 ]
 
+                try:
+                    cl = CustomCloc()
+                    cl.count_env(env)
+                    cloc_data = cl.summary()
+                except Exception as e:
+                    cloc_data = {"error": str(e)}
+
                 db_info = {
                     "name": db_name,
                     "user_count": user_count,
                     "expiration_date": expiration_date or None,
                     "expiration_reason": expiration_reason or None,
                     "installed_modules": module_list,
-                    "cloc": {},
+                    "cloc": cloc_data,
                 }
 
-                # Try to get CLOC
-                try:
-                    cl = cloc.Cloc()
-                    cl.count_customization(env)
-
-                    db_info["cloc"] = {
-                        "output": cl.report(),
-                        "returncode": cl.code,
-                    }
-                except Exception as e:
-                    db_info["cloc"] = {"error": str(e)}
-
                 return db_info
 
         except Exception as e:

glodo_client/utils/__init__.py

@@ -1 +1,2 @@
+from . import cloc
 from . import crypto

glodo_client/utils/cloc.py

@@ -0,0 +1,94 @@
+import ast
+import os
+import pathlib
+
+from manifestoo_core.core_addons import is_core_addon
+from manifestoo_core.odoo_series import OdooSeries, UnsupportedOdooSeries
+
+from odoo import release
+from odoo.modules import Manifest
+from odoo.modules.module import MANIFEST_NAMES
+from odoo.tools.cloc import DEFAULT_EXCLUDE, MAX_FILE_SIZE, VALID_EXTENSION, Cloc
+
+DEFAULT_EXCLUDE_INCLUDE_TESTS = [
+    p for p in DEFAULT_EXCLUDE if not p.startswith(("tests/", "static/tests/"))
+]
+
+
+class CustomCloc(Cloc):
+    def summary(self):
+        return {
+            "code": dict(self.code),
+            "errors": {m: dict(files) for m, files in self.errors.items()},
+        }
+
+    def count_modules(self, env):
+        try:
+            major, minor = release.version_info[:2]
+            series = OdooSeries(f"{major}.{minor}")
+        except (ValueError, UnsupportedOdooSeries):
+            series = None
+
+        domain = [("state", "=", "installed")]
+        if env["ir.module.module"]._fields.get("imported"):
+            domain.append(("imported", "=", False))
+        module_list = env["ir.module.module"].search(domain).mapped("name")
+
+        for module_name in module_list:
+            if series and is_core_addon(module_name, series):
+                continue
+            manifest = Manifest.for_addon(module_name)
+            if not manifest:
+                continue
+            self.count_path(manifest.path)
+
+    # Exact copy of odoo.tools.cloc.Cloc.count_path with DEFAULT_EXCLUDE → DEFAULT_EXCLUDE_INCLUDE_TESTS
+
+    # fmt: off
+    # pylint: disable=broad-except,except-pass
+    # ruff: noqa: E501
+    def count_path(self, path, exclude=None):
+        path = path.rstrip('/')
+        exclude_list = []
+        for i in MANIFEST_NAMES:
+            manifest_path = os.path.join(path, i)
+            try:
+                with open(manifest_path, 'rb') as manifest:
+                    exclude_list.extend(DEFAULT_EXCLUDE_INCLUDE_TESTS)
+                    d = ast.literal_eval(manifest.read().decode('latin1'))
+                    for j in ['cloc_exclude', 'demo', 'demo_xml']:
+                        exclude_list.extend(d.get(j, []))
+                    break
+            except Exception:
+                pass
+        if not exclude:
+            exclude = set()
+        for i in filter(None, exclude_list):
+            assert '..' not in i, (
+                f"Invalid exclusion path '{i}': '..' is not allowed. Use a normalized path."
+            )
+            exclude.update(str(p) for p in pathlib.Path(path).glob(i))
+
+        module_name = os.path.basename(path)
+        self.book(module_name)
+        for root, _dirs, files in os.walk(path):
+            for file_name in files:
+                file_path = os.path.join(root, file_name)
+
+                if file_path in exclude:
+                    continue
+
+                ext = os.path.splitext(file_path)[1].lower()
+                if ext not in VALID_EXTENSION:
+                    continue
+
+                if os.path.getsize(file_path) > MAX_FILE_SIZE:
+                    self.book(module_name, file_path, (-1, "Max file size exceeded"))
+                    continue
+
+                with open(file_path, 'rb') as f:
+                    # Decode using latin1 to avoid error that may raise by decoding with utf8
+                    # The chars not correctly decoded in latin1 have no impact on how many lines will be counted
+                    content = f.read().decode('latin1')
+                self.book(module_name, file_path, self.parse(content, ext))
+    # fmt: on

glodo_server/models/glodo_instance.py

@@ -105,8 +105,22 @@ class GlodoInstance(models.Model):
         readonly=True,
     )
 
+    cloc_excluded_modules = fields.Char(
+        string="CLOC Excluded Modules",
+        help="Comma-separated list of module names (or the literal "
+        "'odoo/studio') to exclude from CLOC totals for this instance. "
+        "Applies to every database under the instance.",
+    )
+
     notes = fields.Text()
 
+    def _parse_excluded_modules(self):
+        self.ensure_one()
+        raw = self.cloc_excluded_modules
+        if not raw:
+            return set()
+        return {part.strip() for part in raw.split(",") if part.strip()}
+
     @api.depends("database_ids")
     def _compute_database_count(self):
         for instance in self:
@@ -325,8 +339,8 @@ def action_sync_info(self):
                 "installed_modules_json": json.dumps(
                     db_data.get("installed_modules", [])
                 ),
-                "cloc_output": db_data.get("cloc", {}).get("output", ""),
             }
+            db_vals.update(Database._cloc_vals_from_payload(db_data.get("cloc") or {}))
 
             if existing:
                 existing.write(db_vals)

glodo_server/models/glodo_instance_database.py

@@ -109,9 +109,22 @@ class GlodoInstanceDatabase(models.Model):
         compute="_compute_installed_modules_html",
     )
 
-    cloc_output = fields.Text(
-        string="CLOC Output",
+    cloc_data_json = fields.Text(
+        string="CLOC (JSON)",
         readonly=True,
+        help="Raw CLOC payload as reported by the remote.",
+    )
+
+    cloc_total = fields.Integer(
+        string="CLOC Total",
+        compute="_compute_cloc_totals",
+        store=True,
+        readonly=True,
+        help="Lines of code counted in custom-module source trees (including "
+        "tests/ and static/tests/) plus studio actions, manual compute fields, "
+        "and imported-module artifacts stored in the database. Excludes core, "
+        "enterprise, and any modules listed in the instance's CLOC Excluded "
+        "Modules.",
     )
 
     last_user_sync = fields.Datetime(
@@ -196,6 +209,37 @@ def _search_expiration_state(self, operator, value):
             )
         return Domain.OR(domains)
 
+    @api.model
+    def _cloc_vals_from_payload(self, cloc_payload):
+        """Return ``{cloc_data_json: ...}`` from a ``cloc`` payload.
+
+        The payload follows the shape emitted by
+        ``glodo_client.utils.cloc.CustomCloc.summary``: a dict with ``code``
+        (per-module LOC, source + customization fused) and ``errors``. The
+        total is derived in ``_compute_cloc_totals`` so it picks up changes
+        to the instance's excluded-modules list without re-syncing.
+        """
+        if not isinstance(cloc_payload, dict) or not cloc_payload:
+            return {"cloc_data_json": False}
+        return {"cloc_data_json": json.dumps(cloc_payload, sort_keys=True)}
+
+    @api.depends("cloc_data_json", "instance_id.cloc_excluded_modules")
+    def _compute_cloc_totals(self):
+        for db in self:
+            total = 0
+            if db.cloc_data_json:
+                try:
+                    payload = json.loads(db.cloc_data_json)
+                except Exception:
+                    payload = {}
+                excluded = db.instance_id._parse_excluded_modules()
+                total = sum(
+                    v
+                    for k, v in (payload.get("code") or {}).items()
+                    if k not in excluded and isinstance(v, int)
+                )
+            db.cloc_total = total
+
     @api.depends("installed_modules_json")
     def _compute_installed_modules_html(self):
         for db in self:

glodo_server/views/glodo_instance_database_views.xml

@@ -96,7 +96,10 @@
                             />
                         </page>
                         <page string="CLOC">
-                            <field name="cloc_output" widget="text" readonly="1" />
+                            <group>
+                                <field name="cloc_total" />
+                            </group>
+                            <field name="cloc_data_json" readonly="1" />
                         </page>
                         <page string="Notes">
                             <field
@@ -129,6 +132,7 @@
                     optional="hide"
                     sum="Inactive Users"
                 />
+                <field name="cloc_total" optional="hide" sum="CLOC Total" />
                 <field name="last_user_sync" />
             </list>
         </field>

glodo_server/views/glodo_instance_views.xml

@@ -134,6 +134,14 @@
                                 </list>
                             </field>
                         </page>
+                        <page string="CLOC">
+                            <group>
+                                <field
+                                    name="cloc_excluded_modules"
+                                    placeholder="e.g. connector_edi, web_cmd_search"
+                                />
+                            </group>
+                        </page>
                         <page string="Notes">
                             <field
                                 name="notes"

requirements.txt

@@ -1,2 +1,3 @@
 # generated from manifests external_dependencies
 cryptography
+manifestoo_core