make script more robust

Author: mvadari

scripts/validate_xls_template.py

@@ -30,6 +30,8 @@
 from dataclasses import dataclass
 from pathlib import Path
 from typing import List, Optional, Tuple
+import urllib.request
+import urllib.error
 
 from markdown_it import MarkdownIt
 from xls_parser import extract_xls_metadata
@@ -86,6 +88,10 @@ class XLSTemplateValidator:
     # Valid category values
     VALID_CATEGORIES = ["Amendment", "System", "Ecosystem", "Meta"]
 
+    # Note: Ledger entry and transaction existence is now determined dynamically
+    # by checking if the documentation exists on xrpl.org rather than
+    # maintaining hardcoded lists.
+
     # Amendment template section patterns and their required subsections
     AMENDMENT_SECTION_TEMPLATES = {
         r"SType:": {
@@ -228,11 +234,11 @@ def _parse_sections(self):
         # Walk through tokens to find headings
         for i, token in enumerate(tokens):
             if token.type == 'heading_open':
-                # Get the heading level (h2 = level 2, h3 = level 3, etc.)
+                # Get the heading level (h1 = level 1, h2 = level 2, etc.)
                 level = int(token.tag[1])
 
-                # Only process h2 (top-level sections)
-                if level != 2:
+                # Skip h1 (document title); process h2+ for structure
+                if level < 2:
                     continue
 
                 # The next token should be 'inline' containing heading text
@@ -396,17 +402,125 @@ def _validate_amendment_structure(self):
         for section in main_sections:
             for pattern, template in self.AMENDMENT_SECTION_TEMPLATES.items():
                 if re.search(pattern, section.title):
+                    # Check if this is a Ledger Entry section for an
+                    # existing ledger entry type
+                    is_existing_ledger_entry = False
+                    if pattern == r"Ledger Entry:":
+                        is_existing_ledger_entry = (
+                            self._is_existing_ledger_entry(section.title)
+                        )
+
+                    # Check if this is a Transaction section for an
+                    # existing transaction type
+                    is_existing_transaction = False
+                    if pattern == r"Transaction:":
+                        is_existing_transaction = (
+                            self._is_existing_transaction(section.title)
+                        )
+
                     self._validate_subsections(
                         section,
                         template["required_subsections"],
-                        template["optional_subsections"]
+                        template["optional_subsections"],
+                        is_existing_ledger_entry,
+                        is_existing_transaction
                     )
 
+    def _is_existing_ledger_entry(self, section_title: str) -> bool:
+        """
+        Check if a Ledger Entry section is for an existing ledger entry.
+
+        Extracts the ledger entry name from the section title and checks
+        if the ledger entry exists on xrpl.org by fetching the URL:
+        https://xrpl.org/docs/references/protocol/ledger-data/ledger-entry-types/[lowercase]
+
+        Args:
+            section_title: The title of the section
+                          (e.g., "Ledger Entry: `AccountRoot`")
+
+        Returns:
+            True if this is an existing ledger entry type, False otherwise
+        """
+        # Extract ledger entry name from title like "Ledger Entry: `Foo`"
+        # or "2. Ledger Entry: `Foo`"
+        match = re.search(r'Ledger Entry:\s*`?([A-Za-z]+)`?', section_title)
+        if not match:
+            return False
+
+        entry_name = match.group(1)
+
+        # Convert to lowercase for URL
+        lowercase_name = entry_name.lower()
+        url = (
+            "https://xrpl.org/docs/references/protocol/"
+            f"ledger-data/ledger-entry-types/{lowercase_name}"
+        )
+
+        try:
+            # Try to fetch the URL
+            req = urllib.request.Request(url, method='HEAD')
+            with urllib.request.urlopen(req, timeout=5) as response:
+                # If we get a 200 response, the ledger entry exists
+                return response.status == 200
+        except urllib.error.HTTPError as e:
+            # 404 means the ledger entry doesn't exist
+            if e.code == 404:
+                return False
+            # Other errors (500, etc.) - assume it exists to be safe
+            return True
+        except (urllib.error.URLError, TimeoutError):
+            # Network error - assume it exists to be safe
+            return True
+
+    def _is_existing_transaction(self, section_title: str) -> bool:
+        """
+        Check if a Transaction section is for an existing transaction type.
+
+        Extracts the transaction name from the section title and checks
+        if the transaction exists on xrpl.org by fetching the URL:
+        https://xrpl.org/docs/references/protocol/transactions/types/[lowercase]
+
+        Args:
+            section_title: The title of the section (e.g., "Transaction: `Payment`")
+
+        Returns:
+            True if this is an existing transaction type, False otherwise
+        """
+        # Extract transaction name from title like "Transaction: `Payment`"
+        # or "3. Transaction: `Payment`"
+        match = re.search(r'Transaction:\s*`?([A-Za-z]+)`?', section_title)
+        if not match:
+            return False
+
+        transaction_name = match.group(1)
+
+        # Convert to lowercase for URL
+        lowercase_name = transaction_name.lower()
+        url = f"https://xrpl.org/docs/references/protocol/transactions/types/{lowercase_name}"
+
+        try:
+            # Try to fetch the URL
+            req = urllib.request.Request(url, method='HEAD')
+            with urllib.request.urlopen(req, timeout=5) as response:
+                # If we get a 200 response, the transaction exists
+                return response.status == 200
+        except urllib.error.HTTPError as e:
+            # 404 means the transaction doesn't exist
+            if e.code == 404:
+                return False
+            # Other errors (500, etc.) - assume it exists to be safe
+            return True
+        except (urllib.error.URLError, TimeoutError):
+            # Network error - assume it exists to be safe
+            return True
+
     def _validate_subsections(
         self,
         parent_section: Section,
         required_subsections: dict,
-        optional_subsections: dict
+        optional_subsections: dict,
+        is_existing_ledger_entry: bool = False,
+        is_existing_transaction: bool = False
     ):
         """Validate that a section has required subsections."""
         # Get all subsections under this parent
@@ -424,8 +538,27 @@ def _validate_subsections(
             if section.level == parent_section.level + 1:
                 subsections.append(section)
 
+        # For existing ledger entries, make certain subsections optional
+        exempted_subsections = set()
+        if is_existing_ledger_entry:
+            exempted_subsections = {
+                "Object Identifier",
+                "Ownership",
+                "Reserves",
+                "Deletion",
+                "RPC Name"
+            }
+
+        # For existing transactions, make Transaction Fee optional
+        if is_existing_transaction:
+            exempted_subsections.add("Transaction Fee")
+
         # Check for required subsections
         for sub_num, sub_title in required_subsections.items():
+            # Skip if this subsection is exempted
+            if sub_title in exempted_subsections:
+                continue
+
             # Find this subsection by title, regardless of its actual number
             found = any(
                 sub_title in s.title

templates/AMENDMENT_TEMPLATE.md

@@ -42,6 +42,8 @@ _[If your specification introduces new ledger entry objects, document each entry
 
 _[If your specification introduces new ledger entry common fields, you can have a section called `Transaction: Common Fields` before listing out any specific transactions.]_
 
+_[**Note for existing ledger entries:** If you are documenting changes to an existing ledger entry type that is already deployed on the XRPL mainnet (e.g., AccountRoot, RippleState, Offer, etc.), the following subsections are **optional**: Ownership (2.4), Reserves (2.5), Deletion (2.6), and RPC Name (2.10). You only need to include these subsections if you are making changes to those aspects of the ledger entry.]_
+
 ### 2.1. Object Identifier
 
 **Key Space:** `0x[XXXX]` _[Specify the 16-bit hex value for the key space]_
@@ -77,15 +79,17 @@ _[Detailed explanation of field behavior, validation rules, etc.]_
 | ---------------- | ------------ | ---------------- |
 | `[lsfFlagName1]` | `0x[Value1]` | `[Description1]` |
 
-### 2.4. Ownership
+### 2.4. Ownership _(Optional if ledger entry already exists on XRPL)_
 
 _[Specify which AccountRoot object owns this ledger entry and how the ownership relationship is established.]_
 
 **Owner:** `[Account field name or "No owner" if this is a global object like FeeSettings]`
 
 **Directory Registration:** `[Describe how this object is registered in the owner's directory, or specify if it's a special case]`
 
-### 2.5. Reserves
+_[Note: This subsection is optional if you are documenting changes to an existing ledger entry type that is already deployed on the XRPL mainnet.]_
+
+### 2.5. Reserves _(Optional if ledger entry already exists on XRPL)_
 
 **Reserve Requirement:** `[Standard/Custom/None]`
 
@@ -95,7 +99,9 @@ _[If Custom]: This ledger entry requires `[X]` reserve units because `[reason]`.
 
 _[If None]: This ledger entry does not require additional reserves because `[reason]`._
 
-### 2.6. Deletion
+_[Note: This subsection is optional if you are documenting changes to an existing ledger entry type that is already deployed on the XRPL mainnet.]_
+
+### 2.6. Deletion _(Optional if ledger entry already exists on XRPL)_
 
 **Deletion Transactions:** `[List transaction types that can delete this object]`
 
@@ -109,6 +115,8 @@ _[If None]: This ledger entry does not require additional reserves because `[rea
 _[If Yes]: This object must be deleted before its owner account can be deleted._
 _[If No]: This object does not prevent its owner account from being deleted._
 
+_[Note: This subsection is optional if you are documenting changes to an existing ledger entry type that is already deployed on the XRPL mainnet.]_
+
 ### 2.7. Pseudo-Account _(Optional)_
 
 _[Only include this section if your ledger entry uses a pseudo-account. Otherwise, delete this subsection.]_
@@ -138,12 +146,14 @@ _[List logical statements that must always be true for this ledger entry. Use `<
 - `[Invariant 2, e.g., "IF <object>.Status == 'Active' THEN <object>.Account != NULL"]`
 - `[Additional invariants as needed]`
 
-### 2.10. RPC Name
+### 2.10. RPC Name _(Optional if ledger entry already exists on XRPL)_
 
 **RPC Type Name:** `[snake_case_name]`
 
 _[This is the name used in `account_objects` and `ledger_data` RPC calls to filter for this object type]_
 
+_[Note: This subsection is optional if you are documenting changes to an existing ledger entry type that is already deployed on the XRPL mainnet.]_
+
 ### 2.11. Example JSON
 
 ```json
@@ -165,6 +175,12 @@ _[If your specification introduces new transactions, document each transaction i
 
 _[If your specification introduces new transaction common fields, you can have a section called `Transaction: Common Fields` before listing out any specific transactions.]_
 
+> **Note for Existing Transactions:** If you are documenting changes to an existing transaction type (one that is already deployed on XRPL mainnet), the following subsection is **optional** unless you are modifying it:
+>
+> - **Transaction Fee** (3.3)
+>
+> For new transaction types, all subsections are required.
+
 > **Naming Convention:** Transaction names should follow the pattern `<LedgerEntryName><Verb>` (e.g., `ExampleSet`, `ExampleDelete`). Most specifications will need at least:
 >
 > - `[Object]Set` or `[Object]Create`: Creates or updates the object
@@ -194,7 +210,9 @@ _[Detailed explanation of field behavior, validation rules, etc.]_
 | --------------- | ------------ | ---------------- |
 | `[tfFlagName1]` | `0x[Value1]` | `[Description1]` |
 
-### 3.3. Transaction Fee
+### 3.3. Transaction Fee _(Optional if transaction already exists on XRPL)_
+
+_[This subsection is optional if you are documenting changes to an existing transaction type. Only include it if you are introducing a new transaction type or modifying the fee structure of an existing one.]_
 
 **Fee Structure:** `[Standard/Custom]`