Merge pull request #281 from P6g9YHK6/main

updates + new pwsh snippet + rustdesk exemples

Author: silversword411

scripts_staging/Backend/Sync TRMM with GIT.py

@@ -93,16 +93,19 @@
     v9.0.3.2 24/04/25 SAN couple of pre-flight fixes
     v9.0.3.3 24/04/25 SAN fix commit errors
     v9.0.4.0 24/04/25 SAN New commit design, decreased importance of uncommited at git check, added emojis ✅, bugfix on git stdout
-
+    v9.0.4.1 28/04/25 SAN Paranoid check added to avoid random deletion, more verbose output on file deletion, moved folder check after git as git require an empty folder when first cloning, couple of pre-flight fixes
+    v9.0.4.2 29/04/25 SAN more explicit part 2 & 3 outputs, added RW check
+    v9.0.4.3 29/04/25 SAN more explicit outputs for git pull & fix other output
+    v9.0.4.4 30/04/25 SAN fix pull
 
 .TODO
-    Handle rights issues when executing git commands
+    move env setup to pre-flight
     Review flow of step 3 for optimisations
-
+    Review the counters for step 3
     Revamp folder structure:
-        Move raws from "scriptsraw" to Category/folder/raws/
+        Move raws from "scriptsraw" to Category_name/raws/
         add "uncategorised" folder
-        remove "scripts" top level folder while keeping snippets
+        remove "scripts" top level folder while keeping snippets and move snippets raws to snippets/raws/
 
     Move ID from json to an array like this and make sure that this array is never overwriten to keep tracks of IDs across instances only add current instance in step 2 if missing:
         "ids": [
@@ -115,6 +118,7 @@
     Delete script support from git ? (dedicated function required at the end of step 2, if json exist but no script matches mark for delete json and use the id of the json to tell the api to delete in trmm)
     Squash commit from minor update json with previous commit
     Add reporting support
+    
 
 """
 
@@ -133,7 +137,6 @@
 
 # Retrieve the git pull branch or default to 'master'
 git_pull_branch = os.getenv('GIT_PULL_BRANCH', 'master')
-if git_pull_branch != 'master': print(f"Git Pull Branch: {git_pull_branch}")
 
 # Retrieve flags from environment variables (default to True unless set to 'false')
 ENABLE_GIT_PULL = os.getenv('ENABLE_GIT_PULL', 'True').lower() != 'false'
@@ -147,35 +150,49 @@
 
 
 def delete_obsolete_files(folder, current_scripts):
-    print(f"Cleaning {folder}...")
-    obsolete = {f for f in folder.rglob('*') if f.is_file() and f.relative_to(folder) not in current_scripts}
-    
+    if not current_scripts:
+        print("❌ ERROR: No valid scripts provided by api. Aborting.")
+        sys.exit(1)
+    if not isinstance(current_scripts, set):
+        print("❌ ERROR: 'current_scripts' must be a set. Aborting.")
+        sys.exit(1)
+
+    print(f"🧹 Cleaning {folder}...")
+
+    all_paths = list(folder.rglob('*'))
+    obsolete = {f for f in all_paths if f.is_file() and f.relative_to(folder) not in current_scripts}
+
     if not obsolete:
-        print("No files missing from the API but still present in the repo.")
+        print("✅ No files missing from the API but still present in the repo.")
+    else:
+        with open("deletion.log", "a") as log:
+            for f in obsolete:
+                action = "🗑️📄 Deleted" if ENABLE_WRITETOFILE else "🗑️📄 Simulated deletion of"
+                try:
+                    if ENABLE_WRITETOFILE:
+                        f.unlink()
+                    print(f"{action} file no longer in the API: {f}")
+                    log.write(f"{action}: {f}\n")
+                except Exception as e:
+                    print(f"⚠️ Error deleting file {f}: {e}")
+                    log.write(f"⚠️ Error deleting {f}: {e}\n")
 
-    for f in obsolete:
-        if ENABLE_WRITETOFILE:
-            try:
-                f.unlink()
-                print(f"Deleted file no longer in the API: {f}")
-            except Exception as e:
-                print(f"Error deleting file {f}: {e}")
-        else:
-            print(f"Simulated deletion of file no longer in the API: {f}")
+    empty_dirs = [d for d in sorted(all_paths, key=lambda p: -len(p.parts)) if d.is_dir() and not any(d.iterdir())]
 
-    empty_dirs = [d for d in sorted(folder.rglob('*'), key=lambda p: -len(p.parts)) if d.is_dir() and not any(d.iterdir())]
     if not empty_dirs:
-        print("No empty directories to remove.")
-
-    for d in empty_dirs:
-        if ENABLE_WRITETOFILE:
-            try:
-                d.rmdir()
-                print(f"Removed empty directory: {d}")
-            except Exception as e:
-                print(f"Could not delete dir {d}: {e}")
-        else:
-            print(f"Simulated removal of empty directory: {d}")
+        print("✅ No empty directories to remove.")
+    else:
+        with open("deletion.log", "a") as log:
+            for d in empty_dirs:
+                action = "🗑️📁 Removed" if ENABLE_WRITETOFILE else "🗑️📁 Simulated removal of"
+                try:
+                    if ENABLE_WRITETOFILE:
+                        d.rmdir()
+                    print(f"{action} empty directory: {d}")
+                    log.write(f"{action}: {d}\n")
+                except Exception as e:
+                    print(f"⚠️ Could not delete dir {d}: {e}")
+                    log.write(f"⚠️ Could not delete dir {d}: {e}\n")
 
 def sanitize_filename(name: str) -> str:
     removed_chars = []
@@ -223,7 +240,7 @@ def process_scripts(scripts, script_folder, script_raw_folder, shell_summary, is
         current.add((folder / fname).relative_to(script_folder))
         current.add((raw_folder / raw_name).relative_to(script_raw_folder))
 
-    print(f"Processed {len(current)} {'snippets' if is_snippet else 'scripts'}.")
+    print(f"Processed {len(current)} {'snippets' if is_snippet else 'scripts'}.\n")
     return current
 
 def compute_hash(file_path):
@@ -237,9 +254,9 @@ def save_file(path, content, is_json=False):
     data = json.dumps(content, indent=4, ensure_ascii=False) if is_json else content
     if ENABLE_WRITETOFILE:
         path.write_text(data, encoding="utf-8")
-        print(f"File saved: {path}")
+        print(f"File saved: {path.relative_to(base_dir) if base_dir else path}")
     else:
-        print(f"File would be saved (simulation): {path}")
+        print(f"File would be saved (simulation): {path.relative_to(base_dir) if base_dir else path}")
 
 
 def pull_from_api(url):
@@ -310,6 +327,7 @@ def update_api_if_needed(match, raw_data, is_snippet):
 
     return False
 
+
 def write_modifications_to_api(base_dir, folders):
     print("Comparing script files with JSON files...")
     mismatches = []
@@ -318,7 +336,7 @@ def write_modifications_to_api(base_dir, folders):
     total_matches = 0
     total_mismatches = 0
     total_updated = 0
-    total_skipped = 0
+    total_not_updated = 0
 
     for folder_key, folder in folders.items():
         is_snippet = folder_key == 'snippetsraw'
@@ -332,15 +350,15 @@ def write_modifications_to_api(base_dir, folders):
                               if p.is_file() and p.stem.lower() == raw_name), None)
             except Exception as e:
                 print(f"Error matching file for {raw_path}: {e}")
-                total_skipped += 1
+                total_not_updated += 1
                 continue
 
             if not match:
-                print(f"No match for {'snippet' if is_snippet else 'script'}: {raw_path}")
-                total_skipped += 1
+                print(f"No match for {'snippet' if is_snippet else 'script'}: {raw_path.relative_to(base_dir)}")
+                total_not_updated += 1 
                 continue
 
-            print(f"Matched {'snippet' if is_snippet else 'script'}: {match} <-> {raw_path}")
+            print(f"Matched {'snippet' if is_snippet else 'script'}: {match.relative_to(base_dir)} <-> {raw_path.relative_to(base_dir)}")
             total_matches += 1
 
             file_hash, code_hash, raw_data = compare_files_and_hashes(match, raw_path)
@@ -364,13 +382,24 @@ def write_modifications_to_api(base_dir, folders):
 
                 if updated:
                     total_updated += 1
+                else:
+                    total_not_updated += 1
+
+    print("\n🔍 Comparison Complete:")
+
+    print(f"🧾 Total files checked: {total_files_checked}")
+    if total_matches > 0:
+        print(f"↔️ Total matches: {total_matches}")
+    if total_mismatches > 0:
+        print(f"🧩 Total mismatches to update: {total_mismatches}")
+    if total_updated > 0:
+        print(f"✅ Total updated: {total_updated}")
+    if total_not_updated > 0:
+        print(f"❌ Total errors: {total_not_updated}")
+
+    if total_matches == total_files_checked:
+        print("✅ Everything is up to date in the api")
 
-    print("\nComparison Complete:")
-    print(f"Total files checked: {total_files_checked}")
-    print(f"Total matches: {total_matches}")
-    print(f"Total mismatches: {total_mismatches}")
-    print(f"Total updates: {total_updated}")
-    print(f"Total skipped: {total_skipped}")
 
 def update_to_api(item_id, payload, is_snippet=False):
     """Update the API with the provided item ID and payload."""
@@ -395,13 +424,27 @@ def update_to_api(item_id, payload, is_snippet=False):
         print(f"Request error for {'snippet' if is_snippet else 'script'} {item_id}: {e}")
 
 def git_pull(base_dir):
-    """Force pull the latest changes from the git repository, discarding local changes."""
-    
+    """Force pull latest changes from the git repository if there are changes, discarding local changes."""
+
     print("Starting pull process...", flush=True)
     try:
+        print(f"Branch to pull: '{git_pull_branch}'")
         print("Fetching latest changes from remote...", flush=True)
         subprocess.check_call(['git', '-C', base_dir, 'fetch', 'origin'], stdout=sys.stdout, stderr=sys.stderr)
 
+        print("Checking for incoming changes...", flush=True)
+        result = subprocess.run(
+            ['git', '-C', base_dir, 'log', f'HEAD..origin/{git_pull_branch}', '--oneline'],
+            capture_output=True, text=True
+        )
+
+        if not result.stdout.strip():
+            print("No changes to pull. Repository is up to date.", flush=True)
+            return
+
+        print("Incoming commits:")
+        print(result.stdout, flush=True)
+
         print(f"Resetting local branch to match 'origin/{git_pull_branch}'...", flush=True)
         subprocess.check_call(['git', '-C', base_dir, 'reset', '--hard', f'origin/{git_pull_branch}'], stdout=sys.stdout, stderr=sys.stderr)
 
@@ -498,13 +541,22 @@ def git_push(base_dir):
             print(f"Changes pushed to branch '{git_pull_branch}'")
 
         else:
-            print("No changes to commit.")
+            print("✅ No changes to commit.")
     except subprocess.CalledProcessError as e:
         print(f"Git operation failed: {e}")
 
 def check_git_health(base_dir):
     """Check the health of the Git repository."""
 
+    # Check the rights to read/write in the directory
+    try:
+        if not os.access(base_dir, os.R_OK | os.W_OK):
+            print(f"❌ Error: No read/write permissions for the directory '{base_dir}'.")
+            return False
+    except Exception as e:
+        print(f"❌ Error: Failed to check permissions for the directory '{base_dir}'. {e}")
+        return False
+
     # Check if 'git' command is available
     try:
         subprocess.check_call(['git', '--version'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
@@ -601,11 +653,12 @@ def check_git_health(base_dir):
         print("❌ Error: Failed to check commit history.")
         return False
 
-    print("✅ Git repository health check passed.")
     return True
 
+
 def pre_flight():
-    global domain, scriptpath, headers
+    global domain, headers, base_dir
+
     domain = os.getenv('DOMAIN')
     api_token = os.getenv('API_TOKEN')
     scriptpath = os.getenv('SCRIPTPATH')
@@ -619,8 +672,13 @@ def pre_flight():
             if var == 'SCRIPTPATH': print(f"  - SCRIPTPATH: The local folder path for Git commands.")
         sys.exit(1)
 
+    #Build headers
     headers = {"X-API-KEY": api_token}
-    domain_for_connection = domain.replace("https://", "").replace("http://", "")
+    #Build base_dir path
+    base_dir = Path(scriptpath).resolve()
+
+    # no http for tcp test or any trailing slash
+    domain_for_connection = domain.replace("https://", "").replace("http://", "").rstrip("/")
 
     try:
         socket.create_connection((domain_for_connection, 443), timeout=5)
@@ -630,11 +688,13 @@ def pre_flight():
         print(f"❌ Error: Unable to connect to {domain} on port 443 - {e} (Obfuscated API Token: {obfuscated})")
         sys.exit(1)
 
+    # Make sure domain starts with https:// and remove any trailing slash
     if not domain.startswith("http://") and not domain.startswith("https://"):
         domain = "https://" + domain
+    domain = domain.rstrip("/")
 
+    #Test api token for read, it is currently not possible to test for write as any request to the api would write empty file.
     obfuscated = api_token[:3] + '*' * (len(api_token) - 6) + api_token[-3:]
-
     try:
         response = requests.get(f"{domain}/scripts/", headers=headers, timeout=5)
         if response.status_code == 200:
@@ -648,6 +708,7 @@ def pre_flight():
 
     return
 
+
 def check_and_create_folders(base_path, subfolders):
     try:
         if not base_path.exists():
@@ -671,21 +732,11 @@ def main():
 
     # 0. Prep: Verify Dependencies, Set Up Environment, and Git Health Check
     print("\n===== Step 0: General Prep =====")
-    
-    # ENV vars & network checks
+
+
+    # ENV vars & network checks & prep vars
     pre_flight()
 
-    # Folder structure check
-    base_dir = Path(scriptpath).resolve()
-    folders = {
-        "scripts": base_dir / "scripts",
-        "scriptsraw": base_dir / "scriptsraw",
-        "snippets": base_dir / "snippets",
-        "snippetsraw": base_dir / "snippetsraw"
-    }
-    check_and_create_folders(base_dir, folders)
-    print("✅ All folders created and verified.")
-    
     # Check the health of the Git repo
     if ENABLE_GIT_PULL or ENABLE_GIT_PUSH:
         if check_git_health(base_dir):
@@ -695,12 +746,21 @@ def main():
             sys.exit(1)
     else:
         print("Skipping Git health check because both pull and push are disabled.")
-    
+
+    # Folder structure check
+    folders = {
+        "scripts": base_dir / "scripts",
+        "scriptsraw": base_dir / "scriptsraw",
+        "snippets": base_dir / "snippets",
+        "snippetsraw": base_dir / "snippetsraw"
+    }
+    check_and_create_folders(base_dir, folders)
+    print("✅ All folders created and verified.")
+
     print("===== End of Step 0: General Prep =====\n")
 
     # 1. Git Pull
     print("\n===== Step 1: Git Pull =====")
-    print(f"Branch to pull: '{git_pull_branch}'")
     if ENABLE_GIT_PULL:
         git_pull(base_dir)
     else:
@@ -716,19 +776,21 @@ def main():
     print("\n===== Step 3: Fetch and Process Scripts and Snippets =====")
     # Initialize counters and sets
     shell_summary, current_scripts = defaultdict(int), set()
-    print("Fetching scripts...")
+    print("Fetching script list...")
     user_defined_scripts = pull_from_api(f"{domain}/scripts/?showHiddenScripts=true")
     user_defined_scripts = [item for item in user_defined_scripts if item.get('script_type') == 'userdefined']
     current_scripts.update(process_scripts(user_defined_scripts, folders["scripts"], folders["scriptsraw"], shell_summary))
 
     # Fetch and process snippets
-    print("Fetching snippets...")
+    print("Fetching snippets list...")
     snippets = pull_from_api(f"{domain}/scripts/snippets/")
     current_scripts.update(process_scripts(snippets, folders["snippets"], folders["snippetsraw"], shell_summary, is_snippet=True))
 
     # Output the total number of scripts exported and provide a summary of the shell counts
     print(f"Total number of scripts exported: {len(current_scripts)}")
-    print("Shell summary:", "\n".join(f"{shell}: {count}" for shell, count in shell_summary.items()))
+    print("Shell summary:")
+    for shell, count in shell_summary.items():
+        print(f"{shell.strip()}: {count}")
 
     # Remove any obsolete files that are no longer existing in the api
     print("\nRemove any obsolete files")

scripts_staging/Lab/RustDesk Get ID.ps1

@@ -0,0 +1,6 @@
+#public
+#grab public id of restdesk to set a custom field
+$ErrorActionPreference= 'silentlycontinue'
+
+cd $env:ProgramFiles\RustDesk\
+.\RustDesk.exe --get-id | out-host