repo_mapper.sh

#!/bin/bash

# Repository Mapper - Maps GitHub organizations and repositories to cybersecurity domains
# This script fetches all orgs/repos, reads READMEs, and creates a categorized mapping

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DATA_DIR="${SCRIPT_DIR}/data"
CACHE_DIR="${DATA_DIR}/cache"
OUTPUT_DIR="${SCRIPT_DIR}/output"
CONFIG_FILE="${SCRIPT_DIR}/config.yaml"

# Create necessary directories
mkdir -p "${DATA_DIR}" "${CACHE_DIR}" "${OUTPUT_DIR}"

# Load excluded orgs from config.yaml if it exists
EXCLUDED_ORGS=()
if [ -f "${CONFIG_FILE}" ]; then
    # Extract exclude_orgs list from YAML (simplified parsing)
    while IFS= read -r line; do
        if [[ $line =~ ^[[:space:]]*-[[:space:]]+\"?([^\"[:space:]]+)\"? ]]; then
            org_name="${BASH_REMATCH[1]}"
            # Skip comment lines
            if [[ ! $org_name =~ ^# ]]; then
                EXCLUDED_ORGS+=("$org_name")
            fi
        fi
    done < <(sed -n '/exclude_orgs:/,/^[[:space:]]*[a-z_]/p' "${CONFIG_FILE}" | grep -E '^\s*-\s+')
fi

echo "=== GitHub Repository Mapper ==="
echo "Starting repository discovery and analysis..."
if [ ${#EXCLUDED_ORGS[@]} -gt 0 ]; then
    echo "⚠ Excluding organizations: ${EXCLUDED_ORGS[*]}"
fi
echo

# Step 1: Fetch all organizations the user belongs to
echo "[1/5] Fetching organizations..."
gh api user/orgs --paginate --jq '.[] | {login, description}' > "${DATA_DIR}/orgs.json" 2>/dev/null || true

# Also get user's personal repos
CURRENT_USER=$(gh api user --jq '.login')
echo "${CURRENT_USER}" > "${DATA_DIR}/current_user.txt"

# Step 2: Fetch all repositories from orgs and personal account
echo "[2/5] Fetching repositories..."

# Initialize repos file
echo "[]" > "${DATA_DIR}/all_repos.json"

# Get personal repos
echo "  - Fetching personal repositories for ${CURRENT_USER}..."
gh repo list "${CURRENT_USER}" --limit 1000 --json name,description,url,isPrivate,updatedAt,primaryLanguage > "${DATA_DIR}/personal_repos.json"

# Get org repos if orgs exist
if [ -s "${DATA_DIR}/orgs.json" ]; then
    cat "${DATA_DIR}/orgs.json" | jq -r '.login' | while read -r org; do
        # Check if org is in excluded list
        skip=false
        for excluded in "${EXCLUDED_ORGS[@]}"; do
            if [ "$org" = "$excluded" ]; then
                echo "  - Skipping excluded organization: ${org}"
                skip=true
                break
            fi
        done

        if [ "$skip" = false ]; then
            echo "  - Fetching repositories for organization: ${org}..."
            gh repo list "${org}" --limit 1000 --json name,description,url,isPrivate,updatedAt,primaryLanguage > "${DATA_DIR}/org_${org}_repos.json" 2>/dev/null || echo "[]" > "${DATA_DIR}/org_${org}_repos.json"
        fi
    done
fi

# Combine all repos into one file
echo "[3/5] Combining repository data..."
jq -s 'add' "${DATA_DIR}"/*_repos.json > "${DATA_DIR}/all_repos.json"

TOTAL_REPOS=$(jq 'length' "${DATA_DIR}/all_repos.json")
echo "  Found ${TOTAL_REPOS} repositories"

# Step 3: Fetch README content for each repository
echo "[4/5] Fetching README files..."
jq -c '.[]' "${DATA_DIR}/all_repos.json" | while read -r repo; do
    REPO_NAME=$(echo "$repo" | jq -r '.name')
    REPO_URL=$(echo "$repo" | jq -r '.url')
    REPO_OWNER=$(echo "$REPO_URL" | sed 's|https://github.com/||' | cut -d'/' -f1)

    README_FILE="${CACHE_DIR}/${REPO_OWNER}_${REPO_NAME}_README.md"

    # Check if README was fetched in last 7 days
    if [ -f "${README_FILE}" ]; then
        FILE_AGE=$(($(date +%s) - $(stat -c %Y "${README_FILE}" 2>/dev/null || stat -f %m "${README_FILE}" 2>/dev/null)))
        if [ ${FILE_AGE} -lt 604800 ]; then
            echo "  - Cached: ${REPO_OWNER}/${REPO_NAME}"
            continue
        fi
    fi

    echo "  - Fetching: ${REPO_OWNER}/${REPO_NAME}"
    gh repo view "${REPO_OWNER}/${REPO_NAME}" --json readme --jq '.readme' > "${README_FILE}" 2>/dev/null || echo "No README available" > "${README_FILE}"
    sleep 0.1  # Rate limiting
done

echo "[5/5] Repository data collection complete!"
echo
echo "Next: Run the recursive README fetcher to get all subdirectory READMEs"
echo "  python3 fetch_all_readmes.py"
echo
echo "Then run the analysis script to categorize repositories"
echo "  python3 analyze_repos.py"