github scraper.py

from xml.etree.ElementTree import tostring

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
import time
import re
import requests
import statistics

s = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=s)

# print("   _____ _ _    _____                                 \n  / ____(_) |  / ____|                                \n | |  __ _| |_| (___   ___ __ _ _ __  _ __   ___ _ __ \n | | |_ | | __|\___ \ / __/ _` | '_ \| '_ \ / _ \ '__|\n | |__| | | |_ ____) | (_| (_| | | | | | | |  __/ |   \n  \_____|_|\__|_____/ \___\__,_|_| |_|_| |_|\___|_|   \n")
print("8\"\"\"\"8         8\"\"\"\"8                                    \n8    \" e eeeee 8      eeee eeeee eeeee eeeee eeee eeeee  \n8e     8   8   8eeeee 8  8 8   8 8   8 8   8 8    8   8  \n88  ee 8e  8e      88 8e   8eee8 8e  8 8e  8 8eee 8eee8e \n88   8 88  88  e   88 88   88  8 88  8 88  8 88   88   8 \n88eee8 88  88  8eee88 88e8 88  8 88  8 88  8 88ee 88   8 \n")
print("starting git scanner \n accepted input: github username , github profile url , github repo link \n in case it didn`t work, try to change the input type" )
target = input("insert:")
print(type(target))

slashCounter = 0

for l in target:
    if l == "/":
        slashCounter = slashCounter + 1

if target.startswith("https://") and slashCounter == 3:
    print(target + "?tab=repositories")
    driver.get(target + "?tab=repositories")
    repo = target
elif target.startswith("https://") and slashCounter > 3:
    print(target)
    driver.get(target)
    repo = target
else:
    print("https://github.com/" + target + "?tab=repositories")
    driver.get("https://github.com/" + target + "?tab=repositories")
    repo = "https://github.com/" + target

print(repo)


# driver.get("https://github.com/novacLearnsCyber?tab=repositories")
# repo = "https://github.com/novacLearnsCyber"


res = driver.find_elements(By.CSS_SELECTOR, 'a[itemprop="name codeRepository"]')
print(f"public repos found: {len(res)}")

repos = [i.text for i in res]
links = []
flink = []
found_secrets = []
ai_flagged = []

patterns = {
    "generic_password":  re.compile(r'(?i)(password|passwd|pwd)\s*[:=]\s*["\']?[^"\'\s]{4,}'),
    "generic_secret":    re.compile(r'(?i)(secret|api[_-]?key|apikey|token)\s*[:=]\s*["\']?[^"\'\s]{8,}'),
    "aws_access_key":    re.compile(r'AKIA[0-9A-Z]{16}'),
    "aws_secret":        re.compile(r'(?i)aws_secret_access_key\s*[:=]\s*["\']?[A-Za-z0-9/+=]{40}'),
    "private_key":       re.compile(r'-----BEGIN (RSA|EC|OPENSSH|DSA)?\s?PRIVATE KEY-----'),
    "slack_token":       re.compile(r'xox[baprs]-[0-9A-Za-z-]{10,}'),
    "openai_key":        re.compile(r'sk-[A-Za-z0-9]{20,}'),
    "jwt":               re.compile(r'eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+'),
    "db_connection_url": re.compile(r'(?i)(postgres|mysql|mongodb(\+srv)?)://[^\s"\']+:[^\s"\']+@[^\s"\']+'),
}


GENERIC_AI_PHRASES = re.compile(
    r'(?i)^\s*#*\s*(initialize|loop through|check if|return the|this function|'
    r'note that|create a|define a|set the|get the|update the|calculate the|'
    r'iterate over|handle the|validate the|process the|ensure that|'
    r'this class|this method|helper function to|used to)\b'
)

EMOJI_PATTERN = re.compile(
    "["
    "\U0001F300-\U0001FAFF"
    "\U00002700-\U000027BF"
    "\U0001F1E6-\U0001F1FF"
    "]+"
)

COMMENT_LINE = re.compile(r'(?:#|//)\s?(.*)$')
DOCSTRING_BLOCK = re.compile(r'"""(.*?)"""', re.DOTALL)


OUTPUT_CALL_LINE = re.compile(
    r'(?i)\b(?:print|println!?|printf!?|echo|console\.log|'
    r'system\.out\.println?|fmt\.print(?:ln|f)?|write-host|puts)\b'
)

STATUS_MARKER_PATTERN = re.compile(
    r'(?i)(\[\d+/\d+\]|\[ok\]|\[!\]|\[error\]|\[success\]|\[done\]|\[info\]|\[warn(?:ing)?\])'
)

GENERIC_STATUS_PHRASES = re.compile(
    r'(?i)(starting|initializ|processing|please wait|in progress|'
    r'successfully|finished|complete[d]?|auto-save|auto save|sincroniz|'
    r'salvez|trimit|verific|incerc sa|reparat automat|conflictele au fost)'
)


def extract_comments(content):
    line_comments = []
    for line in content.splitlines():
        m = COMMENT_LINE.search(line)
        if m and m.group(1).strip():
            line_comments.append(m.group(1).strip())

    docstrings = [d.strip() for d in DOCSTRING_BLOCK.findall(content) if d.strip()]

    return line_comments, docstrings


def count_code_lines(content):
    code_lines = 0
    for line in content.splitlines():
        stripped = line.strip()
        if not stripped:
            continue
        if stripped.startswith("#") or stripped.startswith("//"):
            continue
        code_lines += 1
    return max(code_lines, 1)


def find_trivial_functions_with_docstrings(content):
    flagged = []
    func_pattern = re.compile(
        r'def\s+(\w+)\s*\([^)]*\)\s*:\s*\n\s*"""(.*?)"""\s*\n((?:.*\n){0,3}?)(?=\ndef |\Z)',
        re.MULTILINE
    )
    for match in func_pattern.finditer(content):
        func_name, docstring, body = match.groups()
        body_lines = [l for l in body.splitlines() if l.strip()]
        if len(body_lines) <= 2 and len(docstring.strip()) > 20:
            flagged.append(func_name)
    return flagged


def extract_output_calls(content):

    output_lines = []
    for line in content.splitlines():
        if OUTPUT_CALL_LINE.search(line):
            output_lines.append(line.strip())
    return output_lines


def analyze_verbose_output_style(content, code_lines):

    output_lines = extract_output_calls(content)

    if len(output_lines) < 2:
        return 0, {}

    output_density = len(output_lines) / code_lines

    marker_hits = sum(1 for l in output_lines if STATUS_MARKER_PATTERN.search(l))
    marker_ratio = marker_hits / len(output_lines)

    generic_hits = sum(1 for l in output_lines if GENERIC_STATUS_PHRASES.search(l))
    generic_ratio = generic_hits / len(output_lines)

    emoji_hits = sum(1 for l in output_lines if EMOJI_PATTERN.search(l))

    score = 0
    details = {}

    if output_density > 0.3:
        score += 15
        details["high_output_density"] = round(output_density, 2)

    if marker_ratio > 0.3:
        score += 20
        details["progress_marker_ratio"] = round(marker_ratio, 2)

    if generic_ratio > 0.3:
        score += 15
        details["generic_status_phrase_ratio"] = round(generic_ratio, 2)

    if emoji_hits > 0:
        score += 10
        details["emoji_in_output"] = emoji_hits

    return score, details


def analyze_ai_comment_style(content):

    line_comments, docstrings = extract_comments(content)
    all_comments = line_comments + docstrings
    code_lines = count_code_lines(content)

    score = 0
    details = {}

    # --- comment analysis (needs at least 3 comments to be statistically meaningful) ---
    if len(all_comments) >= 3:
        comment_density = len(all_comments) / code_lines

        generic_hits = sum(1 for c in all_comments if GENERIC_AI_PHRASES.match(c))
        generic_ratio = generic_hits / len(all_comments)

        emoji_hits = sum(1 for c in all_comments if EMOJI_PATTERN.search(c))

        lengths = [len(c) for c in all_comments]
        length_variance = statistics.pvariance(lengths) if len(lengths) > 1 else 0
        low_variance_flag = length_variance < 50 and len(all_comments) >= 5

        proper_punct = sum(
            1 for c in all_comments
            if c and c[0].isupper() and (c.endswith('.') or c.endswith(':'))
        )
        punct_ratio = proper_punct / len(all_comments)

        trivial_docstrings = find_trivial_functions_with_docstrings(content)

        if comment_density > 0.5:
            score += 20
            details["high_density"] = round(comment_density, 2)

        if generic_ratio > 0.3:
            score += 25
            details["generic_phrase_ratio"] = round(generic_ratio, 2)

        if emoji_hits > 0:
            score += 15
            details["emoji_in_comments"] = emoji_hits

        if low_variance_flag:
            score += 20
            details["suspicious_uniformity"] = round(length_variance, 1)

        if punct_ratio > 0.8 and len(all_comments) >= 5:
            score += 10
            details["perfect_punctuation_ratio"] = round(punct_ratio, 2)

        if trivial_docstrings:
            score += 20
            details["docstrings_on_trivial_functions"] = trivial_docstrings

    output_score, output_details = analyze_verbose_output_style(content, code_lines)
    score += output_score
    details.update(output_details)

    if score == 0:
        details["reason"] = "no suspicious signal found"

    return min(score, 100), details


def scan_ai_comments(content, blob_url, threshold=40):
    """Runs the AI style analysis and adds to ai_flagged if it passes the threshold."""
    score, details = analyze_ai_comment_style(content)
    if score >= threshold:
        print(f"[AI?] score {score}/100 in {blob_url} -> {details}")
        ai_flagged.append({
            "file": blob_url,
            "score": score,
            "details": details
        })
    return score, details


def to_raw_url(blob_url):
    # https://github.com/user/repo/blob/branch/path -> https://raw.githubusercontent.com/user/repo/branch/path
    return blob_url.replace("https://github.com/", "https://raw.githubusercontent.com/").replace("/blob/", "/")

def scan_raw(blob_url):
    raw_url = to_raw_url(blob_url)
    try:
        resp = requests.get(raw_url, timeout=10)
        if resp.status_code != 200:
            print(f"raw fetch failed ({resp.status_code}): {raw_url}")
            return

        content = resp.text

        for label, pattern in patterns.items():
            matches = pattern.findall(content)
            if matches:
                print(f"[!] {label} found in {blob_url}")
                found_secrets.append({
                    "file": blob_url,
                    "raw": raw_url,
                    "type": label,
                    "matches": matches
                })

        scan_ai_comments(content, blob_url)

    except Exception as e:
        print(f"error fetching {raw_url}: {e}")




def loop(next_page):

    if "/blob" in next_page:
        scan_raw(next_page)
        print(f"file scanned >>> skip : {next_page}")
        return

    driver.get(next_page)


    empty_check = driver.find_elements(By.XPATH, "//*[contains(text(), 'This repository is empty')]")
    if empty_check:
        print(f"empty repo >>> skip : {next_page}")
        return

    try:
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, "tr.react-directory-row"))
        )
    except Exception:
        print("timeout: no tr.react-directory-row appeared on", next_page)
        return

    rows = driver.find_elements(By.CSS_SELECTOR, "tr.react-directory-row")
    print(f"rows found on {next_page}: {len(rows)}")

    for row in rows:
        try:
            link = row.find_element(By.CSS_SELECTOR, "a.Link--primary")
            name = link.text

            if name.startswith(".git"):
                continue

            url = link.get_attribute("href")
            print("found:", name, url)
            links.append(url)
        except Exception as e:
            print("error on row:", e)
            continue

for r in repos:
    next_page = f"{repo}/{r}"
    flink.append(next_page)
    loop(next_page)

for l in links:
    if "/blob/" in l:
        loop(l)


driver.quit()

print(f"\n=== Total suspicious secrets found: {len(found_secrets)} ===")
for s in found_secrets:
    print(s)

print(f"\n=== Files with AI-suspicious comments: {len(ai_flagged)} ===")
for f in ai_flagged:
    print(f)