build_db.py

import subprocess
import urllib.request
import os
import sys
import tempfile


def main():
    log("Building database...")

    dryrun = False
    if len(sys.argv) >= 2 and sys.argv[1] == "-d":
        log("Dry run")
        dryrun = True

    github_repo = os.getenv("GITHUB_REPOSITORY", "theypsilon/test")
    db_id = os.getenv("DB_ID", None)
    if db_id is None:
        db_id = github_repo

    log("downloading db_operator.py")
    urllib.request.urlretrieve(
        "https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/.github/db_operator.py",
        "/tmp/distribution_db_operator.py",
    )

    db_url = f"https://raw.githubusercontent.com/{github_repo}/db/db.json.zip"
    base_files_url = f"https://raw.githubusercontent.com/{github_repo}/%s/"

    subprocess.run(["rm *.sh"], shell=True, stderr=subprocess.STDOUT)

    run(["python3", "-m", "pip", "install", "Pillow"])

    external_files = "external_files.csv"
    external_repos_check = subprocess.run(
        ["git", "ls-remote", "--heads", "origin", "external_repos_files"],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
    )
    if "refs/heads/external_repos_files" in external_repos_check.stdout:
        run(["git", "fetch", "origin", "external_repos_files"])
        run(["git", "checkout", "FETCH_HEAD", "--", "external_repos_files.csv"])
        external_files = "external_files.csv external_repos_files.csv"
        log("Added external_repos_files.csv from branch external_repos_files!")

    run(
        ["python3", "/tmp/distribution_db_operator.py", "build", "."],
        env={
            "DB_ID": db_id,
            "DB_URL": db_url,
            "DB_JSON_NAME": "db.json",
            "BASE_FILES_URL": base_files_url,
            "FINDER_IGNORE": os.getenv("FINDER_IGNORE", "") + " " + external_files,
            "BROKEN_MRAS_IGNORE": os.getenv("BROKEN_MRAS_IGNORE", "true"),
            "EXTERNAL_FILES": external_files,
        },
    )

    if not dryrun and os.path.exists("db.json") and passes_db_tests(db_id):
        log("Pushing database...")
        run(["zip", "db.json.zip", "db.json"])
        run(["git", "checkout", "--orphan", "db"])
        run(["git", "reset"])
        run(["git", "add", "db.json.zip"])
        run(["git", "commit", "-m", "Creating database"])
        run(["git", "push", "--force", "origin", "db"])

    return 0


def passes_db_tests(db_id):
    log("Testing database...")

    with tempfile.TemporaryDirectory() as temp_folder:
        log("downloading downloader.sh")
        urllib.request.urlretrieve(
            "https://raw.githubusercontent.com/MiSTer-devel/Downloader_MiSTer/main/downloader.sh",
            temp_folder + "/downloader.sh",
        )
        run(["chmod", "+x", "downloader.sh"], cwd=temp_folder)

        downloader_ini_content = f"""
            [MiSTer]
            base_path = {temp_folder}/
            base_system_path = {temp_folder}/
            update_linux = false
            allow_reboot  = 0
            verbose = false
            downloader_retries = 0

            [{db_id}]
            db_url = {os.getcwd()}/db.json
        """
        log("downloader.ini content:")
        log(downloader_ini_content)

        with open(temp_folder + "/downloader.ini", "w") as fini:
            fini.write(downloader_ini_content)

        run(["./downloader.sh"], cwd=temp_folder, env={"DEBUG": "true", "CURL_SSL": ""})
        log("The test went well.")

    return True


def run(commands, env=None, cwd=None):
    log(" ".join(commands))
    if env is not None:
        log("with env:", env)
    if cwd is not None:
        log("with cwd:", cwd)
    subprocess.run(commands, cwd=cwd, env=env, check=True, stderr=subprocess.STDOUT)


def log(*text):
    print(*text, flush=True)


if __name__ == "__main__":
    exit(main())