__mirror-github-to-gitlab.yml

---
# Mirror all repositories in the LizardByte GitHub organization to GitLab.

name: Mirror GitHub to GitLab
permissions: {}

on:
  schedule:
    - cron: '0 3 * * *'
  workflow_dispatch:

concurrency:
  group: mirror-github-to-gitlab
  cancel-in-progress: false

jobs:
  mirror:
    name: Mirror GitHub to GitLab
    permissions:
      contents: read
    runs-on: ubuntu-latest
    steps:
      # Process repositories in one job so project creation, privacy checks, and mirroring stay together.
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

      - name: Get repositories
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
        with:
          github-token: ${{ secrets.GH_BOT_TOKEN }}
          script: |
            const fs = require('fs');
            const ignoredRepositoryNames = JSON.parse(
              fs.readFileSync('.github/repositories-to-ignore.json', 'utf8'),
            );
            if (
              !Array.isArray(ignoredRepositoryNames)
              || !ignoredRepositoryNames.every((name) => typeof name === 'string' && name.length > 0)
            ) {
              throw new Error('Expected an array of non-empty repository names.');
            }
            const ignoredRepositories = new Set(ignoredRepositoryNames);
            const securityAdvisoryIdSegment = '[23456789cfghjmpqrvwx]{4}';
            const securityAdvisoryForkPattern = new RegExp(
              `-ghsa-${securityAdvisoryIdSegment}-${securityAdvisoryIdSegment}-${securityAdvisoryIdSegment}$`,
              'i',
            );
            const shouldIgnoreRepository = (repo) => (
              ignoredRepositories.has(repo.name) || securityAdvisoryForkPattern.test(repo.name)
            );
            const opts = github.rest.repos.listForOrg.endpoint.merge({ org: context.repo.owner });
            const repos = await github.paginate(opts);
            const gitlabTarget = (repo) => {
              const prefixNames = { '.': 'dot-', '-': 'dash-', '_': 'underscore-' };
              const lowerName = repo.name.toLowerCase();
              let targetName = lowerName.replace(
                /^[._-]+/,
                (prefix) => [...prefix].map((character) => prefixNames[character]).join(''),
              );
              targetName = targetName
                .replace(/[^a-z0-9_.-]+/g, '-')
                .replace(/[._-]{2,}/g, '-')
                .replace(/[._-]+$/g, '');

              if (targetName.endsWith('.git') || targetName.endsWith('.atom')) {
                targetName += '-repo';
              }
              if (!targetName) {
                targetName = 'repository';
              }

              const transformed = targetName !== lowerName;
              return {
                targetName: transformed ? targetName : repo.name,
                targetPath: transformed ? `${targetName}-${repo.id}` : repo.name,
              };
            };
            const repositoryData = repos
              .filter((repo) => !shouldIgnoreRepository(repo))
              .map((repo) => ({
                cloneUrl: repo.clone_url,
                name: repo.name,
                sizeKiB: repo.size,
                targetVisibility: repo.visibility === 'public' ? 'public' : 'private',
                ...gitlabTarget(repo),
              }));

            fs.writeFileSync('repositories.json', JSON.stringify(repositoryData), { mode: 0o600 });
            core.info(`Ignored ${repos.length - repositoryData.length} repositories.`);
            core.info(`Prepared ${repositoryData.length} repositories for mirroring.`);

      - name: Mirror repositories
        shell: bash
        env:
          GIT_TERMINAL_PROMPT: '0'
          GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN }}
          GITLAB_API_URL: https://gitlab.com/api/v4
          GITLAB_GROUP: lizardbyte
          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
          GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
        run: |
          set -euo pipefail

          if [[ -z "${GITHUB_TOKEN}" ]] || \
            [[ -z "${GITLAB_TOKEN}" ]] || \
            [[ -z "${GITLAB_SSH_PRIVATE_KEY}" ]]; then
            echo "::error::GH_BOT_TOKEN, GITLAB_TOKEN, and GITLAB_SSH_PRIVATE_KEY must all be configured."
            exit 1
          fi

          repository_file="${GITHUB_WORKSPACE}/repositories.json"
          response_file="$(mktemp)"
          temp_root="$(mktemp -d)"
          trap 'rm -f "${response_file}" "${repository_file}"; rm -rf "${temp_root}"' EXIT

          ssh_dir="${temp_root}/ssh"
          ssh_key="${ssh_dir}/id_ed25519"
          ssh_known_hosts="${ssh_dir}/known_hosts"
          mkdir -m 700 "${ssh_dir}"
          printf '%s\n' "${GITLAB_SSH_PRIVATE_KEY}" > "${ssh_key}"
          chmod 600 "${ssh_key}"
          if ! ssh-keygen -y -P '' -f "${ssh_key}" > /dev/null; then
            echo "::error::GITLAB_SSH_PRIVATE_KEY is not a valid unencrypted SSH private key."
            exit 1
          fi

          # Published at https://docs.gitlab.com/user/gitlab_com/#ssh-known_hosts-entries.
          gitlab_host_key='ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf'
          printf 'gitlab.com %s\n' "${gitlab_host_key}" > "${ssh_known_hosts}"
          chmod 600 "${ssh_known_hosts}"

          gitlab_host_fingerprint="$(ssh-keygen -lf "${ssh_known_hosts}" -E sha256 | awk '{print $2}')"
          if [[ "${gitlab_host_fingerprint}" != \
            'SHA256:eUXGGm1YGsMAS7vkcx6JOJdOGHPem5gQp4taiCfCLB8' ]]; then
            echo "::error::The configured GitLab SSH host key has an unexpected fingerprint."
            exit 1
          fi

          gitlab_ssh_args=(
            -i "${ssh_key}"
            -o BatchMode=yes
            -o IdentitiesOnly=yes
            -o StrictHostKeyChecking=yes
            -o "UserKnownHostsFile=${ssh_known_hosts}"
            -o ConnectTimeout=30
          )
          printf -v gitlab_ssh_command '%q ' ssh "${gitlab_ssh_args[@]}"

          echo "Verifying GitLab SSH authentication..."
          if ! ssh "${gitlab_ssh_args[@]}" -T git@gitlab.com; then
            echo "::error::Unable to authenticate to GitLab with GITLAB_SSH_PRIVATE_KEY."
            exit 1
          fi

          gitlab_request() {
            local method="$1"
            local url="$2"
            local data="${3:-}"
            local curl_args=(
              --silent
              --output "${response_file}"
              --write-out '%{http_code}'
              --request "${method}"
              --header "Accept: application/json"
              --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}"
            )

            if [[ -n "${data}" ]]; then
              curl_args+=(
                --header "Content-Type: application/json"
                --data "${data}"
              )
            fi

            curl "${curl_args[@]}" "${url}"
          }

          wait_for_gitlab_import() {
            local project_id="$1"
            local source_name="$2"
            local deadline="$((SECONDS + 10800))"
            local import_error
            local import_status
            local status

            while ((SECONDS < deadline)); do
              if ! status="$(gitlab_request GET "${GITLAB_API_URL}/projects/${project_id}/import")"; then
                echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab import."
                return 1
              fi
              if [[ "${status}" != "200" ]]; then
                if [[ "${status}" == "403" ]]; then
                  echo "::error title=Mirror failed: ${source_name}::GITLAB_TOKEN needs Import API Read access."
                  return 1
                fi
                echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab import (HTTP ${status})."
                return 1
              fi

              import_status="$(jq -er '.import_status' "${response_file}")"
              case "${import_status}" in
                finished)
                  echo "GitLab server-side import finished."
                  return 0
                  ;;
                scheduled | started)
                  echo "GitLab server-side import status: ${import_status}."
                  sleep 30
                  ;;
                failed)
                  import_error="$(jq -r '.import_error // "No import error detail was provided."' "${response_file}")"
                  import_error="${import_error//$'\r'/ }"
                  import_error="${import_error//$'\n'/ }"
                  echo "GitLab import failed: ${import_error}"
                  echo "::error title=Mirror failed: ${source_name}::GitLab server-side import failed."
                  return 1
                  ;;
                *)
                  echo "::error title=Mirror failed: ${source_name}::Unexpected GitLab import status: ${import_status}."
                  return 1
                  ;;
              esac
            done

            echo "::error title=Mirror failed: ${source_name}::GitLab server-side import timed out after three hours."
            return 1
          }

          encoded_group="$(jq -rn --arg value "${GITLAB_GROUP}" '$value | @uri')"
          if ! status="$(gitlab_request GET "${GITLAB_API_URL}/groups/${encoded_group}")"; then
            echo "::error::Unable to query the GitLab group."
            exit 1
          fi
          if [[ "${status}" != "200" ]]; then
            echo "::error::Unable to query the GitLab group (HTTP ${status})."
            exit 1
          fi
          group_id="$(jq -er '.id' "${response_file}")"

          github_auth="$(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 --wrap=0)"
          echo "::add-mask::${github_auth}"
          github_token_uri="$(jq -rn --arg value "${GITHUB_TOKEN}" '$value | @uri')"
          echo "::add-mask::${github_token_uri}"

          # GitLab.com limits each Git push to 5 GiB. Seed large empty projects through GitLab's import API,
          # using a lower threshold because GitHub's reported repository size is not the resulting pack size.
          gitlab_import_threshold_kib="$((4 * 1024 * 1024))"

          repository_count="$(jq -er 'length' "${repository_file}")"
          successful_repository_count=0
          failed_repository_names=()
          failed_repository_reasons=()
          for ((index = 0; index < repository_count; index++)); do
            repository="$(jq -ec ".[${index}]" "${repository_file}")"
            repository_number="$((index + 1))"
            source_name="$(jq -er '.name' <<< "${repository}")"
            source_clone_url="$(jq -er '.cloneUrl' <<< "${repository}")"
            source_size_kib="$(jq -er '.sizeKiB' <<< "${repository}")"
            target_name="$(jq -er '.targetName' <<< "${repository}")"
            target_path="$(jq -er '.targetPath' <<< "${repository}")"
            target_visibility="$(jq -er '.targetVisibility' <<< "${repository}")"
            description="Mirror of ${source_clone_url}"
            project_path="${GITLAB_GROUP}/${target_path}"
            encoded_project_path="$(jq -rn --arg value "${project_path}" '$value | @uri')"
            repository_log="${temp_root}/repository-${index}.log"

            set +e
            (
              set -euo pipefail
              trap 'echo "::endgroup::"' EXIT

              echo "::group::[${repository_number}/${repository_count}] ${source_name} -> ${project_path}"
              echo "Visibility: ${target_visibility}"
              echo "Checking for the GitLab project..."

              project_created=false
              gitlab_import_started=false

            if ! status="$(gitlab_request GET "${GITLAB_API_URL}/projects/${encoded_project_path}")"; then
              echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab project."
              exit 1
            fi

            if [[ "${status}" == "404" ]] || [[ "${status}" == "301" ]]; then
              if [[ "${status}" == "301" ]]; then
                echo "The exact GitLab path redirects elsewhere; reclaiming ${project_path}."
              else
                echo "Project does not exist; creating ${project_path}..."
              fi

              create_import_url=""
              if ((source_size_kib >= gitlab_import_threshold_kib)); then
                create_import_url="${source_clone_url}"
                if [[ "${target_visibility}" == "private" ]]; then
                  create_import_url="https://x-access-token:${github_token_uri}@${source_clone_url#https://}"
                  echo "::add-mask::${create_import_url}"
                fi
                echo "The large repository will be imported as part of project creation."
              fi

              payload="$(
                jq -nc \
                  --arg description "${description}" \
                  --arg import_url "${create_import_url}" \
                  --arg name "${target_name}" \
                  --arg path "${target_path}" \
                  --argjson namespace_id "${group_id}" \
                  --arg visibility "${target_visibility}" \
                  '{
                    description: $description,
                    name: $name,
                    path: $path,
                    namespace_id: $namespace_id,
                    visibility: $visibility,
                    initialize_with_readme: false
                  } + if $import_url == "" then {} else {import_url: $import_url} end'
              )"
              if ! status="$(gitlab_request POST "${GITLAB_API_URL}/projects" "${payload}")"; then
                echo "::error title=Mirror failed: ${source_name}::Unable to create the GitLab project."
                exit 1
              fi
              if [[ "${status}" != "201" ]]; then
                create_error="$(jq -r '.message // "No response message was provided."' "${response_file}")"
                create_error="${create_error//$'\r'/ }"
                create_error="${create_error//$'\n'/ }"
                echo "GitLab project creation response: ${create_error}"
                error_message="Unable to create the GitLab project (HTTP ${status})."
                echo "::error title=Mirror failed: ${source_name}::${error_message}"
                exit 1
              fi
              project_created=true
              if [[ -n "${create_import_url}" ]]; then
                gitlab_import_started=true
              fi
            elif [[ "${status}" != "200" ]]; then
              echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab project (HTTP ${status})."
              exit 1
            else
              echo "Found existing GitLab project ${project_path}."
            fi

            project_id="$(jq -er '.id' "${response_file}")"
            if [[ "${project_created}" == "true" ]]; then
              project_empty=true
            else
              project_empty="$(jq -r '.empty_repo == true' "${response_file}")"
            fi

            # Make a private source private before changing metadata or pushing any Git data.
            if [[ "${target_visibility}" == "private" ]]; then
              echo "Securing ${project_path} as private before mirroring..."
              privacy_payload="$(jq -nc '{visibility: "private"}')"
              if ! status="$(
                gitlab_request PUT "${GITLAB_API_URL}/projects/${project_id}" "${privacy_payload}"
              )"; then
                echo "::error title=Mirror failed: ${source_name}::Unable to secure the GitLab project."
                exit 1
              fi
              if [[ "${status}" != "200" ]] || \
                [[ "$(jq -er '.visibility' "${response_file}")" != "private" ]]; then
                echo "::error title=Mirror failed: ${source_name}::GitLab privacy verification failed."
                exit 1
              fi
            fi

            echo "Updating and verifying the GitLab project metadata..."
            payload="$(
              jq -nc \
                --arg description "${description}" \
                --arg visibility "${target_visibility}" \
                '{description: $description, visibility: $visibility}'
            )"
            if ! status="$(
              gitlab_request PUT "${GITLAB_API_URL}/projects/${project_id}" "${payload}"
            )"; then
              echo "::error title=Mirror failed: ${source_name}::Unable to update the GitLab project."
              exit 1
            fi
            if [[ "${status}" != "200" ]]; then
              echo "::error title=Mirror failed: ${source_name}::Unable to update the GitLab project (HTTP ${status})."
              exit 1
            fi

            actual_description="$(jq -er '.description // ""' "${response_file}")"
            actual_visibility="$(jq -er '.visibility' "${response_file}")"
            if [[ "${actual_description}" != "${description}" ]]; then
              echo "::error title=Mirror failed: ${source_name}::GitLab description verification failed."
              exit 1
            fi
            if [[ "${actual_visibility}" != "${target_visibility}" ]]; then
              echo "::error title=Mirror failed: ${source_name}::GitLab visibility verification failed."
              exit 1
            fi

            target_clone_url="$(jq -er '.ssh_url_to_repo' "${response_file}")"

            gitlab_incremental_seed=false
            if [[ "${project_empty}" == "true" ]] && \
              ((source_size_kib >= gitlab_import_threshold_kib)) && \
              [[ "${gitlab_import_started}" != "true" ]]; then
              echo "Empty large project detected (${source_size_kib} KiB reported by GitHub)."
              echo "Scheduling a GitLab server-side Git import to avoid the 5 GiB push limit..."
              if [[ "${target_visibility}" == "private" ]]; then
                import_payload="$(
                  jq -nc \
                    --arg import_url "${source_clone_url}" \
                    --arg import_url_user "x-access-token" \
                    --arg import_url_password "${GITHUB_TOKEN}" \
                    '{
                      import_url: $import_url,
                      import_url_user: $import_url_user,
                      import_url_password: $import_url_password
                    }'
                )"
              else
                import_payload="$(
                  jq -nc \
                    --arg import_url "${source_clone_url}" \
                    '{import_url: $import_url}'
                )"
              fi

              if ! status="$(
                gitlab_request POST "${GITLAB_API_URL}/projects/${project_id}/import/git" "${import_payload}"
              )"; then
                echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import."
                exit 1
              fi
              if [[ "${status}" == "403" ]]; then
                echo "::error title=Mirror failed: ${source_name}::GITLAB_TOKEN needs Import API Create access."
                exit 1
              fi

              import_response_message="$(jq -r '.message // "No response message was provided."' "${response_file}")"
              import_response_message="${import_response_message//$'\r'/ }"
              import_response_message="${import_response_message//$'\n'/ }"
              if [[ "${status}" == "409" ]]; then
                case "${import_response_message}" in
                  "Import already in progress")
                    echo "A GitLab import is already in progress; continuing to monitor it."
                    gitlab_import_started=true
                    ;;
                  "Project already has a repository")
                    echo "GitLab has repository storage from an earlier push; using incremental seeding."
                    gitlab_incremental_seed=true
                    ;;
                  *)
                    echo "GitLab import conflict: ${import_response_message}"
                    echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import."
                    exit 1
                    ;;
                esac
              elif [[ "${status}" != "201" ]] && [[ "${status}" != "202" ]]; then
                echo "GitLab import response: ${import_response_message}"
                echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import" \
                  "(HTTP ${status})."
                exit 1
              else
                gitlab_import_started=true
              fi
            elif [[ "${project_empty}" != "true" ]] && \
              ((source_size_kib >= gitlab_import_threshold_kib)); then
              echo "Large project is already populated; using an incremental Git push."
            fi

            mirror_dir="${temp_root}/repository.git"
            echo "Cloning ${source_clone_url} as a mirror..."
            if ! git \
              -c http.https://github.com/.extraheader="AUTHORIZATION: basic ${github_auth}" \
              clone --mirror --progress "${source_clone_url}" "${mirror_dir}"; then
              echo "::error title=Mirror failed: ${source_name}::Unable to clone the GitHub repository."
              exit 1
            fi

            mapfile -t pull_refs < <(
              git -C "${mirror_dir}" for-each-ref --format='%(refname)' refs/pull/
            )
            if ((${#pull_refs[@]} > 0)); then
              echo "Removing ${#pull_refs[@]} GitHub-only pull request refs from the mirror..."
              printf 'delete %s\n' "${pull_refs[@]}" | git -C "${mirror_dir}" update-ref --stdin
            fi

            if [[ "${gitlab_import_started}" == "true" ]]; then
              echo "Waiting for the GitLab server-side import to finish..."
              if ! wait_for_gitlab_import "${project_id}" "${source_name}"; then
                exit 1
              fi
            fi

            mapfile -t mirror_refs < <(
              git -C "${mirror_dir}" for-each-ref --format='%(refname)'
            )
            if ((${#mirror_refs[@]} == 0)); then
              echo "The GitHub repository has no refs to mirror."
              if [[ "${project_empty}" != "true" ]]; then
                echo "::error title=Mirror failed: ${source_name}::The GitHub repository is empty, but" \
                  "the GitLab repository is not. Refusing to leave stale GitLab refs."
                exit 1
              fi
              echo "The GitLab repository is also empty; no Git push is required."
            else
              if [[ "${gitlab_incremental_seed}" == "true" ]]; then
                mapfile -t seed_refs < <(
                  git -C "${mirror_dir}" for-each-ref --format='%(refname)' refs/heads/ refs/tags/
                )
                echo "Seeding ${#seed_refs[@]} branches and tags in separate pushes..."
                for seed_ref in "${seed_refs[@]}"; do
                  echo "Seeding ${seed_ref}..."
                  if ! git \
                    -C "${mirror_dir}" \
                    -c core.sshCommand="${gitlab_ssh_command}" \
                    push --progress "${target_clone_url}" "+${seed_ref}:${seed_ref}"; then
                    echo "::error title=Mirror failed: ${source_name}::Unable to seed ${seed_ref} on GitLab."
                    exit 1
                  fi
                done
              fi

              echo "Pushing the mirror to ${target_clone_url}..."
              if ! git \
                -C "${mirror_dir}" \
                -c core.sshCommand="${gitlab_ssh_command}" \
                push --mirror --progress "${target_clone_url}"; then
                echo "::error title=Mirror failed: ${source_name}::Unable to push the GitLab mirror."
                exit 1
              fi
            fi
            rm -rf "${mirror_dir}"
            echo "Completed mirror for ${source_name}."
            ) 2>&1 | tee "${repository_log}"
            repository_status="${PIPESTATUS[0]}"
            set -e

            rm -rf "${temp_root}/repository.git"
            if ((repository_status == 0)); then
              successful_repository_count="$((successful_repository_count + 1))"
            else
              error_line="$(grep -E '^::error(::| )' "${repository_log}" | tail -n 1 || true)"
              if [[ -n "${error_line}" ]]; then
                failure_reason="${error_line##*::}"
              else
                failure_reason="Repository mirroring failed; inspect its log group."
              fi
              failure_reason="${failure_reason//$'\r'/ }"
              failure_reason="${failure_reason//$'\n'/ }"
              failed_repository_names+=("${source_name}")
              failed_repository_reasons+=("${failure_reason}")
              echo "Recorded mirror failure for ${source_name}; continuing with the next repository."
            fi
            rm -f "${repository_log}"
          done

          failed_repository_count="${#failed_repository_names[@]}"
          {
            echo "## GitHub to GitLab mirror summary"
            echo
            echo "- Successful: ${successful_repository_count}"
            echo "- Failed: ${failed_repository_count}"
            if ((failed_repository_count > 0)); then
              echo
              echo "| Repository | Failure |"
              echo "| --- | --- |"
              for ((failure_index = 0; failure_index < failed_repository_count; failure_index++)); do
                summary_name="${failed_repository_names[$failure_index]//|/\\|}"
                summary_reason="${failed_repository_reasons[$failure_index]//|/\\|}"
                printf '| %s | %s |\n' "\`${summary_name}\`" "${summary_reason}"
              done
            fi
          } >> "${GITHUB_STEP_SUMMARY}"

          if ((failed_repository_count > 0)); then
            echo "::error::Failed to mirror ${failed_repository_count} of ${repository_count} repositories." \
              "See the step summary."
            exit 1
          fi

          echo "Mirrored all ${repository_count} repositories to GitLab."