Skip to content

Mirror GitHub to GitLab #25

Mirror GitHub to GitLab

Mirror GitHub to GitLab #25

---
# Mirror all repositories in the LizardByte GitHub organization to GitLab.
name: Mirror GitHub to GitLab
permissions: {}
on:
schedule:
- cron: '0 3 * * *'
workflow_dispatch:
concurrency:
group: mirror-github-to-gitlab
cancel-in-progress: false
jobs:
mirror:
name: Mirror GitHub to GitLab
permissions:
contents: read
runs-on: ubuntu-latest
steps:
# Process repositories in one job so project creation, privacy checks, and mirroring stay together.
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Get repositories
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.GH_BOT_TOKEN }}
script: |
const fs = require('fs');
const ignoredRepositoryNames = JSON.parse(
fs.readFileSync('.github/repositories-to-ignore.json', 'utf8'),
);
if (
!Array.isArray(ignoredRepositoryNames)
|| !ignoredRepositoryNames.every((name) => typeof name === 'string' && name.length > 0)
) {
throw new Error('Expected an array of non-empty repository names.');
}
const ignoredRepositories = new Set(ignoredRepositoryNames);
const securityAdvisoryIdSegment = '[23456789cfghjmpqrvwx]{4}';
const securityAdvisoryForkPattern = new RegExp(
`-ghsa-${securityAdvisoryIdSegment}-${securityAdvisoryIdSegment}-${securityAdvisoryIdSegment}$`,
'i',
);
const shouldIgnoreRepository = (repo) => (
ignoredRepositories.has(repo.name) || securityAdvisoryForkPattern.test(repo.name)
);
const opts = github.rest.repos.listForOrg.endpoint.merge({ org: context.repo.owner });
const repos = await github.paginate(opts);
const gitlabTarget = (repo) => {
const prefixNames = { '.': 'dot-', '-': 'dash-', '_': 'underscore-' };
const lowerName = repo.name.toLowerCase();
let targetName = lowerName.replace(
/^[._-]+/,
(prefix) => [...prefix].map((character) => prefixNames[character]).join(''),
);
targetName = targetName
.replace(/[^a-z0-9_.-]+/g, '-')
.replace(/[._-]{2,}/g, '-')
.replace(/[._-]+$/g, '');
if (targetName.endsWith('.git') || targetName.endsWith('.atom')) {
targetName += '-repo';
}
if (!targetName) {
targetName = 'repository';
}
const transformed = targetName !== lowerName;
return {
targetName: transformed ? targetName : repo.name,
targetPath: transformed ? `${targetName}-${repo.id}` : repo.name,
};
};
const repositoryData = repos
.filter((repo) => !shouldIgnoreRepository(repo))
.map((repo) => ({
cloneUrl: repo.clone_url,
name: repo.name,
sizeKiB: repo.size,
targetVisibility: repo.visibility === 'public' ? 'public' : 'private',
...gitlabTarget(repo),
}));
fs.writeFileSync('repositories.json', JSON.stringify(repositoryData), { mode: 0o600 });
core.info(`Ignored ${repos.length - repositoryData.length} repositories.`);
core.info(`Prepared ${repositoryData.length} repositories for mirroring.`);
- name: Mirror repositories
shell: bash
env:
GIT_TERMINAL_PROMPT: '0'
GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN }}
GITLAB_API_URL: https://gitlab.com/api/v4
GITLAB_GROUP: lizardbyte
GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
run: |
set -euo pipefail
if [[ -z "${GITHUB_TOKEN}" ]] || \
[[ -z "${GITLAB_TOKEN}" ]] || \
[[ -z "${GITLAB_SSH_PRIVATE_KEY}" ]]; then
echo "::error::GH_BOT_TOKEN, GITLAB_TOKEN, and GITLAB_SSH_PRIVATE_KEY must all be configured."
exit 1
fi
repository_file="${GITHUB_WORKSPACE}/repositories.json"
response_file="$(mktemp)"
temp_root="$(mktemp -d)"
trap 'rm -f "${response_file}" "${repository_file}"; rm -rf "${temp_root}"' EXIT
ssh_dir="${temp_root}/ssh"
ssh_key="${ssh_dir}/id_ed25519"
ssh_known_hosts="${ssh_dir}/known_hosts"
mkdir -m 700 "${ssh_dir}"
printf '%s\n' "${GITLAB_SSH_PRIVATE_KEY}" > "${ssh_key}"
chmod 600 "${ssh_key}"
if ! ssh-keygen -y -P '' -f "${ssh_key}" > /dev/null; then
echo "::error::GITLAB_SSH_PRIVATE_KEY is not a valid unencrypted SSH private key."
exit 1
fi
# Published at https://docs.gitlab.com/user/gitlab_com/#ssh-known_hosts-entries.
gitlab_host_key='ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf'
printf 'gitlab.com %s\n' "${gitlab_host_key}" > "${ssh_known_hosts}"
chmod 600 "${ssh_known_hosts}"
gitlab_host_fingerprint="$(ssh-keygen -lf "${ssh_known_hosts}" -E sha256 | awk '{print $2}')"
if [[ "${gitlab_host_fingerprint}" != \
'SHA256:eUXGGm1YGsMAS7vkcx6JOJdOGHPem5gQp4taiCfCLB8' ]]; then
echo "::error::The configured GitLab SSH host key has an unexpected fingerprint."
exit 1
fi
gitlab_ssh_args=(
-i "${ssh_key}"
-o BatchMode=yes
-o IdentitiesOnly=yes
-o StrictHostKeyChecking=yes
-o "UserKnownHostsFile=${ssh_known_hosts}"
-o ConnectTimeout=30
)
printf -v gitlab_ssh_command '%q ' ssh "${gitlab_ssh_args[@]}"
echo "Verifying GitLab SSH authentication..."
if ! ssh "${gitlab_ssh_args[@]}" -T git@gitlab.com; then
echo "::error::Unable to authenticate to GitLab with GITLAB_SSH_PRIVATE_KEY."
exit 1
fi
gitlab_request() {
local method="$1"
local url="$2"
local data="${3:-}"
local curl_args=(
--silent
--output "${response_file}"
--write-out '%{http_code}'
--request "${method}"
--header "Accept: application/json"
--header "PRIVATE-TOKEN: ${GITLAB_TOKEN}"
)
if [[ -n "${data}" ]]; then
curl_args+=(
--header "Content-Type: application/json"
--data "${data}"
)
fi
curl "${curl_args[@]}" "${url}"
}
wait_for_gitlab_import() {
local project_id="$1"
local source_name="$2"
local deadline="$((SECONDS + 10800))"
local import_error
local import_status
local status
while ((SECONDS < deadline)); do
if ! status="$(gitlab_request GET "${GITLAB_API_URL}/projects/${project_id}/import")"; then
echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab import."
return 1
fi
if [[ "${status}" != "200" ]]; then
if [[ "${status}" == "403" ]]; then
echo "::error title=Mirror failed: ${source_name}::GITLAB_TOKEN needs Import API Read access."
return 1
fi
echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab import (HTTP ${status})."
return 1
fi
import_status="$(jq -er '.import_status' "${response_file}")"
case "${import_status}" in
finished)
echo "GitLab server-side import finished."
return 0
;;
scheduled | started)
echo "GitLab server-side import status: ${import_status}."
sleep 30
;;
failed)
import_error="$(jq -r '.import_error // "No import error detail was provided."' "${response_file}")"
import_error="${import_error//$'\r'/ }"
import_error="${import_error//$'\n'/ }"
echo "GitLab import failed: ${import_error}"
echo "::error title=Mirror failed: ${source_name}::GitLab server-side import failed."
return 1
;;
*)
echo "::error title=Mirror failed: ${source_name}::Unexpected GitLab import status: ${import_status}."
return 1
;;
esac
done
echo "::error title=Mirror failed: ${source_name}::GitLab server-side import timed out after three hours."
return 1
}
encoded_group="$(jq -rn --arg value "${GITLAB_GROUP}" '$value | @uri')"
if ! status="$(gitlab_request GET "${GITLAB_API_URL}/groups/${encoded_group}")"; then
echo "::error::Unable to query the GitLab group."
exit 1
fi
if [[ "${status}" != "200" ]]; then
echo "::error::Unable to query the GitLab group (HTTP ${status})."
exit 1
fi
group_id="$(jq -er '.id' "${response_file}")"
github_auth="$(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 --wrap=0)"
echo "::add-mask::${github_auth}"
github_token_uri="$(jq -rn --arg value "${GITHUB_TOKEN}" '$value | @uri')"
echo "::add-mask::${github_token_uri}"
# GitLab.com limits each Git push to 5 GiB. Seed large empty projects through GitLab's import API,
# using a lower threshold because GitHub's reported repository size is not the resulting pack size.
gitlab_import_threshold_kib="$((4 * 1024 * 1024))"
repository_count="$(jq -er 'length' "${repository_file}")"
successful_repository_count=0
failed_repository_names=()
failed_repository_reasons=()
for ((index = 0; index < repository_count; index++)); do
repository="$(jq -ec ".[${index}]" "${repository_file}")"
repository_number="$((index + 1))"
source_name="$(jq -er '.name' <<< "${repository}")"
source_clone_url="$(jq -er '.cloneUrl' <<< "${repository}")"
source_size_kib="$(jq -er '.sizeKiB' <<< "${repository}")"
target_name="$(jq -er '.targetName' <<< "${repository}")"
target_path="$(jq -er '.targetPath' <<< "${repository}")"
target_visibility="$(jq -er '.targetVisibility' <<< "${repository}")"
description="Mirror of ${source_clone_url}"
project_path="${GITLAB_GROUP}/${target_path}"
encoded_project_path="$(jq -rn --arg value "${project_path}" '$value | @uri')"
repository_log="${temp_root}/repository-${index}.log"
set +e
(
set -euo pipefail
trap 'echo "::endgroup::"' EXIT
echo "::group::[${repository_number}/${repository_count}] ${source_name} -> ${project_path}"
echo "Visibility: ${target_visibility}"
echo "Checking for the GitLab project..."
project_created=false
gitlab_import_started=false
if ! status="$(gitlab_request GET "${GITLAB_API_URL}/projects/${encoded_project_path}")"; then
echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab project."
exit 1
fi
if [[ "${status}" == "404" ]] || [[ "${status}" == "301" ]]; then
if [[ "${status}" == "301" ]]; then
echo "The exact GitLab path redirects elsewhere; reclaiming ${project_path}."
else
echo "Project does not exist; creating ${project_path}..."
fi
create_import_url=""
if ((source_size_kib >= gitlab_import_threshold_kib)); then
create_import_url="${source_clone_url}"
if [[ "${target_visibility}" == "private" ]]; then
create_import_url="https://x-access-token:${github_token_uri}@${source_clone_url#https://}"
echo "::add-mask::${create_import_url}"
fi
echo "The large repository will be imported as part of project creation."
fi
payload="$(
jq -nc \
--arg description "${description}" \
--arg import_url "${create_import_url}" \
--arg name "${target_name}" \
--arg path "${target_path}" \
--argjson namespace_id "${group_id}" \
--arg visibility "${target_visibility}" \
'{
description: $description,
name: $name,
path: $path,
namespace_id: $namespace_id,
visibility: $visibility,
initialize_with_readme: false
} + if $import_url == "" then {} else {import_url: $import_url} end'
)"
if ! status="$(gitlab_request POST "${GITLAB_API_URL}/projects" "${payload}")"; then
echo "::error title=Mirror failed: ${source_name}::Unable to create the GitLab project."
exit 1
fi
if [[ "${status}" != "201" ]]; then
create_error="$(jq -r '.message // "No response message was provided."' "${response_file}")"
create_error="${create_error//$'\r'/ }"
create_error="${create_error//$'\n'/ }"
echo "GitLab project creation response: ${create_error}"
error_message="Unable to create the GitLab project (HTTP ${status})."
echo "::error title=Mirror failed: ${source_name}::${error_message}"
exit 1
fi
project_created=true
if [[ -n "${create_import_url}" ]]; then
gitlab_import_started=true
fi
elif [[ "${status}" != "200" ]]; then
echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab project (HTTP ${status})."
exit 1
else
echo "Found existing GitLab project ${project_path}."
fi
project_id="$(jq -er '.id' "${response_file}")"
if [[ "${project_created}" == "true" ]]; then
project_empty=true
else
project_empty="$(jq -r '.empty_repo == true' "${response_file}")"
fi
# Make a private source private before changing metadata or pushing any Git data.
if [[ "${target_visibility}" == "private" ]]; then
echo "Securing ${project_path} as private before mirroring..."
privacy_payload="$(jq -nc '{visibility: "private"}')"
if ! status="$(
gitlab_request PUT "${GITLAB_API_URL}/projects/${project_id}" "${privacy_payload}"
)"; then
echo "::error title=Mirror failed: ${source_name}::Unable to secure the GitLab project."
exit 1
fi
if [[ "${status}" != "200" ]] || \
[[ "$(jq -er '.visibility' "${response_file}")" != "private" ]]; then
echo "::error title=Mirror failed: ${source_name}::GitLab privacy verification failed."
exit 1
fi
fi
echo "Updating and verifying the GitLab project metadata..."
payload="$(
jq -nc \
--arg description "${description}" \
--arg visibility "${target_visibility}" \
'{description: $description, visibility: $visibility}'
)"
if ! status="$(
gitlab_request PUT "${GITLAB_API_URL}/projects/${project_id}" "${payload}"
)"; then
echo "::error title=Mirror failed: ${source_name}::Unable to update the GitLab project."
exit 1
fi
if [[ "${status}" != "200" ]]; then
echo "::error title=Mirror failed: ${source_name}::Unable to update the GitLab project (HTTP ${status})."
exit 1
fi
actual_description="$(jq -er '.description // ""' "${response_file}")"
actual_visibility="$(jq -er '.visibility' "${response_file}")"
if [[ "${actual_description}" != "${description}" ]]; then
echo "::error title=Mirror failed: ${source_name}::GitLab description verification failed."
exit 1
fi
if [[ "${actual_visibility}" != "${target_visibility}" ]]; then
echo "::error title=Mirror failed: ${source_name}::GitLab visibility verification failed."
exit 1
fi
target_clone_url="$(jq -er '.ssh_url_to_repo' "${response_file}")"
gitlab_incremental_seed=false
if [[ "${project_empty}" == "true" ]] && \
((source_size_kib >= gitlab_import_threshold_kib)) && \
[[ "${gitlab_import_started}" != "true" ]]; then
echo "Empty large project detected (${source_size_kib} KiB reported by GitHub)."
echo "Scheduling a GitLab server-side Git import to avoid the 5 GiB push limit..."
if [[ "${target_visibility}" == "private" ]]; then
import_payload="$(
jq -nc \
--arg import_url "${source_clone_url}" \
--arg import_url_user "x-access-token" \
--arg import_url_password "${GITHUB_TOKEN}" \
'{
import_url: $import_url,
import_url_user: $import_url_user,
import_url_password: $import_url_password
}'
)"
else
import_payload="$(
jq -nc \
--arg import_url "${source_clone_url}" \
'{import_url: $import_url}'
)"
fi
if ! status="$(
gitlab_request POST "${GITLAB_API_URL}/projects/${project_id}/import/git" "${import_payload}"
)"; then
echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import."
exit 1
fi
if [[ "${status}" == "403" ]]; then
echo "::error title=Mirror failed: ${source_name}::GITLAB_TOKEN needs Import API Create access."
exit 1
fi
import_response_message="$(jq -r '.message // "No response message was provided."' "${response_file}")"
import_response_message="${import_response_message//$'\r'/ }"
import_response_message="${import_response_message//$'\n'/ }"
if [[ "${status}" == "409" ]]; then
case "${import_response_message}" in
"Import already in progress")
echo "A GitLab import is already in progress; continuing to monitor it."
gitlab_import_started=true
;;
"Project already has a repository")
echo "GitLab has repository storage from an earlier push; using incremental seeding."
gitlab_incremental_seed=true
;;
*)
echo "GitLab import conflict: ${import_response_message}"
echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import."
exit 1
;;
esac
elif [[ "${status}" != "201" ]] && [[ "${status}" != "202" ]]; then
echo "GitLab import response: ${import_response_message}"
echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import" \
"(HTTP ${status})."
exit 1
else
gitlab_import_started=true
fi
elif [[ "${project_empty}" != "true" ]] && \
((source_size_kib >= gitlab_import_threshold_kib)); then
echo "Large project is already populated; using an incremental Git push."
fi
mirror_dir="${temp_root}/repository.git"
echo "Cloning ${source_clone_url} as a mirror..."
if ! git \
-c http.https://github.com/.extraheader="AUTHORIZATION: basic ${github_auth}" \
clone --mirror --progress "${source_clone_url}" "${mirror_dir}"; then
echo "::error title=Mirror failed: ${source_name}::Unable to clone the GitHub repository."
exit 1
fi
mapfile -t pull_refs < <(
git -C "${mirror_dir}" for-each-ref --format='%(refname)' refs/pull/
)
if ((${#pull_refs[@]} > 0)); then
echo "Removing ${#pull_refs[@]} GitHub-only pull request refs from the mirror..."
printf 'delete %s\n' "${pull_refs[@]}" | git -C "${mirror_dir}" update-ref --stdin
fi
if [[ "${gitlab_import_started}" == "true" ]]; then
echo "Waiting for the GitLab server-side import to finish..."
if ! wait_for_gitlab_import "${project_id}" "${source_name}"; then
exit 1
fi
fi
mapfile -t mirror_refs < <(
git -C "${mirror_dir}" for-each-ref --format='%(refname)'
)
if ((${#mirror_refs[@]} == 0)); then
echo "The GitHub repository has no refs to mirror."
if [[ "${project_empty}" != "true" ]]; then
echo "::error title=Mirror failed: ${source_name}::The GitHub repository is empty, but" \
"the GitLab repository is not. Refusing to leave stale GitLab refs."
exit 1
fi
echo "The GitLab repository is also empty; no Git push is required."
else
if [[ "${gitlab_incremental_seed}" == "true" ]]; then
mapfile -t seed_refs < <(
git -C "${mirror_dir}" for-each-ref --format='%(refname)' refs/heads/ refs/tags/
)
echo "Seeding ${#seed_refs[@]} branches and tags in separate pushes..."
for seed_ref in "${seed_refs[@]}"; do
echo "Seeding ${seed_ref}..."
if ! git \
-C "${mirror_dir}" \
-c core.sshCommand="${gitlab_ssh_command}" \
push --progress "${target_clone_url}" "+${seed_ref}:${seed_ref}"; then
echo "::error title=Mirror failed: ${source_name}::Unable to seed ${seed_ref} on GitLab."
exit 1
fi
done
fi
echo "Pushing the mirror to ${target_clone_url}..."
if ! git \
-C "${mirror_dir}" \
-c core.sshCommand="${gitlab_ssh_command}" \
push --mirror --progress "${target_clone_url}"; then
echo "::error title=Mirror failed: ${source_name}::Unable to push the GitLab mirror."
exit 1
fi
fi
rm -rf "${mirror_dir}"
echo "Completed mirror for ${source_name}."
) 2>&1 | tee "${repository_log}"
repository_status="${PIPESTATUS[0]}"
set -e
rm -rf "${temp_root}/repository.git"
if ((repository_status == 0)); then
successful_repository_count="$((successful_repository_count + 1))"
else
error_line="$(grep -E '^::error(::| )' "${repository_log}" | tail -n 1 || true)"
if [[ -n "${error_line}" ]]; then
failure_reason="${error_line##*::}"
else
failure_reason="Repository mirroring failed; inspect its log group."
fi
failure_reason="${failure_reason//$'\r'/ }"
failure_reason="${failure_reason//$'\n'/ }"
failed_repository_names+=("${source_name}")
failed_repository_reasons+=("${failure_reason}")
echo "Recorded mirror failure for ${source_name}; continuing with the next repository."
fi
rm -f "${repository_log}"
done
failed_repository_count="${#failed_repository_names[@]}"
{
echo "## GitHub to GitLab mirror summary"
echo
echo "- Successful: ${successful_repository_count}"
echo "- Failed: ${failed_repository_count}"
if ((failed_repository_count > 0)); then
echo
echo "| Repository | Failure |"
echo "| --- | --- |"
for ((failure_index = 0; failure_index < failed_repository_count; failure_index++)); do
summary_name="${failed_repository_names[$failure_index]//|/\\|}"
summary_reason="${failed_repository_reasons[$failure_index]//|/\\|}"
printf '| %s | %s |\n' "\`${summary_name}\`" "${summary_reason}"
done
fi
} >> "${GITHUB_STEP_SUMMARY}"
if ((failed_repository_count > 0)); then
echo "::error::Failed to mirror ${failed_repository_count} of ${repository_count} repositories." \
"See the step summary."
exit 1
fi
echo "Mirrored all ${repository_count} repositories to GitLab."