diff --git a/bootstrap.sh b/bootstrap.sh index c897b7e7..eb5e638a 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -197,7 +197,7 @@ step_setup() { step_validate() { echo "Validating Config .. " | tee -a ${log} - ansible-playbook playbooks/validation/validate-schema.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags schema-validation 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/validation/validate-schema.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags schema-validation bash ./validations.sh --global-vars $global_vars --certs-vars $certs_vars 2>&1 | tee -a ${log} step_done } @@ -205,8 +205,8 @@ step_validate() { step_download_content() { echo "Downloading Deps Content .. " | tee -a ${log} # Download control binaries (oc, helm, etc.) first - required by download-content tasks - ansible-playbook playbooks/01-prepare.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags download-control-binaries 2>&1 | tee -a ${log} - ansible-playbook playbooks/01-prepare.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags download-content 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/01-prepare.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags download-control-binaries + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/01-prepare.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags download-content step_done } @@ -215,47 +215,47 @@ step_build_cache() { if [ "$is_disconnected" = false ]; then echo "Connected mode - skipping mirror registry setup" | tee -a ${log} else - ansible-playbook playbooks/02-mirror.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags mirror-registry 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/02-mirror.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags mirror-registry fi - ansible-playbook playbooks/03-deploy.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags configure-abi 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/03-deploy.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags configure-abi step_done } step_acquire_hardware() { echo "Acquiring Hardware .. " | tee -a ${log} # setup content for and boot machines - ansible-playbook playbooks/03-deploy.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags hardware,pre-install-validate 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/03-deploy.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags hardware,pre-install-validate step_done } step_deploy() { echo "Deploying management cluster .. " | tee -a ${log} # deploy Red Hat payload cluster - ansible-playbook playbooks/03-deploy.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags wait-deployment 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/03-deploy.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags wait-deployment step_done } step_post_install() { echo "Post install config.. " | tee -a ${log} # Apply SSL certificates - ansible-playbook playbooks/04-post-install.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags post-install-config 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/04-post-install.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags post-install-config step_done } step_operators() { echo "Deploying management apps .. " | tee -a ${log} # deploy Red Hat payload cluster - ansible-playbook playbooks/05-operators.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags operators 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/05-operators.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags operators step_done } step_day2() { echo "Clair disconnected .." | tee -a ${log} - ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags clair-disconnected 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags clair-disconnected step_done echo "Catalog source ACM policy .." | tee -a ${log} - ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags acm-policy-catalogsources 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars $EXTRA_VARS --tags acm-policy-catalogsources step_done } @@ -268,8 +268,8 @@ step_discovery() { echo "Start discovering nodes.. " | tee -a ${log} if [ -f $cloud_infra_vars ]; then - if ! ansible-playbook -e @$global_vars -e @$certs_vars -e @$cloud_infra_vars $EXTRA_VARS playbooks/07-configure-discovery.yaml 2>&1 | tee -a ${log}; then - echo -e "\\033[31m WARNING! \033[0m Discovery hosts has failed, please check config and rerun: ansible-playbook -e @$global_vars -e @$certs_vars -e @$cloud_infra_vars playbooks/07-configure-discovery.yaml" | tee -a ${log} + if ! ANSIBLE_LOG_PATH=${log} ansible-playbook -e @$global_vars -e @$certs_vars -e @$cloud_infra_vars $EXTRA_VARS playbooks/07-configure-discovery.yaml; then + echo -e "\\033[31m WARNING! \033[0m Discovery hosts has failed, please check config and rerun: ANSIBLE_LOG_PATH=${log} ansible-playbook -e @$global_vars -e @$certs_vars -e @$cloud_infra_vars playbooks/07-configure-discovery.yaml" fi fi step_done diff --git a/defaults/oc_mirror.yaml b/defaults/oc_mirror.yaml new file mode 100644 index 00000000..114562e0 --- /dev/null +++ b/defaults/oc_mirror.yaml @@ -0,0 +1,12 @@ +--- +# Retry and parallelism settings for oc-mirror tasks. +# These are internal defaults and are not meant to be overridden by the user. +ocMirrorParallelImages: 10 +ocMirrorParallelLayers: 10 +ocMirrorRetryTimes: 10 +ocMirrorRetryDelay: 10s +ocMirrorImageTimeout: 40m0s +ocMirrorAnsibleRetries: 10 +ocMirrorAnsibleDelay: 10 +ocMirrorCacheAnsibleRetries: 5 +ocMirrorParallelLayersLocalStorage: 1 diff --git a/operators/quay-operator/quay_disconnected.yaml b/operators/quay-operator/quay_disconnected.yaml index 6c250072..dd0bd2e8 100644 --- a/operators/quay-operator/quay_disconnected.yaml +++ b/operators/quay-operator/quay_disconnected.yaml @@ -27,27 +27,62 @@ src: "../../templates/registries.conf.j2" dest: "{{ lookup('env','HOME') }}/.config/containers/registries.conf" +- name: Ensure oc-mirror log directory exists + ansible.builtin.file: + path: "{{ workingDir }}/logs/" + state: directory + +- name: Set oc-mirror log path + ansible.builtin.set_fact: + _oc_mirror_quay_log: "{{ workingDir }}/logs/oc-mirror.progress.quay.{{ lookup('pipe', 'date +%s') }}.log" + +- name: Show oc-mirror log path + ansible.builtin.debug: + msg: "oc-mirror log file: {{ _oc_mirror_quay_log }}" + - name: Start oc-mirror process - ansible.builtin.shell: | - {{ workingDir }}/bin/oc-mirror --v2 \ - --log-level {{ ocMirrorLogLevel }} \ - --authfile {{ workingDir }}/config/pull-secret.quay.json \ - -c {{ workingDir }}/config/imagesetconfiguration.internal.yaml \ - --workspace file://{{ workingDir }}/config/oc-mirror-workspace-quay \ - docker://registry-quay-quay-enterprise.apps.{{ clusterName }}.{{ baseDomain }} \ - --dest-tls-verify=false \ - --src-tls-verify=false \ - --parallel-images 10 \ - --parallel-layers {{ 1 if quayBackend == 'LocalStorage' else 10 }} \ - --retry-times 10 \ - --retry-delay 0 \ - --image-timeout 40m0s \ - > {{ workingDir }}/logs/oc-mirror.progress.quay.$(date +%s).log 2>&1 - - retries: 10 - delay: 10 - register: r_oc_mirror_quay - until: r_oc_mirror_quay is succeeded + block: + - name: Run oc-mirror to internal Quay + ansible.builtin.shell: | + {{ workingDir }}/bin/oc-mirror --v2 \ + --log-level {{ ocMirrorLogLevel }} \ + --authfile {{ workingDir }}/config/pull-secret.quay.json \ + -c {{ workingDir }}/config/imagesetconfiguration.internal.yaml \ + --workspace file://{{ workingDir }}/config/oc-mirror-workspace-quay \ + docker://registry-quay-quay-enterprise.apps.{{ clusterName }}.{{ baseDomain }} \ + --dest-tls-verify=false \ + --src-tls-verify=false \ + --parallel-images {{ ocMirrorParallelImages }} \ + --parallel-layers {{ ocMirrorParallelLayersLocalStorage if quayBackend == 'LocalStorage' else ocMirrorParallelLayers }} \ + --retry-times {{ ocMirrorRetryTimes }} \ + --retry-delay {{ ocMirrorRetryDelay }} \ + --image-timeout {{ ocMirrorImageTimeout }} \ + > {{ _oc_mirror_quay_log }} 2>&1 + retries: "{{ ocMirrorAnsibleRetries }}" + delay: "{{ ocMirrorAnsibleDelay }}" + register: r_oc_mirror_quay + until: r_oc_mirror_quay is succeeded + + rescue: + - name: Read oc-mirror log tail + ansible.builtin.command: + argv: + - tail + - -n + - "10" + - "{{ _oc_mirror_quay_log }}" + register: _oc_mirror_quay_tail + changed_when: false + failed_when: false + + - name: oc-mirror failed + ansible.builtin.fail: + msg: |- + oc-mirror to internal Quay failed after {{ r_oc_mirror_quay.attempts }} attempts. + + {{ _oc_mirror_quay_tail.stdout | default('Unable to read oc-mirror log tail.') }} + + Full log: {{ _oc_mirror_quay_log }} - name: Delete registries.conf for the oc-mirror process ansible.builtin.file: diff --git a/playbooks/common/load-vars.yaml b/playbooks/common/load-vars.yaml index af1b2ffd..818662f8 100644 --- a/playbooks/common/load-vars.yaml +++ b/playbooks/common/load-vars.yaml @@ -22,6 +22,7 @@ - mirror_registry.yaml - quay_operator.yaml - k8s.yaml + - oc_mirror.yaml loop_control: loop_var: config_file diff --git a/playbooks/tasks/deploy_plugin.yaml b/playbooks/tasks/deploy_plugin.yaml index 0c0c2abc..924cca15 100644 --- a/playbooks/tasks/deploy_plugin.yaml +++ b/playbooks/tasks/deploy_plugin.yaml @@ -124,28 +124,76 @@ - plugin.operators is defined tags: mirror +- name: Ensure oc-mirror log directory exists + ansible.builtin.file: + path: "{{ workingDir }}/logs/" + state: directory + when: + - plugin.mirror | default('none') == 'plugin' + - disconnected | default(true) | bool + tags: mirror + +- name: Set oc-mirror plugin log path + ansible.builtin.set_fact: + _oc_mirror_plugin_log: "{{ workingDir }}/logs/oc-mirror-plugin-{{ plugin_name }}.progress.{{ lookup('pipe', 'date +%s') }}.log" + when: + - plugin.mirror | default('none') == 'plugin' + - disconnected | default(true) | bool + tags: mirror + +- name: Show oc-mirror log path + ansible.builtin.debug: + msg: "oc-mirror log file: {{ _oc_mirror_plugin_log }}" + when: + - plugin.mirror | default('none') == 'plugin' + - disconnected | default(true) | bool + tags: mirror + - name: Run oc-mirror for plugin - ansible.builtin.shell: | - {{ workingDir }}/bin/oc-mirror --v2 \ - --log-level {{ ocMirrorLogLevel }} \ - --authfile "{{ pullSecretPath }}" \ - -c {{ workingDir }}/plugin-{{ plugin_name }}-imageset.yaml \ - --workspace file://{{ workingDir }}/config/oc-mirror-workspace \ - docker://{{ quayHostname }}:8443 \ - --dest-tls-verify=false \ - --parallel-images 10 \ - --parallel-layers 10 \ - --retry-times 10 \ - --retry-delay 0 \ - > {{ workingDir }}/logs/oc-mirror-plugin-{{ plugin_name }}.progress.$(date +%s).log 2>&1 - retries: 10 - delay: 10 - register: r_plugin_mirror - until: r_plugin_mirror is success when: - plugin.mirror | default('none') == 'plugin' - disconnected | default(true) | bool tags: mirror + block: + - name: Run oc-mirror for plugin {{ plugin_name }} + ansible.builtin.shell: | + {{ workingDir }}/bin/oc-mirror --v2 \ + --log-level {{ ocMirrorLogLevel }} \ + --authfile "{{ pullSecretPath }}" \ + -c {{ workingDir }}/plugin-{{ plugin_name }}-imageset.yaml \ + --workspace file://{{ workingDir }}/config/oc-mirror-workspace \ + docker://{{ quayHostname }}:8443 \ + --dest-tls-verify=false \ + --parallel-images {{ ocMirrorParallelImages }} \ + --parallel-layers {{ ocMirrorParallelLayers }} \ + --retry-times {{ ocMirrorRetryTimes }} \ + --retry-delay {{ ocMirrorRetryDelay }} \ + > {{ _oc_mirror_plugin_log }} 2>&1 + retries: "{{ ocMirrorAnsibleRetries }}" + delay: "{{ ocMirrorAnsibleDelay }}" + register: r_plugin_mirror + until: r_plugin_mirror is success + + rescue: + - name: Read oc-mirror log tail + ansible.builtin.command: + argv: + - tail + - -n + - "10" + - "{{ _oc_mirror_plugin_log }}" + register: _oc_mirror_plugin_tail + changed_when: false + failed_when: false + + - name: oc-mirror failed + ansible.builtin.fail: + msg: |- + oc-mirror for plugin {{ plugin_name }} failed after {{ r_plugin_mirror.attempts }} attempts. + + {{ _oc_mirror_plugin_tail.stdout | default('Unable to read oc-mirror log tail.') }} + + Full log: {{ _oc_mirror_plugin_log }} - name: Apply updated mirror manifests to cluster ansible.builtin.shell: | @@ -239,32 +287,75 @@ - r_plugin_mirror is success tags: mirror +- name: Set oc-mirror plugin Quay Enterprise log path + ansible.builtin.set_fact: + _oc_mirror_plugin_quay_log: "{{ workingDir }}/logs/oc-mirror-plugin-{{ plugin_name }}-quay.progress.{{ lookup('pipe', 'date +%s') }}.log" + when: + - plugin.mirror | default('none') == 'plugin' + - disconnected | default(true) | bool + - r_plugin_mirror is defined + - r_plugin_mirror is success + tags: mirror + +- name: Show oc-mirror plugin Quay Enterprise log path + ansible.builtin.debug: + msg: "oc-mirror log file: {{ _oc_mirror_plugin_quay_log }}" + when: + - plugin.mirror | default('none') == 'plugin' + - disconnected | default(true) | bool + - r_plugin_mirror is defined + - r_plugin_mirror is success + tags: mirror + - name: Run oc-mirror for plugin (Quay Enterprise) - ansible.builtin.shell: | - {{ workingDir }}/bin/oc-mirror --v2 \ - --log-level {{ ocMirrorLogLevel }} \ - --authfile {{ workingDir }}/config/pull-secret.quay.json \ - -c {{ workingDir }}/plugin-{{ plugin_name }}-imageset.internal.yaml \ - --workspace file://{{ workingDir }}/config/oc-mirror-workspace-quay \ - docker://registry-quay-quay-enterprise.apps.{{ clusterName }}.{{ baseDomain }} \ - --dest-tls-verify=false \ - --src-tls-verify=false \ - --parallel-images 10 \ - --parallel-layers {{ 1 if quayBackend == 'LocalStorage' else 10 }} \ - --retry-times 10 \ - --retry-delay 0 \ - --image-timeout 40m0s \ - > {{ workingDir }}/logs/oc-mirror-plugin-{{ plugin_name }}-quay.progress.$(date +%s).log 2>&1 - retries: 10 - delay: 10 - register: r_plugin_mirror_quay - until: r_plugin_mirror_quay is success when: - plugin.mirror | default('none') == 'plugin' - disconnected | default(true) | bool - r_plugin_mirror is defined - r_plugin_mirror is success tags: mirror + block: + - name: Run oc-mirror for plugin {{ plugin_name }} (Quay Enterprise) + ansible.builtin.shell: | + {{ workingDir }}/bin/oc-mirror --v2 \ + --log-level {{ ocMirrorLogLevel }} \ + --authfile {{ workingDir }}/config/pull-secret.quay.json \ + -c {{ workingDir }}/plugin-{{ plugin_name }}-imageset.internal.yaml \ + --workspace file://{{ workingDir }}/config/oc-mirror-workspace-quay \ + docker://registry-quay-quay-enterprise.apps.{{ clusterName }}.{{ baseDomain }} \ + --dest-tls-verify=false \ + --src-tls-verify=false \ + --parallel-images {{ ocMirrorParallelImages }} \ + --parallel-layers {{ ocMirrorParallelLayersLocalStorage if quayBackend == 'LocalStorage' else ocMirrorParallelLayers }} \ + --retry-times {{ ocMirrorRetryTimes }} \ + --retry-delay {{ ocMirrorRetryDelay }} \ + --image-timeout {{ ocMirrorImageTimeout }} \ + > {{ _oc_mirror_plugin_quay_log }} 2>&1 + retries: "{{ ocMirrorAnsibleRetries }}" + delay: "{{ ocMirrorAnsibleDelay }}" + register: r_plugin_mirror_quay + until: r_plugin_mirror_quay is success + + rescue: + - name: Read oc-mirror log tail + ansible.builtin.command: + argv: + - tail + - -n + - "10" + - "{{ _oc_mirror_plugin_quay_log }}" + register: _oc_mirror_plugin_quay_tail + changed_when: false + failed_when: false + + - name: oc-mirror failed + ansible.builtin.fail: + msg: |- + oc-mirror for plugin {{ plugin_name }} (Quay Enterprise) failed after {{ r_plugin_mirror_quay.attempts }} attempts. + + {{ _oc_mirror_plugin_quay_tail.stdout | default('Unable to read oc-mirror log tail.') }} + + Full log: {{ _oc_mirror_plugin_quay_log }} - name: Delete registries.conf after plugin Quay Enterprise mirror ansible.builtin.file: diff --git a/playbooks/tasks/mirror_cache.yaml b/playbooks/tasks/mirror_cache.yaml index ebe3d42c..839dc252 100644 --- a/playbooks/tasks/mirror_cache.yaml +++ b/playbooks/tasks/mirror_cache.yaml @@ -34,19 +34,57 @@ path: "{{ workingDir }}/bin/oc-mirror" mode: "0755" +- name: Ensure oc-mirror log directory exists + ansible.builtin.file: + path: "{{ workingDir }}/logs/" + state: directory + +- name: Set oc-mirror cache log path + ansible.builtin.set_fact: + _oc_mirror_cache_log: "{{ workingDir }}/logs/oc-mirror-cache.progress.{{ lookup('pipe', 'date +%s') }}.log" + +- name: Show oc-mirror log path + ansible.builtin.debug: + msg: "oc-mirror log file: {{ _oc_mirror_cache_log }}" + - name: Start oc-mirror process - ansible.builtin.shell: > - {{ workingDir }}/bin/oc-mirror --v2 - --log-level {{ ocMirrorLogLevel }} - --authfile {{ workingDir }}/config/pull-secret-cache.json - --dest-tls-verify=false - -c {{ workingDir }}/config/imagesetconfiguration-cache.yaml - --workspace file://{{ workingDir }}/config/oc-mirror-cache-workspace - --parallel-images 10 --parallel-layers 10 - --retry-times 10 --retry-delay 0 - docker://{{ dc_cache_address }} - > {{ workingDir }}/logs/oc-mirror-cache.progress.$(date +%s).log 2>&1 - retries: 5 - delay: 10 - register: __r_oc_mirror_cache - until: __r_oc_mirror_cache is success + block: + - name: Run oc-mirror to cache registry + ansible.builtin.shell: | + {{ workingDir }}/bin/oc-mirror --v2 \ + --log-level {{ ocMirrorLogLevel }} \ + --authfile {{ workingDir }}/config/pull-secret-cache.json \ + --dest-tls-verify=false \ + -c {{ workingDir }}/config/imagesetconfiguration-cache.yaml \ + --workspace file://{{ workingDir }}/config/oc-mirror-cache-workspace \ + --parallel-images {{ ocMirrorParallelImages }} \ + --parallel-layers {{ ocMirrorParallelLayers }} \ + --retry-times {{ ocMirrorRetryTimes }} \ + --retry-delay {{ ocMirrorRetryDelay }} \ + docker://{{ dc_cache_address }} \ + > {{ _oc_mirror_cache_log }} 2>&1 + retries: "{{ ocMirrorCacheAnsibleRetries }}" + delay: "{{ ocMirrorAnsibleDelay }}" + register: __r_oc_mirror_cache + until: __r_oc_mirror_cache is success + + rescue: + - name: Read oc-mirror log tail + ansible.builtin.command: + argv: + - tail + - -n + - "10" + - "{{ _oc_mirror_cache_log }}" + register: _oc_mirror_cache_tail + changed_when: false + failed_when: false + + - name: oc-mirror failed + ansible.builtin.fail: + msg: |- + oc-mirror to cache registry failed after {{ __r_oc_mirror_cache.attempts }} attempts. + + {{ _oc_mirror_cache_tail.stdout | default('Unable to read oc-mirror log tail.') }} + + Full log: {{ _oc_mirror_cache_log }} diff --git a/playbooks/tasks/mirror_registry.yaml b/playbooks/tasks/mirror_registry.yaml index c55511c0..e9180daa 100644 --- a/playbooks/tasks/mirror_registry.yaml +++ b/playbooks/tasks/mirror_registry.yaml @@ -46,20 +46,58 @@ path: "{{ workingDir }}/bin/oc-mirror" mode: "0755" +- name: Ensure oc-mirror log directory exists + ansible.builtin.file: + path: "{{ workingDir }}/logs/" + state: directory + +- name: Set oc-mirror log path + ansible.builtin.set_fact: + _oc_mirror_log: "{{ workingDir }}/logs/oc-mirror.progress.{{ lookup('pipe', 'date +%s') }}.log" + +- name: Show oc-mirror log path + ansible.builtin.debug: + msg: "oc-mirror log file: {{ _oc_mirror_log }}" + - name: Start oc-mirror process - ansible.builtin.shell: > - {{ workingDir }}/bin/oc-mirror --v2 - {{ '--dry-run' if (mirror_dry_run | default(false) | bool) else '' }} - --log-level {{ ocMirrorLogLevel }} - --authfile {{ pullSecretPath }} - --dest-tls-verify=false - -c {{ workingDir }}/config/imagesetconfiguration.yaml - --workspace file://{{ workingDir }}/config/oc-mirror-workspace - --parallel-images 10 --parallel-layers 10 - --retry-times 10 --retry-delay 0 - docker://{{ quayHostname }}:8443 - > {{ workingDir }}/logs/oc-mirror.progress.$(date +%s).log 2>&1 - retries: "{{ 1 if (mirror_dry_run | default(false) | bool) else 10 }}" - delay: "{{ 0 if (mirror_dry_run | default(false) | bool) else 10 }}" - register: r_oc_mirror - until: r_oc_mirror is success + block: + - name: Run oc-mirror to mirror registry + ansible.builtin.shell: | + {{ workingDir }}/bin/oc-mirror --v2 \ + {{ '--dry-run' if (mirror_dry_run | default(false) | bool) else '' }} \ + --log-level {{ ocMirrorLogLevel }} \ + --authfile {{ pullSecretPath }} \ + --dest-tls-verify=false \ + -c {{ workingDir }}/config/imagesetconfiguration.yaml \ + --workspace file://{{ workingDir }}/config/oc-mirror-workspace \ + --parallel-images {{ ocMirrorParallelImages }} \ + --parallel-layers {{ ocMirrorParallelLayers }} \ + --retry-times {{ ocMirrorRetryTimes }} \ + --retry-delay {{ ocMirrorRetryDelay }} \ + docker://{{ quayHostname }}:8443 \ + > {{ _oc_mirror_log }} 2>&1 + retries: "{{ 1 if (mirror_dry_run | default(false) | bool) else ocMirrorAnsibleRetries }}" + delay: "{{ 0 if (mirror_dry_run | default(false) | bool) else ocMirrorAnsibleDelay }}" + register: r_oc_mirror + until: r_oc_mirror is success + + rescue: + - name: Read oc-mirror log tail + ansible.builtin.command: + argv: + - tail + - -n + - "10" + - "{{ _oc_mirror_log }}" + register: _oc_mirror_tail + changed_when: false + failed_when: false + + - name: oc-mirror failed + ansible.builtin.fail: + msg: |- + oc-mirror to mirror registry failed after {{ r_oc_mirror.attempts }} attempts. + + {{ _oc_mirror_tail.stdout | default('Unable to read oc-mirror log tail.') }} + + Full log: {{ _oc_mirror_log }} diff --git a/playbooks/validation/tasks/defaults_schema_validation.yaml b/playbooks/validation/tasks/defaults_schema_validation.yaml index 632d373e..9479db12 100644 --- a/playbooks/validation/tasks/defaults_schema_validation.yaml +++ b/playbooks/validation/tasks/defaults_schema_validation.yaml @@ -44,6 +44,12 @@ criteria: "{{ lookup('ansible.builtin.file', '../../schemas/quay_operator.yaml') | from_yaml | combine(schema_definitions, recursive=True) | to_json }}" engine: ansible.utils.jsonschema +- name: validate defaults/oc_mirror.yaml schema + ansible.utils.validate: + data: "{{ lookup('ansible.builtin.file', '../../defaults/oc_mirror.yaml') | from_yaml | to_json }}" + criteria: "{{ lookup('ansible.builtin.file', '../../schemas/oc_mirror.yaml') | from_yaml | combine(schema_definitions, recursive=True) | to_json }}" + engine: ansible.utils.jsonschema + - name: validate defaults/k8s.yaml schema ansible.utils.validate: data: "{{ lookup('ansible.builtin.file', '../../defaults/k8s.yaml') | from_yaml | to_json }}" diff --git a/schemas/oc_mirror.yaml b/schemas/oc_mirror.yaml new file mode 100644 index 00000000..18a6ced9 --- /dev/null +++ b/schemas/oc_mirror.yaml @@ -0,0 +1,54 @@ +--- +"$schema": "http://json-schema.org/draft-07/schema" +version: "1.0" +type: object + +additionalProperties: false +properties: + ocMirrorParallelImages: + type: integer + minimum: 1 + description: Number of parallel image downloads for oc-mirror + ocMirrorParallelLayers: + type: integer + minimum: 1 + description: Number of parallel layer downloads for oc-mirror + ocMirrorRetryTimes: + type: integer + minimum: 0 + description: Number of internal retries for oc-mirror + ocMirrorRetryDelay: + type: string + pattern: "^[0-9]+[smh]?$" + description: Delay between oc-mirror internal retries + ocMirrorImageTimeout: + type: string + pattern: "^[0-9]+[smh][0-9]*[smh]?$" + description: Timeout for individual image mirroring + ocMirrorAnsibleRetries: + type: integer + minimum: 1 + description: Number of Ansible-level retries for oc-mirror tasks + ocMirrorAnsibleDelay: + type: integer + minimum: 0 + description: Delay in seconds between Ansible-level retries + ocMirrorCacheAnsibleRetries: + type: integer + minimum: 1 + description: Number of Ansible-level retries for oc-mirror cache tasks + ocMirrorParallelLayersLocalStorage: + type: integer + minimum: 1 + description: Number of parallel layer downloads when using LocalStorage backend + +required: + - ocMirrorParallelImages + - ocMirrorParallelLayers + - ocMirrorRetryTimes + - ocMirrorRetryDelay + - ocMirrorImageTimeout + - ocMirrorAnsibleRetries + - ocMirrorAnsibleDelay + - ocMirrorCacheAnsibleRetries + - ocMirrorParallelLayersLocalStorage diff --git a/sync.sh b/sync.sh index 7bc462fa..59a661af 100755 --- a/sync.sh +++ b/sync.sh @@ -69,26 +69,26 @@ done step_done echo "Validating Config .. " | tee -a ${log} - ansible-playbook playbooks/validation/validate-schema.yaml -e@$global_vars -e@$certs_vars --tags schema-validation 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/validation/validate-schema.yaml -e@$global_vars -e@$certs_vars --tags schema-validation bash ./validations.sh --global-vars $global_vars --certs-vars $certs_vars 2>&1 | tee -a ${log} step_done echo "Building local cache .. " | tee -a ${log} - ansible-playbook playbooks/02-mirror.yaml -e@$global_vars -e@$certs_vars --tags mirror-registry 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/02-mirror.yaml -e@$global_vars -e@$certs_vars --tags mirror-registry step_done echo "Quay disconnected .." | tee -a ${log} - ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars --tags quay-disconnected 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars --tags quay-disconnected step_done echo "Clair disconnected .." | tee -a ${log} - ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars --tags clair-disconnected 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars --tags clair-disconnected step_done echo "ACM ClusterImageSets .." | tee -a ${log} - ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars --tags acm-cis 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars --tags acm-cis step_done echo "OpenShift Pipelines .." | tee -a ${log} - ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars --tags openshift-pipelines 2>&1 | tee -a ${log} + ANSIBLE_LOG_PATH=${log} ansible-playbook playbooks/06-day2.yaml -e@$global_vars -e@$certs_vars --tags openshift-pipelines step_done