Skip to content

Commit 7b0a2e3

Browse files
author
Miguel Angel
committed
am-configure: Configure Pipeline Local FS
- Create a new task file (`configure-pipeline-local-fs.yml`) to configure the Pipeline local FS Space and its locations. - Create a new task file (`configure-create-location.yml`) to be used to create all kinds of locations, checking if the location exists before creating it.
1 parent 195fe0a commit 7b0a2e3

File tree

4 files changed

+301
-0
lines changed

4 files changed

+301
-0
lines changed

‎defaults/main.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,31 @@ archivematica_src_syslog_mcpserver_level: "DEBUG"
220220
# enabled: '0'
221221
# field_name: 'command_id'
222222

223+
#
224+
# Configure locations
225+
#
226+
# For some different space protocols a dictionary will be used to configure locations.
227+
# The dictionary will have 4 keys:
228+
# 1) location_purpose: valid values are:
229+
#
230+
# "AR": AIP RECOVERY
231+
# "AS": AIP STORAGE
232+
# "CP": CURRENTLY PROCESSING
233+
# "DS": DIP STORAGE
234+
# "SD": SWORD DEPOSIT
235+
# "SS": STORAGE SERVICE INTERNAL PROCESSING
236+
# "BL": TRANSFER BACKLOG
237+
# "TS": TRANSFER SOURCE
238+
# "RP": REPLICATOR
239+
#
240+
# 2) location_path: it assumes that the Space absolute path is "/" and removes the "/" when creating the location. So when using
241+
# a different absolute path in Space, please write the relative path from Space path adding a "/" character to
242+
# the beginning of the string.
243+
#
244+
# 3) location_description: any location description
245+
#
246+
# 4) location_default: boolean
247+
223248
#
224249
# Configure SS locations
225250
#
@@ -228,3 +253,15 @@ archivematica_src_syslog_mcpserver_level: "DEBUG"
228253
# - { location_purpose: "AS", location_path: "/aipstore", location_description: "AipStore", location_default: "true" }
229254
# - { location_purpose: "TS", location_path: "/transfer-source/", location_description: "Transfer Source", location_default: "false" }
230255

256+
#
257+
# Configure SS Pipeline Local FileSystem locations. The CP location is mandatory when pipeline is on a server different than the one the SS is on.
258+
# Pipeline Local Filesystems refer to the storage that is local to the Archivematica pipeline,
259+
# but remote to the Storage Service. It's a ssh-based remote space.
260+
# More info:
261+
# https://www.archivematica.org/en/docs/storage-service-0.16/administrators/#pipeline-local-filesystem
262+
#
263+
#archivematica_src_configure_pipeline_localfs_locations:
264+
# - { location_purpose: "CP", location_path: "{{ archivematica_src_shareddir }}", location_description: "Pipeline Local Fylesistem CP", location_default: "false" }
265+
# - { location_purpose: "BL", location_path: "{{ archivematica_src_shareddir }}/www/AIPsStore/transferBacklog", location_description: "Pipeline Local Fylesistem BL", location_default: "false" }
266+
# - { location_purpose: "AS", location_path: "/aipstore", location_description: "AipStore", location_default: "true" }
267+
# - { location_purpose: "TS", location_path: "/transfer-source/", location_description: "Transfer Source", location_default: "false" }

‎tasks/configure-create-location.yml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
---
2+
3+
# This file works as a function to create locations. The arguments are:
4+
# - am_configure_space_uuid
5+
# - am_configure_pipeline_uuid
6+
# - am_configure_location: One record dictionary with fields:
7+
# - location_purpose
8+
# - location_path (absolute path)
9+
# - location_description
10+
# - location_default
11+
#
12+
# For instance, you can call this task file from other task file with:
13+
#
14+
#- name: "Create Pipeline Local Filesystem locations"
15+
# include_tasks: configure-create-location.yml
16+
# vars:
17+
# am_configure_space_uuid: "{{ am_configure_pipelinelocalfs_space_id.stdout }}"
18+
# am_configure_pipeline_uuid: "{{ am_configure_pipelinelocalfs_pipeline_uuid.stdout }}"
19+
# with_items: "{{ archivematica_src_configure_pipeline_localfs_locations }}"
20+
# loop_control:
21+
# loop_var: am_configure_location
22+
23+
- name: "Check if '{{ am_configure_location.location_purpose }}' location with '{{ am_configure_location.location_path }}' path exists in '{{ am_configure_space_uuid }}' space"
24+
shell: >
25+
echo "select location_id from locations_locationpipeline where location_id in
26+
(select uuid from locations_location where purpose='{{ am_configure_location.location_purpose }}'
27+
and relative_path='{{ am_configure_location.location_path | regex_replace('^/', '') }}'
28+
and space_id='{{ am_configure_space_uuid }}');"
29+
| {{ archivematica_src_ss_virtualenv }}/bin/python manage.py dbshell
30+
| tail -n1
31+
args:
32+
chdir: "{{ archivematica_src_ss_app }}"
33+
executable: /bin/bash
34+
environment: "{{ archivematica_src_ss_environment }}"
35+
delegate_to: "{{ archivematica_src_configure_ss_inventory_hostname | default(archivematica_src_configure_ss_url|urlsplit('hostname')) }}"
36+
remote_user: "{{ archivematica_src_configure_ss_ssh_user | default('artefactual') }}"
37+
register: am_configure_check_location_id
38+
39+
40+
- name: "Create '{{ am_configure_location.location_purpose }}' location with '{{ am_configure_location.location_path }}' path exists in '{{ am_configure_space_uuid }}' space"
41+
uri:
42+
url: "{{ archivematica_src_configure_ss_url }}/api/v2/location/"
43+
headers:
44+
Content-Type: "application/json"
45+
Authorization: "ApiKey {{ archivematica_src_configure_ss_user }}:{{ archivematica_src_configure_ss_api_key }}"
46+
body:
47+
pipeline: ["/api/v2/pipeline/{{ am_configure_pipeline_uuid }}/"]
48+
purpose: "{{ am_configure_location.location_purpose }}"
49+
relative_path: "{{ am_configure_location.location_path | regex_replace('^\\/', '') }}"
50+
description: "{{ am_configure_location.location_description }}"
51+
space: "/api/v2/space/{{ am_configure_space_uuid }}/"
52+
default: "{{ am_configure_location.location_default }}"
53+
body_format: json
54+
status_code: 201
55+
method: POST
56+
when: am_configure_check_location_id.stdout == ""
57+
58+
- name: "Get the location id to be replicated when location is a replicator"
59+
shell: >
60+
echo "select id from locations_location where description='{{ am_configure_location.location_replicaof }}';"
61+
| {{ archivematica_src_ss_virtualenv }}/bin/python manage.py dbshell
62+
| tail -n1
63+
args:
64+
chdir: "{{ archivematica_src_ss_app }}"
65+
executable: /bin/bash
66+
environment: "{{ archivematica_src_ss_environment }}"
67+
delegate_to: "{{ archivematica_src_configure_ss_inventory_hostname | default(archivematica_src_configure_ss_url|urlsplit('hostname')) }}"
68+
remote_user: "{{ archivematica_src_configure_ss_ssh_user | default('artefactual') }}"
69+
register: am_configure_replica_location_id
70+
when:
71+
- am_configure_location.location_purpose == "RP"
72+
- am_configure_location.location_replicaof is defined
73+
74+
- name: "Get the location id when location is a replicator"
75+
shell: >
76+
echo "select id from locations_location where description='{{ am_configure_location.location_description }}';"
77+
| {{ archivematica_src_ss_virtualenv }}/bin/python manage.py dbshell
78+
| tail -n1
79+
args:
80+
chdir: "{{ archivematica_src_ss_app }}"
81+
executable: /bin/bash
82+
environment: "{{ archivematica_src_ss_environment }}"
83+
delegate_to: "{{ archivematica_src_configure_ss_inventory_hostname | default(archivematica_src_configure_ss_url|urlsplit('hostname')) }}"
84+
remote_user: "{{ archivematica_src_configure_ss_ssh_user | default('artefactual') }}"
85+
register: am_configure_location_id
86+
when:
87+
- am_configure_location.location_purpose == "RP"
88+
- am_configure_location.location_replicaof is defined
89+
90+
- name: "Configure replication for location with description: '{{ am_configure_location.location_replicaof }}'"
91+
shell: >
92+
echo "insert into locations_location_replicators (from_location_id, to_location_id) values('{{ am_configure_replica_location_id.stdout }}','{{ }}' ;"
93+
| {{ archivematica_src_ss_virtualenv }}/bin/python manage.py dbshell
94+
| tail -n1
95+
args:
96+
chdir: "{{ archivematica_src_ss_app }}"
97+
executable: /bin/bash
98+
environment: "{{ archivematica_src_ss_environment }}"
99+
delegate_to: "{{ archivematica_src_configure_ss_inventory_hostname | default(archivematica_src_configure_ss_url|urlsplit('hostname')) }}"
100+
remote_user: "{{ archivematica_src_configure_ss_ssh_user | default('artefactual') }}"
101+
register: am_configure_replica_location_id
102+
when:
103+
- am_configure_location.location_purpose == "RP"
104+
- am_configure_location.location_replicaof is defined
105+
- am_configure_replica_location_id.stdout != ""
106+
- am_configure_location_id != ""

‎tasks/configure-pipeline-local-fs.yml‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
---
2+
# This tasks file configures the Pipeline Local Filesystem Spaces
3+
# Pipeline Local Filesystems refer to the storage that is local to the Archivematica pipeline,
4+
# but remote to the Storage Service. It's a ssh-based remote space.
5+
# More info:
6+
# https://www.archivematica.org/en/docs/storage-service-0.16/administrators/#pipeline-local-filesystem
7+
8+
9+
# Get pipeline uuid
10+
- name: Get pipeline uuid
11+
become: "yes"
12+
shell: >
13+
echo: "select value from DashboardSettings where name='dashboard_uuid';"
14+
| {{ archivematica_src_am_dashboard_virtualenv }}/bin/python manage.py dbshell
15+
| tail -n1
16+
args:
17+
chdir: "{{ archivematica_src_am_dashboard_app }}"
18+
executable: /bin/bash
19+
environment: "{{ archivematica_src_am_dashboard_environment }}"
20+
register: am_configure_pipelinelocalfs_pipeline_uuid
21+
22+
# Define archivematica_src_configure_pipeline_remote_name
23+
- set_fact:
24+
am_configure_pipelinelocalfs_pipeline_remote_name: "{{ ansible_host }}"
25+
26+
# Redefine archivematica_src_configure_pipeline_remote_name when archivematica_src_configure_am_site_url is defined
27+
- set_fact:
28+
am_configure_pipelinelocalfs_pipeline_remote_name: "{{ archivematica_src_configure_am_site_url|urlsplit('hostname') }}"
29+
when: "archivematica_src_configure_am_site_url is defined"
30+
31+
# Check when Pipeline Local Filesystem already exists
32+
- name: "Get Pipeline Local Filesystem Space ID when it is already configured"
33+
become: "yes"
34+
shell: >
35+
echo "select space_id from locations_pipelinelocalfs where remote_name='{{ am_configure_pipelinelocalfs_pipeline_remote_name }}';"
36+
| {{ archivematica_src_ss_virtualenv }}/bin/python manage.py dbshell
37+
| tail -n1
38+
args:
39+
chdir: "{{ archivematica_src_ss_app }}"
40+
executable: /bin/bash
41+
environment: "{{ archivematica_src_ss_environment }}"
42+
delegate_to: "{{ archivematica_src_configure_ss_inventory_hostname | default(archivematica_src_configure_ss_url|urlsplit('hostname')) }}"
43+
remote_user: "{{ archivematica_src_configure_ss_ssh_user | default('artefactual') }}"
44+
register: am_configure_pipelinelocalfs_space_id
45+
46+
# Create Pipeline Local Filesystem Space when it doesn't exist
47+
48+
- name: "Create Pipeline Local Filesystem Space"
49+
uri:
50+
url: "{{ archivematica_src_configure_ss_url }}/api/v2/space/"
51+
headers:
52+
Content-Type: "application/json"
53+
Authorization: "ApiKey {{ archivematica_src_configure_ss_user }}:{{ archivematica_src_configure_ss_api_key }}"
54+
body:
55+
access_protocol: "PIPE_FS"
56+
path: "/"
57+
staging_path: "/var/archivematica/storage_service"
58+
remote_user: "archivematica"
59+
remote_name: "{{ am_configure_pipelinelocalfs_pipeline_remote_name }}"
60+
rsync_password: ""
61+
assume_rsync_daemon: False
62+
body_format: json
63+
status_code: 201
64+
method: POST
65+
when:
66+
- am_configure_pipelinelocalfs_space_id.stdout is defined
67+
- am_configure_pipelinelocalfs_space_id.stdout == ""
68+
69+
- name: "Get Pipeline Local Filesystem Space ID when it is created"
70+
shell: >
71+
echo "select space_id from locations_pipelinelocalfs where remote_name='{{ am_configure_pipelinelocalfs_pipeline_remote_name }}';"
72+
| {{ archivematica_src_ss_virtualenv }}/bin/python manage.py dbshell
73+
| tail -n1
74+
args:
75+
chdir: "{{ archivematica_src_ss_app }}"
76+
executable: /bin/bash
77+
environment: "{{ archivematica_src_ss_environment }}"
78+
delegate_to: "{{ archivematica_src_configure_ss_inventory_hostname | default(archivematica_src_configure_ss_url|urlsplit('hostname')) }}"
79+
remote_user: "{{ archivematica_src_configure_ss_ssh_user | default('artefactual') }}"
80+
register: am_configure_pipelinelocalfs_space_id_new
81+
when:
82+
- am_configure_pipelinelocalfs_space_id.stdout is defined
83+
- am_configure_pipelinelocalfs_space_id.stdout == ""
84+
85+
86+
# Reassign pipelinelocalfs_space_id when the Space is created
87+
- set_fact: am_configure_pipelinelocalfs_space_id={{ am_configure_pipelinelocalfs_space_id_new }}
88+
when: am_configure_pipelinelocalfs_space_id_new.stdout is defined
89+
90+
91+
- name: "Create Pipeline Local Filesystem locations"
92+
include_tasks: configure-create-location.yml
93+
vars:
94+
am_configure_space_uuid: "{{ am_configure_pipelinelocalfs_space_id.stdout }}"
95+
am_configure_pipeline_uuid: "{{ am_configure_pipelinelocalfs_pipeline_uuid.stdout }}"
96+
with_items: "{{ archivematica_src_configure_pipeline_localfs_locations }}"
97+
loop_control:
98+
loop_var: am_configure_location
99+
100+
# Transfer Backlog and Currently Processing locations must be exactly one per pipeline.
101+
# So it is necessary to remove the pipeline from default locations when these locations_locationpipelinenew locations are added
102+
- name: "Delete pipeline from default LocalFilesystem Currently Procession or Transfer Backlog location when added to Pipeline LocalFS"
103+
shell: >
104+
echo "delete from locations_locationpipeline where pipeline_id='{{ am_configure_pipelinelocalfs_pipeline_uuid.stdout }}' and location_id in
105+
(select uuid from locations_location where purpose='{{ item }}' and space_id in
106+
(select space_id from locations_localfilesystem where id='1'));"
107+
| {{ archivematica_src_ss_virtualenv }}/bin/python manage.py dbshell
108+
| tail -n1
109+
args:
110+
chdir: "{{ archivematica_src_ss_app }}"
111+
executable: /bin/bash
112+
environment: "{{ archivematica_src_ss_environment }}"
113+
delegate_to: "{{ archivematica_src_configure_ss_inventory_hostname | default(archivematica_src_configure_ss_url|urlsplit('hostname')) }}"
114+
remote_user: "{{ archivematica_src_configure_ss_ssh_user | default('artefactual') }}"
115+
when: "item in (archivematica_src_configure_pipeline_localfs_locations | map(attribute='location_purpose') | list)"
116+
loop:
117+
- "BL"
118+
- "CP"
119+
120+
- name: "Create ssh key in SS"
121+
user:
122+
name: "archivematica"
123+
generate_ssh_key: "yes"
124+
ssh_key_file: ".ssh/id_rsa"
125+
delegate_to: "{{ archivematica_src_configure_ss_inventory_hostname | default(archivematica_src_configure_ss_url|urlsplit('hostname')) }}"
126+
remote_user: "{{ archivematica_src_configure_ss_ssh_user | default('artefactual') }}"
127+
register: am_configure_pipelinelocalfs_ss_ssh_key
128+
129+
- name: "Use StrictHostKeyChecking=no ssh option for archivematica user"
130+
lineinfile:
131+
create: "yes"
132+
path: "/var/lib/archivematica/.ssh/config"
133+
owner: "archivematica"
134+
group: "archivematica"
135+
mode: "0600"
136+
line: "StrictHostKeyChecking no"
137+
delegate_to: "{{ archivematica_src_configure_ss_inventory_hostname | default(archivematica_src_configure_ss_url|urlsplit('hostname')) }}"
138+
remote_user: "{{ archivematica_src_configure_ss_ssh_user | default('artefactual') }}"
139+
140+
- name: "Show ssh key"
141+
debug: msg={{ am_configure_pipelinelocalfs_ss_ssh_key.ssh_public_key }}
142+
143+
- name: "Add SS ssh key to pipeline server"
144+
authorized_key:
145+
user: "archivematica"
146+
state: "present"
147+
key: "{{ am_configure_pipelinelocalfs_ss_ssh_key.ssh_public_key }}"

‎tasks/main.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,3 +286,14 @@
286286
when:
287287
- "archivematica_src_install_ss|bool or archivematica_src_install_ss=='rpm'"
288288
- "archivematica_src_configure_gpg is defined"
289+
290+
#
291+
# Configure Pipeline Local Filesystem Space
292+
#
293+
294+
- include: "configure-pipeline-local-fs.yml"
295+
tags:
296+
- "amsrc-configure"
297+
when:
298+
- "archivematica_src_configure_dashboard|bool"
299+
- "archivematica_src_configure_pipeline_localfs_locations is defined"

0 commit comments

Comments
 (0)