Skip to content

Commit 4b58886

Browse files
stdavissteveoh
stdavis
authored and
steveoh
committed
feat: crude restores (truncate and load as well as recreate)
chore: rename restore function project
1 parent 35eb28a commit 4b58886

File tree

9 files changed

+291
-6
lines changed

9 files changed

+291
-6
lines changed

‎.vscode/settings.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"cSpell.words": ["orgid"],
2+
"cSpell.words": ["fromitem", "instafail", "orgid"],
33
"editor.formatOnSave": true,
44
"editor.rulers": [120],
55
"coverage-gutters.showGutterCoverage": false,

‎packages/backup/src/backup/main.py

+10-5
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,11 @@ def backup():
5858
summary = {}
5959
supported_types = [
6060
arcgis.gis.ItemTypeEnum.FEATURE_SERVICE.value,
61-
arcgis.gis.ItemTypeEnum.WEB_EXPERIENCE.value,
62-
arcgis.gis.ItemTypeEnum.WEB_MAP.value,
63-
arcgis.gis.ItemTypeEnum.WEB_SCENE.value,
64-
arcgis.gis.ItemTypeEnum.WEB_MAPPING_APPLICATION.value,
61+
# restoring some of these types below just broken them. Perhaps these can be implemented in the future...
62+
# arcgis.gis.ItemTypeEnum.WEB_EXPERIENCE.value,
63+
# arcgis.gis.ItemTypeEnum.WEB_MAP.value,
64+
# arcgis.gis.ItemTypeEnum.WEB_SCENE.value,
65+
# arcgis.gis.ItemTypeEnum.WEB_MAPPING_APPLICATION.value,
6566
]
6667

6768
while has_more:
@@ -73,7 +74,11 @@ def backup():
7374
)
7475

7576
#: couldn't query or filter multiple types at once, so filtering here
76-
for item in [filteredItem for filteredItem in response["results"] if filteredItem.type in supported_types]:
77+
for item in response["results"]:
78+
if item.type not in supported_types:
79+
print(f"Unsupported item type, skipping: {item.title} ({item.type}, {item.id})")
80+
continue
81+
7782
print(f"Preparing {item.title} ({item.type}, {item.id})")
7883
item_json = dict(item)
7984

‎packages/restore/.gitignore ‎packages/restore-function/.gitignore

File renamed without changes.

‎packages/restore-function/Dockerfile

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
FROM python:3.11-slim
2+
3+
# Allow statements and log messages to immediately appear in the Knative logs
4+
ENV PYTHONUNBUFFERED=True
5+
6+
USER root
7+
RUN useradd -s /bin/bash dummy
8+
9+
# Set the locale
10+
RUN apt-get update && apt-get install -y locales && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 && apt-get install -y gcc && apt-get install -y libkrb5-dev && pip install requests-kerberos
11+
12+
COPY . /app
13+
WORKDIR /app
14+
RUN pip install .
15+
16+
USER dummy
17+
ENTRYPOINT ["restore"]

‎packages/restore-function/pyproject.toml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[tool.ruff]
2+
line-length = 120
3+
ignore = ["E501"]
4+
[tool.black]
5+
line-length = 120
6+
[tool.pytest.ini_options]
7+
minversion = "6.0"
8+
testpaths = [ "tests", "src" ]
9+
norecursedirs = [".env", "data", "maps", ".github", ".vscode"]
10+
console_output_style = "count"
11+
addopts = "--cov-branch --cov=project-moonwalk --cov-report term --cov-report xml:cov.xml --instafail"

‎packages/restore-function/setup.py

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/usr/bin/env python
2+
# -*- encoding: utf-8 -*-
3+
"""
4+
setup.py
5+
A module that installs the restore as a module
6+
"""
7+
8+
from glob import glob
9+
from os.path import basename, splitext
10+
11+
from setuptools import find_packages, setup
12+
13+
setup(
14+
name="moonwalk-restore",
15+
version="1.0.0",
16+
license="MIT",
17+
description="Restore process for project moonwalk",
18+
author="UGRC Developers",
19+
author_email="[email protected]",
20+
url="https://github.com/agrc/project-moonwalk",
21+
packages=find_packages("src"),
22+
package_dir={"": "src"},
23+
py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")],
24+
include_package_data=True,
25+
zip_safe=True,
26+
classifiers=[
27+
# complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers
28+
"Development Status :: 5 - Production/Stable",
29+
"Intended Audience :: Developers",
30+
"Topic :: Utilities",
31+
],
32+
project_urls={
33+
"Issue Tracker": "https://github.com/agrc/project-moonwalk/issues",
34+
},
35+
keywords=["gis"],
36+
install_requires=["arcgis==2.*", "google-cloud-storage==2.*"],
37+
extras_require={
38+
"tests": [
39+
"pytest-cov==5.*",
40+
"pytest-instafail==0.*",
41+
"pytest-mock==3.*",
42+
"pytest-ruff==0.*",
43+
"pytest-watch==4.*",
44+
"pytest==8.*",
45+
"ruff==0.*",
46+
]
47+
},
48+
setup_requires=[
49+
"pytest-runner",
50+
],
51+
entry_points={
52+
"console_scripts": [
53+
"restore = restore.main:restore",
54+
"restore_local = restore.main:local_restore",
55+
]
56+
},
57+
)

‎packages/restore-function/src/restore/__init__.py

Whitespace-only changes.

‎packages/restore-function/src/restore/main.py

+190
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
import json
2+
from pathlib import Path
3+
4+
import arcgis
5+
6+
UPLOAD_ITEM_TITLE = "moonwalk-restore"
7+
8+
9+
def _get_secrets():
10+
"""A helper method for loading secrets from either a GCF mount point or a local secrets folder.
11+
json file
12+
13+
Raises:
14+
FileNotFoundError: If the secrets file can't be found.
15+
16+
Returns:
17+
dict: The secrets .json loaded as a dictionary
18+
"""
19+
20+
secret_folder = Path("/secrets")
21+
22+
#: Try to get the secrets from the Cloud Function mount point
23+
if secret_folder.exists():
24+
return json.loads(Path("/secrets/app/secrets.json").read_text(encoding="utf-8"))
25+
26+
#: Otherwise, try to load a local copy for local development
27+
secret_folder = Path(__file__).parent / "secrets"
28+
if secret_folder.exists():
29+
return json.loads((secret_folder / "secrets.json").read_text(encoding="utf-8"))
30+
31+
raise FileNotFoundError("Secrets folder not found; secrets not loaded.")
32+
33+
34+
def cleanup_restores(gis):
35+
print("Cleaning up any old restores...")
36+
items = gis.content.search(query=f"title:{UPLOAD_ITEM_TITLE}")
37+
for item in items:
38+
item.delete(permanent=True)
39+
40+
41+
def truncate_and_append(item_id, sub_folder, gis, item):
42+
fgdb_item = upload_fgdb(item_id, sub_folder, gis)
43+
44+
collection = arcgis.features.FeatureLayerCollection.fromitem(item)
45+
46+
for layer in collection.layers:
47+
print(f"truncating layer: {layer.properties.name}")
48+
layer.manager.truncate(asynchronous=True, wait=True)
49+
print("appending")
50+
layer.append(
51+
item_id=fgdb_item.id,
52+
upload_format="filegdb",
53+
source_table_name=layer.properties.name,
54+
return_messages=True,
55+
rollback=True,
56+
)
57+
58+
for table in collection.tables:
59+
print(f"truncating table: {table.properties.name}")
60+
table.manager.truncate(asynchronous=True, wait=True)
61+
print("appending")
62+
table.append(
63+
item_id=fgdb_item.id,
64+
upload_format="filegdb",
65+
source_table_name=table.properties.name,
66+
return_messages=True,
67+
rollback=True,
68+
)
69+
70+
fgdb_item.delete(permanent=True)
71+
72+
73+
def upload_fgdb(item_id, sub_folder, gis):
74+
print("uploading fgdb")
75+
zip_path = Path(f"./temp/sample-bucket/{item_id}/{sub_folder}/data.zip")
76+
fgdb_item = gis.content.add(
77+
item_properties={
78+
"type": "File Geodatabase",
79+
"title": UPLOAD_ITEM_TITLE,
80+
"snippet": "temporary upload from moonwalk",
81+
},
82+
data=str(zip_path),
83+
)
84+
85+
return fgdb_item
86+
87+
88+
def recreate_item(item_id, sub_folder, gis):
89+
print("Item not found; creating new item...")
90+
91+
fgdb_item = upload_fgdb(item_id, sub_folder, gis)
92+
93+
original_item_properties = json.loads(
94+
Path(f"./temp/sample-bucket/{item_id}/{sub_folder}/item.json").read_text(encoding="utf-8")
95+
)
96+
97+
print("publishing")
98+
#: todo: should we worry about restoring layer ids?
99+
published_item = fgdb_item.publish(
100+
publish_parameters={"name": original_item_properties.get("name")},
101+
)
102+
success = published_item.reassign_to(
103+
target_owner=original_item_properties.get("owner"),
104+
target_folder=original_item_properties.get("ownerFolder"),
105+
)
106+
if not success:
107+
raise Exception("Failed to reassign item")
108+
109+
#: sharing
110+
published_item.sharing.sharing_level = original_item_properties.get("access")
111+
#: todo: group sharing...
112+
113+
print("updating item")
114+
supported_property_names = [
115+
"description",
116+
"title",
117+
"tags",
118+
"snippet",
119+
"extent",
120+
"accessInformation",
121+
"licenseInfo",
122+
"culture",
123+
"access",
124+
]
125+
supported_properties = {k: v for k, v in original_item_properties.items() if k in supported_property_names}
126+
success = published_item.update(
127+
item_properties=supported_properties,
128+
)
129+
#: metadata?
130+
131+
print("deleting fgdb")
132+
#: get a new item reference since the owner has changed
133+
gis.content.get(fgdb_item.id).delete(permanent=True)
134+
135+
if not success:
136+
raise Exception("Failed to update item")
137+
138+
print(f"new item created: {published_item.id}")
139+
140+
141+
def restore(item_id, sub_folder):
142+
secrets = _get_secrets()
143+
gis = arcgis.GIS(
144+
url=secrets["AGOL_ORG"],
145+
username=secrets["AGOL_USER"],
146+
password=secrets["AGOL_PASSWORD"],
147+
)
148+
149+
cleanup_restores(gis)
150+
151+
print(f"Restoring {item_id} from {sub_folder}...")
152+
153+
item_exists = True
154+
try:
155+
item = arcgis.gis.Item(gis, item_id)
156+
except Exception:
157+
item_exists = False
158+
159+
if item_exists:
160+
if item.type == arcgis.gis.ItemTypeEnum.FEATURE_SERVICE.value:
161+
truncate_and_append(item_id, sub_folder, gis, item)
162+
else:
163+
raise NotImplementedError(f"Unsupported item type: {item.type}")
164+
#: this breaks web maps and experience builder projects, not sure why
165+
# print(f"overwriting item: {item_id} from {sub_folder}")
166+
# success = item.update(
167+
# item_properties=json.loads(
168+
# Path(f"./temp/sample-bucket/{item_id}/{sub_folder}/item.json").read_text(encoding="utf-8")
169+
# ),
170+
# data=str(Path(f"./temp/sample-bucket/{item_id}/{sub_folder}/data.json")),
171+
# )
172+
# if not success:
173+
# print("Failed to update item")
174+
# return
175+
else:
176+
recreate_item(item_id, sub_folder, gis)
177+
178+
print("Restore complete!")
179+
180+
181+
def local_restore():
182+
#: truncate and load
183+
restore("3ac0f9833f7d4335acebd62fe0695635", "short")
184+
185+
#: recreate feature service
186+
# restore("33e9c822af3b4d08844d58169410f9fa", "short")
187+
188+
189+
if __name__ == "__main__":
190+
local_restore()

‎packages/restore-function/src/restore/secrets/secrets.template.json

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"AGOL_ORG": "https://utah.maps.arcgis.com",
3+
"AGOL_PASSWORD": "",
4+
"AGOL_USER": ""
5+
}

0 commit comments

Comments
 (0)