|
| 1 | +""" |
| 2 | +Push changed posts to archive.org |
| 3 | +To use, modify: |
| 4 | +1. BASEURL |
| 5 | +2. Internet Archive API Key, get it at https://archive.org/account/s3.php |
| 6 | + and set them as SAVEPAGENOW_ACCESS_KEY and SAVEPAGENOW_SECRET_KEY environment variables. |
| 7 | + It's recommended to set them as repository secrets. |
| 8 | +""" |
| 9 | + |
| 10 | +import json |
| 11 | +import os |
| 12 | +import traceback |
| 13 | +from typing import Final, Optional, TypedDict |
| 14 | +import yaml |
| 15 | +import requests |
| 16 | + |
| 17 | +ROOTURL: Final[str] = "https://young-lord.github.io/" |
| 18 | +POSTS_BASEURL: Final[str] = ROOTURL + "posts/" |
| 19 | + |
| 20 | +# https://archive.org/details/spn-2-public-api-page-docs-2023-01-22 |
| 21 | +# https://github.com/palewire/savepagenow/blob/main/savepagenow/api.py MIT License |
| 22 | +DEFAULT_USER_AGENT: Final[str] = ( |
| 23 | + "savepagenow (https://github.com/Young-Lord/Young-Lord.github.io/blob/master/.github/workflows/save_page.py)" |
| 24 | +) |
| 25 | + |
| 26 | + |
| 27 | +class WaybackRuntimeError(Exception): |
| 28 | + """A generic error returned by the Wayback Machine.""" |
| 29 | + |
| 30 | + pass |
| 31 | + |
| 32 | + |
| 33 | +class BlockedByRobots(WaybackRuntimeError): |
| 34 | + """Raised when archive.org has been blocked by the site's robots.txt.""" |
| 35 | + |
| 36 | + pass |
| 37 | + |
| 38 | + |
| 39 | +class BadGateway(WaybackRuntimeError): |
| 40 | + """Raised when you receive a 502 bad gateway status code.""" |
| 41 | + |
| 42 | + pass |
| 43 | + |
| 44 | + |
| 45 | +class Unauthorized(WaybackRuntimeError): |
| 46 | + """Raised when you receive a 401 unauthorized status code.""" |
| 47 | + |
| 48 | + pass |
| 49 | + |
| 50 | + |
| 51 | +class Forbidden(WaybackRuntimeError): |
| 52 | + """Raised when you receive a 403 forbidden status code.""" |
| 53 | + |
| 54 | + pass |
| 55 | + |
| 56 | + |
| 57 | +class TooManyRequests(WaybackRuntimeError): |
| 58 | + """Raised when you have exceeded the throttle on request frequency.""" |
| 59 | + |
| 60 | + pass |
| 61 | + |
| 62 | + |
| 63 | +class UnknownError(WaybackRuntimeError): |
| 64 | + """Raised when you receive a 520 unknown status code.""" |
| 65 | + |
| 66 | + pass |
| 67 | + |
| 68 | + |
| 69 | +class WebArchiveReturn(TypedDict): |
| 70 | + url: str |
| 71 | + job_id: str |
| 72 | + |
| 73 | + |
| 74 | +class WebArchiveReturnWithMessage(WebArchiveReturn): |
| 75 | + message: str |
| 76 | + |
| 77 | + |
| 78 | +WebArchiveReturnWithMaybeMessage = WebArchiveReturnWithMessage | WebArchiveReturn |
| 79 | + |
| 80 | + |
| 81 | +def capture( |
| 82 | + target_url: str, |
| 83 | + authenticate: bool = True, |
| 84 | + headers: Optional[dict] = None, |
| 85 | + data: Optional[dict] = None, |
| 86 | +) -> WebArchiveReturnWithMaybeMessage: |
| 87 | + # Put together the URL that will save our request |
| 88 | + domain = "https://web.archive.org" |
| 89 | + request_url = domain + "/save" |
| 90 | + |
| 91 | + if headers is None: |
| 92 | + headers = {} |
| 93 | + user_headers = headers |
| 94 | + if data is None: |
| 95 | + data = {} |
| 96 | + user_data = data |
| 97 | + |
| 98 | + headers = { |
| 99 | + "User-Agent": DEFAULT_USER_AGENT, |
| 100 | + "Accept": "application/json", |
| 101 | + "Content-Type": "application/x-www-form-urlencoded", |
| 102 | + } |
| 103 | + |
| 104 | + data = { |
| 105 | + "url": target_url, |
| 106 | + } |
| 107 | + data.update(user_data) |
| 108 | + |
| 109 | + # Access Keys for Internet Archive API |
| 110 | + # Get it at https://archive.org/account/s3.php |
| 111 | + if authenticate: |
| 112 | + access_key = os.getenv("SAVEPAGENOW_ACCESS_KEY") |
| 113 | + secret_key = os.getenv("SAVEPAGENOW_SECRET_KEY") |
| 114 | + try: |
| 115 | + assert access_key and secret_key |
| 116 | + except AssertionError: |
| 117 | + raise ValueError( |
| 118 | + "You must set SAVEPAGENOW_ACCESS_KEY and SAVEPAGENOW_SECRET_KEY environment variables to use the authenticate flag" |
| 119 | + ) |
| 120 | + headers.update( |
| 121 | + { |
| 122 | + "Authorization": f"LOW {access_key}:{secret_key}", |
| 123 | + } |
| 124 | + ) |
| 125 | + |
| 126 | + headers.update(user_headers) |
| 127 | + |
| 128 | + # Make the request |
| 129 | + response = requests.post(request_url, headers=headers, data=data) |
| 130 | + |
| 131 | + # If it has an error header, raise that. |
| 132 | + has_error_header = "X-Archive-Wayback-Runtime-Error" in response.headers |
| 133 | + if has_error_header: |
| 134 | + error_header = response.headers["X-Archive-Wayback-Runtime-Error"] |
| 135 | + if error_header == "RobotAccessControlException: Blocked By Robots": |
| 136 | + raise BlockedByRobots("archive.org returned blocked by robots.txt error") |
| 137 | + else: |
| 138 | + raise WaybackRuntimeError(error_header) |
| 139 | + |
| 140 | + # If it has an error code, raise that |
| 141 | + status_code = response.status_code |
| 142 | + if status_code == 401: |
| 143 | + raise Unauthorized("Your archive.org access key and/or secret is not valid") |
| 144 | + elif status_code == 403: |
| 145 | + raise Forbidden(response.headers) |
| 146 | + elif status_code == 429: |
| 147 | + traceback.print_exc() |
| 148 | + # raise TooManyRequests(response.headers) |
| 149 | + elif status_code == 502: |
| 150 | + raise BadGateway(response.headers) |
| 151 | + elif status_code == 520: |
| 152 | + raise UnknownError(response.headers) |
| 153 | + return response.json() |
| 154 | + |
| 155 | + |
| 156 | +all_changed_files: list[str] = json.loads(os.environ["all_changed_files"]) |
| 157 | + |
| 158 | +if not all_changed_files: |
| 159 | + print("No blog post changed.") |
| 160 | + exit(0) |
| 161 | + |
| 162 | +FILE_SUFFIX: Final[str] = ".md" |
| 163 | +for file in all_changed_files: |
| 164 | + assert file.endswith(FILE_SUFFIX) |
| 165 | + # https://stackoverflow.com/a/34727830 |
| 166 | + url: str = "" |
| 167 | + with open(file, "r", encoding="utf8") as f: |
| 168 | + front_matter = next(yaml.load_all(f, Loader=yaml.FullLoader)) |
| 169 | + title: str = front_matter["title"] |
| 170 | + if file.startswith("_posts/"): |
| 171 | + slug: str = front_matter["slug"] |
| 172 | + url = POSTS_BASEURL + slug |
| 173 | + else: |
| 174 | + assert "/" not in file # it must be in root dir |
| 175 | + url = ROOTURL + file.removesuffix(FILE_SUFFIX) |
| 176 | + |
| 177 | + ret = capture( |
| 178 | + url, |
| 179 | + data={ |
| 180 | + "capture_outlinks": 1, |
| 181 | + "skip_first_archive": 1, |
| 182 | + "capture_screenshot": 1, |
| 183 | + "delay_wb_availability": 1, |
| 184 | + }, |
| 185 | + ) |
| 186 | + print(f'"{title}" ({url}):') |
| 187 | + if "message" in ret: |
| 188 | + print("\t" + ret["message"]) |
| 189 | + else: |
| 190 | + print("\tno message (that means success)") |
0 commit comments