Skip to content

Commit fc8aab6

Browse files
committed
s3_management: Fix pyright / mypy errors
I was observing these errors in my editor and they were annoying me so I fixed them. I don't expect any functional changes to come out of this. Signed-off-by: Eli Uriegas <eliuriegas@meta.com>
1 parent 3be1e23 commit fc8aab6

1 file changed

Lines changed: 20 additions & 17 deletions

File tree

s3_management/manage.py

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,13 @@
1414
from typing import Dict, Iterable, List, Optional, Set, Type, TypeVar
1515

1616
import boto3
17-
import botocore
18-
from packaging.version import InvalidVersion, parse as _parse_version, Version
17+
import botocore # type: ignore[import]
18+
from packaging.version import (
19+
InvalidVersion,
20+
LegacyVersion,
21+
parse as _parse_version,
22+
Version,
23+
)
1924

2025

2126
S3 = boto3.resource("s3")
@@ -238,15 +243,15 @@ def __lt__(self, other):
238243
return self.key < other.key
239244

240245

241-
def safe_parse_version(ver_str: str) -> Version:
246+
def safe_parse_version(ver_str: str) -> LegacyVersion | Version:
242247
try:
243248
return _parse_version(ver_str)
244249
except InvalidVersion:
245250
return Version("0.0.0")
246251

247252

248253
class S3Index:
249-
def __init__(self: S3IndexType, objects: List[S3Object], prefix: str) -> None:
254+
def __init__(self, objects: List[S3Object], prefix: str) -> None:
250255
self.objects = objects
251256
self.prefix = prefix.rstrip("/")
252257
self.html_name = "index.html"
@@ -256,7 +261,7 @@ def __init__(self: S3IndexType, objects: List[S3Object], prefix: str) -> None:
256261
path.dirname(obj.key) for obj in objects if path.dirname != prefix
257262
}
258263

259-
def nightly_packages_to_show(self: S3IndexType) -> List[S3Object]:
264+
def nightly_packages_to_show(self) -> List[S3Object]:
260265
"""Finding packages to show based on a threshold we specify
261266
262267
Basically takes our S3 packages, normalizes the version for easier
@@ -326,7 +331,7 @@ def get_package_names(self, subdir: Optional[str] = None) -> List[str]:
326331
{self.obj_to_package_name(obj) for obj in self.gen_file_list(subdir)}
327332
)
328333

329-
def normalize_package_version(self: S3IndexType, obj: S3Object) -> str:
334+
def normalize_package_version(self, obj: S3Object) -> str:
330335
# removes the GPU specifier from the package name as well as
331336
# unnecessary things like the file extension, architecture name, etc.
332337
return sub(r"%2B.*", "", "-".join(path.basename(obj.key).split("-")[:2]))
@@ -498,7 +503,7 @@ def compute_sha256(self) -> None:
498503
)
499504

500505
@classmethod
501-
def has_public_read(cls: Type[S3IndexType], key: str) -> bool:
506+
def has_public_read(cls, key: str) -> bool:
502507
def is_all_users_group(o) -> bool:
503508
return (
504509
o.get("Grantee", {}).get("URI")
@@ -512,11 +517,11 @@ def can_read(o) -> bool:
512517
return any(is_all_users_group(x) and can_read(x) for x in acl_grants)
513518

514519
@classmethod
515-
def grant_public_read(cls: Type[S3IndexType], key: str) -> None:
520+
def grant_public_read(cls, key: str) -> None:
516521
CLIENT.put_object_acl(Bucket=BUCKET.name, Key=key, ACL="public-read")
517522

518523
@classmethod
519-
def fetch_object_names(cls: Type[S3IndexType], prefix: str) -> List[str]:
524+
def fetch_object_names(cls, prefix: str) -> List[str]:
520525
obj_names = []
521526
for obj in BUCKET.objects.filter(Prefix=prefix):
522527
is_acceptable = any(
@@ -531,7 +536,7 @@ def fetch_object_names(cls: Type[S3IndexType], prefix: str) -> List[str]:
531536
obj_names.append(obj.key)
532537
return obj_names
533538

534-
def fetch_metadata(self: S3IndexType) -> None:
539+
def fetch_metadata(self) -> None:
535540
# Add PEP 503-compatible hashes to URLs to allow clients to avoid spurious downloads, if possible.
536541
regex_multipart_upload = r"^[A-Za-z0-9+/=]+=-[0-9]+$"
537542
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:
@@ -565,17 +570,17 @@ def fetch_metadata(self: S3IndexType) -> None:
565570
if size := response.get("ContentLength"):
566571
self.objects[idx].size = int(size)
567572

568-
def fetch_pep658(self: S3IndexType) -> None:
573+
def fetch_pep658(self) -> None:
569574
def _fetch_metadata(key: str) -> str:
570575
try:
571576
response = CLIENT.head_object(
572577
Bucket=BUCKET.name, Key=f"{key}.metadata", ChecksumMode="Enabled"
573578
)
574579
sha256 = base64.b64decode(response.get("ChecksumSHA256")).hex()
575580
return sha256
576-
except botocore.exceptions.ClientError as e:
581+
except botocore.exceptions.ClientError as e: # type: ignore
577582
if e.response["Error"]["Code"] == "404":
578-
return None
583+
return ""
579584
raise
580585

581586
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor:
@@ -592,9 +597,7 @@ def _fetch_metadata(key: str) -> str:
592597
self.objects[idx].pep658 = response
593598

594599
@classmethod
595-
def from_S3(
596-
cls: Type[S3IndexType], prefix: str, with_metadata: bool = True
597-
) -> S3IndexType:
600+
def from_S3(cls, prefix: str, with_metadata: bool = True) -> "S3Index":
598601
prefix = prefix.rstrip("/")
599602
obj_names = cls.fetch_object_names(prefix)
600603

@@ -622,7 +625,7 @@ def sanitize_key(key: str) -> str:
622625
return rc
623626

624627
@classmethod
625-
def undelete_prefix(cls: Type[S3IndexType], prefix: str) -> None:
628+
def undelete_prefix(cls, prefix: str) -> None:
626629
paginator = CLIENT.get_paginator("list_object_versions")
627630
for page in paginator.paginate(Bucket=BUCKET.name, Prefix=prefix):
628631
for obj in page.get("DeleteMarkers", []):

0 commit comments

Comments
 (0)