Add ruff flake8-use-pathlib rules

docs/conf.py

@@ -10,14 +10,14 @@
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
+# documentation root, use Path.absolute() to make it absolute, like shown here.
 #
-import os
 import sys
+from pathlib import Path
 
 # import parsons
-# sys.path.insert(0, os.path.abspath('.'))
-sys.path.insert(0, os.path.abspath("../"))
+# sys.path.insert(0, Path(".").absolute())
+sys.path.insert(0, Path("../").absolute())
 
 # -- Project information -----------------------------------------------------
 

parsons/action_kit/action_kit.py

@@ -2,6 +2,7 @@
 import logging
 import math
 import time
+from pathlib import Path
 
 import requests
 
@@ -1359,7 +1360,7 @@ def bulk_upload_csv(
         # self.conn defaults to JSON, but this has to be form/multi-part....
         upload_client = self._conn({"accepts": "application/json"})
         if isinstance(csv_file, str):
-            csv_file = open(csv_file, "rb")
+            csv_file = Path(csv_file).open(mode="rb")
 
         url = self._base_endpoint("upload")
         files = {"upload": csv_file}
@@ -1376,7 +1377,8 @@ def bulk_upload_csv(
                 "id": progress_url.split("/")[-2] if progress_url else None,
                 "progress_url": progress_url,
             }
-            return rv
+
+        return rv
 
     def bulk_upload_table(
         self,

parsons/azure/azure_blob_storage.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from pathlib import Path
 from urllib.parse import urlparse
 
 from azure.core.exceptions import ResourceNotFoundError
@@ -310,11 +311,8 @@ def put_blob(self, container_name, blob_name, local_path, **kwargs):
         # Move all content_settings keys into a ContentSettings object
         content_settings, kwargs_dict = self._get_content_settings_from_dict(kwargs)
 
-        with open(local_path, "rb") as f:
-            data = f.read()
-
-        blob_client = blob_client.upload_blob(
-            data,
+        blob_client.upload_blob(
+            Path(local_path).read_bytes(),
             overwrite=True,
             content_settings=content_settings,
             **kwargs_dict,
@@ -348,7 +346,7 @@ def download_blob(self, container_name, blob_name, local_path=None):
         blob_client = self.get_blob(container_name, blob_name)
 
         logger.info(f"Downloading {blob_name} blob from {container_name} container.")
-        with open(local_path, "wb") as f:
+        with Path(local_path).open(mode="wb") as f:
             blob_client.download_blob().readinto(f)
         logger.info(f"{blob_name} blob saved to {local_path}.")
 

parsons/box/box.py

@@ -19,6 +19,7 @@
 
 import logging
 import tempfile
+from pathlib import Path
 
 import boxsdk
 
@@ -248,7 +249,7 @@ def upload_table_to_folder_id(
         # Create a temp directory in which we will let Parsons create a
         # file. Both will go away automatically when we leave scope.
         with tempfile.TemporaryDirectory() as temp_dir_name:
-            temp_file_path = temp_dir_name + "/table.tmp"
+            temp_file_path = f"{temp_dir_name}/table.tmp"
             if format == "csv":
                 table.to_csv(local_path=temp_file_path)
             elif format == "json":
@@ -286,7 +287,7 @@ def download_file(self, path: str, local_path: str = None) -> str:
 
         file_id = self.get_item_id(path)
 
-        with open(local_path, "wb") as output_file:
+        with Path(local_path).open(mode="wb") as output_file:
             self.client.file(file_id).download_to(output_file)
 
         return local_path
@@ -327,7 +328,7 @@ def get_table_by_file_id(self, file_id, format="csv") -> Table:
         # Temp file will be around as long as enclosing process is running,
         # which we need, because the Table we return will continue to use it.
         output_file_name = create_temp_file()
-        with open(output_file_name, "wb") as output_file:
+        with Path(output_file_name).open(mode="wb") as output_file:
             self.client.file(file_id).download_to(output_file)
 
         if format == "csv":

parsons/catalist/catalist.py

@@ -5,10 +5,10 @@
 
 import base64
 import logging
-import os
 import tempfile
 import time
 import urllib
+from pathlib import Path
 from typing import Dict, List, Optional, Union
 from zipfile import ZipFile
 
@@ -379,15 +379,15 @@ def load_matches(self, id: str) -> Table:
         with ZipFile(temp_file_zip) as zf:
             zf.extractall(path=temp_dir)
 
-        filepath = os.listdir(temp_dir)[0]
+        filepath = next(Path(temp_dir).iterdir())
 
-        result = Table.from_csv(os.path.join(temp_dir, filepath), delimiter="\t")
+        result = Table.from_csv(str(filepath), delimiter="\t")
         return result
 
     def validate_table(self, table: Table, template_id: str = "48827") -> None:
         """Validate table structure and contents."""
         if not template_id == "48827":
-            logger.warn(f"No validator implemented for template {template_id}.")
+            logger.warning(f"No validator implemented for template {template_id}.")
             return
 
         expected_table_columns = [

parsons/databases/mysql/mysql.py

@@ -2,6 +2,7 @@
 import os
 import pickle
 from contextlib import contextmanager
+from pathlib import Path
 
 import mysql.connector as mysql
 import petl
@@ -174,7 +175,7 @@ def query_with_connection(self, sql, connection, parameters=None, commit=True):
                 # all the type information for each field.)
                 temp_file = files.create_temp_file()
 
-                with open(temp_file, "wb") as f:
+                with Path(temp_file).open(mode="wb") as f:
                     # Grab the header
                     pickle.dump(cursor.column_names, f)
 

parsons/databases/postgres/postgres.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from pathlib import Path
 from typing import Optional
 
 from parsons.databases.alchemy import Alchemy
@@ -42,7 +43,7 @@ def __init__(self, username=None, password=None, host=None, db=None, port=5432,
 
         # Check if there is a pgpass file. Psycopg2 will search for this file first when
         # creating a connection.
-        pgpass = os.path.isfile(os.path.expanduser("~/.pgpass"))
+        pgpass = Path("~/.pgpass").expanduser().is_file()
 
         if not any([self.username, self.password, self.host, self.db]) and not pgpass:
             raise ValueError(
@@ -90,8 +91,8 @@ def copy(
 
             sql = f"""COPY "{table_name}" ("{'","'.join(tbl.columns)}") FROM STDIN CSV HEADER;"""
 
-            with self.cursor(connection) as cursor:
-                cursor.copy_expert(sql, open(tbl.to_csv(), "r"))
+            with self.cursor(connection) as cursor, Path(tbl.to_csv()).open() as f:
+                cursor.copy_expert(sql, f)
                 logger.info(f"{tbl.num_rows} rows copied to {table_name}.")
 
     def table(self, table_name):

parsons/databases/postgres/postgres_core.py

@@ -1,6 +1,7 @@
 import logging
 import pickle
 from contextlib import contextmanager
+from pathlib import Path
 from typing import Optional
 
 import petl
@@ -146,7 +147,7 @@ def query_with_connection(self, sql, connection, parameters=None, commit=True):
 
                 temp_file = files.create_temp_file()
 
-                with open(temp_file, "wb") as f:
+                with Path(temp_file).open(mode="wb") as f:
                     # Grab the header
                     header = [i[0] for i in cursor.description]
                     pickle.dump(header, f)

parsons/databases/redshift/redshift.py

@@ -5,6 +5,7 @@
 import pickle
 import random
 from contextlib import contextmanager
+from pathlib import Path
 from typing import List, Optional
 
 import petl
@@ -243,7 +244,7 @@ def query_with_connection(self, sql, connection, parameters=None, commit=True):
 
                 temp_file = files.create_temp_file()
 
-                with open(temp_file, "wb") as f:
+                with Path(temp_file).open(mode="wb") as f:
                     # Grab the header
                     header = [i[0] for i in cursor.description]
                     pickle.dump(header, f)
@@ -949,7 +950,7 @@ def generate_manifest(
         if manifest_key and manifest_bucket:
             # Dump the manifest to a temp JSON file
             manifest_path = files.create_temp_file()
-            with open(manifest_path, "w") as manifest_file_obj:
+            with Path(manifest_path).open(mode="w") as manifest_file_obj:
                 json.dump(manifest, manifest_file_obj, sort_keys=True, indent=4)
 
             # Upload the file to S3

parsons/databases/sqlite/sqlite.py

@@ -111,7 +111,7 @@ def query_with_connection(
             if return_values and cursor.description:
                 temp_file = files.create_temp_file()
 
-                with open(temp_file, "wb") as f:
+                with Path(temp_file).open(mode="wb") as f:
                     # Grab the header
                     header = [i[0] for i in cursor.description]
                     pickle.dump(header, f)

parsons/etl/table.py

@@ -1,6 +1,7 @@
 import logging
 import pickle
 from enum import Enum
+from pathlib import Path
 from typing import Union
 
 import petl
@@ -244,7 +245,7 @@ def materialize_to_file(self, file_path=None):
 
         file_path = file_path or files.create_temp_file()
 
-        with open(file_path, "wb") as handle:
+        with Path(file_path).open(mode="wb") as handle:
             for row in self.table:
                 pickle.dump(list(row), handle)
 

parsons/etl/tofrom.py

@@ -1,6 +1,7 @@
 import gzip
 import io
 import json
+from pathlib import Path
 from typing import Optional
 
 import petl
@@ -901,7 +902,7 @@ def from_s3_csv(
         s3 = S3(aws_access_key_id, aws_secret_access_key)
 
         if from_manifest:
-            with open(s3.get_file(bucket, key)) as fd:
+            with Path(s3.get_file(bucket, key)).open() as fd:
                 manifest = json.load(fd)
 
             s3_keys = [x["url"] for x in manifest["entries"]]

parsons/github/github.py

@@ -1,5 +1,6 @@
 import logging
 from functools import partial, wraps
+from pathlib import Path
 
 import petl
 import requests
@@ -423,8 +424,7 @@ def download_file(self, repo_name, path, branch=None, local_path=None):
                 f"Error downloading {path} from repo {repo_name}: {res.content}"
             )
 
-        with open(local_path, "wb") as f:
-            f.write(res.content)
+        Path(local_path).write_bytes(res.content)
 
         logger.info(f"Downloaded {path} to {local_path}")
 

parsons/google/google_bigquery.py

@@ -5,6 +5,7 @@
 import random
 import uuid
 from contextlib import contextmanager
+from pathlib import Path
 from typing import List, Optional, Union
 
 import google
@@ -850,7 +851,7 @@ def copy_direct(
         )
 
         tmpfile_path = tbl.to_csv()
-        with open(tmpfile_path, mode="rb") as tmpfile:
+        with Path(tmpfile_path).open(mode="rb") as tmpfile:
             load_job = self.client.load_table_from_file(
                 tmpfile,
                 destination=self.get_table_ref(table_name=table_name),
@@ -1533,7 +1534,7 @@ def _fetch_query_results(self, cursor) -> Table:
         # the proper data types (e.g. integer).
         temp_filename = create_temp_file()
 
-        with open(temp_filename, "wb") as temp_file:
+        with Path(temp_filename).open(mode="wb") as temp_file:
             header = [i[0] for i in cursor.description]
             pickle.dump(header, temp_file)
 

parsons/google/google_cloud_storage.py

@@ -4,6 +4,7 @@
 import time
 import uuid
 import zipfile
+from pathlib import Path
 from typing import Optional, Union
 
 import google
@@ -266,7 +267,7 @@ def put_blob(self, bucket_name, blob_name, local_path, **kwargs):
         bucket = self.get_bucket(bucket_name)
         blob = storage.Blob(blob_name, bucket)
 
-        with open(local_path, "rb") as f:
+        with Path(local_path).open(mode="rb") as f:
             blob.upload_from_file(f, **kwargs)
 
         logger.info(f"{blob_name} put in {bucket_name} bucket.")
@@ -296,7 +297,7 @@ def download_blob(self, bucket_name, blob_name, local_path=None):
         blob = storage.Blob(blob_name, bucket)
 
         logger.debug(f"Downloading {blob_name} from {bucket_name} bucket.")
-        with open(local_path, "wb") as f:
+        with Path(local_path).open(mode="wb") as f:
             blob.download_to_file(f, client=self.client)
         logger.debug(f"{blob_name} saved to {local_path}.")
 

parsons/notifications/sendmail.py

@@ -2,7 +2,6 @@
 import io
 import logging
 import mimetypes
-import os
 from abc import ABC, abstractmethod
 from email.encoders import encode_base64
 from email.mime.application import MIMEApplication
@@ -12,6 +11,7 @@
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.utils import parseaddr
+from pathlib import Path
 
 from validate_email import validate_email
 
@@ -148,10 +148,8 @@ def _create_message_attachments(
             elif isinstance(f, io.BytesIO):
                 file_bytes = f.getvalue()
             else:
-                filename = os.path.basename(f)
-                fp = open(f, "rb")
-                file_bytes = fp.read()
-                fp.close()
+                filename = Path(f).name
+                file_bytes = Path(f).read_bytes()
 
             content_type, encoding = mimetypes.guess_type(filename)
             self.log.debug(f"(File: {f}, Content-type: {content_type}, Encoding: {encoding})")

parsons/notifications/slack.py

@@ -1,6 +1,7 @@
 import os
 import time
 import warnings
+from pathlib import Path
 
 import requests
 from slackclient import SlackClient
@@ -220,7 +221,7 @@ def upload_file(
             filetype = filename.split(".")[-1]
 
         mode = "rb" if is_binary else "r"
-        with open(filename, mode) as file_content:
+        with Path(filename).open(mode=mode) as file_content:
             resp = self.client.api_call(
                 "files.upload",
                 channels=channels,