flit_core: refactor path handling to use pathlib

Author: jameshilliard

flit_core/flit_core/buildapi.py

@@ -1,8 +1,5 @@
 """PEP-517 compliant buildsystem API"""
 import logging
-import io
-import os
-import os.path as osp
 from pathlib import Path
 
 from .common import (
@@ -48,36 +45,36 @@ def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None):
     module = Module(ini_info.module, Path.cwd())
     metadata = make_metadata(module, ini_info)
 
-    dist_info = osp.join(metadata_directory,
-                         dist_info_name(metadata.name, metadata.version))
-    os.mkdir(dist_info)
+    dist_info = Path(metadata_directory).joinpath(
+        dist_info_name(metadata.name, metadata.version))
+    dist_info.mkdir()
 
-    with io.open(osp.join(dist_info, 'WHEEL'), 'w', encoding='utf-8') as f:
+    with dist_info.joinpath('WHEEL').open('w', encoding='utf-8') as f:
         _write_wheel_file(f, supports_py2=metadata.supports_py2)
 
-    with io.open(osp.join(dist_info, 'METADATA'), 'w', encoding='utf-8') as f:
+    with dist_info.joinpath('METADATA').open('w', encoding='utf-8') as f:
         metadata.write_metadata_file(f)
 
     if ini_info.entrypoints:
-        with io.open(osp.join(dist_info, 'entry_points.txt'), 'w', encoding='utf-8') as f:
+        with dist_info.joinpath('entry_points.txt').open('w', encoding='utf-8') as f:
             write_entry_points(ini_info.entrypoints, f)
 
-    return osp.basename(dist_info)
+    return Path(dist_info).name
 
 # Metadata for editable are the same as for a wheel
 prepare_metadata_for_build_editable = prepare_metadata_for_build_wheel
 
 def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
     """Builds a wheel, places it in wheel_directory"""
-    info = make_wheel_in(pyproj_toml, Path(wheel_directory))
+    info = make_wheel_in(pyproj_toml, wheel_directory)
     return info.file.name
 
 def build_editable(wheel_directory, config_settings=None, metadata_directory=None):
     """Builds an "editable" wheel, places it in wheel_directory"""
-    info = make_wheel_in(pyproj_toml, Path(wheel_directory), editable=True)
+    info = make_wheel_in(pyproj_toml, wheel_directory, editable=True)
     return info.file.name
 
 def build_sdist(sdist_directory, config_settings=None):
     """Builds an sdist, places it in sdist_directory"""
-    path = SdistBuilder.from_ini_path(pyproj_toml).build(Path(sdist_directory))
+    path = SdistBuilder.from_ini_path(pyproj_toml).build(sdist_directory)
     return path.name

flit_core/flit_core/common.py

@@ -21,11 +21,11 @@ def __init__(self, name, directory=Path()):
         self.name = name
 
         # It must exist either as a .py file or a directory, but not both
-        name_as_path = name.replace('.', os.sep)
-        pkg_dir = directory / name_as_path
-        py_file = directory / (name_as_path+'.py')
-        src_pkg_dir = directory / 'src' / name_as_path
-        src_py_file = directory / 'src' / (name_as_path+'.py')
+        name_as_path = Path(name.replace('.', os.sep))
+        pkg_dir = directory.joinpath(name_as_path)
+        py_file = directory.joinpath(name_as_path).with_suffix('.py')
+        src_pkg_dir = directory.joinpath('src').joinpath(name_as_path)
+        src_py_file = directory.joinpath('src').joinpath(name_as_path).with_suffix('.py')
 
         existing = set()
         if pkg_dir.is_dir():
@@ -57,7 +57,7 @@ def __init__(self, name, directory=Path()):
         elif not existing:
             raise ValueError("No file/folder found for module {}".format(name))
 
-        self.source_dir = directory / self.prefix
+        self.source_dir = directory.joinpath(self.prefix)
 
         if '.' in name:
             self.namespace_package_name = name.rpartition('.')[0]
@@ -66,7 +66,7 @@ def __init__(self, name, directory=Path()):
     @property
     def file(self):
         if self.is_package:
-            return self.path / '__init__.py'
+            return self.path.joinpath('__init__.py')
         else:
             return self.path
 
@@ -76,24 +76,20 @@ def iter_files(self):
         Yields absolute paths - caller may want to make them relative.
         Excludes any __pycache__ and *.pyc files.
         """
-        def _include(path):
-            name = os.path.basename(path)
-            if (name == '__pycache__') or name.endswith('.pyc'):
-                return False
-            return True
+        def _walk(path):
+            files = []
+            for path in path.iterdir():
+                if path.is_file() and path.suffix != '.pyc':
+                    files.append(path)
+                if path.is_dir() and path.name != '__pycache__':
+                    files.extend(_walk(path))
+            return files
 
         if self.is_package:
             # Ensure we sort all files and directories so the order is stable
-            for dirpath, dirs, files in os.walk(str(self.path)):
-                for file in sorted(files):
-                    full_path = os.path.join(dirpath, file)
-                    if _include(full_path):
-                        yield full_path
-
-                dirs[:] = [d for d in sorted(dirs) if _include(d)]
-
+            return sorted(_walk(self.path))
         else:
-            yield str(self.path)
+            return [self.path]
 
 class ProblemInModule(ValueError): pass
 class NoDocstringError(ProblemInModule): pass

flit_core/flit_core/config.py

@@ -3,8 +3,7 @@
 import errno
 import logging
 import os
-import os.path as osp
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 import re
 
 from .vendor import tomli
@@ -177,6 +176,37 @@ def _flatten(d, prefix):
         res.update(_flatten(v, k))
     return res
 
+def posix_normpath(path):
+    """Normalize path, eliminating double slashes, etc."""
+    path = str(path)
+    sep = '/'
+    empty = ''
+    dot = '.'
+    dotdot = '..'
+    if path == empty:
+        return dot
+    initial_slashes = path.startswith(sep)
+    # POSIX allows one or two initial slashes, but treats three or more
+    # as single slash.
+    # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13)
+    if (initial_slashes and
+        path.startswith(sep*2) and not path.startswith(sep*3)):
+        initial_slashes = 2
+    comps = path.split(sep)
+    new_comps = []
+    for comp in comps:
+        if comp in (empty, dot):
+            continue
+        if (comp != dotdot or (not initial_slashes and not new_comps) or
+             (new_comps and new_comps[-1] == dotdot)):
+            new_comps.append(comp)
+        elif new_comps:
+            new_comps.pop()
+    comps = new_comps
+    path = sep.join(comps)
+    if initial_slashes:
+        path = sep*initial_slashes + path
+    return PurePosixPath(path or dot)
 
 def _check_glob_patterns(pats, clude):
     """Check and normalise glob patterns for sdist include/exclude"""
@@ -201,18 +231,18 @@ def _check_glob_patterns(pats, clude):
                 .format(clude, p)
             )
 
-        normp = osp.normpath(p)
+        normp = posix_normpath(PurePosixPath(p))
 
-        if osp.isabs(normp):
+        if normp.is_absolute():
             raise ConfigError(
                 '{} pattern {!r} is an absolute path'.format(clude, p)
             )
-        if osp.normpath(p).startswith('..' + os.sep):
+        if PurePosixPath(normp.parts[0]) == PurePosixPath('..'):
             raise ConfigError(
                 '{} pattern {!r} points out of the directory containing pyproject.toml'
                 .format(clude, p)
             )
-        normed.append(normp)
+        normed.append(str(normp))
 
     return normed
 
@@ -243,10 +273,10 @@ def add_scripts(self, scripts_dict):
 
 
 def description_from_file(rel_path: str, proj_dir: Path, guess_mimetype=True):
-    if osp.isabs(rel_path):
+    if PurePosixPath(rel_path).is_absolute():
         raise ConfigError("Readme path must be relative")
 
-    desc_path = proj_dir / rel_path
+    desc_path = proj_dir.joinpath(rel_path)
     try:
         with desc_path.open('r', encoding='utf-8') as f:
             raw_desc = f.read()

flit_core/flit_core/sdist.py

@@ -1,11 +1,9 @@
 from collections import defaultdict
 from copy import copy
-from glob import glob
 from gzip import GzipFile
 import io
 import logging
 import os
-import os.path as osp
 from pathlib import Path
 from posixpath import join as pjoin
 import tarfile
@@ -43,9 +41,9 @@ def __init__(self, patterns, basedir):
         self.files = set()
 
         for pattern in patterns:
-            for path in sorted(glob(osp.join(basedir, pattern))):
-                rel = osp.relpath(path, basedir)
-                if osp.isdir(path):
+            for path in self.basedir.glob(pattern):
+                rel = path.relative_to(basedir)
+                if rel.is_dir():
                     self.dirs.add(rel)
                 else:
                     self.files.add(rel)
@@ -54,14 +52,14 @@ def match_file(self, rel_path):
         if rel_path in self.files:
             return True
 
-        return any(rel_path.startswith(d + os.sep) for d in self.dirs)
+        return any(d in rel_path.parents for d in self.dirs)
 
     def match_dir(self, rel_path):
         if rel_path in self.dirs:
             return True
 
         # Check if it's a subdirectory of any directory in the list
-        return any(rel_path.startswith(d + os.sep) for d in self.dirs)
+        return any(d in rel_path.parents for d in self.dirs)
 
 
 class SdistBuilder:
@@ -75,12 +73,12 @@ def __init__(self, module, metadata, cfgdir, reqs_by_extra, entrypoints,
                  extra_files, include_patterns=(), exclude_patterns=()):
         self.module = module
         self.metadata = metadata
-        self.cfgdir = cfgdir
+        self.cfgdir = Path(cfgdir)
         self.reqs_by_extra = reqs_by_extra
         self.entrypoints = entrypoints
-        self.extra_files = extra_files
-        self.includes = FilePatterns(include_patterns, str(cfgdir))
-        self.excludes = FilePatterns(exclude_patterns, str(cfgdir))
+        self.extra_files = [Path(p) for p in extra_files]
+        self.includes = FilePatterns(include_patterns, cfgdir)
+        self.excludes = FilePatterns(exclude_patterns, cfgdir)
 
     @classmethod
     def from_ini_path(cls, ini_path: Path):
@@ -112,39 +110,30 @@ def select_files(self):
         This is overridden in flit itself to use information from a VCS to
         include tests, docs, etc. for a 'gold standard' sdist.
         """
-        cfgdir_s = str(self.cfgdir)
         return [
-            osp.relpath(p, cfgdir_s) for p in self.module.iter_files()
+            p.relative_to(self.cfgdir) for p in self.module.iter_files()
         ] + self.extra_files
 
     def apply_includes_excludes(self, files):
-        cfgdir_s = str(self.cfgdir)
-        files = {f for f in files if not self.excludes.match_file(f)}
+        files = {Path(f) for f in files if not self.excludes.match_file(Path(f))}
 
         for f_rel in self.includes.files:
             if not self.excludes.match_file(f_rel):
                 files.add(f_rel)
 
         for rel_d in self.includes.dirs:
-            for dirpath, dirs, dfiles in os.walk(osp.join(cfgdir_s, rel_d)):
-                for file in dfiles:
-                    f_abs = osp.join(dirpath, file)
-                    f_rel = osp.relpath(f_abs, cfgdir_s)
-                    if not self.excludes.match_file(f_rel):
-                        files.add(f_rel)
-
-                # Filter subdirectories before os.walk scans them
-                dirs[:] = [d for d in dirs if not self.excludes.match_dir(
-                    osp.relpath(osp.join(dirpath, d), cfgdir_s)
-                )]
+            for abs_path in self.cfgdir.joinpath(rel_d).glob('**/*'):
+                path = abs_path.relative_to(self.cfgdir)
+                if not self.excludes.match_file(path):
+                    files.add(path)
 
         crucial_files = set(
-            self.extra_files + [str(self.module.file.relative_to(self.cfgdir))]
+            self.extra_files + [self.module.file.relative_to(self.cfgdir)]
         )
         missing_crucial = crucial_files - files
         if missing_crucial:
             raise Exception("Crucial files were excluded from the sdist: {}"
-                            .format(", ".join(missing_crucial)))
+                            .format(", ".join(str(m) for m in missing_crucial)))
 
         return sorted(files)
 
@@ -157,26 +146,27 @@ def dir_name(self):
         return '{}-{}'.format(self.metadata.name, self.metadata.version)
 
     def build(self, target_dir, gen_setup_py=True):
-        os.makedirs(str(target_dir), exist_ok=True)
-        target = target_dir / '{}-{}.tar.gz'.format(
+        target_dir = Path(target_dir)
+        target_dir.mkdir(exist_ok=True)
+        target = target_dir.joinpath('{}-{}.tar.gz'.format(
                 self.metadata.name, self.metadata.version
-        )
+        ))
         source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH', '')
         mtime = int(source_date_epoch) if source_date_epoch else None
-        gz = GzipFile(str(target), mode='wb', mtime=mtime)
-        tf = tarfile.TarFile(str(target), mode='w', fileobj=gz,
+        gz = GzipFile(target, mode='wb', mtime=mtime)
+        tf = tarfile.TarFile(target, mode='w', fileobj=gz,
                              format=tarfile.PAX_FORMAT)
 
         try:
             files_to_add = self.apply_includes_excludes(self.select_files())
 
             for relpath in files_to_add:
-                path = str(self.cfgdir / relpath)
-                ti = tf.gettarinfo(path, arcname=pjoin(self.dir_name, relpath))
+                path = self.cfgdir.joinpath(relpath)
+                ti = tf.gettarinfo(str(path), arcname=pjoin(self.dir_name, relpath))
                 ti = clean_tarinfo(ti, mtime)
 
                 if ti.isreg():
-                    with open(path, 'rb') as f:
+                    with path.open('rb') as f:
                         tf.addfile(ti, f)
                 else:
                     tf.addfile(ti)  # Symlinks & ?

flit_core/flit_core/tests/test_sdist.py

@@ -1,5 +1,4 @@
 from io import BytesIO
-import os.path as osp
 from pathlib import Path
 import tarfile
 from testpath import assert_isfile
@@ -46,6 +45,6 @@ def test_include_exclude():
     )
     files = builder.apply_includes_excludes(builder.select_files())
 
-    assert osp.join('doc', 'test.rst') in files
-    assert osp.join('doc', 'test.txt') not in files
-    assert osp.join('doc', 'subdir', 'test.txt') in files
+    assert Path('doc').joinpath('test.rst') in files
+    assert Path('doc').joinpath('test.txt') not in files
+    assert Path('doc').joinpath('subdir').joinpath('test.txt') in files