Merge pull request #27 from xsuite/release/v0.3.4

Release 0.3.4

Author: freddieknets

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "xaux"
-version = "0.3.3"
+version = "0.3.4"
 description = "Support tools for Xsuite packages"
 authors = ["Frederik F. Van der Veken <[email protected]>",
            "Thomas Pugnat <[email protected]>",
@@ -14,7 +14,7 @@ include = ["LICENSE", "NOTICE"]
 [tool.poetry.dependencies]
 python = ">=3.8"
 
-[tool.poetry.dev-dependencies]
+[poetry.group.dev.dependencies]
 pytest = ">=7.3"
 
 [tool.poetry.extras]

tests/pytest_all_versions.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# files="test_fs.py test_fs_api.py test_fs_afs.py test_fs_eos.py"
+#files='test_fs*.py'
 files=''
 
 for i in 8 9 10 11 12 13

tests/test_general_tools.py

@@ -4,7 +4,7 @@
 # ######################################### #
 
 from subprocess import run, TimeoutExpired
-import numpy as np
+from math import ceil
 from xaux import timestamp, ranID, get_hash, FsPath
 
 
@@ -94,27 +94,27 @@ def test_ranID():
     rans = [ranID(length=l+1) for l in range(40)]
     print(rans)
     for i, ran in enumerate(rans):
-        assert len(ran) == int(np.ceil((i+1)/4)*4)
+        assert len(ran) == int(ceil((i+1)/4)*4)
 
     rans = [ranID(length=20) for _ in range(1000)]
     for ran in rans:
         base64_safe = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
-        assert np.all([c in base64_safe for c in ran])
+        assert all([c in base64_safe for c in ran])
 
     rans = [ranID(length=20, only_alphanumeric=True) for _ in range(1000)]
     for ran in rans:
         alnum = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
-        assert np.all([c in alnum for c in ran])
+        assert all([c in alnum for c in ran])
 
     rans = ranID(length=20, size=1000)
     for ran in rans:
         base64_safe = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
-        assert np.all([c in base64_safe for c in ran])
+        assert all([c in base64_safe for c in ran])
 
     rans = ranID(length=20, size=1000, only_alphanumeric=True)
     for ran in rans:
         alnum = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
-        assert np.all([c in alnum for c in ran])
+        assert all([c in alnum for c in ran])
 
 
 def test_system_lock():

tests/test_version.py

@@ -6,5 +6,5 @@
 from xaux import __version__
 
 def test_version():
-    assert __version__ == '0.3.3'
+    assert __version__ == '0.3.4'
 

xaux/fs/fs.py

@@ -34,6 +34,7 @@ class FsPath:
     is just a regular Path object but with access to the FsPath methods.
     """
     __slots__ = ()
+    __hash__ = Path.__hash__
 
     def __new__(cls, *args):
         from .eos import EosPath, _on_eos

xaux/fs/io.py

@@ -35,7 +35,7 @@
 # cp file1 file2 ... link1  link points to dir1:          copies files into dir1 (potentially overwrites)
 # cp dir1 file1                                           error  "cp: -r not specified; omitting directory 'dir1'"
 # cp -r dir1 file1                                        error  "cp: cannot overwrite non-directory 'file1' with directory 'dir1'"
-# cp -r dir1 dir2           dir2 does not exist:          copies dir1 into new dir2
+# cp -r dir1 dir2           dir2 does not exist:          copies dir1 as new dir2
 # cp -r dir1 dir2           dir2 does exist:              copies dir1 inside dir2
 # links are treated as the file/dir they point to, unless -P is specified, in which case (only for links as source argument) they are treated as separate files
 
@@ -90,15 +90,15 @@ def cp(*args, recursive=False, follow_symlinks=True, **kwargs):
 
     # t_new = time.time(); print(f"Setup done ({int(1e3*(t_new-t_prev))}ms)"); t_prev = t_new
     # Do the copy
-    stdout, stderr = _cp_regular(sources_targets, follow_symlinks)
+    stdout, stderr = _cp_regular(sources_targets, follow_symlinks, **kwargs)
     this_stdout += stdout
     this_stderr += stderr
     # t_new = time.time(); print(f"Regular done ({int(1e3*(t_new-t_prev))}ms)"); t_prev = t_new
-    stdout, stderr = _cp_afs(afs_sources_targets, follow_symlinks)
+    stdout, stderr = _cp_afs(afs_sources_targets, follow_symlinks, **kwargs)
     this_stdout += stdout
     this_stderr += stderr
     # t_new = time.time(); print(f"AFS done ({int(1e3*(t_new-t_prev))}ms)"); t_prev = t_new
-    stdout, stderr = _cp_eos(eos_sources_targets, follow_symlinks)
+    stdout, stderr = _cp_eos(eos_sources_targets, follow_symlinks, **kwargs)
     this_stdout += stdout
     this_stderr += stderr
     # t_new = time.time(); print(f"EOS done ({int(1e3*(t_new-t_prev))}ms)"); t_prev = t_new
@@ -169,7 +169,7 @@ def _loop_sources_and_verify(sources, target):
     return src_target_recursive
 
 
-def _cp_regular(sources_targets, follow_symlinks):
+def _cp_regular(sources_targets, follow_symlinks, **kwargs):
     stdout = ""
     stderr = ""
     for src, target, recursive in sources_targets:
@@ -205,22 +205,42 @@ def _cp_regular(sources_targets, follow_symlinks):
     return stdout, stderr  # An empty error message means success
 
 
-def _cp_afs(sources_targets, follow_symlinks):
+def _cp_afs(sources_targets, follow_symlinks, method=None, **kwargs):
     from xaux.fs import _skip_afs_software, _force_xrdcp
     this_stdout = ""
     this_stderr = ""
     if _skip_afs_software:
         assert not _force_xrdcp
 
+    if method:
+        if method not in ['mount', 'xrdcp']:
+            raise ValueError(f"Invalid method '{method}' for AFS copy. "
+                           + f"Use 'mount' or 'xrdcp'.")
+        elif method == 'xrdcp':
+            if not _xrdcp_installed:
+                raise RuntimeError("xrdcp is not installed. Cannot copy "
+                                + "files to/from AFS using xrdcp.")
+            elif _skip_afs_software:
+                raise RuntimeError("Cannot copy files to/from AFS using xrdcp "
+                                + "because AFS software is skipped.")
+        elif method == 'mount':
+            if not _afs_mounted:
+                raise RuntimeError("AFS is not mounted. Cannot copy files "
+                                + "to/from AFS using mount.")
+            elif _force_xrdcp:
+                raise RuntimeError("Cannot copy files to/from AFS using mount "
+                                + "because xrdcp is forced.")
+
     # We first try to use xrdcp
-    if sources_targets and _xrdcp_installed and not _skip_afs_software:
+    if sources_targets and _xrdcp_installed and not _skip_afs_software \
+    and method != 'mount':
         sources_targets, stdout, stderr = _cp_xrdcp(sources_targets)
         this_stdout += stdout
         this_stderr += stderr
 
     # If xrdcp failed, we try to use the AFS mout
     regular_stderr = ""
-    if sources_targets:
+    if sources_targets and method != 'xrdcp':
         if _force_xrdcp:
             this_stderr += "Skipping AFS mount.\n"
             this_stderr += "Failed to copy files to AFS.\n"
@@ -242,29 +262,63 @@ def _cp_afs(sources_targets, follow_symlinks):
         return this_stdout, this_stderr + regular_stderr
 
 
-def _cp_eos(sources_targets, follow_symlinks):
+def _cp_eos(sources_targets, follow_symlinks, method=None, **kwargs):
     from xaux.fs import _skip_eos_software, _force_eoscmd, _force_xrdcp
     this_stdout = ""
     this_stderr = ""
     if _skip_eos_software:
         assert not _force_eoscmd
         assert not _force_xrdcp
 
+    if method:
+        if method not in ['mount', 'xrdcp', 'eoscmd']:
+            raise ValueError(f"Invalid method '{method}' for EOS copy. "
+                           + f"Use 'mount', 'xrdcp' or 'eoscmd'.")
+        elif method == 'xrdcp':
+            if not _xrdcp_installed:
+                raise RuntimeError("xrdcp is not installed. Cannot copy "
+                                 + "files to/from EOS using xrdcp.")
+            elif _skip_eos_software:
+                raise RuntimeError("Cannot copy files to/from EOS using xrdcp "
+                                 + "because EOS software is skipped.")
+            elif _force_eoscmd:
+                raise RuntimeError("Cannot copy files to/from EOS using xrdcp "
+                                 + "because eoscmd is forced.")
+        elif method == 'eoscmd':
+            if not _eoscmd_installed:
+                raise RuntimeError("eoscmd is not installed. Cannot copy "
+                                 + "files to/from EOS using eoscmd.")
+            elif _skip_eos_software:
+                raise RuntimeError("Cannot copy files to/from EOS using eoscmd "
+                                 + "because EOS software is skipped.")
+            elif _force_xrdcp:
+                raise RuntimeError("Cannot copy files to/from EOS using eoscmd "
+                                 + "because xrdcp is forced.")
+        elif method == 'mount':
+            if not _eos_mounted:
+                raise RuntimeError("EOS is not mounted. Cannot copy files "
+                                 + "to/from EOS using mount.")
+            elif _force_xrdcp or _force_eoscmd:
+                raise RuntimeError("Cannot copy files to/from EOS using mount "
+                                 + "because xrdcp or eoscmd are forced.")
+
     # We first try to use xrdcp
-    if sources_targets and _xrdcp_installed and not _force_eoscmd and not _skip_eos_software:
+    if sources_targets and _xrdcp_installed and not _force_eoscmd and not _skip_eos_software \
+    and method != 'mount' and method != 'eoscmd':
         sources_targets, stdout, stderr = _cp_xrdcp(sources_targets)
         this_stdout += stdout
         this_stderr += stderr
 
     # If xrdcp failed, we try to use eos
-    if sources_targets and _eoscmd_installed and not _force_xrdcp and not _skip_eos_software:
+    if sources_targets and _eoscmd_installed and not _force_xrdcp and not _skip_eos_software \
+    and method != 'mount' and method != 'xrdcp':
         sources_targets, stdout, stderr = _cp_eoscmd(sources_targets)
         this_stdout += stdout
         this_stderr += stderr
 
     # If eos also failed, we try using the EOS mount
     regular_stderr = ""
-    if sources_targets:
+    if sources_targets and method != 'xrdcp' and method != 'eoscmd':
         if _force_xrdcp or _force_eoscmd:
             this_stderr += "Skipping EOS mount.\n"
             this_stderr += "Failed to copy files to EOS.\n"

xaux/general.py

@@ -10,5 +10,5 @@
 # ===================
 # Do not change
 # ===================
-__version__ = '0.3.3'
+__version__ = '0.3.4'
 # ===================

xaux/tools/general_tools.py

@@ -8,8 +8,8 @@
 import atexit
 import base64
 import hashlib
-import pandas as pd
-import numpy as np
+from math import ceil
+from datetime import datetime, timezone
 
 from ..fs import FsPath
 
@@ -29,7 +29,7 @@ def timestamp(*, in_filename=False, ms=False, us=False):
     idx = -3 if ms else -7
     idx = 26 if us else idx
     form = "%Y-%m-%d_%H-%M-%S.%f" if in_filename else "%Y-%m-%d %H:%M:%S.%f"
-    return pd.Timestamp.now(tz='UTC').to_pydatetime().strftime(form)[:idx]
+    return datetime.now(tz=timezone.utc).strftime(form)[:idx]
 
 
 def ranID(*, length=12, size=1, only_alphanumeric=False):
@@ -51,7 +51,7 @@ def ranID(*, length=12, size=1, only_alphanumeric=False):
     if size > 1:
         return [ranID(length=length, only_alphanumeric=only_alphanumeric)
                 for _ in range(size)]
-    length = int(np.ceil(length/4))
+    length = int(ceil(length/4))
     if only_alphanumeric:
         ran = ''
         for _ in range(length):