Merge pull request #290 from dclong/dev

Merge dev into main

Author: dclong

dsutil/__init__.py

@@ -3,4 +3,4 @@
 from . import git
 from . import poetry
 
-__version__ = "0.63.0"
+__version__ = "0.64.0"

dsutil/docker/builder.py

@@ -6,7 +6,6 @@
 import tempfile
 from pathlib import Path
 import time
-import timeit
 import datetime
 from collections import deque, namedtuple
 import shutil
@@ -73,15 +72,6 @@ def _is_image_pushed(msg: dict[str, Any]):
         "total"]
 
 
-def _pull_image_timing(repo: str, tag: str) -> tuple[str, str, str, float]:
-    client = docker.from_env()
-    logger.info("Pulling the Docker image {}:{} ...", repo, tag)
-    seconds = timeit.timeit(
-        lambda: client.images.pull(repo, tag), timer=time.perf_counter_ns, number=1
-    ) / 1E9
-    return repo, tag, "pull", seconds
-
-
 def _ignore_socket(dir_, files):
     dir_ = Path(dir_)
     return [file for file in files if (dir_ / file).is_socket()]
@@ -244,7 +234,12 @@ def _copy_ssh(self, copy_ssh_to: str):
             shutil.copytree(ssh_src, ssh_dst, ignore=_ignore_socket)
             logger.info("~/.ssh has been copied to {}", ssh_dst)
 
-    def build(self, tag_build: str = None, copy_ssh_to: str = "") -> DockerAction:
+    def build(
+        self,
+        tag_build: str = None,
+        copy_ssh_to: str = "",
+        builder: str = "Docker"
+    ) -> DockerAction:
         """Build the Docker image.
 
         :param tag_build: The tag of the Docker image to build.
@@ -266,19 +261,23 @@ def build(self, tag_build: str = None, copy_ssh_to: str = "") -> DockerAction:
             tag_build = "latest"
         logger.info("Building the Docker image {}:{} ...", self._name, tag_build)
         self._update_base_tag(tag_build)
-        client = docker.APIClient(base_url="unix://var/run/docker.sock")
         try:
-            for msg in client.build(
-                path=str(self._path),
-                tag=f"{self._name}:{tag_build}",
-                rm=True,
-                pull=self.is_root(),
-                cache_from=None,
-                decode=True
-            ):
-                if "stream" in msg:
-                    print(msg["stream"], end="")
-            docker.from_env().images.get(f"{self._name}:{tag_build}")
+            if builder == "Docker":
+                for msg in docker.APIClient(base_url="unix://var/run/docker.sock"
+                                           ).build(
+                                               path=str(self._path),
+                                               tag=f"{self._name}:{tag_build}",
+                                               rm=True,
+                                               pull=self.is_root(),
+                                               cache_from=None,
+                                               decode=True
+                                           ):
+                    if "stream" in msg:
+                        print(msg["stream"], end="")
+                docker.from_env().images.get(f"{self._name}:{tag_build}")
+            elif builder == "Kaniko":
+                cmd = f"/kaniko/executor -c {self._path} -d {self._name}:{tag_build}"
+                sp.run(cmd, shell=True, check=True)
         except docker.errors.BuildError as err:
             return DockerAction(
                 succeed=False,
@@ -380,7 +379,8 @@ class DockerImageBuilder:
     def __init__(
         self,
         branch_urls: Union[dict[str, list[str]], str, Path],
-        branch_fallback: str = "dev"
+        branch_fallback: str = "dev",
+        builder: str = "Docker",
     ):
         if isinstance(branch_urls, (str, Path)):
             with open(branch_urls, "r") as fin:
@@ -393,6 +393,7 @@ def __init__(
         self._roots = set()
         self.failures = []
         self._servers = set()
+        self._builder = builder
 
     def _record_docker_servers(self, deps: deque[DockerImage]):
         for dep in deps:
@@ -614,7 +615,7 @@ def _build_image_node(
             branch=node.branch,
             branch_fallback=self._branch_fallback,
             repo_path=self._repo_path
-        ).build(tag_build=tag_build, copy_ssh_to=copy_ssh_to)
+        ).build(tag_build=tag_build, copy_ssh_to=copy_ssh_to, builder=self._builder)
         attr = self._graph.nodes[node]
         attr["build_succeed"] = succeed
         attr["build_err_msg"] = err_msg
@@ -623,7 +624,7 @@ def _build_image_node(
         if not succeed:
             return
         self._tag_image(name, tag, attr)
-        if push:
+        if self._builder == "Docker" and push:
             self._push_images(name, attr["action_time"])
 
     @staticmethod

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dsutil"
-version = "0.63.0"
+version = "0.64.0"
 description = "A utils Python package for data scientists."
 authors = ["Benjamin Du <[email protected]>"]
 

readme.md

@@ -25,7 +25,7 @@ Currently, Python 3.7 and 3.8 are supported.
 
 You can download a copy of the latest release and install it using pip.
 ```bash
-pip3 install --user -U https://github.com/dclong/dsutil/releases/download/v0.63.0/dsutil-0.63.0-py3-none-any.whl
+pip3 install --user -U https://github.com/dclong/dsutil/releases/download/v0.64.0/dsutil-0.64.0-py3-none-any.whl
 ```
 Or you can use the following command to install the latest master branch
 if you have pip 20.0+.
@@ -35,7 +35,7 @@ pip3 install --user -U git+https://github.com/dclong/dsutil@main
 Use one of the following commands if you want to install all components of dsutil. 
 Available additional components are `cv`, `docker`, `pdf`, `jupyter`, `admin` and `all`.
 ```bash
-pip3 install "dsutil[cv] @ https://github.com/dclong/dsutil/releases/download/v0.63.0/dsutil-0.63.0-py3-none-any.whl"
+pip3 install "dsutil[cv] @ https://github.com/dclong/dsutil/releases/download/v0.64.0/dsutil-0.64.0-py3-none-any.whl"
 # or
 pip3 install --user -U "dsutil[all] @ git+https://github.com/dclong/dsutil@main"
 ```