Quail fs integration

.gitignore

@@ -4,6 +4,7 @@ __pycache__/
 *#
 dist/
 build/
+build_fs/
 *.spec
 
 venv

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "iQuailFileServer"]
+	path = iQuailFileServer
+	url = https://github.com/QuailTeam/iQuailFileServer

examples/test_fileserver.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python3
+
+import iquail
+
+if iquail.helper.OS_LINUX:
+	iquail.run(
+		solution=iquail.SolutionFileServer('localhost', '4242',
+                                       './build_fs', '../iQuailFileServer'),
+		#solution=iquail.SolutionFileServer('localhost', '4242', '../iQuailFileServer/build'),
+		installer=iquail.Installer(
+			name='Allum1',
+			publisher='alies',
+			icon='icon.jpg',
+			binary='allum1',
+			console=True,
+			launch_with_quail=True
+		),
+		builder=iquail.builder.Builder(),
+		controller=iquail.ControllerTkinter()
+	)
+else:
+	iquail.run(
+		solution=iquail.SolutionFileServer('192.168.0.13', '4242', '../iQuailFileServer/build'),
+		installer=iquail.Installer(
+			publisher="Michael Moller",
+			name='OpenHardwareMonitor',
+			icon='OpenHardwareMonitor.exe',
+			binary='OpenHardwareMonitor.exe',
+			console=True,
+			launch_with_quail=True,
+			requires_root=True
+		),
+		builder=iquail.builder.Builder(
+			iquail.builder.CmdIcon('icon.ico'),
+			iquail.builder.CmdNoconsole(),
+			side_img_override="side_img.gif",
+		),
+		controller=iquail.ControllerTkinter()
+	)
+

examples/test_fileserver_xonotic.py

@@ -0,0 +1,21 @@
+#!/usr/bin/python3
+
+import iquail
+
+if not iquail.helper.OS_LINUX:
+    raise AssertionError("This test solution is linux only")
+
+iquail.run(
+    solution=iquail.SolutionFileServer('localhost', '4242',
+                                       './build_fs', '../iQuailFileServer'),
+    installer=iquail.Installer(
+        name='Xonotic',
+        publisher='OHM',
+        icon='misc/logos/icons_png/xonotic_512.png',
+        binary='xonotic-linux64-sdl',
+        console=True,
+        launch_with_quail=True
+    ),
+    builder=iquail.builder.Builder(side_img_override='side_img.gif'),
+    controller=iquail.ControllerTkinter()
+)

examples/xonotic.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+
+import iquail
+
+if iquail.helper.OS_LINUX:
+    binary = "xonotic-linux64-sdl"
+    icon = "misc/logos/icons_png/xonotic_512.png"
+
+if iquail.helper.OS_WINDOWS:
+    binary = "xonotic-x86.exe"
+    icon = "xonotic-x86.exe"
+
+iquail.run(
+    solution=iquail.SolutionPacked(path='Xonotic'),
+    installer=iquail.Installer(
+        publisher="OHM",
+        name='Xonotic',
+        icon=icon,
+        binary=binary,
+        console=False,
+        launch_with_quail=False,
+        is_large_solution=True,
+    ),
+
+    # iquail.builder.CmdIcon('icon.ico'),
+    builder=iquail.builder.Builder(
+        iquail.builder.CmdNoconsole(),
+        side_img_override='side_img.gif',
+    ),
+    controller=iquail.ControllerTkinter()
+)

iquail/builder/__init__.py

@@ -5,3 +5,4 @@
 from .cmd_icon import CmdIcon
 from .cmd_noconsole import CmdNoconsole
 from .cmd_integrity import CmdIntegrity
+from .cmd_fileserver_client import CmdFileserverClient

iquail/builder/cmd_fileserver_client.py

@@ -0,0 +1,51 @@
+import os
+import logging
+from .. import helper
+from .cmd_base import CmdBase
+from ..errors import BuilderError
+
+logger = logging.getLogger(__name__)
+
+
+class CmdFileserverClient(CmdBase):
+    """ Compile a Fileserver Client and add it to the executable
+    """
+
+    def __init__(self, build_path, fileserver_path, binary_name):
+        super().__init__()
+        self._binary_name = binary_name
+        self._fileserver_path = fileserver_path
+        if fileserver_path: # Build client
+            self._fileserver_path = os.path.abspath(self._fileserver_path)
+        # else: Client has already been built
+        self._build_path = build_path
+        self._build_path = os.path.abspath(self._build_path)
+        self._client_path = os.path.join(self._build_path, self._binary_name)
+
+    def _run_cmake(self):
+        cmd = 'cmake -B' + self._build_path
+        cmd += ' -S' + self._fileserver_path
+        status = os.system(cmd)
+        if status != 0:
+            raise BuilderError('CmdFileserverClient: CMake failed')
+
+    def _run_make(self):
+        cmd = 'make -j' + str(os.cpu_count())
+        cmd += ' -C ' + self._build_path
+        cmd += ' ' + self._binary_name
+        status = os.system(cmd)
+        if status != 0:
+            raise BuilderError('CmdFileserverClient: Make failed')
+
+    def pre_build(self):
+        if self._fileserver_path == None:
+            return
+        logger.info("Calling CMake:")
+        self._run_cmake()
+        logger.info("Calling Make:")
+        self._run_make()
+
+    def get_build_params(self):
+        params = []
+        params += ['--add-data', self._client_path + os.path.pathsep + '.']
+        return params

iquail/errors.py

@@ -6,3 +6,19 @@ class SolutionUnreachableError(Exception):
 class SolutionNotRemovableError(Exception):
     """Raised if solution is not reachable"""
     pass
+
+class SolutionFileNotFoundError(SolutionUnreachableError):
+    """Raised if solution cannot retreive file"""
+    pass
+
+class SolutionVersionNotFoundError(SolutionUnreachableError):
+    """Raised if distant version cannot be selected"""
+    pass
+
+class SolutionDecompressionError(SolutionUnreachableError):
+    """Raised if distant version cannot be selected"""
+    pass
+
+class BuilderError(Exception):
+    """Raised if solution is not reachable"""
+    pass

iquail/solution/__init__.py

@@ -3,3 +3,4 @@
 from .solution_zip import SolutionZip
 from .solution_packed import SolutionPacked
 from .solution_github import SolutionGitHub
+from .solution_fileserver import SolutionFileServer

iquail/solution/solution_fileserver.py

@@ -0,0 +1,161 @@
+import os
+import sys
+import shutil
+from .solution_base import SolutionBase
+from .solution_fileserver_wrapper import QuailFS
+from ..builder.cmd_fileserver_client import CmdFileserverClient
+from ..errors import SolutionUnreachableError
+from ..errors import SolutionFileNotFoundError
+from ..errors import SolutionVersionNotFoundError
+from ..errors import SolutionDecompressionError
+from ..helper import misc
+from ..helper import OS_WINDOWS
+
+class SolutionFileServer(SolutionBase):
+    def __init__(self, host, port, build_path, fileserver_path=None):
+        super().__init__()
+        self._binary_name = 'iQuailClient'
+        if OS_WINDOWS:
+            self._binary_name += '.exe'
+        self._host = host
+        self._port = port
+        self._fileserver_path = fileserver_path
+        if fileserver_path:
+            self._fileserver_path = os.path.abspath(self._fileserver_path)
+        self._build_path = build_path
+        self._build_path = os.path.abspath(self._build_path)
+        if misc.running_from_script():
+            self._client_bin_path = os.path.join(self._build_path, self._binary_name)
+        else:
+            self._client_bin_path = os.path.join(sys._MEIPASS, self._binary_name)
+        self._tmpdir = None
+        self._serv = None
+        self._files = None
+        self._nbrFiles = 1
+        self._nbrFilesDownloaded = 0
+        self._update = False
+
+    def local(self):
+        return False
+
+    def get_version_string(self):
+        if self._serv == None:
+            self._init_server()
+            version = self._serv.get_version()
+            self._fini_server()
+            return version
+        return self._serv.get_version()
+
+    def _get_patch_version_name(self, curr, last):
+        return curr + '_TO_' + last
+
+    def _init_server(self):
+        self._tmpdir = misc.safe_mkdtemp()
+        self._serv = QuailFS(self._client_bin_path, self._tmpdir)
+        if not self._serv.connect(self._host, self._port):
+            raise SolutionUnreachableError("FileServer.connect() failed: %s" %
+                                           self._serv.get_error())
+
+    def _fini_server(self):
+        self._serv.disconnect()
+        shutil.rmtree(self._tmpdir)
+        self._serv = None
+
+    def open(self):
+        self._init_server()
+        currentVersion = self.get_installed_version()
+        lastVersion = self.get_version_string()
+        if currentVersion != None and currentVersion != lastVersion:
+            self._update = True
+            patchName = self._get_patch_version_name(currentVersion, lastVersion)
+            if not self._serv.set_version(patchName):
+                raise SolutionVersionNotFoundError("FileServer.set_version() failed: %s" %
+                                                   self._serv.get_error())
+        self._nbrFiles = self._serv.get_nbr_files()
+        self._nbrFilesDownloaded = 0
+
+    def close(self):
+        self._fini_server()
+        self._update = False
+
+    def _parse_ls(self, lines):
+        dirs, files = [], []
+        for line in lines:
+            ftype, fsize, fname = line.split(' ', 2)
+            if ftype == 'd':
+                dirs.append(fname)
+            else:
+                files.append(fname)
+        return (dirs, files)
+
+    def _walk_rec(self, root):
+        lines = self._serv.ls(root)
+        dirs, files = self._parse_ls(lines)
+        yield (root, dirs, files)
+        for d in dirs:
+            yield from self._walk_rec(os.path.join(root, d))
+
+    def walk(self):
+        return self._walk_rec('.')
+
+    def _get_tmp_path(self, relpath):
+        return os.path.join(self._tmpdir, relpath)
+
+    def _decompress(self, sourcename, targetname, diffname):
+        def bytes_from_file(filename, chunksize=8192):
+            with open(filename, "rb") as f:
+                while True:
+                    chunk = f.read(chunksize)
+                    if chunk:
+                        for b in chunk:
+                            yield b
+                    else:
+                        break
+        target = open(targetname, 'w+b')
+        source = open(sourcename, 'rb')
+        buffer = b''
+        for byte in bytes_from_file(diffname):
+            if byte == ord(b'\n') and len(buffer):
+                (header, arg) = buffer.split(b':', maxsplit=1)
+                if header == b'INSERT':
+                    (off, _, length) = arg.split()
+                    source.seek(int(off))
+                    target.write(source.read(int(length)))
+                elif header == b'COPY' and len(arg) > 1:
+                    target.write(bytes([arg[1]]))
+                elif header == b'COPY':
+                    target.write(bytes([byte]))
+                buffer = b''
+            else:
+                if buffer == b'\n':
+                    buffer = b''
+                buffer += bytes([byte])
+        target.close()
+        source.close()
+
+    def _try_decompress(self, relpath):
+        source_path = self.retrieve_current_file(relpath)
+        if source_path == None:
+            return
+        diff_path = self._get_tmp_path(relpath)
+        target_path = diff_path
+        diff_path = diff_path + '_diff'
+        os.rename(target_path, diff_path)
+        self._decompress(source_path, target_path, diff_path)
+        os.remove(diff_path)
+
+    def retrieve_file(self, relpath):
+        if not self._serv.get_file(relpath.replace(os.path.sep, '/')):
+            raise SolutionFileNotFoundError('FileServer.get_file() failed')
+        try:
+            self._try_decompress(relpath)
+        except Exception as e:
+            raise SolutionDecompressionError('Unexcpected error in decompression: ' + str(e))
+        self._nbrFilesDownloaded += 1
+        self._update_progress(percent=(100*self._nbrFilesDownloaded)/self._nbrFiles, status='downloading', log=relpath+'\n')
+        return self._get_tmp_path(relpath)
+
+    def builder_cmds(self):
+        cmds = super().builder_cmds() + [CmdFileserverClient(
+            self._build_path, self._fileserver_path, self._binary_name)]
+        return cmds