Fix pip distribution (#812)

MaxRayskiy · Fedr · web-flow · commit 97b7b35be4b3 · 2022-12-19T14:07:56.000Z
* Update pip-build.yml

* fic

* fic

* direct call

* skip unit tests

* another attempt

* win python fix

* test

* test

* add python libs

* reinstall python

* debug

* py libs

* build latest

* return win unit test

* update tests

* undo debug

Co-authored-by: Fedr &lt;fchelnokov@adalisk.com&gt;
diff --git a/.github/workflows/pip-build.yml b/.github/workflows/pip-build.yml
@@ -61,7 +61,9 @@ jobs:
           ${{ matrix.py-cmd }} -m pip install --upgrade -r ./requirements/python.txt 
 
       - name: Install thirdparty libs
-        run: ln -s /usr/local/lib/meshlib-thirdparty-lib/lib ./lib
+        run: |
+          ln -s /usr/local/lib/meshlib-thirdparty-lib/lib ./lib
+          ln -s /usr/local/lib/meshlib-thirdparty-lib/include ./include
 
       - name: Build
         run: ./scripts/build_source.sh
@@ -129,6 +131,9 @@ jobs:
           - py-version: "3.10"
             py-tag: "cp310"
             py-short-version: "310"
+            py-dir: "Python310"
+            py-file: "python310"
+            embed-version: "3.10.0"
           - py-version: "3.11"
             py-tag: "cp311"
             py-short-version: "311"
@@ -149,17 +154,10 @@ jobs:
         run: docker run -d --name meshlib-win --workdir C:\workspace\MeshLib\MeshLib -v ${{ github.workspace }}\..\..:C:\workspace meshrus/meshlib-windows:latest ping -t localhost
 
       - name: Setup python
-        if: ${{ matrix.py-version != '3.10' }}
         run: |
           docker exec meshlib-win choco uninstall python python3 -y
           docker exec meshlib-win choco install python3 --version=${{matrix.py-version}} -y
-          docker exec -w C:\vcpkg\installed\x64-windows-meshrus\lib meshlib-win powershell.exe -c "Remove-Item python310.*"
-          docker exec -w C:\vcpkg\installed\x64-windows-meshrus\bin meshlib-win powershell.exe -c "Remove-Item python310.*"
-          docker exec -w C:\vcpkg\installed\x64-windows-meshrus meshlib-win powershell.exe -c "Copy-Item -Path C:\${{matrix.py-dir}}\${{matrix.py-file}}.dll -Destination bin"
           docker exec -w C:\vcpkg\installed\x64-windows-meshrus meshlib-win powershell.exe -c "Copy-Item -Path C:\${{matrix.py-dir}}\libs\${{matrix.py-file}}.lib -Destination lib"
-          docker exec -w C:\vcpkg\installed\x64-windows-meshrus\include meshlib-win powershell.exe -c "Remove-Item python3.10 -Recurse"
-          docker exec -w C:\vcpkg\installed\x64-windows-meshrus\include meshlib-win powershell.exe -c "mkdir python3.10"
-          docker exec -w C:\vcpkg\installed\x64-windows-meshrus\include meshlib-win powershell.exe -c "Copy-Item -Path C:\${{matrix.py-dir}}\include\* -Destination python3.10 -Recurse"
           docker exec meshlib-win py -${{matrix.py-version}} -m pip install --upgrade pip
           docker exec meshlib-win py -${{matrix.py-version}} -m pip install --upgrade -r ./requirements/python.txt
           docker exec -w C:\workspace\MeshLib\MeshLib\thirdparty\python meshlib-win curl https://www.python.org/ftp/python/${{matrix.embed-version}}/python-${{matrix.embed-version}}-embed-amd64.zip --output python-embed.zip
@@ -168,20 +166,13 @@ jobs:
       - name: Build
         run: |
           docker exec meshlib-win C:\vcpkg\vcpkg.exe integrate install
-          docker exec meshlib-win C:\Temp\vcbuildtools\\MSBuild\Current\Bin\MSBuild.exe source\MeshLib.sln -p:Configuration=Release -p:PythonVersion=${{matrix.py-short-version}} -p:PythonLibPath=C:\${{matrix.py-dir}}\libs\${{matrix.py-file}}.lib -p:PythonDllPath=C:\${{matrix.py-dir}}\${{matrix.py-file}}.dll
-
-      # cant fix embedded python 3.9 for windows yet, but this embedded python is not used in wheel, so we can just disable it this way
-      - name: Embedded Python3.9 fix
-        if: ${{ matrix.py-version == '3.9' }}
-        working-directory: source\x64\Release
-        run: del python39.zip
-        shell: cmd
+          docker exec meshlib-win C:\Temp\vcbuildtools\\MSBuild\Current\Bin\MSBuild.exe source\MeshLib.sln -p:Configuration=Release -p:PythonVersion=${{matrix.py-short-version}} -p:PythonLibPath=C:\${{matrix.py-dir}}\libs\${{matrix.py-file}}.lib -p:PythonDllPath=C:\${{matrix.py-dir}}\${{matrix.py-file}}.dll -p:PythonIncludePath=C:\${{matrix.py-dir}}\include\
 
       - name: Run Test
         working-directory: source\x64\Release
         run: .\MeshViewer.exe -tryHidden -noEventLoop
 
-      - name: Unit Tests Default
+      - name: Unit Tests
         run: docker exec meshlib-win py -3 scripts\run_unit_test_script.py Release
 
       - name: Python Tests
@@ -322,12 +313,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu:20.04", "ubuntu:22.04", "fedora:35", "fedora:37"]
+        os: ["ubuntu:20.04", "ubuntu:22.04", "debian:11-slim", "fedora:37"]
         include:
           - os: "ubuntu:20.04"
             py-version: "3.8"
             py-cmd: "python3.8"
-          - os: "fedora:35"
+          - os: "debian:11-slim"
             py-version: "3.9"
             py-cmd: "python3.9"
           - os: "ubuntu:22.04"
@@ -343,15 +334,9 @@ jobs:
           ref: ${{needs.setup.outputs.version_tag}}
 
       - name: Ubuntu system setup
-        if: ${{ matrix.os == 'ubuntu:20.04' || matrix.os == 'ubuntu:22.04'}}
+        if: ${{ matrix.os == 'ubuntu:20.04' || matrix.os == 'ubuntu:22.04' || matrix.os == 'debian:11-slim' }}
         run: apt -y update && apt -y upgrade && apt -y install curl libssl-dev python3-distutils python3-pip
 
-      - name: Fedora 35 system setup
-        if: ${{ matrix.os == 'fedora:35'}}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.py-version }}
-
       - name: Fedora 37 system setup
         if: ${{matrix.os == 'fedora:37'}}
         run: dnf -y install python3 pip
diff --git a/.github/workflows/release-tests.yml b/.github/workflows/release-tests.yml
@@ -23,43 +23,40 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: ${{ matrix.os }}
-      options: --user root
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu:20.04", "ubuntu:22.04", "fedora:35"]
+        os: ["ubuntu:20.04", "ubuntu:22.04", "debian:11-slim", "fedora:37"]
         include:
           - os: "ubuntu:20.04"
             py-version: "3.8"
             py-cmd: "python3.8"
           - os: "ubuntu:22.04"
             py-version: "3.10"
             py-cmd: "python3.10"
-          - os: "fedora:35"
+          - os: "debian:11-slim"
             py-version: "3.9"
             py-cmd: "python3.9"
+          - os: "fedora:37"
+            py-version: "3.11"
+            py-cmd: "python3.11"
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
           ref: ${{needs.setup.outputs.version_tag}}
 
-      - name: Python setup
-        if: ${{ matrix.os == 'fedora:35'}}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.py-version }}
+      - name: Ubuntu system setup
+        if: ${{ matrix.os == 'ubuntu:20.04' || matrix.os == 'ubuntu:22.04' || matrix.os == 'debian:11-slim' }}
+        run: apt -y update && apt -y upgrade && apt -y install curl libssl-dev python3-distutils python3-pip
 
-      - name: System setup
-        if: ${{ matrix.os != 'fedora:35'}}
-        run: |
-          apt -y update && apt -y upgrade && apt -y install curl libssl-dev python3-distutils python3-pip
+      - name: Fedora 37 system setup
+        if: ${{matrix.os == 'fedora:37'}}
+        run: dnf -y install python3 pip
 
       - name: Pip setup
         run: |
           ${{ matrix.py-cmd }} -m pip install --upgrade pip
-          ${{ matrix.py-cmd }} -m pip uninstall -y meshlib
-          ${{ matrix.py-cmd }} -m pip install --upgrade -r ./requirements/python.txt
 
       - name: Meshlib wheel install
         run:  ${{ matrix.py-cmd }} -m pip install meshlib
@@ -75,7 +72,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        py-version: ["3.8", "3.9", "3.10" ]
+        py-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -89,78 +86,38 @@ jobs:
 
       - name: Pip setup
         run: |
-          py -${{matrix.py-version}} -m pip install --upgrade pip
-          py -${{matrix.py-version}} -m pip uninstall -y meshlib
-          py -${{matrix.py-version}} -m pip install --upgrade -r ./requirements/python.txt     
+          py -${{matrix.py-version}} -m pip install --upgrade pip 
           py -${{matrix.py-version}} -m pip install meshlib
 
       - name: Run Python tests
         working-directory: test_python
         run: py -${{matrix.py-version}} -m pytest -s -v
 
-  install-test:
+  pip-test-macos:
+    needs: setup
     timeout-minutes: 20
-    runs-on: ubuntu-latest
-    container:
-      image: ${{matrix.container}}
-      options: --user root
+    runs-on: macos-12
     strategy:
       fail-fast: false
       matrix:
-        os: [Ubuntu20, Ubuntu22, Fedroa35]
-        include:
-          - os: Ubuntu20
-            container: ubuntu:20.04
-            fileName: "*ubuntu20*"
-            updateCmd: export DEBIAN_FRONTEND=noninteractive && apt-get -y update && apt-get -y upgrade && apt-get -y install sudo
-            installCmd: apt-get -y install ./*.deb
-          - os: Ubuntu22
-            container: ubuntu:22.04
-            fileName: "*ubuntu22*"
-            updateCmd: export DEBIAN_FRONTEND=noninteractive && apt-get -y update && apt-get -y upgrade && apt-get -y install sudo
-            installCmd: apt-get -y install ./*.deb
-          - os: Fedroa35
-            container: fedora:35
-            fileName: "*-dev.rpm"
-            updateCmd: dnf --refresh -y upgrade
-            installCmd: dnf -y install ./*.rpm
+        py-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          ref: ${{needs.setup.outputs.version_tag}}
 
-      - name: Python setup on Fedora
-        if: ${{ matrix.os == 'Fedroa35'}}
-        run: |
-          dnf -y install python3.9 curl
-          curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9
-          python3.9 -m pip install --upgrade -r ./requirements/python.txt
+      - name: Python setup
+        if: ${{ matrix.py-version != '3.11' }}
+        run: brew install python@${{matrix.py-version}}
 
-      - name: System setup
-        if: ${{ matrix.os != 'Fedroa35'}}
+      - name: Pip setup
         run: |
-          apt -y update && apt -y upgrade && apt -y install curl libssl-dev python3-distutils python3-pip
-          python3 -m pip install --upgrade pip
-          python3 -m pip install --upgrade -r ./requirements/python.txt
-
-      - name: Download Release
-        uses: robinraju/release-downloader@v1.6
-        with:
-          repository: MeshInspector/MeshLib
-          latest: true
-          fileName: ${{matrix.fileName}}
-          out-file-path: ""
+          ${{ matrix.py-cmd }} -m pip install --upgrade pip
 
-      - name: Install Release
-        run: |
-          ${{matrix.updateCmd}}
-          ${{matrix.installCmd}}
+      - name: Meshlib wheel install
+        run:  ${{ matrix.py-cmd }} -m pip install meshlib
 
       - name: Run Python tests
-        if: ${{ matrix.os != 'Fedroa35'}}
         working-directory: test_python
-        run: python3 -m pytest -s -v
-
-      - name: Run Python tests on Fedora
-        if: ${{ matrix.os == 'Fedroa35'}}
-        working-directory: test_python
-        run: python3.9 -m pytest -s -v
+        run: ${{ matrix.py-cmd }} -m pytest -s -v
diff --git a/.github/workflows/test-distribution.yml b/.github/workflows/test-distribution.yml
@@ -39,15 +39,19 @@ jobs:
         if: ${{ matrix.os != 'fedora:35' }}
         run: |
           export DEBIAN_FRONTEND=noninteractive && apt-get -y update && apt-get -y upgrade && \
-            apt-get -y install sudo xvfb && \
+            apt-get -y install sudo xvfb curl libssl-dev python3-distutils python3-pip && \
             apt-get -y install ./*${{matrix.name}}*.deb
+          python3 -m pip install --upgrade pip
+          python3 -m pip install --upgrade -r ./requirements/python.txt
 
       - name: Install Fedora Release
         if: ${{ matrix.os == 'fedora:35' }}
         run: |
           dnf --refresh -y upgrade
-          dnf -y install xorg-x11-server-Xvfb mesa-dri-drivers mesa-libGL-devel libXi-devel
+          dnf -y install xorg-x11-server-Xvfb mesa-dri-drivers mesa-libGL-devel libXi-devel python3.9 curl
           dnf -y install ./*.rpm
+          curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9
+          python3.9 -m pip install --upgrade -r ./requirements/python.txt
 
       - name: Debug output
         if: always()
@@ -70,4 +74,13 @@ jobs:
           cmake .. -DCMAKE_CXX_COMPILER=${{matrix.cxx-compiler}} && \
           cmake --build . && \
           ./cmake-example
-          
+
+      - name: Ubuntu python test
+        if: ${{ always() && matrix.os != 'fedora:35' }}
+        working-directory: test_python
+        run: python3 -m pytest -s -v
+
+      - name: Fedora python test
+        if: ${{ always() && matrix.os == 'fedora:35' }}
+        working-directory: test_python
+        run: python3.9 -m pytest -s -v
diff --git a/source/MRMesh/MRMesh.vcxproj b/source/MRMesh/MRMesh.vcxproj
@@ -536,8 +536,8 @@
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
-      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+     <GenerateDebugInformation>true</GenerateDebugInformation>
+      <AdditionalDependencies>ws2_32.lib;%(AdditionalDependencies);$(PythonLibPath)</AdditionalDependencies>
       <DelayLoadDLLs>
       </DelayLoadDLLs>
     </Link>
@@ -573,7 +573,7 @@ copy $(ProjectDir)..\..\thirdparty\python\python310.zip $(TargetDir)python310.zi
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>ws2_32.lib;%(AdditionalDependencies);$(PythonLibPath)</AdditionalDependencies>
       <DelayLoadDLLs>
       </DelayLoadDLLs>
     </Link>
diff --git a/source/common.props b/source/common.props
@@ -25,6 +25,7 @@
   <PropertyGroup Label="Globals">
     <VcpkgCurrentInstalledDir>$(VcpkgRoot)\installed\$(VcpkgTriplet)</VcpkgCurrentInstalledDir>
     <PythonVersion>310</PythonVersion>
+	<PythonIncludePath>$(VcpkgCurrentInstalledDir)\include\python3.10\</PythonIncludePath>
     <PythonLibPath>$(VcpkgCurrentInstalledDir)\lib\python$(PythonVersion).lib</PythonLibPath>
     <PythonDllPath>$(VcpkgCurrentInstalledDir)\bin\python$(PythonVersion).dll</PythonDllPath>
   </PropertyGroup>
@@ -34,7 +35,7 @@
       <LanguageStandard>stdcpplatest</LanguageStandard>
       <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
       <PreprocessorDefinitions>MR_PROJECT_NAME="$(MeshLibProjectName)";NOMINMAX;_CRT_SECURE_NO_DEPRECATE;ImDrawIdx=unsigned;SPDLOG_COMPILED_LIB;SPDLOG_SHARED_LIB;_SILENCE_ALL_CXX17_DEPRECATION_WARNINGS;_SILENCE_CXX23_ALIGNED_STORAGE_DEPRECATION_WARNING;IMGUI_ENABLE_FREETYPE;IMGUI_DISABLE_OBSOLETE_FUNCTIONS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(MeshLibSourceDir);$(VcpkgCurrentInstalledDir)\include\python3.10\;$(MeshLibDir)thirdparty\parallel-hashmap\;$(MeshLibDir)thirdparty\pybind11\include\;$(VcpkgCurrentInstalledDir)\include\suitesparse\;$(MeshLibDir)thirdparty\imgui\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(MeshLibSourceDir);$(PythonIncludePath);$(MeshLibDir)thirdparty\parallel-hashmap\;$(MeshLibDir)thirdparty\pybind11\include\;$(VcpkgCurrentInstalledDir)\include\suitesparse\;$(MeshLibDir)thirdparty\imgui\;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <UseStandardPreprocessor>true</UseStandardPreprocessor>
@@ -75,8 +76,6 @@
       -->
       <DisableSpecificWarnings>4061;4250;4324;4365;4371;4388;4435;4514;4582;4583;4599;4605;4623;4625;4626;4668;4710;4711;4820;4868;5026;5027;5031;5039;5045;5104;5105;5219;5243;5246;5262;5264;26451;%(DisableSpecificWarnings)</DisableSpecificWarnings>
     </ClCompile>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <Link>
       <AdditionalDependencies>$(PythonLibPath);%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
