From f38f2b4733b695e7cc4b4807a3e4ce69a89b561f Mon Sep 17 00:00:00 2001 From: Mauricio Esguerra Date: Wed, 15 Jul 2020 10:57:58 +0200 Subject: [PATCH 1/5] UPDATE: Changes to use the latest LTS for Ubuntu Focal/Fossa Also some changes to make it possible to deply to Fedora28 --- python/manifests/init.pp | 234 ++++++++++++++++++++++++--------------- 1 file changed, 147 insertions(+), 87 deletions(-) diff --git a/python/manifests/init.pp b/python/manifests/init.pp index dc9e528..fd5dc3c 100644 --- a/python/manifests/init.pp +++ b/python/manifests/init.pp @@ -1,105 +1,165 @@ # install packages inside the virtualenv with pip define puppet::install::pip ($pip_package = $title) { - exec { "install-$pip_package": - command => "/env/bin/pip3 install $pip_package", - timeout => 1800, - require => [Package["postgresql", "postgresql-contrib"], Exec["create-virtualenv"]] - } + exec { "install-$pip_package": + command => "/env/bin/pip3 install $pip_package", + timeout => 1800, + require => [Package["postgresql", "postgresql-contrib"], Exec["create-virtualenv"]] + } } class python { - $packages = $operatingsystem ? { - "Ubuntu" => [ - "python3.7", - "python3.7-dev", - "python3.7-venv", - "python3-pip", - # for python2, will be removed - "python-biopython", - "python-openbabel", - "python-rdkit", - "python-yaml", - "libpq-dev", - ], - "CentOS" => [ - "python37", - "python37-devel", - # for python2, will be removed - "python-biopython", - "python-openbabel", - #"python-rdkit", - "PyYAML", - ], - } + $packages = $operatingsystem ? { + "Ubuntu" => [ + "python3", + "python3-dev", + "python3-venv", + "python3-pip", + # for python2, will be removed + "python3-biopython", + "python3-openbabel", + "python3-rdkit", + "python3-yaml", + "libpq-dev", + ], + "CentOS" => [ + "python37", + "python37-devel", + # for python2, will be removed + "python-biopython", + "python-openbabel", + #"python-rdkit", + "PyYAML", + ], + "Fedora" => [ + "python3", + "python3-devel", + "python3-virtualenv", + "python3-pip", + # for python2, will be removed + "python3-biopython", + "python3-openbabel", + "python3-rdkit", + "python3-pyyaml", + "libpq-devel", + ], + } - # install packages - package { $packages: - ensure => present, - require => Exec["update-package-repo"] - } + # install packages + package { $packages: + ensure => present, + require => Exec["update-package-repo"] + } - # create a python3 symlink, because the names of the executable differ between OSes - file { "/usr/local/bin/python3": - ensure => "link", - target => "/usr/bin/python3.7", - require => $operatingsystem ? { - "CentOS" => Package["python37"], - "Ubuntu" => Package["python3.7"], - } + # create a python3 symlink, because the names of the executable differ between OSes + file { "/usr/local/bin/python3": + ensure => "link", + target => "/usr/bin/python3", + require => $operatingsystem ? { + "CentOS" => Package["python3"], + "Ubuntu" => Package["python3"], + "Fedora" => Package["python3"], } + } - # install pip - # exec { "install-pip": - # cwd => "/tmp", - # command => $operatingsystem ? { - # "CentOS" => "wget https://bootstrap.pypa.io/get-pip.py;python3 get-pip.py", - # "Ubuntu" => "apt install -y python3-pip", - # }, - #} + # install pip + # exec { "install-pip": + # cwd => "/tmp", + # command => $operatingsystem ? { + # "CentOS" => "wget https://bootstrap.pypa.io/get-pip.py;python3 get-pip.py", + # "Ubuntu" => "apt install -y python3-pip", + # }, + #} - # create virtualenv - exec { "create-virtualenv": - command => "python3.7 -m venv /env", - } + # create virtualenv + exec { "create-virtualenv": + command => "python3 -m venv /env", + } - $pip_packages = ["ipython", "django", "django-debug-toolbar", "psycopg2-binary", "biopython", "xlrd", "numpy", "PyYAML", - "djangorestframework", "django-rest-swagger", "XlsxWriter", "sphinx", "openpyxl", "xmltodict", "pandas", - "django-polymorphic", "mmtf-python", "scipy", "sklearn", "freesasa", "lxml", "reportlab", "svglib"] - puppet::install::pip { $pip_packages: } + $pip_packages = [ + "ipython", + "django", + "django-debug-toolbar", + "psycopg2-binary", + "biopython", + "xlrd", + "numpy", + "PyYAML", + "djangorestframework", + "django-rest-swagger", + "XlsxWriter", + "sphinx", + "openpyxl", + "xmltodict", + "pandas", + "django-polymorphic", + "mmtf-python", + "scipy", + "sklearn", + "freesasa", + "lxml", + "reportlab", + "svglib", + "jq" + ] - # download and install dssp - exec { "download-dssp": - command => "/usr/bin/wget -q ftp://ftp.cmbi.ru.nl/pub/software/dssp/dssp-2.0.4-linux-amd64 -O /env/bin/dssp", - creates => "/env/bin/dssp", - require => Exec["create-virtualenv"], - } + puppet::install::pip { $pip_packages: } - file { "/env/bin/dssp": - mode => "0755", - require => Exec["download-dssp"], - } + # download and install dssp + exec { "download-dssp": + command => "/usr/bin/wget -q ftp://ftp.cmbi.ru.nl/pub/software/dssp/dssp-2.0.4-linux-amd64 -O /env/bin/dssp", + creates => "/env/bin/dssp", + require => Exec["create-virtualenv"], + } - # download and install MODELLER - exec { "download-modeller": - command => "/usr/bin/wget -q https://salilab.org/modeller/9.23/modeller_9.23-1_amd64.deb -O /env/lib/mod9.23-1_install.deb", - creates => "/env/lib/mod9.23-1_install.deb", - require => Exec["create-virtualenv"], - } - exec { "install-modeller": - command => "sudo env KEY_MODELLER=MODELIRANJE dpkg -i /env/lib/mod9.23-1_install.deb", - require => Exec["download-modeller"], - } - exec { "move-modeller": - command => "sudo mv /usr/lib/python3.7/dist-packages/modeller /env/lib/python3.7/site-packages/", - creates => "/env/lib/python3.7/site-packages/modeller", - require => Exec["install-modeller"], - } - exec { "move_modeller.so": - command => "sudo mv /usr/lib/python3.7/dist-packages/_modeller.so /env/lib/python3.7/site-packages/", - creates => "/env/lib/python3.7/site-packages/_modeller.so", - require => Exec["install-modeller"], - } + file { "/env/bin/dssp": + mode => "0755", + require => Exec["download-dssp"], + } + + # download and install MODELLER for Ubuntu + exec { "download-modeller": + command => "curl https://salilab.org/modeller/9.24/modeller_9.24-1_amd64.deb -o /tmp/modeller_9.24-1_amd64.deb", + creates => "/tmp/modeller_9.24-1_amd64.deb", + require => Exec["create-virtualenv"], + } + exec { "install-modeller": + command => "sudo env KEY_MODELLER=MODELIRANJE dpkg -i /tmp/modeller_9.24-1_amd64.deb", + require => Exec["download-modeller"], + } + exec { "move-modeller": + command => "sudo mv /usr/lib/modeller9.24/modlib/modeller /env/lib/python3.8/site-packages/", + creates => "/env/lib/python3.8/site-packages/modeller", + require => Exec["install-modeller"], + } + exec { "move_modeller.so": + command => "sudo mv /usr/lib/modeller9.24/lib/x86_64-intel8/python3.3/_modeller.so /env/lib/python3.8/site-packages/", + creates => "/env/lib/python3.8/site-packages/_modeller.so", + require => Exec["install-modeller"], + } } + +# # download and install MODELLER for Fedora/RedHat +# exec { "download-modeller": +# command => +# "curl https://salilab.org/modeller/9.24/modeller-9.24-1.x86_64.rpm -o /tmp/modeller-9.24-1.x86_64.rpm", +# creates => "/tmp/modeller-9.24-1.x86_64.rpm", +# require => Exec["create-virtualenv"], +# } +# exec { "install-modeller": +# command => "sudo env KEY_MODELLER=MODELIRANJE rpm -Uvh /tmp/modeller-9.24-1.x86_64.rpm", +# require => Exec["download-modeller"], +# } +# exec { "move-modeller": +# command => "sudo mv /usr/lib/modeller9.24/modlib/modeller /env/lib/python3.8/site-packages/", +# creates => "/env/lib/python3.8/site-packages/modeller", +# require => Exec["install-modeller"], +# } +# exec { "move_modeller.so": +# command => "sudo mv /usr/lib/modeller9.24/lib/x86_64-intel8/python3.3/_modeller.so /env/lib/python3.8/site-packages/", +# creates => "/env/lib/python3.8/site-packages/_modeller.so", +# require => Exec["install-modeller"], +# } +# } From def4228f1922e663a392b21341282ff29f33846c Mon Sep 17 00:00:00 2001 From: Mauricio Esguerra Date: Mon, 27 Jul 2020 10:54:07 +0200 Subject: [PATCH 2/5] UPDATE: Have django-extensions as an installed app. --- python/manifests/init.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/python/manifests/init.pp b/python/manifests/init.pp index fd5dc3c..19049a6 100644 --- a/python/manifests/init.pp +++ b/python/manifests/init.pp @@ -82,6 +82,7 @@ "ipython", "django", "django-debug-toolbar", + "django-extensions", "psycopg2-binary", "biopython", "xlrd", From 54c57b07fc237c9677bc79de6b983a5f3e2c1dd4 Mon Sep 17 00:00:00 2001 From: Mauricio Esguerra Date: Tue, 11 Aug 2020 10:29:15 +0200 Subject: [PATCH 3/5] UPDATE: Try to include added python package chembl_webresource_client --- python/manifests/init.pp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/manifests/init.pp b/python/manifests/init.pp index 19049a6..370971f 100644 --- a/python/manifests/init.pp +++ b/python/manifests/init.pp @@ -103,7 +103,8 @@ "lxml", "reportlab", "svglib", - "jq" + "jq", + "chembl_webresource_client" ] puppet::install::pip { $pip_packages: } From 2e32a965033e44eb3102a2546c3bf332d0cb9d76 Mon Sep 17 00:00:00 2001 From: Mauricio Esguerra Date: Wed, 12 Aug 2020 15:07:52 +0200 Subject: [PATCH 4/5] Add phylip to tools --- tools/manifests/init.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/manifests/init.pp b/tools/manifests/init.pp index 6e89f72..6b70b1c 100644 --- a/tools/manifests/init.pp +++ b/tools/manifests/init.pp @@ -10,6 +10,7 @@ "tig", "clustalo", "ncbi-blast+", + "phylip", ], "CentOS" => [ "expect", From 3ae7b8ac99b2665eab7ef247ac7aa4872f9d8c83 Mon Sep 17 00:00:00 2001 From: Mauricio Esguerra Date: Wed, 2 Sep 2020 09:29:13 +0200 Subject: [PATCH 5/5] UPDATE: Add gunicorn google-api-python-client oauth2client --- python/manifests/init.pp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/python/manifests/init.pp b/python/manifests/init.pp index 370971f..99de91f 100644 --- a/python/manifests/init.pp +++ b/python/manifests/init.pp @@ -1,6 +1,8 @@ # install packages inside the virtualenv with pip -define puppet::install::pip ($pip_package = $title) { +define puppet::install::pip ( + $pip_package = $title +) { exec { "install-$pip_package": command => "/env/bin/pip3 install $pip_package", timeout => 1800, @@ -48,7 +50,7 @@ # install packages package { $packages: - ensure => present, + ensure => present, require => Exec["update-package-repo"] } @@ -82,7 +84,7 @@ "ipython", "django", "django-debug-toolbar", - "django-extensions", + "django-extensions", "psycopg2-binary", "biopython", "xlrd", @@ -104,7 +106,10 @@ "reportlab", "svglib", "jq", - "chembl_webresource_client" + "chembl_webresource_client", + "google-api-python-client", + "oauth2client", + "gunicorn" ] puppet::install::pip { $pip_packages: } @@ -117,7 +122,7 @@ } file { "/env/bin/dssp": - mode => "0755", + mode => "0755", require => Exec["download-dssp"], }