Adjust Vagrant portability testing for automated build

util/devel/test/portability/provision-scripts/chapel-default.sh

@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
 
+PROVISION_SCRIPTS_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+
 if [ ! -d chapel ]
 then
-  ../../provision-scripts/chapel-update.sh
+  $PROVISION_SCRIPTS_DIR/chapel-update.sh
 fi
 
-source ../../provision-scripts/chapel-setmakej.sh
+source $PROVISION_SCRIPTS_DIR/chapel-setmakej.sh
 
-echo "cd chapel && source util/setchplenv.bash && make $MAKEJ && make check"
-cd chapel && source util/setchplenv.bash && make $MAKEJ && make check
+echo "cd chapel && source util/setchplenv.bash && make $MAKEJ && make $MAKEJ check"
+cd chapel && source util/setchplenv.bash && make $MAKEJ && make $MAKEJ check

util/devel/test/portability/provision-scripts/chapel-quickstart.sh

@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
 
+PROVISION_SCRIPTS_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+
 if [ ! -d chapel ]
 then
-  ../../provision-scripts/chapel-update.sh
+  $PROVISION_SCRIPTS_DIR/chapel-update.sh
 fi
 
-source ../../provision-scripts/chapel-setmakej.sh
+source $PROVISION_SCRIPTS_DIR/chapel-setmakej.sh
 
-echo "cd chapel && source util/quickstart/setchplenv.bash && make $MAKEJ && make check"
-cd chapel && source util/quickstart/setchplenv.bash && make $MAKEJ && make check
+echo "cd chapel && source util/quickstart/setchplenv.bash && make $MAKEJ && make $MAKEJ check"
+cd chapel && source util/quickstart/setchplenv.bash && make $MAKEJ && make $MAKEJ check

util/devel/test/portability/provision-scripts/freebsd-repo-fix.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# Adjustment needed to reliably work on network restricted test machines
+
+mkdir -p /usr/local/etc/pkg/repos
+echo 'FreeBSD: { url: "https://pkg.FreeBSD.org/${ABI}/quarterly", mirror_type: "none" }' > /usr/local/etc/pkg/repos/FreeBSD.conf

util/devel/test/portability/provision-scripts/git-clone-chapel.sh

@@ -4,5 +4,5 @@ if [ -d chapel ]
 then
   echo WARNING: chapel directory already exists, not cloning
 else
-  git clone --reference-if-able "${REPO_CACHE_PATH:-/missing}/chapel.git" --depth 1 https://github.com/chapel-lang/chapel
+  git clone --reference-if-able "${REPO_CACHE_PATH:-/missing}/chapel.git" --depth 1 --branch=vagrant-automation https://github.com/arifthpe/chapel
 fi

util/devel/test/portability/provision-scripts/proxy-setup.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+#
+# Persists http_proxy and https_proxy variables in the VM environment if provided.
+
+if [ -z "$http_proxy" ] || [ -z "$https_proxy" ]; then
+    echo "http_proxy or https_proxy was unset, not setting these variables in the VM"
+    exit 0
+fi;
+
+cat << EOF >> /etc/profile
+export http_proxy=$http_proxy
+export https_proxy=$https_proxy
+export HTTP_PROXY=$http_proxy
+export HTTPS_PROXY=$https_proxy
+EOF
+
+cat << EOF >> /etc/csh.cshrc
+export http_proxy=$http_proxy
+export https_proxy=$https_proxy
+export HTTP_PROXY=$http_proxy
+export HTTPS_PROXY=$https_proxy
+EOF
+
+# Hack to work on any sudoers file location
+cat << EOF | EDITOR='tee -a' visudo
+Defaults env_keep += "http_proxy"
+Defaults env_keep += "https_proxy"
+Defaults env_keep += "HTTP_PROXY"
+Defaults env_keep += "HTTPS_PROXY"
+EOF
+
+cat << EOF >> /usr/local/etc/pkg.conf
+pkg_env: {
+    http_proxy: "$http_proxy",
+    https_proxy: "$https_proxy",
+}
+EOF

util/devel/test/portability/vagrant/chapeldefaultcheck.sh

@@ -3,4 +3,4 @@
 # Check if default (util/setchplenv) Chapel build passes make check
 # Prints summary at the end
 
-./tryit.sh 'bash -c '\''cd chapel && source util/setchplenv.bash && export GMAKE=`which gmake` && export MAKE=${GMAKE:-make} && $MAKE && $MAKE check'\'
+./tryit.sh 'bash -c '\''cd chapel && source util/setchplenv.bash && export GMAKE=`which gmake` && export MAKE=${GMAKE:-make} && $MAKE $MAKEJ && $MAKE $MAKEJ check'\'

util/devel/test/portability/vagrant/chapelshowcommit.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Show the current commit of each VM's chapel git repository
+# Show the current commit of VM's chapel git repository
 
 ./tryit.sh "cd chapel && GIT_PAGER=cat git log --oneline -n 1 && GIT_PAGER=cat git diff"
 

util/devel/test/portability/vagrant/chapelupdate.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Update the Chapel git repository in each VM to main
+# Update the Chapel git repository in VM to main
 
 ./tryit.sh "cd chapel && git fetch origin --depth=1 && git checkout origin/main && GIT_PAGER=cat git log --oneline -n 1 && GIT_PAGER=cat git diff"
 

util/devel/test/portability/vagrant/clobber.sh

@@ -1,5 +1,3 @@
 #!/bin/bash
 
-# make clobber in each VM's chapel directory
-
 ./tryit.sh "cd chapel && export GMAKE=`which gmake` && export MAKE=${GMAKE:-make} && $MAKE clobber"

util/devel/test/portability/vagrant/current/bento-freebsd-13.5/Vagrantfile

@@ -13,6 +13,14 @@ Vagrant.configure("2") do |config|
   # export VAGRANT_EXPERIMENTAL="disks"
   config.vm.disk :disk, size: "12GB", primary: true
 
+  config.vm.provision "shell" do |s|
+    s.env =  { http_proxy:ENV['http_proxy'], https_proxy:ENV['https_proxy'] }
+    s.path =  "../../provision-scripts/proxy-setup.sh"
+  end
+
+  config.vm.provision "shell",
+    path: "../../provision-scripts/freebsd-repo-fix.sh"
+
   config.vm.provision "shell",
     path: "../../provision-scripts/freebsd-pkg-deps.sh"
 
@@ -27,7 +35,7 @@ Vagrant.configure("2") do |config|
 
   config.vm.provider "virtualbox" do |vb|
     vb.memory = 4096
-    # vb.cpus = 2
+    vb.cpus = 4
   end
 
 end

util/devel/test/portability/vagrant/current/bento-freebsd-14.2/Vagrantfile

@@ -13,6 +13,14 @@ Vagrant.configure("2") do |config|
   # export VAGRANT_EXPERIMENTAL="disks"
   config.vm.disk :disk, size: "12GB", primary: true
 
+  config.vm.provision "shell" do |s|
+    s.env =  { http_proxy:ENV['http_proxy'], https_proxy:ENV['https_proxy'] }
+    s.path =  "../../provision-scripts/proxy-setup.sh"
+  end
+
+  config.vm.provision "shell",
+    path: "../../provision-scripts/freebsd-repo-fix.sh"
+
   config.vm.provision "shell",
     path: "../../provision-scripts/freebsd-pkg-deps.sh"
 
@@ -27,7 +35,7 @@ Vagrant.configure("2") do |config|
 
   config.vm.provider "virtualbox" do |vb|
     vb.memory = 4096
-    # vb.cpus = 2
+    vb.cpus = 4
   end
 
 end

util/devel/test/portability/vagrant/current/generic-x32-debian12/Vagrantfile

@@ -10,6 +10,17 @@ Vagrant.configure("2") do |config|
   config.vm.box_architecture = "i386"
   config.vm.box_version = "4.3.12"
 
+  config.vm.provision "shell" do |s|
+    s.env =  { http_proxy:ENV['http_proxy'], https_proxy:ENV['https_proxy'] }
+    s.path =  "../../provision-scripts/proxy-setup.sh"
+  end
+
+  # Hack to get this box to grab DNS nameservers from DHCP, rather than using
+  # the public DNS servers it for some reason has hardcoded.
+  config.vm.provision "shell",
+    inline: "sed -i 's/dns-nameserver/#dns-nameserver/' /etc/network/interfaces ;\
+             systemctl restart networking"
+
   config.vm.provision "shell",
     path: "../../provision-scripts/apt-get-deps.sh"
 
@@ -25,7 +36,7 @@ Vagrant.configure("2") do |config|
 
   config.vm.provider "virtualbox" do |vb|
     vb.memory = 4096
-    # vb.cpus = 2
+    vb.cpus = 4
   end
 
 end

util/devel/test/portability/vagrant/current/ubuntu-jammy64-homebrew/Vagrantfile

@@ -8,6 +8,11 @@
 Vagrant.configure("2") do |config|
   config.vm.box = "ubuntu/jammy64"
 
+  config.vm.provision "shell" do |s|
+    s.env =  { http_proxy:ENV['http_proxy'], https_proxy:ENV['https_proxy'] }
+    s.path =  "../../provision-scripts/proxy-setup.sh"
+  end
+
   config.vm.provision "shell", privileged: false,
     path: "../../provision-scripts/apt-get-and-linuxbrew.sh"
 
@@ -19,7 +24,7 @@ Vagrant.configure("2") do |config|
 
   config.vm.provider "virtualbox" do |vb|
     vb.memory = 4096
-    # vb.cpus = 2
+    vb.cpus = 4
   end
 
 end

util/devel/test/portability/vagrant/destroy.sh

@@ -4,10 +4,23 @@
 # Removes space occupied by each VM
 # Don't do this if you have work you want to save in the VMs!
 
+if [ -z "$VM_NAME" ]
+then
+  echo "Please set the VM_NAME environment variable to the name of the VM you want to run this on."
+  exit 1
+fi
+
 for name in current/*
 do
   if [ -f $name/Vagrantfile ]
   then
+    if [ "$VM_NAME" != "all" ]
+    then
+      if [ "$name" != "current/$VM_NAME" ]
+      then
+        continue
+      fi
+    fi
     cd $name
     echo "------------ $name ---- vagrant destroy"
     vagrant destroy -f

util/devel/test/portability/vagrant/gitclean.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-# git clean in each VM's chapel directory
+# git clean in VM's chapel directory
 
 ./tryit.sh "cd chapel && GIT_PAGER=cat git clean -fdx"

util/devel/test/portability/vagrant/halt-all.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Stop any running VMs
+
+DIR=`pwd`
+
+for name in current/*
+do
+  if [ -f $name/Vagrantfile ]
+  then
+    cd $name
+    echo "------------ $name ---- vagrant halt"
+    vagrant halt
+    cd "$DIR"
+  fi
+done

util/devel/test/portability/vagrant/interact-all.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Log in to each VM in turn to run commands manually
+# Assumes VMs are up and leaves them running
+
+for name in current/*
+do
+  if [ -f $name/Vagrantfile ]
+  then
+    cd $name
+    echo "------------ $name ---- vagrant ssh"
+    #vagrant up
+    vagrant ssh
+    #vagrant halt
+    cd ..
+  fi
+done
+

util/devel/test/portability/vagrant/rebuild-all.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Rebuild all of the VMs. This will delete all data on the VMs.
+# Don't do that if you have data you want to keep in any VM!
+
+# enable resizing disks to get
+#  config.vm.disk to work
+#  (to resize the disk for some VMs that come too small)
+export VAGRANT_EXPERIMENTAL="disks"
+
+DIR=`pwd`
+
+# Get new versions of Vagrant boxes
+for name in current/*
+do
+  if [ -f $name/Vagrantfile ]
+  then
+    cd $name
+    echo "------------ $name ---- vagrant box update"
+    # update the box
+    vagrant box update
+    # destroy the current VM
+    # (in case we need to make a new one with a new box)
+    vagrant destroy -f
+    # recreate the VM and install dependencies etc
+    vagrant up
+    # stop the VM
+    vagrant halt
+    cd "$DIR"
+  fi
+done
+
+# Remove any old (now unused) vagrant boxes
+vagrant box prune

util/devel/test/portability/vagrant/tryit.sh

@@ -1,10 +1,19 @@
 #!/bin/bash
 
-# Runs a command (or series of commands) on all the VMs.
-# Launches each VM, runs the command, and then shuts it down.
+# Runs a command (or series of commands) on a VM.
+# Launches the VM, runs the command, and then shuts it down.
 # Prints a summary of the output at the end.
 # Saves all the command output in the file called log.
 
+# Expects the VM name to be given as an environment variable. The special name
+# "all" can be used to run the command on all VMs.
+
+if [ -z "$VM_NAME" ]
+then
+  echo "Please set the VM_NAME environment variable to the name of the VM you want to run this on."
+  exit 1
+fi
+
 LOG=log
 
 declare -a NAME
@@ -35,10 +44,19 @@ do
   fi
 done
 
+exit_status=0
 for name in current/*
 do
   if [ -f $name/Vagrantfile ]
   then
+    # skip if this isn't a desired VM to run on
+    if [ "$VM_NAME" != "all" ]
+    then
+      if [ "$name" != "current/$VM_NAME" ]
+      then
+        continue
+      fi
+    fi
     cd $name
     display=`printf "%${maxlen}s" $name`
     NAME[$i]=$display
@@ -64,6 +82,7 @@ do
       echo "     " $*
       echo
       echo
+      exit_status=1
       #uncomment the below line to make errors fatal
       #exit 1
       ;;
@@ -88,3 +107,5 @@ do
     ((i++))
   fi
 done
+
+exit $exit_status

util/devel/test/portability/vagrant/up-all.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Bring each VM up and leave it running.
+
+for name in current/*
+do
+  if [ -f $name/Vagrantfile ]
+  then
+    cd $name
+    echo "------------ $name ---- vagrant up"
+    vagrant up
+    cd ../..
+  fi
+done
+