Diff Package Versions (#121)

Add build step to compare r package versions.

b/157548958

Author: Philmod

Jenkinsfile

@@ -75,6 +75,32 @@ pipeline {
       }
     }
 
+    stage('Package Versions') {
+      parallel {
+        stage('CPU Diff') {
+          steps {
+            sh '''#!/bin/bash
+            set -exo pipefail
+
+            docker pull gcr.io/kaggle-images/rstats:${PRETEST_TAG}
+            ./diff --target gcr.io/kaggle-images/rstats:${PRETEST_TAG}
+          '''
+          }
+        }
+        stage('GPU Diff') {
+          agent { label 'ephemeral-linux-gpu' }
+          steps {
+            sh '''#!/bin/bash
+            set -exo pipefail
+            
+            docker pull gcr.io/kaggle-private-byod/rstats:${PRETEST_TAG}
+            ./diff --gpu --target gcr.io/kaggle-private-byod/rstats:${PRETEST_TAG}
+          '''
+          }
+        }
+      }
+    }
+
     stage('Label CPU/GPU Staging Images') {
       steps {
         sh '''#!/bin/bash

diff

@@ -0,0 +1,88 @@
+#!/bin/bash
+set -e
+
+usage() {
+cat << EOF
+Usage: $0 [OPTIONS]
+Compare a given Docker image package versions against the prod image.
+
+Options:
+    -g, --gpu       Compare GPU images.
+    -b, --base      The base image to diff against.
+    -t, --target    The image to diff against the base image.
+                    Default is the locally built image.
+EOF
+}
+
+
+BASE_IMAGE_TAG='gcr.io/kaggle-images/rstats:latest'
+BASE_IMAGE_TAG_OVERRIDE=''
+TARGET_IMAGE_TAG='kaggle/rstats-build'
+TARGET_IMAGE_TAG_OVERRIDE=''
+
+while :; do
+    case "$1" in 
+        -h|--help)
+            usage
+            exit
+            ;;
+        -g|--gpu)
+            BASE_IMAGE_TAG='gcr.io/kaggle-private-byod/rstats:latest'
+            TARGET_IMAGE_TAG='kaggle/rstats-gpu-build'
+            ;;
+        -b|--base)
+            if [[ -z "$2" ]]; then
+                usage
+                printf 'ERROR: No IMAGE specified after the %s flag.\n' "$1" >&2
+                exit
+            fi
+            BASE_IMAGE_TAG_OVERRIDE="$2"
+            shift # skip the flag value
+            ;;
+        -t|--target)
+            if [[ -z "$2" ]]; then
+                usage
+                printf 'ERROR: No IMAGE specified after the %s flag.\n' "$1" >&2
+                exit
+            fi
+            TARGET_IMAGE_TAG_OVERRIDE="$2"
+            shift # skip the flag value
+            ;;
+        -?*)
+            usage
+            printf 'ERROR: Unknown option: %s\n' "$1" >&2
+            exit
+            ;;
+        *)            
+            break
+    esac
+
+    shift
+done
+
+if [[ -n "$BASE_IMAGE_TAG_OVERRIDE" ]]; then
+    BASE_IMAGE_TAG="$BASE_IMAGE_TAG_OVERRIDE"
+fi
+
+if [[ -n "$TARGET_IMAGE_TAG_OVERRIDE" ]]; then
+    TARGET_IMAGE_TAG="$TARGET_IMAGE_TAG_OVERRIDE"
+fi
+
+readonly BASE_IMAGE_TAG
+readonly TARGET_IMAGE_TAG
+
+echo "Base: $BASE_IMAGE_TAG"
+echo "Target: $TARGET_IMAGE_TAG"
+
+if [[ "$BASE_IMAGE_TAG" == "gcr.io/"* ]]; then
+    docker pull "$BASE_IMAGE_TAG"
+fi
+
+CMDS=('Rscript /tools/r_list_versions.R | sort')
+for cmd in "${CMDS[@]}"; do
+    echo "== Comparing $cmd =="
+    diff --suppress-common-lines --side-by-side \
+        <(docker run -v $PWD/tools:/tools --rm "$BASE_IMAGE_TAG" /bin/bash -c "$cmd") \
+        <(docker run -v $PWD/tools:/tools --rm "$TARGET_IMAGE_TAG" /bin/bash -c "$cmd") \
+        && echo 'No diff' || true
+done

tools/r_list_versions.R

@@ -0,0 +1,3 @@
+ip <- as.data.frame(installed.packages()[,c(1,3:4)])
+ip <- ip[is.na(ip$Priority),1:2,drop=FALSE]
+write.table(ip, quote=FALSE, sep="==", row.names=FALSE, col.names=FALSE)