tools: add gpg verification using current_ver.dep

Author: islandryu

configure.py

@@ -1771,12 +1771,6 @@ def icu_download(path):
     attemptdownload = nodedownload.candownload(auto_downloads, "icu")
     for icu in icus:
       url = icu['url']
-      (expectHash, hashAlgo, allAlgos) = nodedownload.findHash(icu)
-      if not expectHash:
-        error(f'''Could not find a hash to verify ICU download.
-          {depFile} may be incorrect.
-          For the entry {url},
-          Expected one of these keys: {' '.join(allAlgos)}''')
       local = url.split('/')[-1]
       targetfile = Path(options.download_path, local)
       if not targetfile.is_file():
@@ -1785,14 +1779,28 @@ def icu_download(path):
       else:
         print(f'Re-using existing {targetfile}')
       if targetfile.is_file():
-        print(f'Checking file integrity with {hashAlgo}:\r')
-        gotHash = nodedownload.checkHash(targetfile, hashAlgo)
-        print(f'{hashAlgo}:      {gotHash}  {targetfile}')
-        if expectHash == gotHash:
-          return targetfile
-
-        warn(f'Expected: {expectHash}      *MISMATCH*')
-        warn(f'\n ** Corrupted ZIP? Delete {targetfile} to retry download.\n')
+        if "gpg" in icu:
+          key_url = icu['gpg']["key"]
+          sig_url = icu['gpg']["asc"]
+          try:
+            nodedownload.checkGPG(targetfile, key_url, sig_url)
+            return targetfile
+          except Exception as e:
+            warn(e)
+        else:
+          (expectHash, hashAlgo, allAlgos) = nodedownload.findHash(icu)
+          if not expectHash:
+            error(f'''Could not find a hash to verify ICU download.
+              {depFile} may be incorrect.
+              For the entry {url},
+              Expected one of these keys: {' '.join(allAlgos)}''')
+          print(f'Checking file integrity with {hashAlgo}:\r')
+          gotHash = nodedownload.checkHash(targetfile, hashAlgo)
+          print(f'{hashAlgo}:      {gotHash}  {targetfile}')
+          if expectHash == gotHash:
+            return targetfile
+          warn(f'Expected: {expectHash}      *MISMATCH*')
+          warn(f'\n ** Corrupted ZIP? Delete {targetfile} to retry download.\n')
     return None
   icu_config = {
     'variables': {}

tools/configure.d/nodedownload.py

@@ -7,10 +7,13 @@
 import zipfile
 import tarfile
 import contextlib
+import subprocess
+import tempfile
 try:
-    from urllib.request import FancyURLopener, URLopener
+    from urllib.request import FancyURLopener, URLopener, urlopen
 except ImportError:
-    from urllib import FancyURLopener, URLopener
+    from urllib import FancyURLopener, URLopener, urlopen
+import os
 
 def formatSize(amt):
     """Format a size as a string in MB"""
@@ -68,6 +71,40 @@ def checkHash(targetfile, hashAlgo):
         chunk = f.read(1024)
     return digest.hexdigest()
 
+def checkGPG(targetfile, key_url, sig_url):
+    key_data = download_file(key_url)
+    sig_data = download_file(sig_url)
+    with tempfile.NamedTemporaryFile(delete=False) as f:
+        f.write(key_data)
+        key_file = f.name
+
+    with tempfile.NamedTemporaryFile(delete=False) as f:
+        f.write(sig_data)
+        sig_file = f.name
+    try:
+        cmd = ["gpg", "--import", key_file]
+        subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    except subprocess.CalledProcessError as e:
+        os.remove(key_file)
+        os.remove(sig_file)
+        raise Exception("Failed to import key. Check key url. \n%s" % e.stderr.decode("utf-8"))
+
+    try:
+        cmd = ["gpg", "--verify", sig_file, targetfile]
+        subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        print("Signature is valid")
+        os.remove(key_file)
+        os.remove(sig_file)
+        return True
+    except subprocess.CalledProcessError as e:
+        os.remove(key_file)
+        os.remove(sig_file)
+        raise Exception("Failed to verify signature. Check target file is valid. \n%s" % e.stderr.decode("utf-8"))
+
+def download_file(url):
+    with urlopen(url) as response:
+        return response.read()
+
 def unpack(packedfile, parent_path):
     """Unpacks packedfile into parent_path. Assumes .zip. Returns parent_path"""
     if zipfile.is_zipfile(packedfile):

tools/dep_updaters/update-icu.sh

@@ -57,6 +57,7 @@ if [ -n "$CHECKSUM" ]; then
     echo "Skipped because checksums do not match."
     exit 0
   fi
+  perl -i -pe "s|\"(md5\|gpg)\": .*|\"md5\": \"$CHECKSUM\"|" "$TOOLS_DIR/icu/current_ver.dep"
 else
   echo "Checksum not found"
   echo "check with gpg"
@@ -67,10 +68,11 @@ else
   if gpg --verify signature.asc data.tgz; then
     echo "Signature verified"
     rm data.tgz signature.asc KEYS
+    perl -i -pe "s|\"(gpg\|md5)\": .*|\"gpg\": { \"key\": \"$KEY_URL\", \"asc\": \"$NEW_VERSION_TGZ_ASC_URL\" }|" "$TOOLS_DIR/icu/current_ver.dep"
   else
     echo "Skipped because signature verification failed."
     rm data.tgz signature.asc KEYS
-    exit 1
+    exit 0
   fi
 fi
 
@@ -82,7 +84,6 @@ rm -rf "$DEPS_DIR/icu"
 
 perl -i -pe "s|\"url\": .*|\"url\": \"$NEW_VERSION_TGZ_URL\",|" "$TOOLS_DIR/icu/current_ver.dep"
 
-perl -i -pe "s|\"md5\": .*|\"md5\": \"$CHECKSUM\"|" "$TOOLS_DIR/icu/current_ver.dep"
 
 rm -rf out "$DEPS_DIR/icu" "$DEPS_DIR/icu4c*"