Reapply "Save docker images for C/C++" (#12876) (#12937)

This reverts commit be6c929.

Author: jonathanmetzman

infra/build/functions/build_and_run_coverage_test.py

@@ -39,6 +39,9 @@ class TestRequestCoverageBuilds(fake_filesystem_unittest.TestCase):
   def setUp(self):
     self.maxDiff = None  # pylint: disable=invalid-name
     self.setUpPyfakefs()
+    self.patcher = mock.patch('build_lib.get_unique_build_step_image_id',
+                              return_value='UNIQUE_ID')
+    self.mock_function = self.patcher.start()
 
   @mock.patch('build_lib.get_signed_url', return_value='test_url')
   @mock.patch('build_lib.download_corpora_steps',

infra/build/functions/build_lib.py

@@ -385,6 +385,10 @@ def _make_image_name_architecture_specific(image_name, architecture):
   return f'{image_name}-{architecture.lower()}'
 
 
+def get_unique_build_step_image_id():
+  return uuid.uuid4()
+
+
 def get_docker_build_step(image_names,
                           directory,
                           use_buildkit_cache=False,
@@ -410,13 +414,14 @@ def get_docker_build_step(image_names,
   if cache_image:
     args.extend(['--build-arg', f'CACHE_IMAGE={cache_image}'])
 
-  for image_name in image_names:
+  for image_name in sorted(image_names):
     args.extend(['--tag', image_name])
 
   step = {
       'name': DOCKER_TOOL_IMAGE,
       'args': args,
       'dir': directory,
+      'id': f'build-{get_unique_build_step_image_id()}',
   }
   # Handle buildkit args
   # Note that we mutate "args" after making it a value in step.
@@ -462,10 +467,11 @@ def get_project_image_steps(  # pylint: disable=too-many-arguments
     steps.extend(get_pull_test_images_steps(config.test_image_suffix))
   src_root = 'oss-fuzz' if not experiment else '.'
 
-  docker_build_step = get_docker_build_step([image],
-                                            os.path.join('projects', name),
-                                            src_root=src_root,
-                                            cache_image=cache_image)
+  docker_build_step = get_docker_build_step(
+      [image, _get_unsafe_name(name)],
+      os.path.join('projects', name),
+      src_root=src_root,
+      cache_image=cache_image)
   steps.append(docker_build_step)
   if srcmap:
     srcmap_step_id = get_srcmap_step_id()
@@ -477,7 +483,7 @@ def get_project_image_steps(  # pylint: disable=too-many-arguments
         ],
         'env': [
             'OSSFUZZ_REVISION=$REVISION_ID',
-            'FUZZING_LANGUAGE=%s' % language,
+            f'FUZZING_LANGUAGE={language}',
         ],
         'id': srcmap_step_id
     }])
@@ -498,15 +504,30 @@ def get_project_image_steps(  # pylint: disable=too-many-arguments
             'args': ['buildx', 'use', builder_name]
         },
     ])
-    docker_build_arm_step = get_docker_build_step([image],
-                                                  os.path.join(
-                                                      'projects', name),
-                                                  architecture=_ARM64)
+    docker_build_arm_step = get_docker_build_step(
+        [image, _get_unsafe_name(name)],
+        os.path.join('projects', name),
+        architecture=_ARM64)
     steps.append(docker_build_arm_step)
 
+  if (config.build_type == 'fuzzing' and language in ('c', 'c++')):
+    # Push so that historical bugs are reproducible.
+    push_step = {
+        'name': 'gcr.io/cloud-builders/docker',
+        'args': ['push', _get_unsafe_name(name)],
+        'id': 'push-image',
+        'waitFor': [docker_build_step['id']],
+        'allowFailure': True
+    }
+    steps.append(push_step)
+
   return steps
 
 
+def _get_unsafe_name(name):
+  return f'us-central1-docker.pkg.dev/oss-fuzz/unsafe/{name}'
+
+
 def get_logs_url(build_id):
   """Returns url that displays the build logs."""
   return (

infra/build/functions/build_project.py

@@ -19,8 +19,6 @@
 Usage: build_project.py <project_dir>
 """
 
-from __future__ import print_function
-
 import argparse
 from dataclasses import dataclass
 import datetime
@@ -77,6 +75,7 @@ class Config:
   experiment: bool = False
   # TODO(ochang): This should be different per engine+sanitizer combination.
   upload_build_logs: str = None
+  build_type: str = None
 
 
 WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')
@@ -476,7 +475,6 @@ def get_build_steps_for_project(project,
           upload_steps = get_upload_steps(project, build, timestamp,
                                           config.testing)
           build_steps.extend(upload_steps)
-
   return build_steps
 
 
@@ -629,15 +627,16 @@ def get_args(description):
   return parser.parse_args()
 
 
-def create_config_from_commandline(args):
+def create_config(args, build_type):
   """Create a Config object from parsed command line |args|."""
   upload = not args.experiment
   return Config(testing=args.testing,
                 test_image_suffix=args.test_image_suffix,
                 branch=args.branch,
                 parallel=args.parallel,
                 upload=upload,
-                experiment=args.experiment)
+                experiment=args.experiment,
+                build_type=build_type)
 
 
 def build_script_main(script_description, get_build_steps_func, build_type):
@@ -650,7 +649,7 @@ def build_script_main(script_description, get_build_steps_func, build_type):
 
   credentials = oauth2client.client.GoogleCredentials.get_application_default()
   error = False
-  config = create_config_from_commandline(args)
+  config = create_config(args, build_type)
   for project_name in args.projects:
     logging.info('Getting steps for: "%s".', project_name)
     try:

infra/build/functions/test_data/expected_build_steps.json

@@ -14,9 +14,12 @@
       "build",
       "--tag",
       "gcr.io/oss-fuzz/test-project",
+      "--tag",
+      "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project",
       "."
     ],
-    "dir": "oss-fuzz/projects/test-project"
+    "dir": "oss-fuzz/projects/test-project",
+    "id": "build-UNIQUE_ID"
   },
   {
     "name": "gcr.io/oss-fuzz/test-project",
@@ -68,9 +71,12 @@
       "--load",
       "--tag",
       "gcr.io/oss-fuzz/test-project-aarch64",
+      "--tag",
+      "us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project-aarch64",
       "."
     ],
-    "dir": "oss-fuzz/projects/test-project"
+    "dir": "oss-fuzz/projects/test-project",
+    "id": "build-UNIQUE_ID"
   },
   {
     "name": "gcr.io/cloud-builders/docker",

infra/build/functions/test_data/expected_centipede_build_steps.json

@@ -13,10 +13,13 @@
     "args": [
       "build",
       "--tag",
-      "gcr.io/oss-fuzz/test-project",
+	"gcr.io/oss-fuzz/test-project",
+      "--tag",
+	"us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project",
       "."
     ],
-    "dir": "oss-fuzz/projects/test-project"
+    "dir": "oss-fuzz/projects/test-project",
+    "id": "build-UNIQUE_ID"
   },
   {
     "name": "gcr.io/oss-fuzz/test-project",

infra/build/functions/test_data/expected_coverage_build_steps.json

@@ -14,9 +14,12 @@
       "build",
       "--tag",
       "gcr.io/oss-fuzz/test-project",
+      "--tag",
+	"us-central1-docker.pkg.dev/oss-fuzz/unsafe/test-project",
       "."
     ],
-    "dir": "oss-fuzz/projects/test-project"
+    "dir": "oss-fuzz/projects/test-project",
+    "id": "build-UNIQUE_ID"
   },
   {
     "name": "gcr.io/oss-fuzz/test-project",

infra/build/functions/test_data/expected_trial_build_steps.json

@@ -167,9 +167,12 @@
       "build",
       "--tag",
       "gcr.io/oss-fuzz/skcms",
+      "--tag",
+      "us-central1-docker.pkg.dev/oss-fuzz/unsafe/skcms",
       "."
     ],
-    "dir": "oss-fuzz/projects/skcms"
+    "dir": "oss-fuzz/projects/skcms",
+    "id": "build-UNIQUE_ID"
   },
   {
     "name": "gcr.io/oss-fuzz/skcms",