Add internal_index_image_copy and internal_index_image_copy_resolved attributes

Added attributes to response of Add and Rm requests. Their values will be the
pullspec pointing to the default IIB output repo. The values will be same as
that of index_image and index_image_resolved respectively when overwrite_from_index
is NOT provided when submitting the request.

Refers to CLOUDDST-13288

Author: yashvardhannanavati

iib/web/api_v1.py

@@ -554,6 +554,8 @@ def patch_request(request_id):
         'from_index_resolved',
         'index_image',
         'index_image_resolved',
+        'internal_index_image_copy',
+        'internal_index_image_copy_resolved',
         'source_from_index_resolved',
         'target_index_resolved',
     )

iib/web/migrations/versions/3283f52e7329_add_internal_index_image_copy_to_add_and_rm.py

@@ -0,0 +1,83 @@
+"""Add internal_index_image_copy to Add and Rm.
+
+Revision ID: 3283f52e7329
+Revises: 5188702409d9
+Create Date: 2022-06-10 01:41:18.583209
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3283f52e7329'
+down_revision = '5188702409d9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('request_add', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('internal_index_image_copy_id', sa.Integer(), nullable=True))
+        batch_op.add_column(
+            sa.Column('internal_index_image_copy_resolved_id', sa.Integer(), nullable=True)
+        )
+        batch_op.create_foreign_key(
+            "internal_index_image_copy_id_fkey", 'image', ['internal_index_image_copy_id'], ['id']
+        )
+        batch_op.create_foreign_key(
+            "internal_index_image_copy_resolved_id_fkey",
+            'image',
+            ['internal_index_image_copy_resolved_id'],
+            ['id'],
+        )
+
+    with op.batch_alter_table('request_create_empty_index', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('internal_index_image_copy_id', sa.Integer(), nullable=True))
+        batch_op.add_column(
+            sa.Column('internal_index_image_copy_resolved_id', sa.Integer(), nullable=True)
+        )
+        batch_op.create_foreign_key(
+            "internal_index_image_copy_id_fkey", 'image', ['internal_index_image_copy_id'], ['id']
+        )
+        batch_op.create_foreign_key(
+            "internal_index_image_copy_resolved_id_fkey",
+            'image',
+            ['internal_index_image_copy_resolved_id'],
+            ['id'],
+        )
+
+    with op.batch_alter_table('request_rm', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('internal_index_image_copy_id', sa.Integer(), nullable=True))
+        batch_op.add_column(
+            sa.Column('internal_index_image_copy_resolved_id', sa.Integer(), nullable=True)
+        )
+        batch_op.create_foreign_key(
+            "internal_index_image_copy_resolved_id_fkey",
+            'image',
+            ['internal_index_image_copy_resolved_id'],
+            ['id'],
+        )
+        batch_op.create_foreign_key(
+            "internal_index_image_copy_id_fkey", 'image', ['internal_index_image_copy_id'], ['id']
+        )
+
+
+def downgrade():
+    with op.batch_alter_table('request_rm', schema=None) as batch_op:
+        batch_op.drop_constraint("internal_index_image_copy_resolved_id_fkey", type_='foreignkey')
+        batch_op.drop_constraint("internal_index_image_copy_id_fkey", type_='foreignkey')
+        batch_op.drop_column('internal_index_image_copy_resolved_id')
+        batch_op.drop_column('internal_index_image_copy_id')
+
+    with op.batch_alter_table('request_create_empty_index', schema=None) as batch_op:
+        batch_op.drop_constraint("internal_index_image_copy_resolved_id_fkey", type_='foreignkey')
+        batch_op.drop_constraint("internal_index_image_copy_id_fkey", type_='foreignkey')
+        batch_op.drop_column('internal_index_image_copy_resolved_id')
+        batch_op.drop_column('internal_index_image_copy_id')
+
+    with op.batch_alter_table('request_add', schema=None) as batch_op:
+        batch_op.drop_constraint("internal_index_image_copy_resolved_id_fkey", type_='foreignkey')
+        batch_op.drop_constraint("internal_index_image_copy_id_fkey", type_='foreignkey')
+        batch_op.drop_column('internal_index_image_copy_resolved_id')
+        batch_op.drop_column('internal_index_image_copy_id')

iib/web/models.py

@@ -683,6 +683,8 @@ def get_request_query_options(verbose=False):
         joinedload(RequestAdd.from_index_resolved),
         joinedload(RequestAdd.index_image),
         joinedload(RequestAdd.index_image_resolved),
+        joinedload(RequestAdd.internal_index_image_copy),
+        joinedload(RequestAdd.internal_index_image_copy_resolved),
         joinedload(RequestAdd.build_tags),
         joinedload(RequestRegenerateBundle.bundle_image),
         joinedload(RequestRegenerateBundle.from_bundle_image),
@@ -693,6 +695,8 @@ def get_request_query_options(verbose=False):
         joinedload(RequestRm.from_index_resolved),
         joinedload(RequestRm.index_image),
         joinedload(RequestRm.index_image_resolved),
+        joinedload(RequestRm.internal_index_image_copy),
+        joinedload(RequestRm.internal_index_image_copy_resolved),
         joinedload(RequestRm.operators),
         joinedload(RequestRm.build_tags),
         joinedload(RequestMergeIndexImage.build_tags),
@@ -773,6 +777,30 @@ def index_image_resolved(cls):
         """Return the relationship to the built index image."""
         return db.relationship('Image', foreign_keys=[cls.index_image_resolved_id], uselist=False)
 
+    @declared_attr
+    def internal_index_image_copy_id(cls):
+        """Return the ID of IIB's internal copy of the built index image."""
+        return db.Column(db.Integer, db.ForeignKey('image.id'))
+
+    @declared_attr
+    def internal_index_image_copy(cls):
+        """Return the relationship to IIB's internal copy of the built index image."""
+        return db.relationship(
+            'Image', foreign_keys=[cls.internal_index_image_copy_id], uselist=False
+        )
+
+    @declared_attr
+    def internal_index_image_copy_resolved_id(cls):
+        """Return the ID of resolved IIB's internal copy of the built index image."""
+        return db.Column(db.Integer, db.ForeignKey('image.id'))
+
+    @declared_attr
+    def internal_index_image_copy_resolved(cls):
+        """Return the relationship to resolved IIB's internal copy of the built index image."""
+        return db.relationship(
+            'Image', foreign_keys=[cls.internal_index_image_copy_resolved_id], uselist=False
+        )
+
     @declared_attr
     def distribution_scope(cls):
         """Return the distribution_scope for the request."""
@@ -907,6 +935,12 @@ def get_common_index_image_json(self):
             'from_index_resolved': getattr(self.from_index_resolved, 'pull_specification', None),
             'index_image': getattr(self.index_image, 'pull_specification', None),
             'index_image_resolved': getattr(self.index_image_resolved, 'pull_specification', None),
+            'internal_index_image_copy': getattr(
+                self.internal_index_image_copy, 'pull_specification', None
+            ),
+            'internal_index_image_copy_resolved': getattr(
+                self.internal_index_image_copy_resolved, 'pull_specification', None
+            ),
             'organization': None,
             'removed_operators': [],
             'distribution_scope': self.distribution_scope,
@@ -928,6 +962,8 @@ def get_index_image_mutable_keys(self):
             'from_index_resolved',
             'index_image',
             'index_image_resolved',
+            'internal_index_image_copy',
+            'internal_index_image_copy_resolved',
         }
 
 
@@ -1639,6 +1675,8 @@ def to_json(self, verbose=True):
         rv.pop('deprecation_list')
         rv.pop('removed_operators')
         rv.pop('build_tags')
+        rv.pop('internal_index_image_copy')
+        rv.pop('internal_index_image_copy_resolved')
         rv['labels'] = self.labels
         return rv
 
@@ -1653,4 +1691,6 @@ def get_mutable_keys(self):
         rv.update(self.get_index_image_mutable_keys())
         rv.update('labels')
         rv.remove('from_bundle_image_resolved')
+        rv.remove('internal_index_image_copy')
+        rv.remove('internal_index_image_copy_resolved')
         return rv

iib/web/static/api_v1.yaml

@@ -784,6 +784,20 @@ components:
             index_image_resolved:
               type: string
               example: 'quay.io/iib-stage/iib@sha256:abcdef012356789'
+            internal_index_image_copy:
+              description: >
+                The pullspec of the internal copy of the index image built by IIB.
+                It will have the same value as index_image when overwrite_from_index
+                is not provided.
+              type: string
+              example: 'quay.io/iib-stage/iib:5'
+            internal_index_image_copy_resolved:
+              description: >
+                The resolved pullspec of the internal copy of the index image built by IIB.
+                It will have the same value as index_image_resolved when overwrite_from_index
+                is not provided.
+              type: string
+              example: 'quay.io/iib-stage/iib@sha256:abcdef012356789'
             distribution_scope:
               description: >
                 The scope of distribution for the index created by the request.

iib/workers/tasks/build.py

@@ -243,6 +243,8 @@ def _update_index_image_pull_spec(
         with set_registry_token(overwrite_from_index_token, index_image):
             index_image_resolved = get_resolved_image(index_image)
         payload['index_image_resolved'] = index_image_resolved
+        payload['internal_index_image_copy'] = output_pull_spec
+        payload['internal_index_image_copy_resolved'] = get_resolved_image(output_pull_spec)
 
     update_request(request_id, payload, exc_msg='Failed setting the index image on the request')
 

tests/test_web/test_api_v1.py

@@ -27,6 +27,12 @@ def test_get_build(app, auth_env, client, db):
             'quay.io/namespace/from_index@sha256:defghi'
         )
         request.index_image = Image.get_or_create('quay.io/namespace/index@sha256:fghijk')
+        request.internal_index_image_copy = Image.get_or_create(
+            'quay.io/namespace/internal_index:tag'
+        )
+        request.internal_index_image_copy_resolved = Image.get_or_create(
+            'quay.io/namespace/internal_index@sha256:something'
+        )
         request.add_architecture('amd64')
         request.add_architecture('s390x')
         request.add_state('complete', 'Completed successfully')
@@ -57,6 +63,8 @@ def test_get_build(app, auth_env, client, db):
         'id': 1,
         'index_image': 'quay.io/namespace/index@sha256:fghijk',
         'index_image_resolved': None,
+        'internal_index_image_copy': 'quay.io/namespace/internal_index:tag',
+        'internal_index_image_copy_resolved': 'quay.io/namespace/internal_index@sha256:something',
         'logs': {
             'url': 'http://localhost/api/v1/builds/1/logs',
             'expiration': '2020-02-15T17:03:00Z',
@@ -680,6 +688,8 @@ def test_add_bundle_success(
         'id': 1,
         'index_image': None,
         'index_image_resolved': None,
+        'internal_index_image_copy': None,
+        'internal_index_image_copy_resolved': None,
         'removed_operators': [],
         'request_type': 'add',
         'state': 'in_progress',
@@ -989,6 +999,8 @@ def test_patch_request_add_success(
         'index_image': 'index:image',
         'index_image_resolved': 'index:image-resolved',
         'binary_image_resolved': 'binary-image@sha256:1234',
+        'internal_index_image_copy': 'quay.io/namespace/internal_index:tag',
+        'internal_index_image_copy_resolved': 'quay.io/namespace/internal_index@sha256:something',
     }
 
     if distribution_scope:
@@ -1010,6 +1022,8 @@ def test_patch_request_add_success(
         'id': minimal_request_add.id,
         'index_image': 'index:image',
         'index_image_resolved': 'index:image-resolved',
+        'internal_index_image_copy': 'quay.io/namespace/internal_index:tag',
+        'internal_index_image_copy_resolved': 'quay.io/namespace/internal_index@sha256:something',
         'logs': {
             'url': 'http://localhost/api/v1/builds/1/logs',
             'expiration': '2020-02-15T17:03:00Z',
@@ -1059,6 +1073,8 @@ def test_patch_request_rm_success(mock_smfsc, db, minimal_request_rm, worker_aut
         'index_image': 'index:image',
         'index_image_resolved': 'index:image-resolved',
         'binary_image_resolved': 'binary-image@sha256:1234',
+        'internal_index_image_copy': 'quay.io/namespace/internal_index:tag',
+        'internal_index_image_copy_resolved': 'quay.io/namespace/internal_index@sha256:something',
     }
 
     response_json = {
@@ -1077,6 +1093,8 @@ def test_patch_request_rm_success(mock_smfsc, db, minimal_request_rm, worker_aut
         'id': minimal_request_rm.id,
         'index_image': 'index:image',
         'index_image_resolved': 'index:image-resolved',
+        'internal_index_image_copy': 'quay.io/namespace/internal_index:tag',
+        'internal_index_image_copy_resolved': 'quay.io/namespace/internal_index@sha256:something',
         'logs': {
             'url': 'http://localhost/api/v1/builds/1/logs',
             'expiration': '2020-02-15T17:03:00Z',
@@ -1204,6 +1222,8 @@ def test_remove_operator_success(mock_smfsc, mock_rm, db, auth_env, client):
         'id': 1,
         'index_image': None,
         'index_image_resolved': None,
+        'internal_index_image_copy': None,
+        'internal_index_image_copy_resolved': None,
         'logs': {
             'url': 'http://localhost/api/v1/builds/1/logs',
             'expiration': '2020-02-15T17:03:00Z',

tests/test_workers/test_tasks/test_build.py

@@ -260,7 +260,13 @@ def test_update_index_image_pull_spec(
     mock_ur.assert_called_once()
     update_request_payload = mock_ur.call_args[0][1]
     if add_or_rm:
-        assert update_request_payload.keys() == {'arches', 'index_image', 'index_image_resolved'}
+        assert update_request_payload.keys() == {
+            'arches',
+            'index_image',
+            'index_image_resolved',
+            'internal_index_image_copy',
+            'internal_index_image_copy_resolved',
+        }
     else:
         assert update_request_payload.keys() == {'arches', 'index_image'}
     assert update_request_payload['index_image'] == expected_pull_spec