From 9fe51dbb89806260cd9ff3a6ab52c6fbf282fab1 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 25 Jun 2026 12:19:50 +0100 Subject: [PATCH 1/3] Migrate Syndication to use v2 version of the dynamo lib --- metadata-editor/app/lib/MetadataSqsMessageConsumer.scala | 2 +- metadata-editor/app/lib/Syndication.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata-editor/app/lib/MetadataSqsMessageConsumer.scala b/metadata-editor/app/lib/MetadataSqsMessageConsumer.scala index 3dc414102fb..c11fd896d05 100644 --- a/metadata-editor/app/lib/MetadataSqsMessageConsumer.scala +++ b/metadata-editor/app/lib/MetadataSqsMessageConsumer.scala @@ -15,6 +15,6 @@ class MetadataSqsMessageConsumer(config: EditsConfig, metadataEditorMetrics: Met } def processDeletedImage(message: JsValue) = Future { - withImageId(message)(id => store.deleteItem(id)) + withImageId(message)(id => store.deleteItemV2(id)) } } diff --git a/metadata-editor/app/lib/Syndication.scala b/metadata-editor/app/lib/Syndication.scala index d8a6f767fbf..e67583f2477 100644 --- a/metadata-editor/app/lib/Syndication.scala +++ b/metadata-editor/app/lib/Syndication.scala @@ -48,8 +48,8 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { (implicit ec: ExecutionContext): Future[Unit] = publishChangedSyndicationRightsForPhotoshoot[Unit](id, unchangedPhotoshoot = false) { () => for { - edits <- editsStore.removeKey(id, Edits.Photoshoot) - _ <- editsStore.removeKey(id, Edits.PhotoshootTitle) + edits <- editsStore.removeKeyV2(id, Edits.Photoshoot) + _ <- editsStore.removeKeyV2(id, Edits.PhotoshootTitle) _ = publish(id, UpdateImagePhotoshootMetadata)(edits) } yield () } From df56fe084b878647b911a001b3c83b2ab5a9a88a Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 25 Jun 2026 16:10:36 +0100 Subject: [PATCH 2/3] Use batch v2 --- .../gu/mediaservice/lib/aws/DynamoDB.scala | 57 ++++++++++++++++++- metadata-editor/app/lib/Syndication.scala | 2 +- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala index 63e763fcaa6..dddf531214b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala @@ -15,8 +15,9 @@ import play.api.libs.json._ import software.amazon.awssdk.enhanced.dynamodb._ import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument import software.amazon.awssdk.enhanced.dynamodb.model +import software.amazon.awssdk.enhanced.dynamodb.model.{BatchGetItemEnhancedRequest, ReadBatch} import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{UpdateItemRequest, AttributeValue => AttributeValueV2, ReturnValue => ReturnValueV2} +import software.amazon.awssdk.services.dynamodb.model.{BatchGetItemRequest, UpdateItemRequest, AttributeValue => AttributeValueV2, KeysAndAttributes => KeysAndAttributesV2, ReturnValue => ReturnValueV2} import scala.annotation.tailrec import scala.concurrent.{ExecutionContext, Future} @@ -131,6 +132,60 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti def setAddV2(id: String, key: String, value: List[String])(implicit ex: ExecutionContext): Future[JsObject] = Future { updateV2(id, DynamoDB.addExpr(key, lastModifiedKey), AttributeValueV2.fromSs(value.asJava)) } + + def batchGetV2(ids: List[String], attributeKey: String)(implicit ex: ExecutionContext, rjs: Reads[T]): Future[Map[String, T]] = { + val chunks = + ids.grouped(100).toList.zipWithIndex + + Future + .traverse(chunks) { case (chunk, idx) => + logger.info(s"Fetching records for chunk $idx of ${chunks.size}") + Future { + + val readBatchBuilder = + ReadBatch.builder(classOf[EnhancedDocument]) + .mappedTableResource(table2) + + chunk.foreach { id => + readBatchBuilder.addGetItem( + Key.builder() + .partitionValue(id) + .build() + ) + } + + val results = + dynamo2.batchGetItem( + BatchGetItemEnhancedRequest.builder() + .readBatches(readBatchBuilder.build()) + .build() + ) + + results + .resultsForTable(table2) + .asScala + .toList + .flatMap { doc => + + logger.info(s"Obtained document $doc") + + val json = asJsObject(doc) + + val maybeT = + (json \ attributeKey).asOpt[T] + + logger.info(s"Obtained a T of $maybeT from json $json") + + maybeT.map( + doc.getString(IdKey) -> _ + ) + } + .toMap + } + } + .map(_.foldLeft(Map.empty[String, T])(_ ++ _)) + } + def batchGet(ids: List[String], attributeKey: String) (implicit ex: ExecutionContext, rjs: Reads[T]): Future[Map[String, T]] = { val keyChunkList = ids diff --git a/metadata-editor/app/lib/Syndication.scala b/metadata-editor/app/lib/Syndication.scala index e67583f2477..a3111bab729 100644 --- a/metadata-editor/app/lib/Syndication.scala +++ b/metadata-editor/app/lib/Syndication.scala @@ -135,7 +135,7 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { def getAllImageRightsInPhotoshoot(photoshoot: Photoshoot) (implicit ec: ExecutionContext): Future[Map[String, SyndicationRights]] = for { imageIds <- getImagesInPhotoshoot(photoshoot) - allNonInferredRights <- syndicationStore.batchGet(imageIds, syndicationRightsFieldName) + allNonInferredRights <- syndicationStore.batchGetV2(imageIds, syndicationRightsFieldName) } yield { logger.info(s"Found non-inferred rights for ${allNonInferredRights.size} of ${imageIds.size} images in photoshoot ${photoshoot.title}") val mostRecentInferrableRightsMaybe = getMostRecentInferrableSyndicationRights(allNonInferredRights.values.toList) From 98dc0ab64890b634a5517d01e0616dc6b930862f Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 25 Jun 2026 16:19:01 +0100 Subject: [PATCH 3/3] Upgrade the scan for v2 --- .../gu/mediaservice/lib/aws/DynamoDB.scala | 32 ++++++++++++++++++- metadata-editor/app/lib/Syndication.scala | 2 +- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala index dddf531214b..e6fc62f6961 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala @@ -17,7 +17,7 @@ import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument import software.amazon.awssdk.enhanced.dynamodb.model import software.amazon.awssdk.enhanced.dynamodb.model.{BatchGetItemEnhancedRequest, ReadBatch} import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{BatchGetItemRequest, UpdateItemRequest, AttributeValue => AttributeValueV2, KeysAndAttributes => KeysAndAttributesV2, ReturnValue => ReturnValueV2} +import software.amazon.awssdk.services.dynamodb.model.{UpdateItemRequest, AttributeValue => AttributeValueV2, ReturnValue => ReturnValueV2, QueryRequest => QueryRequestV2} import scala.annotation.tailrec import scala.concurrent.{ExecutionContext, Future} @@ -255,6 +255,36 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti items map (a => a.getString("id")) } + def scanForIdV2( + indexName: String, + keyName: String, + key: String + )(implicit ex: ExecutionContext): Future[List[String]] = + Future { + + val response = + client2.query( + QueryRequestV2.builder() + .tableName(tableName) + .indexName(indexName) + .keyConditionExpression(s"$keyName = :key") + .expressionAttributeValues( + Map( + ":key" -> + AttributeValueV2.builder() + .s(key) + .build() + ).asJava + ) + .projectionExpression("id") + .build() + ) + + response.items().asScala.toList.map { item => + item.get("id").s() + } + } + private def updateRequestBuilder(id: String, expression: String) = { UpdateItemRequest.builder() .key(Map(IdKey -> AttributeValueV2.fromS(id)).asJava) diff --git a/metadata-editor/app/lib/Syndication.scala b/metadata-editor/app/lib/Syndication.scala index a3111bab729..1fcbc9f4b13 100644 --- a/metadata-editor/app/lib/Syndication.scala +++ b/metadata-editor/app/lib/Syndication.scala @@ -144,7 +144,7 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { def getImagesInPhotoshoot(photoshoot: Photoshoot) (implicit ec: ExecutionContext): Future[List[String]] = - editsStore.scanForId(config.editsTablePhotoshootIndex, Edits.PhotoshootTitle, photoshoot.title) + editsStore.scanForIdV2(config.editsTablePhotoshootIndex, Edits.PhotoshootTitle, photoshoot.title) .recover { case NoItemFound => Nil } def getChangedRights(before: Map[String, SyndicationRights], after: Map[String, SyndicationRights]): Map[String, Option[SyndicationRights]] = {