Use trails from s3

common/app/agents/DeeplyReadAgent.scala

@@ -1,73 +1,32 @@
 package agents
 
-import com.gu.contentapi.client.model.v1.{Content, ElementType}
-import com.gu.contentapi.client.utils.CapiModelEnrichment.RenderingFormat
 import common._
-import contentapi.ContentApiClient
-import layout.DiscussionSettings
-import model.ContentFormat
-import services.{FaciaContentConvert, OphanApi}
+import conf.Configuration
+import model.dotcomrendering.Trail
+import services.S3Async
 
 import scala.concurrent.{ExecutionContext, Future}
-import scala.util.control.NonFatal
-import model.dotcomrendering.Trail
 
-class DeeplyReadAgent(contentApiClient: ContentApiClient, ophanApi: OphanApi) extends GuLogging {
+object DeeplyReadS3Agent extends S3Async {
+  override lazy val bucket = Configuration.cache.bucket;
+  lazy val stage: String = Configuration.environment.stage.toUpperCase
+}
 
-  private val deeplyReadItems = Box[Map[Edition, Seq[Trail]]](Map.empty)
+class DeeplyReadAgent extends GuLogging {
 
-  def removeStartingSlash(path: String): String = {
-    if (path.startsWith("/")) path.stripPrefix("/") else path
-  }
+  private val deeplyReadItems = Box[Map[Edition, Seq[Trail]]](Map.empty)
 
   def refresh()(implicit ec: ExecutionContext): Future[Unit] = {
     log.debug(s"Deeply Read Agent refresh()")
-    /*
-      We query Ophan for the deeply read URLs and use them to queryCapi
-      then use this information to create a sequence of trails that we cache
-      using a Box structure.
-     */
     Future
       .sequence(Edition.allEditions.map { edition =>
-        ophanApi
-          .getDeeplyRead(edition)
-          .flatMap { ophanDeeplyReadItems =>
-            log.debug(s"Fetched ${ophanDeeplyReadItems.size} Deeply Read items for ${edition.displayName}")
-            val constructedTrail: Seq[Future[Option[Trail]]] = ophanDeeplyReadItems.map { ophanItem =>
-              log.debug(s"CAPI lookup for Ophan deeply read item: ${ophanItem.toString}")
-              val path = removeStartingSlash(ophanItem.path)
-              log.debug(s"CAPI Lookup for path: $path")
-              val capiRequest = contentApiClient
-                .item(path)
-                .showTags("all")
-                .showFields("all")
-                .showReferences("none")
-                .showAtoms("none")
-
-              contentApiClient
-                .getResponse(capiRequest)
-                .map { res =>
-                  res.content.flatMap { capiData =>
-                    log.debug(s"Retrieved CAPI data for Deeply Read item: ${path}")
-                    deeplyReadUrlToTrail(capiData)
-                  }
-                }
-                .recover { case NonFatal(e) =>
-                  log.error(s"Error retrieving CAPI data for Deeply Read item: ${path}. ${e.getMessage}")
-                  None
-                }
-            }
-            Future
-              .sequence(constructedTrail)
-              .map { maybeTrails =>
-                (edition, maybeTrails.flatten.take(10))
-              }
-
-          }
-          .recover { e =>
-            log.error(s"Failed to fetch Deeply Read items for ${edition.displayName}. ${e.getMessage()}")
-            (edition, Seq.empty)
-          }
+        DeeplyReadS3Agent
+          .getObjectAsJson[Seq[Trail]](
+            s"${DeeplyReadS3Agent.stage}/deeply-read/${edition.id.toLowerCase()}.json",
+          )
+          .map(trailsList => {
+            edition -> trailsList.take(10)
+          })
       })
       .map(trailsList => {
         val map = trailsList.toMap
@@ -84,47 +43,6 @@ class DeeplyReadAgent(contentApiClient: ContentApiClient, ophanApi: OphanApi) ex
       })
   }
 
-  def correctPillar(pillar: String): String = if (pillar == "arts") "culture" else pillar
-
-  def deeplyReadUrlToTrail(content: Content): Option[Trail] = {
-
-    val contentFormat: ContentFormat = ContentFormat(content.design, content.theme, content.display)
-
-    for {
-      webPublicationDate <- content.webPublicationDate
-      fields <- content.fields
-      linkText <- fields.trailText
-      pillar <- content.pillarName
-      headline <- fields.headline
-      shortUrl <- fields.shortUrl
-    } yield Trail(
-      url = content.webUrl,
-      linkText = linkText,
-      showByline = false,
-      byline = fields.byline,
-      masterImage = None,
-      image = fields.thumbnail,
-      carouselImages = Map.empty,
-      ageWarning = None,
-      isLiveBlog = fields.liveBloggingNow.getOrElse(false),
-      pillar = correctPillar(pillar.toLowerCase),
-      designType = content.`type`.toString,
-      format = contentFormat,
-      webPublicationDate = webPublicationDate.toString(),
-      headline = headline,
-      mediaType = None,
-      shortUrl = shortUrl,
-      kickerText = None,
-      starRating = None,
-      avatarUrl = None,
-      branding = None,
-      discussion = DiscussionSettings.fromTrail(FaciaContentConvert.contentToFaciaContent(content)),
-      trailText = content.fields.flatMap(_.trailText),
-      galleryCount =
-        content.elements.map(_.count(el => el.`type` == ElementType.Image && el.relation == "gallery")).filter(_ > 0),
-    )
-  }
-
   def getTrails(edition: Edition)(implicit ec: ExecutionContext): Seq[Trail] = {
     val updatedTrails = deeplyReadItems.get().getOrElse(edition, Seq.empty)
     if (updatedTrails.isEmpty) {

common/app/common/configuration.scala

@@ -711,6 +711,10 @@ class GuardianConfiguration extends GuLogging {
     lazy val host = configuration.getStringProperty("newsletterApi.host")
     lazy val origin = configuration.getStringProperty("newsletterApi.origin")
   }
+
+  object cache {
+    lazy val bucket = configuration.getMandatoryStringProperty("cache.bucket")
+  }
 }
 
 object ManifestData {

common/app/model/dotcomrendering/Trail.scala

@@ -1,14 +1,21 @@
 package model.dotcomrendering
 
 import com.github.nscala_time.time.Imports.DateTimeZone
-import com.gu.commercial.branding.{Branding, BrandingType, Dimensions, Logo => CommercialLogo}
+import com.gu.commercial.branding.{
+  Branding,
+  BrandingType,
+  Dimensions,
+  Foundation,
+  PaidContent,
+  Sponsored,
+  Logo => CommercialLogo,
+}
 import common.{Edition, LinkTo}
 import implicits.FaciaContentFrontendHelpers.FaciaContentFrontendHelper
 import layout.DiscussionSettings
-import model.dotcomrendering.DotcomRenderingUtils.withoutNull
 import model.{Article, ContentFormat, ImageMedia, Pillar}
 import model.pressed.PressedContent
-import play.api.libs.json.{Json, OWrites, Writes}
+import play.api.libs.json.{JsNull, JsObject, JsResult, JsValue, Json, OFormat, Reads, Writes}
 import play.api.mvc.RequestHeader
 import views.support.{ImageProfile, ImgSrc, Item300, Item460, RemoveOuterParaHtml}
 
@@ -40,50 +47,116 @@ case class Trail(
 
 object Trail {
 
-  implicit val brandingTypeWrites: Writes[BrandingType] = new Writes[BrandingType] {
-    def writes(bt: BrandingType) = {
-      Json.obj(
-        "name" -> bt.name,
-      )
+  implicit val carouselImagesReads: Reads[Map[String, Option[String]]] =
+    Reads[Map[String, Option[String]]] { json =>
+      json.validate[JsObject].map {
+        _.fields.map { case (k, v) => k -> v.asOpt[String] }.toMap
+      }
     }
+
+  implicit val brandingTypeFormat: OFormat[BrandingType] = new OFormat[BrandingType] {
+    def reads(json: JsValue): JsResult[BrandingType] =
+      (json \ "name").validate[String].map {
+        case PaidContent.name => PaidContent
+        case Foundation.name  => Foundation
+        case _                => Sponsored
+      }
+    def writes(bt: BrandingType): JsObject = Json.obj("name" -> bt.name)
   }
 
-  implicit val dimensionsWrites: OWrites[Dimensions] = Json.writes[Dimensions]
-
-  implicit val logoWrites: OWrites[CommercialLogo] = Json.writes[CommercialLogo]
-
-  implicit val brandingWrites: OWrites[Branding] = Json.writes[Branding]
-
-  implicit val discussionWrites: OWrites[DiscussionSettings] = Json.writes[DiscussionSettings]
-
-  implicit val OnwardItemWrites: Writes[Trail] = Writes { trail =>
-    val jsObject = Json.obj(
-      "url" -> trail.url,
-      "linkText" -> trail.linkText,
-      "showByline" -> trail.showByline,
-      "byline" -> trail.byline,
-      "masterImage" -> trail.masterImage,
-      "image" -> trail.image,
-      "carouselImages" -> trail.carouselImages,
-      "ageWarning" -> trail.ageWarning,
-      "isLiveBlog" -> trail.isLiveBlog,
-      "pillar" -> trail.pillar,
-      "designType" -> trail.designType,
-      "format" -> trail.format,
-      "webPublicationDate" -> trail.webPublicationDate,
-      "headline" -> trail.headline,
-      "mediaType" -> trail.mediaType,
-      "shortUrl" -> trail.shortUrl,
-      "kickerText" -> trail.kickerText,
-      "starRating" -> trail.starRating,
-      "avatarUrl" -> trail.avatarUrl,
-      "branding" -> trail.branding,
-      "discussion" -> trail.discussion,
-      "trailText" -> trail.trailText,
-      "galleryCount" -> trail.galleryCount,
-    )
+  implicit val dimensionsFormat: OFormat[Dimensions] = Json.format[Dimensions]
+
+  implicit val logoFormat: OFormat[CommercialLogo] = Json.format[CommercialLogo]
+
+  implicit val brandingFormat: OFormat[Branding] = Json.format[Branding]
 
-    withoutNull(jsObject)
+  implicit val discussionSettingsFormat: OFormat[DiscussionSettings] = Json.format[DiscussionSettings]
+
+  implicit val contentFormatFormat: OFormat[ContentFormat] = new OFormat[ContentFormat] {
+    def reads(json: JsValue): JsResult[ContentFormat] = ContentFormat.contentFormatReads.reads(json)
+    def writes(cf: ContentFormat): JsObject = ContentFormat.contentFormatWrites.writes(cf).as[JsObject]
+  }
+
+  implicit val trailFormat: OFormat[Trail] = new OFormat[Trail] {
+    override def reads(json: JsValue): JsResult[Trail] =
+      for {
+        url <- (json \ "url").validate[String]
+        linkText <- (json \ "linkText").validate[String]
+        showByline <- (json \ "showByline").validate[Boolean]
+        byline <- (json \ "byline").validateOpt[String]
+        masterImage <- (json \ "masterImage").validateOpt[String]
+        image <- (json \ "image").validateOpt[String]
+        carouselImages <- (json \ "carouselImages").validate[Map[String, Option[String]]]
+        ageWarning <- (json \ "ageWarning").validateOpt[String]
+        isLiveBlog <- (json \ "isLiveBlog").validate[Boolean]
+        pillar <- (json \ "pillar").validate[String]
+        designType <- (json \ "designType").validate[String]
+        format <- (json \ "format").validate[ContentFormat]
+        webPublicationDate <- (json \ "webPublicationDate").validate[String]
+        headline <- (json \ "headline").validate[String]
+        mediaType <- (json \ "mediaType").validateOpt[String]
+        shortUrl <- (json \ "shortUrl").validate[String]
+        kickerText <- (json \ "kickerText").validateOpt[String]
+        starRating <- (json \ "starRating").validateOpt[Int]
+        avatarUrl <- (json \ "avatarUrl").validateOpt[String]
+        branding <- (json \ "branding").validateOpt[Branding]
+        discussion <- (json \ "discussion").validate[DiscussionSettings]
+        trailText <- (json \ "trailText").validateOpt[String]
+        galleryCount <- (json \ "galleryCount").validateOpt[Int]
+      } yield Trail(
+        url,
+        linkText,
+        showByline,
+        byline,
+        masterImage,
+        image,
+        carouselImages,
+        ageWarning,
+        isLiveBlog,
+        pillar,
+        designType,
+        format,
+        webPublicationDate,
+        headline,
+        mediaType,
+        shortUrl,
+        kickerText,
+        starRating,
+        avatarUrl,
+        branding,
+        discussion,
+        trailText,
+        galleryCount,
+      )
+
+    override def writes(trail: Trail): JsObject = {
+      val obj = Json.obj(
+        "url" -> trail.url,
+        "linkText" -> trail.linkText,
+        "showByline" -> trail.showByline,
+        "byline" -> trail.byline,
+        "masterImage" -> trail.masterImage,
+        "image" -> trail.image,
+        "carouselImages" -> trail.carouselImages,
+        "ageWarning" -> trail.ageWarning,
+        "isLiveBlog" -> trail.isLiveBlog,
+        "pillar" -> trail.pillar,
+        "designType" -> trail.designType,
+        "format" -> trail.format,
+        "webPublicationDate" -> trail.webPublicationDate,
+        "headline" -> trail.headline,
+        "mediaType" -> trail.mediaType,
+        "shortUrl" -> trail.shortUrl,
+        "kickerText" -> trail.kickerText,
+        "starRating" -> trail.starRating,
+        "avatarUrl" -> trail.avatarUrl,
+        "branding" -> trail.branding,
+        "discussion" -> trail.discussion,
+        "trailText" -> trail.trailText,
+        "galleryCount" -> trail.galleryCount,
+      )
+      JsObject(obj.fields.filterNot(_._2 == JsNull))
+    }
   }
 
   // We ideally want this to be replaced by something else in the near future. Probably

common/app/services/S3Async.scala

@@ -0,0 +1,60 @@
+package services
+
+import com.gu.etagcaching.aws.s3.ObjectId
+import common.GuLogging
+import conf.Configuration
+import play.api.libs.json.{JsError, JsSuccess, Json, Reads}
+import services.S3.logS3ExceptionWithDevHint
+import software.amazon.awssdk.core.async.AsyncResponseTransformer
+import software.amazon.awssdk.services.s3.model.{GetObjectRequest, GetObjectResponse, NoSuchKeyException, S3Exception}
+import utils.AWSv2
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.jdk.FutureConverters._
+import scala.io.Codec
+
+trait S3Async extends GuLogging {
+
+  lazy val bucket: String = Configuration.aws.frontendStoreBucket
+
+  lazy private val client = AWSv2.S3Async
+
+  def handleS3Errors[T](key: String)(future: Future[T])(implicit ec: ExecutionContext): Future[T] = {
+    val objectId = ObjectId(bucket, key)
+    future.recoverWith {
+      case e: NoSuchKeyException =>
+        log.warn(s"not found at ${objectId.s3Uri}")
+        Future.failed(e)
+      case e: S3Exception =>
+        logS3ExceptionWithDevHint(objectId, e)
+        Future.failed(e)
+    }
+  }
+
+  private def getResponse(
+      key: String,
+  )(implicit codec: Codec, ec: ExecutionContext): Future[(GetObjectResponse, String)] = {
+    val request = GetObjectRequest.builder().bucket(bucket).key(key).build()
+    val responseFutureJava = client.getObject(request, AsyncResponseTransformer.toBytes[GetObjectResponse]())
+
+    responseFutureJava.asScala.map { responseBytes =>
+      val objectResponse = responseBytes.response()
+      log.debug(s"S3 got ${objectResponse.contentLength} bytes from $key")
+      val content = new String(responseBytes.asByteArray(), codec.charSet)
+      (objectResponse, content)
+    }
+  }
+
+  def getObjectAsJson[T: Reads](key: String)(implicit ec: ExecutionContext): Future[T] = {
+    val futureResponse = getResponse(key)(Codec.UTF8, ec).map(_._2).flatMap { jsonString =>
+      val parsedJson = Json.parse(jsonString)
+      parsedJson.validate[T] match {
+        case JsSuccess(parsedObject, _) =>
+          Future.successful(parsedObject)
+        case JsError(errors) =>
+          Future.failed(new RuntimeException(s"Failed to parse JSON for key $key. Errors: $errors"))
+      }
+    }
+    handleS3Errors(key)(futureResponse)
+  }
+}

common/conf/env/DEVINFRA.properties

@@ -9,6 +9,7 @@ content.api.host=https://content.guardianapis.com
 pa.api.key=none
 
 aws.bucket=none
+cache.bucket=none
 
 ophan.api.host=http://api.ophan.co.uk/api
 ophan.api.key=none