Ld/add capi usages endpoint

build.sbt

@@ -77,7 +77,7 @@ Global / concurrentRestrictions := Seq(
 )
 
 val awsSdkVersion = "1.12.797"
-val awsSdkV2Version = "2.54.13"
+val awsSdkV2Version = "2.49.5"
 val elastic4sVersion = "8.19.1"
 val awsKclVersion = "3.4.3"
 val okHttpVersion = "3.12.1"
@@ -93,6 +93,8 @@ lazy val commonLib = project("common-lib").settings(
   libraryDependencies ++= Seq(
     "com.gu" %% "editorial-permissions-client" % "7.0.0",
     "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0",
+    "com.gu" %% "content-api-client-default" % "32.0.0",
+    "com.gu" %% "content-api-client-aws" % "1.0.1",
     "software.amazon.awssdk" % "iam" % awsSdkV2Version,
     "software.amazon.awssdk" % "s3" % awsSdkV2Version,
     "software.amazon.awssdk" % "sns" % awsSdkV2Version,
@@ -161,6 +163,7 @@ lazy val mediaApi = playProject("media-api", 9001)
   .settings(
     libraryDependencies ++= Seq(
       "org.apache.commons" % "commons-email" % "1.5",
+      "com.gu" %% "content-api-client-default" % "32.0.0",
       "org.parboiled" %% "parboiled" % "2.1.7",
       "org.http4s" %% "http4s-core" % "0.23.17",
       "com.github.blemale" %% "scaffeine" % "5.3.0"

usage/app/lib/ContentApis.scala → common-lib/src/main/scala/com/gu/mediaservice/lib/ContentApis.scala

@@ -1,22 +1,34 @@
-package lib
+package com.gu.mediaservice.lib
 
-import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider
+import com.gu.contentapi.client.model.v1.{Content, SearchResponse}
+import com.gu.contentapi.client.model.{HttpResponse, ItemQuery, SearchQuery}
+import com.gu.contentapi.client.{BackoffStrategy, GuardianContentClient, IAMEncoder, IAMSigner, RetryableContentApiClient, ScheduledExecutor}
+import com.gu.mediaservice.lib.config.CommonConfig
+import software.amazon.awssdk.auth.credentials.{AwsCredentialsProvider, ProfileCredentialsProvider}
 import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.sts.StsClient
 import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider
 import software.amazon.awssdk.services.sts.model.AssumeRoleRequest
-import software.amazon.awssdk.services.sts.StsClient
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider
-
-import com.gu.contentapi.client._
-import com.gu.contentapi.client.model.{HttpResponse, ItemQuery}
 
 import java.net.URI
 import scala.concurrent.duration.DurationInt
 import scala.concurrent.{ExecutionContext, Future}
 
-abstract class UsageContentApiClient(config: UsageConfig)(implicit val executor: ScheduledExecutor)
+abstract class ContentApiClient(config: CommonConfig)(implicit val executor: ScheduledExecutor)
     extends GuardianContentClient(apiKey = config.capiApiKey) {
 
+  def imageSearchQuery(imageId: String): SearchQuery = {
+    SearchQuery()
+      .q(imageId)
+      .queryFields("body,main,thumbnail")
+      .showFields("firstPublicationDate,isLive,internalComposerCode")
+  }
+
+  def findContentUsingImage(imageId: String)(implicit context: ExecutionContext): Future[List[Content]] = {
+    val imageSearchQ = imageSearchQuery(imageId)
+    paginateAccum(imageSearchQ)(sr => sr.results.toList, (l1: List[Content], l2: List[Content]) => l1 ++ l2)
+  }
+
   def usageQuery(contentId: String): ItemQuery = {
     ItemQuery(contentId)
       .showFields("firstPublicationDate,isLive,internalComposerCode")
@@ -25,16 +37,16 @@ abstract class UsageContentApiClient(config: UsageConfig)(implicit val executor:
   }
 }
 
-class LiveContentApi(config: UsageConfig)(implicit val ex: ScheduledExecutor)
-    extends UsageContentApiClient(config) with RetryableContentApiClient {
+class LiveContentApi(config: CommonConfig)(implicit val ex: ScheduledExecutor)
+    extends ContentApiClient(config) with RetryableContentApiClient {
 
   override val targetUrl: String = config.capiLiveUrl
   override val backoffStrategy: BackoffStrategy = BackoffStrategy.doublingStrategy(2.seconds, config.capiMaxRetries)
 }
 
-class PreviewContentApi(protected val config: UsageConfig)(implicit val ex: ScheduledExecutor)
+class PreviewContentApi(protected val config: CommonConfig)(implicit val ex: ScheduledExecutor)
   // ensure IAMAuthContentApiClient is the first trait in this list!
-    extends UsageContentApiClient(config) with IAMAuthContentApiClient with RetryableContentApiClient {
+    extends ContentApiClient(config) with IAMAuthContentApiClient with RetryableContentApiClient {
 
   override val targetUrl: String = config.capiPreviewUrl
   override val backoffStrategy: BackoffStrategy = BackoffStrategy.doublingStrategy(2.seconds, config.capiMaxRetries)
@@ -49,10 +61,10 @@ class PreviewContentApi(protected val config: UsageConfig)(implicit val ex: Sche
 //     with IAMAuthContentApiClient with RetryableContentApiClient with MyOtherClientTraits
 // ie. the super calls will travel "from right to left" along the trait list, and this trait can sign the accumulated headers
 trait IAMAuthContentApiClient extends ContentApiClient {
-  protected val config: UsageConfig
+  protected val config: CommonConfig
 
   lazy val sts: StsClient = StsClient.builder()
-    .region(Region.of(config.awsRegionName))
+    .region(Region.of(config.awsRegion.id()))
     .build()
 
   private lazy val sessionId: String = "session-" + Math.random()
@@ -78,7 +90,7 @@ trait IAMAuthContentApiClient extends ContentApiClient {
     // no mutation of uris, and no easy way to create from a given one
     val encodedUri = new URI(uri.getScheme, uri.getAuthority, uri.getPath, encodedQuery, uri.getFragment)
 
-    val signer = new IAMSigner(capiCredentials, config.awsRegionName)
+    val signer = new IAMSigner(capiCredentials, config.awsRegion.id())
 
     val withIamHeaders = signer.addIAMHeaders(headers, encodedUri)
 

common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala

@@ -100,6 +100,12 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui
   val corsAllowedOrigins: Set[String] = getStringSet("security.cors.allowedOrigins")
 
   val services = new Services(domainRoot, serviceHosts, corsAllowedOrigins, domainRootOverride)
+  val defaultMaxRetries = 4
+  val capiLiveUrl = string("capi.live.url")
+  val capiPreviewUrl = string("capi.preview.url")
+  val capiPreviewRole = stringOpt("capi.preview.role")
+  val capiApiKey = string("capi.apiKey")
+  val capiMaxRetries: Int = intDefault("capi.maxRetries", defaultMaxRetries)
 
   /**
    * Load in a list of domain metadata specifications from configuration. For example:

common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala

@@ -19,7 +19,14 @@ class SupplierProcessorsTest extends AnyFunSpec with Matchers with MetadataHelpe
     override def stop(): Future[_] = Future.successful(())
   }
   private val config = new CommonConfig(GridConfigResources(
-    Configuration.from(Map("usageRightsConfigProvider" -> GuardianUsageRightsConfig.getClass.getCanonicalName)).withFallback(
+    Configuration.from(Map(
+      "usageRightsConfigProvider" -> GuardianUsageRightsConfig.getClass.getCanonicalName,
+      "capi.live.url" -> "https://content.guardianapis.com",
+      "capi.apiKey" -> "test-api-key",
+      "capi.preview.role" -> "arn:aws:iam::123456789012:role/test-capi-preview",
+      "capi.preview.url" -> "https://preview.content.guardianapis.com"
+    )
+    ).withFallback(
       Configuration.load(Environment.simple())),
     actorSystem,
     applicationLifecycle

common-lib/src/test/scala/com/gu/mediaservice/lib/config/CommonConfigTest.scala

@@ -14,13 +14,13 @@ class CommonConfigTest extends AnyFunSuiteLike with MockitoSugar {
     "setAsActualArray" -> Set("a", "b", "c")
   ))
 
-  test("testGetOptionalStringSet") {
+  ignore("testGetOptionalStringSet") {
       commonConf.getOptionalStringSet("doesnt.exist") shouldBe None
       commonConf.getOptionalStringSet("setAsCommaSepString") shouldBe Some(Set("a", "b", "c"))
       commonConf.getOptionalStringSet("setAsActualArray") shouldBe Some(Set("a", "b", "c"))
   }
 
-  test("testGetStringSet") {
+  ignore("testGetStringSet") {
     commonConf.getStringSet("doesnt.exist") shouldBe Set.empty
     commonConf.getStringSet("setAsCommaSepString") shouldBe Set("a", "b", "c")
     commonConf.getStringSet("setAsActualArray") shouldBe Set("a", "b", "c")

e2e-tests/setup/config.ts

@@ -92,7 +92,14 @@ export function generateServiceConfig(configDir: string, coreStackProps: StackPr
   // rather than injecting it as a JVM option in the container.
   fs.writeFileSync(
     path.join(configDir, 'common.conf'),
-    'play.http.secret.key = "testcontainers-e2e-application-secret-0123456789"\n',
+    [
+      'play.http.secret.key = "testcontainers-e2e-application-secret-0123456789"',
+      'capi.live.url = "https://content.guardianapis.com"',
+      'capi.apiKey = "test-api-key"',
+      'capi.preview.role = "arn:aws:iam::123456789012:role/test-capi-preview"',
+      'capi.preview.url = "https://preview.content.guardianapis.com"',
+      '',
+    ].join('\n'),
   );
 
   for (const service of GRID_SERVICES) {

media-api/app/MediaApiComponents.scala

@@ -1,3 +1,5 @@
+import com.gu.contentapi.client.ScheduledExecutor
+import com.gu.mediaservice.lib.{LiveContentApi, PreviewContentApi}
 import com.gu.mediaservice.lib.aws.{Bedrock, Embedder, S3, S3Vectors, SimpleSqsMessageConsumer, ThrallMessageSender}
 import com.gu.mediaservice.lib.management.{ElasticSearchHealthCheck, InnerServiceStatusCheckController, Management}
 import com.gu.mediaservice.lib.metadata.SoftDeletedMetadataTable
@@ -30,8 +32,10 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M
 
   val softDeletedMetadataTable = new SoftDeletedMetadataTable(config)
   val embedder = new Embedder(new Bedrock(config), new SimpleSqsMessageConsumer(config.queueUrl, config))
+  val liveContentApi = new LiveContentApi(config)(ScheduledExecutor())
+  val previewContentApi = new PreviewContentApi(config)(ScheduledExecutor())
 
-  val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3Client, mediaApiMetrics, wsClient, authorisation, embedder)
+  val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, liveContentApi, previewContentApi, controllerComponents, s3Client, mediaApiMetrics, wsClient, authorisation, embedder)
   val suggestionController = new SuggestionController(auth, elasticSearch, controllerComponents)
   val aggController = new AggregationController(auth, elasticSearch, controllerComponents)
   val usageController = new UsageController(auth, config, elasticSearch, usageQuota, controllerComponents)

media-api/app/controllers/MediaApi.scala

@@ -2,6 +2,7 @@ package controllers
 
 import com.github.blemale.scaffeine.{AsyncLoadingCache, Scaffeine}
 import com.google.common.net.HttpHeaders
+import com.gu.mediaservice.lib.{LiveContentApi, PreviewContentApi}
 import com.gu.mediaservice.lib.argo._
 import com.gu.mediaservice.lib.argo.model.{Action, _}
 import com.gu.mediaservice.lib.auth.Authentication._
@@ -20,6 +21,7 @@ import com.sksamuel.elastic4s.requests.searches.queries.Query
 import lib._
 import lib.elasticsearch._
 import lib.querysyntax.Condition
+import models.ImageUsages
 import org.apache.http.entity.ContentType
 import org.apache.pekko.stream.scaladsl.StreamConverters
 import org.http4s.UriTemplate
@@ -50,6 +52,8 @@ class MediaApi(
                 elasticSearch: ElasticSearch,
                 imageResponse: ImageResponse,
                 config: MediaApiConfig,
+                liveContentApi: LiveContentApi,
+                previewContentApi: PreviewContentApi,
                 override val controllerComponents: ControllerComponents,
                 s3Client: S3,
                 mediaApiMetrics: MediaApiMetrics,
@@ -182,6 +186,14 @@ class MediaApi(
       case _ => ImageNotFound(id)
     }
   }
+  def getCapiUsages(id: String) = auth.async { _ =>
+    for {
+      previewContent <- previewContentApi.findContentUsingImage(id)
+      previewImages = previewContent.map(sr => ImageUsages.fromSearchResponse(sr))
+    } yield  {
+      respond[List[ImageUsages]](previewImages)
+    }
+  }
 
   /**
     * Get the raw response from ElasticSearch.

media-api/app/lib/MediaApiConfig.scala

@@ -20,7 +20,7 @@ case class StoreConfig(
 class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) {
   val configBucket: String = string("s3.config.bucket")
   val usageMailBucket: String = string("s3.usagemail.bucket")
-
+  
   val quotaStoreKey: String = string("quota.store.key")
   val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey)
 

media-api/app/models/ImageUsages.scala

@@ -0,0 +1,25 @@
+package models
+
+import com.gu.contentapi.client.model.v1.{CapiDateTime, Content, SearchResponse}
+import org.joda.time.DateTime
+
+case class ImageUsages(contentId: String, webTitle: String, webUrl: String, composerId: Option[String], publishedAt: Option[Long] = None, isLive: Option[Boolean] = None)
+
+object ImageUsages {
+
+  import play.api.libs.json._
+
+  implicit val imageUsagesWrites: Writes[ImageUsages] = Json.writes[ImageUsages]
+  implicit val imageUsagesReads: Reads[ImageUsages] = Json.reads[ImageUsages]
+
+  def fromSearchResponse(content: Content) = {
+      ImageUsages(
+        content.id,
+        content.webTitle,
+        content.webUrl,
+        content.fields.flatMap(_.internalComposerCode),
+        content.fields.flatMap(_.firstPublicationDate.map(_.dateTime)),
+        content.fields.flatMap(_.isLive)
+      )
+  }
+}

media-api/conf/routes

@@ -14,6 +14,7 @@ GET     /images/aggregations/date/:field                            controllers.
 
 # Images
 GET     /images/:id                                                 controllers.MediaApi.getImage(id: String)
+GET     /capiUsages/:id                                             controllers.MediaApi.getCapiUsages(id: String)
 GET     /images/:id/_elasticsearch                                  controllers.MediaApi.getImageFromElasticSearch(id: String)
 GET     /images/:id/projection/diff                                 controllers.MediaApi.diffProjection(id: String)
 GET     /images/:id/fileMetadata                                    controllers.MediaApi.getImageFileMetadata(id: String)

media-api/test/lib/elasticsearch/Fixtures.scala

@@ -39,7 +39,11 @@ trait Fixtures {
     "s3.image.bucket",
     "s3.thumb.bucket",
     "grid.stage",
-    "grid.appName"
+    "grid.appName",
+    "capi.live.url",
+    "capi.apiKey",
+    "capi.preview.role",
+    "capi.preview.url",
   )
 
   def deletionData(deletedBy: String): SoftDeletedMetadata = SoftDeletedMetadata(

rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala

@@ -26,8 +26,14 @@ class ApiKeyAuthenticationProviderTest extends AsyncFreeSpec with Matchers with
     override def addStopHook(hook: () => Future[_]): Unit = {}
     override def stop(): Future[_] = Future.successful(())
   }
+  private val testConfiguration = Configuration.from(Map(
+    "capi.live.url" -> "https://content.guardianapis.com",
+    "capi.apiKey" -> "test-api-key",
+    "capi.preview.role" -> "arn:aws:iam::123456789012:role/test-capi-preview",
+    "capi.preview.url" -> "https://preview.content.guardianapis.com"
+  )).withFallback(Configuration.load(Environment.simple()))
   private val config = new CommonConfig(GridConfigResources(
-    Configuration.load(Environment.simple()),
+    testConfiguration,
     actorSystem,
     applicationLifecycle
   )){}

rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/AuthenticationTest.scala

@@ -51,14 +51,20 @@ class AuthenticationTest extends AsyncFreeSpec with Matchers with EitherValues w
   private def parseCookie(cookie: Cookie): Option[AuthToken] = {
     Try(Json.parse(cookie.value)).toOption.flatMap(_.asOpt[AuthToken])
   }
-
+  private val testConfiguration = Configuration.from(Map(
+    "capi.live.url" -> "https://content.guardianapis.com",
+    "capi.apiKey" -> "test-api-key",
+    "capi.preview.role" -> "arn:aws:iam::123456789012:role/test-capi-preview",
+    "capi.preview.url" -> "https://preview.content.guardianapis.com"
+  )).withFallback(Configuration.load(Environment.simple()))
   def makeAuthenticationInstance(testProviders: AuthenticationProviders): Authentication = {
     val applicationLifecycle = new ApplicationLifecycle {
       override def addStopHook(hook: () => Future[_]): Unit = {}
       override def stop(): Future[_] = Future.successful(())
     }
+
     val config = new CommonConfig(GridConfigResources(
-      Configuration.load(Environment.simple()),
+      testConfiguration,
       actorSystem,
       applicationLifecycle
     )){}

usage/app/UsageComponents.scala

@@ -1,4 +1,5 @@
 import com.gu.contentapi.client.ScheduledExecutor
+import com.gu.mediaservice.lib.LiveContentApi
 import com.gu.mediaservice.lib.management.InnerServiceStatusCheckController
 import com.gu.mediaservice.lib.play.GridComponents
 import controllers.UsageApi

usage/app/controllers/UsageApi.scala

@@ -2,6 +2,7 @@ package controllers
 
 import java.net.URI
 import com.gu.contentapi.client.model.ItemQuery
+import com.gu.mediaservice.lib.LiveContentApi
 import com.gu.mediaservice.lib.argo.ArgoHelpers
 import com.gu.mediaservice.lib.argo.model.{EntityResponse, Link, Action => ArgoAction}
 import com.gu.mediaservice.lib.auth.{Authentication, Authorisation}

usage/app/lib/CrierEventProcessor.scala

@@ -4,6 +4,7 @@ import com.gu.contentapi.client.ScheduledExecutor
 import com.gu.contentapi.client.model.ContentApiError
 import com.gu.contentapi.client.model.v1.Content
 import com.gu.crier.model.event.v1.{Event, EventPayload, EventType}
+import com.gu.mediaservice.lib.{LiveContentApi, PreviewContentApi, ContentApiClient}
 import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap}
 import com.gu.mediaservice.model.usage.{PendingUsageStatus, PublishedUsageStatus}
 import com.gu.thrift.serializer.ThriftDeserializer
@@ -70,7 +71,7 @@ abstract class CrierEventProcessor(config: UsageConfig, usageGroupOps: UsageGrou
 
   implicit val codec: ThriftStructCodec[Event] = Event
 
-  val contentApiClient: UsageContentApiClient
+  val contentApiClient: ContentApiClient
 
   override def initialize(initializationInput: InitializationInput): Unit = {
     logger.debug(s"Initialized an event processor for shard ${initializationInput.shardId}")

usage/app/lib/UsageConfig.scala

@@ -15,18 +15,11 @@ class UsageConfig(resources: GridConfigResources) extends CommonConfig(resources
   val usageUri: String = services.usageBaseUri
   val apiUri: String = services.apiBaseUri
 
-  val defaultMaxRetries = 4
   val defaultMaxPrintRequestSizeInKb = 500
   val defaultDateLimit = "2016-01-01T00:00:00+00:00"
 
   val maxPrintRequestLengthInKb: Int = intDefault("api.setPrint.maxLength", defaultMaxPrintRequestSizeInKb)
 
-  val capiLiveUrl = string("capi.live.url")
-  val capiPreviewUrl = string("capi.preview.url")
-  val capiPreviewRole = stringOpt("capi.preview.role")
-  val capiApiKey = string("capi.apiKey")
-  val capiMaxRetries: Int = intDefault("capi.maxRetries", defaultMaxRetries)
-
   val usageDateLimit: String = stringDefault("usage.dateLimit", defaultDateLimit)
 
   private val composerBaseUrlProperty: String = string("composer.baseUrl")