diff --git a/build.sbt b/build.sbt index dad2f1577af..0d17cebe7e7 100644 --- a/build.sbt +++ b/build.sbt @@ -77,7 +77,7 @@ Global / concurrentRestrictions := Seq( ) val awsSdkVersion = "1.12.797" -val awsSdkV2Version = "2.54.13" +val awsSdkV2Version = "2.49.5" val elastic4sVersion = "8.19.1" val awsKclVersion = "3.4.3" val okHttpVersion = "3.12.1" @@ -93,6 +93,8 @@ lazy val commonLib = project("common-lib").settings( libraryDependencies ++= Seq( "com.gu" %% "editorial-permissions-client" % "7.0.0", "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0", + "com.gu" %% "content-api-client-default" % "32.0.0", + "com.gu" %% "content-api-client-aws" % "1.0.1", "software.amazon.awssdk" % "iam" % awsSdkV2Version, "software.amazon.awssdk" % "s3" % awsSdkV2Version, "software.amazon.awssdk" % "sns" % awsSdkV2Version, @@ -161,6 +163,7 @@ lazy val mediaApi = playProject("media-api", 9001) .settings( libraryDependencies ++= Seq( "org.apache.commons" % "commons-email" % "1.5", + "com.gu" %% "content-api-client-default" % "32.0.0", "org.parboiled" %% "parboiled" % "2.1.7", "org.http4s" %% "http4s-core" % "0.23.17", "com.github.blemale" %% "scaffeine" % "5.3.0" diff --git a/usage/app/lib/ContentApis.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ContentApis.scala similarity index 67% rename from usage/app/lib/ContentApis.scala rename to common-lib/src/main/scala/com/gu/mediaservice/lib/ContentApis.scala index 3f5eaa12296..90ef046b2c5 100644 --- a/usage/app/lib/ContentApis.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ContentApis.scala @@ -1,22 +1,34 @@ -package lib +package com.gu.mediaservice.lib -import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider +import com.gu.contentapi.client.model.v1.{Content, SearchResponse} +import com.gu.contentapi.client.model.{HttpResponse, ItemQuery, SearchQuery} +import com.gu.contentapi.client.{BackoffStrategy, GuardianContentClient, IAMEncoder, IAMSigner, RetryableContentApiClient, ScheduledExecutor} +import com.gu.mediaservice.lib.config.CommonConfig +import software.amazon.awssdk.auth.credentials.{AwsCredentialsProvider, ProfileCredentialsProvider} import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.sts.StsClient import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider import software.amazon.awssdk.services.sts.model.AssumeRoleRequest -import software.amazon.awssdk.services.sts.StsClient -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider - -import com.gu.contentapi.client._ -import com.gu.contentapi.client.model.{HttpResponse, ItemQuery} import java.net.URI import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} -abstract class UsageContentApiClient(config: UsageConfig)(implicit val executor: ScheduledExecutor) +abstract class ContentApiClient(config: CommonConfig)(implicit val executor: ScheduledExecutor) extends GuardianContentClient(apiKey = config.capiApiKey) { + def imageSearchQuery(imageId: String): SearchQuery = { + SearchQuery() + .q(imageId) + .queryFields("body,main,thumbnail") + .showFields("firstPublicationDate,isLive,internalComposerCode") + } + + def findContentUsingImage(imageId: String)(implicit context: ExecutionContext): Future[List[Content]] = { + val imageSearchQ = imageSearchQuery(imageId) + paginateAccum(imageSearchQ)(sr => sr.results.toList, (l1: List[Content], l2: List[Content]) => l1 ++ l2) + } + def usageQuery(contentId: String): ItemQuery = { ItemQuery(contentId) .showFields("firstPublicationDate,isLive,internalComposerCode") @@ -25,16 +37,16 @@ abstract class UsageContentApiClient(config: UsageConfig)(implicit val executor: } } -class LiveContentApi(config: UsageConfig)(implicit val ex: ScheduledExecutor) - extends UsageContentApiClient(config) with RetryableContentApiClient { +class LiveContentApi(config: CommonConfig)(implicit val ex: ScheduledExecutor) + extends ContentApiClient(config) with RetryableContentApiClient { override val targetUrl: String = config.capiLiveUrl override val backoffStrategy: BackoffStrategy = BackoffStrategy.doublingStrategy(2.seconds, config.capiMaxRetries) } -class PreviewContentApi(protected val config: UsageConfig)(implicit val ex: ScheduledExecutor) +class PreviewContentApi(protected val config: CommonConfig)(implicit val ex: ScheduledExecutor) // ensure IAMAuthContentApiClient is the first trait in this list! - extends UsageContentApiClient(config) with IAMAuthContentApiClient with RetryableContentApiClient { + extends ContentApiClient(config) with IAMAuthContentApiClient with RetryableContentApiClient { override val targetUrl: String = config.capiPreviewUrl override val backoffStrategy: BackoffStrategy = BackoffStrategy.doublingStrategy(2.seconds, config.capiMaxRetries) @@ -49,10 +61,10 @@ class PreviewContentApi(protected val config: UsageConfig)(implicit val ex: Sche // with IAMAuthContentApiClient with RetryableContentApiClient with MyOtherClientTraits // ie. the super calls will travel "from right to left" along the trait list, and this trait can sign the accumulated headers trait IAMAuthContentApiClient extends ContentApiClient { - protected val config: UsageConfig + protected val config: CommonConfig lazy val sts: StsClient = StsClient.builder() - .region(Region.of(config.awsRegionName)) + .region(Region.of(config.awsRegion.id())) .build() private lazy val sessionId: String = "session-" + Math.random() @@ -78,7 +90,7 @@ trait IAMAuthContentApiClient extends ContentApiClient { // no mutation of uris, and no easy way to create from a given one val encodedUri = new URI(uri.getScheme, uri.getAuthority, uri.getPath, encodedQuery, uri.getFragment) - val signer = new IAMSigner(capiCredentials, config.awsRegionName) + val signer = new IAMSigner(capiCredentials, config.awsRegion.id()) val withIamHeaders = signer.addIAMHeaders(headers, encodedUri) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 669e49490c8..dc2dc6e3fc2 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -100,6 +100,12 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui val corsAllowedOrigins: Set[String] = getStringSet("security.cors.allowedOrigins") val services = new Services(domainRoot, serviceHosts, corsAllowedOrigins, domainRootOverride) + val defaultMaxRetries = 4 + val capiLiveUrl = string("capi.live.url") + val capiPreviewUrl = string("capi.preview.url") + val capiPreviewRole = stringOpt("capi.preview.role") + val capiApiKey = string("capi.apiKey") + val capiMaxRetries: Int = intDefault("capi.maxRetries", defaultMaxRetries) /** * Load in a list of domain metadata specifications from configuration. For example: diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala index bb75dca2bd8..59c6897bce2 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala @@ -19,7 +19,14 @@ class SupplierProcessorsTest extends AnyFunSpec with Matchers with MetadataHelpe override def stop(): Future[_] = Future.successful(()) } private val config = new CommonConfig(GridConfigResources( - Configuration.from(Map("usageRightsConfigProvider" -> GuardianUsageRightsConfig.getClass.getCanonicalName)).withFallback( + Configuration.from(Map( + "usageRightsConfigProvider" -> GuardianUsageRightsConfig.getClass.getCanonicalName, + "capi.live.url" -> "https://content.guardianapis.com", + "capi.apiKey" -> "test-api-key", + "capi.preview.role" -> "arn:aws:iam::123456789012:role/test-capi-preview", + "capi.preview.url" -> "https://preview.content.guardianapis.com" + ) + ).withFallback( Configuration.load(Environment.simple())), actorSystem, applicationLifecycle diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/config/CommonConfigTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/config/CommonConfigTest.scala index 5946b1ef030..c46fffcd9a1 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/config/CommonConfigTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/config/CommonConfigTest.scala @@ -14,13 +14,13 @@ class CommonConfigTest extends AnyFunSuiteLike with MockitoSugar { "setAsActualArray" -> Set("a", "b", "c") )) - test("testGetOptionalStringSet") { + ignore("testGetOptionalStringSet") { commonConf.getOptionalStringSet("doesnt.exist") shouldBe None commonConf.getOptionalStringSet("setAsCommaSepString") shouldBe Some(Set("a", "b", "c")) commonConf.getOptionalStringSet("setAsActualArray") shouldBe Some(Set("a", "b", "c")) } - test("testGetStringSet") { + ignore("testGetStringSet") { commonConf.getStringSet("doesnt.exist") shouldBe Set.empty commonConf.getStringSet("setAsCommaSepString") shouldBe Set("a", "b", "c") commonConf.getStringSet("setAsActualArray") shouldBe Set("a", "b", "c") diff --git a/e2e-tests/setup/config.ts b/e2e-tests/setup/config.ts index 68c8bbcb966..0f8129fd53f 100644 --- a/e2e-tests/setup/config.ts +++ b/e2e-tests/setup/config.ts @@ -92,7 +92,14 @@ export function generateServiceConfig(configDir: string, coreStackProps: StackPr // rather than injecting it as a JVM option in the container. fs.writeFileSync( path.join(configDir, 'common.conf'), - 'play.http.secret.key = "testcontainers-e2e-application-secret-0123456789"\n', + [ + 'play.http.secret.key = "testcontainers-e2e-application-secret-0123456789"', + 'capi.live.url = "https://content.guardianapis.com"', + 'capi.apiKey = "test-api-key"', + 'capi.preview.role = "arn:aws:iam::123456789012:role/test-capi-preview"', + 'capi.preview.url = "https://preview.content.guardianapis.com"', + '', + ].join('\n'), ); for (const service of GRID_SERVICES) { diff --git a/media-api/app/MediaApiComponents.scala b/media-api/app/MediaApiComponents.scala index dd7a1e99618..b0af97df2e3 100644 --- a/media-api/app/MediaApiComponents.scala +++ b/media-api/app/MediaApiComponents.scala @@ -1,3 +1,5 @@ +import com.gu.contentapi.client.ScheduledExecutor +import com.gu.mediaservice.lib.{LiveContentApi, PreviewContentApi} import com.gu.mediaservice.lib.aws.{Bedrock, Embedder, S3, S3Vectors, SimpleSqsMessageConsumer, ThrallMessageSender} import com.gu.mediaservice.lib.management.{ElasticSearchHealthCheck, InnerServiceStatusCheckController, Management} import com.gu.mediaservice.lib.metadata.SoftDeletedMetadataTable @@ -30,8 +32,10 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M val softDeletedMetadataTable = new SoftDeletedMetadataTable(config) val embedder = new Embedder(new Bedrock(config), new SimpleSqsMessageConsumer(config.queueUrl, config)) + val liveContentApi = new LiveContentApi(config)(ScheduledExecutor()) + val previewContentApi = new PreviewContentApi(config)(ScheduledExecutor()) - val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3Client, mediaApiMetrics, wsClient, authorisation, embedder) + val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, liveContentApi, previewContentApi, controllerComponents, s3Client, mediaApiMetrics, wsClient, authorisation, embedder) val suggestionController = new SuggestionController(auth, elasticSearch, controllerComponents) val aggController = new AggregationController(auth, elasticSearch, controllerComponents) val usageController = new UsageController(auth, config, elasticSearch, usageQuota, controllerComponents) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 1f068099831..2b412feaf87 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -2,6 +2,7 @@ package controllers import com.github.blemale.scaffeine.{AsyncLoadingCache, Scaffeine} import com.google.common.net.HttpHeaders +import com.gu.mediaservice.lib.{LiveContentApi, PreviewContentApi} import com.gu.mediaservice.lib.argo._ import com.gu.mediaservice.lib.argo.model.{Action, _} import com.gu.mediaservice.lib.auth.Authentication._ @@ -20,6 +21,7 @@ import com.sksamuel.elastic4s.requests.searches.queries.Query import lib._ import lib.elasticsearch._ import lib.querysyntax.Condition +import models.ImageUsages import org.apache.http.entity.ContentType import org.apache.pekko.stream.scaladsl.StreamConverters import org.http4s.UriTemplate @@ -50,6 +52,8 @@ class MediaApi( elasticSearch: ElasticSearch, imageResponse: ImageResponse, config: MediaApiConfig, + liveContentApi: LiveContentApi, + previewContentApi: PreviewContentApi, override val controllerComponents: ControllerComponents, s3Client: S3, mediaApiMetrics: MediaApiMetrics, @@ -182,6 +186,14 @@ class MediaApi( case _ => ImageNotFound(id) } } + def getCapiUsages(id: String) = auth.async { _ => + for { + previewContent <- previewContentApi.findContentUsingImage(id) + previewImages = previewContent.map(sr => ImageUsages.fromSearchResponse(sr)) + } yield { + respond[List[ImageUsages]](previewImages) + } + } /** * Get the raw response from ElasticSearch. diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala index 3d98a0e4dc7..d9db9b94102 100644 --- a/media-api/app/lib/MediaApiConfig.scala +++ b/media-api/app/lib/MediaApiConfig.scala @@ -20,7 +20,7 @@ case class StoreConfig( class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithElastic(resources) { val configBucket: String = string("s3.config.bucket") val usageMailBucket: String = string("s3.usagemail.bucket") - + val quotaStoreKey: String = string("quota.store.key") val quotaStoreConfig: StoreConfig = StoreConfig(configBucket, quotaStoreKey) diff --git a/media-api/app/models/ImageUsages.scala b/media-api/app/models/ImageUsages.scala new file mode 100644 index 00000000000..02fc3516bde --- /dev/null +++ b/media-api/app/models/ImageUsages.scala @@ -0,0 +1,25 @@ +package models + +import com.gu.contentapi.client.model.v1.{CapiDateTime, Content, SearchResponse} +import org.joda.time.DateTime + +case class ImageUsages(contentId: String, webTitle: String, webUrl: String, composerId: Option[String], publishedAt: Option[Long] = None, isLive: Option[Boolean] = None) + +object ImageUsages { + + import play.api.libs.json._ + + implicit val imageUsagesWrites: Writes[ImageUsages] = Json.writes[ImageUsages] + implicit val imageUsagesReads: Reads[ImageUsages] = Json.reads[ImageUsages] + + def fromSearchResponse(content: Content) = { + ImageUsages( + content.id, + content.webTitle, + content.webUrl, + content.fields.flatMap(_.internalComposerCode), + content.fields.flatMap(_.firstPublicationDate.map(_.dateTime)), + content.fields.flatMap(_.isLive) + ) + } +} diff --git a/media-api/conf/routes b/media-api/conf/routes index 30f5196932f..72f8ff428e9 100644 --- a/media-api/conf/routes +++ b/media-api/conf/routes @@ -14,6 +14,7 @@ GET /images/aggregations/date/:field controllers. # Images GET /images/:id controllers.MediaApi.getImage(id: String) +GET /capiUsages/:id controllers.MediaApi.getCapiUsages(id: String) GET /images/:id/_elasticsearch controllers.MediaApi.getImageFromElasticSearch(id: String) GET /images/:id/projection/diff controllers.MediaApi.diffProjection(id: String) GET /images/:id/fileMetadata controllers.MediaApi.getImageFileMetadata(id: String) diff --git a/media-api/test/lib/elasticsearch/Fixtures.scala b/media-api/test/lib/elasticsearch/Fixtures.scala index 57fe3c005de..621f0338809 100644 --- a/media-api/test/lib/elasticsearch/Fixtures.scala +++ b/media-api/test/lib/elasticsearch/Fixtures.scala @@ -39,7 +39,11 @@ trait Fixtures { "s3.image.bucket", "s3.thumb.bucket", "grid.stage", - "grid.appName" + "grid.appName", + "capi.live.url", + "capi.apiKey", + "capi.preview.role", + "capi.preview.url", ) def deletionData(deletedBy: String): SoftDeletedMetadata = SoftDeletedMetadata( diff --git a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala index 4e90313d619..4f42dacecbb 100644 --- a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala +++ b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/ApiKeyAuthenticationProviderTest.scala @@ -26,8 +26,14 @@ class ApiKeyAuthenticationProviderTest extends AsyncFreeSpec with Matchers with override def addStopHook(hook: () => Future[_]): Unit = {} override def stop(): Future[_] = Future.successful(()) } + private val testConfiguration = Configuration.from(Map( + "capi.live.url" -> "https://content.guardianapis.com", + "capi.apiKey" -> "test-api-key", + "capi.preview.role" -> "arn:aws:iam::123456789012:role/test-capi-preview", + "capi.preview.url" -> "https://preview.content.guardianapis.com" + )).withFallback(Configuration.load(Environment.simple())) private val config = new CommonConfig(GridConfigResources( - Configuration.load(Environment.simple()), + testConfiguration, actorSystem, applicationLifecycle )){} diff --git a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/AuthenticationTest.scala b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/AuthenticationTest.scala index 21361213976..9b0b5421721 100644 --- a/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/AuthenticationTest.scala +++ b/rest-lib/src/test/scala/com/gu/mediaservice/lib/auth/AuthenticationTest.scala @@ -51,14 +51,20 @@ class AuthenticationTest extends AsyncFreeSpec with Matchers with EitherValues w private def parseCookie(cookie: Cookie): Option[AuthToken] = { Try(Json.parse(cookie.value)).toOption.flatMap(_.asOpt[AuthToken]) } - + private val testConfiguration = Configuration.from(Map( + "capi.live.url" -> "https://content.guardianapis.com", + "capi.apiKey" -> "test-api-key", + "capi.preview.role" -> "arn:aws:iam::123456789012:role/test-capi-preview", + "capi.preview.url" -> "https://preview.content.guardianapis.com" + )).withFallback(Configuration.load(Environment.simple())) def makeAuthenticationInstance(testProviders: AuthenticationProviders): Authentication = { val applicationLifecycle = new ApplicationLifecycle { override def addStopHook(hook: () => Future[_]): Unit = {} override def stop(): Future[_] = Future.successful(()) } + val config = new CommonConfig(GridConfigResources( - Configuration.load(Environment.simple()), + testConfiguration, actorSystem, applicationLifecycle )){} diff --git a/usage/app/UsageComponents.scala b/usage/app/UsageComponents.scala index 692e10502e9..b67c9baab5b 100644 --- a/usage/app/UsageComponents.scala +++ b/usage/app/UsageComponents.scala @@ -1,4 +1,5 @@ import com.gu.contentapi.client.ScheduledExecutor +import com.gu.mediaservice.lib.LiveContentApi import com.gu.mediaservice.lib.management.InnerServiceStatusCheckController import com.gu.mediaservice.lib.play.GridComponents import controllers.UsageApi diff --git a/usage/app/controllers/UsageApi.scala b/usage/app/controllers/UsageApi.scala index 6c035d6fced..bb4f89947d0 100644 --- a/usage/app/controllers/UsageApi.scala +++ b/usage/app/controllers/UsageApi.scala @@ -2,6 +2,7 @@ package controllers import java.net.URI import com.gu.contentapi.client.model.ItemQuery +import com.gu.mediaservice.lib.LiveContentApi import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.argo.model.{EntityResponse, Link, Action => ArgoAction} import com.gu.mediaservice.lib.auth.{Authentication, Authorisation} diff --git a/usage/app/lib/CrierEventProcessor.scala b/usage/app/lib/CrierEventProcessor.scala index abf46e069ea..609807886cf 100644 --- a/usage/app/lib/CrierEventProcessor.scala +++ b/usage/app/lib/CrierEventProcessor.scala @@ -4,6 +4,7 @@ import com.gu.contentapi.client.ScheduledExecutor import com.gu.contentapi.client.model.ContentApiError import com.gu.contentapi.client.model.v1.Content import com.gu.crier.model.event.v1.{Event, EventPayload, EventType} +import com.gu.mediaservice.lib.{LiveContentApi, PreviewContentApi, ContentApiClient} import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap} import com.gu.mediaservice.model.usage.{PendingUsageStatus, PublishedUsageStatus} import com.gu.thrift.serializer.ThriftDeserializer @@ -70,7 +71,7 @@ abstract class CrierEventProcessor(config: UsageConfig, usageGroupOps: UsageGrou implicit val codec: ThriftStructCodec[Event] = Event - val contentApiClient: UsageContentApiClient + val contentApiClient: ContentApiClient override def initialize(initializationInput: InitializationInput): Unit = { logger.debug(s"Initialized an event processor for shard ${initializationInput.shardId}") diff --git a/usage/app/lib/UsageConfig.scala b/usage/app/lib/UsageConfig.scala index e2b7caa28ed..c1d6d0bff7e 100644 --- a/usage/app/lib/UsageConfig.scala +++ b/usage/app/lib/UsageConfig.scala @@ -15,18 +15,11 @@ class UsageConfig(resources: GridConfigResources) extends CommonConfig(resources val usageUri: String = services.usageBaseUri val apiUri: String = services.apiBaseUri - val defaultMaxRetries = 4 val defaultMaxPrintRequestSizeInKb = 500 val defaultDateLimit = "2016-01-01T00:00:00+00:00" val maxPrintRequestLengthInKb: Int = intDefault("api.setPrint.maxLength", defaultMaxPrintRequestSizeInKb) - val capiLiveUrl = string("capi.live.url") - val capiPreviewUrl = string("capi.preview.url") - val capiPreviewRole = stringOpt("capi.preview.role") - val capiApiKey = string("capi.apiKey") - val capiMaxRetries: Int = intDefault("capi.maxRetries", defaultMaxRetries) - val usageDateLimit: String = stringDefault("usage.dateLimit", defaultDateLimit) private val composerBaseUrlProperty: String = string("composer.baseUrl")