diff --git a/build.sbt b/build.sbt index dad2f1577a..82947ea1db 100644 --- a/build.sbt +++ b/build.sbt @@ -161,6 +161,9 @@ lazy val mediaApi = playProject("media-api", 9001) .settings( libraryDependencies ++= Seq( "org.apache.commons" % "commons-email" % "1.5", + "com.gu" %% "content-api-client-default" % "32.0.0", + "com.gu" %% "content-api-client-aws" % "1.0.1", + "software.amazon.awssdk" % "sts" % awsSdkV2Version, "org.parboiled" %% "parboiled" % "2.1.7", "org.http4s" %% "http4s-core" % "0.23.17", "com.github.blemale" %% "scaffeine" % "5.3.0" diff --git a/dev/script/generate-config/service-config.js b/dev/script/generate-config/service-config.js index b5820bba78..4f8f1b49b8 100644 --- a/dev/script/generate-config/service-config.js +++ b/dev/script/generate-config/service-config.js @@ -222,6 +222,11 @@ function getMediaApiConfig(config) { |} |metrics.request.enabled=false |syndication.review.useRuntimeFieldsFix=true + |capi.preview.url="previewurl" + |capi.preview.role="previewrole" + |capi.maxRetries=3 + |capi.apiKey="apikey" + |composer.domain="composerdomain" |`; } diff --git a/media-api/app/MediaApiComponents.scala b/media-api/app/MediaApiComponents.scala index dd7a1e9961..904972030c 100644 --- a/media-api/app/MediaApiComponents.scala +++ b/media-api/app/MediaApiComponents.scala @@ -1,3 +1,4 @@ +import com.gu.contentapi.client.ScheduledExecutor import com.gu.mediaservice.lib.aws.{Bedrock, Embedder, S3, S3Vectors, SimpleSqsMessageConsumer, ThrallMessageSender} import com.gu.mediaservice.lib.management.{ElasticSearchHealthCheck, InnerServiceStatusCheckController, Management} import com.gu.mediaservice.lib.metadata.SoftDeletedMetadataTable @@ -30,8 +31,9 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M val softDeletedMetadataTable = new SoftDeletedMetadataTable(config) val embedder = new Embedder(new Bedrock(config), new SimpleSqsMessageConsumer(config.queueUrl, config)) + val previewContentApi = new PreviewContentApi(config)(ScheduledExecutor()) - val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, controllerComponents, s3Client, mediaApiMetrics, wsClient, authorisation, embedder) + val mediaApi = new MediaApi(auth, messageSender, softDeletedMetadataTable, elasticSearch, imageResponse, config, previewContentApi, controllerComponents, s3Client, mediaApiMetrics, wsClient, authorisation, embedder) val suggestionController = new SuggestionController(auth, elasticSearch, controllerComponents) val aggController = new AggregationController(auth, elasticSearch, controllerComponents) val usageController = new UsageController(auth, config, elasticSearch, usageQuota, controllerComponents) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 1f06809983..1106241f03 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -20,6 +20,7 @@ import com.sksamuel.elastic4s.requests.searches.queries.Query import lib._ import lib.elasticsearch._ import lib.querysyntax.Condition +import models.UsagesInContent import org.apache.http.entity.ContentType import org.apache.pekko.stream.scaladsl.StreamConverters import org.http4s.UriTemplate @@ -50,6 +51,7 @@ class MediaApi( elasticSearch: ElasticSearch, imageResponse: ImageResponse, config: MediaApiConfig, + previewContentApi: PreviewContentApi, override val controllerComponents: ControllerComponents, s3Client: S3, mediaApiMetrics: MediaApiMetrics, @@ -118,6 +120,7 @@ class MediaApi( val maybeLoaderLink: Option[Link] = Some(Link("loader", config.loaderUri)).filter(_ => userCanUpload) val maybeArchiveLink: Option[Link] = Some(Link("archive", s"${config.metadataUri}/metadata/{id}/archived")).filter(_ => userCanArchive) + val maybeCapiUsagesLink: Option[Link] = Some(Link("capiUsages", s"${config.rootUri}/capiUsages/{id}")).filter(_ => config.takedownEnabled) val indexLinks = List( searchLink, Link("image", s"${config.rootUri}/images/{id}"), @@ -135,7 +138,7 @@ class MediaApi( Link("syndicate-image", s"${config.rootUri}/images/{id}/{partnerName}/{startPending}/syndicateImage"), Link("undelete", s"${config.rootUri}/images/{id}/undelete"), Link("usage", config.usageUri), - ) ++ maybeLoaderLink.toList ++ maybeArchiveLink.toList + ) ++ maybeLoaderLink.toList ++ maybeArchiveLink.toList ++ maybeCapiUsagesLink.toList respond(indexData, indexLinks) } @@ -182,6 +185,15 @@ class MediaApi( case _ => ImageNotFound(id) } } + def getCapiUsages(id: String) = auth.async { _ => + val composerDomain = config.composerDomain + for { + previewContent <- previewContentApi.findContentUsingImage(id) + previewImages = previewContent.map(sr => UsagesInContent.fromSearchResponse(sr, composerDomain)) + } yield { + respond[List[UsagesInContent]](previewImages) + } + } /** * Get the raw response from ElasticSearch. diff --git a/media-api/app/lib/ContentApi.scala b/media-api/app/lib/ContentApi.scala new file mode 100644 index 0000000000..9bac90beac --- /dev/null +++ b/media-api/app/lib/ContentApi.scala @@ -0,0 +1,85 @@ +package lib + +import com.gu.contentapi.client.model.v1.Content +import com.gu.contentapi.client.model.{HttpResponse, SearchQuery} +import com.gu.contentapi.client._ +import software.amazon.awssdk.auth.credentials.{AwsCredentialsProvider, ProfileCredentialsProvider} +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.sts.StsClient +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest + +import java.net.URI +import scala.concurrent.duration.DurationInt +import scala.concurrent.{ExecutionContext, Future} + +abstract class ContentApiClient(config: MediaApiConfig)(implicit val executor: ScheduledExecutor) + extends GuardianContentClient(apiKey = config.capiApiKey) { + + def imageSearchQuery(imageId: String): SearchQuery = { + SearchQuery() + .q(imageId) + .queryFields("body,main,thumbnail") + .showFields("firstPublicationDate,isLive,internalComposerCode") + } + + def findContentUsingImage(imageId: String)(implicit context: ExecutionContext): Future[List[Content]] = { + val imageSearchQ = imageSearchQuery(imageId) + paginateAccum(imageSearchQ)(sr => sr.results.toList, (l1: List[Content], l2: List[Content]) => l1 ++ l2) + } + +} + +class PreviewContentApi(protected val config: MediaApiConfig)(implicit val ex: ScheduledExecutor) + // ensure IAMAuthContentApiClient is the first trait in this list! + extends ContentApiClient(config) with IAMAuthContentApiClient with RetryableContentApiClient { + + override val targetUrl: String = config.capiPreviewUrl + override val backoffStrategy: BackoffStrategy = BackoffStrategy.doublingStrategy(2.seconds, config.capiMaxRetries) +} + +// order of mixing is important. Some client traits (notably RetryableContentApiClient!) +// also override get, adding header(s) (and could potentially edit the uri too) before calling super.get(). Those +// traits must be executed BEFORE this trait, so that the get override in this trait +// receives the headers that will actually be sent over the wire. +// so any class mixing this in should have it first in the list of traits, eg. +// class MyCapiClient extends GuardianContentApiClient(apiKey) +// with IAMAuthContentApiClient with RetryableContentApiClient with MyOtherClientTraits +// ie. the super calls will travel "from right to left" along the trait list, and this trait can sign the accumulated headers +trait IAMAuthContentApiClient extends ContentApiClient { + protected val config: MediaApiConfig + + lazy val sts: StsClient = StsClient.builder() + .region(Region.of(config.awsRegionName)) + .build() + + private lazy val sessionId: String = "session-" + Math.random() + lazy val capiCredentials: AwsCredentialsProvider = + config.capiPreviewRole.map(arn => { + + val assumeRoleRequest = AssumeRoleRequest.builder().roleArn(arn).roleSessionName(sessionId).build() + + StsAssumeRoleCredentialsProvider.builder() + .refreshRequest(assumeRoleRequest) + .stsClient(sts) + .build() + }).getOrElse(ProfileCredentialsProvider.create("capi")) // will be used if stream is ever run locally (unusual) + + abstract override def get( + url: String, + headers: Map[String, String] + )(implicit context: ExecutionContext): Future[HttpResponse] = { + + val uri = new URI(url) + val encodedQuery = IAMEncoder.encodeParams(uri.getQuery) + + // no mutation of uris, and no easy way to create from a given one + val encodedUri = new URI(uri.getScheme, uri.getAuthority, uri.getPath, encodedQuery, uri.getFragment) + + val signer = new IAMSigner(capiCredentials, config.awsRegion.id()) + + val withIamHeaders = signer.addIAMHeaders(headers, encodedUri) + + super.get(encodedUri.toString, withIamHeaders) + } +} diff --git a/media-api/app/lib/MediaApiConfig.scala b/media-api/app/lib/MediaApiConfig.scala index 3d98a0e4dc..2d14e6dbab 100644 --- a/media-api/app/lib/MediaApiConfig.scala +++ b/media-api/app/lib/MediaApiConfig.scala @@ -26,6 +26,7 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla //Lazy allows this to be empty and not break things unless used somewhere lazy val imgPublishingBucket = string("publishing.image.bucket") + val awsRegionName = string("aws.region") val imageBucket: String = string("s3.image.bucket") val thumbBucket: String = string("s3.thumb.bucket") @@ -56,7 +57,12 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla val collectionsUri: String = services.collectionsBaseUri val requiredMetadata = NonEmptyList("credit", "description", "usageRights") - + val capiPreviewUrl = string("capi.preview.url") + val capiPreviewRole = stringOpt("capi.preview.role") + val defaultMaxRetries = 4 + val capiMaxRetries: Int = intDefault("capi.maxRetries", defaultMaxRetries) + val capiApiKey = string("capi.apiKey") + val composerDomain = string("composer.domain") val syndicationStartDate: Option[DateTime] = Try { stringOpt("syndication.start").map(d => DateTime.parse(d).withTimeAtStartOfDay()) }.toOption.flatten @@ -73,6 +79,7 @@ class MediaApiConfig(resources: GridConfigResources) extends CommonConfigWithEla configuration.getOptional[Map[String, String]]("usageRestrictions").getOrElse(Map.empty) val restrictDownload: Boolean = boolean("restrictDownload") + val takedownEnabled: Boolean = booleanOpt("image.takedown.enabled").getOrElse(false) val queueUrl: String = stringOpt("sqs.embedder.queue.url").getOrElse("") diff --git a/media-api/app/models/UsagesInContent.scala b/media-api/app/models/UsagesInContent.scala new file mode 100644 index 0000000000..b6efc1bd1f --- /dev/null +++ b/media-api/app/models/UsagesInContent.scala @@ -0,0 +1,25 @@ +package models + +import com.gu.contentapi.client.model.v1.Content +import lib.MediaApiConfig + +case class UsagesInContent(contentId: String, webTitle: String, webUrl: String, composerUrl: Option[String], publishedAt: Option[Long] = None, isLive: Option[Boolean] = None) + +object UsagesInContent { + + import play.api.libs.json._ + + implicit val imageUsagesWrites: Writes[UsagesInContent] = Json.writes[UsagesInContent] + implicit val imageUsagesReads: Reads[UsagesInContent] = Json.reads[UsagesInContent] + + def fromSearchResponse(content: Content, composerDomain: String) = { + UsagesInContent( + content.id, + content.webTitle, + content.webUrl, + content.fields.flatMap(_.internalComposerCode.map(code => s"${composerDomain}content/${code}")), + content.fields.flatMap(_.firstPublicationDate.map(_.dateTime)), + content.fields.flatMap(_.isLive) + ) + } +} diff --git a/media-api/conf/routes b/media-api/conf/routes index 30f5196932..72f8ff428e 100644 --- a/media-api/conf/routes +++ b/media-api/conf/routes @@ -14,6 +14,7 @@ GET /images/aggregations/date/:field controllers. # Images GET /images/:id controllers.MediaApi.getImage(id: String) +GET /capiUsages/:id controllers.MediaApi.getCapiUsages(id: String) GET /images/:id/_elasticsearch controllers.MediaApi.getImageFromElasticSearch(id: String) GET /images/:id/projection/diff controllers.MediaApi.diffProjection(id: String) GET /images/:id/fileMetadata controllers.MediaApi.getImageFileMetadata(id: String) diff --git a/media-api/test/lib/elasticsearch/Fixtures.scala b/media-api/test/lib/elasticsearch/Fixtures.scala index 57fe3c005d..bdc5ca3fb9 100644 --- a/media-api/test/lib/elasticsearch/Fixtures.scala +++ b/media-api/test/lib/elasticsearch/Fixtures.scala @@ -21,6 +21,7 @@ trait Fixtures { "usageRights" -> Map( "applicable" -> List() ), + "aws.region" -> "eu-west-1", "usageRightsConfigProvider" -> "com.gu.mediaservice.lib.config.RuntimeUsageRightsConfig" ) val NOT_USED_IN_TEST = "not used in test" @@ -39,7 +40,12 @@ trait Fixtures { "s3.image.bucket", "s3.thumb.bucket", "grid.stage", - "grid.appName" + "grid.appName", + "capi.live.url", + "capi.apiKey", + "capi.preview.role", + "capi.preview.url", + "composer.domain" ) def deletionData(deletedBy: String): SoftDeletedMetadata = SoftDeletedMetadata(