CMR-7648: Add storage extension (#397)

* Add storage extension

* Remove importing StacExtension

Author: EthanGrants

src/domains/__tests__/collections.spec.ts

@@ -164,11 +164,13 @@ describe("collectionsToStac", () => {
             href: "s3://op-test-protected/SOME_DATA.002",
             title: "op_test_protected_SOME_DATA_002",
             roles: ["data"],
+            "storage:refs": ["aws"],
           },
           s3_op_test_public_SOME_DATA_V003: {
             href: "s3://op-test-public/SOME_DATA.V003",
             title: "op_test_public_SOME_DATA_V003",
             roles: ["data"],
+            "storage:refs": ["aws"],
           },
           s3_credentials: {
             href: "https://data.lpdaac.earthdatacloud.nasa.gov/s3credentials",
@@ -204,18 +206,75 @@ describe("collectionsToStac", () => {
             "https://data.lpdaac.earthdatacloud.nasa.gov/s3credentialsREADME",
         };
 
+        const stacCollection: any = collectionToStac(base);
+        expect(stacCollection).to.have.deep.property("assets", {
+          s3_op_test_protected_SOME_DATA_002: {
+            href: "s3://op-test-protected/SOME_DATA.002",
+            title: "op_test_protected_SOME_DATA_002",
+            roles: ["data"],
+            "storage:refs": ["aws"],
+          },
+          s3_op_test_public_SOME_DATA_V003: {
+            href: "s3://op-test-public/SOME_DATA.V003",
+            title: "op_test_public_SOME_DATA_V003",
+            roles: ["data"],
+            "storage:refs": ["aws"],
+          },
+          s3_credentials: {
+            href: "https://data.lpdaac.earthdatacloud.nasa.gov/s3credentials",
+            roles: ["metadata"],
+            title: "S3 credentials API endpoint",
+          },
+          s3_credentials_documentation: {
+            href: "https://data.lpdaac.earthdatacloud.nasa.gov/s3credentialsREADME",
+            roles: ["metadata"],
+            title: "S3 credentials API endpoint documentation",
+          },
+          metadata: {
+            href: "undefined/search/concepts/C00000000-TEST_PROV.xml",
+            roles: ["metadata"],
+            title: "CMR XML metadata for C00000000-TEST_PROV",
+            type: "application/xml",
+          },
+        });
+        expect(stacCollection).to.have.deep.property("storage:schemes", {
+          aws: {
+            type: "aws-s3",
+            platform: "https://{bucket}.s3.{region}.amazonaws.com",
+            bucket: "op-test-protected",
+            region: "us-west-2",
+          },
+        });
+      });
+    });
+    describe("given multiple S3 links with public and then protected buckets", () => {
+      it("should return a STAC collection with storage:refs preferring the protected bucket", () => {
+        const [base] = generateCollections(1);
+        base.directDistributionInformation = {
+          region: "us-west-2",
+          s3BucketAndObjectPrefixNames: [
+            "s3://op-test-public/SOME_DATA.V003",
+            "s3://op-test-protected/SOME_DATA.002",
+          ],
+          s3CredentialsApiEndpoint: "https://data.lpdaac.earthdatacloud.nasa.gov/s3credentials",
+          s3CredentialsApiDocumentationUrl:
+            "https://data.lpdaac.earthdatacloud.nasa.gov/s3credentialsREADME",
+        };
+
         const stacCollection: any = collectionToStac(base);
 
         expect(stacCollection).to.have.deep.property("assets", {
           s3_op_test_protected_SOME_DATA_002: {
             href: "s3://op-test-protected/SOME_DATA.002",
             title: "op_test_protected_SOME_DATA_002",
             roles: ["data"],
+            "storage:refs": ["aws"],
           },
           s3_op_test_public_SOME_DATA_V003: {
             href: "s3://op-test-public/SOME_DATA.V003",
             title: "op_test_public_SOME_DATA_V003",
             roles: ["data"],
+            "storage:refs": ["aws"],
           },
           s3_credentials: {
             href: "https://data.lpdaac.earthdatacloud.nasa.gov/s3credentials",
@@ -234,6 +293,15 @@ describe("collectionsToStac", () => {
             type: "application/xml",
           },
         });
+
+        expect(stacCollection).to.have.deep.property("storage:schemes", {
+          aws: {
+            type: "aws-s3",
+            platform: "https://{bucket}.s3.{region}.amazonaws.com",
+            bucket: "op-test-protected", // select protected even if public bucket comes first
+            region: "us-west-2",
+          },
+        });
       });
     });
   });

src/domains/__tests__/items.spec.ts

@@ -3,6 +3,7 @@ const { expect } = chai;
 
 import { granuleToStac } from "../items";
 import { generateGranules } from "../../utils/testUtils";
+import { RelatedUrlType, UrlContentType } from "../../models/GraphQLModels";
 
 const [baseGranule] = generateGranules(1);
 
@@ -268,4 +269,112 @@ describe("granuleToStac", () => {
       });
     });
   });
+  describe("given it provides AWS S3 links in related urls", () => {
+    it("should return a STACItem", () => {
+      const [granule] = generateGranules(1);
+      granule.relatedUrls?.push(
+        {
+          urlContentType: UrlContentType.DISTRIBUTION_URL,
+          url: "s3://myBucket/path/to/object/A.01.tif",
+          description: "S3 Download",
+          type: RelatedUrlType.GET_DATA_VIA_DIRECT_ACCESS,
+        },
+        {
+          urlContentType: UrlContentType.DISTRIBUTION_URL,
+          url: "s3://myBucket/path/to/object/A.02.tif",
+          description: "S3 Download",
+          type: RelatedUrlType.GET_DATA_VIA_DIRECT_ACCESS,
+        }
+      );
+      const item = granuleToStac(granule);
+      expect(item).to.deep.equal({
+        type: "Feature",
+        id: granule.title,
+        stac_version: "1.0.0",
+        stac_extensions: ["https://stac-extensions.github.io/storage/v2.0.0/schema.json"],
+        bbox: [-180, -90, 180, 90],
+        geometry: null,
+        properties: {
+          datetime: "2009-09-14T00:00:00.000Z",
+          start_datetime: "2009-09-14T00:00:00.000Z",
+          end_datetime: "2010-09-14T00:00:00.000Z",
+          "storage:schemes": {
+            aws: {
+              bucket: "myBucket",
+              platform: "https://{bucket}.s3.{region}.amazonaws.com",
+              region: "us-west-2",
+              type: "aws-s3",
+            },
+          },
+        },
+        collection: "TEST_COLLECTION_1",
+        links: [
+          {
+            href: "undefined/search/concepts/G000000000-TEST_PROV.json",
+            rel: "via",
+            title: "CMR JSON metadata for item",
+            type: "application/json",
+          },
+          {
+            href: "undefined/search/concepts/G000000000-TEST_PROV.umm_json",
+            rel: "via",
+            title: "CMR UMM_JSON metadata for item",
+            type: "application/vnd.nasa.cmr.umm+json",
+          },
+        ],
+        assets: {
+          B09: {
+            description: "Browse image for Earthdata Search",
+            href: "ftp://e4ftl014.cr.usgs.gov/MODIS_Composites/MOTA/.B09.tif",
+            title: "Direct Download [0]",
+            roles: ["data"],
+          },
+          asset_key: {
+            description: "Example of bad url data",
+            href: "ftp://e4ftl015/ExampleBadUrl",
+            title: "Direct Download [1]",
+            roles: ["data"],
+          },
+          s3_01: {
+            href: "s3://myBucket/path/to/object/A.01.tif",
+            title: "S3 Direct Download [0]",
+            description: "S3 Download",
+            roles: ["data"],
+            "storage:refs": ["aws"],
+          },
+          s3_02: {
+            href: "s3://myBucket/path/to/object/A.02.tif",
+            title: "S3 Direct Download [1]",
+            description: "S3 Download",
+            roles: ["data"],
+            "storage:refs": ["aws"],
+          },
+          provider_metadata: {
+            href: "ftp://e4ftl01.cr.usgs.gov/MODIS_Composites/MOTA/MCD43A4.005/2009.09.14/MCD43A4.A2009257.h29v03.005.2009276045828.hdf.xml",
+            description: "metadata",
+            title: "Provider Metadata",
+            roles: ["metadata"],
+          },
+          thumbnail_0: {
+            href: "ftp://e4ftl01.cr.usgs.gov/MODIS_Composites/MOTA/MCD43A4.005/2009.09.14/MCD43A4.A2009257.h29v03.005.2009276045828.vinr.img",
+            title: "Thumbnail [0]",
+            description: "Browse image for Earthdata Search",
+            roles: ["thumbnail"],
+          },
+          thumbnail_1: {
+            href: "ftp://e4ftl012/ExampleBadUrl",
+            title: "Thumbnail [1]",
+            description: "Browse image for Earthdata Search",
+            roles: ["thumbnail"],
+          },
+          metadata: {
+            href: "undefined/search/concepts/G000000000-TEST_PROV.xml",
+            roles: ["metadata"],
+            title: "CMR XML metadata for G000000000-TEST_PROV",
+            type: "application/xml",
+          },
+        },
+      });
+    });
+  });
 });

src/domains/__tests__/stac.spec.ts

@@ -2,7 +2,7 @@ import chai from "chai";
 const { expect } = chai;
 import { stringify as stringifyQuery } from "qs";
 
-import { buildQuery, sortByToSortKeys, browseAssets } from "../stac";
+import { buildQuery, sortByToSortKeys, browseAssets, s3downloadAssets } from "../stac";
 import { RelatedUrlType, UrlContentType } from "../../models/GraphQLModels";
 import { SortObject } from "../../models/StacModels";
 
@@ -570,6 +570,45 @@ describe("browseAssets", () => {
   });
 });
 
+describe("s3downloadAssets", () => {
+  it("should extract S3 URL from relatedUrls", () => {
+    const granule = {
+      relatedUrls: [
+        {
+          type: RelatedUrlType.GET_DATA_VIA_DIRECT_ACCESS,
+          url: "s3://bucket-name/path/to/granule",
+          description: "S3 URL for granule",
+        },
+        {
+          type: RelatedUrlType.GET_DATA,
+          url: "https://not-s3.com/file",
+          description: "Not S3 URL",
+        },
+      ],
+    };
+    const result = s3downloadAssets(granule as any);
+    expect(result).to.have.deep.property("s3_asset_key", {
+      href: "s3://bucket-name/path/to/granule",
+      title: "S3 Direct Download",
+      roles: ["data"],
+      description: "S3 URL for granule",
+      "storage:refs": ["aws"],
+    });
+  });
+  it("should return undefined if no S3 URL is found", () => {
+    const granule = {
+      relatedUrls: [
+        {
+          type: RelatedUrlType.GET_DATA,
+          url: "https://not-s3.com/file",
+        },
+      ],
+    };
+    const result = s3downloadAssets(granule as any);
+    expect(result).to.deep.equal({});
+  });
+});
+
 describe("conversions to GraphQL", () => {
   describe("collections", () => {
     describe("given cmr-stac 1.0 style collection identifiers", () => {

src/domains/collections.ts

@@ -16,7 +16,13 @@ import {
 
 import { cmrSpatialToExtent } from "./bounding-box";
 
-import { CMR_QUERY_MAX, extractAssets, paginateQuery } from "./stac";
+import {
+  CMR_QUERY_MAX,
+  extractAssets,
+  paginateQuery,
+  deriveExtensions,
+  generateStorageExtension,
+} from "./stac";
 
 const CMR_ROOT = process.env.CMR_URL;
 const STAC_VERSION = process.env.STAC_VERSION ?? "1.0.0";
@@ -232,13 +238,29 @@ const generateProviders = (collection: Collection) => [
 ];
 
 /**
- * Convert a GraphQL collection item into a STACCollection.
+ * Return the storage extension schema and attributes for a collection
+ */
+const storageExtension = (collection: Collection) => {
+  const s3Buckets = (collection.directDistributionInformation?.s3BucketAndObjectPrefixNames ?? [])
+    .flatMap((s3Buckets) => s3Buckets.split(","))
+    .map((s3Bucket) => s3Bucket.trim())
+    .filter((s3Bucket) => s3Bucket)
+    .map((s3Bucket) => (s3Bucket.startsWith("s3://") ? s3Bucket : `s3://${s3Bucket}`));
+  if (s3Buckets.length == 0) return;
+  return generateStorageExtension(s3Buckets);
+};
+
+/* Convert a GraphQL collection item into a STACCollection.
  */
 export const collectionToStac = (collection: Collection): STACCollection => {
   const { entryId, description, title } = collection;
 
   const { license, licenseLink } = extractLicense(collection);
 
+  const { extensions, attributes: extensionAttributes } = deriveExtensions(collection, [
+    storageExtension,
+  ]);
+
   const assets = extractAssets(collection);
   const extent = createExtent(collection);
   const keywords = createKeywords(collection);
@@ -252,13 +274,15 @@ export const collectionToStac = (collection: Collection): STACCollection => {
     title,
     description,
     stac_version: STAC_VERSION,
+    stac_extensions: extensions,
     extent,
     assets,
     providers,
     links,
     license,
     keywords,
     summaries,
+    ...extensionAttributes,
   } as STACCollection;
 };