[Feature] Create script to backload API data for a given date

.github/workflows/ci.yml

@@ -70,8 +70,8 @@ jobs:
       - lint
       - audit_dependencies
       - test
-    if: github.ref == 'refs/heads/develop'
-    uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
+    if: github.ref == 'refs/heads/bugfix/backload_ga4_api_data'
+    uses: 18F/analytics-reporter/.github/workflows/deploy.yml@bugfix/backload_ga4_api_data
     with:
       ANALYTICS_GA4_CALL_RETRY_COUNT: ${{ vars.ANALYTICS_GA4_CALL_RETRY_COUNT_DEV }}
       ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }}

deploy/backload.js

@@ -0,0 +1,92 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const { program } = require("commander");
+const { formatISO, parseISO, differenceInDays, subDays } = require("date-fns");
+const apiReports = require("../reports/api.json");
+const { exec } = require("child_process");
+const logger = require("../src/logger").initialize();
+
+/**
+ * Script to backload DAP API data for a particular date.
+ *
+ * Example run command:
+ * dotenv -e .env.analytics node -- ./backload.js --date 2024-01-01
+ */
+
+program
+  .option("-d, --date <string>", "date to run the backload for in ISO format")
+  .option(
+    "-e, --env <string>",
+    "dotenv file to load for the process",
+    ".env.analytics",
+  );
+
+program.parse();
+
+const options = program.opts();
+require("dotenv").config({ path: options.env });
+
+const scriptRootPath = `${process.env.ANALYTICS_ROOT_PATH}/deploy`;
+
+const runScriptWithLogName = async (scriptPath, scriptLoggingName) => {
+  logger.info(`Beginning: ${scriptLoggingName}`);
+  logger.info(`File path: ${scriptPath}`);
+  const promise = new Promise((resolve) => {
+    const childProcess = exec(scriptPath);
+    childProcess.stdout.on("data", (data) => {
+      console.log(data.toString().trim());
+    });
+
+    childProcess.stderr.on("data", (data) => {
+      console.log(data.toString().trim());
+    });
+
+    childProcess.on("close", () => {
+      resolve();
+    });
+  });
+
+  await promise;
+  fs.unlinkSync("reports/api.json");
+  fs.renameSync("reports/api.original.json", "reports/api.json");
+};
+
+const api_run = () => {
+  return runScriptWithLogName(`${scriptRootPath}/api.sh`, "api.sh");
+};
+
+(async () => {
+  logger.info("===========================================");
+  logger.info("======= STARTING ANALYTICS-REPORTER =======");
+  logger.info(`Running /deploy/backload.js for: ${options.date}`);
+  logger.info("===========================================");
+
+  const scriptTargetDate = parseISO(options.date);
+  const today = new Date();
+  let totalDaysAgo = differenceInDays(today, scriptTargetDate);
+
+  while (totalDaysAgo >= 1) {
+    const iterationTargetDate = subDays(new Date(), totalDaysAgo);
+
+    const iterationDaysAgo = differenceInDays(today, iterationTargetDate) + 1;
+    const modifiedApiJsonString = JSON.stringify(apiReports).replaceAll(
+      '"yesterday"',
+      `"${iterationDaysAgo}daysAgo"`,
+    );
+
+    fs.unlinkSync("reports/api.json");
+    fs.writeFileSync("reports/api.original.json", JSON.stringify(apiReports));
+    fs.writeFileSync("reports/api.json", modifiedApiJsonString);
+    fs.writeFileSync("reports/api.new.json", modifiedApiJsonString);
+
+    logger.info(
+      `Running API reports for ${formatISO(iterationTargetDate, { representation: "date" })}...`,
+    );
+    await api_run();
+    totalDaysAgo = totalDaysAgo - 1;
+    logger.info(
+      `API reports for ${formatISO(iterationTargetDate, { representation: "date" })} complete`,
+    );
+  }
+})();

deploy/publisher.js

@@ -17,26 +17,10 @@ const Bree = require("bree");
 const bree = new Bree({
   logger,
   jobs: [
-    // Runs `../jobs/realtime.js` 1 millisecond after the process starts and
-    // then every 15 minutes going forward.
+    // Runs `../jobs/backload_api.js` immediately on process start
     {
-      name: "realtime",
+      name: "backload_api",
       timeout: "1",
-      interval: "15m",
-    },
-    // Runs `../jobs/daily.js` 1 minute after the process starts and then at
-    // 10:01 AM every day going forward.
-    {
-      name: "daily",
-      timeout: "1m",
-      interval: "at 10:01 am",
-    },
-    // Runs `../jobs/api.js` 2 minutes after the process starts and then at
-    // 10:02 AM every day going forward.
-    {
-      name: "api",
-      timeout: "2m",
-      interval: "at 10:02 am",
     },
   ],
 });

index.js

@@ -136,6 +136,7 @@ async function runQueuePublish(options = {}) {
 
   for (const agency of agencies) {
     for (const reportConfig of reportConfigs) {
+      agency.agencyName = agency.agencyName || "";
       process.env.AGENCY_NAME = agency.agencyName;
       const reportLogger = Logger.initialize({
         agencyName: appConfig.agencyLogName,
@@ -156,7 +157,7 @@ async function runQueuePublish(options = {}) {
         );
         if (messageId) {
           reportLogger.info(
-            `Created message in queue: ${queueClient.name} with message ID: ${messageId}`,
+            `Created job in queue: ${appConfig.messageQueueName} for ${reportConfig.query.dateRanges[0].startDate}`,
           );
         } else {
           reportLogger.info(

jobs/backload_api.js

@@ -0,0 +1,59 @@
+process.env.ANALYTICS_REPORTS_PATH = "reports/api.json";
+process.env.ANALYTICS_SCRIPT_NAME = "backload_api.js";
+
+const fs = require("fs");
+const { formatISO, parseISO, differenceInDays, subDays } = require("date-fns");
+
+const apiReports = require("../reports/api.json");
+const { runQueuePublish } = require("../index.js");
+const options = {
+  frequency: "daily",
+  debug: true,
+  "write-to-database": true,
+  agenciesFile: `${process.env.ANALYTICS_ROOT_PATH}/deploy/agencies.json`,
+};
+const logger = require("../src/logger.js").initialize();
+
+(async () => {
+  const scriptTargetDate = parseISO("2024-07-01");
+  const today = new Date();
+  let totalDaysAgo = differenceInDays(today, scriptTargetDate);
+
+  while (totalDaysAgo >= 1) {
+    const iterationTargetDate = subDays(new Date(), totalDaysAgo);
+
+    const iterationDaysAgo = differenceInDays(today, iterationTargetDate) + 1;
+    const modifiedApiJsonString = JSON.stringify(apiReports).replaceAll(
+      '"yesterday"',
+      `"${iterationDaysAgo}daysAgo"`,
+    );
+
+    fs.unlinkSync("reports/api.json");
+    fs.writeFileSync("reports/api.original.json", JSON.stringify(apiReports));
+    fs.writeFileSync("reports/api.json", modifiedApiJsonString);
+    fs.writeFileSync("reports/api.new.json", modifiedApiJsonString);
+
+    logger.info(
+      `Running API reports for ${formatISO(iterationTargetDate, { representation: "date" })}...`,
+    );
+
+    try {
+      await runQueuePublish(options);
+      logger.info(`Job completed: ${process.env.ANALYTICS_SCRIPT_NAME}`);
+    } catch (e) {
+      logger.error(
+        `Job exited with error: ${process.env.ANALYTICS_SCRIPT_NAME}`,
+      );
+      logger.error(e);
+      throw e;
+    }
+
+    totalDaysAgo = totalDaysAgo - 1;
+    logger.info(
+      `API reports for ${formatISO(iterationTargetDate, { representation: "date" })} complete`,
+    );
+
+    fs.unlinkSync("reports/api.json");
+    fs.renameSync("reports/api.original.json", "reports/api.json");
+  }
+})();

package.json

@@ -74,21 +74,23 @@
     "@smithy/node-http-handler": "^3.0.0",
     "@snyk/protect": "^1.1269.0",
     "bree": "^9.2.4",
+    "dotenv": "^16.4.5",
     "fast-csv": "^4.3.6",
     "googleapis": "^140.0.0",
     "max-listeners-exceeded-warning": "^0.0.1",
     "minimist": "^1.2.8",
     "p-retry": "^6.2.0",
     "pg-boss": "^9.0.3",
     "proxy-agent": "^6.4.0",
-    "winston": "^3.11.0"
+    "winston": "^3.11.0",
+    "commander": "^12.1.0",
+    "date-fns": "^3.6.0"
   },
   "devDependencies": {
     "@cucumber/cucumber": "^10.3.1",
     "@eslint/js": "^8.57.0",
     "chai": "^4.4.0",
     "chai-as-promised": "^8.0.1",
-    "dotenv": "^16.4.5",
     "dotenv-cli": "^7.4.3",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",