Merge remote-tracking branch 'origin'

Adding tunable parameters for data export.

Author: rogonion

src/API/.envrc.template

@@ -39,3 +39,5 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 export DATA_TEMPLATES_FOLDER="$SCRIPT_DIR/templates"
 export FULL_OCCURRENCE_DATA_FOLDER="$SCRIPT_DIR/../OccurrenceGeoJob/.tmp/target" # Set to value of TARGET_DIRECTORY of OccurrenceGeoJob. Ensure job has been ran first.
 export DATABASE_GUIDE_FILE_NAME="Vector_Atlas_Database_Guide.pdf" # Must be located inside the '$DATA_TEMPLATES_FOLDER/Vector Atlas'
+export DATA_EXPORT_BATCH_SIZE=200 # Tune the batch size based on deployment environment.
+export DATA_EXPORT_YIELD_AFTER=5 # After how many batches should setImmediate be called. May be useful for giving node time to clean-up memory and process other requests.

src/API/src/config/config.ts

@@ -60,6 +60,18 @@ const config = convict({
     default: 'Vector_Atlas_Database_Guide.pdf',
     env: 'DATABASE_GUIDE_FILE_NAME',
   },
+  dataExportBatchSize: {
+    type: Number,
+    doc: 'The max batch size (number of rows) during data export. Tune based on observed deployment environment factors e.g, database/network latency higher give node more room to be idle thus memory pressure is low.',
+    default: 200,
+    env: 'DATA_EXPORT_BATCH_SIZE',
+  },
+  dataExportYieldAfter: {
+    type: Number,
+    doc: 'After how many batches should setImmediate be called. May be useful for giving node time to clean-up memory and process other requests',
+    default: 5,
+    env: 'DATA_EXPORT_YIELD_AFTER',
+  },
 });
 
 export default config;

src/API/src/db/shared/dynamic-export.service-v2.ts

@@ -167,6 +167,7 @@ export class DynamicExportServiceV2<T = Occurrence> {
     columns: RawTemplateFieldMap[],
     targetFilePath: string | null = null,
     pageSize = 200,
+    yieldAfter = 5,
     exportJob: ExportJob = null,
     updateProgressCallback?: (jobId: string, progress: number) => void,
     saveToDisk = true,
@@ -179,6 +180,8 @@ export class DynamicExportServiceV2<T = Occurrence> {
       fs.mkdirSync(exportDir, { recursive: true });
     }
 
+    console.log('Using pageSize of', pageSize);
+
     const finalPath =
       targetFilePath ||
       path.join(exportDir, `va_export_${exportJob?.id || Date.now()}.xlsx`);
@@ -258,9 +261,9 @@ export class DynamicExportServiceV2<T = Occurrence> {
         updateProgressCallback(exportJob.id, progress);
       }
 
-      // yield every 5 pages (~1,000 rows)
+      // yield every x pages
       // Potentially gives other HTTP requests and health checks an immediate turn on the event loop
-      if (page % 5 === 0) {
+      if (page % yieldAfter === 0) {
         await new Promise((resolve) => setImmediate(resolve));
       }
     }

src/API/src/exports/exports.processor-v2.ts

@@ -14,7 +14,7 @@ import {
   RAW_TEMPLATE_FIELD_EXCLUDED,
   RAW_TEMPLATE_FIELD_MAPPING,
 } from 'src/db/occurrence/template-mapping';
-import { extractFileNameFromBlobUrl } from 'src/utils';
+import { extractFileNameFromBlobUrl, maskEmail } from 'src/utils';
 
 @Injectable()
 @Processor('exports')
@@ -94,7 +94,8 @@ export class ExportsProcessorV2 extends WorkerHost {
         JSON.stringify(sanitizedFilters),
       );
 
-      const take = 200;
+      const take = config.get('dataExportBatchSize');
+      const yieldAfter = config.get('dataExportYieldAfter');
       const saveToDisk = true; // CRITICAL: Forces ExcelJS streaming writer to write straight to disk
 
       // 2. STREAM EXCEL DIRECTLY TO DISK
@@ -103,6 +104,7 @@ export class ExportsProcessorV2 extends WorkerHost {
         RAW_TEMPLATE_FIELD_MAPPING,
         excelFilePath, // Output file target
         take,
+        yieldAfter,
         exportJob,
         (jobId, progress) => {
           this.exportsService.updateProgress(jobId, progress);
@@ -200,7 +202,9 @@ export class ExportsProcessorV2 extends WorkerHost {
         );
 
         console.log(
-          `Notification email sent to ${updatedExportJob.downloaderEmail}`,
+          `Notification email sent to ${maskEmail(
+            updatedExportJob.downloaderEmail,
+          )}`,
         );
       }
 

src/API/src/utils.ts

@@ -3,6 +3,17 @@ import { sanitize } from './dataset-upload/utils';
 import path from 'path';
 import { isKeyObject } from 'util/types';
 
+import crypto from 'crypto';
+
+export function maskEmail(email: string) {
+  if (!email || typeof email !== 'string') return 'unknown-user';
+
+  return crypto
+    .createHash('sha256')
+    .update(email.trim().toLowerCase())
+    .digest('hex');
+}
+
 export const isEmpty = (object) =>
   Object.values(object).every((x) => x === null || x === '' || x === undefined);