diff --git a/.github/workflows/transformations_aws_compliance_premium.yml b/.github/workflows/transformations_aws_compliance_premium.yml index b66a8b642..20e519bbc 100644 --- a/.github/workflows/transformations_aws_compliance_premium.yml +++ b/.github/workflows/transformations_aws_compliance_premium.yml @@ -156,6 +156,7 @@ jobs: if: needs.prepare.outputs.s3 == 'true' run: | cloudquery migrate tests/s3.yml + ./tests/create_athena_tables.sh uv run dbt run --target dev-athena --profiles-dir ./tests - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 diff --git a/transformations/aws/asset-inventory-free/tests/clickhouse.yml b/transformations/aws/asset-inventory-free/tests/clickhouse.yml index e926a5105..9bf924ad8 100644 --- a/transformations/aws/asset-inventory-free/tests/clickhouse.yml +++ b/transformations/aws/asset-inventory-free/tests/clickhouse.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v34.9.0" # latest version of source aws plugin + version: "v35.0.0" # latest version of source aws plugin destinations: ["clickhouse"] tables: ["*"] --- diff --git a/transformations/aws/asset-inventory-free/tests/postgres.yml b/transformations/aws/asset-inventory-free/tests/postgres.yml index 7b908eade..12a3ea43f 100644 --- a/transformations/aws/asset-inventory-free/tests/postgres.yml +++ b/transformations/aws/asset-inventory-free/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v34.9.0" # latest version of source aws plugin + version: "v35.0.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/asset-inventory-free/tests/snowflake.yml b/transformations/aws/asset-inventory-free/tests/snowflake.yml index 55e8e326a..b669a126c 100644 --- a/transformations/aws/asset-inventory-free/tests/snowflake.yml +++ b/transformations/aws/asset-inventory-free/tests/snowflake.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v34.9.0" # latest version of source aws plugin + version: "v35.0.0" # latest version of source aws plugin destinations: ["snowflake"] tables: ["*"] --- diff --git a/transformations/aws/compliance-premium/README.md b/transformations/aws/compliance-premium/README.md index b8f9bbb68..47a20ba41 100644 --- a/transformations/aws/compliance-premium/README.md +++ b/transformations/aws/compliance-premium/README.md @@ -134,7 +134,7 @@ spec: "aws_iam_user_policies", "aws_cloudwatchlogs_metric_filters", "aws_cloudtrail_trails", - "aws_cloudwatch_alarms", + "aws_cloudwatch_metric_alarms", "aws_ec2_flow_logs", "aws_iam_virtual_mfa_devices", "aws_ec2_vpcs", @@ -224,7 +224,7 @@ tables: ["aws_iam_password_policies", "aws_iam_user_policies", "aws_cloudwatchlogs_metric_filters", "aws_cloudtrail_trails", -"aws_cloudwatch_alarms", +"aws_cloudwatch_metric_alarms", "aws_ec2_flow_logs", "aws_iam_virtual_mfa_devices", "aws_ec2_vpcs", @@ -261,7 +261,7 @@ tables: ["aws_securityhub_hubs", "aws_iam_user_access_keys", "aws_cloudtrail_trails", "aws_efs_filesystems", -"aws_cloudwatch_alarms", +"aws_cloudwatch_metric_alarms", "aws_accessanalyzer_analyzers", "aws_ec2_vpcs", "aws_s3_bucket_versionings", @@ -313,7 +313,7 @@ tables: ["aws_securityhub_hubs", "aws_iam_user_access_keys", "aws_cloudtrail_trails", "aws_efs_filesystems", -"aws_cloudwatch_alarms", +"aws_cloudwatch_metric_alarms", "aws_accessanalyzer_analyzers", "aws_ec2_vpcs", "aws_s3_bucket_versionings", @@ -493,7 +493,7 @@ tables: ["aws_ec2_eips", "aws_iam_policies", "aws_s3_bucket_encryption_rules", "aws_cloudtrail_trails", -"aws_cloudwatch_alarms", +"aws_cloudwatch_metric_alarms", "aws_ec2_vpcs", "aws_iam_accounts", "aws_rds_cluster_snapshots", diff --git a/transformations/aws/compliance-premium/dbt_project.yml b/transformations/aws/compliance-premium/dbt_project.yml index 9210ef923..0860cdc10 100644 --- a/transformations/aws/compliance-premium/dbt_project.yml +++ b/transformations/aws/compliance-premium/dbt_project.yml @@ -44,7 +44,7 @@ vars: - aws_ec2_security_groups - aws_sns_subscriptions - aws_cloudwatchlogs_metric_filters - - aws_cloudwatch_alarms + - aws_cloudwatch_metric_alarms cis_v2_0_0: - aws_iam_password_policies - aws_iam_user_attached_policies @@ -89,7 +89,7 @@ vars: - aws_sns_subscriptions - aws_ec2_network_acls - aws_cloudwatchlogs_metric_filters - - aws_cloudwatch_alarms + - aws_cloudwatch_metric_alarms - aws_iam_policy_default_versions cis_v3_0_0: - aws_iam_password_policies @@ -133,7 +133,7 @@ vars: - aws_cloudwatchlogs_metric_filters - aws_sns_subscriptions - aws_ec2_network_acls - - aws_cloudwatch_alarms + - aws_cloudwatch_metric_alarms - aws_kms_key_rotation_statuses - aws_iam_policy_default_versions foundational_security: @@ -299,7 +299,7 @@ vars: - aws_s3_accounts - aws_ec2_security_groups - aws_sns_subscriptions - - aws_cloudwatch_alarms + - aws_cloudwatch_metric_alarms - aws_cloudwatchlogs_metric_filters - aws_kms_key_rotation_statuses imds_v2: @@ -450,5 +450,5 @@ vars: - aws_s3_bucket_lifecycles - aws_cloudwatchlogs_metric_filters - aws_sns_subscriptions - - aws_cloudwatch_alarms + - aws_cloudwatch_metric_alarms - aws_kms_key_rotation_statuses diff --git a/transformations/aws/compliance-premium/tests/bigquery.yml b/transformations/aws/compliance-premium/tests/bigquery.yml index 209e920aa..23747649f 100644 --- a/transformations/aws/compliance-premium/tests/bigquery.yml +++ b/transformations/aws/compliance-premium/tests/bigquery.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v34.9.0" # latest version of source aws plugin + version: "v35.0.0" # latest version of source aws plugin destinations: ["bigquery"] tables: ["*"] --- diff --git a/transformations/aws/compliance-premium/tests/create_athena_tables.sh b/transformations/aws/compliance-premium/tests/create_athena_tables.sh new file mode 100755 index 000000000..dc1e3cf49 --- /dev/null +++ b/transformations/aws/compliance-premium/tests/create_athena_tables.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +DATABASE="athena_integration" +DATA_LOCATION="s3://cq-integration-tests/aws-compliance-athena/data" +OUTPUT_LOCATION="s3://cq-integration-tests/dbt/" + +run_query() { + local query="$1" + local execution_id + execution_id=$(aws athena start-query-execution \ + --query-string "$query" \ + --query-execution-context Database="$DATABASE" \ + --result-configuration OutputLocation="$OUTPUT_LOCATION" \ + --output text --query QueryExecutionId) + + while true; do + local state + state=$(aws athena get-query-execution --query-execution-id "$execution_id" --output text --query QueryExecution.Status.State) + case "$state" in + SUCCEEDED) return 0 ;; + FAILED | CANCELLED) + aws athena get-query-execution --query-execution-id "$execution_id" --output text --query QueryExecution.Status.StateChangeReason >&2 + return 1 + ;; + esac + sleep 2 + done +} + +run_query "CREATE EXTERNAL TABLE IF NOT EXISTS aws_cloudwatch_metric_alarms ( + account_id string, + region string, + arn string, + alarm_name string, + actions_enabled boolean, + alarm_actions array, + metrics string +) +STORED AS PARQUET +LOCATION '${DATA_LOCATION}/aws_cloudwatch_metric_alarms/'" diff --git a/transformations/aws/compliance-premium/tests/postgres.yml b/transformations/aws/compliance-premium/tests/postgres.yml index 7b908eade..12a3ea43f 100644 --- a/transformations/aws/compliance-premium/tests/postgres.yml +++ b/transformations/aws/compliance-premium/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v34.9.0" # latest version of source aws plugin + version: "v35.0.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/compliance-premium/tests/s3.yml b/transformations/aws/compliance-premium/tests/s3.yml index 0898b1ab0..a0935166b 100644 --- a/transformations/aws/compliance-premium/tests/s3.yml +++ b/transformations/aws/compliance-premium/tests/s3.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v34.9.0" # latest version of source aws plugin + version: "v35.0.0" # latest version of source aws plugin destinations: ["s3"] tables: ["*"] --- diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index 171b65d3d..631ede2d2 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v34.9.0" # latest version of source aws plugin + version: "v35.0.0" # latest version of source aws plugin destinations: ["snowflake"] tables: ["*"] --- diff --git a/transformations/aws/cost/models/compute_optimizer/ebs_volume_optimization_recommendations.sql b/transformations/aws/cost/models/compute_optimizer/ebs_volume_optimization_recommendations.sql index b89ec84de..408644e42 100644 --- a/transformations/aws/cost/models/compute_optimizer/ebs_volume_optimization_recommendations.sql +++ b/transformations/aws/cost/models/compute_optimizer/ebs_volume_optimization_recommendations.sql @@ -6,7 +6,7 @@ as well as recommendations for optimizing performance and potentially reducing c SELECT r.account_id, - volume_arn, + r.volume_arn, eev.attachments ->> 'associatedResource' as ecs_attached, eev.encrypted, eev.fast_restored, diff --git a/transformations/aws/cost/tests/postgres.yml b/transformations/aws/cost/tests/postgres.yml index 7b908eade..12a3ea43f 100644 --- a/transformations/aws/cost/tests/postgres.yml +++ b/transformations/aws/cost/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v34.9.0" # latest version of source aws plugin + version: "v35.0.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/data-resilience/tests/postgres.yml b/transformations/aws/data-resilience/tests/postgres.yml index 7b908eade..12a3ea43f 100644 --- a/transformations/aws/data-resilience/tests/postgres.yml +++ b/transformations/aws/data-resilience/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v34.9.0" # latest version of source aws plugin + version: "v35.0.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/encryption/tests/postgres.yml b/transformations/aws/encryption/tests/postgres.yml index 7b908eade..12a3ea43f 100644 --- a/transformations/aws/encryption/tests/postgres.yml +++ b/transformations/aws/encryption/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v34.9.0" # latest version of source aws plugin + version: "v35.0.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/macros/cloudwatch/alarm_actions_disabled.sql b/transformations/aws/macros/cloudwatch/alarm_actions_disabled.sql index dece169bd..9b5f2f4dd 100644 --- a/transformations/aws/macros/cloudwatch/alarm_actions_disabled.sql +++ b/transformations/aws/macros/cloudwatch/alarm_actions_disabled.sql @@ -12,6 +12,6 @@ select account_id, arn as resource_id, 'fail' as status -from aws_cloudwatch_alarms +from aws_cloudwatch_metric_alarms where actions_enabled = false or array_length(alarm_actions, 1) = 0{% endmacro %} diff --git a/transformations/aws/macros/log_metric_filter_and_alarm.sql b/transformations/aws/macros/log_metric_filter_and_alarm.sql index ceb838d05..bc96c14a5 100644 --- a/transformations/aws/macros/log_metric_filter_and_alarm.sql +++ b/transformations/aws/macros/log_metric_filter_and_alarm.sql @@ -7,7 +7,7 @@ {% macro postgres__log_metric_filter_and_alarm() %} with af as ( select distinct a.arn, a.actions_enabled, a.alarm_actions, m->'MetricStat'->'Metric'->>'MetricName' as metric_name -- TODO check - from aws_cloudwatch_alarms a, jsonb_array_elements(a.metrics) as m + from aws_cloudwatch_metric_alarms a, jsonb_array_elements(a.metrics) as m ), tes as ( select trail_arn from aws_cloudtrail_trail_event_selectors @@ -37,7 +37,7 @@ where t.is_multi_region_trail = TRUE {% macro bigquery__log_metric_filter_and_alarm() %} with af as ( select distinct a.arn, a.actions_enabled, ARRAY_TO_STRING(a.alarm_actions, ',') as alarm_actions, JSON_VALUE(m.MetricStat.Metric.MetricName) as metric_name - from {{ full_table_name("aws_cloudwatch_alarms") }} a, + from {{ full_table_name("aws_cloudwatch_metric_alarms") }} a, UNNEST(JSON_QUERY_ARRAY(metrics)) as m ), tes as ( @@ -72,7 +72,7 @@ where t.is_multi_region_trail = TRUE {% macro snowflake__log_metric_filter_and_alarm() %} with af as ( select distinct a.arn, a.actions_enabled, a.alarm_actions, m.value:MetricStat:Metric:MetricName as metric_name -- TODO check - from aws_cloudwatch_alarms a, + from aws_cloudwatch_metric_alarms a, LATERAL FLATTEN (metrics) as m ), aes as ( @@ -119,7 +119,7 @@ WITH af AS ( a.alarm_actions, json_extract_scalar(m, '$.MetricStat.Metric.MetricName') AS metric_name FROM - aws_cloudwatch_alarms a + aws_cloudwatch_metric_alarms a CROSS JOIN UNNEST(cast(json_extract(a.metrics, '$') as array(json))) as t(m) ),