Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ jobs:
if: needs.prepare.outputs.s3 == 'true'
run: |
cloudquery migrate tests/s3.yml
./tests/create_athena_tables.sh
uv run dbt run --target dev-athena --profiles-dir ./tests
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ spec:
name: aws
path: cloudquery/aws
registry: cloudquery
version: "v34.9.0" # latest version of source aws plugin
version: "v35.0.0" # latest version of source aws plugin
destinations: ["clickhouse"]
tables: ["*"]
---
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ spec:
name: aws
path: cloudquery/aws
registry: cloudquery
version: "v34.9.0" # latest version of source aws plugin
version: "v35.0.0" # latest version of source aws plugin
destinations: ["postgresql"]
tables: ["*"]
---
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ spec:
name: aws
path: cloudquery/aws
registry: cloudquery
version: "v34.9.0" # latest version of source aws plugin
version: "v35.0.0" # latest version of source aws plugin
destinations: ["snowflake"]
tables: ["*"]
---
Expand Down
10 changes: 5 additions & 5 deletions transformations/aws/compliance-premium/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ spec:
"aws_iam_user_policies",
"aws_cloudwatchlogs_metric_filters",
"aws_cloudtrail_trails",
"aws_cloudwatch_alarms",
"aws_cloudwatch_metric_alarms",
"aws_ec2_flow_logs",
"aws_iam_virtual_mfa_devices",
"aws_ec2_vpcs",
Expand Down Expand Up @@ -224,7 +224,7 @@ tables: ["aws_iam_password_policies",
"aws_iam_user_policies",
"aws_cloudwatchlogs_metric_filters",
"aws_cloudtrail_trails",
"aws_cloudwatch_alarms",
"aws_cloudwatch_metric_alarms",
"aws_ec2_flow_logs",
"aws_iam_virtual_mfa_devices",
"aws_ec2_vpcs",
Expand Down Expand Up @@ -261,7 +261,7 @@ tables: ["aws_securityhub_hubs",
"aws_iam_user_access_keys",
"aws_cloudtrail_trails",
"aws_efs_filesystems",
"aws_cloudwatch_alarms",
"aws_cloudwatch_metric_alarms",
"aws_accessanalyzer_analyzers",
"aws_ec2_vpcs",
"aws_s3_bucket_versionings",
Expand Down Expand Up @@ -313,7 +313,7 @@ tables: ["aws_securityhub_hubs",
"aws_iam_user_access_keys",
"aws_cloudtrail_trails",
"aws_efs_filesystems",
"aws_cloudwatch_alarms",
"aws_cloudwatch_metric_alarms",
"aws_accessanalyzer_analyzers",
"aws_ec2_vpcs",
"aws_s3_bucket_versionings",
Expand Down Expand Up @@ -493,7 +493,7 @@ tables: ["aws_ec2_eips",
"aws_iam_policies",
"aws_s3_bucket_encryption_rules",
"aws_cloudtrail_trails",
"aws_cloudwatch_alarms",
"aws_cloudwatch_metric_alarms",
"aws_ec2_vpcs",
"aws_iam_accounts",
"aws_rds_cluster_snapshots",
Expand Down
10 changes: 5 additions & 5 deletions transformations/aws/compliance-premium/dbt_project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ vars:
- aws_ec2_security_groups
- aws_sns_subscriptions
- aws_cloudwatchlogs_metric_filters
- aws_cloudwatch_alarms
- aws_cloudwatch_metric_alarms
cis_v2_0_0:
- aws_iam_password_policies
- aws_iam_user_attached_policies
Expand Down Expand Up @@ -89,7 +89,7 @@ vars:
- aws_sns_subscriptions
- aws_ec2_network_acls
- aws_cloudwatchlogs_metric_filters
- aws_cloudwatch_alarms
- aws_cloudwatch_metric_alarms
- aws_iam_policy_default_versions
cis_v3_0_0:
- aws_iam_password_policies
Expand Down Expand Up @@ -133,7 +133,7 @@ vars:
- aws_cloudwatchlogs_metric_filters
- aws_sns_subscriptions
- aws_ec2_network_acls
- aws_cloudwatch_alarms
- aws_cloudwatch_metric_alarms
- aws_kms_key_rotation_statuses
- aws_iam_policy_default_versions
foundational_security:
Expand Down Expand Up @@ -299,7 +299,7 @@ vars:
- aws_s3_accounts
- aws_ec2_security_groups
- aws_sns_subscriptions
- aws_cloudwatch_alarms
- aws_cloudwatch_metric_alarms
- aws_cloudwatchlogs_metric_filters
- aws_kms_key_rotation_statuses
imds_v2:
Expand Down Expand Up @@ -450,5 +450,5 @@ vars:
- aws_s3_bucket_lifecycles
- aws_cloudwatchlogs_metric_filters
- aws_sns_subscriptions
- aws_cloudwatch_alarms
- aws_cloudwatch_metric_alarms
- aws_kms_key_rotation_statuses
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ spec:
name: aws
path: cloudquery/aws
registry: cloudquery
version: "v34.9.0" # latest version of source aws plugin
version: "v35.0.0" # latest version of source aws plugin
destinations: ["bigquery"]
tables: ["*"]
---
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
#!/usr/bin/env bash
set -euo pipefail

DATABASE="athena_integration"
DATA_LOCATION="s3://cq-integration-tests/aws-compliance-athena/data"
OUTPUT_LOCATION="s3://cq-integration-tests/dbt/"

run_query() {
local query="$1"
local execution_id
execution_id=$(aws athena start-query-execution \
--query-string "$query" \
--query-execution-context Database="$DATABASE" \
--result-configuration OutputLocation="$OUTPUT_LOCATION" \
--output text --query QueryExecutionId)

while true; do
local state
state=$(aws athena get-query-execution --query-execution-id "$execution_id" --output text --query QueryExecution.Status.State)
case "$state" in
SUCCEEDED) return 0 ;;
FAILED | CANCELLED)
aws athena get-query-execution --query-execution-id "$execution_id" --output text --query QueryExecution.Status.StateChangeReason >&2
return 1
;;
esac
sleep 2
done
}

run_query "CREATE EXTERNAL TABLE IF NOT EXISTS aws_cloudwatch_metric_alarms (
account_id string,
region string,
arn string,
alarm_name string,
actions_enabled boolean,
alarm_actions array<string>,
metrics string
)
STORED AS PARQUET
LOCATION '${DATA_LOCATION}/aws_cloudwatch_metric_alarms/'"
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ spec:
name: aws
path: cloudquery/aws
registry: cloudquery
version: "v34.9.0" # latest version of source aws plugin
version: "v35.0.0" # latest version of source aws plugin
destinations: ["postgresql"]
tables: ["*"]
---
Expand Down
2 changes: 1 addition & 1 deletion transformations/aws/compliance-premium/tests/s3.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ spec:
name: aws
path: cloudquery/aws
registry: cloudquery
version: "v34.9.0" # latest version of source aws plugin
version: "v35.0.0" # latest version of source aws plugin
destinations: ["s3"]
tables: ["*"]
---
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ spec:
name: aws
path: cloudquery/aws
registry: cloudquery
version: "v34.9.0" # latest version of source aws plugin
version: "v35.0.0" # latest version of source aws plugin
destinations: ["snowflake"]
tables: ["*"]
---
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ as well as recommendations for optimizing performance and potentially reducing c

SELECT
r.account_id,
volume_arn,
r.volume_arn,
eev.attachments ->> 'associatedResource' as ecs_attached,
eev.encrypted,
eev.fast_restored,
Expand Down
2 changes: 1 addition & 1 deletion transformations/aws/cost/tests/postgres.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ spec:
name: aws
path: cloudquery/aws
registry: cloudquery
version: "v34.9.0" # latest version of source aws plugin
version: "v35.0.0" # latest version of source aws plugin
destinations: ["postgresql"]
tables: ["*"]
---
Expand Down
2 changes: 1 addition & 1 deletion transformations/aws/data-resilience/tests/postgres.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ spec:
name: aws
path: cloudquery/aws
registry: cloudquery
version: "v34.9.0" # latest version of source aws plugin
version: "v35.0.0" # latest version of source aws plugin
destinations: ["postgresql"]
tables: ["*"]
---
Expand Down
2 changes: 1 addition & 1 deletion transformations/aws/encryption/tests/postgres.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ spec:
name: aws
path: cloudquery/aws
registry: cloudquery
version: "v34.9.0" # latest version of source aws plugin
version: "v35.0.0" # latest version of source aws plugin
destinations: ["postgresql"]
tables: ["*"]
---
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@ select
account_id,
arn as resource_id,
'fail' as status
from aws_cloudwatch_alarms
from aws_cloudwatch_metric_alarms
where actions_enabled = false
or array_length(alarm_actions, 1) = 0{% endmacro %}
8 changes: 4 additions & 4 deletions transformations/aws/macros/log_metric_filter_and_alarm.sql
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
{% macro postgres__log_metric_filter_and_alarm() %}
with af as (
select distinct a.arn, a.actions_enabled, a.alarm_actions, m->'MetricStat'->'Metric'->>'MetricName' as metric_name -- TODO check
from aws_cloudwatch_alarms a, jsonb_array_elements(a.metrics) as m
from aws_cloudwatch_metric_alarms a, jsonb_array_elements(a.metrics) as m
),
tes as (
select trail_arn from aws_cloudtrail_trail_event_selectors
Expand Down Expand Up @@ -37,7 +37,7 @@ where t.is_multi_region_trail = TRUE
{% macro bigquery__log_metric_filter_and_alarm() %}
with af as (
select distinct a.arn, a.actions_enabled, ARRAY_TO_STRING(a.alarm_actions, ',') as alarm_actions, JSON_VALUE(m.MetricStat.Metric.MetricName) as metric_name
from {{ full_table_name("aws_cloudwatch_alarms") }} a,
from {{ full_table_name("aws_cloudwatch_metric_alarms") }} a,
UNNEST(JSON_QUERY_ARRAY(metrics)) as m
),
tes as (
Expand Down Expand Up @@ -72,7 +72,7 @@ where t.is_multi_region_trail = TRUE
{% macro snowflake__log_metric_filter_and_alarm() %}
with af as (
select distinct a.arn, a.actions_enabled, a.alarm_actions, m.value:MetricStat:Metric:MetricName as metric_name -- TODO check
from aws_cloudwatch_alarms a,
from aws_cloudwatch_metric_alarms a,
LATERAL FLATTEN (metrics) as m
),
aes as (
Expand Down Expand Up @@ -119,7 +119,7 @@ WITH af AS (
a.alarm_actions,
json_extract_scalar(m, '$.MetricStat.Metric.MetricName') AS metric_name
FROM
aws_cloudwatch_alarms a
aws_cloudwatch_metric_alarms a
CROSS JOIN
UNNEST(cast(json_extract(a.metrics, '$') as array(json))) as t(m)
),
Expand Down
Loading