feat(mqtt): add mapache user + drop user_data from ignore_changes

Two coupled changes:

1. New mqtt_user_mapache (default "mapache") with a 32-char
   random_password, written as a third line in /etc/nanomq_pwd.conf via
   user-data. Intended for the mapache services fleet beyond gr26 —
   query, foreman, any future MQTT publishers. Keeps the credential
   distinct from gr26 so it can rotate independently of CAN ingest.

2. Drop user_data from lifecycle.ignore_changes (keep ami). The old
   guard meant adding users, ACL tweaks, or any user-data edit required
   an explicit `terraform taint` / `-replace` to actually deploy — easy
   to forget, and we've already hit the trap twice. nanomq carries no
   persistent state, so a ~90s replacement on legitimate config changes
   is acceptable; paho clients auto-reconnect. Postgres + ClickHouse
   modules keep user_data ignored because they own data volumes.

Side effect: the apply that lands this PR will replace the running
gr-mqtt instance (new user-data → instance recreate). EIP, SG, and all
three random_password values survive in state.

Author: BK1031

infra/environments/prod/outputs.tf

@@ -65,6 +65,12 @@ output "mqtt_password_tcm26" {
   sensitive   = true
 }
 
+output "mqtt_password_mapache" {
+  description = "Generated MQTT password for the mapache services fleet user. Read with `terraform output -raw mqtt_password_mapache` → mapache-secrets/MQTT_MAPACHE_PASSWORD or wherever the consumer reads it."
+  value       = module.mqtt.mqtt_password_mapache
+  sensitive   = true
+}
+
 output "clickhouse_private_ip" {
   description = "Private IP of the ClickHouse EC2. In-cluster pods connect to this on 8123 (HTTP) / 9000 (native)."
   value       = module.clickhouse.private_ip

infra/modules/mqtt-ec2/main.tf

@@ -49,6 +49,15 @@ resource "random_password" "mqtt_tcm26" {
   special = false
 }
 
+# Credential for mapache services (the in-cluster Go services beyond gr26
+# — e.g. query, foreman, future publishers). Distinct from gr26's own
+# user so the service-fleet credential can rotate independently of the
+# CAN-ingest pipeline.
+resource "random_password" "mqtt_mapache" {
+  length  = 32
+  special = false
+}
+
 resource "aws_security_group" "this" {
   name        = var.name
   description = "NanoMQ for ${var.name}"
@@ -107,18 +116,22 @@ resource "aws_instance" "this" {
   }
 
   user_data = templatefile("${path.module}/user-data.sh.tftpl", {
-    nanomq_version      = var.nanomq_version
-    mqtt_user           = var.mqtt_user
-    mqtt_password       = random_password.mqtt.result
-    mqtt_user_tcm26     = var.mqtt_user_tcm26
-    mqtt_password_tcm26 = random_password.mqtt_tcm26.result
+    nanomq_version        = var.nanomq_version
+    mqtt_user             = var.mqtt_user
+    mqtt_password         = random_password.mqtt.result
+    mqtt_user_tcm26       = var.mqtt_user_tcm26
+    mqtt_password_tcm26   = random_password.mqtt_tcm26.result
+    mqtt_user_mapache     = var.mqtt_user_mapache
+    mqtt_password_mapache = random_password.mqtt_mapache.result
   })
 
-  # Don't recycle the instance on user-data churn. nanomq carries no
-  # persistent state we care about across replacements, so AMI bumps
-  # are also benign — `terraform taint` to intentionally replace.
+  # user_data is intentionally NOT in ignore_changes: nanomq carries no
+  # persistent state, so legitimate config edits (new user, ACL change)
+  # should flow through a normal `terraform apply` and trigger the ~90s
+  # broker downtime willingly. Keeping `ami` ignored so unrelated AL2023
+  # AMI churn doesn't silently roll the instance.
   lifecycle {
-    ignore_changes = [user_data, ami]
+    ignore_changes = [ami]
   }
 
   tags = {

infra/modules/mqtt-ec2/outputs.tf

@@ -44,3 +44,14 @@ output "mqtt_password_tcm26" {
   value       = random_password.mqtt_tcm26.result
   sensitive   = true
 }
+
+output "mqtt_user_mapache" {
+  description = "MQTT username for mapache services beyond gr26. Pair with mqtt_password_mapache."
+  value       = var.mqtt_user_mapache
+}
+
+output "mqtt_password_mapache" {
+  description = "Generated MQTT password for mapache services. Read via `terraform output -raw mqtt_password_mapache` and put into the relevant k8s Secret key."
+  value       = random_password.mqtt_mapache.result
+  sensitive   = true
+}

infra/modules/mqtt-ec2/user-data.sh.tftpl

@@ -25,6 +25,7 @@ EOF
 cat >/etc/nanomq_pwd.conf <<EOF
 "${mqtt_user}": "${mqtt_password}"
 "${mqtt_user_tcm26}": "${mqtt_password_tcm26}"
+"${mqtt_user_mapache}": "${mqtt_password_mapache}"
 EOF
 chmod 600 /etc/nanomq_pwd.conf
 

infra/modules/mqtt-ec2/variables.tf

@@ -42,6 +42,12 @@ variable "mqtt_user_tcm26" {
   default     = "tcm26"
 }
 
+variable "mqtt_user_mapache" {
+  description = "MQTT username for mapache services beyond gr26 (query, foreman, future publishers). Paired with the module-generated mqtt_password_mapache. Distinct from mqtt_user so the service-fleet credential can rotate independently of the CAN-ingest pipeline."
+  type        = string
+  default     = "mapache"
+}
+
 variable "allowed_security_group_ids" {
   description = "Security group IDs allowed to connect on port 1883. Typically the EKS node SG."
   type        = list(string)