Infrastructure as Code for Monitoring: Automating Observability

provider "datadog" {
api_key = var.datadog _api_key
app_key = var.datadog _app_key
}

# Define a monitor
resource "datadog_monitor" "cpu_monitor" {
name = "CPU Usage Monitor for ${var.environment}"
type = "metric alert"
message = "CPU usage high on {{host.name}} in $ {var.environment}"
escalation_message = "CPU usage still high on {{host.name}} - escalating"
query = "avg(last_5m) :avg:system. cpu.user {environment: ${var.environment}} by {host} > ${var.cpu_threshold}"
monitor_thresholds {
critical = var.cpu_threshold
warning = var.cpu _warning_threshold
}
notify_no_data = false
renotify_interval = 60
tags = ["service: ${var.service_name}" , "team:${var.team}", "environment: ${var.environment}"]
}
# Monitoring dashboard
resource "datadog_dashboard" "service_dashboard" {
title = "${var.service_name} Dashboard - ${var.environment}"
description = "Dashboard for monitoring ${var.service_name} in ${var.environment}"
layout_type = "ordered"
widget {
# Dashboard widget configuration
# ...
}
# Additional widgets
# ...
}

tasks:

- name: Install monitoring agent package
package:
name: monitoring-agent
state: present

- name: Configure monitoring agent
template:
src: templates /agent.conf.j2
dest: /etc/monitoring -agent/agent.conf
owner: monitoring
group: monitoring
mode: '0644'
notify: Restart monitoring agent

- name: Enable and start monitoring agent
service:
name: monitoring-agent
state: started
enabled: yes

handlers:
- name: Restart monitoring agent
service:
name: monitoring-agent
state: restarted

---
# Service monitor definition
apiVersion: monitoring .coreos.com/v1
kind: ServiceMonitor
metadata:
name: frontend -service-monitor
namespace: monitoring
labels:
team: frontend
spec:
selector:
matchLabels:
app: frontend
endpoints:
- port: web
interval: 15s
path: /metrics

- name: pagerduty
type: pagerduty
service_key: "{{ pagerduty_service_key }}"

- name: email_alerts
type: email
addresses:
- "team@ example.com"
- "oncall@ example.com"

alert_routes:
- match:
severity: critical
routes:
- team_slack
- pagerduty

- match:
severity: warning
routes:
- team_slack

- match:
severity: info
routes:
- email_alerts

# Define common alert pattern
module "service_alerts" {
source = "./modules /service-alerts"
service_name = var.service_name
environment = var.environment
# Environment-specific thresholds
error_rate_threshold = lookup (local. error_thresholds, var.environment, 0.05)
latency_threshold = lookup (local. latency_thresholds, var.environment, 500)
availability_threshold = lookup (local. availability_thresholds, var.environment, 99.9)
# Environment-specific notification targets
notification_channels = var.environment == "production"
? ["slack-prod", "pagerduty"]
: ["slack-dev"]
}

# Local variable maps for environment-specific settings
locals {
error_thresholds = {
development = 0.10 # 10% error rate allowed in dev
staging = 0.08 # 8% in staging
production = 0.02 # Only 2% in production
}
latency_thresholds = {
development = 1000 # 1000ms in dev
staging = 750 # 750ms in staging
production = 300 # 300ms in production
}
availability_thresholds = {
development = 95.0 # 95% in dev
staging = 99.0 # 99% in staging
production = 99.95 # 99.95% in production
}
}

import yaml
import json
import requests
from jinja2 import Template

# Load service inventory
with open ('service_inventory. yaml', 'r') as file:
services = yaml.safe_ load(file)

# Load dashboard template
with open ('dashboard_template .json.j2', 'r') as file:
template = Template (file.read())

# Generate dashboards for each service
for service in services:
# Render dashboard with service details
dashboard_json = template.render(
service_name=service ['name'],
environment=service ['environment'],
team=service ['team'],
metrics= service.get ('metrics', []),
slo_targets= service.get ('slo_targets', {})
)

# Convert to Python dict
dashboard = json.loads (dashboard_json)

# Add/modify panels based on service-specific needs
if 'database' in service.get ('dependencies', []):
dashboard ['panels'] .append (db_performance_panel)

# Deploy dashboard to Grafana
response = requests.post (
f"{grafana_url} /api /dashboards/db",
headers= {
"Authorization": f"Bearer {grafana_api_key}",
"Content-Type": "application/json"
},
json={"dashboard": dashboard, "overwrite": True}
)

if response .status_code == 200:
print(f"Successfully deployed dashboard for {service['name']}")
else:
print(f"Failed to deploy dashboard for {service['name']}: {response.text}")

# Example abstract dashboard definition in YAML
# This can be converted to specific formats (Grafana, Datadog, etc.)

dashboard:

title: "Service Overview - {{ service_name }}"
description: "Performance dashboard for {{ service_name }}"
variables:

- name: environment
type: custom
options: ["production", "staging", "development"]
default: "production"

rows:

- title: "Health Overview"
panels:

- title: "Uptime"
type: stat
metrics:
- query: "sum(up {service=' {{ service_name }}', environment = '$environment'}) /count(up {service= '{{ service_name }}', environment ='$environment'}) *100"
unit: percentage
thresholds:
- value: 0
color: red
- value: 99
color: green

- title: "Request Rate"
type: graph
metrics:
- query: "sum (rate (http_requests_total {service= '{{ service_name }}', environment= '$environment'} [5m]))"
legend: "Requests / sec"
unit: requests per second

- title: "Performance"
panels:

- title: "Latency"
type: graph
metrics:
- query: "histogram_quantile (0.95, sum (rate (http_request _duration_seconds _bucket {service= '{{ service_name }}', environment= '$environment'} [5m])) by (le))"
legend: "p95 Latency"
- query: "histogram_quantile (0.50, sum (rate (http_request _duration_seconds _bucket {service= '{{ service_name }}', environment= '$environment'} [5m])) by (le))"
legend: "p50 Latency"
unit: seconds

monitoring/
├── common/ # Shared monitoring components
├── alert_templates/ # Reusable alert definitions
├── dashboard_templates/ # Reusable dashboard templates
└── defaults/ # Default thresholds and settings

├── environments/ # Environment-specific configurations
├── production/ # Production environment
├── main.tf # Main configuration file
├── variables.tf # Variable definitions
└── terraform.tfvars # Environment values
├── staging/
└── development/

├── services/ # Service-specific monitoring
├── api/ # API service monitoring
├── alerts.tf
├── dashboards.tf
└── variables.tf
├── database/
└── frontend/

├── modules/ # Reusable monitoring modules
├── service_monitoring/ # Standard service monitoring
├── database_monitoring/ # Database-specific monitoring
└── slo_monitoring/ # SLO monitoring module

└── README.md # Documentation

# Example Terraform version constraints for monitoring providers

terraform {
required_version = ">&= 1.0.0, < 2.0.0"

required_providers {
datadog = {
source = "DataDog/datadog"
version = " 3.20.0"
}
grafana = {
source = "grafana/grafana"
version = " 1.28.0"
}
prometheus = {
source = "prometheus /prometheus"
version = " 0.14.0"
}
}
}

# Module versioning
module " api_monitoring" {
source = "github.com /organization/ monitoring-modules //service_monitoring? ref=v1.2.0"
service_name = "api"
environment = var.environment
}

# Example reusable monitoring component in YAML
# This could be applied to multiple services

name: http_service_monitoring
description: "Standard monitoring for HTTP-based services"
parameters:

service_name:
description: "Name of the service to monitor"
type: string
required: true

environment:
description: "Deployment environment"
type: string
required: true

error_threshold:
description: "Error rate threshold for alerting"
type: number
default: 0.05

latency_threshold_ms:
description: "Latency threshold in milliseconds"
type: number
default: 500

components:

# Uptime monitoring
- type: uptime_check
target: "https:// {{ service_name }}. {{ environment }} .example.com /health"
interval: 30s

# Error rate alerting
- type: alert_rule
name: "{{ service_name }} - High Error Rate"
query: "sum (rate (http_server_ requests_total {service= '{{ service_name }}', status=~'5..', environment= '{{ environment }}'} [5m])) / sum (rate (http_server _requests_total {service= '{{ service_name }}', environment= '{{ environment }}'} [5m])) > {{ error_threshold }}"
duration: 5m
severity: critical

# Latency alerting
- type: alert_rule
name: "{{ service_name }} - High Latency"
query: "histogram_quantile (0.95, sum (rate(http_request _duration_seconds _bucket {service= '{{ service_name }}', environment= '{{ environment }}'} [5m])) by (le)) > {{ latency _threshold_ms / 1000 }}"
duration: 5m
severity: warning

# Standard dashboard
- type: dashboard
name: "{{ service_name }} - {{ environment }}"
template: "http_service _dashboard"
variables:
service_name: "{{ service_name }}"
environment: "{{ environment }}"

- name: 'validate _cpu_alert'
type: 'alert_test'
alert: 'high _cpu_usage'
values:
- series: 'system.cpu.user {host= test-host}'
values: [0.5, 0.6, 0.7, 0.8, 0.9] # Values below threshold
expect:
triggered: false

- name: 'validate_cpu _alert_firing'
type: 'alert_test'
alert: 'high_cpu_usage'
values:
- series: 'system.cpu.user {host= test-host}'
values: [0.8, 0.85, 0.9, 0.92, 0.95] # Values above threshold
expect:
triggered: true

- name: 'validate_ dashboard _variables'
type: 'dashboard_test'
dashboard: 'service _dashboard'
variables:
- name: 'environment'
values: ["production", "staging", "development"]
required: true
- name: 'service'
values: [] # Should be dynamically populated
required: true

- name: 'validate_ latency_query'
type: 'query_test'
query: 'histogram_quantile (0.95, sum (rate (http_request_ duration_seconds_ bucket {service="api"} [5m])) by (le))'
expect:
valid: true
returns_data: true

on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
paths:
- 'monitoring/**'

jobs:
validate:
name: Validate Monitoring Code
runs-on: ubuntu-latest
steps:

- name: Checkout code
uses: actions/checkout@v3

- name: Setup Terraform
uses: hashicorp /setup-terraform@v2
with:
terraform_version: 1.3.x

- name: Terraform Init
run: |
cd monitoring /environments /development
terraform init

- name: Terraform Validate
run: |
cd monitoring /environments /development
terraform validate

- name: Terraform Format Check
run: terraform fmt -check -recursive monitoring/

- name: Terraform Plan
run: |
cd monitoring /environments /development
terraform plan-out = plan.out

- name: Monitoring-specific tests
run: |
python scripts /test_monitoring .py

- name: Alert coverage verification
run: |
python scripts /verify_alert_coverage.py

import requests
import time
import random
from prometheus_client import start_http_server, Counter, Gauge, Histogram
# Metrics that will be monitored
requests_total = Counter ('http_requests_total', 'Total HTTP Requests', ['status', 'endpoint'])
request_duration = Histogram ('http_request_ duration_seconds', 'HTTP request duration in seconds',
['endpoint'], buckets=(0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0))
cpu_usage = Gauge ('system _cpu_usage', 'System CPU Usage')
memory_usage = Gauge ('system_memory _usage_bytes', 'System Memory Usage in Bytes')

# Start metrics server
start_http_server (8000)

# Simulate normal traffic pattern
def simulate_normal_traffic():
for _ in range(100):
endpoint = random.choice ([ '/api/users' , '/api/products' , '/api/orders'])
status = '200' if random.random() < 0.95 else '500'
requests_total.labels (status=status, endpoint = endpoint).inc()

with request_ duration .labels (endpoint = endpoint) .time():
time.sleep (random.uniform (0.01, 0.1))

cpu_usage.set (random.uniform (0.1, 0.4))
memory_ usage.set (random.uniform (1e8, 5e8))
time.sleep (0.1)

# Simulate an incident
def simulate_incident():
print("Simulating incident - increased error rate and latency")

for _ in range(50):
endpoint = random.choice ([ '/api/users' , '/api/products', '/api/orders'])
status = '500' if random.random() < 0.3 else '200'
requests _total.labels (status=status, endpoint=endpoint) .inc()

with request_duration .labels (endpoint=endpoint) .time():
time.sleep (random.uniform (0.1, 0.5))

cpu_usage.set (random.uniform (0.7, 0.95))
memory_usage.set (random.uniform (7e8, 9e8))
time.sleep (0.1)

# Simulate recovery
def simulate _recovery():
print ("Simulating recovery - returning to normal metrics")

for _ in range(50):
endpoint = random.choice ([ '/api/users', '/api/products', '/api/orders'])
error_probability = 0.3 - (0.004 * _)
status = '500' if random.random() < error_probability else '200'
requests_total .labels (status=status, endpoint= endpoint) .inc()

latency_factor = 0.5 - (0.008 * _)
with request_duration .labels (endpoint= endpoint). time():
time.sleep (random.uniform (0.1, max(0.1, latency_factor)))

cpu_factor = 0.95 - (0.01 * _)
cpu_usage.set (random.uniform (0.4, max(0.4, cpu_factor)))

memory_factor = 9e8 - (1e7 * _)
memory_usage.set (random.uniform (4e8, max(4e8, memory_factor)))
time.sleep(0.1)

# Main test sequence
print("Starting monitoring test simulation")
print("Phase 1: Normal traffic patterns")
simulate _normal_traffic()

print ("Phase 2: Incident simulation")
simulate_incident ()

print ("Phase 3: Recovery")
simulate_recovery ()

print ("Phase 4: Normal operation")
simulate _normal_traffic ()

print("Test simulation complete - check if your monitoring detected the incident")

stages:
- validate
- deploy-dev
- test-dev
- deploy-staging
- test-staging
- approve-production
- deploy-production
- verify-production

validate:
script:
- terraform validate
- run- monitoring -tests.sh
artifacts:
paths:
- monitoring -plan.json

deploy-dev:
stage: deploy-dev
script:
- cd environments /development
- terraform apply -auto-approve
dependencies:
- validate
environment:
name: development

test-dev:
stage: test-dev
script:
- verify-monitoring -deployment.sh development
- run-monitoring -simulation.sh development
dependencies:
- deploy-dev

# Similar steps for staging environment

approve -production:
stage: approve-production
type: manual
script:
- echo "Deployment to production approved"
dependencies:
- test-staging

deploy-production:
stage: deploy-production
script:
- cd environments /production
- terraform apply -auto-approve
dependencies:
- approve-production
environment:
name: production

verify-production:
stage: verify-production
script:
- verify- monitoring- deployment.sh production
- run-synthetic -tests.sh production
dependencies:
- deploy-production

import requests
import json
import os
import subprocess
import smtplib
from email.message import EmailMessage

# Get the expected configuration from version control
def get_expected _configuration():
subprocess.run (["git", "pull", "origin", "main"], check=True)

result = subprocess.run (
["terraform", "output", "-json", "monitoring _configuration"],
capture_output=True,
text=True,
check=True
)

return json.loads (result.stdout)

# Get the actual configuration from the monitoring system
def get_actual _configuration (api_url, api_key):
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}

alerts_response = requests.get (f"{api_url}/alerts", headers = headers)
alerts = alerts_ response.json()

dashboards_response = requests.get (f"{api_url} /dashboards", headers = headers)
dashboards = dashboards _response.json()

return {
"alerts": alerts,
"dashboards": dashboards
}

# Compare configurations and detect differences
def detect_drift(expected, actual):
drift = {
"alerts": {"missing": [], "modified": [], "unexpected": []},
"dashboards": {"missing": [], "modified": [], "unexpected": []}
}

expected_alerts = {alert ["id"] : alert for alert in expected["alerts"]}
actual_alerts = {alert ["id"] : alert for alert in actual ["alerts"]}

for alert_id, expected_alert in expected_alerts.items():
if alert_id not in actual_alerts:
drift ["alerts"] ["missing"] .append (expected_alert)
elif not configurations_match (expected_alert, actual_alerts [alert_id]):
drift["alerts"] ["modified"] .append ({
"expected": expected _alert,
"actual": actual_alerts [alert_id]
})

for alert_id, actual_alert in actual_alerts. items():
if alert_id not in expected_alerts:
drift["alerts"] ["unexpected"] .append(actual_alert)

return drift

def has_significant_drift (drift):
return (len(drift ["alerts"] ["missing"]) > 0 or
len(drift ["alerts"] ["modified"]) > 0 or
len(drift ["dashboards"] ["missing"]) > 0 or
len(drift ["dashboards"] ["modified"]) > 0)

def notify_drift(drift, recipients):
msg = EmailMessage()
msg ['Subject'] = 'Monitoring Configuration Drift Detected'
msg ['From'] = 'monitoring @example.com'
msg ['To'] = ', '.join (recipients)

body = "Monitoring configuration drift has been detected:\n\n"
if drift ["alerts"] ["missing"] :
body += f"Missing alerts: {len(drift ['alerts'] ['missing'])} \n"
for alert in drift ["alerts"] ["missing"]:
body += f"- {alert['name']} (ID: {alert ['id']})\n"

if drift ["alerts"] ["modified"]:
body += f"\nModified alerts: {len(drift['alerts'] ['modified'])}\n"
for change in drift["alerts"] ["modified"]:
body += f"- {change ['expected'] ['name']} (ID: {change ['expected'] ['id']})\n"

body += "\nPlease investigate this drift and reconcile the configuration."
msg. set_content (body)

with smtplib.SMTP ('smtp.example.com', 587) as smtp:
smtp.starttls()
smtp.login ('monitoring @example.com', os.environ['SMTP_PASSWORD'])
smtp .send_message (msg)

def check_monitoring_drift():
expected = get_expected_configuration()
actual = get_actual _configuration (os.environ ['MONITORING_API_URL'], os.environ ['MONITORING_API_KEY'])
drift = detect_drift (expected, actual)

if has_significant_drift (drift):
print ("Significant monitoring configuration drift detected!")
notify _drift(drift, ["devops@example.com", "monitoring@example.com" ])
return 1
else:
print ("No significant monitoring configuration drift detected.")
return 0

if name == "main":
exit (check_monitoring _drift())

# Frontend monitoring
resource "datadog_monitor" "frontend_performance" {
name = "${var. service_name} Frontend Performance - ${var.environment}"
type = "query alert"
message = "Frontend performance degraded on ${var.service_name} in ${var.environment}. Check user experience metrics."
query = "avg (last_15m) :avg: rum.performance. load_event {service: ${var.service_name}, env:${var.environment}} > ${var. frontend_load _threshold}"
monitor _thresholds {
critical = var.frontend _load_threshold
warning = var.frontend _load_warning threshold
}
require_full_window = false
notify_no_data = false
tags = ["service: ${var.service_name}", "team:${var.team}", "env: ${var.environment}", "component:frontend"]
}
# API monitoring - endpoints
resource "datadog_monitor" "api_availability" {
name = "${var. service_name} API Availability - ${var.environment}"
type = "metric alert"
message = "API endpoint availability has dropped below threshold on ${var.service_name} in ${var.environment}."
query = "sum (last_5m): avg:api .endpoint.availability {service: ${var.service_name} ,env: ${var.environment}} by {endpoint} < ${var.api availability_threshold}"
monitor_thresholds {
critical = var.api _availability_threshold
warning = var.api _availability_warning _threshold
}
require _full_window = false
notify _no_data = true
no_data timeframe = 10
tags = ["service: ${var.service_name}", "team:${var.team}", "env: ${var.environment}", "component:api"]
}
# User journey synthetic test
resource "datadog synthetics_test" "user_journey" {
name = "${var.service_name} Critical User Journey - ${var.environment}"
type = "browser"
status = "live"
locations = ["aws: ${var. primary_region}", "aws: ${var. secondary_region}"]
request_definition {
method = "GET"
url = "https:// ${var.environment == "production" ? "" : "${var.environment} ."}${var .domain_name}"
}
assertion {
type = "statusCode"
operator = "is"
target = "200"
}
browser_step { name: "Login" }
browser_step { name: "Navigate to product" }
browser_step { name: "Add to cart" }
browser_step { name: "Checkout" }
options_list {
tick_every = 900
retry {
count = 2
interval = 300
}
monitor_options {
renotify_interval = 120
}
}
tags = ["service: ${var. service_name}", "journey:checkout", "env: ${var.environment}"]
}

monitoring:
- name : product- catalog-availability
type : availability
endpoint : /api/products
frequency : 1m
locations :
- ${primary _region}
- ${secondary _region}
thresholds :
availability : 99.9%
response_time : 500ms

- name : cart- functionality
type : synthetic
frequency : 5m
locations :
- ${primary _region}
steps :
- name : Navigate to product
action : navigate
url : https://${domain} /products/featured
- name : Add to cart
action : click
selector : button. add-to-cart
- name : View cart
action : navigate
url : https://$ {domain}/cart
- name : Verify product in cart
action : assert
selector : .cart-item
assertion : exists
alerts :
- channel : slack -${team}
severity : critical

- name : checkout-process
type : transaction
frequency : 15m
locations :
- ${primary_region}
steps :
- endpoint : /api/cart
method : GET
validate : status == 200
- endpoint : /api/checkout /start
method : POST
body : ${checkout_ payload_template}
validate : status == 200
- endpoint : /api/checkout /payment
method : POST
body : ${payment_ payload_template}
validate : status == 200 && body.contains("success")
- endpoint : /api/orders /latest
method : GET
validate : status == 200 && body.contains ("processing")
thresholds :
success_rate : 99.5%
total_duration : 3000ms
alerts :
- channel : slack- ${team}
severity : critical
- channel : pagerduty- ${team}
severity : critical

- name : conversion-rate
type : business_metric
query : "sum:ecommerce. checkout.completed {env:${env}} .as_count() / sum: ecommerce. checkout.started {env:${env}} .as_count() * 100"
frequency : 15m
window : 1h
thresholds :
critical : < ${conversion _critical_threshold}
warning : < ${conversion _warning_threshold}
alerts :
- channel : slack-business
severity : high
- channel : email-reports
severity : high

# SLA monitoring
module "sla_monitoring" {
source = "./modules /sla-monitoring"

service_name = var. service_name
environment = var. environment
availability_target = var.sla _availability_target
response_time_target = var.sla _response_time_target
error_rate_target = var.sla _error_rate_target
measurement_window = "30d"

notification_channels = {
critical = ["slack-sre", "pagerduty-team"]
warning = ["slack-sre"]
info = ["slack-sre"]
}

dashboard_name = "${var.service_name} - SLA Compliance"
}

# Business metrics
resource "datadog_monitor" "conversion_rate" {
name = "${var.service_name} Conversion Rate - ${var.environment}"
type = "query alert"
message = "Conversion rate has dropped below critical threshold for ${var.service_name} in ${var.environment}."

query = "min(last_1h):( sum:ecommerce .checkout.completed {service: ${var.service_name}, env: ${var.environment}} .as_count() / sum:ecommerce .checkout.started {service: ${var.service_name} ,env: ${var.environment}}. as_count() * 100 ) < ${var .conversion _critical _threshold}"

monitor_thresholds {
critical = var.conversion _critical_threshold
warning = var.conversion _warning_threshold
}

require_full_window = false
notify_no_data = false

tags = ["service: ${var.service_name }", "team: ${var.team}", "env:${var.environment}", "metric:business"]
}

resource "datadog_monitor" "revenue_tracking" {
name = "${var.service_name} Hourly Revenue - ${var.environment}"
type = "query alert"
message = "Hourly revenue has dropped significantly for ${var.service_name} in ${var.environment}."

query = "avg(last_1h): avg:business. revenue.hourly {service: ${var.service_name} , env: ${var.environment}} < ${var.revenue_threshold}"

monitor _thresholds {
critical = var.revenue _threshold
warning = var.revenue _warning_threshold
}

require_full_window = false
notify_no_data = true
no_data_timeframe = 60
evaluation_delay = 900
new_host_delay = 300
message_include_links = true

tags = ["service: ${var.service_name} ", "team:${var.team}", "env:${var.environment}", "metric:business"]
}

resource "datadog_dashboard" "customer_satisfaction" {
title = "${var.service_name} Customer Satisfaction Correlation - ${var.environment}"
description = "Correlation between technical performance and customer satisfaction"
layout_type = "ordered"

widget {
timeseries _definition {
title = "Response Time vs Customer Satisfaction"
request {
q = "avg: http.request .response_time {service: ${var.service_name}, env: ${var.environment}}"
type = "line"
}
request {
q = "avg: business.customer .satisfaction {service: ${var.service_name}, env: ${var.environment}}"
type = "line"
display_type = "line"
style {
line_type = "solid"
line_width = "normal"
palette = "cool"
}
yaxis = "right"
}
}
}

widget {
timeseries _definition {
title = "Error Rate vs Support Ticket Volume"
request {
q = "sum: http.request .errors {service: ${var.service_name} ,env: ${var.environment}} / sum: http.request .count {service: ${var.service_name} ,env: ${var.environment}} * 100"
type = "line"
}
request {
q = "sum: business.support .tickets {service: ${var.service_name} ,env: ${var.environment}} .rollup (sum, 3600)"
type = "line"
display_type = "line"
style {
line_type = "solid"
line_width = "normal"
palette = "warm"
}
yaxis = "right"
}
}
}
}

# AWS resources monitoring
module "aws_monitoring" {
source = "./modules /aws-monitoring"
environment = var.environment
notification_topic = var.notification_topic
# EC2 monitoring configuration
ec2_monitoring = {
cpu_threshold = 80
memory_threshold = 85
status_check_enable = true
instance_tags = var.monitored _instance_tags
}
# RDS monitoring configuration
rds_monitoring = {
cpu_threshold = 75
storage_threshold = 85
connections _threshold = var.db_max _connections * 0.85
replica_lag _threshold = 300
instances = var.monitored _db_instances
}
# ELB monitoring configuration
elb_monitoring = {
latency _threshold = 0.5
error _rate_threshold = 5
healthy hosts_percent = 75
load_balancers = var.monitored load_balancers
}
}
# GCP resources monitoring
module "gcp_monitoring" {
source = "./modules /gcp-monitoring"
project_id = var.gcp_project_id
environment = var.environment
notification channel = var.gcp notification_channel
# Compute Engine monitoring
compute monitoring = {
cpu threshold = 80
memory threshold = 85
disk threshold = 90
instance_filter = "labels.environment = ${var.environment}"
}
# Cloud SQL monitoring
cloudsql _monitoring = {
cpu _threshold = 75
memory _threshold = 80
disk _usage_threshold = 85
instances = var.monitored _sql_instances
}
}
# Cross-cloud dashboard
resource "grafana_dashboard" "multi cloud_overview" {
config_json = templatefile ("${path.module} /templates /multi_cloud dashboard.json", {
environment = var.environment
aws_region = var.aws_region
gcp_project_id = var.gcp_project_id
service_name = var.service_name
})
folder = var.grafana _folder_id
}
# Cost monitoring alerts
resource "datadog_monitor" "aws_cost_anomaly" {
name = "AWS Cost Anomaly - ${var.environment}"
type = "query alert"
message = "AWS cost has increased significantly for ${var.environment}. Please investigate potential cost optimization opportunities."
query = "avg (last_1d) :anomalies (avg:aws. billing.estimated _charges {account_id:$ {var.aws _account_id}}, 'basic', 2, direction='above')"
monitor _thresholds {
critical = 1
}
require _full_window = false
notify _no_data = false
tags = ["provider:aws", "team: ${var.team}", "env: ${var.environment}", "metric:cost" ]
}
resource "datadog_monitor" "gcp_cost _anomaly" {
name = "GCP Cost Anomaly - ${var.environment}"
type = "query alert"
message = "GCP cost has increased significantly for ${var.environment}. Please investigate potential cost optimization opportunities."
query = "avg (last_1d) :anomalies (avg:gcp. billing.cost {project_id: ${var.gcp _project_id}}, 'basic', 2, direction='above')"
monitor_thresholds {
critical = 1
}
require _full_window = false
notify _no_data = false
tags = ["provider:gcp", "team: ${var.team}", "env:$ {var.environment}", "metric:cost"]
}

# Example Kubernetes monitoring configuration in YAML

apiVersion: monitoring. coreos.com/v1
kind: ServiceMonitor
metadata:
name: api- service-monitor
namespace: monitoring
labels:
release: prometheus
spec:
selector:
matchLabels:
app: api-service
namespaceSelector:
matchNames:
- application
endpoints:
- port: metrics
interval: 15s
path: /metrics

---

apiVersion: monitoring. coreos.com/v1
kind: PrometheusRule
metadata:
name: kubernetes-alerts
namespace: monitoring
labels:
release: prometheus
spec:
groups:
- name: kubernetes-resources
rules:

- alert: PodHighCPUUsage
expr: sum (rate (container_cpu_ usage_seconds_ total {{container! ="POD",container! =""}}[5m])) by (namespace, pod) / sum (kube_pod_ container_resource_ limits_cpu_cores) by (namespace, pod) > 0.85
for: 10m
labels:
severity: warning
team: operations
annotations:
summary: "High CPU usage for pod {{ $labels.pod }} in namespace {{ $labels.namespace }}"
description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has been using more than 85% of its CPU limit for the last 10 minutes."

- alert: PodHigh MemoryUsage
expr: sum (container_memory _working_set bytes {{container! ="POD",container! =""}}) by (namespace, pod) / sum (kube_pod container_resource _limits_memory _bytes) by (namespace, pod) > 0.85
for: 10m
labels:
severity: warning
team: operations
annotations:
summary: "High memory usage for pod {{ $labels.pod }} in namespace {{ $labels.namespace }}"
description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} has been using more than 85% of its memory limit for the last 10 minutes."

- alert: PodCrashLooping
expr: rate (kube_pod_ container_status _restarts_total [15m]) > 0
for: 10m
labels:
severity: critical
team: operations
annotations:
summary: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} is crash looping"
description: "Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} is restarting frequently. Check logs for more details."

- name: kubernetes -services
rules:

- alert: KubernetesServiceDown
expr: kube_service _spec_type {type="ClusterIP"} unless on (namespace, service) (kube_service _spec_type {type="ClusterIP"} and kube_endpoint _address_available {{}} > 0)
for: 5m
labels:
severity: critical
team: operations
annotations:
summary: "Service {{ $labels.service }} in namespace {{ $labels.namespace }} has no endpoints"
description: "Service {{ $labels.service }} in namespace {{ $labels.namespace }} has no available endpoints. Check pods and deployments."

- alert: HighErrorRate
expr: sum (rate (istio_requests _total {{response _code=~ "5.*"}}[5m])) by (destination_service, destination _service_namespace) / sum (rate (istio_requests _total [5m])) by (destination_service , destination _service_namespace) > 0.05
for: 5m
labels:
severity: critical
team: application
annotations:
summary: "High error rate for service {{ $labels. destination_service }} in namespace {{ $labels. destination_service _namespace }}"
description: "Service {{ $labels. destination_service }} in namespace {{ $labels. destination_service _namespace }} has more than 5% error rate over the last 5 minutes."

# Example Terraform configuration for serverless monitoring

# AWS Lambda monitoring
module "lambda_monitoring" {
source = "./modules/lambda-monitoring"
environment = var.environment
notification_topic = var.notification_topic

common_parameters = {
error_rate_threshold = 0.05
duration_threshold_ms = 1000
throttle_threshold = 5
concurrent _executions_threshold = var.max_concurrent _executions * 0.8
}

functions = {
"api-handler" = {
duration_ threshold_ms = 500
invocation pattern = "scheduled"
},
"data-processor" = {
duration_threshold_ms = 10000
memory utilization _threshold = 0.8
invocation_pattern = "event-driven"
},
"notification -sender" = {
error_rate _threshold = 0.02
invocation _pattern = "event-driven"
}
}

enable_cold_ start_monitoring = true
cold_start_ threshold_ms = 1000
cold_start_ percentage_threshold = 0.1

enable_cost_monitoring = true
daily_cost_threshold = var.lambda_daily _cost_threshold
monthly_cost_threshold = var.lambda_monthly _cost_threshold
}

# Azure Functions monitoring
module "azure_functions_ monitoring" {
source = "./modules/azure- functions-monitoring"

resource_group_name = var.resource_group_name
app_name = var.function_app_name
environment = var.environment
action_group_id = var.action_group_id

metrics _configuration = {
execution_count _threshold = 1000
execution_units _threshold = 5000
error_percentage _threshold = 5
average_duration _threshold_ms = 1000
}

enable _health_probe = true
health _probe_interval = "00:05:00"
health _probe_timeout = "00:00:30"

enable _log_alerts = true
log _alert_configurations = [
{
name = "Function AppException"
query = "traces | where customDimensions.Category startswith 'Function' and severityLevel == 3"
threshold = 5
frequency = 5
time_window = 30
severity = 2
},
{
name = "Function ExecutionTimeout"
query = "traces | where message contains 'Execution timeout' and customDimensions.Category startswith 'Function'"
threshold = 1
frequency = 5
time_window = 60
severity = 1
}
]
}

# Google Cloud Functions monitoring
resource "google_monitoring _alert_policy" "function_error_rate" {
display_name = "Cloud Function Error Rate - ${var.environment}"
combiner = "OR"
conditions {
display_name = "Error rate for ${var.function_name}"
condition_threshold {
filter = "resource.type = "cloud_function" AND resource.labels .function_name = "${var.function_name}" AND metric.type = "cloudfunctions .googleapis.com /function /execution_count" AND metric. labels.status = "error""
duration = "300s"
comparison = "COMPARISON_GT"
aggregations {
alignment_period = "60s"
per_series_aligner = "ALIGN_RATE"
}
threshold_value = 0.05
denominator_filter = "resource.type = "cloud_function" AND resource.labels. function_name = "${var.function_name}" AND metric.type = "cloudfunctions. googleapis.com /function /execution_count""
denominator _aggregations {
alignment _period = "60s"
per_series _aligner = "ALIGN_RATE"
}
}
}
notification_channels = [var.notification_channel_id]
documentation {
content = "The error rate for Cloud Function ${var.function_name} has exceeded 5% over the last 5 minutes."
mime_type = "text/markdown"
}
}

resource "google_monitoring_dashboard" "serverless_dashboard" {
dashboard_json = templatefile ("${path.module} /templates /serverless_dashboard.json" , {
project_id = var.project_id
function_name = var.function_name
environment = var.environment
})
}