Integration configuration for AWS Logs
Configure log forwarding from AWS CloudWatch to Dynatrace using Terraform.
AWS CloudWatch Logs example
AWS log forwarding enables the streaming of logs from Amazon CloudWatch into Dynatrace logs via an ActiveGate. For more details, refer to CloudWatch Logs.
Below is a Terraform configuration example for setting up the CloudWatch log forwarder.
variable "stack_name" {type = stringdescription = "Dynatrace log forwarder name"default = "dynatracelogs"}variable "dynatrace_environment_url" {description = "URL to Dynatrace environment"}variable "dynatrace_api_key" {description = "Dynatrace API key"sensitive = true}variable "verify_ssl_target_active_gate" {description = "Verify SSL certificate for target ActiveGate"default = false}variable "max_log_content_length" {description = "Maximum log content length"default = 65536}variable "use_existing_active_gate" {description = "Use existing ActiveGate"default = true}variable "dynatrace_paas_token" {description = "Dynatrace PaaS token"sensitive = truedefault = "dummyToken"}variable "tenant_id" {description = "Tenant ID"default = "not_provided"}variable "deploy_ag_with_vpc" {type = booldescription = "Deploy ActiveGate with VPC"default = false}data "aws_ssm_parameter" "latest_amazon_linux_ami_id" {name = "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2"}data "aws_region" "current" {}data "aws_caller_identity" "current" {}data "aws_availability_zones" "available" {}resource "aws_vpc" "vpc" {count = var.deploy_ag_with_vpc ? 1 : 0cidr_block = "172.31.0.0/16"enable_dns_support = trueenable_dns_hostnames = true}resource "aws_subnet" "public_subnet" {count = var.deploy_ag_with_vpc ? 1 : 0vpc_id = aws_vpc.vpc[count.index].idcidr_block = "172.31.1.0/27"availability_zone = data.aws_availability_zones.available.names[0]map_public_ip_on_launch = true}resource "aws_subnet" "private_subnet" {count = var.deploy_ag_with_vpc ? 1 : 0vpc_id = aws_vpc.vpc[count.index].idavailability_zone = data.aws_availability_zones.available.names[0]cidr_block = "172.31.2.0/27"map_public_ip_on_launch = false}resource "aws_internet_gateway" "igw" {count = var.deploy_ag_with_vpc ? 1 : 0vpc_id = aws_vpc.vpc[count.index].id}resource "aws_route_table" "public_route_table" {count = var.deploy_ag_with_vpc ? 1 : 0vpc_id = aws_vpc.vpc[count.index].id}resource "aws_route" "public_route_internet" {count = var.deploy_ag_with_vpc ? 1 : 0route_table_id = aws_route_table.public_route_table.*.id[count.index]destination_cidr_block = "0.0.0.0/0"gateway_id = aws_internet_gateway.igw[count.index].id}resource "aws_route_table_association" "public_subnet_route_table_assoc" {count = var.deploy_ag_with_vpc ? 1 : 0subnet_id = aws_subnet.public_subnet[count.index].idroute_table_id = aws_route_table.public_route_table.*.id[count.index]}resource "aws_security_group" "security_group" {count = var.deploy_ag_with_vpc ? 1 : 0vpc_id = aws_vpc.vpc[count.index].iddescription = "Allow ActiveGate ingress"}resource "aws_security_group_rule" "active_gate_inbound" {count = var.deploy_ag_with_vpc ? 1 : 0type = "ingress"from_port = 9999to_port = 9999protocol = "tcp"security_group_id = aws_security_group.security_group.*.id[count.index]source_security_group_id = aws_security_group.security_group.*.id[count.index]}resource "aws_instance" "ec2_active_gate" {count = var.use_existing_active_gate ? 0 : 1ami = data.aws_ssm_parameter.latest_amazon_linux_ami_id.valueinstance_type = "t3.small"subnet_id = aws_subnet.public_subnet[count.index].idsecurity_groups = [aws_security_group.security_group[count.index].id]tags = {Name = "${var.stack_name}-active-gate"}user_data = <<-EOF#!/bin/bash -xewget -O Dynatrace-ActiveGate-Linux-x86.sh "${var.dynatrace_environment_url}/api/v1/deployment/installer/gateway/unix/latest?arch=x86&flavor=default" --header="Authorization: Api-Token ${var.dynatrace_paas_token}"wget https://ca.dynatrace.com/dt-root.cert.pem( echo 'Content-Type: multipart/signed; protocol="application/x-pkcs7-signature"; micalg="sha-256"; boundary="--SIGNED-INSTALLER"'; echo ; echo ; echo '----SIGNED-INSTALLER' ; cat Dynatrace-ActiveGate-Linux-x86.sh ) | openssl cms -verify -CAfile dt-root.cert.pem > /dev/null/bin/sh Dynatrace-ActiveGate-Linux-x86.shecho "[aws_monitoring]aws_monitoring_enabled = false[azure_monitoring]azure_monitoring_enabled = false[cloudfoundry_monitoring]cloudfoundry_monitoring_enabled = false[kubernetes_monitoring]kubernetes_monitoring_enabled = false[vmware_monitoring]vmware_monitoring_enabled = false[rpm]rpm_enabled = false[beacon_forwarder]beacon_forwarder_enabled = false[extension_controller]extension_controller_enabled = false[dbAgent]dbAgent_enabled = false[metrics_ingest]metrics_ingest_enabled = false[collector]MSGrouter = false" >> /var/lib/dynatrace/gateway/config/custom.propertiessystemctl restart dynatracegatewaytouch /home/ec2-user/userdata-ag-installation-successEOF}resource "aws_iam_role" "lambda_role" {name = "${var.stack_name}-lambda-role"assume_role_policy = jsonencode({Version = "2012-10-17"Statement = [{Effect = "Allow"Principal = {Service = "lambda.amazonaws.com"}Action = "sts:AssumeRole"}]})managed_policy_arns = ["arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"]inline_policy {name = "cloudwatch_put_metric_data"policy = jsonencode({Version = "2012-10-17"Statement = [{Effect = "Allow"Action = "cloudwatch:PutMetricData"Resource = "*"}]})}lifecycle {ignore_changes = [managed_policy_arns, inline_policy]}}resource "aws_lambda_function" "lambda" {filename = "dynatrace-aws-log-forwarder-lambda.zip"function_name = "${var.stack_name}-function"role = aws_iam_role.lambda_role.arnhandler = "index.handler"runtime = "python3.8"memory_size = 256timeout = 60environment {variables = {DEBUG = falseDYNATRACE_API_KEY = var.dynatrace_api_keyDYNATRACE_ENV_URL = var.deploy_ag_with_vpc ? "https://${element(aws_instance.ec2_active_gate.*.private_ip, 0)}:9999/e/${var.tenant_id}" : var.dynatrace_environment_urlVERIFY_SSL = var.verify_ssl_target_active_gateMAX_LOG_CONTENT_LENGTH = var.max_log_content_lengthCLOUD_LOG_FORWARDER = "${data.aws_caller_identity.current.account_id}:${data.aws_region.current.name}:${var.stack_name}"}}}resource "aws_s3_bucket" "delivery_bucket" {bucket = "${var.stack_name}-delivery-bucket"force_destroy = true}resource "aws_s3_bucket_lifecycle_configuration" "example" {bucket = aws_s3_bucket.delivery_bucket.idrule {id = "rule-1"status = "Enabled"expiration {days = 7}}}resource "aws_s3_bucket_public_access_block" "delivery_bucket_access" {bucket = aws_s3_bucket.delivery_bucket.idblock_public_acls = trueblock_public_policy = trueignore_public_acls = truerestrict_public_buckets = true}resource "aws_iam_role" "delivery_stream_role" {name = "${var.stack_name}-delivery-stream-role"assume_role_policy = jsonencode({Version = "2012-10-17"Statement = [{Effect = "Allow"Principal = {Service = "firehose.amazonaws.com"}Action = "sts:AssumeRole"}]})inline_policy {name = "firehose_delivery_policy"policy = jsonencode({Version = "2012-10-17"Statement = [{Effect = "Allow"Action = ["s3:AbortMultipartUpload","s3:GetBucketLocation","s3:GetObject","s3:ListBucket","s3:ListBucketMultipartUploads","s3:PutObject"]Resource = [aws_s3_bucket.delivery_bucket.arn,"${aws_s3_bucket.delivery_bucket.arn}/*"]}]})}}resource "aws_kinesis_firehose_delivery_stream" "firehose_log_streams" {name = "${var.stack_name}-firehose-delivery-stream"destination = "extended_s3"extended_s3_configuration {bucket_arn = aws_s3_bucket.delivery_bucket.arnrole_arn = aws_iam_role.delivery_stream_role.arnbuffering_interval = 60buffering_size = 5compression_format = "GZIP"error_output_prefix = "error-"prefix = "success-"processing_configuration {enabled = trueprocessors {type = "Lambda"parameters {parameter_name = "LambdaArn"parameter_value = aws_lambda_function.lambda.arn}}}}}resource "aws_iam_policy" "firehose_lambda_invocation_policy" {name = "firehose_lambda_invocation_policy"description = "Policy for Firehose Lambda invocation"policy = jsonencode({Version = "2012-10-17"Statement = [{Effect = "Allow"Action = ["lambda:InvokeFunction","lambda:GetFunctionConfiguration",]Resource = [aws_lambda_function.lambda.arn,]}]})}resource "aws_iam_role_policy_attachment" "delivery_stream_role_lambda_policy_attachment" {role = aws_iam_role.delivery_stream_role.namepolicy_arn = aws_iam_policy.firehose_lambda_invocation_policy.arn}resource "aws_iam_role" "cloudwatch_logs_role" {name = "${var.stack_name}-cloud-watch-role"assume_role_policy = jsonencode({Version = "2012-10-17"Statement = [{Effect = "Allow"Principal = {Service = "logs.${data.aws_region.current.name}.amazonaws.com"}Action = "sts:AssumeRole"}]})description = "Role for subscription filters (to write to Firehose)"}resource "aws_iam_policy" "cloudwatch_logs_policy" {name = "${var.stack_name}-cloud-watch-policy"policy = jsonencode({Version = "2012-10-17"Statement = [{Effect = "Allow"Action = ["firehose:PutRecord","firehose:PutRecordBatch",]Resource = aws_kinesis_firehose_delivery_stream.firehose_log_streams.arn}]})}resource "aws_iam_role_policy_attachment" "cloudwatch_logs_policy_attachment" {role = aws_iam_role.cloudwatch_logs_role.namepolicy_arn = aws_iam_policy.cloudwatch_logs_policy.arn}resource "aws_cloudwatch_dashboard" "self_monitoring_dashboard" {dashboard_name = "DynatraceLogForwarder-SelfMonitoring-${data.aws_region.current.name}-${var.stack_name}"dashboard_body = jsonencode({"widgets": [{"height": 6,"width": 12,"y": 6,"x": 0,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Kinesis record age", "function_name", "${aws_lambda_function.lambda.function_name}", { "stat": "Minimum" } ],[ "...", { "stat": "Average" } ],[ "..." ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","stat": "Maximum","period": 60,"liveData": true,"setPeriodToTimeRange": true,"legend": {"position": "bottom"},"title": "Kinesis - record age"}},{"height": 6,"width": 12,"y": 18,"x": 12,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Kinesis record.data decompressed size", "function_name", "${aws_lambda_function.lambda.function_name}" ],[ "...", { "stat": "Average" } ],[ "...", { "stat": "Maximum" } ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","stat": "Minimum","period": 60,"title": "Kinesis - record.data decompressed size"}},{"height": 6,"width": 12,"y": 18,"x": 0,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Kinesis record.data compressed size", "function_name", "${aws_lambda_function.lambda.function_name}" ],[ "...", { "stat": "Average" } ],[ "...", { "stat": "Maximum" } ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","stat": "Minimum","period": 60,"title": "Kinesis - record.data compressed size"}},{"height": 6,"width": 12,"y": 24,"x": 0,"type": "metric","properties": {"metrics": [[ { "expression": "SEARCH('{AWS/Logs,FilterName,LogGroupName,DestinationType} FilterName=\"${var.stack_name}\" MetricName=\"ForwardedLogEvents\"', 'Sum', 60)", "id": "e1", "region": "${data.aws_region.current.name}" } ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","period": 300,"stat": "Average","title": "Log Groups - log entries received from CloudWatch"}},{"height": 6,"width": 12,"y": 30,"x": 12,"type": "metric","properties": {"metrics": [[ { "expression": "SEARCH('{AWS/Logs,FilterName,LogGroupName,DestinationType} FilterName=\"${var.stack_name}\" MetricName=\"ForwardedBytes\"', 'Sum', 60)", "id": "e1", "region": "${data.aws_region.current.name}" } ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","period": 300,"stat": "Average","title": "Log Groups - bytes received from CloudWatch"}},{"height": 6,"width": 12,"y": 30,"x": 0,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Batches prepared", "function_name", "${aws_lambda_function.lambda.function_name}" ],[ ".", "Batches delivered", ".", "." ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","stat": "Sum","period": 60,"title": "Delivery - batches"}},{"height": 6,"width": 12,"y": 36,"x": 0,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Log entries prepared", "function_name", "${aws_lambda_function.lambda.function_name}" ],[ ".", "Log entries delivered", ".", "." ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","stat": "Sum","period": 60,"title": "Delivery - log entries"}},{"height": 6,"width": 12,"y": 36,"x": 12,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Data volume prepared", "function_name", "${aws_lambda_function.lambda.function_name}" ],[ ".", "Data volume delivered", ".", "." ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","title": "Delivery - data volume","period": 60,"stat": "Sum"}},{"height": 6,"width": 12,"y": 48,"x": 0,"type": "metric","properties": {"metrics": [[ { "expression": "SEARCH('{DT/LogsStreaming,function_name,status_code} function_name=\"${aws_lambda_function.lambda.function_name}\" MetricName=\"Requests status code count\"', 'Sum', 60)", "id": "e1", "region": "${data.aws_region.current.name}" } ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","title": "Requests - status codes","period": 300,"stat": "Average"}},{"height": 6,"width": 12,"y": 42,"x": 0,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Requests duration", "function_name", "${aws_lambda_function.lambda.function_name}", { "stat": "Minimum" } ],[ "..." ],[ "...", { "stat": "Maximum" } ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","period": 60,"stat": "Average","title": "Requests - durations"}},{"height": 6,"width": 12,"y": 48,"x": 12,"type": "metric","properties": {"metrics": [[ { "expression": "SEARCH('{DT/LogsStreaming,function_name,type} function_name=\"${aws_lambda_function.lambda.function_name}\" MetricName=\"Issues\"', 'Sum', 60)", "id": "e1", "region": "${data.aws_region.current.name}" } ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","stat": "Sum","period": 60,"title": "Delivery - issues"}},{"height": 6,"width": 12,"y": 42,"x": 12,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Log attr trimmed", "function_name", "${aws_lambda_function.lambda.function_name}" ],[ ".", "Log content trimmed", ".", "." ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","period": 60,"title": "Logs - trimmed","stat": "Sum"}},{"height": 6,"width": 12,"y": 6,"x": 12,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Log age min", "function_name", "${aws_lambda_function.lambda.function_name}", { "stat": "Minimum" } ],[ "DT/LogsStreaming", "Log age avg", "function_name", "${aws_lambda_function.lambda.function_name}" ],[ "DT/LogsStreaming", "Log age max", "function_name", "${aws_lambda_function.lambda.function_name}", { "stat": "Maximum" } ]],"view": "timeSeries","stacked": false,"title": "Logs - age","region": "${data.aws_region.current.name}","period": 60,"stat": "Average"}},{"height": 6,"width": 12,"y": 0,"x": 0,"type": "metric","properties": {"view": "timeSeries","stacked": false,"metrics": [[ "AWS/Lambda", "Invocations", "FunctionName", "${aws_lambda_function.lambda.function_name}" ]],"region": "${data.aws_region.current.name}","title": "Lambda - invocations"}},{"height": 6,"width": 12,"y": 0,"x": 12,"type": "metric","properties": {"view": "timeSeries","stacked": false,"metrics": [[ "AWS/Lambda", "Duration", "FunctionName", "${aws_lambda_function.lambda.function_name}" ]],"region": "${data.aws_region.current.name}","title": "Lambda - duration"}},{"height": 6,"width": 12,"y": 12,"x": 12,"type": "metric","properties": {"metrics": [[ "AWS/Lambda", "Errors", "FunctionName", "${aws_lambda_function.lambda.function_name}" ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","stat": "Sum","period": 60,"title": "Lambda errors"}},{"height": 6,"width": 12,"y": 12,"x": 0,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Kinesis record age", "function_name", "${aws_lambda_function.lambda.function_name}", { "label": "Kinesis records" } ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","stat": "SampleCount","period": 60,"title": "Kinesis - records number"}},{"height": 6,"width": 12,"y": 24,"x": 12,"type": "metric","properties": {"metrics": [[ "DT/LogsStreaming", "Kinesis record.data decompressed size", "function_name", "${aws_lambda_function.lambda.function_name}" ]],"view": "timeSeries","stacked": false,"region": "${data.aws_region.current.name}","stat": "Sum","period": 60,"title": "Kinesis - sum record.data decompressed size"}}]})}