Integration configuration for Azure and AWS Logs

1locals {
2  stack_name = "dynatrace-aws-logs"
3  # tags are optional
4  tags = {
5    source = "terraform"
6  }
7
8  # To discover potential names for log groups the command
9  #   aws logs describe-log-groups --output text --query "logGroups[].[logGroupName]"
10  # will be helpful
11  log_groups = [
12    "/aws/lambda/my-lambda",
13    "/aws/apigateway/my-api"
14  ]
15
16  # https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html
17  # empty string matches everything
18  filter_pattern = ""
19}
20
21resource "local_file" "dynatrace_aws_log_forwarder" {
22  filename = "${path.module}/dynatrace-aws-log-forwarder.zip"
23  source   = "https://github.com/dynatrace-oss/dynatrace-aws-log-forwarder/releases/latest/download/dynatrace-aws-log-forwarder.zip"
24}
25
26resource "archive_file" "dynatrace_aws_log_forwarder" {
27  type        = "zip"
28  # You may want to choose a different folder to extract to here
29  output_path = "${path.module}"
30  source_file = file(local_file.dynatrace_aws_log_forwarder.filename)
31}
32
33# ----------------------------------------------------------------------------------------------------------------------
34# Lambda Function code needs to be available via S3 Bucket
35# ----------------------------------------------------------------------------------------------------------------------
36resource "aws_s3_bucket" "dynatrace_aws_log_forwarder" {
37  bucket = "dynatrace_aws_log_forwarder"
38}
39
40resource "aws_s3_bucket_object" "dynatrace_aws_log_forwarder" {
41  bucket = aws_s3_bucket.dynatrace_aws_log_forwarder.id
42  key    = "dynatrace-aws-log-forwarder.zip"
43  source = "${archive_file.dynatrace_aws_log_forwarder.output_path}/dynatrace-aws-log-forwarder-lambda.zip"
44}
45
46# ----------------------------------------------------------------------------------------------------------------------
47# Create Stack
48# ----------------------------------------------------------------------------------------------------------------------
49resource "aws_cloudformation_stack" "dynatrace_aws_log_forwarder" {
50  name = local.stack_name
51  capabilities = ["CAPABILITY_IAM"]
52  tags = local.tags
53  parameters = {
54    # The URL to Your Dynatrace SaaS logs ingest target.
55    # If you choose new ActiveGate deployment (UseExistingActiveGate = false), provide Tenant URL (https://<your_environment_ID>.live.dynatrace.com).
56    # If you choose to use existing ActiveGate (UseExistingActiveGate = true), provide ActiveGate endpoint:
57    #  - for Public ActiveGate: https://<your_environment_ID>.live.dynatrace.com
58    #  - for Environment ActiveGate: https://<active_gate_address>:9999/e/<environment_id> (e.g. https://22.111.98.222:9999/e/abc12345)
59    DynatraceEnvironmentUrl = var.target_url
60    # Set to "" if you choose to new an existing ActiveGate (UseExistingActiveGate = false). It will not be used.
61    # If you choose to use an existing ActiveGate (UseExistingActiveGate = true)
62    #  - for Public ActiveGate: https://<TenantId>.live.dynatrace.com
63    #  - for Environment ActiveGate: https://<active_gate_address>:9999/e/<TenantId> (e.g. https://22.111.98.222:9999/e/abc12345)
64    TenantId = "tenant-id"
65    # Dynatrace API token. Integration requires API v1 Log import Token permission.
66    DynatraceApiKey = var.target_api_token
67    # Enables checking SSL certificate of the target Active Gate
68    # By default (if this option is not provided) certificates aren't validated
69    VerifySSLTargetActiveGate = false
70    # If you choose new ActiveGate deployment, set to 'false'
71    # In such case, a new EC2 with ActiveGate will be added to log forwarder deployment (enclosed in VPC with log forwarder)
72    # If you choose to use existing ActiveGate (either Public AG or Environment AG), set to 'true'.
73    UseExistingActiveGate = true
74    # Only needed when UseExistingActiveGate = false
75    # PaaS token generated in Integration/Platform as a Service.
76    # Used for ActiveGate installation.
77    DynatracePaasToken = var.target_paas_token
78    # Defines the max log length after which a log will be truncated
79    # For values over 8192 there's also a change in Dynatrace settings needed
80    # For that you need to contact Dynatrace One.
81    MaxLogContentLength = 8192
82  }
83
84  ###############################################################
85  # This is VERY messy - definitely room for improvement
86  # When executing `dynatrace-aws-logs.sh deploy` it executes two things in a row:
87  #   * `aws cloudformation deploy` creates a stack using the template file `dynatrace-aws-log-forwarder-template.yaml`
88  #       - The Lambda Function defined within `dynatrace-aws-log-forwarder-template.yaml` contains a source code only a skeleton
89  #   * `aws lambda update-function-code` immediately afterwards updates the source code contained within `dynatrace-aws-log-forwarder-lambda.zip`
90  # The AWS Terraform Provider doesn't offer a counterpart for `aws lambda update-function-code`
91  #   - The resource `aws_lambda_function` would CREATE a Lambda Function for us
92  #   - But in that case we'd have to replicate the settings of it from `dynatrace-aws-log-forwarder-template.yaml`
93  # The current solution replaces within the template the skeleton code
94  #     ZipFile: |
95  #        def handler(event, context):
96  #          raise Exception("Dynatrace Logs Lambda has not been uploaded")
97  # with
98  #     S3Bucket: ${aws_s3_bucket.dynatrace_aws_log_forwarder.id}
99  #     S3Key: ${aws_s3_bucket_object.dynatrace_aws_log_forwarder.key}
100  # in other words, we're referring to the zip archive that has been previously made available via S3 Bucket
101  # That way there is no counterpart for `aws lambda update-function-code` necessary
102  template_body = replace(file("dynatrace-aws-log-forwarder-template.yaml"), "        ZipFile: |\n          def handler(event, context):\n            raise Exception(\"Dynatrace Logs Lambda has not been uploaded\")", "        S3Bucket: ${aws_s3_bucket.dynatrace_aws_log_forwarder.id}\n        S3Key: ${aws_s3_bucket_object.dynatrace_aws_log_forwarder.key}")
103}
104
105resource "aws_cloudwatch_log_subscription_filter" "example" {
106  for_each = local.log_groups
107
108  name            = local.stack_name
109  role_arn        = aws_cloudformation_stack.dynatrace_aws_log_forwarder.outputs["CloudWatchLogsRoleArn"]
110  log_group_name  = each.key
111  filter_pattern  = local.filter_pattern
112  destination_arn = aws_cloudformation_stack.dynatrace_aws_log_forwarder.outputs["FirehoseArn"]
113}
114
115# aws logs put-subscription-filter --log-group-name "$LOG_GROUP" --filter-name "$SUBSCRIPTION_FILTER_NAME" --filter-pattern "$FILTER_PATTERN"  --role-arn "$ROLE_ARN"

1locals {
2  deployment_name  = "deployment_name"
3  target_url       = "target_url"
4  target_api_token = "target_api_token"
5}
6
7# ######################################################################################
8# CREATE RESOURCE GROUP
9# ######################################################################################
10
11resource "azurerm_resource_group" "resgrp" {
12  name     = "example-resources"
13  location = "East US"
14}
15
16# ######################################################################################
17# SET UP AZURE EVENT HUBS INSTANCE
18# ######################################################################################
19
20resource "azurerm_eventhub_namespace" "evthubns" {
21  name                = "acceptanceTestEventHubNamespace"
22  location            = azurerm_resource_group.resgrp.location
23  resource_group_name = azurerm_resource_group.resgrp.name
24  sku                 = "Basic"
25  capacity            = 2
26
27  tags = {
28    environment = "Production"
29  }
30}
31
32resource "azurerm_eventhub" "evthub" {
33  name                = "acceptanceTestEventHub"
34  namespace_name      = azurerm_eventhub_namespace.evthubns.name
35  resource_group_name = azurerm_resource_group.resgrp.name
36  partition_count     = 2
37  message_retention   = 2
38}
39
40resource "azurerm_eventhub_namespace_authorization_rule" "evthubnsauthrule" {
41  name                = "example-auth-rule"
42  namespace_name      = azurerm_eventhub_namespace.evthubns.name
43  resource_group_name = azurerm_eventhub_namespace.evthubns.resource_group_name
44  listen              = true
45  send                = false
46  manage              = false
47}
48
49# ######################################################################################
50# STORAGE ACCOUNT
51# ######################################################################################
52
53resource "azurerm_storage_account" "storage" {
54  name                     = "storageaccountname"
55  resource_group_name      = azurerm_resource_group.resgrp.name
56  location                 = azurerm_resource_group.resgrp.location
57  account_tier             = "Standard"
58  account_replication_type = "LRS"
59}
60
61data "azurerm_client_config" "current" {}
62
63resource "azurerm_key_vault" "example" {
64  name                       = "examplekeyvault"
65  location                   = azurerm_resource_group.resgrp.location
66  resource_group_name        = azurerm_resource_group.resgrp.name
67  tenant_id                  = data.azurerm_client_config.current.tenant_id
68  soft_delete_retention_days = 7
69  purge_protection_enabled   = false
70  sku_name                   = "standard"
71}
72
73# ######################################################################################
74# CONFIGURE MONITORING SETTINGS - ROUTE TO EVENT HUB INSTANCE
75# ######################################################################################
76
77resource "azurerm_monitor_diagnostic_setting" "example" {
78  name                           = "example-diagnostic-setting"
79  target_resource_id             = azurerm_storage_account.storacc.id
80  eventhub_authorization_rule_id = azurerm_eventhub_namespace_authorization_rule.evthubnsauthrule.authorization_rule_id
81
82  log {
83    category = "AuditEvent"
84    enabled  = true
85
86    retention_policy {
87      enabled = true
88      days    = 30
89    }
90  }
91}
92
93# #################################################################################################################
94# -----------------------------------------------------------------------------------------------------------------
95#
96#   EVERYTHING FROM HERE ON REPLICATES WHAT IS USUALLY DONE BY
97#   https://github.com/dynatrace-oss/dynatrace-azure-log-forwarder/releases/latest/download/dynatrace-azure-logs.sh
98#
99# -----------------------------------------------------------------------------------------------------------------
100# #################################################################################################################
101
102
103# ######################################################################################
104# DEPLOYMENT GROUP
105# ######################################################################################
106
107data "http" "template" {
108  url = "https://github.com/dynatrace-oss/dynatrace-azure-log-forwarder/releases/download/release-0.1.3/dynatrace-azure-forwarder.json"
109}
110
111resource "azurerm_resource_group_template_deployment" "example" {
112  name                = "example-deployment"
113  resource_group_name = azurerm_resource_group.resgrp.name
114  deployment_mode     = "Incremental" # Unsure here. Should this be "Complete"?
115  template_content    = data.http.template.body
116  parameters = jsondecode({
117    "forwarderName" : "${local.deployment_name}",
118    "targetUrl" : "${local.target_url}",
119    "targetAPIToken" : "${local.target_api_token}",
120    "eventHubConnectionString" : "${azurerm_eventhub_namespace_authorization_rule.evthubnsauthrule.primary_connection_string}",
121    "eventHubName" : "${azurerm_eventhub.evthub.name}",
122    "requireValidCertificate" : true,
123    "selfMonitoringEnabled" : false,
124    "deployActiveGateContainer" : false,
125    "targetPaasToken" : "",
126    "filterConfig" : "",
127    "resourceTags" : "\"LogsForwarderDeployment\":\"${local.deployment_name}\""
128  })
129}
130
131resource "azurerm_app_service_plan" "example" {
132  name                = "example-app-service-plan"
133  location            = azurerm_resource_group.resgrp.location
134  resource_group_name = azurerm_resource_group.resgrp.name
135  sku {
136    tier = "Standard"
137    size = "S1"
138  }
139}
140
141# wait some time to allow functionapp to warmup ?
142resource "azurerm_web_app" "example" {
143  name                = "${local.deployment_name}-function"
144  location            = azurerm_resource_group.resgrp.location
145  resource_group_name = azurerm_resource_group.resgrp.name
146  app_service_plan_id = azurerm_app_service_plan.example.id
147
148  site_config {
149    dotnet_framework_version = "v4.0"
150  }
151}
152
153resource "azurerm_storage_container" "container" {
154  name                  = "webapp-container"
155  storage_account_name  = azurerm_storage_account.storage.name
156  container_access_type = "private"
157}
158
159resource "azurerm_storage_blob" "log_forwarder_blob" {
160  name                   = "dynatrace-azure-log-forwarder.zip"
161  storage_account_name   = azurerm_storage_account.storage.name
162  storage_container_name = azurerm_storage_container.container.name
163  type                   = "Block"
164  source                 = "https://github.com/dynatrace-oss/dynatrace-azure-log-forwarder/releases/download/release-0.1.3/dynatrace-azure-log-forwarder.zip"
165}
166
167resource "azurerm_app_service" "example" {
168  name                = "${local.deployment_name}-function"
169  location            = azurerm_resource_group.resgrp.location
170  resource_group_name = azurerm_resource_group.resgrp.name
171  app_service_plan_id = azurerm_app_service_plan.example.id
172
173  site_config {
174    always_on                = true
175    dotnet_framework_version = "v4.0"
176  }
177
178  storage_account {
179    name           = azurerm_storage_account.storage.name
180    type           = "AzureBlob"
181    account_key    = azurerm_storage_account.storage.primary_access_key
182    container_name = azurerm_storage_container.container.name
183    path           = "site/wwwroot"
184  }
185}