Connecting log data to traces

1{
2  "severity": "error",
3  "time": 1638957438023,
4  "pid": 1,
5  "hostname": "paymentservice-788946fdcd-42lgq",
6  "name": "paymentservice-charge",
7  "dt.trace_id": "d04b42bc9f4b6ecdbf6bc9f4b6ecdbc",
8  "dt.span_id": "9adc716eb808d428",
9  "dt.entity.process_group_instance": "PROCESS_GROUP_INSTANCE-27204EFED3D8466E",
10  "message": "Unsupported card type for cardNumber=************0454"
11}

1<?xml version="1.0" encoding="windows-1252" standalone="no"?>
2<record>
3    <date>2021-08-24T14:41:36.565218700Z</date>
4    <millis>1629816096565</millis>
5    <nanos>218700</nanos>
6    <sequence>0</sequence>
7    <logger>com.apm.testapp.logging.jul.XMLLoggingSample</logger>
8    <level>INFO</level>
9    <class>com.apm.testapp.logging.jul.BaseLoggingSample</class>
10    <method>info</method>
11    <thread>1</thread>
12    <message>Update completed successfully.</message>
13    <dt.trace_id>513fcd4e9b08792fcd4e9b08792</dt.trace_id>
14    <dt.span_id>125840e3125840e3</dt.span_id>
15    <dt.entity.process_group_instance>PROCESS_GROUP_INSTANCE-27204EFED3D8466E</dt.entity.process_group_instance>
16</record>

1127.0.0.1 - [21/Oct/2021:10:33:28 +0200] GET /index.htm HTTP/1.1 404 597 [!dt dt.trace_id=aa764ee37ebaa764ee37eaa764ee37e,dt.span_id=b93ede8b93ede8, dt.entity.process_group_instance=PROCESS_GROUP_INSTANCE-27204EFED3D8466E]

1log_format custom '$remote_addr - [$time_local] $request $status $body_bytes_sent [!dt dt.trace_id=$dt_trace_id,dt.span_id=$dt_span_id,dt.trace_sampled=$dt_trace_sampled]';
2access_log logs/access.log custom;

1127.0.0.1 - [22/Mar/2022:08:50:45 +0100] GET /index.htm HTTP/1.1 200 30 [!dt dt.trace_id=b9e5c9ec08be5fab5071d76f427be7da,dt.span_id=43c5bb9432593963,dt.trace_sampled=true]
2127.0.0.1 - [22/Mar/2022:08:50:45 +0100] GET /index.htm HTTP/1.1 200 30 [!dt dt.trace_id=01e52950b145d97bf22345e68c5e6c58,dt.span_id=de819d856eecb236,dt.trace_sampled=true]

1log_format custom '$remote_addr - [$time_local] $request $status $body_bytes_sent [!dt dt.trace_id=$trace_id,dt.span_id=$span_id]'; access_log logs/access.log custom

1127.0.0.1 - [21/Oct/2021:10:33:28 +0200] GET /index.htm HTTP/1.1 404 597 [!dt dt.trace_id=e1c0afeb0b8a91d7748139aa764ee37e,dt.span_id=e5e6748fab93ede8]
2127.0.0.1 - [21/Oct/2021:10:33:31 +0200] GET /index.html HTTP/1.1 200 1056 [!dt dt.trace_id=81fe7816ba6c38f7aa09aef3684cd941,dt.span_id=3bdacc466ae073cd]

1<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} dt.trace_id=%X{dt.trace_id} dt.span_id=%X{dt.span_id} dt.entity.process_group_instance=%X{dt.entity.process_group_instance} - %msg%n"/>

1<appender name="COMPOSITEJSONENCODER" class="ch.qos.logback.core.FileAppender">
2  <file>compositejsonencoder.log</file>
3  <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
4    <providers>
5      <timestamp>
6        <fieldName>timestamp</fieldName>
7        <timeZone>UTC</timeZone>
8      </timestamp>
9      <loggerName>
10        <fieldName>logger</fieldName>
11      </loggerName>
12      <logLevel>
13        <fieldName>level</fieldName>
14      </logLevel>
15      <threadName>
16        <fieldName>thread</fieldName>
17      </threadName>
18      <mdc>
19        <includeMdcKeyName>dt.span_id</includeMdcKeyName>
20        <includeMdcKeyName>dt.trace_id</includeMdcKeyName>
21        <includeMdcKeyName>dt.entity.host</includeMdcKeyName>
22      </mdc>
23      <stackTrace>
24        <fieldName>stackTrace</fieldName>
25        <!-- maxLength - limit the length of the stack trace -->
26        <throwableConverter class="net.logstash.logback.stacktrace.ShortenedThrowableConverter">
27          <maxDepthPerThrowable>200</maxDepthPerThrowable>
28          <maxLength>14000</maxLength>
29          <rootCauseFirst>true</rootCauseFirst>
30        </throwableConverter>
31      </stackTrace>
32      <message />
33      <throwableClassName>
34        <fieldName>exceptionClass</fieldName>
35      </throwableClassName>
36    </providers>
37  </encoder>
38</appender>

1main-snippet: load_module /opt/dynatrace/oneagent/agent/bin/current/linux-musl-x86-64/liboneagentnginx.so;
2log-format-upstream: '$remote_addr - $remote_user [$time_local] "$request" [!dt dt.trace_id=$dt_trace_id,dt.span_id=$dt_span_id,dt.trace_sampled=$dt_trace_sampled] $status $body_bytes_sent  "$http_referer" "$http_user_agent" $request_length'

1import json
2
3from opentelemetry import trace as OpenTelemetry
4from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
5    OTLPSpanExporter,
6)
7from opentelemetry.sdk.resources import Resource
8from opentelemetry.sdk.trace import TracerProvider, sampling
9from opentelemetry.sdk.trace.export import (
10    BatchSpanProcessor,
11)
12
13merged = dict()
14for name in ["dt_metadata_e617c525669e072eebe3d0f08212e8f2.json", "/var/lib/dynatrace/enrichment/dt_metadata.json"]:
15    try:
16        data = ''
17        with open(name) as f:
18          data = json.load(f if name.startswith("/var") else open(f.read()))
19        merged.update(data)
20    except:
21        pass
22
23merged.update({
24    "service.name": "python-quickstart", #TODO Replace with the name of your application
25    "service.version": "1.0.1", #TODO Replace with the version of your application
26})
27resource = Resource.create(merged)
28
29tracer_provider = TracerProvider(sampler=sampling.ALWAYS_ON, resource=resource)
30OpenTelemetry.set_tracer_provider(tracer_provider)
31
32tracer_provider.add_span_processor(
33    BatchSpanProcessor(OTLPSpanExporter(
34        endpoint="http://localhost:14499/otlp/v1/traces"
35    )))

1const winston = require("winston");
2const Transport = require("winston-transport");
3
4class CustomTransport extends Transport {
5  log(info, next) {
6    let myLogLine = `MyLogLine: ${info.timestamp} level=${info.level}: ${info.message}`;
7    // this is important as above line only picks timestamp, level and message but nothing else from metadata
8    if (info["dt.trace_id"]) {
9      myLogLine = `[!dt dt.trace_id=${info["dt.trace_id"]},dt.span_id=${info["dt.span_id"]},dt.trace_sampled=${info["dt.trace_sampled"]}] ${myLogLine}`;
10    }
11    console.log(myLogLine);
12    next();
13  }
14}
15
16const logger = winston.createLogger({
17  level: "info",
18  format: winston.format.timestamp(),
19  transports: [
20    new CustomTransport(),
21    // this transport includes all metadata (including dynatrace added traceId,..)
22    new winston.transport.Console({
23      format: winston.format.simple()
24    })
25  ]
26})