Tempo 面试题

# tempo-distributed Helm Chart values.yaml 关键配置
distributor:
  replicas: 3
  config:
    receivers:
      otlp:
        protocols:
          grpc: {}
          http: {}
ingester:
  replicas: 3
  config:
    trace_idle_period: 10s
    max_block_duration: 5m
querier:
  replicas: 3
  config:
    search:
      max_duration: 0  # 0 = 禁用搜索

# Grafana datasources.yaml
apiVersion: 1
datasources:
  - name: Loki
    type: loki
    uid: loki
    url: http://loki:3100
  - name: Tempo
    type: tempo
    uid: tempo
    url: http://tempo:3100
    jsonData:
      tracesToLogsV2:
        datasourceUid: loki
        spanStartTimeShift: -1h
        spanEndTimeShift: 1h
        filterByTraceID: true
        filterBySpanID: true
      tracesToMetrics:
        datasourceUid: prometheus
        spanStartTimeShift: -1h
        spanEndTimeShift: 1h
        queries:
          - name: "Request rate"
            query: "sum(rate(http_server_requests_seconds_count{service=`$service`}[5m]))"

# 1. 通过 Loki 日志定位错误
{service="checkout"} |= "error" | json | traceID="abc123"

# 2. Grafana 自动从日志提取 traceID，跳转 Tempo
# 3. Tempo 返回完整调用链
# 4. 点击 Span 中的 service.name，跳转 Prometheus 查看该服务指标

# Grafana Tempo datasource
apiVersion: 1
datasources:
  - name: Tempo
    type: tempo
    uid: tempo
    url: http://tempo:3100
    jsonData:
      httpMethod: GET
      tracesToLogsV2:
        datasourceUid: loki
        spanStartTimeShift: -1h    # 日志时间窗口起点偏移
        spanEndTimeShift: 1h       # 日志时间窗口终点偏移
        filterByTraceID: true      # 通过 traceID 过滤
        filterBySpanID: true       # 通过 spanID 过滤
        customQuery: true          # 启用自定义 LogQL
        query: '{service=`$service`} | json | traceID="$${traceId}"'
      serviceMap:
        datasourceUid: prometheus
      nodeGraph:
        enabled: true
      search:
        maxDuration: 5m            # 限制 TraceQL 搜索时间范围

# Python OTel SDK 自动注入 traceID 到日志
from opentelemetry import trace
from opentelemetry.instrumentation.logging import LoggingInstrumentor

# 启用日志注入（自动添加 trace_id / span_id 字段）
LoggingInstrumentor().instrument(set_logging_format=True)

import logging
logger = logging.getLogger(__name__)
logger.error("payment failed", extra={
    "order_id": "12345",
    "amount": 99.9,
    # OTel 自动添加:
    # "otelTraceID": "abc123...",
    # "otelSpanID": "def456...",
})

{
  "timestamp": "2026-06-06T10:30:00Z",
  "level": "error",
  "service": "checkout",
  "message": "payment failed",
  "traceID": "abc123def456",
  "spanID": "789ghi",
  "order_id": "12345"
}

# OpenTelemetry SDK 自动关联 exemplar
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from prometheus_client import Counter

trace.set_tracer_provider(TracerProvider())
tracer = trace.get_tracer(__name__)

request_counter = Counter('http_requests_total', 'Total requests')

# Prometheus 抓取指标时携带 traceID exemplar
with tracer.start_as_current_span("handle_request") as span:
    request_counter.labels(method="GET", status="200").inc(
        exemplar={"trace_id": span.get_span_context().trace_id}
    )

# otel-collector-config.yaml
processors:
  tail_sampling:
    decision_wait: 10s              # 等待 10s 收集所有 Span
    num_traces: 50000               # 内存中维护的 Trace 数
    expected_new_traces_per_sec: 1000
    policies:
      # 错误请求 100% 保留
      - name: errors
        type: status_code
        status_code: {status_codes: [ERROR]}
      # P99 延迟 > 2s 的请求 100% 保留
      - name: slow-traces
        type: latency
        latency: {threshold_ms: 2000}
      # 健康请求采样 1%
      - name: health-sampled
        type: probabilistic
        probabilistic: {sampling_percentage: 1}

exporters:
  otlp/tempo:
    endpoint: tempo:4317
    tls: {insecure: true}

# tempo.yaml - 接收端限制
distributor:
  receivers:
    otlp:
      protocols:
        grpc: {}
  # 限制单租户每秒 Span 数
  span_limits:
    max_recv_msg_size: 16777216  # 16MB

ingester:
  config:
    # 单个 Trace 最大 Span 数（防止单 Trace 撑爆存储）
    max_traces_per_user: 10000
    # 批处理周期
    flush_check_period: 10s
    trace_idle_period: 10s

# pyroscope.yaml
pyroscopedb:
  data_path: /var/lib/pyroscope

# 启用 trace ingestion
tracing:
  enabled: true
  sampling_fraction: 0.1  # 10% Trace 携带 Profile

# Tempo 地址
otlp:
  endpoint: tempo:4317

import (
    "github.com/grafana/pyroscope-go"
    "go.opentelemetry.io/otel"
)

func main() {
    // 启动 Pyroscope agent，集成 OTel Trace
    pyroscope.Start(pyroscope.Config{
        ApplicationName: "checkout-service",
        ServerAddress:   "http://pyroscope:4040",
        Tags: map[string]string{
            "service.version": "v1.2.3",
            "region":          "us-east-1",
        },
    })

    // 创建带 Pyroscope Profile 的 Span
    tracer := otel.Tracer("checkout")
    ctx, span := tracer.Start(context.Background(), "process-order")
    defer span.End()

    // 关联 Profile 到 Span
    pyroscope.TagWrapper(ctx, func(ctx context.Context) {
        // 业务逻辑，Pyroscope 自动捕获 CPU/Mem Profile
        processOrder(ctx)
    })
}

# Grafana Explore 配置关联面板
panels:
  - title: "Slow Request Trace"
    type: trace
    datasource: {type: tempo, uid: tempo}
  - title: "CPU Flamegraph for /checkout"
    type: flamegraph
    datasource: {type: pyroscope, uid: pyroscope}
    targets:
      - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds{service="checkout"}'

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc: {endpoint: 0.0.0.0:4317}
  jaeger:
    protocols:
      thrift_http: {endpoint: 0.0.0.0:14268}
      grpc: {endpoint: 0.0.0.0:14250}

processors:
  batch:
    timeout: 10s
    send_batch_size: 10000
  # 转换资源属性
  resource:
    attributes:
      - key: deployment.environment
        from_attribute: env
        action: insert

exporters:
  otlp/tempo:
    endpoint: tempo-distributor:4317
    tls: {insecure: true}
    sending_queue: {enabled: true, num_consumers: 10}
  otlp/jaeger-legacy:
    endpoint: jaeger-collector:14250
    tls: {insecure: true}

service:
  pipelines:
    traces:
      receivers: [otlp, jaeger]
      processors: [batch, resource]
      exporters: [otlp/tempo, otlp/jaeger-legacy]  # 双写过渡期

# 阶段 1: 双写（应用同时发送 Jaeger 和 OTel）
# OpenTelemetry SDK 配置 OTLP exporter
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter

provider = TracerProvider()
provider.add_span_processor(
    BatchSpanProcessor(OTLPSpanExporter(endpoint="http://otel-collector:4317"))
)
trace.set_tracer_provider(provider)

# 阶段 2: 完全切换到 OTLP，移除 Jaeger Client
# 删除 jaeger-client 依赖，OTel SDK 自动处理

# tempo.yaml
distributor:
  receivers:
    jaeger:
      protocols:
        thrift_http: {endpoint: 0.0.0.0:14268}
        grpc: {endpoint: 0.0.0.0:14250}
        thrift_compact: {endpoint: 0.0.0.0:6831}
        thrift_binary: {endpoint: 0.0.0.0:6832}
    zipkin:
      endpoint: 0.0.0.0:9411
    otlp:
      protocols:
        grpc: {endpoint: 0.0.0.0:4317}
        http: {endpoint: 0.0.0.0:4318}

# 阶段 1: 保留 Jaeger 数据源，新增加 Tempo
datasources:
  - name: Jaeger-Legacy
    type: jaeger
    url: http://jaeger-query:16686
  - name: Tempo
    type: tempo
    url: http://tempo:3100

# 阶段 2: 统一使用 Tempo，旧数据源通过 Tempo 的 search 功能查询历史

# 基础结构
{ <span-set-filter> } [ <attribute-filter> ] [ <pipeline-filter> ]

# 示例
{ resource.service.name = "checkout" }                     # 过滤服务
{ span.http.status_code = 500 }                             # 过滤状态码
{ resource.service.name = "checkout" && duration > 2s }      # 组合条件
{ resource.service.name = "checkout" && span.http.method = "POST" && duration > 500ms }

# parent 关系：查找所有父 Span 为 "POST /api" 的子 Span
{ parent.name = "POST /api" }

# descendant 关系：查找所有后裔 Span
{ descendant.name =~ "SELECT.*" }

# combine 关系：联合多个 Span 集
{ resource.service.name = "frontend" } && { resource.service.name = "backend" }

# 链式 Span 过滤：root 是 HTTP，descendant 包含 DB 查询
{ name = "HTTP GET /api" } >> { name =~ "SELECT.*" }

# 计算 P99 延迟
{ resource.service.name = "checkout" } | quantile_over_time(duration, 0.99)
# 注：quantile_over_time 是 Tempo 支持的内置 TraceQL 聚合函数，非 PromQL

# 计算错误率
{ resource.service.name = "checkout" && status = error } | count() / count()

# 按服务名分组的错误数
{ status = error } | sum by (resource.service.name)(count())

# Top 10 最慢 Span
{ name = "db.query" } | topk(10, max(duration))

# 1. 先过滤再聚合
{ resource.service.name = "checkout" && status = error } | topk(5, max(duration))

# 2. select 投影
{ resource.service.name = "checkout" } | select(span.http.url, duration)

# 3. 条件过滤
{ resource.service.name = "checkout" } | status = error

# 1. 查找过去 1 小时所有超时请求的 Trace
{ name = "HTTP GET" && duration > 5s }

# 2. 查找包含 MySQL 慢查询的 Trace
{ descendant.name =~ "SELECT.*" && descendant.duration > 1s }

# 3. 查找特定用户 ID 的所有 Trace
{ span.user.id = "12345" }

# 4. 错误链路的服务拓扑分析
{ status = error && resource.service.name =~ ".*" } | sum by (resource.service.name)(count())

# 5. P95 延迟按服务名分组
{ resource.service.name =~ ".+" } | quantile_over_time(duration, 0.95) by (resource.service.name)

# 6. 查找跨服务调用的 root span
{ kind = server && parent = nil && duration > 1s }

{
  "datasource": { "type": "tempo", "uid": "tempo" },
  "targets": [{
    "queryType": "traceql",
    "query": "{ resource.service.name = \"checkout\" && status = error && duration > 1s }"
  }],
  "type": "traces"
}

# tempo.yaml
storage:
  trace:
    backend: s3
    s3:
      bucket: tempo-traces-prod
      endpoint: s3.us-east-1.amazonaws.com
      region: us-east-1
      access_key: ${AWS_ACCESS_KEY_ID}
      secret_key: ${AWS_SECRET_ACCESS_KEY}
      # 生产环境关键配置
      forcepathstyle: false         # AWS S3 使用 virtual-hosted-style
      enable_dual_stack: true       # IPv4/IPv6 双栈
    # 块生命周期
    block:
      bloom_filter:
        enabled: true                # 启用 Bloom Filter 加速搜索
        false_positive_rate: 0.05   # 5% 误判率
      v2_in_buffer_bytes: 5242880    # 5MB 块大小
      v2_out_buffer_bytes: 10485760  # 10MB 写缓冲
      v2_prefetched_blocks: 10       # 预取块数
    wal:
      path: /var/tempo/wal           # 写前日志（WAL）路径
    pool:
      max_workers: 100               # 写入并发
      queue_depth: 10000             # 写队列深度

# 压缩器
compactor:
  compaction:
    block_retention: 336h           # 块保留 14 天
    compacted_block_retention: 168h # 压缩后保留 7 天
    max_compaction_objects: 1000000
  # 异步删除过期块
  deletion:
    marker_file_extension: .tombstone

storage:
  trace:
    backend: gcs
    gcs:
      bucket_name: tempo-traces-prod
      service_account: /var/secrets/gcs/key.json
      chunk_buffer_size: 10MiB
      enable_dual_stack: true

storage:
  trace:
    backend: azure
    azure:
      container_name: tempo-traces
      storage_account_name: ${AZURE_STORAGE_ACCOUNT}
      storage_account_key: ${AZURE_STORAGE_KEY}
      endpoint_suffix: core.windows.net
      # 或者使用 managed identity
      # user_assigned_id: /subscriptions/.../userAssignedIdentities/...

# S3 生命周期策略（自动转储到 Glacier）
lifecycle:
  - id: tempo-tiering
    enabled: true
    rules:
      - status: enabled
        transitions:
          - days: 7
            storage_class: STANDARD_IA        # 7 天后转 Infrequent Access
          - days: 30
            storage_class: GLACIER_IR          # 30 天后转 Glacier Instant Retrieval
          - days: 90
            storage_class: DEEP_ARCHIVE        # 90 天后转 Deep Archive
        expiration:
          days: 365
        noncurrent_version_expiration:
          noncurrent_days: 30

# tempo-multi-tenant.yaml
multitenancy:
  enabled: true
  tenant_settings:
    "tenant-a":
      usage_group: high-volume
    "tenant-b":
      usage_group: low-volume

# 每个租户独立 bucket
storage:
  trace:
    backend: s3
    s3:
      bucket: "tempo-${tenant_id}"   # 占位符

# S3 跨区域复制
replication:
  role: arn:aws:iam::123456789012:role/tempo-replication
  rules:
    - id: tempo-dr
      status: enabled
      destination:
        bucket: arn:aws:s3:::tempo-traces-dr
        storage_class: STANDARD_IA
      filter:
        prefix: ""

# IAM 最小权限策略
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "s3:GetObject",
        "s3:PutObject",
        "s3:DeleteObject",
        "s3:ListBucket"
      ],
      "Resource": [
        "arn:aws:s3:::tempo-traces-prod",
        "arn:aws:s3:::tempo-traces-prod/*"
      ]
    }
  ]
}

# 启用服务端加密
encryption:
  sse:
    type: aws:kms
    kms_key_id: arn:aws:kms:us-east-1:123456789012:key/abcd-1234

# Prometheus 告警规则
groups:
  - name: tempo-storage
    rules:
      - alert: TempoIngesterWriteErrors
        expr: rate(tempo_ingester_traces_created_total[5m]) == 0
        for: 10m
        annotations:
          summary: "Tempo Ingester 写入失败"
      - alert: TempoCompactorLag
        expr: time() - tempo_compactor_marker_last_modified_timestamp_seconds > 3600
        for: 30m
        annotations:
          summary: "Compactor 压缩滞后超过 1 小时"
      - alert: TempoBlockSize
        expr: histogram_quantile(0.99, rate(tempo_querier_query_duration_seconds_bucket[5m])) > 5
        for: 15m
        annotations:
          summary: "P99 查询延迟超过 5s"

# tempo.yaml
metrics_generator:
  registry:
    external_labels:
      source: tempo
      cluster: prod
  storage:
    path: /var/tempo/metrics
  # Remote Write 目标
  remote_write:
    - url: http://mimir:9009/api/v1/push
      headers:
        X-Scope-OrgID: tenant-1
      send_exemplars: true
  # 处理器配置
  processor:
    service_graphs:
      enable_grpc_histogram: true
      histogram_buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
    span_filters:
      - 'span.kind = server'  # 仅处理 server Span（避免重复计数）
    # 忽略内部服务
    dimensions:
      - service.name
      - span.http.method
      - span.http.status_code

  # 指标生成频率
  collection_interval: 30s
  # 启用注册中心（用于服务名解析）
  propagation:
    use_is_now: true

# 伪代码：Metrics Generator 解析 Span 关系
def process_trace(trace):
    for span in trace.spans:
        # 提取父子关系
        parent_span = find_parent(span, trace.spans)
        if parent_span and parent_span.service != span.service:
            # 跨服务调用：生成 client → server 边
            edge = Edge(
                client=parent_span.service,
                server=span.service,
                span_name=span.name,
                duration=span.duration,
                status=span.status_code
            )
            edges.append(edge)
    return edges

{
  "type": "nodeGraph",
  "title": "Service Map",
  "datasource": { "type": "prometheus", "uid": "prometheus" },
  "targets": [{
    "expr": "sum by (client, server) (rate(traces_service_graph_request_total[5m]))",
    "legendFormat": "{{client}} → {{server}}"
  }],
  "options": {
    "graph": {
      "enableHierarchicalLayout": true
    }
  }
}

# Rate: 每秒请求数
sum by (server) (rate(traces_service_graph_request_total[5m]))

# Error: 错误率
sum by (server) (rate(traces_service_graph_request_failed_total[5m]))
  / sum by (server) (rate(traces_service_graph_request_total[5m]))

# Duration: P99 延迟
histogram_quantile(0.99,
  sum by (server, le) (
    rate(traces_service_graph_request_duration_seconds_bucket[5m])
  )
)

# Mimir / Prometheus 配置启用 exemplars
remote_write:
  - url: http://mimir:9009/api/v1/push
    send_exemplars: true

# Grafana 面板：点击指标的 exemplar 跳转到 Tempo
# 体验：发现 P99 异常 → 点击 exemplar → 查看具体 Trace → 定位 Span

metrics_generator:
  # 限制单个 Trace 处理的最大 Span 数
  max_spans_per_trace: 1000
  # 限制并发处理的 Trace 块数
  concurrent_processes: 4
  # 跳过特定服务的 Span
  ignore_services:
    - internal-health-check
    - kube-probe

# 维度对齐：确保 Trace 标签与指标标签一致
processors:
  resource:
    attributes:
      - key: service.namespace
        from_attribute: k8s_namespace
        action: insert
      - key: deployment.environment
        from_attribute: env
        action: insert

T+0:  黑色星期五开始，订单服务 Span 流量从 1万/s 飙升至 10万/s
T+5m: Ingester 内存使用从 4GB 升至 14GB（接近 16GB limit）
T+8m: Ingester Pod OOMKilled，Pod 自动重启
T+9m: 重启后 WAL 恢复，但内存再次迅速上升
T+12m: 第二个 Pod OOMKilled
T+15m: 写入路径中断，Distributor 缓冲区溢出，客户端报错
T+20m: 运维手动扩容 Ingester 至 6 副本，临时恢复

# 内存使用趋势
container_memory_working_set_bytes{pod=~"tempo-ingester.*"}

# 关键 Tempo 指标
tempo_ingester_bytes_received_total        # 接收字节数
tempo_ingester_traces_created_total         # 成功创建 Trace 数
tempo_ingester_live_traces                  # 内存中活跃 Trace 数
tempo_ingester_flush_queue_length           # 刷盘队列长度

# 通过 Tempo API 查询单 Trace 大小
curl -H "X-Scope-OrgID: tenant-1" \
  "http://tempo:3100/api/traces/abc123def456" | wc -c

# 检查超大 Trace
tempo_ingester_live_traces > 1000  # 活跃 Trace 数

# 1. 单个 Trace Span 数量爆炸
# 订单服务的 checkout 流程包含 500+ 个 DB Span（每条订单项一次 SQL）
# 10 万订单/分钟 × 500 Span/Trace = 5000 万 Span/分钟

# 2. Ingester 批处理窗口过长
ingester:
  config:
    trace_idle_period: 30s      # 等待 30s 才刷盘
    max_block_duration: 1h      # 块最大 1 小时
    # 内存累积量 = 30s × 10万/s × 平均 2KB/Span = 6GB

# 3. 缺少 Span 限制
ingester:
  config:
    max_traces_per_user: 10000  # 上限过低
    # 实际单租户有 20 万活跃 Trace

# 1. 调整 Ingester 配置
ingester:
  replicas: 6                    # 扩容到 6 副本
  resources:
    requests:
      memory: 16Gi
      cpu: 4
    limits:
      memory: 24Gi               # 提高内存上限
  config:
    trace_idle_period: 10s       # 缩短至 10s
    max_block_duration: 5m       # 缩短至 5 分钟
    max_traces_per_user: 50000   # 调高上限
    # 增加并发刷盘
    flush_check_period: 1s
    concurrent_flushes: 4

# 2. 客户端限制 Trace 体积
# OTel SDK 配置 BatchSpanProcessor
span_limits:
  max_attributes_per_span: 32
  max_events_per_span: 64
  max_links_per_span: 32
  attribute_count_limit: 1000
  max_span_attribute_length: 1024

# 3. 减少冗余 Span
# 在 OTel Collector 中过滤
processors:
  tail_sampling:
    policies:
      - name: drop-noisy-spans
        type: string_attribute
        string_attribute:
          key: span.name
          values: [health.check, db.ping]
          invert_match: false  # 丢弃这些 Span

ingester:
  config:
    wal:
      path: /var/tempo/wal
      # WAL 配置：OOm 后重启可恢复未刷盘数据
      max_block_duration: 1h
      flush_on_shutdown: true

# Prometheus 告警规则
groups:
  - name: tempo-ingester
    rules:
      - alert: TempoIngesterHighMemory
        expr: |
          container_memory_working_set_bytes{pod=~"tempo-ingester.*"}
          / container_spec_memory_limit_bytes{pod=~"tempo-ingester.*"} > 0.8
        for: 5m
        annotations:
          summary: "Ingester 内存使用超过 80%"
      - alert: TempoIngesterFlushLag
        expr: |
          time() - tempo_ingester_last_flushed_timestamp_seconds > 60
        for: 5m
        annotations:
          summary: "Ingester 刷盘滞后超过 1 分钟"
      - alert: TempoTraceDropRate
        expr: |
          rate(tempo_distributor_spans_rejected_total[5m]) > 0
        for: 1m
        annotations:
          summary: "Distributor 正在拒绝 Span"

{
  "type": "traces",
  "title": "Slow Checkout Traces",
  "datasource": { "type": "tempo", "uid": "tempo" },
  "targets": [{
    "queryType": "traceql",
    "query": "{ resource.service.name = \"checkout\" && duration > 2s }",
    "limit": 20
  }],
  "options": {
    "showTraceActions": true,
    "showSpanFilterActions": true
  }
}

# Tempo 数据源启用 Service Map
datasources:
  - name: Tempo
    type: tempo
    uid: tempo
    jsonData:
      serviceMap:
        datasourceUid: prometheus    # Service Graph 指标所在 Prometheus
        # Service Graph 指标名
        metricsQuery:
          rateQuery: 'sum by (client, server)(rate(traces_service_graph_request_total[$__rate_interval]))'
          errorRateQuery: |
            sum by (client, server)(
              rate(traces_service_graph_request_failed_total[$__rate_interval])
            ) / sum by (client, server)(
              rate(traces_service_graph_request_total[$__rate_interval])
            )
          latencyQuery: |
            histogram_quantile(0.95,
              sum by (client, server, le)(
                rate(traces_service_graph_request_duration_seconds_bucket[$__rate_interval])
              )
            )

{
  "type": "nodeGraph",
  "title": "Service Call Graph",
  "datasource": { "type": "tempo", "uid": "tempo" },
  "targets": [{
    "queryType": "traceql",
    "query": "{ resource.service.name = \"frontend\" && duration > 1s }"
  }],
  "options": {
    "graph": {
      "enableHierarchicalLayout": true,
      "showStats": true
    }
  }
}

# 查找所有包含 MySQL 慢查询的 Trace
{ resource.service.name =~ ".+" } >> { name =~ "SELECT.*" && duration > 500ms }

# 查找错误链路（任意 Span 报错）
{ status = error && resource.service.name = "checkout" }

# 查找 HTTP 5xx 错误
{ span.http.status_code >= 500 }

# 查找跨地域调用（region 标签差异）
{ resource.cloud.region = "us-east-1" } >> { resource.cloud.region = "eu-west-1" }

# 查找未捕获的异常
{ status = error && !span.error.type =~ "BusinessError.*" }

# Prometheus 数据源配置
datasources:
  - name: Prometheus
    type: prometheus
    uid: prometheus
    url: http://mimir:9009/prometheus
    jsonData:
      exemplarTraceIdDestinations:
        - name: traceID
          datasourceUid: tempo

# 指标面板启用 exemplar 显示
{
  "type": "timeseries",
  "targets": [{
    "expr": "histogram_quantile(0.99, sum by (le)(rate(http_request_duration_seconds_bucket[5m])))",
    "exemplar": true    # 启用 exemplar
  }],
  "options": {
    "showExemplars": true
  }
}

# Tempo 告警规则
apiVersion: 1
groups:
  - orgId: 1
    name: tempo-alerts
    folder: Observability
    interval: 1m
    rules:
      - uid: tempo-high-error-rate
        title: "High Error Rate from Traces"
        condition: C
        data:
          - refId: A
            datasourceUid: mimir
            model:
              expr: |
                sum(rate(traces_service_graph_request_failed_total{
                  service="checkout"
                }[5m]))
                / sum(rate(traces_service_graph_request_total{
                  service="checkout"
                }[5m]))
              instant: true
          - refId: C
            datasourceUid: __expr__
            model:
              type: threshold
              expression: A
              conditions:
                - evaluator: {type: gt, params: [0.05]}
        for: 5m
        annotations:
          summary: "Checkout 服务错误率超过 5%"

      - uid: tempo-p99-latency
        title: "High P99 Latency"
        condition: C
        data:
          - refId: A
            datasourceUid: mimir
            model:
              expr: |
                histogram_quantile(0.99,
                  sum by (le)(
                    rate(traces_service_graph_request_duration_seconds_bucket{
                      service="checkout"
                    }[5m])
                  )
                )
          - refId: C
            datasourceUid: __expr__
            model:
              type: threshold
              expression: A
              conditions:
                - evaluator: {type: gt, params: [2]}
        for: 10m
        annotations:
          summary: "Checkout P99 延迟超过 2s"

{
  "type": "flamegraph",
  "title": "Trace Duration Distribution",
  "datasource": { "type": "tempo", "uid": "tempo" },
  "targets": [{
    "queryType": "traceql",
    "query": "{ resource.service.name = \"checkout\" && duration > 1s }"
  }],
  "options": {
    "flamegraph": {
      "showAllFrames": true,
      "sortBy": "spanName"
    }
  }
}

# dashboard.json
{
  "title": "LGTM Observability Dashboard",
  "panels": [
    {
      "id": 1,
      "title": "Service Map",
      "type": "nodeGraph",
      "gridPos": {x: 0, y: 0, w: 24, h: 10}
    },
    {
      "id": 2,
      "title": "RED Metrics (Rate)",
      "type": "timeseries",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": {x: 0, y: 10, w: 12, h: 8}
    },
    {
      "id": 3,
      "title": "Error Logs",
      "type": "logs",
      "datasource": { "type": "loki", "uid": "loki" },
      "gridPos": {x: 12, y: 10, w: 12, h: 8}
    },
    {
      "id": 4,
      "title": "Recent Traces",
      "type": "traces",
      "datasource": { "type": "tempo", "uid": "tempo" },
      "gridPos": {x: 0, y: 18, w: 24, h: 10}
    }
  ]
}