Observability в Python/FastAPI: три столпа наблюдаемости для production

@app.post("/checkout")
async def checkout(order: OrderCreate):
    user = await get_user(order.user_id)           # 50ms... или 5s?
    inventory = await check_inventory(order.items)  # Работает... или нет?
    payment = await process_payment(order)           # Успех... но какой ценой?
    await send_confirmation(user.email)              # Отправилось... кому?
    return {"order_id": payment.order_id}

pip install structlog

# app/logging_config.py
import structlog
import logging
import sys
 
def setup_logging(json_logs: bool = True, log_level: str = "INFO"):
    """Настройка structlog для production и development."""
 
    # Общие процессоры для всех режимов
    shared_processors: list[structlog.types.Processor] = [
        structlog.contextvars.merge_contextvars,    # Контекст из contextvars
        structlog.stdlib.add_logger_name,            # Имя логгера
        structlog.stdlib.add_log_level,              # Уровень лога
        structlog.processors.TimeStamper(fmt="iso"), # ISO timestamp
        structlog.processors.StackInfoRenderer(),    # Stack trace
        structlog.processors.UnicodeDecoder(),       # Unicode
    ]
 
    if json_logs:
        # Production: JSON для Loki
        renderer = structlog.processors.JSONRenderer()
    else:
        # Development: читаемый вывод в консоль
        renderer = structlog.dev.ConsoleRenderer(colors=True)
 
    structlog.configure(
        processors=[
            *shared_processors,
            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
        ],
        logger_factory=structlog.stdlib.LoggerFactory(),
        wrapper_class=structlog.stdlib.BoundLogger,
        cache_logger_on_first_use=True,
    )
 
    formatter = structlog.stdlib.ProcessorFormatter(
        processors=[
            structlog.stdlib.ProcessorFormatter.remove_processors_meta,
            renderer,
        ],
    )
 
    handler = logging.StreamHandler(sys.stdout)
    handler.setFormatter(formatter)
 
    root_logger = logging.getLogger()
    root_logger.handlers.clear()
    root_logger.addHandler(handler)
    root_logger.setLevel(log_level)

# app/main.py
import structlog
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request
from app.logging_config import setup_logging
 
@asynccontextmanager
async def lifespan(app: FastAPI):
    setup_logging(json_logs=True)
    yield
 
app = FastAPI(lifespan=lifespan)
logger = structlog.stdlib.get_logger()
 
@app.middleware("http")
async def logging_middleware(request: Request, call_next):
    """Добавляет request_id и контекст в каждый лог."""
    import uuid, time
 
    request_id = request.headers.get("X-Request-ID", str(uuid.uuid4()))
    start_time = time.perf_counter()
 
    # Привязываем контекст — все логи внутри запроса получат эти поля
    structlog.contextvars.clear_contextvars()
    structlog.contextvars.bind_contextvars(
        request_id=request_id,
        method=request.method,
        path=request.url.path,
        client_ip=request.client.host if request.client else None,
    )
 
    try:
        response = await call_next(request)
        duration_ms = (time.perf_counter() - start_time) * 1000
 
        await logger.ainfo(
            "request_completed",
            status_code=response.status_code,
            duration_ms=round(duration_ms, 2),
        )
        response.headers["X-Request-ID"] = request_id
        return response
 
    except Exception as exc:
        duration_ms = (time.perf_counter() - start_time) * 1000
        await logger.aerror(
            "request_failed",
            error=str(exc),
            error_type=type(exc).__name__,
            duration_ms=round(duration_ms, 2),
        )
        raise

{
  "event": "request_completed",
  "request_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
  "method": "POST",
  "path": "/checkout",
  "client_ip": "10.0.1.15",
  "status_code": 200,
  "duration_ms": 142.35,
  "logger": "app.main",
  "level": "info",
  "timestamp": "2026-02-15T10:30:00.000000Z"
}

# docker-compose.observability.yml
services:
  loki:
    image: grafana/loki:3.4
    ports:
      - "3100:3100"
    volumes:
      - loki-data:/loki
    command: -config.file=/etc/loki/local-config.yaml
 
  promtail:
    image: grafana/promtail:3.4
    volumes:
      - /var/log:/var/log
      - ./promtail-config.yml:/etc/promtail/config.yml
      - /var/run/docker.sock:/var/run/docker.sock
    command: -config.file=/etc/promtail/config.yml
    depends_on:
      - loki
 
volumes:
  loki-data:

# promtail-config.yml
server:
  http_listen_port: 9080
 
positions:
  filename: /tmp/positions.yaml
 
clients:
  - url: http://loki:3100/loki/api/v1/push
 
scrape_configs:
  - job_name: docker
    docker_sd_configs:
      - host: unix:///var/run/docker.sock
        refresh_interval: 5s
    relabel_configs:
      # Метки из Docker labels
      - source_labels: ["__meta_docker_container_name"]
        target_label: "container"
      - source_labels:
          ["__meta_docker_container_label_com_docker_compose_service"]
        target_label: "service"
    pipeline_stages:
      # Парсим JSON-логи
      - json:
          expressions:
            level: level
            request_id: request_id
            event: event
      - labels:
          level:
          event:

# Все ошибки за последний час
{service="fastapi-app"} |= `"level":"error"` | json
 
# Медленные запросы (> 1 секунда)
{service="fastapi-app"} | json | duration_ms > 1000
 
# Ошибки конкретного пользователя
{service="fastapi-app"} | json | user_id = "12345" and level = "error"
 
# Количество ошибок по типу за 5 минут
sum by (error_type) (
  count_over_time(
    {service="fastapi-app"} | json | level = "error" [5m]
  )
)

# pip install python-logging-loki
import logging
import logging_loki
 
loki_handler = logging_loki.LokiHandler(
    url="http://loki:3100/loki/api/v1/push",
    tags={"service": "fastapi-app", "environment": "production"},
    version="1",
)
 
logger = logging.getLogger("app")
logger.addHandler(loki_handler)

pip install prometheus-client prometheus-fastapi-instrumentator

# app/main.py
from fastapi import FastAPI
from prometheus_fastapi_instrumentator import Instrumentator
 
app = FastAPI()
 
# Автоматически добавляет метрики:
# - http_requests_total (counter)
# - http_request_duration_seconds (histogram)
# - http_request_size_bytes (summary)
# - http_response_size_bytes (summary)
# - http_requests_in_progress (gauge)
Instrumentator().instrument(app).expose(app, endpoint="/metrics")

# app/metrics.py
from prometheus_client import Counter, Histogram, Gauge
 
# Бизнес-метрики
ORDERS_TOTAL = Counter(
    "orders_total",
    "Total number of orders",
    ["status", "payment_method"],
)
 
ORDER_AMOUNT = Histogram(
    "order_amount_rub",
    "Order amount in rubles",
    buckets=[100, 500, 1000, 5000, 10000, 50000, 100000],
)
 
ACTIVE_USERS = Gauge(
    "active_users",
    "Number of currently active users",
)
 
# Метрики внешних зависимостей
DB_QUERY_DURATION = Histogram(
    "db_query_duration_seconds",
    "Database query duration",
    ["query_type", "table"],
    buckets=[0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0],
)
 
EXTERNAL_API_REQUESTS = Counter(
    "external_api_requests_total",
    "External API requests",
    ["service", "method", "status"],
)

# app/routes/orders.py
import time
from app.metrics import ORDERS_TOTAL, ORDER_AMOUNT, DB_QUERY_DURATION
 
@app.post("/orders")
async def create_order(order: OrderCreate):
    # Замеряем время запроса к БД
    start = time.perf_counter()
    db_order = await db.orders.insert(order.dict())
    DB_QUERY_DURATION.labels(
        query_type="insert", table="orders"
    ).observe(time.perf_counter() - start)
 
    # Инкрементируем бизнес-метрику
    ORDERS_TOTAL.labels(
        status="created",
        payment_method=order.payment_method,
    ).inc()
 
    ORDER_AMOUNT.observe(order.amount)
 
    return db_order

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s
 
scrape_configs:
  - job_name: "fastapi-app"
    static_configs:
      - targets: ["fastapi-app:8000"]
    metrics_path: "/metrics"
    scrape_interval: 10s
 
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]

services:
  prometheus:
    image: prom/prometheus:v3.2
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus-data:/prometheus
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--storage.tsdb.retention.time=30d"
 
volumes:
  prometheus-data:

# Текущий RPS (requests per second)
rate(http_requests_total{service="fastapi-app"}[5m])
 
# p95 латентность за последние 5 минут
histogram_quantile(0.95,
  rate(http_request_duration_seconds_bucket{service="fastapi-app"}[5m])
)
 
# Error rate (процент ошибок)
sum(rate(http_requests_total{status=~"5.."}[5m]))
/
sum(rate(http_requests_total[5m]))
* 100
 
# Скорость роста заказов в час
increase(orders_total{status="created"}[1h])
 
# Средний размер заказа за день
rate(order_amount_rub_sum[1d]) / rate(order_amount_rub_count[1d])

# alert_rules.yml — лежит рядом с prometheus.yml
groups:
  - name: fastapi-alerts
    rules:
      # Высокий error rate
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m]))
          /
          sum(rate(http_requests_total[5m]))
          > 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Error rate выше 5%"
          description: "{{ $value | humanizePercentage }} ошибок за последние 5 минут"
 
      # Высокая латентность
      - alert: HighLatency
        expr: |
          histogram_quantile(0.95,
            rate(http_request_duration_seconds_bucket[5m])
          ) > 1.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "p95 латентность выше 1 секунды"
 
      # База данных медленная
      - alert: SlowDatabaseQueries
        expr: |
          histogram_quantile(0.95,
            rate(db_query_duration_seconds_bucket[5m])
          ) > 0.5
        for: 3m
        labels:
          severity: warning
        annotations:
          summary: "p95 запросов к БД выше 500ms"

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s
 
# Подключаем файл с правилами алертов
rule_files:
  - "alert_rules.yml"
 
# Указываем, куда отправлять сработавшие алерты
alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]
 
scrape_configs:
  - job_name: "fastapi-app"
    static_configs:
      - targets: ["fastapi-app:8000"]
    metrics_path: "/metrics"

# alertmanager.yml — конфигурация маршрутизации уведомлений
global:
  resolve_timeout: 5m
 
route:
  receiver: "default"
  group_by: ["alertname", "severity"]
  group_wait: 30s # Ждём 30s перед первой отправкой (группировка)
  group_interval: 5m # Интервал между повторными уведомлениями группы
  repeat_interval: 4h # Повтор, если алерт всё ещё активен
 
  routes:
    # Critical → Slack + PagerDuty
    - match:
        severity: critical
      receiver: "slack-critical"
      repeat_interval: 30m
 
    # Warning → только Slack
    - match:
        severity: warning
      receiver: "slack-warnings"
 
receivers:
  - name: "default"
    slack_configs:
      - api_url: "https://hooks.slack.com/services/T.../B.../xxx"
        channel: "#alerts"
 
  - name: "slack-critical"
    slack_configs:
      - api_url: "https://hooks.slack.com/services/T.../B.../xxx"
        channel: "#alerts-critical"
        title: "{{ .CommonAnnotations.summary }}"
        text: "{{ .CommonAnnotations.description }}"
 
  - name: "slack-warnings"
    slack_configs:
      - api_url: "https://hooks.slack.com/services/T.../B.../xxx"
        channel: "#alerts-warnings"

services:
  alertmanager:
    image: prom/alertmanager:v0.28
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml
 
  prometheus:
    image: prom/prometheus:v3.2
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./alert_rules.yml:/etc/prometheus/alert_rules.yml
    depends_on:
      - alertmanager

pip install opentelemetry-api \
    opentelemetry-sdk \
    opentelemetry-exporter-otlp \
    opentelemetry-instrumentation-fastapi \
    opentelemetry-instrumentation-httpx \
    opentelemetry-instrumentation-sqlalchemy \
    opentelemetry-instrumentation-redis

# app/telemetry.py
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource, SERVICE_NAME, SERVICE_VERSION
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
from opentelemetry.instrumentation.redis import RedisInstrumentor
 
def setup_telemetry(app, service_name: str = "fastapi-app"):
    """Инициализация OpenTelemetry с экспортом в OTLP (Jaeger/Tempo)."""
 
    resource = Resource.create({
        SERVICE_NAME: service_name,
        SERVICE_VERSION: "1.0.0",
        "deployment.environment": "production",
    })
 
    # TracerProvider — центральный объект для создания трейсов
    provider = TracerProvider(resource=resource)
 
    # Экспорт через OTLP (gRPC) — универсальный протокол
    otlp_exporter = OTLPSpanExporter(
        endpoint="http://jaeger:4317",  # или Tempo, или OTel Collector
        insecure=True,
    )
 
    # BatchSpanProcessor — буферизация и отправка пачками
    provider.add_span_processor(
        BatchSpanProcessor(
            otlp_exporter,
            max_queue_size=2048,
            max_export_batch_size=512,
            schedule_delay_millis=5000,
        )
    )
 
    trace.set_tracer_provider(provider)
 
    # Автоматическая инструментация
    FastAPIInstrumentor.instrument_app(app)
    HTTPXClientInstrumentor().instrument()  # httpx — async HTTP клиент
    SQLAlchemyInstrumentor().instrument()   # SQLAlchemy queries
    RedisInstrumentor().instrument()         # Redis commands

# app/main.py
from contextlib import asynccontextmanager
from fastapi import FastAPI
from app.telemetry import setup_telemetry
 
@asynccontextmanager
async def lifespan(app: FastAPI):
    setup_telemetry(app)
    yield
 
app = FastAPI(lifespan=lifespan)

# app/services/payment.py
from opentelemetry import trace
 
tracer = trace.get_tracer(__name__)
 
async def process_payment(order_id: int, amount: float, method: str):
    """Обработка платежа с детальным tracing."""
 
    with tracer.start_as_current_span(
        "process_payment",
        attributes={
            "payment.order_id": order_id,
            "payment.amount": amount,
            "payment.method": method,
            "payment.currency": "RUB",
        },
    ) as span:
        # Шаг 1: валидация
        with tracer.start_as_current_span("validate_payment"):
            validate_payment_data(order_id, amount, method)
 
        # Шаг 2: вызов платёжного шлюза
        with tracer.start_as_current_span(
            "call_payment_gateway",
            attributes={"gateway.name": "stripe"},
        ) as gateway_span:
            try:
                result = await payment_gateway.charge(
                    amount=amount,
                    method=method,
                )
                gateway_span.set_attribute("gateway.transaction_id", result.tx_id)
            except PaymentError as e:
                gateway_span.set_status(
                    trace.StatusCode.ERROR,
                    description=str(e),
                )
                gateway_span.record_exception(e)
                raise
 
        # Шаг 3: сохранение в БД
        with tracer.start_as_current_span("save_payment_record"):
            await db.payments.insert({
                "order_id": order_id,
                "amount": amount,
                "tx_id": result.tx_id,
                "status": "completed",
            })
 
        span.set_attribute("payment.status", "completed")
        span.set_attribute("payment.transaction_id", result.tx_id)
 
        return result

import httpx
 
async def check_inventory(items: list[dict]):
    # trace_id автоматически добавляется в заголовки
    async with httpx.AsyncClient() as client:
        response = await client.post(
            "http://inventory-service/check",
            json={"items": items},
        )
        return response.json()

# docker-compose.observability.yml (дополнение)
services:
  jaeger:
    image: jaegertracing/jaeger:2.4
    ports:
      - "16686:16686" # UI
      - "4317:4317" # OTLP gRPC
      - "4318:4318" # OTLP HTTP
    environment:
      - COLLECTOR_OTLP_ENABLED=true

services:
  tempo:
    image: grafana/tempo:2.7
    ports:
      - "4317:4317" # OTLP gRPC
      - "4318:4318" # OTLP HTTP
    volumes:
      - ./tempo-config.yml:/etc/tempo/config.yml
      - tempo-data:/var/tempo
    command: ["-config.file=/etc/tempo/config.yml"]

# tempo-config.yml
server:
  http_listen_port: 3200
 
distributor:
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:4317
        http:
          endpoint: 0.0.0.0:4318
 
storage:
  trace:
    backend: local
    local:
      path: /var/tempo/blocks

# app/logging_config.py — добавляем trace context в structlog
from opentelemetry import trace
 
def add_trace_context(logger, method_name, event_dict):
    """Процессор structlog: добавляет trace_id и span_id в каждый лог."""
    span = trace.get_current_span()
    if span and span.is_recording():
        ctx = span.get_span_context()
        event_dict["trace_id"] = format(ctx.trace_id, "032x")
        event_dict["span_id"] = format(ctx.span_id, "016x")
    return event_dict
 
# Добавить в shared_processors:
shared_processors = [
    structlog.contextvars.merge_contextvars,
    add_trace_context,          # <-- trace context в каждом логе
    structlog.stdlib.add_log_level,
    structlog.processors.TimeStamper(fmt="iso"),
    # ...
]

{
  "event": "payment_processed",
  "trace_id": "0af7651916cd43dd8448eb211c80319c",
  "span_id": "b7ad6b7169203331",
  "order_id": 12345,
  "amount": 5000,
  "level": "info",
  "timestamp": "2026-02-15T10:30:00.123456Z"
}

from prometheus_client import Histogram
from opentelemetry import trace
 
REQUEST_DURATION = Histogram(
    "http_request_duration_seconds",
    "Request duration",
    ["method", "path"],
)
 
@app.middleware("http")
async def metrics_middleware(request: Request, call_next):
    start = time.perf_counter()
    response = await call_next(request)
    duration = time.perf_counter() - start
 
    # Добавляем exemplar с trace_id
    span = trace.get_current_span()
    exemplar = {}
    if span and span.is_recording():
        ctx = span.get_span_context()
        exemplar = {"trace_id": format(ctx.trace_id, "032x")}
 
    REQUEST_DURATION.labels(
        method=request.method,
        path=request.url.path,
    ).observe(duration, exemplar=exemplar)
 
    return response

# docker-compose.observability.yml (Grafana)
services:
  grafana:
    image: grafana/grafana:11.5
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_FEATURE_TOGGLES_ENABLE=traceToLogs,traceToMetrics
    volumes:
      - grafana-data:/var/grafana
      - ./grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml
 
volumes:
  grafana-data:

# grafana-datasources.yml
apiVersion: 1
 
datasources:
  - name: Prometheus
    type: prometheus
    url: http://prometheus:9090
    isDefault: true
 
  - name: Loki
    type: loki
    url: http://loki:3100
    jsonData:
      derivedFields:
        - name: trace_id
          matcherRegex: '"trace_id":"(\w+)"'
          url: "$${__value.raw}"
          datasourceUid: tempo
          urlDisplayLabel: "View Trace"
 
  - name: Tempo
    type: tempo
    uid: tempo
    url: http://tempo:3200
    jsonData:
      tracesToLogs:
        datasourceUid: loki
        filterByTraceID: true
        filterBySpanID: true
      tracesToMetrics:
        datasourceUid: prometheus

services:
  # === Приложение ===
  fastapi-app:
    build: .
    ports:
      - "8000:8000"
    environment:
      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
      - OTEL_SERVICE_NAME=fastapi-app
    depends_on:
      - otel-collector
 
  # === OpenTelemetry Collector ===
  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.120.0
    ports:
      - "4317:4317" # OTLP gRPC
      - "4318:4318" # OTLP HTTP
      - "8888:8888" # Collector metrics
    volumes:
      - ./otel-collector-config.yml:/etc/otel/config.yml
    command: ["--config=/etc/otel/config.yml"]
 
  # === Трейсы ===
  jaeger:
    image: jaegertracing/jaeger:2.4
    ports:
      - "16686:16686"
 
  # === Метрики ===
  prometheus:
    image: prom/prometheus:v3.2
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
 
  # === Логи ===
  loki:
    image: grafana/loki:3.4
    ports:
      - "3100:3100"
 
  promtail:
    image: grafana/promtail:3.4
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./promtail-config.yml:/etc/promtail/config.yml
 
  # === Визуализация ===
  grafana:
    image: grafana/grafana:11.5
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin

# otel-collector-config.yml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318
 
processors:
  batch:
    timeout: 5s
    send_batch_size: 1024
 
  # Tail-based sampling: сохраняем 100% ошибок, 10% успешных
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: errors
        type: status_code
        status_code: { status_codes: [ERROR] }
      - name: slow-requests
        type: latency
        latency: { threshold_ms: 1000 }
      - name: probabilistic
        type: probabilistic
        probabilistic: { sampling_percentage: 10 }
 
exporters:
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true
 
  prometheus:
    endpoint: 0.0.0.0:8889
 
  loki:
    endpoint: http://loki:3100/loki/api/v1/push
 
service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [tail_sampling, batch]
      exporters: [otlp/jaeger]
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [prometheus]

from opentelemetry.sdk.trace.sampling import TraceIdRatioBased, ParentBased
 
# Сохраняем 10% трейсов (случайная выборка)
sampler = ParentBased(
    root=TraceIdRatioBased(0.1),
)
 
provider = TracerProvider(
    resource=resource,
    sampler=sampler,
)

# В otel-collector-config.yml (см. выше)
processors:
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: errors-policy
        type: status_code
        status_code: { status_codes: [ERROR] }
      - name: high-latency
        type: latency
        latency: { threshold_ms: 2000 }
      - name: sample-rest
        type: probabilistic
        probabilistic: { sampling_percentage: 5 }

# Span naming: <verb>_<object> или <service>.<method>
# ✅ Хорошо
"process_payment"
"inventory.check_stock"
"db.query.select_user"
 
# ❌ Плохо
"handler"              # Слишком общее
"POST /api/v1/orders"  # URL как имя — высокая кардинальность
"doStuff"              # Непонятно

# HTTP
span.set_attribute("http.method", "POST")
span.set_attribute("http.url", "https://api.example.com/orders")
span.set_attribute("http.status_code", 200)
 
# Database
span.set_attribute("db.system", "postgresql")
span.set_attribute("db.statement", "SELECT * FROM users WHERE id = $1")
span.set_attribute("db.operation", "SELECT")
 
# Бизнес-контекст
span.set_attribute("order.id", 12345)
span.set_attribute("user.id", "u-abc-123")
span.set_attribute("payment.amount", 5000.0)

# DEBUG — диагностика в development (не включать в production)
logger.debug("cache_lookup", key="user:123", hit=True)
 
# INFO — значимые бизнес-события
logger.info("order_created", order_id=123, amount=5000)
 
# WARNING — аномалии, которые не являются ошибкой
logger.warning("slow_query", duration_ms=1500, query="SELECT ...")
 
# ERROR — ошибки, требующие внимания
logger.error("payment_failed", order_id=123, error="timeout")
 
# CRITICAL — сервис не может работать
logger.critical("database_unavailable", host="db-primary")

from prometheus_client import Counter, Histogram, Gauge
 
# 1. Latency — время обработки запросов
REQUEST_DURATION = Histogram(
    "http_request_duration_seconds",
    "Request latency",
    ["method", "endpoint", "status"],
)
 
# 2. Traffic — объём запросов
REQUEST_COUNT = Counter(
    "http_requests_total",
    "Total requests",
    ["method", "endpoint", "status"],
)
 
# 3. Errors — количество и процент ошибок
ERROR_COUNT = Counter(
    "http_errors_total",
    "Total errors",
    ["method", "endpoint", "error_type"],
)
 
# 4. Saturation — насколько загружен сервис
IN_PROGRESS = Gauge(
    "http_requests_in_progress",
    "Currently processing requests",
)
 
DB_POOL_USAGE = Gauge(
    "db_connection_pool_usage_ratio",
    "Database connection pool usage (0-1)",
)

# ❌ Никогда
logger.info("user_login", password=user.password, token=jwt_token)
logger.info("payment", card_number=card.number)
 
# ✅ Правильно
logger.info("user_login", user_id=user.id)
logger.info("payment", card_last4=card.number[-4:], amount=amount)