Chaos Engineering под нагрузкой

# chaos/network-latency.yaml
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: api-network-delay
  namespace: default
spec:
  action: delay # Добавляем задержку
  mode: all # Все pods с label
  selector:
    labelSelectors:
      app: api-service
  delay:
    latency: "200ms" # Задержка 200ms
    correlation: "50" # 50% корреляция (более реалистично)
    jitter: "50ms" # Jitter ±50ms
  duration: "5m" # Длительность chaos
  scheduler:
    cron: "@every 10m" # Повторять каждые 10 минут

// chaos-network-latency.js
import http from "k6/http";
import { check, sleep } from "k6";
import { Trend, Rate } from "k6/metrics";
 
const latencyBeforeChaos = new Trend("latency_before_chaos");
const latencyDuringChaos = new Trend("latency_during_chaos");
const latencyAfterChaos = new Trend("latency_after_chaos");
const errorsDuringChaos = new Rate("errors_during_chaos");
 
export const options = {
  scenarios: {
    chaos_test: {
      executor: "constant-arrival-rate",
      rate: 100, // 100 RPS
      duration: "15m",
      preAllocatedVUs: 50,
      maxVUs: 200,
    },
  },
  thresholds: {
    latency_before_chaos: ["p(95)<500"], // Без chaos
    latency_during_chaos: ["p(95)<700"], // С chaos допускаем +200ms
    latency_after_chaos: ["p(95)<500"], // Recovery
    errors_during_chaos: ["rate<0.05"], // < 5% ошибок во время chaos
    http_req_failed: ["rate<0.03"], // < 3% общая ошибка
  },
};
 
const BASE_URL = __ENV.BASE_URL || "http://api-service:8080";
const CHAOS_START_TIME = 5 * 60 * 1000; // Chaos начинается через 5 минут
const CHAOS_DURATION = 5 * 60 * 1000; // Chaos длится 5 минут
 
export default function () {
  const now = Date.now() - __ENV.TEST_START_TIME;
  const chaosActive =
    now > CHAOS_START_TIME && now < CHAOS_START_TIME + CHAOS_DURATION;
 
  const res = http.get(`${BASE_URL}/api/products`);
 
  // Записываем метрики в зависимости от фазы
  if (now < CHAOS_START_TIME) {
    latencyBeforeChaos.add(res.timings.duration);
  } else if (chaosActive) {
    latencyDuringChaos.add(res.timings.duration);
    errorsDuringChaos.add(res.status >= 400 || res.status === 0);
  } else {
    latencyAfterChaos.add(res.timings.duration);
  }
 
  check(res, {
    "status 200": (r) => r.status === 200,
  });
 
  sleep(0.1);
}

# 1. Применить Chaos Mesh манифест (запланировано на 5 минут после старта)
kubectl apply -f chaos/network-latency.yaml
 
# 2. Запустить k6 тест
TEST_START_TIME=$(date +%s%3N) k6 run chaos-network-latency.js
 
# 3. Наблюдать метрики в Grafana:
# - latency_before_chaos: p95 ~300ms
# - latency_during_chaos: p95 ~500ms (+200ms от chaos)
# - latency_after_chaos: p95 ~300ms (recovery)

latency_before_chaos....: avg=285ms p(95)=320ms p(99)=380ms
latency_during_chaos....: avg=485ms p(95)=520ms p(99)=600ms  ← +200ms chaos
latency_after_chaos.....: avg=290ms p(95)=325ms p(99)=390ms  ← recovery
errors_during_chaos.....: 1.2% (некоторые запросы timeout)

# chaos/pod-kill.yaml
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: api-pod-kill
  namespace: default
spec:
  action: pod-kill # Убить pod
  mode: one # Убить один случайный pod
  selector:
    labelSelectors:
      app: api-service
  scheduler:
    cron: "@every 2m" # Убивать pod каждые 2 минуты

// chaos-pod-kill.js
import http from "k6/http";
import { check, sleep } from "k6";
import { Rate, Trend } from "k6/metrics";
 
const podRestartDetected = new Rate("pod_restart_detected");
const recoveryTime = new Trend("recovery_time");
const errorsRate = new Rate("errors_rate");
 
let lastErrorTime = null;
let lastSuccessTime = null;
 
export const options = {
  scenarios: {
    pod_chaos: {
      executor: "constant-arrival-rate",
      rate: 50,
      duration: "10m",
      preAllocatedVUs: 25,
      maxVUs: 100,
    },
  },
  thresholds: {
    pod_restart_detected: ["rate>0"], // Должен быть хотя бы 1 restart
    recovery_time: ["p(95)<5000"], // Recovery < 5s
    errors_rate: ["rate<0.10"], // < 10% ошибок (допускаем spike)
  },
};
 
const BASE_URL = __ENV.BASE_URL || "http://api-service:8080";
 
export default function () {
  const res = http.get(`${BASE_URL}/api/health`);
 
  const success = res.status === 200;
  errorsRate.add(!success);
 
  if (!success) {
    if (lastSuccessTime !== null) {
      podRestartDetected.add(1);
      lastErrorTime = Date.now();
    }
  } else {
    if (lastErrorTime !== null) {
      const recovery = Date.now() - lastErrorTime;
      recoveryTime.add(recovery);
      console.log(`Pod recovered in ${recovery}ms`);
      lastErrorTime = null;
    }
    lastSuccessTime = Date.now();
  }
 
  sleep(1);
}

pod_restart_detected....: 5 restarts detected
recovery_time...........: avg=2.1s min=1.2s p(95)=4.5s max=6.8s
errors_rate.............: 8.5% (spike во время pod restart)

# chaos/packet-loss.yaml
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: api-packet-loss
  namespace: default
spec:
  action: loss # Packet loss
  mode: all
  selector:
    labelSelectors:
      app: api-service
  loss:
    loss: "10" # 10% packet loss
    correlation: "25" # 25% корреляция
  duration: "3m"

// chaos-packet-loss.js
import http from "k6/http";
import { check, sleep } from "k6";
import { Rate, Counter } from "k6/metrics";
 
const timeouts = new Counter("timeouts");
const retries = new Counter("retries");
const successAfterRetry = new Rate("success_after_retry");
 
export const options = {
  scenarios: {
    packet_loss_test: {
      executor: "constant-arrival-rate",
      rate: 50,
      duration: "8m",
      preAllocatedVUs: 25,
      maxVUs: 100,
    },
  },
  thresholds: {
    timeouts: ["count<50"], // < 50 timeouts за тест
    retries: ["count<100"], // < 100 retries
    success_after_retry: ["rate>0.80"], // 80% retries успешны
  },
};
 
const BASE_URL = __ENV.BASE_URL || "http://api-service:8080";
 
export default function () {
  let res = http.get(`${BASE_URL}/api/products`, {
    timeout: "5s", // Timeout 5s
  });
 
  if (res.status === 0) {
    // Timeout или network error
    timeouts.add(1);
 
    // Retry once
    retries.add(1);
    sleep(0.5);
 
    res = http.get(`${BASE_URL}/api/products`, {
      timeout: "5s",
    });
 
    successAfterRetry.add(res.status === 200);
  }
 
  check(res, {
    "status 200": (r) => r.status === 200,
  });
 
  sleep(1);
}

# Собираем k6 с расширением
xk6 build --with github.com/grafana/xk6-disruptor

import { ServiceDisruptor } from "k6/x/disruptor";
 
export const options = {
  scenarios: {
    disrupt: {
      executor: "shared-iterations",
      vus: 1,
      iterations: 1,
      exec: "disrupt",
    },
    load: {
      executor: "constant-arrival-rate",
      rate: 50,
      duration: "5m",
      preAllocatedVUs: 25,
      maxVUs: 100,
      exec: "loadTest",
      startTime: "10s", // Начать после inject
    },
  },
};
 
// Inject chaos
export function disrupt() {
  const disruptor = new ServiceDisruptor("api-service", "default");
 
  const fault = {
    averageDelay: "200ms",
    errorRate: 0.1, // 10% errors
    statusCode: 500,
  };
 
  disruptor.injectHTTPFaults(fault, "5m");
}
 
// Load test
export function loadTest() {
  const res = http.get(`${BASE_URL}/api/products`);
 
  check(res, {
    "status 200": (r) => r.status === 200,
  });
 
  sleep(1);
}

# chaos/graduated-network-delay.yaml
apiVersion: chaos-mesh.org/v1alpha1
kind: Schedule
metadata:
  name: graduated-network-chaos
spec:
  schedule: "@every 15m"
  type: NetworkChaos
  networkChaos:
    - name: mild-delay
      delay:
        latency: "50ms"
      duration: "5m"
    - name: moderate-delay
      delay:
        latency: "200ms"
      duration: "5m"
    - name: severe-delay
      delay:
        latency: "500ms"
      duration: "5m"

import { Trend, Rate } from "k6/metrics";
 
const latency = new Trend("latency");
const errors = new Rate("errors");
const circuitBreakerOpen = new Rate("circuit_breaker_open");
 
export default function () {
  const res = http.get(`${BASE_URL}/api/products`);
 
  latency.add(res.timings.duration);
  errors.add(res.status >= 400);
 
  // Детектим circuit breaker
  if (res.status === 503) {
    circuitBreakerOpen.add(1);
    console.log("Circuit breaker detected!");
  } else {
    circuitBreakerOpen.add(0);
  }
}

# Если chaos пошел не так — немедленно удалить
kubectl delete -f chaos/network-latency.yaml
 
# Или через Chaos Mesh UI: Pause/Delete experiment