DevOps自动化运维平台架构设计实战

平台架构全景

┌─────────────────────────────────────────────────────────────────┐
│                    统一门户层 (Portal)                           │
│   Web控制台 │ 移动端App │ CLI工具 │ API Gateway                 │
├─────────────────────────────────────────────────────────────────┤
│                    平台服务层 (Service)                          │
│   项目管理 │ 流水线编排 │ 环境管理 │ 制品管理 │ 审批中心          │
├─────────────────────────────────────────────────────────────────┤
│                    自动化引擎层 (Engine)                         │
│   CI引擎 │ CD引擎 │ IaC引擎 │ 配置中心 │ 安全扫描引擎            │
├─────────────────────────────────────────────────────────────────┤
│                    数据与可观测层 (Observability)                 │
│   Prometheus+Grafana │ ELK Stack │ Jaeger链路追踪 │ 审计日志     │
├─────────────────────────────────────────────────────────────────┤
│                    基础设施层 (Infrastructure)                    │
│   物理机/VM │ Kubernetes集群 │ 网络/存储 │ 云资源(Terraform)     │
└─────────────────────────────────────────────────────────────────┘
      

# Terraform目录结构设计
infra/
├── modules/                  # 可复用模块
│   ├── vpc/                  # 网络模块
│   ├── k8s-cluster/          # K8s集群模块
│   ├── rds/                  # 数据库模块
│   ├── redis/                # Redis模块
│   ├── elb/                  # 负载均衡模块
│   └── security-group/       # 安全组模块
├── environments/             # 环境配置
│   ├── dev/
│   │   ├── main.tf
│   │   ├── variables.tf
│   │   └── terraform.tfvars
│   ├── staging/
│   ├── production/
│   └── dr/                   # 灾备环境
├── backend/                  # 状态管理
│   └── s3-backend.tf
└── global/                   # 全局资源（IAM、DNS等）
    └── dns.tf
      

可观测性三支柱

• Metrics（指标）：Prometheus采集 + Thanos长期存储，Grafana统一可视化。部署Prometheus联邦架构，中心集群聚合所有子集群指标。
• Logs（日志）：Filebeat采集 → Kafka缓冲 → Logstash处理 → Elasticsearch存储 → Kibana展示。日志保留策略：热数据7天、温数据30天、冷数据1年。
• Traces（链路追踪）：Jaeger + OpenTelemetry SDK，覆盖Java/Go/Python微服务调用链。追踪数据采样率：生产环境1%，故障时自动提升至100%。

# 配置层次设计
config/
├── base/                     # 基线配置（所有环境共享）
│   ├── deployment.yaml       # K8s Deployment模板
│   ├── service.yaml          # Service定义
│   ├── hpa.yaml              # 弹性伸缩策略
│   └── networkpolicy.yaml    # 网络策略
├── overlays/                 # 环境差异化配置
│   ├── dev/
│   │   ├── kustomization.yaml
│   │   ├── replicas-patch.yaml       # dev: 2副本
│   │   └── resource-limits.yaml      # dev: 资源限制宽松
│   ├── staging/
│   │   ├── kustomization.yaml
│   │   ├── replicas-patch.yaml       # staging: 3副本
│   │   └── configmap-staging.yaml    # staging专用配置
│   └── production/
│       ├── kustomization.yaml
│       ├── replicas-patch.yaml       # prod: 10副本
│       ├── resource-limits.yaml      # prod: 严格资源限制
│       ├── pdb.yaml                  # Pod反亲和性
│       └── configmap-prod.yaml       # 生产配置（加密存储）
└── helm-values/              # Helm Chart值文件（可选）
    ├── dev.yaml
    ├── staging.yaml
    └── production.yaml
      

# 回滚决策规则（PromQL）
# 错误率超过1%则触发回滚
sum(rate(http_requests_total{status=~"5.."}[2m]))
/ sum(rate(http_requests_total[2m])) > 0.01

# P99延迟超过阈值
histogram_quantile(0.99,
  sum(rate(http_request_duration_seconds_bucket[2m])) by (le)
) > 2.0

# Pod重启次数异常
increase(kube_pod_container_status_restarts_total[5m]) > 0

# 健康检查失败率
sum(rate(probe_http_status_code{status!="200"}[1m])) > 0
      

// 回滚执行逻辑（伪代码）
func ExecuteRollback(app string, targetRevision string) error {
    // 1. 获取目标版本的完整资源快照
    snapshot := GetRevisionSnapshot(app, targetRevision)

    // 2. 创建回滚事务
    tx := NewRollbackTransaction(app, snapshot)

    // 3. 依次回滚关联资源（按依赖顺序）
    resources := TopologicalSort(snapshot.Resources)
    for _, res := range resources {
        if err := tx.ApplyResource(res); err != nil {
            tx.Rollback()  // 任一资源回滚失败则整体回退
            return fmt.Errorf("rollback failed at %s: %v", res.Name, err)
        }
    }

    // 4. 验证回滚后状态
    if err := ValidatePostRollback(app, snapshot); err != nil {
        // 记录回滚后异常，触发人工介入
        AlertRollbackAnomaly(app, err)
    }

    // 5. 记录审计日志
    AuditLog("ROLLBACK", app, targetRevision, "SUCCESS")
    return nil
}
      

# Flagger金丝雀分析配置
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
  name: order-service
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: order-service
  service:
    port: 8080
    targetPort: 8080
  analysis:
    interval: 1m
    threshold: 5
    maxWeight: 50
    stepWeight: 10
    metrics:
      - name: error-rate
        thresholdRange:
          max: 0.01
        interval: 1m
      - name: latency-p99
        thresholdRange:
          max: 2000
        interval: 1m
      - name: order-success-rate    # 业务自定义指标
        thresholdRange:
          min: 0.995
        interval: 2m
    webhooks:
      - name: load-test
        type: rollout
        url: http://flagger-loadtester/
        metadata:
          cmd: "hey -z 1m -q 100 -c 10 http://order-service:8080/api/orders"
      - name: acceptance-test
        type: pre-rollout
        url: http://flagger-loadtester/
        metadata:
          type: bash
          cmd: "curl -sf http://order-service:8080/healthz"
      

# .gitlab-ci.yml 标准模板
stages:
  - build
  - test
  - security
  - package
  - publish
  - deploy

variables:
  DOCKER_REGISTRY: "harbor.internal.com"
  MAVEN_OPTS: "-Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository"

# 构建阶段
compile:
  stage: build
  image: maven:3.8-openjdk-17
  script:
    - mvn clean compile -DskipTests -B
  cache:
    key: "${CI_COMMIT_REF_SLUG}-maven"
    paths:
      - .m2/repository/
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

# 单元测试 + 覆盖率
unit-test:
  stage: test
  image: maven:3.8-openjdk-17
  script:
    - mvn test -B
    - awk -F"," '{ instructions += $4 + $5; covered += $5 } END
        { print covered/instructions * 100 }'
        target/site/jacoco/jacoco.csv > coverage.txt
  coverage: '/\d+\.\d+/'
  artifacts:
    reports:
      junit: target/surefire-reports/TEST-*.xml
    paths:
      - target/site/jacoco/

# 安全扫描
sast-scan:
  stage: security
  image: returntocorp/semgrep
  script:
    - semgrep --config=auto --json --output=semgrep-report.json .
  artifacts:
    paths:
      - semgrep-report.json
  allow_failure: false

trivy-scan:
  stage: security
  image: aquasec/trivy
  script:
    - trivy image --exit-code 1 --severity HIGH,CRITICAL
        ${DOCKER_REGISTRY}/${CI_PROJECT_PATH}:${CI_COMMIT_SHORT_SHA}

# Docker镜像构建
docker-build:
  stage: package
  image: docker:24.0
  services:
    - docker:24.0-dind
  script:
    - |
      docker build \
        --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
        --build-arg VCS_REF=${CI_COMMIT_SHORT_SHA} \
        --cache-from ${DOCKER_REGISTRY}/${CI_PROJECT_PATH}:cache \
        -t ${DOCKER_REGISTRY}/${CI_PROJECT_PATH}:${CI_COMMIT_SHORT_SHA} \
        -t ${DOCKER_REGISTRY}/${CI_PROJECT_PATH}:latest \
        --push .
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

# 部署到开发环境
deploy-dev:
  stage: deploy
  image: bitnami/kubectl
  script:
    - kubectl config use-context dev-cluster
    - kubectl set image deployment/${APP_NAME}
        ${APP_NAME}=${DOCKER_REGISTRY}/${CI_PROJECT_PATH}:${CI_COMMIT_SHORT_SHA}
        -n dev
    - kubectl rollout status deployment/${APP_NAME} -n dev --timeout=300s
  environment:
    name: dev
    url: https://dev.internal.com
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      when: manual
      

# 多阶段构建 + 非root用户 + 安全加固
FROM eclipse-temurin:17-jdk-alpine AS builder
WORKDIR /app
COPY pom.xml .
COPY .mvn/ .mvn/
COPY mvnw .
RUN ./mvnw dependency:go-offline -B
COPY src ./src
RUN ./mvnw package -DskipTests -B

FROM eclipse-temurin:17-jre-alpine
LABEL maintainer="platform-team@company.com"
LABEL org.opencontainers.image.source="https://gitlab.internal.com/${PROJECT}"

# 创建非root用户
RUN addgroup -S appgroup && adduser -S appuser -G appgroup

# 设置时区
RUN apk add --no-cache tzdata && \
    cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    echo "Asia/Shanghai" > /etc/timezone

WORKDIR /app
COPY --from=builder /app/target/*.jar app.jar

# 安全加固：只读文件系统 + 限制能力
USER appuser
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=3s --start-period=60s --retries=3 \
    CMD wget -qO- http://localhost:8080/actuator/health || exit 1

ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-XX:MaxRAMPercentage=75.0",
            "-Djava.security.egd=file:/dev/./urandom",
            "-Dspring.profiles.active=${SPRING_PROFILES_ACTIVE}",
            "-jar", "app.jar"]
      

# Ansible动态 inventory + 批量管理示例
# group_vars/production.yml
nginx_workers: 8
redis_maxmemory: "8gb"
keepalived_priority_map:
  master: 100
  backup: 90

# roles/nginx/tasks/main.yml（关键任务片段）
- name: Validate Nginx configuration
  command: nginx -t
  changed_when: false
  register: nginx_test

- name: Reload Nginx (graceful)
  systemd:
    name: nginx
    state: reloaded
  when: nginx_test.rc == 0
  notify: verify nginx is running

- name: Rollback on failure
  block:
    - name: Deploy new configuration
      template:
        src: nginx.conf.j2
        dest: /etc/nginx/nginx.conf
        backup: yes
    - name: Test and reload
      systemd:
        name: nginx
        state: reloaded
  rescue:
    - name: Restore backup configuration
      copy:
        src: ""
        dest: /etc/nginx/nginx.conf
      notify: reload nginx
    - name: Alert rollback event
      uri:
        url: "http://alert-manager/api/alert"
        method: POST
        body: {"service": "nginx", "event": "config_rollback"}
      

# 中心Prometheus联邦配置
scrape_configs:
  - job_name: 'federate'
    scrape_interval: 15s
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
        - '{job=~"kubernetes-.*"}'
        - '{job=~"node-.*"}'
        - '{__name__=~"up|http_requests_total|http_request_duration_seconds.*"}'
    static_configs:
      - targets:
        - prometheus-cluster1:9090
        - prometheus-cluster2:9090
        - prometheus-cluster3:9090
        - prometheus-infra:9090   # 基础设施监控
        - prometheus-middleware:9090  # 中间件监控

# Thanos Sidecar + Store配置
thanos:
  sidecar:
    objstore:
      config:
        type: S3
        config:
          bucket: thanos-metrics
          endpoint: s3.internal.com
          access_key: ${AWS_ACCESS_KEY_ID}
          secret_key: ${AWS_SECRET_ACCESS_KEY}
  store:
    retention: 90d
    compaction: true
      

经验教训清单

• K8s版本升级：曾在没有充分测试的情况下从1.20升级到1.22，导致一个依赖废弃API的服务集群级故障。教训：K8s升级必须建立专门的Canary集群先行验证。
• Prometheus存储膨胀：初期未规划长期存储方案，Prometheus本地存储撑爆导致监控中断24小时。教训：从第一天就接入Thanos或Mimir等长期存储方案。
• 密钥管理：曾将数据库密码明文存储在GitLab CI变量中，在审计中被标记为安全漏洞。教训：引入Vault进行集中密钥管理，所有敏感数据运行时注入。
• 过度工程化：早期自研了一套复杂的编排引擎，最终发现Argo CD + Flagger已能满足90%的需求，自研引擎维护成本高且功能不如开源方案完善。教训：先用好开源工具，有明确缺口再自研。