5 台服务器（五）：Spring Cloud 上线与验收

JDK             21
Spring Boot     3.5.x
Spring Cloud    2025.0.x
Spring Cloud Alibaba  2025.0.0.0
Nacos Server/Client   3.2.4

<dependencyManagement>
  <dependencies>
    <dependency>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-dependencies</artifactId>
      <version>3.5.6</version>
      <type>pom</type><scope>import</scope>
    </dependency>
    <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-dependencies</artifactId>
      <version>2025.0.2</version>
      <type>pom</type><scope>import</scope>
    </dependency>
    <dependency>
      <groupId>com.alibaba.cloud</groupId>
      <artifactId>spring-cloud-alibaba-dependencies</artifactId>
      <version>2025.0.0.0</version>
      <type>pom</type><scope>import</scope>
    </dependency>
  </dependencies>
</dependencyManagement>

kubectl -n middleware exec -it mysql-0 -- mysql -uroot -p'Mysql@2026!ChangeMe' -e "
CREATE DATABASE IF NOT EXISTS nacos_config DEFAULT CHARSET utf8mb4;
CREATE DATABASE IF NOT EXISTS shop DEFAULT CHARSET utf8mb4;
"
# Nacos 建表语句从这个地址拿，对应版本 3.2.4
# https://github.com/alibaba/nacos/blob/3.2.4/distribution/conf/mysql-schema.sql
kubectl -n middleware exec -i mysql-0 -- mysql -uroot -p'Mysql@2026!ChangeMe' nacos_config < mysql-schema.sql

# /root/nacos.yaml 关键片段
apiVersion: v1
kind: Service
metadata:
  name: nacos-headless
  namespace: middleware
spec:
  clusterIP: None
  selector: { app: nacos }
  ports:
  - { name: http,  port: 8848, targetPort: 8848 }
  - { name: grpc,  port: 9848, targetPort: 9848 }
  - { name: jraft, port: 9849, targetPort: 9849 }
---
apiVersion: v1
kind: Service
metadata:
  name: nacos
  namespace: middleware
spec:
  selector: { app: nacos }
  ports:
  - { name: http, port: 8848, targetPort: 8848 }
  - { name: grpc, port: 9848, targetPort: 9848 }
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: nacos
  namespace: middleware
spec:
  serviceName: nacos-headless
  replicas: 3
  podManagementPolicy: Parallel
  selector:
    matchLabels: { app: nacos }
  template:
    metadata:
      labels: { app: nacos }
    spec:
      containers:
      - name: nacos
        image: nacos/nacos-server:3.2.4
        env:
        - name: MODE
          value: cluster
        - name: NACOS_SERVERS
          value: "nacos-0.nacos-headless.middleware.svc.cluster.local:8848 nacos-1.nacos-headless.middleware.svc.cluster.local:8848 nacos-2.nacos-headless.middleware.svc.cluster.local:8848"
        - name: SPRING_DATASOURCE_PLATFORM
          value: mysql
        - name: MYSQL_SERVICE_HOST
          value: mysql.middleware.svc.cluster.local
        - name: MYSQL_SERVICE_DB_NAME
          value: nacos_config
        - name: MYSQL_SERVICE_USER
          value: root
        - name: MYSQL_SERVICE_PASSWORD
          valueFrom:
            secretKeyRef: { name: nacos-secret, key: mysql-password }
        - name: NACOS_AUTH_ENABLE
          value: "true"
        ports:
        - { containerPort: 8848, name: http }
        - { containerPort: 9848, name: grpc }
        - { containerPort: 9849, name: jraft }
        readinessProbe:
          httpGet: { path: /nacos/v3/admin/core/state/health, port: 8080 }
          initialDelaySeconds: 40
          periodSeconds: 10
          failureThreshold: 15
        livenessProbe:
          httpGet: { path: /nacos/v3/admin/core/state/health, port: 8080 }
          initialDelaySeconds: 90
          periodSeconds: 20
          failureThreshold: 10
        resources:
          requests: { cpu: 500m, memory: 2Gi }
          limits:   { cpu: 2,    memory: 3Gi }
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: nfs-client
      resources:
        requests:
          storage: 20Gi

# Dockerfile（分层版）
FROM eclipse-temurin:21-jre-alpine AS extract
WORKDIR /build
ARG JAR_FILE=target/*.jar
COPY ${JAR_FILE} app.jar
RUN java -Djarmode=layertools -jar app.jar extract

FROM eclipse-temurin:21-jre-alpine
RUN apk add --no-cache tzdata curl \
 && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
 && echo "Asia/Shanghai" > /etc/timezone
WORKDIR /app
COPY --from=extract /build/dependencies/ ./
COPY --from=extract /build/spring-boot-loader/ ./
COPY --from=extract /build/snapshot-dependencies/ ./
COPY --from=extract /build/application/ ./
ENV JAVA_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=70.0 \
-XX:+UseG1GC -XX:MaxGCPauseMillis=200 \
-XX:+ExitOnOutOfMemoryError \
-Djava.security.egd=file:/dev/./urandom \
-Dfile.encoding=UTF-8"
EXPOSE 8080
ENTRYPOINT ["sh","-c","exec java $JAVA_OPTS org.springframework.boot.loader.launch.JarLauncher"]

# Dockerfile（简单版，本地调试用）
FROM eclipse-temurin:21-jre-alpine
RUN apk add --no-cache tzdata curl \
 && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
WORKDIR /app
COPY target/*.jar app.jar
ENV JAVA_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=70.0 -XX:+UseG1GC -XX:+ExitOnOutOfMemoryError -Dfile.encoding=UTF-8"
EXPOSE 8080
ENTRYPOINT ["sh","-c","exec java $JAVA_OPTS -jar /app/app.jar"]

# shop-order.yaml 关键片段
apiVersion: v1
kind: ConfigMap
metadata:
  name: shop-common-env
  namespace: prod
data:
  SPRING_PROFILES_ACTIVE: "prod"
  SPRING_CLOUD_NACOS_SERVER_ADDR: "nacos.middleware.svc.cluster.local:8848"
  SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE: "prod"
  SPRING_CLOUD_NACOS_CONFIG_NAMESPACE: "prod"
  SPRING_DATASOURCE_URL: "jdbc:mysql://mysql.middleware.svc.cluster.local:3306/shop?useUnicode=true&characterEncoding=utf8&serverTimezone=Asia/Shanghai&useSSL=false"
  SPRING_DATA_REDIS_HOST: "redis.middleware.svc.cluster.local"
  SPRING_DATA_REDIS_PORT: "6379"
  SERVER_SHUTDOWN: "graceful"
  SPRING_LIFECYCLE_TIMEOUT_PER_SHUTDOWN_PHASE: "30s"
  MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: "health,info,prometheus"
  MANAGEMENT_ENDPOINT_HEALTH_PROBES_ENABLED: "true"
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: shop-order
  namespace: prod
spec:
  replicas: 2
  revisionHistoryLimit: 5
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels: { app: shop-order }
  template:
    metadata:
      labels:
        app: shop-order
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/path: /actuator/prometheus
        prometheus.io/port: "8080"
    spec:
      terminationGracePeriodSeconds: 60
      imagePullSecrets:
      - name: harbor-pull
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: DoNotSchedule
        labelSelector:
          matchLabels: { app: shop-order }
      containers:
      - name: shop-order
        image: harbor.apps.lxhhub.top/shop/shop-order:20260909-a1b2c3d
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
        envFrom:
        - configMapRef: { name: shop-common-env }
        - secretRef: { name: shop-secret }
        env:
        - name: POD_IP
          valueFrom:
            fieldRef: { fieldPath: status.podIP }
        - name: SPRING_CLOUD_NACOS_DISCOVERY_IP
          value: $(POD_IP)
        startupProbe:
          httpGet: { path: /actuator/health/liveness, port: 8080 }
          failureThreshold: 60
          periodSeconds: 5
        readinessProbe:
          httpGet: { path: /actuator/health/readiness, port: 8080 }
          periodSeconds: 5
          failureThreshold: 3
        livenessProbe:
          httpGet: { path: /actuator/health/liveness, port: 8080 }
          periodSeconds: 15
          failureThreshold: 3
        lifecycle:
          preStop:
            exec:
              command: ["sh", "-c", "sleep 25"]
        resources:
          requests: { cpu: 500m, memory: 1Gi }
          limits:   { cpu: 2,    memory: 2Gi }
---
apiVersion: v1
kind: Service
metadata:
  name: shop-order
  namespace: prod
spec:
  selector: { app: shop-order }
  ports:
  - port: 8080
    targetPort: 8080

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: shop-gateway
  namespace: prod
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: "20m"
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "10"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "120"
spec:
  ingressClassName: nginx
  rules:
  - host: shop.apps.lxhhub.top
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: shop-gateway
            port: { number: 8080 }

shop-gitops/
├── base/
│   ├── kustomization.yaml
│   ├── configmap.yaml
│   ├── secret.yaml
│   ├── shop-gateway.yaml
│   └── ...
└── overlays/
    ├── prod/
    │   ├── kustomization.yaml     # kustomize edit set image 改的就是这里
    │   └── replica-patch.yaml
    └── test/
        └── kustomization.yaml

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: shop-prod
  namespace: argocd
spec:
  project: default
  source:
    repoURL: http://git.apps.lxhhub.top/gitea-admin/shop-gitops.git
    targetRevision: main
    path: overlays/prod
  destination:
    server: https://kubernetes.default.svc
    namespace: prod
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
    - CreateNamespace=true
    - ApplyOutOfSyncOnly=true
    retry:
      limit: 5
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m

# 方式一：Argo CD 回滚到上一版
argocd app history shop-prod
argocd app rollback shop-prod <REVISION>

# 方式二：Git revert
git revert HEAD && git push

kubectl get nodes -o wide                      # 5 台 Ready
kubectl get pod -A                             # 无 Pending / CrashLoopBackOff
kubectl get --raw='/readyz?verbose' | tail -20
etcdctl-prod endpoint health --write-out=table # 3 个 true
kubectl -n kube-system logs -l k8s-app=kube-dns --tail=20
kubectl top nodes                              # metrics-server 正常

# 停掉持有 VIP 的 master 上的 apiserver，VIP 应漂走，kubectl 仍可用
systemctl stop kube-apiserver
kubectl get nodes

# 停掉一个 master 整机，etcd 应仍可写（3 节点坏 1 个仍然 quorum）
kubectl create sa test-ha && kubectl delete sa test-ha

# 恢复后 etcd 成员应自动追平
etcdctl-prod endpoint status --write-out=table

kubectl run dns-test --image=busybox:1.37 -it --rm --restart=Never -- \
  nslookup kubernetes.default.svc.cluster.local
kubectl run dns-test2 --image=busybox:1.37 -it --rm --restart=Never -- \
  nslookup shop-order.prod.svc.cluster.local

# Pod 之间跨节点互通
kubectl -n prod exec -it <pod1> -- curl -s http://shop-order:8080/actuator/health

kubectl -n prod get pod -o wide
kubectl -n prod get hpa
curl -H "Host: shop.apps.lxhhub.top" http://10.0.0.14/actuator/health

# 压测触发 HPA
kubectl run -i --tty load --rm --image=busybox:1.37 --restart=Never -- \
  /bin/sh -c "while true; do wget -q -O- http://shop-order.prod:8080/api/order/list; done"
kubectl -n prod get hpa -w

etcdctl-prod snapshot save /backup/etcd-$(date +%Y%m%d).db
etcdctl-prod snapshot status /backup/etcd-$(date +%Y%m%d).db --write-out=table

cat > /etc/systemd/system/etcd-backup.service <<'EOF'
[Unit]
Description=etcd snapshot backup
[Service]
Type=oneshot
ExecStart=/bin/bash -c 'etcdctl-prod snapshot save /backup/etcd-$(date +%%Y%%m%%d-%%H%%M).db'
EOF
cat > /etc/systemd/system/etcd-backup.timer <<'EOF'
[Unit]
Description=Daily etcd backup
[Timer]
OnCalendar=*-*-* 02:30:00
Persistent=true
[Install]
WantedBy=timers.target
EOF
systemctl enable --now etcd-backup.timer