5 台服务器，从二进制装 K8s 到 Spring Cloud 上线

cat > /root/init-node.sh <<'SCRIPT'
#!/bin/bash
set -e
HOST=$1
[ -z "$HOST" ] && { echo "用法: $0 <主机名>"; exit 1; }

hostnamectl set-hostname ${HOST}

cat >> /etc/hosts <<'EOF'
10.0.0.10  k8s-vip
10.0.0.11  k8s-m1
10.0.0.12  k8s-m2
10.0.0.13  k8s-m3
10.0.0.14  k8s-n1
10.0.0.15  k8s-n2
EOF

# 防火墙。有严格的合规要求就别关，改成放行 6443/2379/2380/10250/179/8472 等端口
systemctl disable --now firewalld || true

setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

swapoff -a
sed -i '/ swap /s/^/#/' /etc/fstab

dnf install -y ipvsadm ipset conntrack-tools socat ebtables chrony \
               curl wget tar jq bash-completion nfs-utils

systemctl enable --now chronyd

cat > /etc/modules-load.d/k8s.conf <<'EOF'
overlay
br_netfilter
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF
modprobe overlay
modprobe br_netfilter
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh

cat > /etc/sysctl.d/99-k8s.conf <<'EOF'
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
net.ipv4.ip_local_port_range        = 1024 65535
net.netfilter.nf_conntrack_max      = 1000000
net.netfilter.nf_conntrack_tcp_timeout_established = 86400
net.core.somaxconn                  = 32768
net.ipv4.neigh.default.gc_thresh1   = 8192
net.ipv4.neigh.default.gc_thresh2   = 32768
net.ipv4.neigh.default.gc_thresh3   = 65536
fs.file-max                         = 1000000
fs.inotify.max_user_instances       = 8192
fs.inotify.max_user_watches         = 524288
vm.swappiness                       = 0
vm.max_map_count                    = 262144
EOF
sysctl --system

cat > /etc/security/limits.d/99-k8s.conf <<'EOF'
*       soft    nofile    1048576
*       hard    nofile    1048576
*       soft    nproc     655360
*       hard    nproc     655360
*       soft    memlock   unlimited
*       hard    memlock   unlimited
EOF

mkdir -p /etc/kubernetes/pki /var/lib/kubelet /var/log/kubernetes
echo "init done: ${HOST}"
SCRIPT

chmod +x /root/init-node.sh

# 各自执行
/root/init-node.sh k8s-m1   # 10.0.0.11
/root/init-node.sh k8s-m2   # 10.0.0.12
/root/init-node.sh k8s-m3   # 10.0.0.13
/root/init-node.sh k8s-n1   # 10.0.0.14
/root/init-node.sh k8s-n2   # 10.0.0.15

reboot

getenforce                 # Permissive 或 Disabled
free -h | grep Swap        # 必须是 0
chronyc tracking | grep Leap  # 时间同步正常
hostname                   # 五台各不相同

cat > /root/install-containerd.sh <<'SCRIPT'
#!/bin/bash
set -e
CD_VER=2.3.3
RUNC_VER=1.3.0      # 换成官方 release 页最新的 1.3.x
CNI_VER=1.7.0       # 换成官方 release 页最新的 1.7.x
PAUSE_VER=3.10

cd /usr/local/src

# containerd
wget -q https://github.com/containerd/containerd/releases/download/v${CD_VER}/containerd-${CD_VER}-linux-amd64.tar.gz
tar -C /usr/local -xzf containerd-${CD_VER}-linux-amd64.tar.gz

# runc
wget -q -O /usr/local/sbin/runc https://github.com/opencontainers/runc/releases/download/v${RUNC_VER}/runc.amd64
chmod +x /usr/local/sbin/runc

# CNI plugins
mkdir -p /opt/cni/bin /etc/cni/net.d
wget -q https://github.com/containernetworking/plugins/releases/download/v${CNI_VER}/cni-plugins-linux-amd64-v${CNI_VER}.tgz
tar -C /opt/cni/bin -xzf cni-plugins-linux-amd64-v${CNI_VER}.tgz

# systemd unit
cat > /usr/lib/systemd/system/containerd.service <<'EOF'
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=1048576
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF

mkdir -p /etc/containerd /etc/containerd/certs.d

cat > /etc/containerd/config.toml <<EOF
version = 3
root = "/var/lib/containerd"
state = "/run/containerd"

[grpc]
  max_recv_message_size = 16777216
  max_send_message_size = 16777216

[debug]
  level = "warn"

[plugins."io.containerd.cri.v1.images"]
  snapshotter = "overlayfs"
  [plugins."io.containerd.cri.v1.images".pinned_images]
    sandbox = "registry.k8s.io/pause:${PAUSE_VER}"

[plugins."io.containerd.cri.v1.runtime"]
  [plugins."io.containerd.cri.v1.runtime".containerd]
    default_runtime_name = "runc"
    [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc]
      runtime_type = "io.containerd.runc.v2"
      [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.runc.options]
        SystemdCgroup = true

[plugins."io.containerd.cri.v1.runtime".cni]
  bin_dir = "/opt/cni/bin"
  conf_dir = "/etc/cni/net.d"
EOF

systemctl daemon-reload
systemctl enable --now containerd
systemctl is-active containerd
SCRIPT

chmod +x /root/install-containerd.sh && /root/install-containerd.sh

# 公共镜像加速，按你的网络环境选一个或几个
mkdir -p /etc/containerd/certs.d/docker.io
cat > /etc/containerd/certs.d/docker.io/hosts.toml <<'EOF'
server = "https://registry-1.docker.io"
[host."https://docker.m.daocloud.io"]
  capabilities = ["pull", "resolve"]
[host."https://registry-1.docker.io"]
  capabilities = ["pull", "resolve"]
EOF

# 内网 Harbor，HTTP
mkdir -p /etc/containerd/certs.d/harbor.apps.lxhhub.top
cat > /etc/containerd/certs.d/harbor.apps.lxhhub.top/hosts.toml <<'EOF'
server = "http://harbor.apps.lxhhub.top"
[host."http://harbor.apps.lxhhub.top"]
  capabilities = ["pull", "resolve", "push"]
  skip_verify = true
EOF

systemctl restart containerd

CRICTL_VER=v1.32.0
wget -q https://github.com/kubernetes-sigs/crictl/releases/download/${CRICTL_VER}/crictl-${CRICTL_VER}-linux-amd64.tar.gz
tar -C /usr/local/bin -xzf crictl-${CRICTL_VER}-linux-amd64.tar.gz

cat > /etc/crictl.yaml <<'EOF'
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

crictl info | head -20
crictl pull registry.k8s.io/pause:3.10

# m1 上执行
wget -q -O /usr/local/bin/cfssl     https://github.com/cloudflare/cfssl/releases/download/v1.6.5/cfssl_1.6.5_linux_amd64
wget -q -O /usr/local/bin/cfssljson https://github.com/cloudflare/cfssl/releases/download/v1.6.5/cfssljson_1.6.5_linux_amd64
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson

mkdir -p /root/ssl && cd /root/ssl

cat > ca-config.json <<'EOF'
{
  "signing": {
    "default": { "expiry": "87600h" },
    "profiles": {
      "kubernetes": {
        "expiry": "87600h",
        "usages": ["signing", "key encipherment", "server auth", "client auth"]
      },
      "etcd": {
        "expiry": "87600h",
        "usages": ["signing", "key encipherment", "server auth", "client auth"]
      }
    }
  }
}
EOF

cat > etcd-ca-csr.json <<'EOF'
{
  "CN": "etcd-ca",
  "key": { "algo": "rsa", "size": 4096 },
  "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "etcd", "OU": "Ops" }]
}
EOF

cat > k8s-ca-csr.json <<'EOF'
{
  "CN": "kubernetes-ca",
  "key": { "algo": "rsa", "size": 4096 },
  "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "Kubernetes", "OU": "Ops" }]
}
EOF

cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca
cfssl gencert -initca k8s-ca-csr.json  | cfssljson -bare k8s-ca

cat > etcd-server-csr.json <<'EOF'
{
  "CN": "etcd-server",
  "hosts": [
    "127.0.0.1",
    "10.0.0.11", "10.0.0.12", "10.0.0.13",
    "k8s-m1", "k8s-m2", "k8s-m3",
    "k8s-m1.apps.lxhhub.top", "k8s-m2.apps.lxhhub.top", "k8s-m3.apps.lxhhub.top"
  ],
  "key": { "algo": "rsa", "size": 2048 },
  "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "etcd", "OU": "Ops" }]
}
EOF

cat > etcd-client-csr.json <<'EOF'
{
  "CN": "etcd-client",
  "key": { "algo": "rsa", "size": 2048 },
  "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "etcd", "OU": "Ops" }]
}
EOF

cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=etcd \
  etcd-server-csr.json | cfssljson -bare etcd-server
cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=etcd \
  etcd-client-csr.json | cfssljson -bare etcd-client

cat > apiserver-csr.json <<'EOF'
{
  "CN": "kube-apiserver",
  "hosts": [
    "127.0.0.1",
    "10.0.0.10", "10.0.0.11", "10.0.0.12", "10.0.0.13",
    "10.96.0.1",
    "k8s-vip", "k8s-m1", "k8s-m2", "k8s-m3",
    "kubernetes", "kubernetes.default", "kubernetes.default.svc",
    "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"
  ],
  "key": { "algo": "rsa", "size": 2048 },
  "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "Kubernetes", "OU": "Ops" }]
}
EOF

cfssl gencert -ca=k8s-ca.pem -ca-key=k8s-ca-key.pem -config=ca-config.json -profile=kubernetes \
  apiserver-csr.json | cfssljson -bare apiserver

# apiserver -> kubelet
cat > apiserver-kubelet-client-csr.json <<'EOF'
{
  "CN": "apiserver-kubelet-client",
  "key": { "algo": "rsa", "size": 2048 },
  "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "system:masters", "OU": "Ops" }]
}
EOF
cfssl gencert -ca=k8s-ca.pem -ca-key=k8s-ca-key.pem -config=ca-config.json -profile=kubernetes \
  apiserver-kubelet-client-csr.json | cfssljson -bare apiserver-kubelet-client

# controller-manager / scheduler / admin / kube-proxy
gen_client () {
  cat > ${1}-csr.json <<EOF
{
  "CN": "${2}",
  "key": { "algo": "rsa", "size": 2048 },
  "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "${3}", "OU": "Ops" }]
}
EOF
  cfssl gencert -ca=k8s-ca.pem -ca-key=k8s-ca-key.pem -config=ca-config.json -profile=kubernetes \
    ${1}-csr.json | cfssljson -bare ${1}
}

gen_client kube-controller-manager  system:kube-controller-manager  system:kube-controller-manager
gen_client kube-scheduler           system:kube-scheduler           system:kube-scheduler
gen_client admin                    admin                           system:masters
gen_client kube-proxy               system:kube-proxy               system:node-proxier

cat > front-proxy-ca-csr.json <<'EOF'
{
  "CN": "front-proxy-ca",
  "key": { "algo": "rsa", "size": 4096 },
  "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "Kubernetes", "OU": "Ops" }]
}
EOF
cfssl gencert -initca front-proxy-ca-csr.json | cfssljson -bare front-proxy-ca

cat > front-proxy-client-csr.json <<'EOF'
{
  "CN": "front-proxy-client",
  "key": { "algo": "rsa", "size": 2048 },
  "names": [{ "C": "CN", "ST": "Beijing", "L": "Beijing", "O": "Kubernetes", "OU": "Ops" }]
}
EOF
cfssl gencert -ca=front-proxy-ca.pem -ca-key=front-proxy-ca-key.pem -config=ca-config.json \
  -profile=kubernetes front-proxy-client-csr.json | cfssljson -bare front-proxy-client

openssl genrsa -out sa.key 2048
openssl rsa -in sa.key -pubout -out sa.pub

cd /root/ssl

# 三个 master：etcd 证书 + k8s 证书 + sa 密钥
for h in k8s-m1 k8s-m2 k8s-m3; do
  ssh $h "mkdir -p /etc/etcd/ssl /etc/kubernetes/pki"
  scp etcd-ca.pem etcd-server.pem etcd-server-key.pem etcd-client.pem etcd-client-key.pem \
      $h:/etc/etcd/ssl/
  scp k8s-ca.pem k8s-ca-key.pem apiserver.pem apiserver-key.pem \
      apiserver-kubelet-client.pem apiserver-kubelet-client-key.pem \
      front-proxy-ca.pem front-proxy-client.pem front-proxy-client-key.pem \
      sa.key sa.pub $h:/etc/kubernetes/pki/
done

# 两个 worker：只需要 CA 公钥
for h in k8s-n1 k8s-n2; do
  ssh $h "mkdir -p /etc/kubernetes/pki"
  scp k8s-ca.pem $h:/etc/kubernetes/pki/
done

ETCD_VER=v3.6.13
wget -q https://github.com/etcd-io/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz
tar -xzf etcd-${ETCD_VER}-linux-amd64.tar.gz
cp etcd-${ETCD_VER}-linux-amd64/etcd{,-ctl} /usr/local/bin/
etcd --version

cat > /root/install-etcd.sh <<'SCRIPT'
#!/bin/bash
set -e
NODE_IP=$1
NODE_NAME=$(hostname)

cat > /usr/lib/systemd/system/etcd.service <<EOF
[Unit]
Description=Etcd Server
Documentation=https://etcd.io
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd \\
  --name=${NODE_NAME} \\
  --data-dir=/var/lib/etcd \\
  --wal-dir=/var/lib/etcd/wal \\
  --listen-peer-urls=https://${NODE_IP}:2380 \\
  --listen-client-urls=https://${NODE_IP}:2379,https://127.0.0.1:2379 \\
  --advertise-client-urls=https://${NODE_IP}:2379 \\
  --initial-advertise-peer-urls=https://${NODE_IP}:2380 \\
  --initial-cluster=k8s-m1=https://10.0.0.11:2380,k8s-m2=https://10.0.0.12:2380,k8s-m3=https://10.0.0.13:2380 \\
  --initial-cluster-state=new \\
  --initial-cluster-token=etcd-k8s-cluster \\
  --cert-file=/etc/etcd/ssl/etcd-server.pem \\
  --key-file=/etc/etcd/ssl/etcd-server-key.pem \\
  --peer-cert-file=/etc/etcd/ssl/etcd-server.pem \\
  --peer-key-file=/etc/etcd/ssl/etcd-server-key.pem \\
  --trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem \\
  --peer-trusted-ca-file=/etc/etcd/ssl/etcd-ca.pem \\
  --client-cert-auth=true \\
  --peer-client-cert-auth=true \\
  --snapshot-count=10000 \\
  --quota-backend-bytes=8589934592 \\
  --auto-compaction-mode=revision \\
  --auto-compaction-retention=1000 \\
  --v2-deprecation=write-only \\
  --log-level=info \\
  --logger=zap
Restart=always
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable --now etcd
sleep 3
systemctl is-active etcd
SCRIPT

chmod +x /root/install-etcd.sh
/root/install-etcd.sh 10.0.0.11   # m1

export ETCDCTL_API=3
etcdctl --endpoints=https://10.0.0.11:2379,https://10.0.0.12:2379,https://10.0.0.13:2379 \
  --cacert=/etc/etcd/ssl/etcd-ca.pem \
  --cert=/etc/etcd/ssl/etcd-client.pem \
  --key=/etc/etcd/ssl/etcd-client-key.pem \
  endpoint health --write-out=table

etcdctl --endpoints=https://10.0.0.11:2379 \
  --cacert=/etc/etcd/ssl/etcd-ca.pem \
  --cert=/etc/etcd/ssl/etcd-client.pem \
  --key=/etc/etcd/ssl/etcd-client-key.pem \
  member list --write-out=table

cat >> /root/.bashrc <<'EOF'
export ETCDCTL_API=3
alias etcdctl-prod='etcdctl --endpoints=https://10.0.0.11:2379,https://10.0.0.12:2379,https://10.0.0.13:2379 --cacert=/etc/etcd/ssl/etcd-ca.pem --cert=/etc/etcd/ssl/etcd-client.pem --key=/etc/etcd/ssl/etcd-client-key.pem'
EOF
source /root/.bashrc

K8S_VER=v1.36.4
wget -q https://dl.k8s.io/${K8S_VER}/kubernetes-server-linux-amd64.tar.gz
tar -xzf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin
cp kube-apiserver kube-controller-manager kube-scheduler kubelet kube-proxy kubectl /usr/local/bin/
chmod +x /usr/local/bin/kube*
kube-apiserver --version

cd /root/ssl
KUBE_APISERVER="https://10.0.0.10:16443"   # 走 VIP，后面 haproxy 起在这个端口

# 通用函数
gen_kubeconfig () {
  local NAME=$1 CN=$2
  kubectl config set-cluster kubernetes \
    --certificate-authority=k8s-ca.pem \
    --embed-certs=true \
    --server=${KUBE_APISERVER} \
    --kubeconfig=${NAME}.kubeconfig
  kubectl config set-credentials ${CN} \
    --client-certificate=${NAME}.pem \
    --client-key=${NAME}-key.pem \
    --embed-certs=true \
    --kubeconfig=${NAME}.kubeconfig
  kubectl config set-context ${CN}@kubernetes \
    --cluster=kubernetes --user=${CN} \
    --kubeconfig=${NAME}.kubeconfig
  kubectl config use-context ${CN}@kubernetes --kubeconfig=${NAME}.kubeconfig
}

gen_kubeconfig kube-controller-manager system:kube-controller-manager
gen_kubeconfig kube-scheduler           system:kube-scheduler
gen_kubeconfig admin                    admin
gen_kubeconfig kube-proxy               system:kube-proxy

TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' \n' | head -c 6).$(head -c 24 /dev/urandom | base64 | tr -d '=+/' | head -c 16)
echo "bootstrap token: ${TOKEN}"

cat > token.csv <<EOF
${TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

kubectl config set-cluster kubernetes \
  --certificate-authority=k8s-ca.pem --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=bootstrap-kubelet.kubeconfig
kubectl config set-credentials kubelet-bootstrap \
  --token=${TOKEN} \
  --kubeconfig=bootstrap-kubelet.kubeconfig
kubectl config set-context kubelet-bootstrap@kubernetes \
  --cluster=kubernetes --user=kubelet-bootstrap \
  --kubeconfig=bootstrap-kubelet.kubeconfig
kubectl config use-context kubelet-bootstrap@kubernetes \
  --kubeconfig=bootstrap-kubelet.kubeconfig

for h in k8s-m1 k8s-m2 k8s-m3; do
  scp token.csv bootstrap-kubelet.kubeconfig \
      kube-controller-manager.kubeconfig kube-scheduler.kubeconfig \
      admin.kubeconfig kube-proxy.kubeconfig $h:/etc/kubernetes/
done

cat > /usr/lib/systemd/system/kube-apiserver.service <<'EOF'
[Unit]
Description=Kubernetes API Server
Documentation=https://kubernetes.io
After=network.target etcd.service

[Service]
ExecStart=/usr/local/bin/kube-apiserver \
  --advertise-address=NODE_IP_PLACEHOLDER \
  --bind-address=0.0.0.0 \
  --secure-port=6443 \
  --allow-privileged=true \
  --authorization-mode=Node,RBAC \
  --enable-admission-plugins=NodeRestriction \
  --enable-bootstrap-token-auth=true \
  --token-auth-file=/etc/kubernetes/token.csv \
  --service-cluster-ip-range=10.96.0.0/12 \
  --service-node-port-range=30000-32767 \
  --etcd-servers=https://10.0.0.11:2379,https://10.0.0.12:2379,https://10.0.0.13:2379 \
  --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \
  --etcd-certfile=/etc/etcd/ssl/etcd-client.pem \
  --etcd-keyfile=/etc/etcd/ssl/etcd-client-key.pem \
  --client-ca-file=/etc/kubernetes/pki/k8s-ca.pem \
  --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \
  --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \
  --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.pem \
  --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client-key.pem \
  --service-account-key-file=/etc/kubernetes/pki/sa.pub \
  --service-account-signing-key-file=/etc/kubernetes/pki/sa.key \
  --service-account-issuer=https://kubernetes.default.svc.cluster.local \
  --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
  --requestheader-username-headers=X-Remote-User \
  --requestheader-group-headers=X-Remote-Group \
  --requestheader-extra-headers-prefix=X-Remote-Extra- \
  --requestheader-allowed-names=front-proxy-client \
  --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \
  --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \
  --enable-aggregator-routing=true \
  --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
  --max-mutating-requests-inflight=2000 \
  --max-requests-inflight=4000 \
  --default-not-ready-toleration-seconds=30 \
  --default-unreachable-toleration-seconds=30 \
  --audit-log-maxage=15 --audit-log-maxbackup=5 --audit-log-maxsize=200 \
  --audit-log-path=/var/log/kubernetes/apiserver-audit.log \
  --v=2
Restart=always
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

sed -i "s/NODE_IP_PLACEHOLDER/$(hostname -i | awk '{print $1}')/" /usr/lib/systemd/system/kube-apiserver.service
systemctl daemon-reload && systemctl enable --now kube-apiserver

cat > /usr/lib/systemd/system/kube-controller-manager.service <<'EOF'
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://kubernetes.io
After=network.target kube-apiserver.service

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
  --kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \
  --authentication-kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \
  --authorization-kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \
  --client-ca-file=/etc/kubernetes/pki/k8s-ca.pem \
  --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
  --root-ca-file=/etc/kubernetes/pki/k8s-ca.pem \
  --service-account-private-key-file=/etc/kubernetes/pki/sa.key \
  --cluster-signing-cert-file=/etc/kubernetes/pki/k8s-ca.pem \
  --cluster-signing-key-file=/etc/kubernetes/pki/k8s-ca-key.pem \
  --cluster-signing-duration=87600h \
  --cluster-name=kubernetes \
  --cluster-cidr=172.16.0.0/16 \
  --service-cluster-ip-range=10.96.0.0/12 \
  --allocate-node-cidrs=true \
  --controllers=*,bootstrapsigner,tokencleaner \
  --use-service-account-credentials=true \
  --node-monitor-period=5s \
  --node-monitor-grace-period=40s \
  --terminated-pod-gc-threshold=100 \
  --leader-elect=true \
  --leader-elect-lease-duration=15s \
  --leader-elect-renew-deadline=10s \
  --leader-elect-retry-period=2s \
  --v=2
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable --now kube-controller-manager

cat > /etc/kubernetes/kube-scheduler.yaml <<'EOF'
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
clientConnection:
  kubeconfig: /etc/kubernetes/kube-scheduler.kubeconfig
leaderElection:
  leaderElect: true
  leaseDuration: 15s
  renewDeadline: 10s
  retryPeriod: 2s
percentageOfNodesToScore: 100
EOF

cat > /usr/lib/systemd/system/kube-scheduler.service <<'EOF'
[Unit]
Description=Kubernetes Scheduler
Documentation=https://kubernetes.io
After=network.target kube-apiserver.service

[Service]
ExecStart=/usr/local/bin/kube-scheduler \
  --config=/etc/kubernetes/kube-scheduler.yaml \
  --v=2
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable --now kube-scheduler

mkdir -p ~/.kube
cp /etc/kubernetes/admin.kubeconfig ~/.kube/config
kubectl completion bash > /etc/bash_completion.d/kubectl
source /etc/bash_completion.d/kubectl

kubectl get cs    # 老命令可能已废弃，改用下面这条
kubectl get --raw='/readyz?verbose' | grep -E 'etcd|readyz'

cat > /root/rbac-bootstrap.yaml <<'EOF'
# 允许 bootstrap token 发起 CSR
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kubelet-bootstrap
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:node-bootstrapper
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:kubelet-bootstrap
---
# 自动批准首次 CSR
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: auto-approve-csrs-for-group
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:kubelet-bootstrap
---
# 自动批准 kubelet 客户端证书续期
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: auto-approve-renewals-for-nodes
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:nodes
---
# 自动批准 kubelet server 证书（serverTLSBootstrap 打开时需要）
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: auto-approve-server-renewals-for-nodes
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeserver
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:nodes
---
# apiserver 访问 kubelet 的 logs/exec/metrics
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:kube-apiserver-to-kubelet
rules:
- apiGroups: [""]
  resources: ["nodes/proxy", "nodes/stats", "nodes/log", "nodes/spec", "nodes/metrics"]
  verbs: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:kube-apiserver
  namespace: ""
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kube-apiserver-to-kubelet
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: User
  name: kube-apiserver
EOF

kubectl apply -f /root/rbac-bootstrap.yaml

dnf install -y haproxy keepalived

cat > /etc/haproxy/haproxy.cfg <<'EOF'
global
    log         127.0.0.1 local2
    maxconn     32768
    user        haproxy
    group       haproxy
    daemon

defaults
    mode                    tcp
    log                     global
    option                  tcplog
    timeout connect         10s
    timeout client          1m
    timeout server          1m

frontend k8s-apiserver
    bind *:16443
    mode tcp
    default_backend k8s-apiserver

backend k8s-apiserver
    mode tcp
    balance roundrobin
    option tcp-check
    server k8s-m1 10.0.0.11:6443 check inter 2000 fall 3 rise 2
    server k8s-m2 10.0.0.12:6443 check inter 2000 fall 3 rise 2
    server k8s-m3 10.0.0.13:6443 check inter 2000 fall 3 rise 2
EOF

cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
    router_id LVS_K8S_$(hostname | tr 'a-z' 'A-Z')
    script_user root
    enable_script_security
}

vrrp_script check_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 3
    weight -30
    fall 2
    rise 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0              # 改成你的网卡名
    virtual_router_id 51
    priority 100                # m1=100 m2=99 m3=98
    advert_int 1
    nopreempt
    authentication {
        auth_type PASS
        auth_pass k8s_vip_2026
    }
    virtual_ipaddress {
        10.0.0.10/24 dev eth0
    }
    track_script {
        check_apiserver
    }
}
EOF

cat > /etc/keepalived/check_apiserver.sh <<'EOF'
#!/bin/bash
if curl -sk --max-time 2 https://127.0.0.1:16443/readyz >/dev/null 2>&1; then
  exit 0
fi
exit 1
EOF
chmod +x /etc/keepalived/check_apiserver.sh

systemctl enable --now haproxy keepalived

ip addr show | grep 10.0.0.10        # 只在某台上出现
curl -sk https://10.0.0.10:16443/readyz
systemctl stop kube-apiserver        # 在持有 VIP 的机器上停掉，看 VIP 是否漂走

cat > /etc/kubernetes/kubelet-config.yaml <<'EOF'
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 0
authentication:
  anonymous:
    enabled: false
  webhook:
    enabled: true
    cacheTTL: 2m0s
  x509:
    clientCAFile: /etc/kubernetes/pki/k8s-ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
containerLogMaxFiles: 5
containerLogMaxSize: 100Mi
maxPods: 110
failSwapOn: true
serializeImagePulls: false
registryPullQPS: 10
registryBurst: 20
rotateCertificates: true
serverTLSBootstrap: true
evictionHard:
  memory.available: 500Mi
  nodefs.available: 10%
  imagefs.available: 15%
  nodefs.inodesFree: 5%
evictionSoft:
  memory.available: 1Gi
evictionSoftGracePeriod:
  memory.available: 1m30s
kubeReserved:
  cpu: 200m
  memory: 1Gi
  ephemeral-storage: 1Gi
systemReserved:
  cpu: 200m
  memory: 1Gi
  ephemeral-storage: 1Gi
EOF

cat > /usr/lib/systemd/system/kubelet.service <<'EOF'
[Unit]
Description=Kubernetes Kubelet
Documentation=https://kubernetes.io
After=containerd.service
Requires=containerd.service

[Service]
ExecStart=/usr/local/bin/kubelet \
  --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig \
  --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
  --config=/etc/kubernetes/kubelet-config.yaml \
  --cert-dir=/etc/kubernetes/pki \
  --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
  --node-ip=NODE_IP_PLACEHOLDER \
  --hostname-override=HOSTNAME_PLACEHOLDER \
  --pod-infra-container-image=registry.k8s.io/pause:3.10 \
  --root-dir=/var/lib/kubelet \
  --v=2
Restart=always
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

sed -i "s/NODE_IP_PLACEHOLDER/$(hostname -i | awk '{print $1}')/" /usr/lib/systemd/system/kubelet.service
sed -i "s/HOSTNAME_PLACEHOLDER/$(hostname)/" /usr/lib/systemd/system/kubelet.service

systemctl daemon-reload && systemctl enable --now kubelet

# m1 上执行
scp /etc/kubernetes/bootstrap-kubelet.kubeconfig /etc/kubernetes/pki/k8s-ca.pem k8s-n1:/etc/kubernetes/
scp /etc/kubernetes/bootstrap-kubelet.kubeconfig /etc/kubernetes/pki/k8s-ca.pem k8s-n2:/etc/kubernetes/
# n1/n2 上把 k8s-ca.pem 放到 /etc/kubernetes/pki/

kubectl get csr
# 会出现 Pending -> Approved,Issued
# 因为开了 serverTLSBootstrap，还会多一个 selfnodeserver 的 CSR，同样会自动批准

cat > /etc/kubernetes/kube-proxy-config.yaml <<'EOF'
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
clientConnection:
  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
clusterCIDR: 172.16.0.0/16
mode: ipvs
healthzBindAddress: 0.0.0.0:10256
metricsBindAddress: 0.0.0.0:10249
ipvs:
  scheduler: rr
  strictARP: true
  tcpTimeout: 0s
  tcpFinTimeout: 0s
  udpTimeout: 0s
conntrack:
  maxPerCore: 32768
  min: 131072
  tcpEstablishedTimeout: 0s
  tcpCloseWaitTimeout: 0s
EOF

cat > /usr/lib/systemd/system/kube-proxy.service <<'EOF'
[Unit]
Description=Kubernetes Kube Proxy
Documentation=https://kubernetes.io
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-proxy \
  --config=/etc/kubernetes/kube-proxy-config.yaml \
  --hostname-override=HOSTNAME_PLACEHOLDER \
  --v=2
Restart=always
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

sed -i "s/HOSTNAME_PLACEHOLDER/$(hostname)/" /usr/lib/systemd/system/kube-proxy.service
systemctl daemon-reload && systemctl enable --now kube-proxy

scp /etc/kubernetes/kube-proxy.kubeconfig k8s-n1:/etc/kubernetes/
scp /etc/kubernetes/kube-proxy.kubeconfig k8s-n2:/etc/kubernetes/

kubectl label node k8s-m1 node-role.kubernetes.io/control-plane=
kubectl label node k8s-m2 node-role.kubernetes.io/control-plane=
kubectl label node k8s-m3 node-role.kubernetes.io/control-plane=
kubectl label node k8s-n1 node-role.kubernetes.io/worker=
kubectl label node k8s-n2 node-role.kubernetes.io/worker=

kubectl taint nodes k8s-m1 node-role.kubernetes.io/control-plane:NoSchedule
kubectl taint nodes k8s-m2 node-role.kubernetes.io/control-plane:NoSchedule
kubectl taint nodes k8s-m3 node-role.kubernetes.io/control-plane:NoSchedule

kubectl get nodes -o wide

CALICO_VER=v3.32.2
wget -q https://github.com/projectcalico/calico/releases/download/${CALICO_VER}/release-${CALICO_VER}.tgz
tar -xzf release-${CALICO_VER}.tgz
cd release-${CALICO_VER}/manifests/

# IP 池必须和 kube-controller-manager 的 --cluster-cidr 一致
sed -i 's|# - name: CALICO_IPV4POOL_CIDR|  - name: CALICO_IPV4POOL_CIDR|; s|#   value: "192.168.0.0/16"|  value: "172.16.0.0/16"|' calico.yaml
grep -A1 'CALICO_IPV4POOL_CIDR' calico.yaml

kubectl apply -f calico.yaml
kubectl -n kube-system get pod -l k8s-app=calico-node -w

sed -i '/# - name: IP_AUTODETECTION_METHOD/{n; s|#   value: "first-found"|  value: "interface=eth0"|}' calico.yaml
sed -i 's|# - name: IP_AUTODETECTION_METHOD|  - name: IP_AUTODETECTION_METHOD|' calico.yaml

wget -q -O /usr/local/bin/calicoctl https://github.com/projectcalico/calico/releases/download/${CALICO_VER}/calicoctl-linux-amd64
chmod +x /usr/local/bin/calicoctl
calicoctl node status

cat > /root/coredns.yaml <<'EOF'
apiVersion: v1
kind: ServiceAccount
metadata:
  name: coredns
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:coredns
rules:
- apiGroups: [""]
  resources: ["endpoints", "services", "pods", "namespaces"]
  verbs: ["list", "watch"]
- apiGroups: ["discovery.k8s.io"]
  resources: ["endpointslices"]
  verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:coredns
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:coredns
subjects:
- kind: ServiceAccount
  name: coredns
  namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health {
          lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          pods insecure
          fallthrough in-addr.arpa ip6.arpa
          ttl 30
        }
        prometheus :9153
        forward . /etc/resolv.conf {
          max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
spec:
  replicas: 2
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  selector:
    matchLabels:
      k8s-app: kube-dns
  template:
    metadata:
      labels:
        k8s-app: kube-dns
    spec:
      priorityClassName: system-cluster-critical
      serviceAccountName: coredns
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: k8s-app
                operator: In
                values: ["kube-dns"]
            topologyKey: kubernetes.io/hostname
      containers:
      - name: coredns
        image: registry.k8s.io/coredns/coredns:v1.13.2
        imagePullPolicy: IfNotPresent
        args: ["-conf", "/etc/coredns/Corefile"]
        resources:
          limits:
            memory: 512Mi
          requests:
            cpu: 100m
            memory: 128Mi
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9153
          name: metrics
          protocol: TCP
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /ready
            port: 8181
            scheme: HTTP
          periodSeconds: 2
          timeoutSeconds: 3
        volumeMounts:
        - name: config-volume
          mountPath: /etc/coredns
          readOnly: true
      volumes:
      - name: config-volume
        configMap:
          name: coredns
          items:
          - key: Corefile
            path: Corefile
---
apiVersion: v1
kind: Service
metadata:
  name: kube-dns
  namespace: kube-system
  annotations:
    prometheus.io/port: "9153"
    prometheus.io/scrape: "true"
  labels:
    k8s-app: kube-dns
spec:
  selector:
    k8s-app: kube-dns
  clusterIP: 10.96.0.10
  ports:
  - name: dns
    port: 53
    protocol: UDP
    targetPort: 53
  - name: dns-tcp
    port: 53
    protocol: TCP
    targetPort: 53
  - name: metrics
    port: 9153
    protocol: TCP
    targetPort: 9153
EOF

kubectl apply -f /root/coredns.yaml

kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

# 二进制集群的 kubelet 证书不是集群 CA 签发的场景，加上这个参数
kubectl -n kube-system patch deployment metrics-server --type='json' -p='[
  {"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"},
  {"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname"}
]'

kubectl top nodes
kubectl top pod -A

helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm repo update

helm upgrade --install ingress-nginx ingress-nginx/ingress-nginx \
  -n ingress-nginx --create-namespace \
  --set controller.kind=DaemonSet \
  --set controller.hostNetwork=true \
  --set controller.dnsPolicy=ClusterFirstWithHostNet \
  --set controller.service.type=ClusterIP \
  --set controller.ingressClassResource.default=true \
  --set controller.metrics.enabled=true \
  --set controller.config.proxy-body-size=100m \
  --set controller.config.use-forwarded-headers=true

kubectl -n ingress-nginx get pod -o wide

# m1 上
mkdir -p /data/nfs && chmod 777 /data/nfs
dnf install -y nfs-utils
systemctl enable --now nfs-server

cat > /etc/exports <<'EOF'
/data/nfs 10.0.0.0/24(rw,sync,no_root_squash,no_subtree_check)
EOF
exportfs -arv
showmount -e 10.0.0.11

# n1/n2 上验证
showmount -e 10.0.0.11

helm repo add nfs-subdir-external-provisioner \
  https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/

helm upgrade --install nfs-client nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
  -n kube-system \
  --set nfs.server=10.0.0.11 \
  --set nfs.path=/data/nfs \
  --set storageClass.name=nfs-client \
  --set storageClass.defaultClass=true \
  --set storageClass.reclaimPolicy=Retain \
  --set storageClass.archiveOnDelete=false \
  --set replicaCount=1

kubectl get sc

HELM_VER=v3.17.3
wget -q https://get.helm.sh/helm-${HELM_VER}-linux-amd64.tar.gz
tar -xzf helm-${HELM_VER}-linux-amd64.tar.gz
mv linux-amd64/helm /usr/local/bin/
helm version

for ns in devops harbor git cicd observability logging middleware prod; do
  kubectl create ns $ns --dry-run=client -o yaml | kubectl apply -f -
done

helm repo add harbor https://helm.goharbor.io
helm repo update

cat > /root/harbor-values.yaml <<'EOF'
expose:
  type: ingress
  tls:
    enabled: false
  ingress:
    className: nginx
    hosts:
      core: harbor.apps.lxhhub.top
    annotations:
      nginx.ingress.kubernetes.io/proxy-body-size: "0"
      nginx.ingress.kubernetes.io/proxy-read-timeout: "900"
      nginx.ingress.kubernetes.io/proxy-send-timeout: "900"

externalURL: http://harbor.apps.lxhhub.top
harborAdminPassword: "Harbor@2026!ChangeMe"

persistence:
  enabled: true
  resourcePolicy: "keep"
  persistentVolumeClaim:
    registry:
      storageClass: "nfs-client"
      size: 200Gi
    jobservice:
      jobLog:
        storageClass: "nfs-client"
        size: 10Gi
    database:
      storageClass: "nfs-client"
      size: 10Gi
    redis:
      storageClass: "nfs-client"
      size: 10Gi
    trivy:
      storageClass: "nfs-client"
      size: 10Gi

notary:
  enabled: false
trivy:
  enabled: true
EOF

helm upgrade --install harbor harbor/harbor \
  -n harbor -f /root/harbor-values.yaml --wait --timeout 15m

helm package ./charts/shop && helm push shop-0.1.0.tgz oci://harbor.apps.lxhhub.top/shop/charts

kubectl create secret docker-registry harbor-pull \
  -n prod \
  --docker-server=harbor.apps.lxhhub.top \
  --docker-username='robot$shop-puller' \
  --docker-password='<token>' \
  --dry-run=client -o yaml | kubectl apply -f -

kubectl -n prod patch serviceaccount default \
  -p '{"imagePullSecrets":[{"name":"harbor-pull"}]}'

helm repo add gitea https://dl.gitea.com/charts
helm repo update

cat > /root/gitea-values.yaml <<'EOF'
ingress:
  enabled: true
  className: nginx
  hosts:
  - host: git.apps.lxhhub.top
    paths:
    - path: /
      pathType: Prefix
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: "512m"

persistence:
  enabled: true
  storageClass: nfs-client
  size: 50Gi

gitea:
  admin:
    username: gitea-admin
    password: "Gitea@2026!ChangeMe"
    email: admin@lxhhub.top
  config:
    server:
      ROOT_URL: http://git.apps.lxhhub.top/
      SSH_DOMAIN: git.apps.lxhhub.top
    repository:
      ROOT: /data/git/repositories
    service:
      DISABLE_REGISTRATION: true
    actions:
      ENABLED: true

redis-cluster:
  enabled: false
postgresql:
  enabled: true
  persistence:
    enabled: true
    storageClass: nfs-client
    size: 10Gi
  auth:
    database: gitea
EOF

helm upgrade --install gitea gitea/gitea -n git -f /root/gitea-values.yaml --wait --timeout 10m

cat > /root/nexus.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nexus-data
  namespace: devops
spec:
  accessModes: ["ReadWriteOnce"]
  storageClassName: nfs-client
  resources:
    requests:
      storage: 200Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nexus
  namespace: devops
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nexus
  template:
    metadata:
      labels:
        app: nexus
    spec:
      initContainers:
      - name: fix-perm
        image: busybox:1.37
        command: ["sh", "-c", "chown -R 200:200 /nexus-data"]
        volumeMounts:
        - name: data
          mountPath: /nexus-data
      containers:
      - name: nexus
        image: sonatype/nexus3:3.83.1
        ports:
        - containerPort: 8081
        env:
        - name: INSTALL4J_ADD_VM_PARAMS
          value: "-Xms2g -Xmx2g -XX:MaxDirectMemorySize=2g -Djava.util.prefs.userRoot=/nexus-data/javaprefs"
        readinessProbe:
          httpGet: { path: /service/rest/v1/status, port: 8081 }
          initialDelaySeconds: 90
          periodSeconds: 15
          failureThreshold: 20
        livenessProbe:
          httpGet: { path: /service/rest/v1/status, port: 8081 }
          initialDelaySeconds: 180
          periodSeconds: 30
        resources:
          requests: { cpu: 1, memory: 4Gi }
          limits:   { cpu: 2, memory: 5Gi }
        volumeMounts:
        - name: data
          mountPath: /nexus-data
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: nexus-data
---
apiVersion: v1
kind: Service
metadata:
  name: nexus
  namespace: devops
spec:
  selector: { app: nexus }
  ports:
  - port: 8081
    targetPort: 8081
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nexus
  namespace: devops
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: "1g"
spec:
  ingressClassName: nginx
  rules:
  - host: nexus.apps.lxhhub.top
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: nexus
            port: { number: 8081 }
EOF

kubectl apply -f /root/nexus.yaml

helm repo add jenkins https://charts.jenkins.io
helm repo update

cat > /root/jenkins-values.yaml <<'EOF'
controller:
  image: "jenkins/jenkins"
  tag: "lts-jdk21"
  imagePullPolicy: "IfNotPresent"
  numExecutors: 0          # 构建全丢给 agent pod
  adminUser: "admin"
  adminPassword: "Jenkins@2026!ChangeMe"
  ingress:
    enabled: true
    className: nginx
    hostName: jenkins.apps.lxhhub.top
    annotations:
      nginx.ingress.kubernetes.io/proxy-body-size: "100m"
  additionalPlugins:
  - kubernetes:latest
  - workflow-aggregator:latest
  - git:latest
  - configuration-as-code:latest
  - credentials-binding:latest
  - sonar:latest
  - docker-workflow:latest
  - blueocean:latest
  installLatestPlugins: false
  JCasC:
    enabled: false
  persistence:
    enabled: true
    storageClass: nfs-client
    size: 50Gi
  resources:
    requests: { cpu: 500m, memory: 2Gi }
    limits:   { cpu: 2,    memory: 4Gi }
  serviceType: ClusterIP
  healthProbes:
    livenessFailureThreshold: 12

agent:
  enabled: true
  image: "jenkins/inbound-agent"
  tag: "latest-jdk21"
  resources:
    requests: { cpu: 500m, memory: 1Gi }
    limits:   { cpu: 2,    memory: 3Gi }
EOF

helm upgrade --install jenkins jenkins/jenkins -n cicd -f /root/jenkins-values.yaml --wait --timeout 15m

kubectl -n cicd create pvc maven-repo --storage-class=nfs-client --access-mode=ReadWriteMany

helm repo add sonarqube https://SonarSource.github.io/helm-chart-sonarqube
helm repo update

helm upgrade --install sonarqube sonarqube/sonarqube -n devops \
  --set edition=community \
  --set ingress.enabled=true \
  --set ingress.className=nginx \
  --set ingress.hosts[0].name=sonar.apps.lxhhub.top \
  --set persistence.enabled=true \
  --set persistence.storageClass=nfs-client \
  --set persistence.size=20Gi \
  --set postgresql.persistence.enabled=true \
  --set postgresql.persistence.storageClass=nfs-client \
  --wait --timeout 15m

helm repo add argo https://argoproj.github.io/argo-helm
helm repo update

cat > /root/argocd-values.yaml <<'EOF'
configs:
  params:
    server.insecure: true
  repositories:
    gitea-shop:
      url: http://git.apps.lxhhub.top/gitea-admin/shop-gitops.git

server:
  ingress:
    enabled: true
    ingressClassName: nginx
    hostname: argocd.apps.lxhhub.top
    extraAnnotations:
      nginx.ingress.kubernetes.io/ssl-passthrough: "false"
      nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
  resources:
    requests: { cpu: 250m, memory: 512Mi }

controller:
  resources:
    requests: { cpu: 250m, memory: 1Gi }

repoServer:
  resources:
    requests: { cpu: 250m, memory: 1Gi }
EOF

helm upgrade --install argocd argo/argo-cd -n argocd --create-namespace \
  -f /root/argocd-values.yaml --wait --timeout 10m

# 初始密码
kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' | base64 -d; echo

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

cat > /root/prom-values.yaml <<'EOF'
prometheus:
  prometheusSpec:
    retention: 15d
    retentionSize: 45GiB
    scrapeInterval: 30s
    evaluationInterval: 30s
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: nfs-client
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 100Gi
    resources:
      requests: { cpu: 500m, memory: 2Gi }
      limits:   { cpu: 2,    memory: 6Gi }

grafana:
  enabled: true
  adminPassword: "Grafana@2026!ChangeMe"
  ingress:
    enabled: true
    ingressClassName: nginx
    hosts: ["grafana.apps.lxhhub.top"]
  persistence:
    enabled: true
    storageClassName: nfs-client
    size: 10Gi
  grafana.ini:
    date_formats:
      default_timezone: Asia/Shanghai

alertmanager:
  alertmanagerSpec:
    storage:
      volumeClaimTemplate:
        spec:
          storageClassName: nfs-client
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 10Gi

kubeStateMetrics:
  enabled: true
nodeExporter:
  enabled: true
prometheusOperator:
  enabled: true
EOF

helm upgrade --install kube-prom prometheus-community/kube-prometheus-stack \
  -n observability -f /root/prom-values.yaml --wait --timeout 15m

kubectl -n observability create secret generic etcd-client-certs \
  --from-file=ca.crt=/etc/etcd/ssl/etcd-ca.pem \
  --from-file=tls.crt=/etc/etcd/ssl/etcd-client.pem \
  --from-file=tls.key=/etc/etcd/ssl/etcd-client-key.pem

# /root/prom-etcd.yaml
prometheus:
  prometheusSpec:
    secrets: ["etcd-client-certs"]
    additionalScrapeConfigs:
    - job_name: 'etcd'
      scheme: https
      tls_config:
        ca_file: /etc/prometheus/secrets/etcd-client-certs/ca.crt
        cert_file: /etc/prometheus/secrets/etcd-client-certs/tls.crt
        key_file: /etc/prometheus/secrets/etcd-client-certs/tls.key
      static_configs:
      - targets: ['10.0.0.11:2379','10.0.0.12:2379','10.0.0.13:2379']

helm upgrade kube-prom prometheus-community/kube-prometheus-stack \
  -n observability -f /root/prom-values.yaml -f /root/prom-etcd.yaml

helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

cat > /root/loki-values.yaml <<'EOF'
deploymentMode: SingleBinary
loki:
  auth_enabled: false
  commonConfig:
    replication_factor: 1
  storage:
    type: filesystem
    bucketNames:
      chunks: chunks
      ruler: ruler
      admin: admin
  limits_config:
    retention_period: 15d
  schemaConfig:
    configs:
    - from: "2024-01-01"
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h
  pattern_ingester:
    enabled: false

singleBinary:
  replicas: 1
  persistence:
    enabled: true
    storageClass: nfs-client
    size: 100Gi
  resources:
    requests: { cpu: 500m, memory: 1Gi }
    limits:   { cpu: 2,    memory: 3Gi }

gateway:
  enabled: true
monitoring:
  selfMonitoring:
    enabled: false
    grafanaAgent:
      installOperator: false
  lokiCanary:
    enabled: false
test:
  enabled: false
EOF

helm upgrade --install loki grafana/loki -n logging -f /root/loki-values.yaml --wait --timeout 10m

cat > /root/alloy-values.yaml <<'EOF'
alloy:
  configMap:
    create: true
    content: |
      logging {
        level = "info"
      }

      discovery.kubernetes "pods" {
        role = "pod"
      }

      discovery.relabel "pod_logs" {
        targets = discovery.kubernetes.pods.targets

        rule {
          source_labels = ["__meta_kubernetes_namespace"]
          action        = "replace"
          target_label  = "namespace"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_name"]
          action        = "replace"
          target_label  = "pod"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_container_name"]
          action        = "replace"
          target_label  = "container"
        }
        rule {
          source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
          action        = "replace"
          separator     = "/"
          replacement   = "/var/log/pods/*$1/*.log"
          target_label  = "__path__"
        }
        rule {
          action       = "labelmap"
          regex        = "__meta_kubernetes_pod_label_(.+)"
        }
      }

      loki.source.file "pod_logs" {
        targets    = discovery.relabel.pod_logs.output
        forward_to = [loki.write.default.receiver]
      }

      loki.write "default" {
        endpoint {
          url = "http://loki-gateway.logging.svc.cluster.local/loki/api/v1/push"
        }
      }

  mounts:
    varlog: true

controller:
  type: daemonset
EOF

helm upgrade --install alloy grafana/alloy -n logging -f /root/alloy-values.yaml --wait --timeout 10m

{namespace="prod", app="shop-order"} |= "ERROR"

helm repo add skywalking https://apache.jfrog.io/artifactory/skywalking-helm
helm repo update
# 如果这个源哪天不通了，直接从 GitHub 拉 chart 源码装：
# git clone https://github.com/apache/skywalking-helm && cd skywalking-helm/chart/skywalking

helm upgrade --install skywalking skywalking/skywalking -n observability \
  --set oap.image.tag=10.2.0 \
  --set oap.storageType=elasticsearch \
  --set oap.replicas=1 \
  --set ui.image.tag=10.2.0 \
  --set elasticsearch.enabled=true \
  --set elasticsearch.imageTag=8.15.2 \
  --set elasticsearch.persistence.enabled=true \
  --set elasticsearch.persistence.storageClass=nfs-client \
  --wait --timeout 20m

JDK             21
Spring Boot     3.5.x
Spring Cloud    2025.0.x
Spring Cloud Alibaba  2025.0.0.0
Nacos Server/Client   3.2.4

<dependencyManagement>
  <dependencies>
    <dependency>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-dependencies</artifactId>
      <version>3.5.6</version>
      <type>pom</type><scope>import</scope>
    </dependency>
    <dependency>
      <groupId>org.springframework.cloud</groupId>
      <artifactId>spring-cloud-dependencies</artifactId>
      <version>2025.0.2</version>
      <type>pom</type><scope>import</scope>
    </dependency>
    <dependency>
      <groupId>com.alibaba.cloud</groupId>
      <artifactId>spring-cloud-alibaba-dependencies</artifactId>
      <version>2025.0.0.0</version>
      <type>pom</type><scope>import</scope>
    </dependency>
  </dependencies>
</dependencyManagement>

# /root/mysql.yaml
apiVersion: v1
kind: Secret
metadata:
  name: mysql-secret
  namespace: middleware
type: Opaque
stringData:
  root-password: "Mysql@2026!ChangeMe"
---
apiVersion: v1
kind: Service
metadata:
  name: mysql
  namespace: middleware
spec:
  clusterIP: None
  selector: { app: mysql }
  ports:
  - port: 3306
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mysql
  namespace: middleware
spec:
  serviceName: mysql
  replicas: 1
  selector:
    matchLabels: { app: mysql }
  template:
    metadata:
      labels: { app: mysql }
    spec:
      containers:
      - name: mysql
        image: mysql:8.4
        env:
        - name: MYSQL_ROOT_PASSWORD
          valueFrom:
            secretKeyRef: { name: mysql-secret, key: root-password }
        - name: TZ
          value: Asia/Shanghai
        args:
        - --character-set-server=utf8mb4
        - --collation-server=utf8mb4_unicode_ci
        - --default-time-zone=+08:00
        - --max_connections=1000
        - --innodb_buffer_pool_size=2G
        ports:
        - containerPort: 3306
        readinessProbe:
          exec:
            command: ["mysqladmin", "ping", "-h", "127.0.0.1", "-p$(MYSQL_ROOT_PASSWORD)"]
          initialDelaySeconds: 30
          periodSeconds: 10
        resources:
          requests: { cpu: 1, memory: 4Gi }
          limits:   { cpu: 2, memory: 6Gi }
        volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: nfs-client
      resources:
        requests:
          storage: 100Gi

kubectl -n middleware exec -it mysql-0 -- mysql -uroot -p'Mysql@2026!ChangeMe' -e "
CREATE DATABASE IF NOT EXISTS nacos_config DEFAULT CHARSET utf8mb4;
CREATE DATABASE IF NOT EXISTS shop DEFAULT CHARSET utf8mb4;
"
# Nacos 建表语句从这个地址拿，对应版本 3.2.4
# https://github.com/alibaba/nacos/blob/3.2.4/distribution/conf/mysql-schema.sql
kubectl -n middleware exec -i mysql-0 -- mysql -uroot -p'Mysql@2026!ChangeMe' nacos_config < mysql-schema.sql

# /root/redis.yaml
apiVersion: v1
kind: Secret
metadata:
  name: redis-secret
  namespace: middleware
type: Opaque
stringData:
  password: "Redis@2026!ChangeMe"
---
apiVersion: v1
kind: Service
metadata:
  name: redis
  namespace: middleware
spec:
  selector: { app: redis }
  ports:
  - port: 6379
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: middleware
spec:
  serviceName: redis
  replicas: 1
  selector:
    matchLabels: { app: redis }
  template:
    metadata:
      labels: { app: redis }
    spec:
      containers:
      - name: redis
        image: redis:7.4-alpine
        command:
        - sh
        - -c
        - redis-server --requirepass $(REDIS_PASSWORD) --appendonly yes --maxmemory 2gb --maxmemory-policy allkeys-lru
        env:
        - name: REDIS_PASSWORD
          valueFrom:
            secretKeyRef: { name: redis-secret, key: password }
        ports:
        - containerPort: 6379
        readinessProbe:
          exec:
            command: ["redis-cli", "-a", "$(REDIS_PASSWORD)", "ping"]
          initialDelaySeconds: 15
          periodSeconds: 10
        resources:
          requests: { cpu: 200m, memory: 512Mi }
          limits:   { cpu: 1,    memory: 3Gi }
        volumeMounts:
        - name: data
          mountPath: /data
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: nfs-client
      resources:
        requests:
          storage: 20Gi

# /root/nacos.yaml
apiVersion: v1
kind: Secret
metadata:
  name: nacos-secret
  namespace: middleware
type: Opaque
stringData:
  mysql-password: "Mysql@2026!ChangeMe"
  auth-token: "NacosAuthTokenSecretKeyMustBeLongerThan32Chars2026"
  identity-key: "serverIdentity"
  identity-value: "security"
---
apiVersion: v1
kind: Service
metadata:
  name: nacos-headless
  namespace: middleware
spec:
  clusterIP: None
  selector: { app: nacos }
  ports:
  - { name: http,   port: 8848, targetPort: 8848 }
  - { name: grpc,   port: 9848, targetPort: 9848 }
  - { name: jraft,  port: 9849, targetPort: 9849 }
  - { name: console, port: 8080, targetPort: 8080 }
---
apiVersion: v1
kind: Service
metadata:
  name: nacos
  namespace: middleware
spec:
  selector: { app: nacos }
  ports:
  - { name: http,   port: 8848, targetPort: 8848 }
  - { name: grpc,   port: 9848, targetPort: 9848 }
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: nacos
  namespace: middleware
spec:
  serviceName: nacos-headless
  replicas: 3
  podManagementPolicy: Parallel
  selector:
    matchLabels: { app: nacos }
  template:
    metadata:
      labels: { app: nacos }
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              topologyKey: kubernetes.io/hostname
              labelSelector:
                matchExpressions:
                - { key: app, operator: In, values: ["nacos"] }
      containers:
      - name: nacos
        image: nacos/nacos-server:3.2.4
        env:
        - name: MODE
          value: cluster
        - name: NACOS_SERVERS
          value: "nacos-0.nacos-headless.middleware.svc.cluster.local:8848 nacos-1.nacos-headless.middleware.svc.cluster.local:8848 nacos-2.nacos-headless.middleware.svc.cluster.local:8848"
        - name: SPRING_DATASOURCE_PLATFORM
          value: mysql
        - name: MYSQL_SERVICE_HOST
          value: mysql.middleware.svc.cluster.local
        - name: MYSQL_SERVICE_PORT
          value: "3306"
        - name: MYSQL_SERVICE_DB_NAME
          value: nacos_config
        - name: MYSQL_SERVICE_USER
          value: root
        - name: MYSQL_SERVICE_PASSWORD
          valueFrom:
            secretKeyRef: { name: nacos-secret, key: mysql-password }
        - name: MYSQL_SERVICE_DB_PARAM
          value: "characterEncoding=utf8&connectTimeout=10000&socketTimeout=30000&allowPublicKeyRetrieval=true&useSSL=false"
        - name: NACOS_AUTH_ENABLE
          value: "true"
        - name: NACOS_AUTH_TOKEN
          valueFrom:
            secretKeyRef: { name: nacos-secret, key: auth-token }
        - name: NACOS_AUTH_IDENTITY_KEY
          valueFrom:
            secretKeyRef: { name: nacos-secret, key: identity-key }
        - name: NACOS_AUTH_IDENTITY_VALUE
          valueFrom:
            secretKeyRef: { name: nacos-secret, key: identity-value }
        - name: JVM_XMS
          value: 1g
        - name: JVM_XMX
          value: 1g
        - name: JVM_XMN
          value: 512m
        ports:
        - { containerPort: 8848, name: http }
        - { containerPort: 9848, name: grpc }
        - { containerPort: 9849, name: jraft }
        - { containerPort: 8080, name: console }
        readinessProbe:
          httpGet: { path: /nacos/v3/admin/core/state/health, port: 8080 }
          initialDelaySeconds: 40
          periodSeconds: 10
          failureThreshold: 15
        livenessProbe:
          httpGet: { path: /nacos/v3/admin/core/state/health, port: 8080 }
          initialDelaySeconds: 90
          periodSeconds: 20
          failureThreshold: 10
        resources:
          requests: { cpu: 500m, memory: 2Gi }
          limits:   { cpu: 2,    memory: 3Gi }
        volumeMounts:
        - name: data
          mountPath: /home/nacos/data
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: nfs-client
      resources:
        requests:
          storage: 20Gi

kubectl apply --dry-run=client -f nacos.yaml

# Dockerfile
FROM eclipse-temurin:21-jre-alpine AS extract
WORKDIR /build
ARG JAR_FILE=target/*.jar
COPY ${JAR_FILE} app.jar
RUN java -Djarmode=layertools -jar app.jar extract

FROM eclipse-temurin:21-jre-alpine
RUN apk add --no-cache tzdata curl \
 && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
 && echo "Asia/Shanghai" > /etc/timezone
WORKDIR /app
COPY --from=extract /build/dependencies/ ./
COPY --from=extract /build/spring-boot-loader/ ./
COPY --from=extract /build/snapshot-dependencies/ ./
COPY --from=extract /build/application/ ./
ENV JAVA_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=70.0 \
-XX:+UseG1GC -XX:MaxGCPauseMillis=200 \
-XX:+ExitOnOutOfMemoryError \
-Djava.security.egd=file:/dev/./urandom \
-Dfile.encoding=UTF-8"
EXPOSE 8080
ENTRYPOINT ["sh","-c","exec java $JAVA_OPTS org.springframework.boot.loader.launch.JarLauncher"]

FROM eclipse-temurin:21-jre-alpine
RUN apk add --no-cache tzdata curl \
 && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
WORKDIR /app
COPY target/*.jar app.jar
ENV JAVA_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=70.0 -XX:+UseG1GC -XX:+ExitOnOutOfMemoryError -Dfile.encoding=UTF-8"
EXPOSE 8080
ENTRYPOINT ["sh","-c","exec java $JAVA_OPTS -jar /app/app.jar"]

# shop-order.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: shop-common-env
  namespace: prod
data:
  SPRING_PROFILES_ACTIVE: "prod"
  SPRING_CLOUD_NACOS_SERVER_ADDR: "nacos.middleware.svc.cluster.local:8848"
  SPRING_CLOUD_NACOS_DISCOVERY_NAMESPACE: "prod"
  SPRING_CLOUD_NACOS_CONFIG_NAMESPACE: "prod"
  SPRING_CLOUD_NACOS_USERNAME: "nacos"
  SPRING_DATASOURCE_URL: "jdbc:mysql://mysql.middleware.svc.cluster.local:3306/shop?useUnicode=true&characterEncoding=utf8&serverTimezone=Asia/Shanghai&useSSL=false"
  SPRING_DATA_REDIS_HOST: "redis.middleware.svc.cluster.local"
  SPRING_DATA_REDIS_PORT: "6379"
  SERVER_SHUTDOWN: "graceful"
  SPRING_LIFECYCLE_TIMEOUT_PER_SHUTDOWN_PHASE: "30s"
  MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: "health,info,prometheus"
  MANAGEMENT_ENDPOINT_HEALTH_PROBES_ENABLED: "true"
---
apiVersion: v1
kind: Secret
metadata:
  name: shop-secret
  namespace: prod
type: Opaque
stringData:
  SPRING_DATASOURCE_USERNAME: "root"
  SPRING_DATASOURCE_PASSWORD: "Mysql@2026!ChangeMe"
  SPRING_DATA_REDIS_PASSWORD: "Redis@2026!ChangeMe"
  SPRING_CLOUD_NACOS_PASSWORD: "Nacos@2026!ChangeMe"
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: shop-order
  namespace: prod
spec:
  replicas: 2
  revisionHistoryLimit: 5
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels: { app: shop-order }
  template:
    metadata:
      labels:
        app: shop-order
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/path: /actuator/prometheus
        prometheus.io/port: "8080"
    spec:
      terminationGracePeriodSeconds: 60
      imagePullSecrets:
      - name: harbor-pull
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: DoNotSchedule
        labelSelector:
          matchLabels: { app: shop-order }
      containers:
      - name: shop-order
        image: harbor.apps.lxhhub.top/shop/shop-order:20260909-a1b2c3d
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
        envFrom:
        - configMapRef: { name: shop-common-env }
        - secretRef: { name: shop-secret }
        env:
        - name: POD_IP
          valueFrom:
            fieldRef: { fieldPath: status.podIP }
        - name: SPRING_CLOUD_NACOS_DISCOVERY_IP
          value: $(POD_IP)
        startupProbe:
          httpGet: { path: /actuator/health/liveness, port: 8080 }
          failureThreshold: 60
          periodSeconds: 5
        readinessProbe:
          httpGet: { path: /actuator/health/readiness, port: 8080 }
          periodSeconds: 5
          failureThreshold: 3
        livenessProbe:
          httpGet: { path: /actuator/health/liveness, port: 8080 }
          periodSeconds: 15
          failureThreshold: 3
        lifecycle:
          preStop:
            exec:
              command: ["sh", "-c", "sleep 25"]
        resources:
          requests: { cpu: 500m, memory: 1Gi }
          limits:   { cpu: 2,    memory: 2Gi }
---
apiVersion: v1
kind: Service
metadata:
  name: shop-order
  namespace: prod
spec:
  selector: { app: shop-order }
  ports:
  - port: 8080
    targetPort: 8080
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: shop-order
  namespace: prod
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: shop-order
  minReplicas: 2
  maxReplicas: 8
  metrics:
  - type: Resource
    resource:
      name: cpu
      target: { type: Utilization, averageUtilization: 70 }
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: shop-order
  namespace: prod
spec:
  minAvailable: 1
  selector:
    matchLabels: { app: shop-order }

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: shop-gateway
  namespace: prod
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: "20m"
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "10"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "120"
spec:
  ingressClassName: nginx
  rules:
  - host: shop.apps.lxhhub.top
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: shop-gateway
            port: { number: 8080 }

// Jenkinsfile
pipeline {
  agent {
    kubernetes {
      cloud 'kubernetes'
      defaultContainer 'maven'
      yaml '''
apiVersion: v1
kind: Pod
spec:
  serviceAccountName: jenkins
  containers:
  - name: maven
    image: maven:3.9.9-eclipse-temurin-21
    command: ["cat"]
    tty: true
    volumeMounts:
    - name: m2
      mountPath: /root/.m2
    - name: settings
      mountPath: /root/.m2/settings.xml
      subPath: settings.xml
  - name: kaniko
    image: gcr.io/kaniko-project/executor:debug
    command: ["cat"]
    tty: true
    volumeMounts:
    - name: docker-config
      mountPath: /kaniko/.docker
  volumes:
  - name: m2
    persistentVolumeClaim:
      claimName: maven-repo
  - name: settings
    configMap:
      name: maven-settings
  - name: docker-config
    configMap:
      name: docker-config
'''
    }
  }

  environment {
    HARBOR   = 'harbor.apps.lxhhub.top'
    NAMESPACE= 'shop'
    SERVICES = 'shop-gateway shop-auth shop-user shop-product shop-order'
  }

  stages {
    stage('Checkout') {
      steps {
        checkout scm
        script {
          env.GIT_SHA = sh(script: 'git rev-parse --short=7 HEAD', returnStdout: true).trim()
          env.IMAGE_TAG = "${BUILD_NUMBER}-${GIT_SHA}"
        }
        echo "本次构建标签: ${IMAGE_TAG}"
      }
    }

    stage('Unit Test') {
      steps {
        container('maven') {
          sh 'mvn -B -ntp clean test'
        }
      }
      post {
        always {
          junit allowEmptyResults: true, testResults: '**/target/surefire-reports/*.xml'
        }
      }
    }

    stage('Package') {
      steps {
        container('maven') {
          sh 'mvn -B -ntp package -DskipTests'
        }
      }
    }

    stage('SonarQube') {
      when { expression { return false } }   // 装了 Sonar 就改成 true
      steps {
        container('maven') {
          withSonarQubeEnv('sonarqube') {
            sh 'mvn -B -ntp sonar:sonar'
          }
        }
      }
    }

    stage('Build & Push Images') {
      steps {
        container('kaniko') {
          script {
            def jobs = [:]
            env.SERVICES.split(' ').each { svc ->
              jobs[svc] = {
                sh """
                  /kaniko/executor \
                    --context ${WORKSPACE}/${svc} \
                    --dockerfile ${WORKSPACE}/${svc}/Dockerfile \
                    --destination ${HARBOR}/${NAMESPACE}/${svc}:${IMAGE_TAG} \
                    --destination ${HARBOR}/${NAMESPACE}/${svc}:latest \
                    --insecure --insecure-pull --skip-tls-verify \
                    --cache=true --cache-repo ${HARBOR}/${NAMESPACE}/kaniko-cache \
                    --snapshot-mode=redo \
                    --reproducible
                """
              }
            }
            parallel jobs
          }
        }
      }
    }

    stage('Update GitOps Repo') {
      steps {
        container('maven') {
          withCredentials([usernamePassword(credentialsId: 'gitea-cred',
                           usernameVariable: 'GIT_USER', passwordVariable: 'GIT_PASS')]) {
            sh '''
              git clone http://${GIT_USER}:${GIT_PASS}@git.apps.lxhhub.top/gitea-admin/shop-gitops.git
              cd shop-gitops
              cd overlays/prod
              for svc in ${SERVICES}; do
                kustomize edit set image ${HARBOR}/${NAMESPACE}/${svc}:${IMAGE_TAG}
              done
              git config user.email "jenkins@lxhhub.top"
              git config user.name  "jenkins"
              git commit -am "chore: bump image to ${IMAGE_TAG}" || echo "没有变化"
              git push origin main
            '''
          }
        }
      }
    }
  }

  post {
    success { echo "发布已提交给 Argo CD，镜像标签 ${IMAGE_TAG}" }
    failure { echo "构建失败，集群未发生变更" }
  }
}

shop-gitops/
├── base/
│   ├── kustomization.yaml
│   ├── configmap.yaml
│   ├── secret.yaml
│   ├── shop-gateway.yaml
│   └── ...
└── overlays/
    ├── prod/
    │   ├── kustomization.yaml     # kustomize edit set image 改的就是这里
    │   └── replica-patch.yaml
    └── test/
        └── kustomization.yaml

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: shop-prod
  namespace: argocd
spec:
  project: default
  source:
    repoURL: http://git.apps.lxhhub.top/gitea-admin/shop-gitops.git
    targetRevision: main
    path: overlays/prod
  destination:
    server: https://kubernetes.default.svc
    namespace: prod
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
    - CreateNamespace=true
    - ApplyOutOfSyncOnly=true
    retry:
      limit: 5
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m

# 方式一：Argo CD 回滚到上一版
argocd app history shop-prod
argocd app rollback shop-prod <REVISION>

# 方式二：Git revert，更推荐，因为 Git 才是事实来源
git revert HEAD && git push

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: shop-gateway-canary
  namespace: prod
  annotations:
    nginx.ingress.kubernetes.io/canary: "true"
    nginx.ingress.kubernetes.io/canary-weight: "10"
spec:
  ingressClassName: nginx
  rules:
  - host: shop.apps.lxhhub.top
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: shop-gateway-canary
            port: { number: 8080 }

kubectl get nodes -o wide                      # 5 台 Ready
kubectl get pod -A                             # 无 Pending / CrashLoopBackOff
kubectl get --raw='/readyz?verbose' | tail -20
etcdctl-prod endpoint health --write-out=table # 3 个 true
kubectl -n kube-system logs -l k8s-app=kube-dns --tail=20
kubectl top nodes                              # metrics-server 正常

# 停掉持有 VIP 的 master 上的 apiserver，VIP 应漂走，kubectl 仍可用
systemctl stop kube-apiserver
kubectl get nodes

# 停掉一个 master 整机，etcd 应仍可写（3 节点坏 1 个仍然 quorum）
kubectl create sa test-ha && kubectl delete sa test-ha

# 恢复后 etcd 成员应自动追平
etcdctl-prod endpoint status --write-out=table

kubectl run dns-test --image=busybox:1.37 -it --rm --restart=Never -- \
  nslookup kubernetes.default.svc.cluster.local
kubectl run dns-test2 --image=busybox:1.37 -it --rm --restart=Never -- \
  nslookup shop-order.prod.svc.cluster.local

# Pod 之间跨节点互通
kubectl -n prod exec -it <pod1> -- curl -s http://shop-order:8080/actuator/health

kubectl -n prod get pod -o wide
kubectl -n prod get hpa
curl -H "Host: shop.apps.lxhhub.top" http://10.0.0.14/actuator/health

# 压测触发 HPA
kubectl run -i --tty load --rm --image=busybox:1.37 --restart=Never -- \
  /bin/sh -c "while true; do wget -q -O- http://shop-order.prod:8080/api/order/list; done"
kubectl -n prod get hpa -w

etcdctl-prod snapshot save /backup/etcd-$(date +%Y%m%d).db
etcdctl-prod snapshot status /backup/etcd-$(date +%Y%m%d).db --write-out=table

# 加个定时
cat > /etc/systemd/system/etcd-backup.service <<'EOF'
[Unit]
Description=etcd snapshot backup
[Service]
Type=oneshot
ExecStart=/bin/bash -c 'etcdctl-prod snapshot save /backup/etcd-$(date +%%Y%%m%%d-%%H%%M).db'
EOF
cat > /etc/systemd/system/etcd-backup.timer <<'EOF'
[Unit]
Description=Daily etcd backup
[Timer]
OnCalendar=*-*-* 02:30:00
Persistent=true
[Install]
WantedBy=timers.target
EOF
systemctl enable --now etcd-backup.timer