SealOS 面试题

# 传统方式（10+ 步骤）
kubeadm init --pod-network-cidr=10.244.0.0/16
kubectl apply -f calico.yaml

# SealOS
sealos run labring/kubernetes:v1.28.0 labring/calico:v3.26.0 \
  --masters 192.168.1.10 \
  --nodes 192.168.1.11,192.168.1.12

# Clusterfile
apiVersion: sealos.io/v1beta1
kind: Cluster
metadata:
  name: mycluster
spec:
  hosts:
  - ips:
    - 192.168.1.10
    - 192.168.1.11
    - 192.168.1.12
    roles: [master]
  - ips:
    - 192.168.1.13
    - 192.168.1.14
    roles: [node]
  image: labring/kubernetes:v1.28.0
  ssh:
    user: root
    passwd: your-password  # 或使用私钥
    port: 22

graph TD
    subgraph Master["3 个 Master 节点"]
        M1["Master-1 (192.168.1.10) + kube-apiserver"]
        M2["Master-2 (192.168.1.11) + kube-apiserver"]
        M3["Master-3 (192.168.1.12) + kube-apiserver"]
    end
    LV["lvscare (VIP: 10.103.0.1:6443)"] -->|"LVS 负载均衡"| M1
    LV -->|"LVS 负载均衡"| M2
    LV -->|"LVS 负载均衡"| M3
    Node["Node 节点 kubelet"] -->|"连接 VIP"| LV

sealos run labring/kubernetes:v1.28.0 \
  --masters 192.168.1.10,192.168.1.11,192.168.1.12 \
  --nodes 192.168.1.13,192.168.1.14

# 停止一个 Master 的 API Server
systemctl stop kube-apiserver

# 集群仍正常
kubectl get nodes  # 仍然可用

# lvscare 自动摘除故障 API Server

# 查看可用镜像
sealos images

# Kubernetes 基础镜像
labring/kubernetes:v1.28.0       # K8s 核心组件
labring/calico:v3.26.0           # CNI
labring/helm:v3.13.0             # Helm
labring/ingress-nginx:v1.9.0     # Ingress

graph TD
    Root["labring/kubernetes:v1.28.0"]
    Root --> Reg["registry/（内置镜像仓库内容）"]
    Reg --> APIS["kube-apiserver"]
    Reg --> CM["kube-controller-manager"]
    Reg --> Sched["kube-scheduler"]
    Reg --> Proxy["kube-proxy"]
    Reg --> Etcd["etcd"]
    Reg --> DNS["coredns"]
    Reg --> Pause["pause"]
    Root --> Img["images/（镜像元数据）"]
    Img --> Shim["shim"]
    Root --> KF["Kubefile（镜像构建文件，类比 Dockerfile）"]
    Root --> Mani["manifests/（初始化清单）"]
    Mani --> Init["init.yaml"]

# Kubefile
FROM labring/kubernetes:v1.28.0

# 添加 CNI 插件
COPY images/calico images/

# 添加用户定义的 DaemonSet
COPY manifests/ manifests/

# 添加应用
CMD ["kubectl apply -f manifests/"]

sealos build -t labring/mycluster:v1.0 -f Kubefile .

# 添加 Master 节点
sealos add --masters 192.168.1.15

# 添加 Node 节点
sealos add --nodes 192.168.1.16,192.168.1.17

# 删除 Master 节点
sealos delete --masters 192.168.1.15

# 删除 Node 节点
sealos delete --nodes 192.168.1.16

graph TD
    S1["1. SealOS CLI 连接到新节点"] --> S2["2. 分发 Kubernetes 二进制和配置文件"]
    S2 --> S3["3. 分发集群证书（kubelet、kube-proxy）"]
    S3 --> S4["4. 启动 kubelet 服务"]
    S4 --> S5["5. 节点加入集群"]
    S5 --> S6["6. 更新 lvscare 负载均衡配置"]

# 查看当前版本
kubectl version

# 升级到指定版本
sealos upgrade --image labring/kubernetes:v1.29.0

# 指定升级策略
sealos upgrade --image labring/kubernetes:v1.29.0 \
  --parallel 2 \                    # 并行升级节点数
  --force                           # 强制升级

graph TD
    S1["1. 拉取新版本镜像"] --> S2["2. 升级 Master 节点（逐个）"]
    S2 --> S2a["排空（drain）节点"]
    S2a --> S2b["升级 kubelet/kubeadm"]
    S2b --> S2c["升级 API Server 等控制面组件"]
    S2c --> S2d["恢复调度（uncordon）"]
    S2d --> S3["3. 升级 Node 节点（并行，可配置）"]
    S3 --> S4["4. 更新集群版本状态"]

# 备份集群（包含 etcd）
sealos backup --name my-backup

# 指定备份存储
sealos backup --name my-backup \
  --path /data/backup/k8s

# 从备份恢复
sealos restore --name my-backup

# 从指定路径恢复
sealos restore --name my-backup --path /data/backup/k8s

# 安装 MySQL 集群
sealos run labring/mysql-cluster:8.0

# 安装 Redis 集群
sealos run labring/redis-cluster:7.0

# 安装 Prometheus 监控栈
sealos run labring/kube-prometheus-stack:v0.70.0

# Kubefile
FROM scratch
COPY manifests manifests/
COPY registry registry/
CMD ["kubectl apply -f manifests/"]
# manifests/mysql-cluster.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: mysql-cluster
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mysql
  namespace: mysql-cluster
spec:
  replicas: 3
  serviceName: mysql
  template:
    spec:
      containers:
      - name: mysql
        image: mysql:8.0

# Clusterfile SSH 配置
spec:
  ssh:
    user: root
    # 方式 1：密码
    passwd: "your-password"
    # 方式 2：私钥
    pk: /root/.ssh/id_rsa
    pkPasswd: "key-password"  # 密钥密码（可选）
    port: 22

# 指定密码，SealOS 自动配置所有节点 SSH 互信
sealos run labring/kubernetes:v1.28.0 \
  --masters 192.168.1.10 \
  --nodes 192.168.1.11,192.168.1.12 \
  --user root \
  --passwd 'your-password'

apiVersion: sealos.io/v1beta1
kind: Cluster
metadata:
  name: prod-cluster
spec:
  hosts:
  - ips: [192.168.1.10, 192.168.1.11, 192.168.1.12]
    roles: [master]
  - ips: [192.168.1.13, 192.168.1.14, 192.168.1.15]
    roles: [node]
  image: labring/kubernetes:v1.28.0
  ssh:
    user: root
    passwd: "password"
    port: 22
apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: kubeadm
spec:
  path: etc/init.sh
  data: |
    # 自定义初始化脚本
    sysctl -w net.bridge.bridge-nf-call-iptables=1
    sysctl -w net.ipv4.ip_forward=1    
apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: calico
spec:
  path: manifests/calico-custom.yaml
  data: |
    # 自定义 CNI 配置    

# SealOS 安装后查看组件
kubectl -n kube-system get pods
# kube-apiserver          # 静态 Pod
# kube-controller-manager # 静态 Pod
# kube-scheduler          # 静态 Pod
# etcd                    # 静态 Pod
# coredns                 # Deployment

# 阿里云 ECS 一键创建集群
sealos run labring/kubernetes:v1.28.0 \
  --masters 192.168.1.10,192.168.1.11 \
  --nodes 192.168.1.12,192.168.1.13 \
  --provider aliyun

# AWS EC2
sealos run labring/kubernetes:v1.28.0 \
  --masters 10.0.1.10,10.0.1.11 \
  --nodes 10.0.1.12,10.0.1.13 \
  --provider aws

# 查看 etcd 成员
kubectl -n kube-system exec etcd-master-0 -- etcdctl member list

# backup etcd（通过 sealos backup 已包含）
sealos backup

# 手动 etcd 快照
kubectl -n kube-system exec etcd-master-0 -- \
  etcdctl snapshot save /var/lib/etcd/snapshot.db

# 仓库地址（默认）
# registry.{cluster-name}.svc.cluster.local:5000
# 或本地端口 5000

# 推送自定义镜像
docker tag myapp:latest localhost:5000/myapp:latest
docker push localhost:5000/myapp:latest

# 节点上 image-cri-shim 自动将仓库地址替换为本地仓库
# 加速镜像拉取，减少外网带宽

graph LR
    Pod["Pod 创建"] --> Kubelet["kubelet 拉取镜像"]
    Kubelet --> Shim["image-cri-shim"]
    Shim --> Check{"检查镜像是否在本地仓库"}
    Check -->|"存在"| Local["直接拉取本地（内网）"]
    Check -->|"不存在"| Remote["从外网拉取"]
    Remote --> Sync["同步到本地仓库"]

# 查看证书信息
openssl x509 -in /etc/kubernetes/pkg/ca.crt -text -noout

# 证书列表
# /etc/kubernetes/pki/
#   apiserver.crt / apiserver.key
#   apiserver-kubelet-client.crt / apiserver-kubelet-client.key
#   ca.crt / ca.key
#   front-proxy-ca.crt / front-proxy-ca.key
#   sa.key / sa.pub
#   etcd/ca.crt / etcd/server.crt

# 管理多个集群的配置文件
# - ~/.sealos/cluster/prod-cluster.yaml
# - ~/.sealos/cluster/staging-cluster.yaml
# - ~/.sealos/cluster/dev-cluster.yaml

# 切换到不同集群
export SEALOS_CLUSTER=staging-cluster
sealos status

# 或使用 kubeconfig
export KUBECONFIG=/root/.sealos/staging-cluster/etc/kubeconfig

# 在联网环境拉取镜像
sealos pull labring/kubernetes:v1.28.0
sealos pull labring/calico:v3.26.0

# 保存为 tar 包
sealos save -o k8s-offline.tar labring/kubernetes:v1.28.0
sealos save -o calico-offline.tar labring/calico:v3.26.0

# 在离线环境导入
sealos load -i k8s-offline.tar
sealos load -i calico-offline.tar

# 部署离线集群
sealos run labring/kubernetes:v1.28.0 labring/calico:v3.26.0 \
  --masters 192.168.1.10 \
  --nodes 192.168.1.11,192.168.1.12

# 集群诊断
sealos debug

# 收集诊断日志
sealos debug --output /tmp/cluster-debug.tar.gz

# 节点健康检查
sealos status

# 查看节点状态
sealos nodes ls

graph TD
    Root["labring/kubernetes:v1.28.0"]
    Root --> L0["Layer 0: Base OS 依赖（libseccomp、socat、conntrack 等）"]
    Root --> L1["Layer 1: CRI 运行时（containerd 二进制 + systemd 配置）"]
    Root --> L2["Layer 2: K8s 组件（kubelet、kubeadm、kubectl 二进制）"]
    Root --> L3["Layer 3: 容器镜像（kube-apiserver、etcd、coredns 等）"]
    Root --> L4["Layer 4: 集群配置（Clusterfile 模板、kubeadm 配置）"]
    Root --> L5["Layer 5: 初始化脚本与管理工具"]
    Root --> LN["Layer N: 自定义扩展（CNI、Ingress、监控等）"]

# Kubefile 类比 Dockerfile，集群镜像构建文件
FROM labring/kubernetes:v1.28.0

# COPY 指令添加文件到镜像层
COPY ./runtimes/containerd /usr/local/bin/
COPY ./cni/custom-calico ./images/

# RUN 在构建时执行命令
RUN wget https://github.com/containerd/containerd/releases/download/v1.7.0/containerd-1.7.0-linux-amd64.tar.gz

# CMD 集群部署完成后执行的命令
CMD ["kubectl apply -f manifests/custom-resources.yaml"]

# 构建镜像
sealos build -t myrepo/kubernetes:v1.28.0-custom -f Kubefile .

# 推送到远程仓库
sealos push myrepo/kubernetes:v1.28.0-custom

# 从远程仓库拉取
sealos pull myrepo/kubernetes:v1.28.0-custom

# Clusterfile 中指定运行时
apiVersion: sealos.io/v1beta1
kind: Cluster
metadata:
  name: containerd-cluster
spec:
  hosts:
  - ips: [192.168.1.10, 192.168.1.11]
    roles: [master]
  - ips: [192.168.1.12, 192.168.1.13]
    roles: [node]
  image: labring/kubernetes:v1.28.0
apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: containerd-config
spec:
  path: etc/containerd-config.toml
  data: |
    version = 2
    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
      runtime_type = "io.containerd.runc.v2"
    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
      SystemdCgroup = true
    [plugins."io.containerd.grpc.v1.cri"]
      sandbox_image = "registry.k8s.io/pause:3.9"    

apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: cri-o-runtime
spec:
  path: etc/crio-setup.sh
  data: |
    #!/bin/bash
    # 安装 cri-o
    OS=xUbuntu_22.04
    CRIO_VERSION=1.28
    echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /" \
      > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
    echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /" \
      > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
    apt-get update && apt-get install -y cri-o cri-o-runc
    systemctl enable crio --now    

apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: docker-runtime
spec:
  path: etc/kubeadm-init.sh
  data: |
    # 在 kubeadm 初始化前安装 cri-dockerd
    wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.4/cri-dockerd_0.3.4.3-0.ubuntu-jammy_amd64.deb
    dpkg -i cri-dockerd_0.3.4.3-0.ubuntu-jammy_amd64.deb
    # 修改 kubeadm 配置使用 cri-dockerd socket
    sed -i 's|/run/containerd/containerd.sock|/run/cri-dockerd.sock|' /etc/kubeadm.yml    

# Calico
sealos run labring/kubernetes:v1.28.0 labring/calico:v3.26.0 \
  --masters 192.168.1.10,192.168.1.11 \
  --nodes 192.168.1.12,192.168.1.13

# Cilium
sealos run labring/kubernetes:v1.28.0 labring/cilium:v1.14.0 \
  --masters 192.168.1.10 \
  --nodes 192.168.1.11,192.168.1.12

apiVersion: sealos.io/v1beta1
kind: Cluster
metadata:
  name: cilium-cluster
spec:
  hosts:
  - ips: [192.168.1.10, 192.168.1.11]
    roles: [master]
  - ips: [192.168.1.12, 192.168.1.13]
    roles: [node]
  image: labring/kubernetes:v1.28.0
apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: cilium-custom
spec:
  strategy: merge
  path: manifests/cilium-values.yaml
  data: |
    # 在集群初始化前应用自定义 CNI
    apiVersion: v1
    kind: ConfigMap
    metadata:
      name: cilium-config
    data:
      enable-ipv4: "true"
      enable-bpf-masquerade: "true"
      enable-bandwidth-manager: "true"
      kube-proxy-replacement: "strict"
      ipam-mode: "cluster-pool"
      cluster-pool-ipv4-cidr: "10.42.0.0/16"    

# Kubefile
FROM labring/kubernetes:v1.28.0

# 添加 Cilium 镜像到内置仓库
COPY images/cilium-operator.tar ./registry/
COPY images/cilium-agent.tar ./registry/

# 添加 Cilium Helm Chart
COPY charts/cilium ./charts/

# 部署 Cilium
CMD ["helm install cilium ./charts/cilium \
  --namespace kube-system \
  --set ipam.mode=kubernetes \
  --set kubeProxyReplacement=strict \
  --set hubble.relay.enabled=true \
  --set hubble.ui.enabled=true"]

sealos build -t myrepo/kubernetes-cilium:v1.28.0 -f Kubefile .
sealos run myrepo/kubernetes-cilium:v1.28.0 \
  --masters 192.168.1.10,192.168.1.11 \
  --nodes 192.168.1.12

graph TD
    Kubelet["kubelet"] --> Shim["image-cri-shim"]
    Shim --> Reg["内置 Registry (localhost:5000)"]
    Reg -->|"缓存未命中时"| Harbor["外部 Harbor (harbor.company.com)"]
    Harbor --> Scan["镜像安全扫描（Trivy / Clair）"]
    Harbor --> Sign["镜像签名验证（Notary / Cosign）"]
    Harbor --> RBAC["访问控制（RBAC + LDAP）"]
    Harbor --> Rep["镜像复制与同步"]

# 内置 Registry 配置代理到 Harbor
apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: registry-proxy
spec:
  path: etc/registry-config.yml
  data: |
    version: 0.1
    proxy:
      remoteurl: https://harbor.company.com
      username: sealos-proxy
      password: Harbor12345
    storage:
      filesystem:
        rootdirectory: /var/lib/registry    

# 重启 Registry 使配置生效
kubectl -n sealos-system rollout restart deployment/registry

# Harbor 侧创建复制规则
# 将 labring/kubernetes 等镜像从 Docker Hub 同步到 Harbor
# 项目：public → 目标：harbor.company.com/sealos

# SealOS 指向 Harbor 仓库
sealos run harbor.company.com/sealos/kubernetes:v1.28.0 \
  harbor.company.com/sealos/calico:v3.26.0 \
  --masters 192.168.1.10 \
  --nodes 192.168.1.11,192.168.1.12

# 查看证书有效期
kubeadm certs check-expiration

# SealOS 生成的证书
# /etc/kubernetes/pki/ca.crt              100 年有效期
# /etc/kubernetes/pki/apiserver.crt       100 年有效期
# /etc/kubernetes/pki/apiserver-kubelet-client.crt  100 年有效期

# 续签所有证书（传统 kubeadm 方式）
kubeadm certs renew all

# 续签指定证书
kubeadm certs renew apiserver
kubeadm certs renew apiserver-kubelet-client

# 重启控制面组件使证书生效
crictl pods | grep kube-apiserver | awk '{print $1}' | xargs crictl stopp

# 在集群部署模板中集成 cert-manager
# Kubefile
FROM labring/kubernetes:v1.28.0

COPY images/cert-manager ./registry/
COPY manifests/cert-manager.yaml ./manifests/

CMD ["sh", "-c", "
  kubectl apply -f manifests/cert-manager.yaml &&
  kubectl wait --for=condition=Available deployment/cert-manager -n cert-manager --timeout=300s
"]

# 自动续签 kubelet 服务证书
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: kubelet-serving
  namespace: kube-system
spec:
  secretName: kubelet-serving-cert
  duration: 2160h  # 90 天
  renewBefore: 360h  # 到期前 15 天自动续签
  issuerRef:
    name: ca-issuer
    kind: Issuer
  dnsNames:
  - "*.cluster.local"

# 1. 备份现有证书
cp -r /etc/kubernetes/pki /etc/kubernetes/pki.bak

# 2. 生成新 CA
kubeadm init phase certs all --config kubeadm-config.yaml

# 3. 更新所有组件证书
kubeadm init phase kubeconfig all --config kubeadm-config.yaml

# 4. 滚动重启控制面
crictl pods --namespace kube-system | grep -E 'kube-apiserver|kube-controller|kube-scheduler' \
  | awk '{print $1}' | xargs -I {} crictl stopp {}

# 移除故障节点
sealos delete --nodes 192.168.1.14

# 加入新节点替换
sealos add --nodes 192.168.1.15

# 1. 从集群中移除故障 Master
sealos delete --masters 192.168.1.11

# 2. 清理残留的 etcd 成员
kubectl -n kube-system exec -it etcd-192.168.1.10 -- etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  member remove <故障节点ID>

# 3. 加入新 Master 节点
sealos add --masters 192.168.1.16

# 场景：多数 etcd 节点故障，从快照恢复
# 1. 停止所有控制面组件
mv /etc/kubernetes/manifests /etc/kubernetes/manifests.bak

# 2. 从快照恢复 etcd 数据
ETCDCTL_API=3 etcdctl snapshot restore /backup/etcd-snapshot.db \
  --name master-0 \
  --initial-cluster "master-0=https://192.168.1.10:2380" \
  --initial-advertise-peer-urls https://192.168.1.10:2380 \
  --data-dir /var/lib/etcd-restore

# 3. 替换数据目录并启动 etcd
mv /var/lib/etcd /var/lib/etcd.bak
mv /var/lib/etcd-restore /var/lib/etcd

# 4. 恢复控制面组件
mv /etc/kubernetes/manifests.bak /etc/kubernetes/manifests

# 5. 重建其他 Master 节点
sealos add --masters 192.168.1.11,192.168.1.12

graph TD
    subgraph Remove["删除节点流程"]
        R1["1. kubectl drain --ignore-daemonsets --delete-emptydir-data"]
        R2["2. kubectl delete node"]
        R3["3. SSH 到目标节点，kubeadm reset"]
        R4["4. 清理网络配置和 iptables 规则"]
        R5["5. 如果该节点是 etcd 成员，从 etcd 集群中移除"]
        R1 --> R2 --> R3 --> R4 --> R5
    end
    subgraph Add["添加节点流程"]
        A1["1. SSH 连接到新节点"]
        A2["2. 安装 CRI 运行时（containerd）"]
        A3["3. 分发 K8s 二进制与配置文件"]
        A4["4. 分发集群证书"]
        A5["5. kubeadm join 或 kubeadm init（Master）"]
        A6["6. 更新 lvscare 负载均衡配置"]
        A7["7. 等待节点 Ready"]
        A1 --> A2 --> A3 --> A4 --> A5 --> A6 --> A7
    end

# 在 amd64 机器上构建
sealos build -t myrepo/kubernetes:v1.28.0-amd64 -f Kubefile .

# 在 arm64 机器上构建
sealos build -t myrepo/kubernetes:v1.28.0-arm64 -f Kubefile .

# 创建 manifest list
sealos manifest create myrepo/kubernetes:v1.28.0
sealos manifest add myrepo/kubernetes:v1.28.0 \
  myrepo/kubernetes:v1.28.0-amd64 \
  myrepo/kubernetes:v1.28.0-arm64

# 推送 manifest list
sealos manifest push myrepo/kubernetes:v1.28.0

# Clusterfile 混合架构集群
apiVersion: sealos.io/v1beta1
kind: Cluster
metadata:
  name: hybrid-cluster
spec:
  hosts:
  # amd64 Master 节点
  - ips: [192.168.1.10, 192.168.1.11]
    roles: [master]
    arch: amd64
  # arm64 Worker 节点
  - ips: [192.168.1.20, 192.168.1.21, 192.168.1.22]
    roles: [node]
    arch: arm64
  image: myrepo/kubernetes:v1.28.0
  ssh:
    user: root
    passwd: "password"

apiVersion: apps/v1
kind: Deployment
metadata:
  name: multi-arch-app
spec:
  template:
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/arch
                operator: In
                values: [amd64]
      containers:
      - name: app
        image: myapp:v1.0  # 多架构镜像

# 生产环境 Clusterfile 示例
apiVersion: sealos.io/v1beta1
kind: Cluster
metadata:
  name: prod-k8s-cluster
spec:
  hosts:
  # 3 个 Master（跨机架/可用区）
  - ips: [10.10.1.10, 10.10.2.10, 10.10.3.10]
    roles: [master]
  # N 个 Worker（按业务分组）
  - ips: [10.10.1.21, 10.10.1.22, 10.10.1.23]
    roles: [node]
    labels:
      node-role: general
  - ips: [10.10.2.21, 10.10.2.22]
    roles: [node]
    labels:
      node-role: high-memory
  image: labring/kubernetes:v1.28.0
  ssh:
    user: root
    pk: /root/.ssh/id_rsa
    port: 22

apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: kernel-params
spec:
  path: etc/sysctl.sh
  data: |
    #!/bin/bash
    cat <<EOF > /etc/sysctl.d/99-kubernetes.conf
    net.bridge.bridge-nf-call-iptables = 1
    net.bridge.bridge-nf-call-ip6tables = 1
    net.ipv4.ip_forward = 1
    net.ipv4.tcp_tw_reuse = 1
    net.ipv4.ip_local_port_range = 1024 65000
    fs.file-max = 2097152
    fs.inotify.max_user_instances = 8192
    fs.inotify.max_user_watches = 524288
    vm.swappiness = 0
    vm.overcommit_memory = 1
    kernel.panic = 10
    kernel.panic_on_oops = 1
    EOF
    sysctl --system    

# 定时 etcd 备份（CronJob）
# 每 6 小时执行一次 etcd 快照
0 */6 * * * sealos backup --name prod-backup-$(date +\%Y\%m\%d-\%H\%M)

# 备份保留策略
# 最近 7 天的每日备份 + 最近 4 周的每周备份

# Clusterfile 配置 GPU 节点
apiVersion: sealos.io/v1beta1
kind: Cluster
metadata:
  name: gpu-cluster
spec:
  hosts:
  - ips: [192.168.1.10, 192.168.1.11]
    roles: [master]
  - ips: [192.168.1.12]
    roles: [node]
    labels:
      accelerator: nvidia-t4
  - ips: [192.168.1.13, 192.168.1.14]
    roles: [node]
    labels:
      accelerator: nvidia-a100
  image: labring/kubernetes:v1.28.0
apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: nvidia-gpu-setup
spec:
  path: etc/gpu-setup.sh
  data: |
    #!/bin/bash
    # 安装 NVIDIA 驱动
    distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
    curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add -
    curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list \
      > /etc/apt/sources.list.d/nvidia-docker.list

    apt-get update && apt-get install -y nvidia-container-toolkit

    # 配置 containerd 支持 GPU
    nvidia-ctk runtime configure --runtime=containerd
    systemctl restart containerd    
apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: nvidia-device-plugin
spec:
  strategy: merge
  path: manifests/nvidia-plugin.yaml
  data: |
    apiVersion: apps/v1
    kind: DaemonSet
    metadata:
      name: nvidia-device-plugin-daemonset
      namespace: kube-system
    spec:
      selector:
        matchLabels:
          name: nvidia-device-plugin-ds
      template:
        metadata:
          labels:
            name: nvidia-device-plugin-ds
        spec:
          tolerations:
          - key: nvidia.com/gpu
            operator: Exists
            effect: NoSchedule
          containers:
          - name: nvidia-device-plugin-ctr
            image: nvcr.io/nvidia/k8s-device-plugin:v0.14.0
            securityContext:
              allowPrivilegeEscalation: false
              capabilities:
                drop: ["ALL"]    

apiVersion: v1
kind: Pod
metadata:
  name: gpu-training-job
spec:
  nodeSelector:
    accelerator: nvidia-t4
  containers:
  - name: training
    image: nvcr.io/nvidia/pytorch:23.10-py3
    resources:
      limits:
        nvidia.com/gpu: 2  # 申请 2 块 GPU
    env:
    - name: NVIDIA_VISIBLE_DEVICES
      value: all

graph TD
    subgraph Host["主机层"]
        Driver["NVIDIA Driver ≥ 470.x"]
        Toolkit["nvidia-container-toolkit"]
    end
    subgraph Runtime["容器运行时层"]
        Containerd["containerd (nvidia-container-runtime 作为 runtime)"]
    end
    subgraph K8s["Kubernetes 层"]
        Plugin["nvidia-device-plugin (DaemonSet)"]
    end
    subgraph Schedule["调度层"]
        Selector["nodeSelector / nodeAffinity (accelerator label)"]
        Limits["Resource Limits (nvidia.com/gpu)"]
    end
    Host --> Runtime --> K8s --> Schedule

apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: os-hardening
spec:
  path: etc/os-hardening.sh
  data: |
    #!/bin/bash
    # 禁用不必要的服务
    systemctl disable --now cups avahi-daemon bluetooth

    # 配置审计规则
    cat <<EOF > /etc/audit/rules.d/k8s.rules
    -w /etc/kubernetes/ -p wa -k kubernetes_config
    -w /var/lib/kubelet/ -p wa -k kubelet_data
    -w /etc/cni/ -p wa -k cni_config
    EOF

    # 限制 SSH 访问
    sed -i 's/^#PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
    sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config

    # 配置防火墙
    ufw default deny incoming
    ufw default allow outgoing
    ufw allow 22/tcp
    ufw allow 6443/tcp
    ufw allow 2379:2380/tcp
    ufw allow 10250/tcp
    ufw enable    

# kubeadm 安全参数
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
apiServer:
  extraArgs:
    # 禁止匿名访问
    anonymous-auth: "false"
    # 启用审计日志
    audit-log-path: /var/log/kubernetes/audit.log
    audit-log-maxage: "30"
    audit-log-maxbackup: "10"
    audit-log-maxsize: "100"
    # 启用加密 Provider
    encryption-provider-config: /etc/kubernetes/pki/encryption-config.yaml
    # 启用 Node 鉴权
    authorization-mode: Node,RBAC
    # TLS 最低版本
    tls-min-version: VersionTLS12

apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: etcd-encryption
spec:
  path: etc/encryption-config.yaml
  data: |
    apiVersion: apiserver.config.k8s.io/v1
    kind: EncryptionConfiguration
    resources:
    - resources:
      - secrets
      - configmaps
      providers:
      - aescbc:
          keys:
          - name: key1
            secret: <base64-encoded-32-byte-key>
      - identity: {}    

apiVersion: v1
kind: Namespace
metadata:
  name: production
  labels:
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/warn: restricted
apiVersion: sealos.io/v1beta1
kind: Config
metadata:
  name: network-policy-default-deny
spec:
  path: manifests/default-deny-netpol.yaml
  data: |
    apiVersion: networking.k8s.io/v1
    kind: NetworkPolicy
    metadata:
      name: default-deny-all
      namespace: production
    spec:
      podSelector: {}
      policyTypes:
      - Ingress
      - Egress