使用斐讯 N1 和 OpenWrt 搭建 K3s 集群

环境说明
#

软件版本
#

K3s 版本：v1.19.7+k3s1
Docker 版本：19.03.13 (N1)，19.03.12 (OpenWrt)
写盘工具：balenaEtcher

硬件配置
#

IP 地址	机型	配置	操作系统	角色
192.168.8.1	占美（机型不详）	4C 2G（CPU N2940）	OpenWrt（X86_64 自编译）	node/agent
192.168.8.112	斐讯 N1	4C 2G	Armbian（5.0.2-aml-s905）	master/server

OpenWrt 自编译
#

需要自编译的原因：之前使用 esir 打包的高大全 OpenWrt 固件部署 K3s 时发现，内核没有开启 vxlan 特性，导致部署失败。如果您使用的 OpenWrt 固件已开启 vxlan 特性，可跳过此步骤。

检查方法：

root@OpenWrt:~# lsmod | grep -i vxlan  # 有输出表示具有 vxlan 特性
ip6_udp_tunnel         16384  1 vxlan
udp_tunnel             16384  1 vxlan
vxlan                  53248  0

感谢 esir 等大佬提供的 GitHub Actions CI 工作流
仓库地址
视频教程

Fork 仓库代码
#

克隆代码
#

git clone https://github.com.cnpmjs.org/cdryzun/AutoBuild-OpenWrt.git

cd AutoBuild-OpenWrt

修改配置文件
#

修改工作流文件
#

以 x86_64 平台为例：

code .github/workflows/Build_OP_x86_64.yml  # 编辑 CI 构建脚本

# 取消注释以下内容，注意空格
push:
  branches:
    - master

修改构建配置
#

code x86_64.config

参考资料
#

CONFIG_PACKAGE_grub2-efi=y
CONFIG_EFI_IMAGES=y
CONFIG_TARGET_IMAGES_GZIP=y
CONFIG_TARGET_KERNEL_PARTSIZE=16
CONFIG_TARGET_ROOTFS_PARTSIZE=300
# CONFIG_GRUB_CONSOLE is not set
CONFIG_PACKAGE_luci-app-docker=y
CONFIG_PACKAGE_luci-i18n-docker-zh-cn=y
CONFIG_PACKAGE_luci-app-ssr-plus=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Shadowsocks=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Simple_obfs=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_V2ray_plugin=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_V2ray=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Trojan=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_Kcptun=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_ShadowsocksR_Server=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_ShadowsocksR_Socks=y
CONFIG_PACKAGE_luci-app-ssr-plus_INCLUDE_NaiveProxy=y
CONFIG_PACKAGE_luci-theme-argon=y
CONFIG_PACKAGE_luci-app-mwan3=y
CONFIG_PACKAGE_luci-app-mwan3helper=y
CONFIG_PACKAGE_luci-i18n-mwan3-zh-cn=y
CONFIG_PACKAGE_luci-i18n-mwan3helper-zh-cn=y
CONFIG_PACKAGE_luci-app-syncdial=y
CONFIG_PACKAGE_luci-app-sfe=y
CONFIG_PACKAGE_luci-app-ttyd=y
CONFIG_PACKAGE_luci-i18n-ttyd-zh-cn=y
CONFIG_PACKAGE_luci-app-webadmin=y
CONFIG_PACKAGE_luci-i18n-webadmin-zh-cn=y
CONFIG_PACKAGE_luci-app-zerotier=y
CONFIG_PACKAGE_luci-app-frpc=y
CONFIG_PACKAGE_luci-app-kodexplorer=y
CONFIG_PACKAGE_luci-app-nfs=y
CONFIG_PACKAGE_ss=y
CONFIG_PACKAGE_luci-app-nps=y
CONFIG_PACKAGE_luci-app-ntpc=y
CONFIG_PACKAGE_luci-app-rclone=y
CONFIG_PACKAGE_luci-app-radicale=y
CONFIG_PACKAGE_luci-app-qbittorrent=y
CONFIG_PACKAGE_luci-app-samba=y
CONFIG_PACKAGE_luci-app-wol=y
CONFIG_PACKAGE_luci-app-xlnetacc=y
CONFIG_PACKAGE_luci-app-bird1-ipv4=y
CONFIG_PACKAGE_luci-app-bird1-ipv6=y
CONFIG_PACKAGE_luci-app-commands=y
CONFIG_PACKAGE_luci-app-diskman=y
CONFIG_PACKAGE_luci-app-dnscrypt-proxy=y
CONFIG_PACKAGE_luci-app-dnsforwarder=y
CONFIG_PACKAGE_luci-app-familycloud=y
CONFIG_PACKAGE_luci-app-hd-idle=y
CONFIG_PACKAGE_luci-app-netdata=y
CONFIG_PACKAGE_ipv6helper=y
CONFIG_PACKAGE_dnsmasq_full_auth=y
CONFIG_PACKAGE_dnsmasq_full_conntrack=y
CONFIG_PACKAGE_dnsmasq_full_dnssec=y
CONFIG_PACKAGE_curl=y
CONFIG_PACKAGE_htop=y
CONFIG_PACKAGE_wget=y
CONFIG_PACKAGE_lrzsz=y
CONFIG_PACKAGE_vim=y
CONFIG_PACKAGE_zip=y
CONFIG_PACKAGE_kmod-kvm-amd=y
CONFIG_PACKAGE_kmod-kvm-intel=y
CONFIG_PACKAGE_kmod-kvm-x86=y
CONFIG_OPENSSL_ENGINE_CRYPTO=y
CONFIG_OPENSSL_ENGINE_DIGEST=y
CONFIG_OPENSSL_WITH_CAMELLIA=y
CONFIG_OPENSSL_WITH_COMPRESSION=y
CONFIG_OPENSSL_WITH_DTLS=y
CONFIG_OPENSSL_WITH_EC2M=y
CONFIG_OPENSSL_WITH_ERROR_MESSAGES=y
CONFIG_OPENSSL_WITH_GOST=y
CONFIG_OPENSSL_WITH_IDEA=y
CONFIG_OPENSSL_WITH_MDC2=y
CONFIG_OPENSSL_WITH_RFC3779=y
CONFIG_OPENSSL_WITH_SEED=y
CONFIG_OPENSSL_WITH_WHIRLPOOL=y
CONFIG_OPENVPN_openssl_ENABLE_DEF_AUTH=y
CONFIG_OPENVPN_openssl_ENABLE_FRAGMENT=y
CONFIG_OPENVPN_openssl_ENABLE_LZ4=y
CONFIG_OPENVPN_openssl_ENABLE_LZO=y
CONFIG_OPENVPN_openssl_ENABLE_MULTIHOME=y
CONFIG_OPENVPN_openssl_ENABLE_PF=y
CONFIG_OPENVPN_openssl_ENABLE_PORT_SHARE=y
CONFIG_OPENVPN_openssl_ENABLE_SERVER=y
CONFIG_OPENVPN_openssl_ENABLE_SMALL=y
CONFIG_KERNEL_BUILD_USER="treeSir Playground"
CONFIG_GRUB_TITLE="OpenWrt AutoBuild by treeSirPlayground"
CONFIG_PACKAGE_kmod-usb-ohci=y
CONFIG_PACKAGE_kmod-usb-ohci-pci=y
CONFIG_PACKAGE_kmod-usb-storage-uas=y
CONFIG_PACKAGE_kmod-usb-uhci=y
CONFIG_PACKAGE_kmod-sdhci=y
CONFIG_PACKAGE_kmod-usb-ehci=y
CONFIG_PACKAGE_kmod-usb2=y
CONFIG_PACKAGE_kmod-usb2-pci=y
CONFIG_PACKAGE_luci-app-passwall=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Brook=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Trojan=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_Trojan_GO=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_simple-obfs=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_v2ray-plugin=y
CONFIG_PACKAGE_luci-app-passwall_INCLUDE_https_dns_proxy=n
# CONFIG_PACKAGE_naiveproxy=y
# CONFIG_PACKAGE_nspr=y
# CONFIG_PACKAGE_shadowsocks-libev-ss-server=y
# CONFIG_PACKAGE_ssocks=y
# CONFIG_PACKAGE_ssocksd=y
# CONFIG_PACKAGE_trojan-go=y
# CONFIG_PACKAGE_trojan-plus=y
# CONFIG_PACKAGE_chinadns-ng=y
CONFIG_TARGET_x86=y
CONFIG_TARGET_x86_64=y
CONFIG_TARGET_x86_64_Generic=y
CONFIG_FEED_luci=y
CONFIG_FEED_packages=y
CONFIG_FEED_routing=y
CONFIG_FEED_telephony=y
CONFIG_IPTABLES_CONNLABEL=y
CONFIG_IPTABLES_NFTABLES=y
CONFIG_KERNEL_BLK_CGROUP=y
CONFIG_KERNEL_CFS_BANDWIDTH=y
CONFIG_KERNEL_CGROUPS=y
CONFIG_KERNEL_CGROUP_CPUACCT=y
CONFIG_KERNEL_CGROUP_DEVICE=y
CONFIG_KERNEL_CGROUP_FREEZER=y
CONFIG_KERNEL_CGROUP_PIDS=y
CONFIG_KERNEL_CGROUP_SCHED=y
CONFIG_KERNEL_CPUSETS=y
CONFIG_KERNEL_DEVPTS_MULTIPLE_INSTANCES=y
CONFIG_KERNEL_FAIR_GROUP_SCHED=y
CONFIG_KERNEL_FREEZER=y
CONFIG_KERNEL_IPC_NS=y
CONFIG_KERNEL_KEYS=y
CONFIG_KERNEL_LXC_MISC=y
CONFIG_KERNEL_MEMCG=y
CONFIG_KERNEL_MM_OWNER=y
CONFIG_KERNEL_NAMESPACES=y
CONFIG_KERNEL_NETPRIO_CGROUP=y
CONFIG_KERNEL_NET_CLS_CGROUP=y
CONFIG_KERNEL_NET_NS=y
CONFIG_KERNEL_PID_NS=y
CONFIG_KERNEL_POSIX_MQUEUE=y
CONFIG_KERNEL_PROC_PID_CPUSET=y
CONFIG_KERNEL_RESOURCE_COUNTERS=y
CONFIG_KERNEL_SECCOMP=y
CONFIG_KERNEL_SECCOMP_FILTER=y
CONFIG_KERNEL_USER_NS=y
CONFIG_KERNEL_UTS_NS=y
CONFIG_PACKAGE_block-mount=y
CONFIG_PACKAGE_ip-bridge=y
CONFIG_PACKAGE_ip-full=y
CONFIG_PACKAGE_ipset=y
CONFIG_PACKAGE_iptables-mod-conntrack-extra=y
CONFIG_PACKAGE_iptables-mod-extra=y
CONFIG_PACKAGE_iptables-mod-ipopt=y
CONFIG_PACKAGE_kmod-asn1-decoder=y
CONFIG_PACKAGE_kmod-br-netfilter=y
CONFIG_PACKAGE_kmod-crypto-aead=y
CONFIG_PACKAGE_kmod-crypto-authenc=y
CONFIG_PACKAGE_kmod-crypto-crc32c=y
CONFIG_PACKAGE_kmod-crypto-hash=y
CONFIG_PACKAGE_kmod-crypto-hw-ccp=y
CONFIG_PACKAGE_kmod-crypto-manager=y
CONFIG_PACKAGE_kmod-crypto-null=y
CONFIG_PACKAGE_kmod-crypto-pcompress=y
CONFIG_PACKAGE_kmod-crypto-rsa=y
CONFIG_PACKAGE_kmod-crypto-sha1=y
CONFIG_PACKAGE_kmod-crypto-sha256=y
CONFIG_PACKAGE_kmod-fuse=y
CONFIG_PACKAGE_kmod-ikconfig=y
CONFIG_PACKAGE_kmod-ipt-conntrack-extra=y
CONFIG_PACKAGE_kmod-ipt-extra=y
CONFIG_PACKAGE_kmod-ipt-ipopt=y
CONFIG_PACKAGE_kmod-ipt-ipset=y
CONFIG_PACKAGE_kmod-ipt-raw=y
CONFIG_PACKAGE_kmod-iptunnel=y
CONFIG_PACKAGE_kmod-leds-gpio=y
CONFIG_PACKAGE_kmod-lib-crc32c=y
CONFIG_PACKAGE_kmod-nf-conntrack-netlink=y
CONFIG_PACKAGE_kmod-nf-ipvs=y
CONFIG_PACKAGE_kmod-nfnetlink=y
CONFIG_PACKAGE_kmod-nls-base=y
CONFIG_PACKAGE_kmod-random-core=y
CONFIG_PACKAGE_kmod-softdog=y
CONFIG_PACKAGE_kmod-sound-core=y
CONFIG_PACKAGE_kmod-udptunnel4=y
CONFIG_PACKAGE_kmod-udptunnel6=y
CONFIG_PACKAGE_kmod-usb-core=y
CONFIG_PACKAGE_kmod-usb3=y
CONFIG_PACKAGE_kmod-veth=y
CONFIG_PACKAGE_kmod-vxlan=y
CONFIG_PACKAGE_kmod-ppp=y
CONFIG_PACKAGE_kmod-pppox=y
CONFIG_PACKAGE_libipset=y
CONFIG_PACKAGE_libmnl=y
CONFIG_PACKAGE_libnetfilter-conntrack=y
CONFIG_PACKAGE_libnfnetlink=y
CONFIG_PACKAGE_libnftnl=y
CONFIG_TARGET_ROOTFS_PARTSIZE=1024
# CONFIG_PACKAGE_iptables-mod-conntrack-label is not set
# CONFIG_PACKAGE_kmod-ipt-conntrack-label is not set
CONFIG_PACKAGE_kmod-lib-crc-ccitt=y

示例配置，仅供参考使用

构建固件
#

触发构建
#

提交代码前，检查 Actions 是否已开启。

git add .github/workflows/Build_OP_x86_64.yml
git add ./x86_64.config

git commit -m "build custom openwrt"

git push

提交完成后，等待构建结束。

下载固件
#

构建完成后下载 OpenWrt 固件文件：

点击后向下滚动，点击进行下载：

省略 OpenWrt 写入步骤。下载固件后解压，进入 OpenWrt/targets/x86/64 目录即可看到固件文件。

N1 K3s Master 部署
#

N1 使用 Armbian 系统，同样省略写入步骤。N1 刷入 Armbian 系统的资料网上很多，此处不再赘述。

Docker 配置文件
#

以下为 N1 Docker 配置文件，不一定适合所有环境，可参考部分配置使用：

 cat /etc/docker/daemon.json 
{
  "experimental": false,
  "registry-mirrors": [
    "https://7bezldxe.mirror.aliyuncs.com"
  ],
  "oom-score-adjust": -1000,
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m",
    "max-file": "3"
  },
  "max-concurrent-downloads": 10,
  "max-concurrent-uploads": 10,
  "features": {
    "buildkit": true
  },
  "insecure-registries": [
    "idocker.io"
  ]
}

配置项简短说明:
“log-driver”: “json-file” 设置json 格式日志
“oom-score-adjust”: -1000 防止容器被内核 oom
“log-opts” 设置容器日志大小
“max-concurrent-downloads”: 10 并行下载容器数量
“max-concurrent-uploads”: 10 并行上传
“storage-driver”: “overlay2” 设置存储驱动为 overlay2
“bip” 容器默认的网段
“registry-mirrors” 配置镜像下载加速这里使用的是阿里云的
“insecure-registries” 信任的私服地址

如若修改了配置，记得重启一下服务查看是否生效

service docker restart

docker info

部署前检查
#

注意：以下内容是部署完成后的打印结果，部署前的检查步骤未做记录。

root@n1:/# k3s check-config

Verifying binaries in /var/lib/rancher/k3s/data/ad8f0f93ebb9db5c507884fcdec249d73dd348293dac194e01462c57815cca46/bin:
- sha256sum: good
- links: good

System:
- /sbin iptables v1.6.1: older than v1.8
- swap: should be disabled
- routes: default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed

Limits:
- /proc/sys/kernel/keys/root_maxkeys: 1000000

info: reading kernel config from /proc/config.gz ...

Generally Necessary:
- cgroup hierarchy: properly mounted [/sys/fs/cgroup]
- CONFIG_NAMESPACES: enabled
- CONFIG_NET_NS: enabled
- CONFIG_PID_NS: enabled
- CONFIG_IPC_NS: enabled
- CONFIG_UTS_NS: enabled
- CONFIG_CGROUPS: enabled
- CONFIG_CGROUP_CPUACCT: enabled
- CONFIG_CGROUP_DEVICE: enabled
- CONFIG_CGROUP_FREEZER: enabled
- CONFIG_CGROUP_SCHED: enabled
- CONFIG_CPUSETS: enabled
- CONFIG_MEMCG: enabled
- CONFIG_KEYS: enabled
- CONFIG_VETH: enabled (as module)
- CONFIG_BRIDGE: enabled (as module)
- CONFIG_BRIDGE_NETFILTER: enabled (as module)
- CONFIG_IP_NF_FILTER: enabled (as module)
- CONFIG_IP_NF_TARGET_MASQUERADE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_CONNTRACK: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_IPVS: enabled (as module)
- CONFIG_IP_NF_NAT: enabled (as module)
- CONFIG_NF_NAT: enabled (as module)
- CONFIG_POSIX_MQUEUE: enabled

Optional Features:
- CONFIG_USER_NS: enabled
- CONFIG_SECCOMP: enabled
- CONFIG_CGROUP_PIDS: enabled
- CONFIG_BLK_CGROUP: enabled
- CONFIG_BLK_DEV_THROTTLING: enabled
- CONFIG_CGROUP_PERF: enabled
- CONFIG_CGROUP_HUGETLB: enabled
- CONFIG_NET_CLS_CGROUP: enabled (as module)
- CONFIG_CGROUP_NET_PRIO: enabled
- CONFIG_CFS_BANDWIDTH: enabled
- CONFIG_FAIR_GROUP_SCHED: enabled
- CONFIG_RT_GROUP_SCHED: enabled
- CONFIG_IP_NF_TARGET_REDIRECT: enabled (as module)
- CONFIG_IP_SET: enabled (as module)
- CONFIG_IP_VS: enabled (as module)
- CONFIG_IP_VS_NFCT: enabled
- CONFIG_IP_VS_PROTO_TCP: enabled
- CONFIG_IP_VS_PROTO_UDP: enabled
- CONFIG_IP_VS_RR: enabled (as module)
- CONFIG_EXT4_FS: enabled
- CONFIG_EXT4_FS_POSIX_ACL: enabled
- CONFIG_EXT4_FS_SECURITY: enabled
- Network Drivers:
  - "overlay":
    - CONFIG_VXLAN: enabled (as module)
      Optional (for encrypted networks):
      - CONFIG_CRYPTO: enabled
      - CONFIG_CRYPTO_AEAD: enabled
      - CONFIG_CRYPTO_GCM: enabled (as module)
      - CONFIG_CRYPTO_SEQIV: enabled
      - CONFIG_CRYPTO_GHASH: enabled (as module)
      - CONFIG_XFRM: enabled
      - CONFIG_XFRM_USER: enabled (as module)
      - CONFIG_XFRM_ALGO: enabled (as module)
      - CONFIG_INET_ESP: enabled (as module)
      - CONFIG_INET_XFRM_MODE_TRANSPORT: enabled
- Storage Drivers:
  - "overlay":
    - CONFIG_OVERLAY_FS: enabled

STATUS: pass

部署 K3s
#

官方提供了部署脚本，部署非常方便：

export INSTALL_K3S_VERSION=v1.19.7+k3s1

export INSTALL_K3S_EXEC="--docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666"

curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | INSTALL_K3S_MIRROR=cn sh -

root@n1:~# kubectl get nodes
NAME   STATUS   ROLES    AGE   VERSION
n1     Ready    master   6s    v1.19.7+k3s1

OpenWrt K3s Agent 部署
#

在 OpenWrt 中部署 K3s 时不能使用官方提供的自动部署脚本，运行脚本会提示错误：[ERROR] Can not find systemd or openrc to use as a process supervisor for k3s。好在 GitHub 上有大佬提供了 OpenWrt K3s 的安装包。

Docker 配置
#

以下为 OpenWrt 中的 Docker 配置，配置项说明请参考上面 N1 中的 Docker 配置说明：

cat /etc/docker/daemon.json
{
"data-root": "/data/docker-root",
"log-level": "warn",
"oom-score-adjust": -1000,
"log-driver": "json-file",
  "log-opts": {
   "max-size": "100m",
   "max-file": "3"
},
"max-concurrent-downloads": 10,
"max-concurrent-uploads": 10,
"insecure-registries": ["idocker.io"],
"registry-mirrors": ["https://7bezldxe.mirror.aliyuncs.com"],
"storage-driver": "overlay2",
"storage-opts": [
   "overlay2.override_kernel_check=true"
]
}

⚠️ 注意：配置修改后需要重启 Docker 服务才会生效
/etc/init.d/dockerd restart

K3s Agent 部署
#

安装包 GitHub 仓库地址

下载安装包
#

wget https://github.com/discordianfish/k3s-openwrt/releases/download/9/k3s_1.17.4+k3s1_x86_64.opk

opkg install ./k3s_1.17.4+k3s1_x86_64.opk

root@OpenWrt:/opt/tools# /usr/bin/k3s --version
k3s version v1.17.4+k3s1 (3eee8ac3)

升级 K3s 版本
#

默认安装包中 K3s 版本为 v1.17.4+k3s1，与 N1 上的 Master 版本不匹配，需要升级：

wget https://github.com/k3s-io/k3s/releases/download/v1.19.7%2Bk3s1/k3s

chmod a+x k3s

mv /usr/bin/k3s /usr/bin/k3s.old

cp -a k3s /usr/bin/k3s

root@OpenWrt:/opt/tools# k3s --version
k3s version v1.19.7+k3s1 (5a00e38d)

部署前检查
#

注意：以下内容同样是部署完成后的打印结果

root@OpenWrt:~# k3s check-config

Verifying binaries in /var/lib/rancher/k3s/data/30740d1d67da51fe92b10367ecce4d580e552c634ad4a6c4dd13297ffd1f3edd/bin:
- sha256sum: good
- links: good

System:
- /usr/sbin iptables v1.8.4 (legacy): ok
- swap: disabled
- routes: default CIDRs 10.42.0.0/16 or 10.43.0.0/16 already routed

Limits:
- /proc/sys/kernel/keys/root_maxkeys: 1000000

info: reading kernel config from /proc/config.gz ...

Generally Necessary:
- cgroup hierarchy: properly mounted [/sys/fs/cgroup]
- CONFIG_NAMESPACES: enabled
- CONFIG_NET_NS: enabled
- CONFIG_PID_NS: enabled
- CONFIG_IPC_NS: enabled
- CONFIG_UTS_NS: enabled
- CONFIG_CGROUPS: enabled
- CONFIG_CGROUP_CPUACCT: enabled
- CONFIG_CGROUP_DEVICE: enabled
- CONFIG_CGROUP_FREEZER: enabled
- CONFIG_CGROUP_SCHED: enabled
- CONFIG_CPUSETS: enabled
- CONFIG_MEMCG: enabled
- CONFIG_KEYS: enabled
- CONFIG_VETH: enabled (as module)
- CONFIG_BRIDGE: enabled
- CONFIG_BRIDGE_NETFILTER: enabled (as module)
- CONFIG_IP_NF_FILTER: enabled (as module)
- CONFIG_IP_NF_TARGET_MASQUERADE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_ADDRTYPE: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_CONNTRACK: enabled (as module)
- CONFIG_NETFILTER_XT_MATCH_IPVS: enabled (as module)
- CONFIG_IP_NF_NAT: enabled (as module)
- CONFIG_NF_NAT: enabled (as module)
- CONFIG_POSIX_MQUEUE: enabled

Optional Features:
- CONFIG_USER_NS: enabled
- CONFIG_SECCOMP: enabled
- CONFIG_CGROUP_PIDS: enabled
- CONFIG_BLK_CGROUP: enabled
- CONFIG_BLK_DEV_THROTTLING: missing
- CONFIG_CGROUP_PERF: missing
- CONFIG_CGROUP_HUGETLB: missing
- CONFIG_NET_CLS_CGROUP: enabled
- CONFIG_CGROUP_NET_PRIO: enabled
- CONFIG_CFS_BANDWIDTH: enabled
- CONFIG_FAIR_GROUP_SCHED: enabled
- CONFIG_RT_GROUP_SCHED: enabled
- CONFIG_IP_NF_TARGET_REDIRECT: enabled (as module)
- CONFIG_IP_SET: enabled (as module)
- CONFIG_IP_VS: enabled (as module)
- CONFIG_IP_VS_NFCT: enabled
- CONFIG_IP_VS_PROTO_TCP: enabled
- CONFIG_IP_VS_PROTO_UDP: enabled
- CONFIG_IP_VS_RR: enabled (as module)
- CONFIG_EXT4_FS: enabled
- CONFIG_EXT4_FS_POSIX_ACL: missing
- CONFIG_EXT4_FS_SECURITY: missing
    enable these ext4 configs if you are using ext3 or ext4 as backing filesystem
- Network Drivers:
  - "overlay":
    - CONFIG_VXLAN: enabled (as module)
      Optional (for encrypted networks):
      - CONFIG_CRYPTO: enabled
      - CONFIG_CRYPTO_AEAD: enabled
      - CONFIG_CRYPTO_GCM: missing
      - CONFIG_CRYPTO_SEQIV: missing
      - CONFIG_CRYPTO_GHASH: missing
      - CONFIG_XFRM: enabled
      - CONFIG_XFRM_USER: enabled (as module)
      - CONFIG_XFRM_ALGO: enabled (as module)
      - CONFIG_INET_ESP: enabled (as module)
      - CONFIG_INET_XFRM_MODE_TRANSPORT: missing
- Storage Drivers:
  - "overlay":
    - CONFIG_OVERLAY_FS: enabled

STATUS: pass

第一次部署检查时，显示有两项未通过，与此链接中描述类似。

修改 K3s 服务配置
#

参考：K3s 中文文档

默认安装后的配置文件是 /etc/config/k3s，该文件默认不存在，需要手动创建。

--token 后的配置来自 N1 Master 中的 /var/lib/rancher/k3s/server/node-token 文件：

root@n1:/# cat /var/lib/rancher/k3s/server/node-token
K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85

服务配置文件 /etc/config/k3s：

cat /etc/config/k3s
config globals 'globals'
        option opts '--server https://192.168.8.112:6443 --token K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85 --docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666'
        option root '/application/k3s'

配置文件可根据实际使用情况进行增删

服务启动脚本 /etc/init.d/k3s：

修改说明：下面的启动脚本中，将 uci_get 替换为 uci get，将 --server 修改为 --agent。如果部署的是 Server 端，此处可以不做修改，且 /etc/config/k3s 中也需要修改为 Server 端的配置项。

root@OpenWrt:~# cat /etc/init.d/k3s
#!/bin/sh /etc/rc.common

START=60
STOP=20

PIDFILE=/var/run/k3s.pid
EXEC="/usr/bin/k3s"

ensure_cgroup_mount() {
  # Unmount /sys/fs/cgroup if mounted as cgroup
  grep -q ' /sys/fs/cgroup cgroup' /proc/self/mounts && umount /sys/fs/cgroup

  grep -q ' /sys/fs/cgroup tmpfs' /proc/self/mounts \
    || mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup

  for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
    mnt="/sys/fs/cgroup/$sys"
    grep -q "cgroup $mnt " /proc/self/mounts && continue
    mkdir -p "$mnt"
    mount -n -t cgroup -o $sys cgroup "$mnt"
  done
}

start() {
  ensure_cgroup_mount
  start-stop-daemon -S -b -x "$EXEC" -m -p "$PIDFILE" \
    -- agent $(uci get k3s.globals.opts) \
    --data-dir $(uci get k3s.globals.root)
}

stop() {
  start-stop-daemon -K -p "$PIDFILE"
}

启动服务
#

/etc/init.d/k3s start  # 启动命令

root@OpenWrt:~# ps -ef | grep k3s
 3451 root      867m S    {k3s-agent} /usr/bin/k3s agent   # 启动成功后，会有进程在后台运行
21059 root      1096 S    grep k3s

root@n1:/# kubectl get nodes
NAME      STATUS   ROLES    AGE    VERSION
n1        Ready    master   2d3h   v1.19.7+k3s1
openwrt   Ready    <none>   2d2h   v1.19.7+k3s1

如果运行启动命令后没有响应，可以前台手动运行进行调试：

k3s agent --server https://192.168.8.112:6443 --token K106ccb265579f1e400312844312787c9ce4dd8528b6c58e26894e953661c9a907ef5d2::server:484323236840cb1c3bea454570f85 --docker --kube-apiserver-arg service-node-port-range=40000-65000 --no-deploy traefik --write-kubeconfig ~/.kube/config --write-kubeconfig-mode 666

问题记录
#

启动 Agent 时显示以下错误：

level=error msg="Node password rejected, duplicate hostname or contents of '/etc/rancher/node/password' may not match server node-passwd entry, try enabling a unique node name with the --with-node-id flag"

问题原因：之前部署失败，但 Server 端仍保留 Agent 端的 server id。当 Agent 端再次注册到 Server 端时，Server 端通过主机名匹配，发现 ID 不匹配导致冲突。

解决方法：在 Server 端删除旧的 server id 或更改 Agent 端的主机名（推荐在 Server 端删除）。

cat /etc/rancher/node/password  # agent
92a18abb46

cat /var/lib/rancher/k3s/server/cred/node-passwd
898170dfe92a18abb46,n1,n1,
9185b3215d3afcab9,openwrt,openwrt,  # 删除此行

OpenWrt 防火墙配置
#

编辑 /etc/config/network 配置文件，添加以下内容：

config interface 'k8s'
	option proto 'none'
	option ifname 'cni0'

编辑 /etc/config/firewall，添加以下内容：

config zone
        option name 'k8s'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'ACCEPT'
        option network 'k8s'

重启 OpenWrt 系统使配置生效：

reboot

部署应用测试
#

部署一个 Deployment 类型的 Nginx 服务，测试 K3s 集群完整性：

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: nginx
spec:
  ports:
    - protocol: TCP
      name: web
      port: 80
  selector:
    app: nginx

---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 4
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: nginx
          image: nginx:1.16.1
          ports:
            - name: web
              containerPort: 80
EOF

watch kubectl get pod -o wide

NAME                    READY   STATUS    RESTARTS   AGE    IP           NODE      NOMINATED NODE   READINESS GATES
nginx-bcf5bbd7d-n2lfx   1/1     Running   0          3m2s   10.42.0.14   n1        <none>           <none>
nginx-bcf5bbd7d-zhj6j   1/1     Running   0          78s    10.42.0.15   n1        <none>           <none>
nginx-bcf5bbd7d-sxlvp   1/1     Running   0          78s    10.42.1.17   openwrt   <none>           <none>
nginx-bcf5bbd7d-bsxtn   1/1     Running   0          78s    10.42.1.18   openwrt   <none>           <none>

可以看到 N1 和 OpenWrt 分别启动了 2 个容器。现在分别在两台机器上使用 curl 测试对方主机中 Pod 的 IP，检查网络连通性。

N1 测试 curl 10.42.1.17 和 10.42.1.18：