k8s

容器国产化适配

龙芯k8s适配

Posted by 爱折腾的工程师 on Saturday, April 25, 2020

环境

  • cpu架构：mips64

  • 操作系统: CentOS 7

  • kernel版本：3.10.0-693.lns7.8.mips64el

  • docker： 18.06.3.ce

  • kubernetes: 1.15.9-0

部署准备

配置静态hosts解析

  1. 禁用selinux，防火墙，swap

    [root@compute ~]# sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
[root@compute ~]# setenforce 0

[root@compute ~]# getenforce 
Disabled

    关闭firewalld

    [root@compute ~]# systemctl status firewalld.service 
● firewalld.service - firewalld - dynamic firewall daemon
   Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
   Active: inactive (dead)
     Docs: man:firewalld(1)

    临时关闭swag

    [root@master ~]# swapoff -a

    编辑/etc/fstab,禁用swap分区的挂载

    [root@compute ~]# vim /etc/fstab
#UUID=c5b3070e-4f41-4282-bf1c-359b90ebfbb7 swap                    swap    defaults        0 0

    swappiness参数调整

    [root@master ~]# vim /etc/sysctl.d/k8s.conf
vm.swappiness=0
[root@master ~]# sysctl -p /etc/sysctl.d/k8s.conf

  2. master和node节点安装以下软件包

    下载地址：http://ftp.loongnix.org/os/loongnix-server/1.7/virt/mips64el/kubernetes115/

    # wget -c http://ftp.loongnix.org/os/loongnix-server/1.7/virt/mips64el/kubernetes115/kubectl-1.15.9-0.mips64el.rpm \
    http://ftp.loongnix.org/os/loongnix-server/1.7/virt/mips64el/kubernetes115/kubernetes-cni-0.7.5-0.mips64el.rpm \
    http://ftp.loongnix.org/os/loongnix-server/1.7/virt/mips64el/kubernetes115/kubelet-1.15.9-0.mips64el.rpm \
    http://ftp.loongnix.org/os/loongnix-server/1.7/virt/mips64el/kubernetes115/kubeadm-1.15.9-0.mips64el.rpm

    # rpm -ivh kubeadm-1.15.9-0.mips64el.rpm \
    kubectl-1.15.9-0.mips64el.rpm \
    kubelet-1.15.9-0.mips64el.rpm \
    kubernetes-cni-0.7.5-0.mips64el.rpm

  3. 所有节点安装docker

    # yum install -y docker-ce socat

    # vim /usr/lib/systemd/system/docker.service 
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=-/etc/sysconfig/docker
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=1048576
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=always
StartLimitBurst=3
StartLimitInterval=60s

[Install]
WantedBy=multi-user.target

    # systemctl daemon-reload 
# systemctl start docker
# systemctl enable docker.service

# systemctl enable kubelet.service

部署

kubeadm init --pod-network-cidr=10.244.0.0/16 \
    --apiserver-advertise-address=<你的apiserver-ip> \
    --kubernetes-version=v1.15.9 \
    --image-repository docker.io/loongnixk8s

只有一个节点，去除污点，让master也可以跑负载

kubectl taint node compute node-role.kubernetes.io/master-

安装flannel

# mkdir -p /etc/cni/net.d/

[root@compute ~]# vim kube-flannel.yml 
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
rules:
  - apiGroups:
      - ""
    resources:
      - pods
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes/status
    verbs:
      - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "host-gw"
      }
    }
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      hostNetwork: true
      nodeSelector:
        beta.kubernetes.io/arch: mips64le
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: loongnixk8s/flannel-mips64le:0.10.0 
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: loongnixk8s/flannel-mips64le:0.10.0 
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: true
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg

[root@compute ~]# kubectl apply -f kube-flannel.yml 

[root@compute ~]# kubectl get nodes
NAME      STATUS   ROLES    AGE   VERSION
compute   Ready    master   14m   v1.15.9

测试

# kubectl run nginx --image=loongnixk8s/nginx:1.17.7 --replicas=1

[root@compute ~]# kubectl get po
NAME                     READY   STATUS    RESTARTS   AGE
nginx-7c4577b99b-qsl4b   1/1     Running   0          77s

[root@compute ~]# kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort
service/nginx exposed

[root@compute ~]# kubectl get svc nginx
NAME    TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE
nginx   NodePort   10.97.240.70   <none>        88:30296/TCP   5s

[root@compute ~]# curl http://<your-apiserver-ip>:30296/
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

参考链接

「真诚赞赏，手留余香」

爱折腾的工程师

真诚赞赏，手留余香

使用微信扫描二维码完成支付

CATALOG
    FEATURED TAGS
    devops go k8s llm openstack tkestack 练车