在K8s上搭建MinIO集群（使用SSL）

系统运维 2023-09-22 三掌柜 手机阅读
Scan me!

部署MinIO Operator

# 下载MinIO的K8s插件
curl https://github.com/minio/operator/releases/download/v5.0.9/kubectl-minio_5.0.9_linux_amd64 -o kubectl-minio
 
# 添加执行权限
chmod +x kubectl-minio
 
# 移动到目录
mv kubectl-minio /usr/local/bin/
 
# 查看插件的版本
[root@node1 ~]# kubectl minio version
v5.0.9
[root@node1 ~]#

初始化 MinIO K8s Operator

执行kubectl minio init命令进行初始化，默认资源将会部署到minio-operator名称空间下

# --console-tls以tls方式部署暴露Operator console
 
[root@node1 ~]# kubectl minio init --console-tls
namespace/minio-operator created
serviceaccount/minio-operator created
clusterrole.rbac.authorization.k8s.io/minio-operator-role created
clusterrolebinding.rbac.authorization.k8s.io/minio-operator-binding created
customresourcedefinition.apiextensions.k8s.io/tenants.minio.min.io created
customresourcedefinition.apiextensions.k8s.io/policybindings.sts.min.io created
service/operator created
service/sts created
deployment.apps/minio-operator created
serviceaccount/console-sa created
secret/console-sa-secret created
clusterrole.rbac.authorization.k8s.io/console-sa-role created
clusterrolebinding.rbac.authorization.k8s.io/console-sa-binding created
configmap/console-env created
service/console created
deployment.apps/console created
-----------------
 
To open Operator UI, start a port forward using this command:
 
kubectl minio proxy -n minio-operator

验证查看资源的部署情况

[root@node1 ~]# kubectl get all --namespace minio-operator
NAME                                 READY   STATUS    RESTARTS   AGE
pod/console-6dc9887578-hccsx         1/1     Running   0          2m49s
pod/minio-operator-c9567668c-9rjw2   1/1     Running   0          2m49s
pod/minio-operator-c9567668c-nftmn   1/1     Running   0          2m49s
 
NAME               TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)             AGE
service/console    ClusterIP   10.103.70.66             9090/TCP,9443/TCP   2m49s
service/operator   ClusterIP   10.105.129.185           4221/TCP            2m50s
service/sts        ClusterIP   10.107.119.42            4223/TCP            2m50s
 
NAME                             READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/console          1/1     1            1           2m49s
deployment.apps/minio-operator   2/2     2            2           2m50s
 
NAME                                       DESIRED   CURRENT   READY   AGE
replicaset.apps/console-6dc9887578         1         1         1       2m49s
replicaset.apps/minio-operator-c9567668c   2         2         2       2m50s

临时启用流量转发来管理操作MinIO Operator console

[root@node1 ~]# kubectl minio proxy
Starting port forward of the Console UI.
 
To connect open a browser and go to http://localhost:9090
 
Current JWT to login: eyJhbGciOiJSUzI1NiIsImtpZCI6IkVoT2cwNVEwVDZGMWhCLXltRmZkMnJZczdLS0lTejZMalV3b3o4azNPdzQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJtaW5pby1vcGVyYXRvciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJjb25zb2xlLXNhLXNlY3fdfdfIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJjb25zb2xlLXNhIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiMTI4NTYzYjctZGU1My00MWE0LTk0ZjItNDNiM2NiNTVjNWVjIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Om1pbmlvLW9wZXJhdG9yOmNvbnNvbGUtc2EifQ.daFEPrOnBzbZvoERrkikNlNR8F3PGnKfLMXCK2oWB6LrPpo7Zdkw3fdfdubzBzKKK4u8Mvi3HFGp49E-S_GXPulaOcwy-JKoeYkTbe-wOOQMreeyZO5ENPiuYXVAuB-GRJhneQwTxKXc6uUQDQ3awHTeNlRokPuvy59tqkrGcQjiX5JOHVCOgzb9QHRSvF5uKV2GIprG9Se2kV18UKJxR3t5OQ5-EqSn35reZdckrtEi2U2e_JWY8SORHl6WsQVmYWricRpT5HlYZ_6SI23kwHjO3oXCeGWSdE-Ea-trF5dUXn06x3bVVSIoLqByW1MWBPgpB7_s_w
 
Forwarding from 0.0.0.0:9090 -> 9090

使用上面输出的JWT登陆访问页面

image.png

image.png

其他的一些选项可以根据自己的需要进行配置，配置完成后点击创建即可

创建后会有用户名和密码的提示

Snipaste_2023-09-22_21-09-38.png

在面板中还可以看到资源的使用情况

Snipaste_2023-09-22_21-16-24.png

查看该Tenant在k8s名称空间中的部署情况

[root@node1 ~]# kubectl get all -n my-minio -owide
NAME                    READY   STATUS    RESTARTS   AGE     IP            NODE      NOMINATED NODE   READINESS GATES
pod/my-minio-pool-0-0   2/2     Running   0          2m56s   10.244.4.40   node4                
pod/my-minio-pool-0-1   2/2     Running   0          2m56s   10.244.6.36   desktop              
pod/my-minio-pool-0-2   2/2     Running   0          2m56s   10.244.5.35   node5                
pod/my-minio-pool-0-3   2/2     Running   0          2m56s   10.244.3.67   node3                
 
NAME                       TYPE           CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE     SELECTOR
service/minio              LoadBalancer   10.98.230.179        443:30541/TCP    2m57s   v1.min.io/tenant=my-minio
service/my-minio-console   LoadBalancer   10.100.146.17        9443:31951/TCP   2m57s   v1.min.io/tenant=my-minio
service/my-minio-hl        ClusterIP      None                    9000/TCP         2m57s   v1.min.io/tenant=my-minio
 
NAME                               READY   AGE     CONTAINERS      IMAGES
statefulset.apps/my-minio-pool-0   4/4     2m56s   minio,sidecar   minio/minio:RELEASE.2023-09-20T22-49-55Z,minio/operator:v5.0.9

访问服务验证

使用前面创建Tenant时候的用户密码登陆，可以看到连接是ssl连接的

image.png

image.png

遇到的错误

在使用命令行客户端进行连接的时候会提示连接失败

[root@node1 ~]# mc alias set my-tenant https://10.100.146.17:9443 "QO9o1LMYiW7n6RNe" "fc09mCdOcr5tMVBltCPdTXKW3z4jZUn3" --api s3v4
mc:  Unable to initialize new alias from the provided credentials. Get "https://10.100.146.17:9443": tls: failed to verify certificate: x509: cannot validate certificate for 10.100.146.17 because it doesn't contain any IP SANs.

参考文章

  • min.io/docs/minio/…