|
|
@@ -0,0 +1,476 @@
|
|
|
+---
|
|
|
+# Source: rook-ceph-cluster/templates/cephblockpool.yaml
|
|
|
+apiVersion: storage.k8s.io/v1
|
|
|
+kind: StorageClass
|
|
|
+metadata:
|
|
|
+ name: ceph-block
|
|
|
+ annotations:
|
|
|
+ storageclass.kubernetes.io/is-default-class: "true"
|
|
|
+provisioner: rook-ceph.rbd.csi.ceph.com
|
|
|
+parameters:
|
|
|
+ pool: ceph-blockpool
|
|
|
+ clusterID: rook-ceph
|
|
|
+
|
|
|
+ csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
|
|
|
+ csi.storage.k8s.io/controller-expand-secret-namespace: 'rook-ceph'
|
|
|
+ csi.storage.k8s.io/fstype: ext4
|
|
|
+ csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
|
|
|
+ csi.storage.k8s.io/node-stage-secret-namespace: 'rook-ceph'
|
|
|
+ csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
|
|
|
+ csi.storage.k8s.io/provisioner-secret-namespace: 'rook-ceph'
|
|
|
+ imageFeatures: layering
|
|
|
+ imageFormat: "2"
|
|
|
+
|
|
|
+reclaimPolicy: Delete
|
|
|
+allowVolumeExpansion: true
|
|
|
+volumeBindingMode: Immediate
|
|
|
+---
|
|
|
+# Source: rook-ceph-cluster/templates/cephfilesystem.yaml
|
|
|
+apiVersion: storage.k8s.io/v1
|
|
|
+kind: StorageClass
|
|
|
+metadata:
|
|
|
+ name: ceph-filesystem
|
|
|
+ annotations:
|
|
|
+ storageclass.kubernetes.io/is-default-class: "false"
|
|
|
+provisioner: rook-ceph.cephfs.csi.ceph.com
|
|
|
+parameters:
|
|
|
+ fsName: ceph-filesystem
|
|
|
+ pool: ceph-filesystem-data0
|
|
|
+ clusterID: rook-ceph
|
|
|
+
|
|
|
+ csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
|
|
|
+ csi.storage.k8s.io/controller-expand-secret-namespace: 'rook-ceph'
|
|
|
+ csi.storage.k8s.io/fstype: ext4
|
|
|
+ csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
|
|
|
+ csi.storage.k8s.io/node-stage-secret-namespace: 'rook-ceph'
|
|
|
+ csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
|
|
|
+ csi.storage.k8s.io/provisioner-secret-namespace: 'rook-ceph'
|
|
|
+
|
|
|
+reclaimPolicy: Delete
|
|
|
+allowVolumeExpansion: true
|
|
|
+volumeBindingMode: Immediate
|
|
|
+---
|
|
|
+# Source: rook-ceph-cluster/templates/cephobjectstore.yaml
|
|
|
+apiVersion: storage.k8s.io/v1
|
|
|
+kind: StorageClass
|
|
|
+metadata:
|
|
|
+ name: ceph-bucket
|
|
|
+provisioner: rook-ceph.ceph.rook.io/bucket
|
|
|
+reclaimPolicy: Delete
|
|
|
+volumeBindingMode: Immediate
|
|
|
+parameters:
|
|
|
+ objectStoreName: ceph-objectstore
|
|
|
+ objectStoreNamespace: rook-ceph
|
|
|
+
|
|
|
+ region: us-east-1
|
|
|
+---
|
|
|
+# Source: rook-ceph-cluster/templates/deployment.yaml
|
|
|
+apiVersion: apps/v1
|
|
|
+kind: Deployment
|
|
|
+metadata:
|
|
|
+ name: rook-ceph-tools
|
|
|
+ labels:
|
|
|
+ app: rook-ceph-tools
|
|
|
+spec:
|
|
|
+ replicas: 1
|
|
|
+ selector:
|
|
|
+ matchLabels:
|
|
|
+ app: rook-ceph-tools
|
|
|
+ template:
|
|
|
+ metadata:
|
|
|
+ labels:
|
|
|
+ app: rook-ceph-tools
|
|
|
+ spec:
|
|
|
+ dnsPolicy: ClusterFirstWithHostNet
|
|
|
+ containers:
|
|
|
+ - name: rook-ceph-tools
|
|
|
+ image: quay.io/ceph/ceph:v18.2.0
|
|
|
+ command:
|
|
|
+ - /bin/bash
|
|
|
+ - -c
|
|
|
+ - |
|
|
|
+ # Replicate the script from toolbox.sh inline so the ceph image
|
|
|
+ # can be run directly, instead of requiring the rook toolbox
|
|
|
+ CEPH_CONFIG="/etc/ceph/ceph.conf"
|
|
|
+ MON_CONFIG="/etc/rook/mon-endpoints"
|
|
|
+ KEYRING_FILE="/etc/ceph/keyring"
|
|
|
+
|
|
|
+ # create a ceph config file in its default location so ceph/rados tools can be used
|
|
|
+ # without specifying any arguments
|
|
|
+ write_endpoints() {
|
|
|
+ endpoints=$(cat ${MON_CONFIG})
|
|
|
+
|
|
|
+ # filter out the mon names
|
|
|
+ # external cluster can have numbers or hyphens in mon names, handling them in regex
|
|
|
+ # shellcheck disable=SC2001
|
|
|
+ mon_endpoints=$(echo "${endpoints}"| sed 's/[a-z0-9_-]\+=//g')
|
|
|
+
|
|
|
+ DATE=$(date)
|
|
|
+ echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
|
|
|
+ cat <<EOF > ${CEPH_CONFIG}
|
|
|
+ [global]
|
|
|
+ mon_host = ${mon_endpoints}
|
|
|
+
|
|
|
+ [client.admin]
|
|
|
+ keyring = ${KEYRING_FILE}
|
|
|
+ EOF
|
|
|
+ }
|
|
|
+
|
|
|
+ # watch the endpoints config file and update if the mon endpoints ever change
|
|
|
+ watch_endpoints() {
|
|
|
+ # get the timestamp for the target of the soft link
|
|
|
+ real_path=$(realpath ${MON_CONFIG})
|
|
|
+ initial_time=$(stat -c %Z "${real_path}")
|
|
|
+ while true; do
|
|
|
+ real_path=$(realpath ${MON_CONFIG})
|
|
|
+ latest_time=$(stat -c %Z "${real_path}")
|
|
|
+
|
|
|
+ if [[ "${latest_time}" != "${initial_time}" ]]; then
|
|
|
+ write_endpoints
|
|
|
+ initial_time=${latest_time}
|
|
|
+ fi
|
|
|
+
|
|
|
+ sleep 10
|
|
|
+ done
|
|
|
+ }
|
|
|
+
|
|
|
+ # read the secret from an env var (for backward compatibility), or from the secret file
|
|
|
+ ceph_secret=${ROOK_CEPH_SECRET}
|
|
|
+ if [[ "$ceph_secret" == "" ]]; then
|
|
|
+ ceph_secret=$(cat /var/lib/rook-ceph-mon/secret.keyring)
|
|
|
+ fi
|
|
|
+
|
|
|
+ # create the keyring file
|
|
|
+ cat <<EOF > ${KEYRING_FILE}
|
|
|
+ [${ROOK_CEPH_USERNAME}]
|
|
|
+ key = ${ROOK_CEPH_SECRET}
|
|
|
+ key = ${ceph_secret}
|
|
|
+ EOF
|
|
|
+
|
|
|
+ # write the initial config file
|
|
|
+ write_endpoints
|
|
|
+
|
|
|
+ # continuously update the mon endpoints if they fail over
|
|
|
+ watch_endpoints
|
|
|
+ imagePullPolicy: IfNotPresent
|
|
|
+ tty: true
|
|
|
+ env:
|
|
|
+ - name: ROOK_CEPH_USERNAME
|
|
|
+ valueFrom:
|
|
|
+ secretKeyRef:
|
|
|
+ name: rook-ceph-mon
|
|
|
+ key: ceph-username
|
|
|
+ resources:
|
|
|
+ limits:
|
|
|
+ cpu: 500m
|
|
|
+ memory: 1Gi
|
|
|
+ requests:
|
|
|
+ cpu: 100m
|
|
|
+ memory: 128Mi
|
|
|
+ volumeMounts:
|
|
|
+ - mountPath: /etc/ceph
|
|
|
+ name: ceph-config
|
|
|
+ - name: mon-endpoint-volume
|
|
|
+ mountPath: /etc/rook
|
|
|
+ - name: ceph-admin-secret
|
|
|
+ mountPath: /var/lib/rook-ceph-mon
|
|
|
+ volumes:
|
|
|
+ - name: ceph-admin-secret
|
|
|
+ secret:
|
|
|
+ secretName: rook-ceph-mon
|
|
|
+ optional: false
|
|
|
+ items:
|
|
|
+ - key: ceph-secret
|
|
|
+ path: secret.keyring
|
|
|
+ - name: mon-endpoint-volume
|
|
|
+ configMap:
|
|
|
+ name: rook-ceph-mon-endpoints
|
|
|
+ items:
|
|
|
+ - key: data
|
|
|
+ path: mon-endpoints
|
|
|
+ - name: ceph-config
|
|
|
+ emptyDir: {}
|
|
|
+ tolerations:
|
|
|
+ - key: "node.kubernetes.io/unreachable"
|
|
|
+ operator: "Exists"
|
|
|
+ effect: "NoExecute"
|
|
|
+ tolerationSeconds: 5
|
|
|
+---
|
|
|
+# Source: rook-ceph-cluster/templates/securityContextConstraints.yaml
|
|
|
+# scc for the Rook and Ceph daemons
|
|
|
+# for creating cluster in openshift
|
|
|
+---
|
|
|
+# Source: rook-ceph-cluster/templates/volumesnapshotclass.yaml
|
|
|
+---
|
|
|
+---
|
|
|
+# Source: rook-ceph-cluster/templates/cephblockpool.yaml
|
|
|
+apiVersion: ceph.rook.io/v1
|
|
|
+kind: CephBlockPool
|
|
|
+metadata:
|
|
|
+ name: ceph-blockpool
|
|
|
+spec:
|
|
|
+ failureDomain: host
|
|
|
+ replicated:
|
|
|
+ size: 3
|
|
|
+---
|
|
|
+# Source: rook-ceph-cluster/templates/cephcluster.yaml
|
|
|
+apiVersion: ceph.rook.io/v1
|
|
|
+kind: CephCluster
|
|
|
+metadata:
|
|
|
+ name: rook-ceph
|
|
|
+spec:
|
|
|
+ monitoring:
|
|
|
+ enabled: false
|
|
|
+
|
|
|
+ cephVersion:
|
|
|
+ allowUnsupported: false
|
|
|
+ image: quay.io/ceph/ceph:v18.2.0
|
|
|
+ cleanupPolicy:
|
|
|
+ allowUninstallWithVolumes: false
|
|
|
+ confirmation: ""
|
|
|
+ sanitizeDisks:
|
|
|
+ dataSource: zero
|
|
|
+ iteration: 1
|
|
|
+ method: quick
|
|
|
+ continueUpgradeAfterChecksEvenIfNotHealthy: false
|
|
|
+ crashCollector:
|
|
|
+ disable: true
|
|
|
+ dashboard:
|
|
|
+ enabled: true
|
|
|
+ port: 8080
|
|
|
+ ssl: false
|
|
|
+ urlPrefix: /ceph-dashboard
|
|
|
+ dataDirHostPath: /var/lib/rook
|
|
|
+ disruptionManagement:
|
|
|
+ managePodBudgets: true
|
|
|
+ osdMaintenanceTimeout: 30
|
|
|
+ pgHealthCheckTimeout: 0
|
|
|
+ healthCheck:
|
|
|
+ daemonHealth:
|
|
|
+ mon:
|
|
|
+ disabled: false
|
|
|
+ interval: 45s
|
|
|
+ osd:
|
|
|
+ disabled: false
|
|
|
+ interval: 60s
|
|
|
+ status:
|
|
|
+ disabled: false
|
|
|
+ interval: 60s
|
|
|
+ livenessProbe:
|
|
|
+ mgr:
|
|
|
+ disabled: false
|
|
|
+ mon:
|
|
|
+ disabled: false
|
|
|
+ osd:
|
|
|
+ disabled: false
|
|
|
+ logCollector:
|
|
|
+ enabled: true
|
|
|
+ maxLogSize: 500M
|
|
|
+ periodicity: daily
|
|
|
+ mgr:
|
|
|
+ allowMultiplePerNode: false
|
|
|
+ count: 2
|
|
|
+ modules:
|
|
|
+ - enabled: true
|
|
|
+ name: pg_autoscaler
|
|
|
+ - enabled: true
|
|
|
+ name: rook
|
|
|
+ mon:
|
|
|
+ allowMultiplePerNode: false
|
|
|
+ count: 3
|
|
|
+ network:
|
|
|
+ connections:
|
|
|
+ compression:
|
|
|
+ enabled: false
|
|
|
+ encryption:
|
|
|
+ enabled: false
|
|
|
+ hostNetwork: true
|
|
|
+ provider: host
|
|
|
+ requireMsgr2: false
|
|
|
+ hostNetwork: true
|
|
|
+ placement:
|
|
|
+ all:
|
|
|
+ nodeAffinity:
|
|
|
+ requiredDuringSchedulingIgnoredDuringExecution:
|
|
|
+ nodeSelectorTerms:
|
|
|
+ - matchExpressions:
|
|
|
+ - key: all
|
|
|
+ operator: In
|
|
|
+ values:
|
|
|
+ - all
|
|
|
+ podAffinity: null
|
|
|
+ podAntiAffinity: null
|
|
|
+ tolerations:
|
|
|
+ - key: all
|
|
|
+ operator: Exists
|
|
|
+ topologySpreadConstraints: null
|
|
|
+ mgr:
|
|
|
+ nodeAffinity:
|
|
|
+ requiredDuringSchedulingIgnoredDuringExecution:
|
|
|
+ nodeSelectorTerms:
|
|
|
+ - matchExpressions:
|
|
|
+ - key: mgr
|
|
|
+ operator: In
|
|
|
+ values:
|
|
|
+ - mgr
|
|
|
+ podAffinity: null
|
|
|
+ podAntiAffinity: null
|
|
|
+ tolerations:
|
|
|
+ - key: mgr
|
|
|
+ operator: Exists
|
|
|
+ topologySpreadConstraints: null
|
|
|
+ mon:
|
|
|
+ nodeAffinity:
|
|
|
+ requiredDuringSchedulingIgnoredDuringExecution:
|
|
|
+ nodeSelectorTerms:
|
|
|
+ - matchExpressions:
|
|
|
+ - key: mon
|
|
|
+ operator: In
|
|
|
+ values:
|
|
|
+ - mon
|
|
|
+ podAffinity: null
|
|
|
+ podAntiAffinity: null
|
|
|
+ tolerations:
|
|
|
+ - key: mon
|
|
|
+ operator: Exists
|
|
|
+ topologySpreadConstraints: null
|
|
|
+ osd:
|
|
|
+ nodeAffinity:
|
|
|
+ requiredDuringSchedulingIgnoredDuringExecution:
|
|
|
+ nodeSelectorTerms:
|
|
|
+ - matchExpressions:
|
|
|
+ - key: osd
|
|
|
+ operator: In
|
|
|
+ values:
|
|
|
+ - osd
|
|
|
+ podAffinity: null
|
|
|
+ podAntiAffinity: null
|
|
|
+ tolerations:
|
|
|
+ - key: osd
|
|
|
+ operator: Exists
|
|
|
+ topologySpreadConstraints: null
|
|
|
+ priorityClassNames:
|
|
|
+ mgr: system-cluster-critical
|
|
|
+ mon: system-node-critical
|
|
|
+ osd: system-node-critical
|
|
|
+ removeOSDsIfOutAndSafeToRemove: false
|
|
|
+ resources:
|
|
|
+ cleanup:
|
|
|
+ limits:
|
|
|
+ cpu: 500m
|
|
|
+ memory: 1Gi
|
|
|
+ requests:
|
|
|
+ cpu: 500m
|
|
|
+ memory: 100Mi
|
|
|
+ crashcollector:
|
|
|
+ limits:
|
|
|
+ cpu: 500m
|
|
|
+ memory: 60Mi
|
|
|
+ requests:
|
|
|
+ cpu: 100m
|
|
|
+ memory: 60Mi
|
|
|
+ exporter:
|
|
|
+ limits:
|
|
|
+ cpu: 250m
|
|
|
+ memory: 128Mi
|
|
|
+ requests:
|
|
|
+ cpu: 50m
|
|
|
+ memory: 50Mi
|
|
|
+ logcollector:
|
|
|
+ limits:
|
|
|
+ cpu: 500m
|
|
|
+ memory: 1Gi
|
|
|
+ requests:
|
|
|
+ cpu: 100m
|
|
|
+ memory: 100Mi
|
|
|
+ mgr:
|
|
|
+ limits:
|
|
|
+ cpu: 1000m
|
|
|
+ memory: 1Gi
|
|
|
+ requests:
|
|
|
+ cpu: 500m
|
|
|
+ memory: 512Mi
|
|
|
+ mgr-sidecar:
|
|
|
+ limits:
|
|
|
+ cpu: 500m
|
|
|
+ memory: 100Mi
|
|
|
+ requests:
|
|
|
+ cpu: 100m
|
|
|
+ memory: 40Mi
|
|
|
+ mon:
|
|
|
+ limits:
|
|
|
+ cpu: 2000m
|
|
|
+ memory: 2Gi
|
|
|
+ requests:
|
|
|
+ cpu: 1000m
|
|
|
+ memory: 1Gi
|
|
|
+ osd:
|
|
|
+ limits:
|
|
|
+ cpu: 2000m
|
|
|
+ memory: 2560Mi
|
|
|
+ requests:
|
|
|
+ cpu: 1000m
|
|
|
+ memory: 2560Mi
|
|
|
+ prepareosd:
|
|
|
+ requests:
|
|
|
+ cpu: 500m
|
|
|
+ memory: 50Mi
|
|
|
+ skipUpgradeChecks: false
|
|
|
+ storage:
|
|
|
+ useAllDevices: true
|
|
|
+ useAllNodes: true
|
|
|
+ waitTimeoutForHealthyOSDInMinutes: 10
|
|
|
+---
|
|
|
+# Source: rook-ceph-cluster/templates/cephfilesystem.yaml
|
|
|
+apiVersion: ceph.rook.io/v1
|
|
|
+kind: CephFilesystem
|
|
|
+metadata:
|
|
|
+ name: ceph-filesystem
|
|
|
+spec:
|
|
|
+ dataPools:
|
|
|
+ - failureDomain: host
|
|
|
+ name: data0
|
|
|
+ replicated:
|
|
|
+ size: 3
|
|
|
+ metadataPool:
|
|
|
+ replicated:
|
|
|
+ size: 3
|
|
|
+ metadataServer:
|
|
|
+ activeCount: 1
|
|
|
+ activeStandby: true
|
|
|
+ priorityClassName: system-cluster-critical
|
|
|
+ resources:
|
|
|
+ limits:
|
|
|
+ cpu: 2000m
|
|
|
+ memory: 4Gi
|
|
|
+ requests:
|
|
|
+ cpu: 1000m
|
|
|
+ memory: 4Gi
|
|
|
+---
|
|
|
+# Source: rook-ceph-cluster/templates/cephobjectstore.yaml
|
|
|
+apiVersion: ceph.rook.io/v1
|
|
|
+kind: CephObjectStore
|
|
|
+metadata:
|
|
|
+ name: ceph-objectstore
|
|
|
+spec:
|
|
|
+ dataPool:
|
|
|
+ erasureCoded:
|
|
|
+ codingChunks: 1
|
|
|
+ dataChunks: 2
|
|
|
+ failureDomain: host
|
|
|
+ gateway:
|
|
|
+ instances: 1
|
|
|
+ port: 80
|
|
|
+ priorityClassName: system-cluster-critical
|
|
|
+ resources:
|
|
|
+ limits:
|
|
|
+ cpu: 2000m
|
|
|
+ memory: 2Gi
|
|
|
+ requests:
|
|
|
+ cpu: 1000m
|
|
|
+ memory: 1Gi
|
|
|
+ metadataPool:
|
|
|
+ failureDomain: host
|
|
|
+ replicated:
|
|
|
+ size: 3
|
|
|
+ preservePoolsOnDelete: true
|
