在 GKE 上探索 Apache Solr Operator v0.3.0

kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.45.0/deploy/static/provider/cloud/deploy.yaml

kubectl get pods -l app.kubernetes.io/name=ingress-nginx -n ingress-nginx \
  --field-selector status.phase=Running

NAME                                        READY   STATUS    RESTARTS   AGE
ingress-nginx-controller-6c94f69c74-fxzp7   1/1     Running   0          6m23s

kubectl create ns sop030
kubectl config set-context --current --namespace=sop030

helm repo add apache-solr https://solr.apache.org/charts
helm repo update

kubectl create -f https://solr.apache.org/operator/downloads/crds/v0.3.0/all-with-dependencies.yaml

helm upgrade --install solr-operator apache-solr/solr-operator \
  --version 0.3.0

kubectl get pod -l control-plane=solr-operator
kubectl describe pod -l control-plane=solr-operator

kubectl get pod -l component=zookeeper-operator

# get a list of all CRDs in the cluster
kubectl get crds

# get details about the SolrCloud CRD Spec
kubectl explain solrclouds.spec
kubectl explain solrclouds.spec.solrImage

# get details about the SolrCloud CRD Status
kubectl explain solrclouds.status

apiVersion: solr.apache.org/v1beta1
kind: SolrCloud
metadata:
  name: explore
spec:
  customSolrKubeOptions:
    podOptions:
      resources:
        limits:
          memory: 3Gi
        requests:
          cpu: 700m
          memory: 3Gi
  dataStorage:
    persistent:
      pvcTemplate:
        spec:
          resources:
            requests:
              storage: 2Gi
      reclaimPolicy: Delete
  replicas: 3
  solrImage:
    repository: solr
    tag: 8.8.2
  solrJavaMem: -Xms500M -Xmx500M
  updateStrategy:
    method: StatefulSet
  zookeeperRef:
    provided:
      chroot: /explore
      image:
        pullPolicy: IfNotPresent
        repository: pravega/zookeeper
        tag: 0.2.9
      persistence:
        reclaimPolicy: Delete
        spec:
          accessModes:
          - ReadWriteOnce
          resources:
            requests:
              storage: 2Gi
      replicas: 3
      zookeeperPodPolicy:
        resources:
          limits:
            memory: 500Mi
          requests:
            cpu: 250m
            memory: 500Mi

kubectl logs -l control-plane=solr-operator -f

kubectl apply -f explore-SolrCloud.yaml

$ kubectl get solrclouds

NAME     VERSION   TARGETVERSION   DESIREDNODES   NODES   READYNODES   AGE
explore  8.8.2                     3              3       3            73s

$ kubectl get solrclouds explore -o yaml

kubectl get sts explore -o yaml

  updateStrategy:
    method: StatefulSet

---
apiVersion: v1
kind: Secret
metadata:
  name: pkcs12-keystore-password
stringData:
  password-key: Test1234

---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
  name: selfsigned-issuer
spec:
  selfSigned: {}

---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: explore-selfsigned-cert
spec:
  subject:
    organizations: ["self"]
  dnsNames:
    - localhost
  secretName: explore-selfsigned-cert-tls
  issuerRef:
    name: selfsigned-issuer
  keystores:
    pkcs12:
      create: true
      passwordSecretRef:
        key: password-key
        name: pkcs12-keystore-password

  keystores:
    pkcs12:
      create: true

kubectl get secret explore-selfsigned-cert-tls -o yaml

spec:
  ...

  solrAddressability:
    commonServicePort: 443
    external:
      domainName: YOUR_DOMAIN_NAME_HERE
      method: Ingress
      nodePortOverride: 443
      useExternalAddress: false
    podPort: 8983

  solrTLS:
    restartOnTLSSecretUpdate: true
    pkcs12Secret:
      name: explore-selfsigned-cert-tls
      key: keystore.p12
    keyStorePasswordSecret:
      name: pkcs12-keystore-password
      key: password-key

kubectl apply -f explore-SolrCloud.yaml

curl https://127.0.0.1:8983/solr/admin/info/system -k

---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
  name: acme-letsencrypt-issuer
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: *** REDACTED ***
    privateKeySecretRef:
      name: acme-letsencrypt-issuer-pk
    solvers:
    - dns01:
        cloudDNS:
          project: GCP_PROJECT
          serviceAccountSecretRef:
            name: clouddns-dns01-solver-svc-acct
            key: key.json

---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: explore-solr-tls-cert
spec:
  dnsNames:
  - YOUR_DOMAIN_NAME_HERE
  issuerRef:
    kind: Issuer
    name: acme-letsencrypt-issuer
  keystores:
    pkcs12:
      create: true
      passwordSecretRef:
        key: password-key
        name: pkcs12-keystore-password
  secretName: explore-solr-tls-letsencrypt
  subject:
    countries:
    - USA
    organizationalUnits:
    - k8s
    organizations:
    - solr

kubectl logs -l app.kubernetes.io/name=cert-manager -n cert-manager

spec:
  ...

  solrTLS:
    pkcs12Secret:
      name: explore-solr-tls-letsencrypt
      key: keystore.p12

spec:
  ...

  solrSecurity:
    authenticationType: Basic

kubectl get secret explore-solrcloud-security-bootstrap \
  -o jsonpath='{.data.admin}' | base64 --decode

kubectl get nodes -L topology.kubernetes.io/zone

spec:
  ...

  customSolrKubeOptions:
    podOptions:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: "technology"
                  operator: In
                  values:
                  - solr-cloud
                - key: "solr-cloud"
                  operator: In
                  values:
                  - explore
              topologyKey: topology.kubernetes.io/zone
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                - key: "technology"
                  operator: In
                  values:
                  - solr-cloud
                - key: "solr-cloud"
                  operator: In
                  values:
                  - explore
              topologyKey: kubernetes.io/hostname

kubectl get po -l solr-cloud=explore,technology=solr-cloud \
  -o json | jq -r '.items | sort_by(.spec.nodeName)[] | [.spec.nodeName] | @tsv' | uniq | wc -l

{"node": "#ANY", "shard": "#EACH", "replica":"<2"},

{"replica":"#EQUAL", "shard":"#EACH", "sysprop.availability_zone":"#EACH"},

spec:
  ...

  customSolrKubeOptions:
    podOptions:
      initContainers: # additional init containers for the Solr pods
        - name: set-zone # GKE specific, avoids giving get nodes permission to the service account
          image: curlimages/curl:latest
          command:
            - '/bin/sh'
            - '-c'
            - |
              zone=$(curl -sS http://metadata.google.internal/computeMetadata/v1/instance/zone -H 'Metadata-Flavor: Google')
              zone=${zone##*/}
              if [ "${zone}" != "" ]; then
                echo "export SOLR_OPTS=\"\${SOLR_OPTS} -Davailability_zone=${zone}\"" > /docker-entrypoint-initdb.d/set-zone.sh
              fi
          volumeMounts:
            - name: initdb
              mountPath: /docker-entrypoint-initdb.d
      volumes:
      - defaultContainerMount:
          mountPath: /docker-entrypoint-initdb.d
          name: initdb
        name: initdb
        source:
          emptyDir: {}

spec:
  ...

  updateStrategy:
    managed:
      maxPodsUnavailable: 2
      maxShardReplicasUnavailable: 2
    method: Managed

 solrReference:
    cloud:
      name: "explore"
    basicAuthSecret: explore-solrcloud-basic-auth
 solrTLS:
    restartOnTLSSecretUpdate: true
    pkcs12Secret:
      name: explore-selfsigned-cert-tls
      key: keystore.p12
    keyStorePasswordSecret:
      name: pkcs12-keystore-password
      key: password-key

kubectl get svc -l solr-prometheus-exporter=explore-prom-exporter

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: solr-metrics
  labels:
    release: prometheus-stack
spec:
  selector:
    matchLabels:
      solr-prometheus-exporter: explore-prom-exporter
  namespaceSelector:
    matchNames:
    - sop030
  endpoints:
  - port: solr-metrics
    interval: 15s

kubectl expose deployment prometheus-stack-grafana --type=LoadBalancer \
   --name=grafana -n monitoring

kubectl -n monitoring get service grafana \
  -o jsonpath='{.status.loadBalancer.ingress[0].ip}'

wget -q -O grafana-solr-dashboard.json \
  "https://raw.githubusercontent.com/apache/lucene-solr/branch_8x/solr/contrib/prometheus-exporter/conf/grafana-solr-dashboard.json"

shards.preference=node.sysprop:sysprop.availability_zone,replica.location:local

---
apiVersion: v1
kind: Secret
metadata:
  name: pkcs12-keystore-password
stringData:
  password-key: Test1234

---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
  name: selfsigned-issuer
spec:
  selfSigned: {}

---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: explore-selfsigned-cert
spec:
  subject:
    organizations: ["self"]
  dnsNames:
    - localhost
  secretName: explore-selfsigned-cert-tls
  issuerRef:
    name: selfsigned-issuer
  keystores:
    pkcs12:
      create: true
      passwordSecretRef:
        key: password-key
        name: pkcs12-keystore-password

---
apiVersion: solr.apache.org/v1beta1
kind: SolrCloud
metadata:
  name: explore
spec:
  customSolrKubeOptions:
    podOptions:
      resources:
        limits:
          memory: 3Gi
        requests:
          cpu: 700m
          memory: 3Gi
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: "technology"
                  operator: In
                  values:
                  - solr-cloud
                - key: "solr-cloud"
                  operator: In
                  values:
                  - explore
              topologyKey: topology.kubernetes.io/zone
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                - key: "technology"
                  operator: In
                  values:
                  - solr-cloud
                - key: "solr-cloud"
                  operator: In
                  values:
                  - explore
              topologyKey: kubernetes.io/hostname
      initContainers: # additional init containers for the Solr pods
        - name: set-zone # GKE specific, avoids giving get nodes permission to the service account
          image: curlimages/curl:latest
          command:
            - '/bin/sh'
            - '-c'
            - |
              zone=$(curl -sS http://metadata.google.internal/computeMetadata/v1/instance/zone -H 'Metadata-Flavor: Google')
              zone=${zone##*/}
              if [ "${zone}" != "" ]; then
                echo "export SOLR_OPTS=\"\${SOLR_OPTS} -Davailability_zone=${zone}\"" > /docker-entrypoint-initdb.d/set-zone.sh
              fi
          volumeMounts:
            - name: initdb
              mountPath: /docker-entrypoint-initdb.d
      volumes:
      - defaultContainerMount:
          mountPath: /docker-entrypoint-initdb.d
          name: initdb
        name: initdb
        source:
          emptyDir: {}

  dataStorage:
    persistent:
      pvcTemplate:
        spec:
          resources:
            requests:
              storage: 2Gi
      reclaimPolicy: Delete
  replicas: 3
  solrImage:
    repository: solr
    tag: 8.8.2
  solrJavaMem: -Xms500M -Xmx510M
  updateStrategy:
    managed:
      maxPodsUnavailable: 2
      maxShardReplicasUnavailable: 2
    method: Managed

  solrAddressability:
    commonServicePort: 443
    external:
      domainName: YOUR_DOMAIN_NAME_HERE
      method: Ingress
      nodePortOverride: 443
      useExternalAddress: false
    podPort: 8983

  solrTLS:
    restartOnTLSSecretUpdate: true
    pkcs12Secret:
      name: explore-selfsigned-cert-tls
      key: keystore.p12
    keyStorePasswordSecret:
      name: pkcs12-keystore-password
      key: password-key

  solrSecurity:
    authenticationType: Basic

  zookeeperRef:
    provided:
      chroot: /explore
      image:
        pullPolicy: IfNotPresent
        repository: pravega/zookeeper
        tag: 0.2.9
      persistence:
        reclaimPolicy: Delete
        spec:
          accessModes:
          - ReadWriteOnce
          resources:
            requests:
              storage: 2Gi
      replicas: 3
      zookeeperPodPolicy:
        resources:
          limits:
            memory: 500Mi
          requests:
            cpu: 250m
            memory: 500Mi

---
apiVersion: solr.apache.org/v1beta1
kind: SolrPrometheusExporter
metadata:
  labels:
    controller-tools.k8s.io: "1.0"
  name: explore-prom-exporter
spec:
  customKubeOptions:
    podOptions:
      resources:
        requests:
          cpu: 300m
          memory: 800Mi
  solrReference:
    cloud:
      name: "explore"
    basicAuthSecret: explore-solrcloud-basic-auth
    solrTLS:
      restartOnTLSSecretUpdate: true
      pkcs12Secret:
        name: explore-selfsigned-cert-tls
        key: keystore.p12
      keyStorePasswordSecret:
        name: pkcs12-keystore-password
        key: password-key
  numThreads: 6
  image:
    repository: solr
    tag: 8.8.2

---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: solr-metrics
  labels:
    release: prometheus-stack
spec:
  selector:
    matchLabels:
      solr-prometheus-exporter: explore-prom-exporter
  namespaceSelector:
    matchNames:
    - sop030
  endpoints:
  - port: solr-metrics
    interval: 15s