K8SPXC-1071 - ProxySQL - Add support for percona-scheduler-admin

proxysql/Dockerfile

@@ -11,7 +11,7 @@ FROM redhat/ubi8-minimal
 LABEL org.opencontainers.image.authors="[email protected]"
 
 ENV PXC_VERSION 8.0.30-22.1
-ENV PROXYSQL_VERSION 2.4.8-1.1
+ENV PROXYSQL_VERSION 2.5.1-1.1
 ENV OS_VER el8
 ENV FULL_PERCONA_VERSION "$PXC_VERSION.$OS_VER"
 ENV FULL_PROXYSQL_VERSION "$PROXYSQL_VERSION.$OS_VER"
@@ -81,8 +81,8 @@ RUN cp /usr/share/doc/proxysql2/LICENSE /licenses/LICENSE.proxysql
 
 COPY dockerdir /
 COPY --from=go_builder /go/src/github.com/percona/percona-xtradb-cluster-operator/src/peer-list /usr/bin/
-RUN chown 1001:1001 /etc/proxysql/proxysql.cnf /etc/proxysql-admin.cnf; \
-    chmod 664 /etc/proxysql/proxysql.cnf /etc/proxysql-admin.cnf
+RUN chown 1001:1001 /etc/proxysql/proxysql.cnf /etc/proxysql-admin.cnf /etc/config.toml; \
+    chmod 664 /etc/proxysql/proxysql.cnf /etc/proxysql-admin.cnf /etc/config.toml
 
 USER 1001
 

proxysql/Dockerfile.k8s

@@ -6,6 +6,19 @@ RUN export GO111MODULE=off \
     && curl -Lf -o /go/src/github.com/percona/percona-xtradb-cluster-operator/src/peer-list.go https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/main/cmd/peer-list/main.go \
     && go build peer-list.go
 
+FROM golang:1.19 AS go_builder_pxc_scheduler
+
+RUN git clone --branch=FR-72-Shunned https://github.com/percona/pxc_scheduler_handler.git /go/src/github.com/percona/pxc_scheduler_handler/
+
+WORKDIR /go/src/github.com/percona/pxc_scheduler_handler
+
+RUN go get github.com/Tusamarco/toml \
+    && go get github.com/go-sql-driver/mysql \
+    && go get github.com/sirupsen/logrus \
+    && go get golang.org/x/text/language \
+    && go get golang.org/x/text/message \
+    && go build -o pxc_scheduler_handler .
+
 FROM redhat/ubi8-minimal AS ubi8
 
 # Please don't remove old-style LABEL since it's needed for RedHat certification
@@ -97,8 +110,12 @@ RUN cp /usr/share/doc/proxysql2/LICENSE /licenses/LICENSE.proxysql
 
 COPY dockerdir /
 COPY --from=go_builder /go/src/github.com/percona/percona-xtradb-cluster-operator/src/peer-list /usr/bin/
-RUN chown 1001:1001 /etc/proxysql/proxysql.cnf /etc/proxysql-admin.cnf; \
-    chmod 664 /etc/proxysql/proxysql.cnf /etc/proxysql-admin.cnf
+
+RUN rm -rf /usr/bin/pxc_scheduler_handler
+COPY --from=go_builder_pxc_scheduler /go/src/github.com/percona/pxc_scheduler_handler/pxc_scheduler_handler /usr/bin/
+
+RUN chown 1001:1001 /etc/proxysql/proxysql.cnf /etc/proxysql-admin.cnf /etc/config.toml; \
+    chmod 664 /etc/proxysql/proxysql.cnf /etc/proxysql-admin.cnf /etc/config.toml
 
 USER 1001
 

proxysql/dockerdir/entrypoint.sh

@@ -3,11 +3,25 @@
 set -o xtrace
 
 PROXY_CFG=/etc/proxysql/proxysql.cnf
-PROXY_ADMIN_CFG=/etc/proxysql-admin.cnf
+# Percona scheduler
+PERCONA_SCHEDULER_CFG=/etc/config.toml
+TEMP_PROXY_SCHEDULER_CFG=$(mktemp)
+# internal scheduler
+PROXY_SCHEDULER_CFG=/etc/proxysql-admin.cnf
 
 MYSQL_INTERFACES='0.0.0.0:3306;0.0.0.0:33062'
 CLUSTER_PORT='33062'
-sed "s/#export WRITERS_ARE_READERS=.*$/export WRITERS_ARE_READERS='yes'/g" ${PROXY_ADMIN_CFG} 1<>${PROXY_ADMIN_CFG}
+# Percona scheduler
+sed "s/^writerIsAlsoReader.*=.*$/writerIsAlsoReader = 1/" ${PERCONA_SCHEDULER_CFG} | \
+sed "s/^hgW.*=.*$/hgW = 11/" | \
+sed "s/^hgR.*=.*$/hgR = 10/" | \
+sed "s/^clustered.*=.*false$/clustered = true/" | \
+sed 's/logLevel = "info"/logLevel = "debug"/g'  | \
+sed '/lockClusterTimeout/a lockrefreshtime=450'  | \
+sed "s/^failBack.*=.*false$/failBack = true/" > ${TEMP_PROXY_SCHEDULER_CFG}
+cp -f ${TEMP_PROXY_SCHEDULER_CFG} ${PERCONA_SCHEDULER_CFG}
+# internal scheduler
+sed "s/#export WRITERS_ARE_READERS=.*$/export WRITERS_ARE_READERS='yes'/g" ${PROXY_SCHEDULER_CFG} 1<> ${PROXY_SCHEDULER_CFG}
 
 sed "s/interfaces=\"0.0.0.0:3306\"/interfaces=\"${MYSQL_INTERFACES:-0.0.0.0:3306}\"/g" ${PROXY_CFG} 1<>${PROXY_CFG}
 sed "s/stacksize=1048576/stacksize=${MYSQL_STACKSIZE:-1048576}/g" ${PROXY_CFG} 1<>${PROXY_CFG}
@@ -18,54 +32,77 @@ OPERATOR_PASSWORD_ESCAPED=$(sed 's/[][\-\!\#\$\%\&\(\)\*\+\,\.\:\;\<\=\>\?\@\^\_
 MONITOR_PASSWORD_ESCAPED=$(sed 's/[][\-\!\#\$\%\&\(\)\*\+\,\.\:\;\<\=\>\?\@\^\_\~\{\}]/\\&/g' <<<"${MONITOR_PASSWORD}")
 PROXY_ADMIN_PASSWORD_ESCAPED=$(sed 's/[][\-\!\#\$\%\&\(\)\*\+\,\.\:\;\<\=\>\?\@\^\_\~\{\}]/\\&/g' <<<"${PROXY_ADMIN_PASSWORD}")
 
-sed "s/\"admin:admin\"/\"${PROXY_ADMIN_USER:-admin}:${PROXY_ADMIN_PASSWORD_ESCAPED:-admin}\"/g" ${PROXY_CFG} 1<>${PROXY_CFG}
-sed "s/cluster_username=\"admin\"/cluster_username=\"${PROXY_ADMIN_USER:-admin}\"/g" ${PROXY_CFG} 1<>${PROXY_CFG}
-sed "s/cluster_password=\"admin\"/cluster_password=\"${PROXY_ADMIN_PASSWORD_ESCAPED:-admin}\"/g" ${PROXY_CFG} 1<>${PROXY_CFG}
-sed "s/monitor_password=\"monitor\"/monitor_password=\"${MONITOR_PASSWORD_ESCAPED:-monitor}\"/g" ${PROXY_CFG} 1<>${PROXY_CFG}
-sed "s/PROXYSQL_USERNAME='admin'/PROXYSQL_USERNAME='${PROXY_ADMIN_USER:-admin}'/g" ${PROXY_ADMIN_CFG} 1<>${PROXY_ADMIN_CFG}
-sed "s/PROXYSQL_PASSWORD='admin'/PROXYSQL_PASSWORD='${PROXY_ADMIN_PASSWORD_ESCAPED:-admin}'/g" ${PROXY_ADMIN_CFG} 1<>${PROXY_ADMIN_CFG}
-sed "s/CLUSTER_USERNAME='admin'/CLUSTER_USERNAME='operator'/g" ${PROXY_ADMIN_CFG} 1<>${PROXY_ADMIN_CFG}
-sed "s/CLUSTER_PASSWORD='admin'/CLUSTER_PASSWORD='${OPERATOR_PASSWORD_ESCAPED:-operator}'/g" ${PROXY_ADMIN_CFG} 1<>${PROXY_ADMIN_CFG}
-sed "s/CLUSTER_PORT='3306'/CLUSTER_PORT='${CLUSTER_PORT:-3306}'/g" ${PROXY_ADMIN_CFG} 1<>${PROXY_ADMIN_CFG}
-sed "s/MONITOR_USERNAME='monitor'/MONITOR_USERNAME='monitor'/g" ${PROXY_ADMIN_CFG} 1<>${PROXY_ADMIN_CFG}
-sed "s/MONITOR_PASSWORD='monitor'/MONITOR_PASSWORD='${MONITOR_PASSWORD_ESCAPED:-monitor}'/g" ${PROXY_ADMIN_CFG} 1<>${PROXY_ADMIN_CFG}
+sed "s/\"admin:admin\"/\"${PROXY_ADMIN_USER:-admin}:${PROXY_ADMIN_PASSWORD_ESCAPED:-admin}\"/g"  ${PROXY_CFG} 1<> ${PROXY_CFG}
+sed "s/cluster_username=\"admin\"/cluster_username=\"${PROXY_ADMIN_USER:-admin}\"/g"     ${PROXY_CFG} 1<> ${PROXY_CFG}
+sed "s/cluster_password=\"admin\"/cluster_password=\"${PROXY_ADMIN_PASSWORD_ESCAPED:-admin}\"/g" ${PROXY_CFG} 1<> ${PROXY_CFG}
+sed "s/monitor_password=\"monitor\"/monitor_password=\"${MONITOR_PASSWORD_ESCAPED:-monitor}\"/g" ${PROXY_CFG} 1<> ${PROXY_CFG}
+
+# Percona scheduler
+sed "s/^host.*=.*\"$/host = '$(hostname -f)'/" ${PERCONA_SCHEDULER_CFG} | \
+sed "s/^user.*=.*\"$/user = '${PROXY_ADMIN_USER:-admin}'/" | \
+sed "s/^password.*=.*\"$/password = '${PROXY_ADMIN_PASSWORD_ESCAPED:-admin}'/" | \
+sed "s/^clusterUserPassword.*=.*\"$/clusterUserPassword='${OPERATOR_PASSWORD_ESCAPED:-operator}'/" | \
+sed "s/^clusterUser.*=.*\"$/clusterUser = 'operator'/" | \
+sed "s/^clusterPort.*=.*\"$/clusterPort='${CLUSTER_PORT:-3306}'/" | \
+sed "s/^monitorUserPassword.*=.*\"$/monitorUserPassword='${MONITOR_PASSWORD_ESCAPED:-monitor}'/" | \
+sed "s/^monitorUser.*=.*\"$/monitorUser='monitor'/" > ${TEMP_PROXY_SCHEDULER_CFG}
+cp -f ${TEMP_PROXY_SCHEDULER_CFG} ${PERCONA_SCHEDULER_CFG}
+# internal scheduler
+sed "s/PROXYSQL_USERNAME='admin'/PROXYSQL_USERNAME='${PROXY_ADMIN_USER:-admin}'/g"             ${PROXY_SCHEDULER_CFG} 1<> ${PROXY_SCHEDULER_CFG}
+sed "s/PROXYSQL_PASSWORD='admin'/PROXYSQL_PASSWORD='${PROXY_ADMIN_PASSWORD_ESCAPED:-admin}'/g" ${PROXY_SCHEDULER_CFG} 1<> ${PROXY_SCHEDULER_CFG}
+sed "s/CLUSTER_USERNAME='admin'/CLUSTER_USERNAME='operator'/g"                                 ${PROXY_SCHEDULER_CFG} 1<> ${PROXY_SCHEDULER_CFG}
+sed "s/CLUSTER_PASSWORD='admin'/CLUSTER_PASSWORD='${OPERATOR_PASSWORD_ESCAPED:-operator}'/g"   ${PROXY_SCHEDULER_CFG} 1<> ${PROXY_SCHEDULER_CFG}
+sed "s/CLUSTER_PORT='3306'/CLUSTER_PORT='${CLUSTER_PORT:-3306}'/g"                             ${PROXY_SCHEDULER_CFG} 1<> ${PROXY_SCHEDULER_CFG}
+sed "s/MONITOR_USERNAME='monitor'/MONITOR_USERNAME='monitor'/g"                                ${PROXY_SCHEDULER_CFG} 1<> ${PROXY_SCHEDULER_CFG}
+sed "s/MONITOR_PASSWORD='monitor'/MONITOR_PASSWORD='${MONITOR_PASSWORD_ESCAPED:-monitor}'/g"   ${PROXY_SCHEDULER_CFG} 1<> ${PROXY_SCHEDULER_CFG}
+
 set -o xtrace
 
 ## SSL/TLS support
 CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
 if [ -f "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" ]; then
-	CA=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
+    CA=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
 fi
 SSL_DIR=${SSL_DIR:-/etc/proxysql/ssl}
 if [ -f "${SSL_DIR}/ca.crt" ]; then
-	CA=${SSL_DIR}/ca.crt
+    CA=${SSL_DIR}/ca.crt
+    sed "s:^sslCertificatePath.*= .*\"$:sslCertificatePath = \"${SSL_DIR}\":" ${PERCONA_SCHEDULER_CFG} > ${TEMP_PROXY_SCHEDULER_CFG}
+    cp -f ${TEMP_PROXY_SCHEDULER_CFG} ${PERCONA_SCHEDULER_CFG}
 fi
 SSL_INTERNAL_DIR=${SSL_INTERNAL_DIR:-/etc/proxysql/ssl-internal}
 if [ -f "${SSL_INTERNAL_DIR}/ca.crt" ]; then
-	CA=${SSL_INTERNAL_DIR}/ca.crt
+    CA=${SSL_INTERNAL_DIR}/ca.crt
+    sed "s:^sslCertificatePath.*= .*\"$:sslCertificatePath = \"${SSL_INTERNAL_DIR}\":" ${PERCONA_SCHEDULER_CFG} > ${TEMP_PROXY_SCHEDULER_CFG}
+    cp -f ${TEMP_PROXY_SCHEDULER_CFG} ${PERCONA_SCHEDULER_CFG}
 fi
 
 KEY=${SSL_DIR}/tls.key
 CERT=${SSL_DIR}/tls.crt
 if [ -f "${SSL_INTERNAL_DIR}/tls.key" ] && [ -f "${SSL_INTERNAL_DIR}/tls.crt" ]; then
-	KEY=${SSL_INTERNAL_DIR}/tls.key
-	CERT=${SSL_INTERNAL_DIR}/tls.crt
+    KEY=${SSL_INTERNAL_DIR}/tls.key
+    CERT=${SSL_INTERNAL_DIR}/tls.crt
 fi
 
 if [ -f "$CA" ] && [ -f "$KEY" ] && [ -f "$CERT" ] && [ -n "$PXC_SERVICE" ]; then
-	sed "s^have_ssl=false^have_ssl=true^" ${PROXY_CFG} 1<>${PROXY_CFG}
-	sed "s^ssl_p2s_ca=\"\"^ssl_p2s_ca=\"$CA\"^" ${PROXY_CFG} 1<>${PROXY_CFG}
-	sed "s^ssl_p2s_ca=\"\"^ssl_p2s_ca=\"$CA\"^" ${PROXY_CFG} 1<>${PROXY_CFG}
-	sed "s^ssl_p2s_key=\"\"^ssl_p2s_key=\"$KEY\"^" ${PROXY_CFG} 1<>${PROXY_CFG}
-	sed "s^ssl_p2s_cert=\"\"^ssl_p2s_cert=\"$CERT\"^" ${PROXY_CFG} 1<>${PROXY_CFG}
+    sed "s^have_ssl=false^have_ssl=true^"                   ${PROXY_CFG} 1<> ${PROXY_CFG}
+    sed "s^ssl_p2s_ca=\"\"^ssl_p2s_ca=\"$CA\"^"             ${PROXY_CFG} 1<> ${PROXY_CFG}
+    sed "s^ssl_p2s_ca=\"\"^ssl_p2s_ca=\"$CA\"^"             ${PROXY_CFG} 1<> ${PROXY_CFG}
+    sed "s^ssl_p2s_key=\"\"^ssl_p2s_key=\"$KEY\"^"          ${PROXY_CFG} 1<> ${PROXY_CFG}
+    sed "s^ssl_p2s_cert=\"\"^ssl_p2s_cert=\"$CERT\"^"       ${PROXY_CFG} 1<> ${PROXY_CFG}
+    # Percona scheduler
+    sed "s:^sslCa.*=.*\"$:sslCa = \"${CA##*/}\":" ${PERCONA_SCHEDULER_CFG} | \
+    sed "s:^sslKey.*=.*\"$:sslKey = \"${KEY##*/}\":" | \
+    sed "s:^sslClient.*=.*\"$:sslClient = \"${CERT##*/}\":" > ${TEMP_PROXY_SCHEDULER_CFG}
+    cp -f ${TEMP_PROXY_SCHEDULER_CFG} ${PERCONA_SCHEDULER_CFG}
+    rm ${TEMP_PROXY_SCHEDULER_CFG}
 fi
 
 if [ -f "${SSL_DIR}/tls.key" ] && [ -f "${SSL_DIR}/tls.crt" ]; then
-	cp "${SSL_DIR}/tls.key" /var/lib/proxysql/proxysql-key.pem
-	cp "${SSL_DIR}/tls.crt" /var/lib/proxysql/proxysql-cert.pem
+    cp "${SSL_DIR}/tls.key" /var/lib/proxysql/proxysql-key.pem
+    cp "${SSL_DIR}/tls.crt" /var/lib/proxysql/proxysql-cert.pem
 fi
 if [ -f "${SSL_DIR}/ca.crt" ]; then
-	cp "${SSL_DIR}/ca.crt" /var/lib/proxysql/proxysql-ca.pem
+    cp "${SSL_DIR}/ca.crt" /var/lib/proxysql/proxysql-ca.pem
 fi
 
 test -e /opt/percona/hookscript/hook.sh && source /opt/percona/hookscript/hook.sh

proxysql/dockerdir/usr/bin/add_proxysql_nodes.sh

@@ -15,7 +15,12 @@ function proxysql_admin_exec() {
 function wait_for_proxysql() {
     local server="$1"
     echo "Waiting for host $server to be online..."
-    while [ "$(proxysql_admin_exec "$server" 'SELECT MAX(active) FROM runtime_mysql_galera_hostgroups')" != "1" ]
+    if [ "${PXC_HANDLER}" == "scheduler" ]; then
+        PROXYSQL_TABLE="runtime_scheduler"
+    else
+        PROXYSQL_TABLE="runtime_mysql_galera_hostgroups"
+    fi
+    while [ "$(proxysql_admin_exec "$server" "SELECT MAX(active) FROM ${PROXYSQL_TABLE}")" != "1" ]
     do
         echo "ProxySQL is not up yet... sleeping ..."
         sleep 1

proxysql/dockerdir/usr/bin/add_pxc_nodes.sh

@@ -39,6 +39,14 @@ function wait_for_proxy() {
     done
 }
 
+function exit_if_no_lock() {
+    lock=$(proxysql_admin_exec "127.0.0.1" "SELECT comment FROM runtime_proxysql_servers WHERE hostname LIKE '$HOSTNAME.%'")
+    if [ "${lock}" == "" ]; then
+        echo "I don't have the lock. Do nothing."
+        exit 0
+    fi
+}
+
 function main() {
     echo "Running $0"
 
@@ -55,35 +63,88 @@ function main() {
     wait_for_proxy
 
     SSL_ARG=""
+    temp=$(mktemp)
     if [ "$(proxysql_admin_exec "127.0.0.1" 'SELECT variable_value FROM global_variables WHERE variable_name="mysql-have_ssl"')" = "true" ]; then
-        SSL_ARG="--use-ssl=yes"
+        if [ "${PXC_HANDLER}" == "scheduler" ]; then
+            sed "s/^useSSL.*=.*$/useSSL=1/" /etc/config.toml > ${temp} && cp -f ${temp} /etc/config.toml
+        else
+            SSL_ARG="--use-ssl=yes"
+        fi
     fi
 
-    sed "s/WRITE_NODE=.*/WRITE_NODE='$pod_zero.$service:3306'/g" /etc/proxysql-admin.cnf 1<> /etc/proxysql-admin.cnf
-
-    proxysql-admin \
-        --config-file=/etc/proxysql-admin.cnf \
-        --cluster-hostname="$first_host" \
-        --enable \
-        --update-cluster \
-        --force \
-        --remove-all-servers \
-        --disable-updates \
-        --force \
-        $SSL_ARG
-
-    proxysql-admin \
-        --config-file=/etc/proxysql-admin.cnf \
-        --cluster-hostname="$first_host" \
-        --sync-multi-cluster-users \
-        --add-query-rule \
-        --disable-updates \
-        --force 
-
-    proxysql-admin \
-        --config-file=/etc/proxysql-admin.cnf \
-        --cluster-hostname="$first_host" \
-        --update-mysql-version
+    if [ "${PXC_HANDLER}" == "scheduler" ]; then
+        sed "s/^clusterHost.*=.*\"$/clusterHost=\"$first_host\"/" /etc/config.toml > ${temp} && cp -f ${temp} /etc/config.toml
+        rm ${temp}
+
+        set +o errexit
+        if proxysql-admin --config-file=/etc/proxysql-admin.cnf --is-enabled >/dev/null 2>&1; then
+            echo "Cleaning setup from proxysql-admin..."
+            proxysql-admin --config-file=/etc/proxysql-admin.cnf --disable
+        fi
+        set -o errexit
+
+        if [ "$(proxysql_admin_exec "127.0.0.1" 'SELECT count(*) FROM mysql_servers')" -eq 0 ]; then
+            percona-scheduler-admin \
+                --config-file=/etc/config.toml \
+                --write-node="$pod_zero.$service:3306" \
+                --enable \
+                --force
+        else
+            exit_if_no_lock
+
+            percona-scheduler-admin \
+                --config-file=/etc/config.toml \
+                --write-node="$pod_zero.$service:3306" \
+                --update-cluster \
+                --remove-all-servers \
+                --force
+
+            proxysql_admin_exec "127.0.0.1" 'LOAD MYSQL SERVERS TO RUNTIME;'
+        fi
+
+        exit_if_no_lock
+
+        percona-scheduler-admin \
+            --config-file=/etc/config.toml \
+            --write-node="$pod_zero.$service:3306" \
+            --sync-multi-cluster-users \
+            --add-query-rule \
+            --force
+
+        percona-scheduler-admin \
+            --config-file=/etc/config.toml \
+            --update-mysql-version
+    else
+        set +o errexit
+        if percona-scheduler-admin --config-file=/etc/config.toml --is-enabled >/dev/null 2>&1; then
+            echo "Cleaning setup from percona-scheduler-admin..."
+            percona-scheduler-admin --config-file=/etc/config.toml --disable
+        fi
+        set -o errexit
+
+        proxysql-admin \
+            --config-file=/etc/proxysql-admin.cnf \
+            --cluster-hostname="$first_host" \
+            --enable \
+            --update-cluster \
+            --force \
+            --remove-all-servers \
+            --disable-updates \
+            $SSL_ARG
+
+        proxysql-admin \
+            --config-file=/etc/proxysql-admin.cnf \
+            --cluster-hostname="$first_host" \
+            --sync-multi-cluster-users \
+            --add-query-rule \
+            --disable-updates \
+            --force
+
+        proxysql-admin \
+            --config-file=/etc/proxysql-admin.cnf \
+            --cluster-hostname="$first_host" \
+            --update-mysql-version
+    fi
 
     echo "All done!"
 }