From 9d73c346b7ea5510f2f62cfd0d1f2e1e3f8f1a90 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Wed, 3 Sep 2025 13:48:38 -0700 Subject: [PATCH] new modified startup script for central controller docker images with support for configuring bigtable, pubsub, redis --- .../Dockerfile.conda | 2 +- ext/central-controller-docker/main-new.sh | 166 ++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) create mode 100755 ext/central-controller-docker/main-new.sh diff --git a/ext/central-controller-docker/Dockerfile.conda b/ext/central-controller-docker/Dockerfile.conda index 311dd8705..844e01ec2 100644 --- a/ext/central-controller-docker/Dockerfile.conda +++ b/ext/central-controller-docker/Dockerfile.conda @@ -17,7 +17,7 @@ ADD build/zerotier-one /usr/local/bin/zerotier-one RUN chmod a+x /usr/local/bin/zerotier-one RUN echo "/opt/conda/envs/central_controller/lib" > /etc/ld.so.conf.d/conda-central-controller.conf && ldconfig -ADD ext/central-controller-docker/main.sh / +ADD ext/central-controller-docker/main-new.sh /main.sh RUN chmod a+x /main.sh ENTRYPOINT ["/main.sh"] diff --git a/ext/central-controller-docker/main-new.sh b/ext/central-controller-docker/main-new.sh new file mode 100755 index 000000000..f3e48b324 --- /dev/null +++ b/ext/central-controller-docker/main-new.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +conda activate central_controller + +if [ -z "$ZT_DB_HOST" ]; then + echo '*** FAILED: ZT_DB_HOST environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_PORT" ]; then + echo '*** FAILED: ZT_DB_PORT environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_NAME" ]; then + echo '*** FAILED: ZT_DB_NAME environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_USER" ]; then + echo '*** FAILED: ZT_DB_USER environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_PASSWORD" ]; then + echo '*** FAILED: ZT_DB_PASSWORD environment variable not defined' + exit 1 +fi + +REDIS="" +if [ "$ZT_USE_REDIS" == "true" ]; then + if [ -z "$ZT_REDIS_HOST" ]; then + echo '*** FAILED: ZT_REDIS_HOST environment variable not defined' + exit 1 + fi + + if [ -z "$ZT_REDIS_PORT" ]; then + echo '*** FAILED: ZT_REDIS_PORT enivronment variable not defined' + exit 1 + fi + + if [ -z "$ZT_REDIS_CLUSTER_MODE" ]; then + echo '*** FAILED: ZT_REDIS_CLUSTER_MODE environment variable not defined' + exit 1 + fi + + REDIS=", \"redis\": { + \"hostname\": \"${ZT_REDIS_HOST}\", + \"port\": ${ZT_REDIS_PORT}, + \"clusterMode\": ${ZT_REDIS_CLUSTER_MODE}, + \"password\": \"${ZT_REDIS_PASSWORD}\" + } + " +else + REDIS="\"redis\": null" +fi + +mkdir -p /var/lib/zerotier-one + +pushd /var/lib/zerotier-one +if [ -d "$ZT_IDENTITY_PATH" ]; then + echo '*** Using existing ZT identity from path $ZT_IDENTITY_PATH' + + ln -s $ZT_IDENTITY_PATH/identity.public identity.public + ln -s $ZT_IDENTITY_PATH/identity.secret identity.secret + if [ -L "$ZT_IDENTITY_PATH/authtoken.secret" ] && [ -e "$ZT_IDENTITY_PATH/authtoken.secret" ]; then + ln -s $ZT_IDENTITY_PATH/authtoken.secret authtoken.secret + ln -s $ZT_IDENTITY_PATH/authtoken.secret metricstoken.secret + fi +fi +popd + +DEFAULT_PORT=9993 +DEFAULT_LB_MODE=false + +APP_NAME="controller-$(cat /var/lib/zerotier-one/identity.public | cut -d ':' -f 1)" + +BIGTABLE_CONF="" +if [ "$ZT_USE_BIGTABLE" == "true" ]; then + if [ -z "$ZT_BIGTABLE_PROJECT" ] || [ -z "$ZT_BIGTABLE_INSTANCE" ] || [ -z "$ZT_BIGTABLE_TABLE" ]; then + echo '*** FAILED: ZT_BIGTABLE_PROJECT, ZT_BIGTABLE_INSTANCE, and ZT_BIGTABLE_TABLE environment variables must all be defined to use Bigtable as a controller backend' + exit 1 + fi + + BIGTABLE_CONF=", \"bigtable\": { + \"project\": \"${ZT_BIGTABLE_PROJECT}\", + \"instance\": \"${ZT_BIGTABLE_INSTANCE}\", + \"table\": \"${ZT_BIGTABLE_TABLE}\" + }, + " +fi + + +PUBSUB_CONF="" +if [ "$ZT_USE_PUBSUB" == "true" ]; then + if [ -z "$ZT_PUBSUB_PROJECT" ]; then + echo '*** FAILED: ZT_PUBSUB_PROJECT environment variable must be defined to use PubSub as a controller backend' + exit 1 + fi + + PUBSUB_CONF=", \"pubsub\": { + \"project\": \"${ZT_CTL_PUBSUB_PROJECT}\" + } + " +fi + +echo "{ + \"settings\": { + \"controllerDbPath\": \"postgres:host=${ZT_DB_HOST} port=${ZT_DB_PORT} dbname=${ZT_DB_NAME} user=${ZT_DB_USER} password=${ZT_DB_PASSWORD} application_name=${APP_NAME} sslmode=prefer sslcert=${DB_CLIENT_CERT} sslkey=${DB_CLIENT_KEY} sslrootcert=${DB_SERVER_CA}\", + \"portMappingEnabled\": true, + \"softwareUpdate\": \"disable\", + \"interfacePrefixBlacklist\": [ + \"inot\", + \"nat64\" + ], + \"lowBandwidthMode\": ${ZT_LB_MODE:-$DEFAULT_LB_MODE}, + \"ssoRedirectURL\": \"${ZT_SSO_REDIRECT_URL}\", + \"allowManagementFrom\": [\"127.0.0.1\", \"::1\", \"10.0.0.0/8\"], + \"otel\": { + \"exporterEndpoint\": \"${ZT_EXPORTER_ENDPOINT}\", + \"exporterSampleRate\": ${ZT_EXPORTER_SAMPLE_RATE:-0} + } + ${REDIS} + }, + \"controller\": { + \"listenMode\": \"${ZT_LISTEN_MODE:-pgsql}\", + \"statusMode\": \"`${ZT_STATUS_MODE:-pgsql}\" + ${REDIS} + ${BIGTABLE_CONF} + ${PUBSUB_CONF} + } +} +" > /var/lib/zerotier-one/local.conf + +if [ -n "$DB_SERVER_CA" ]; then + echo "secret list" + chmod 600 /secrets/db/*.pem + ls -l /secrets/db/ + until /usr/bin/pg_isready -h ${ZT_DB_HOST} -p ${ZT_DB_PORT} -d "sslmode=prefer sslcert=${DB_CLIENT_CERT} sslkey=${DB_CLIENT_KEY} sslrootcert=${DB_SERVER_CA}"; do + echo "Waiting for PostgreSQL..."; + sleep 2; + done +else + until /usr/bin/pg_isready -h ${ZT_DB_HOST} -p ${ZT_DB_PORT}; do + echo "Waiting for PostgreSQL..."; + sleep 2; + done +fi + + +echo "Migrating database (if needed)..." +if [ -n "$DB_SERVER_CA" ]; then + /usr/local/bin/migrate -source file:///migrations -database "postgres://$ZT_DB_USER:$ZT_DB_PASSWORD@$ZT_DB_HOST:$ZT_DB_PORT/$ZT_DB_NAME?x-migrations-table=controller_migrations&sslmode=verify-full&sslrootcert=$DB_SERVER_CA&sslcert=$DB_CLIENT_CERT&sslkey=$DB_CLIENT_KEY" up +else + /usr/local/bin/migrate -source file:///migrations -database "postgres://$ZT_DB_USER:$ZT_DB_PASSWORD@$ZT_DB_HOST:$ZT_DB_PORT/$ZT_DB_NAME?x-migrations-table=controller_migrations&sslmode=disable" up +fi + +if [ -n "$ZT_TEMPORAL_HOST" ] && [ -n "$ZT_TEMPORAL_PORT" ]; then + echo "waiting for temporal..." + while ! nc -z ${ZT_TEMPORAL_HOST} ${ZT_TEMPORAL_PORT}; do + echo "waiting..."; + sleep 1; + done + echo "Temporal is up" +fi + +export GLIBCXX_FORCE_NEW=1 +export GLIBCPP_FORCE_NEW=1 +export LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.2" +exec /usr/local/bin/zerotier-one -p${ZT_CONTROLLER_PORT:-$DEFAULT_PORT} /var/lib/zerotier-one