diff --git a/docker-compose.yml b/docker-compose.yml index 5fa7283..9090eb1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,15 +14,20 @@ services: context: shutter args: ASSETS_VERSION: shutter-gnosis-1000-set1.3 # $NETWORK-10*$CHAIN_ID-set-$VERSION - UPSTREAM_VERSION: v1.3.6 + UPSTREAM_VERSION: v1.3.10 KEYPER_CONFIG_DIR: /keyper/config SHUTTER_CHAIN_DIR: /chain STAKER_SCRIPTS_VERSION: v0.1.0 restart: unless-stopped environment: - SHUTTER_GNOSIS_NODE_PRIVATEKEY: "" - SHUTTER_PUSH_METRICS_ENABLED: false - KEYPER_NAME: "" + - SHUTTER_GNOSIS_NODE_PRIVATEKEY="" + - SHUTTER_PUSH_METRICS_ENABLED=false + - KEYPER_NAME="" + - SHUTTER_PUSH_LOGS_ENABLED=false + - PUSHGATEWAY_USERNAME="" + - PUSHGATEWAY_PASSWORD="" + - ETHEREUM_WS= + - BEACON_HTTP= volumes: - chain:/chain - keyper-config:/keyper/config @@ -34,11 +39,11 @@ services: ASSETS_VERSION: shutter-gnosis-1000-set1.3 # $NETWORK-10*$CHAIN_ID-set-$VERSION restart: on-failure environment: - SHUTTER_PUSH_METRICS_ENABLED: false - KEYPER_NAME: "" - PUSHGATEWAY_URL: "https://keyperingest.metrics.shutter.network/api/v1/write" - PUSHGATEWAY_USERNAME: "" - PUSHGATEWAY_PASSWORD: "" + - SHUTTER_PUSH_METRICS_ENABLED=false + - KEYPER_NAME="" + - PUSHGATEWAY_URL="https://keyperingest.metrics.shutter.network/api/v1/write" + - PUSHGATEWAY_USERNAME="" + - PUSHGATEWAY_PASSWORD="" volumes: - metrics-config:/config diff --git a/setup-wizard.yml b/setup-wizard.yml index d134348..bf5a839 100644 --- a/setup-wizard.yml +++ b/setup-wizard.yml @@ -21,6 +21,39 @@ fields: required: false secret: true + + - id: external_ws_rpc + title: External WS RPC + description: | + A websocket connection to an external ethereum RPC (e.g. 'wss://some.external.url' or 'ws://1.2.3.4:8545'). If this is given, shutter will use this RPC to connect to the network. If not, shutter will try to use an RPC service on this dappnode. + target: + type: environment + name: ETHEREUM_WS + service: shutter + required: false + + - id: external_beacon_http_rpc + title: External Beacon HTTP RPC + description: | + A HTTP connection to an external ethereum beacon RPC (e.g. 'https://some.external.url' or 'http://1.2.3.4:4000'). If this is given, shutter will use this beacon api to connect to the network. If not, shutter will try to use an beacon api service on this dappnode. + target: + type: environment + name: BEACON_HTTP + service: shutter + required: false + + - id: enable_push_logs + title: Enable Push Logs + description: | + Enable the push logs feature to send logs to an external server controlled by Shutter. + target: + type: environment + name: SHUTTER_PUSH_LOGS_ENABLED + service: [shutter, metrics] + enum: + - "true" + - "false" + - id: enable_push_metrics title: Enable Push Metrics description: | @@ -51,7 +84,7 @@ fields: target: type: environment name: PUSHGATEWAY_USERNAME - service: metrics + service: [metrics, shutter] required: false if: { enable_push_metrics: { "enum": ["true"] } } @@ -62,7 +95,7 @@ fields: target: type: environment name: PUSHGATEWAY_PASSWORD - service: metrics + service: [metrics, shutter] required: false secret: true if: { enable_push_metrics: { "enum": ["true"] } } diff --git a/shutter/Dockerfile b/shutter/Dockerfile index c7784c9..963ada8 100644 --- a/shutter/Dockerfile +++ b/shutter/Dockerfile @@ -41,6 +41,7 @@ ADD ${STAKER_SCRIPTS_URL}/dvt_lsd_tools.sh /etc/profile.d/ COPY go-shutter-settings ${SHUTTER_SETTINGS_SRC_DIR} COPY supervisord.conf /etc/supervisord.conf +COPY promtail_config.yaml /etc/promtail_config.yaml RUN go build -C ${SHUTTER_SETTINGS_SRC_DIR} -o /usr/local/bin/go_shutter_settings @@ -48,8 +49,32 @@ RUN mkdir -p ${KEYPER_CONFIG_DIR} ${SHUTTER_CHAIN_DIR} ${ASSETS_DIR} /opt/superv chmod +rx /etc/profile.d/dvt_lsd_tools.sh COPY scripts /usr/local/bin/ +RUN chmod +x /usr/local/bin/*.sh COPY --from=assets ${ASSETS_DIR}/ ${ASSETS_DIR}/ +# For pushing logs to loki +RUN apt-get -y install wget gpg +RUN mkdir -p /etc/apt/keyrings/ +RUN wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor > /etc/apt/keyrings/grafana.gpg +RUN echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | tee /etc/apt/sources.list.d/grafana.list +# promtail & rotatelogs (from apache2) +RUN apt-get update && apt-get -y install promtail apache2 + +RUN apt-get update && apt-get install -y \ + ca-certificates \ + curl + +ARG NODE_VERSION=22.14.0 +ARG NODE_PACKAGE=node-v$NODE_VERSION-linux-x64 +ARG NODE_HOME=/opt/$NODE_PACKAGE + +ENV NODE_PATH $NODE_HOME/lib/node_modules +ENV PATH $NODE_HOME/bin:$PATH + +RUN curl https://nodejs.org/dist/v$NODE_VERSION/$NODE_PACKAGE.tar.gz | tar -xzC /opt/ + +RUN npm install -g wscat + # Placed here to rebuild less layers ENV CHAIN_PORT=${CHAIN_PORT} \ SHUTTER_P2P_LISTENADDRESSES="/ip4/0.0.0.0/tcp/${KEYPER_PORT},/ip4/0.0.0.0/udp/${KEYPER_PORT}/quic-v1" \ diff --git a/shutter/promtail_config.yaml b/shutter/promtail_config.yaml new file mode 100644 index 0000000..f241855 --- /dev/null +++ b/shutter/promtail_config.yaml @@ -0,0 +1,46 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +client: + url: https://logs.metrics.shutter.network/insert/loki/api/v1/push + basic_auth: + username: ${PUSHGATEWAY_USERNAME} + password: ${PUSHGATEWAY_PASSWORD} + +scrape_configs: + - job_name: configure + pipeline_stages: + - docker: + static_configs: + - targets: + - localhost + labels: + job: configure + host: ${KEYPER_NAME} + __path__: /tmp/configure.log + + - job_name: keyper + pipeline_stages: + - docker: + static_configs: + - targets: + - localhost + labels: + job: keyper + host: ${KEYPER_NAME} + __path__: /tmp/keyper.log + + - job_name: chain + pipeline_stages: + - docker: + static_configs: + - targets: + - localhost + labels: + job: chain + host: ${KEYPER_NAME} + __path__: /tmp/chain.log diff --git a/shutter/scripts/configure_keyper.sh b/shutter/scripts/configure_keyper.sh index e2363ae..bdc5de3 100755 --- a/shutter/scripts/configure_keyper.sh +++ b/shutter/scripts/configure_keyper.sh @@ -6,6 +6,43 @@ # shellcheck disable=SC1091 . "${ASSETS_DIR}/variables.env" +NODE_VERSION=22.14.0 +NODE_PACKAGE=node-v$NODE_VERSION-linux-x64 +NODE_HOME=/opt/$NODE_PACKAGE + +NODE_PATH=$NODE_HOME/lib/node_modules +PATH=$NODE_HOME/bin:$PATH + +function test_ethereum_url() { + # FIXME: This is a workaround for the issue with the staker-scripts@v0.1.1 not setting get_execution_ws_url_from_global_env correctly in the environment variables. + # Git Issue: https://github.com/dappnode/staker-package-scripts/issues/11 + export SHUTTER_GNOSIS_NODE_ETHEREUMURL=${ETHEREUM_WS:-$(get_execution_ws_url_from_global_env ${NETWORK} ${SUPPORTED_NETWORKS})} + RESULT=$(wscat -c "$SHUTTER_GNOSIS_NODE_ETHEREUMURL" -x '{"jsonrpc": "2.0", "method": "eth_syncing", "params": [], "id": 1}') + if [[ $RESULT =~ '"id":1' ]]; then return 0; else + export SHUTTER_GNOSIS_NODE_ETHEREUMURL=ws://execution.${NETWORK}.dncore.dappnode:8545 + RESULT=$(wscat -c "$SHUTTER_GNOSIS_NODE_ETHEREUMURL" -x '{"jsonrpc": "2.0", "method": "eth_syncing", "params": [], "id": 1}') + if [[ $RESULT =~ '"id":1' ]]; then return 0; else + echo "Could not find DAppNode RPC/WS url for this package!" + echo "Please configure 'ETHEREUM_WS' to point to an applicable websocket RPC service." + exit 1 + fi + fi +} + +function test_beacon_url() { + export SHUTTER_BEACONAPIURL=${BEACON_HTTP:-$(get_beacon_api_url_from_global_env "$NETWORK" "$SUPPORTED_NETWORKS")} + RESULT=$(curl -X GET "${SHUTTER_BEACONAPIURL}/eth/v1/beacon/genesis" -H "Accept: application/json") + if [[ $RESULT =~ '"genesis_time"' ]]; then return 0; else + export SHUTTER_BEACONAPIURL=http://beacon-chain.${NETWORK}.dncore.dappnode:4000 + RESULT=$(curl -X GET "${SHUTTER_BEACONAPIURL}/eth/v1/beacon/genesis" -H "Accept: application/json") + if [[ $RESULT =~ '"genesis_time"' ]]; then return 0; else + echo "Could not find DAppNode Beacon API url for this package!" + echo "Please configure 'BEACON_HTTP' to point to an applicable HTTP API service." + exit 1; + fi + fi +} + echo "[INFO | configure] Calculating keyper configuration values..." SUPPORTED_NETWORKS="gnosis" @@ -16,9 +53,16 @@ if [[ ! "$SHUTTER_P2P_LISTENADDRESSES" =~ ^\[.*\]$ ]]; then fi export SHUTTER_P2P_ADVERTISEADDRESSES="[\"/ip4/${_DAPPNODE_GLOBAL_PUBLIC_IP}/tcp/${KEYPER_PORT}\", \"/ip4/${_DAPPNODE_GLOBAL_PUBLIC_IP}/udp/${KEYPER_PORT}/quic-v1\"]" -export SHUTTER_BEACONAPIURL=$(get_beacon_api_url_from_global_env "$NETWORK" "$SUPPORTED_NETWORKS") + + +test_beacon_url +echo "[DEBUG | configure] SHUTTER_BEACONAPIURL is ${SHUTTER_BEACONAPIURL}" + export SHUTTER_GNOSIS_NODE_CONTRACTSURL=http://execution.gnosis.dncore.dappnode:8545 -export SHUTTER_GNOSIS_NODE_ETHEREUMURL=$(get_execution_ws_url_from_global_env "$NETWORK" "$SUPPORTED_NETWORKS") + +test_ethereum_url +echo "[DEBUG | configure] SHUTTER_GNOSIS_NODE_ETHEREUMURL is ${SHUTTER_GNOSIS_NODE_ETHEREUMURL}" + export VALIDATOR_PUBLIC_KEY=$(cat "${SHUTTER_CHAIN_DIR}/config/priv_validator_pubkey.hex") export SHUTTER_DISCOVERY_NAMESPACE="${_ASSETS_DISCOVERY_NAME_PREFIX}-${_ASSETS_INSTANCE_ID}" export SHUTTER_METRICS_ENABLED=${SHUTTER_PUSH_METRICS_ENABLED} diff --git a/shutter/scripts/run_chain.sh b/shutter/scripts/run_chain.sh index 3167447..c5e7265 100755 --- a/shutter/scripts/run_chain.sh +++ b/shutter/scripts/run_chain.sh @@ -3,8 +3,12 @@ run_chain() { echo "[INFO | chain] Starting chain..." - - $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE" + if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]]; + then + $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE" |& rotatelogs -n 1 -e -c /tmp/chain.log 5M + else + $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE" + fi } run_chain diff --git a/shutter/scripts/run_configure.sh b/shutter/scripts/run_configure.sh new file mode 100644 index 0000000..484dbf6 --- /dev/null +++ b/shutter/scripts/run_configure.sh @@ -0,0 +1,9 @@ +#!/bin/bash + + +if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]]; +then + configure.sh |& rotatelogs -n 1 -e -c /tmp/configure.log 5M +else + configure.sh +fi diff --git a/shutter/scripts/run_keyper.sh b/shutter/scripts/run_keyper.sh index b6f792c..b6484b0 100755 --- a/shutter/scripts/run_keyper.sh +++ b/shutter/scripts/run_keyper.sh @@ -18,7 +18,12 @@ perform_chain_healthcheck() { } run_keyper() { - $SHUTTER_BIN gnosiskeyper --config "$KEYPER_CONFIG_FILE" + if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]]; + then + $SHUTTER_BIN gnosiskeyper --config "$KEYPER_CONFIG_FILE" |& rotatelogs -n 1 -e -c /tmp/keyper.log 5M + else + $SHUTTER_BIN gnosiskeyper --config "$KEYPER_CONFIG_FILE" + fi } perform_chain_healthcheck diff --git a/shutter/scripts/run_promtail.sh b/shutter/scripts/run_promtail.sh new file mode 100644 index 0000000..f335d42 --- /dev/null +++ b/shutter/scripts/run_promtail.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +run_promtail() { + if [[ $SHUTTER_PUSH_LOGS_ENABLED=true ]]; + then + promtail -config.expand-env=true -log-config-reverse-order -print-config-stderr -config.file /etc/promtail_config.yaml + else + tail -f /dev/null + fi +} + +run_promtail diff --git a/shutter/supervisord.conf b/shutter/supervisord.conf index bfc6ccc..4c6b47c 100644 --- a/shutter/supervisord.conf +++ b/shutter/supervisord.conf @@ -18,7 +18,7 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface serverurl=unix:///var/run/supervisor.sock [program:configure] -command = configure.sh +command = run_configure.sh priority = 1 autostart = true autorestart = false @@ -47,4 +47,16 @@ autorestart = true stdout_logfile = /dev/stdout stdout_logfile_maxbytes = 0 stderr_logfile = /dev/stderr -stderr_logfile_maxbytes = 0 \ No newline at end of file +stderr_logfile_maxbytes = 0 + +[program:promtail] +command = run_promtail.sh ; we ingest 'rotatelogs' managed logfiles -- there is potential for some missed log lines. +priority = 4 +autostart = %(ENV_SHUTTER_PUSH_LOGS_ENABLED)s +startretries = 9999 ; A large number enough to cover node updates +autorestart = %(ENV_SHUTTER_PUSH_LOGS_ENABLED)s +environment = KEYPER_NAME="%(ENV_KEYPER_NAME)s",PUSHGATEWAY_USERNAME="%(ENV_PUSHGATEWAY_USERNAME)s",PUSHGATEWAY_PASSWORD="%(ENV_PUSHGATEWAY_PASSWORD)s",SHUTTER_PUSH_LOGS_ENABLED="%(ENV_SHUTTER_PUSH_LOGS_ENABLED)s" +stdout_logfile = /dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile = /dev/stderr +stderr_logfile_maxbytes = 0