diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..e69de29 diff --git a/debian/conffiles b/debian/conffiles new file mode 100644 index 0000000..d567f72 --- /dev/null +++ b/debian/conffiles @@ -0,0 +1,3 @@ +/etc/randao/.env +/etc/randao/wallet.json +/etc/randao/wallet.seed \ No newline at end of file diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..e69de29 diff --git a/debian/etc/systemd/system/randao.service b/debian/etc/systemd/system/randao.service new file mode 100644 index 0000000..eca398f --- /dev/null +++ b/debian/etc/systemd/system/randao.service @@ -0,0 +1,22 @@ +[Unit] +Description=RANDAO Provider +# When running manually, use the --no-block flag: +# systemctl start --no-block randao.service +Requires=docker.service +After=docker.service +#Wants-randao.timer + +[Service] +Type=simple +User=randao_service +Group=randao_service +WorkingDirectory=/home/randao/RandaoProvider/docker-compose +#ExecStart=/usr/bin/docker compose --env-file /etc/randao/.env up --pull=always +ExecStart=/usr/bin/docker compose -f docker-compose.yml -f docker-compose.appliance.yml --env-file /etc/randao/.env up --pull=always +ExecStop=/usr/bin/docker compose down +TimeoutStartSec=0 +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/debian/etc/systemd/system/randao.timer b/debian/etc/systemd/system/randao.timer new file mode 100644 index 0000000..958852a --- /dev/null +++ b/debian/etc/systemd/system/randao.timer @@ -0,0 +1,22 @@ +Description=Timer to periodically restart RANDAO Provider for latest image pull +#Requires=randao.service +#After=randao.service + +[Timer] +# Restart every day at a random time within the first hour of the day +# OnCalendar=daily +# RandomizedDelaySec=1h +# Persistent=true + +# OR, restart every 12 hours (e.g., 00:00, 12:00) with a random delay +OnCalendar=*-*-* 00,12:00:00 +RandomizedDelaySec=30min +Persistent=true + +# OR, restart every 8 hours from when the service last became active +#OnUnitActiveSec=8h +#RandomizedDelaySec=30min +#AccuracySec=1min # Optional: Reduce timer inaccuracy from default (often 1min) + +[Install] +WantedBy=timers.target diff --git a/debian/etc/update-motd.d/66-randao b/debian/etc/update-motd.d/66-randao new file mode 100644 index 0000000..4a80554 --- /dev/null +++ b/debian/etc/update-motd.d/66-randao @@ -0,0 +1,59 @@ +#!/bin/bash +THIS_SCRIPT="randao" +MOTD_DISABLE="" + +[[ -f /etc/default/armbian-motd ]] && . /etc/default/armbian-motd + +for f in $MOTD_DISABLE; do + [[ $f == $THIS_SCRIPT ]] && exit 0 +done + +## FORMATTING VARIABLES +# --- ANSI Color Codes --- +RED=$'\e[31m' +GREEN=$'\e[32m' +BLUE=$'\e[94m' +YELLOW=$'\e[93m' +GOLD=$'\e[38;5;214m' +ORANGE=$'\e[38;5;208m' +MAGENTA=$'\e[35m' +CYAN=$'\e[36m' + +# if using color with effects, use color first and then the effect. The color codes above reset the effect to none. +BOLD=$'\e[1m' # bold +ITAL=$'\e[3m' # italics +ULINE=$'\e[4m' # underline +XOUT=$'\e[9m' # crossed out +REV=$'\e[7m' # reversed +NC=$'\e[0m' # No Color (resets to default) + + +#servicelist=("randao.service" "randao.timer") +servicelist=("randao.timer") + +for service in "${servicelist[@]}"; do + if [[ -f "/etc/systemd/system/${service}" ]]; then + serviceEnabled=$(systemctl is-enabled $service) + if [[ ${serviceEnabled} == "enabled" ]]; then + serviceEnabled="${BOLD}${GREEN}enabled${NC}" + serviceMessage="" + else + serviceEnabled="${BOLD}${YELLOW}disabled${NC}" + serviceMessage="To enable ${BOLD}${service}${NC}: sudo systemctl enable ${service}" + fi + + serviceActive=$(systemctl is-active $service) + if [[ ${serviceActive} == "active" ]]; then + # service is active + serviceActive="${BOLD}${GREEN}active${NC}" + serviceMessage+="" + else + # service is in-active + serviceActive="${BOLD}${YELLOW}inactive${NC}" + serviceMessage+="\nThen start ${BOLD}${service}${NC}: sudo systemctl start ${service}" + fi + + echo -e "${BOLD}${service} is $serviceEnabled and $serviceActive." + echo -e "${serviceMessage}\n" + fi +done diff --git a/debian/install b/debian/install new file mode 100644 index 0000000..ffaaef1 --- /dev/null +++ b/debian/install @@ -0,0 +1,15 @@ +# Source files from your Git repo (relative to repo root) +# Destination on target system (relative to /) + +# Systemd service and timer files +debian/etc/systemd/system/randao.service /etc/systemd/system/ +debian/etc/systemd/system/randao.timer /etc/systemd/system/ +debian/etc/update-motd.d/66-randao /etc/update-motd.d/66-randao + +# Docker Compose project files +docker-compose/ /opt/randao-provider/ +orchestrator/ /opt/randao-provider/ +puzzle-generator/ /opt/randao-provider/ +requester/ /opt/randao-provider/ +LICENSE /opt/randao-provider/ +README.md /opt/randao-provider/ \ No newline at end of file diff --git a/debian/postinst b/debian/postinst new file mode 100644 index 0000000..c97fde1 --- /dev/null +++ b/debian/postinst @@ -0,0 +1,99 @@ +#!/bin/sh +# postinst script for randao-provider Debian package + +set -e # Exit immediately if a command exits with a non-zero status + +# --- 1. Define Paths and Logging --- +LOG_FILE="/var/log/randao-provider-postinst.log" +USERNAME="randao_service" +GROUPNAME="randao_service" +DOCKER_GROUP="docker" +ETC_RANDAO_DIR="/etc/randao" +APP_ROOT_DIR="/opt/randao-provider" + +# Initialize log file with secure permissions +touch "$LOG_FILE" +chmod 600 "$LOG_FILE" + +log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - postinst: $1" >> "$LOG_FILE" +} + +log_message "Starting randao-provider post-installation script." + +# --- 2. Create the dedicated system user and group --- +log_message "Checking for user '$USERNAME' and group '$GROUPNAME'." +if ! id -u "$USERNAME" >/dev/null 2>&1; then + log_message "Creating system user '$USERNAME' with a dynamic UID." + # Let the system pick a safe UID automatically + adduser --system --no-create-home --group "$USERNAME" + log_message "User '$USERNAME' created." +else + log_message "User '$USERNAME' already exists. Skipping creation." +fi + +# Add the user to the docker group if it exists +if getent group "$DOCKER_GROUP" >/dev/null 2>&1; then + if ! getent group "$DOCKER_GROUP" | grep -q "\b$USERNAME\b"; then + log_message "Adding user '$USERNAME' to group '$DOCKER_GROUP'." + usermod -aG "$DOCKER_GROUP" "$USERNAME" + log_message "User '$USERNAME' added to '$DOCKER_GROUP'." + else + log_message "User '$USERNAME' is already in group '$DOCKER_GROUP'." + fi +else + log_message "WARNING: Group '$DOCKER_GROUP' does not exist. The service may not function correctly." +fi + +# --- 3. Manage Configuration --- +# NOTE: This section is ideally replaced by using a 'conffiles' file. +# The logic is kept here assuming you are not using conffiles yet. +log_message "Ensuring config directory '$ETC_RANDAO_DIR' exists." +mkdir -p "$ETC_RANDAO_DIR" +chown root:root "$ETC_RANDAO_DIR" +chmod 700 "$ETC_RANDAO_DIR" # Only root can access the directory listing + +TEMPLATE_DIR="$APP_ROOT_DIR/docker-compose/templates" + +# Safely copy example config files if they don't already exist +if [ ! -f "$ETC_RANDAO_DIR/.env" ]; then + log_message "Copying example.env to $ETC_RANDAO_DIR/.env." + cp "$TEMPLATE_DIR/example.env" "$ETC_RANDAO_DIR/.env" +fi +if [ ! -f "$ETC_RANDAO_DIR/wallet.json" ]; then + log_message "Copying example.wallet.json to $ETC_RANDAO_DIR/wallet.json." + cp "$TEMPLATE_DIR/example.wallet.json" "$ETC_RANDAO_DIR/wallet.json" +fi + +# --- 4. Set Secure Permissions for Configuration Files --- +# Set permissions on any config file that exists in the directory. +log_message "Setting ownership and permissions for config files in '$ETC_RANDAO_DIR'." +if [ -f "$ETC_RANDAO_DIR/.env" ]; then + chown root:"$GROUPNAME" "$ETC_RANDAO_DIR/.env" + chmod 640 "$ETC_RANDAO_DIR/.env" # root:rw, group:r, other:--- +fi +if [ -f "$ETC_RANDAO_DIR/wallet.json" ]; then + chown root:"$GROUPNAME" "$ETC_RANDAO_DIR/wallet.json" + chmod 640 "$ETC_RANDAO_DIR/wallet.json" +fi +if [ -f "$ETC_RANDAO_DIR/wallet.seed" ]; then + chown root:"$GROUPNAME" "$ETC_RANDAO_DIR/wallet.seed" + chmod 640 "$ETC_RANDAO_DIR/wallet.seed" +fi + +log_message "Permissions set. User '$USERNAME' (in group '$GROUPNAME') has read-access to configs." +log_message "IMPORTANT: Remember to edit configuration in /etc/randao/ with your actual secrets." + +# --- 5. Enable and Start Systemd Units --- +# NOTE: This section is ideally removed in favor of deb-helper in debian/rules. +# The logic is kept here assuming you are not using deb-helper yet. +log_message "Reloading systemd daemon, then enabling and starting randao.timer." +systemctl daemon-reload +systemctl enable randao.timer +systemctl start --no-block randao.timer +log_message "Systemd randao.timer has been enabled and started." + + +log_message "Randao Provider post-installation script finished." + +exit 0 \ No newline at end of file diff --git a/debian/prerm b/debian/prerm new file mode 100644 index 0000000..e69de29 diff --git a/debian/rules b/debian/rules new file mode 100644 index 0000000..6a675de --- /dev/null +++ b/debian/rules @@ -0,0 +1,10 @@ +#!/usr/bin/make -f + +%: + dh $@ + +# This block overrides the default 'dh_auto_configure' step. +# It tells deb-helper to run the 'configure' script but with +# an extra option. After this step, the normal sequence continues. +override_dh_auto_configure: + dh_auto_configure -- --with-extra-feature \ No newline at end of file diff --git a/docker-compose/.env.example b/docker-compose/.env.example index ece7ec9..24a371e 100644 --- a/docker-compose/.env.example +++ b/docker-compose/.env.example @@ -3,15 +3,8 @@ DB_PASSWORD=mypassword DB_NAME=mydatabase DOCKER_NETWORK=backend LOG_CONSOLE_LEVEL=3 -SEED_PHRASE="Create a NEW wallet and enter the 12 - 24 words here" -WALLET_JSON = '{ - "kty": "RSA", - "e": "test", - "n": "test", - "d": "test", - "p": "test", - "q": "test", - "dp": "test", - "dq": "test", - "qi": "test" -}' +## Enable ONE of the wallet methods below: +#SEED_FILE_PATH=/app/config/wallet.seed # path corresponds to the container volume mounted in docker-compose.yml file +#WALLET_JSON_FILE_PATH=/app/config/wallet.json # path corresponds to the container volume mounted in docker-compose.yml file +#SEED_PHRASE="Create a NEW wallet and enter the 12 - 24 words here" +#WALLET_JSON = '{ "kty": "RSA", "e": "test", "n": "test", "d": "test", "p": "test", "q": "test", "dp": "test", "dq": "test", "qi": "test" }' diff --git a/docker-compose/QuickStart.md b/docker-compose/QuickStart.md new file mode 100644 index 0000000..e2b9ead --- /dev/null +++ b/docker-compose/QuickStart.md @@ -0,0 +1,104 @@ +# **Randao Provider: Quick Start Guide** + +This guide provides a quick way to get the Randao Provider running on your local machine using Docker Compose. This is ideal for development, testing, or non-appliance deployments. +**Assumptions:** + +- You have **Docker Desktop** (Windows/macOS) or **Docker Engine** (Linux) installed and running. +- You have basic command-line knowledge. +- You are using the official randao/orchestrator image, which has been pre-built with the necessary wallet management modifications. +- **You have a compatible Arweave wallet (JWK file or mnemonic seed phrase) ready.** + +## **1\. Get the Project Files** + +First, clone the Randao Provider repository from GitHub: +```sh +git clone https://github.com/RandAOLabs/Randomness-Provider.git randao-provider +``` + +Now, navigate into the Docker Compose directory: +``sh +cd randao-provider/docker-compose/ +`` + +## **2\. Prepare Configuration Files** + +You need to create/edit two essential configuration files: .env (for database and logging) and your wallet key file (wallet.json or wallet.seed). + +### **2.1. Create .env (Environment Variables)** + +This file stores your database credentials and other settings. + +1. Copy the example .env file: + cp .env.example .env + +2. Open the newly created .env file in a text editor (e.g., nano .env or code .env) and **fill in your desired values** for the database user, password, and name. You can keep the defaults if running locally for testing. + \# .env + DB\_USER=myuser + DB\_PASSWORD=mypassword + DB\_NAME=mydatabase + DOCKER\_NETWORK=backend + LOG\_CONSOLE\_LEVEL=3 \# Set to 7 for verbose (DEBUG) logs + + +### **2.2. Create Wallet Key File (wallet.json or wallet.seed)** + +This file contains your Arweave wallet's private key (JWK) or mnemonic seed phrase. The application will prioritize reading from wallet.json (JWK) if both are present. If neither file is found, it will fall back to environment variables. We recommend using Wander as the Chrome plugin integrates easily with our [Provider Portal](https://providers_randao.ar.io/providers). + +#### **Option A: Using wallet.json (JWK)** + +1. Copy the example wallet.json file: + cp wallet.json.example wallet.json + +2. Open wallet.json in a text editor and **replace its content with your actual Arweave wallet's JWK (JSON Web Key) data.** + **⚠️ IMPORTANT SECURITY WARNING ⚠️** + + - **NEVER use the example wallet content for a real wallet.** Always generate your own unique Arweave wallet. + - **Keep your wallet.json file secure.** Do not share it or commit it to public repositories. + - On Linux/macOS, it is highly recommended to set strict permissions: + chmod 600 wallet.json + +#### **Option B: Using wallet.seed (Mnemonic Seed Phrase)** + +1. Copy the example wallet.seed file: + cp wallet.seed.example wallet.seed + +2. Open wallet.seed in a text editor and **replace its content with your actual Arweave wallet's mnemonic seed phrase.** The seed phrase must be 12, 18, or 24 words, separated by single spaces, with no extra characters. + **⚠️ IMPORTANT SECURITY WARNING ⚠️** + + - **NEVER use the example seed phrase for a real wallet.** Always generate your own unique Arweave wallet. + - **Keep your wallet.seed file secure.** Do not share it or commit it to public repositories. + - On Linux/macOS, it is highly recommended to set strict permissions: + chmod 600 wallet.seed + +### **2.3. Alternative (Less Secure Fallback): Environment Variables** + +If you prefer not to create wallet.json or wallet.seed files, you can instead add the wallet content directly into your .env file using the WALLET\_JSON or SEED\_PHRASE environment variables. The application will fall back to these if it cannot read from the mounted files. + +- **For JWK:** Add WALLET\_JSON='{"your\_jwk\_content\_here"}' to your .env file. +- **For Seed Phrase:** Add SEED\_PHRASE="your seed phrase words here" to your .env file. + +However, **this method is less secure as environment variables are easily inspectable.** + +## **3\. Run the Randao Provider** + +Now you can start your Docker Compose stack. This command will automatically pull the necessary Docker images and set up your services. +docker compose up \-d \--pull=always + +- \--pull=always: Ensures that Docker always checks for and pulls the latest versions of the images from Docker Hub. +- up: Starts the services in the foreground, showing their logs directly in your terminal. +- \-d: run the container in the background (detached mode) + + +## **4\. Monitor Logs** + +To see the real-time output from your running services (especially for debugging wallet initialization): + +`docker compose logs \-f` + +## **5\. Stop the Randao Provider** + +To stop and remove the running containers, networks, and volumes (excluding named volumes like pgdata): + +`docker compose down` + +You should now have your Randao Provider up and running locally\! \ No newline at end of file diff --git a/docker-compose/docker-compose.appliance.yml b/docker-compose/docker-compose.appliance.yml new file mode 100644 index 0000000..10c20a8 --- /dev/null +++ b/docker-compose/docker-compose.appliance.yml @@ -0,0 +1,28 @@ +# /home/randao/Randomness-Provider.git/docker-compose/docker-compose.appliance.yml +services: + orchestrator: + volumes: + # Override wallet mounts to point to /etc/randao/ for appliance security + - /etc/randao/wallet.json:/app/config/wallet.json:ro + - /etc/randao/wallet.seed:/app/config/wallet.seed:ro # If used + deploy: # Appliance-specific orchestrator deploy rules + resources: + limits: + cpus: '1.5' # Allow orchestrator up to 1.5 cores + memory: 300M # Allow up to 300MB RAM + reservations: + cpus: '0.5' # Reserve 0.5 of a core + memory: 192M + + postgres: # Appliance-specific postgres deploy rules and config mount + volumes: + - ./postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro # <--- ADDED HERE + shm_size: '64m' # <--- ADDED HERE (or adjust as needed) + deploy: + resources: + limits: + cpus: '0.4' # Limit Postgres to 40% of one core + memory: 150M # Limit Postgres to 150MB RAM + reservations: + cpus: '0.1' # Reserve 10% of one core + memory: 64M \ No newline at end of file diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index f46a33a..b3c35bd 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -1,3 +1,4 @@ +# /home/randao/RandaoProvider/docker-compose/docker-compose.yml (Base) services: postgres: image: postgres:13-alpine @@ -11,6 +12,7 @@ services: - backend volumes: - pgdata:/var/lib/postgresql/data + # REMOVED: - ./postgres/postgresql.conf:/etc/postgresql/postgresql.conf:ro healthcheck: test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-myuser} -d ${DB_NAME:-mydatabase}"] interval: 10s @@ -21,11 +23,12 @@ services: options: max-size: "100m" max-file: "5" + # REMOVED: shm_size: '64m' - Docker's default shm_size (64MB) will be used + # REMOVED: NO deploy section here; no device-specific limits in the base file. orchestrator: - image: randao/orchestrator:latest - # pull_policy: always # Ensure the latest image is always pulled - restart: unless-stopped # Restart on crash (or on exit with non-zero code) + image: hottoddie/orchestrator:custom-file-wallet # Or your chosen stable tag (e.g., appliance-stable) + pull_policy: always depends_on: postgres: condition: service_healthy @@ -35,13 +38,16 @@ services: DB_USER: ${DB_USER:-myuser} DB_PASSWORD: ${DB_PASSWORD:-mypassword} DB_NAME: ${DB_NAME:-mydatabase} - SEED_PHRASE: ${SEED_PHRASE} + WALLET_JSON_FILE_PATH: /app/config/wallet.json + SEED_FILE_PATH: /app/config/wallet.seed WALLET_JSON: ${WALLET_JSON} DOCKER_NETWORK: backend networks: - backend volumes: - /var/run/docker.sock:/var/run/docker.sock + - ./wallet.json:/app/config/wallet.json # Default/Standalone wallet.json mount + - ./wallet.seed:/app/config/wallet.seed # Default/Standalone wallet.seed mount logging: driver: json-file options: @@ -55,4 +61,4 @@ networks: volumes: pgdata: - driver: local + driver: local \ No newline at end of file diff --git a/docker-compose/postgres/postgresql.conf b/docker-compose/postgres/postgresql.conf new file mode 100644 index 0000000..c0cb761 --- /dev/null +++ b/docker-compose/postgres/postgresql.conf @@ -0,0 +1,26 @@ +# /home/randao/RandomnProvider/docker-compose/postgres/postgresql.conf +# --- Memory --- +# Total available RAM is 512MB (0.5GB) +# Target for Postgres container is 150MB limit +shared_buffers = 40MB # ~25% of Postgres container's 150MB limit +work_mem = 1MB # For sorting/hashing per operation (start low) +maintenance_work_mem = 16MB # For VACUUM, CREATE INDEX (lower than default) +effective_cache_size = 60MB # Estimate of OS + shared_buffers cache (roughly 40% of container limit) +wal_buffers = 2MB # Smaller WAL buffers for low write loads + +# --- Connections --- +max_connections = 20 # Limit connections to reduce per-connection memory overhead + # Adjust based on orchestrator's needs. + +# --- Checkpointing --- +# Aim to reduce write spikes for low I/O systems +checkpoint_timeout = 10min # Increase from default 5min +max_wal_size = 256MB # Equivalent to ~16 checkpoint segments (16MB each) +checkpoint_completion_target = 0.9 # Spread out checkpoint writes more + +# --- Autovacuum --- +# Autovacuum can be resource-intensive; aggressive settings +# might be needed to keep table bloat down, but tune carefully. +# autovacuum_max_workers = 1 # Reduce workers +# autovacuum_vacuum_scale_factor = 0.05 # Vacuum more frequently on small tables +# autovacuum_analyze_scale_factor = 0.02 # Analyze more frequently diff --git a/docker-compose/wallet.json.example b/docker-compose/wallet.json.example new file mode 100644 index 0000000..72ca0b9 --- /dev/null +++ b/docker-compose/wallet.json.example @@ -0,0 +1,11 @@ +{ + "kty": "RSA", + "e": "AQAB", + "n": "zxP9Y4b1... (truncated for brevity)...eB8zP6Q", + "d": "Jk-9sR1b... (truncated for brevity)...k_1gQ2", + "p": "9h7x0zY0... (truncated for brevity)...wD4q", + "q": "8x6w9yZ8... (truncated for brevity)...vH9k", + "dp": "2o1p3q4r... (truncated for brevity)...sU5t", + "dq": "5t6u7v8w... (truncated for brevity)...xB9y", + "qi": "1a2b3c4d... (truncated for brevity)...gF0h" +} \ No newline at end of file diff --git a/docker-compose/wallet.seed.example b/docker-compose/wallet.seed.example new file mode 100644 index 0000000..f490c8b --- /dev/null +++ b/docker-compose/wallet.seed.example @@ -0,0 +1 @@ +example caution example caution example caution example caution example caution example caution \ No newline at end of file diff --git a/orchestrator/.dockerignore b/orchestrator/.dockerignore index a0bedda..c4f9ed1 100644 --- a/orchestrator/.dockerignore +++ b/orchestrator/.dockerignore @@ -1,3 +1,4 @@ node_modules .git dist +debian \ No newline at end of file diff --git a/orchestrator/docs/todd-builder.md b/orchestrator/docs/todd-builder.md new file mode 100644 index 0000000..8f7c4ab --- /dev/null +++ b/orchestrator/docs/todd-builder.md @@ -0,0 +1,37 @@ + +# Navigate to your Docker Compose project directory (on your amd64 machine) +cd /path/to/your/local/RandaoProvider/docker-compose/ + +# Export version as an environment variable +export VERSION=v1.0.12 # You can change this value to any version you want + +# Build the Docker image with the version tag +# docker build -t hottoddie/orchestrator:custom-file-wallet -t hottoddie/orchestrator:$VERSION . + +# Log in to Docker +docker login + +# Push the image with the version tag +docker push randao/orchestrator:latest +docker push randao/orchestrator:$VERSION + +# Create and use buildx builder +docker buildx create --use +docker buildx inspect --bootstrap + + + + + +# Build for multiple platforms (including amd64 for testing and arm64 for Orange Pi Zero 3) + + docker buildx build \ + --platform linux/amd64,linux/arm64,linux/arm/v7 \ + -t hottoddie/orchestrator:custom-file-wallet \ + --push \ + -f ../orchestrator/Dockerfile \ + ../orchestrator/ \ No newline at end of file diff --git a/orchestrator/src/walletUtils.ts b/orchestrator/src/walletUtils.ts index b6f7147..2a0b23a 100644 --- a/orchestrator/src/walletUtils.ts +++ b/orchestrator/src/walletUtils.ts @@ -5,216 +5,236 @@ import { isOneOf, isString } from "typed-assert"; import { wordlists, mnemonicToSeed } from "bip39-web-crypto"; import logger from "./logger"; import Arweave from "arweave"; +import * as fs from 'fs/promises'; -// Initialize Arweave +// --- Configuration --- + +// ⭐ Recommendation: Externalize Arweave configuration for flexibility. const arweave = Arweave.init({ - host: "arweave.net", - port: 443, - protocol: "https", + host: process.env.ARWEAVE_HOST || "arweave.net", + port: process.env.ARWEAVE_PORT ? parseInt(process.env.ARWEAVE_PORT, 10) : 443, + protocol: process.env.ARWEAVE_PROTOCOL || "https", }); -// Global wallet storage +// ⭐ Recommendation: Avoid magic numbers by defining them as constants. +const STRONG_PASSWORD_LEVEL = 3; // Corresponds to 'Strong' in check-password-strength + +// ⭐ Recommendation: Create a Set for efficient mnemonic validation (O(1) lookup). +const englishWordlistSet = new Set(wordlists.english); + + +// --- Global State --- let globalWallet: JWKInterface | null = null; -let walletSource: 'seed_phrase' | 'json' | null = null; +let walletSource: 'seed_file' | 'json_file' | 'seed_phrase' | 'json_string' | null = null; -/** - * Credits to arweave.app for the mnemonic wallet generation - * - * https://github.com/jfbeats/ArweaveWebWallet/blob/master/src/functions/Wallets.ts - * https://github.com/jfbeats/ArweaveWebWallet/blob/master/src/functions/Crypto.ts - */ + +// --- Core Crypto Functions --- /** - * Generate a JWK from a mnemonic seedphrase + * Generate a JWK from a mnemonic seedphrase. * - * @param mnemonic Mnemonic seedphrase to generate wallet from - * @returns Wallet JWK + * @param mnemonic Mnemonic seedphrase to generate wallet from. + * @returns Wallet JWK. */ export async function jwkFromMnemonic(mnemonic: string): Promise { - // TODO: We use `mnemonicToSeed()` from `bip39-web-crypto` here instead of using `getKeyPairFromMnemonic`, which - // internally uses `bip39`. Instead, we should just be using `getKeyPairFromMnemonic` and lazy load this dependency: - // - // For additional context, see https://www.notion.so/community-labs/Human-Crypto-Keys-reported-Bug-d3a8910dabb6460da814def62665181a + // TODO: As noted in the original code, this should be replaced. + // Use `getKeyPairFromMnemonic` from `human-crypto-keys` for a more direct and efficient implementation. + // See: https://www.notion.so/community-labs/Human-Crypto-Keys-reported-Bug-d3a8910dabb6460da814def62665181a const seedBuffer = await mnemonicToSeed(mnemonic); + // Recommendation: Investigate and fix the need for @ts-ignore. + // The type of `seedBuffer` (likely Uint8Array) might differ from what `getKeyPairFromSeed` expects in Node.js (e.g., a Buffer). + // A potential fix could be `Buffer.from(seedBuffer)`. const { privateKey } = await getKeyPairFromSeed( //@ts-ignore seedBuffer, - { - id: "rsa", - modulusLength: 4096, - }, + { id: "rsa", modulusLength: 4096 }, { privateKeyFormat: "pkcs8-der" }, ); + + // Recommendation: Investigate and fix the need for `as any`. const jwk = await pkcs8ToJwk(privateKey as any); return jwk; } /** - * Convert a PKCS8 private key to a JWK + * Convert a PKCS8 private key to a JWK using Node.js's native crypto. * - * @param privateKey PKCS8 private key to convert - * @returns JWK + * @param privateKey PKCS8 private key to convert. + * @returns JWK. */ export async function pkcs8ToJwk(privateKey: Uint8Array): Promise { - // Need to adapt for Node.js environment as the original uses window.crypto const crypto = require('crypto').webcrypto; - + const key = await crypto.subtle.importKey( - "pkcs8", - privateKey, - { name: "RSA-PSS", hash: "SHA-256" }, - true, + "pkcs8", + privateKey, + { name: "RSA-PSS", hash: "SHA-256" }, + true, ["sign"] ); - + const jwk = await crypto.subtle.exportKey("jwk", key); + // The explicit mapping is safe but could potentially be simplified + // if Arweave's JWKInterface is compatible with the standard JsonWebKey type. return { - kty: jwk.kty!, - e: jwk.e!, - n: jwk.n!, - d: jwk.d, - p: jwk.p, - q: jwk.q, - dp: jwk.dp, - dq: jwk.dq, - qi: jwk.qi, + kty: jwk.kty!, e: jwk.e!, n: jwk.n!, + d: jwk.d, p: jwk.p, q: jwk.q, + dp: jwk.dp, dq: jwk.dq, qi: jwk.qi, }; } +// --- Validation Functions --- + /** - * Check password strength + * Check if a password is rated as "Strong". * - * @param password Password to check + * @param password Password to check. */ export function checkPasswordValid(password: string): boolean { const strength = passwordStrength(password); - - return strength.id === 3; + return strength.id === STRONG_PASSWORD_LEVEL; } /** - * Validate if a string is a valid mnemonic phrase - * - * @param mnemonic Mnemonic to validate - * @returns Length of the mnemonic if valid + * Validate if a string is a valid BIP-39 mnemonic phrase using an efficient Set lookup. + * * @param mnemonic Mnemonic to validate. + * @returns `true` if the mnemonic is valid, otherwise `false`. */ -export function isValidMnemonic(mnemonic: string): number { - isString(mnemonic, "Mnemonic has to be a string."); - - const words = mnemonic.split(" "); - - isOneOf(words.length, [12, 18, 24], "Invalid mnemonic length."); - - const wordlist = wordlists.english; - - for (const word of words) { - isOneOf(word, wordlist, "Invalid word in mnemonic."); +export function isValidMnemonic(mnemonic: string): boolean { + try { + isString(mnemonic, "Mnemonic has to be a string."); + const words = mnemonic.trim().split(" "); + isOneOf(words.length, [12, 18, 24], "Invalid mnemonic length."); + + for (const word of words) { + if (!englishWordlistSet.has(word)) { + logger.warn(`Invalid word found in mnemonic: "${word}"`); + return false; + } + } + return true; + } catch (error) { + logger.error("Mnemonic validation failed", error); + return false; } - - return words.length; } + +// --- Wallet Initialization & Access --- + /** - * Initialize wallet from environment variables - * Prioritizes SEED_PHRASE over WALLET_JSON if both are present - * - * @returns JWK wallet object + * ⭐ Recommendation: Refactored `initializeWallet`. + * Initializes the global wallet by trying a series of sources in a defined order of priority. + * This approach is more modular and easier to maintain. + * + * @returns JWK wallet object. */ export async function initializeWallet(): Promise { - // If wallet is already initialized, return it if (globalWallet) { + logger.debug("[DEBUG] Wallet already initialized. Returning existing wallet."); return globalWallet; } - const hasSeedPhrase = !!process.env.SEED_PHRASE; - const hasWalletJson = !!process.env.WALLET_JSON; + // Define wallet sources in order of priority. + // The 'path' property for env var sources is just for logging clarity. + const sources = [ + { + type: 'seed_file', + path: process.env.SEED_FILE_PATH, + enabled: !!process.env.SEED_FILE_PATH, + load: async () => { + logger.debug(`[DEBUG] Attempting to load wallet from SEED_FILE_PATH: ${process.env.SEED_FILE_PATH}`); + const seed = await fs.readFile(process.env.SEED_FILE_PATH!, 'utf8'); + logger.debug(`[DEBUG] Read seed string from file (first 50 chars): "${seed.trim().substring(0, 50)}..."`); + if (!isValidMnemonic(seed)) throw new Error("Invalid mnemonic format in file."); + return jwkFromMnemonic(seed.trim()); + }, + }, + { + type: 'json_file', + path: process.env.WALLET_JSON_FILE_PATH, + enabled: !!process.env.WALLET_JSON_FILE_PATH, + load: async () => { + logger.debug(`[DEBUG] Attempting to load wallet from WALLET_JSON_FILE_PATH: ${process.env.WALLET_JSON_FILE_PATH}`); + const jsonString = await fs.readFile(process.env.WALLET_JSON_FILE_PATH!, 'utf8'); + logger.debug(`[DEBUG] Read JSON string from file (first 50 chars): "${jsonString.trim().substring(0, 50)}..."`); + return JSON.parse(jsonString); + }, + }, + { + type: 'seed_phrase', + path: "env var", // Placeholder for logging + enabled: !!process.env.SEED_PHRASE, + load: async () => { + logger.debug(`[DEBUG] Attempting to load wallet from SEED_PHRASE env var.`); + const seed = process.env.SEED_PHRASE!; + logger.debug(`[DEBUG] SEED_PHRASE env var content (first 50 chars): "${seed.trim().substring(0, 50)}..."`); + if (!isValidMnemonic(seed)) throw new Error("Invalid mnemonic format in env var."); + return jwkFromMnemonic(seed.trim()); + }, + }, + { + type: 'json_string', + path: "env var", // Placeholder for logging + enabled: !!process.env.WALLET_JSON, + load: async () => { + logger.debug(`[DEBUG] Attempting to load wallet from WALLET_JSON env var.`); + const jsonString = process.env.WALLET_JSON!; + logger.debug(`[DEBUG] WALLET_JSON env var content (first 50 chars): "${jsonString.trim().substring(0, 50)}..."`); + return JSON.parse(jsonString); + }, + }, + ] as const; // <--- This 'as const' is critical for type inference + + for (const source of sources) { + if (source.enabled) { + logger.debug(`[DEBUG] Checking wallet source: ${source.type.toUpperCase()}`); + try { + const wallet = await source.load(); // wallet is JWKInterface + const address = await arweave.wallets.jwkToAddress(wallet); + + logger.info(`Wallet initialized from ${source.type.toUpperCase()} with address: ${address}`); + if (source.path && source.path !== "env var") { // Log path only if it's a file path + logger.info(`Wallet source path: ${source.path}`); + } - if (hasSeedPhrase && hasWalletJson) { - logger.info("Both SEED_PHRASE and WALLET_JSON are provided. Prioritizing SEED_PHRASE."); - } else if (!hasSeedPhrase && !hasWalletJson) { - throw new Error("No wallet configuration found. Please provide either SEED_PHRASE or WALLET_JSON in environment variables."); - } + globalWallet = wallet; // Assign to globalWallet for future calls + walletSource = source.type; // This assignment is now type-safe due to 'as const' - let wallet: JWKInterface; - let seedPhraseAddress: string | undefined; - let jsonAddress: string | undefined; + return wallet; // <--- Directly return 'wallet' which is guaranteed JWKInterface - // Try to load wallet from seed phrase if available - if (hasSeedPhrase) { - try { - const seedPhrase = process.env.SEED_PHRASE!; - if (!isValidMnemonic(seedPhrase)) { - throw new Error("Invalid seed phrase format"); - } - - wallet = await jwkFromMnemonic(seedPhrase); - seedPhraseAddress = await arweave.wallets.jwkToAddress(wallet); - - // If we have both, also get the JSON wallet address for comparison - if (hasWalletJson) { - try { - const jsonWallet = JSON.parse(process.env.WALLET_JSON!); - jsonAddress = await arweave.wallets.jwkToAddress(jsonWallet); - } catch (error) { - logger.error("Failed to parse WALLET_JSON", error); - } - } - - walletSource = 'seed_phrase'; - globalWallet = wallet; - logger.info(`Using wallet from SEED_PHRASE with address: ${seedPhraseAddress}`); - - if (jsonAddress) { - logger.info(`WALLET_JSON address (not used): ${jsonAddress}`); - } - } catch (error) { - logger.error("Failed to initialize wallet from SEED_PHRASE", error); - - // Fall back to JSON if available - if (hasWalletJson) { - logger.info("Falling back to WALLET_JSON"); - } else { - throw error; + } catch (error) { + logger.error(`Failed to load wallet from ${source.type.toUpperCase()} (${source.path || 'no path specified'}): ${error instanceof Error ? error.message : String(error)}`); + logger.debug(`[DEBUG] Full error details for ${source.type.toUpperCase()}:`, error); + // Fall through to the next source. } + } else { + logger.debug(`[DEBUG] Wallet source ${source.type.toUpperCase()} is not enabled or not configured.`); } } - // If we haven't successfully initialized the wallet from seed phrase, try JSON - if (!globalWallet && hasWalletJson) { - try { - wallet = JSON.parse(process.env.WALLET_JSON!); - jsonAddress = await arweave.wallets.jwkToAddress(wallet); - - walletSource = 'json'; - globalWallet = wallet; - logger.info(`Using wallet from WALLET_JSON with address: ${jsonAddress}`); - } catch (error) { - throw new Error(`Failed to initialize wallet from WALLET_JSON: ${error}`); - } - } - - return globalWallet!; + // If the loop completes without successfully returning a wallet, + // then we throw an error as no wallet could be initialized. + throw new Error("No wallet configuration could be successfully initialized from any source. Please check environment variables and file paths."); } /** - * Get wallet address - * - * @returns Provider ID/wallet address + * Get the initialized wallet's address. + * * @returns The wallet address string. */ export async function getWalletAddress(): Promise { const wallet = await initializeWallet(); - return await arweave.wallets.jwkToAddress(wallet); + return arweave.wallets.jwkToAddress(wallet); } /** - * Get the wallet for signing transactions - * - * @returns JWK wallet object + * Get the initialized wallet for signing transactions. + * * @returns The JWK wallet object. */ export async function getWallet(): Promise { - return await initializeWallet(); -} + return initializeWallet(); +} \ No newline at end of file diff --git a/todd-updates.md b/todd-updates.md new file mode 100644 index 0000000..1722644 --- /dev/null +++ b/todd-updates.md @@ -0,0 +1,252 @@ +# Randao Provider Configuration Guide + +This document outlines how to deploy the Randao Provider application using Docker Compose, covering configurations for both standard user environments (e.g., Windows, macOS, Linux desktop) and dedicated Linux service/appliance environments. + +----- + +## 1\. Core Concepts & Files + +The Randao Provider deployment relies on these key files: + + * **`docker-compose.yml`**: The **base** Docker Compose file. It defines the core services (orchestrator, PostgreSQL), their dependencies, and common environment variables. It uses relative paths for `wallet.json` and `postgresql.conf` for portability and serves as the default configuration for standard user environments. + * **`.env`**: A plain text file storing environment variables (database credentials, network settings). This file is sourced by Docker Compose. + * **`wallet.json`**: Your Arweave wallet's JWK (JSON Web Key) file. This contains sensitive private key information. + * **`wallet.seed`** (Optional): If you use a mnemonic seed phrase instead of a JWK. + * **`docker-compose.appliance.yml`**: An **override** Docker Compose file specifically for appliance deployments. It defines absolute paths for sensitive files (like `wallet.json`) and appliance-specific resource limits. + * **`postgres/postgresql.conf`**: Custom PostgreSQL configuration for resource-constrained environments. + * **Wallet Management**: The application's source code (specifically `walletUtils.ts`) has been modified to prioritize reading wallet information securely from mounted files (e.g., `wallet.json` or `wallet.seed`), falling back to environment variables (`WALLET_JSON` or `SEED_PHRASE`) if files aren't found. This guide assumes you are using an image that includes these modifications. + +----- + +## 2\. Why Use Wallet Files Instead of Environment Variables? + +Using dedicated files (like `wallet.json` or `wallet.seed`) for sensitive wallet information is **strongly preferred for security reasons** over passing this data directly as environment variables (`WALLET_JSON` or `SEED_PHRASE`). + +Here's why: + + * **Reduced Visibility (Primary Reason):** + * **Environment Variables (`WALLET_JSON`, `SEED_PHRASE`):** These are notoriously insecure for sensitive data. Anyone with access to the Docker host (even a non-root user with `docker` group access) can easily inspect a running container's environment variables using the `docker inspect ` command. This means your full wallet private key could be displayed in plain text in the command's output. + * **Files (`wallet.json`, `wallet.seed`):** When you mount a file into a container (e.g., `/etc/randao/wallet.json` into `/app/config/wallet.json`), the file's content is not directly exposed as an environment variable of the running process. An attacker would need filesystem access to `/etc/randao/wallet.json` on the host (which can be protected with strict permissions), *and* potentially shell access *inside* the container, to read the file. + * **Principle of Least Privilege (Filesystem):** You can set very tight file permissions on the host (e.g., `chmod 640` or `600`) for `wallet.json` and `wallet.seed`. This allows only the necessary user (e.g., `root` for ownership, `randao_service` user for read access via group) to access the file, further limiting exposure. + * **Best Practice:** Mounting sensitive data as files is the industry-standard best practice for secret management in containerized environments (e.g., Docker Secrets in Swarm mode, Kubernetes Secrets mounted as volumes). + * **Logging:** Environment variables can sometimes inadvertently end up in logs if the application or logging system isn't carefully configured. File contents are less prone to this leakage. + +While the application supports falling back to environment variables for convenience, **using the file-based method for your wallet is always the more secure choice, especially for production or appliance deployments.** + +----- + +## 3\. Setting Up the Project Directory + +Begin by cloning the Randao Provider repository from GitHub and organizing your configuration files. + +### **3.1. Clone the Repository:** + +```bash +git clone https://github.com/RandAOLabs/Randomness-Provider.git your-randao-provider-repo +``` + +### **3.2. Navigate to the Docker Compose Directory:** + +```bash +cd your-randao-provider-repo/docker-compose/ +``` + +### **3.3. Project Directory Structure:** + +Your directory should look similar to this: + +``` +your-randao-provider-repo/ +├── docker-compose/ +│ ├── docker-compose.yml # Base Docker Compose file +│ ├── docker-compose.appliance.yml # Appliance-specific overrides +│ ├── postgres/ +│ │ └── postgresql.conf # Custom Postgres config +│ ├── .env.example # Example .env file (for users to copy) +│ ├── wallet.json.example # Example wallet.json (for users to copy) +│ └── wallet.seed.example # Example wallet.seed (optional) +├── orchestrator/ # Contains Dockerfile and walletUtils.ts (source, not used directly by docker compose up) +│ └── Dockerfile +│ └── src/walletUtils.ts +├── LICENSE +└── README.md +``` + +----- + +## 4\. Configuration for a Standard Docker User (e.g., Windows, macOS, Linux Desktop) + +This setup is for users who want to run the provider locally without systemd integration, using pre-built Docker images. + +### **4.1. Prerequisites:** + + * **Docker Desktop** (Windows/macOS) or **Docker Engine** (Linux) installed and running. + * Access to the command line/terminal. + +### **4.2. Setup Steps:** + +1. **Navigate to the `docker-compose` directory** (if not already there): + + ```bash + cd your-randao-provider-repo/docker-compose/ + ``` + +2. **Create `.env` file:** + Copy the example `.env` file and **fill in your database credentials**. + + ```bash + cp .env.example .env + # Open .env in a text editor and fill in DB_USER, DB_PASSWORD, DB_NAME, DOCKER_NETWORK, LOG_CONSOLE_LEVEL + # Example .env content: + # DB_USER=myuser + # DB_PASSWORD=mypassword + # DB_NAME=mydatabase + # DOCKER_NETWORK=backend + # LOG_CONSOLE_LEVEL=7 + ``` + +3. **Create `wallet.json` (or `wallet.seed`):** + Copy the example wallet file and **paste your actual Arweave wallet's JWK content** (or seed phrase) into it. + + ```bash + cp wallet.json.example wallet.json + # Open wallet.json in a text editor and paste your JWK content. + # On Linux/macOS, set permissions for security: + chmod 600 wallet.json + ``` + + * **Fallback Option:** If you prefer not to create `wallet.json` directly, you can put `WALLET_JSON='{"your_jwk_content"}'` directly into your `.env` file. The application code will fall back to this environment variable if it cannot read `wallet.json` from the mounted file. **Note that this fallback is less secure.** + +4. **Run the Docker Compose Stack:** + Use the base `docker-compose.yml`. `docker compose` will pull the necessary images. + + ```bash + docker compose up --pull=always + ``` + + * `--pull=always`: Ensures the latest image versions are pulled from Docker Hub. + * `up`: Starts the services in the foreground. Add `-d` to run in detached mode (background). + +5. **Monitor Logs:** + + ```bash + docker compose logs -f + ``` + +----- + +## 5\. Configuration for a Linux Service / Appliance + +This setup provides robust, automated management via `systemd`, enhanced security, and consistent updates, using pre-built Docker images. + +### **5.1. Prerequisites:** + + * **Debian/Ubuntu** (or similar Linux distribution) installed. + * **Docker Engine** and **Docker Compose V2** installed. + * **`randao_service` system user** created (e.g., `sudo adduser --system --no-create-home --group --uid 888 randao_service`). + * `randao_service` user added to the `docker` group (e.g., `sudo usermod -aG docker randao_service`). + * **Swap space** configured (highly recommended for low-RAM devices like H3). + * **Ownership and permissions** for the project directory set for `randao_service`. + ```bash + sudo chown -R randao_service:randao_service /home/randao/Randomness-Provider.git/ + sudo chmod -R u=rwX,go=rX /home/randao/Randomness-Provider.git/ + ``` + +### **5.2. Setup Steps:** + +1. **Place Sensitive Configuration Files in `/etc/randao/`:** + These files are managed by `root` but readable by `randao_service`. This is the **preferred and most secure location** for appliance secrets. + + ```bash + # Create the directory + sudo mkdir -p /etc/randao/ + + # Copy your actual .env and wallet.json files from your local setup or provisioning source + # Example (assuming they are temporarily available in /tmp/ during provisioning): + sudo cp /tmp/.env /etc/randao/.env + sudo cp /tmp/wallet.json /etc/randao/wallet.json + sudo cp /tmp/wallet.seed /etc/randao/wallet.seed # If using seed file + + # Set ownership and permissions for the directory + sudo chown root:root /etc/randao/ + sudo chmod 700 /etc/randao/ # Root only access to the directory itself + + # Set ownership and permissions for the files + sudo chown root:randao_service /etc/randao/.env + sudo chmod 640 /etc/randao/.env # Root R/W, randao_service group R, others no access + + sudo chown root:randao_service /etc/randao/wallet.json + sudo chmod 640 /etc/randao/wallet.json + + # If wallet.seed is used + sudo chown root:randao_service /etc/randao/wallet.seed + sudo chmod 640 /etc/randao/wallet.seed + ``` + +2. **Place `docker-compose` Project Files:** + Copy the cloned repository contents to a system location like `/home/randao/Randomness-Provider.git/`. + + ```bash + # Example: + sudo cp -r /path/to/your/cloned-repo/Randomness-Provider.git /home/randao/ + ``` + +3. **Create Systemd Service Unit (`randao.service`):** + Create `/etc/systemd/system/randao.service` with the following content: + + ```ini + # /etc/systemd/system/randao.service + [Unit] + Description=RANDAO Provider + Documentation=https://github.com/RandAOLabs/Randomness-Provider + Requires=docker.service + After=network-online.target docker.service + + [Service] + Type=simple + User=randao_service + Group=randao_service + WorkingDirectory=/home/randao/Randomness-Provider.git/docker-compose + ExecStart=/usr/bin/docker compose -f docker-compose.yml -f docker-compose.appliance.yml --env-file /etc/randao/.env up --pull=always + ExecStop=/usr/bin/docker compose down + TimeoutStartSec=0 + Restart=on-failure + RestartSec=5s + + [Install] + WantedBy=multi-user.target + ``` + +4. **Create Systemd Timer Unit (`randao.timer`):** + Create `/etc/systemd/system/randao.timer` with the following content for periodic updates: + + ```ini + # /etc/systemd/system/randao.timer + [Unit] + Description=Timer to periodically restart RANDAO Provider for latest image pull + + [Timer] + OnCalendar=*-*-* 00,12:00:00 # Restart every day at midnight and noon UTC + RandomizedDelaySec=30min # Add a random delay to prevent stampedes + Persistent=true # Trigger on boot if a scheduled run was missed + OnBootSec=10s # Start 10 seconds after system boot (initial run) + + [Install] + WantedBy=timers.target + ``` + +5. **Enable & Start Services:** + + ```bash + sudo systemctl daemon-reload # Reload systemd to recognize new units + sudo systemctl enable randao.timer # Enable the timer for autostart on reboot + sudo systemctl start randao.timer # Start the timer immediately + # The timer will then trigger randao.service (e.g., after 10 seconds due to OnBootSec) + ``` + +6. **Monitor Logs:** + + ```bash + sudo journalctl -u randao.service -f # Monitor real-time logs from your service + sudo systemctl status randao.timer # Check timer's status and next activation + ``` \ No newline at end of file