diff --git a/.github/scripts/expand-workflow.sh b/.github/scripts/expand-workflow.sh new file mode 100755 index 00000000..41818327 --- /dev/null +++ b/.github/scripts/expand-workflow.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +print_help() { + cat <<'EOF' +expand-includes.sh — simple textual include expander + +USAGE: + expand-includes.sh + expand-includes.sh -h | --help + +DESCRIPTION: + Reads and writes the expanded result to stdout. + + Lines matching the following form are replaced by the contents + of the referenced file: + + include: "path/to/file.yml.in" # optional comment + + Rules: + - The filepath MUST be in double quotes. + - Leading whitespace (spaces and/or tabs) is preserved and applied + to every line of the included file. + - Trailing comments starting with '#' are ignored. + - Whitespace around 'include:' and the filename is ignored. + - Included files may themselves contain include: directives. + - Expansion is purely textual; no YAML parsing is performed. + +EOF +} + +expand_file() { + local file="$1" + + while IFS= read -r line || [[ -n "$line" ]]; do + # Remove trailing comments for matching (keep original for output) + local stripped="${line%%#*}" + + # Match YAML include line: + # + # include: "path" + # + if [[ "$stripped" =~ ^([[:space:]]*)include[[:space:]]*:[[:space:]]*\"([^\"]+)\"[[:space:]]*$ ]]; then + local indent="${BASH_REMATCH[1]}" + local include_file="${BASH_REMATCH[2]}" + + if [[ ! -f "$include_file" ]]; then + echo "ERROR: included file not found: $include_file" >&2 + return 1 + fi + + # Recursively expand included file (no subshells) + if ! mapfile -t inc_lines < <(expand_file "$include_file"); then + return 1 + fi + + for inc_line in "${inc_lines[@]}"; do + printf "%s%s\n" "$indent" "$inc_line" + done + + else + echo "$line" + fi + done < "$file" +} + + +# ---- main ---- + +if [[ $# -ne 1 ]]; then + print_help >&2 + exit 1 +fi + +case "$1" in + -h|--help) + print_help + exit 0 + ;; +esac + +printf "# Warning: generated file, do not edit. Generated by '%s %s'\n" "$0" "$*" + +expand_file "$1" diff --git a/.github/workflow-src/Makefile b/.github/workflow-src/Makefile new file mode 100644 index 00000000..f0806b9c --- /dev/null +++ b/.github/workflow-src/Makefile @@ -0,0 +1,21 @@ +SHELL := /usr/bin/env bash + +SRC_DIR := . +OUT_DIR := ../workflows +EXPAND := ../scripts/expand-workflow.sh + +# All input files (recursively) +INPUTS := $(shell find $(SRC_DIR) -type f) + +# All top-level workflow inputs (one-to-one with outputs) +WORKFLOW_SRCS := $(wildcard $(SRC_DIR)/*.yml.in) +WORKFLOWS := $(patsubst $(SRC_DIR)/%.yml.in, $(OUT_DIR)/%.yml, $(WORKFLOW_SRCS)) + +.PHONY: all +all: $(WORKFLOWS) + +# Rule: output depends on *all* inputs and the expand script +$(OUT_DIR)/%.yml: $(SRC_DIR)/%.yml.in $(INPUTS) $(EXPAND) + @echo "Generating $@ from $<" + @mkdir -p $(OUT_DIR) + @$(EXPAND) $< > $@ diff --git a/.github/workflow-src/cjose.yml.in b/.github/workflow-src/cjose.yml.in new file mode 100644 index 00000000..1b16b48c --- /dev/null +++ b/.github/workflow-src/cjose.yml.in @@ -0,0 +1,46 @@ +name: cjose Tests +include: "./common/wp-trigger.yml.in" + +jobs: + include: "./common/wp-build.yml.in" + + test_cjose: + include: "./common/wp-testenv.yml.in" + + strategy: + matrix: + cjose_ref: [ 'v0.6.2.1' ] + force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] + include: "./common/wp-matrix.yml.in" + + steps: + include: "./common/wp-install.yml.in" + + - name: Install cjose dependencies + run: | + apt-get update + apt-get install -y git build-essential autoconf automake \ + libtool pkg-config libjansson-dev check ca-certificates dpkg-dev + + - name: Download cjose + uses: actions/checkout@v4 + with: + repository: OpenIDC/cjose + ref: ${{ matrix.cjose_ref }} + path: cjose + fetch-depth: 1 + + - name: Build cjose + working-directory: cjose + run: | + ./configure CFLAGS="-Wno-error=deprecated-declarations" + make + + - name: Run cjose tests + working-directory: cjose + run: | + export ${{ matrix.force_fail }} + make test 2>&1 | tee cjose-test.log + TEST_RESULT=$(grep -q "FAIL: check_cjose" cjose-test.log && echo "1" || echo "0") + echo "TEST_RESULT = $TEST_RESULT" + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} cjose diff --git a/.github/workflow-src/common/wp-build.yml.in b/.github/workflow-src/common/wp-build.yml.in new file mode 100644 index 00000000..1ea158e3 --- /dev/null +++ b/.github/workflow-src/common/wp-build.yml.in @@ -0,0 +1,10 @@ +build_wolfprovider: + uses: ./.github/workflows/build-wolfprovider.yml + with: + wolfssl_ref: ${{ matrix.wolfssl_ref }} + openssl_ref: ${{ matrix.openssl_ref }} + fips_ref: ${{ matrix.fips_ref }} + replace_default: ${{ matrix.replace_default }} + strategy: + matrix: + include: "./common/wp-matrix.yml.in" diff --git a/.github/workflow-src/common/wp-install.yml.in b/.github/workflow-src/common/wp-install.yml.in new file mode 100644 index 00000000..f6b31cc8 --- /dev/null +++ b/.github/workflow-src/common/wp-install.yml.in @@ -0,0 +1,38 @@ +- name: Setup env vars + run: | + # These paths must match the ones in build-wolfprovider.yml + PACKAGES_PATH=/tmp + echo "PACKAGES_PATH=$PACKAGES_PATH" >> "$GITHUB_ENV" + echo "WOLFSSL_PACKAGES_PATH=\${PACKAGES_PATH}/wolfssl-packages" >> "$GITHUB_ENV" + echo "OPENSSL_PACKAGES_PATH=\${PACKAGES_PATH}/openssl-packages" >> "$GITHUB_ENV" + echo "WOLFPROV_PACKAGES_PATH=\${PACKAGES_PATH}/wolfprov-packages" >> "$GITHUB_ENV" + +- name: Checkout wolfProvider + uses: actions/checkout@v4 + with: + fetch-depth: 1 + +- name: Download packages from build job + uses: actions/download-artifact@v4 + with: + name: debian-packages-${{ matrix.fips_ref }}${{ matrix.replace_default && '-replace-default' || '' }}-${{ matrix.wolfssl_ref }}-${{ matrix.openssl_ref }} + path: ${{ env.PACKAGES_PATH }} + +- name: Install wolfSSL/OpenSSL/wolfprov packages + run: | + apt install --reinstall -y \ + ${{ env.WOLFSSL_PACKAGES_PATH }}/libwolfssl_*.deb + + apt install --reinstall -y --allow-downgrades --allow-change-held-packages \ + ${{ env.OPENSSL_PACKAGES_PATH }}/openssl_*.deb \ + ${{ env.OPENSSL_PACKAGES_PATH }}/libssl3_*.deb \ + ${{ env.OPENSSL_PACKAGES_PATH }}/libssl-dev_*.deb + + apt install --reinstall -y \ + ${{ env.WOLFPROV_PACKAGES_PATH }}/libwolfprov_*.deb + +- name: Verify wolfProvider is properly installed + run: | + $GITHUB_WORKSPACE/scripts/verify-install.sh \ + ${{ matrix.replace_default && '--replace-default' || '' }} \ + ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} diff --git a/.github/workflow-src/common/wp-matrix.yml.in b/.github/workflow-src/common/wp-matrix.yml.in new file mode 100644 index 00000000..f8148b93 --- /dev/null +++ b/.github/workflow-src/common/wp-matrix.yml.in @@ -0,0 +1,4 @@ +wolfssl_ref: [ 'v5.8.4-stable' ] +openssl_ref: [ 'openssl-3.5.4' ] +fips_ref: [ 'FIPS', 'non-FIPS' ] +replace_default: [ true ] diff --git a/.github/workflow-src/common/wp-testenv.yml.in b/.github/workflow-src/common/wp-testenv.yml.in new file mode 100644 index 00000000..16be14d7 --- /dev/null +++ b/.github/workflow-src/common/wp-testenv.yml.in @@ -0,0 +1,10 @@ +runs-on: ubuntu-22.04 +needs: build_wolfprovider +# Run inside Debian Bookworm to match packaging environment +container: + image: debian:bookworm + env: + DEBIAN_FRONTEND: noninteractive + +# This should be a safe limit for the tests to run. +timeout-minutes: 20 \ No newline at end of file diff --git a/.github/workflow-src/common/wp-trigger.yml.in b/.github/workflow-src/common/wp-trigger.yml.in new file mode 100644 index 00000000..783adca5 --- /dev/null +++ b/.github/workflow-src/common/wp-trigger.yml.in @@ -0,0 +1,9 @@ +on: + push: + branches: [ 'master', 'main', 'release/**' ] + pull_request: + branches: [ '*' ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true \ No newline at end of file diff --git a/.github/workflow-src/curl.yml.in b/.github/workflow-src/curl.yml.in new file mode 100644 index 00000000..cb0cf6fa --- /dev/null +++ b/.github/workflow-src/curl.yml.in @@ -0,0 +1,57 @@ +name: Curl Tests + +include: "./common/wp-trigger.yml.in" + +jobs: + include: "./common/wp-build.yml.in" + + test_curl: + include: "./common/wp-testenv.yml.in" + + strategy: + matrix: + curl_ref: [ 'curl-8_4_0', 'curl-7_88_1' ] + force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] + include: "./common/wp-matrix.yml.in" + steps: + include: "./common/wp-install.yml.in" + + - name: Install curl dependencies + run: | + apt-get update + apt-get install -y nghttp2 libpsl5 libpsl-dev python3-impacket \ + build-essential autoconf automake libtool + + - name: Build curl + uses: wolfSSL/actions-build-autotools-project@v1 + with: + repository: curl/curl + path: curl + ref: ${{ matrix.curl_ref }} + configure: --with-openssl + check: false + + - name: Generate certificates for curl master force-fail tests + run: | + if [ "${{ matrix.force_fail }}" = "WOLFPROV_FORCE_FAIL=1" ] && + [ "${{ matrix.curl_ref }}" = "master" ]; then + cd curl/tests/certs + make test-ca.cacert + cd ../.. + fi + - name: Test curl with wolfProvider + working-directory: curl + shell: bash + run: | + set +o pipefail # ignore errors from make check + export ${{ matrix.force_fail }} + export CURL_REF=${{ matrix.curl_ref }} + + # Tests rely on $USER being set + export USER=testuser + + # Run tests and save output to test.log + make -j$(nproc) test-ci 2>&1 | tee curl-test.log + # Capture the test result using PIPESTATUS (Bash only) + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} curl diff --git a/.github/workflow-src/readme.md b/.github/workflow-src/readme.md new file mode 100644 index 00000000..279392f4 --- /dev/null +++ b/.github/workflow-src/readme.md @@ -0,0 +1,18 @@ +# Github workflow pre-processing + +## TLDR +Run `make` from this directory to regenerate the yamls in the `.github/workflows` directory. Then commit those files without further modification. + +## Details +In order to unify large chunks of code, we pre-process the workflow yamls to insert common blocks of code. + +The files `*.yml.in` are the pre-processed workflow files. Manually edit these, not the generated ones in `.github/workflows`. Run `make` here to regenerate any outdated workflows. + +The workflow `generated-workflows.yml` ensures that the pre-processing has been done. + +### Syntax +Designed to be as lightweight and yaml-like as possible, insert the following line structure into a `.yml.in` file. + +` include: "path/to/file.yml.in"` + +Any leading whitespace indentation is preserved. diff --git a/.github/workflows/build-wolfprovider.yml b/.github/workflows/build-wolfprovider.yml index 319c4927..080804b5 100644 --- a/.github/workflows/build-wolfprovider.yml +++ b/.github/workflows/build-wolfprovider.yml @@ -15,7 +15,6 @@ on: replace_default: required: false type: boolean - default: false jobs: build_wolfprovider_common: @@ -42,6 +41,7 @@ jobs: OPENSSL_PACKAGES_PATH: /tmp/openssl-packages WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages DEBS_PATH: debs + PACKAGE_NAME: debian-packages-${{ inputs.fips_ref }}${{ inputs.replace_default && '-replace-default' || '' }}-${{ inputs.wolfssl_ref }}-${{ inputs.openssl_ref }} steps: # Check if artifact already exists from another job in the same workflow run # When multiple matrix jobs run in parallel, the first one to finish uploads the artifact @@ -51,7 +51,7 @@ jobs: continue-on-error: true uses: actions/download-artifact@v4 with: - name: debian-packages-${{ inputs.fips_ref }}${{ inputs.replace_default && '-replace-default' || '' }}-${{ inputs.wolfssl_ref }}-${{ inputs.openssl_ref }} + name: ${{ env.PACKAGE_NAME }} # Download pre-built packages from debs branch - name: Checkout debs branch @@ -199,7 +199,7 @@ jobs: if: steps.check_artifact.outcome != 'success' uses: actions/upload-artifact@v4 with: - name: debian-packages-${{ inputs.fips_ref }}${{ inputs.replace_default && '-replace-default' || '' }}-${{ inputs.wolfssl_ref }}-${{ inputs.openssl_ref }} + name: ${{ env.PACKAGE_NAME }} path: | ${{ env.WOLFSSL_PACKAGES_PATH }} ${{ env.OPENSSL_PACKAGES_PATH }} diff --git a/.github/workflows/cjose.yml b/.github/workflows/cjose.yml index 778a4225..fd18338c 100644 --- a/.github/workflows/cjose.yml +++ b/.github/workflows/cjose.yml @@ -1,6 +1,5 @@ +# Warning: generated file, do not edit. Generated by '../scripts/expand-workflow.sh cjose.yml.in' name: cjose Tests - -# START OF COMMON SECTION on: push: branches: [ 'master', 'main', 'release/**' ] @@ -10,7 +9,6 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -# END OF COMMON SECTION jobs: build_wolfprovider: @@ -35,58 +33,65 @@ jobs: image: debian:bookworm env: DEBIAN_FRONTEND: noninteractive + # This should be a safe limit for the tests to run. timeout-minutes: 20 + strategy: matrix: - # Dont test osp master since it might be unstable cjose_ref: [ 'v0.6.2.1' ] + force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] wolfssl_ref: [ 'v5.8.4-stable' ] openssl_ref: [ 'openssl-3.5.4' ] fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] - env: - WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages - OPENSSL_PACKAGES_PATH: /tmp/openssl-packages - WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages + steps: - - name: Install cjose dependencies + - name: Setup env vars run: | - apt-get update - apt-get install -y git build-essential autoconf automake \ - libtool pkg-config libjansson-dev check ca-certificates dpkg-dev - + # These paths must match the ones in build-wolfprovider.yml + PACKAGES_PATH=/tmp + echo "PACKAGES_PATH=$PACKAGES_PATH" >> "$GITHUB_ENV" + echo "WOLFSSL_PACKAGES_PATH=\${PACKAGES_PATH}/wolfssl-packages" >> "$GITHUB_ENV" + echo "OPENSSL_PACKAGES_PATH=\${PACKAGES_PATH}/openssl-packages" >> "$GITHUB_ENV" + echo "WOLFPROV_PACKAGES_PATH=\${PACKAGES_PATH}/wolfprov-packages" >> "$GITHUB_ENV" + - name: Checkout wolfProvider uses: actions/checkout@v4 with: fetch-depth: 1 - + - name: Download packages from build job uses: actions/download-artifact@v4 with: name: debian-packages-${{ matrix.fips_ref }}${{ matrix.replace_default && '-replace-default' || '' }}-${{ matrix.wolfssl_ref }}-${{ matrix.openssl_ref }} - path: /tmp - + path: ${{ env.PACKAGES_PATH }} + - name: Install wolfSSL/OpenSSL/wolfprov packages run: | apt install --reinstall -y \ ${{ env.WOLFSSL_PACKAGES_PATH }}/libwolfssl_*.deb - + apt install --reinstall -y --allow-downgrades --allow-change-held-packages \ ${{ env.OPENSSL_PACKAGES_PATH }}/openssl_*.deb \ ${{ env.OPENSSL_PACKAGES_PATH }}/libssl3_*.deb \ ${{ env.OPENSSL_PACKAGES_PATH }}/libssl-dev_*.deb - + apt install --reinstall -y \ ${{ env.WOLFPROV_PACKAGES_PATH }}/libwolfprov_*.deb - + - name: Verify wolfProvider is properly installed run: | $GITHUB_WORKSPACE/scripts/verify-install.sh \ ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} + - name: Install cjose dependencies + run: | + apt-get update + apt-get install -y git build-essential autoconf automake \ + libtool pkg-config libjansson-dev check ca-certificates dpkg-dev + - name: Download cjose uses: actions/checkout@v4 with: @@ -99,15 +104,12 @@ jobs: working-directory: cjose run: | ./configure CFLAGS="-Wno-error=deprecated-declarations" - - # Build cjose make - name: Run cjose tests working-directory: cjose run: | export ${{ matrix.force_fail }} - make test 2>&1 | tee cjose-test.log TEST_RESULT=$(grep -q "FAIL: check_cjose" cjose-test.log && echo "1" || echo "0") echo "TEST_RESULT = $TEST_RESULT" diff --git a/.github/workflows/curl.yml b/.github/workflows/curl.yml index 6c5ee31b..85bf9464 100644 --- a/.github/workflows/curl.yml +++ b/.github/workflows/curl.yml @@ -1,6 +1,6 @@ +# Warning: generated file, do not edit. Generated by '../scripts/expand-workflow.sh curl.yml.in' name: Curl Tests -# START OF COMMON SECTION on: push: branches: [ 'master', 'main', 'release/**' ] @@ -10,7 +10,6 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -# END OF COMMON SECTION jobs: build_wolfprovider: @@ -30,56 +29,64 @@ jobs: test_curl: runs-on: ubuntu-22.04 needs: build_wolfprovider + # Run inside Debian Bookworm to match packaging environment container: image: debian:bookworm env: DEBIAN_FRONTEND: noninteractive + # This should be a safe limit for the tests to run. timeout-minutes: 20 + strategy: matrix: curl_ref: [ 'curl-8_4_0', 'curl-7_88_1' ] + force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] wolfssl_ref: [ 'v5.8.4-stable' ] openssl_ref: [ 'openssl-3.5.4' ] fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] - env: - WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages - OPENSSL_PACKAGES_PATH: /tmp/openssl-packages - WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages steps: + - name: Setup env vars + run: | + # These paths must match the ones in build-wolfprovider.yml + PACKAGES_PATH=/tmp + echo "PACKAGES_PATH=$PACKAGES_PATH" >> "$GITHUB_ENV" + echo "WOLFSSL_PACKAGES_PATH=\${PACKAGES_PATH}/wolfssl-packages" >> "$GITHUB_ENV" + echo "OPENSSL_PACKAGES_PATH=\${PACKAGES_PATH}/openssl-packages" >> "$GITHUB_ENV" + echo "WOLFPROV_PACKAGES_PATH=\${PACKAGES_PATH}/wolfprov-packages" >> "$GITHUB_ENV" + - name: Checkout wolfProvider uses: actions/checkout@v4 with: fetch-depth: 1 - + - name: Download packages from build job uses: actions/download-artifact@v4 with: name: debian-packages-${{ matrix.fips_ref }}${{ matrix.replace_default && '-replace-default' || '' }}-${{ matrix.wolfssl_ref }}-${{ matrix.openssl_ref }} - path: /tmp - + path: ${{ env.PACKAGES_PATH }} + - name: Install wolfSSL/OpenSSL/wolfprov packages run: | - apt install --reinstall -y --allow-downgrades --allow-change-held-packages \ + apt install --reinstall -y \ ${{ env.WOLFSSL_PACKAGES_PATH }}/libwolfssl_*.deb - + apt install --reinstall -y --allow-downgrades --allow-change-held-packages \ ${{ env.OPENSSL_PACKAGES_PATH }}/openssl_*.deb \ ${{ env.OPENSSL_PACKAGES_PATH }}/libssl3_*.deb \ ${{ env.OPENSSL_PACKAGES_PATH }}/libssl-dev_*.deb - - apt install --reinstall -y --allow-downgrades --allow-change-held-packages \ + + apt install --reinstall -y \ ${{ env.WOLFPROV_PACKAGES_PATH }}/libwolfprov_*.deb - + - name: Verify wolfProvider is properly installed run: | $GITHUB_WORKSPACE/scripts/verify-install.sh \ ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies + - name: Install curl dependencies run: | apt-get update apt-get install -y nghttp2 libpsl5 libpsl-dev python3-impacket \ diff --git a/.github/workflows/generated-workflows.yml b/.github/workflows/generated-workflows.yml new file mode 100644 index 00000000..a0f6370b --- /dev/null +++ b/.github/workflows/generated-workflows.yml @@ -0,0 +1,28 @@ +name: Generated workflow sync + +on: + pull_request: + +# This workflow fails if any generated workflows are not up to date. +# Re-run 'make' in .github/workflow-src to update them. +jobs: + check-workflows: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential + + - name: Generate workflows + run: | + cd .github/workflow-src + make + + - name: Verify no diffs + run: | + git diff --exit-code .github/workflows