diff --git a/.github/workflows/pull-request-develop.yml b/.github/workflows/pull-request-develop.yml index 933c037b8..77ea12c2c 100644 --- a/.github/workflows/pull-request-develop.yml +++ b/.github/workflows/pull-request-develop.yml @@ -113,7 +113,7 @@ jobs: image: redis:6.2-alpine ports: - 6379:6379 - lighnet: + lightnet: image: o1labs/mina-local-network:compatible-latest-lightnet env: RUN_ARCHIVE_NODE: true @@ -126,6 +126,15 @@ jobs: - 8181:8181 # archive endpoints - 8282:8282 + minio: + # fixme: let's not depend on external unofficial image + image: lazybit/minio + ports: + - 9000:9000 + env: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + options: --name=minio --health-cmd "curl http://localhost:9000/minio/health/live" steps: - uses: actions/checkout@v3 @@ -151,4 +160,4 @@ jobs: polling-interval-ms: 5000 - name: "Integration tests" - run: npm run test:integration \ No newline at end of file + run: npm run test:integration diff --git a/package-lock.json b/package-lock.json index f5626d39d..3d20a5e90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8094,6 +8094,13 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/@zxing/text-encoding": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@zxing/text-encoding/-/text-encoding-0.9.0.tgz", + "integrity": "sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==", + "license": "(Unlicense OR Apache-2.0)", + "optional": true + }, "node_modules/abbrev": { "version": "2.0.0", "license": "ISC", @@ -8600,7 +8607,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", - "dev": true, "license": "MIT", "dependencies": { "possible-typed-array-names": "^1.0.0" @@ -8942,6 +8948,15 @@ "license": "MIT", "peer": true }, + "node_modules/block-stream2": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/block-stream2/-/block-stream2-2.1.0.tgz", + "integrity": "sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==", + "license": "MIT", + "dependencies": { + "readable-stream": "^3.4.0" + } + }, "node_modules/body": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/body/-/body-5.1.0.tgz", @@ -9051,6 +9066,12 @@ "node": ">=8" } }, + "node_modules/browser-or-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", + "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==", + "license": "MIT" + }, "node_modules/browserslist": { "version": "4.23.1", "funding": [ @@ -9122,6 +9143,15 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "license": "MIT" @@ -9347,7 +9377,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", - "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.0", @@ -10763,7 +10792,6 @@ }, "node_modules/define-data-property": { "version": "1.1.4", - "dev": true, "license": "MIT", "dependencies": { "es-define-property": "^1.0.0", @@ -12532,6 +12560,24 @@ "fast-decode-uri-component": "^1.0.1" } }, + "node_modules/fast-xml-parser": { + "version": "4.5.3", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.3.tgz", + "integrity": "sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^1.1.1" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.17.1", "license": "ISC", @@ -12850,7 +12896,6 @@ "version": "0.3.5", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", - "dev": true, "license": "MIT", "dependencies": { "is-callable": "^1.2.7" @@ -14525,7 +14570,6 @@ }, "node_modules/has-property-descriptors": { "version": "1.0.2", - "dev": true, "license": "MIT", "dependencies": { "es-define-property": "^1.0.0" @@ -15881,6 +15925,22 @@ "node": ">=0.10.0" } }, + "node_modules/is-arguments": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", + "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-array-buffer": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", @@ -15980,7 +16040,6 @@ }, "node_modules/is-callable": { "version": "1.2.7", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -16370,7 +16429,6 @@ "version": "1.1.15", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", - "dev": true, "license": "MIT", "dependencies": { "which-typed-array": "^1.1.16" @@ -20573,6 +20631,91 @@ "node": ">= 6" } }, + "node_modules/minio": { + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/minio/-/minio-8.0.6.tgz", + "integrity": "sha512-sOeh2/b/XprRmEtYsnNRFtOqNRTPDvYtMWh+spWlfsuCV/+IdxNeKVUMKLqI7b5Dr07ZqCPuaRGU/rB9pZYVdQ==", + "license": "Apache-2.0", + "dependencies": { + "async": "^3.2.4", + "block-stream2": "^2.1.0", + "browser-or-node": "^2.1.1", + "buffer-crc32": "^1.0.0", + "eventemitter3": "^5.0.1", + "fast-xml-parser": "^4.4.1", + "ipaddr.js": "^2.0.1", + "lodash": "^4.17.21", + "mime-types": "^2.1.35", + "query-string": "^7.1.3", + "stream-json": "^1.8.0", + "through2": "^4.0.2", + "web-encoding": "^1.1.5", + "xml2js": "^0.5.0 || ^0.6.2" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, + "node_modules/minio/node_modules/decode-uri-component": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/minio/node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", + "license": "MIT" + }, + "node_modules/minio/node_modules/filter-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", + "integrity": "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/minio/node_modules/ipaddr.js": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.3.0.tgz", + "integrity": "sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/minio/node_modules/query-string": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz", + "integrity": "sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==", + "license": "MIT", + "dependencies": { + "decode-uri-component": "^0.2.2", + "filter-obj": "^1.1.0", + "split-on-first": "^1.0.0", + "strict-uri-encode": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minio/node_modules/split-on-first": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", + "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/minipass": { "version": "7.1.2", "license": "ISC", @@ -23307,7 +23450,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -24394,7 +24536,6 @@ }, "node_modules/readable-stream": { "version": "3.6.2", - "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -24890,6 +25031,15 @@ "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", "optional": true }, + "node_modules/sax": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.4.tgz", + "integrity": "sha512-1n3r/tGXO6b6VXMdFT54SHzT9ytu9yr7TaELowdYpMqY/Ao7EnlQGmAQ1+RatX7Tkkdm6hONI2owqNx2aZj5Sw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, "node_modules/scheduler": { "version": "0.23.2", "license": "MIT", @@ -25052,7 +25202,6 @@ }, "node_modules/set-function-length": { "version": "1.2.2", - "dev": true, "license": "MIT", "dependencies": { "define-data-property": "^1.1.4", @@ -25754,15 +25903,38 @@ "node": ">= 0.4" } }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==", + "license": "BSD-3-Clause" + }, + "node_modules/stream-json": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", + "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==", + "license": "BSD-3-Clause", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, "node_modules/streamsearch": { "version": "1.1.0", "engines": { "node": ">=10.0.0" } }, + "node_modules/strict-uri-encode": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", + "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/string_decoder": { "version": "1.3.0", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -26011,6 +26183,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.1.2.tgz", + "integrity": "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/strong-log-transformer": { "version": "2.1.0", "dev": true, @@ -26534,7 +26718,6 @@ }, "node_modules/through2": { "version": "4.0.2", - "dev": true, "license": "MIT", "dependencies": { "readable-stream": "3" @@ -27639,6 +27822,19 @@ } } }, + "node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "license": "MIT" @@ -27800,6 +27996,18 @@ "defaults": "^1.0.3" } }, + "node_modules/web-encoding": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", + "integrity": "sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA==", + "license": "MIT", + "dependencies": { + "util": "^0.12.3" + }, + "optionalDependencies": { + "@zxing/text-encoding": "0.9.0" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "license": "BSD-2-Clause" @@ -27922,7 +28130,6 @@ "version": "1.1.19", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.19.tgz", "integrity": "sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw==", - "dev": true, "license": "MIT", "dependencies": { "available-typed-arrays": "^1.0.7", @@ -28221,6 +28428,28 @@ } } }, + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "license": "MIT", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + }, "node_modules/xtend": { "version": "4.0.2", "license": "MIT", @@ -28440,6 +28669,7 @@ "version": "0.1.1-develop.833+397881ed", "license": "MIT", "dependencies": { + "cachedir": "^2.4.0", "lodash": "^4.17.21", "loglevel": "^1.8.1", "reflect-metadata": "^0.1.13", @@ -28455,6 +28685,15 @@ "tsyringe": "^4.10.0" } }, + "packages/common/node_modules/cachedir": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.4.0.tgz", + "integrity": "sha512-9EtFOZR8g22CL7BWjJ9BUx1+A/djkofnyW3aOXZORNW2kxoUpx2h+uN2cOqwPmFhnpVmxg+KW2OjOSgChTEvsQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "packages/deployment": { "name": "@proto-kit/deployment", "version": "0.1.1-develop.833+397881ed", @@ -28462,11 +28701,13 @@ "dependencies": { "@types/yargs": "^17.0.29", "loglevel": "^1.8.1", + "minio": "^8.0.6", "reflect-metadata": "^0.1.13", "yargs": "^17.7.2" }, "devDependencies": { - "@jest/globals": "^29.5.0" + "@jest/globals": "^29.5.0", + "cachedir": "^2.4.0" }, "peerDependencies": { "@proto-kit/common": "*", @@ -28478,6 +28719,16 @@ "tsyringe": "^4.10.0" } }, + "packages/deployment/node_modules/cachedir": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.4.0.tgz", + "integrity": "sha512-9EtFOZR8g22CL7BWjJ9BUx1+A/djkofnyW3aOXZORNW2kxoUpx2h+uN2cOqwPmFhnpVmxg+KW2OjOSgChTEvsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "packages/explorer": { "name": "@proto-kit/explorer", "version": "0.1.0", @@ -28780,7 +29031,8 @@ "reflect-metadata": "^0.1.13" }, "devDependencies": { - "@jest/globals": "^29.5.0" + "@jest/globals": "^29.5.0", + "cachedir": "^2.4.0" }, "peerDependencies": { "@proto-kit/api": "*", @@ -28795,6 +29047,16 @@ "o1js": "^2.10.0", "tsyringe": "^4.10.0" } + }, + "packages/stack/node_modules/cachedir": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.4.0.tgz", + "integrity": "sha512-9EtFOZR8g22CL7BWjJ9BUx1+A/djkofnyW3aOXZORNW2kxoUpx2h+uN2cOqwPmFhnpVmxg+KW2OjOSgChTEvsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } } } } diff --git a/packages/common/package.json b/packages/common/package.json index c94147b3a..2a3b8d49d 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -17,6 +17,7 @@ "access": "public" }, "dependencies": { + "cachedir": "^2.4.0", "lodash": "^4.17.21", "loglevel": "^1.8.1", "reflect-metadata": "^0.1.13", diff --git a/packages/common/src/cache/CacheManifest.ts b/packages/common/src/cache/CacheManifest.ts new file mode 100644 index 000000000..f01ae6cf5 --- /dev/null +++ b/packages/common/src/cache/CacheManifest.ts @@ -0,0 +1,57 @@ +import path from "node:path"; +import fs from "node:fs"; + +import { injectable, singleton } from "tsyringe"; +import cachedir from "cachedir"; + +@injectable() +@singleton() +export class CacheManifest { + public manifestFile(): string { + return path.format({ + dir: cachedir("o1js"), + name: "protokit-cache-manifest", + ext: "json", + }); + } + + manifestRead = false; + + manifest: string[] = []; + + private readManifest(): string[] { + const file = this.manifestFile(); + if (fs.existsSync(file)) { + return JSON.parse(fs.readFileSync(file).toString()); + } + return []; + } + + private ensureManifestRead() { + if (!this.manifestRead) { + this.manifest = this.readManifest(); + this.manifestRead = true; + } + } + + public getManifest() { + this.ensureManifestRead(); + + return this.manifest; + } + + public writeToManifest(program: string) { + this.ensureManifestRead(); + + if (!this.manifest.includes(program)) { + this.manifest.push(program); + fs.writeFileSync(this.manifestFile(), JSON.stringify(this.manifest)); + } + } + + public includes(program: string): boolean { + this.ensureManifestRead(); + + return this.manifest.includes(program); + } +} diff --git a/packages/common/src/cache/ProxyCache.ts b/packages/common/src/cache/ProxyCache.ts new file mode 100644 index 000000000..6c5a97f43 --- /dev/null +++ b/packages/common/src/cache/ProxyCache.ts @@ -0,0 +1,30 @@ +import { CacheHeader, Cache as O1Cache } from "o1js"; + +export class ProxyCache implements O1Cache { + private realCache = O1Cache.FileSystemDefault; + + private log: string[] = []; + + canWrite = true; + + debug = false; + + cacheDirectory = this.realCache.cacheDirectory; + + public read(header: CacheHeader): Uint8Array | undefined { + return this.realCache.read(header); + } + + public write(header: CacheHeader, value: Uint8Array): void { + this.log.push(header.persistentId); + return this.realCache.write(header, value); + } + + public getLog() { + return this.log; + } + + public startLog() { + this.log = []; + } +} diff --git a/packages/common/src/cache/RemoteCache.ts b/packages/common/src/cache/RemoteCache.ts new file mode 100644 index 000000000..9fc73e50d --- /dev/null +++ b/packages/common/src/cache/RemoteCache.ts @@ -0,0 +1,9 @@ +import { Readable } from "node:stream"; + +export interface RemoteCache { + storeObject(program: string, object: string, file: Readable): Promise; + + getObjects(program: string): Promise; + + readObject(program: string, object: string): Promise; +} diff --git a/packages/common/src/cache/RemoteCacheCompiler.ts b/packages/common/src/cache/RemoteCacheCompiler.ts new file mode 100644 index 000000000..71c98893a --- /dev/null +++ b/packages/common/src/cache/RemoteCacheCompiler.ts @@ -0,0 +1,131 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { inject, injectable, Lifecycle, scoped } from "tsyringe"; +import cachedir from "cachedir"; + +import { log } from "../log"; +import { mapSequential } from "../utils"; +import { + CompileArtifact, + PlainZkProgram, +} from "../zkProgrammable/ZkProgrammable"; + +import { RemoteCache } from "./RemoteCache"; +import { ProxyCache } from "./ProxyCache"; +import { CacheManifest } from "./CacheManifest"; + +@injectable() +@scoped(Lifecycle.ContainerScoped) +export class RemoteCacheCompiler { + public constructor( + @inject("RemoteCache", { isOptional: true }) + private readonly remoteCache: RemoteCache | undefined, + private readonly manifest: CacheManifest + ) { + if (remoteCache === undefined) { + log.debug( + "No remote cache configured, only using local file system cache for circuits" + ); + } else { + log.debug("Using remote cache for circuit caching"); + } + } + + private getFileFromObjectName(object: string): string { + const dir = cachedir("o1js"); + + return path.format({ + dir, + base: object, + }); + } + + private async download(remoteCache: RemoteCache, name: string) { + const objects = await remoteCache.getObjects(name); + + log.debug(`Downloading ${objects.length} cached objects for ${name}`); + + await mapSequential(objects, async (object) => { + const readable = await remoteCache.readObject(name, object); + const file = this.getFileFromObjectName(object); + + const writeStream = fs.createWriteStream(file); + readable.pipe(writeStream); + + await new Promise((res) => { + writeStream.on("close", res); + }); + }); + } + + private async uploadFile( + remoteCache: RemoteCache, + program: string, + fileName: string + ) { + const file = path.resolve(cachedir("o1js"), fileName); + const readStream = fs.createReadStream(file); + await remoteCache.storeObject(program, fileName, readStream); + } + + private async upload( + remoteCache: RemoteCache, + name: string, + identifiers: string[] + ) { + await mapSequential(identifiers, async (identifier) => { + await this.uploadFile(remoteCache, name, identifier); + await this.uploadFile(remoteCache, name, `${identifier}.header`); + }); + } + + private isSRSFile(file: string): boolean { + return file.includes("srs-") || file.includes("lagrange-"); + } + + private async compileWithRemoteCache( + remoteCache: RemoteCache, + program: Pick + ) { + const { name } = program; + + if (!this.manifest.includes("srs")) { + await this.download(remoteCache, "srs"); + } + + if (!this.manifest.includes(name)) { + await this.download(remoteCache, name); + } + + const cache = new ProxyCache(); + cache.startLog(); + + const result = await program.compile({ + cache, + }); + + const files = cache.getLog(); + log.debug("Uploading files", files); + + const srsFiles = files.filter((file) => this.isSRSFile(file)); + await this.upload(remoteCache, "srs", srsFiles); + this.manifest.writeToManifest("srs"); + + const circuitFiles = files.filter((file) => !this.isSRSFile(file)); + await this.upload(remoteCache, name, circuitFiles); + this.manifest.writeToManifest(name); + + return result; + } + + public async compileWithCache( + program: Pick + ): Promise { + if (this.remoteCache !== undefined) { + return await this.compileWithRemoteCache(this.remoteCache, program); + } else { + return await program.compile(); + } + } +} diff --git a/packages/common/src/compiling/AtomicCompileHelper.ts b/packages/common/src/compiling/AtomicCompileHelper.ts index 8d881941f..47f4272af 100644 --- a/packages/common/src/compiling/AtomicCompileHelper.ts +++ b/packages/common/src/compiling/AtomicCompileHelper.ts @@ -1,3 +1,5 @@ +import { inject, injectable } from "tsyringe"; + import { AreProofsEnabled, CompileArtifact, @@ -6,6 +8,7 @@ import { import { isSubtypeOfName } from "../utils"; import { TypedClass } from "../types"; import { log } from "../log"; +import { RemoteCacheCompiler } from "../cache/RemoteCacheCompiler"; export type ArtifactRecord = Record; @@ -14,8 +17,13 @@ export type CompileTarget = { compile: () => Promise; }; +@injectable() export class AtomicCompileHelper { - public constructor(private readonly areProofsEnabled: AreProofsEnabled) {} + public constructor( + @inject("AreProofsEnabled") + private readonly areProofsEnabled: AreProofsEnabled, + private readonly remoteCacheCompiler: RemoteCacheCompiler + ) {} private compilationPromises: { [key: string]: Promise; @@ -44,7 +52,8 @@ export class AtomicCompileHelper { ) ) { log.time(`Compiling ${name}`); - this.compilationPromises[name] = contract.compile(); + this.compilationPromises[name] = + this.remoteCacheCompiler.compileWithCache(contract); newPromise = true; } else { log.debug(`Compiling ${name} - mock`); diff --git a/packages/common/src/compiling/CompileRegistry.ts b/packages/common/src/compiling/CompileRegistry.ts index a85766ffe..86e249bfa 100644 --- a/packages/common/src/compiling/CompileRegistry.ts +++ b/packages/common/src/compiling/CompileRegistry.ts @@ -1,9 +1,6 @@ -import { inject, injectable, singleton } from "tsyringe"; +import { injectable, singleton } from "tsyringe"; -import { - AreProofsEnabled, - CompileArtifact, -} from "../zkProgrammable/ZkProgrammable"; +import { CompileArtifact } from "../zkProgrammable/ZkProgrammable"; import { ArtifactRecord, @@ -19,14 +16,7 @@ import { @injectable() @singleton() export class CompileRegistry { - public constructor( - @inject("AreProofsEnabled") - private readonly areProofsEnabled: AreProofsEnabled - ) { - this.compiler = new AtomicCompileHelper(this.areProofsEnabled); - } - - private compiler: AtomicCompileHelper; + public constructor(private readonly compiler: AtomicCompileHelper) {} private artifacts: ArtifactRecord = {}; diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index fa82cc1f2..57cf8778d 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -29,3 +29,7 @@ export * from "./compiling/AtomicCompileHelper"; export * from "./compiling/CompileRegistry"; export * from "./compiling/CompilableModule"; export * from "./compiling/services/ChildVerificationKeyService"; +export * from "./cache/RemoteCache"; +export * from "./cache/RemoteCacheCompiler"; +export * from "./cache/CacheManifest"; +export * from "./cache/ProxyCache"; diff --git a/packages/common/src/zkProgrammable/ZkProgrammable.ts b/packages/common/src/zkProgrammable/ZkProgrammable.ts index 3106a6f92..1736af95f 100644 --- a/packages/common/src/zkProgrammable/ZkProgrammable.ts +++ b/packages/common/src/zkProgrammable/ZkProgrammable.ts @@ -1,4 +1,11 @@ -import { ZkProgram, FlexibleProvablePure, Proof, Field, Provable } from "o1js"; +import { + ZkProgram, + FlexibleProvablePure, + Proof, + Field, + Provable, + Cache as O1Cache, +} from "o1js"; import { Memoize } from "typescript-memoize"; import { log } from "../log"; @@ -30,7 +37,14 @@ export interface Verify { } export interface Compile { - (): Promise; + (options?: { + cache?: O1Cache; + forceRecompile?: boolean; + proofsEnabled?: boolean; + withRuntimeTables?: boolean; + numChunks?: number; + lazyMode?: boolean; + }): Promise; } export interface PlainZkProgram< @@ -93,10 +107,10 @@ export const MOCK_VERIFICATION_KEY = dummyVerificationKey(); export function compileToMockable( compile: Compile, { areProofsEnabled }: AreProofsEnabled -): () => Promise { - return async () => { +): Compile { + return async (...args) => { if (areProofsEnabled) { - return await compile(); + return await compile(...args); } return { diff --git a/packages/deployment/docker/docker-compose.yml b/packages/deployment/docker/docker-compose.yml index 999c284d3..3dc4cf715 100644 --- a/packages/deployment/docker/docker-compose.yml +++ b/packages/deployment/docker/docker-compose.yml @@ -10,4 +10,6 @@ include: - ./lightnet/docker-compose.yml - - ./monitoring/docker-compose.yml \ No newline at end of file + - ./monitoring/docker-compose.yml + + - ./minio/docker-compose.yml diff --git a/packages/deployment/docker/minio/docker-compose.yml b/packages/deployment/docker/minio/docker-compose.yml new file mode 100644 index 000000000..907a57a99 --- /dev/null +++ b/packages/deployment/docker/minio/docker-compose.yml @@ -0,0 +1,16 @@ +# Copyright Broadcom, Inc. All Rights Reserved. +# SPDX-License-Identifier: APACHE-2.0 + +services: + minio: + image: quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z + command: server --console-address ":9001" /mnt/data + ports: + - '9000:9000' + - '9001:9001' + volumes: + - '../data/minio:/mnt/data' + +volumes: + minio_data: + driver: local diff --git a/packages/deployment/package.json b/packages/deployment/package.json index 94f86fcac..360e6777e 100644 --- a/packages/deployment/package.json +++ b/packages/deployment/package.json @@ -10,7 +10,8 @@ "lint": "eslint ./src ./test", "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", - "test:watch": "npm run test:file -- ./test/** --watch" + "test:watch": "npm run test:file -- ./test/** --watch", + "test:integration": "npm run test:file -- ./test-integration/** --runInBand" }, "main": "dist/index.js", "publishConfig": { @@ -19,6 +20,7 @@ "dependencies": { "@types/yargs": "^17.0.29", "loglevel": "^1.8.1", + "minio": "^8.0.6", "reflect-metadata": "^0.1.13", "yargs": "^17.7.2" }, @@ -32,7 +34,8 @@ "tsyringe": "^4.10.0" }, "devDependencies": { - "@jest/globals": "^29.5.0" + "@jest/globals": "^29.5.0", + "cachedir": "^2.4.0" }, "gitHead": "397881ed5d8f98f5005bcd7be7f5a12b3bc6f956" } diff --git a/packages/deployment/src/cache/S3RemoteCache.ts b/packages/deployment/src/cache/S3RemoteCache.ts new file mode 100644 index 000000000..efeca6fc8 --- /dev/null +++ b/packages/deployment/src/cache/S3RemoteCache.ts @@ -0,0 +1,92 @@ +import { Readable } from "stream"; +import { finished } from "node:stream/promises"; + +import * as Minio from "minio"; +import { RemoteCache } from "@proto-kit/common"; +import { SequencerModule, sequencerModule } from "@proto-kit/sequencer"; + +export type S3Config = { + client: Minio.ClientOptions; + bucketName: string; +}; + +const PREFIX_DELIMITER = "/"; + +@sequencerModule() +export class S3RemoteCache + extends SequencerModule + implements RemoteCache +{ + client?: Minio.Client; + + private getObjectName(program: string, object: string): string { + return `${program}${PREFIX_DELIMITER}${object}`; + } + + private assertValidObjectName(name: string) { + if (name.includes("/")) { + throw new Error("Object name can't contain slashes (/)"); + } + } + + private async ensureBucketExists() { + const bucketExists = await this.client!.bucketExists( + this.config.bucketName + ); + + if (!bucketExists) { + await this.client!.makeBucket(this.config.bucketName); + } + } + + public async storeObject( + program: string, + object: string, + file: Readable + ): Promise { + this.assertValidObjectName(object); + this.assertValidObjectName(program); + + await this.client!.putObject( + this.config.bucketName, + this.getObjectName(program, object), + file, + undefined, + {} + ); + } + + public async getObjects(program: string): Promise { + this.assertValidObjectName(program); + + const stream = this.client!.listObjectsV2( + this.config.bucketName, + program + PREFIX_DELIMITER + ); + + const results: Minio.BucketItem[] = []; + stream.on("data", (data) => { + results.push(data); + }); + + await finished(stream); + + return results.map((result) => result.name!.split("/")[1]); + } + + public async readObject(program: string, object: string): Promise { + this.assertValidObjectName(object); + this.assertValidObjectName(program); + + return await this.client!.getObject( + this.config.bucketName, + this.getObjectName(program, object) + ); + } + + async start(): Promise { + this.client = new Minio.Client(this.config.client); + + await this.ensureBucketExists(); + } +} diff --git a/packages/deployment/src/index.ts b/packages/deployment/src/index.ts index 9f8b0a609..cbc5458d5 100644 --- a/packages/deployment/src/index.ts +++ b/packages/deployment/src/index.ts @@ -1,2 +1,3 @@ export * from "./queue/BullQueue"; export * from "./environment/Environment"; +export * from "./cache/S3RemoteCache"; diff --git a/packages/deployment/test-integration/s3.test.ts b/packages/deployment/test-integration/s3.test.ts new file mode 100644 index 000000000..7d7a52de7 --- /dev/null +++ b/packages/deployment/test-integration/s3.test.ts @@ -0,0 +1,65 @@ +import * as fs from "node:fs"; + +import Minio from "minio"; +import { Field, ZkProgram } from "o1js"; +import cachedir from "cachedir"; +import { CacheManifest, log, RemoteCacheCompiler } from "@proto-kit/common"; + +import { S3RemoteCache } from "../src/cache/S3RemoteCache"; + +const program = ZkProgram({ + name: "cache-test-zkprogram", + publicInput: Field, + publicOutput: Field, + methods: { + increment: { + privateInputs: [], + async method(input: Field) { + return { publicOutput: input.add(1).add(input) }; + }, + }, + }, +}); + +describe("s3", () => { + const config: Minio.ClientOptions = { + endPoint: "localhost", + port: 9000, + useSSL: false, + accessKey: "minioadmin", + secretKey: "minioadmin", + }; + + function clearCache() { + fs.rmSync(cachedir("o1js"), { force: true, recursive: true }); + fs.mkdirSync(cachedir("o1js")); + } + + beforeAll(() => { + clearCache(); + + log.setLevel("DEBUG"); + }); + + it("should upload artifacts on first compile", async () => { + const cache = new S3RemoteCache(); + cache.config = { + client: config, + bucketName: "cache-test-bucket", + }; + await cache.start(); + + const compiler = new RemoteCacheCompiler(cache, new CacheManifest()); + + await compiler.compileWithCache(program); + + const srsObjects = await cache.getObjects("srs"); + expect(srsObjects.length).toBe(6); + + const programObjects = await cache.getObjects("cache-test-zkprogram"); + expect(programObjects.length).toBe(8); + }, 100000); + + // Not a lot else we can test this way, since o1js keeps the prover, therefore recompiling + // doesn't do anything regarding the cache... +}); diff --git a/packages/deployment/test-integration/tsconfig.json b/packages/deployment/test-integration/tsconfig.json new file mode 100644 index 000000000..a0c8c946b --- /dev/null +++ b/packages/deployment/test-integration/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "./../../../tsconfig.json", + "compilerOptions": { + "experimentalDecorators": true + }, + "include": ["./**/*.ts", "./*.ts"] +} diff --git a/packages/sequencer/package.json b/packages/sequencer/package.json index 79370f6cc..2fe6b483e 100644 --- a/packages/sequencer/package.json +++ b/packages/sequencer/package.json @@ -11,7 +11,7 @@ "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch", - "integration": "npm run test:file -- ./test-integration/** --runInBand", + "test:integration": "npm run test:file -- ./test-integration/** --runInBand", "start": "npm run build && node --experimental-vm-modules --experimental-wasm-modules --es-module-specifier-resolution=node ./dist/src/entry.js" }, "main": "dist/index.js", diff --git a/packages/sequencer/test-integration/workers/modules.ts b/packages/sequencer/test-integration/workers/modules.ts index d65204c04..ba1b3477f 100644 --- a/packages/sequencer/test-integration/workers/modules.ts +++ b/packages/sequencer/test-integration/workers/modules.ts @@ -4,7 +4,7 @@ import { Protocol } from "@proto-kit/protocol"; import { VanillaProtocolModules } from "@proto-kit/library"; import { ModulesConfig } from "@proto-kit/common"; // eslint-disable-next-line import/no-extraneous-dependencies -import { BullQueueConfig } from "@proto-kit/deployment"; +import { BullQueueConfig, S3Config } from "@proto-kit/deployment"; import { ProvenBalance } from "../../test/integration/mocks/ProvenBalance"; import { ProtocolStateTestHook } from "../../test/integration/mocks/ProtocolStateTestHook"; @@ -29,6 +29,7 @@ export const runtimeProtocolConfig: ModulesConfig<{ Protocol: { AccountState: {}, BlockProver: {}, + TransactionProver: {}, StateTransitionProver: {}, BlockHeight: {}, LastStateRoot: {}, @@ -44,3 +45,14 @@ export const BullConfig: BullQueueConfig = { db: 1, }, }; + +export const RemoteCacheConfig: S3Config = { + bucketName: "worker-test-proven", + client: { + endPoint: "localhost", + port: 9000, + useSSL: false, + accessKey: "minioadmin", + secretKey: "minioadmin", + }, +}; diff --git a/packages/sequencer/test-integration/workers/workers-proven.test.ts b/packages/sequencer/test-integration/workers/workers-proven.test.ts index f914473e4..251f87f84 100644 --- a/packages/sequencer/test-integration/workers/workers-proven.test.ts +++ b/packages/sequencer/test-integration/workers/workers-proven.test.ts @@ -2,7 +2,7 @@ import "reflect-metadata"; import { container } from "tsyringe"; import { PrivateKey, UInt64 } from "o1js"; import { expectDefined, log } from "@proto-kit/common"; -import { BullQueue } from "@proto-kit/deployment"; +import { BullQueue, S3RemoteCache } from "@proto-kit/deployment"; import { BatchProducerModule, @@ -20,6 +20,7 @@ import { BlockTestService } from "../../test/integration/services/BlockTestServi import { BullConfig, protocolClass, + RemoteCacheConfig, runtimeClass, runtimeProtocolConfig, } from "./modules"; @@ -27,6 +28,7 @@ import { ChildProcessWorker } from "./ChildProcessWorker"; const timeout = 300000; +// true const proofsEnabled = false; describe("worker-proven", () => { @@ -63,6 +65,7 @@ describe("worker-proven", () => { TaskQueue: BullQueue, FeeStrategy: ConstantFeeStrategy, SequencerStartupModule, + RemoteCache: S3RemoteCache, }); const app = AppChain.from({ @@ -82,6 +85,7 @@ describe("worker-proven", () => { TaskQueue: BullConfig, FeeStrategy: {}, SequencerStartupModule: {}, + RemoteCache: RemoteCacheConfig, }, ...runtimeProtocolConfig, }); diff --git a/packages/sequencer/test/integration/Proven.test.ts b/packages/sequencer/test/integration/Proven.test.ts index bca7f53af..77f95dbf2 100644 --- a/packages/sequencer/test/integration/Proven.test.ts +++ b/packages/sequencer/test/integration/Proven.test.ts @@ -163,7 +163,8 @@ describe.skip("Proven", () => { const vkService = new ChildVerificationKeyService(); const proofs = new InMemoryAreProofsEnabled(); proofs.setProofsEnabled(true); - const registry = new CompileRegistry(proofs); + const registry = + appChain.sequencer.dependencyContainer.resolve(CompileRegistry); registry.addArtifactsRaw({ BlockProver: { verificationKey: MOCK_VERIFICATION_KEY, diff --git a/packages/stack/package.json b/packages/stack/package.json index 2851234e1..adf6a640c 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -11,6 +11,7 @@ "test:file": "node --experimental-vm-modules ../../node_modules/jest/bin/jest.js", "test": "npm run test:file -- ./test/**", "test:watch": "npm run test:file -- ./test/** --watch", + "test:integration": "npm run test:file -- ./test-integration/** --runInBand", "graphql": "npm run build && node --experimental-vm-modules --experimental-wasm-modules --es-module-specifier-resolution=node ./dist/scripts/graphql/run-graphql.js", "start": "node --experimental-vm-modules --es-module-specifier-resolution=node ./dist/start.js" }, @@ -32,7 +33,8 @@ "tsyringe": "^4.10.0" }, "devDependencies": { - "@jest/globals": "^29.5.0" + "@jest/globals": "^29.5.0", + "cachedir": "^2.4.0" }, "dependencies": { "reflect-metadata": "^0.1.13" diff --git a/packages/stack/test-integration/caching/caching.test.ts b/packages/stack/test-integration/caching/caching.test.ts new file mode 100644 index 000000000..896cdf645 --- /dev/null +++ b/packages/stack/test-integration/caching/caching.test.ts @@ -0,0 +1,160 @@ +import "reflect-metadata"; +import { + BlockStorageNetworkStateModule, + ClientAppChain, + InMemoryBlockExplorer, + InMemorySigner, + InMemoryTransactionSender, + StateServiceQueryModule, +} from "@proto-kit/sdk"; +import { PrivateKey } from "o1js"; +import { Runtime } from "@proto-kit/module"; +import { Protocol } from "@proto-kit/protocol"; +import { + VanillaProtocolModules, + VanillaRuntimeModules, +} from "@proto-kit/library"; +import { log } from "@proto-kit/common"; +import { + BatchProducerModule, + BlockProducerModule, + InMemoryDatabase, + LocalTaskQueue, + LocalTaskWorkerModule, + ManualBlockTrigger, + NoopBaseLayer, + PrivateMempool, + Sequencer, + SequencerStartupModule, + VanillaTaskWorkerModules, +} from "@proto-kit/sequencer"; +import { container } from "tsyringe"; +import { S3RemoteCache } from "@proto-kit/deployment"; + +import { TestBalances } from "../../src"; +import * as fs from "node:fs"; +import cachedir from "cachedir"; + +export async function startAppChain() { + const appChain = ClientAppChain.from({ + Runtime: Runtime.from( + VanillaRuntimeModules.with({ + Balances: TestBalances, + }) + ), + + Protocol: Protocol.from(VanillaProtocolModules.with({})), + + Sequencer: Sequencer.from({ + Database: InMemoryDatabase, + + Mempool: PrivateMempool, + LocalTaskWorkerModule: LocalTaskWorkerModule.from( + VanillaTaskWorkerModules.withoutSettlement() + ), + + BaseLayer: NoopBaseLayer, + BatchProducerModule, + BlockProducerModule, + BlockTrigger: ManualBlockTrigger, + TaskQueue: LocalTaskQueue, + RemoteCache: S3RemoteCache, + + SequencerStartupModule, + }), + + Signer: InMemorySigner, + TransactionSender: InMemoryTransactionSender, + QueryTransportModule: StateServiceQueryModule, + NetworkStateTransportModule: BlockStorageNetworkStateModule, + BlockExplorerTransportModule: InMemoryBlockExplorer, + }); + + appChain.configure({ + Runtime: { + Balances: {}, + }, + + Protocol: { + ...Protocol.defaultConfig(), + TransactionFee: { + tokenId: 0n, + feeRecipient: PrivateKey.random().toPublicKey().toBase58(), + baseFee: 0n, + methods: {}, + perWeightUnitFee: 0n, + }, + }, + + Sequencer: { + SequencerStartupModule: {}, + Database: {}, + + Mempool: {}, + BatchProducerModule: {}, + LocalTaskWorkerModule: VanillaTaskWorkerModules.defaultConfig(), + BaseLayer: {}, + TaskQueue: {}, + + BlockProducerModule: { + allowEmptyBlock: true, + }, + + BlockTrigger: {}, + + RemoteCache: { + client: { + endPoint: "localhost", + port: 9000, + useSSL: false, + accessKey: "minioadmin", + secretKey: "minioadmin", + }, + bucketName: "caching-stack-test", + }, + }, + + TransactionSender: {}, + QueryTransportModule: {}, + NetworkStateTransportModule: {}, + + Signer: { + signer: PrivateKey.random(), + }, + BlockExplorerTransportModule: {}, + }); + + await appChain.start(true, container.createChildContainer()); + + return appChain; +} + +function clearCache() { + fs.rmSync(cachedir("o1js"), { force: true, recursive: true }); + fs.mkdirSync(cachedir("o1js")); +} + +// This test is convered by workers-proven.test.ts +describe.skip("caching stack test", () => { + let appchain: Awaited>; + + beforeAll(async () => { + log.setLevel("DEBUG"); + clearCache(); + appchain = await startAppChain(); + }, 500000); + + afterAll(async () => { + await appchain.close(); + }); + + it("should have compiled and pushed to cache", async () => { + const cache = appchain.sequencer.resolve("RemoteCache"); + + const blockProverObjects = await cache.getObjects("blockprover"); + expect(blockProverObjects.length).toBe(6); + + const srsObjects = await cache.getObjects("srs"); + expect(srsObjects.length).toBeGreaterThan(8); + }, 500000); +}); diff --git a/packages/stack/test-integration/tsconfig.json b/packages/stack/test-integration/tsconfig.json new file mode 100644 index 000000000..a0c8c946b --- /dev/null +++ b/packages/stack/test-integration/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "./../../../tsconfig.json", + "compilerOptions": { + "experimentalDecorators": true + }, + "include": ["./**/*.ts", "./*.ts"] +}