Dynamic block building & jit transaction fetching

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Added
 
+- Introduced dynamic block building and JIT transaction fetching [#394](https://github.com/proto-kit/framework/pull/394)
 - Introduced block explorer [#381](https://github.com/proto-kit/framework/pull/381)
 - Added CircuitAnalysisModule for easy analysis of protocol circuits [#379](https://github.com/proto-kit/framework/pull/379)
 - Separated settlement and bridging functionally, so now settlement can be used without bridging [#376](https://github.com/proto-kit/framework/pull/376)

packages/api/src/graphql/modules/MempoolResolver.ts

@@ -161,7 +161,10 @@ export class MempoolResolver extends GraphqlModule {
       "Returns the hashes of all transactions that are currently inside the mempool",
   })
   public async transactions() {
-    const txs = await this.transactionStorage.getPendingUserTransactions();
+    const txs = await this.transactionStorage.getPendingUserTransactions(
+      0,
+      1000
+    );
     return txs.map((x) => x.hash().toString());
   }
 }

packages/indexer/src/IndexerNotifier.ts

@@ -81,7 +81,7 @@ export class IndexerNotifier extends SequencerModule<Record<never, never>> {
 
         await txQueue.addTask(task);
       } catch (err) {
-        console.error("Failed to add pending-tx task", err);
+        log.error("Failed to add pending-tx task", err);
       }
     });
     this.sequencer.events.on("batch-produced", async (batch) => {

packages/library/src/hooks/RuntimeFeeAnalyzerService.ts

@@ -79,8 +79,8 @@ export class RuntimeFeeAnalyzerService extends ConfigurableModule<RuntimeFeeAnal
       transaction: RuntimeTransaction.dummyTransaction(),
       networkState: NetworkState.empty(),
     });
+    context.clear();
 
-    container.resolve(RuntimeMethodExecutionContext).clear();
     let methodCounter = 0;
     const [values, indexes] =
       await this.runtime.zkProgrammable.zkProgram.reduce<

packages/library/src/hooks/TransactionFeeHook.ts

@@ -126,35 +126,30 @@ export class TransactionFeeHook extends ProvableTransactionHook<TransactionFeeHo
    *
    * @param executionData
    */
-  public async beforeTransaction(
-    executionData: BeforeTransactionHookArguments
-  ): Promise<void> {
+  public async beforeTransaction({
+    transaction: { transaction },
+  }: BeforeTransactionHookArguments): Promise<void> {
     const feeConfig = Provable.witness(MethodFeeConfigData, () =>
-      this.feeAnalyzer.getFeeConfig(
-        executionData.transaction.methodId.toBigInt()
-      )
+      this.feeAnalyzer.getFeeConfig(transaction.methodId.toBigInt())
     );
     const witness = Provable.witness(
       RuntimeFeeAnalyzerService.getWitnessType(),
-      () =>
-        this.feeAnalyzer.getWitness(
-          executionData.transaction.methodId.toBigInt()
-        )
+      () => this.feeAnalyzer.getWitness(transaction.methodId.toBigInt())
     );
 
     const root = Field(this.feeAnalyzer.getRoot());
     const calculatedRoot = witness.calculateRoot(feeConfig.hash());
 
     root.assertEquals(calculatedRoot, errors.invalidFeeTreeRoot());
     feeConfig.methodId.assertEquals(
-      executionData.transaction.methodId,
+      transaction.methodId,
       errors.invalidFeeConfigMethodId()
     );
 
     const fee = this.getFee(feeConfig);
 
     await this.transferFee(
-      executionData.transaction.sender,
+      transaction.sender,
       UInt64.Unsafe.fromField(fee.value)
     );
   }
@@ -163,23 +158,26 @@ export class TransactionFeeHook extends ProvableTransactionHook<TransactionFeeHo
     noop();
   }
 
-  public async removeTransactionWhen(
-    args: BeforeTransactionHookArguments
-  ): Promise<boolean> {
+  public async removeTransactionWhen({
+    transaction,
+  }: BeforeTransactionHookArguments): Promise<boolean> {
     const feeConfig = this.feeAnalyzer.getFeeConfig(
-      args.transaction.methodId.toBigInt()
+      transaction.transaction.methodId.toBigInt()
     );
 
     const fee = this.getFee(feeConfig);
 
     const tokenId = new TokenId(this.config.tokenId);
-    const feeRecipient = PublicKey.fromBase58(this.config.feeRecipient);
 
     const balanceAvailable = await this.balances.balances.get({
       tokenId,
-      address: feeRecipient,
+      address: transaction.transaction.sender.value,
     });
 
-    return balanceAvailable.orElse(Balance.from(0)).lessThan(fee).toBoolean();
+    return balanceAvailable
+      .orElse(Balance.from(0))
+      .lessThan(fee)
+      .or(transaction.isMessage)
+      .toBoolean();
   }
 }

packages/module/test/method/MethodParameterEncoder.test.ts

@@ -1,3 +1,4 @@
+import "reflect-metadata";
 import {
   Struct,
   Field,

packages/module/test/method/runtimeMethod-fail.test.ts

@@ -1,3 +1,4 @@
+import "reflect-metadata";
 import { Bool, Field, PublicKey, Struct, ZkProgram } from "o1js";
 import { noop } from "@proto-kit/common";
 

packages/persistance/src/services/prisma/PrismaTransactionStorage.ts

@@ -19,7 +19,10 @@ export class PrismaTransactionStorage implements TransactionStorage {
   ) {}
 
   @trace("db.txs.get")
-  public async getPendingUserTransactions(): Promise<PendingTransaction[]> {
+  public async getPendingUserTransactions(
+    offset: number,
+    limit?: number
+  ): Promise<PendingTransaction[]> {
     const { prismaClient } = this.connection;
 
     const txs = await prismaClient.transaction.findMany({
@@ -31,6 +34,8 @@ export class PrismaTransactionStorage implements TransactionStorage {
           equals: false,
         },
       },
+      skip: offset,
+      take: limit,
     });
     return txs.map((tx) => this.transactionMapper.mapIn(tx));
   }

packages/persistance/test-integration/PrismaBlockProduction.test.ts

@@ -251,7 +251,7 @@ describe("prisma integration", () => {
         PrismaTransactionStorage
       );
 
-      const txs = await txResolver.getPendingUserTransactions();
+      const txs = await txResolver.getPendingUserTransactions(0);
 
       expectDefined(transaction.transaction);
 

packages/protocol/src/hooks/AccountStateHook.ts

@@ -22,7 +22,7 @@ export class AccountStateHook extends ProvableTransactionHook {
   );
 
   public async beforeTransaction({
-    transaction,
+    transaction: { transaction },
   }: BeforeTransactionHookArguments) {
     const sender = transaction.sender.value;
 
@@ -57,7 +57,7 @@ export class AccountStateHook extends ProvableTransactionHook {
 
   // Under these conditions we want the tx removed from the mempool.
   public async removeTransactionWhen({
-    transaction,
+    transaction: { transaction },
   }: BeforeTransactionHookArguments): Promise<boolean> {
     const sender = transaction.sender.value;
 

packages/protocol/src/index.ts

@@ -6,7 +6,7 @@ export * from "./model/StateTransitionProvableBatch";
 export * from "./model/Option";
 export * from "./model/Path";
 export * from "./model/network/NetworkState";
-export * from "./model/transaction/SignedTransaction";
+export * from "./model/transaction/AuthorizedTransaction";
 export * from "./model/transaction/RuntimeTransaction";
 export * from "./model/transaction/ValueOption";
 export * from "./model/MethodPublicOutput";

packages/protocol/src/model/transaction/SignedTransaction.ts → packages/protocol/src/model/transaction/AuthorizedTransaction.ts

@@ -2,26 +2,30 @@ import { Bool, Field, Scalar, Signature, Struct, UInt64 } from "o1js";
 
 import { RuntimeTransaction } from "./RuntimeTransaction";
 
-export class SignedTransaction extends Struct({
+export class AuthorizedTransaction extends Struct({
   transaction: RuntimeTransaction,
   signature: Signature,
+  isMessage: Bool,
 }) {
   public static getSignatureData(args: {
     methodId: Field;
     nonce: UInt64;
     argsHash: Field;
   }): Field[] {
+    // No isMessage here - we don't sign that
     return [args.methodId, ...args.nonce.value.toFields(), args.argsHash];
   }
 
-  public static dummy(): SignedTransaction {
-    return new SignedTransaction({
+  public static dummy(): AuthorizedTransaction {
+    return new AuthorizedTransaction({
       transaction: RuntimeTransaction.dummyTransaction(),
 
       signature: Signature.fromObject({
         s: Scalar.from(0),
         r: Field(0),
       }),
+
+      isMessage: Bool(false),
     });
   }
 
@@ -31,17 +35,16 @@ export class SignedTransaction extends Struct({
 
   public getSignatureData(): Field[] {
     const { methodId, argsHash, nonce } = this.transaction;
-    return SignedTransaction.getSignatureData({
+    return AuthorizedTransaction.getSignatureData({
       nonce: nonce.value,
       methodId,
       argsHash,
     });
   }
 
-  public validateSignature(): Bool {
-    return this.signature.verify(
-      this.transaction.sender.value,
-      this.getSignatureData()
-    );
+  public validateAuthorization(): Bool {
+    return this.signature
+      .verify(this.transaction.sender.value, this.getSignatureData())
+      .or(this.isMessage);
   }
 }

packages/protocol/src/protocol/ProvableTransactionHook.ts

@@ -1,13 +1,10 @@
 import { NoConfig } from "@proto-kit/common";
-import { Field, Signature } from "o1js";
+import { Field } from "o1js";
 
-import { RuntimeTransaction } from "../model/transaction/RuntimeTransaction";
 import { NetworkState } from "../model/network/NetworkState";
 import { MethodPublicOutput } from "../model/MethodPublicOutput";
-import {
-  TransactionProverState,
-  TransactionProverTransactionArguments,
-} from "../prover/transaction/TransactionProvable";
+import { TransactionProverState } from "../prover/transaction/TransactionProvable";
+import { AuthorizedTransaction } from "../model/transaction/AuthorizedTransaction";
 
 import { TransitioningProtocolModule } from "./TransitioningProtocolModule";
 
@@ -32,34 +29,29 @@ export function toProvableHookTransactionState(
 }
 
 export function toBeforeTransactionHookArgument(
-  executionData: Omit<
-    TransactionProverTransactionArguments,
-    "verificationKeyAttestation"
-  >,
+  authorizedTransaction: AuthorizedTransaction,
   networkState: NetworkState,
   state: Parameters<typeof toProvableHookTransactionState>[0]
 ): BeforeTransactionHookArguments {
-  const { transaction, signature } = executionData;
-
   return {
     networkState,
-    transaction,
-    signature,
+    transaction: authorizedTransaction,
     prover: toProvableHookTransactionState(state),
   };
 }
 
 export function toAfterTransactionHookArgument(
-  executionData: Omit<
-    TransactionProverTransactionArguments,
-    "verificationKeyAttestation"
-  >,
+  authorizedTransaction: AuthorizedTransaction,
   networkState: NetworkState,
   state: Parameters<typeof toProvableHookTransactionState>[0],
   runtimeResult: MethodPublicOutput
 ): AfterTransactionHookArguments {
   return {
-    ...toBeforeTransactionHookArgument(executionData, networkState, state),
+    ...toBeforeTransactionHookArgument(
+      authorizedTransaction,
+      networkState,
+      state
+    ),
     runtimeResult,
   };
 }
@@ -75,8 +67,7 @@ export type TransactionResult = Omit<
 >;
 
 export interface BeforeTransactionHookArguments {
-  transaction: RuntimeTransaction;
-  signature: Signature;
+  transaction: AuthorizedTransaction;
   networkState: NetworkState;
   prover: ProvableHookTransactionState;
 }

packages/protocol/src/prover/transaction/TransactionProver.ts

@@ -24,7 +24,7 @@ import {
 import { StateServiceProvider } from "../../state/StateServiceProvider";
 import { RuntimeVerificationKeyRootService } from "../block/services/RuntimeVerificationKeyRootService";
 import { addTransactionToBundle, executeHooks } from "../utils";
-import { SignedTransaction } from "../../model/transaction/SignedTransaction";
+import { AuthorizedTransaction } from "../../model/transaction/AuthorizedTransaction";
 import {
   MethodVKConfigData,
   MinimalVKTreeService,
@@ -115,8 +115,14 @@ export class TransactionProverZkProgrammable extends ZkProgrammable<
 
     const { isMessage } = runtimeOutput;
 
+    const authorizedTransaction = new AuthorizedTransaction({
+      transaction,
+      signature,
+      isMessage,
+    });
+
     const beforeTxHookArguments = toBeforeTransactionHookArgument(
-      executionData,
+      authorizedTransaction,
       networkState,
       state
     );
@@ -140,7 +146,7 @@ export class TransactionProverZkProgrammable extends ZkProgrammable<
 
     // Apply afterTransaction hook state transitions
     const afterTxHookArguments = toAfterTransactionHookArgument(
-      executionData,
+      authorizedTransaction,
       networkState,
       state,
       runtimeOutput
@@ -165,14 +171,10 @@ export class TransactionProverZkProgrammable extends ZkProgrammable<
       "Transactions provided in AppProof and BlockProof do not match"
     );
 
-    // Check transaction signature
-    new SignedTransaction({
-      transaction,
-      signature,
-    })
-      .validateSignature()
-      .or(isMessage)
-      .assertTrue("Transaction signature not valid");
+    // Check transaction signature or isMessage
+    authorizedTransaction
+      .validateAuthorization()
+      .assertTrue("Transaction authorization not valid");
 
     // Validate layout of transaction witness
     transaction.assertTransactionType(isMessage);
@@ -210,7 +212,10 @@ export class TransactionProverZkProgrammable extends ZkProgrammable<
     isMessage: Bool
   ) {
     const { batch, rawStatus } = await executeHooks(
-      hookArguments,
+      {
+        transaction: hookArguments.transaction.transaction,
+        networkState: hookArguments.networkState,
+      },
       `${type}Transaction`,
       async () => {
         for (const module of this.transactionHooks) {

packages/protocol/test/BlockProver.test.ts

@@ -10,7 +10,7 @@ import "reflect-metadata";
 import {
   MethodPublicOutput,
   NetworkState,
-  SignedTransaction,
+  AuthorizedTransaction,
   StateTransitionProverPublicInput,
   StateTransitionProverPublicOutput,
 } from "../src";