diff --git a/pyproject.toml b/pyproject.toml index b7839300..f54aca79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Python SDK that enables developers to build and deploy LangGraph readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" dependencies = [ - "uipath>=2.4.0, <2.5.0", + "uipath", "uipath-runtime>=0.4.0, <0.5.0", "langgraph>=1.0.0, <2.0.0", "langchain-core>=1.2.5, <2.0.0", diff --git a/samples/tool-calling-suspend-resume/README.md b/samples/tool-calling-suspend-resume/README.md new file mode 100644 index 00000000..24949144 --- /dev/null +++ b/samples/tool-calling-suspend-resume/README.md @@ -0,0 +1,85 @@ +# Tool-Calling Suspend/Resume Agent + +A simple agent demonstrating the suspend/resume pattern for RPA process invocations using LangGraph's `interrupt()` function. + +## Overview + +This agent calls `interrupt(InvokeProcess(...))` to suspend execution while waiting for an external RPA process to complete. The evaluation runtime detects the suspension and extracts trigger information for resumption. + +## Features + +- Single-node graph demonstrating tool-calling suspend pattern +- Uses LangGraph's `interrupt()` to suspend execution +- Proper `InvokeProcess` structure for RPA invocation +- Includes checkpointer (required for interrupts) +- Comprehensive evaluation sets with trajectory validation + +## Running Evaluations + +```bash +# Navigate to the sample directory +cd samples/tool-calling-suspend-resume + +# Run evaluation +uv run uipath eval graph evaluations/eval-sets/test_suspend_resume.json +``` + +## Evaluation Sets + +### `test_suspend_resume.json` +Tests the actual suspend/resume behavior: +- Validates agent calls `interrupt()` with proper `InvokeProcess` structure +- Checks for suspension indicators in logs +- Uses both LLM-based trajectory evaluator and contains-based evaluator + +### `test_with_evaluators.json` +Tests evaluator execution after completion: +- Modifies the agent to complete without suspending +- Validates that evaluators run and produce scores +- Useful for verifying evaluator configuration + +## Architecture + +``` +graph.py (LangGraph Agent) + ↓ +invoke_process_node → interrupt(InvokeProcess(...)) + ↓ +SUSPENDS execution + ↓ +Runtime detects suspension + ↓ +Extracts triggers + ↓ +Skips evaluators (run after resume) +``` + +## Key Components + +- **graph.py**: Main agent with single node that calls `interrupt()` +- **evaluations/**: Evaluation sets and evaluator configurations + - **eval-sets/**: Test cases for suspend and evaluator testing + - **evaluators/**: LLM trajectory and contains evaluator configs +- **pyproject.toml**: Package metadata +- **uipath.json**: Agent configuration + +## How It Works + +1. **Agent Execution**: Agent runs and reaches `interrupt(InvokeProcess(...))` +2. **Suspension**: LangGraph raises interrupt, runtime detects `SUSPENDED` status +3. **Trigger Extraction**: Runtime extracts trigger with process details +4. **Evaluator Skip**: Evaluators are skipped during suspension +5. **Resume** (when implemented): Process completes, agent resumes +6. **Evaluator Execution**: Evaluators run on final output + +## Expected Output + +When running the evaluation, you should see: +``` +🔴 DETECTED SUSPENSION → Runtime detects status change +📋 Extracted N trigger(s) → Shows trigger details +⏭️ Skipping evaluators → Explains why no evaluation +✅ Passing through triggers → Shows trigger propagation +``` + +The evaluation result will show `status: SUSPENDED` with trigger information in the output JSON. diff --git a/samples/tool-calling-suspend-resume/evaluations/eval-sets/test_simple.json b/samples/tool-calling-suspend-resume/evaluations/eval-sets/test_simple.json new file mode 100644 index 00000000..3f36abb3 --- /dev/null +++ b/samples/tool-calling-suspend-resume/evaluations/eval-sets/test_simple.json @@ -0,0 +1,23 @@ +{ + "version": "1.0", + "id": "test-simple-suspend", + "name": "Simple Suspend Test", + "description": "Basic test for suspend/resume pattern without LLM evaluator", + "evaluatorRefs": [ + "SuspendCheckEvaluator" + ], + "evaluations": [ + { + "id": "test_suspend_basic", + "name": "Basic suspend test", + "inputs": { + "query": "Test suspend and resume with RPA process" + }, + "evaluationCriterias": { + "SuspendCheckEvaluator": { + "searchText": "SUSPENDING EXECUTION" + } + } + } + ] +} diff --git a/samples/tool-calling-suspend-resume/evaluations/eval-sets/test_suspend_resume.json b/samples/tool-calling-suspend-resume/evaluations/eval-sets/test_suspend_resume.json new file mode 100644 index 00000000..acbca4af --- /dev/null +++ b/samples/tool-calling-suspend-resume/evaluations/eval-sets/test_suspend_resume.json @@ -0,0 +1,42 @@ +{ + "version": "1.0", + "id": "test-suspend-resume-eval", + "name": "Suspend Resume Test", + "description": "Test evaluation set for suspend/resume pattern with RPA process invocation", + "evaluatorRefs": [ + "SuspendResumeTrajectoryEvaluator", + "SuspendCheckEvaluator" + ], + "evaluations": [ + { + "id": "test_suspend_basic", + "name": "Basic suspend test", + "inputs": { + "query": "Test suspend and resume with RPA process" + }, + "evaluationCriterias": { + "SuspendResumeTrajectoryEvaluator": { + "expectedAgentBehavior": "The agent should call interrupt() to suspend execution while waiting for the TestProcess RPA process to complete. The execution trace should contain a GraphInterrupt exception with InvokeProcess details including process name 'TestProcess' and query parameter." + }, + "SuspendCheckEvaluator": { + "searchText": "SUSPENDING EXECUTION" + } + } + }, + { + "id": "test_suspend_complex", + "name": "Complex query suspend test", + "inputs": { + "query": "Process complex data structure with multiple parameters" + }, + "evaluationCriterias": { + "SuspendResumeTrajectoryEvaluator": { + "expectedAgentBehavior": "The agent should call interrupt() to suspend execution with InvokeProcess request for TestProcess. The trace should show the suspend happened in the invoke_process node with proper InvokeProcess structure." + }, + "SuspendCheckEvaluator": { + "searchText": "SUSPENDING EXECUTION" + } + } + } + ] +} diff --git a/samples/tool-calling-suspend-resume/evaluations/evaluators/suspend-check.json b/samples/tool-calling-suspend-resume/evaluations/evaluators/suspend-check.json new file mode 100644 index 00000000..cf298e16 --- /dev/null +++ b/samples/tool-calling-suspend-resume/evaluations/evaluators/suspend-check.json @@ -0,0 +1,15 @@ +{ + "version": "1.0", + "id": "SuspendCheckEvaluator", + "description": "Checks if the execution contains suspend-related indicators in the output.", + "evaluatorTypeId": "uipath-contains", + "evaluatorConfig": { + "name": "SuspendCheckEvaluator", + "targetOutputKey": "logs", + "negated": false, + "ignoreCase": false, + "defaultEvaluationCriteria": { + "searchText": "SUSPENDING EXECUTION" + } + } +} diff --git a/samples/tool-calling-suspend-resume/evaluations/evaluators/suspend-resume-trajectory.json b/samples/tool-calling-suspend-resume/evaluations/evaluators/suspend-resume-trajectory.json new file mode 100644 index 00000000..ec48e22e --- /dev/null +++ b/samples/tool-calling-suspend-resume/evaluations/evaluators/suspend-resume-trajectory.json @@ -0,0 +1,15 @@ +{ + "version": "1.0", + "id": "SuspendResumeTrajectoryEvaluator", + "description": "Evaluates the agent's execution trajectory for proper suspend/resume pattern with RPA process invocation.", + "evaluatorTypeId": "uipath-llm-judge-trajectory-similarity", + "evaluatorConfig": { + "name": "SuspendResumeTrajectoryEvaluator", + "model": "gpt-4.1-2025-04-14", + "prompt": "Evaluate the agent's execution trajectory for proper suspend/resume pattern implementation.\n\nExpected Agent Behavior: {{ExpectedAgentBehavior}}\nAgent Run History: {{AgentRunHistory}}\n\nAssess whether the agent:\n1. Called interrupt() to suspend execution at the appropriate point\n2. Created a proper InvokeProcess request with correct structure\n3. Included all necessary parameters (process name, query, etc.)\n4. Followed the expected execution flow for suspend/resume pattern\n\nProvide a score from 0-100 based on how well the agent followed the expected trajectory and properly implemented the suspend/resume pattern.", + "temperature": 0.0, + "defaultEvaluationCriteria": { + "expectedAgentBehavior": "The agent should call interrupt() to suspend execution while waiting for an RPA process to complete, with proper InvokeProcess details in the trace." + } + } +} diff --git a/samples/tool-calling-suspend-resume/graph.py b/samples/tool-calling-suspend-resume/graph.py new file mode 100644 index 00000000..e691c22d --- /dev/null +++ b/samples/tool-calling-suspend-resume/graph.py @@ -0,0 +1,94 @@ +"""Tool-calling agent demonstrating suspend/resume with RPA process invocation.""" + +import logging + +from langgraph.constants import END, START +from langgraph.graph import StateGraph +from langgraph.types import interrupt +from pydantic import BaseModel +from uipath.platform.common import InvokeProcess + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +logger = logging.getLogger(__name__) + + +class Input(BaseModel): + """Input for the test agent.""" + + query: str + + +class Output(BaseModel): + """Output from the test agent.""" + + result: str + + +class State(BaseModel): + """Internal state for the agent.""" + + query: str + result: str = "" + + +async def invoke_process_node(state: State) -> State: + """Node that invokes an RPA process and suspends execution.""" + logger.info("=" * 80) + logger.info("AGENT NODE: Starting invoke_process_node") + logger.info(f"AGENT NODE: Received query: {state.query}") + + # Create an InvokeProcess request + invoke_request = InvokeProcess( + name="TestProcess", + input_arguments={"query": state.query, "data": "test_data"}, + process_folder_path="Shared", + ) + + logger.info( + f"AGENT NODE: Created InvokeProcess request: {invoke_request.model_dump()}" + ) + logger.info("🔴 AGENT NODE: About to call interrupt() - SUSPENDING EXECUTION") + logger.info("=" * 80) + + # Interrupt execution and capture the process output + # The runtime will detect this and return SUSPENDED status + # When resumed, the return value will contain the process output + process_output = interrupt(invoke_request) + + # This code won't execute until the process completes and execution resumes + logger.info("=" * 80) + logger.info("🟢 AGENT NODE: Execution RESUMED after interrupt()") + logger.info("AGENT NODE: RPA process has completed") + logger.info(f"AGENT NODE: Process output: {process_output}") + logger.info(f"AGENT NODE: Returning result for query: {state.query}") + logger.info("=" * 80) + + # Extract result from process output + result = ( + process_output.get("result", "Process completed") + if isinstance(process_output, dict) + else str(process_output) + ) + return State(query=state.query, result=result) + + +# Build the graph +builder = StateGraph(state_schema=State) + +# Add single node that invokes the process +builder.add_node("invoke_process", invoke_process_node) + +# Connect: START -> invoke_process -> END +builder.add_edge(START, "invoke_process") +builder.add_edge("invoke_process", END) + +# Compile with checkpointer (required for interrupts to work) +from langgraph.checkpoint.memory import MemorySaver + +checkpointer = MemorySaver() +graph = builder.compile(checkpointer=checkpointer) diff --git a/samples/tool-calling-suspend-resume/langgraph.json b/samples/tool-calling-suspend-resume/langgraph.json new file mode 100644 index 00000000..96fd332e --- /dev/null +++ b/samples/tool-calling-suspend-resume/langgraph.json @@ -0,0 +1,6 @@ +{ + "dependencies": ["."], + "graphs": { + "graph": "./graph.py:graph" + } +} diff --git a/samples/tool-calling-suspend-resume/pyproject.toml b/samples/tool-calling-suspend-resume/pyproject.toml new file mode 100644 index 00000000..fdbee7b5 --- /dev/null +++ b/samples/tool-calling-suspend-resume/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "tool-calling-suspend-resume" +version = "0.1.0" +description = "Tool-calling agent demonstrating suspend/resume with RPA process invocation" +requires-python = ">=3.11" +dependencies = [] diff --git a/samples/tool-calling-suspend-resume/uipath.json b/samples/tool-calling-suspend-resume/uipath.json new file mode 100644 index 00000000..30c00e94 --- /dev/null +++ b/samples/tool-calling-suspend-resume/uipath.json @@ -0,0 +1,5 @@ +{ + "runtimeOptions": { + "isConversational": false + } +} diff --git a/uv.lock b/uv.lock index eda76354..02683b19 100644 --- a/uv.lock +++ b/uv.lock @@ -3347,7 +3347,7 @@ requires-dist = [ { name = "openinference-instrumentation-langchain", specifier = ">=0.1.56" }, { name = "pydantic-settings", specifier = ">=2.6.0" }, { name = "python-dotenv", specifier = ">=1.0.1" }, - { name = "uipath", specifier = ">=2.4.0,<2.5.0" }, + { name = "uipath" }, { name = "uipath-runtime", specifier = ">=0.4.0,<0.5.0" }, ] provides-extras = ["vertex", "bedrock"]