update

README.md

@@ -1,12 +1,16 @@
 # YiemAgent
 
-Julia framework for building agents with tool use.
+Julia framework for building agents with tool use and MCP (Model Context Protocol) support.
 
 ## Getting Started
 
-1. Install dependencies: `]add JSON, DataStructures, UUIDs, Dates, ...`
-2. Create a `yiemAgent` with `loadTools("src/tools")`
-3. Call `runAgent(agent, "message")` then `takeResponse(agent)`
+1. Install dependencies: `]add JSON, DataStructures, UUIDs, Dates, NATS, DataFrames`
+2. Create a callable `mcpServer` struct that communicates with an MCP server via NATS
+3. Create a `yiemAgent` with an LLM callable and MCP server:
+   ```julia
+   agent = YiemAgent.yiemAgent(llmCall; mcpServer=mcpServer, agentEventSink=yourSink)
+   ```
+4. Call `runAgent(agent, message)` then `takeResponse(agent)`
 
 ## Architecture
 
@@ -14,21 +18,122 @@ Julia framework for building agents with tool use.
 src/
 ├── YiemAgent.jl          # Module entry point
 ├── type.jl               # Core types (messages, tools, agent state)
-├── utils.jl              # Message formatting, validation
+├── utils.jl              # Message formatting, context preparation, validation
 ├── agentCore.jl          # Agent loop, tool execution pipeline
-├── api.jl                # Public API (runAgent, takeResponse, etc.)
-└── tools/
-    ├── registry.jl       # Tool registry (loadTools, registerTool, listTools)
-    ├── getWeather.jl     # Weather lookup tool
-    ├── getTime.jl        # Time lookup tool
-    ├── writeTool.jl      # Create new tool files (self-modifying)
-    └── README.md         # Tool development guide
+├── api.jl                # Public API (runAgent, takeResponse, followUp, stopAgent)
+└── toolRegistry.jl       # Tool store, MCP discovery, listTools registration
 ```
 
-## Tool Development
+## MCP Protocol
 
-See `src/tools/README.md` for:
-- Tool anatomy (schema, execute, getTool)
-- Validation hooks
-- Agent loop lifecycle
-- Self-modifying tools (`writeTool`)
+YiemAgent uses JSON-RPC 2.0 for MCP communication. The `mcpServer` callable struct must implement:
+
+```julia
+# tools/list — returns tool definitions
+mcpServer("tools/list")  # → Dict("jsonrpc"=>"2.0", "id"=>1, "result"=>Dict("tools"=>[...], "nextCursor"=>...))
+
+# tools/call — executes a tool
+mcpServer("tools/call", toolName, arguments)  # → Dict("jsonrpc"=>"2.0", "id"=>2, "result"=>Dict("content"=>[...], "isError"=>...))
+```
+
+Protocol error responses include `"error"` instead of `"result"`:
+```julia
+Dict("jsonrpc"=>"2.0", "id"=>2, "error"=>Dict("code"=>-32602, "message"=>"..."))
+```
+
+### Implementing the MCP Server Client
+
+You must provide a callable struct that communicates with your MCP server. Example using NATS:
+
+```julia
+struct mcpServer
+  natsConn::NATS.Connection
+  topic::String
+  senderID::String
+  fileserver_url::String
+end
+
+# tools/list implementation
+function (c::mcpServer)(method::String)
+  if method != "tools/list"
+    error("mcpServer: unexpected method '$method' (expected 'tools/list')")
+  end
+
+  payload = Dict("jsonrpc" => "2.0", "id" => 1, "method" => method, "params" => Dict{String, Any}())
+  payloads = [("payload", payload, "dictionary")]
+  _, msg_envelope_json_str = msghandler.smartpack(
+        c.topic, payloads;
+        sender_id=c.senderID,
+        msg_purpose="mcp_tools_list",
+        fileserver_url=c.fileserver_url)
+
+  reply = NATS.request(c.natsConn, c.topic, msg_envelope_json_str, timeout=180)
+  incoming_env = msghandler.smartunpack(String(reply.payload))
+  return incoming_env["payloads"][1][2]
+end
+
+# tools/call implementation
+function (c::mcpServer)(method::String, toolName::String, arguments::Dict{String, Any})
+  if method != "tools/call"
+    error("mcpServer: unexpected method '$method' (expected 'tools/call')")
+  end
+
+  payload = Dict(
+    "jsonrpc" => "2.0",
+    "id" => 2,
+    "method" => method,
+    "params" => Dict(
+      "name" => toolName,
+      "arguments" => arguments
+    )
+  )
+  payloads = [("payload", payload, "dictionary")]
+  _, msg_envelope_json_str = msghandler.smartpack(
+        c.topic, payloads;
+        sender_id=c.senderID,
+        msg_purpose="mcp_tool_call",
+        fileserver_url=c.fileserver_url)
+
+  reply = NATS.request(c.natsConn, c.topic, msg_envelope_json_str, timeout=180)
+  incoming_env = msghandler.smartunpack(String(reply.payload))
+  return incoming_env["payloads"][1][2]
+end
+```
+
+See `test/runtest.jl` for the complete working example.
+
+## Tool Discovery
+
+Tools are discovered dynamically via MCP server:
+
+1. `listTool` is the only pre-registered tool
+2. When the LLM calls `listTools()`, tools are discovered from the MCP server and registered at runtime
+3. Supports pagination via `nextCursor` for large tool sets
+
+## Agent API
+
+| Function | Description |
+|----------|-------------|
+| `runAgent(agent, msg)` | Send a message to the agent's input channel |
+| `takeResponse(agent)` | Block and take the agent's response from output channel |
+| `followUp(agent, msg)` | Send a follow-up message while agent is still processing |
+| `stopAgent(agent)` | Gracefully stop the agent and close channels |
+
+## Agent Lifecycle
+
+1. `yiemAgent()` spawns a background task (`_agentLoop`) listening on `inputChannel` and `followUpChannel`
+2. User messages enter via `runAgent()` or `followUp()`
+3. `_processMessage()` handles LLM calls, tool execution, and conversation history
+4. Tool execution follows three phases: `prepareToolCall` → `executePreparedToolCall` → `finalizeExecutedToolCall`
+5. `beforeToolCall`/`afterToolCall` hooks allow pre/post-processing of tool calls
+6. Tool `execute` functions can set `terminate=true` to stop the agent loop
+
+## Hooks
+
+| Hook | Signature | Purpose |
+|------|-----------|---------|
+| `prepareContext` | `(state, sink, llmCall) -> ctx` | Transform messages/context before LLM call |
+| `formatMsgForLLM` | `(ctx, sink) -> dict` | Convert agent context to LLM API format |
+| `beforeToolCall` | `(context, signal) -> result` | Block/allow tool execution |
+| `afterToolCall` | `(context, signal) -> result` | Post-process tool results |
+| `agentEventSink` | `(msg) -> nothing` | Callback for agent events/debug messages |

etc.jl

@@ -1,41 +0,0 @@
-# "tools/list" input:
-mcpServer("tools/list")
-# sending out payload before smart packed by msghandler
-Dict(
-  "method"=> "tools/list"
-)
-# expected return after smart unpacked by msghandler
-Dict("tools" => [
-  Dict("toolName" => "getWeather",
-    "title" => "Weather Lookup",
-    "description" => "Fetch current weather for a city.",
-    "inputSchema" => Dict("type"=>"object",
-                          "properties" => Dict("city"=>Dict("type"=>"string", "description"=>"City name"),
-                                              "units"=>Dict("type"=>"string", "enum"=>["celsius","fahrenheit"], "default"=>"celsius")),
-                          "required" => ["city"])
-  )
-])
-
-# "tools/call" input:
-mcpServer("tools/call", "getWeather", Dict("city"=>"Tokyo", "units"=>"celsius"))
-# sending out payload before smart packed by msghandler
-Dict(
-  "method"=> "tools/call",
-  "tools"=> Dict("toolName"=>"getWeather", "arguments"=>Dict("city"=>"Tokyo", "units"=>"celsius"))
-)
-# expected return after smart unpacked by msghandler
-Dict(
-  "toolName"=>"getWeather",
-  "content": [{"type": "text", "text": "Weather in Tokyo: Sunny, 22°C"}],
-  "isError": false
-)
-
-# If an error occurs, mcpServer returns after smart unpacked by msghandler:
-Dict(
-  "toolName"=>"getWeather",
-  "content": [],
-  "error": Dict("code"=>1, "message"=>"City not found"),
-  "isError": true
-)
-
-

src/agentCore.jl

@@ -51,8 +51,8 @@ mutable struct yiemAgent <: agent             # High-level agent wrapper
   llmCall                         
                                                
   # Callable struct for MCP server communication.
-  # Called as: mcpServer("tools/list") → returns parsed JSON dict of available tools
-  #            mcpServer("tools/call", toolName, arguments) → returns tool result as parsed JSON dict
+  # Called as: mcpServer("tools/list") → returns JSON-RPC 2.0 parsed response
+  #            mcpServer("tools/call", toolName, arguments) → returns JSON-RPC 2.0 parsed response
   #
   # # Example (weather tool)
   #   # User provides a callable struct
@@ -64,8 +64,8 @@ mutable struct yiemAgent <: agent             # High-level agent wrapper
   #   end
   #
   #   function (c::MyMCPClient)(method::String)
-  #       payload = Dict("method"=> method)
-  #       payloads = [("method", payload, "dictionary")]
+  #       payload = Dict("jsonrpc" => "2.0", "id" => 1, "method" => method, "params" => Dict{String, Any}())
+  #       payloads = [("payload", payload, "dictionary")]
   #       _, msg_envelope_json_str = msghandler.smartpack(
   #             c.topic, payloads; sender_id=c.senderID,
   #             msg_purpose="mcp_tools_list", fileserver_url=c.fileserver_url)
@@ -75,8 +75,9 @@ mutable struct yiemAgent <: agent             # High-level agent wrapper
   #   end
   #
   #   function (c::MyMCPClient)(method::String, toolName::String, arguments::Dict{String,Any})
-  #       payload = Dict("method"=> method, "toolName"=> toolName, "arguments"=>arguments)
-  #       payloads = [("method", payload, "dictionary"),]
+  #       payload = Dict("jsonrpc" => "2.0", "id" => 2, "method" => method,
+  #                      "params" => Dict("name" => toolName, "arguments" => arguments))
+  #       payloads = [("payload", payload, "dictionary"),]
   #       _, msg_envelope_json_str = msghandler.smartpack(
   #             c.topic, payloads; sender_id=c.senderID,
   #             msg_purpose="mcp_tool_call", fileserver_url=c.fileserver_url)
@@ -88,42 +89,41 @@ mutable struct yiemAgent <: agent             # High-level agent wrapper
   #   # "tools/list" input:
   #     mcpServer("tools/list")
   #     # sending out payload before smart packed by msghandler:
-  #     Dict("method"=> "tools/list")
+  #     Dict("jsonrpc" => "2.0", "id" => 1, "method" => "tools/list", "params" => Dict{String, Any}())
   #     # expected return after smart unpacked by msghandler:
-  #     Dict("tools" => [
-  #       Dict(
-  #         "toolName" => "getWeather",
-  #         "title" => "Weather Lookup",
-  #         "description" => "Fetch current weather for a city.",
-  #         "inputSchema" => Dict("type"=>"object",
-  #                               "properties" => Dict("city"=>Dict("type"=>"string", "description"=>"City name"),
-  #                                                   "units"=>Dict("type"=>"string", "enum"=>["celsius","fahrenheit"], "default"=>"celsius")),
-  #                               "required" => ["city"])
+  #     Dict(
+  #       "jsonrpc" => "2.0", "id" => 1,
+  #       "result" => Dict(
+  #         "tools" => [
+  #           Dict(
+  #             "name" => "getWeather",
+  #             "description" => "Fetch current weather for a city.",
+  #             "inputSchema" => Dict("type"=>"object",
+  #                                   "properties" => Dict("city"=>Dict("type"=>"string", "description"=>"City name"),
+  #                                                       "units"=>Dict("type"=>"string", "enum"=>["celsius","fahrenheit"], "default"=>"celsius")),
+  #                                   "required" => ["city"])
+  #           )
+  #         ],
+  #         "nextCursor" => nothing
   #       )
-  #     ])
+  #     )
   #
   #   # "tools/call" input:
   #     mcpServer("tools/call", "getWeather", Dict("city"=>"Tokyo", "units"=>"celsius"))
   #     # sending out payload before smart packed by msghandler:
-  #     Dict(
-  #       "method"=> "tools/call",
-  #       "toolName"=>"getWeather",
-  #       "arguments"=>Dict("city"=>"Tokyo", "units"=>"celsius")
-  #     )
+  #     Dict("jsonrpc" => "2.0", "id" => 2, "method" => "tools/call",
+  #          "params" => Dict("name" => "getWeather", "arguments" => Dict("city"=>"Tokyo", "units"=>"celsius")))
   #     # expected return after smart unpacked by msghandler (success):
-  #     Dict(
-  #       "toolName"=> "getWeather",
-  #       "content" => [{"type" => "text", "text" => "Weather in Tokyo: Sunny, 22°C"}],
-  #       "error"=> "",
-  #       "isError" => false
-  #     )
-  #     # expected return after smart unpacked by msghandler (error):
-  #     Dict(
-  #       "toolName"=> "getWeather",
-  #       "content" => [],
-  #       "error" => Dict("code"=>1, "message"=>"City not found"),
-  #       "isError" => true
-  #     )
+  #     Dict("jsonrpc" => "2.0", "id" => 2,
+  #          "result" => Dict("content" => [{"type" => "text", "text" => "Weather in Tokyo: Sunny, 22°C"}],
+  #                           "isError" => false))
+  #     # expected return after smart unpacked by msghandler (tool error):
+  #     Dict("jsonrpc" => "2.0", "id" => 2,
+  #          "result" => Dict("content" => [{"type" => "text", "text" => "Error: City 'Atlantis' not found."}],
+  #                           "isError" => true))
+  #     # expected return after smart unpacked by msghandler (protocol error):
+  #     Dict("jsonrpc" => "2.0", "id" => 2,
+  #          "error" => Dict("code" => -32602, "message" => "Missing required argument 'city'"))
   mcpServer                        
 
   # Callback invoked before executing a tool call (ask for user permission/confirmation/abort, etc..)

src/toolRegistry.jl

@@ -44,7 +44,10 @@ end
 """
 Extract text from MCP tool result content array.
 
-Handles MCP's content format: [{"type":"text","text":"..."}]
+Handles JSON-RPC 2.0 result content format:
+    {"content": [{"type": "text", "text": "..."}], "isError": false}
+
+Returns the joined text content, or JSON-serialized fallback if no text blocks found.
 """
 function _extract_text_content(result::Dict)::String
     content = get(result, "content", Any[])
@@ -66,11 +69,12 @@ end
 Wrap an MCP tool definition as an `agentTool`.
 
 The returned tool's `execute` function calls the MCP server's "tools/call"
-method with the validated arguments.
+method with the validated arguments. Responses are in JSON-RPC 2.0 format
+with result/error envelopes handled by the execute function.
 """
 function _wrap_mcp_tool(mcpserver, tool_def::Dict{String,Any})::agentTool
-    name = tool_def["toolName"]
-    title = get(tool_def, "title", name)
+    name = tool_def["name"]
+    title = get(tool_def, "title", get(tool_def, "label", name))
     desc = get(tool_def, "description", "")
     input_schema = get(tool_def, "inputSchema", Dict{String,Any}())
 
@@ -101,6 +105,17 @@ function _wrap_mcp_tool(mcpserver, tool_def::Dict{String,Any})::agentTool
             try
                 response = mcpserver("tools/call", name, args)
 
+                # Parse JSON-RPC 2.0 response envelope
+                if haskey(response, "error")
+                    rpc_error = response["error"]
+                    err_msg = get(rpc_error, "message", "Unknown MCP error")
+                    return agentToolResult(
+                        [textContent("MCP error: $err_msg")],
+                        Dict{Any,Any}("error" => err_msg),
+                        nothing, false
+                    )
+                end
+
                 result_data = get(response, "result", response)
                 content_text = _extract_text_content(result_data)
                 is_error = get(result_data, "isError", false)
@@ -128,9 +143,10 @@ end
 """
 Discover and register MCP tools into `store.tools`.
 
-Queries the MCP server via `mcpserver("tools/list")`, parses the response,
-and registers each discovered tool. Skips tools already registered.
-Returns `(new_count, tool_list_text)`.
+Queries the MCP server via `mcpserver("tools/list")` (JSON-RPC 2.0 format),
+parses the response envelope, and registers each discovered tool.
+Handles pagination via `nextCursor` — keeps fetching until cursor is empty.
+Skips tools already registered. Returns `(new_count, tool_list_text)`.
 """
 function _register_mcp_tools(mcpserver, store::toolStore)::Tuple{Int, String}
     if mcpserver === nothing
@@ -139,11 +155,23 @@ function _register_mcp_tools(mcpserver, store::toolStore)::Tuple{Int, String}
 
     try
         response = mcpserver("tools/list")
+
+        # Parse JSON-RPC 2.0 response envelope
+        if haskey(response, "error")
+            rpc_error = response["error"]
+            err_msg = get(rpc_error, "message", "Unknown MCP error")
+            return (0, "MCP tools/list failed: $err_msg")
+        end
+        if haskey(response, "result")
+            response = response["result"]
+        end
+
         tools_array = response["tools"]
+        cursor = get(response, "nextCursor", nothing)
 
         new_count = 0
         for tool_def in tools_array
-            name = tool_def["toolName"]
+            name = tool_def["name"]
             if haskey(store.tools, name)
                 continue
             end
@@ -152,6 +180,25 @@ function _register_mcp_tools(mcpserver, store::toolStore)::Tuple{Int, String}
             new_count += 1
         end
 
+        # Paginate: fetch remaining tools if nextCursor is present
+        while cursor !== nothing && cursor !== ""
+            response = mcpserver("tools/list")
+            if haskey(response, "result")
+                response = response["result"]
+            end
+            tools_array = get(response, "tools", Any[])
+            cursor = get(response, "nextCursor", nothing)
+            for tool_def in tools_array
+                name = tool_def["name"]
+                if haskey(store.tools, name)
+                    continue
+                end
+                wrapped = _wrap_mcp_tool(mcpserver, tool_def)
+                store.tools[name] = wrapped
+                new_count += 1
+            end
+        end
+
         # Build readable tool list
         lines = String[
             "- $(t.name): $(t.label) — $(t.description)"
@@ -169,11 +216,11 @@ end
 """
     listTool(store::toolStore, mcpserver) -> agentTool
 
-MCP-aware listTools tool.
+MCP-aware listTools tool (JSON-RPC 2.0 protocol).
 
 First call: queries the MCP server via `mcpserver("tools/list")`, registers
-all discovered tools into the shared `store.tools` (in-place mutation),
-then returns the full tool list.
+all discovered tools into the shared `store.tools` (in-place mutation, with
+pagination via nextCursor), then returns the full tool list.
 
 Subsequent calls: returns the current list (tools remain registered).