YiemAgent/learning/07-AGENTHARNESS.md

AgentCore.jl - AgentHarness Deep Dive

AgentHarness Architecture

┌─────────────────────────────────────────────────────────────────────────────┐
│                        AgentHarness Layer                                   │
└─────────────────────────────────────────────────────────────────────────────┐

┌─────────────────────────────────────────────────────────────────────────────┐
│  AgentHarness = Agent + Session + Resources                                 │
│                                                                              │
│  ┌───────────────────────────────────────────────────────────────────────┐  │
│  │  AgentHarness                                                         │  │
│  │  - Manages Agent instances                                            │  │
│  │  - Provides Session persistence                                       │  │
│  │  - Manages resources (skills, prompt templates)                      │  │
│  │  - Handles extension hooks                                            │  │
│  │  - Coordinates tool execution with context                            │  │
│  └───────────────────────────────────────────────────────────────────────┘  │
│                              │                                                │
│        ┌─────────────────────┼─────────────────────┐                        │
│        ▼                     ▼                     ▼                        │
│   ┌──────────────┐    ┌──────────────┐    ┌──────────────┐                │
│   │   Agent      │    │  SessionRepo │    │   Resources  │                │
│   │  (state,     │    │  (create,    │    │  (skills,    │                │
│   │   events)    │    │   open,      │    │   templates) │                │
│   └──────────────┘    │   list)      │    └──────────────┘                │
│                       └──────────────┘                                     │
│                                     │                                        │
│                                     ▼                                        │
│                            ┌──────────────┐                                │
│                            │ Session      │                                │
│                            │ (history,    │                                │
│                            │  branching)  │                                │
│                            └──────────────┘                                │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  AgentHarnessEvent System                                                   │
└─────────────────────────────────────────────────────────────────────────────┘

AgentEvent (from Agent)
├─ AgentHarnessOwnEvent
│  ├─ BeforeAgentStartEvent
│  ├─ ContextEvent
│  ├─ BeforeProviderRequestEvent
│  ├─ BeforeProviderPayloadEvent
│  ├─ AfterProviderResponseEvent
│  ├─ ToolCallEvent
│  ├─ ToolResultEvent
│  ├─ SessionBeforeCompactEvent
│  ├─ SessionCompactEvent
│  ├─ SessionBeforeTreeEvent
│  ├─ SessionTreeEvent
│  ├─ ModelUpdateEvent
│  ├─ ThinkingLevelUpdateEvent
│  ├─ ToolsUpdateEvent
│  ├─ ResourcesUpdateEvent
│  └─ ... (other session events)

└─ AgentEvent (from AgentLoop)
   ├─ AgentStartEvent / AgentEndEvent
   ├─ TurnStartEvent / TurnEndEvent
   ├─ MessageStartEvent / MessageEndEvent
   └─ ToolExecutionStartEvent / ToolExecutionEndEvent

AgentHarness Components

1. AgentHarnessOptions

mutable struct AgentHarnessOptions{
    TC, TSkill<:Skill, TPromptTemplate<:PromptTemplate, TTool<:AgentHarnessTool
}
    session::Session
    models::Any
    tools::Union{Vector{TTool}, Nothing}
    resources::Union{AgentHarnessResources{TSkill, TPromptTemplate}, Nothing}
    system_prompt::Union{AgentHarnessSystemPrompt{TC, TSkill, TPromptTemplate, TTool}, Nothing}
    stream_options::Union{AgentHarnessStreamOptions, Nothing}
    retry::Union{Any, Nothing}
    model::Model
    thinking_level::Union{ThinkingLevel, Nothing}
    active_tool_names::Union{Vector{String}, Nothing}
    steering_mode::Union{QueueMode, Nothing}
    follow_up_mode::Union{QueueMode, Nothing}
    tool_context::Union{AgentHarnessToolContextSource{TC}, Nothing}
end

Purpose: Configure AgentHarness with all necessary options

Key fields:

• session: Session instance for persistence
• models: Available models
• tools: Agent tools
• resources: Skills and prompt templates
• system_prompt: System prompt (string or function)
• stream_options: LLM streaming options
• model: Default model
• thinking_level: Default thinking level
• active_tool_names: Active tools
• tool_context: Context source for tools

2. AgentHarnessResources

mutable struct AgentHarnessResources{TSkill<:Skill, TPromptTemplate<:PromptTemplate}
    promptTemplates::Union{Vector{TPromptTemplate}, Nothing}
    skills::Union{Vector{TSkill}, Nothing}
end

Purpose: Load and manage skills and prompt templates

3. Skill

mutable struct Skill
    name::String
    description::String
    content::String
    filePath::String
    disableModelInvocation::Bool
end

Purpose: Define specialized instructions for specific tasks

Format:

<!-- SKILL.md -->
{
  "name": "File Operations",
  "description": "Handle file system operations",
  "disable-model-invocation": false
}
---

# File Operations Skill

This skill provides instructions for working with files...

4. PromptTemplate

mutable struct PromptTemplate
    name::String
    description::Union{String, Nothing}
    content::String
end

Purpose: Reusable prompt snippets with arguments

Format:

<!-- template.md -->
{
  "description": "Generate commit message"
}
---

Generate a git commit message for:
$1
$ARGUMENTS

5. AgentHarnessStreamOptions

mutable struct AgentHarnessStreamOptions
    transport::Union{String, Nothing}
    timeout_ms::Union{Int64, Nothing}
    max_retries::Union{Int64, Nothing}
    max_retry_delay_ms::Union{Int64, Nothing}
    headers::Union{Dict{String, String}, Nothing}
    metadata::Union{Dict{String, Any}, Nothing}
    cache_retention::Union{String, Nothing}
end

Purpose: Configure LLM API call options

SessionRepo Interface

abstract type SessionRepo<
    TMetadata<:SessionMetadata,
    TCreateOptions,
    TListOptions
> end

Repo Methods

# Create new session
create(repo::SessionRepo, options::TCreateOptions)::Promise{Session}

# Open existing session
open(repo::SessionRepo, metadata::TMetadata)::Promise{Session}

# List sessions
list(repo::SessionRepo, options::TListOptions)::Promise{Vector{TMetadata}}

# Delete session
delete(repo::SessionRepo, metadata::TMetadata)::Promise{Nothing}

# Fork session (create branch)
fork(repo::SessionRepo, source::TMetadata, options::Dict{String, Any})::Promise{Session}

JsonlSessionRepo

# JSONL-based session repository
# - Sessions stored as JSONL files
# - Supports create, open, list, delete, fork
# - Branch navigation via session tree

Extension Hooks

Hook Types

# Before agent starts
BeforeAgentStartEvent
├─ prompt: String
├─ images: Union{Vector{ImageContent}, Nothing}
├─ system_prompt: String
└─ resources: AgentHarnessResources

BeforeAgentStartResult
├─ messages: Union{Vector{AgentMessage}, Nothing}
└─ system_prompt: Union{String, Nothing}

# Context event
ContextEvent
└─ messages: Vector{AgentMessage}

ContextResult
└─ messages: Vector{AgentMessage}

# Before LLM request
BeforeProviderRequestEvent
├─ model: Model
├─ session_id: String
└─ stream_options: AgentHarnessStreamOptions

BeforeProviderRequestResult
└─ stream_options: Union{AgentHarnessStreamOptionsPatch, Nothing}

# Before LLM payload
BeforeProviderPayloadEvent
├─ model: Model
└─ payload: Any

BeforeProviderPayloadResult
└─ payload: Any

# After LLM response
AfterProviderResponseEvent
├─ status: Int64
└─ headers: Dict{String, String}

# Tool call
ToolCallEvent
├─ tool_call_id: String
├─ tool_name: String
└─ input: Dict{String, Any}

ToolCallResult
├─ block: Union{Bool, Nothing}
└─ reason: Union{String, Nothing}

# Tool result
ToolResultEvent
├─ tool_call_id: String
├─ tool_name: String
├─ input: Dict{String, Any}
├─ content: Vector{MessageContent}
├─ details: Any
├─ is_error: Bool
└─ usage: Union{Usage, Nothing}

ToolResultPatch
├─ content: Union{Vector{MessageContent}, Nothing}
├─ details: Union{Any, Nothing}
├─ is_error: Union{Bool, Nothing}
├─ usage: Union{Usage, Nothing}
└─ terminate: Union{Bool, Nothing}

# Session compaction
SessionBeforeCompactEvent
├─ preparation: Any
├─ branch_entries: Vector{SessionTreeEntry}
├─ custom_instructions: Union{String, Nothing}
└─ signal: Any

SessionBeforeCompactResult
├─ cancel: Union{Bool, Nothing}
└─ compaction: Union{CompactResult, Nothing}

SessionCompactEvent
├─ compaction_entry: CompactionEntry
└─ from_hook: Bool

# Session tree (branching)
SessionBeforeTreeEvent
├─ preparation: Any
└─ signal: Any

SessionBeforeTreeResult
├─ cancel: Union{Bool, Nothing}
├─ summary: Union{Dict{String, Any}, Nothing}
├─ custom_instructions: Union{String, Nothing}
├─ replace_instructions: Union{Bool, Nothing}
└─ label: Union{String, Nothing}

SessionTreeEvent
├─ new_leaf_id: Union{String, Nothing}
├─ old_leaf_id: Union{String, Nothing}
├─ summary_entry: Union{BranchSummaryEntry, Nothing}
└─ from_hook: Union{Bool, Nothing}

Hook Usage Examples

BeforeAgentStartHook

function beforeAgentStart(event, signal)
    # Modify system prompt based on context
    new_system_prompt = "$(event.system_prompt)\n\nUser prefers concise responses."
    
    # Prepend initial messages
    initial_messages = [
        UserMessage("user", [TextContent("Context: $(event.prompt)")], timestamp),
    ]
    
    return BeforeAgentStartResult(
        initial_messages,
        new_system_prompt,
    )
end

# Configure harness
harness = AgentHarness(Dict(
    :beforeAgentStart => beforeAgentStart,
))

BeforeProviderPayloadHook

function beforeProviderPayload(event, signal)
    # Modify LLM payload before sending
    payload = event.payload
    
    # Add custom metadata
    payload.metadata = merge(payload.metadata, Dict(
        "session_id" => event.session_id,
        "timestamp" => Dates.now(),
    ))
    
    return BeforeProviderPayloadResult(payload)
end

ToolCallHook

function toolCall(event, signal)
    # Block dangerous tool calls
    if event.tool_name == "bash" && contains(event.input["command"], "rm -rf /")
        return ToolCallResult(true, "Blocking dangerous command")
    end
    
    # Log tool execution
    println("Tool call: $(event.tool_name)")
    
    return nothing  # Allow execution
end

BeforeCompactHook

function beforeCompact(event, signal)
    # Add custom instructions for compaction
    custom_instructions = """
    Focus on retaining user preferences and key decisions.
    Omit verbose tool outputs that don't add value.
    """
    
    return SessionBeforeCompactResult(
        false,  # Don't cancel
        Dict(
            "summary" => "Custom compaction with focus on user intent",
            "custom_instructions" => custom_instructions,
        ),
    )
end

Tool Context

AgentHarnessToolContextSource

mutable struct AgentHarnessToolContextSource{TContext}
    context::Union{TContext, Function}
end

Purpose: Provide context to tools during execution

Tool Execution Context

# Tools receive context from AgentHarness
tool.execute(
    tool_call_id,
    params,
    signal,
    on_update,
    context,  # From AgentHarnessToolContextSource
)

# Context can be:
# - Static value
# - Function that returns value

Complete Example

using AgentCore

# 1. Create skills
skills, skill_diagnostics = loadSkills(
    execution_env,
    "/path/to/skills",
)

# 2. Create prompt templates
templates, template_diagnostics = loadPromptTemplates(
    execution_env,
    "/path/to/templates",
)

# 3. Create resources
resources = AgentHarnessResources(
    templates,
    skills,
)

# 4. Create session repo
repo = JsonlSessionRepo(
    "/path/to/sessions",
)

# 5. Create session
session = create(repo, Dict(
    "cwd" => "/path/to/project",
    "metadata" => Dict("project" => "my-project"),
))

# 6. Configure tools
bash_tool = createBashTool()
read_tool = createReadTool()

tools = [bash_tool, read_tool]

# 7. Configure hooks
hooks = Dict(
    :beforeAgentStart => beforeAgentStartHook,
    :beforeProviderPayload => beforePayloadHook,
    :toolCall => toolCallHook,
)

# 8. Create harness
harness = AgentHarness(Dict(
    :session => session,
    :models => models,
    :tools => tools,
    :resources => resources,
    :system_prompt => "You are a helpful assistant.",
    :model => Model(...),
    :thinking_level => THINKING_MEDIUM,
    :active_tool_names => ["bash", "read"],
    :steering_mode => QUEUE_ONE_AT_A_TIME,
    :follow_up_mode => QUEUE_ONE_AT_A_TIME,
    :tool_context => AgentHarnessToolContextSource(context),
    :stream_options => AgentHarnessStreamOptions(
        transport = "auto",
        timeout_ms = 30000,
        max_retries = 3,
    ),
))

# 9. Subscribe to events
subscribe(harness) do event, signal
    if event isa BeforeAgentStartEvent
        println("Agent starting...")
    elseif event isa MessageEndEvent
        println("Message: $(event.message)")
    end
end

# 10. Run conversation
harness.prompt("What files are in the current directory?")

# 11. Wait for completion
wait_for_idle(harness)

# 12. Manage branches
session.moveTo(some_entry_id)  # Fork from entry

Hook Execution Flow

User Code
    │
    ├─► AgentHarness.prompt()
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  BeforeAgentStartEvent                                                 │
│  ├─ User prompt                                                        │
│  ├─ System prompt                                                      │
│  └─ Resources                                                          │
│        │                                                                │
│        └─► beforeAgentStart hook (optional)                            │
│            └─► BeforeAgentStartResult (optional modifications)         │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  Agent.createLoopConfig()                                              │
│  └─► Merge options with hooks                                          │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  Agent.prompt()                                                        │
│  └─► Start AgentLoop                                                   │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  AgentLoop.agentLoop()                                                 │
│        │                                                                │
│        ├─► transform_context hook (optional)                           │
│        └─► convert_to_llm()                                            │
│            └─► Message[] for LLM API                                   │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  BeforeProviderRequestEvent                                            │
│  ├─ Model                                                              │
│  ├─ Session ID                                                         │
│  └─ Stream Options                                                     │
│        │                                                                │
│        └─► beforeProviderRequest hook (optional)                       │
│            └─► BeforeProviderRequestResult (optional modifications)    │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  StreamFn (LLM API call)                                               │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  AfterProviderResponseEvent                                            │
│  ├─ Status code                                                        │
│  └─ Response headers                                                   │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  BeforeProviderPayloadEvent                                            │
│  ├─ Model                                                              │
│  └─ Payload (before sending)                                           │
│        │                                                                │
│        └─► beforeProviderPayload hook (optional)                       │
│            └─► BeforeProviderPayloadResult (optional modifications)    │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  LLM API Request                                                       │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  Assistant Message (streaming)                                         │
│        │                                                                │
│        ├─► Text deltas                                                 │
│        └─► Tool calls                                                  │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  Tool Execution (for each tool call)                                   │
│        │                                                                │
│        ├─► before_tool_call hook (Agent)                               │
│        ├─► toolCall hook (Harness - optional)                          │
│        │       └─► ToolCallResult (can block execution)               │
│        ├─► prepareToolCall()                                           │
│        ├─► execute()                                                   │
│        │       └─► Tool execution with context                        │
│        ├─► after_tool_call hook (Agent)                                │
│        └─► toolResult hook (Harness - optional)                        │
│                └─► ToolResultPatch (can modify result)                │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  AgentLoop continues with tool results                                 │
│        │                                                                │
│        ├─► Next LLM call with tool results                             │
│        └─► Or end of conversation                                      │
└────────────────────────────────────────────────────────────────────────┘
    │
    ▼
┌────────────────────────────────────────────────────────────────────────┐
│  AgentEndEvent                                                         │
│  └─► Final messages in session                                         │
└────────────────────────────────────────────────────────────────────────┘

Session Management with Harness

# Create harness with session repo
repo = JsonlSessionRepo("/path/to/sessions")

# Create session
session = create(repo, Dict(
    "cwd" => "/path/to/project",
    "metadata" => Dict("name" => "my-session"),
))

# Or open existing session
metadata = JsonlSessionMetadata(...)
session = open(repo, metadata)

# List sessions
sessions = list(repo, Dict())
for meta in sessions
    println("Session: $(meta.id)")
end

# Delete session
delete(repo, metadata)

# Fork session (branch)
forked_session = fork(repo, source_metadata, Dict(
    "summary" => "Branch for feature X",
))

Resources Management

# Load skills from directory
skills, diagnostics = loadSkills(
    execution_env,
    "/path/to/skills",
)

# Load prompt templates from directory
templates, diagnostics = loadPromptTemplates(
    execution_env,
    "/path/to/templates",
)

# Create resources
resources = AgentHarnessResources(
    templates,
    skills,
)

# Use in harness
harness = AgentHarness(Dict(
    :resources => resources,
))

Best Practices

1. Use hooks for logging and validation

   • beforeAgentStart for initialization
   • beforeProviderPayload for custom metadata
   • toolCall for blocking dangerous operations

2. Organize skills by domain

   • File operations
   • Database queries
   • HTTP requests
   • Git operations

3. Use templates for common patterns

   • Commit message generation
   • Code review instructions
   • Testing prompts

4. Manage sessions carefully

   • Compact periodically
   • Use branches for exploration
   • Clean up old sessions

5. Monitor resource usage

   • Track token counts
   • Watch API costs
   • Optimize tool execution

Troubleshooting

Hook not being called

# Check hook is registered
if isnothing(harness.beforeAgentStart)
    println("Hook not registered")
end

Session not persisting

# Check repo is configured
if isnothing(harness.repo)
    println("No repo configured")
end

Resources not loading

# Check diagnostics
for diag in skill_diagnostics
    println("Skill warning: $(diag.message)")
end