YiemAgent/learning/04-TYPES_MESSAGES.md

AgentCore.jl - Types and Messages Deep Dive

Core Type Hierarchy

┌─────────────────────────────────────────────────────────────────────────────┐
│                          Type Hierarchy                                     │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  ThinkingLevel (Enum)                                                      │
│  - THINKING_OFF                                                            │
│  - THINKING_MINIMAL                                                        │
│  - THINKING_LOW                                                            │
│  - THINKING_MEDIUM                                                         │
│  - THINKING_HIGH                                                           │
│  - THINKING_XHIGH                                                          │
│  - THINKING_MAX                                                            │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  ToolExecutionMode (Enum)                                                  │
│  - EXECUTION_SEQUENTIAL  (Tools run one at a time)                        │
│  - EXECUTION_PARALLEL    (Tools run concurrently)                          │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  QueueMode (Enum)                                                          │
│  - QUEUE_ALL             (Drain all messages at once)                     │
│  - QUEUE_ONE_AT_A_TIME   (Process one message at a time)                  │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  MessageContent (Abstract Type)                                            │
│  ├── TextContent (String)                                                  │
│  └── ImageContent (data::String, mime_type::String)                       │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  Message (Abstract Type)                                                   │
│  ├── UserMessage                                                           │
│  │   └─ role: "user", content: Message[], timestamp: Int64                │
│  ├── AssistantMessage                                                      │
│  │   └─ role: "assistant", content: Message[], api, provider, model,      │
│  │        usage: Usage, stop_reason, error_message, timestamp             │
│  └── ToolResultMessage                                                     │
│      └─ role: "toolResult", tool_call_id, tool_name, content, details,    │
│           usage, added_tool_names, is_error, timestamp                    │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  AgentMessage (Abstract Type)                                              │
│  └─ Union of all message types above + custom types                       │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  AgentTool                                                                 │
│  - name: String                                                            │
│  - label: String                                                           │
│  - description: String                                                     │
│  - parameters: Any                                                         │
│  - execute: Function                                                       │
│  - prepare_arguments: Union{Function, Nothing}                            │
│  - execution_mode: Union{ToolExecutionMode, Nothing}                     │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  AgentContext                                                              │
│  - system_prompt: String                                                   │
│  - messages: Vector{AgentMessage}                                          │
│  - tools: Union{Vector{AgentTool}, Nothing}                                │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  AgentEvent (Abstract Type)                                                │
│  ├── AgentStartEvent / AgentEndEvent                                       │
│  ├── TurnStartEvent / TurnEndEvent                                         │
│  ├── MessageStartEvent / MessageEndEvent                                   │
│  ├── MessageUpdateEvent                                                    │
│  ├── ToolExecutionStartEvent / ToolExecutionEndEvent                      │
│  └── ToolExecutionUpdateEvent                                             │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  Usage & ModelCost                                                         │
│  Usage: input, output, cache_read, cache_write, total_tokens, cost        │
│  ModelCost: input, output, cache_read, cache_write (all Float64)          │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│  Model                                                                     │
│  - id, name, api, provider, base_url, reasoning: Bool                     │
│  - input: Vector{String}                                                   │
│  - cost: ModelCost                                                         │
│  - context_window, max_tokens: Int64                                       │
└─────────────────────────────────────────────────────────────────────────────┘

Message Types

UserMessage

struct UserMessage <: Message
    role::String              # "user"
    content::Vector{MessageContent}
    timestamp::Timestamp      # Int64 (Unix timestamp)
end

Usage:

# Simple text message
UserMessage(
    "user",
    [TextContent("Hello, how are you?")],
    Int64(Dates.now(Dates.UTC).datetime)
)

# With multiple content types
UserMessage(
    "user",
    [
        TextContent("Analyze this image"),
        ImageContent(data_base64, "image/png")
    ],
    timestamp
)

AssistantMessage

struct AssistantMessage <: Message
    role::String              # "assistant"
    content::Vector{MessageContent}
    api::String               # API identifier
    provider::String          # Provider name
    model::String             # Model ID
    usage::Usage
    stop_reason::String       # "done", "error", "aborted", "length", etc.
    error_message::Union{String, Nothing}
    timestamp::Timestamp
end

Content can include:

• TextContent
• ToolCall

AssistantMessage(
    "assistant",
    [
        TextContent("I'll check the directory for you."),
        ToolCall(
            "tool",
            "tc_123",
            "bash",
            Dict("command" => "ls -la"),
            nothing
        ),
        ToolCall(
            "tool",
            "tc_456",
            "read",
            Dict("path" => "README.md"),
            nothing
        )
    ],
    "openai",
    "openai",
    "gpt-4",
    Usage(100, 50, 0, 0, 150, UsageCost(0.001, 0.002, 0.0, 0.0, 0.003)),
    "done",
    nothing,
    timestamp
)

ToolResultMessage

struct ToolResultMessage <: Message
    role::String              # "toolResult"
    tool_call_id::String      # Reference to original ToolCall
    tool_name::String         # Name of tool that executed
    content::Vector{MessageContent}
    details::Any              # Additional tool-specific details
    usage::Union{Usage, Nothing}
    added_tool_names::Union{Vector{String}, Nothing}
    is_error::Bool            # True if tool execution failed
    timestamp::Timestamp
end

Usage:

ToolResultMessage(
    "toolResult",
    "tc_123",
    "bash",
    [TextContent("file1.md\nfile2.md\n")],
    BashToolDetails(...),
    nothing,
    nothing,
    false,
    timestamp
)

AgentTool Structure

struct AgentTool{TParameters, TDetails}
    name::String
    label::String
    description::String
    parameters::TParameters
    execute::Function
    prepare_arguments::Union{Function, Nothing}
    execution_mode::Union{ToolExecutionMode, Nothing}
end

Parameters:

• name: Unique identifier for the tool
• label: Display name
• description: What the tool does
• parameters: JSON schema for tool arguments
• execute: Main execution function
• prepare_arguments: Optional preprocessing
• execution_mode: Sequential or parallel

Tool Execution Function Signature

execute::Function(
    tool_call_id::String,
    params::Dict{String, Any},
    signal::Union{Any, Nothing},  # Abort signal
    on_update::Function,          # Callback for streaming updates
    context::Any,                 # Tool context
)::AgentToolResult

Returns:

AgentToolResult(
    content::Vector{MessageContent},  # Result content
    details::T,                       # Tool-specific details
    usage::Union{Usage, Nothing},     # Usage statistics
    added_tool_names::Union{Vector{String}, Nothing},
    terminate::Union{Bool, Nothing},  # If true, stop agent after this
)

AgentContext

struct AgentContext
    system_prompt::String
    messages::Vector{AgentMessage}
    tools::Union{Vector{AgentTool}, Nothing}
end

Purpose: Read-only snapshot of agent state for LLM calls

Usage in AgentLoop:

function streamAssistantResponse(
    context::AgentContext,  # Contains messages, tools, system prompt
    config::AgentLoopConfig,
    ...
)::AssistantMessage
    # Convert to LLM format
    llm_messages = config.convert_to_llm(context.messages)
    
    # Create context for API
    llm_context = Context(
        context.system_prompt,
        llm_messages,
        context.tools,
    )
    
    # Call LLM
    return stream_function(context.model, llm_context, config)
end

Event Types

Agent Lifecycle Events

struct AgentStartEvent <: AgentEvent end
struct AgentEndEvent <: AgentEvent
    messages::Vector{AgentMessage}
end

Turn Events

struct TurnStartEvent <: AgentEvent end
struct TurnEndEvent <: AgentEvent
    message::AgentMessage
    tool_results::Vector{ToolResultMessage}
end

Message Events

struct MessageStartEvent <: AgentEvent
    message::AgentMessage
end
struct MessageUpdateEvent <: AgentEvent
    message::AgentMessage
    assistant_message_event::Any  # Partial message event
end
struct MessageEndEvent <: AgentEvent
    message::AgentMessage
end

Tool Execution Events

struct ToolExecutionStartEvent <: AgentEvent
    tool_call_id::String
    tool_name::String
    args::Any
end
struct ToolExecutionUpdateEvent <: AgentEvent
    tool_call_id::String
    tool_name::String
    args::Any
    partial_result::Any
end
struct ToolExecutionEndEvent <: AgentEvent
    tool_call_id::String
    tool_name::String
    result::Any
    is_error::Bool
end

Usage Statistics

struct Usage
    input::Int64           # Input tokens
    output::Int64          # Output tokens
    cache_read::Int64      # Cache read tokens
    cache_write::Int64     # Cache write tokens
    total_tokens::Int64    # Total tokens
    cost::UsageCost
end

struct UsageCost
    input::Float64
    output::Float64
    cache_read::Float64
    cache_write::Float64
    total::Float64
end

Example:

Usage(
    1000,      # input tokens
    200,       # output tokens
    500,       # cache read tokens
    0,         # cache write tokens
    1700,      # total tokens
    UsageCost(
        0.0005,  # input cost ($0.50 per 1M tokens)
        0.0015,  # output cost ($1.50 per 1M tokens)
        0.00025, # cache read cost
        0.0,     # cache write cost
        0.0035   # total cost
    )
)

Model Type

struct Model{Api}
    id::String                    # Model identifier (e.g., "gpt-4")
    name::String                  # Model name (e.g., "GPT-4")
    api::Api                      # API type (String, Symbol, or custom type)
    provider::String              # Provider name (e.g., "openai")
    base_url::String              # API base URL
    reasoning::Bool               # Whether model supports reasoning
    input::Vector{String}         # Input modes (e.g., ["text", "image"])
    cost::ModelCost
    context_window::Int64         # Max context window (e.g., 128000)
    max_tokens::Int64             # Max output tokens
end

struct ModelCost
    input::Float64
    output::Float64
    cache_read::Float64
    cache_write::Float64
end

ToolCall Type

struct ToolCall
    type::String           # "tool"
    id::String             # Unique ID for this tool call
    name::String           # Tool name to call
    arguments::Dict{String, Any}  # Tool arguments as JSON-like Dict
    partial_json::Union{String, Nothing}  # Partial JSON string
end

Example:

ToolCall(
    "tool",
    "call_abc123",
    "bash",
    Dict(
        "command" => "ls -la",
        "timeout" => 30
    ),
    nothing
)

Custom Message Types

BashExecutionMessage

mutable struct BashExecutionMessage
    role::String              # "custom"
    command::String
    output::String
    exit_code::Union{Int64, Nothing}
    cancelled::Bool
    truncated::Bool
    full_output_path::Union{String, Nothing}
    timestamp::Timestamp
    exclude_from_context::Bool
end

CompactionSummaryMessage

mutable struct CompactionSummaryMessage
    role::String              # "compactionSummary"
    summary::String           # Summary of compacted history
    tokens_before::Int64      # Context size before compaction
    timestamp::Timestamp
end

BranchSummaryMessage

mutable struct BranchSummaryMessage
    role::String              # "branchSummary"
    summary::String           # Summary of branch history
    from_id::String           # Branch point ID
    timestamp::Timestamp
end

AgentState

mutable struct AgentState
    system_prompt::String
    model::Model
    thinking_level::ThinkingLevel
    tools::Vector{AgentTool}
    messages::Vector{AgentMessage}
    is_streaming::Bool
    streaming_message::Union{AgentMessage, Nothing}
    pending_tool_calls::Set{String}
    error_message::Union{String, Nothing}
end

Purpose: Runtime state of the Agent

Note: AgentState is mutable and used internally by Agent

Key Conversion Functions

convertToLlm()

function convertToLlm(messages::Vector{AgentMessage})::Vector{Message}
    result::Vector{Message} = Message[]
    
    for m in messages
        converted = convertToLlmMessage(m)
        if !isnothing(converted)
            push!(result, converted)
        end
    end
    
    return result
end

Purpose: Transform AgentMessage[] to Message[] for LLM API

Example:

# Input: AgentMessage[]
[
    UserMessage(...),
    AssistantMessage(...),
    ToolResultMessage(...),
    BashExecutionMessage(...),  # Will be converted to UserMessage
    CompactionSummaryMessage(...),  # Will be converted to UserMessage
]

# Output: Message[]
[
    UserMessage(...),
    AssistantMessage(...),
    ToolResultMessage(...),
    UserMessage(...),  # Converted from BashExecutionMessage
    UserMessage(...),  # Converted from CompactionSummaryMessage
]

Default convertToLlmMessage Implementations

function convertToLlmMessage(m::BashExecutionMessage)
    if m.exclude_from_context
        return nothing
    end
    return UserMessage("user", [TextContent(bashExecutionToText(m))], m.timestamp)
end

function convertToLlmMessage(m::CompactionSummaryMessage)
    text = COMPACTION_SUMMARY_PREFIX * m.summary * COMPACTION_SUMMARY_SUFFIX
    return UserMessage("user", [TextContent(text)], m.timestamp)
end

function convertToLlmMessage(m::BranchSummaryMessage)
    text = BRANCH_SUMMARY_PREFIX * m.summary * BRANCH_SUMMARY_SUFFIX
    return UserMessage("user", [TextContent(text)], m.timestamp)
end

function convertToLlmMessage(m::UserMessage)
    return m  # Pass through
end

function convertToLlmMessage(m::AssistantMessage)
    return m  # Pass through
end

function convertToLlmMessage(m::ToolResultMessage)
    return m  # Pass through
end

Summary

The type system in AgentCore.jl provides:

1. Strong typing for different message types
2. Extensibility through abstract types and multiple dispatch
3. Clear separation between internal (AgentMessage) and external (Message) formats
4. Rich metadata in Usage and Model types for cost tracking
5. Event-driven architecture through Event types
6. Tool execution flexibility through Tool types with hooks

All types are designed for:

• Interoperability with LLM APIs
• Extensibility for custom message types
• Performance with immutable structs where possible
• Debuggability through rich event system