src/YiemAgent.jl

@@ -16,8 +16,8 @@ module YiemAgent
   include("llmfunction.jl")
   using .llmfunction
 
-  include("core.jl")
-  using .core
+  include("agentCore.jl")
+  using .agentCore
 
   include("api.jl")
   using .api

src/agentCore.jl

@@ -0,0 +1,203 @@
+module agentCore
+
+# export prompt
+
+using JSON, DataStructures, Dates, UUIDs, HTTP, Random, PrettyPrinting, Serialization,
+      DataFrames, Serde
+using GeneralUtils
+using ..type, ..util, ..llmfunction
+
+# ---------------------------------------------- 100 --------------------------------------------- #
+
+"""
+Private agent loop. Runs in a background `@spawn` task.
+
+Waits on `input_ch` and `followUpQueue`, processing whichever has a message first.
+On each iteration, dispatches the message through `_process_message` and sends the result
+to `output_ch`. Exits on `:shutdown` signal.
+
+# Arguments
+- `agent::yiemAgent`: The agent whose loop to run
+
+# Returns
+- `nothing` — the loop runs until `:shutdown` is received or an error occurs
+
+# Notes
+- This function is automatically spawned as a background task when a `yiemAgent` is created.
+- On any error, logs the error with `@error` and exits the loop.
+- Message priority: `input_ch` messages are checked before `followUpQueue` messages.
+
+# Examples
+```jldoctest
+julia> # Called automatically by yiemAgent constructor
+```
+"""
+function _agent_loop(agent::yiemAgent)  #WORKING
+  try
+    while true
+      # Wait on either channel — the one with a message fires first
+      # Wait on either channel — the one with a message is taken first
+      msg = nothing
+      while msg === nothing
+        if isready(agent.input_ch)
+          msg = take!(agent.input_ch)
+        else
+          yield()
+        end
+      end
+
+      # Check for shutdown signal
+      if msg === :shutdown
+        #TODO make sure every running tools ended properly
+        break
+      end
+
+      #TODO convert raw user msg to userMessage type
+
+      #TODO add userMessage to agent._state.messages
+
+
+      if isready(agent.followUpQueue)
+          msg = take!(agent.followUpQueue)
+      end
+
+      # @spawn. Dispatch message through the processing pipeline.
+      result = _process_message(agent, msg)
+
+      # Send response to user
+      put!(agent.output_ch, result)
+    end
+  catch e
+    # On any error, send error response and exit the loop
+    @error "Agent loop failed" error=e
+  end
+end
+
+"""
+Process a single message through the agent pipeline.
+
+This is the core processing function where LLM calls, tool execution, and response generation
+should be implemented. Currently a placeholder that echoes back the received message.
+
+# Arguments
+- `agent::yiemAgent`: The agent processing the message
+- `msg`: The message to process (from `input_ch` or `followUpQueue`)
+
+# Returns
+- An `assistantMessage` instance with the processed response
+
+# Notes
+- Implement the full processing pipeline:
+  1. Add `msg` to `agent._state.messages`
+  2. Call `agent.formatMsgForLLM(agent._state)` 1 to format for LLM
+  3. If `agent.preprocessMessages` is set, call it on the formatted messages
+  4. Call the LLM (blocking — the task waits here)
+  5. If agent has tools, handle tool calls in a loop
+  6. Build `assistantMessage` and return it
+
+# Examples
+```jldoctest
+julia> # Currently returns a placeholder echo response
+```
+"""
+function _process_message(agent::yiemAgent, msg)
+  # WORKING Replace with actual processing logic
+  # check steering message
+
+
+
+
+
+
+  # 1. call agent.
+  # 2. Call agent.formatMsgForLLM(agent._state) to format for LLM
+  # 3. If preprocessMessages is set, call agent.preprocessMessages(...)
+  # 4. Call the LLM (blocking — the task waits here)
+  # 5. If agent has tools, handle tool calls in a loop
+  # 6. Build assistantMessage and return it
+
+  # Placeholder: echo back the message as a simple response
+  @warn "TODO: implement _process_message"
+  return assistantMessage(
+    role="assistant",
+    content=[textContent("Received: $(msg)")],
+    api="", model="", usage=nothing,
+    stopReason="end_turn",
+    errorMessage=nothing,
+    timestamp=now(),
+  )
+end
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+end # end of module

src/api.jl

@@ -13,7 +13,26 @@ using ..type, ..util, ..llmfunction
 
 """
 Send a message to the agent's input channel.
+
 Blocks if the input channel buffer is full (capacity 16 by default).
+The agent processes messages from `input_ch` in the background task.
+
+# Arguments
+- `agent::yiemAgent`: The agent instance to send a message to
+- `msg`: The message to send (any type accepted by the agent's processing pipeline)
+
+# Returns
+- The same `agent` instance for chaining
+
+# Notes
+- Use `take_response(agent)` to receive the agent's response after sending a message.
+- Use `follow_up(agent, msg)` to send messages while the agent is still processing.
+
+# Examples
+```jldoctest
+julia> run_agent(agent, "Hello!")
+yiemAgent(...)
+```
 """
 function run_agent(agent::yiemAgent, msg)
   put!(agent.input_ch, msg)
@@ -22,7 +41,23 @@ end
 
 """
 Take a response from the agent's output channel.
+
 Blocks until the agent sends a response.
+
+# Arguments
+- `agent::yiemAgent`: The agent instance to receive a response from
+
+# Returns
+- An `assistantMessage` instance representing the agent's response
+
+# Notes
+- Use `run_agent(agent, msg)` to send a message before calling this function.
+
+# Examples
+```jldoctest
+julia> response = take_response(agent)
+assistantMessage(...)
+```
 """
 function take_response(agent::yiemAgent)
   return take!(agent.output_ch)
@@ -30,8 +65,27 @@ end
 
 """
 Send a follow-up message while the agent is still processing.
-Follow-up messages are processed after all input_ch messages
+
+Follow-up messages are queued and processed after all `input_ch` messages
 and before any tool call results are sent.
+
+# Arguments
+- `agent::yiemAgent`: The agent instance to send a follow-up message to
+- `msg`: The follow-up message to send
+
+# Returns
+- The same `agent` instance for chaining
+
+# Notes
+- Use `run_agent(agent, msg)` for the primary message and `follow_up(agent, msg)` for additional
+  messages while the agent is processing.
+- Follow-up messages are buffered in a separate channel (capacity 32 by default).
+
+# Examples
+```jldoctest
+julia> follow_up(agent, "Also consider red wines")
+yiemAgent(...)
+```
 """
 function follow_up(agent::yiemAgent, msg)
   put!(agent.followUpQueue, msg)
@@ -40,7 +94,25 @@ end
 
 """
 Gracefully stop the agent.
-Sends a :shutdown signal, waits for the task to finish, then closes all channels.
+
+Sends a `:shutdown` signal to the input channel, waits for the background task to finish,
+then closes all channels (`input_ch`, `output_ch`, `followUpQueue`).
+
+# Arguments
+- `agent::yiemAgent`: The agent instance to stop
+
+# Returns
+- `nothing`
+
+# Notes
+- After calling `stop_agent`, the agent is no longer usable. A new agent must be created
+  for further interaction.
+- If the background task throws a `TaskFailedException`, it is rethrown.
+
+# Examples
+```jldoctest
+julia> stop_agent(agent)
+```
 """
 function stop_agent(agent::yiemAgent)
   put!(agent.input_ch, :shutdown)

src/core.jl

@@ -1,147 +0,0 @@
-module core
-
-# export prompt
-
-using JSON, DataStructures, Dates, UUIDs, HTTP, Random, PrettyPrinting, Serialization,
-      DataFrames, Serde
-using GeneralUtils
-using ..type, ..util, ..llmfunction
-
-# ---------------------------------------------- 100 --------------------------------------------- #
-
-"""
-Private agent loop. Runs in a background @task.
-Waits on input_ch and followUpQueue, processing whichever has a message first.
-"""
-function _agent_loop(agent::yiemAgent)
-  try
-    while true
-      # Wait on either channel — the one with a message fires first
-      # Wait on either channel — the one with a message is taken first
-      msg = nothing
-      while msg === nothing
-        if isready(agent.input_ch)
-          msg = take!(agent.input_ch)
-        elseif isready(agent.followUpQueue)
-          msg = take!(agent.followUpQueue)
-        else
-          yield()
-        end
-      end
-
-      # Check for shutdown signal
-      if msg === :shutdown
-        break
-      end
-
-      # Dispatch message through the processing pipeline
-      result = _process_message(agent, msg)
-
-      # Send response to user
-      put!(agent.output_ch, result)
-    end
-  catch e
-    # On any error, send error response and exit the loop
-    @error "Agent loop failed" error=e
-  end
-end
-
-"""
-Process a single message through the agent pipeline.
-This is where you add your LLM call, tool execution, etc.
-"""
-function _process_message(agent::yiemAgent, msg)
-  # WORKING Replace with actual processing logic
-  # 
-  # 1. Add msg to agent._state.messages
-  # 2. Call agent.formatMsgForLLM(agent._state) to format for LLM
-  # 3. If preprocessMessages is set, call agent.preprocessMessages(...)
-  # 4. Call the LLM (blocking — the task waits here)
-  # 5. If agent has tools, handle tool calls in a loop
-  # 6. Build assistantMessage and return it
-
-  # Placeholder: echo back the message as a simple response
-  @warn "TODO: implement _process_message"
-  return assistantMessage(
-    role="assistant",
-    content=[textContent("Received: $(msg)")],
-    api="", model="", usage=nothing,
-    stopReason="end_turn",
-    errorMessage=nothing,
-    timestamp=now(),
-  )
-end
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-end # end of module

src/llmfunction.jl

@@ -12,27 +12,29 @@ using ..type, ..util
 # ---------------------------------------------- 100 --------------------------------------------- #
 
 
-""" Chatbox for chatting with virtual wine customer.
+"""
+Chat with a virtual wine customer for recommendations.
+
+Formats the input for the configured LLM model and communicates via MQTT
+to receive a response tuple containing text, selection, reward, and terminal status.
 
 # Arguments
-  - `a::T1`
-    one of Yiem's agent
-  - `input::T2`
-    text to be send to virtual wine customer
-  
-# Return
-  - `response::String`
-    response of virtual wine customer
-# Example
-```jldoctest
-julia> 
-```
+- `a::T1`: An agent instance (subtype of `agent`) with `config` containing `externalservice`
+  and `mqttServerInfo` keys
+- `input::T2`: Text to send to the virtual wine customer LLM
+
+# Returns
+- `Union{Tuple{String, Number, Number, Bool}, Tuple{String, Nothing, Number, Bool}}`: A tuple of
+  `(response_text, select, reward, isterminal)` where `select` may be `Nothing`
+
+# Notes
+- Requires `a.config["externalservice"]["virtualWineCustomer_1"]` with `llminfo` and `mqtttopic`.
+- Requires `a.config["mqttServerInfo"]` with `broker` and `port`.
+- Only supports `llama3instruct` model name (other models throw an error).
+- Uses `GeneralUtils.sendReceiveMqttMsg` with a 120-second timeout.
 
 # TODO
-  - [] update docstring
-  - [] add reccommend() to compare wine
-
-# Signature
+- Add `recommend()` to compare wines
 """
 function virtualWineUserRecommendbox(a::T1, input
   )::Union{Tuple{String, Number, Number, Bool}, Tuple{String, Nothing, Number, Bool}} where {T1<:agent}
@@ -73,27 +75,36 @@ end
 
 
 
-""" Chatbox for chatting with virtual wine customer.
+"""
+Chatbox for conversing with a virtual wine customer AI.
+
+Formats the chat history with a system prompt, sends it via MQTT to a text2text instruct
+LLM service, and parses the JSON response. Retries up to 5 times on failure.
 
 # Arguments
-  - `a::T1`
-    one of Yiem's agent
-  - `input::T2`
-    text to be send to virtual wine customer
-  
-# Return
-  - `response::String`
-    response of virtual wine customer
-# Example
+- `config::T1`: Configuration dictionary (subtype of `AbstractDict`) containing:
+  - `externalservice["text2text"]["mqtttopic"]`: MQTT topic for the LLM service
+  - `mqttServerInfo["broker"]`: MQTT broker address
+  - `mqttServerInfo["port"]`: MQTT broker port
+- `input::T2`: Current sommelier message text (subtype of `AbstractString`)
+- `virtualCustomerChatHistory`: Chat history vector of dictionaries with `"name"` and `"text"` keys
+
+# Returns
+- `Union{Tuple{String, Number, Number, Bool}, Tuple{String, Nothing, Number, Bool}}`: A tuple of
+  `(text, select, reward, isterminal)` representing the virtual customer's response
+
+# Notes
+- The system prompt defines the virtual customer's persona and response format.
+- Chat history role names are transformed: "user" → "you", "assistant" → "sommelier".
+- Uses `jsoncorrection` to fix malformed LLM JSON responses.
+- Retries up to 5 times on error before throwing.
+- Uses `formatLLMtext` with `"llama3instruct"` format.
+
+# Examples
 ```jldoctest
-julia> 
+julia> result = YiemAgent.virtualWineUserChatbox(config, sommelier_msg, history)
+("I'd like something under $50", nothing, 0, false)
 ```
-
-# TODO
-  - [] update docs 
-  - [x] write a prompt for virtual customer 
-
-# Signature
 """
 function virtualWineUserChatbox(config::T1, input::T2, virtualCustomerChatHistory
   )::Union{Tuple{String, Number, Number, Bool}, Tuple{String, Nothing, Number, Bool}} where {T1<:AbstractDict, T2<:AbstractString}
@@ -265,24 +276,36 @@ pushfirst!(virtualCustomerChatHistory, Dict("name"=> "system", "text"=> systemms
   error("virtualWineUserChatbox failed to get a response")
 end
 
-""" Search wine in stock.
+"""
+Search for wines in stock.
+
+Executes a wine search via SQL (either through SQLLLM or direct query), optionally fetching
+and base64-encoding bottle images for each result.
 
 # Arguments
-  - `a::T1` 
-    one of ChatAgent's agent.
-  - `thoughtdict::AbstractDict`
-# Return
-  A JSON string of available wine
+- `a::T`: An agent instance (subtype of `agent`) with context containing `executeSQL`,
+  `pg_conn_str`, and `agentconfig`
+- `thoughtdict::AbstractDict`: A dictionary containing `action_input` (the search query string)
 
-# Example
+# Keyword Arguments
+- `useSQLLLM::Bool=false`: Whether to use SQLLLM for the query instead of direct SQL generation
+
+# Returns
+- `NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}}`: A named tuple with:
+  - `thoughtdict`: The input thoughtdict with `action_result` populated
+  - `result_raw`: Vector of wine dictionaries with image data, or `nothing`
+
+# Notes
+- When `useSQLLLM=false`, performs hard SQL filtering followed by vector search.
+- Fetches bottle images from `http://192.168.88.106:8080/` and encodes as base64.
+- Uses `wine_search_term_classification` to extract search conditions.
+- Requires PostgreSQL connection info from `a.context.agentconfig["externalservice"]["sommpanion_db"]`.
+
+# Examples
 ```jldoctest
-julia> using ChatAgent
-julia> agent = YiemAgent.sommelier(...)
-julia> thoughtdict = 
-  OrderedDict{String, Any}(
-    "plan" => "The user is asking a very specific question about a wine (Brunello di Montalcino from Tenuta CastelGiocondo). Although the policy suggests gathering budget, wine type, and occasion, the user has provided enough specific information (name, region, producer) to attempt a direct search in the database. I will use the SEARCH_WINE_DATABASE action to check if this specific wine is in our inventory.", 
-    "action_name" => "SEARCH_WINE_DATABASE", 
-    "action_input" => "Brunello di Montalcino from Tenuta CastelGiocondo")
+julia> thoughtdict = OrderedDict("action_input" => "red wine under 50");
+julia> result = YiemAgent.search_wine_database!(agent, thoughtdict)
+(thoughtdict=OrderedDict{String, Any}(...), result_raw=[Dict(...), ...])
 ```
 """
 function search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool=false
@@ -371,6 +394,39 @@ function search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool=
 end
 
 
+"""
+Generate an SQL query from a natural language search term.
+
+Uses an LLM to generate SQL based on the database schema relevant to the search term,
+with validation and retry logic.
+
+# Arguments
+- `a::T`: An agent instance (subtype of `agent`) with context containing:
+  - `find_related_tables_for_user_question`: Function to find relevant database tables
+  - `pg_conn_str`: PostgreSQL connection string
+  - `text2textInstructLLM`: LLM function for text generation
+- `searchterm::String`: Natural language search term to convert to SQL
+
+# Keyword Arguments
+- `maxattempt::Int=10`: Maximum number of attempts to get a valid SQL response from the LLM
+
+# Returns
+- `String`: A valid SQL query string
+
+# Notes
+- Uses `gemma-4-E4B-it-UD-Q4_K_XL` model for SQL generation.
+- Dynamically fetches only relevant table schemas for the search term.
+- Validates LLM response contains required keys: "plan", "action_name", "action_input".
+- `action_name` must be "RUNSQL"; `action_input` must not contain "RUNSQL".
+- Strips triple backticks and extracts SQL from fenced code blocks.
+- Throws on failure after `maxattempt` retries.
+
+# Examples
+```jldoctest
+julia> sql = YiemAgent.generatesql(agent, "red wine under 50")
+"SELECT ... WHERE w.wine_type = 'red' AND rw.price < 50;"
+```
+"""
 function generatesql(a::T, searchterm::String,
   ; maxattempt=10
   )::String where {T<:agent}
@@ -655,6 +711,40 @@ julia> thoughtdict =
 ```
 julia> predefined_wine_search_sql(agent, thoughtdict["action_input"])
 """
+"""
+Classify a wine search term into hard SQL conditions and vector search words.
+
+Uses an LLM to extract database column conditions from a natural language search term,
+then classifies them into hard conditions (for SQL WHERE clauses) and vector search
+entries (for approximate matching).
+
+# Arguments
+- `a::T`: An agent instance (subtype of `agent`) with context containing:
+  - `find_related_tables_for_user_question`: Function to find relevant tables
+  - `pg_conn_str`: PostgreSQL connection string
+  - `text2textInstructLLM`: LLM function for text generation
+- `searchterm::String`: Natural language search term to classify
+
+# Keyword Arguments
+- `maxattempt::Int=10`: Maximum number of attempts to get a valid classification from the LLM
+
+# Returns
+- `NamedTuple{(:hard_conditions, :vector_search), Tuple{Vector{JSON.Object{String, Any}}, Vector{JSON.Object{String, Any}}}}`:
+  - `hard_conditions`: Entries with standard operators (=, <>, !=, >, <, >=, <=) for SQL WHERE clauses
+  - `vector_search`: Entries with non-standard operators (LIKE, IN, IS NULL, etc.) for vector search
+
+# Notes
+- Uses `gemma-4-E4B-it-UD-Q4_K_XL` model with JSON schema response format.
+- Applies fuzzy correction for "fuzzy_correction" bucket columns via `resolve_entity`.
+- Uses `classify_column` to determine the column type bucket.
+- Uses `harvest_entity_catalog` and `resolve_entity` (threshold=0.9) for fuzzy matching.
+
+# Examples
+```jldoctest
+julia> result = YiemAgent.wine_search_term_classification(agent, "dry red wine under 30")
+(hard_conditions=[...], vector_search=[...])
+```
+"""
 function wine_search_term_classification(a::T, searchterm::String,
   ; maxattempt=10
   ) where {T<:agent}
@@ -857,6 +947,35 @@ function wine_search_term_classification(a::T, searchterm::String,
   error("SQLLLM DecisionMaker() failed to generate a thought \n", response)
 end
 
+"""
+Build a SQL query from pre-classified wine search conditions.
+
+Constructs a JOIN query across `wine`, `retailer_wine`, and `retailer` tables with
+dynamic WHERE clauses based on the provided conditions.
+
+# Arguments
+- `conditions::Vector{JSON.Object{String, Any}}`: Vector of condition objects, each containing:
+  - `table_name`: One of "wine", "retailer_wine" (other tables are skipped)
+  - `column_name`: The column to filter on
+  - `operator`: SQL comparison operator (=, <>, !=, >, <, >=, <=)
+  - `value`: The value to compare against (number or string)
+
+# Returns
+- `String`: A complete SQL query with WHERE clause
+
+# Notes
+- Supports table aliases: "wine" → "w", "retailer_wine" → "rw"
+- Automatically handles numeric vs string value types in SQL formatting
+- Strings are single-quote escaped (replaces "'" with "''")
+- Returns a query with no WHERE clause if conditions vector is empty
+
+# Examples
+```jldoctest
+julia> cond = [JSON.Object{String, Any}("table_name"=>"wine", "column_name"=>"wine_type", "operator"=>"=", "value"=>"red")];
+julia> YiemAgent.predefined_wine_search_sql(cond)
+"SELECT ... FROM wine AS w JOIN ... WHERE w.wine_type = 'red';"
+```
+"""
 function predefined_wine_search_sql(conditions::Vector{JSON.Object{String, Any}})::String
     # 1. Base SQL structure
     base_query = 
@@ -934,6 +1053,36 @@ JOIN retailer AS r ON rw.retailer_id = r.retailer_id
     return string(base_query, where_sql, ";")
 end
 
+"""
+Execute a SQL query against the database and return formatted results.
+
+Adds `ORDER BY RANDOM() LIMIT 2` for non-LIMITed queries, removes `DISTINCT`, and returns
+either a formatted string result or the DataFrame.
+
+# Arguments
+- `executeSQL::Function`: A function that executes SQL and returns results (e.g., PostgreSQL connection)
+- `sql::T`: The SQL query string to execute (subtype of `AbstractString`)
+
+# Returns
+- `NamedTuple{(:result_str, :result_raw, :success, :errormsg)}`: A named tuple with:
+  - `result_str::Union{String, Nothing}`: Formatted string representation of results
+  - `result_raw::Union{DataFrame, Nothing}`: The DataFrame result, or `nothing`
+  - `success::Bool`: Whether the query executed successfully
+  - `errormsg::Union{String, Nothing}`: Error message if failed, or `nothing`
+
+# Notes
+- Removes `DISTINCT` keyword before execution (incompatible with `RANDOM()`)
+- Appends `ORDER BY RANDOM() LIMIT 2` if query doesn't have `LIMIT` and ends with `;`
+- Returns "No records found" message if zero rows
+- Returns column count warning if more than 30 columns
+- Randomly samples 2 rows if result has more than 2 rows
+
+# Examples
+```jldoctest
+julia> result = YiemAgent.SQLexecution(execute_fn, "SELECT * FROM wine;")
+(result_str="...", result_raw=DataFrame(...), success=true, errormsg=nothing)
+```
+"""
 function SQLexecution(executeSQL::Function, sql::T
   )::NamedTuple where {T<:AbstractString}
 
@@ -984,6 +1133,26 @@ function SQLexecution(executeSQL::Function, sql::T
   end
 end
 
+"""
+DEPRECATED: Search for wines in stock (legacy implementation).
+
+Use `search_wine_database!` instead. This function uses the older approach with
+`extractWineAttributes_1` and `extractWineAttributes_2` for attribute extraction.
+
+# Arguments
+- `a::T`: An agent instance (subtype of `agent`)
+- `thoughtdict::AbstractDict`: Dictionary containing `action_input` (search query)
+
+# Keyword Arguments
+- `useSQLLLM::Bool=false`: Whether to use SQLLLM for the query
+
+# Returns
+- `NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}}`
+
+# Notes
+- DEPRECATED: Use `search_wine_database!` for the current implementation.
+- Calls `extractWineAttributes_1` and `extractWineAttributes_2` for attribute extraction.
+"""
 function DEPRECIATED_search_wine_database!(a::T, thoughtdict::AbstractDict; useSQLLLM::Bool=false
   )::NamedTuple{(:thoughtdict, :result_raw), Tuple{OrderedDict, Any}} where {T<:agent}
   
@@ -1044,16 +1213,38 @@ function DEPRECIATED_search_wine_database!(a::T, thoughtdict::AbstractDict; useS
 end
 
 """
+Extract wine attributes from a user's search query.
+
+Uses an LLM to parse natural language input into structured wine attributes including
+name, winery, vintage, country, type, grape, price range, occasion, and food pairing.
 
 # Arguments
-  - `v::Integer`
-    dummy variable
-    
-# Return
+- `a::T1`: An agent instance (subtype of `agent`) with context containing:
+  - `text2textInstructLLM`: LLM function for text generation
+  - `pg_conn_str`: PostgreSQL connection string
+  - `id`: Agent identifier
+- `input::T2`: User's search query string (subtype of `AbstractString`)
 
-# Example
+# Keyword Arguments
+- `maxattempt::Int=10`: Maximum number of attempts to get a valid response from the LLM
+
+# Returns
+- `String`: Comma-separated list of extracted attributes in the format `"key: value, key: value"`
+  Attributes with "N/A", empty, or "none" values are excluded.
+
+# Notes
+- Uses `gemma-4-E4B-it-UD-Q4_K_XL` model with temperature 0.7.
+- Extracts: wine_name, winery, vintage, country, wine_type, grape_varietal, tasting_notes,
+  wine_price_min, wine_price_max, occasion, food_to_be_paired_with_wine
+- Validates response contains all required keys via `checkAgentResponse_JSON`.
+- Applies fuzzy entity resolution via `harvest_entity_catalog` and `resolve_entity` (threshold=0.9).
+- Strips "(some comment)" patterns from values.
+- Removes keys: thought, tasting_notes, occasion, food_to_be_paired_with_wine, vintage from final output.
+
+# Examples
 ```jldoctest
-julia> 
+julia> YiemAgent.extractWineAttributes_1(agent, "red wine from Napa under 50")
+"wine_name:N/A, winery:N/A, vintage:N/A, country:United States, wine_type:red, grape_varietal:N/A, wine_price_min:0, wine_price_max:50"
 ```
 """
 function extractWineAttributes_1(a::T1, input::T2; maxattempt=10
@@ -1225,8 +1416,40 @@ function extractWineAttributes_1(a::T1, input::T2; maxattempt=10
 end
 
 """
- - TODO "French dry white wines with medium bod" the LLM does not recognize sweetness. use LLM self questioning to solve.
- - TODO French Syrah, Viognier, under 100. LLM extract intensiry of 3-5. why?
+Extract wine intensity, sweetness, tannin, and acidity attributes from a query.
+
+Uses an LLM with a conversion table to map descriptive words (e.g., "medium-bodied", "low acidity")
+to integer ranges on a 1-5 scale.
+
+# Arguments
+- `a::T1`: An agent instance (subtype of `agent`) with context containing:
+  - `text2textInstructLLM`: LLM function for text generation
+  - `id`: Agent identifier
+- `input::T2`: User's query string containing descriptive wine preferences
+  (subtype of `AbstractString`)
+
+# Returns
+- `String`: Comma-separated list of extracted attributes in the format
+  `"key: value, key: value"`. Only includes numeric values or non-N/A strings.
+
+# Notes
+- Uses `gemma-4-E4B-it-UD-Q4_K_XL` model with temperature 0.7.
+- Extracts: sweetness, acidity, tannin, intensity (each with min/max values).
+- Applies `remove_french_accents` to the LLM response.
+- Extracts thinking via `extractthink` before JSON parsing.
+- Validates response contains all required keys via `checkAgentResponse_JSON`.
+- Removes keyword fields (sweetness_keyword, acidity_keyword, etc.) from final output.
+- Only includes values that are numbers or non-"N/A" strings.
+
+# TODO
+- "French dry white wines with medium bod" — the LLM does not recognize sweetness. Use LLM self-questioning to solve.
+- French Syrah, Viognier, under 100 — LLM extracts intensity of 3-5. Investigate why.
+
+# Examples
+```jldoctest
+julia> YiemAgent.extractWineAttributes_2(agent, "medium-bodied, low acidity, medium tannin")
+"acidity_min:1, acidity_max:2, tannin_min:3, tannin_max:4, intensity_min:3, intensity_max:4"
+```
 """
 function extractWineAttributes_2(a::T1, input::T2)::String where {T1<:agent, T2<:AbstractString}
 
@@ -1423,12 +1646,67 @@ end
 
 
 
+"""
+Get a simplified DDL schema for a PostgreSQL table with sample values.
+
+Establishes a new connection and delegates to the connection-based overload.
+
+# Arguments
+- `pg_conn_str::String`: PostgreSQL connection string
+- `table_name::String`: Name of the table to get schema for
+
+# Keyword Arguments
+- `schema_name::String="public"`: PostgreSQL schema name
+
+# Returns
+- `String`: Formatted DDL schema string with sample values
+
+# Notes
+- Delegates to `get_db_table_schema_simple_with_samples(conn, table_name, ...)` after creating
+  a new `LibPQ.Connection`.
+
+# Examples
+```jldoctest
+julia> schema = YiemAgent.get_db_table_schema_simple_with_samples(conn_str, "wine")
+"CREATE TABLE public.wine (\n  wine_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),\n  ...\n);"
+```
+"""
 function get_db_table_schema_simple_with_samples(pg_conn_str::String, table_name::String; 
   schema_name::String="public")::String
   conn = LibPQ.Connection(pg_conn_str)
   return get_db_table_schema_simple_with_samples(conn, table_name; schema_name=schema_name)
 end
 
+"""
+Get a simplified DDL schema for a PostgreSQL table with sample values.
+
+Queries the PostgreSQL catalog for column metadata, fetches sample values, and builds
+a DDL string with inline comments for each column.
+
+# Arguments
+- `conn`: An active PostgreSQL connection (e.g., `LibPQ.Connection`)
+- `table_name::String`: Name of the table to get schema for
+
+# Keyword Arguments
+- `schema_name::String="public"`: PostgreSQL schema name
+- `sample_count::Int=3`: Number of non-null sample values to fetch per column
+
+# Returns
+- `String`: Formatted DDL schema string in the style of `CREATE TABLE` statements
+  with sample values as inline comments
+
+# Notes
+- Queries `pg_attribute`, `pg_class`, `pg_namespace`, `pg_attrdef`, and `pg_constraint`
+  for column metadata, defaults, and constraints.
+- Fetches up to `sample_count` non-null values per column via `json_agg`.
+- Throws an error if the table is not found.
+
+# Examples
+```jldoctest
+julia> schema = YiemAgent.get_db_table_schema_simple_with_samples(conn, "wine")
+"CREATE TABLE public.wine (\n  wine_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),\n  ...\n);"
+```
+"""
 function get_db_table_schema_simple_with_samples(conn, table_name::String; schema_name::String="public", sample_count::Int=3)::String
     # 1. SQL query for catalog metadata
     meta_sql = """

src/type.jl

@@ -194,6 +194,21 @@ end
 # Tool types
 # ============================================================================
 
+"""
+A tool available to the agent.
+
+# Arguments
+- `name::String`: Tool identifier
+- `label::String`: Human-readable tool name
+- `description::String`: What the tool does
+- `parameters::TParameters`: Tool parameters schema (JSON schema)
+- `execute::Function`: Tool execution function
+- `prepareArguments::Union{Function, Nothing}`: Optional argument preparation callback
+- `executionMode::Union{toolExecutionMode, Nothing}`: Override: run tool calls sequentially or in parallel
+
+# Returns
+- A new `agentTool` instance
+"""
 struct agentTool{TParameters, TDetails}       # A tool available to the agent
     name::String                              # Tool identifier
     label::String                             # Human-readable tool name
@@ -209,6 +224,17 @@ end
 # Agent context
 # ============================================================================
 
+"""
+Snapshot of the agent's conversation context.
+
+# Arguments
+- `systemPrompt::String`: System prompt for the agent
+- `messages::Vector{agentMessage}`: Conversation messages
+- `tools::Union{Vector{agentTool}, Nothing}`: Available tools
+
+# Returns
+- A new `agentContext` instance
+"""
 struct agentContext                           # Snapshot of the agent's conversation context
     systemPrompt::String                      # System prompt for the agent
     messages::Vector{agentMessage}            # Conversation messages
@@ -266,9 +292,6 @@ function agentState(
     )
 end
 
-# ============================================================================
-# Tool call types
-# ============================================================================
 
 struct toolCall                               # A tool invocation from the LLM
     type::String                              # Always "function"
@@ -278,20 +301,25 @@ struct toolCall                               # A tool invocation from the LLM
 end
 
 
-# ============================================================================
-# Next turn context
-# ============================================================================
+"""
+Context for preparing the next conversation turn.
 
-struct nextTurnContext                        # Context for preparing the next conversation turn
+# Arguments
+- `message::assistantMessage`: The assistant's message that just completed
+- `toolResults::Vector{toolResultMessage}`: Tool results from this turn
+- `context::agentContext`: Current conversation context
+- `newMessages::Vector{agentMessage}`: Messages to append to the context
+
+# Returns
+- A new `prepareNextTurnContext` instance
+"""
+struct prepareNextTurnContext                        # Context for preparing the next conversation turn
     message::assistantMessage                 # The assistant's message that just completed
     toolResults::Vector{toolResultMessage}    # Tool results from this turn
     context::agentContext                     # Current conversation context
     newMessages::Vector{agentMessage}         # Messages to append to the context
 end
 
-# ============================================================================
-# llmModel types
-# ============================================================================
 
 struct modelCost                              # Model pricing per 1M tokens
     input::Float64                            # Price per 1M input tokens

src/util.jl

@@ -10,47 +10,26 @@ using ..type
 
 # ---------------------------------------------- 100 --------------------------------------------- #
 
-""" Clear agent chat history.
+"""
+Clear agent chat history.
+
+Empties the conversation history, short-term memory, events log, and chatbox.
 
 # Arguments
-  - `a::agent`
-    an agent
+- `a::T`: An agent instance (subtype of `agent`)
 
-# Return
-  - nothing
+# Returns
+- `nothing`
 
-# Example
+# Notes
+- Does not clear long-term memory; use `[PENDING] clear memory` when implemented.
+
+# Examples
 ```jldoctest
-julia> using YiemAgent, MQTTClient, GeneralUtils
-julia> client, connection = MakeConnection("test.mosquitto.org", 1883)
-julia> connect(client, connection)
-julia> msgMeta = GeneralUtils.generate_msgMeta("testtopic")
-julia> agentConfig = Dict(
-           "receiveprompt"=>Dict(
-             "mqtttopic"=> "testtopic/receive",
-           ),
-           "receiveinternal"=>Dict(  
-             "mqtttopic"=> "testtopic/internal",
-           ),
-           "text2text"=>Dict(
-             "mqtttopic"=> "testtopic/text2text",
-           ),
-         )
-julia> a = YiemAgent.sommelier(
-          client,
-          msgMeta,
-          agentConfig,
-          )
-julia> YiemAgent.addNewMessage(a, "user", "hello")
-julia> YiemAgent.clearhistory(a)
+julia> YiemAgent.clearhistory(agent)
 ```
-
-# TODO
-  - [PENDING] clear memory
-
-# Signature
 """
-function clearhistory(a::T)  where {T<:agent}
+function clearhistory(a::T)  where {T<:agent}
   empty!(a.chathistory)
   empty!(a.memory["shortmem"])
   empty!(a.memory["events"])
@@ -58,40 +37,29 @@ function clearhistory(a::T)  where {T<:agent}
 end
 
 
-""" Add new message to agent.
+"""
+Add a new message to the agent's conversation history.
 
-  messages => Dict(
-    "role" => "user",
-    "content" => [
-      Dict("type" => "text", "text" => "Describe this image for me"),
-      Dict(
-        "type" => "image_url",
-        "image_url" => Dict("url" => data_uri)
-      )
-    ]
-  )
+Automatically summarizes the oldest messages if the history exceeds `maximumMsg`.
 
-  Arguments\n
-  -----
-    a::agent
-      an agent
-    role::String
-      message sender role i.e. system, user or assistant
-    text::String
-      message text
+# Arguments
+- `a::T1`: An agent instance (subtype of `agent`)
+- `name::String`: Message sender role (e.g. "system", "user", "assistant")
+- `userinput::T2`: Message dictionary to append (must contain "name" and "text" keys)
 
-  Return\n
-  -----
-    nothing
+# Keyword Arguments
+- `maximumMsg::Integer=30`: Maximum number of messages before summarization kicks in
 
-  Example\n
-  -----
-  ```jldoctest
+# Returns
+- `nothing`
 
-  ```
+# Notes
+- When history length exceeds `maximumMsg`, the oldest messages are summarized automatically.
 
-  Signature\n
-  -----
+# Examples
+```jldoctest
+julia> YiemAgent.addNewMessage(agent, "user", Dict("name" => "user", "text" => "hello"))
+```
 """
 function addNewMessage(a::T1, name::String, userinput::T2; 
                       maximumMsg::Integer=30) where {T1<:agent, T2<:AbstractDict}
@@ -168,6 +136,23 @@ function chatHistoryToText(vecd::Vector; withkey=true, range=nothing)::String
 end
 
 
+"""
+Convert a vector of wine dictionaries to a formatted text string.
+
+# Arguments
+- `vecd::Vector`: A vector of dictionaries, each representing a wine with key-value pairs
+
+# Returns
+- A formatted string where each wine is numbered and each key-value pair is comma-separated
+  in the format: `"1) key1:value1,key2:value2 key3:value3 2) ..."`
+
+# Examples
+```jldoctest
+julia> vecd = [Dict("wine_name" => "Chateau A", "price" => "50")]
+julia> YiemAgent.availableWineToText(vecd)
+"1) wine_name:Chateau A,price:50 "
+```
+"""
 function availableWineToText(vecd::Vector)::String
   # Initialize an empty string to hold the final text
   rowtext = ""
@@ -189,34 +174,30 @@ end
 
 
 
-""" Create a dictionary representing an event with optional details.
+"""
+Create a dictionary representing an event with optional details.
 
-# Arguments
-  - `event_description::Union{String, Nothing}`
-    A description of the event
-  - `timestamp::Union{DateTime, Nothing}`
-    The time when the event occurred
-  - `subject::Union{String, Nothing}`
-    The subject or entity associated with the event
-  - `thought::Union{AbstractDict, Nothing}`
-    Any associated thoughts or metadata
-  - `action_name::Union{String, Nothing}`
-    The name of the action performed (e.g., "CHAT", "CHECKINVENTORY")
-  - `action_input::Union{String, Nothing}`
-    Input or parameters for the action
-  - `location::Union{String, Nothing}`
-    Where the event took place
-  - `equipment_used::Union{String, Nothing}`
-    Equipment involved in the event
-  - `material_used::Union{String, Nothing}`
-    Materials used during the event
-  - `outcome::Union{String, Nothing}`
-    The result or consequence of the event after action execution
-  - `note::Union{String, Nothing}`
-    Additional notes or comments
+# Keyword Arguments
+- `event_description::Union{String, Nothing}`: A description of the event
+- `timestamp::Union{DateTime, Nothing}`: The time when the event occurred
+- `subject::Union{String, Nothing}`: The subject or entity associated with the event
+- `thought::Union{AbstractDict, Nothing}`: Any associated thoughts or metadata
+- `action_name::Union{String, Nothing}`: The name of the action performed (e.g., "CHAT", "CHECKINVENTORY")
+- `action_input::Union{String, Nothing}`: Input or parameters for the action
+- `location::Union{String, Nothing}`: Where the event took place
+- `equipment_used::Union{String, Nothing}`: Equipment involved in the event
+- `material_used::Union{String, Nothing}`: Materials used during the event
+- `observation::Union{String, Nothing}`: Observation of the event
+- `note::Union{String, Nothing}`: Additional notes or comments
 
 # Returns
-  A dictionary with event details as symbol-keyed key-value pairs
+- A `Dict{String, Any}` with event details as string-keyed key-value pairs
+
+# Examples
+```jldoctest
+julia> YiemAgent.eventdict(action_name="CHAT", action_input="hello")
+Dict{String, Any} with 11 entries: ...
+```
 """
 function eventdict(;
   event_description::Union{String, Nothing}=nothing,
@@ -250,31 +231,31 @@ function eventdict(;
 end
 
 
-""" Create a formatted timeline string from a sequence of events.
+"""
+Create a formatted timeline string from a sequence of events.
 
 # Arguments
-  - `events::T1`
-    Vector of event dictionaries containing subject, action_input and optional outcome fields
-    Each event dictionary should have the following keys:
-    - :subject - The subject or entity performing the action
-    - :action_input - The action or input performed by the subject
-    - :observation - (Optional) The result or outcome of the action
+- `events::T1`: Vector of event dictionaries. Each must have `action_name` and `action_input` keys,
+  and optionally `subject` and `observation` keys.
+
+# Keyword Arguments
+- `eventindex::Union{UnitRange, Nothing}=nothing`: Optional range of event indices to include.
+  If `nothing`, all events are included.
 
 # Returns
-  - `timeline::String`
-    A formatted string representing the events with their subjects, actions, and optional outcomes
-    Format: "{index}) {subject}> {action_input} {outcome}\n" for each event
-
-# Example
-
-events = [
-  Dict("subject" => "User", "action_input" => "Hello", "observation" => nothing),
-  Dict("subject" => "Assistant", "action_input" => "Hi there!", "observation" => "with a smile")
-]
-timeline = createTimeline(events)
-# 1) User> Hello
-# 2) Assistant> Hi there! with a smile
+- `timeline::String`: A formatted string where each event appears on its own line in the format:
+  `"Event_{index} {subject}> action_name: {action_name}, action_input: {action_input}"`
+  If `observation` is present, it is appended.
 
+# Examples
+```jldoctest
+julia> events = [
+         Dict("subject" => "User", "action_input" => "Hello", "action_name" => "CHAT", "observation" => nothing),
+         Dict("subject" => "Assistant", "action_input" => "Hi there!", "action_name" => "CHAT", "observation" => "with a smile")
+       ];
+julia> YiemAgent.createTimeline(events)
+"Event_1 User> action_name: CHAT, action_input: Hello\\nEvent_2 Assistant> action_name: CHAT, action_input: Hi there!\\n"
+```
 """
 function createTimeline(events::T1; eventindex::Union{UnitRange, Nothing}=nothing
   ) where {T1<:AbstractVector}
@@ -308,6 +289,29 @@ function createTimeline(events::T1; eventindex::Union{UnitRange, Nothing}=nothin
   return timeline
 end
 
+"""
+Create a formatted event log from a sequence of events.
+
+# Arguments
+- `events::T1`: Vector of event dictionaries. Each must have `subject`, `action_name`, `action_input`,
+  and optionally `observation` keys.
+
+# Keyword Arguments
+- `index::Union{UnitRange, Nothing}=nothing`: Optional range of event indices to include.
+  If `nothing`, all events are included.
+
+# Returns
+- A `Vector{Dict{String, String}}` where each dictionary has `"name"` (from event subject) and
+  `"text"` (formatted action description) keys.
+
+# Examples
+```jldoctest
+julia> events = [Dict("subject" => "User", "action_name" => "CHAT", "action_input" => "hello", "observation" => nothing)];
+julia> log = YiemAgent.createEventsLog(events);
+julia> log[1]["name"]
+"User"
+```
+"""
 function createEventsLog(events::T1; index::Union{UnitRange, Nothing}=nothing
   ) where {T1<:AbstractVector}
   # Initialize empty log array
@@ -347,6 +351,27 @@ function createEventsLog(events::T1; index::Union{UnitRange, Nothing}=nothing
 end
 
 
+"""
+Create a formatted chat log from a sequence of chat entries.
+
+# Arguments
+- `chatdict::T1`: Vector of chat entry dictionaries. Each must have `"name"` and `"text"` keys.
+
+# Keyword Arguments
+- `index::Union{UnitRange, Nothing}=nothing`: Optional range of entry indices to include.
+  If `nothing`, all entries are included.
+
+# Returns
+- A `Vector{Dict{String, String}}` where each dictionary has `"name"` and `"text"` keys
+  copied from the corresponding input entry.
+
+# Examples
+```jldoctest
+julia> chats = [Dict("name" => "user", "text" => "hello"), Dict("name" => "assistant", "text" => "hi")];
+julia> YiemAgent.createChatLog(chats)[1]["name"]
+"user"
+```
+"""
 function createChatLog(chatdict::T1; index::Union{UnitRange, Nothing}=nothing
   ) where {T1<:AbstractVector}
   # Initialize empty log array
@@ -373,6 +398,29 @@ function createChatLog(chatdict::T1; index::Union{UnitRange, Nothing}=nothing
 end
 
 
+"""
+Check if an agent's text response contains all required header keywords.
+
+Validates that the response includes all required keywords without duplications.
+
+# Arguments
+- `response::String`: The agent's text response to validate
+- `requiredHeader::T`: Array of required keyword strings (subtype of `Array{String}`)
+
+# Returns
+- `Tuple{Bool, Union{String, Nothing}}`: A two-element tuple where:
+  - First element: `true` if all required keywords are present and not duplicated, `false` otherwise
+  - Second element: An error description string if validation failed, or `nothing` if passed
+
+# Notes
+- Uses `GeneralUtils.detectKeywordVariation` for flexible keyword matching.
+
+# Examples
+```jldoctest
+julia> ispass, err = YiemAgent.checkAgentResponse_text("hello world", ["hello"])
+(true, nothing)
+```
+"""
 function checkAgentResponse_text(response::String, requiredHeader::T
   )::Tuple where {T<:Array{String}}
   detected_kw = GeneralUtils.detectKeywordVariation(requiredHeader, response)