2063 lines
61 KiB
Julia
Executable File
2063 lines
61 KiB
Julia
Executable File
module interface
|
||
|
||
|
||
export agentReact, agentReflex,
|
||
addNewMessage, clearMessage, removeLatestMsg, conversation, directconversation,
|
||
writeEvaluationGuideline, grading, analyze, selfReflext,
|
||
formulateUserResponse, extractinfo, updateEnvState, chat_mistral_openorca,
|
||
recap, readKeywordMemory
|
||
|
||
using JSON3, DataStructures, Dates, UUIDs, HTTP, Random
|
||
using CommUtils, GeneralUtils
|
||
using ..type, ..utils, ..llmfunction
|
||
|
||
# ---------------------------------------------------------------------------- #
|
||
# pythoncall setting #
|
||
# ---------------------------------------------------------------------------- #
|
||
# Ref: https://github.com/JuliaPy/PythonCall.jl/issues/252
|
||
# by setting the following variables, PythonCall.jl will use:
|
||
# 1. system's python and packages installed by system (via apt install)
|
||
# or 2. conda python and packages installed by conda
|
||
# if these setting are not set (comment out), PythonCall will use its own python and packages that
|
||
# installed by CondaPkg.jl (from env_preparation.jl)
|
||
# ENV["JULIA_CONDAPKG_BACKEND"] = "Null" # set condapkg backend = none
|
||
# systemPython = split(read(`which python`, String), "\n")[1] # system's python path
|
||
# ENV["JULIA_PYTHONCALL_EXE"] = systemPython # find python location with $> which python ex. raw"/root/conda/bin/python"
|
||
|
||
# using PythonCall
|
||
# const py_agents = PythonCall.pynew()
|
||
# const py_llms = PythonCall.pynew()
|
||
# function __init__()
|
||
# # PythonCall.pycopy!(py_cv2, pyimport("cv2"))
|
||
|
||
# # equivalent to from urllib.request import urlopen in python
|
||
# PythonCall.pycopy!(py_agents, pyimport("langchain.agents"))
|
||
# PythonCall.pycopy!(py_llms, pyimport("langchain.llms"))
|
||
# end
|
||
|
||
#------------------------------------------------------------------------------------------------100
|
||
|
||
|
||
""" Add new message to agent.
|
||
|
||
Arguments:
|
||
|
||
Return:
|
||
|
||
```jldoctest
|
||
julia> addNewMessage(agent1, "user", "Where should I go to buy snacks")
|
||
```
|
||
"""
|
||
function addNewMessage(a::T1, role::String, content::T2) where {T1<:agent, T2<:AbstractString}
|
||
if role ∉ a.availableRole # guard against typo
|
||
error("role is not in agent.availableRole $(@__LINE__)")
|
||
end
|
||
|
||
# check whether user messages exceed limit
|
||
userMsg = 0
|
||
for i in a.messages
|
||
if i[:role] == "user"
|
||
userMsg += 1
|
||
end
|
||
end
|
||
messageleft = 0
|
||
|
||
if userMsg > a.maxUserMsg # delete all conversation
|
||
clearMessage(a)
|
||
messageleft = a.maxUserMsg
|
||
else
|
||
userMsg += 1
|
||
d = Dict(:role=> role, :content=> content, :timestamp=> Dates.now())
|
||
push!(a.messages, d)
|
||
messageleft = a.maxUserMsg - userMsg
|
||
end
|
||
|
||
return messageleft
|
||
end
|
||
|
||
function clearMessage(a::T) where {T<:agent}
|
||
for i in eachindex(a.messages)
|
||
if length(a.messages) > 0
|
||
pop!(a.messages)
|
||
else
|
||
break
|
||
end
|
||
end
|
||
memory::Dict{Symbol, Any} = Dict(
|
||
:shortterm=> OrderedDict{String, Any}(),
|
||
:longterm=> OrderedDict{String, Any}(),
|
||
:log=> OrderedDict{String, Any}(), # span from user stimulus -> multiples attempts -> final respond
|
||
)
|
||
@show a.messages
|
||
end
|
||
|
||
function removeLatestMsg(a::T) where {T<:agent}
|
||
if length(a.messages) > 1
|
||
pop!(a.messages)
|
||
end
|
||
end
|
||
|
||
function chat_mistral_openorca(a::agentReflex)
|
||
"""
|
||
general prompt format:
|
||
|
||
"
|
||
<|system|>
|
||
{role}
|
||
{tools}
|
||
{thinkingFormat}
|
||
{context}
|
||
<|im_end|>
|
||
<|im_start|>user
|
||
{usermsg}
|
||
<|im_end|>
|
||
<|im_start|>assistant
|
||
|
||
"
|
||
|
||
Note:
|
||
{context} =
|
||
"
|
||
{earlierConversation}
|
||
{env state}
|
||
{shortterm memory}
|
||
{longterm memory}
|
||
"
|
||
"""
|
||
|
||
conversation = messagesToString(a.messages)
|
||
|
||
aboutYourself =
|
||
"""
|
||
Your name is $(a.agentName)
|
||
$(a.roles[a.role])
|
||
"""
|
||
|
||
prompt =
|
||
"""
|
||
<|system|>
|
||
<About yourself>
|
||
$aboutYourself
|
||
</About yourself>
|
||
</s>
|
||
$conversation
|
||
<|assistant|>
|
||
"""
|
||
|
||
response = sendReceivePrompt(a, prompt, timeout=180, stopword=["<|", "</"])
|
||
response = split(response, "<|")[1]
|
||
response = split(response, "</")[1]
|
||
|
||
return response
|
||
end
|
||
|
||
|
||
function planner_mistral_openorca(a::agentReflex)
|
||
"""
|
||
general prompt format:
|
||
|
||
"
|
||
<|system|>
|
||
{role}
|
||
{tools}
|
||
{thinkingFormat}
|
||
<|im_end|>
|
||
{context}
|
||
<|im_start|>user
|
||
{usermsg}
|
||
<|im_end|>
|
||
<|im_start|>assistant
|
||
|
||
"
|
||
|
||
Note:
|
||
{context} =
|
||
"
|
||
{earlierConversation}
|
||
{env state}
|
||
{shortterm memory}
|
||
{longterm memory}
|
||
"
|
||
"""
|
||
|
||
conversation = messagesToString(a.messages)
|
||
toollines = ""
|
||
for (toolname, v) in a.tools
|
||
if toolname ∉ [""]
|
||
# toolline = "$toolname: $(v[:description]) $(v[:input]) $(v[:output])\n"
|
||
toolline = "$toolname: $(v[:description])\n"
|
||
toollines *= toolline
|
||
end
|
||
end
|
||
|
||
# skip objective and plan because LLM is going to generate new plan
|
||
shorttermMemory = dictToString(a.memory[:shortterm], skiplist=["Objective:", "Plan 1:"])
|
||
|
||
aboutYourself =
|
||
"""
|
||
Your name is $(a.agentName)
|
||
$(a.roles[a.role])
|
||
$(a.roleSpecificInstruction[a.role])
|
||
"""
|
||
|
||
assistant_plan_prompt =
|
||
"""
|
||
<|system|>
|
||
<About yourself>
|
||
$aboutYourself
|
||
</About yourself>
|
||
<You have access to the following tools>
|
||
$toollines
|
||
</You have access to the following tools
|
||
<Your earlier work>
|
||
$shorttermMemory
|
||
</Your earlier work>
|
||
<Your job>
|
||
Plan: first you should always think about your conversation with the user and your earlier work thoroughly then extract and devise a complete, task by task, plan to achieve your objective (pay attention to correct numeral calculation and commonsense).
|
||
Keyword memory: using JSON format, list all variables in the plan you need to find out
|
||
P.S.1 each task of the plan should be a single action.
|
||
</Your job>
|
||
<Example>
|
||
Plan:
|
||
1. Ask the user about how many miles per day they drive
|
||
2. Ask the user about what stuff they usually carry with
|
||
3. Ask the user about preferred type of car they want to buy (sedan, sport, SUV, etc)
|
||
8. Ask the user about their price range
|
||
9. Use inventory tool to find cars that match the user's preferences and are within their price range
|
||
10. Use finalanswer tool to present the recommended car to the user.
|
||
Keyword memory: {"mile per day": null, "carry item": null, "car type": null, "price range": null}
|
||
</Example>
|
||
</s>
|
||
$conversation
|
||
<|assistant|>
|
||
|
||
"""
|
||
|
||
response = sendReceivePrompt(a, assistant_plan_prompt, max_tokens=1024, temperature=0.1,
|
||
timeout=180, stopword=["<|user|>", "</"])
|
||
response = split(response, "<|")[1]
|
||
response = split(response, "</")[1]
|
||
response = split(response, "\n\n")[1]
|
||
@show response
|
||
|
||
headerToDetect = ["Plan:", "Keyword memory:",]
|
||
headers = detectCharacters(response, headerToDetect)
|
||
@show headers
|
||
chunkedtext = chunktext(response, headers)
|
||
a.memory[:c] = chunkedtext
|
||
|
||
plan = chunkedtext["Plan:"]
|
||
keywordmem = JSON3.read(chunkedtext["Keyword memory:"]) # use JSON to change from json string to dict
|
||
|
||
for (k, v) in keywordmem
|
||
a.memory[:keyword][String(k)] = v
|
||
end
|
||
|
||
return plan
|
||
end
|
||
|
||
""" Update the current plan.
|
||
"""
|
||
function updatePlan(a::agentReflex)
|
||
# conversation = messagesToString_nomark(a.messages)
|
||
|
||
toollines = ""
|
||
for (toolname, v) in a.tools
|
||
if toolname ∉ ["askbox"]
|
||
toolline = "$toolname: $(v[:description]) $(v[:input]) $(v[:output])\n"
|
||
toollines *= toolline
|
||
end
|
||
end
|
||
|
||
work = dictToString(a.memory[:shortterm])
|
||
|
||
prompt =
|
||
"""
|
||
<s>
|
||
<|system|>
|
||
$(a.roles[a.role])
|
||
Request the user’s input for the following info initially, and use alternative sources of information only if they are unable to provide it:
|
||
- occasion
|
||
- type of food ask the user
|
||
- user's personal taste of wine
|
||
- ambient temperature at the serving location
|
||
- wine price range
|
||
- wines we have in stock (use tools to get the info)
|
||
You provide a personalized recommendation of up to two wines based on the user's info above, and you describe the benefits of each wine in detail.
|
||
|
||
You have access to the following tools:
|
||
$toollines
|
||
|
||
Your work:
|
||
$work
|
||
|
||
Your job is to update the plan using available info from your work.
|
||
P.S. do not update if no info available.
|
||
|
||
For example:
|
||
Plan: 1. Ask the user for their food type.
|
||
Obs: It will be Thai dishes.
|
||
Updated plan: 1. Ask the user for their food type (Thai dishes).
|
||
</s>
|
||
Updated plan:
|
||
"""
|
||
|
||
result = sendReceivePrompt(a, prompt, max_tokens=1024, temperature=0.1)
|
||
@show updatedPlan = result
|
||
a.memory[:shortterm]["Plan 1:"] = result
|
||
|
||
end
|
||
|
||
# function selfAwareness(a::agentReflex)
|
||
|
||
# getonlykeys = ["Actinput", "Obs"]
|
||
# worknoplan = similar(a.memory[:shortterm])
|
||
# for (k, v) in a.memory[:shortterm]
|
||
# count = 0
|
||
# for i in getonlykeys
|
||
# if occursin(i, k)
|
||
# count += 1
|
||
# end
|
||
# end
|
||
# if count != 0
|
||
# worknoplan[k] = v
|
||
# end
|
||
# end
|
||
|
||
# work = dictToString(worknoplan)
|
||
|
||
# aboutYourself =
|
||
# """
|
||
# Your name is $(a.agentName)
|
||
# $(a.roles[a.role])
|
||
|
||
# """
|
||
|
||
# prompt =
|
||
# """
|
||
# <|system|>
|
||
# <About yourself>
|
||
# $aboutYourself
|
||
# $(a.roleSpecificInstruction[a.role])
|
||
# </About yourself>
|
||
# <Your earlier work>
|
||
# $work
|
||
# </Your earlier work>
|
||
# <Your keyword memory>
|
||
# $(JSON3.write(a.memory[:keyword]))
|
||
# </Your keyword memory>
|
||
# <Your job>
|
||
# Use the following format strictly:
|
||
# Info extraction: repeat all important info from the latest observed result thoroughly
|
||
# Info mapping: based on extracted info, explicitly state what each info could match which keyword memory's key
|
||
# Info matching: using JSON format, what key in my memory matches which info
|
||
# </Your job>
|
||
|
||
# <Example>
|
||
# <Your earlier work>
|
||
# The user wants to buy an electric SUV car under 20000 dollars.
|
||
# </Your earlier work>
|
||
# <Your keyword memory>
|
||
# {"car type": null, "color": null, "financing": null}
|
||
# </Your keyword memory>
|
||
# Info extraction:
|
||
# - The user is buying an electric SUV car.
|
||
# Info mapping:
|
||
# - SUV could matches "car type" key
|
||
# - electric could matches "engine type" key
|
||
# Info matching: {"car type": "SUV", "engine type": "electric motor", "color": null, "financing": null}
|
||
# </Example>
|
||
# </s>
|
||
# <|assistant|>
|
||
# Info extraction:
|
||
# """
|
||
# response = sendReceivePrompt(a, prompt, max_tokens=1024, temperature=0.2, timeout=180,
|
||
# stopword=["/n/n", "END", "End", "Obs", "<|", "</"])
|
||
# response = split(response, "<|")[1]
|
||
# response = split(response, "</")[1]
|
||
# response = "Info extraction:" * response
|
||
|
||
# println("")
|
||
# @show selfaware_1 = response
|
||
|
||
# headerToDetect = ["Info extraction:", "Info mapping:", "Info matching:", "Actinput"]
|
||
# headers = detectCharacters(response, headerToDetect)
|
||
|
||
# # headers[1:2] is for when LLM generate more than a paire of "Info extraction" and "Info matching", discard the rest
|
||
# chunkedtext = chunktext(response, headers[1:3])
|
||
# println("")
|
||
# _infomatch = chunkedtext["Info matching:"]
|
||
# _infomatch = GeneralUtils.getStringBetweenCharacters(_infomatch, '{', '}', endCharLocation="next")
|
||
# infomatch = copy(JSON3.read(_infomatch))
|
||
|
||
# println("")
|
||
# @show chunkedtext
|
||
|
||
# keywordMemoryUpdate!(a.memory[:keyword], infomatch)
|
||
|
||
# response = "What I know about user:" * JSON3.write(a.memory[:keyword]) # * response
|
||
# println("")
|
||
# @show selfaware_2 = response
|
||
|
||
|
||
# return response
|
||
# end
|
||
|
||
function selfAwareness(a::agentReflex)
|
||
|
||
getonlykeys = ["Actinput", "Obs"]
|
||
worknoplan = similar(a.memory[:shortterm])
|
||
worklength = length(a.memory[:shortterm])
|
||
for (i, (k, v)) in enumerate(a.memory[:shortterm])
|
||
if i >= worklength - 1
|
||
count = 0
|
||
for i in getonlykeys
|
||
if occursin(i, k)
|
||
count += 1
|
||
end
|
||
end
|
||
if count != 0
|
||
worknoplan[k] = v
|
||
end
|
||
end
|
||
end
|
||
|
||
println("")
|
||
@show worknoplan
|
||
|
||
work = dictToString(worknoplan)
|
||
|
||
aboutYourself =
|
||
"""
|
||
Your name is $(a.agentName)
|
||
$(a.roles[a.role])
|
||
|
||
"""
|
||
#WORKING may be I need to use "- $k is $v" because LLM skip the key for sweetness
|
||
prompt =
|
||
"""
|
||
<s>
|
||
<|system|>
|
||
<About yourself>
|
||
$aboutYourself
|
||
$(a.roleSpecificInstruction[a.role])
|
||
</About yourself>
|
||
<Your job>
|
||
Use the following format strictly:
|
||
Info extraction: repeat all important info from the latest observed result thoroughly
|
||
Info mapping: based on extracted info, explicitly state what each info could match which keyword memory's key
|
||
Info matching: using JSON format, what key in my memory matches which info
|
||
</Your job>
|
||
</|system|>
|
||
<Example>
|
||
<Your earlier work>
|
||
The user wants to buy an electric SUV car under 20000 dollars.
|
||
</Your earlier work>
|
||
<Your keyword memory>
|
||
{\"car type\": null, \"engine type\": null, \"price\": null, \"color\": null, \"financing\": null}
|
||
</Your keyword memory>
|
||
<|assistant|>
|
||
Info extraction:
|
||
- The user is buying an electric SUV car.
|
||
- price is under 20000 dollars
|
||
Info mapping:
|
||
- "SUV" could matches "car type" key
|
||
- "electric" could matches "engine type" key
|
||
- "under 20000 dollars" could match "price" key
|
||
Info matching: {\"car type\": \"SUV\", \"engine type\": \"electric motor\", \"price\": \"under 20000\", \"color\": null, \"financing\": null}
|
||
</|assistant|>
|
||
</Example>
|
||
</s>
|
||
<Your earlier work>
|
||
$work
|
||
</Your earlier work>
|
||
<Your keyword memory>
|
||
$(JSON3.write(a.memory[:keyword]))
|
||
</Your keyword memory>
|
||
<|assistant|>
|
||
Info extraction:
|
||
"""
|
||
response = sendReceivePrompt(a, prompt, max_tokens=1024, temperature=0.2, timeout=180,
|
||
stopword=["/n/n", "END", "End", "Obs", "<|", "</"])
|
||
response = split(response, "<|")[1]
|
||
response = split(response, "</")[1]
|
||
response = split(response, "</|assistant|>")[1]
|
||
response = "Info extraction:" * response
|
||
|
||
println("")
|
||
@show selfaware_1 = response
|
||
|
||
headerToDetect = ["Info extraction:", "Info mapping:", "Info matching:", "Actinput"]
|
||
headers = detectCharacters(response, headerToDetect)
|
||
|
||
# headers[1:2] is for when LLM generate more than a paire of "Info extraction" and "Info matching", discard the rest
|
||
chunkedtext = chunktext(response, headers[1:3])
|
||
println("")
|
||
_infomatch = chunkedtext["Info matching:"]
|
||
_infomatch = GeneralUtils.getStringBetweenCharacters(_infomatch, '{', '}', endCharLocation="end")
|
||
infomatch = GeneralUtils.JSON3read_stringKey(_infomatch)
|
||
# infomatch = copy(JSON3.read(_infomatch))
|
||
|
||
println("")
|
||
@show chunkedtext
|
||
|
||
println("")
|
||
@show infomatch
|
||
|
||
keywordMemoryUpdate!(a.memory[:keyword], infomatch)
|
||
|
||
response = "What I know about user:" * JSON3.write(a.memory[:keyword]) # * response
|
||
println("")
|
||
@show selfaware_2 = response
|
||
|
||
|
||
return response
|
||
end
|
||
|
||
# function actor_mistral_openorca(a::agentReflex, selfaware=nothing)
|
||
# getonlykeys = ["Actinput", "Obs"]
|
||
# worknoplan = similar(a.memory[:shortterm])
|
||
# for (k, v) in a.memory[:shortterm]
|
||
# count = 0
|
||
# for i in getonlykeys
|
||
# if occursin(i, k)
|
||
# count += 1
|
||
# end
|
||
# end
|
||
# if count != 0
|
||
# worknoplan[k] = v
|
||
# end
|
||
# end
|
||
|
||
# work = dictToString(worknoplan)
|
||
|
||
|
||
|
||
# """
|
||
# general prompt format:
|
||
|
||
# "
|
||
# <|system|>
|
||
# {role}
|
||
# {tools}
|
||
# {thinkingFormat}
|
||
# <|im_end|>
|
||
# {context}
|
||
# <|im_start|>user
|
||
# {usermsg}
|
||
# <|im_end|>
|
||
# <|im_start|>assistant
|
||
|
||
# "
|
||
|
||
# Note:
|
||
# {context} =
|
||
# "
|
||
# {earlierConversation}
|
||
# {env state}
|
||
# {shortterm memory}
|
||
# {longterm memory}
|
||
# "
|
||
# """
|
||
|
||
# toolslist = []
|
||
# toolnames = ""
|
||
# toollines = ""
|
||
# for (toolname, v) in a.tools
|
||
# toolline = "$toolname: $(v[:description]) $(v[:input]) $(v[:output])\n"
|
||
# toollines *= toolline
|
||
# toolnames *= "$toolname, "
|
||
# push!(toolslist, toolname)
|
||
# end
|
||
|
||
# thought = "Thought: you should always think about what to do according to the plan (pay attention to correct numeral calculation and commonsense and do one thing at a time.)"
|
||
# startword = "Thought:"
|
||
# if selfaware !== nothing
|
||
# "
|
||
# Thought: based on what you know, you should focus on what you need to improve first then follow your plan to decide what to do next. (P.S. 1) let's think a single step. 2) pay attention to correct numeral calculation and commonsense.)
|
||
# "
|
||
# end
|
||
# # your should request the missing information first before making a decision
|
||
# aboutYourself =
|
||
# """
|
||
# Your name is $(a.agentName)
|
||
# $(a.roles[a.role])
|
||
# """
|
||
|
||
# winestocksearchresult = nothing
|
||
# if haskey(a.memory, :winestocksearchresult) && a.memory[:winestockResult] !== nothing
|
||
# winestocksearchresult =
|
||
# """
|
||
# <winestock search result>
|
||
# $(a.memory[:winestocksearchresult])
|
||
# </winestock search result>
|
||
# """
|
||
# else
|
||
# winestocksearchresult = "\n"
|
||
# end
|
||
|
||
# prompt =
|
||
# """
|
||
# <|system|>
|
||
# <About yourself>
|
||
# $aboutYourself
|
||
# </About yourself>
|
||
# <You have access to the following tools>
|
||
# $toollines
|
||
# </You have access to the following tools>
|
||
# <Your plan>
|
||
# $(a.memory[:shortterm]["Plan 1:"])
|
||
# </Your plan>
|
||
# <What I know about the user>
|
||
# $(JSON3.write(a.memory[:keyword]))
|
||
# </What I know about the user>
|
||
# <Your job>
|
||
# Use the following format:
|
||
# $thought
|
||
# Act: based on your thought what action to choose?, must be one of [{toolnames}].
|
||
# Actinput: your input to the action (pay attention to the tool's input)
|
||
# Obs: observed result of the action
|
||
# </Your job>
|
||
# <Example>
|
||
# <What I know about the user>
|
||
# $(readKeywordMemory(a))
|
||
# </What I know about the user>
|
||
# Thought: based on what you know, I think he also need to know whether there are any charging station near by his house. I should search the internet to get this info.
|
||
# Act: internetsearch
|
||
# Actinput: {\"internetsearch\": \"EV charging station near Bangkok\"}
|
||
# </Example>
|
||
# </s>
|
||
# <|assistant|>
|
||
# $startword
|
||
# """
|
||
|
||
# prompt = replace(prompt, "{toolnames}" => toolnames)
|
||
|
||
# println("")
|
||
# @show actor_prompt = prompt
|
||
|
||
# response = nothing
|
||
# chunkedtext = nothing
|
||
# latestTask = nothing
|
||
|
||
# while true # while Thought or Act is empty, run actor again
|
||
|
||
# response = sendReceivePrompt(a, prompt, max_tokens=1024, temperature=0.4, timeout=300,
|
||
# stopword=["Thought:", "Obs:", "<|system|>", "</s>", "<|end|>"],
|
||
# seed=rand(1000000:2000000))
|
||
# println("")
|
||
# @show actor_raw = response
|
||
|
||
# response = splittext(response, ["/n/n", "END", "End","obs", "Obs", "<|im_end|>"])
|
||
# response = split(response, "<|")[1]
|
||
# response = split(response, "</")[1]
|
||
# # response = split(response, "Thought:")[end]
|
||
|
||
# latestTask = shortMemLatestTask(a.memory[:shortterm]) +1
|
||
|
||
# response = startword * response
|
||
|
||
# headerToDetect = ["Plan:", "Self-awareness:", "Thought:",
|
||
# "Act:", "Actinput:", "Obs:",
|
||
# "Answer:", "Conclusion:", "Summary:"]
|
||
|
||
# # replace headers with headers with correct attempt and task number
|
||
# response = replaceHeaders(response, headerToDetect, latestTask)
|
||
# response = split(response, "<|")[1]
|
||
# response = split(response, "</")[1]
|
||
|
||
# # sometime LLM use wrong keyword. use regex to detect "actinput5:" and replace it with "Actinput"
|
||
# regexmatch = match(r"actinput\d+:", response)
|
||
# respone = regexmatch !== nothing ? response = replace(response, match=>"Actinput:") : response
|
||
# response = replace(response, "actinput:"=>"Actinput:")
|
||
|
||
# println("")
|
||
# @show actor_response = response
|
||
|
||
# headerToDetect = ["Plan $(a.attempt):",
|
||
# "Self-awareness $latestTask:",
|
||
# "Thought $latestTask:",
|
||
# "Act $latestTask:",
|
||
# "Actinput $latestTask:",
|
||
# "Obs $latestTask:",
|
||
# "Check $latestTask:",]
|
||
# headers = detectCharacters(response, headerToDetect)
|
||
# chunkedtext = chunktext(response, headers)
|
||
|
||
# # assuming length more than 10 character means LLM has valid thinking
|
||
# check_1 = haskey(chunkedtext, "Thought $latestTask:")
|
||
# check_2 = haskey(chunkedtext, "Act $latestTask:")
|
||
# check_3 = haskey(chunkedtext, "Actinput $latestTask:")
|
||
|
||
# # check for a valid toolname
|
||
# check_4 = false
|
||
# for i in toolslist
|
||
# if occursin(i, chunkedtext["Act $latestTask:"])
|
||
# check_4 = true
|
||
# break
|
||
# end
|
||
# end
|
||
|
||
# # check for empty Thought
|
||
# check_5 = length(chunkedtext["Thought $latestTask:"]) > 5
|
||
# # check for empty Actinput
|
||
# check_6 = nothing
|
||
# try
|
||
# check_6 = length(chunkedtext["Actinput $latestTask:"]) > 5
|
||
# catch
|
||
# println("")
|
||
# @show response
|
||
# println("")
|
||
# @show chunkedtext
|
||
# a.memory[:chunkedtext] = chunkedtext
|
||
# end
|
||
|
||
|
||
# # check whether the act has valid json
|
||
# check_7 = true
|
||
# if occursin('{', response)
|
||
# try
|
||
# act = GeneralUtils.getStringBetweenCharacters(response, '{', '}', endCharLocation="end")
|
||
# act = JSON3.read(act)
|
||
# check_7 = true
|
||
# catch
|
||
# check_7 = false
|
||
# end
|
||
# end
|
||
|
||
# # print all check_1 to check_6
|
||
# println("check_1: $check_1, check_2: $check_2, check_3: $check_3, check_4: $check_4,
|
||
# check_5: $check_5, check_6: $check_6, check_7: $check_7")
|
||
# if check_1 && check_2 && check_3 && check_4 && check_5 && check_6 && check_7
|
||
|
||
# #TODO paraphrase selfaware
|
||
# break
|
||
# end
|
||
# @show retrying_actor = response
|
||
# end
|
||
|
||
# toolname = toolNameBeingCalled(chunkedtext["Act $latestTask:"], a.tools)
|
||
|
||
# # change trailing number to continue a.memory[:shortterm]
|
||
# headerToDetect = ["Question:", "Plan:", "Self-awareness:", "Thought:",
|
||
# "Act:", "Actinput:", "Obs:", "...",
|
||
# "Answer:", "Conclusion:", "Summary:"]
|
||
# response = replaceHeaders(response, headerToDetect, latestTask)
|
||
# println("")
|
||
# @show actor_response_1 = response
|
||
# headerToDetect = ["Plan $(a.attempt):",
|
||
# "Thought $latestTask:",
|
||
# "Act $latestTask:",
|
||
# "Actinput $latestTask:",
|
||
# "Obs $latestTask:",
|
||
# "Check $latestTask:",]
|
||
# headers = detectCharacters(response, headerToDetect)
|
||
# chunkedtext = chunktext(response, headers)
|
||
# println("")
|
||
# @show chunkedtext
|
||
|
||
|
||
# toolinput = chunkedtext["Actinput $latestTask:"]
|
||
|
||
|
||
# # because tools has JSON input but sometime LLM output is not JSON, we need to check.
|
||
# if occursin("{", toolinput)
|
||
# act = GeneralUtils.getStringBetweenCharacters(response, '{', '}', endCharLocation="end")
|
||
# act = copy(JSON3.read(act))
|
||
# chunkedtext["Actinput $latestTask:"] = JSON3.write(act[Symbol(toolname)])
|
||
# toolinput = act[Symbol(toolname)]
|
||
# end
|
||
|
||
|
||
# chunkedtext["Act $latestTask:"] = toolname
|
||
|
||
# return (toolname=toolname, toolinput=toolinput, chunkedtext=chunkedtext, selfaware=selfaware)
|
||
# end
|
||
|
||
function actor_mistral_openorca(a::agentReflex, selfaware=nothing)
|
||
|
||
toolslist = []
|
||
toolnames = ""
|
||
toollines = ""
|
||
for (toolname, v) in a.tools
|
||
toolline = "$toolname: $(v[:description]) $(v[:input]) $(v[:output])\n"
|
||
toollines *= toolline
|
||
toolnames *= "$toolname, "
|
||
push!(toolslist, toolname)
|
||
end
|
||
|
||
thought = "Thought: you should always think about what to do according to the plan (pay attention to correct numeral calculation and commonsense and do one thing at a time.)"
|
||
startword = "Thought:"
|
||
if selfaware !== nothing
|
||
"
|
||
Thought: based on what you know, you should focus on what you need to improve first then follow your plan to decide what to do next. (P.S. 1) let's think a single step. 2) pay attention to correct numeral calculation and commonsense.)
|
||
"
|
||
end
|
||
# your should request the missing information first before making a decision
|
||
aboutYourself =
|
||
"""
|
||
Your name is $(a.agentName)
|
||
$(a.roles[a.role])
|
||
"""
|
||
|
||
winestocksearchresult = nothing
|
||
if haskey(a.memory, :winestocksearchresult) && a.memory[:winestockResult] !== nothing
|
||
winestocksearchresult =
|
||
"""
|
||
<winestock search result>
|
||
$(a.memory[:winestocksearchresult])
|
||
</winestock search result>
|
||
"""
|
||
else
|
||
winestocksearchresult = "\n"
|
||
end
|
||
|
||
keywordmemory = ""
|
||
for (k, v) in a.memory[:keyword]
|
||
if v === nothing
|
||
keywordmemory *= "- I have no info on $k \n"
|
||
else
|
||
keywordmemory *= "- $k is $v \n"
|
||
end
|
||
end
|
||
|
||
"""
|
||
- Car type is SUV
|
||
- Brand is Lexus
|
||
- Price is 20k dollar
|
||
- No info on the car color yet
|
||
- No info on the financing method yet
|
||
- Luxury level is high
|
||
{\"car type\": "SUV",\"brand\":\"Lexus\",\"price\":\"200000\",\"color\": null,\"financing method\": null, \"luxury level\":\"high\"}
|
||
"""
|
||
|
||
prompt =
|
||
"""
|
||
<s>
|
||
<|system|>
|
||
<About yourself>
|
||
Your name is $(a.agentName)
|
||
$(a.roles[a.role])
|
||
</About yourself>
|
||
<You have access to the following tools>
|
||
$toollines
|
||
</You have access to the following tools>
|
||
<Your job>
|
||
Use the following format:
|
||
Thought: based on what you know about the user, where are you at according to the plan? what to do next?. (PS. 1. let's think only one thing at a time. 2. pay attention to correct numeral calculation and commonsense.)
|
||
Act: based on your thought what action to choose?, must be one of [{toolnames}].
|
||
Actinput: your input to the action using JSON format (pay attention to the tool's input)
|
||
Obs: observed result of the action
|
||
</Your job>
|
||
</|system|>
|
||
<Example 1>
|
||
<my plan>
|
||
I'll ask the user for car type, brand, price, color, financing method and luxury level before I'll give them any advice.
|
||
</my plan>
|
||
<What I know about the user>
|
||
- Car type is SUV
|
||
- Brand is Lexus
|
||
- Price is 20k dollar
|
||
- No info on the car color yet
|
||
- No info on the financing method yet
|
||
- Luxury level is high
|
||
</What I know about the user>
|
||
<|assistant|>
|
||
Thought: after checking what I know about the user against my plan, I still don't know the color and financing method yet. Next, I need to know what color the user like.
|
||
Act: askbox
|
||
Actinput:
|
||
</|assistant|>
|
||
</Example 1>
|
||
</s>
|
||
<My plan>
|
||
$(a.memory[:shortterm]["Plan 1:"])
|
||
</My plan>
|
||
<What I know about the user>
|
||
$keywordmemory)
|
||
</What I know about the user>
|
||
<|assistant|>
|
||
Thought:
|
||
"""
|
||
|
||
prompt = replace(prompt, "{toolnames}" => toolnames)
|
||
|
||
println("")
|
||
@show actor_prompt = prompt
|
||
|
||
response = nothing
|
||
chunkedtext = nothing
|
||
latestTask = nothing
|
||
|
||
while true # while Thought or Act is empty, run actor again
|
||
|
||
response = sendReceivePrompt(a, prompt, max_tokens=1024, temperature=0.4, timeout=300,
|
||
stopword=["Thought:", "Obs:", "<|system|>", "</s>", "<|end|>"],
|
||
seed=rand(1000000:2000000))
|
||
println("")
|
||
@show actor_raw = response
|
||
|
||
response = splittext(response, ["/n/n", "END", "End","obs", "Obs", "<|im_end|>"])
|
||
response = split(response, "<|")[1]
|
||
response = split(response, "</")[1]
|
||
# response = split(response, "Thought:")[end]
|
||
|
||
latestTask = shortMemLatestTask(a.memory[:shortterm]) +1
|
||
|
||
response = startword * response
|
||
|
||
headerToDetect = ["Plan:", "Self-awareness:", "Thought:",
|
||
"Act:", "Actinput:", "Obs:",
|
||
"Answer:", "Conclusion:", "Summary:"]
|
||
|
||
# replace headers with headers with correct attempt and task number
|
||
response = replaceHeaders(response, headerToDetect, latestTask)
|
||
response = split(response, "<|")[1]
|
||
response = split(response, "</")[1]
|
||
|
||
# sometime LLM use wrong keyword. use regex to detect "actinput5:" and replace it with "Actinput"
|
||
regexmatch = match(r"actinput\d+:", response)
|
||
respone = regexmatch !== nothing ? response = replace(response, match=>"Actinput:") : response
|
||
response = replace(response, "actinput:"=>"Actinput:")
|
||
|
||
println("")
|
||
@show actor_response = response
|
||
|
||
headerToDetect = ["Plan $(a.attempt):",
|
||
"Self-awareness $latestTask:",
|
||
"Thought $latestTask:",
|
||
"Act $latestTask:",
|
||
"Actinput $latestTask:",
|
||
"Obs $latestTask:",
|
||
"Check $latestTask:",]
|
||
headers = detectCharacters(response, headerToDetect)
|
||
chunkedtext = chunktext(response, headers)
|
||
|
||
# assuming length more than 10 character means LLM has valid thinking
|
||
check_1 = haskey(chunkedtext, "Thought $latestTask:")
|
||
check_2 = haskey(chunkedtext, "Act $latestTask:")
|
||
check_3 = haskey(chunkedtext, "Actinput $latestTask:")
|
||
|
||
# check for a valid toolname
|
||
check_4 = false
|
||
for i in toolslist
|
||
if occursin(i, chunkedtext["Act $latestTask:"])
|
||
check_4 = true
|
||
break
|
||
end
|
||
end
|
||
|
||
# check for empty Thought
|
||
check_5 = length(chunkedtext["Thought $latestTask:"]) > 5
|
||
# check for empty Actinput
|
||
check_6 = nothing
|
||
try
|
||
check_6 = length(chunkedtext["Actinput $latestTask:"]) > 5
|
||
catch
|
||
println("")
|
||
@show response
|
||
println("")
|
||
@show chunkedtext
|
||
a.memory[:chunkedtext] = chunkedtext
|
||
end
|
||
|
||
|
||
# check whether the act has valid json
|
||
check_7 = true
|
||
if occursin('{', response)
|
||
try
|
||
act = GeneralUtils.getStringBetweenCharacters(response, '{', '}', endCharLocation="end")
|
||
println("")
|
||
@show act
|
||
act = JSON3.read(act)
|
||
check_7 = true
|
||
catch
|
||
check_7 = false
|
||
end
|
||
end
|
||
|
||
if check_1 && check_2 && check_3 && check_4 && check_5 && check_6 && check_7
|
||
break
|
||
end
|
||
# print all check_1 to check_6
|
||
println("")
|
||
println("check_1: $check_1, check_2: $check_2, check_3: $check_3, check_4: $check_4,
|
||
check_5: $check_5, check_6: $check_6, check_7: $check_7")
|
||
@show retrying_actor = response
|
||
end
|
||
|
||
toolname = toolNameBeingCalled(chunkedtext["Act $latestTask:"], a.tools)
|
||
|
||
# change trailing number to continue a.memory[:shortterm]
|
||
headerToDetect = ["Question:", "Plan:", "Self-awareness:", "Thought:",
|
||
"Act:", "Actinput:", "Obs:", "...",
|
||
"Answer:", "Conclusion:", "Summary:"]
|
||
response = replaceHeaders(response, headerToDetect, latestTask)
|
||
println("")
|
||
@show actor_response_1 = response
|
||
headerToDetect = ["Plan $(a.attempt):",
|
||
"Thought $latestTask:",
|
||
"Act $latestTask:",
|
||
"Actinput $latestTask:",
|
||
"Obs $latestTask:",
|
||
"Check $latestTask:",]
|
||
headers = detectCharacters(response, headerToDetect)
|
||
chunkedtext = chunktext(response, headers)
|
||
chunkedtext["Act $latestTask:"] = toolname
|
||
|
||
println("")
|
||
@show chunkedtext
|
||
|
||
toolinput = chunkedtext["Actinput $latestTask:"]
|
||
|
||
# # because tools has JSON input but sometime LLM output is not JSON, we need to check.
|
||
# if occursin("{", toolinput)
|
||
# act = GeneralUtils.getStringBetweenCharacters(response, '{', '}', endCharLocation="end")
|
||
# act = copy(JSON3.read(act))
|
||
|
||
# println("")
|
||
# @show act
|
||
|
||
# chunkedtext["Actinput $latestTask:"] = JSON3.write(act[Symbol(toolname)])
|
||
# a.memory[:c] = chunkedtext
|
||
# toolinput = act[Symbol(toolname)]
|
||
# end
|
||
|
||
|
||
|
||
return (toolname=toolname, toolinput=toolinput, chunkedtext=chunkedtext, selfaware=selfaware)
|
||
end
|
||
|
||
|
||
"""
|
||
Chat with llm.
|
||
|
||
```jldoctest
|
||
julia> using JSON3, UUIDs, Dates, FileIO, CommUtils, ChatAgent
|
||
julia> mqttClientSpec = (
|
||
clientName= "someclient", # name of this client
|
||
clientID= "$(uuid4())",
|
||
broker= "mqtt.yiem.cc",
|
||
pubtopic= (imgAI="img/api/v0.0.1/gpu/request",
|
||
txtAI="txt/api/v0.1.0/gpu/request"),
|
||
subtopic= (imgAI="agent/api/v0.1.0/img/response",
|
||
txtAI="agent/api/v0.1.0/txt/response"),
|
||
keepalive= 30,
|
||
)
|
||
julia> msgMeta = Dict(
|
||
:msgPurpose=> "updateStatus",
|
||
:from=> "agent",
|
||
:to=> "llmAI",
|
||
:requestresponse=> "request",
|
||
:sendto=> "", # destination topic
|
||
:replyTo=> "agent/api/v0.1.0/txt/response", # requester ask responseer to send reply to this topic
|
||
:repondToMsgId=> "", # responseer is responseing to this msg id
|
||
:taskstatus=> "", # "complete", "fail", "waiting" or other status
|
||
:timestamp=> Dates.now(),
|
||
:msgId=> "$(uuid4())",
|
||
)
|
||
julia> newAgent = ChatAgent.agentReact(
|
||
"Jene",
|
||
mqttClientSpec,
|
||
role=:assistant_react,
|
||
msgMeta=msgMeta
|
||
)
|
||
julia> response = ChatAgent.conversation(newAgent, "Hi! how are you?")
|
||
```
|
||
# """
|
||
function conversation(a::agentReflex, usermsg::String; attemptlimit::Int=3)
|
||
a.attemptlimit = attemptlimit
|
||
workstate = nothing
|
||
response = nothing
|
||
|
||
_ = addNewMessage(a, "user", usermsg)
|
||
isuseplan = isUsePlans(a)
|
||
# newinfo = extractinfo(a, usermsg)
|
||
# a.env = newinfo !== nothing ? updateEnvState(a, newinfo) : a.env
|
||
@show isuseplan
|
||
|
||
if isuseplan # use plan before responding
|
||
if haskey(a.memory[:shortterm], "User:") == false #TODO should change role if user want to buy wine.
|
||
a.memory[:shortterm]["User:"] = usermsg
|
||
end
|
||
workstate, response = work(a)
|
||
end
|
||
|
||
# if LLM using askbox, use returning msg form askbox as conversation response
|
||
if workstate == "askbox" || workstate == "formulatedUserResponse"
|
||
#TODO paraphrase msg so that it is human friendlier word.
|
||
else
|
||
response = chat_mistral_openorca(a)
|
||
response = split(response, "\n\n")[1]
|
||
response = split(response, "\n\n")[1]
|
||
end
|
||
|
||
response = removeTrailingCharacters(response)
|
||
_ = addNewMessage(a, "assistant", response)
|
||
|
||
return response
|
||
end
|
||
|
||
"""
|
||
Continuously run llm functions except when llm is getting Answer: or askbox.
|
||
There are many work() depend on thinking mode.
|
||
"""
|
||
function work(a::agentReflex)
|
||
workstate = nothing
|
||
response = nothing
|
||
|
||
# user answering LLM -> Obs
|
||
if length(a.memory[:shortterm]) > 1
|
||
latestTask = shortMemLatestTask(a.memory[:shortterm])
|
||
if haskey(a.memory[:shortterm], "Act $latestTask:")
|
||
if occursin("askbox", a.memory[:shortterm]["Act $latestTask:"])
|
||
a.memory[:shortterm]["Obs $latestTask:"] = "(user response) " * a.messages[end][:content]
|
||
end
|
||
end
|
||
end
|
||
|
||
while true # Work loop
|
||
objective = nothing
|
||
|
||
# make new plan
|
||
if !haskey(a.memory[:shortterm], "Plan 1:")
|
||
plan = planner_mistral_openorca(a)
|
||
a.memory[:shortterm]["Plan $(a.attempt):"] = plan
|
||
a.memory[:log]["Plan $(a.attempt):"] = plan
|
||
a.task = 1 # reset because new plan is created
|
||
println("")
|
||
@show plan
|
||
println("")
|
||
@show a.attempt
|
||
end
|
||
|
||
if a.attempt <= a.attemptlimit
|
||
toolname = nothing
|
||
toolinput = nothing
|
||
|
||
# enter actor loop
|
||
actorstate, msgToUser = actor(a)
|
||
|
||
if actorstate == "askbox"
|
||
response = msgToUser
|
||
workstate = actorstate
|
||
break
|
||
elseif actorstate == "formulateFinalResponse"
|
||
println("all tasks done")
|
||
|
||
response = formulateUserResponse(a)
|
||
|
||
println("")
|
||
formulatedresponse = response
|
||
@show formulatedresponse
|
||
|
||
a.memory[:shortterm]["response $(a.attempt):"] = response
|
||
a.memory[:log]["response $(a.attempt):"] = response
|
||
|
||
# evaluate. if score > 6/10 good enough.
|
||
guideline = writeEvaluationGuideline(a)
|
||
|
||
println("")
|
||
@show guideline
|
||
|
||
score = grading(a, guideline, response)
|
||
@show score
|
||
if score > 5 # good enough answer
|
||
println("")
|
||
formulatedresponse_final = response
|
||
@show formulatedresponse_final
|
||
workstate = "formulatedUserResponse"
|
||
a.memory[:shortterm] = OrderedDict{String, Any}()
|
||
a.memory[:log] = OrderedDict{String, Any}()
|
||
break
|
||
else # self evaluate and reflect then try again
|
||
analysis = analyze(a)
|
||
println("")
|
||
@show analysis
|
||
|
||
lessonwithcontext = selfReflext(a, analysis)
|
||
|
||
println("")
|
||
@show lessonwithcontext
|
||
|
||
headerToDetect = ["Lesson:", "Context:", ]
|
||
headers = detectCharacters(lessonwithcontext, headerToDetect)
|
||
chunkedtext = chunktext(lessonwithcontext, headers)
|
||
|
||
a.memory[:longterm][chunkedtext["Context:"]] = chunkedtext["Lesson:"]
|
||
a.attempt += 1
|
||
a.task = 0
|
||
a.memory[:shortterm] = OrderedDict{String, Any}()
|
||
a.memory[:log] = OrderedDict{String, Any}()
|
||
println("")
|
||
println("RETRY $(a.attempt +1)")
|
||
println("")
|
||
end
|
||
else
|
||
error("undefied condition, actorstate $actorstate $(@__LINE__)")
|
||
break
|
||
end
|
||
else
|
||
error("attempt limit reach")
|
||
break
|
||
end
|
||
end
|
||
|
||
# good enough answer
|
||
return workstate, response
|
||
end
|
||
|
||
|
||
"""
|
||
Actor function.
|
||
|
||
Arguments:
|
||
a, one of ChatAgent's agent.
|
||
plan, a task by task plan to response
|
||
|
||
Return:
|
||
case 1) if actor complete the plan successfully.
|
||
actorState = "all tasks done" inidicates that all task in plan were done.
|
||
msgToUser = nothing.
|
||
case 2) if actor needs to talk to user for more context
|
||
actorState = "askbox"
|
||
msgToUser = "message from assistant to user"
|
||
|
||
"""
|
||
function actor(a::agentReflex)
|
||
|
||
actorState = nothing
|
||
msgToUser = nothing
|
||
|
||
|
||
# totaltasks = checkTotalTaskInPlan(a)
|
||
|
||
|
||
while true # Actor loop
|
||
# check whether the current task is completed, skip evaluation if memory has only "Plan 1:"
|
||
# taskrecap = ""
|
||
# if length(keys(a.memory[:shortterm])) != 1
|
||
# taskrecap = recap(a)
|
||
# end
|
||
# println("")
|
||
# @show taskrecap
|
||
latestTask = shortMemLatestTask(a.memory[:shortterm]) +1
|
||
println(">>> working")
|
||
# work
|
||
selfaware = nothing
|
||
if length(a.memory[:shortterm]) > 2 # must have User:, Plan:, Thought:, Act:, Actinput: already
|
||
selfaware = selfAwareness(a)
|
||
end
|
||
|
||
actorResult = actor_mistral_openorca(a, selfaware)
|
||
toolname, toolinput, chunkedtext, selfaware = actorResult
|
||
println("")
|
||
@show toolname
|
||
@show toolinput
|
||
println(typeof(toolinput))
|
||
|
||
if toolname == "askbox" # chat with user
|
||
msgToUser = askbox(toolinput)
|
||
actorState = toolname
|
||
|
||
#WORKING add only a single Q1 to memory because LLM need to ask the user only 1 question at a time
|
||
latestTask = shortMemLatestTask(a.memory[:shortterm]) +1
|
||
chunkedtext["Actinput $latestTask:"] = msgToUser
|
||
addShortMem!(a.memory[:shortterm], chunkedtext)
|
||
|
||
break
|
||
elseif toolname == "finalanswer"
|
||
addShortMem!(a.memory[:shortterm], chunkedtext)
|
||
println(">>> already done")
|
||
actorState = "formulateFinalResponse"
|
||
error(5555)
|
||
break
|
||
else # function call
|
||
addShortMem!(a.memory[:shortterm], chunkedtext)
|
||
f = a.tools[toolname][:func]
|
||
toolresult = f(a, actorResult)
|
||
@show toolresult
|
||
if toolname == ""
|
||
a.memory[:shortterm]["Obs $latestTask:"] = "I found wines in <winestock search result>"
|
||
a.memory[:winestockResult] = toolresult
|
||
a.memory[:log]["Obs $latestTask:"] = "winestock search done"
|
||
else
|
||
a.memory[:shortterm]["Obs $latestTask:"] = toolresult
|
||
a.memory[:log]["Obs $latestTask:"] = toolresult
|
||
end
|
||
end
|
||
|
||
|
||
|
||
|
||
|
||
end
|
||
|
||
return actorState, msgToUser
|
||
end
|
||
|
||
|
||
|
||
""" Write evaluation guideline.
|
||
|
||
Arguments:
|
||
a, one of ChatAgent's agent.
|
||
usermsg, stimulus e.g. question, task and etc.
|
||
|
||
Return:
|
||
An evaluation guideline used to guage AI's work.
|
||
|
||
Example:
|
||
|
||
```jldoctest
|
||
julia> using ChatAgent, CommUtils
|
||
julia> agent = ChatAgent.agentReflex("Jene")
|
||
julia> usermsg = "What's AMD latest product?"
|
||
"
|
||
julia> evaluationGuideLine = writeEvaluationGuideline(agent, usermsg)
|
||
```
|
||
"""
|
||
function writeEvaluationGuideline(a::agentReflex)
|
||
prompt =
|
||
"""
|
||
<|system|>
|
||
$(a.roles[a.role])
|
||
<You have access to the following tools>
|
||
askbox: Useful for when you need to ask a customer for more context. Input should be a conversation to customer.
|
||
wikisearch: Useful for when you need to search an encyclopedia Input is keywords and not a question.
|
||
</You have access to the following tools>
|
||
<Your job>
|
||
1. Write an evaluation guideline for wine recommendation in order to be able to evaluate your response.
|
||
2. An example of what the response should be.
|
||
</Your job>
|
||
</s>
|
||
<|assistant|>
|
||
|
||
"""
|
||
|
||
response = sendReceivePrompt(a, prompt)
|
||
return response
|
||
end
|
||
|
||
|
||
|
||
""" Determine a score out of 10 according to evaluation guideline.
|
||
|
||
Arguments:
|
||
a, one of ChatAgent's agent.
|
||
guidelines, an evaluation guideline.
|
||
shorttermMemory, a short term memory that logs what happened.
|
||
|
||
Return:
|
||
A score out of 10 based on guideline.
|
||
|
||
Example:
|
||
|
||
```jldoctest
|
||
julia> using ChatAgent, CommUtils
|
||
julia> agent = ChatAgent.agentReflex("Jene")
|
||
julia> shorttermMemory = OrderedDict{String, Any}(
|
||
"user" => "What's the latest AMD GPU?",
|
||
"Plan 1:" => " To answer this question, I will need to search for the latest AMD GPU using the wikisearch tool.\n",
|
||
"Act 1:" => " wikisearch\n",
|
||
"Actinput 1:" => " amd gpu latest\n",
|
||
"Obs 1:" => "No info available for your search query.",
|
||
"Act 2:" => " wikisearch\n",
|
||
"Actinput 2:" => " amd graphics card latest\n",
|
||
"Obs 2:" => "No info available for your search query.")
|
||
julia> guideline = "\nEvaluation Guideline:\n1. Check if the user's question has been understood correctly.\n2. Evaluate the tasks taken to provide the information requested by the user.\n3. Assess whether the correct tools were used for the task.\n4. Determine if the user's request was successfully fulfilled.\n5. Identify any potential improvements or alternative approaches that could be used in the future.\n\nThe response should include:\n1. A clear understanding of the user's question.\n2. The tasks taken to provide the information requested by the user.\n3. An evaluation of whether the correct tools were used for the task.\n4. A confirmation or explanation if the user's request was successfully fulfilled.\n5. Any potential improvements or alternative approaches that could be used in the future."
|
||
julia> score = grading(agent, guideline, shorttermMemory)
|
||
```
|
||
"""
|
||
function grading(a, guideline::T, text::T) where {T<:AbstractString}
|
||
prompt =
|
||
"""
|
||
<|system|>
|
||
<You have access to the following tools>
|
||
askbox: Useful for when you need to ask a customer for more context. Input should be a conversation to customer.
|
||
wikisearch: Useful for when you need to search an encyclopedia Input is keywords and not a question.
|
||
</You have access to the following tools>
|
||
<Guideline>
|
||
$guideline
|
||
</Guideline>
|
||
<Your response>
|
||
$text
|
||
</Your response>
|
||
|
||
<Your job>
|
||
Evaluate your response using the evaluation guideline then give yourself a score out of 9 for your response.
|
||
</Your job>
|
||
<Example>
|
||
{"Evaluate": "My response is detailed with good comparison between options.", "Score": 6}
|
||
</Example>
|
||
</s>
|
||
<|assistant|>
|
||
{
|
||
"""
|
||
println("")
|
||
prompt_grading = prompt
|
||
@show prompt_grading
|
||
println("")
|
||
score = nothing
|
||
while true
|
||
response = sendReceivePrompt(a, prompt, timeout=180)
|
||
try
|
||
response = "{" * split(response, "}")[1] * "}"
|
||
@show response
|
||
@show jsonresponse = JSON3.read(response)
|
||
score = jsonresponse["Score"]
|
||
break
|
||
catch
|
||
println("retry grading")
|
||
end
|
||
end
|
||
|
||
return score
|
||
end
|
||
|
||
|
||
|
||
""" Analize work.
|
||
|
||
Arguments:
|
||
a, one of ChatAgent's agent.
|
||
|
||
Return:
|
||
A report of analized work.
|
||
|
||
Example:
|
||
|
||
```jldoctest
|
||
julia> using ChatAgent, CommUtils
|
||
julia> agent = ChatAgent.agentReflex("Jene")
|
||
julia> shorttermMemory = OrderedDict{String, Any}(
|
||
"user:" => "What's the latest AMD GPU?",
|
||
"Plan 1:" => " To answer this question, I will need to search for the latest AMD GPU using the wikisearch tool.\n",
|
||
"Act 1:" => " wikisearch\n",
|
||
"Actinput 1:" => " amd gpu latest\n",
|
||
"Obs 1:" => "No info available for your search query.",
|
||
"Act 2:" => " wikisearch\n",
|
||
"Actinput 2:" => " amd graphics card latest\n",
|
||
"Obs 2:" => "No info available for your search query.")
|
||
julia> report = analyze(agent, shorttermMemory)
|
||
```
|
||
"""
|
||
function analyze(a)
|
||
shorttermMemory = dictToString(a.memory[:shortterm])
|
||
prompt =
|
||
"""
|
||
<|system|>
|
||
<you have access to the following tools>
|
||
askbox: Useful for when you need to ask a customer for more context. Input should be a conversation to customer.
|
||
wikisearch: Useful for when you need to search an encyclopedia Input is keywords and not a question.
|
||
</you have access to the following tools>
|
||
<your earlier work>
|
||
$shorttermMemory
|
||
</your earlier work>
|
||
<your job>
|
||
You job is to do each of the following in detail to analize your work.
|
||
1. What happened?
|
||
2. List all relationships, each with cause and effect.
|
||
3. Look at each relationship, figure out why it behaved that way.
|
||
4. What could you do to improve the response?
|
||
</your job>
|
||
</s>
|
||
<|assistant|>
|
||
|
||
"""
|
||
|
||
response = sendReceivePrompt(a, prompt, max_tokens=1024, timeout=180)
|
||
|
||
return response
|
||
end
|
||
|
||
|
||
""" Write a lesson drawn from evaluation.
|
||
|
||
Arguments:
|
||
a, one of ChatAgent's agent.
|
||
report, a report resulted from analyzing shorttermMemory
|
||
|
||
Return:
|
||
A lesson.
|
||
|
||
Example:
|
||
|
||
```jldoctest
|
||
julia> using ChatAgent, CommUtils
|
||
julia> agent = ChatAgent.agentReflex("Jene")
|
||
julia> report =
|
||
"What happened: I tried to search for AMD's latest product using the wikisearch tool,
|
||
but no information was available in the search results.
|
||
Cause and effect relationships:
|
||
1. Searching \"AMD latest product\" -> No info available.
|
||
2. Searching \"most recent product release\" -> No info available.
|
||
3. Searching \"latest product\" -> No info available.
|
||
Analysis of each relationship:
|
||
1. The search for \"AMD latest product\" did not provide any information because the wikisearch tool could not find relevant results for that query.
|
||
2. The search for \"most recent product release\" also did not yield any results, indicating that there might be no recent product releases available or that the information is not accessible through the wikisearch tool.
|
||
3. The search for \"latest product\" similarly resulted in no information being found, suggesting that either the latest product is not listed on the encyclopedia or it is not easily identifiable using the wikisearch tool.
|
||
Improvements: To improve the response, I could try searching for AMD's products on a different
|
||
source or search engine to find the most recent product release. Additionally, I could ask
|
||
the user for more context or clarify their question to better understand what they are
|
||
looking for."
|
||
julia> lesson = selfReflext(agent, report)
|
||
```
|
||
"""
|
||
function selfReflext(a, analysis::T) where {T<:AbstractString}
|
||
prompt =
|
||
"""
|
||
<|system|>
|
||
<You have access to the following tools>
|
||
askbox: Useful for when you need to ask a customer for more context. Input should be a conversation to customer.
|
||
wikisearch: Useful for when you need to search an encyclopedia Input is keywords and not a question.
|
||
</You have access to the following tools>
|
||
<your report>
|
||
$analysis
|
||
</your report>
|
||
<your job are>
|
||
1. Lesson: what lesson could you learn from your report?.
|
||
2. Context: what is the context this lesson could apply to?
|
||
</your job are>
|
||
</s>
|
||
<|assistant|>
|
||
|
||
"""
|
||
|
||
response = sendReceivePrompt(a, prompt, max_tokens=1024)
|
||
return response
|
||
end
|
||
|
||
|
||
""" Formulate a response from work for user's stimulus.
|
||
|
||
Arguments:
|
||
a, one of ChatAgent's agent.
|
||
|
||
Return:
|
||
A response for user's stimulus.
|
||
|
||
Example:
|
||
```jldoctest
|
||
julia> using ChatAgent, CommUtils
|
||
julia> agent = ChatAgent.agentReflex("Jene")
|
||
julia> shorttermMemory = OrderedDict{String, Any}(
|
||
"user:" => "What's the latest AMD GPU?",
|
||
"Plan 1:" => " To answer this question, I will need to search for the latest AMD GPU using the wikisearch tool.\n",
|
||
"Act 1:" => " wikisearch\n",
|
||
"Actinput 1:" => " amd gpu latest\n",
|
||
"Obs 1:" => "No info available for your search query.",
|
||
"Act 2:" => " wikisearch\n",
|
||
"Actinput 2:" => " amd graphics card latest\n",
|
||
"Obs 2:" => "No info available for your search query.")
|
||
|
||
julia> report = formulateUserResponse(agent, shorttermMemory)
|
||
```
|
||
"""
|
||
function formulateUserResponse(a)
|
||
conversation = messagesToString_nomark(a.messages, addressAIas="I")
|
||
work = dictToString(a.memory[:shortterm])
|
||
|
||
prompt =
|
||
"""
|
||
<|system|>
|
||
<symbol meaning>
|
||
Plan: a plan
|
||
Thought: your thought
|
||
Act: the action you took
|
||
Actinput: the input to the action
|
||
Obs: the result of the action
|
||
</symbol meaning>
|
||
|
||
<your talk with the user>
|
||
$conversation
|
||
</your talk with the user>
|
||
|
||
<your earlier work>
|
||
$work
|
||
</your earlier work>
|
||
|
||
<your job>
|
||
Based on your talk with the user and your work, present a response that compares and justifies each option in great detail to the user.
|
||
</your job>
|
||
</s>
|
||
<|assistant|>
|
||
Recommendation:
|
||
"""
|
||
response = sendReceivePrompt(a, prompt, max_tokens=1024, timeout=300)
|
||
return response
|
||
end
|
||
|
||
# function formulateUserResponse(a)
|
||
# conversation = messagesToString_nomark(a.messages, addressAIas="I")
|
||
# work = dictToString(a.memory[:shortterm])
|
||
|
||
# prompt =
|
||
# """
|
||
# <|system|>
|
||
# Symbol:
|
||
# Plan: a plan
|
||
# Thought: your thought
|
||
# Act: the action you took
|
||
# Actinput: the input to the action
|
||
# Obs: the result of the action
|
||
|
||
# Your talk with the user:
|
||
# $conversation
|
||
|
||
# Your work:
|
||
# $work
|
||
|
||
# From your talk with the user and your work, formulate a response for the user .
|
||
# </s>
|
||
# <|assistant|>
|
||
# response:
|
||
# """
|
||
# response = sendReceivePrompt(a, prompt)
|
||
# return response
|
||
# end
|
||
|
||
|
||
""" Extract important info from text into key-value pair text.
|
||
|
||
Arguments:
|
||
a, one of ChatAgent's agent.
|
||
text, a text you want to extract info
|
||
|
||
Return:
|
||
key-value pair text.
|
||
|
||
Example:
|
||
```jldoctest
|
||
julia> using ChatAgent
|
||
julia> agent = ChatAgent.agentReflex("Jene")
|
||
julia> text = "We are holding a wedding party at the beach."
|
||
julia> extract(agent, text)
|
||
"location=beach, event=wedding party"
|
||
```
|
||
"""
|
||
function extractinfo(a, text::T) where {T<:AbstractString}
|
||
# determine whether there are any important info in an input text
|
||
prompt =
|
||
"""
|
||
<|system|>
|
||
<user's message>
|
||
$text
|
||
</user's message>
|
||
<your job>
|
||
Determine whether there are important info in the user's message. Answer: {Yes/No/Not sure}
|
||
</your job>
|
||
</s>
|
||
Answer:
|
||
"""
|
||
response = sendReceivePrompt(a, prompt, temperature=0.0)
|
||
if occursin("Yes", response)
|
||
prompt =
|
||
"""
|
||
<|system|>
|
||
<user's message>
|
||
$text
|
||
</user's message>
|
||
<your job>
|
||
Extract important info from the user's message into keys and values using this format: key=value,.
|
||
p.s.1 you can extract many key-value pairs.
|
||
</your job>
|
||
</s>
|
||
|
||
"""
|
||
|
||
response = sendReceivePrompt(a, prompt, temperature=0.0)
|
||
return response
|
||
else
|
||
return nothing
|
||
end
|
||
end
|
||
|
||
|
||
""" Update important info from key-value pair text into another key-value pair text.
|
||
|
||
Arguments:
|
||
a, one of ChatAgent's agent
|
||
text, a key-value pair text
|
||
|
||
Return:
|
||
updated key-value pair text
|
||
|
||
Example:
|
||
```jldoctest
|
||
julia> using ChatAgent
|
||
julia> agent = ChatAgent.agentReflex("Jene")
|
||
julia> currentinfo = "location=beach, event=wedding party"
|
||
julia> newinfo = "wine_type=full body, dry and medium tannin\nprice_range=50 dollars"
|
||
julia> updateEnvState(agent, currentinfo, newinfo)
|
||
" location=beach, event=wedding party, wine_type=full body, dry and medium tannin, price_range=50 dollars"
|
||
```
|
||
"""
|
||
function updateEnvState(a, newinfo)
|
||
prompt =
|
||
"""
|
||
<|im_start|>system
|
||
Current state:
|
||
$(a.env)
|
||
|
||
New info:
|
||
$newinfo
|
||
|
||
Your job is to update or add information from new info into the current state which use key-value format.
|
||
<|im_end|>
|
||
Updated Current State:\n
|
||
"""
|
||
|
||
response = sendReceivePrompt(a, prompt, temperature=0.0)
|
||
return response
|
||
end
|
||
|
||
|
||
|
||
""" Determine whether LLM should go to next task.
|
||
|
||
Arguments:
|
||
a, one of ChatAgent's agent.
|
||
|
||
Return:
|
||
"Yes" or "no" decision to go next task.
|
||
|
||
Example:
|
||
```jldoctest
|
||
julia> using ChatAgent, CommUtils
|
||
julia> agent = ChatAgent.agentReflex("Jene")
|
||
julia> shorttermMemory = OrderedDict{String, Any}(
|
||
"user:" => "What's the latest AMD GPU?",
|
||
"Plan 1:" => " To answer this question, I will need to search for the latest AMD GPU using the wikisearch tool.\n",
|
||
"Act 1:" => " wikisearch\n",
|
||
"Actinput 1:" => " amd gpu latest\n",
|
||
"Obs 1:" => "No info available for your search query.",
|
||
"Act 2:" => " wikisearch\n",
|
||
"Actinput 2:" => " amd graphics card latest\n",
|
||
"Obs 2:" => "No info available for your search query.")
|
||
|
||
julia> decision = checkTaskCompletion(agent)
|
||
"Yes"
|
||
```
|
||
"""
|
||
function checkTaskCompletion(a)
|
||
@show a.memory[:shortterm]["Plan 1:"]
|
||
# stimulus = a.memory[:shortterm]["user:"]
|
||
work = dictToString(a.memory[:shortterm])
|
||
|
||
prompt =
|
||
"""
|
||
<|system|>
|
||
<Symbol meaning>
|
||
Plan: a plan
|
||
Thought: your thought
|
||
Act: the action you took
|
||
Actinput: the input to the action
|
||
Obs: the result of the action
|
||
</Symbol meaning>
|
||
|
||
<Your earlier work>
|
||
$work
|
||
</Your earlier work>
|
||
<Your job>
|
||
Check whether each task of your plan has been completed.
|
||
</Your job>
|
||
|
||
<Example 1>
|
||
Task 1 of the plan: Ask user about their preferred topping of a pizza.
|
||
Obs: I love Malvasia.
|
||
assistant: After checking all my work's observed results, I can't find any relevant info that the user tell me what is their preferred topping in pizza. Thus, task 1 isn't done yet.
|
||
Task 2 of the plan: Ask user if they have any preferred type of car.
|
||
Obs: I like a semi truck.
|
||
assistant: After checking all my work's observed results, I found that the user like a semi truck. Thus, task 2 is done.
|
||
Task 3 of the plan: How much you are looking to spend for a new house?
|
||
Obs: 50K THB.
|
||
assistant: After checking all my work's observed results, I found that the user have a budget of 50,000 Baht. Thus, task 3 is done.
|
||
</Example 1>
|
||
|
||
Let's think step by step.
|
||
</s>
|
||
<|assistant|> After
|
||
"""
|
||
response = nothing
|
||
_response = nothing
|
||
_response = sendReceivePrompt(a, prompt, max_tokens=1024)
|
||
@show checkTaskCompletion_raw = _response
|
||
_response = split(_response, "</")[1]
|
||
_response = split(_response, "\n\n")[1]
|
||
# response = "I " * split(_response, "{")[1] # sometime response have more than 1 {answer: done}
|
||
|
||
decision = nothing
|
||
# if occursin("done", response)
|
||
# decision = true
|
||
# else
|
||
# decision = false
|
||
# end
|
||
|
||
return decision, response
|
||
end
|
||
|
||
|
||
function recap(a)
|
||
# stimulus = a.memory[:shortterm]["user:"]
|
||
getonlykeys = ["Actinput", "Obs"]
|
||
worknoplan = similar(a.memory[:shortterm])
|
||
for (k, v) in a.memory[:shortterm]
|
||
count = 0
|
||
for i in getonlykeys
|
||
if occursin(i, k)
|
||
count += 1
|
||
end
|
||
end
|
||
if count != 0
|
||
worknoplan[k] = v
|
||
end
|
||
end
|
||
|
||
work = dictToString(worknoplan)
|
||
|
||
toolnames = ""
|
||
toollines = ""
|
||
for (toolname, v) in a.tools
|
||
toolline = "$toolname: $(v[:description]) $(v[:input]) $(v[:output])\n"
|
||
toollines *= toolline
|
||
toolnames *= "$toolname, "
|
||
end
|
||
|
||
# prompt =
|
||
# """
|
||
# <|system|>
|
||
# <Symbol meaning>
|
||
# Plan: a plan
|
||
# Thought: your thought
|
||
# Act: the action you took
|
||
# Actinput: the input to the action
|
||
# Obs: the result of the action
|
||
# </Symbol meaning>
|
||
|
||
# <Your earlier work>
|
||
# $work
|
||
# </Your earlier work>
|
||
# <Your job>
|
||
# Recap: list all your observed results in detail
|
||
# </Your job>
|
||
|
||
# Let's think step by step.
|
||
# </s>
|
||
# <|assistant|>
|
||
# Recap:
|
||
# """
|
||
|
||
|
||
|
||
prompt =
|
||
"""
|
||
<|system|>
|
||
$(a.roles[a.role])
|
||
<Symbol meaning>
|
||
Plan: a plan
|
||
Thought: your thought
|
||
Act: the action you took
|
||
Actinput: the input to the action
|
||
Obs: the result of the action
|
||
</Symbol meaning>
|
||
<You have access to the following tools>
|
||
$toollines
|
||
</You have access to the following tools>
|
||
<Your earlier work>
|
||
$work
|
||
</Your earlier work>
|
||
<Your job>
|
||
Extract info: extract each info in details from your earlier work according to the Actinput context.
|
||
</Your job>
|
||
|
||
Let's think step by step.
|
||
</s>
|
||
<|assistant|>
|
||
Extracted info:
|
||
"""
|
||
aware = "Self-awareness: map the info from the recap to the plan's tasks then state your mapping."
|
||
response = sendReceivePrompt(a, prompt, max_tokens=1024, temperature=0.0)
|
||
response = split(response, "</")[1]
|
||
response = split(response, "<|")[1]
|
||
response = split(response, "\n\n")[1]
|
||
|
||
return response
|
||
end
|
||
|
||
""" Direct conversation is not an agent, messages does not pass through logic loop
|
||
but goes directly to LLM.
|
||
"""
|
||
function directconversation(a::agentReflex, usermsg::String)
|
||
response = nothing
|
||
|
||
_ = addNewMessage(a, "user", usermsg)
|
||
|
||
response = chat_mistral_openorca(a)
|
||
response = removeTrailingCharacters(response)
|
||
_ = addNewMessage(a, "assistant", response)
|
||
return response
|
||
end
|
||
|
||
|
||
""" Convert keyword memory into a string.
|
||
|
||
Arguments\n
|
||
a : one of ChatAgent's agent.
|
||
keywordmemory : a dictionary of keyword memory.
|
||
|
||
Return\n
|
||
result : a string of LLM readout from keyword memory
|
||
|
||
Example:
|
||
```jldoctest
|
||
julia> using ChatAgent, CommUtils
|
||
julia> a = ChatAgent.agentReflex("Jene")
|
||
julia> keywordmemory = OrderedDict{String, Any}(
|
||
"food type" => nothing,
|
||
"tannin level" => "low to medium",
|
||
"intensity level" => "medium-bodied",
|
||
"acidity level" => nothing,
|
||
"price range" => "fifteen dollars",
|
||
"wine type" => "Red",
|
||
"sweetness level" => "dry",
|
||
)
|
||
|
||
julia> readout = readKeywordMemory(a, keywordmemory=keywordmemory)
|
||
" - The user did not provide food type yet
|
||
- The user prefers a low to medium tannin level
|
||
- The user prefers a medium-bodied intensity level
|
||
- The user did not provide acidity level yet
|
||
- The user prefers price range is fifteen dollars
|
||
- The user prefers a Red wine type
|
||
- The user prefers a dry sweetness level"
|
||
```
|
||
"""
|
||
function readKeywordMemory(a; keywordmemory::Union{AbstractDict, Nothing}=nothing)
|
||
|
||
keywordmemory = keywordmemory !== nothing ? keywordmemory : a.memory[:keyword]
|
||
result = ""
|
||
|
||
if !isempty(keywordmemory)
|
||
new_keywordmemory = deepcopy(keywordmemory)
|
||
@show keywordmemory
|
||
|
||
|
||
# prepare reversed dict for pop! coz I need to preserve key order
|
||
reversed_keywordmemory = Dict()
|
||
while length(new_keywordmemory) > 0
|
||
k, v = pop!(new_keywordmemory)
|
||
reversed_keywordmemory[k] = v
|
||
end
|
||
|
||
while length(reversed_keywordmemory) > 0
|
||
tempdict = OrderedDict()
|
||
for i in 1:4
|
||
if length(reversed_keywordmemory) == 0
|
||
break
|
||
else
|
||
k, v = pop!(reversed_keywordmemory)
|
||
tempdict[k] = v
|
||
end
|
||
end
|
||
|
||
# ask LLM to read tempdict
|
||
jsonstr = JSON3.write(tempdict)
|
||
prompt =
|
||
"""
|
||
<s>
|
||
<|system|>
|
||
<About yourself>
|
||
Your name is $(a.agentName)
|
||
$(a.roles[a.role])
|
||
</About yourself>
|
||
<Your job>
|
||
Readout all the key and its value pairs in memory, one by one. Do not say anything else.
|
||
</Your job>
|
||
</|system|>
|
||
<Example 1>
|
||
<Memory>
|
||
{\"car type\": "SUV",\"brand\":\"Lexus\",\"price\":\"20k dollar\",\"color\": null,\"financing method\": null, \"luxury level\":\"high\"}
|
||
</Memory>
|
||
<|assistant|>
|
||
- Car type is SUV
|
||
- Brand is Lexus
|
||
- Price is 20k dollar
|
||
- No info on the car color yet
|
||
- No info on the financing method yet
|
||
- Luxury level is high
|
||
</|assistant|>
|
||
</Example 1>
|
||
</s>
|
||
<Memory>
|
||
User preference: $jsonstr
|
||
</Memory>
|
||
<|assistant|>
|
||
"""
|
||
|
||
response = sendReceivePrompt(a, prompt, max_tokens=512, temperature=0.0)
|
||
response = split(response, "</|assistant|>")[1]
|
||
|
||
# store LLM readout string to result
|
||
result = result * response
|
||
end
|
||
end
|
||
|
||
return result
|
||
end
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
end # module |