update
This commit is contained in:
+88
-78
@@ -4,7 +4,7 @@ export listAllTable_json, listAllTable_str, tableinfo, getdata, finalAnswerBox,
|
||||
getTableNameFromSQL, extractContent_dataframe, SQLexecution, compareState
|
||||
|
||||
using HTTP, JSON3, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV,
|
||||
DataStructures, StatsBase
|
||||
DataStructures, StatsBase, Dates
|
||||
using GeneralUtils, LLMMCTS
|
||||
using ..util
|
||||
|
||||
@@ -812,8 +812,8 @@ julia>
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function compareState(query, highValueStateList)
|
||||
|
||||
function compareState(question::String, highValueStateList, text2textInstructLLM::Function)
|
||||
println(typeof(highValueStateList))
|
||||
systemmsg =
|
||||
"""
|
||||
<Your profile>
|
||||
@@ -825,14 +825,13 @@ function compareState(query, highValueStateList)
|
||||
- Identify and select the most accurate and relevant response from these multiple results for the user
|
||||
</Your mission>
|
||||
<At each round of conversation, you will be given the following>
|
||||
- The user's question
|
||||
- The user's attempted actions and their corresponding results
|
||||
Question: the question the user is trying to answer
|
||||
Attempt: the user's attempted actions and their corresponding results
|
||||
</At each round of conversation, you will be given the following>
|
||||
<You should then respond to the user with the following>
|
||||
Comparison: a comparison of the results from each attempt
|
||||
Comparison: a comparison of all results from all attempts
|
||||
Rationale: a brief explanation of why the selected response is the most accurate and relevant
|
||||
Selected_response_number: the number the selected response in the list of results
|
||||
|
||||
Selected_response_number: the number the selected response in the list of results (e.g., 1, 2, 3, ...)
|
||||
</You should then respond to the user with the following>
|
||||
<You should only respond in format as described below>
|
||||
Comparison: ...
|
||||
@@ -855,89 +854,100 @@ function compareState(query, highValueStateList)
|
||||
Let's begin!
|
||||
"""
|
||||
|
||||
# thoughthistory = ""
|
||||
# for (k, v) in state[:thoughtHistory]
|
||||
# thoughthistory *= "$k: $v\n"
|
||||
# end
|
||||
potentialSolution = []
|
||||
keys = [:action_input, :observation]
|
||||
# extract the last action_name, action_input, observation of each state in highValueStateList and store them in a dictionary then push into potentialSolution
|
||||
for state in highValueStateList
|
||||
thoughtHistory = state[:thoughtHistory]
|
||||
_, currentstate_latestIndice =
|
||||
GeneralUtils.findHighestIndexKey(thoughtHistory, keys[1])
|
||||
latestKeys = makekey.(keys, currentstate_latestIndice)
|
||||
d = Dict()
|
||||
# get the last action_name, action_input, observation of currentstate
|
||||
for (i,v) in enumerate(keys)
|
||||
d[v] = thoughtHistory[latestKeys[i]]
|
||||
end
|
||||
push!(potentialSolution, d)
|
||||
end
|
||||
|
||||
# errornote = ""
|
||||
"""
|
||||
# put potential solutions from potentialSolution into the following form
|
||||
Attempt 1
|
||||
action_name:
|
||||
action_input:
|
||||
observation:
|
||||
Attempt 2
|
||||
action_name:
|
||||
action_input:
|
||||
observation:
|
||||
...
|
||||
"""
|
||||
potentialSolutionStr = ""
|
||||
for (i, state) in enumerate(potentialSolution)
|
||||
potentialSolutionStr *= "Attempt $i\n"
|
||||
for k in keys
|
||||
potentialSolutionStr *= "$k: $(state[k])\n"
|
||||
println("")
|
||||
end
|
||||
end
|
||||
|
||||
# for attempt in 1:10
|
||||
# errorFlag = false
|
||||
errornote = ""
|
||||
|
||||
# usermsg =
|
||||
# """
|
||||
# Trajectory: $thoughthistory
|
||||
# Error_note: $errornote
|
||||
# """
|
||||
for attempt in 1:10
|
||||
errorFlag = false
|
||||
|
||||
# _prompt =
|
||||
# [
|
||||
# Dict(:name=> "system", :text=> systemmsg),
|
||||
# Dict(:name=> "user", :text=> usermsg)
|
||||
# ]
|
||||
usermsg =
|
||||
"""
|
||||
Question: $question
|
||||
Attempts: $potentialSolutionStr
|
||||
"""
|
||||
|
||||
# # put in model format
|
||||
# prompt = GeneralUtils.formatLLMtext(_prompt; formatname="qwen")
|
||||
_prompt =
|
||||
[
|
||||
Dict(:name=> "system", :text=> systemmsg),
|
||||
Dict(:name=> "user", :text=> usermsg)
|
||||
]
|
||||
|
||||
# header = ["Trajectory_evaluation:", "Answer_evaluation:", "Accepted_as_answer:", "Score:", "Suggestion:"]
|
||||
# dictkey = ["trajectory_evaluation", "answer_evaluation", "accepted_as_answer", "score", "suggestion"]
|
||||
# put in model format
|
||||
prompt = GeneralUtils.formatLLMtext(_prompt; formatname="qwen")
|
||||
|
||||
# response = text2textInstructLLM(prompt)
|
||||
header = ["Comparison:", "Rationale:", "Selected_response_number:"]
|
||||
dictkey = ["comparison", "rationale", "selected_response_number"]
|
||||
|
||||
# # sometime LLM output something like **Comprehension**: which is not expected
|
||||
# response = replace(response, "**"=>"")
|
||||
# response = replace(response, "***"=>"")
|
||||
response = text2textInstructLLM(prompt)
|
||||
|
||||
# # make sure every header is in the response
|
||||
# for i in header
|
||||
# detected = GeneralUtils.detect_keyword(i, response)
|
||||
# if detected === nothing
|
||||
# errornote = "Your previous response didn't provide $i"
|
||||
# errorFlag = true
|
||||
# end
|
||||
# end
|
||||
# if errorFlag
|
||||
# continue # skip to the next iteration
|
||||
# end
|
||||
# sometime LLM output something like **Comprehension**: which is not expected
|
||||
response = replace(response, "**"=>"")
|
||||
response = replace(response, "***"=>"")
|
||||
|
||||
# responsedict = GeneralUtils.textToDict(response, header;
|
||||
# dictKey=dictkey, symbolkey=true)
|
||||
# make sure every header is in the response
|
||||
for i in header
|
||||
detected = GeneralUtils.detect_keyword(i, response)
|
||||
if detected === nothing
|
||||
errornote = "Your previous response didn't provide $i"
|
||||
errorFlag = true
|
||||
end
|
||||
end
|
||||
if errorFlag
|
||||
continue # skip to the next iteration
|
||||
end
|
||||
|
||||
responsedict = GeneralUtils.textToDict(response, header;
|
||||
dictKey=dictkey, symbolkey=true)
|
||||
|
||||
# responsedict[:score] = responsedict[:score][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
|
||||
# try
|
||||
# responsedict[:score] = parse(Int, responsedict[:score]) # convert string "5" into integer 5
|
||||
# catch
|
||||
# continue
|
||||
# end
|
||||
responsedict[:selected_response_number] = responsedict[:selected_response_number][1] # some time "6\nThe trajectories are incomplete" is generated but I only need the number.
|
||||
try
|
||||
responsedict[:selected_response_number] = parse(Int, responsedict[:selected_response_number]) # convert string "5" into integer 5
|
||||
catch
|
||||
continue
|
||||
end
|
||||
|
||||
# accepted_as_answer::AbstractString = responsedict[:accepted_as_answer]
|
||||
println("\n~~~ compareState() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
pprintln(Dict(responsedict))
|
||||
|
||||
# if accepted_as_answer ∉ ["Yes", "No"] # [PENDING] add errornote into the prompt
|
||||
# error("generated accepted_as_answer has wrong format")
|
||||
# end
|
||||
|
||||
# # add to state here instead to in transition() because the latter causes julia extension crash (a bug in julia extension)
|
||||
# state[:evaluation] = "$(responsedict[:trajectory_evaluation]) $(responsedict[:answer_evaluation])"
|
||||
# state[:evaluationscore] = responsedict[:score]
|
||||
# state[:accepted_as_answer] = responsedict[:accepted_as_answer]
|
||||
# state[:suggestion] = responsedict[:suggestion]
|
||||
|
||||
# # mark as terminal state when the answer is achieved
|
||||
# if accepted_as_answer == "Yes"
|
||||
|
||||
# # mark the state as terminal state because the evaluation say so.
|
||||
# state[:isterminal] = true
|
||||
|
||||
# # evaluation score as reward because different answers hold different value for the user.
|
||||
# state[:reward] = responsedict[:score]
|
||||
# end
|
||||
# println("\n~~~ Evaluator() ", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
# pprintln(Dict(responsedict))
|
||||
|
||||
# return responsedict[:score]
|
||||
# end
|
||||
# error("Evaluator failed to generate an evaluation, Response: \n$response\n<|End of error|>")
|
||||
return responsedict[:selected_response_number]
|
||||
end
|
||||
error("compareState failed to generate an evaluation, Response: \n$response\n<|End of error|>", @__FILE__, ":", @__LINE__, " $(Dates.now())")
|
||||
end
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user