ton/SQLLLM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

update

Browse Source
This commit is contained in:
narawat
2026-06-29 20:55:52 +07:00
parent bc81033924
commit c085a445e2
8 changed files with 137 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Manifest.toml
+103 -13
View File
@@ -2,7 +2,31 @@
julia_version = "1.12.6"
manifest_format = "2.0"
project_hash = "2fa6a293fb2eebf613585700dd4002877d4bff24"
project_hash = "6e0efa362e5748de9ce219e0376be2f93d464376"
[[deps.Accessors]]
deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"]
git-tree-sha1 = "7063ad1083578215c7c4bf410368150abe8d5524"
uuid = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"
version = "0.1.45"
[deps.Accessors.extensions]
AxisKeysExt = "AxisKeys"
IntervalSetsExt = "IntervalSets"
LinearAlgebraExt = "LinearAlgebra"
StaticArraysExt = "StaticArrays"
StructArraysExt = "StructArrays"
TestExt = "Test"
UnitfulExt = "Unitful"
[deps.Accessors.weakdeps]
AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5"
IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
[[deps.AliasTables]]
deps = ["PtrArrays", "Random"]
@@ -66,6 +90,11 @@ git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9"
uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
version = "0.7.8"
[[deps.CommonSolve]]
git-tree-sha1 = "99ee296f88c12485402e37c2fd025f95ae097637"
uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
version = "0.2.9"
[[deps.Compat]]
deps = ["TOML", "UUIDs"]
git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad"
@@ -86,6 +115,30 @@ deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
version = "1.3.0+1"
[[deps.CompositionsBase]]
git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad"
uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b"
version = "0.1.2"
weakdeps = ["InverseFunctions"]
[deps.CompositionsBase.extensions]
CompositionsBaseInverseFunctionsExt = "InverseFunctions"
[[deps.ConstructionBase]]
git-tree-sha1 = "b4b092499347b18a015186eae3042f72267106cb"
uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9"
version = "1.6.0"
[deps.ConstructionBase.extensions]
ConstructionBaseIntervalSetsExt = "IntervalSets"
ConstructionBaseLinearAlgebraExt = "LinearAlgebra"
ConstructionBaseStaticArraysExt = "StaticArrays"
[deps.ConstructionBase.weakdeps]
IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
[[deps.Crayons]]
git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
@@ -134,10 +187,10 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
version = "1.11.0"
[[deps.Distributions]]
deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"]
git-tree-sha1 = "3c8a0a9a6d4a10bdfb6b751bd2b6051ed3e25fd4"
deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "Roots", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"]
git-tree-sha1 = "cd3c5ac74cd3923c8945c6a81518c46abd0e73a3"
uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
version = "0.25.127"
version = "0.25.129"
[deps.Distributions.extensions]
DistributionsChainRulesCoreExt = "ChainRulesCore"
@@ -217,17 +270,17 @@ version = "1.11.0"
[[deps.GeneralUtils]]
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"]
git-tree-sha1 = "8720a31344bc85ad610ae12f7e1247de22070765"
git-tree-sha1 = "f1bad7621e6ac2d235adc8c593b0aff87bd1d93e"
repo-rev = "main"
repo-url = "https://git.yiem.cc/ton/GeneralUtils"
uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
version = "0.3.2"
version = "0.4.2"
[[deps.HTTP]]
deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"]
git-tree-sha1 = "a95f80749437ffb42948848d2d2ec81a5050ef4b"
git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
version = "2.4.0"
version = "2.5.0"
[[deps.HashArrayMappedTries]]
git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae"
@@ -273,6 +326,16 @@ git-tree-sha1 = "d6fe00b123e32ddd17231b35d69a6394e696fd5a"
uuid = "d8418881-c3e1-53bb-8760-2df7ec849ed5"
version = "1.11.0"
[[deps.InverseFunctions]]
git-tree-sha1 = "a779299d77cd080bf77b97535acecd73e1c5e5cb"
uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
version = "0.1.17"
weakdeps = ["Dates", "Test"]
[deps.InverseFunctions.extensions]
InverseFunctionsDatesExt = "Dates"
InverseFunctionsTestExt = "Test"
[[deps.InvertedIndices]]
git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe"
uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f"
@@ -424,6 +487,11 @@ git-tree-sha1 = "3733419e9a71156b389f3e331672d2e95436783f"
uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b"
version = "3.6.2"
[[deps.MacroTools]]
git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.16"
[[deps.Markdown]]
deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
@@ -522,9 +590,9 @@ version = "1.8.2"
[[deps.PDMats]]
deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
git-tree-sha1 = "e4cff168707d441cd6bf3ff7e4832bdf34278e4a"
git-tree-sha1 = "26766d4b5f1a410c218a19b85a672c6edb693c65"
uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
version = "0.11.37"
version = "0.11.40"
weakdeps = ["StatsBase"]
[deps.PDMats.extensions]
@@ -657,12 +725,34 @@ git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8"
uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
version = "0.5.1+0"
[[deps.Roots]]
deps = ["Accessors", "CommonSolve", "Printf"]
git-tree-sha1 = "91cfb1cb4f6e27557cc2df798a31eff6089a41eb"
uuid = "f2b01f46-fcfa-551c-844a-d8ac1e96c665"
version = "3.0.0"
[deps.Roots.extensions]
RootsChainRulesCoreExt = "ChainRulesCore"
RootsForwardDiffExt = "ForwardDiff"
RootsIntervalRootFindingExt = "IntervalRootFinding"
RootsSymPyExt = "SymPy"
RootsSymPyPythonCallExt = "SymPyPythonCall"
RootsUnitfulExt = "Unitful"
[deps.Roots.weakdeps]
ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
IntervalRootFinding = "d2bf35a9-74e0-55ec-b149-d360ff49b807"
SymPy = "24249f21-da20-56a4-8eb1-6a02cf4ae2e6"
SymPyPythonCall = "bc8888f7-b21e-4b7c-a06a-5d9c9496438c"
Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0"
[[deps.SQLLLM]]
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "FileIO", "HTTP", "JSON3", "LLMMCTS", "LibPQ", "PrettyPrinting", "Random", "Revise", "StatsBase", "Tables", "URIs", "UUIDs"]
deps = ["CSV", "DataFrames", "DataStructures", "Dates", "FileIO", "GeneralUtils", "HTTP", "JSON", "LLMMCTS", "LibPQ", "PrettyPrinting", "Random", "Revise", "StatsBase", "Tables", "URIs", "UUIDs"]
path = "."
uuid = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3"
version = "0.2.4"
@@ -823,9 +913,9 @@ version = "1.0.1"
[[deps.Tables]]
deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"]
git-tree-sha1 = "f2c1efbc8f3a609aadf318094f8fc5204bdaf344"
git-tree-sha1 = "0f38a06c83f0007bbab3cf911262841c9a0f07e0"
uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
version = "1.12.1"
version = "1.13.0"
[[deps.Tar]]
deps = ["ArgTools", "SHA"]
Project.toml
+4 -1
View File
@@ -9,8 +9,9 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
LLMMCTS = "d76c5a4d-449e-4835-8cc4-dd86ec44f241"
LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
@@ -23,3 +24,5 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
[compat]
Dates = "1.11.0"
GeneralUtils = "0.4.2"
JSON = "1.6.1"
README.md
View File
src/interface.jl
+8 -35
View File
@@ -2,7 +2,7 @@ module interface
export decisionMaker, evaluator, reflector, transition, query
using LibPQ, DataStructures, JSON3, UUIDs, PrettyPrinting, Dates
using LibPQ, DataStructures, JSON, UUIDs, PrettyPrinting, Dates
using GeneralUtils, LLMMCTS
using ..util, ..llmfunction
@@ -106,33 +106,6 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt=10
)::Dict{String, Any} where {T1<:AbstractDict, T2<:Function}
# lessonDict =
# if isfile("lesson.json")
# lessonDict = copy(JSON3.read("lesson.json"))
# else
# lessonDict = nothing
# end
# lessonDict = nothing
# lesson =
# if lessonDict === nothing
# ""
# else
# """
# You have attempted to help the user before and failed, either because your reasoning for the
# recommendation was incorrect or your response did not exactly match the user expectation.
# The following lesson(s) give a plan to avoid failing to help the user in the same way you
# did previously. Use them to improve your strategy to help the user.
# Here are some lessons in JSON format:
# $(JSON3.write(lessonDict))
# When providing the thought and action for the current trial, that into account these failed
# trajectories and make sure not to repeat the same mistakes and incorrect answers.
# """
# end
systemmsg =
"""
You are a helpful assistant that find the data from a database to satisfy the user's question.
@@ -239,7 +212,7 @@ $(state["suggestion"])
responsedict = nothing
try
responsedict = copy(JSON3.read(response))
responsedict = copy(JSON.parse(response))
catch
println("\nERROR YiemAgent generatechat() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
continue
@@ -310,7 +283,7 @@ responsedict["thoughthistory"] = state["thoughtHistory"]
responsedict["think"] = think
# # read sessionId
# sessionid = JSON3.read("/appfolder/app/sessionid.json")
# sessionid = JSON.parse("/appfolder/app/sessionid.json")
# # save to filename ./log/decisionlog.txt
# println("saving SQLLLM decisionMaker() to disk")
# filename = "agent_decision_log_$(sessionid[:id]).json"
@@ -324,7 +297,7 @@ responsedict["thoughthistory"] = state["thoughtHistory"]
# end
# else
# # read the file and append new data
# decisionlist = copy(JSON3.read(filepath))
# decisionlist = copy(JSON.parse(filepath))
# push!(decisionlist, responsedict)
# println("Appending new data to file $filepath")
# open(filepath, "w") do io
@@ -343,7 +316,7 @@ end
# # lessonDict =
# # if isfile("lesson.json")
# # lessonDict = copy(JSON3.read("lesson.json"))
# # lessonDict = copy(JSON.parse("lesson.json"))
# # else
# # lessonDict = nothing
# # end
@@ -754,7 +727,7 @@ function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFo
# responsedict[:think] = think
# # read sessionId
# sessionid = JSON3.read("/appfolder/app/sessionid.json")
# sessionid = JSON.parse("/appfolder/app/sessionid.json")
# # save to filename ./log/decisionlog.txt
# println("saving SQLLLM evaluator() to disk")
# filename = "agent_evaluator_log_$(sessionid[:id]).json"
@@ -768,7 +741,7 @@ function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFo
# end
# else
# # read the file and append new data
# decisionlist = copy(JSON3.read(filepath))
# decisionlist = copy(JSON.parse(filepath))
# push!(decisionlist, responsedict)
# println("Appending new data to file $filepath")
# open(filepath, "w") do io
@@ -917,7 +890,7 @@ function reflector(config::T1, state::T2)::String where {T1<:AbstractDict, T2<:A
error("Not valid JSON")
end
reflectionDict = copy(JSON3.read(responseJsonStr))
reflectionDict = copy(JSON.parse(responseJsonStr))
# check if dict has all required value
dummya::AbstractString = reflectionDict[:reflection]
src/llmfunction.jl
+3 -3
View File
@@ -3,7 +3,7 @@ module llmfunction
export listAllTable_json, listAllTable_str, tableinfo, getdata, finalAnswerBox,
getTableNameFromSQL, extractContent_dataframe, SQLexecution, compareState
using HTTP, JSON3, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV,
using HTTP, JSON, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV,
DataStructures, StatsBase, Dates
using GeneralUtils, LLMMCTS
using ..util
@@ -189,7 +189,7 @@ function tableinfo(executeSQL::Function, tablenames::T
end
end
if !isempty(notExistingTable)
result = "Error, the following tables does not exist in the database: $(JSON3.write(notExistingTable))"
result = "Error, the following tables does not exist in the database: $(JSON.json(notExistingTable))"
return (result=result,)
end
@@ -779,7 +779,7 @@ function getTableNameFromSQL(sql::T, text2textInstructLLM::Function,
response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
responsedict = GeneralUtils.textToDict(response, header;
dictKey=dictkey, symbolkey=false)
response = copy(JSON3.read(responsedict["table_name"]))
response = copy(JSON.parse(responsedict["table_name"]))
return response
catch e
test/runtests.jl
+6 -6
View File
@@ -1,9 +1,9 @@
using Revise
using LibPQ, Dates, JSON3, PrettyPrinting, UUIDs, DataFrames, DataStructures, Base64
using LibPQ, Dates, JSON, PrettyPrinting, UUIDs, DataFrames, DataStructures, Base64
using GeneralUtils, SQLLLM
config = JSON3.read("/appfolder/app/dev/YiemAgent/test/config.json")
config = JSON.parse("/appfolder/app/dev/YiemAgent/test/config.json")
function executeSQL(sql::T) where {T<:AbstractString}
host = config[:externalservice][:wineDB][:host]
@@ -173,7 +173,7 @@ function similarSommelierDecision(recentevents::T1; maxdistance::Integer=3
println("\n~~~ found similar decision. row id $rowid, distance $distance ", @__FILE__, " ", @__LINE__)
output_b64 = df[1, :function_output_base64] # pick the closest match
_output_str = String(base64decode(output_b64))
output = copy(JSON3.read(_output_str))
output = copy(JSON.parse(_output_str))
return output
else
println("\n~~~ similar decision not found, max distance $maxdistance ", @__FILE__, " ", @__LINE__)
@@ -193,7 +193,7 @@ function insertSommelierDecision(recentevents::T1, decision::T2; maxdistance::In
if row == 0 || distance > maxdistance # no close enough SQL stored in the database
recentevents_embedding = getEmbedding(recentevents)[1]
recentevents = replace(recentevents, "'" => "")
decision_json = JSON3.write(decision)
decision_json = JSON.json(decision)
decision_base64 = base64encode(decision_json)
decision = replace(decision_json, "'" => "")
@@ -212,14 +212,14 @@ sessionId = GeneralUtils.uuid4snakecase()
d = Dict(:id => sessionId)
filepath = "/appfolder/app/sessionid.json"
open(filepath, "w") do io
JSON3.pretty(io, d)
JSON.pretty(io, d)
end
# query = "How many German wines do you have?"
# highValueStateList = copy(JSON3.read("/appfolder/app/highValueState_1.json"))
# highValueStateList = copy(JSON.parse("/appfolder/app/highValueState_1.json"))
# selectedState = SQLLLM.compareState(query, highValueStateList, text2textInstructLLM)
test/test_1.jl
+2 -2
View File
@@ -1,5 +1,5 @@
using Revise
using LibPQ, JSON3, PrettyPrinting, UUIDs, DataFrames, DataStructures, Dates, MQTTClient, Random
using LibPQ, JSON, PrettyPrinting, UUIDs, DataFrames, DataStructures, Dates, MQTTClient, Random
using SQLLLM, GeneralUtils
@@ -39,7 +39,7 @@ df = DataFrame(result)
config = copy(JSON3.read("config.json"))
config = copy(JSON.parse("config.json"))
msgMeta = GeneralUtils.generate_msgMeta(
config[:externalservice][:text2textinstruct][:mqtttopic];
test/test_2.jl
+2 -2
View File
@@ -1,8 +1,8 @@
# using Revise
# using SQLLLM, LLMMCTS, DataStructures, JSON3
# using SQLLLM, LLMMCTS, DataStructures, JSON
# query = "How many German wines do you have?"
# highValueStateList = copy(JSON3.read("/appfolder/app/highValueState_1.json"))
# highValueStateList = copy(JSON.parse("/appfolder/app/highValueState_1.json"))
# selectedState = SQLLLM.compareState(query, highValueStateList)