update

Manifest.toml

@@ -2,7 +2,31 @@
 
 julia_version = "1.12.6"
 manifest_format = "2.0"
-project_hash = "2fa6a293fb2eebf613585700dd4002877d4bff24"
+project_hash = "6e0efa362e5748de9ce219e0376be2f93d464376"
+
+[[deps.Accessors]]
+deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "MacroTools"]
+git-tree-sha1 = "7063ad1083578215c7c4bf410368150abe8d5524"
+uuid = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"
+version = "0.1.45"
+
+    [deps.Accessors.extensions]
+    AxisKeysExt = "AxisKeys"
+    IntervalSetsExt = "IntervalSets"
+    LinearAlgebraExt = "LinearAlgebra"
+    StaticArraysExt = "StaticArrays"
+    StructArraysExt = "StructArrays"
+    TestExt = "Test"
+    UnitfulExt = "Unitful"
+
+    [deps.Accessors.weakdeps]
+    AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5"
+    IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
+    LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+    StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
+    Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+    Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
 
 [[deps.AliasTables]]
 deps = ["PtrArrays", "Random"]
@@ -66,6 +90,11 @@ git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9"
 uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
 version = "0.7.8"
 
+[[deps.CommonSolve]]
+git-tree-sha1 = "99ee296f88c12485402e37c2fd025f95ae097637"
+uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
+version = "0.2.9"
+
 [[deps.Compat]]
 deps = ["TOML", "UUIDs"]
 git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad"
@@ -86,6 +115,30 @@ deps = ["Artifacts", "Libdl"]
 uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
 version = "1.3.0+1"
 
+[[deps.CompositionsBase]]
+git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad"
+uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b"
+version = "0.1.2"
+weakdeps = ["InverseFunctions"]
+
+    [deps.CompositionsBase.extensions]
+    CompositionsBaseInverseFunctionsExt = "InverseFunctions"
+
+[[deps.ConstructionBase]]
+git-tree-sha1 = "b4b092499347b18a015186eae3042f72267106cb"
+uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9"
+version = "1.6.0"
+
+    [deps.ConstructionBase.extensions]
+    ConstructionBaseIntervalSetsExt = "IntervalSets"
+    ConstructionBaseLinearAlgebraExt = "LinearAlgebra"
+    ConstructionBaseStaticArraysExt = "StaticArrays"
+
+    [deps.ConstructionBase.weakdeps]
+    IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953"
+    LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+    StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+
 [[deps.Crayons]]
 git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
 uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
@@ -134,10 +187,10 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 version = "1.11.0"
 
 [[deps.Distributions]]
-deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"]
+deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "Roots", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"]
-git-tree-sha1 = "3c8a0a9a6d4a10bdfb6b751bd2b6051ed3e25fd4"
+git-tree-sha1 = "cd3c5ac74cd3923c8945c6a81518c46abd0e73a3"
 uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
-version = "0.25.127"
+version = "0.25.129"
 
     [deps.Distributions.extensions]
     DistributionsChainRulesCoreExt = "ChainRulesCore"
@@ -217,17 +270,17 @@ version = "1.11.0"
 
 [[deps.GeneralUtils]]
 deps = ["CSV", "DataFrames", "DataStructures", "Dates", "Distributions", "JSON", "NATS", "PrettyPrinting", "Random", "Revise", "SHA", "UUIDs"]
-git-tree-sha1 = "8720a31344bc85ad610ae12f7e1247de22070765"
+git-tree-sha1 = "f1bad7621e6ac2d235adc8c593b0aff87bd1d93e"
 repo-rev = "main"
 repo-url = "https://git.yiem.cc/ton/GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.3.2"
+version = "0.4.2"
 
 [[deps.HTTP]]
 deps = ["Base64", "CodecZlib", "Dates", "EnumX", "PrecompileTools", "Random", "Reseau", "SHA", "URIs", "UUIDs", "Zlib_jll"]
-git-tree-sha1 = "a95f80749437ffb42948848d2d2ec81a5050ef4b"
+git-tree-sha1 = "69343dd8afb1671b84c3aa2dda511238d0919a55"
 uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-version = "2.4.0"
+version = "2.5.0"
 
 [[deps.HashArrayMappedTries]]
 git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae"
@@ -273,6 +326,16 @@ git-tree-sha1 = "d6fe00b123e32ddd17231b35d69a6394e696fd5a"
 uuid = "d8418881-c3e1-53bb-8760-2df7ec849ed5"
 version = "1.11.0"
 
+[[deps.InverseFunctions]]
+git-tree-sha1 = "a779299d77cd080bf77b97535acecd73e1c5e5cb"
+uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
+version = "0.1.17"
+weakdeps = ["Dates", "Test"]
+
+    [deps.InverseFunctions.extensions]
+    InverseFunctionsDatesExt = "Dates"
+    InverseFunctionsTestExt = "Test"
+
 [[deps.InvertedIndices]]
 git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe"
 uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f"
@@ -424,6 +487,11 @@ git-tree-sha1 = "3733419e9a71156b389f3e331672d2e95436783f"
 uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b"
 version = "3.6.2"
 
+[[deps.MacroTools]]
+git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522"
+uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+version = "0.5.16"
+
 [[deps.Markdown]]
 deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
@@ -522,9 +590,9 @@ version = "1.8.2"
 
 [[deps.PDMats]]
 deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
-git-tree-sha1 = "e4cff168707d441cd6bf3ff7e4832bdf34278e4a"
+git-tree-sha1 = "26766d4b5f1a410c218a19b85a672c6edb693c65"
 uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
-version = "0.11.37"
+version = "0.11.40"
 weakdeps = ["StatsBase"]
 
     [deps.PDMats.extensions]
@@ -657,12 +725,34 @@ git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8"
 uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
 version = "0.5.1+0"
 
+[[deps.Roots]]
+deps = ["Accessors", "CommonSolve", "Printf"]
+git-tree-sha1 = "91cfb1cb4f6e27557cc2df798a31eff6089a41eb"
+uuid = "f2b01f46-fcfa-551c-844a-d8ac1e96c665"
+version = "3.0.0"
+
+    [deps.Roots.extensions]
+    RootsChainRulesCoreExt = "ChainRulesCore"
+    RootsForwardDiffExt = "ForwardDiff"
+    RootsIntervalRootFindingExt = "IntervalRootFinding"
+    RootsSymPyExt = "SymPy"
+    RootsSymPyPythonCallExt = "SymPyPythonCall"
+    RootsUnitfulExt = "Unitful"
+
+    [deps.Roots.weakdeps]
+    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+    ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+    IntervalRootFinding = "d2bf35a9-74e0-55ec-b149-d360ff49b807"
+    SymPy = "24249f21-da20-56a4-8eb1-6a02cf4ae2e6"
+    SymPyPythonCall = "bc8888f7-b21e-4b7c-a06a-5d9c9496438c"
+    Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
+
 [[deps.SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 version = "0.7.0"
 
 [[deps.SQLLLM]]
-deps = ["CSV", "DataFrames", "DataStructures", "Dates", "FileIO", "HTTP", "JSON3", "LLMMCTS", "LibPQ", "PrettyPrinting", "Random", "Revise", "StatsBase", "Tables", "URIs", "UUIDs"]
+deps = ["CSV", "DataFrames", "DataStructures", "Dates", "FileIO", "GeneralUtils", "HTTP", "JSON", "LLMMCTS", "LibPQ", "PrettyPrinting", "Random", "Revise", "StatsBase", "Tables", "URIs", "UUIDs"]
 path = "."
 uuid = "2ebc79c7-cc10-4a3a-9665-d2e1d61e63d3"
 version = "0.2.4"
@@ -823,9 +913,9 @@ version = "1.0.1"
 
 [[deps.Tables]]
 deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"]
-git-tree-sha1 = "f2c1efbc8f3a609aadf318094f8fc5204bdaf344"
+git-tree-sha1 = "0f38a06c83f0007bbab3cf911262841c9a0f07e0"
 uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
-version = "1.12.1"
+version = "1.13.0"
 
 [[deps.Tar]]
 deps = ["ArgTools", "SHA"]

Project.toml

@@ -9,8 +9,9 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
+GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LLMMCTS = "d76c5a4d-449e-4835-8cc4-dd86ec44f241"
 LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
 PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
@@ -23,3 +24,5 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
 Dates = "1.11.0"
+GeneralUtils = "0.4.2"
+JSON = "1.6.1"

src/interface.jl

@@ -2,7 +2,7 @@ module interface
 
 export decisionMaker, evaluator, reflector, transition, query
 
-using LibPQ, DataStructures, JSON3, UUIDs, PrettyPrinting, Dates
+using LibPQ, DataStructures, JSON, UUIDs, PrettyPrinting, Dates
 using GeneralUtils, LLMMCTS
 using ..util, ..llmfunction
 
@@ -106,33 +106,6 @@ function decisionMaker(state::T1, additionalinfo, text2textInstructLLM::Function
   ; querySQLVectorDBF::Union{T2, Nothing}=nothing, maxattempt=10
   )::Dict{String, Any} where {T1<:AbstractDict, T2<:Function}
 
-  # lessonDict =
-  # if isfile("lesson.json")
-  #   lessonDict = copy(JSON3.read("lesson.json"))
-  # else
-  #   lessonDict = nothing
-  # end
-
-  # lessonDict = nothing
-
-  # lesson =
-  # if lessonDict === nothing
-  #   ""
-  # else
-  #   """
-  #   You have attempted to help the user before and failed, either because your reasoning for the 
-  #   recommendation was incorrect or your response did not exactly match the user expectation. 
-  #   The following lesson(s) give a plan to avoid failing to help the user in the same way you 
-  #   did previously. Use them to improve your strategy to help the user.
-
-  #   Here are some lessons in JSON format:
-  #   $(JSON3.write(lessonDict))
-
-  #   When providing the thought and action for the current trial, that into account these failed 
-  #   trajectories and make sure not to repeat the same mistakes and incorrect answers. 
-  #   """
-  # end
-
   systemmsg = 
   """
     You are a helpful assistant that find the data from a database to satisfy the user's question. 
@@ -239,7 +212,7 @@ $(state["suggestion"])
 
     responsedict = nothing
     try
-      responsedict = copy(JSON3.read(response))  
+      responsedict = copy(JSON.parse(response))  
     catch
       println("\nERROR YiemAgent generatechat() failed to parse response: $response", @__FILE__, ":", @__LINE__, " $(Dates.now())")
       continue
@@ -310,7 +283,7 @@ responsedict["thoughthistory"] = state["thoughtHistory"]
     responsedict["think"] = think
 
     # # read sessionId
-    # sessionid = JSON3.read("/appfolder/app/sessionid.json")
+    # sessionid = JSON.parse("/appfolder/app/sessionid.json")
     # # save to filename ./log/decisionlog.txt
     # println("saving SQLLLM decisionMaker() to disk")
     # filename = "agent_decision_log_$(sessionid[:id]).json"
@@ -324,7 +297,7 @@ responsedict["thoughthistory"] = state["thoughtHistory"]
     #   end
     # else
     #   # read the file and append new data
-    #   decisionlist = copy(JSON3.read(filepath))
+    #   decisionlist = copy(JSON.parse(filepath))
     #   push!(decisionlist, responsedict)
     #   println("Appending new data to file $filepath")
     #   open(filepath, "w") do io
@@ -343,7 +316,7 @@ end
 
 #   # lessonDict =
 #   # if isfile("lesson.json")
-#   #   lessonDict = copy(JSON3.read("lesson.json"))
+#   #   lessonDict = copy(JSON.parse("lesson.json"))
 #   # else
 #   #   lessonDict = nothing
 #   # end
@@ -754,7 +727,7 @@ function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFo
     # responsedict[:think] = think
 
     # # read sessionId
-    # sessionid = JSON3.read("/appfolder/app/sessionid.json")
+    # sessionid = JSON.parse("/appfolder/app/sessionid.json")
     # # save to filename ./log/decisionlog.txt
     # println("saving SQLLLM evaluator() to disk")
     # filename = "agent_evaluator_log_$(sessionid[:id]).json"
@@ -768,7 +741,7 @@ function evaluator(state::T1, thoughtDict, text2textInstructLLM::Function, llmFo
     #   end
     # else
     #   # read the file and append new data
-    #   decisionlist = copy(JSON3.read(filepath))
+    #   decisionlist = copy(JSON.parse(filepath))
     #   push!(decisionlist, responsedict)
     #   println("Appending new data to file $filepath")
     #   open(filepath, "w") do io
@@ -917,7 +890,7 @@ function reflector(config::T1, state::T2)::String where {T1<:AbstractDict, T2<:A
         error("Not valid JSON")
       end
 
-      reflectionDict = copy(JSON3.read(responseJsonStr)) 
+      reflectionDict = copy(JSON.parse(responseJsonStr)) 
 
       # check if dict has all required value
       dummya::AbstractString = reflectionDict[:reflection]

src/llmfunction.jl

@@ -3,7 +3,7 @@ module llmfunction
 export listAllTable_json, listAllTable_str, tableinfo, getdata, finalAnswerBox,
   getTableNameFromSQL, extractContent_dataframe, SQLexecution, compareState
 
-using HTTP, JSON3, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV,
+using HTTP, JSON, URIs, Random, PrettyPrinting, UUIDs, LibPQ, Tables, DataFrames, CSV,
   DataStructures, StatsBase, Dates
 using GeneralUtils, LLMMCTS
 using ..util
@@ -189,7 +189,7 @@ function tableinfo(executeSQL::Function, tablenames::T
     end
   end
   if !isempty(notExistingTable)
-    result = "Error, the following tables does not exist in the database: $(JSON3.write(notExistingTable))"
+    result = "Error, the following tables does not exist in the database: $(JSON.json(notExistingTable))"
     return (result=result,)
   end
 
@@ -779,7 +779,7 @@ function getTableNameFromSQL(sql::T, text2textInstructLLM::Function,
       response = GeneralUtils.deFormatLLMtext(response, llmFormatName)
       responsedict = GeneralUtils.textToDict(response, header; 
                                             dictKey=dictkey, symbolkey=false)
-      response = copy(JSON3.read(responsedict["table_name"]))
+      response = copy(JSON.parse(responsedict["table_name"]))
 
       return response
     catch e

test/runtests.jl

@@ -1,9 +1,9 @@
 using Revise
-using LibPQ, Dates, JSON3, PrettyPrinting, UUIDs, DataFrames, DataStructures, Base64
+using LibPQ, Dates, JSON, PrettyPrinting, UUIDs, DataFrames, DataStructures, Base64
 using GeneralUtils, SQLLLM
 
 
-config = JSON3.read("/appfolder/app/dev/YiemAgent/test/config.json")
+config = JSON.parse("/appfolder/app/dev/YiemAgent/test/config.json")
 
 function executeSQL(sql::T) where {T<:AbstractString}
   host = config[:externalservice][:wineDB][:host]
@@ -173,7 +173,7 @@ function similarSommelierDecision(recentevents::T1; maxdistance::Integer=3
     println("\n~~~ found similar decision. row id $rowid, distance $distance ", @__FILE__, " ", @__LINE__)
     output_b64 = df[1, :function_output_base64]   # pick the closest match
     _output_str = String(base64decode(output_b64))
-    output = copy(JSON3.read(_output_str))
+    output = copy(JSON.parse(_output_str))
     return output
   else
     println("\n~~~ similar decision not found, max distance $maxdistance ", @__FILE__, " ", @__LINE__)
@@ -193,7 +193,7 @@ function insertSommelierDecision(recentevents::T1, decision::T2; maxdistance::In
   if row == 0 || distance > maxdistance # no close enough SQL stored in the database
     recentevents_embedding = getEmbedding(recentevents)[1]
     recentevents = replace(recentevents, "'" => "")
-    decision_json = JSON3.write(decision)
+    decision_json = JSON.json(decision)
     decision_base64 = base64encode(decision_json)
     decision = replace(decision_json, "'" => "")
     
@@ -212,14 +212,14 @@ sessionId = GeneralUtils.uuid4snakecase()
 d = Dict(:id => sessionId)
 filepath = "/appfolder/app/sessionid.json"
 open(filepath, "w") do io
-  JSON3.pretty(io, d)
+  JSON.pretty(io, d)
 end
 
 
 
 
 # query = "How many German wines do you have?"
-# highValueStateList = copy(JSON3.read("/appfolder/app/highValueState_1.json"))
+# highValueStateList = copy(JSON.parse("/appfolder/app/highValueState_1.json"))
 # selectedState = SQLLLM.compareState(query, highValueStateList, text2textInstructLLM)
 
 

test/test_1.jl

@@ -1,5 +1,5 @@
 using Revise
-using LibPQ, JSON3, PrettyPrinting, UUIDs, DataFrames, DataStructures, Dates, MQTTClient, Random
+using LibPQ, JSON, PrettyPrinting, UUIDs, DataFrames, DataStructures, Dates, MQTTClient, Random
 using SQLLLM, GeneralUtils
 
 
@@ -39,7 +39,7 @@ df = DataFrame(result)
 
 
 
-config = copy(JSON3.read("config.json"))
+config = copy(JSON.parse("config.json"))
 
 msgMeta = GeneralUtils.generate_msgMeta(
     config[:externalservice][:text2textinstruct][:mqtttopic];

test/test_2.jl

@@ -1,8 +1,8 @@
 # using Revise
-# using SQLLLM, LLMMCTS, DataStructures, JSON3
+# using SQLLLM, LLMMCTS, DataStructures, JSON
 
 # query = "How many German wines do you have?"
-# highValueStateList = copy(JSON3.read("/appfolder/app/highValueState_1.json"))
+# highValueStateList = copy(JSON.parse("/appfolder/app/highValueState_1.json"))
 # selectedState = SQLLLM.compareState(query, highValueStateList)