update
This commit is contained in:
@@ -0,0 +1,355 @@
|
||||
|
||||
|
||||
using LibPQ
|
||||
using DataFrames
|
||||
|
||||
"""
|
||||
extract_vector_metadata(pg_conn_str::String) -> DataFrame
|
||||
|
||||
Queries PostgreSQL system catalogs to extract a rich semantic text map of every
|
||||
column in the database. Returns a DataFrame designed for vector embedding generation.
|
||||
"""
|
||||
function extract_vector_metadata(pg_conn_str::String)
|
||||
conn = LibPQ.Connection(pg_conn_str)
|
||||
|
||||
# This direct SQL query pulls the column specifications along with column-level descriptions
|
||||
query = """
|
||||
SELECT
|
||||
c.relname AS table_name,
|
||||
a.attname AS column_name,
|
||||
format_type(a.atttypid, a.atttypmod) AS data_type,
|
||||
COALESCE(d.description, '') AS column_description,
|
||||
CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key,
|
||||
CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key
|
||||
FROM pg_attribute a
|
||||
JOIN pg_class c ON c.oid = a.attrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
-- Join to fetch column comments/descriptions
|
||||
LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
|
||||
-- Check if column is part of a Primary Key
|
||||
LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid
|
||||
AND pk.contype = 'p'
|
||||
AND a.attnum = ANY(pk.conkey)
|
||||
-- Check if column is part of a Foreign Key
|
||||
LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid
|
||||
AND fk.contype = 'f'
|
||||
AND a.attnum = ANY(fk.conkey)
|
||||
WHERE
|
||||
n.nspname = 'public' -- Only user schemas
|
||||
AND c.relkind = 'r' -- Only standard tables
|
||||
AND a.attnum > 0 -- Skip system hidden columns
|
||||
AND NOT a.attisdropped; -- Skip dropped columns
|
||||
"""
|
||||
|
||||
try
|
||||
# Execute and format into a clean DataFrame
|
||||
result = execute(conn, query)
|
||||
df = DataFrame(result)
|
||||
|
||||
# Create a unique document ID for each vector row
|
||||
df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)]
|
||||
|
||||
return df
|
||||
finally
|
||||
close(conn)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
"""
|
||||
generate_embedding_payloads(df::DataFrame) -> Vector{Dict}
|
||||
|
||||
Transforms the metadata DataFrame into structured text strings optimal for
|
||||
vector space mapping.
|
||||
"""
|
||||
function generate_embedding_payloads(df::DataFrame)
|
||||
payloads = Dict[]
|
||||
|
||||
for row in eachrow(df)
|
||||
# 1. Build a rich text description summarizing the column's role
|
||||
text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)"
|
||||
|
||||
if row.is_primary_key
|
||||
text_payload *= " [PRIMARY KEY]"
|
||||
end
|
||||
if row.is_foreign_key
|
||||
text_payload *= " [FOREIGN KEY RELATIONAL LINK]"
|
||||
end
|
||||
|
||||
# Append business descriptions if they exist in the database comments
|
||||
if !isempty(strip(row.column_description))
|
||||
text_payload *= " | Description: $(row.column_description)"
|
||||
else
|
||||
text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture."
|
||||
end
|
||||
|
||||
# 2. Package everything neatly to be passed to your vector store client
|
||||
push!(payloads, Dict(
|
||||
"id" => row.vector_id,
|
||||
"text_content" => text_payload,
|
||||
"metadata" => Dict(
|
||||
"table" => row.table_name,
|
||||
"column" => row.column_name,
|
||||
"type" => row.data_type
|
||||
)
|
||||
))
|
||||
end
|
||||
|
||||
return payloads
|
||||
end
|
||||
|
||||
|
||||
"""
|
||||
resolve_semantic_cluster(vector_hits::Vector{String}, g::SimpleGraph, table_to_id::Dict{String, Int}, id_to_table::Dict{Int, String}) -> Vector{String}
|
||||
|
||||
Takes a scattered array of semantically matched tables from Stage 1, navigates
|
||||
the undirected network structure, and isolates the minimum interconnected subgraph
|
||||
required to weave ALL hits into a single valid SQL query.
|
||||
"""
|
||||
function resolve_semantic_cluster(
|
||||
vector_hits::Vector{String},
|
||||
g::SimpleGraph,
|
||||
table_to_id::Dict{String, Int},
|
||||
id_to_table::Dict{Int, String}
|
||||
)
|
||||
# Filter out hits that don't exist in our actual database graph mapping
|
||||
valid_node_ids = Int[]
|
||||
for hit in vector_hits
|
||||
if haskey(table_to_id, hit)
|
||||
push!(valid_node_ids, table_to_id[hit])
|
||||
else
|
||||
@warn "Vector hit '$hit' does not map to an existing database table."
|
||||
end
|
||||
end
|
||||
|
||||
unique!(valid_node_ids)
|
||||
|
||||
# Edge Case Handlers
|
||||
if isempty(valid_node_ids)
|
||||
return String[]
|
||||
elseif length(valid_node_ids) == 1
|
||||
return [id_to_table[valid_node_ids[1]]]
|
||||
end
|
||||
|
||||
# The Isolated Subgraph Set to build our final context
|
||||
schema_subgraph_nodes = Set{Int}()
|
||||
|
||||
# Phase A: Select an initial anchor component. We use the highest-ranked vector hit.
|
||||
anchor_node = valid_node_ids[1]
|
||||
push!(schema_subgraph_nodes, anchor_node)
|
||||
|
||||
# Phase B: Sequentially route paths to all other semantic coordinates
|
||||
for target_node in valid_node_ids[2:end]
|
||||
# Skip if an earlier loop trajectory already naturally absorbed this table
|
||||
if target_node in schema_subgraph_nodes
|
||||
continue
|
||||
end
|
||||
|
||||
# Calculate the shortest path tree from the CURRENT state of our subgraph
|
||||
# We find the shortest path from the target back to ANY node currently in our tree
|
||||
shortest_paths = dijkstra_shortest_paths(g, target_node)
|
||||
|
||||
# Find which node currently in our subgraph is closest to the target node
|
||||
closest_subgraph_node = 0
|
||||
min_distance = Inf
|
||||
|
||||
for subgraph_node in schema_subgraph_nodes
|
||||
dist = shortest_paths.dists[subgraph_node]
|
||||
if dist < min_distance
|
||||
min_distance = dist
|
||||
closest_subgraph_node = subgraph_node
|
||||
end
|
||||
end
|
||||
|
||||
# Reconstruct the path from the target node to the closest point on our existing tree
|
||||
if closest_subgraph_node != 0
|
||||
curr = closest_subgraph_node
|
||||
while curr != 0
|
||||
push!(schema_subgraph_nodes, curr)
|
||||
curr = shortest_paths.parents[curr]
|
||||
if curr == target_node
|
||||
push!(schema_subgraph_nodes, target_node)
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Map the unique structural nodes back to clean table names
|
||||
return [id_to_table[node_id] for node_id in schema_subgraph_nodes]
|
||||
end
|
||||
|
||||
function get_embedding(nats_conn::NATS.Connection, text::AbstractArray{String})
|
||||
documents_dict = Dict("documents" => text)
|
||||
payloads = [("documents", documents_dict, "dictionary")]
|
||||
_, msg_envelope_json_str = msghandler.smartpack(
|
||||
config["externalservice"]["servicesloadbalancer"]["nats"],
|
||||
payloads;
|
||||
msg_purpose="embedding",
|
||||
broker_url=config["nats_server_info"]["url"],
|
||||
fileserver_url=config["externalservice"]["fileserver"]["url"])
|
||||
|
||||
reply = NATS.request(nats_conn,
|
||||
config["externalservice"]["servicesloadbalancer"]["nats"],
|
||||
msg_envelope_json_str, timeout=120)
|
||||
incoming_env_json_str = String(reply.payload)
|
||||
incoming_env = msghandler.smartunpack(incoming_env_json_str)
|
||||
embedding_response = incoming_env["payloads"][1][2]
|
||||
|
||||
return embedding_response
|
||||
end
|
||||
|
||||
nats_conn = NATS.connect(config["nats_server_info"]["url"])
|
||||
|
||||
# Run the extractor
|
||||
metadata_df = extract_vector_metadata(pg_conn_str)
|
||||
embedding_ready = generate_embedding_payloads(metadata_df)
|
||||
|
||||
println(embedding_ready[1]["text_content"])
|
||||
# Output: "Table: join_table | Column: seller_id | Type: integer [PRIMARY KEY] [FOREIGN KEY RELATIONAL LINK] | Description: Links unique sellers to their corresponding product items."
|
||||
|
||||
embedding_ready_2 = [i["text_content"] for i in embedding_ready]
|
||||
table_embedding = get_embedding(nats_conn, embedding_ready_2)
|
||||
|
||||
user_question =
|
||||
"""
|
||||
Retrieves ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"] of wines that match the following criteria - {wine_name: Montrachet Grand Cru, winery: Domaine Jacques Prieur, region: Montrachet, country: France, , retailer_name: Yiem Wines Ltd, retailerid: f54eab6b-7650-4448-b009-c53f3efbcc3b}
|
||||
"""
|
||||
user_question_embedding = get_embedding(nats_conn, [user_question])
|
||||
|
||||
|
||||
using Distances
|
||||
similarity = 1 - cosine_dist(Float64.(table_embedding["data"][1]["embedding"]),
|
||||
Float64.(user_question_embedding["data"][1]["embedding"])
|
||||
)
|
||||
user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"])
|
||||
user_question_similarity = []
|
||||
for i in table_embedding["data"]
|
||||
i_data = i["embedding"]
|
||||
i_float = Float64.(i_data)
|
||||
r = 1 - cosine_dist(i_float, user_question_embedding)
|
||||
push!(user_question_similarity, r)
|
||||
end
|
||||
|
||||
new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity))
|
||||
sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min
|
||||
|
||||
# top 20 of sorted_df get this tables
|
||||
vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"]
|
||||
|
||||
g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str)
|
||||
|
||||
# tables that I should put schema in LLM context
|
||||
optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
|
||||
|
||||
|
||||
|
||||
function related_tables_for_user_question()
|
||||
|
||||
|
||||
|
||||
|
||||
end
|
||||
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
|
||||
# Agent 3 (The Entity Resolver): Instantly runs a fast, local token search (like BM25) to map messy user text (like HandOld) to the exact database string (Hand Old Bar & Grill) before the SQL is drafted.
|
||||
|
||||
using StringDistances
|
||||
|
||||
"""
|
||||
harvest_entity_catalog(conn_str::String, table::String, column::String) -> Vector{String}
|
||||
|
||||
Pulls unique, clean text strings from a specific entity column to build a local index.
|
||||
"""
|
||||
function harvest_entity_catalog(conn_str::String, table::String, column::String)
|
||||
conn = LibPQ.Connection(conn_str)
|
||||
|
||||
# We only care about unique, non-null values to keep the index fast and dense
|
||||
query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;"
|
||||
|
||||
try
|
||||
df = DataFrame(execute(conn, query))
|
||||
# Return as a clean array of strings
|
||||
return String.(strip.(df[:, 1]))
|
||||
finally
|
||||
close(conn)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
"""
|
||||
resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.6) -> String
|
||||
|
||||
Parses user text, matches it against the real database catalog, and returns
|
||||
the exact string found in the database. Returns an empty string if no confident match.
|
||||
"""
|
||||
function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5)
|
||||
best_match = ""
|
||||
highest_score = 0.0
|
||||
|
||||
# Normalize input text to ensure case-insensitive matching
|
||||
clean_input = lowercase(strip(messy_input))
|
||||
|
||||
for real_string in catalog
|
||||
clean_real = lowercase(real_string)
|
||||
|
||||
# Calculate phonetic/structural similarity score (0.0 to 1.0)
|
||||
# JaroWinkler is optimized for short strings, names, and partial acronyms
|
||||
score = compare(clean_real, clean_input, JaroWinkler())
|
||||
|
||||
# Substring/Token fallback: handle cases like "HandOld" matching "Hand Old Bar & Grill"
|
||||
# We strip spaces to check if the user just compressed words together
|
||||
if contains(replace(clean_real, " " => ""), clean_input)
|
||||
score = max(score, 0.85)
|
||||
end
|
||||
|
||||
if score > highest_score
|
||||
highest_score = score
|
||||
best_match = real_string
|
||||
end
|
||||
end
|
||||
|
||||
# Only return if we cross our safety confidence barrier
|
||||
if highest_score >= threshold
|
||||
return best_match
|
||||
end
|
||||
|
||||
return "" # No confident match found
|
||||
end
|
||||
|
||||
|
||||
|
||||
winery_catalog = harvest_entity_catalog(conn_str, "wine", "winery")
|
||||
# Let's assume the catalog contains: ["Hand Old Bar & Grill", "Bangkok Diner", "Phuket Seafood"]
|
||||
|
||||
# 2. The user asks a messy question with a typo and compressed text
|
||||
user_question = "What are the total sales at HandOld last week?"
|
||||
|
||||
# 3. Agent 3 isolates potential nouns or scans the question against the index
|
||||
# We look for words that don't match standard english dictionary tokens, or check the full string segments
|
||||
detected_entity = "Jacob"
|
||||
|
||||
# 4. Run the resolution engine
|
||||
exact_db_string = resolve_entity(detected_entity, winery_catalog)
|
||||
# "United States"
|
||||
|
||||
println("Messy Input: ", detected_entity)
|
||||
println("Resolved Engine Value: ", exact_db_string)
|
||||
# Output: "Hand Old Bar & Grill"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+276
-338
@@ -1,103 +1,12 @@
|
||||
module llmUtil
|
||||
|
||||
export formatLLMtext, formatLLMtext_llama3instruct, jsoncorrection, deFormatLLMtext, extractthink,
|
||||
export formatLLMtext, extractthink,
|
||||
checkAgentResponse_JSON, clean_json_response
|
||||
|
||||
using UUIDs, JSON, Dates
|
||||
using UUIDs, JSON, Dates, DataFrames
|
||||
using GeneralUtils
|
||||
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
#[PENDING] update code to use JSON
|
||||
|
||||
""" Convert a single chat dictionary into LLM model instruct format.
|
||||
|
||||
# Llama 3 instruct format example
|
||||
<|begin_of_text|>
|
||||
<|start_header_id|>system<|end_header_id|>
|
||||
You are a helpful assistant.
|
||||
<|eot_id|>
|
||||
<|start_header_id|>user<|end_header_id|>
|
||||
Get me an icecream.
|
||||
<|eot_id|>
|
||||
<|start_header_id|>assistant<|end_header_id|>
|
||||
Go buy it yourself at 7-11.
|
||||
<|eot_id|>
|
||||
|
||||
# Arguments
|
||||
- `name::T`
|
||||
message owner name e.f. "system", "user" or "assistant"
|
||||
- `text::T`
|
||||
|
||||
# Return
|
||||
- `formattedtext::String`
|
||||
text formatted to model format
|
||||
|
||||
# Example
|
||||
```jldoctest
|
||||
julia> using Revise
|
||||
julia> using YiemAgent
|
||||
julia> d = Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",)
|
||||
julia> formattedtext = YiemAgent.formatLLMtext_llama3instruct(d[:name], d[:text])
|
||||
"<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n"
|
||||
```
|
||||
|
||||
Signature
|
||||
"""
|
||||
function formatLLMtext_llama3instruct(name::T, text::T;
|
||||
assistantStarter::Bool=false) where {T<:AbstractString}
|
||||
formattedtext =
|
||||
if name == "system"
|
||||
"""
|
||||
<|start_header_id|>$name<|end_header_id|>
|
||||
$text
|
||||
<|eot_id|>
|
||||
"""
|
||||
else
|
||||
"""
|
||||
<|start_header_id|>$name<|end_header_id|>
|
||||
$text
|
||||
<|eot_id|>
|
||||
"""
|
||||
end
|
||||
|
||||
if assistantStarter
|
||||
formattedtext *=
|
||||
"""
|
||||
<|start_header_id|>assistant<|end_header_id|>
|
||||
"""
|
||||
end
|
||||
|
||||
return formattedtext
|
||||
end
|
||||
|
||||
|
||||
function formatLLMtext_qwen(name::T, text::T;
|
||||
assistantStarter::Bool=false) where {T<:AbstractString}
|
||||
formattedtext =
|
||||
if name == "system"
|
||||
"""
|
||||
<|im_start|>$name
|
||||
$text
|
||||
<|im_end|>
|
||||
"""
|
||||
else
|
||||
"""
|
||||
<|im_start|>$name
|
||||
$text
|
||||
<|im_end|>
|
||||
"""
|
||||
end
|
||||
|
||||
if assistantStarter
|
||||
formattedtext *=
|
||||
"""
|
||||
<|im_start|>assistant
|
||||
"""
|
||||
end
|
||||
|
||||
return formattedtext
|
||||
end
|
||||
|
||||
|
||||
function formatLLMtext_qwen3(name::T, text::T;
|
||||
assistantStarter::Bool=false) where {T<:AbstractString}
|
||||
@@ -127,59 +36,6 @@ function formatLLMtext_qwen3(name::T, text::T;
|
||||
end
|
||||
|
||||
|
||||
function formatLLMtext_phi4(name::T, text::T;
|
||||
assistantStarter::Bool=false) where {T<:AbstractString}
|
||||
formattedtext =
|
||||
if name == "system"
|
||||
"""
|
||||
<|system|>
|
||||
$text
|
||||
<|end|>
|
||||
"""
|
||||
else
|
||||
"""
|
||||
<|assistant|>
|
||||
$text
|
||||
<|end|>
|
||||
"""
|
||||
end
|
||||
|
||||
if assistantStarter
|
||||
formattedtext *=
|
||||
"""
|
||||
<|assistant|>
|
||||
"""
|
||||
end
|
||||
|
||||
return formattedtext
|
||||
end
|
||||
|
||||
|
||||
function formatLLMtext_granite3(name::T, text::T;
|
||||
assistantStarter::Bool=false) where {T<:AbstractString}
|
||||
formattedtext =
|
||||
if name == "system"
|
||||
"""
|
||||
<|start_of_role|>system<|end_of_role|>{$text}<|end_of_text|>
|
||||
"""
|
||||
else
|
||||
"""
|
||||
<|start_of_role|>$name<|end_of_role|>{$text}<|end_of_text|>
|
||||
"""
|
||||
end
|
||||
|
||||
if assistantStarter
|
||||
formattedtext *=
|
||||
"""
|
||||
<|start_of_role|>assistant<|end_of_role|>{
|
||||
"""
|
||||
end
|
||||
|
||||
return formattedtext
|
||||
end
|
||||
|
||||
|
||||
|
||||
""" Convert a vector of chat message dictionaries into LLM model instruct format.
|
||||
|
||||
# Arguments
|
||||
@@ -194,13 +50,13 @@ end
|
||||
# Example
|
||||
```jldoctest
|
||||
julia> using Revise
|
||||
julia> using YiemAgent
|
||||
julia> using GeneralUtils
|
||||
julia> chatmessage = [
|
||||
Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",),
|
||||
Dict(:name=> "user",:text=> "list me all planets in our solar system.",),
|
||||
Dict(:name=> "assistant",:text=> "I'm sorry. I don't know. You tell me.",),
|
||||
]
|
||||
julia> formattedtext = YiemAgent.formatLLMtext(chatmessage, "llama3instruct")
|
||||
julia> formattedtext = GeneralUtils.formatLLMtext(chatmessage, "llama3instruct")
|
||||
"<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n <|start_header_id|>user<|end_header_id|>\n list me all planets in our solar system.\n <|eot_id|>\n <|start_header_id|>assistant<|end_header_id|>\n I'm sorry. I don't know. You tell me.\n <|eot_id|>\n"
|
||||
```
|
||||
"""
|
||||
@@ -237,192 +93,6 @@ function formatLLMtext(messages::Vector{Dict{Symbol, T}}, formatname::String
|
||||
return str
|
||||
end
|
||||
|
||||
""" Revert LLM-format response back into regular text.
|
||||
|
||||
# Arguments
|
||||
- `text::String`
|
||||
The LLM formatted string to be converted.
|
||||
|
||||
# Return
|
||||
- `normalText::String`
|
||||
The original plain text extracted from the given LLM-formatted string.
|
||||
|
||||
# Example
|
||||
```jldoctest
|
||||
julia> using Revise
|
||||
julia> using YiemAgent
|
||||
julia> response = "<|begin_of_text|>This is a sample system instruction.<|eot_id|>"
|
||||
julia> normalText = YiemAgent.deFormatLLMtext(response, "granite3")
|
||||
"This is a sample system instruction."
|
||||
```
|
||||
"""
|
||||
function deFormatLLMtext(text::String, formatname::String; includethink::Bool=false
|
||||
)::String
|
||||
f =
|
||||
if formatname == "granite3"
|
||||
deFormatLLMtext_granite3
|
||||
elseif formatname == "qwen3"
|
||||
deFormatLLMtext_qwen3
|
||||
else
|
||||
error("$formatname template not define yet")
|
||||
end
|
||||
|
||||
r = f(text)
|
||||
result = r === nothing ? text : r
|
||||
return result
|
||||
end
|
||||
|
||||
|
||||
""" Revert LLM-format response back into regular text for Granite 3 format.
|
||||
|
||||
# Arguments
|
||||
- `text::String`
|
||||
The LLM formatted string to be converted.
|
||||
|
||||
# Return
|
||||
- `normalText::Union{Nothing, String}`
|
||||
The original plain text extracted from the given LLM-formatted string.
|
||||
Returns nothing if the text is not in Granite 3 format.
|
||||
|
||||
# Example
|
||||
```jldoctest
|
||||
julia> using Revise
|
||||
julia> using YiemAgent
|
||||
julia> response = "{This is a sample LLM response.}"
|
||||
julia> normalText = YiemAgent.deFormatLLMtext(response, "granite3")
|
||||
"This is a sample LLM response."
|
||||
"""
|
||||
function deFormatLLMtext_granite3(text::String)::Union{Nothing, String}
|
||||
# check if '{' and '}' are in the text because it's a special format for the LLM response
|
||||
if contains(text, "<|im_start|>assistant")
|
||||
# get the text between '{' and '}'
|
||||
text_between_braces = GeneralUtils.extractTextBetweenCharacter(text, '{', '}')[1]
|
||||
return text_between_braces
|
||||
elseif text[end] == '}'
|
||||
text = "{$text"
|
||||
text_between_braces = GeneralUtils.extractTextBetweenCharacter(text, '{', '}')[1]
|
||||
else
|
||||
return nothing
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
function deFormatLLMtext_qwen3(text::String)::Union{Nothing, String}
|
||||
return text
|
||||
end
|
||||
|
||||
# function deFormatLLMtext_qwen3(text::String; includethink::Bool=false)::Union{Nothing, String}
|
||||
# think = nothing
|
||||
# str = nothing
|
||||
|
||||
# if occursin("<think>", text)
|
||||
# r = GeneralUtils.extractTextBetweenString(text, "<think>", "</think>")
|
||||
# if r[:success]
|
||||
# think = r[:text]
|
||||
# end
|
||||
# str = string(split(text, "</think>")[2])
|
||||
# end
|
||||
|
||||
# if includethink == true && occursin("<think>", text)
|
||||
# result = "ModelThought: $think $str"
|
||||
# return result
|
||||
# elseif includethink == false && occursin("<think>", text)
|
||||
# result = str
|
||||
# return result
|
||||
# else
|
||||
# return text
|
||||
# end
|
||||
# end
|
||||
|
||||
|
||||
""" Attemp to correct LLM response's incorrect JSON response.
|
||||
|
||||
# Arguments
|
||||
- `a::T1`
|
||||
one of Yiem's agent
|
||||
- `input::T2`
|
||||
text to be send to virtual wine customer
|
||||
|
||||
# Return
|
||||
- `correctjson::String`
|
||||
corrected json string
|
||||
|
||||
# Example
|
||||
```jldoctest
|
||||
julia>
|
||||
```
|
||||
|
||||
# Signature
|
||||
"""
|
||||
function jsoncorrection(config::T1, input::T2, correctJsonExample::T3;
|
||||
maxattempt::Integer=3
|
||||
) where {T1<:AbstractDict, T2<:AbstractString, T3<:AbstractString}
|
||||
|
||||
incorrectjson = deepcopy(input)
|
||||
correctjson = nothing
|
||||
|
||||
for attempt in 1:maxattempt
|
||||
try
|
||||
d = copy(JSON3.read(incorrectjson))
|
||||
correctjson = incorrectjson
|
||||
return correctjson
|
||||
catch e
|
||||
@warn "Attempting to correct JSON string. Attempt $attempt"
|
||||
e = """$e"""
|
||||
if occursin("EOF", e)
|
||||
e = split(e, "EOF")[1] * "EOF"
|
||||
end
|
||||
incorrectjson = deepcopy(input)
|
||||
_prompt =
|
||||
"""
|
||||
Your goal are:
|
||||
1) Use the expected JSON format as a guideline to check why the given JSON string failed to load and provide a corrected version that can be loaded by Python's json.load function.
|
||||
2) Provide Corrected JSON string only. Do not provide any other info.
|
||||
|
||||
$correctJsonExample
|
||||
|
||||
Let's begin!
|
||||
Given JSON string: $incorrectjson
|
||||
The given JSON string failed to load previously because: $e
|
||||
Corrected JSON string:
|
||||
"""
|
||||
|
||||
# apply LLM specific instruct format
|
||||
externalService = config[:externalservice][:text2textinstruct]
|
||||
llminfo = externalService[:llminfo]
|
||||
prompt =
|
||||
if llminfo[:name] == "llama3instruct"
|
||||
formatLLMtext_llama3instruct("system", _prompt)
|
||||
else
|
||||
error("llm model name is not defied yet $(@__LINE__)")
|
||||
end
|
||||
|
||||
# send formatted input to user using GeneralUtils.sendReceiveMqttMsg
|
||||
msgMeta = GeneralUtils.generate_msgMeta(
|
||||
externalService[:mqtttopic],
|
||||
senderName= "jsoncorrection",
|
||||
senderId= uuid4snakecase(),
|
||||
receiverName= "text2textinstruct",
|
||||
mqttBroker= config[:mqttServerInfo][:broker],
|
||||
mqttBrokerPort= config[:mqttServerInfo][:port],
|
||||
)
|
||||
|
||||
outgoingMsg = Dict(
|
||||
:msgMeta=> msgMeta,
|
||||
:payload=> Dict(
|
||||
:text=> prompt,
|
||||
:kwargs=> Dict(
|
||||
:max_tokens=> 512,
|
||||
:stop=> ["<|eot_id|>"],
|
||||
)
|
||||
)
|
||||
)
|
||||
result = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
|
||||
incorrectjson = result[:response][:text]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
function extractthink(text::String)
|
||||
think = nothing
|
||||
@@ -470,18 +140,18 @@ The validation logic checks:
|
||||
# Example
|
||||
|
||||
```julia
|
||||
julia> using YiemAgent
|
||||
julia> using GeneralUtils
|
||||
julia> requiredKeys = ["wine_name", "price", "rating"]
|
||||
julia> response = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98)
|
||||
julia> YiemAgent.checkAgentResponse_JSON(response, requiredKeys)
|
||||
julia> GeneralUtils.checkAgentResponse_JSON(response, requiredKeys)
|
||||
(true, nothing)
|
||||
|
||||
julia> response_missing = Dict("wine_name"=>"Château Margaux", "price"=>250.0)
|
||||
julia> YiemAgent.checkAgentResponse_JSON(response_missing, requiredKeys)
|
||||
julia> GeneralUtils.checkAgentResponse_JSON(response_missing, requiredKeys)
|
||||
(false, "rating are missing from your previous response")
|
||||
|
||||
julia> response_extra = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98, "extra_field"=>"data")
|
||||
julia> YiemAgent.checkAgentResponse_JSON(response_extra, requiredKeys)
|
||||
julia> GeneralUtils.checkAgentResponse_JSON(response_extra, requiredKeys)
|
||||
(false, "Your previous attempt has duplicated points according to the required response format")
|
||||
```
|
||||
"""
|
||||
@@ -550,12 +220,280 @@ function clean_json_response(text::String)
|
||||
end
|
||||
|
||||
|
||||
""" Extract vector metadata from PostgreSQL database.
|
||||
|
||||
Queries PostgreSQL system catalogs to extract column metadata including table names,
|
||||
column names, data types, descriptions, and constraint information (primary/foreign keys).
|
||||
Returns a DataFrame designed for vector embedding generation.
|
||||
|
||||
# Arguments
|
||||
- `pg_conn_str::String`
|
||||
PostgreSQL connection string (e.g., "postgresql://user:pass@host:port/dbname")
|
||||
|
||||
# Return
|
||||
- `DataFrame`
|
||||
A DataFrame with columns:
|
||||
- `table_name`: Name of the table
|
||||
- `column_name`: Name of the column
|
||||
- `data_type`: PostgreSQL data type with typemod
|
||||
- `column_description`: Column's comment/description (empty string if none)
|
||||
- `is_primary_key`: Boolean indicating if column is part of primary key
|
||||
- `is_foreign_key`: Boolean indicating if column is part of foreign key
|
||||
- `vector_id`: Generated unique identifier in format `col_{table_name}_{column_name}`
|
||||
|
||||
# Example
|
||||
```julia
|
||||
julia> using GeneralUtils
|
||||
julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb"
|
||||
julia> df = GeneralUtils.extract_vector_metadata(pg_conn)
|
||||
DataFrame
|
||||
6 rows × 7 columns
|
||||
table_name column_name data_type column_description is_primary_key is_foreign_key vector_id
|
||||
─────────────┬───────────┬───────────┬───────────────────┬───────────────┬───────────────┬────────────────────────
|
||||
users id integer User ID true false col_users_id
|
||||
users name text User name false false col_users_name
|
||||
users email text User email false false col_users_email
|
||||
products id integer Product ID true false col_products_id
|
||||
products price numeric Product price false false col_products_price
|
||||
products user_id integer Reference to user false true col_products_user_id
|
||||
```
|
||||
"""
|
||||
function extract_vector_metadata(pg_conn_str::String)::DataFrame
|
||||
conn = LibPQ.Connection(pg_conn_str)
|
||||
|
||||
# This direct SQL query pulls the column specifications along with column-level descriptions
|
||||
query =
|
||||
"""
|
||||
SELECT
|
||||
c.relname AS column_name,
|
||||
format_type(a.atttypid, a.atttypmod) AS data_type,
|
||||
COALESCE(d.description, '') AS column_description,
|
||||
CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key,
|
||||
CASE WHEN AS table_name,
|
||||
a.attnamefk.contype = 'f' THEN true ELSE false END AS is_foreign_key
|
||||
FROM pg_attribute a
|
||||
JOIN pg_class c ON c.oid = a.attrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
-- Join to fetch column comments/descriptions
|
||||
LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
|
||||
-- Check if column is part of a Primary Key
|
||||
LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid
|
||||
AND pk.contype = 'p'
|
||||
AND a.attnum = ANY(pk.conkey)
|
||||
-- Check if column is part of a Foreign Key
|
||||
LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid
|
||||
AND fk.contype = 'f'
|
||||
AND a.attnum = ANY(fk.conkey)
|
||||
WHERE
|
||||
n.nspname = 'public' -- Only user schemas
|
||||
AND c.relkind = 'r' -- Only standard tables
|
||||
AND a.attnum > 0 -- Skip system hidden columns
|
||||
AND NOT a.attisdropped; -- Skip dropped columns
|
||||
"""
|
||||
|
||||
try
|
||||
# Execute and format into a clean DataFrame
|
||||
result = execute(conn, query)
|
||||
df = DataFrame(result)
|
||||
|
||||
# Create a unique document ID for each vector row
|
||||
df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)]
|
||||
|
||||
return df
|
||||
finally
|
||||
close(conn)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
""" Generate embedding payloads from vector metadata DataFrame.
|
||||
|
||||
Transforms the metadata DataFrame from `extract_vector_metadata` into a vector of
|
||||
dictionaries structured for vector embedding storage and retrieval.
|
||||
|
||||
# Arguments
|
||||
- `df::DataFrame`
|
||||
A DataFrame with columns from `extract_vector_metadata`: `table_name`, `column_name`,
|
||||
`data_type`, `column_description`, `is_primary_key`, `is_foreign_key`, `vector_id`.
|
||||
|
||||
# Return
|
||||
- `Vector{Dict}`
|
||||
A vector of dictionaries with keys:
|
||||
- `id`: The vector ID from `vector_id`
|
||||
- `text_content`: A structured text string combining table, column, type, constraint
|
||||
markers, and description
|
||||
- `metadata`: A dictionary containing `table`, `column`, and `type`
|
||||
|
||||
# Details
|
||||
The function constructs rich text payloads by:
|
||||
1. Building a base string with table name, column name, and data type
|
||||
2. Appending constraint markers `[PRIMARY KEY]` or `[FOREIGN KEY RELATIONAL LINK]`
|
||||
3. Adding column description if available, otherwise generating a default description
|
||||
|
||||
# Example
|
||||
```julia
|
||||
julia> using GeneralUtils
|
||||
julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb"
|
||||
julia> df = GeneralUtils.extract_vector_metadata(pg_conn)
|
||||
julia> payloads = GeneralUtils.generate_embedding_payloads(df)
|
||||
3-element Vector{Dict}:
|
||||
Dict("id" => "col_users_id", "text_content" => "Table: users | Column: id | Type: integer [PRIMARY KEY] | Description: User ID", "metadata" => Dict("table" => "users", "column" => "id", "type" => "integer"))
|
||||
Dict("id" => "col_users_name", "text_content" => "Table: users | Column: name | Type: text | Description: User name", "metadata" => Dict("table" => "users", "column" => "name", "type" => "text"))
|
||||
Dict("id" => "col_users_email", "text_content" => "Table: users | Column: email | Type: text | Description: User email", "metadata" => Dict("table" => "users", "column" => "email", "type" => "text"))
|
||||
```
|
||||
"""
|
||||
function generate_embedding_payloads(df::DataFrame)::Vector{Dict}
|
||||
payloads = Dict[]
|
||||
|
||||
for row in eachrow(df)
|
||||
# 1. Build a rich text description summarizing the column's role
|
||||
text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)"
|
||||
|
||||
if row.is_primary_key
|
||||
text_payload *= " [PRIMARY KEY]"
|
||||
end
|
||||
if row.is_foreign_key
|
||||
text_payload *= " [FOREIGN KEY RELATIONAL LINK]"
|
||||
end
|
||||
|
||||
# Append business descriptions if they exist in the database comments
|
||||
if !isempty(strip(row.column_description))
|
||||
text_payload *= " | Description: $(row.column_description)"
|
||||
else
|
||||
text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture."
|
||||
end
|
||||
|
||||
# 2. Package everything neatly to be passed to your vector store client
|
||||
push!(payloads, Dict(
|
||||
"id" => row.vector_id,
|
||||
"text_content" => text_payload,
|
||||
"metadata" => Dict(
|
||||
"table" => row.table_name,
|
||||
"column" => row.column_name,
|
||||
"type" => row.data_type
|
||||
)
|
||||
))
|
||||
end
|
||||
|
||||
return payloads
|
||||
end
|
||||
|
||||
|
||||
""" Resolve semantic cluster from vector hits using graph traversal.
|
||||
|
||||
Finds the minimum interconnected subgraph that connects all semantically matched tables
|
||||
from vector search results, enabling construction of valid SQL queries across related tables.
|
||||
|
||||
# Arguments
|
||||
- `vector_hits::Vector{String}`
|
||||
A vector of table names matched semantically from Stage 1 vector search.
|
||||
- `g::SimpleGraph`
|
||||
An undirected graph representing table relationships (nodes=tables, edges=foreign key relations).
|
||||
- `table_to_id::Dict{String, Int}`
|
||||
Mapping from table names to node IDs in the graph.
|
||||
- `id_to_table::Dict{Int, String}`
|
||||
Reverse mapping from node IDs to table names.
|
||||
|
||||
# Return
|
||||
- `Vector{String}`
|
||||
A vector of table names representing the minimum subgraph that connects all input vector hits.
|
||||
The order reflects traversal path from anchor node to connected components.
|
||||
|
||||
# Details
|
||||
The algorithm:
|
||||
1. Validates vector hits against the graph's table mapping
|
||||
2. Handles edge cases: empty hits, single table (returns as-is)
|
||||
3. Uses the highest-ranked vector hit as anchor node
|
||||
4. For each remaining hit, finds shortest path to current subgraph using Dijkstra's algorithm
|
||||
5. Builds minimal connected subgraph containing all hits
|
||||
6. Returns table names in traversal order
|
||||
|
||||
# Example
|
||||
```julia
|
||||
julia> using GeneralUtils, Graphs
|
||||
julia> g = SimpleGraph(5)
|
||||
julia> add_edge!(g, 1, 2)
|
||||
julia> add_edge!(g, 2, 3)
|
||||
julia> add_edge!(g, 3, 4)
|
||||
julia> table_to_id = Dict("users" => 1, "orders" => 2, "payments" => 3, "products" => 4, "inventory" => 5)
|
||||
julia> id_to_table = Dict(1 => "users", 2 => "orders", 3 => "payments", 4 => "products", 5 => "inventory")
|
||||
julia> vector_hits = ["users", "products"]
|
||||
julia> GeneralUtils.resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
|
||||
["users", "orders", "products"]
|
||||
```
|
||||
"""
|
||||
function resolve_semantic_cluster(
|
||||
vector_hits::Vector{String},
|
||||
g::SimpleGraph,
|
||||
table_to_id::Dict{String, Int},
|
||||
id_to_table::Dict{Int, String}
|
||||
)::Vector{String}
|
||||
# Filter out hits that don't exist in our actual database graph mapping
|
||||
valid_node_ids = Int[]
|
||||
for hit in vector_hits
|
||||
if haskey(table_to_id, hit)
|
||||
push!(valid_node_ids, table_to_id[hit])
|
||||
else
|
||||
@warn "Vector hit '$hit' does not map to an existing database table."
|
||||
end
|
||||
end
|
||||
|
||||
unique!(valid_node_ids)
|
||||
|
||||
# Edge Case Handlers
|
||||
if isempty(valid_node_ids)
|
||||
return String[]
|
||||
elseif length(valid_node_ids) == 1
|
||||
return [id_to_table[valid_node_ids[1]]]
|
||||
end
|
||||
|
||||
# The Isolated Subgraph Set to build our final context
|
||||
schema_subgraph_nodes = Set{Int}()
|
||||
|
||||
# Phase A: Select an initial anchor component. We use the highest-ranked vector hit.
|
||||
anchor_node = valid_node_ids[1]
|
||||
push!(schema_subgraph_nodes, anchor_node)
|
||||
|
||||
# Phase B: Sequentially route paths to all other semantic coordinates
|
||||
for target_node in valid_node_ids[2:end]
|
||||
# Skip if an earlier loop trajectory already naturally absorbed this table
|
||||
if target_node in schema_subgraph_nodes
|
||||
continue
|
||||
end
|
||||
|
||||
# Calculate the shortest path tree from the CURRENT state of our subgraph
|
||||
# We find the shortest path from the target back to ANY node currently in our tree
|
||||
shortest_paths = dijkstra_shortest_paths(g, target_node)
|
||||
|
||||
# Find which node currently in our subgraph is closest to the target node
|
||||
closest_subgraph_node = 0
|
||||
min_distance = Inf
|
||||
|
||||
for subgraph_node in schema_subgraph_nodes
|
||||
dist = shortest_paths.dists[subgraph_node]
|
||||
if dist < min_distance
|
||||
min_distance = dist
|
||||
closest_subgraph_node = subgraph_node
|
||||
end
|
||||
end
|
||||
|
||||
# Reconstruct the path from the target node to the closest point on our existing tree
|
||||
if closest_subgraph_node != 0
|
||||
curr = closest_subgraph_node
|
||||
while curr != 0
|
||||
push!(schema_subgraph_nodes, curr)
|
||||
curr = shortest_paths.parents[curr]
|
||||
if curr == target_node
|
||||
push!(schema_subgraph_nodes, target_node)
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Map the unique structural nodes back to clean table names
|
||||
return [id_to_table[node_id] for node_id in schema_subgraph_nodes]
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user