ton/GeneralUtils
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

update

Browse Source
This commit is contained in:
narawat
2026-07-13 21:06:09 +07:00
parent c56fc7366c
commit edad442242
2 changed files with 631 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llmUtil.jl
+276 -338
View File
@@ -1,103 +1,12 @@
module llmUtil
export formatLLMtext, formatLLMtext_llama3instruct, jsoncorrection, deFormatLLMtext, extractthink,
export formatLLMtext, extractthink,
checkAgentResponse_JSON, clean_json_response
using UUIDs, JSON, Dates
using UUIDs, JSON, Dates, DataFrames
using GeneralUtils
# ---------------------------------------------- 100 --------------------------------------------- #
#[PENDING] update code to use JSON
""" Convert a single chat dictionary into LLM model instruct format.
# Llama 3 instruct format example
<|begin_of_text|>
<|start_header_id|>system<|end_header_id|>
You are a helpful assistant.
<|eot_id|>
<|start_header_id|>user<|end_header_id|>
Get me an icecream.
<|eot_id|>
<|start_header_id|>assistant<|end_header_id|>
Go buy it yourself at 7-11.
<|eot_id|>
# Arguments
- `name::T`
message owner name e.f. "system", "user" or "assistant"
- `text::T`
# Return
- `formattedtext::String`
text formatted to model format
# Example
```jldoctest
julia> using Revise
julia> using YiemAgent
julia> d = Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",)
julia> formattedtext = YiemAgent.formatLLMtext_llama3instruct(d[:name], d[:text])
"<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n"
```
Signature
"""
function formatLLMtext_llama3instruct(name::T, text::T;
assistantStarter::Bool=false) where {T<:AbstractString}
formattedtext =
if name == "system"
"""
<|start_header_id|>$name<|end_header_id|>
$text
<|eot_id|>
"""
else
"""
<|start_header_id|>$name<|end_header_id|>
$text
<|eot_id|>
"""
end
if assistantStarter
formattedtext *=
"""
<|start_header_id|>assistant<|end_header_id|>
"""
end
return formattedtext
end
function formatLLMtext_qwen(name::T, text::T;
assistantStarter::Bool=false) where {T<:AbstractString}
formattedtext =
if name == "system"
"""
<|im_start|>$name
$text
<|im_end|>
"""
else
"""
<|im_start|>$name
$text
<|im_end|>
"""
end
if assistantStarter
formattedtext *=
"""
<|im_start|>assistant
"""
end
return formattedtext
end
function formatLLMtext_qwen3(name::T, text::T;
assistantStarter::Bool=false) where {T<:AbstractString}
@@ -127,59 +36,6 @@ function formatLLMtext_qwen3(name::T, text::T;
end
function formatLLMtext_phi4(name::T, text::T;
assistantStarter::Bool=false) where {T<:AbstractString}
formattedtext =
if name == "system"
"""
<|system|>
$text
<|end|>
"""
else
"""
<|assistant|>
$text
<|end|>
"""
end
if assistantStarter
formattedtext *=
"""
<|assistant|>
"""
end
return formattedtext
end
function formatLLMtext_granite3(name::T, text::T;
assistantStarter::Bool=false) where {T<:AbstractString}
formattedtext =
if name == "system"
"""
<|start_of_role|>system<|end_of_role|>{$text}<|end_of_text|>
"""
else
"""
<|start_of_role|>$name<|end_of_role|>{$text}<|end_of_text|>
"""
end
if assistantStarter
formattedtext *=
"""
<|start_of_role|>assistant<|end_of_role|>{
"""
end
return formattedtext
end
""" Convert a vector of chat message dictionaries into LLM model instruct format.
# Arguments
@@ -194,13 +50,13 @@ end
# Example
```jldoctest
julia> using Revise
julia> using YiemAgent
julia> using GeneralUtils
julia> chatmessage = [
Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",),
Dict(:name=> "user",:text=> "list me all planets in our solar system.",),
Dict(:name=> "assistant",:text=> "I'm sorry. I don't know. You tell me.",),
]
julia> formattedtext = YiemAgent.formatLLMtext(chatmessage, "llama3instruct")
julia> formattedtext = GeneralUtils.formatLLMtext(chatmessage, "llama3instruct")
"<|begin_of_text|>\n <|start_header_id|>system<|end_header_id|>\n You are a helpful, respectful and honest assistant.\n <|eot_id|>\n <|start_header_id|>user<|end_header_id|>\n list me all planets in our solar system.\n <|eot_id|>\n <|start_header_id|>assistant<|end_header_id|>\n I'm sorry. I don't know. You tell me.\n <|eot_id|>\n"
```
"""
@@ -237,192 +93,6 @@ function formatLLMtext(messages::Vector{Dict{Symbol, T}}, formatname::String
return str
end
""" Revert LLM-format response back into regular text.
# Arguments
- `text::String`
The LLM formatted string to be converted.
# Return
- `normalText::String`
The original plain text extracted from the given LLM-formatted string.
# Example
```jldoctest
julia> using Revise
julia> using YiemAgent
julia> response = "<|begin_of_text|>This is a sample system instruction.<|eot_id|>"
julia> normalText = YiemAgent.deFormatLLMtext(response, "granite3")
"This is a sample system instruction."
```
"""
function deFormatLLMtext(text::String, formatname::String; includethink::Bool=false
)::String
f =
if formatname == "granite3"
deFormatLLMtext_granite3
elseif formatname == "qwen3"
deFormatLLMtext_qwen3
else
error("$formatname template not define yet")
end
r = f(text)
result = r === nothing ? text : r
return result
end
""" Revert LLM-format response back into regular text for Granite 3 format.
# Arguments
- `text::String`
The LLM formatted string to be converted.
# Return
- `normalText::Union{Nothing, String}`
The original plain text extracted from the given LLM-formatted string.
Returns nothing if the text is not in Granite 3 format.
# Example
```jldoctest
julia> using Revise
julia> using YiemAgent
julia> response = "{This is a sample LLM response.}"
julia> normalText = YiemAgent.deFormatLLMtext(response, "granite3")
"This is a sample LLM response."
"""
function deFormatLLMtext_granite3(text::String)::Union{Nothing, String}
# check if '{' and '}' are in the text because it's a special format for the LLM response
if contains(text, "<|im_start|>assistant")
# get the text between '{' and '}'
text_between_braces = GeneralUtils.extractTextBetweenCharacter(text, '{', '}')[1]
return text_between_braces
elseif text[end] == '}'
text = "{$text"
text_between_braces = GeneralUtils.extractTextBetweenCharacter(text, '{', '}')[1]
else
return nothing
end
end
function deFormatLLMtext_qwen3(text::String)::Union{Nothing, String}
return text
end
# function deFormatLLMtext_qwen3(text::String; includethink::Bool=false)::Union{Nothing, String}
# think = nothing
# str = nothing
# if occursin("<think>", text)
# r = GeneralUtils.extractTextBetweenString(text, "<think>", "</think>")
# if r[:success]
# think = r[:text]
# end
# str = string(split(text, "</think>")[2])
# end
# if includethink == true && occursin("<think>", text)
# result = "ModelThought: $think $str"
# return result
# elseif includethink == false && occursin("<think>", text)
# result = str
# return result
# else
# return text
# end
# end
""" Attemp to correct LLM response's incorrect JSON response.
# Arguments
- `a::T1`
one of Yiem's agent
- `input::T2`
text to be send to virtual wine customer
# Return
- `correctjson::String`
corrected json string
# Example
```jldoctest
julia>
```
# Signature
"""
function jsoncorrection(config::T1, input::T2, correctJsonExample::T3;
maxattempt::Integer=3
) where {T1<:AbstractDict, T2<:AbstractString, T3<:AbstractString}
incorrectjson = deepcopy(input)
correctjson = nothing
for attempt in 1:maxattempt
try
d = copy(JSON3.read(incorrectjson))
correctjson = incorrectjson
return correctjson
catch e
@warn "Attempting to correct JSON string. Attempt $attempt"
e = """$e"""
if occursin("EOF", e)
e = split(e, "EOF")[1] * "EOF"
end
incorrectjson = deepcopy(input)
_prompt =
"""
Your goal are:
1) Use the expected JSON format as a guideline to check why the given JSON string failed to load and provide a corrected version that can be loaded by Python's json.load function.
2) Provide Corrected JSON string only. Do not provide any other info.
$correctJsonExample
Let's begin!
Given JSON string: $incorrectjson
The given JSON string failed to load previously because: $e
Corrected JSON string:
"""
# apply LLM specific instruct format
externalService = config[:externalservice][:text2textinstruct]
llminfo = externalService[:llminfo]
prompt =
if llminfo[:name] == "llama3instruct"
formatLLMtext_llama3instruct("system", _prompt)
else
error("llm model name is not defied yet $(@__LINE__)")
end
# send formatted input to user using GeneralUtils.sendReceiveMqttMsg
msgMeta = GeneralUtils.generate_msgMeta(
externalService[:mqtttopic],
senderName= "jsoncorrection",
senderId= uuid4snakecase(),
receiverName= "text2textinstruct",
mqttBroker= config[:mqttServerInfo][:broker],
mqttBrokerPort= config[:mqttServerInfo][:port],
)
outgoingMsg = Dict(
:msgMeta=> msgMeta,
:payload=> Dict(
:text=> prompt,
:kwargs=> Dict(
:max_tokens=> 512,
:stop=> ["<|eot_id|>"],
)
)
)
result = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
incorrectjson = result[:response][:text]
end
end
end
function extractthink(text::String)
think = nothing
@@ -470,18 +140,18 @@ The validation logic checks:
# Example
```julia
julia> using YiemAgent
julia> using GeneralUtils
julia> requiredKeys = ["wine_name", "price", "rating"]
julia> response = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98)
julia> YiemAgent.checkAgentResponse_JSON(response, requiredKeys)
julia> GeneralUtils.checkAgentResponse_JSON(response, requiredKeys)
(true, nothing)
julia> response_missing = Dict("wine_name"=>"Château Margaux", "price"=>250.0)
julia> YiemAgent.checkAgentResponse_JSON(response_missing, requiredKeys)
julia> GeneralUtils.checkAgentResponse_JSON(response_missing, requiredKeys)
(false, "rating are missing from your previous response")
julia> response_extra = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98, "extra_field"=>"data")
julia> YiemAgent.checkAgentResponse_JSON(response_extra, requiredKeys)
julia> GeneralUtils.checkAgentResponse_JSON(response_extra, requiredKeys)
(false, "Your previous attempt has duplicated points according to the required response format")
```
"""
@@ -550,12 +220,280 @@ function clean_json_response(text::String)
end
""" Extract vector metadata from PostgreSQL database.
Queries PostgreSQL system catalogs to extract column metadata including table names,
column names, data types, descriptions, and constraint information (primary/foreign keys).
Returns a DataFrame designed for vector embedding generation.
# Arguments
- `pg_conn_str::String`
PostgreSQL connection string (e.g., "postgresql://user:pass@host:port/dbname")
# Return
- `DataFrame`
A DataFrame with columns:
- `table_name`: Name of the table
- `column_name`: Name of the column
- `data_type`: PostgreSQL data type with typemod
- `column_description`: Column's comment/description (empty string if none)
- `is_primary_key`: Boolean indicating if column is part of primary key
- `is_foreign_key`: Boolean indicating if column is part of foreign key
- `vector_id`: Generated unique identifier in format `col_{table_name}_{column_name}`
# Example
```julia
julia> using GeneralUtils
julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb"
julia> df = GeneralUtils.extract_vector_metadata(pg_conn)
DataFrame
6 rows × 7 columns
table_name column_name data_type column_description is_primary_key is_foreign_key vector_id
─────────────┬───────────┬───────────┬───────────────────┬───────────────┬───────────────┬────────────────────────
users id integer User ID true false col_users_id
users name text User name false false col_users_name
users email text User email false false col_users_email
products id integer Product ID true false col_products_id
products price numeric Product price false false col_products_price
products user_id integer Reference to user false true col_products_user_id
```
"""
function extract_vector_metadata(pg_conn_str::String)::DataFrame
conn = LibPQ.Connection(pg_conn_str)
# This direct SQL query pulls the column specifications along with column-level descriptions
query =
"""
SELECT
c.relname AS column_name,
format_type(a.atttypid, a.atttypmod) AS data_type,
COALESCE(d.description, '') AS column_description,
CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key,
CASE WHEN AS table_name,
a.attnamefk.contype = 'f' THEN true ELSE false END AS is_foreign_key
FROM pg_attribute a
JOIN pg_class c ON c.oid = a.attrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
-- Join to fetch column comments/descriptions
LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
-- Check if column is part of a Primary Key
LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid
AND pk.contype = 'p'
AND a.attnum = ANY(pk.conkey)
-- Check if column is part of a Foreign Key
LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid
AND fk.contype = 'f'
AND a.attnum = ANY(fk.conkey)
WHERE
n.nspname = 'public' -- Only user schemas
AND c.relkind = 'r' -- Only standard tables
AND a.attnum > 0 -- Skip system hidden columns
AND NOT a.attisdropped; -- Skip dropped columns
"""
try
# Execute and format into a clean DataFrame
result = execute(conn, query)
df = DataFrame(result)
# Create a unique document ID for each vector row
df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)]
return df
finally
close(conn)
end
end
""" Generate embedding payloads from vector metadata DataFrame.
Transforms the metadata DataFrame from `extract_vector_metadata` into a vector of
dictionaries structured for vector embedding storage and retrieval.
# Arguments
- `df::DataFrame`
A DataFrame with columns from `extract_vector_metadata`: `table_name`, `column_name`,
`data_type`, `column_description`, `is_primary_key`, `is_foreign_key`, `vector_id`.
# Return
- `Vector{Dict}`
A vector of dictionaries with keys:
- `id`: The vector ID from `vector_id`
- `text_content`: A structured text string combining table, column, type, constraint
markers, and description
- `metadata`: A dictionary containing `table`, `column`, and `type`
# Details
The function constructs rich text payloads by:
1. Building a base string with table name, column name, and data type
2. Appending constraint markers `[PRIMARY KEY]` or `[FOREIGN KEY RELATIONAL LINK]`
3. Adding column description if available, otherwise generating a default description
# Example
```julia
julia> using GeneralUtils
julia> pg_conn = "postgresql://user:pass@localhost:5432/mydb"
julia> df = GeneralUtils.extract_vector_metadata(pg_conn)
julia> payloads = GeneralUtils.generate_embedding_payloads(df)
3-element Vector{Dict}:
Dict("id" => "col_users_id", "text_content" => "Table: users | Column: id | Type: integer [PRIMARY KEY] | Description: User ID", "metadata" => Dict("table" => "users", "column" => "id", "type" => "integer"))
Dict("id" => "col_users_name", "text_content" => "Table: users | Column: name | Type: text | Description: User name", "metadata" => Dict("table" => "users", "column" => "name", "type" => "text"))
Dict("id" => "col_users_email", "text_content" => "Table: users | Column: email | Type: text | Description: User email", "metadata" => Dict("table" => "users", "column" => "email", "type" => "text"))
```
"""
function generate_embedding_payloads(df::DataFrame)::Vector{Dict}
payloads = Dict[]
for row in eachrow(df)
# 1. Build a rich text description summarizing the column's role
text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)"
if row.is_primary_key
text_payload *= " [PRIMARY KEY]"
end
if row.is_foreign_key
text_payload *= " [FOREIGN KEY RELATIONAL LINK]"
end
# Append business descriptions if they exist in the database comments
if !isempty(strip(row.column_description))
text_payload *= " | Description: $(row.column_description)"
else
text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture."
end
# 2. Package everything neatly to be passed to your vector store client
push!(payloads, Dict(
"id" => row.vector_id,
"text_content" => text_payload,
"metadata" => Dict(
"table" => row.table_name,
"column" => row.column_name,
"type" => row.data_type
)
))
end
return payloads
end
""" Resolve semantic cluster from vector hits using graph traversal.
Finds the minimum interconnected subgraph that connects all semantically matched tables
from vector search results, enabling construction of valid SQL queries across related tables.
# Arguments
- `vector_hits::Vector{String}`
A vector of table names matched semantically from Stage 1 vector search.
- `g::SimpleGraph`
An undirected graph representing table relationships (nodes=tables, edges=foreign key relations).
- `table_to_id::Dict{String, Int}`
Mapping from table names to node IDs in the graph.
- `id_to_table::Dict{Int, String}`
Reverse mapping from node IDs to table names.
# Return
- `Vector{String}`
A vector of table names representing the minimum subgraph that connects all input vector hits.
The order reflects traversal path from anchor node to connected components.
# Details
The algorithm:
1. Validates vector hits against the graph's table mapping
2. Handles edge cases: empty hits, single table (returns as-is)
3. Uses the highest-ranked vector hit as anchor node
4. For each remaining hit, finds shortest path to current subgraph using Dijkstra's algorithm
5. Builds minimal connected subgraph containing all hits
6. Returns table names in traversal order
# Example
```julia
julia> using GeneralUtils, Graphs
julia> g = SimpleGraph(5)
julia> add_edge!(g, 1, 2)
julia> add_edge!(g, 2, 3)
julia> add_edge!(g, 3, 4)
julia> table_to_id = Dict("users" => 1, "orders" => 2, "payments" => 3, "products" => 4, "inventory" => 5)
julia> id_to_table = Dict(1 => "users", 2 => "orders", 3 => "payments", 4 => "products", 5 => "inventory")
julia> vector_hits = ["users", "products"]
julia> GeneralUtils.resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
["users", "orders", "products"]
```
"""
function resolve_semantic_cluster(
vector_hits::Vector{String},
g::SimpleGraph,
table_to_id::Dict{String, Int},
id_to_table::Dict{Int, String}
)::Vector{String}
# Filter out hits that don't exist in our actual database graph mapping
valid_node_ids = Int[]
for hit in vector_hits
if haskey(table_to_id, hit)
push!(valid_node_ids, table_to_id[hit])
else
@warn "Vector hit '$hit' does not map to an existing database table."
end
end
unique!(valid_node_ids)
# Edge Case Handlers
if isempty(valid_node_ids)
return String[]
elseif length(valid_node_ids) == 1
return [id_to_table[valid_node_ids[1]]]
end
# The Isolated Subgraph Set to build our final context
schema_subgraph_nodes = Set{Int}()
# Phase A: Select an initial anchor component. We use the highest-ranked vector hit.
anchor_node = valid_node_ids[1]
push!(schema_subgraph_nodes, anchor_node)
# Phase B: Sequentially route paths to all other semantic coordinates
for target_node in valid_node_ids[2:end]
# Skip if an earlier loop trajectory already naturally absorbed this table
if target_node in schema_subgraph_nodes
continue
end
# Calculate the shortest path tree from the CURRENT state of our subgraph
# We find the shortest path from the target back to ANY node currently in our tree
shortest_paths = dijkstra_shortest_paths(g, target_node)
# Find which node currently in our subgraph is closest to the target node
closest_subgraph_node = 0
min_distance = Inf
for subgraph_node in schema_subgraph_nodes
dist = shortest_paths.dists[subgraph_node]
if dist < min_distance
min_distance = dist
closest_subgraph_node = subgraph_node
end
end
# Reconstruct the path from the target node to the closest point on our existing tree
if closest_subgraph_node != 0
curr = closest_subgraph_node
while curr != 0
push!(schema_subgraph_nodes, curr)
curr = shortest_paths.parents[curr]
if curr == target_node
push!(schema_subgraph_nodes, target_node)
break
end
end
end
end
# Map the unique structural nodes back to clean table names
return [id_to_table[node_id] for node_id in schema_subgraph_nodes]
end