update

Reviewed-on: #7

Project.toml

@@ -1,7 +1,7 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
+version = "0.4.10"
 authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
-version = "0.2.2"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
@@ -9,9 +9,23 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
-MQTTClient = "985f35cc-2c3d-4943-b8c1-f0931d5f0959"
+Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
+HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
+NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
 PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[compat]
+Graphs = "1.14.0"
+HTTP = "2.5.0 - 2.9.9"
+JSON = "1.3.0 - 1.9.9"
+LibPQ = "1.18.0"
+NATS = "0.1.0"
+Revise = "3.13.2"
+StringDistances = "1.0.0"

README.md

@@ -0,0 +1,5 @@
+Todo:
+  - [WORKING] update with JSON
+
+  Change from previous version:
+  - replace JSON3 with JSON

codesnippet/nats.jl

@@ -0,0 +1,79 @@
+
+using NATS, JSON3
+connection  = NATS.connect("nats.yiem.cc:4222")
+sub1 = NATS.reply(connection, "some_subject"; queue_group="group1") do msg
+                  payload = copy(JSON3.read(msg.payload))
+                  println(payload)
+                  println(msg.reply_to)
+                  # publish(connection, msg.reply_to, "ACK")
+                  return JSON3.write(Dict(:a=>"wassup"))
+              end
+
+
+
+
+
+
+
+
+
+
+using NATS, JSON3, GeneralUtils
+connection  = NATS.connect("nats.yiem.cc:4222")
+
+msgMeta = GeneralUtils.generate_msgMeta(
+  "text2textinstruct_medium.inference.api.v1";
+  msgPurpose= "inference",
+  senderName= "yiemagent",
+  senderId= GeneralUtils.uuid4snakecase(),
+  receiverName= "text2textinstruct",
+)
+
+llmHttpTimeout = 60
+outgoingMsg = Dict(
+    :msgMeta=> msgMeta,
+    :payload=> Dict(
+      :text=> "Wassup buddy!",
+      :kwargs=> Dict(
+        :max_tokens=> 2048,
+        :stop=> ["<|im_end|>"],
+        :temperature=> 0.2,
+      ),
+      :llmHttpTimeout=>llmHttpTimeout,
+    )
+  )
+
+r = NATS.request(String, connection, "text2textinstruct_medium.inference.api.v1", 
+                JSON3.write(outgoingMsg); timer=Timer(llmHttpTimeout))
+
+
+
+
+
+
+
+
+
+
+
+using NATS, JSON3, GeneralUtils
+connection  = NATS.connect("nats.yiem.cc:4222")
+
+msgMeta = GeneralUtils.generate_msgMeta(
+  "tonpc.containerServices",
+  msgPurpose="reset container",
+  senderName= "",
+)
+
+outgoingMsg = Dict(
+  :msgMeta=> msgMeta,
+  :payload=> "docker container restart ollama-instance-2",
+)
+
+# may be I can't use NATS request inside NATS reply??
+r = NATS.request(String, connection, msgMeta[:sendTopic], JSON3.write(outgoingMsg); timer=Timer(10))
+
+
+
+
+

etc.jl

@@ -0,0 +1,293 @@
+
+
+
+using LibPQ, JSON, Graphs, DataFrames
+config = JSON.parsefile("./appconfig.json")
+host_url, _port = split(config["externalservice"]["sommpanion_db"]["url"], ':')
+port = parse(Int, _port)
+dbname = "winedb"
+user = config["externalservice"]["sommpanion_db"]["user"]
+password = config["externalservice"]["sommpanion_db"]["password"]
+pg_conn_str = "host=$host_url port=$port dbname=$dbname user=$user password=$password"
+db_connection = LibPQ.Connection(pg_conn_str)
+
+
+
+
+
+# ---------------------------------------------- 100 --------------------------------------------- #
+
+
+using LibPQ
+using DataFrames
+
+"""
+    extract_column_metadata(pg_conn_str::String) -> DataFrame
+
+Queries PostgreSQL system catalogs to extract a rich semantic text map of every 
+column in the database. Returns a DataFrame designed for vector embedding generation.
+"""
+function extract_column_metadata(pg_conn_str::String)
+    conn = LibPQ.Connection(pg_conn_str)
+    
+    # This direct SQL query pulls the column specifications along with column-level descriptions
+    query = """
+        SELECT 
+            c.relname AS table_name,
+            a.attname AS column_name,
+            format_type(a.atttypid, a.atttypmod) AS data_type,
+            COALESCE(d.description, '') AS column_description,
+            CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key,
+            CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key
+        FROM pg_attribute a
+        JOIN pg_class c ON c.oid = a.attrelid
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        -- Join to fetch column comments/descriptions
+        LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
+        -- Check if column is part of a Primary Key
+        LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid 
+            AND pk.contype = 'p' 
+            AND a.attnum = ANY(pk.conkey)
+        -- Check if column is part of a Foreign Key
+        LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid 
+            AND fk.contype = 'f' 
+            AND a.attnum = ANY(fk.conkey)
+        WHERE 
+            n.nspname = 'public'          -- Only user schemas
+            AND c.relkind = 'r'            -- Only standard tables
+            AND a.attnum > 0               -- Skip system hidden columns
+            AND NOT a.attisdropped;        -- Skip dropped columns
+    """
+    
+    try
+        # Execute and format into a clean DataFrame
+        result = execute(conn, query)
+        df = DataFrame(result)
+        
+        # Create a unique document ID for each vector row
+        df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)]
+        
+        return df
+    finally
+        close(conn)
+    end
+end
+
+
+
+"""
+    generate_embedding_payloads(df::DataFrame) -> Vector{Dict}
+
+Transforms the metadata DataFrame into structured text strings optimal for 
+vector space mapping.
+"""
+function generate_embedding_payloads(df::DataFrame)
+    payloads = Dict[]
+    
+    for row in eachrow(df)
+        # 1. Build a rich text description summarizing the column's role
+        text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)"
+        
+        if row.is_primary_key
+            text_payload *= " [PRIMARY KEY]"
+        end
+        if row.is_foreign_key
+            text_payload *= " [FOREIGN KEY RELATIONAL LINK]"
+        end
+        
+        # Append business descriptions if they exist in the database comments
+        if !isempty(strip(row.column_description))
+            text_payload *= " | Description: $(row.column_description)"
+        else
+            text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture."
+        end
+        
+        # 2. Package everything neatly to be passed to your vector store client
+        push!(payloads, Dict(
+            "id" => row.vector_id,
+            "text_content" => text_payload,
+            "metadata" => Dict(
+                "table" => row.table_name,
+                "column" => row.column_name,
+                "type" => row.data_type
+            )
+        ))
+    end
+    
+    return payloads
+end
+
+
+"""
+    resolve_semantic_cluster(vector_hits::Vector{String}, g::SimpleGraph, table_to_id::Dict{String, Int}, id_to_table::Dict{Int, String}) -> Vector{String}
+
+Takes a scattered array of semantically matched tables from Stage 1, navigates 
+the undirected network structure, and isolates the minimum interconnected subgraph 
+required to weave ALL hits into a single valid SQL query.
+"""
+function resolve_semantic_cluster(
+    vector_hits::Vector{String}, 
+    g::SimpleGraph, 
+    table_to_id::Dict{String, Int}, 
+    id_to_table::Dict{Int, String}
+)
+    # Filter out hits that don't exist in our actual database graph mapping
+    valid_node_ids = Int[]
+    for hit in vector_hits
+        if haskey(table_to_id, hit)
+            push!(valid_node_ids, table_to_id[hit])
+        else
+            @warn "Vector hit '$hit' does not map to an existing database table."
+        end
+    end
+    
+    unique!(valid_node_ids)
+    
+    # Edge Case Handlers
+    if isempty(valid_node_ids)
+        return String[]
+    elseif length(valid_node_ids) == 1
+        return [id_to_table[valid_node_ids[1]]]
+    end
+
+    # The Isolated Subgraph Set to build our final context
+    schema_subgraph_nodes = Set{Int}()
+    
+    # Phase A: Select an initial anchor component. We use the highest-ranked vector hit.
+    anchor_node = valid_node_ids[1]
+    push!(schema_subgraph_nodes, anchor_node)
+    
+    # Phase B: Sequentially route paths to all other semantic coordinates
+    for target_node in valid_node_ids[2:end]
+        # Skip if an earlier loop trajectory already naturally absorbed this table
+        if target_node in schema_subgraph_nodes
+            continue
+        end
+        
+        # Calculate the shortest path tree from the CURRENT state of our subgraph
+        # We find the shortest path from the target back to ANY node currently in our tree
+        shortest_paths = dijkstra_shortest_paths(g, target_node)
+        
+        # Find which node currently in our subgraph is closest to the target node
+        closest_subgraph_node = 0
+        min_distance = Inf
+        
+        for subgraph_node in schema_subgraph_nodes
+            dist = shortest_paths.dists[subgraph_node]
+            if dist < min_distance
+                min_distance = dist
+                closest_subgraph_node = subgraph_node
+            end
+        end
+        
+        # Reconstruct the path from the target node to the closest point on our existing tree
+        if closest_subgraph_node != 0
+            curr = closest_subgraph_node
+            while curr != 0
+                push!(schema_subgraph_nodes, curr)
+                curr = shortest_paths.parents[curr]
+                if curr == target_node
+                    push!(schema_subgraph_nodes, target_node)
+                    break
+                end
+            end
+        end
+    end
+    
+    # Map the unique structural nodes back to clean table names
+    return [id_to_table[node_id] for node_id in schema_subgraph_nodes]
+end
+
+function get_embedding(nats_conn::NATS.Connection, text::AbstractArray{String})
+  documents_dict = Dict("documents" => text)
+  payloads = [("documents", documents_dict, "dictionary")]
+  _, msg_envelope_json_str = msghandler.smartpack(
+    config["externalservice"]["servicesloadbalancer"]["nats"], 
+    payloads;   
+    msg_purpose="embedding",  
+    broker_url=config["nats_server_info"]["url"],
+    fileserver_url=config["externalservice"]["fileserver"]["url"])
+
+  reply = NATS.request(nats_conn, 
+                      config["externalservice"]["servicesloadbalancer"]["nats"], 
+                      msg_envelope_json_str, timeout=120)
+  incoming_env_json_str = String(reply.payload)
+  incoming_env = msghandler.smartunpack(incoming_env_json_str)
+  embedding_response = incoming_env["payloads"][1][2]
+
+  return embedding_response
+end
+
+nats_conn  = NATS.connect(config["nats_server_info"]["url"])
+
+# Run the extractor
+metadata_df = extract_column_metadata(pg_conn_str)
+embedding_ready = generate_embedding_payloads(metadata_df)
+
+println(embedding_ready[1]["text_content"])
+# Output: "Table: join_table | Column: seller_id | Type: integer [PRIMARY KEY] [FOREIGN KEY RELATIONAL LINK] | Description: Links unique sellers to their corresponding product items."
+
+# use only text content
+embedding_ready_2 = [i["text_content"] for i in embedding_ready]
+table_embedding = get_embedding(nats_conn, embedding_ready_2)
+
+user_question = 
+  """
+  Retrieves ["winery", "wine_name", "wine_id", "vintage", "region", "country", "wine_type", "grape", "serving_temperature", "sweetness", "intensity", "tannin", "acidity", "tasting_notes", "price", "currency", "image_url", "retailer_name", "retailer_id"] of wines that match the following criteria - {wine_name: Montrachet Grand Cru, winery: Domaine Jacques Prieur, region: Montrachet, country: France, , retailer_name: Yiem Wines Ltd, retailerid: f54eab6b-7650-4448-b009-c53f3efbcc3b}
+  """
+user_question_embedding = get_embedding(nats_conn, [user_question])
+
+
+using Distances
+# similarity = 1 - Distances.cosine_dist(Float64.(table_embedding["data"][1]["embedding"]), 
+#                             Float64.(user_question_embedding["data"][1]["embedding"])
+#                             )
+user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"])
+user_question_similarity = []
+for i in table_embedding["data"]
+  i_data = i["embedding"]
+  i_float = Float64.(i_data)
+  r = 1 - Distances.cosine_dist(i_float, user_question_embedding)
+  push!(user_question_similarity, r)
+end
+
+new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity))
+sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min
+
+# top 20 of sorted_df get this tables
+vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"]
+
+g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str)
+
+# tables that I should put schema in LLM context
+optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
+
+
+
+function find_related_tables_for_user_question(question::String)
+  metadata_df = extract_column_metadata(pg_conn_str)
+  embedding_ready = generate_embedding_payloads(metadata_df)
+
+  # use only text content
+  embedding_ready_2 = [i["text_content"] for i in embedding_ready]
+  table_embedding = get_embedding(nats_conn, embedding_ready_2)
+  user_question_embedding = Float64.(user_question_embedding["data"][1]["embedding"])
+  user_question_similarity = []
+  for i in table_embedding["data"]
+    i_data = i["embedding"]
+    i_float = Float64.(i_data)
+    r = 1 - Distances.cosine_dist(i_float, user_question_embedding)
+    push!(user_question_similarity, r)
+  end
+
+  new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity))
+  sorted_df = sort(new_df, :user_question_similarity, rev=true) # sort max to min
+
+  #WORKING extract top 20 rows of sorted_df to get tables that related to user question
+  vector_hits = ["retailer_wine", "wine", "wine_food", "retailer"]
+
+  g, id_to_table, table_to_id = harvest_undirected_schema_graph(pg_conn_str)
+
+  # tables that I should put schema in LLM context
+  return optimized_context = resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
+end

src/GeneralUtils.jl

@@ -2,7 +2,7 @@ module GeneralUtils
     
 
 export  # struct
-        mqttClientInstance,
+        # mqttClientInstance,
         # function
         noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue 
 
@@ -24,16 +24,7 @@ using .interface
 
 #------------------------------------------------------------------------------------------------100
 
-""" version 0.0.4
-    Todo:
-        - [*1] cartesianAssign for different matrix dimension
 
-    Change from version: 0.0.3
-        - 
-    
-    All features
-        
-"""
 
 
 

src/dbUtil.jl

@@ -2,13 +2,45 @@ module dbUtil
   
 export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
 
-using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames,
+using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
       SHA
 using ..util
-
+#[PENDING] update code to use JSON
 # ---------------------------------------------- 100 --------------------------------------------- #
 
 
+"""
+dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format
+
+This function takes a dictionary and converts it into a PostgreSQL-compatible key-value string
+format suitable for storage in a TEXT field. The output format uses curly braces with comma-separated
+key-value pairs, where string values are quoted.
+
+# Function Workflow:
+1. Iterates through dictionary key-value pairs
+2. Handles nested dictionaries by recursively converting them
+3. Wraps string values in double quotes
+4. Formats numeric and other values without quotes
+5. Returns a PostgreSQL-compatible key-value string enclosed in curly braces
+
+# Arguments:
+  - `dict::Dict` - Dictionary containing key-value pairs to convert
+
+# Return:
+  - A String in PostgreSQL key-value format: "{key1: value1, key2: \"value2\", ...}"
+
+# Example
+```jldoctest
+julia> data = Dict{String, Any}(
+           "name" => "John",
+           "age" => 30,
+           "city" => "New York"
+       );
+
+julia> dictToPostgresKeyValueString(data)
+"{\"name\": \"John\", \"age\": 30, \"city\": \"New York\"}"
+```
+"""
 function dictToPostgresKeyValueString(dict)
   parts = []
   for (k, v) in dict
@@ -26,61 +58,71 @@ end
 
 
 
-""" Get characters between specified characters.
+""" generateInsertSQL - Generate SQL INSERT statement from dictionary data
 
-# Arguments
-  - `text::T`
-    a text being searched
-  - `startChar::Char`
-    start character
-  - `endChar::Char`
-    end character
-# Keyword Arguments
-  - `endCharLocation::String`
-    end character position after startChar. Can be "next" or "end". "next" means the closed 
-    endChar just after startChar. "end" means the furthest endChar. 
-  - `includeChar::Bool` 
-    whether to include the startChar and endChar. Default is true
-# Return
-  the characters between specified characters.
+This function constructs a SQL INSERT statement by extracting values for specified columns
+from a dictionary and formatting them into a valid PostgreSQL INSERT query.
+
+# Function Workflow:
+1. Iterates through the dictionary key-value pairs
+2. Filters keys to only include those present in `columnToInsert`
+3. Collects column names and their corresponding values
+4. Constructs the final SQL INSERT statement
+
+# Arguments:
+  - `table_name::String` - Name of the database table to insert into
+  - `columnToInsert::Vector{Symbol}` - List of column names to include in the INSERT statement
+  - `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs for the insert
+
+# Return:
+  - A String containing the SQL INSERT statement
 
 # Example
 ```jldoctest
-julia> using Revise
-julia> using GeneralUtils
-julia> insert_data = Dict(
-      :grape => "NA",
-      :acidity => "0",
-      :tannin => "0",
-      :country => "NA",
-      :description => "NA",
-      :region => "NA",
-      :winery => "ccc",
-      :intensity => "0",
-      :sweetness => "0",
-      :tasting_notes => "NA",
-      :wine_name => "new_wine",
-      :wine_id => "9e1deb6a-d57f-4d2c-abbe-da813f4e91ad",
-      :wine_type => "NA",
-      :other_attributes => "{\"attribute3\":{\"attribute5\":666,\"attribute4\":\"text\"},\"attribute1\":\"hello world\",\"attribute2\":555}",
-      :fizziness => "0",
-      :serving_temperature => "0",
-      :additional_search_term => "{NA1,NA2}")
-```
-# TODO
-  - [] update docs
+julia> using UUIDs
 
-# Signature
+# Insert a single record with specific columns
+table_name = "wine"
+columnToInsert = [:acidity, :tannin, :country, :region, :winery]
+data = Dict{Symbol, Any}(
+    :grape => "Cabernet Sauvignon",
+    :acidity => "medium",                 # using descriptive scale (low/medium/full)
+    :tannin => "medium-plus",             # common wine descriptor
+    :country => "France",
+    :description => "A rich and structured red wine with notes of blackcurrant, cedar, and subtle oak.",
+    :region => "Bordeaux",
+    :winery => "Château Margaux",
+    :intensity => "medium",               # intensity is usually low/medium/full
+    :sweetness => "dry",                  # dry/medium-dry/medium/medium-sweet/sweet
+    :tasting_notes => "Blackberry, graphite, tobacco, vanilla, and subtle earth.",
+    :wine_name => "Château Margaux Grand Cru",
+    :wine_id => "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a",  # UUID-like (valid hex)
+    :wine_type => "Red",
+    :other_attributes => Dict{String, Any}(
+        "vintage" => 2018,
+        "alcohol_percent" => 13.5,
+        "ph" => 3.6,
+        " aging_years" => 24,              # years in barrel
+        " producer_code" => "CM-GRAND"
+    ),
+    :fizziness => "still",
+    :serving_temperature => "16–18°C",
+    :additional_search_term => ["Cabernet", "Bordeaux red", "premium wine", "CabSav"]
+)
+
+julia> generateInsertSQL(table_name, columnToInsert, data)
+"INSERT INTO wine (acidity, tannin, country, region, winery) VALUES ('medium', 'medium-plus', 'France', 'Bordeaux', 'Château Margaux');"
+```
 """
-function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, 
-  insert_data::Dict{Symbol, Any})
+function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, data::Dict{Symbol, Any})
   columns = String[]
   values = String[]
 
-  for (key, value) in insert_data
+  for (key, value) in data
     if key ∈ columnToInsert
       push!(columns, string(key))
-      push!(values, "'$value'") #[] number should not wrapped in ''
+      value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+      push!(values, value_str)
     end
   end
   
@@ -89,113 +131,101 @@ function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol},
   
   return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
 end
-# function generateInsertSQL(table_name::String, insert_data::Dict{Symbol, Any})
-#   columns = String[]
-#   values = String[]
 
-#   for (key, value) in insert_data
-#       push!(columns, string(key))
-#       if key == :other_attributes
-#           push!(values, "'$value'")
-#       else
-#           push!(values, "'$value'")
-#       end
-#   end
+function generateInsertSQL(table_name::String, data::AbstractDict{String, Any})
+  columns = String[]
+  values = String[]
 
-#   columns_str = join(columns, ", ")
-#   values_str = join(values, ", ")
-  
-#   return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
-# end
-
-  
-
-
-
-"""
-example:
-
-insert_data = Dict(
-    :grape => "NA",
-    :acidity => "0",
-    :tannin => "0",
-    :country => "NA",
-    :description => "NA",
-    :region => "NA",
-    :winery => "ccc",
-    :intensity => "0",
-    :sweetness => "0",
-    :tasting_notes => "NA",
-    :wine_name => "new_wine",
-    :wine_id => "9e1deb6a-d57f-4d2c-abbe-da813f4e91ad",
-    :wine_type => "NA",
-    :other_attributes => "{\"attribute3\":{\"attribute5\":666,\"attribute4\":\"text\"},\"attribute1\":\"hello world\",\"attribute2\":555}",
-    :fizziness => "0",
-    :serving_temperature => "0",
-    :additional_search_term => "{NA1,NA2}")
-
-id_keys is the primary key columns
-"""
-# function generateUpdateSQL(table_name::String, update_data::Dict{Symbol, Any}, id_keys::Vector{Symbol})
-#   set_clauses = String[]
-#   where_clauses = String[]
-  
-#   for (key, value) in update_data
-#       if key in id_keys
-#           push!(where_clauses, "$key = '$value'")
-#       else
-#           if key == :other_attributes
-#               push!(set_clauses, "$key = '$value'")
-#           else
-#               push!(set_clauses, "$key = '$value'")
-#           end
-#       end
-#   end
-  
-#   set_clause = join(set_clauses, ", ")
-#   where_clause = join(where_clauses, " AND ")
-  
-#   return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
-# end
-
-function generateUpdateSQL(table_name::String, columnToUpdate::Vector{Symbol}, 
-  updatedata::Dict{Symbol, Any}, id_keys::Vector{Symbol})
-  
-  set_clauses = String[]
-  where_clauses = String[]
-  
-  for (key, value) in updatedata
-      if key in id_keys
-          push!(where_clauses, "$key = '$value'")
-      else
-        if key ∈ columnToUpdate # update only specified columns
-          push!(set_clauses, "$key = '$value'")
-        end
-      end
+  for (key, value) in data
+    push!(columns, string(key))
+    value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+    push!(values, value_str)
   end
   
-  set_clause = join(set_clauses, ", ")
-  where_clause = join(where_clauses, " AND ")
+  columns_str = join(columns, ", ")
+  values_str = join(values, ", ")
     
-  return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
+    return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
 end
 
 
+# ---------------------------------------------- 100 --------------------------------------------- #
 
+""" generateUpdateSQL - Generate SQL UPDATE statement from dictionary data
 
+This function constructs a SQL UPDATE statement by updating multiple columns
+based on a primary key condition.
 
+# Arguments:
+  - `table_name::String` - Name of the database table to update
+  - `pk_column::Symbol` - The primary key column name
+  - `pk_value` - The primary key value (used in WHERE clause)
+  - `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs to update
 
+# Return:
+  - A String containing the SQL UPDATE statement
 
+# Example
+```jldoctest
+julia> using UUIDs
 
+# Update multiple columns using a dictionary
+table_name = "wine"
+pk_column = :wine_id
+pk_value = "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a"
+data = Dict{Symbol, Any}(
+    :acidity => "full",
+    :tannin => "medium",
+    :country => "Italy"
+)
 
+julia> generateUpdateSQL(table_name, pk_column, pk_value, data)
+"UPDATE wine SET acidity = 'full', tannin = 'medium', country = 'Italy' WHERE wine_id = '8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a';"
+```
+"""
+function generateUpdateSQL(table_name::String, pk_column::String, pk_value, 
+                          data::AbstractDict{String, Any})
+    # Build SET clause
+    set_parts = String[]
+    for (key, value) in data
+        if key ∉ [pk_column]
+          value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+          push!(set_parts, "$(string(key)) = $value_str")
+        end
+    end
     
+    set_clause = join(set_parts, ", ")
     
+    # Handle primary key value
+    pk_val_str = isa(pk_value, AbstractString) ? "'$pk_value'" : "$pk_value"
     
+    return "UPDATE $table_name SET $set_clause WHERE $pk_column = $pk_val_str;"
+end
 
+function generateUpdateSQL(table_name::String, pk_dict::AbstractDict{String, Any}, 
+                          data::AbstractDict{String, Any})
+    # Build SET clause
+    set_parts = String[]
+    for (key, value) in data
+        if key ∉ keys(pk_dict)
+            value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+            push!(set_parts, "$(string(key)) = $value_str")
+        end
+    end
     
+    set_clause = join(set_parts, ", ")
     
+    # Build WHERE clause for composite keys
+    where_parts = String[]
+    for (col, val) in pk_dict
+        val_str = isa(val, AbstractString) ? "'$val'" : "$val"
+        push!(where_parts, "$(string(col)) = $val_str")
+    end
     
+    where_clause = join(where_parts, " AND ")
     
+    return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
+end
 
 
 end   # module

src/llmUtil.jl

@@ -1,95 +1,16 @@
 module llmUtil
 
-export formatLLMtext, formatLLMtext_llama3instruct, jsoncorrection
+export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response,
+        extract_column_metadata, generate_embedding_payloads, resolve_semantic_cluster,
+        harvest_entity_catalog, resolve_entity, harvest_db_undirected_schema_graph
 
-using UUIDs, JSON3, Dates
-using GeneralUtils
+using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs, LibPQ
+using ..util
 
 # ---------------------------------------------- 100 --------------------------------------------- #
 
-
-""" Convert a single chat dictionary into LLM model instruct format.
-
-# Llama 3 instruct format example
-  <|begin_of_text|>
-  <|start_header_id|>system<|end_header_id|>
-  You are a helpful assistant.
-  <|eot_id|>
-  <|start_header_id|>user<|end_header_id|>
-  Get me an icecream.
-  <|eot_id|>
-  <|start_header_id|>assistant<|end_header_id|>
-  Go buy it yourself at 7-11.
-  <|eot_id|>
-
-# Arguments
-  - `name::T`
-    message owner name e.f. "system", "user" or "assistant"
-  - `text::T`
-
-# Return
-  - `formattedtext::String`
-    text formatted to model format
-
-# Example
-```jldoctest
-julia> using Revise
-julia> using YiemAgent
-julia> d = Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",)
-julia> formattedtext = YiemAgent.formatLLMtext_llama3instruct(d[:name], d[:text])
-"<|begin_of_text|>\n  <|start_header_id|>system<|end_header_id|>\n  You are a helpful, respectful and honest assistant.\n  <|eot_id|>\n"
-```
-
-Signature
-"""
-function formatLLMtext_llama3instruct(name::T, text::T;
-  assistantStarter::Bool=true) where {T<:AbstractString}
-  formattedtext = 
-  if name == "system"
-    """
-    <|start_header_id|>$name<|end_header_id|>
-    $text
-    <|eot_id|>
-    """
-  else
-    """
-    <|start_header_id|>$name<|end_header_id|>
-    $text
-    <|eot_id|>
-    """
-  end
-
-  if assistantStarter
-    formattedtext *=
-      """
-      <|start_header_id|>assistant<|end_header_id|>
-      """
-  end
-
-  return formattedtext
-end
-# function formatLLMtext_llama3instruct(name::T, text::T) where {T<:AbstractString}
-#   formattedtext = 
-#   if name == "system"
-#     """
-#     <|begin_of_text|>
-#     <|start_header_id|>$name<|end_header_id|>
-#     $text
-#     <|eot_id|>
-#     """
-#   else
-#     """
-#     <|start_header_id|>$name<|end_header_id|>
-#     $text
-#     <|eot_id|>
-#     """
-#   end
-
-#   return formattedtext
-# end
-
-function formatLLMtext_qwen(name::T, text::T;
-  assistantStarter::Bool=true) where {T<:AbstractString}
+function formatLLMtext_qwen3(name::T, text::T;
+  assistantStarter::Bool=false) where {T<:AbstractString}
   formattedtext = 
   if name == "system"
     """
@@ -116,14 +37,13 @@ function formatLLMtext_qwen(name::T, text::T;
 end
 
 
-""" Convert a chat messages in vector of dictionary into LLM model instruct format.
+""" Convert a vector of chat message dictionaries into LLM model instruct format.
 
 # Arguments
   - `messages::Vector{Dict{Symbol, T}}`
-    message owner name e.f. "system", "user" or "assistant"
+    A vector of dictionaries where each dictionary contains the keys `:name` (the name of the message owner) and `:text` (the text of the message).
   - `formatname::T`
-    format name to be used
-
+    The name of the format to be used for converting the chat messages.
 # Return
   - `formattedtext::String`
     text formatted to model format
@@ -131,147 +51,699 @@ end
 # Example
 ```jldoctest
 julia> using Revise
-julia> using YiemAgent
+julia> using GeneralUtils
 julia> chatmessage = [
           Dict(:name=> "system",:text=> "You are a helpful, respectful and honest assistant.",),
           Dict(:name=> "user",:text=> "list me all planets in our solar system.",),
           Dict(:name=> "assistant",:text=> "I'm sorry. I don't know. You tell me.",),
         ]
-julia> formattedtext = YiemAgent.formatLLMtext(chatmessage, "llama3instruct")
+julia> formattedtext = GeneralUtils.formatLLMtext(chatmessage, "llama3instruct")
 "<|begin_of_text|>\n  <|start_header_id|>system<|end_header_id|>\n  You are a helpful, respectful and honest assistant.\n  <|eot_id|>\n  <|start_header_id|>user<|end_header_id|>\n  list me all planets in our solar system.\n  <|eot_id|>\n  <|start_header_id|>assistant<|end_header_id|>\n  I'm sorry. I don't know. You tell me.\n  <|eot_id|>\n"
 ```
-
-# Signature
 """
-function formatLLMtext(messages::Vector{Dict{Symbol, T}}; formatname::String="llama3instruct"
-  )::String where {T<:Any}
-  f = if formatname == "llama3instruct"
-    formatLLMtext_llama3instruct
-  elseif formatname == "mistral"
-    # not define yet
-  elseif formatname == "phi3instruct"
-    # not define yet
-  elseif formatname == "qwen"
-    formatLLMtext_qwen
-  else
-    error("$formatname template not define yet")
-  end
+function formatLLMtext(messages::Vector{Dict{Symbol, T}}, formatname::String
+  )::String where {T<:AbstractString}
+  f = 
+    if formatname == "llama3instruct"
+      formatLLMtext_llama3instruct
+    elseif formatname == "mistral"
+      # not define yet
+    elseif formatname == "phi3instruct"
+      # not define yet
+    elseif formatname == "qwen"
+      formatLLMtext_qwen
+    elseif formatname == "qwen3"
+      formatLLMtext_qwen3
+    elseif formatname == "phi4"
+      formatLLMtext_phi4
+    elseif formatname == "granite3"
+      formatLLMtext_granite3
+    else
+      error("$formatname template not define yet")
+    end
 
   str = ""
-  for t in messages
-    str *= f(t[:name], t[:text])
+  for (i, t) in enumerate(messages)
+    if i < length(messages)
+      str *= f(t[:name], t[:text])
+    else
+      str *= f(t[:name], t[:text]; assistantStarter=true)
+    end
   end
 
   return str
 end
 
 
-""" Attemp to correct LLM response's incorrect JSON response.
-
-# Arguments
-  - `a::T1`
-    one of Yiem's agent
-  - `input::T2`
-    text to be send to virtual wine customer
-  
-# Return
-  - `correctjson::String`
-    corrected json string  
-
-# Example
-```jldoctest
-julia> 
-```
-
-# Signature
-"""
-function jsoncorrection(config::T1, input::T2, correctJsonExample::T3;
-  maxattempt::Integer=3
-  ) where {T1<:AbstractDict, T2<:AbstractString, T3<:AbstractString}
-
-  incorrectjson = deepcopy(input)
-  correctjson = nothing
-
-  for attempt in 1:maxattempt
-    try
-      d = copy(JSON3.read(incorrectjson))
-      correctjson = incorrectjson
-      return correctjson
-    catch e
-      @warn "Attempting to correct JSON string. Attempt $attempt"
-      e = """$e"""
-      if occursin("EOF", e)
-        e = split(e, "EOF")[1] * "EOF"
-      end
-      incorrectjson = deepcopy(input)
-      _prompt =
-      """
-      Your goal are:
-      1) Use the expected JSON format as a guideline to check why the given JSON string failed to load and provide a corrected version that can be loaded by Python's json.load function.
-      2) Provide Corrected JSON string only. Do not provide any other info.
-
-      $correctJsonExample
-
-      Let's begin!
-      Given JSON string: $incorrectjson
-      The given JSON string failed to load previously because: $e
-      Corrected JSON string:
-      """
-
-      # apply LLM specific instruct format
-      externalService = config[:externalservice][:text2textinstruct]
-      llminfo = externalService[:llminfo]
-      prompt = 
-      if llminfo[:name] == "llama3instruct"
-        formatLLMtext_llama3instruct("system", _prompt)
-      else
-        error("llm model name is not defied yet $(@__LINE__)")
-      end
-
-      # send formatted input to user using GeneralUtils.sendReceiveMqttMsg
-      msgMeta = GeneralUtils.generate_msgMeta(
-        externalService[:mqtttopic],
-        senderName= "jsoncorrection",
-        senderId= uuid4snakecase(),
-        receiverName= "text2textinstruct",
-        mqttBroker= config[:mqttServerInfo][:broker],
-        mqttBrokerPort= config[:mqttServerInfo][:port],
-      )
-
-      outgoingMsg = Dict(
-        :msgMeta=> msgMeta,
-        :payload=> Dict(
-          :text=> prompt,
-          :kwargs=> Dict(
-            :max_tokens=> 512,
-            :stop=> ["<|eot_id|>"],
-            )
-          )
-        )
-      result = GeneralUtils.sendReceiveMqttMsg(outgoingMsg; timeout=120)
-      incorrectjson = result[:response][:text]
+function extractthink(text::String)
+  think = nothing
+  str = nothing
+  if occursin("<think>", text)
+    r = GeneralUtils.extractTextBetweenString(text, "<think>", "</think>")
+    if r[:success]
+      think = r[:text]
     end
+    str = string(split(text, "</think>")[2])
+  else
+    str = text
   end
+  return think, str
 end
 
 
 
+""" Validate that an agent's JSON response contains all required keys and no extra keys.
+
+The function checks if `responsedict` contains exactly the keys specified in `requiredKeys`
+with no duplicates and no missing keys. It is designed to validate structured agent responses
+against an expected schema.
+
+# Arguments
+- `responsedict::Dict`
+  A dictionary containing the agent's JSON response. Must be a plain `Dict` or
+  dictionary-like object with String keys.
+- `requiredKeys::T` where `T<:Array{String}`
+  An array of required key names that must be present in `responsedict`.
+
+# Return
+- `Tuple{Bool, Union{String, Nothing}}`
+  A tuple where the first element indicates whether validation passed (`true`) or failed (`false`),
+  and the second element contains an error message if validation failed, or `nothing` if it passed.
+
+# Details
+The validation logic checks:
+1. **Duplicate keys**: If `responsedict` contains more keys than `requiredKeys`, validation fails
+   because the agent included extra/unexpected keys.
+2. **Missing keys**: If any key in `requiredKeys` is absent from `responsedict`, validation fails
+   and the specific missing keys are listed in the error message.
+3. **Valid response**: If all required keys are present and no extra keys exist, validation passes.
+
+# Example
+
+```julia
+julia> using GeneralUtils
+julia> requiredKeys = ["wine_name", "price", "rating"]
+julia> response = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98)
+julia> GeneralUtils.checkAgentResponse_JSON(response, requiredKeys)
+(true, nothing)
+
+julia> response_missing = Dict("wine_name"=>"Château Margaux", "price"=>250.0)
+julia> GeneralUtils.checkAgentResponse_JSON(response_missing, requiredKeys)
+(false, "rating are missing from your previous response")
+
+julia> response_extra = Dict("wine_name"=>"Château Margaux", "price"=>250.0, "rating"=>98, "extra_field"=>"data")
+julia> GeneralUtils.checkAgentResponse_JSON(response_extra, requiredKeys)
+(false, "Your previous attempt has duplicated points according to the required response format")
+```
+"""
+function checkAgentResponse_JSON(responsedict::T1, requiredKeys::T2
+  )::Tuple where {T1<:AbstractDict, T2<:Array{String}}
+  _responsedictKey = keys(responsedict)
+  responsedictKey = [i for i in _responsedictKey] # convert into a list
+  is_requiredKeys_in_responsedictKey = [i ∈ responsedictKey for i in requiredKeys]
+  ispass = false
+  errormsg = nothing
+  if length(is_requiredKeys_in_responsedictKey) > length(requiredKeys)
+    errormsg = "Your previous attempt has duplicated points according to the required response format"
+    ispass = false
+  elseif !all(is_requiredKeys_in_responsedictKey)
+    zeroind = findall(x -> x == 0, is_requiredKeys_in_responsedictKey)
+    missingkeys = [requiredKeys[i] for i in zeroind]
+    errormsg = "$missingkeys are missing from your previous response"
+    ispass = false
+  else
+    ispass = true
+  end
+  return (ispass, errormsg)
+end
+
+
+""" Convert a plain text string containing key-value pairs into a JSON-formatted string.
+
+This function takes text containing key-value pairs (typically extracted from LLM responses)
+and wraps them in proper JSON braces to create a valid JSON string. It cleans the input
+by removing common formatting artifacts like braces, code block markers, and language
+specifiers before wrapping the content.
+
+# Arguments
+- `text::String`
+  A string containing key-value pairs, typically in the format `key: value, key: value`.
+  May contain leading/trailing braces, code block markers (```), or language specifiers
+  (e.g., `json`) that will be removed.
+
+# Return
+- `String`
+  A JSON-formatted string with the key-value pairs wrapped in `{}` braces.
+
+# Notes
+- The function removes `{`, `}`, `````, and `json` from the input before wrapping.
+- The output is always wrapped in curly braces to create valid JSON structure.
+- This is typically used to sanitize LLM responses that contain key-value data
+  but may include formatting artifacts.
+
+# Examples
+```jldoctest
+julia> text = "thought: Hello, action: CHATBOX, input: How can I help?"
+julia> clean_json_response(text)
+"{thought: Hello, action: CHATBOX, input: How can I help?}"
+
+julia> text = "{thought: Hello, action: CHATBOX}"
+julia> clean_json_response(text)
+"{thought: Hello, action: CHATBOX}"
+
+julia> text = "```json{thought: Hello, action: CHATBOX}```"
+julia> clean_json_response(text)
+"{thought: Hello, action: CHATBOX}"
+```
+"""
+function clean_json_response(text::String)
+  removelist = ["{", "}", "```", "json", "\n"]
+  return '{' * removestring(text, removelist) * '}'
+end
 
 
 
+""" Harvest database schema as undirected graph from PostgreSQL.
+
+Extracts table structure and relationships from a PostgreSQL database by querying
+system catalogs to build a graph representation of tables and their relationships.
+
+# Arguments
+- `pg_conn_str::String`
+  PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret")
+
+# Return
+- `g::SimpleGraph`: An undirected graph where nodes represent tables and edges represent
+  relationships (foreign keys or shared ID column patterns).
+- `id_to_table::Dict{Int, String}`: Maps numerical node IDs (1..n) to actual table names.
+- `table_to_id::Dict{String, Int}`: Reverse mapping from table names to graph node IDs.
+
+# Details
+The function extracts schema information using two strategies:
+1. **Explicit Foreign Keys**: Queries `pg_constraint` for actual foreign key relationships
+2. **Fallback Name Matching**: Infers relationships from shared column naming patterns
+   (e.g., `seller_id`, `product_id` columns across tables)
+
+# Example
+```julia
+julia> using GeneralUtils
+julia> pg_conn_str = "host=localhost port=5432 dbname=winedb user=admin password=secret"
+julia> g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str)
+julia> vertices(g)
+10
+```
+"""
+function harvest_db_undirected_schema_graph(pg_conn_str)   
+  conn = LibPQ.Connection(pg_conn_str) 
+    try
+      # 1. Fetch all user tables
+      table_query = """
+          SELECT c.relname AS table_name, c.oid
+          FROM pg_class c
+          JOIN pg_namespace n ON n.oid = c.relnamespace
+          WHERE n.nspname = 'public' AND c.relkind = 'r';
+      """
+      table_df = DataFrame(execute(conn, table_query))
+      table_names = table_df.table_name
+      
+      num_tables = length(table_names)
+      table_to_id = Dict{String, Int}(name => i for (i, name) in enumerate(table_names))
+      id_to_table = Dict{Int, String}(i => name for (i, name) in enumerate(table_names))
+      
+      # CRITICAL: Use SimpleGraph (Undirected) so pathfinding can traverse both ways
+      g = SimpleGraph(num_tables)
+      
+      # 2. Strategy A: Extract Explicit Foreign Keys
+      fk_query = """
+          SELECT 
+              conrelid::regclass::text AS source_table,
+              confrelid::regclass::text AS target_table
+          FROM pg_constraint c
+          JOIN pg_namespace n ON n.oid = c.connamespace
+          WHERE c.contype = 'f' AND n.nspname = 'public';
+      """
+      fk_df = DataFrame(execute(conn, fk_query))
+      
+      for row in eachrow(fk_df)
+          src = split(replace(row.source_table, "\"" => ""), '.')[end]
+          tgt = split(replace(row.target_table, "\"" => ""), '.')[end]
+          
+          if haskey(table_to_id, src) && haskey(table_to_id, tgt)
+              add_edge!(g, table_to_id[src], table_to_id[tgt])
+          end
+      end
+      
+      # 3. Strategy B: Fallback Name Matching (For missing explicit constraints)
+      # Fetch all columns for all tables
+      col_query = """
+          SELECT a.attname AS column_name, c.relname AS table_name
+          FROM pg_attribute a
+          JOIN pg_class c ON c.oid = a.attrelid
+          JOIN pg_namespace n ON n.oid = c.relnamespace
+          WHERE n.nspname = 'public' AND c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped;
+      """
+      col_df = DataFrame(execute(conn, col_query))
+      
+      # Group columns by name to see who shares what fields
+      for gdf in groupby(col_df, :column_name)
+        col_name = gdf.column_name[1]
+        
+        # We only infer connections on ID fields (e.g., seller_id, product_id)
+        # Avoid matching generic names like 'id', 'created_at', or 'name'
+        if endswith(lowercase(col_name), "_id") && col_name != "id"
+          sharing_tables = gdf.table_name
+          
+          # Connect all tables that share this ID column
+          for i in eachindex(sharing_tables), j in (i+1):length(sharing_tables)
+            t1 = sharing_tables[i]
+            t2 = sharing_tables[j]
+            
+            if haskey(table_to_id, t1) && haskey(table_to_id, t2)
+              add_edge!(g, table_to_id[t1], table_to_id[t2])
+            end
+          end
+        end
+      end
+      
+      return g, id_to_table, table_to_id
+
+    finally
+      close(conn)
+    end
+end
 
 
 
+""" Extract vector metadata from PostgreSQL database.
+
+Queries PostgreSQL system catalogs to extract column metadata including table names,
+column names, data types, descriptions, and constraint information (primary/foreign keys).
+Returns a DataFrame designed for vector embedding generation.
+
+# Arguments
+- `pg_conn_str::String`
+  PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret")
+
+# Return
+- `DataFrame`
+  A DataFrame with columns:
+  - `table_name`: Name of the table
+  - `column_name`: Name of the column
+  - `data_type`: PostgreSQL data type with typemod
+  - `column_description`: Column's comment/description (empty string if none)
+  - `is_primary_key`: Boolean indicating if column is part of primary key
+  - `is_foreign_key`: Boolean indicating if column is part of foreign key
+  - `vector_id`: Generated unique identifier in format `col_{table_name}_{column_name}`
+
+# Example
+```julia
+julia> using GeneralUtils
+julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
+julia> df = GeneralUtils.extract_column_metadata(pg_conn)
+DataFrame
+6 rows × 7 columns
+table_name  column_name  data_type  column_description  is_primary_key  is_foreign_key  vector_id
+─────────────┬───────────┬───────────┬───────────────────┬───────────────┬───────────────┬────────────────────────
+users       id           integer    User ID             true            false         col_users_id
+users       name         text       User name           false           false         col_users_name
+users       email        text       User email          false           false         col_users_email
+products    id           integer    Product ID          true            false         col_products_id
+products    price        numeric    Product price       false           false         col_products_price
+products    user_id      integer    Reference to user   false           true          col_products_user_id
+```
+"""
+function extract_column_metadata(pg_conn_str::String)::DataFrame
+  conn = LibPQ.Connection(pg_conn_str)
+  return extract_column_metadata(conn)
+end
+
+function extract_column_metadata(conn::LibPQ.Connection)::DataFrame
+  # This direct SQL query pulls the column specifications along with column-level descriptions
+  query = """
+        SELECT 
+            c.relname AS table_name,
+            a.attname AS column_name,
+            format_type(a.atttypid, a.atttypmod) AS data_type,
+            COALESCE(d.description, '') AS column_description,
+            CASE WHEN pk.contype = 'p' THEN true ELSE false END AS is_primary_key,
+            CASE WHEN fk.contype = 'f' THEN true ELSE false END AS is_foreign_key
+        FROM pg_attribute a
+        JOIN pg_class c ON c.oid = a.attrelid
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        -- Join to fetch column comments/descriptions
+        LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
+        -- Check if column is part of a Primary Key
+        LEFT JOIN pg_constraint pk ON pk.conrelid = c.oid 
+            AND pk.contype = 'p' 
+            AND a.attnum = ANY(pk.conkey)
+        -- Check if column is part of a Foreign Key
+        LEFT JOIN pg_constraint fk ON fk.conrelid = c.oid 
+            AND fk.contype = 'f' 
+            AND a.attnum = ANY(fk.conkey)
+        WHERE 
+            n.nspname = 'public'          -- Only user schemas
+            AND c.relkind = 'r'            -- Only standard tables
+            AND a.attnum > 0               -- Skip system hidden columns
+            AND NOT a.attisdropped;        -- Skip dropped columns
+    """
+  
+  try
+      # Execute and format into a clean DataFrame
+      result = LibPQ.execute(conn, query)
+      df = DataFrame(result)
+      
+      # Create a unique document ID for each vector row
+      df.vector_id = ["col_\$(row.table_name)_\$(row.column_name)" for row in eachrow(df)]
+      
+      return df
+  finally
+      close(conn)
+  end
+end
 
 
+""" Generate embedding payloads from vector metadata DataFrame.
+
+Transforms the metadata DataFrame from `extract_column_metadata` into a vector of
+dictionaries structured for vector embedding storage and retrieval.
+
+# Arguments
+- `df::DataFrame`
+  A DataFrame with columns from `extract_column_metadata`: `table_name`, `column_name`,
+  `data_type`, `column_description`, `is_primary_key`, `is_foreign_key`, `vector_id`.
+
+# Return
+- `Vector{Dict}`
+  A vector of dictionaries with keys:
+  - `id`: The vector ID from `vector_id`
+  - `text_content`: A structured text string combining table, column, type, constraint
+    markers, and description
+  - `metadata`: A dictionary containing `table`, `column`, and `type`
+
+# Details
+The function constructs rich text payloads by:
+1. Building a base string with table name, column name, and data type
+2. Appending constraint markers `[PRIMARY KEY]` or `[FOREIGN KEY RELATIONAL LINK]`
+3. Adding column description if available, otherwise generating a default description
+
+# Example
+```julia
+julia> using GeneralUtils
+julia> pg_conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
+julia> df = GeneralUtils.extract_column_metadata(pg_conn)
+julia> payloads = GeneralUtils.generate_embedding_payloads(df)
+3-element Vector{Dict}:
+ Dict("id" => "col_users_id", "text_content" => "Table: users | Column: id | Type: integer [PRIMARY KEY] | Description: User ID", "metadata" => Dict("table" => "users", "column" => "id", "type" => "integer"))
+ Dict("id" => "col_users_name", "text_content" => "Table: users | Column: name | Type: text | Description: User name", "metadata" => Dict("table" => "users", "column" => "name", "type" => "text"))
+ Dict("id" => "col_users_email", "text_content" => "Table: users | Column: email | Type: text | Description: User email", "metadata" => Dict("table" => "users", "column" => "email", "type" => "text"))
+```
+"""
+function generate_embedding_payloads(df::DataFrame)::Vector{Dict}
+    payloads = Dict[]
+    
+    for row in eachrow(df)
+        # 1. Build a rich text description summarizing the column's role
+        text_payload = "Table: $(row.table_name) | Column: $(row.column_name) | Type: $(row.data_type)"
+        
+        if row.is_primary_key
+            text_payload *= " [PRIMARY KEY]"
+        end
+        if row.is_foreign_key
+            text_payload *= " [FOREIGN KEY RELATIONAL LINK]"
+        end
+        
+        # Append business descriptions if they exist in the database comments
+        if !isempty(strip(row.column_description))
+            text_payload *= " | Description: $(row.column_description)"
+        else
+            text_payload *= " | Description: Represents $(row.column_name) data fields within the $(row.table_name) architecture."
+        end
+        
+        # 2. Package everything neatly to be passed to your vector store client
+        push!(payloads, Dict(
+            "id" => row.vector_id,
+            "text_content" => text_payload,
+            "metadata" => Dict(
+                "table" => row.table_name,
+                "column" => row.column_name,
+                "type" => row.data_type
+            )
+        ))
+    end
+    
+    return payloads
+end
 
 
+""" Resolve semantic cluster from vector hits using graph traversal.
+
+Finds the minimum interconnected subgraph that connects all semantically matched tables
+from vector search results, enabling construction of valid SQL queries across related tables.
+
+# Arguments
+- `vector_hits::Vector{String}`
+  A vector of table names matched semantically from Stage 1 vector search.
+- `g::SimpleGraph`
+  An undirected graph representing table relationships (nodes=tables, edges=foreign key relations).
+- `table_to_id::Dict{String, Int}`
+  Mapping from table names to node IDs in the graph.
+- `id_to_table::Dict{Int, String}`
+  Reverse mapping from node IDs to table names.
+
+# Return
+- `Vector{String}`
+  A vector of table names representing the minimum subgraph that connects all input vector hits.
+  The order reflects traversal path from anchor node to connected components.
+
+# Details
+The algorithm:
+1. Validates vector hits against the graph's table mapping
+2. Handles edge cases: empty hits, single table (returns as-is)
+3. Uses the highest-ranked vector hit as anchor node
+4. For each remaining hit, finds shortest path to current subgraph using Dijkstra's algorithm
+5. Builds minimal connected subgraph containing all hits
+6. Returns table names in traversal order
+
+# Example
+```julia
+julia> using GeneralUtils, Graphs
+julia> g = SimpleGraph(5)
+julia> add_edge!(g, 1, 2)
+julia> add_edge!(g, 2, 3)
+julia> add_edge!(g, 3, 4)
+julia> table_to_id = Dict("users" => 1, "orders" => 2, "payments" => 3, "products" => 4, "inventory" => 5)
+julia> id_to_table = Dict(1 => "users", 2 => "orders", 3 => "payments", 4 => "products", 5 => "inventory")
+julia> vector_hits = ["users", "products"]
+julia> GeneralUtils.resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
+["users", "orders", "products"]
+```
+"""
+function resolve_semantic_cluster(
+    vector_hits::Vector{String}, 
+    g::SimpleGraph, 
+    table_to_id::Dict{String, Int}, 
+    id_to_table::Dict{Int, String}
+  )::Vector{String}
+    # Filter out hits that don't exist in our actual database graph mapping
+    valid_node_ids = Int[]
+    for hit in vector_hits
+        if haskey(table_to_id, hit)
+            push!(valid_node_ids, table_to_id[hit])
+        else
+            @warn "Vector hit '$hit' does not map to an existing database table."
+        end
+    end
+    
+    unique!(valid_node_ids)
+    
+    # Edge Case Handlers
+    if isempty(valid_node_ids)
+        return String[]
+    elseif length(valid_node_ids) == 1
+        return [id_to_table[valid_node_ids[1]]]
+    end
+
+    # The Isolated Subgraph Set to build our final context
+    schema_subgraph_nodes = Set{Int}()
+    
+    # Phase A: Select an initial anchor component. We use the highest-ranked vector hit.
+    anchor_node = valid_node_ids[1]
+    push!(schema_subgraph_nodes, anchor_node)
+    
+    # Phase B: Sequentially route paths to all other semantic coordinates
+    for target_node in valid_node_ids[2:end]
+        # Skip if an earlier loop trajectory already naturally absorbed this table
+        if target_node in schema_subgraph_nodes
+            continue
+        end
+        
+        # Calculate the shortest path tree from the CURRENT state of our subgraph
+        # We find the shortest path from the target back to ANY node currently in our tree
+        shortest_paths = dijkstra_shortest_paths(g, target_node)
+        
+        # Find which node currently in our subgraph is closest to the target node
+        closest_subgraph_node = 0
+        min_distance = Inf
+        
+        for subgraph_node in schema_subgraph_nodes
+            dist = shortest_paths.dists[subgraph_node]
+            if dist < min_distance
+                min_distance = dist
+                closest_subgraph_node = subgraph_node
+            end
+        end
+        
+        # Reconstruct the path from the target node to the closest point on our existing tree
+        if closest_subgraph_node != 0
+            curr = closest_subgraph_node
+            while curr != 0
+                push!(schema_subgraph_nodes, curr)
+                curr = shortest_paths.parents[curr]
+                if curr == target_node
+                    push!(schema_subgraph_nodes, target_node)
+                    break
+                end
+            end
+        end
+    end
+    
+    # Map the unique structural nodes back to clean table names
+    return [id_to_table[node_id] for node_id in schema_subgraph_nodes]
+end
 
 
+""" Harvest entity catalog from database column.
+
+Extracts unique, non-null values from a specific column to build a local index for 
+semantic search or entity resolution.
+
+# Arguments
+- `pg_conn_str::String`
+  PostgreSQL connection string in LibPQ format (e.g., "host=hostname port=5432 dbname=database user=username password=secret")
+- `table::String`
+  Table name to query
+- `column::String`
+  Column name containing entity values
+
+# Return
+- `Vector{String}`
+  A vector of unique, stripped strings from the specified column. Empty strings
+  are removed via `strip()`.
+
+# Details
+The function:
+1. Connects to PostgreSQL database
+2. Executes `SELECT DISTINCT column FROM table WHERE column IS NOT NULL`
+3. Converts result to DataFrame
+4. Strips whitespace from each value and converts to String
+5. Returns clean vector of unique entity values
+
+# Example
+```julia
+julia> using GeneralUtils
+julia> conn = "host=localhost port=5432 dbname=winedb user=admin password=secret"
+julia> fruits = GeneralUtils.harvest_entity_catalog(conn, "products", "fruit_name")
+["Apple", "Banana", "Orange", "Mango"]
+```
+"""
+function harvest_entity_catalog(pg_conn_str::String, table::String, column::String)::Vector{String}
+  conn = LibPQ.Connection(pg_conn_str)
+  return harvest_entity_catalog(conn, table, column)
+end
+
+function harvest_entity_catalog(conn::LibPQ.Connection, table::String, column::String)::Vector{String}
+  
+  # We only care about unique, non-null values to keep the index fast and dense
+  query = "SELECT DISTINCT $(column) FROM $(table) WHERE $(column) IS NOT NULL;"
+  
+  try
+    df = DataFrame(LibPQ.execute(conn, query))
+    # Return as a clean array of strings
+    return String.(strip.(df[:, 1]))
+  catch
+    return String[]
+  finally
+    close(conn)
+  end
+end
 
 
+""" Resolve entity name from messy input using fuzzy string matching.
 
+Matches user-provided text against a reference catalog using Jaro-Winkler similarity
+and returns the closest matching exact string from the database catalog.
 
+# Arguments
+- `messy_input::String`
+  The user input text that may contain typos, compressed words, or variations.
+- `catalog::Vector{String}`
+  A vector of valid, exact entity strings from the database.
+
+# Keyword Arguments
+- `threshold::Float64` (default: `0.5`)
+  Minimum similarity score (0.0 to 1.0) required to return a match. Lower values
+  allow more lenient matching; higher values require closer matches.
+
+# Return
+- `String`
+  The exact matching string from `catalog` if similarity score ≥ threshold,
+  otherwise an empty string `""`.
+
+# Details
+The function:
+1. Normalizes input to lowercase and strips whitespace
+2. Computes Jaro-Winkler similarity score against each catalog entry
+3. Applies substring fallback: if compressed words match (e.g., "HandOld" → "Hand Old Bar & Grill"),
+   boosts score to 0.85
+4. Returns the highest-scoring catalog entry if score ≥ threshold, else empty string
+
+# Example
+```julia
+julia> using GeneralUtils
+julia> catalog = ["Hand Old Bar & Grill", "Hand Old", "Wine Cellar"]
+julia> GeneralUtils.resolve_entity("HandOld", catalog; threshold=0.5)
+"Hand Old Bar & Grill"
+
+julia> GeneralUtils.resolve_entity("Wine Cellar", catalog; threshold=0.5)
+"Wine Cellar"
+
+julia> GeneralUtils.resolve_entity("Unknown Place", catalog; threshold=0.5)
+""
+```
+"""
+function resolve_entity(messy_input::String, catalog::Vector{String}; threshold=0.5)::String
+    best_match = ""
+    highest_score = 0.0
+    
+    # Normalize input text to ensure case-insensitive matching
+    clean_input = lowercase(strip(messy_input))
+    
+    for real_string in catalog
+        clean_real = lowercase(real_string)
+        
+        # Calculate phonetic/structural similarity score (0.0 to 1.0)
+        # JaroWinkler is optimized for short strings, names, and partial acronyms
+        score = compare(clean_real, clean_input, JaroWinkler())
+        
+        # Substring/Token fallback: handle cases like "HandOld" matching "Hand Old Bar & Grill"
+        # We strip spaces to check if the user just compressed words together
+        if contains(replace(clean_real, " " => ""), clean_input)
+            score = max(score, 0.85) 
+        end
+        
+        if score > highest_score
+            highest_score = score
+            best_match = real_string
+        end
+    end
+    
+    # Only return if we cross our safety confidence barrier
+    if highest_score >= threshold
+        return best_match
+    end
+    
+    return "" # No confident match found
+end
 
 
 

src/util.jl

@@ -1,47 +1,52 @@
 module util
   
 export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, replaceDictKeys,
-        findMatchingDictKey, textToDict, randstring, randstrings, timeout,
+        findMatchingDictKey, randstring, randstrings, timeout,
         dataframeToCSV, dfToVectorDict, disintegrate_vectorDict, getDataFrameValue, dfRowtoString, 
-        dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, 
+        dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, issomething,
         dictToString_numbering, extract_triple_backtick_text, 
-        countGivenWords, remove_french_accents, detect_keyword, extractTextBetweenCharacter,
-        convertCamelSnakeKebabCase
+        countGivenWords, remove_french_accents, removestring,
+        extractTextBetweenCharacter, extractTextBetweenString, 
+        convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex
 
-using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames
+using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames
 
 # ---------------------------------------------- 100 --------------------------------------------- #
 
-""" Compute time different between start time and stop time in a given unit. 
- Unit can be "milliseconds", "seconds", "minutes", "hours".
+""" Computes the time difference between two `DateTime` values and returns the
+result in a specified unit: milliseconds, seconds, minutes, or hours.
 
 # Arguments
-  - `starttime::DateTime` 
-    start time
-  - `stoptime::DateTime`  
-    stop time
-  - `unit::String`  
-    unit of time difference
+- `starttime::DateTime`
+  The starting `DateTime` value.
+- `stoptime::DateTime`
+  The ending `DateTime` value.
+- `unit::String`
+  The unit for the result. Must be one of: `"milliseconds"`, `"seconds"`,
+  `"minutes"`, `"hours"`.
 
 # Return
-  - time difference in given unit
+- `Integer`: The time difference converted to the specified unit.
 
-# Example
+# Notes
+- The function computes `stoptime - starttime` and converts the result to the
+  requested unit using integer division.
+- Errors with `ArgumentError` if an invalid unit is specified.
+
+# Examples
 ```jldoctest
-julia> using Revise
 julia> using GeneralUtils, Dates
 julia> a = Dates.now()
-julia> b = a + Dates.Day(5) # add 5 days
-julia> GeneralUtils.timedifference(a, b, "hours")
+julia> b = a + Dates.Day(5)
+julia> timedifference(a, b, "hours")
 120
 ```
-
-# Signature
 """
 function timedifference(starttime::DateTime, stoptime::DateTime, unit::String)::Integer
   diff = stoptime - starttime
   unit = lowercase(unit)
   
+    # Check the unit and calculate the time difference accordingly
   if unit == "milliseconds"
       return diff.value
   elseif unit == "seconds"
@@ -182,21 +187,27 @@ end
 
 
 
-""" Get uuid4 with snake case
+""" Generates a UUID4 (version 4) identifier and converts it to snake case by
+replacing hyphens with underscores.
+
+# Arguments
+- This function takes no arguments.
 
 # Return
-  - `uuid4::String`
-    uuid4 with snake case
+- `String`: A UUID4 string with underscores instead of hyphens (e.g.,
+  `"0f6e4f_568c_4df4_8c79_1d7a58072f4a"`).
 
-# Example
+# Notes
+- Uses the `uuid4()` function from the UUIDs standard library to generate a
+  random UUID.
+- The underscore character replaces all hyphens in the UUID string.
+
+# Examples
 ```jldoctest
-julia> using Revise
 julia> using GeneralUtils
-julia> GeneralUtils.uuid4snakecase()
+julia> uuid4snakecase()
 "0f6e4f_568c_4df4_8c79_1d7a58072f4a"
 ```
-
-# Signature
 """
 function uuid4snakecase()::String
   _id = string(uuid4())
@@ -205,32 +216,37 @@ function uuid4snakecase()::String
 end
 
 
-""" Replace a dictionary key with the new key
+""" Replaces keys in a dictionary according to a mapping, returning a new
+dictionary with updated keys while preserving the original values.
 
 # Arguments
-  - `d::Dict` 
-    The input dictionary that you want to modify
-  - `replacementMap::Dict`
-    A dictionary that maps old keys to new keys
+- `d::Dict`
+  The input dictionary to modify.
+- `replacementMap::Dict`
+  A dictionary mapping old keys to new keys. Keys not present in this map are
+  left unchanged.
 
 # Return
-  - `newDict::Dict`
-    new dictionary with the replaced keys
+- `Dict`: A new dictionary with replaced keys. Values are preserved from the
+  original dictionary.
 
-# Example
+# Notes
+- The function creates a new dictionary rather than modifying the input in
+  place.
+- Keys not found in `replacementMap` are copied to the result with their
+  original keys unchanged.
+
+# Examples
 ```jldoctest
-julia> using Revise
 julia> using GeneralUtils
 julia> d = Dict(:a => 1, :b => 2, :c => 3)
 julia> replacement_map = Dict(:a => :x, :b => :y)
-julia> new_dict = GeneralUtils.replaceDictKeys(d, replacement_map)
+julia> replaceDictKeys(d, replacement_map)
 Dict{Any, Any} with 3 entries:
   :y => 2
   :c => 3
   :x => 1
 ```
-
-# Signature
 """
 function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict
   newDict = Dict()
@@ -242,101 +258,6 @@ function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict
 end
 
 
-""" Convert text into a dictionary with a given keywords. This function use keywords to slice
-  a given text into the following format: KW1|kw1_text|KW2|kw2_text|KW3|kw3_text.
-  The left most string which has no keyword will be discarded. WARNING, ordering is important
-
-# Arguments
-  - `text::String`
-    A text to be converted.
-  - `keywords::Vector{String}`
-    A list of keywords to be used to slice the text. 
-    These keywords also be the resulting dict keys.
-# Keyword Arguments
-  - `rightmarker::String`
-    A maker used to make a word to be unique. Ex, A keyword "plan" with rightmarker ":", 
-    the function will search for "plan:" otherwise the function will search for "plan".
-    The marker will not be in the resulting dict keys.
-  - `symbolkey::Bool`
-    If true, resulting dict's key will be Symbols, otherwise string.
-  - `lowercasekey::Bool`
-    set resulting dict's key to be lowercase
-
-# Return
-  - `d::OrderedDict`
-
-# Example
-```jldoctest
-julia> text = "TODAY thought: what to do plan: wake up and going out action: 1. wake up 2. eat 3. sleep"
-julia> sample_keywords = ["thought", "plan", "action"]
-julia> resultdict = GeneralUtils.textToDict(text, sample_keywords; rightmarker=":", symbolkey=true)
-julia> println(resultdict)
-OrderedCollections.OrderedDict{Any, Any}(:thought => "what to do", 
-                                        :plan => "wake up and going out", 
-                                        :action => "1. wake up 2. eat 3. sleep")
-```
-
-# Signature
-"""
-function textToDict(text::String, detectKeywords::Vector{String}; 
-    dictKey::Union{Vector{String}, Nothing}=nothing, 
-    symbolkey::Bool=false, lowercasekey::Bool=false
-    )::OrderedDict
-
-  # make sure this function detect variation of a work e.g. agent, Agent, AGENT
-  kw = []
-  # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
-  for keyword in detectKeywords
-    detected = detect_keyword(keyword, text)
-    if detected !== nothing
-      push!(kw, detected)
-    else
-      error("Keyword $keyword not found in text.")
-    end
-  end
-
-  od1, od2 =
-  if symbolkey
-    OrderedDict{Symbol, Any}(), OrderedDict{Symbol, Any}()
-  else
-    OrderedDict{String, Any}(), OrderedDict{String, Any}()
-  end
-
-  remainingtext = text
-  dictKey_ = reverse(dictKey)
-  
-  # process text from back to front
-  for (i,keyword) in enumerate(reverse(kw))
-    # Find the position of the keyword in the text
-    keywordidx = findlast(keyword, remainingtext)
-    dKey = dictKey_[i]
-    
-    if keywordidx !== nothing
-      substr = remainingtext[keywordidx[end]+1:end]
-      str = string(strip(substr))  # Removes both leading and trailing whitespace.
-      _key = lowercasekey == true ? lowercase(dKey) : dKey
-      key = symbolkey == true ? Symbol(_key) : _key
-      od1[key] = str
-      remainingtext = remainingtext[1:keywordidx[1]-1]
-    else
-      error("""keyword "$keyword" not found in the provided text: $text </end of error note>""")
-    end
-  end
-
-  # correct the order
-  ks = reverse([i for i in keys(od1)])
-  for k in ks
-    k = symbolkey == true ? Symbol(k) : k
-    od2[k] = od1[k]
-  end
-  
-  return od2
-end
-
-
-
-
-
 """ Generate a random string
 
 # Arguments
@@ -387,36 +308,43 @@ end
 
 
 
-""" Execute a function with timer.
+""" Executes a function with a timeout mechanism. If the function does not
+complete within the specified time, it is interrupted and a timeout message
+is returned.
 
 # Arguments
-  - `f::Function`
-      a function to run
-  - `timeoutwindow::Integer``
-      timeout in seconds
+- `f::Function`
+  The function to execute.
+- `timeoutwindow::Integer`
+  The timeout duration in seconds.
 
-# Keyword Argument
-  - `fargs`
-    arguments for the function
-  - `timeoutmsg::String`
-    time out message
+# Keyword Arguments
+- `fargs`
+  Arguments to pass to the function `f`. If `nothing`, the function is called
+  without arguments.
+- `timeoutmsg::String`
+  The message to return if the function times out. Defaults to `"task timed out"`.
 
 # Return
-  - task result otherwise timeout message
+- The result of the function if it completes within the timeout, otherwise the
+  `timeoutmsg` string.
 
-# Example
-  ```jldoctest
+# Notes
+- Uses Julia's `@task`, `schedule`, and `Timer` to implement non-blocking
+  execution with interruption via `Base.throwto`.
+- Errors with `InterruptException` if the function exceeds the timeout.
+
+# Examples
+```jldoctest
 julia> function testfunc(x)
-          sleep(x)
-          return "task done"
-        end
+           sleep(x)
+           return "task done"
+       end
 julia> result = timeout(testfunc, 10; fargs=20)
 "task timed out"
 julia> result = timeout(testfunc, 20; fargs=10)
 "task done"
 ```
-
-#  Signature
 """
 function timeout(f::Function, timeoutwindow::Integer; fargs=nothing, timeoutmsg="task timed out")
   tsk = @task f(fargs)
@@ -433,23 +361,26 @@ end
 
 
 
-""" Convert a dataframe into CSV.
+""" Converts a DataFrame to a CSV string representation using the CSV.jl package.
 
 # Arguments
-  - `df::DataFrame`
-    A connection object to Postgres database
+- `df::DataFrame`
+  The DataFrame to convert to CSV format.
 
 # Return
-  - `result::String`
+- `String`: The DataFrame contents as a CSV-formatted string.
 
-# Example
+# Notes
+- Uses `CSV.write` with an `IOBuffer` to capture the output as a string.
+- The returned string contains the full CSV representation including headers.
+
+# Examples
 ```jldoctest
 julia> using DataFrames, GeneralUtils
 julia> df = DataFrame(A=1:3, B=5:7, fixed=1)
-julia> result = GeneralUtils.dataframeToCSV(df)
+julia> dataframeToCSV(df)
+"1,5,1\n2,6,1\n3,7,1\n"
 ```
-
-# Signature
 """
 function dataframeToCSV(df::DataFrame)
   # Create an IOBuffer to capture the output
@@ -471,7 +402,7 @@ end
 
 #  Example
   ```jldoctest
-  julia> using DataFrames, JSON3, GeneralUtils
+  julia> using DataFrames, GeneralUtils
   julia> df = DataFrame(A = [1, 2, 3], B = ["apple", "banana", "cherry"])
   julia> vectorDict = GeneralUtils.dfToVectorDict(df)
   [Dict{String, Any}("B" => "apple", "A" => 1), 
@@ -509,7 +440,7 @@ end
 
 # Example
   ```jldoctest
-  julia> using GeneralUtils, Dates, JSON3, UUIDs
+  julia> using GeneralUtils, Dates, UUIDs
   julia> vecDict = [Dict("a" => i) for i in 1:10]
   julia> d = GeneralUtils.disintegrate_vectorDict(vecDict, 3)
   julia> println(d[:data])
@@ -519,8 +450,6 @@ end
   3 => [Dict("a"=>7), Dict("a"=>8), Dict("a"=>9)]
   4 => [Dict("a"=>10)]
   ```
-
-# Signature
 """
 function disintegrate_vectorDict(data::Vector, partsize::Integer
   )
@@ -566,8 +495,6 @@ end
   julia> getDataFrameValue(df[1, :], :name)
   "Alice"
   ```
-
-# Signature
 """
 getDataFrameValue(row::DataFrameRow, key::Symbol) = row.:($key)
 
@@ -636,8 +563,6 @@ end
   julia> dfToString(df)
   "1) name: Alice, age: 25\n2) name: Bob, age: 30"
   ```
-
-# Signature
 """
 function dfToString(df::DataFrame)
   dfstr = ""
@@ -661,7 +586,7 @@ end
 
 # Example
   ```jldoctest
-  julia> using DataFrames
+  julia> using DataFrames, GeneralUtils
 
   julia> df = DataFrame(name=["Alice", "Bob"], age=[25, 30])
   2×2 DataFrame
@@ -671,13 +596,11 @@ end
    │ 1   │ Alice   25    
    │ 2   │ Bob     30    
 
-  julia> dataframe_to_json_list(df)
+  julia> GeneralUtils.dataframe_to_json_list(df)
   2-element Vector{String}:
   "{\"name\":\"Alice\",\"age\":25}"
   "{\"name\":\"Bob\",\"age\":30}"
   ```
-
-# Signature
 """
 function dataframe_to_json_list(df::DataFrame)::Vector{String}
   json_list = []
@@ -711,8 +634,6 @@ end
   julia> dict_to_string(od)
   "1) name: Alice, 2) age: 25"
   ```
-
-# Signature
 """
 function dictToString(od::T) where {T<:AbstractDict}
   items = []
@@ -770,48 +691,19 @@ function extract_triple_backtick_text(input::String)::Vector{String}
 end
 
 
-"""
-    detect_keyword(keyword::String, text::String) -> Union{Nothing, String}
+wordwindow(word::String, startindex::Integer)::UnitRange = startindex:startindex + length(word) -1
 
-Detects if a keyword exists in the text in different case variations (lowercase, uppercase first letter, or all uppercase).
-
-# Arguments:
-- `keyword::String` The keyword to search for
-- `text::String` The text to search in
-
-# Returns:
-- `Union{Nothing, String}` Returns the matched keyword variation if found, otherwise returns nothing
-
-# Examples:
-  ```jldoctest
-  julia> detect_keyword("test", "This is a Test case")
-  "Test"
-
-  julia> detect_keyword("error", "NO ERRORS FOUND")
-  "ERRORS"
-
-  julia> detect_keyword("missing", "complete data")
-  nothing
-  ```
-
-# Signature
-"""
-function detect_keyword(keyword::String, text::String)::Union{Nothing, String}
-  # Define the keyword variations to search for
-  keyword_variations = [keyword, uppercasefirst(keyword), uppercase(keyword), lowercase(keyword)]
-  
-  # Check if any of the keyword variations are in the text
-  for variation in keyword_variations
-      if occursin(variation, text)
-          return variation
-      end
+function cuttext(range, text)
+  # check whether range is outside text boundary
+  if range.start > length(text) || range.stop > length(text)
+    return nothing
+  else
+    return text[range]
   end
-  
-  # Return nothing if no variation is found
-  return nothing
 end
 
 
+
 """
     countGivenWords(text::String, words::Vector{String}) -> Dict{String, Int}
 
@@ -855,27 +747,39 @@ end
 
 
 
-"""
-    remove_french_accents(text::String) -> String
+""" Remove French accents from the given text.
 
-Remove French accents from the given text.
+The function replaces accented French characters with their non-accented
+counterparts using a dictionary mapping. Supported accented characters
+include: à, â, ä, á, é, è, ê, ë, î, ï, í, ñ, ô, ö, ò, ó, ù, û, ü, ÿ, ç,
+and their uppercase variants. The apostrophe character `’` is removed
+completely.
 
 # Arguments
-- `text::String` The input string containing French accents.
+- `text::AbstractString`
+  The input string containing French accented characters.
 
-# Returns
-- `String` The input string with all French accents removed.
+# Return
+- `AbstractString`: A new string with all French accents replaced by their
+  non-accented equivalents.
+
+# Notes
+- The function creates a character list and replaces each accented character
+  according to an internal dictionary mapping.
+- Does **not** mutate the input; it allocates a new string.
 
 # Examples
-  ```jldoctest
-  julia> remove_french_accents("Café")
-  "Cafe"
+```jldoctest
+julia> using GeneralUtils
+julia> remove_french_accents("Café")
+"Cafe"
 
-  julia> remove_french_accents("L'été est beau.")
-  "L'ete est beau."
-  ```
+julia> remove_french_accents("L'été est beau.")
+"L'ete est beau."
 
-# Signature
+julia> remove_french_accents("Noël, naïve, François")
+"Noel, naive, Francois"
+```
 """
 function remove_french_accents(text::AbstractString)::AbstractString
   textcharlist = [i for i in text]
@@ -924,9 +828,11 @@ Extracts and returns the text that is enclosed between two specified characters
 
 # Examples
 ```jldoctest
-julia> text = "Hello [World]!"
+julia> text = "Hello [World]! [Yay]"
 julia> extracted_text = extractTextBetweenCharacter(text, '[', ']')
-println(extracted_text)  # Output: "World"
+2-element Vector{Any}:
+ "World"
+ "Yay"
 ```
 """
 function extractTextBetweenCharacter(text::String, startchar::Char, endchar::Char)
@@ -950,6 +856,29 @@ function extractTextBetweenCharacter(text::String, startchar::Char, endchar::Cha
 end
 
 
+function extractTextBetweenString(text::String, startstr::String, endstr::String)
+  # check whether startstr is in the text or not
+  isStartStr = split(text, startstr)
+  if length(isStartStr) > 2
+    return (success=false, error="There are more than one occurrences of the start string '$startstr' in the text. Text must has only one start string", errorcode=2, result=nothing)
+  elseif length(isStartStr) == 1
+    return (success=false, error="There are no start string '$startstr' in the text. Text must has only one start string", errorcode=1, result=nothing)
+  end
+
+  # check whether endstr is in the text or not
+  isEndStr = split(text, endstr)
+  if length(isEndStr) > 2
+    return (success=false, error="There are more than one occurrences of the end string '$endstr' in the text. Text must has only one end string", errorcode=3, result=nothing)
+  elseif length(isStartStr) == 1
+    return (success=false, error="There are no end string '$endstr' in the text. Text must has only one end string", errorcode=4, result=nothing)
+  end
+
+  s = string(split(isStartStr[2], endstr)[1])
+  
+  return (success=true, error=nothing, errorcode=0, text=s)
+end
+
+
 """
 Determines if the given string follows camel case naming convention.
 
@@ -1075,10 +1004,173 @@ function convertCamelSnakeKebabCase(text::T, tocase::Symbol)::String where {T<:A
 end
 
 
+""" Check if a value is not `nothing`.
+
+# Arguments
+- `x`: The value to check
+
+# Returns
+- `Bool`: `true` if `x` is not `nothing`, `false` otherwise
+
+# Examples
+```jldoctest
+julia> issomething(1)
+true
+julia> issomething(nothing)
+false
+julia> issomething("test")
+true
+````
+"""
+function issomething(x)
+  return x === nothing ? false : true
+end
 
 
+""" Adjust a given range to fit within the bounds of a vector's length.
+
+# Arguments
+  - `v::T1`
+    the input vector to check against
+  - `range::UnitRange`
+    the original range to be adjusted
+
+# Return
+  - `adjusted_range::UnitRange`
+    a range that is constrained to the vector's length, preventing out-of-bounds indexing
+
+# Example
+
+julia> v = [1, 2, 3, 4, 5]
+julia> fitrange(v, 3:10)
+3:5
+
+"""
+function fitrange(v::T1, range::UnitRange) where {T1<:AbstractVector}
+  totalelements = length(v)
+
+  startind =
+    # check if user put start range greater than total event
+    if range.start > totalelements
+      totalelements
+    else
+      range.start
+    end
+
+  stopind = 
+    if range.stop > totalelements
+      totalelements
+    else
+      range.stop
+    end
+
+  return startind:stopind
+end
 
 
+""" Find a unit range for a vector given a number of the most recent elements of interest.
+
+# Arguments
+  - `vectorLength::Integer`
+    the length of the vector to generate range from
+  - `n::Integer`
+    the number of most recent elements to include in range
+
+
+# Return
+  - `UnitRange`
+    a range representing the n most recent elements of a vector with length vectorLength
+
+# Example
+```jldoctest
+julia> a = [1, 2, 3, 4, 5]
+julia> recentElementsIndex(length(a), 3)
+3:5
+julia> recentElementsIndex(length(a), 0)
+5:5
+```
+"""
+function recentElementsIndex(vectorlength::Integer, n::Integer; includelatest::Bool=false)
+  if n == 0
+    error("n must be greater than 0")
+  end
+  
+  if includelatest
+    start = max(1, vectorlength - n + 1)
+    return start:vectorlength
+  else
+    startind = max(1, vectorlength - n)
+    endind = vectorlength -1
+    return startind:endind
+  end
+end
+
+
+""" Find a unit range for a vector excluding the most recent elements.
+
+# Arguments
+    - `vectorlength::Integer`
+      the length of the vector to generate range from
+    - `n::Integer`
+      the number of most recent elements to exclude from range
+
+# Return
+  - `UnitRange`
+    a range representing the elements of the vector excluding the last `n` elements
+
+# Example
+```jldoctest
+julia> a = [1, 2, 3, 4, 5]
+julia> nonRecentElementsIndex(length(a), 3)
+1:2
+julia> nonRecentElementsIndex(length(a), 1)
+1:4
+julia> nonRecentElementsIndex(length(a), 0)
+1:5
+```
+"""
+function nonRecentElementsIndex(vectorlength::Integer, n::Integer)
+  if n < 0
+      error("n must be non-negative")
+  end
+  if n > vectorlength
+      return 1:0  # empty range
+  end
+  return 1:(vectorlength-n)
+end
+
+""" Remove specified substrings from text.
+
+Removes all occurrences of each string in `removelist` from the input text
+by repeatedly replacing them with empty strings until none remain.
+
+# Arguments
+- `text::String`
+  The input string to modify.
+- `removelist::Vector{String}`
+  A vector of substrings to remove from the text.
+
+# Return
+- `String`: The text with all specified substrings removed.
+
+# Examples
+```jldoctest
+julia> using GeneralUtils
+julia> removestring("hello world", ["l", " "])
+"heoword"
+julia> removestring("foo bar baz", ["bar", " "])
+"foobaz"
+```
+"""
+function removestring(text::String, removelist::Vector{String})::String
+  for i in removelist
+    while occursin(i, text)
+      text = replace(text, i => "")
+    end
+  end
+
+  return string(text)
+end
 
 
 

test/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "julia.environmentPath": "/appfolder/app/dev/GeneralUtils/test"
+}

test/Manifest.toml

@@ -0,0 +1,41 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.11.4"
+manifest_format = "2.0"
+project_hash = "71d91126b5a1fb1020e1098d9d492de2a4438fd2"
+
+[[deps.Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+version = "1.11.0"
+
+[[deps.InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+version = "1.11.0"
+
+[[deps.Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+version = "1.11.0"
+
+[[deps.Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+version = "1.11.0"
+
+[[deps.Random]]
+deps = ["SHA"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+version = "1.11.0"
+
+[[deps.SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
+
+[[deps.Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+version = "1.11.0"
+
+[[deps.Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+version = "1.11.0"

test/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/runtest.jl

@@ -1,44 +0,0 @@
-using Revise
-using GeneralUtils, MQTTClient, JSON3
-
-mqttMsgReceiveTopic = ["/receivetopic_1", "/receivetopic_2"]
-mqttMsgReceiveChannel = (ch1=Channel(8), ch2=Channel(32))
-keepaliveChannel = Channel(8)
-function onMsgCallback(topic, payload)
-            jobj = JSON3.read(String(payload))
-            incomingMqttMsg = copy(jobj)  # convert json object into julia dictionary recursively
-            
-            if occursin("topic_1", topic)
-              put!(mqttMsgReceiveChannel[:ch1], incomingMqttMsg)
-            elseif occursin("topic_2", topic)
-              put!(mqttMsgReceiveChannel[:ch2], incomingMqttMsg)
-            elseif occursin("keepalive", topic)
-              put!(keepaliveChannel, incomingMqttMsg)
-            else
-              println("undefined condition ", @__FILE__, " ", @__LINE__)
-            end
-          end
-mqttInstance = GeneralUtils.mqttClientInstance_v2(
-                          "mqtt.yiem.cc",
-                          mqttMsgReceiveTopic,
-                          mqttMsgReceiveChannel,
-                          keepaliveChannel,
-                          onMsgCallback
-                        )
-
-
-_ = GeneralUtils.checkMqttConnection!(mqttInstance)
-
-
-println("GeneralUtils test done")
-
-
-
-
-
-
-
-
-
-
-

test/runtests.jl

@@ -0,0 +1,39 @@
+using Test
+using GeneralUtils: detect_keyword
+
+@testset "detect_keyword tests" begin
+    @test detect_keyword(["test"], "this is a test") == Dict("test" => 1)
+    
+    @test detect_keyword(["hello", "world"], "hello world hello") == Dict("hello" => 2, "world" => 1)
+    
+    @test detect_keyword(["cat"], "category") == Dict("cat" => 1)
+    
+    @test detect_keyword(["cat"], "category"; mode="individual") == Dict("cat" => 0)
+    
+    @test detect_keyword(["dog"], "dogs and cats"; mode="individual", delimiter=[' ']) == Dict("dog" => 0)
+    
+    @test detect_keyword(["test"], "test.case"; mode="individual", delimiter=['.']) == Dict("test" => 1)
+    
+    @test detect_keyword(["word"], "") == Dict("word" => 0)
+    
+    @test detect_keyword(String[], "some text") == Dict{String, Integer}()
+    
+    @test detect_keyword(["a", "b"], "a.b\nc"; delimiter=['.', '\n']) == Dict("a" => 1, "b" => 1)
+    
+    multiline_text = """
+    first line
+    second line
+    first word
+    """
+    @test detect_keyword(["first"], multiline_text) == Dict("first" => 2)
+    
+    @test detect_keyword(["word"], "word"; mode="individual") == Dict("word" => 1)
+    
+    @test detect_keyword(["test"], "testing.test.tester"; mode="individual", delimiter=['.']) == Dict("test" => 1)
+end
+
+
+
+
+
+