update

Reviewed-on: #12

Project.toml

@@ -1,6 +1,6 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.5.0"
+version = "0.5.11"
 authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
 
 [deps]

src/dbUtil.jl

@@ -1,13 +1,107 @@
 module dbUtil
   
-export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
+export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity,
+        execute_postgres_sql
 
 using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
-      SHA
+      SHA, NATS, LibPQ
 using ..util
 #[PENDING] update code to use JSON
 # ---------------------------------------------- 100 --------------------------------------------- #
 
+""" Execute SQL against a PostgreSQL database using LibPQ connection string.
+
+# Arguments
+- `pg_conn_str::AbstractString`: PostgreSQL connection string in format "host=... port=... dbname=... user=... password=..."
+- `sql::AbstractString`: SQL query to execute
+
+# Returns
+- `LibPQ.Result` on success, `nothing` on failure
+
+# Example
+```julia
+pg_conn_str = "host=localhost port=5432 dbname=mydb user=myuser password=mypass"
+sql = "SELECT * FROM wine;"
+result = execute_postgres_sql(pg_conn_str, sql)
+```
+"""
+function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
+  db_connection = LibPQ.Connection(pg_conn_str)
+  result = nothing
+  try
+    result = LibPQ.execute(db_connection, sql)  
+  catch e
+    @error e
+    LibPQ.close(db_connection)
+  end
+  
+  LibPQ.close(db_connection)
+  return result
+end
+
+
+""" find_text_vector_similarity
+
+Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity.
+
+This function computes an embedding for the input text using the provided embedding function,
+then queries the database to find records with the most similar vector representations using
+PostgreSQL's cosine similarity operator (`<->`).
+
+# Arguments
+- `text::AbstractString`: The input text to find similar records for
+- `tablename::AbstractString`: Name of the database table containing the embedding column
+- `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings
+- `executesql::Function`: Function that executes SQL queries and returns results
+- `get_embedding::Function`: Function that generates embeddings for text inputs
+
+# Keyword Arguments
+- `limit::Integer=1`: Maximum number of similar records to return
+
+# Returns
+- `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column
+  where smaller values indicate higher similarity
+
+# Example
+```julia
+# Assume you have embedding and SQL execution functions
+text = "a rich structured red wine"
+tablename = "wine"
+embeddingColumnName = "description_embedding"
+
+df = find_text_vector_similarity(
+    text, tablename, embeddingColumnName,
+    executesql, get_embedding;
+    limit = 5
+)
+
+# Result contains columns from the table plus a 'distance' column
+# where distance = 1 - cosine_similarity (smaller = more similar)
+```
+"""
+function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3,
+  executesql::Function, get_embedding::Function; 
+  limit::Integer=1
+  )::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
+  # get embedding from LLM service
+  _embedding = get_embedding([text])
+  _embedding = _embedding["data"][1]["embedding"]
+  _embedding = "$_embedding"
+
+  embedding = _embedding[4:end] # remove 'Any' from Any[...]
+
+  # check whether there is close enough vector already store in executesql. if no, add, else skip 
+  sql = """
+        SELECT *, $embeddingColumnName <-> '$embedding' as distance
+        FROM $tablename
+        ORDER BY distance LIMIT $limit;
+        """
+  response = executesql(sql)
+  df = DataFrame(response)
+
+  return df
+end
+
 
 """
 dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format

src/llmUtil.jl

@@ -3,7 +3,7 @@ module llmUtil
 export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response,
         extract_column_metadata, generate_embedding_payloads, resolve_semantic_cluster,
         harvest_entity_catalog, resolve_entity, harvest_db_undirected_schema_graph,
-        get_db_table_schema
+        get_db_table_schema, get_db_table_schema_simple
 
 using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs, LibPQ
 using ..util
@@ -1016,7 +1016,115 @@ function get_db_table_schema(conn::LibPQ.Connection, table_name::String)::DataFr
 end
 
 
+""" Generate simplified DDL statement for a PostgreSQL table.
 
+Extracts table structure from PostgreSQL and returns a clean CREATE TABLE statement
+without NULL/NOT NULL constraints, useful for schema documentation or migration purposes.
+
+# Arguments
+- `conn::LibPQ.Connection`
+  A PostgreSQL connection object created via `LibPQ.Connection()`.
+- `table_name::String`
+  The name of the table to extract.
+- `schema_name::String` (default: `"public"`)
+  The schema containing the table.
+
+# Return
+- `String`
+  A CREATE TABLE DDL statement containing:
+  - Column definitions with names, types, and DEFAULT values
+  - Table-level constraints (PRIMARY KEY, FOREIGN KEY, UNIQUE, CHECK)
+  - Excludes NULL/NOT NULL specifications for cleaner output
+
+# Details
+The function:
+1. Queries PostgreSQL system catalogs (`pg_attribute`, `pg_class`, `pg_namespace`)
+2. Extracts column names, data types, and default values
+3. Captures table-level constraints via `pg_constraint`
+4. Omits NULL/NOT NULL checks to produce a simplified schema definition
+5. Returns properly formatted DDL with quoted identifiers
+
+# Example
+```julia
+julia> using GeneralUtils, LibPQ
+julia> conn = LibPQ.Connection("host=localhost port=5432 dbname=winedb user=admin password=secret")
+julia> ddl = GeneralUtils.get_db_table_schema_simple(conn, "wine")
+"CREATE TABLE \"public\".\"wine\" (
+    \"id\" integer DEFAULT nextval('wine_id_seq'::regclass),
+    \"wine_name\" text,
+    \"year\" integer,
+    \"price\" numeric
+);"
+```
+"""
+function get_db_table_schema_simple(pg_conn_str::String, table_name::String; 
+  schema_name::String="public")::String
+  conn = LibPQ.Connection(pg_conn_str)
+  return get_db_table_schema_simple(conn, table_name; schema_name=schema_name)
+end
+
+function get_db_table_schema_simple(conn, table_name::String; schema_name::String="public")::String
+    # 1. SQL query tailored to omit nullability checks
+    sql = """
+    SELECT 
+        a.attname AS column_name,
+        format_type(a.atttypid, a.atttypmod) AS data_type,
+        pg_get_expr(def.adbin, def.adrelid) AS default_value,
+        COALESCE(
+            (SELECT pg_get_constraintdef(p.oid) 
+             FROM pg_catalog.pg_constraint p 
+             WHERE p.conrelid = c.oid AND a.attnum = ANY(p.conkey) 
+             LIMIT 1), ''
+        ) AS constraint_definition
+    FROM 
+        pg_catalog.pg_attribute a
+    JOIN 
+        pg_catalog.pg_class c ON a.attrelid = c.oid
+    JOIN 
+        pg_catalog.pg_namespace n ON c.relnamespace = n.oid
+    LEFT JOIN 
+        pg_catalog.pg_attrdef def ON def.adrelid = c.oid AND def.adnum = a.attnum
+    WHERE 
+        c.relname = \$1
+        AND n.nspname = \$2
+        AND a.attnum > 0 
+        AND NOT a.attisdropped
+    ORDER BY 
+        a.attnum;
+    """
+
+    result = execute(conn, sql, [table_name, schema_name])
+    
+    if length(result) == 0
+        error("Table '$schema_name.$table_name' not found.")
+    end
+
+    ddl_lines = String[]
+    constraints = String[]
+
+    for row in result
+        col_name   = row.column_name
+        data_type  = row.data_type
+        
+        # Handle the default value if it exists
+        default_val = ismissing(row.default_value) ? "" : " DEFAULT " * row.default_value
+        
+        # Build the column definition line (without NULL/NOT NULL)
+        col_def = "    \"$col_name\" $data_type$default_val"
+        push!(ddl_lines, col_def)
+
+        # Handle table-level constraints
+        con_def = ismissing(row.constraint_definition) ? "" : row.constraint_definition
+        if !isempty(con_def) && !(con_def in constraints)
+            push!(constraints, "    " * con_def)
+        end
+    end
+
+    all_definitions = vcat(ddl_lines, constraints)
+    body = join(all_definitions, ",\n")
+    
+    return "CREATE TABLE \"$schema_name\".\"$table_name\" (\n$body\n);"
+end