module dbUtil export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity, execute_postgres_sql using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, SHA, NATS, LibPQ using ..util #[PENDING] update code to use JSON # ---------------------------------------------- 100 --------------------------------------------- # """ Execute SQL against a PostgreSQL database using LibPQ connection string. # Arguments - `pg_conn_str::AbstractString`: PostgreSQL connection string in format "host=... port=... dbname=... user=... password=..." - `sql::AbstractString`: SQL query to execute # Returns - `LibPQ.Result` on success, `nothing` on failure # Example ```julia pg_conn_str = "host=localhost port=5432 dbname=mydb user=myuser password=mypass" sql = "SELECT * FROM wine;" result = execute_postgres_sql(pg_conn_str, sql) ``` """ function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString} db_connection = LibPQ.Connection(pg_conn_str) result = nothing try result = LibPQ.execute(db_connection, sql) catch e @error e LibPQ.close(db_connection) end LibPQ.close(db_connection) return result end """ find_text_vector_similarity Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity. This function computes an embedding for the input text using the provided embedding function, then queries the database to find records with the most similar vector representations using PostgreSQL's cosine similarity operator (`<->`). # Arguments - `text::AbstractString`: The input text to find similar records for - `tablename::AbstractString`: Name of the database table containing the embedding column - `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings - `executesql::Function`: Function that executes SQL queries and returns results - `get_embedding::Function`: Function that generates embeddings for text inputs # Keyword Arguments - `limit::Integer=1`: Maximum number of similar records to return # Returns - `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column where smaller values indicate higher similarity # Example ```julia # Assume you have embedding and SQL execution functions text = "a rich structured red wine" tablename = "wine" embeddingColumnName = "description_embedding" df = find_text_vector_similarity( text, tablename, embeddingColumnName, executesql, get_embedding; limit = 5 ) # Result contains columns from the table plus a 'distance' column # where distance = 1 - cosine_similarity (smaller = more similar) ``` """ function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3, executesql::Function, get_embedding; limit::Integer=1 )::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString} # get embedding from LLM service _embedding = get_embedding([text]) _embedding = _embedding["data"][1]["embedding"] _embedding = "$_embedding" embedding = _embedding[4:end] # remove 'Any' from Any[...] # check whether there is close enough vector already store in executesql. if no, add, else skip sql = """ SELECT *, $embeddingColumnName <-> '$embedding' as distance FROM $tablename ORDER BY distance LIMIT $limit; """ response = executesql(sql) df = DataFrame(response) return df end """ dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format This function takes a dictionary and converts it into a PostgreSQL-compatible key-value string format suitable for storage in a TEXT field. The output format uses curly braces with comma-separated key-value pairs, where string values are quoted. # Function Workflow: 1. Iterates through dictionary key-value pairs 2. Handles nested dictionaries by recursively converting them 3. Wraps string values in double quotes 4. Formats numeric and other values without quotes 5. Returns a PostgreSQL-compatible key-value string enclosed in curly braces # Arguments: - `dict::Dict` - Dictionary containing key-value pairs to convert # Return: - A String in PostgreSQL key-value format: "{key1: value1, key2: \"value2\", ...}" # Example ```jldoctest julia> data = Dict{String, Any}( "name" => "John", "age" => 30, "city" => "New York" ); julia> dictToPostgresKeyValueString(data) "{\"name\": \"John\", \"age\": 30, \"city\": \"New York\"}" ``` """ function dictToPostgresKeyValueString(dict) parts = [] for (k, v) in dict if isa(v, Dict) push!(parts, "\"$k\": " * dict_to_string(v)) elseif isa(v, AbstractString) push!(parts, "\"$k\": \"$v\"") else push!(parts, "\"$k\": $v") end end return "{" * join(parts, ", ") * "}" end """ generateInsertSQL - Generate SQL INSERT statement from dictionary data This function constructs a SQL INSERT statement by extracting values for specified columns from a dictionary and formatting them into a valid PostgreSQL INSERT query. # Function Workflow: 1. Iterates through the dictionary key-value pairs 2. Filters keys to only include those present in `columnToInsert` 3. Collects column names and their corresponding values 4. Constructs the final SQL INSERT statement # Arguments: - `table_name::String` - Name of the database table to insert into - `columnToInsert::Vector{Symbol}` - List of column names to include in the INSERT statement - `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs for the insert # Return: - A String containing the SQL INSERT statement # Example ```jldoctest julia> using UUIDs # Insert a single record with specific columns table_name = "wine" columnToInsert = [:acidity, :tannin, :country, :region, :winery] data = Dict{Symbol, Any}( :grape => "Cabernet Sauvignon", :acidity => "medium", # using descriptive scale (low/medium/full) :tannin => "medium-plus", # common wine descriptor :country => "France", :description => "A rich and structured red wine with notes of blackcurrant, cedar, and subtle oak.", :region => "Bordeaux", :winery => "Château Margaux", :intensity => "medium", # intensity is usually low/medium/full :sweetness => "dry", # dry/medium-dry/medium/medium-sweet/sweet :tasting_notes => "Blackberry, graphite, tobacco, vanilla, and subtle earth.", :wine_name => "Château Margaux Grand Cru", :wine_id => "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a", # UUID-like (valid hex) :wine_type => "Red", :other_attributes => Dict{String, Any}( "vintage" => 2018, "alcohol_percent" => 13.5, "ph" => 3.6, " aging_years" => 24, # years in barrel " producer_code" => "CM-GRAND" ), :fizziness => "still", :serving_temperature => "16–18°C", :additional_search_term => ["Cabernet", "Bordeaux red", "premium wine", "CabSav"] ) julia> generateInsertSQL(table_name, columnToInsert, data) "INSERT INTO wine (acidity, tannin, country, region, winery) VALUES ('medium', 'medium-plus', 'France', 'Bordeaux', 'Château Margaux');" ``` """ function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, data::Dict{Symbol, Any}) columns = String[] values = String[] for (key, value) in data if key ∈ columnToInsert push!(columns, string(key)) value_str = isa(value, AbstractString) ? "'$value'" : "$value" push!(values, value_str) end end columns_str = join(columns, ", ") values_str = join(values, ", ") return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);" end function generateInsertSQL(table_name::String, data::AbstractDict{String, Any}) columns = String[] values = String[] for (key, value) in data push!(columns, string(key)) value_str = isa(value, AbstractString) ? "'$value'" : "$value" push!(values, value_str) end columns_str = join(columns, ", ") values_str = join(values, ", ") return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);" end # ---------------------------------------------- 100 --------------------------------------------- # """ generateUpdateSQL - Generate SQL UPDATE statement from dictionary data This function constructs a SQL UPDATE statement by updating multiple columns based on a primary key condition. # Arguments: - `table_name::String` - Name of the database table to update - `pk_column::Symbol` - The primary key column name - `pk_value` - The primary key value (used in WHERE clause) - `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs to update # Return: - A String containing the SQL UPDATE statement # Example ```jldoctest julia> using UUIDs # Update multiple columns using a dictionary table_name = "wine" pk_column = :wine_id pk_value = "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a" data = Dict{Symbol, Any}( :acidity => "full", :tannin => "medium", :country => "Italy" ) julia> generateUpdateSQL(table_name, pk_column, pk_value, data) "UPDATE wine SET acidity = 'full', tannin = 'medium', country = 'Italy' WHERE wine_id = '8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a';" ``` """ function generateUpdateSQL(table_name::String, pk_column::String, pk_value, data::AbstractDict{String, Any}) # Build SET clause set_parts = String[] for (key, value) in data if key ∉ [pk_column] value_str = isa(value, AbstractString) ? "'$value'" : "$value" push!(set_parts, "$(string(key)) = $value_str") end end set_clause = join(set_parts, ", ") # Handle primary key value pk_val_str = isa(pk_value, AbstractString) ? "'$pk_value'" : "$pk_value" return "UPDATE $table_name SET $set_clause WHERE $pk_column = $pk_val_str;" end function generateUpdateSQL(table_name::String, pk_dict::AbstractDict{String, Any}, data::AbstractDict{String, Any}) # Build SET clause set_parts = String[] for (key, value) in data if key ∉ keys(pk_dict) value_str = isa(value, AbstractString) ? "'$value'" : "$value" push!(set_parts, "$(string(key)) = $value_str") end end set_clause = join(set_parts, ", ") # Build WHERE clause for composite keys where_parts = String[] for (col, val) in pk_dict val_str = isa(val, AbstractString) ? "'$val'" : "$val" push!(where_parts, "$(string(col)) = $val_str") end where_clause = join(where_parts, " AND ") return "UPDATE $table_name SET $set_clause WHERE $where_clause;" end end # module