325 lines
11 KiB
Julia
325 lines
11 KiB
Julia
module dbUtil
|
||
|
||
export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity,
|
||
execute_postgres_sql
|
||
|
||
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
|
||
SHA, NATS, LibPQ
|
||
using ..util
|
||
#[PENDING] update code to use JSON
|
||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||
|
||
""" Execute SQL against a PostgreSQL database using LibPQ connection string.
|
||
|
||
# Arguments
|
||
- `pg_conn_str::AbstractString`: PostgreSQL connection string in format "host=... port=... dbname=... user=... password=..."
|
||
- `sql::AbstractString`: SQL query to execute
|
||
|
||
# Returns
|
||
- `LibPQ.Result` on success, `nothing` on failure
|
||
|
||
# Example
|
||
```julia
|
||
pg_conn_str = "host=localhost port=5432 dbname=mydb user=myuser password=mypass"
|
||
sql = "SELECT * FROM wine;"
|
||
result = execute_postgres_sql(pg_conn_str, sql)
|
||
```
|
||
"""
|
||
function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
|
||
db_connection = LibPQ.Connection(pg_conn_str)
|
||
result = nothing
|
||
try
|
||
result = LibPQ.execute(db_connection, sql)
|
||
catch e
|
||
@error e
|
||
LibPQ.close(db_connection)
|
||
end
|
||
|
||
LibPQ.close(db_connection)
|
||
return result
|
||
end
|
||
|
||
|
||
""" find_text_vector_similarity
|
||
|
||
Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity.
|
||
|
||
This function computes an embedding for the input text using the provided embedding function,
|
||
then queries the database to find records with the most similar vector representations using
|
||
PostgreSQL's cosine similarity operator (`<->`).
|
||
|
||
# Arguments
|
||
- `text::AbstractString`: The input text to find similar records for
|
||
- `tablename::AbstractString`: Name of the database table containing the embedding column
|
||
- `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings
|
||
- `executesql::Function`: Function that executes SQL queries and returns results
|
||
- `get_embedding::Function`: Function that generates embeddings for text inputs
|
||
|
||
# Keyword Arguments
|
||
- `limit::Integer=1`: Maximum number of similar records to return
|
||
|
||
# Returns
|
||
- `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column
|
||
where smaller values indicate higher similarity
|
||
|
||
# Example
|
||
```julia
|
||
# Assume you have embedding and SQL execution functions
|
||
text = "a rich structured red wine"
|
||
tablename = "wine"
|
||
embeddingColumnName = "description_embedding"
|
||
|
||
df = find_text_vector_similarity(
|
||
text, tablename, embeddingColumnName,
|
||
executesql, get_embedding;
|
||
limit = 5
|
||
)
|
||
|
||
# Result contains columns from the table plus a 'distance' column
|
||
# where distance = 1 - cosine_similarity (smaller = more similar)
|
||
```
|
||
"""
|
||
function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3,
|
||
executesql::Function, get_embedding;
|
||
limit::Integer=1
|
||
)::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
|
||
# get embedding from LLM service
|
||
_embedding = get_embedding([text])
|
||
_embedding = _embedding["data"][1]["embedding"]
|
||
_embedding = "$_embedding"
|
||
|
||
embedding = _embedding[4:end] # remove 'Any' from Any[...]
|
||
|
||
# check whether there is close enough vector already store in executesql. if no, add, else skip
|
||
sql = """
|
||
SELECT *, $embeddingColumnName <-> '$embedding' as distance
|
||
FROM $tablename
|
||
ORDER BY distance LIMIT $limit;
|
||
"""
|
||
response = executesql(sql)
|
||
df = DataFrame(response)
|
||
|
||
return df
|
||
end
|
||
|
||
|
||
"""
|
||
dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format
|
||
|
||
This function takes a dictionary and converts it into a PostgreSQL-compatible key-value string
|
||
format suitable for storage in a TEXT field. The output format uses curly braces with comma-separated
|
||
key-value pairs, where string values are quoted.
|
||
|
||
# Function Workflow:
|
||
1. Iterates through dictionary key-value pairs
|
||
2. Handles nested dictionaries by recursively converting them
|
||
3. Wraps string values in double quotes
|
||
4. Formats numeric and other values without quotes
|
||
5. Returns a PostgreSQL-compatible key-value string enclosed in curly braces
|
||
|
||
# Arguments:
|
||
- `dict::Dict` - Dictionary containing key-value pairs to convert
|
||
|
||
# Return:
|
||
- A String in PostgreSQL key-value format: "{key1: value1, key2: \"value2\", ...}"
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia> data = Dict{String, Any}(
|
||
"name" => "John",
|
||
"age" => 30,
|
||
"city" => "New York"
|
||
);
|
||
|
||
julia> dictToPostgresKeyValueString(data)
|
||
"{\"name\": \"John\", \"age\": 30, \"city\": \"New York\"}"
|
||
```
|
||
"""
|
||
function dictToPostgresKeyValueString(dict)
|
||
parts = []
|
||
for (k, v) in dict
|
||
if isa(v, Dict)
|
||
push!(parts, "\"$k\": " * dict_to_string(v))
|
||
elseif isa(v, AbstractString)
|
||
push!(parts, "\"$k\": \"$v\"")
|
||
else
|
||
push!(parts, "\"$k\": $v")
|
||
end
|
||
end
|
||
return "{" * join(parts, ", ") * "}"
|
||
end
|
||
|
||
|
||
|
||
|
||
""" generateInsertSQL - Generate SQL INSERT statement from dictionary data
|
||
|
||
This function constructs a SQL INSERT statement by extracting values for specified columns
|
||
from a dictionary and formatting them into a valid PostgreSQL INSERT query.
|
||
|
||
# Function Workflow:
|
||
1. Iterates through the dictionary key-value pairs
|
||
2. Filters keys to only include those present in `columnToInsert`
|
||
3. Collects column names and their corresponding values
|
||
4. Constructs the final SQL INSERT statement
|
||
|
||
# Arguments:
|
||
- `table_name::String` - Name of the database table to insert into
|
||
- `columnToInsert::Vector{Symbol}` - List of column names to include in the INSERT statement
|
||
- `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs for the insert
|
||
|
||
# Return:
|
||
- A String containing the SQL INSERT statement
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia> using UUIDs
|
||
|
||
# Insert a single record with specific columns
|
||
table_name = "wine"
|
||
columnToInsert = [:acidity, :tannin, :country, :region, :winery]
|
||
data = Dict{Symbol, Any}(
|
||
:grape => "Cabernet Sauvignon",
|
||
:acidity => "medium", # using descriptive scale (low/medium/full)
|
||
:tannin => "medium-plus", # common wine descriptor
|
||
:country => "France",
|
||
:description => "A rich and structured red wine with notes of blackcurrant, cedar, and subtle oak.",
|
||
:region => "Bordeaux",
|
||
:winery => "Château Margaux",
|
||
:intensity => "medium", # intensity is usually low/medium/full
|
||
:sweetness => "dry", # dry/medium-dry/medium/medium-sweet/sweet
|
||
:tasting_notes => "Blackberry, graphite, tobacco, vanilla, and subtle earth.",
|
||
:wine_name => "Château Margaux Grand Cru",
|
||
:wine_id => "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a", # UUID-like (valid hex)
|
||
:wine_type => "Red",
|
||
:other_attributes => Dict{String, Any}(
|
||
"vintage" => 2018,
|
||
"alcohol_percent" => 13.5,
|
||
"ph" => 3.6,
|
||
" aging_years" => 24, # years in barrel
|
||
" producer_code" => "CM-GRAND"
|
||
),
|
||
:fizziness => "still",
|
||
:serving_temperature => "16–18°C",
|
||
:additional_search_term => ["Cabernet", "Bordeaux red", "premium wine", "CabSav"]
|
||
)
|
||
|
||
julia> generateInsertSQL(table_name, columnToInsert, data)
|
||
"INSERT INTO wine (acidity, tannin, country, region, winery) VALUES ('medium', 'medium-plus', 'France', 'Bordeaux', 'Château Margaux');"
|
||
```
|
||
"""
|
||
function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, data::Dict{Symbol, Any})
|
||
columns = String[]
|
||
values = String[]
|
||
|
||
for (key, value) in data
|
||
if key ∈ columnToInsert
|
||
push!(columns, string(key))
|
||
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||
push!(values, value_str)
|
||
end
|
||
end
|
||
|
||
columns_str = join(columns, ", ")
|
||
values_str = join(values, ", ")
|
||
|
||
return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
|
||
end
|
||
|
||
function generateInsertSQL(table_name::String, data::AbstractDict{String, Any})
|
||
columns = String[]
|
||
values = String[]
|
||
|
||
for (key, value) in data
|
||
push!(columns, string(key))
|
||
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||
push!(values, value_str)
|
||
end
|
||
|
||
columns_str = join(columns, ", ")
|
||
values_str = join(values, ", ")
|
||
|
||
return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
|
||
end
|
||
|
||
|
||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||
|
||
""" generateUpdateSQL - Generate SQL UPDATE statement from dictionary data
|
||
|
||
This function constructs a SQL UPDATE statement by updating multiple columns
|
||
based on a primary key condition.
|
||
|
||
# Arguments:
|
||
- `table_name::String` - Name of the database table to update
|
||
- `pk_column::Symbol` - The primary key column name
|
||
- `pk_value` - The primary key value (used in WHERE clause)
|
||
- `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs to update
|
||
|
||
# Return:
|
||
- A String containing the SQL UPDATE statement
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia> using UUIDs
|
||
|
||
# Update multiple columns using a dictionary
|
||
table_name = "wine"
|
||
pk_column = :wine_id
|
||
pk_value = "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a"
|
||
data = Dict{Symbol, Any}(
|
||
:acidity => "full",
|
||
:tannin => "medium",
|
||
:country => "Italy"
|
||
)
|
||
|
||
julia> generateUpdateSQL(table_name, pk_column, pk_value, data)
|
||
"UPDATE wine SET acidity = 'full', tannin = 'medium', country = 'Italy' WHERE wine_id = '8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a';"
|
||
```
|
||
"""
|
||
function generateUpdateSQL(table_name::String, pk_column::String, pk_value,
|
||
data::AbstractDict{String, Any})
|
||
# Build SET clause
|
||
set_parts = String[]
|
||
for (key, value) in data
|
||
if key ∉ [pk_column]
|
||
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||
push!(set_parts, "$(string(key)) = $value_str")
|
||
end
|
||
end
|
||
|
||
set_clause = join(set_parts, ", ")
|
||
|
||
# Handle primary key value
|
||
pk_val_str = isa(pk_value, AbstractString) ? "'$pk_value'" : "$pk_value"
|
||
|
||
return "UPDATE $table_name SET $set_clause WHERE $pk_column = $pk_val_str;"
|
||
end
|
||
|
||
function generateUpdateSQL(table_name::String, pk_dict::AbstractDict{String, Any},
|
||
data::AbstractDict{String, Any})
|
||
# Build SET clause
|
||
set_parts = String[]
|
||
for (key, value) in data
|
||
if key ∉ keys(pk_dict)
|
||
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||
push!(set_parts, "$(string(key)) = $value_str")
|
||
end
|
||
end
|
||
|
||
set_clause = join(set_parts, ", ")
|
||
|
||
# Build WHERE clause for composite keys
|
||
where_parts = String[]
|
||
for (col, val) in pk_dict
|
||
val_str = isa(val, AbstractString) ? "'$val'" : "$val"
|
||
push!(where_parts, "$(string(col)) = $val_str")
|
||
end
|
||
|
||
where_clause = join(where_parts, " AND ")
|
||
|
||
return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
|
||
end
|
||
|
||
|
||
end # module |