312 lines
10 KiB
Julia
312 lines
10 KiB
Julia
module dbUtil
|
||
|
||
export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
|
||
|
||
using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
|
||
SHA
|
||
using ..util
|
||
#[PENDING] update code to use JSON
|
||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||
|
||
""" Execute SQL against a PostgreSQL database using LibPQ connection string.
|
||
|
||
# Arguments
|
||
- `pg_conn_str::AbstractString`: PostgreSQL connection string in format "host=... port=... dbname=... user=... password=..."
|
||
- `sql::AbstractString`: SQL query to execute
|
||
|
||
# Returns
|
||
- `LibPQ.Result` on success, `nothing` on failure
|
||
|
||
# Example
|
||
```julia
|
||
pg_conn_str = "host=localhost port=5432 dbname=mydb user=myuser password=mypass"
|
||
sql = "SELECT * FROM wine;"
|
||
result = execute_postgres_sql(pg_conn_str, sql)
|
||
```
|
||
"""
|
||
function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
|
||
db_connection = LibPQ.Connection(pg_conn_str)
|
||
result = nothing
|
||
try
|
||
result = LibPQ.execute(db_connection, sql)
|
||
catch e
|
||
@error e
|
||
LibPQ.close(db_connection)
|
||
end
|
||
|
||
LibPQ.close(db_connection)
|
||
return result
|
||
end
|
||
|
||
"""
|
||
get_embedding_nats(nats_server_url::String, texts::Vector{String})
|
||
|
||
Fetch embeddings for a list of texts from a NATS-based embedding service.
|
||
|
||
# Arguments
|
||
- `nats_server_url::String`: NATS server connection URL
|
||
- `texts::Vector{String}`: List of text strings to generate embeddings for
|
||
|
||
# Returns
|
||
- `Vector` of embeddings, where each embedding is a vector of floats (Vector{Float32} or Vector{Float64})
|
||
|
||
# Example
|
||
```julia
|
||
nats_server_url = "nats://localhost:4222"
|
||
texts = ["Hello world", "Another text"]
|
||
embeddings = get_embedding_nats(nats_server_url, texts)
|
||
```
|
||
"""
|
||
function get_embedding_nats(nats_server_url::String, texts::Vector{String})
|
||
nats_conn = NATS.connect(nats_server_url)
|
||
return get_embedding_nats(nats_conn, texts)
|
||
end
|
||
|
||
function get_embedding_nats(nats_conn::NATS.Connection, texts::Vector{String})
|
||
println("Generating embeddings for $(length(texts)) texts...")
|
||
documents_dict = Dict("documents" => texts)
|
||
payloads = [("documents", documents_dict, "dictionary")]
|
||
_, msg_envelope_json_str = msghandler.smartpack(
|
||
config["externalservice"]["servicesloadbalancer"]["nats"],
|
||
payloads;
|
||
msg_purpose="embedding",
|
||
broker_url=config["nats_server_info"]["url"],
|
||
fileserver_url=config["externalservice"]["fileserver"]["url"])
|
||
|
||
reply = NATS.request(nats_conn,
|
||
config["externalservice"]["servicesloadbalancer"]["nats"],
|
||
msg_envelope_json_str, timeout=300)
|
||
incoming_env_json_str = String(reply.payload)
|
||
incoming_env = msghandler.smartunpack(incoming_env_json_str)
|
||
embedding_response = incoming_env["payloads"][1][2]
|
||
|
||
result = []
|
||
for i in embedding_response["data"]
|
||
embedding_vector = i["embedding"]
|
||
push!(result, embedding_vector)
|
||
end
|
||
return result
|
||
end
|
||
|
||
|
||
|
||
"""
|
||
dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format
|
||
|
||
This function takes a dictionary and converts it into a PostgreSQL-compatible key-value string
|
||
format suitable for storage in a TEXT field. The output format uses curly braces with comma-separated
|
||
key-value pairs, where string values are quoted.
|
||
|
||
# Function Workflow:
|
||
1. Iterates through dictionary key-value pairs
|
||
2. Handles nested dictionaries by recursively converting them
|
||
3. Wraps string values in double quotes
|
||
4. Formats numeric and other values without quotes
|
||
5. Returns a PostgreSQL-compatible key-value string enclosed in curly braces
|
||
|
||
# Arguments:
|
||
- `dict::Dict` - Dictionary containing key-value pairs to convert
|
||
|
||
# Return:
|
||
- A String in PostgreSQL key-value format: "{key1: value1, key2: \"value2\", ...}"
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia> data = Dict{String, Any}(
|
||
"name" => "John",
|
||
"age" => 30,
|
||
"city" => "New York"
|
||
);
|
||
|
||
julia> dictToPostgresKeyValueString(data)
|
||
"{\"name\": \"John\", \"age\": 30, \"city\": \"New York\"}"
|
||
```
|
||
"""
|
||
function dictToPostgresKeyValueString(dict)
|
||
parts = []
|
||
for (k, v) in dict
|
||
if isa(v, Dict)
|
||
push!(parts, "\"$k\": " * dict_to_string(v))
|
||
elseif isa(v, AbstractString)
|
||
push!(parts, "\"$k\": \"$v\"")
|
||
else
|
||
push!(parts, "\"$k\": $v")
|
||
end
|
||
end
|
||
return "{" * join(parts, ", ") * "}"
|
||
end
|
||
|
||
|
||
|
||
|
||
""" generateInsertSQL - Generate SQL INSERT statement from dictionary data
|
||
|
||
This function constructs a SQL INSERT statement by extracting values for specified columns
|
||
from a dictionary and formatting them into a valid PostgreSQL INSERT query.
|
||
|
||
# Function Workflow:
|
||
1. Iterates through the dictionary key-value pairs
|
||
2. Filters keys to only include those present in `columnToInsert`
|
||
3. Collects column names and their corresponding values
|
||
4. Constructs the final SQL INSERT statement
|
||
|
||
# Arguments:
|
||
- `table_name::String` - Name of the database table to insert into
|
||
- `columnToInsert::Vector{Symbol}` - List of column names to include in the INSERT statement
|
||
- `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs for the insert
|
||
|
||
# Return:
|
||
- A String containing the SQL INSERT statement
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia> using UUIDs
|
||
|
||
# Insert a single record with specific columns
|
||
table_name = "wine"
|
||
columnToInsert = [:acidity, :tannin, :country, :region, :winery]
|
||
data = Dict{Symbol, Any}(
|
||
:grape => "Cabernet Sauvignon",
|
||
:acidity => "medium", # using descriptive scale (low/medium/full)
|
||
:tannin => "medium-plus", # common wine descriptor
|
||
:country => "France",
|
||
:description => "A rich and structured red wine with notes of blackcurrant, cedar, and subtle oak.",
|
||
:region => "Bordeaux",
|
||
:winery => "Château Margaux",
|
||
:intensity => "medium", # intensity is usually low/medium/full
|
||
:sweetness => "dry", # dry/medium-dry/medium/medium-sweet/sweet
|
||
:tasting_notes => "Blackberry, graphite, tobacco, vanilla, and subtle earth.",
|
||
:wine_name => "Château Margaux Grand Cru",
|
||
:wine_id => "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a", # UUID-like (valid hex)
|
||
:wine_type => "Red",
|
||
:other_attributes => Dict{String, Any}(
|
||
"vintage" => 2018,
|
||
"alcohol_percent" => 13.5,
|
||
"ph" => 3.6,
|
||
" aging_years" => 24, # years in barrel
|
||
" producer_code" => "CM-GRAND"
|
||
),
|
||
:fizziness => "still",
|
||
:serving_temperature => "16–18°C",
|
||
:additional_search_term => ["Cabernet", "Bordeaux red", "premium wine", "CabSav"]
|
||
)
|
||
|
||
julia> generateInsertSQL(table_name, columnToInsert, data)
|
||
"INSERT INTO wine (acidity, tannin, country, region, winery) VALUES ('medium', 'medium-plus', 'France', 'Bordeaux', 'Château Margaux');"
|
||
```
|
||
"""
|
||
function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, data::Dict{Symbol, Any})
|
||
columns = String[]
|
||
values = String[]
|
||
|
||
for (key, value) in data
|
||
if key ∈ columnToInsert
|
||
push!(columns, string(key))
|
||
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||
push!(values, value_str)
|
||
end
|
||
end
|
||
|
||
columns_str = join(columns, ", ")
|
||
values_str = join(values, ", ")
|
||
|
||
return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
|
||
end
|
||
|
||
function generateInsertSQL(table_name::String, data::AbstractDict{String, Any})
|
||
columns = String[]
|
||
values = String[]
|
||
|
||
for (key, value) in data
|
||
push!(columns, string(key))
|
||
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||
push!(values, value_str)
|
||
end
|
||
|
||
columns_str = join(columns, ", ")
|
||
values_str = join(values, ", ")
|
||
|
||
return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
|
||
end
|
||
|
||
|
||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||
|
||
""" generateUpdateSQL - Generate SQL UPDATE statement from dictionary data
|
||
|
||
This function constructs a SQL UPDATE statement by updating multiple columns
|
||
based on a primary key condition.
|
||
|
||
# Arguments:
|
||
- `table_name::String` - Name of the database table to update
|
||
- `pk_column::Symbol` - The primary key column name
|
||
- `pk_value` - The primary key value (used in WHERE clause)
|
||
- `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs to update
|
||
|
||
# Return:
|
||
- A String containing the SQL UPDATE statement
|
||
|
||
# Example
|
||
```jldoctest
|
||
julia> using UUIDs
|
||
|
||
# Update multiple columns using a dictionary
|
||
table_name = "wine"
|
||
pk_column = :wine_id
|
||
pk_value = "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a"
|
||
data = Dict{Symbol, Any}(
|
||
:acidity => "full",
|
||
:tannin => "medium",
|
||
:country => "Italy"
|
||
)
|
||
|
||
julia> generateUpdateSQL(table_name, pk_column, pk_value, data)
|
||
"UPDATE wine SET acidity = 'full', tannin = 'medium', country = 'Italy' WHERE wine_id = '8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a';"
|
||
```
|
||
"""
|
||
function generateUpdateSQL(table_name::String, pk_column::String, pk_value,
|
||
data::AbstractDict{String, Any})
|
||
# Build SET clause
|
||
set_parts = String[]
|
||
for (key, value) in data
|
||
if key ∉ [pk_column]
|
||
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||
push!(set_parts, "$(string(key)) = $value_str")
|
||
end
|
||
end
|
||
|
||
set_clause = join(set_parts, ", ")
|
||
|
||
# Handle primary key value
|
||
pk_val_str = isa(pk_value, AbstractString) ? "'$pk_value'" : "$pk_value"
|
||
|
||
return "UPDATE $table_name SET $set_clause WHERE $pk_column = $pk_val_str;"
|
||
end
|
||
|
||
function generateUpdateSQL(table_name::String, pk_dict::AbstractDict{String, Any},
|
||
data::AbstractDict{String, Any})
|
||
# Build SET clause
|
||
set_parts = String[]
|
||
for (key, value) in data
|
||
if key ∉ keys(pk_dict)
|
||
value_str = isa(value, AbstractString) ? "'$value'" : "$value"
|
||
push!(set_parts, "$(string(key)) = $value_str")
|
||
end
|
||
end
|
||
|
||
set_clause = join(set_parts, ", ")
|
||
|
||
# Build WHERE clause for composite keys
|
||
where_parts = String[]
|
||
for (col, val) in pk_dict
|
||
val_str = isa(val, AbstractString) ? "'$val'" : "$val"
|
||
push!(where_parts, "$(string(col)) = $val_str")
|
||
end
|
||
|
||
where_clause = join(where_parts, " AND ")
|
||
|
||
return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
|
||
end
|
||
|
||
|
||
end # module |