diff --git a/Project.toml b/Project.toml index 175ee0e..eb6b085 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "GeneralUtils" uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" -version = "0.5.10" +version = "0.5.11" authors = ["tonaerospace "] [deps] diff --git a/src/dbUtil.jl b/src/dbUtil.jl index 3c3733b..24cf890 100644 --- a/src/dbUtil.jl +++ b/src/dbUtil.jl @@ -1,7 +1,7 @@ module dbUtil export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity, - execute_postgres_sql, get_embedding_nats + execute_postgres_sql using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames, SHA, NATS, LibPQ @@ -39,60 +39,6 @@ function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString} return result end -""" -get_embedding_nats(nats_server_url::String, texts::Vector{String}, sendto_topic::String, fileserver_url::String) - -Fetch embeddings for a list of texts from a NATS-based embedding service. - -# Arguments -- `nats_server_url::String`: NATS server connection URL -- `texts::Vector{String}`: List of text strings to generate embeddings for -- `sendto_topic::String`: NATS topic to send the embedding request to -- `fileserver_url::String`: URL of the file server for additional data - -# Returns -- `Vector` of embeddings, where each embedding is a vector of floats (Vector{Float32} or Vector{Float64}) - -# Example -```julia -nats_server_url = "nats://localhost:4222" -texts = ["Hello world", "Another text"] -sendto_topic = "embedding.service" -fileserver_url = "http://localhost:8080" -embeddings = get_embedding_nats(nats_server_url, texts, sendto_topic, fileserver_url) -``` -""" -function get_embedding_nats(nats_server_url::String, texts::Vector{String}, sendto_topic::String, - fileserver_url::String) - nats_conn = NATS.connect(nats_server_url) - return get_embedding_nats(nats_conn, texts, sendto_topic, fileserver_url) -end - -function get_embedding_nats(nats_conn::NATS.Connection, texts::Vector{String}, sendto_topic::String, - fileserver_url::String) - println("Generating embeddings for $(length(texts)) texts...") - documents_dict = Dict("documents" => texts) - payloads = [("documents", documents_dict, "dictionary")] - _, msg_envelope_json_str = msghandler.smartpack( - sendto_topic, - payloads; - msg_purpose="embedding", - fileserver_url=fileserver_url) - - reply = NATS.request(nats_conn, - sendto_topic, - msg_envelope_json_str, timeout=300) - incoming_env_json_str = String(reply.payload) - incoming_env = msghandler.smartunpack(incoming_env_json_str) - embedding_response = incoming_env["payloads"][1][2] - - result = [] - for i in embedding_response["data"] - embedding_vector = i["embedding"] - push!(result, embedding_vector) - end - return result -end """ find_text_vector_similarity