update

Project.toml

@@ -1,6 +1,6 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.5.3"
+version = "0.5.11"
 authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
 
 [deps]

src/dbUtil.jl

@@ -1,9 +1,10 @@
 module dbUtil
   
-export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
+export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity,
+        execute_postgres_sql
 
 using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
-      SHA
+      SHA, NATS, LibPQ
 using ..util
 #[PENDING] update code to use JSON
 # ---------------------------------------------- 100 --------------------------------------------- #
@@ -38,57 +39,69 @@ function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
   return result
 end
 
-"""
-get_embedding_nats(nats_server_url::String, texts::Vector{String})
 
-Fetch embeddings for a list of texts from a NATS-based embedding service.
+""" find_text_vector_similarity
+
+Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity.
+
+This function computes an embedding for the input text using the provided embedding function,
+then queries the database to find records with the most similar vector representations using
+PostgreSQL's cosine similarity operator (`<->`).
 
 # Arguments
-- `nats_server_url::String`: NATS server connection URL
+- `text::AbstractString`: The input text to find similar records for
-- `texts::Vector{String}`: List of text strings to generate embeddings for
+- `tablename::AbstractString`: Name of the database table containing the embedding column
+- `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings
+- `executesql::Function`: Function that executes SQL queries and returns results
+- `get_embedding::Function`: Function that generates embeddings for text inputs
+
+# Keyword Arguments
+- `limit::Integer=1`: Maximum number of similar records to return
 
 # Returns
-- `Vector` of embeddings, where each embedding is a vector of floats (Vector{Float32} or Vector{Float64})
+- `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column
+  where smaller values indicate higher similarity
 
 # Example
 ```julia
-nats_server_url = "nats://localhost:4222"
+# Assume you have embedding and SQL execution functions
-texts = ["Hello world", "Another text"]
+text = "a rich structured red wine"
-embeddings = get_embedding_nats(nats_server_url, texts)
+tablename = "wine"
+embeddingColumnName = "description_embedding"
+
+df = find_text_vector_similarity(
+    text, tablename, embeddingColumnName,
+    executesql, get_embedding;
+    limit = 5
+)
+
+# Result contains columns from the table plus a 'distance' column
+# where distance = 1 - cosine_similarity (smaller = more similar)
 ```
 """
-function get_embedding_nats(nats_server_url::String, texts::Vector{String})
+function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3,
-  nats_conn = NATS.connect(nats_server_url)
+  executesql::Function, get_embedding; 
-  return get_embedding_nats(nats_conn, texts)
+  limit::Integer=1
+  )::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
+  # get embedding from LLM service
+  _embedding = get_embedding([text])
+  _embedding = _embedding["data"][1]["embedding"]
+  _embedding = "$_embedding"
+
+  embedding = _embedding[4:end] # remove 'Any' from Any[...]
+
+  # check whether there is close enough vector already store in executesql. if no, add, else skip 
+  sql = """
+        SELECT *, $embeddingColumnName <-> '$embedding' as distance
+        FROM $tablename
+        ORDER BY distance LIMIT $limit;
+        """
+  response = executesql(sql)
+  df = DataFrame(response)
+
+  return df
 end
 
-function get_embedding_nats(nats_conn::NATS.Connection, texts::Vector{String})
-  println("Generating embeddings for $(length(texts)) texts...")
-  documents_dict = Dict("documents" => texts)
-  payloads = [("documents", documents_dict, "dictionary")]
-  _, msg_envelope_json_str = msghandler.smartpack(
-    config["externalservice"]["servicesloadbalancer"]["nats"], 
-    payloads;   
-    msg_purpose="embedding",  
-    broker_url=config["nats_server_info"]["url"],
-    fileserver_url=config["externalservice"]["fileserver"]["url"])
-
-  reply = NATS.request(nats_conn, 
-                      config["externalservice"]["servicesloadbalancer"]["nats"], 
-                      msg_envelope_json_str, timeout=300)
-  incoming_env_json_str = String(reply.payload)
-  incoming_env = msghandler.smartunpack(incoming_env_json_str)
-  embedding_response = incoming_env["payloads"][1][2]
-
-  result = []
-  for i in embedding_response["data"]
-    embedding_vector = i["embedding"]
-    push!(result, embedding_vector)
-  end
-  return result
-end
-
-
 
 """
 dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format