update

Project.toml

@@ -1,6 +1,6 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.5.2"
+version = "0.5.11"
 authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
 
 [deps]

src/dbUtil.jl

@@ -1,9 +1,10 @@
 module dbUtil
   
-export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
+export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity,
+        execute_postgres_sql
 
 using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
-      SHA
+      SHA, NATS, LibPQ
 using ..util
 #[PENDING] update code to use JSON
 # ---------------------------------------------- 100 --------------------------------------------- #
@@ -39,6 +40,69 @@ function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
 end
 
 
+""" find_text_vector_similarity
+
+Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity.
+
+This function computes an embedding for the input text using the provided embedding function,
+then queries the database to find records with the most similar vector representations using
+PostgreSQL's cosine similarity operator (`<->`).
+
+# Arguments
+- `text::AbstractString`: The input text to find similar records for
+- `tablename::AbstractString`: Name of the database table containing the embedding column
+- `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings
+- `executesql::Function`: Function that executes SQL queries and returns results
+- `get_embedding::Function`: Function that generates embeddings for text inputs
+
+# Keyword Arguments
+- `limit::Integer=1`: Maximum number of similar records to return
+
+# Returns
+- `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column
+  where smaller values indicate higher similarity
+
+# Example
+```julia
+# Assume you have embedding and SQL execution functions
+text = "a rich structured red wine"
+tablename = "wine"
+embeddingColumnName = "description_embedding"
+
+df = find_text_vector_similarity(
+    text, tablename, embeddingColumnName,
+    executesql, get_embedding;
+    limit = 5
+)
+
+# Result contains columns from the table plus a 'distance' column
+# where distance = 1 - cosine_similarity (smaller = more similar)
+```
+"""
+function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3,
+  executesql::Function, get_embedding::Function; 
+  limit::Integer=1
+  )::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
+  # get embedding from LLM service
+  _embedding = get_embedding([text])
+  _embedding = _embedding["data"][1]["embedding"]
+  _embedding = "$_embedding"
+
+  embedding = _embedding[4:end] # remove 'Any' from Any[...]
+
+  # check whether there is close enough vector already store in executesql. if no, add, else skip 
+  sql = """
+        SELECT *, $embeddingColumnName <-> '$embedding' as distance
+        FROM $tablename
+        ORDER BY distance LIMIT $limit;
+        """
+  response = executesql(sql)
+  df = DataFrame(response)
+
+  return df
+end
+
+
 """
 dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format