add garageS3 feature

Reviewed-on: #12

Project.toml

@@ -1,6 +1,6 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.5.1"
+version = "0.6.0"
 authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
 
 [deps]

src/GeneralUtils.jl

@@ -21,6 +21,8 @@ using .llmUtil
 include("interface.jl")
 using .interface
 
+include("garageS3.jl")
+using .garageS3
 
 #------------------------------------------------------------------------------------------------100
 

src/dbUtil.jl

@@ -1,13 +1,107 @@
 module dbUtil
   
-export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
+export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity,
+        execute_postgres_sql
 
 using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
-      SHA
+      SHA, NATS, LibPQ
 using ..util
 #[PENDING] update code to use JSON
 # ---------------------------------------------- 100 --------------------------------------------- #
 
+""" Execute SQL against a PostgreSQL database using LibPQ connection string.
+
+# Arguments
+- `pg_conn_str::AbstractString`: PostgreSQL connection string in format "host=... port=... dbname=... user=... password=..."
+- `sql::AbstractString`: SQL query to execute
+
+# Returns
+- `LibPQ.Result` on success, `nothing` on failure
+
+# Example
+```julia
+pg_conn_str = "host=localhost port=5432 dbname=mydb user=myuser password=mypass"
+sql = "SELECT * FROM wine;"
+result = execute_postgres_sql(pg_conn_str, sql)
+```
+"""
+function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
+  db_connection = LibPQ.Connection(pg_conn_str)
+  result = nothing
+  try
+    result = LibPQ.execute(db_connection, sql)  
+  catch e
+    @error e
+    LibPQ.close(db_connection)
+  end
+  
+  LibPQ.close(db_connection)
+  return result
+end
+
+
+""" find_text_vector_similarity
+
+Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity.
+
+This function computes an embedding for the input text using the provided embedding function,
+then queries the database to find records with the most similar vector representations using
+PostgreSQL's cosine similarity operator (`<->`).
+
+# Arguments
+- `text::AbstractString`: The input text to find similar records for
+- `tablename::AbstractString`: Name of the database table containing the embedding column
+- `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings
+- `executesql::Function`: Function that executes SQL queries and returns results
+- `get_embedding::Function`: Function that generates embeddings for text inputs
+
+# Keyword Arguments
+- `limit::Integer=1`: Maximum number of similar records to return
+
+# Returns
+- `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column
+  where smaller values indicate higher similarity
+
+# Example
+```julia
+# Assume you have embedding and SQL execution functions
+text = "a rich structured red wine"
+tablename = "wine"
+embeddingColumnName = "description_embedding"
+
+df = find_text_vector_similarity(
+    text, tablename, embeddingColumnName,
+    executesql, get_embedding;
+    limit = 5
+)
+
+# Result contains columns from the table plus a 'distance' column
+# where distance = 1 - cosine_similarity (smaller = more similar)
+```
+"""
+function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3,
+  executesql::Function, get_embedding; 
+  limit::Integer=1
+  )::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
+  # get embedding from LLM service
+  _embedding = get_embedding([text])
+  _embedding = _embedding["data"][1]["embedding"]
+  _embedding = "$_embedding"
+
+  embedding = _embedding[4:end] # remove 'Any' from Any[...]
+
+  # check whether there is close enough vector already store in executesql. if no, add, else skip 
+  sql = """
+        SELECT *, $embeddingColumnName <-> '$embedding' as distance
+        FROM $tablename
+        ORDER BY distance LIMIT $limit;
+        """
+  response = executesql(sql)
+  df = DataFrame(response)
+
+  return df
+end
+
 
 """
 dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format

src/garageS3.jl

@@ -0,0 +1,137 @@
+module GarageS3
+
+export
+    GarageStorage,
+    put_file,
+    get_file,
+    list_files,
+    delete_file
+
+using AWS
+using AWSS3
+# ---------------------------------------------- 100 --------------------------------------------- #
+
+# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
+# this file use local AWS config
+
+""" Example
+const storage = GarageStorage(
+    "https://s3-api.yiem.cc",
+    "GKb080154a2e5b19100b1b2c6e",     # key ID  (create at garage-ui.yiem.cc)
+    "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",   # key ID's secret key
+    "sommpanion-s3"
+)
+
+# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
+put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
+keys = list_files(storage)
+data = String(get_file(storage, "users-1004.json"))
+
+println("Read back: ", data)
+println("Bucket keys: ", keys)
+
+# test with curl
+curl -v \
+  -H 'Host: sommpanion-s3.s3-web.yiem.cc' \
+  http://192.168.88.106:3902/users-1002.json
+
+
+# test with a browser
+https://s3-web.mydomain.com/my_bucket_name/users-1002.json
+
+"""
+
+# ===================================================================
+# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
+# ===================================================================
+
+struct SimpleCredentials
+    access_key_id::String
+    secret_key::String
+    token::String
+end
+
+struct GarageConfig <: AWS.AbstractAWSConfig
+    endpoint::String
+    region::String
+    credentials::SimpleCredentials
+end
+
+# Required extensions for AWS.jl pipeline
+AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
+AWS.credentials(c::SimpleCredentials) = c
+AWS.check_credentials(c::SimpleCredentials) = c
+AWS.region(aws::GarageConfig) = aws.region
+AWS.credentials(aws::GarageConfig) = aws.credentials
+
+function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
+    return strip(aws.endpoint, '/')
+end
+
+function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
+    endpoint = strip(aws.endpoint, '/')
+    resource_path = startswith(resource, '/') ? resource : "/" * resource
+    return string(endpoint, resource_path)
+end
+
+# ===================================================================
+# 2. Thread-Safe Storage Client Wrapper
+# ===================================================================
+
+struct GarageStorage
+    config::GarageConfig
+    bucket::String
+end
+
+function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
+    creds = SimpleCredentials(key, secret, "")
+    config = GarageConfig(endpoint, region, creds)
+    return GarageStorage(config, bucket)
+end
+
+# Write object to Garage
+function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}})
+    # Check if the key contains a slash (virtual folder character)
+    if occursin('/', key)
+        @warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
+        return nothing
+    end
+    
+    # Proceed if the key is flat
+    s3_put(storage.config, storage.bucket, key, data)
+    println("Successfully uploaded: ", key)
+end
+
+# Read object from Garage
+function get_file(storage::GarageStorage, key::String)::Vector{UInt8}
+    return s3_get(storage.config, storage.bucket, key)
+end
+
+# List keys in bucket (Thread-safe & explicit credentials)
+function list_files(storage::GarageStorage)
+    # Approach 2: s3_list_objects returns a Vector of Dicts with object details
+    objects = s3_list_objects(storage.config, storage.bucket)
+    return [obj["Key"] for obj in objects]
+end
+
+# Delete object from Garage
+function delete_file(storage::GarageStorage, key::String)
+    s3_delete(storage.config, storage.bucket, key)
+end
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+end # module GarageS3