add garageS3 feature

Reviewed-on: #12

Project.toml

@@ -1,6 +1,6 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.5.0"
+version = "0.6.0"
 authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
 
 [deps]

src/GeneralUtils.jl

@@ -21,6 +21,8 @@ using .llmUtil
 include("interface.jl")
 using .interface
 
+include("garageS3.jl")
+using .garageS3
 
 #------------------------------------------------------------------------------------------------100
 

src/dbUtil.jl

@@ -1,13 +1,107 @@
 module dbUtil
   
-export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
+export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL, find_text_vector_similarity,
+        execute_postgres_sql
 
 using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
-      SHA
+      SHA, NATS, LibPQ
 using ..util
 #[PENDING] update code to use JSON
 # ---------------------------------------------- 100 --------------------------------------------- #
 
+""" Execute SQL against a PostgreSQL database using LibPQ connection string.
+
+# Arguments
+- `pg_conn_str::AbstractString`: PostgreSQL connection string in format "host=... port=... dbname=... user=... password=..."
+- `sql::AbstractString`: SQL query to execute
+
+# Returns
+- `LibPQ.Result` on success, `nothing` on failure
+
+# Example
+```julia
+pg_conn_str = "host=localhost port=5432 dbname=mydb user=myuser password=mypass"
+sql = "SELECT * FROM wine;"
+result = execute_postgres_sql(pg_conn_str, sql)
+```
+"""
+function execute_postgres_sql(pg_conn_str::T, sql::T) where {T<:AbstractString}
+  db_connection = LibPQ.Connection(pg_conn_str)
+  result = nothing
+  try
+    result = LibPQ.execute(db_connection, sql)  
+  catch e
+    @error e
+    LibPQ.close(db_connection)
+  end
+  
+  LibPQ.close(db_connection)
+  return result
+end
+
+
+""" find_text_vector_similarity
+
+Find the most similar text records in a PostgreSQL database using vector embeddings and cosine similarity.
+
+This function computes an embedding for the input text using the provided embedding function,
+then queries the database to find records with the most similar vector representations using
+PostgreSQL's cosine similarity operator (`<->`).
+
+# Arguments
+- `text::AbstractString`: The input text to find similar records for
+- `tablename::AbstractString`: Name of the database table containing the embedding column
+- `embeddingColumnName::AbstractString`: Name of the column storing vector embeddings
+- `executesql::Function`: Function that executes SQL queries and returns results
+- `get_embedding::Function`: Function that generates embeddings for text inputs
+
+# Keyword Arguments
+- `limit::Integer=1`: Maximum number of similar records to return
+
+# Returns
+- `DataFrame`: Database records ordered by similarity (most similar first), including a `distance` column
+  where smaller values indicate higher similarity
+
+# Example
+```julia
+# Assume you have embedding and SQL execution functions
+text = "a rich structured red wine"
+tablename = "wine"
+embeddingColumnName = "description_embedding"
+
+df = find_text_vector_similarity(
+    text, tablename, embeddingColumnName,
+    executesql, get_embedding;
+    limit = 5
+)
+
+# Result contains columns from the table plus a 'distance' column
+# where distance = 1 - cosine_similarity (smaller = more similar)
+```
+"""
+function find_text_vector_similarity(text::T1, tablename::T2, embeddingColumnName::T3,
+  executesql::Function, get_embedding; 
+  limit::Integer=1
+  )::DataFrame where {T1<:AbstractString, T2<:AbstractString, T3<:AbstractString}
+  # get embedding from LLM service
+  _embedding = get_embedding([text])
+  _embedding = _embedding["data"][1]["embedding"]
+  _embedding = "$_embedding"
+
+  embedding = _embedding[4:end] # remove 'Any' from Any[...]
+
+  # check whether there is close enough vector already store in executesql. if no, add, else skip 
+  sql = """
+        SELECT *, $embeddingColumnName <-> '$embedding' as distance
+        FROM $tablename
+        ORDER BY distance LIMIT $limit;
+        """
+  response = executesql(sql)
+  df = DataFrame(response)
+
+  return df
+end
+
 
 """
 dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format

src/garageS3.jl

@@ -0,0 +1,137 @@
+module GarageS3
+
+export
+    GarageStorage,
+    put_file,
+    get_file,
+    list_files,
+    delete_file
+
+using AWS
+using AWSS3
+# ---------------------------------------------- 100 --------------------------------------------- #
+
+# Garage uses Path-Style routing (https://s3-api.my-domain.com/bucket/key).
+# this file use local AWS config
+
+""" Example
+const storage = GarageStorage(
+    "https://s3-api.yiem.cc",
+    "GKb080154a2e5b19100b1b2c6e",     # key ID  (create at garage-ui.yiem.cc)
+    "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",   # key ID's secret key
+    "sommpanion-s3"
+)
+
+# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
+put_file(storage, "users-1005.json", "{\"status\": \"active\"}")
+keys = list_files(storage)
+data = String(get_file(storage, "users-1004.json"))
+
+println("Read back: ", data)
+println("Bucket keys: ", keys)
+
+# test with curl
+curl -v \
+  -H 'Host: sommpanion-s3.s3-web.yiem.cc' \
+  http://192.168.88.106:3902/users-1002.json
+
+
+# test with a browser
+https://s3-web.mydomain.com/my_bucket_name/users-1002.json
+
+"""
+
+# ===================================================================
+# 1. Custom GarageConfig Definition (Thread-Safe, No Global State)
+# ===================================================================
+
+struct SimpleCredentials
+    access_key_id::String
+    secret_key::String
+    token::String
+end
+
+struct GarageConfig <: AWS.AbstractAWSConfig
+    endpoint::String
+    region::String
+    credentials::SimpleCredentials
+end
+
+# Required extensions for AWS.jl pipeline
+AWS.refresh!(c::SimpleCredentials; force::Bool=false) = c
+AWS.credentials(c::SimpleCredentials) = c
+AWS.check_credentials(c::SimpleCredentials) = c
+AWS.region(aws::GarageConfig) = aws.region
+AWS.credentials(aws::GarageConfig) = aws.credentials
+
+function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
+    return strip(aws.endpoint, '/')
+end
+
+function AWS.generate_service_url(aws::GarageConfig, service::String, resource::String)
+    endpoint = strip(aws.endpoint, '/')
+    resource_path = startswith(resource, '/') ? resource : "/" * resource
+    return string(endpoint, resource_path)
+end
+
+# ===================================================================
+# 2. Thread-Safe Storage Client Wrapper
+# ===================================================================
+
+struct GarageStorage
+    config::GarageConfig
+    bucket::String
+end
+
+function GarageStorage(endpoint::String, key::String, secret::String, bucket::String; region::String="garage")
+    creds = SimpleCredentials(key, secret, "")
+    config = GarageConfig(endpoint, region, creds)
+    return GarageStorage(config, bucket)
+end
+
+# Write object to Garage
+function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}})
+    # Check if the key contains a slash (virtual folder character)
+    if occursin('/', key)
+        @warn "Upload aborted! Slashes ('/') are not allowed in keys ('$key') to prevent S3 listing issues. Use a flat naming convention instead (e.g., 'users-1002.json')."
+        return nothing
+    end
+    
+    # Proceed if the key is flat
+    s3_put(storage.config, storage.bucket, key, data)
+    println("Successfully uploaded: ", key)
+end
+
+# Read object from Garage
+function get_file(storage::GarageStorage, key::String)::Vector{UInt8}
+    return s3_get(storage.config, storage.bucket, key)
+end
+
+# List keys in bucket (Thread-safe & explicit credentials)
+function list_files(storage::GarageStorage)
+    # Approach 2: s3_list_objects returns a Vector of Dicts with object details
+    objects = s3_list_objects(storage.config, storage.bucket)
+    return [obj["Key"] for obj in objects]
+end
+
+# Delete object from Garage
+function delete_file(storage::GarageStorage, key::String)
+    s3_delete(storage.config, storage.bucket, key)
+end
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+end # module GarageS3

src/llmUtil.jl

@@ -3,7 +3,7 @@ module llmUtil
 export formatLLMtext, extractthink, checkAgentResponse_JSON, clean_json_response,
         extract_column_metadata, generate_embedding_payloads, resolve_semantic_cluster,
         harvest_entity_catalog, resolve_entity, harvest_db_undirected_schema_graph,
-        get_db_table_schema
+        get_db_table_schema, get_db_table_schema_simple
 
 using UUIDs, JSON, Dates, DataFrames, StringDistances, Graphs, LibPQ
 using ..util
@@ -1016,7 +1016,115 @@ function get_db_table_schema(conn::LibPQ.Connection, table_name::String)::DataFr
 end
 
 
+""" Generate simplified DDL statement for a PostgreSQL table.
 
+Extracts table structure from PostgreSQL and returns a clean CREATE TABLE statement
+without NULL/NOT NULL constraints, useful for schema documentation or migration purposes.
+
+# Arguments
+- `conn::LibPQ.Connection`
+  A PostgreSQL connection object created via `LibPQ.Connection()`.
+- `table_name::String`
+  The name of the table to extract.
+- `schema_name::String` (default: `"public"`)
+  The schema containing the table.
+
+# Return
+- `String`
+  A CREATE TABLE DDL statement containing:
+  - Column definitions with names, types, and DEFAULT values
+  - Table-level constraints (PRIMARY KEY, FOREIGN KEY, UNIQUE, CHECK)
+  - Excludes NULL/NOT NULL specifications for cleaner output
+
+# Details
+The function:
+1. Queries PostgreSQL system catalogs (`pg_attribute`, `pg_class`, `pg_namespace`)
+2. Extracts column names, data types, and default values
+3. Captures table-level constraints via `pg_constraint`
+4. Omits NULL/NOT NULL checks to produce a simplified schema definition
+5. Returns properly formatted DDL with quoted identifiers
+
+# Example
+```julia
+julia> using GeneralUtils, LibPQ
+julia> conn = LibPQ.Connection("host=localhost port=5432 dbname=winedb user=admin password=secret")
+julia> ddl = GeneralUtils.get_db_table_schema_simple(conn, "wine")
+"CREATE TABLE \"public\".\"wine\" (
+    \"id\" integer DEFAULT nextval('wine_id_seq'::regclass),
+    \"wine_name\" text,
+    \"year\" integer,
+    \"price\" numeric
+);"
+```
+"""
+function get_db_table_schema_simple(pg_conn_str::String, table_name::String; 
+  schema_name::String="public")::String
+  conn = LibPQ.Connection(pg_conn_str)
+  return get_db_table_schema_simple(conn, table_name; schema_name=schema_name)
+end
+
+function get_db_table_schema_simple(conn, table_name::String; schema_name::String="public")::String
+    # 1. SQL query tailored to omit nullability checks
+    sql = """
+    SELECT 
+        a.attname AS column_name,
+        format_type(a.atttypid, a.atttypmod) AS data_type,
+        pg_get_expr(def.adbin, def.adrelid) AS default_value,
+        COALESCE(
+            (SELECT pg_get_constraintdef(p.oid) 
+             FROM pg_catalog.pg_constraint p 
+             WHERE p.conrelid = c.oid AND a.attnum = ANY(p.conkey) 
+             LIMIT 1), ''
+        ) AS constraint_definition
+    FROM 
+        pg_catalog.pg_attribute a
+    JOIN 
+        pg_catalog.pg_class c ON a.attrelid = c.oid
+    JOIN 
+        pg_catalog.pg_namespace n ON c.relnamespace = n.oid
+    LEFT JOIN 
+        pg_catalog.pg_attrdef def ON def.adrelid = c.oid AND def.adnum = a.attnum
+    WHERE 
+        c.relname = \$1
+        AND n.nspname = \$2
+        AND a.attnum > 0 
+        AND NOT a.attisdropped
+    ORDER BY 
+        a.attnum;
+    """
+
+    result = execute(conn, sql, [table_name, schema_name])
+    
+    if length(result) == 0
+        error("Table '$schema_name.$table_name' not found.")
+    end
+
+    ddl_lines = String[]
+    constraints = String[]
+
+    for row in result
+        col_name   = row.column_name
+        data_type  = row.data_type
+        
+        # Handle the default value if it exists
+        default_val = ismissing(row.default_value) ? "" : " DEFAULT " * row.default_value
+        
+        # Build the column definition line (without NULL/NOT NULL)
+        col_def = "    \"$col_name\" $data_type$default_val"
+        push!(ddl_lines, col_def)
+
+        # Handle table-level constraints
+        con_def = ismissing(row.constraint_definition) ? "" : row.constraint_definition
+        if !isempty(con_def) && !(con_def in constraints)
+            push!(constraints, "    " * con_def)
+        end
+    end
+
+    all_definitions = vcat(ddl_lines, constraints)
+    body = join(all_definitions, ",\n")
+    
+    return "CREATE TABLE \"$schema_name\".\"$table_name\" (\n$body\n);"
+end