update

src/llmUtil.jl

@@ -252,6 +252,45 @@ julia> g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_g
 julia> vertices(g)
 10
 ```
+
+# Integration Guide: Finding Related Tables for User Questions
+This function is designed to work together with `extract_column_metadata` and `resolve_semantic_cluster` to answer user questions by identifying related tables:
+
+```julia
+# Step 1: Extract column metadata and generate embeddings for semantic search
+pg_conn_str = "host=localhost port=5432 dbname=winedb user=admin password=secret"
+metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str)
+embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df)
+
+# Use only text content for embedding
+embedding_ready_2 = [i["text_content"] for i in embedding_ready]
+table_embedding = get_embedding(embedding_ready_2)
+
+# Get embedding for user question
+_user_question_embedding = get_embedding([question])
+user_question_embedding = Float64.(_user_question_embedding["data"][1]["embedding"])
+
+# Calculate similarity between question and all columns
+user_question_similarity = []
+for i in table_embedding["data"]
+  i_data = i["embedding"]
+  i_float = Float64.(i_data)
+  r = 1 - Distances.cosine_dist(i_float, user_question_embedding)
+  push!(user_question_similarity, r)
+end
+
+# Step 2: Find top related tables
+new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity))
+sorted_df = sort(new_df, :user_question_similarity, rev=true)
+_top_20_tables = unique(sorted_df[1:20, :table_name])
+top_20_tables = [i for i in _top_20_tables]
+
+# Step 3: Build schema graph and resolve table relationships
+g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str)
+table_relationship = GeneralUtils.resolve_semantic_cluster(top_20_tables, g, table_to_id, id_to_table)
+
+# table_relationship now contains tables in the order they should be joined
+```
 """
 function harvest_db_undirected_schema_graph(pg_conn_str)   
   conn = LibPQ.Connection(pg_conn_str) 
@@ -371,6 +410,21 @@ products    id           integer    Product ID          true            false
 products    price        numeric    Product price       false           false         col_products_price
 products    user_id      integer    Reference to user   false           true          col_products_user_id
 ```
+
+# Integration Guide
+This function is the first step in the semantic table discovery pipeline. Use it with `generate_embedding_payloads` to prepare data for vector similarity search:
+
+```julia
+# Step 1: Extract metadata
+metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str)
+
+# Step 2: Generate embedding payloads for semantic search
+embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df)
+
+# Use text_content field for embedding
+text_content_list = [i["text_content"] for i in embedding_ready]
+embeddings = get_embedding(text_content_list)
+```
 """
 function extract_column_metadata(pg_conn_str::String)::DataFrame
   conn = LibPQ.Connection(pg_conn_str)
@@ -457,6 +511,30 @@ julia> payloads = GeneralUtils.generate_embedding_payloads(df)
  Dict("id" => "col_users_name", "text_content" => "Table: users | Column: name | Type: text | Description: User name", "metadata" => Dict("table" => "users", "column" => "name", "type" => "text"))
  Dict("id" => "col_users_email", "text_content" => "Table: users | Column: email | Type: text | Description: User email", "metadata" => Dict("table" => "users", "column" => "email", "type" => "text"))
 ```
+
+# Integration Guide
+This function transforms column metadata into a format suitable for semantic search.
+Use `text_content` field to generate embeddings, then compare against user question embeddings:
+
+```julia
+# 1. Extract metadata
+metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str)
+
+# 2. Generate payloads
+embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df)
+
+# 3. Extract text content for embedding (only this field is used for similarity)
+text_content_list = [i["text_content"] for i in embedding_ready]
+
+# 4. Generate embeddings for all columns
+column_embeddings = get_embedding(text_content_list)
+
+# 5. Generate embedding for user question
+question_embedding = get_embedding([user_question])
+
+# 6. Calculate cosine similarity to find most relevant columns/tables
+# (See harvest_db_undirected_schema_graph for full integration example)
+```
 """
 function generate_embedding_payloads(df::DataFrame)::Vector{Dict}
     payloads = Dict[]
@@ -537,6 +615,50 @@ julia> vector_hits = ["users", "products"]
 julia> GeneralUtils.resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table)
 ["users", "orders", "products"]
 ```
+
+# Integration Guide: Understanding Table Relationships
+This function determines how tables from vector search results are connected in the database schema.
+The returned list represents the optimal join order for constructing SQL queries.
+
+**How tables are linked:**
+- **Explicit Foreign Keys**: Direct relationships defined by `FOREIGN KEY` constraints in PostgreSQL
+- **Implicit Relationships**: Tables sharing similar column naming patterns (e.g., `user_id` in both `users` and `orders` tables)
+
+**Example output interpretation:**
+```julia
+# Input: Top 20 tables from semantic search
+top_tables = ["users", "products", "payments"]
+
+# Output: Tables in join order
+table_relationship = ["users", "orders", "payments"]
+# Interpretation: 
+#   1. Start with 'users' table
+#   2. Join 'orders' via foreign key (likely users.id -> orders.user_id)
+#   3. Join 'payments' via foreign key (likely orders.id -> payments.order_id)
+```
+
+**Full workflow example:**
+```julia
+# Step 1: Get column embeddings and find semantically related tables
+metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str)
+embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df)
+text_content = [i["text_content"] for i in embedding_ready]
+table_embedding = get_embedding(text_content)
+
+# Step 2: Calculate similarity with user question
+question_embedding = Float64.(get_embedding([question])["data"][1]["embedding"])
+similarities = [1 - Distances.cosine_dist(Float64.(i["embedding"]), question_embedding) 
+                for i in table_embedding["data"]]
+
+# Step 3: Extract top related tables
+top_tables = unique(metadata_df[sortperm(similarities)[1:20], :table_name])
+
+# Step 4: Build schema graph and resolve relationships
+g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str)
+related_tables = GeneralUtils.resolve_semantic_cluster(top_tables, g, table_to_id, id_to_table)
+
+# related_tables now contains tables in optimal join order for SQL query construction
+```
 """
 function resolve_semantic_cluster(
     vector_hits::Vector{String},