diff --git a/src/llmUtil.jl b/src/llmUtil.jl index 5545822..f2c603b 100644 --- a/src/llmUtil.jl +++ b/src/llmUtil.jl @@ -252,6 +252,45 @@ julia> g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_g julia> vertices(g) 10 ``` + +# Integration Guide: Finding Related Tables for User Questions +This function is designed to work together with `extract_column_metadata` and `resolve_semantic_cluster` to answer user questions by identifying related tables: + +```julia +# Step 1: Extract column metadata and generate embeddings for semantic search +pg_conn_str = "host=localhost port=5432 dbname=winedb user=admin password=secret" +metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str) +embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df) + +# Use only text content for embedding +embedding_ready_2 = [i["text_content"] for i in embedding_ready] +table_embedding = get_embedding(embedding_ready_2) + +# Get embedding for user question +_user_question_embedding = get_embedding([question]) +user_question_embedding = Float64.(_user_question_embedding["data"][1]["embedding"]) + +# Calculate similarity between question and all columns +user_question_similarity = [] +for i in table_embedding["data"] + i_data = i["embedding"] + i_float = Float64.(i_data) + r = 1 - Distances.cosine_dist(i_float, user_question_embedding) + push!(user_question_similarity, r) +end + +# Step 2: Find top related tables +new_df = hcat(metadata_df, DataFrame(user_question_similarity = user_question_similarity)) +sorted_df = sort(new_df, :user_question_similarity, rev=true) +_top_20_tables = unique(sorted_df[1:20, :table_name]) +top_20_tables = [i for i in _top_20_tables] + +# Step 3: Build schema graph and resolve table relationships +g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str) +table_relationship = GeneralUtils.resolve_semantic_cluster(top_20_tables, g, table_to_id, id_to_table) + +# table_relationship now contains tables in the order they should be joined +``` """ function harvest_db_undirected_schema_graph(pg_conn_str) conn = LibPQ.Connection(pg_conn_str) @@ -371,6 +410,21 @@ products id integer Product ID true false products price numeric Product price false false col_products_price products user_id integer Reference to user false true col_products_user_id ``` + +# Integration Guide +This function is the first step in the semantic table discovery pipeline. Use it with `generate_embedding_payloads` to prepare data for vector similarity search: + +```julia +# Step 1: Extract metadata +metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str) + +# Step 2: Generate embedding payloads for semantic search +embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df) + +# Use text_content field for embedding +text_content_list = [i["text_content"] for i in embedding_ready] +embeddings = get_embedding(text_content_list) +``` """ function extract_column_metadata(pg_conn_str::String)::DataFrame conn = LibPQ.Connection(pg_conn_str) @@ -457,6 +511,30 @@ julia> payloads = GeneralUtils.generate_embedding_payloads(df) Dict("id" => "col_users_name", "text_content" => "Table: users | Column: name | Type: text | Description: User name", "metadata" => Dict("table" => "users", "column" => "name", "type" => "text")) Dict("id" => "col_users_email", "text_content" => "Table: users | Column: email | Type: text | Description: User email", "metadata" => Dict("table" => "users", "column" => "email", "type" => "text")) ``` + +# Integration Guide +This function transforms column metadata into a format suitable for semantic search. +Use `text_content` field to generate embeddings, then compare against user question embeddings: + +```julia +# 1. Extract metadata +metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str) + +# 2. Generate payloads +embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df) + +# 3. Extract text content for embedding (only this field is used for similarity) +text_content_list = [i["text_content"] for i in embedding_ready] + +# 4. Generate embeddings for all columns +column_embeddings = get_embedding(text_content_list) + +# 5. Generate embedding for user question +question_embedding = get_embedding([user_question]) + +# 6. Calculate cosine similarity to find most relevant columns/tables +# (See harvest_db_undirected_schema_graph for full integration example) +``` """ function generate_embedding_payloads(df::DataFrame)::Vector{Dict} payloads = Dict[] @@ -537,6 +615,50 @@ julia> vector_hits = ["users", "products"] julia> GeneralUtils.resolve_semantic_cluster(vector_hits, g, table_to_id, id_to_table) ["users", "orders", "products"] ``` + +# Integration Guide: Understanding Table Relationships +This function determines how tables from vector search results are connected in the database schema. +The returned list represents the optimal join order for constructing SQL queries. + +**How tables are linked:** +- **Explicit Foreign Keys**: Direct relationships defined by `FOREIGN KEY` constraints in PostgreSQL +- **Implicit Relationships**: Tables sharing similar column naming patterns (e.g., `user_id` in both `users` and `orders` tables) + +**Example output interpretation:** +```julia +# Input: Top 20 tables from semantic search +top_tables = ["users", "products", "payments"] + +# Output: Tables in join order +table_relationship = ["users", "orders", "payments"] +# Interpretation: +# 1. Start with 'users' table +# 2. Join 'orders' via foreign key (likely users.id -> orders.user_id) +# 3. Join 'payments' via foreign key (likely orders.id -> payments.order_id) +``` + +**Full workflow example:** +```julia +# Step 1: Get column embeddings and find semantically related tables +metadata_df = GeneralUtils.extract_column_metadata(pg_conn_str) +embedding_ready = GeneralUtils.generate_embedding_payloads(metadata_df) +text_content = [i["text_content"] for i in embedding_ready] +table_embedding = get_embedding(text_content) + +# Step 2: Calculate similarity with user question +question_embedding = Float64.(get_embedding([question])["data"][1]["embedding"]) +similarities = [1 - Distances.cosine_dist(Float64.(i["embedding"]), question_embedding) + for i in table_embedding["data"]] + +# Step 3: Extract top related tables +top_tables = unique(metadata_df[sortperm(similarities)[1:20], :table_name]) + +# Step 4: Build schema graph and resolve relationships +g, id_to_table, table_to_id = GeneralUtils.harvest_db_undirected_schema_graph(pg_conn_str) +related_tables = GeneralUtils.resolve_semantic_cluster(top_tables, g, table_to_id, id_to_table) + +# related_tables now contains tables in optimal join order for SQL query construction +``` """ function resolve_semantic_cluster( vector_hits::Vector{String},