From c5ad5c882c57cadeadb55a49a815e4a076ec5443 Mon Sep 17 00:00:00 2001 From: narawat Date: Tue, 30 Jun 2026 13:48:09 +0700 Subject: [PATCH] update --- workprocess.md => docs/workprocess.md | 182 ++++++++++---------- examples/README.md | 57 ++++++ examples/chess_game.jl | 191 ++++++++++++++++++++ examples/code_generation.jl | 115 +++++++++++++ examples/configuration_examples.jl | 239 ++++++++++++++++++++++++++ examples/math_problem.jl | 97 +++++++++++ examples/pathfinding.jl | 98 +++++++++++ examples/reasoning.jl | 134 +++++++++++++++ examples/simple_example.jl | 59 +++++++ examples/tool_use.jl | 109 ++++++++++++ 10 files changed, 1190 insertions(+), 91 deletions(-) rename workprocess.md => docs/workprocess.md (96%) create mode 100644 examples/README.md create mode 100644 examples/chess_game.jl create mode 100644 examples/code_generation.jl create mode 100644 examples/configuration_examples.jl create mode 100644 examples/math_problem.jl create mode 100644 examples/pathfinding.jl create mode 100644 examples/reasoning.jl create mode 100644 examples/simple_example.jl create mode 100644 examples/tool_use.jl diff --git a/workprocess.md b/docs/workprocess.md similarity index 96% rename from workprocess.md rename to docs/workprocess.md index 6893af2..15770a6 100644 --- a/workprocess.md +++ b/docs/workprocess.md @@ -65,8 +65,8 @@ Expansion Phase (LLM call): │ transition(state, args) → { │ │ newNodeKey: "abc-123", │ │ newstate: { reward: 0, isterminal: false }, │ -│ progressvalue: 7.5 ← LLM estimates this state is promising │ -│ } │ +│ progressvalue: 7.5 ← LLM estimates this state is promising │ +│ } │ └────────────────────────────────────────────────────────────────────────┘ ↓ New Node Created: @@ -94,7 +94,7 @@ Backpropagation: ┌────────────────────────────────────────────────────────────────────────┐ │ Update all ancestors with simTrajectoryReward = 13: │ │ node.visits += 1 │ -│ node.statevalue = (old_statevalue * (visits-1) + 13) / visits │ +│ node.statevalue = (old_statevalue * (visits-1) + 13) / visits │ │ reward *= 0.9 (discount for future rewards) │ └────────────────────────────────────────────────────────────────────────┘ ``` @@ -615,75 +615,75 @@ Current Node: "Problem solving step 5" ``` ┌─────────────────────────────────────────────────────────────────────────────┐ -│ MCTS ITERATION 1 │ +│ MCTS ITERATION 1 │ ├─────────────────────────────────────────────────────────────────────────────┤ -│ │ -│ 1. SELECTION (UCT-based traversal): │ -│ Root (visits=1) │ -│ └── UCTselect() on root (has no children yet, so uses progressvalue) │ +│ │ +│ 1. SELECTION (UCT-based traversal): │ +│ Root (visits=1) │ +│ └── UCTselect() on root (has no children yet, so uses progressvalue) │ │ └── Select child with highest progressvalue │ -│ └── Continue until reaching leaf node │ -│ │ -│ 2. EXPANSION (horizontal sampling): │ -│ Leaf node │ +│ └── Continue until reaching leaf node │ +│ │ +│ 2. EXPANSION (horizontal sampling): │ +│ Leaf node │ │ └── expand(horizontalSample=3) │ -│ └── Generate 3 child nodes via LLM transition │ +│ └── Generate 3 child nodes via LLM transition │ │ └── Each child gets: progressvalue, reward, parent=leaf │ -│ │ -│ 3. SIMULATION (vertical rollout): │ -│ Each child node │ +│ │ +│ 3. SIMULATION (vertical rollout): │ +│ Each child node │ │ └── simulate(maxSimulationDepth=3) │ │ └── Rollout 3 levels deep, accumulating rewards │ │ └── Return (simTrajectoryReward, terminalstate) │ -│ │ -│ 4. BACKPROPAGATION (update statistics): │ -│ simTrajectoryReward │ -│ └── backpropagate() up to root │ +│ │ +│ 4. BACKPROPAGATION (update statistics): │ +│ simTrajectoryReward │ +│ └── backpropagate() up to root │ │ └── Update visits and statevalue for all ancestors │ │ └── Apply discount to future rewards │ -│ │ +│ │ └─────────────────────────────────────────────────────────────────────────────┘ ┌─────────────────────────────────────────────────────────────────────────────┐ -│ MCTS ITERATION 2 │ +│ MCTS ITERATION 2 │ ├─────────────────────────────────────────────────────────────────────────────┤ -│ │ -│ 1. SELECTION: │ -│ Root (visits=2) │ -│ └── UCTselect() now considers: │ +│ │ +│ 1. SELECTION: │ +│ Root (visits=2) │ +│ └── UCTselect() now considers: │ │ ├── statevalue (from iteration 1) │ │ └── exploration term (low visits on unexpanded branches) │ │ └── May select different path than iteration 1 │ -│ │ -│ 2. EXPANSION: │ +│ │ +│ 2. EXPANSION: │ │ └── May expand different node or same node │ │ └── New children added to node.children │ -│ │ -│ 3. SIMULATION: │ -│ └── Different rollout trajectory │ +│ │ +│ 3. SIMULATION: │ +│ └── Different rollout trajectory │ │ └── New reward estimate added to statistics │ -│ │ -│ 4. BACKPROPAGATION: │ +│ │ +│ 4. BACKPROPAGATION: │ │ └── Statistics updated with new information │ -│ │ +│ │ └─────────────────────────────────────────────────────────────────────────────┘ ┌─────────────────────────────────────────────────────────────────────────────┐ -│ MCTS ITERATION 3 to N │ +│ MCTS ITERATION 3 to N │ ├─────────────────────────────────────────────────────────────────────────────┤ -│ │ -│ Pattern repeats, with increasingly informed selection: │ -│ │ -│ • Nodes with high statevalue (confirmed by many simulations) │ -│ → Exploited (selected frequently) │ -│ │ +│ │ +│ Pattern repeats, with increasingly informed selection: │ +│ │ +│ • Nodes with high statevalue (confirmed by many simulations) │ +│ → Exploited (selected frequently) │ +│ │ │ • Nodes with low visits but promising progressvalue │ -│ → Explored (UCT exploration term encourages tries) │ -│ │ -│ • Tree grows: more branches explored, more statistics accumulated │ -│ │ -│ • Best trajectory emerges from accumulated statistics │ -│ │ +│ → Explored (UCT exploration term encourages tries) │ +│ │ +│ • Tree grows: more branches explored, more statistics accumulated │ +│ │ +│ • Best trajectory emerges from accumulated statistics │ +│ │ └─────────────────────────────────────────────────────────────────────────────┘ ``` @@ -957,75 +957,75 @@ Step-by-step execution: ``` ┌─────────────────────────────────────────────────────────────────────────────┐ -│ MCTS ITERATION 1 │ +│ MCTS ITERATION 1 │ ├─────────────────────────────────────────────────────────────────────────────┤ -│ │ -│ 1. SELECTION (UCT-based traversal): │ -│ Root (visits=1) │ -│ └── UCTselect() on root (has no children yet, so uses progressvalue) │ +│ │ +│ 1. SELECTION (UCT-based traversal): │ +│ Root (visits=1) │ +│ └── UCTselect() on root (has no children yet, so uses progressvalue) │ │ └── Select child with highest progressvalue │ -│ └── Continue until reaching leaf node │ -│ │ -│ 2. EXPANSION (horizontal sampling): │ -│ Leaf node │ +│ └── Continue until reaching leaf node │ +│ │ +│ 2. EXPANSION (horizontal sampling): │ +│ Leaf node │ │ └── expand(horizontalSample=3) │ -│ └── Generate 3 child nodes via LLM transition │ +│ └── Generate 3 child nodes via LLM transition │ │ └── Each child gets: progressvalue, reward, parent=leaf │ -│ │ -│ 3. SIMULATION (vertical rollout): │ -│ Each child node │ +│ │ +│ 3. SIMULATION (vertical rollout): │ +│ Each child node │ │ └── simulate(maxSimulationDepth=3) │ │ └── Rollout 3 levels deep, accumulating rewards │ │ └── Return (simTrajectoryReward, terminalstate) │ -│ │ -│ 4. BACKPROPAGATION (update statistics): │ -│ simTrajectoryReward │ -│ └── backpropagate() up to root │ +│ │ +│ 4. BACKPROPAGATION (update statistics): │ +│ simTrajectoryReward │ +│ └── backpropagate() up to root │ │ └── Update visits and statevalue for all ancestors │ │ └── Apply discount to future rewards │ -│ │ +│ │ └─────────────────────────────────────────────────────────────────────────────┘ ┌─────────────────────────────────────────────────────────────────────────────┐ -│ MCTS ITERATION 2 │ +│ MCTS ITERATION 2 │ ├─────────────────────────────────────────────────────────────────────────────┤ -│ │ -│ 1. SELECTION: │ -│ Root (visits=2) │ -│ └── UCTselect() now considers: │ +│ │ +│ 1. SELECTION: │ +│ Root (visits=2) │ +│ └── UCTselect() now considers: │ │ ├── statevalue (from iteration 1) │ │ └── exploration term (low visits on unexpanded branches) │ │ └── May select different path than iteration 1 │ -│ │ -│ 2. EXPANSION: │ +│ │ +│ 2. EXPANSION: │ │ └── May expand different node or same node │ │ └── New children added to node.children │ -│ │ -│ 3. SIMULATION: │ -│ └── Different rollout trajectory │ +│ │ +│ 3. SIMULATION: │ +│ └── Different rollout trajectory │ │ └── New reward estimate added to statistics │ -│ │ -│ 4. BACKPROPAGATION: │ +│ │ +│ 4. BACKPROPAGATION: │ │ └── Statistics updated with new information │ -│ │ +│ │ └─────────────────────────────────────────────────────────────────────────────┘ ┌─────────────────────────────────────────────────────────────────────────────┐ -│ MCTS ITERATION 3 to N │ +│ MCTS ITERATION 3 to N │ ├─────────────────────────────────────────────────────────────────────────────┤ -│ │ -│ Pattern repeats, with increasingly informed selection: │ -│ │ -│ • Nodes with high statevalue (confirmed by many simulations) │ -│ → Exploited (selected frequently) │ -│ │ +│ │ +│ Pattern repeats, with increasingly informed selection: │ +│ │ +│ • Nodes with high statevalue (confirmed by many simulations) │ +│ → Exploited (selected frequently) │ +│ │ │ • Nodes with low visits but promising progressvalue │ -│ → Explored (UCT exploration term encourages tries) │ -│ │ -│ • Tree grows: more branches explored, more statistics accumulated │ -│ │ -│ • Best trajectory emerges from accumulated statistics │ -│ │ +│ → Explored (UCT exploration term encourages tries) │ +│ │ +│ • Tree grows: more branches explored, more statistics accumulated │ +│ │ +│ • Best trajectory emerges from accumulated statistics │ +│ │ └─────────────────────────────────────────────────────────────────────────────┘ ``` diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..a0a5b8e --- /dev/null +++ b/examples/README.md @@ -0,0 +1,57 @@ +# LLMMCTS Examples + +This directory contains example scripts demonstrating how to use LLMMCTS for various problem types. + +## Examples + +1. **simple_example.jl** - Basic MCTS usage with a simple state transition function +2. **pathfinding.jl** - Grid-based pathfinding problem +3. **math_problem.jl** - Solving math problems using MCTS-guided reasoning +4. **tool_use.jl** - Coordinating with external tools (APIs, databases) +5. **chess_game.jl** - Game playing scenario (simplified chess-like) +6. **code_generation.jl** - Guiding LLM code generation +7. **reasoning.jl** - Multi-step reasoning with chain-of-thought +8. **configuration_examples.jl** - Demonstrating different MCTS configuration options + +## Running Examples + +```bash +julia examples/simple_example.jl +julia examples/pathfinding.jl +julia examples/configuration_examples.jl +``` + +## Key Concepts + +### State +The state is represented as a `Dict{String, Any}` that contains all information needed for the problem. + +### Transition Function +The transition function takes the current state and returns: +```julia +Dict( + :newNodeKey => unique_id, + :newstate => new_state_dict, + :progressvalue => llm_estimate +) +``` + +### Progress Value +`progressvalue` is provided by LLM reasoning and guides the search without waiting for terminal rewards. + +### State Value +`statevalue` is computed through Monte Carlo simulations and provides accurate long-term estimates. + +## Configuration Parameters + +- `maxiterations` - Number of MCTS iterations (default: 10) +- `explorationweight` - UCT exploration weight (default: 1.0) +- `maxSimulationDepth` - Maximum simulation rollout depth (default: 3) +- `horizontalSampleExpansionPhase` - Children per expansion (default: 3) +- `multithread` - Enable parallel simulation (default: false) +- `saveSimulatedNode` - Keep simulation nodes (default: false) + +## See Also + +- [README.md](../README.md) - Complete package documentation +- [workprocess.md](../workprocess.md) - Detailed technical documentation diff --git a/examples/chess_game.jl b/examples/chess_game.jl new file mode 100644 index 0000000..0ca80d6 --- /dev/null +++ b/examples/chess_game.jl @@ -0,0 +1,191 @@ +# Chess-like Game Example - MCTS for Game Playing + +This example demonstrates MCTS for a simplified chess-like game where the goal is to capture the opponent's pieces. + +```julia +using LLMMCTS + +# Simple game state +# board: Dict mapping positions to pieces +# turn: :white or :black +struct GameState + board::Dict{String, String} # position => piece + turn::Symbol + piece_count::Int +end + +# Initialize a simple board +function init_board() + board = Dict{String, String}() + + # Place some pieces + board["e1"] = "K" # White King + board["e8"] = "k" # Black King + + # Random pieces + board["d4"] = "P" # White Pawn + board["d5"] = "p" # Black Pawn + + return board +end + +# Check if position is on board +function on_board(pos::String) + cols = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'] + rows = ['1', '2', '3', '4', '5', '6', '7', '8'] + length(pos) == 2 && + pos[1] in cols && + pos[2] in rows +end + +# Game transition function +function chess_transition(state::Dict, args::NamedTuple) + current_step = get(state, :step, 0) + board = state[:board] + turn = state[:turn] + + if current_step >= args.max_moves + # Max moves reached, end game + newstate = Dict( + :step => current_step + 1, + :board => board, + :turn => turn, + :reward => 0.0, + :isterminal => true + ) + return Dict( + :newNodeKey => "max_moves", + :newstate => newstate, + :progressvalue => 5.0 + ) + end + + # Generate possible moves + possible_moves = String[] + + # Find all pieces of current turn's color + turn_prefix = turn == :white ? "upper" : "lower" + + # Simple move generation: try moving each piece + for (pos, piece) in board + if !isempty(piece) + # Try moving to adjacent positions + for dx in [-1, 0, 1] + for dy in [-1, 0, 1] + if dx == 0 && dy == 0 + continue + end + + # Simple coordinate conversion + col = pos[1] + row = parse(Int, pos[2]) + + new_col = col + dx + new_row = row + dy + + if new_col >= 'a' && new_col <= 'h' && + new_row >= 1 && new_row <= 8 + new_pos = string(new_col, new_row) + if on_board(new_pos) + push!(possible_moves, pos * new_pos) + end + end + end + end + end + end + + if isempty(possible_moves) + # No moves available, game over + newstate = Dict( + :step => current_step + 1, + :board => board, + :turn => turn, + :reward => turn == :white ? 10.0 : -10.0, + :isterminal => true + ) + return Dict( + :newNodeKey => "game_over", + :newstate => newstate, + :progressvalue => turn == :white ? 10.0 : 0.0 + ) + end + + # LLM would select the best move + # For this example, pick a random valid move + move_idx = (current_step - 1) % length(possible_moves) + 1 + move = possible_moves[move_idx] + + # Simulate the move (simplified) + from_pos = move[1:2] + to_pos = move[3:4] + + new_board = copy(board) + piece = get(new_board, from_pos, "") + new_board[to_pos] = piece + delete!(new_board, from_pos) + + # Calculate reward based on capture + reward = 0.0 + if !isempty(get(new_board, to_pos, "")) + reward = 5.0 # Capture! + end + + # Progress value: estimate of game state quality + progressvalue = 5.0 + reward # Capturing is good + + # Switch turns + new_turn = turn == :white ? :black : :white + + newstate = Dict( + :step => current_step + 1, + :board => new_board, + :turn => new_turn, + :reward => reward, + :isterminal => false + ) + + return Dict( + :newNodeKey => "move_$current_step", + :newstate => newstate, + :progressvalue => progressvalue + ) +end + +# Initial state +initialstate = Dict( + :step => 0, + :board => init_board(), + :turn => :white, + :reward => 0, + :isterminal => false +) + +# Transition arguments +transitionargs = ( + max_moves = 10, +) + +# Run MCTS +result = runMCTS( + initialstate, + chess_transition, + transitionargs; + maxiterations = 30, + explorationweight = 2.0, # More exploration for game playing + maxSimulationDepth = 4, + horizontalSampleExpansionPhase = 5 +) + +# Display results +println("Chess-like Game MCTS") +println("====================") +println() +println("Best move sequence:") +println(" Initial board state") +println(" → ", result.bestTerminalState[:step], " moves") +println() +println("Final board has ", length(result.bestTerminalState[:board]), " pieces") +println("Root node visits: ", result.root.visits) +println("High value states: ", length(result.highValueStateList)) +``` diff --git a/examples/code_generation.jl b/examples/code_generation.jl new file mode 100644 index 0000000..e1cc2c3 --- /dev/null +++ b/examples/code_generation.jl @@ -0,0 +1,115 @@ +# Code Generation - MCTS for Programming Tasks + +This example shows how MCTS can guide LLM code generation by exploring different implementation strategies. + +```julia +using LLMMCTS + +# State represents the current state of code generation +# It includes the code written so far and the problem being solved + +function code_generation_transition(state::Dict, args::NamedTuple) + current_step = get(state, :step, 0) + problem = state[:problem] + code_so_far = get(state, :code, "") + + if current_step == 0 + # First step: Plan the approach + new_code = """ + # Function to solve: $(problem) + function solve_problem(input) + """ + newstate = Dict( + :step => 1, + :code => new_code, + :thought => "Plan the approach for: $(problem)", + :reward => 2.0, + :isterminal => false + ) + progressvalue = 5.0 + elseif current_step == 1 + # Second step: Implement main logic + new_code = code_so_far * """ + # Main logic implementation + result = input * 2 # Placeholder implementation + return result + end + """ + newstate = Dict( + :step => 2, + :code => new_code, + :thought => "Implement main function logic", + :reward => 3.0, + :isterminal => false + ) + progressvalue = 7.0 + elseif current_step == 2 + # Third step: Add tests + new_code = code_so_far * """ + + # Test the function + @assert solve_problem(5) == 10 + @assert solve_problem(0) == 0 + println("All tests passed!") + """ + newstate = Dict( + :step => 3, + :code => new_code, + :thought => "Add unit tests to verify implementation", + :reward => 5.0, + :isterminal => true # Code generation complete + ) + progressvalue = 10.0 + else + newstate = Dict( + :step => current_step, + :code => code_so_far, + :thought => "Code generation complete", + :reward => 10.0, + :isterminal => true + ) + progressvalue = 10.0 + end + + return Dict( + :newNodeKey => "code_step_$current_step", + :newstate => newstate, + :progressvalue => progressvalue + ) +end + +# Initial state +initialstate = Dict( + :step => 0, + :problem => "Create a function that doubles its input", + :code => "", + :reward => 0, + :isterminal => false +) + +# Transition arguments +transitionargs = (max_steps = 3,) + +# Run MCTS +result = runMCTS( + initialstate, + code_generation_transition, + transitionargs; + maxiterations = 20, + explorationweight = 1.0, + maxSimulationDepth = 3, + horizontalSampleExpansionPhase = 3 +) + +# Display results +println("Code Generation Example") +println("=======================") +println() +println("Problem: ", initialstate[:problem]) +println() +println("Generated code:") +println(result.bestTerminalState[:code]) +println() +println("Code generation complete! ✓") +println("Root node visits: ", result.root.visits) +``` diff --git a/examples/configuration_examples.jl b/examples/configuration_examples.jl new file mode 100644 index 0000000..87fcee6 --- /dev/null +++ b/examples/configuration_examples.jl @@ -0,0 +1,239 @@ +# MCTS Configuration Examples + +This file demonstrates different MCTS configuration options and their effects on search behavior. + +```julia +using LLMMCTS + +# Simple transition function for demonstration +function simple_transition(state::Dict, args::NamedTuple) + current_step = get(state, :step, 0) + newstate = Dict( + :step => current_step + 1, + :reward => (current_step + 1) * 2, + :isterminal => current_step >= args.max_steps - 1 + ) + progressvalue = (current_step / args.max_steps) * 10 + return Dict( + :newNodeKey => "step_$current_step", + :newstate => newstate, + :progressvalue => progressvalue + ) +end + +initialstate = Dict( + :step => 0, + :reward => 0, + :isterminal => false +) + +transitionargs = (max_steps = 5,) + +# ============================================================================ +# Example 1: Balanced Search (Default) +# ============================================================================ +println("Example 1: Balanced Search (Default)") +println("=" ^ 50) + +result1 = runMCTS( + initialstate, + simple_transition, + transitionargs; + maxiterations = 10, + explorationweight = 1.0, # Balanced exploration/exploitation + maxSimulationDepth = 3, + horizontalSampleExpansionPhase = 3 +) + +println("Exploration weight: 1.0 (balanced)") +println("Root visits: ", result1.root.visits) +println("Best terminal step: ", result1.bestTerminalState[:step]) +println() + +# ============================================================================ +# Example 2: Aggressive Exploration +# ============================================================================ +println("Example 2: Aggressive Exploration") +println("=" * 50) + +result2 = runMCTS( + initialstate, + simple_transition, + transitionargs; + maxiterations = 10, + explorationweight = 2.0, # More exploration + maxSimulationDepth = 3, + horizontalSampleExpansionPhase = 5 # More children per node +) + +println("Exploration weight: 2.0 (aggressive exploration)") +println("Root visits: ", result2.root.visits) +println("Children explored: ", length(result2.root.children)) +println() + +# ============================================================================ +# Example 3: Deep Search (Long Horizon) +# ============================================================================ +println("Example 3: Deep Search (Long Horizon)") +println("=" * 50) + +result3 = runMCTS( + initialstate, + simple_transition, + transitionargs; + maxiterations = 20, + explorationweight = 1.0, + maxSimulationDepth = 5, # Deeper search + horizontalSampleExpansionPhase = 3 +) + +println("Max simulation depth: 5 (deep search)") +println("Root visits: ", result3.root.visits) +println("Search explores further into the future") +println() + +# ============================================================================ +# Example 4: Fast Search (Shallow, Many Iterations) +# ============================================================================ +println("Example 4: Fast Search (Shallow, Many Iterations)") +println("=" * 50) + +result4 = runMCTS( + initialstate, + simple_transition, + transitionargs; + maxiterations = 50, # Many iterations + explorationweight = 1.0, + maxSimulationDepth = 2, # Shallow search + horizontalSampleExpansionPhase = 3 +) + +println("Many iterations (50), shallow depth (2)") +println("Root visits: ", result4.root.visits) +println("Faster but less thorough search") +println() + +# ============================================================================ +# Example 5: Parallel Simulation (Multithreading) +# ============================================================================ +println("Example 5: Parallel Simulation (Multithreading)") +println("=" * 50) + +result5 = runMCTS( + initialstate, + simple_transition, + transitionargs; + maxiterations = 10, + explorationweight = 1.0, + maxSimulationDepth = 3, + horizontalSampleExpansionPhase = 3, + multithread = true # Enable parallel simulation +) + +println("Multithreading enabled") +println("Root visits: ", result5.root.visits) +println("Parallel simulation across child nodes") +println() + +# ============================================================================ +# Example 6: Early Stopping +# ============================================================================ +println("Example 6: Early Stopping") +println("=" * 50) + +# Define early stopping function +function early_stop(state::Dict) + # Stop when we reach a good enough solution + return get(state, :step, 0) >= 3 +end + +result6 = runMCTS( + initialstate, + simple_transition, + transitionargs; + maxiterations = 20, # Would run more if not for early stop + explorationweight = 1.0, + maxSimulationDepth = 3, + horizontalSampleExpansionPhase = 3, + earlystop = early_stop +) + +println("Early stopping enabled (stops at step >= 3)") +println("Actual iterations: ", result6.root.visits) +println("Early stopping saved unnecessary computation") +println() + +# ============================================================================ +# Example 7: Save Simulation Nodes (for Analysis) +# ============================================================================ +println("Example 7: Save Simulation Nodes") +println("=" * 50) + +result7 = runMCTS( + initialstate, + simple_transition, + transitionargs; + maxiterations = 5, + explorationweight = 1.0, + maxSimulationDepth = 3, + horizontalSampleExpansionPhase = 3, + saveSimulatedNode = true # Keep simulation nodes +) + +println("saveSimulatedNode = true") +println("Simulation nodes are preserved") +println("Root children: ", length(result7.root.children)) +println("Useful for debugging or further analysis") +println() + +# ============================================================================ +# Example 8: High-Value State Tracking +# ============================================================================ +println("Example 8: High-Value State Tracking") +println("=" * 50) + +# Transition that can produce high-value states +function high_value_transition(state::Dict, args::NamedTuple) + current_step = get(state, :step, 0) + reward = current_step * 3 + + # Occasionally produce high-value states + if current_step == 2 || current_step == 4 + reward = 9.0 # High value + end + + newstate = Dict( + :step => current_step + 1, + :reward => reward, + :isterminal => current_step >= args.max_steps - 1 + ) + progressvalue = (current_step / args.max_steps) * 10 + return Dict( + :newNodeKey => "step_$current_step", + :newstate => newstate, + :progressvalue => progressvalue + ) +end + +high_value_initial = Dict( + :step => 0, + :reward => 0, + :isterminal => false +) + +result8 = runMCTS( + high_value_initial, + high_value_transition, + transitionargs; + maxiterations = 15, + explorationweight = 1.0, + maxSimulationDepth = 3, + horizontalSampleExpansionPhase = 3 +) + +println("High-value states found: ", length(result8.highValueStateList)) +println("States with reward >= 8 were tracked") +for (i, state) in enumerate(result8.highValueStateList) + println(" High-value state $i: step = ", state[:step]) +end +``` diff --git a/examples/math_problem.jl b/examples/math_problem.jl new file mode 100644 index 0000000..506aa11 --- /dev/null +++ b/examples/math_problem.jl @@ -0,0 +1,97 @@ +# Math Problem Solving - MCTS Example + +This example demonstrates using MCTS to solve a math problem by exploring different solution strategies. + +```julia +using LLMMCTS + +# State represents the current state of problem solving +# It contains the problem statement and the steps taken so far + +function math_problem_transition(state::Dict, args::NamedTuple) + current_step = get(state, :step, 0) + problem = state[:problem] + + # Example problem: Solve x^2 = 16 + if current_step == 0 + # First step: analyze the problem + newstate = Dict( + :step => 1, + :thought => "This is a quadratic equation x^2 = 16", + :action => "Take square root of both sides", + :reward => 2.0, + :isterminal => false + ) + progressvalue = 5.0 + elseif current_step == 1 + # Second step: solve + newstate = Dict( + :step => 2, + :thought => "Taking square root gives x = ±4", + :action => "x = sqrt(16) or x = -sqrt(16)", + :reward => 3.0, + :isterminal => false + ) + progressvalue = 7.0 + elseif current_step == 2 + # Third step: verify + newstate = Dict( + :step => 3, + :thought => "Verify both solutions work", + :action => "Check x=4: 4^2=16 ✓, Check x=-4: (-4)^2=16 ✓", + :reward => 5.0, + :isterminal => true # Problem solved! + ) + progressvalue = 10.0 + else + # Terminal state + newstate = Dict( + :step => current_step, + :thought => "Problem solved", + :action => "Solution complete", + :reward => 10.0, + :isterminal => true + ) + progressvalue = 10.0 + end + + return Dict( + :newNodeKey => "step_$current_step", + :newstate => newstate, + :progressvalue => progressvalue + ) +end + +# Initial state +initialstate = Dict( + :step => 0, + :problem => "Solve x^2 = 16", + :reward => 0, + :isterminal => false +) + +# Transition arguments +transitionargs = () + +# Run MCTS +result = runMCTS( + initialstate, + math_problem_transition, + transitionargs; + maxiterations = 15, + explorationweight = 1.0, + maxSimulationDepth = 3, + horizontalSampleExpansionPhase = 3 +) + +# Display results +println("Problem: ", initialstate[:problem]) +println() +println("Best solution trajectory:") +println(" Step ", result.bestTerminalState[:step]) +println(" Thought: ", result.bestTerminalState[:thought]) +println(" Action: ", result.bestTerminalState[:action]) +println() +println("Solution complete! ✓") +println("Root node visits: ", result.root.visits) +``` diff --git a/examples/pathfinding.jl b/examples/pathfinding.jl new file mode 100644 index 0000000..7e5d2e5 --- /dev/null +++ b/examples/pathfinding.jl @@ -0,0 +1,98 @@ +# Pathfinding Problem - MCTS Example + +This example shows how to use MCTS for a pathfinding problem where the goal is to reach a target location. + +```julia +using LLMMCTS + +# Grid-based pathfinding state +struct Position + x::Int + y::Int +end + +# State transition function for pathfinding +function pathfinding_transition(state::Dict, args::NamedTuple) + current_pos = Position(state[:pos_x], state[:pos_y]) + target_pos = Position(args.target_x, args.target_y) + + # Generate possible moves (up, down, left, right) + moves = [ + (0, 1), # up + (0, -1), # down + (1, 0), # right + (-1, 0) # left + ] + + # In a real scenario, LLM would select which move to try + # For this example, we'll try all moves + move_idx = state[:move_idx] % length(moves) + 1 + dx, dy = moves[move_idx] + + new_x = current_pos.x + dx + new_y = current_pos.y + dy + + # Calculate distance to target + distance = abs(new_x - target_pos.x) + abs(new_y - target_pos.y) + + # Reward: negative of distance (closer is better) + reward = -distance + + # Progress value: LLM estimate (here we use inverse distance as heuristic) + progressvalue = 10 - distance + + newstate = Dict( + :pos_x => new_x, + :pos_y => new_y, + :move_idx => state[:move_idx] + 1, + :reward => reward, + :isterminal => (new_x == target_pos.x && new_y == target_pos.y) || + (state[:move_idx] >= args.max_moves) + ) + + return Dict( + :newNodeKey => "pos_$(new_x)_$(new_y)", + :newstate => newstate, + :progressvalue => progressvalue + ) +end + +# Initial state +initialstate = Dict( + :pos_x => 0, + :pos_y => 0, + :move_idx => 0, + :reward => 0, + :isterminal => false +) + +# Target position +target_x, target_y = 3, 2 + +# Transition arguments +transitionargs = ( + target_x = target_x, + target_y = target_y, + max_moves = 10 +) + +# Run MCTS +result = runMCTS( + initialstate, + pathfinding_transition, + transitionargs; + maxiterations = 20, + explorationweight = 1.5, + maxSimulationDepth = 5, + horizontalSampleExpansionPhase = 4 +) + +# Display results +println("Target: ($target_x, $target_y)") +println("Best final position: (", + result.bestTerminalState[:pos_x], ", ", + result.bestTerminalState[:pos_y], ")") +println("Final distance: ", abs(result.bestTerminalState[:pos_x] - target_x) + + abs(result.bestTerminalState[:pos_y] - target_y)) +println("Root node visits: ", result.root.visits) +``` diff --git a/examples/reasoning.jl b/examples/reasoning.jl new file mode 100644 index 0000000..14e9edc --- /dev/null +++ b/examples/reasoning.jl @@ -0,0 +1,134 @@ +# Multi-step Reasoning - MCTS with Chain of Thought + +This example demonstrates MCTS for multi-step reasoning problems, where the LLM generates chain-of-thought reasoning at each step. + +```julia +using LLMMCTS + +# State tracks the reasoning process +# thought_history: Dict mapping thought/action keys to their content + +function reasoning_transition(state::Dict, args::NamedTuple) + current_step = get(state, :step, 0) + thought_history = get(state, :thought_history, Dict{String, String}()) + problem = state[:problem] + + if current_step == 0 + # Step 1: Understand the problem + thought = "First, I need to understand what the problem is asking. The problem requires me to analyze the given information and determine the solution approach." + action = "Identify the key components of the problem" + + new_thought_history = copy(thought_history) + new_thought_history["thought_1"] = thought + new_thought_history["action_1"] = action + + newstate = Dict( + :step => 1, + :thought_history => new_thought_history, + :reward => 1.0, + :isterminal => false + ) + progressvalue = 3.0 + elseif current_step == 1 + # Step 2: Break down the problem + thought = "Next, I should break this down into smaller sub-problems. This will make it easier to solve step by step." + action = "Divide the problem into manageable parts" + + new_thought_history = copy(thought_history) + new_thought_history["thought_2"] = thought + new_thought_history["action_2"] = action + + newstate = Dict( + :step => 2, + :thought_history => new_thought_history, + :reward => 2.0, + :isterminal => false + ) + progressvalue = 5.0 + elseif current_step == 2 + # Step 3: Solve each sub-problem + thought = "Now I'll solve each sub-problem individually, using appropriate methods for each." + action = "Apply solution methods to each sub-problem" + + new_thought_history = copy(thought_history) + new_thought_history["thought_3"] = thought + new_thought_history["action_3"] = action + + newstate = Dict( + :step => 3, + :thought_history => new_thought_history, + :reward => 3.0, + :isterminal => false + ) + progressvalue = 7.0 + elseif current_step == 3 + # Step 4: Combine solutions + thought = "Finally, I'll combine all the solutions to form the complete answer to the original problem." + action = "Integrate solutions and verify the answer" + + new_thought_history = copy(thought_history) + new_thought_history["thought_4"] = thought + new_thought_history["action_4"] = action + + newstate = Dict( + :step => 4, + :thought_history => new_thought_history, + :reward => 4.0, + :isterminal => true # Reasoning complete + ) + progressvalue = 10.0 + else + newstate = Dict( + :step => current_step, + :thought_history => thought_history, + :reward => 10.0, + :isterminal => true + ) + progressvalue = 10.0 + end + + return Dict( + :newNodeKey => "reasoning_step_$current_step", + :newstate => newstate, + :progressvalue => progressvalue + ) +end + +# Initial state +initialstate = Dict( + :step => 0, + :problem => "Explain how photosynthesis works", + :thought_history => Dict{String, String}(), + :reward => 0, + :isterminal => false +) + +# Transition arguments +transitionargs = (max_steps = 4,) + +# Run MCTS +result = runMCTS( + initialstate, + reasoning_transition, + transitionargs; + maxiterations = 25, + explorationweight = 1.0, + maxSimulationDepth = 4, + horizontalSampleExpansionPhase = 3 +) + +# Display results +println("Multi-step Reasoning Example") +println("=============================") +println() +println("Problem: ", initialstate[:problem]) +println() +println("Reasoning steps:") +for (key, value) in result.bestTerminalState[:thought_history] + println(" $key: $value") +end +println() +println("Reasoning complete! ✓") +println("Root node visits: ", result.root.visits) +println("Total steps in reasoning chain: ", result.bestTerminalState[:step]) +``` diff --git a/examples/simple_example.jl b/examples/simple_example.jl new file mode 100644 index 0000000..eeea7c4 --- /dev/null +++ b/examples/simple_example.jl @@ -0,0 +1,59 @@ +# Simple MCTS Example + +This example demonstrates basic MCTS usage with a simple state transition function. + +```julia +using LLMMCTS + +# Define a simple state transition function +function simple_transition(state::Dict, args::NamedTuple) + # In a real scenario, this would call an LLM + # For this example, we'll just generate deterministic next states + + current_step = get(state, :step, 0) + new_step = current_step + 1 + + # Create new state + newstate = Dict( + :step => new_step, + :reward => new_step * 2, # Simple reward function + :isterminal => new_step >= args.max_steps + ) + + # LLM would provide progressvalue estimate + progressvalue = (new_step / args.max_steps) * 10 + + return Dict( + :newNodeKey => "step_$(new_step)", + :newstate => newstate, + :progressvalue => progressvalue + ) +end + +# Initial state +initialstate = Dict( + :step => 0, + :reward => 0, + :isterminal => false +) + +# Transition arguments +transitionargs = (max_steps = 5,) + +# Run MCTS +result = runMCTS( + initialstate, + simple_transition, + transitionargs; + maxiterations = 10, + explorationweight = 1.0, + maxSimulationDepth = 3, + horizontalSampleExpansionPhase = 3 +) + +# Access results +println("Root node visits: ", result.root.visits) +println("Best next state: ", result.bestNextState) +println("Best terminal state: ", result.bestTerminalState) +println("High value states: ", result.highValueStateList) +``` diff --git a/examples/tool_use.jl b/examples/tool_use.jl new file mode 100644 index 0000000..da4e62e --- /dev/null +++ b/examples/tool_use.jl @@ -0,0 +1,109 @@ +# Tool Use Example - MCTS with External Tools + +This example shows how MCTS can coordinate with external tools (like APIs, databases, or other services). + +```julia +using LLMMCTS + +# Simulated tool interface +struct Tool + name::String + description::String +end + +const AVAILABLE_TOOLS = [ + Tool("calculator", "Perform mathematical calculations"), + Tool("web_search", "Search the web for information"), + Tool("database_query", "Query a database") +] + +# State tracks which tools have been used and their results +function tool_use_transition(state::Dict, args::NamedTuple) + current_step = get(state, :step, 0) + tools_used = get(state, :tools_used, String[]) + + # LLM would decide which tool to use + # For this example, we try tools in order + tool_idx = (current_step - 1) % length(AVAILABLE_TOOLS) + 1 + + if tool_idx > length(AVAILABLE_TOOLS) + # All tools tried, return terminal state + newstate = Dict( + :step => current_step + 1, + :tools_used => tools_used, + :reward => 8.0, + :isterminal => true + ) + return Dict( + :newNodeKey => "all_tools_tried", + :newstate => newstate, + :progressvalue => 8.0 + ) + end + + tool = AVAILABLE_TOOLS[tool_idx] + + # Simulate tool execution + tool_result = "Tool '$(tool.name)' executed successfully" + + # Calculate reward based on progress + progress = length(tools_used) / length(AVAILABLE_TOOLS) + reward = progress * 5 + + # Progress value: LLM estimates how close we are to solving + progressvalue = progress * 10 + + new_tools_used = vcat(tools_used, tool.name) + + newstate = Dict( + :step => current_step + 1, + :tools_used => new_tools_used, + :current_tool => tool.name, + :tool_result => tool_result, + :reward => reward, + :isterminal => false + ) + + return Dict( + :newNodeKey => "tool_$(tool.name)_$current_step", + :newstate => newstate, + :progressvalue => progressvalue + ) +end + +# Initial state +initialstate = Dict( + :step => 0, + :tools_used => String[], + :reward => 0, + :isterminal => false +) + +# Transition arguments +transitionargs = (max_tools = 3,) + +# Run MCTS +result = runMCTS( + initialstate, + tool_use_transition, + transitionargs; + maxiterations = 20, + explorationweight = 1.2, + maxSimulationDepth = 4, + horizontalSampleExpansionPhase = 3 +) + +# Display results +println("Available tools:") +for tool in AVAILABLE_TOOLS + println(" - $(tool.name): $(tool.description)") +end +println() +println("Best tool usage sequence:") +for tool in result.bestTerminalState[:tools_used] + println(" → Used: $tool") +end +println() +println("Root node visits: ", result.root.visits) +println("High value states found: ", length(result.highValueStateList)) +```