ton/LLMMCTS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

update

Browse Source
This commit is contained in:
narawat
2026-06-30 13:48:09 +07:00
parent 6cd58ffd14
commit c5ad5c882c
10 changed files with 1190 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workprocess.md → docs/workprocess.md
+91 -91
View File
@@ -65,8 +65,8 @@ Expansion Phase (LLM call):
│ transition(state, args) → { │
│ newNodeKey: "abc-123", │
│ newstate: { reward: 0, isterminal: false }, │
│ progressvalue: 7.5 ← LLM estimates this state is promising │
│ } │
│ progressvalue: 7.5 ← LLM estimates this state is promising │
│ } │
└────────────────────────────────────────────────────────────────────────┘
↓
New Node Created:
@@ -94,7 +94,7 @@ Backpropagation:
┌────────────────────────────────────────────────────────────────────────┐
│ Update all ancestors with simTrajectoryReward = 13: │
│ node.visits += 1 │
│ node.statevalue = (old_statevalue * (visits-1) + 13) / visits │
│ node.statevalue = (old_statevalue * (visits-1) + 13) / visits │
│ reward *= 0.9 (discount for future rewards) │
└────────────────────────────────────────────────────────────────────────┘
```
@@ -615,75 +615,75 @@ Current Node: "Problem solving step 5"
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ MCTS ITERATION 1 │
│ MCTS ITERATION 1 │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ 1. SELECTION (UCT-based traversal): │
│ Root (visits=1) │
│ └── UCTselect() on root (has no children yet, so uses progressvalue) │
│ │
│ 1. SELECTION (UCT-based traversal): │
│ Root (visits=1) │
│ └── UCTselect() on root (has no children yet, so uses progressvalue) │
│ └── Select child with highest progressvalue │
│ └── Continue until reaching leaf node │
│ │
│ 2. EXPANSION (horizontal sampling): │
│ Leaf node │
│ └── Continue until reaching leaf node │
│ │
│ 2. EXPANSION (horizontal sampling): │
│ Leaf node │
│ └── expand(horizontalSample=3) │
│ └── Generate 3 child nodes via LLM transition │
│ └── Generate 3 child nodes via LLM transition │
│ └── Each child gets: progressvalue, reward, parent=leaf │
│ │
│ 3. SIMULATION (vertical rollout): │
│ Each child node │
│ │
│ 3. SIMULATION (vertical rollout): │
│ Each child node │
│ └── simulate(maxSimulationDepth=3) │
│ └── Rollout 3 levels deep, accumulating rewards │
│ └── Return (simTrajectoryReward, terminalstate) │
│ │
│ 4. BACKPROPAGATION (update statistics): │
│ simTrajectoryReward │
│ └── backpropagate() up to root │
│ │
│ 4. BACKPROPAGATION (update statistics): │
│ simTrajectoryReward │
│ └── backpropagate() up to root │
│ └── Update visits and statevalue for all ancestors │
│ └── Apply discount to future rewards │
│ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────────────────┐
│ MCTS ITERATION 2 │
│ MCTS ITERATION 2 │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ 1. SELECTION: │
│ Root (visits=2) │
│ └── UCTselect() now considers: │
│ │
│ 1. SELECTION: │
│ Root (visits=2) │
│ └── UCTselect() now considers: │
│ ├── statevalue (from iteration 1) │
│ └── exploration term (low visits on unexpanded branches) │
│ └── May select different path than iteration 1 │
│ │
│ 2. EXPANSION: │
│ │
│ 2. EXPANSION: │
│ └── May expand different node or same node │
│ └── New children added to node.children │
│ │
│ 3. SIMULATION: │
│ └── Different rollout trajectory │
│ │
│ 3. SIMULATION: │
│ └── Different rollout trajectory │
│ └── New reward estimate added to statistics │
│ │
│ 4. BACKPROPAGATION: │
│ │
│ 4. BACKPROPAGATION: │
│ └── Statistics updated with new information │
│ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────────────────┐
│ MCTS ITERATION 3 to N │
│ MCTS ITERATION 3 to N │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ Pattern repeats, with increasingly informed selection: │
│ │
│ • Nodes with high statevalue (confirmed by many simulations) │
│ → Exploited (selected frequently) │
│ │
│ │
│ Pattern repeats, with increasingly informed selection: │
│ │
│ • Nodes with high statevalue (confirmed by many simulations) │
│ → Exploited (selected frequently) │
│ │
│ • Nodes with low visits but promising progressvalue │
│ → Explored (UCT exploration term encourages tries) │
│ │
│ • Tree grows: more branches explored, more statistics accumulated │
│ │
│ • Best trajectory emerges from accumulated statistics │
│ │
│ → Explored (UCT exploration term encourages tries) │
│ │
│ • Tree grows: more branches explored, more statistics accumulated │
│ │
│ • Best trajectory emerges from accumulated statistics │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
@@ -957,75 +957,75 @@ Step-by-step execution:
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ MCTS ITERATION 1 │
│ MCTS ITERATION 1 │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ 1. SELECTION (UCT-based traversal): │
│ Root (visits=1) │
│ └── UCTselect() on root (has no children yet, so uses progressvalue) │
│ │
│ 1. SELECTION (UCT-based traversal): │
│ Root (visits=1) │
│ └── UCTselect() on root (has no children yet, so uses progressvalue) │
│ └── Select child with highest progressvalue │
│ └── Continue until reaching leaf node │
│ │
│ 2. EXPANSION (horizontal sampling): │
│ Leaf node │
│ └── Continue until reaching leaf node │
│ │
│ 2. EXPANSION (horizontal sampling): │
│ Leaf node │
│ └── expand(horizontalSample=3) │
│ └── Generate 3 child nodes via LLM transition │
│ └── Generate 3 child nodes via LLM transition │
│ └── Each child gets: progressvalue, reward, parent=leaf │
│ │
│ 3. SIMULATION (vertical rollout): │
│ Each child node │
│ │
│ 3. SIMULATION (vertical rollout): │
│ Each child node │
│ └── simulate(maxSimulationDepth=3) │
│ └── Rollout 3 levels deep, accumulating rewards │
│ └── Return (simTrajectoryReward, terminalstate) │
│ │
│ 4. BACKPROPAGATION (update statistics): │
│ simTrajectoryReward │
│ └── backpropagate() up to root │
│ │
│ 4. BACKPROPAGATION (update statistics): │
│ simTrajectoryReward │
│ └── backpropagate() up to root │
│ └── Update visits and statevalue for all ancestors │
│ └── Apply discount to future rewards │
│ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────────────────┐
│ MCTS ITERATION 2 │
│ MCTS ITERATION 2 │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ 1. SELECTION: │
│ Root (visits=2) │
│ └── UCTselect() now considers: │
│ │
│ 1. SELECTION: │
│ Root (visits=2) │
│ └── UCTselect() now considers: │
│ ├── statevalue (from iteration 1) │
│ └── exploration term (low visits on unexpanded branches) │
│ └── May select different path than iteration 1 │
│ │
│ 2. EXPANSION: │
│ │
│ 2. EXPANSION: │
│ └── May expand different node or same node │
│ └── New children added to node.children │
│ │
│ 3. SIMULATION: │
│ └── Different rollout trajectory │
│ │
│ 3. SIMULATION: │
│ └── Different rollout trajectory │
│ └── New reward estimate added to statistics │
│ │
│ 4. BACKPROPAGATION: │
│ │
│ 4. BACKPROPAGATION: │
│ └── Statistics updated with new information │
│ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────────────────┐
│ MCTS ITERATION 3 to N │
│ MCTS ITERATION 3 to N │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ Pattern repeats, with increasingly informed selection: │
│ │
│ • Nodes with high statevalue (confirmed by many simulations) │
│ → Exploited (selected frequently) │
│ │
│ │
│ Pattern repeats, with increasingly informed selection: │
│ │
│ • Nodes with high statevalue (confirmed by many simulations) │
│ → Exploited (selected frequently) │
│ │
│ • Nodes with low visits but promising progressvalue │
│ → Explored (UCT exploration term encourages tries) │
│ │
│ • Tree grows: more branches explored, more statistics accumulated │
│ │
│ • Best trajectory emerges from accumulated statistics │
│ │
│ → Explored (UCT exploration term encourages tries) │
│ │
│ • Tree grows: more branches explored, more statistics accumulated │
│ │
│ • Best trajectory emerges from accumulated statistics │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
examples/README.md
+57
View File
@@ -0,0 +1,57 @@
# LLMMCTS Examples
This directory contains example scripts demonstrating how to use LLMMCTS for various problem types.
## Examples
1. **simple_example.jl** - Basic MCTS usage with a simple state transition function
2. **pathfinding.jl** - Grid-based pathfinding problem
3. **math_problem.jl** - Solving math problems using MCTS-guided reasoning
4. **tool_use.jl** - Coordinating with external tools (APIs, databases)
5. **chess_game.jl** - Game playing scenario (simplified chess-like)
6. **code_generation.jl** - Guiding LLM code generation
7. **reasoning.jl** - Multi-step reasoning with chain-of-thought
8. **configuration_examples.jl** - Demonstrating different MCTS configuration options
## Running Examples
```bash
julia examples/simple_example.jl
julia examples/pathfinding.jl
julia examples/configuration_examples.jl
```
## Key Concepts
### State
The state is represented as a `Dict{String, Any}` that contains all information needed for the problem.
### Transition Function
The transition function takes the current state and returns:
```julia
Dict(
:newNodeKey => unique_id,
:newstate => new_state_dict,
:progressvalue => llm_estimate
)
```
### Progress Value
`progressvalue` is provided by LLM reasoning and guides the search without waiting for terminal rewards.
### State Value
`statevalue` is computed through Monte Carlo simulations and provides accurate long-term estimates.
## Configuration Parameters
- `maxiterations` - Number of MCTS iterations (default: 10)
- `explorationweight` - UCT exploration weight (default: 1.0)
- `maxSimulationDepth` - Maximum simulation rollout depth (default: 3)
- `horizontalSampleExpansionPhase` - Children per expansion (default: 3)
- `multithread` - Enable parallel simulation (default: false)
- `saveSimulatedNode` - Keep simulation nodes (default: false)
## See Also
- [README.md](../README.md) - Complete package documentation
- [workprocess.md](../workprocess.md) - Detailed technical documentation
examples/chess_game.jl
+191
View File
@@ -0,0 +1,191 @@
# Chess-like Game Example - MCTS for Game Playing
This example demonstrates MCTS for a simplified chess-like game where the goal is to capture the opponent's pieces.
```julia
using LLMMCTS
# Simple game state
# board: Dict mapping positions to pieces
# turn: :white or :black
struct GameState
board::Dict{String, String} # position => piece
turn::Symbol
piece_count::Int
end
# Initialize a simple board
function init_board()
board = Dict{String, String}()
# Place some pieces
board["e1"] = "K" # White King
board["e8"] = "k" # Black King
# Random pieces
board["d4"] = "P" # White Pawn
board["d5"] = "p" # Black Pawn
return board
end
# Check if position is on board
function on_board(pos::String)
cols = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']
rows = ['1', '2', '3', '4', '5', '6', '7', '8']
length(pos) == 2 &&
pos[1] in cols &&
pos[2] in rows
end
# Game transition function
function chess_transition(state::Dict, args::NamedTuple)
current_step = get(state, :step, 0)
board = state[:board]
turn = state[:turn]
if current_step >= args.max_moves
# Max moves reached, end game
newstate = Dict(
:step => current_step + 1,
:board => board,
:turn => turn,
:reward => 0.0,
:isterminal => true
)
return Dict(
:newNodeKey => "max_moves",
:newstate => newstate,
:progressvalue => 5.0
)
end
# Generate possible moves
possible_moves = String[]
# Find all pieces of current turn's color
turn_prefix = turn == :white ? "upper" : "lower"
# Simple move generation: try moving each piece
for (pos, piece) in board
if !isempty(piece)
# Try moving to adjacent positions
for dx in [-1, 0, 1]
for dy in [-1, 0, 1]
if dx == 0 && dy == 0
continue
end
# Simple coordinate conversion
col = pos[1]
row = parse(Int, pos[2])
new_col = col + dx
new_row = row + dy
if new_col >= 'a' && new_col <= 'h' &&
new_row >= 1 && new_row <= 8
new_pos = string(new_col, new_row)
if on_board(new_pos)
push!(possible_moves, pos * new_pos)
end
end
end
end
end
end
if isempty(possible_moves)
# No moves available, game over
newstate = Dict(
:step => current_step + 1,
:board => board,
:turn => turn,
:reward => turn == :white ? 10.0 : -10.0,
:isterminal => true
)
return Dict(
:newNodeKey => "game_over",
:newstate => newstate,
:progressvalue => turn == :white ? 10.0 : 0.0
)
end
# LLM would select the best move
# For this example, pick a random valid move
move_idx = (current_step - 1) % length(possible_moves) + 1
move = possible_moves[move_idx]
# Simulate the move (simplified)
from_pos = move[1:2]
to_pos = move[3:4]
new_board = copy(board)
piece = get(new_board, from_pos, "")
new_board[to_pos] = piece
delete!(new_board, from_pos)
# Calculate reward based on capture
reward = 0.0
if !isempty(get(new_board, to_pos, ""))
reward = 5.0 # Capture!
end
# Progress value: estimate of game state quality
progressvalue = 5.0 + reward # Capturing is good
# Switch turns
new_turn = turn == :white ? :black : :white
newstate = Dict(
:step => current_step + 1,
:board => new_board,
:turn => new_turn,
:reward => reward,
:isterminal => false
)
return Dict(
:newNodeKey => "move_$current_step",
:newstate => newstate,
:progressvalue => progressvalue
)
end
# Initial state
initialstate = Dict(
:step => 0,
:board => init_board(),
:turn => :white,
:reward => 0,
:isterminal => false
)
# Transition arguments
transitionargs = (
max_moves = 10,
)
# Run MCTS
result = runMCTS(
initialstate,
chess_transition,
transitionargs;
maxiterations = 30,
explorationweight = 2.0, # More exploration for game playing
maxSimulationDepth = 4,
horizontalSampleExpansionPhase = 5
)
# Display results
println("Chess-like Game MCTS")
println("====================")
println()
println("Best move sequence:")
println(" Initial board state")
println(" → ", result.bestTerminalState[:step], " moves")
println()
println("Final board has ", length(result.bestTerminalState[:board]), " pieces")
println("Root node visits: ", result.root.visits)
println("High value states: ", length(result.highValueStateList))
```
examples/code_generation.jl
+115
View File
@@ -0,0 +1,115 @@
# Code Generation - MCTS for Programming Tasks
This example shows how MCTS can guide LLM code generation by exploring different implementation strategies.
```julia
using LLMMCTS
# State represents the current state of code generation
# It includes the code written so far and the problem being solved
function code_generation_transition(state::Dict, args::NamedTuple)
current_step = get(state, :step, 0)
problem = state[:problem]
code_so_far = get(state, :code, "")
if current_step == 0
# First step: Plan the approach
new_code = """
# Function to solve: $(problem)
function solve_problem(input)
"""
newstate = Dict(
:step => 1,
:code => new_code,
:thought => "Plan the approach for: $(problem)",
:reward => 2.0,
:isterminal => false
)
progressvalue = 5.0
elseif current_step == 1
# Second step: Implement main logic
new_code = code_so_far * """
# Main logic implementation
result = input * 2 # Placeholder implementation
return result
end
"""
newstate = Dict(
:step => 2,
:code => new_code,
:thought => "Implement main function logic",
:reward => 3.0,
:isterminal => false
)
progressvalue = 7.0
elseif current_step == 2
# Third step: Add tests
new_code = code_so_far * """
# Test the function
@assert solve_problem(5) == 10
@assert solve_problem(0) == 0
println("All tests passed!")
"""
newstate = Dict(
:step => 3,
:code => new_code,
:thought => "Add unit tests to verify implementation",
:reward => 5.0,
:isterminal => true # Code generation complete
)
progressvalue = 10.0
else
newstate = Dict(
:step => current_step,
:code => code_so_far,
:thought => "Code generation complete",
:reward => 10.0,
:isterminal => true
)
progressvalue = 10.0
end
return Dict(
:newNodeKey => "code_step_$current_step",
:newstate => newstate,
:progressvalue => progressvalue
)
end
# Initial state
initialstate = Dict(
:step => 0,
:problem => "Create a function that doubles its input",
:code => "",
:reward => 0,
:isterminal => false
)
# Transition arguments
transitionargs = (max_steps = 3,)
# Run MCTS
result = runMCTS(
initialstate,
code_generation_transition,
transitionargs;
maxiterations = 20,
explorationweight = 1.0,
maxSimulationDepth = 3,
horizontalSampleExpansionPhase = 3
)
# Display results
println("Code Generation Example")
println("=======================")
println()
println("Problem: ", initialstate[:problem])
println()
println("Generated code:")
println(result.bestTerminalState[:code])
println()
println("Code generation complete! ✓")
println("Root node visits: ", result.root.visits)
```
examples/configuration_examples.jl
+239
View File
@@ -0,0 +1,239 @@
# MCTS Configuration Examples
This file demonstrates different MCTS configuration options and their effects on search behavior.
```julia
using LLMMCTS
# Simple transition function for demonstration
function simple_transition(state::Dict, args::NamedTuple)
current_step = get(state, :step, 0)
newstate = Dict(
:step => current_step + 1,
:reward => (current_step + 1) * 2,
:isterminal => current_step >= args.max_steps - 1
)
progressvalue = (current_step / args.max_steps) * 10
return Dict(
:newNodeKey => "step_$current_step",
:newstate => newstate,
:progressvalue => progressvalue
)
end
initialstate = Dict(
:step => 0,
:reward => 0,
:isterminal => false
)
transitionargs = (max_steps = 5,)
# ============================================================================
# Example 1: Balanced Search (Default)
# ============================================================================
println("Example 1: Balanced Search (Default)")
println("=" ^ 50)
result1 = runMCTS(
initialstate,
simple_transition,
transitionargs;
maxiterations = 10,
explorationweight = 1.0, # Balanced exploration/exploitation
maxSimulationDepth = 3,
horizontalSampleExpansionPhase = 3
)
println("Exploration weight: 1.0 (balanced)")
println("Root visits: ", result1.root.visits)
println("Best terminal step: ", result1.bestTerminalState[:step])
println()
# ============================================================================
# Example 2: Aggressive Exploration
# ============================================================================
println("Example 2: Aggressive Exploration")
println("=" * 50)
result2 = runMCTS(
initialstate,
simple_transition,
transitionargs;
maxiterations = 10,
explorationweight = 2.0, # More exploration
maxSimulationDepth = 3,
horizontalSampleExpansionPhase = 5 # More children per node
)
println("Exploration weight: 2.0 (aggressive exploration)")
println("Root visits: ", result2.root.visits)
println("Children explored: ", length(result2.root.children))
println()
# ============================================================================
# Example 3: Deep Search (Long Horizon)
# ============================================================================
println("Example 3: Deep Search (Long Horizon)")
println("=" * 50)
result3 = runMCTS(
initialstate,
simple_transition,
transitionargs;
maxiterations = 20,
explorationweight = 1.0,
maxSimulationDepth = 5, # Deeper search
horizontalSampleExpansionPhase = 3
)
println("Max simulation depth: 5 (deep search)")
println("Root visits: ", result3.root.visits)
println("Search explores further into the future")
println()
# ============================================================================
# Example 4: Fast Search (Shallow, Many Iterations)
# ============================================================================
println("Example 4: Fast Search (Shallow, Many Iterations)")
println("=" * 50)
result4 = runMCTS(
initialstate,
simple_transition,
transitionargs;
maxiterations = 50, # Many iterations
explorationweight = 1.0,
maxSimulationDepth = 2, # Shallow search
horizontalSampleExpansionPhase = 3
)
println("Many iterations (50), shallow depth (2)")
println("Root visits: ", result4.root.visits)
println("Faster but less thorough search")
println()
# ============================================================================
# Example 5: Parallel Simulation (Multithreading)
# ============================================================================
println("Example 5: Parallel Simulation (Multithreading)")
println("=" * 50)
result5 = runMCTS(
initialstate,
simple_transition,
transitionargs;
maxiterations = 10,
explorationweight = 1.0,
maxSimulationDepth = 3,
horizontalSampleExpansionPhase = 3,
multithread = true # Enable parallel simulation
)
println("Multithreading enabled")
println("Root visits: ", result5.root.visits)
println("Parallel simulation across child nodes")
println()
# ============================================================================
# Example 6: Early Stopping
# ============================================================================
println("Example 6: Early Stopping")
println("=" * 50)
# Define early stopping function
function early_stop(state::Dict)
# Stop when we reach a good enough solution
return get(state, :step, 0) >= 3
end
result6 = runMCTS(
initialstate,
simple_transition,
transitionargs;
maxiterations = 20, # Would run more if not for early stop
explorationweight = 1.0,
maxSimulationDepth = 3,
horizontalSampleExpansionPhase = 3,
earlystop = early_stop
)
println("Early stopping enabled (stops at step >= 3)")
println("Actual iterations: ", result6.root.visits)
println("Early stopping saved unnecessary computation")
println()
# ============================================================================
# Example 7: Save Simulation Nodes (for Analysis)
# ============================================================================
println("Example 7: Save Simulation Nodes")
println("=" * 50)
result7 = runMCTS(
initialstate,
simple_transition,
transitionargs;
maxiterations = 5,
explorationweight = 1.0,
maxSimulationDepth = 3,
horizontalSampleExpansionPhase = 3,
saveSimulatedNode = true # Keep simulation nodes
)
println("saveSimulatedNode = true")
println("Simulation nodes are preserved")
println("Root children: ", length(result7.root.children))
println("Useful for debugging or further analysis")
println()
# ============================================================================
# Example 8: High-Value State Tracking
# ============================================================================
println("Example 8: High-Value State Tracking")
println("=" * 50)
# Transition that can produce high-value states
function high_value_transition(state::Dict, args::NamedTuple)
current_step = get(state, :step, 0)
reward = current_step * 3
# Occasionally produce high-value states
if current_step == 2 || current_step == 4
reward = 9.0 # High value
end
newstate = Dict(
:step => current_step + 1,
:reward => reward,
:isterminal => current_step >= args.max_steps - 1
)
progressvalue = (current_step / args.max_steps) * 10
return Dict(
:newNodeKey => "step_$current_step",
:newstate => newstate,
:progressvalue => progressvalue
)
end
high_value_initial = Dict(
:step => 0,
:reward => 0,
:isterminal => false
)
result8 = runMCTS(
high_value_initial,
high_value_transition,
transitionargs;
maxiterations = 15,
explorationweight = 1.0,
maxSimulationDepth = 3,
horizontalSampleExpansionPhase = 3
)
println("High-value states found: ", length(result8.highValueStateList))
println("States with reward >= 8 were tracked")
for (i, state) in enumerate(result8.highValueStateList)
println(" High-value state $i: step = ", state[:step])
end
```
examples/math_problem.jl
+97
View File
@@ -0,0 +1,97 @@
# Math Problem Solving - MCTS Example
This example demonstrates using MCTS to solve a math problem by exploring different solution strategies.
```julia
using LLMMCTS
# State represents the current state of problem solving
# It contains the problem statement and the steps taken so far
function math_problem_transition(state::Dict, args::NamedTuple)
current_step = get(state, :step, 0)
problem = state[:problem]
# Example problem: Solve x^2 = 16
if current_step == 0
# First step: analyze the problem
newstate = Dict(
:step => 1,
:thought => "This is a quadratic equation x^2 = 16",
:action => "Take square root of both sides",
:reward => 2.0,
:isterminal => false
)
progressvalue = 5.0
elseif current_step == 1
# Second step: solve
newstate = Dict(
:step => 2,
:thought => "Taking square root gives x = ±4",
:action => "x = sqrt(16) or x = -sqrt(16)",
:reward => 3.0,
:isterminal => false
)
progressvalue = 7.0
elseif current_step == 2
# Third step: verify
newstate = Dict(
:step => 3,
:thought => "Verify both solutions work",
:action => "Check x=4: 4^2=16 ✓, Check x=-4: (-4)^2=16 ✓",
:reward => 5.0,
:isterminal => true # Problem solved!
)
progressvalue = 10.0
else
# Terminal state
newstate = Dict(
:step => current_step,
:thought => "Problem solved",
:action => "Solution complete",
:reward => 10.0,
:isterminal => true
)
progressvalue = 10.0
end
return Dict(
:newNodeKey => "step_$current_step",
:newstate => newstate,
:progressvalue => progressvalue
)
end
# Initial state
initialstate = Dict(
:step => 0,
:problem => "Solve x^2 = 16",
:reward => 0,
:isterminal => false
)
# Transition arguments
transitionargs = ()
# Run MCTS
result = runMCTS(
initialstate,
math_problem_transition,
transitionargs;
maxiterations = 15,
explorationweight = 1.0,
maxSimulationDepth = 3,
horizontalSampleExpansionPhase = 3
)
# Display results
println("Problem: ", initialstate[:problem])
println()
println("Best solution trajectory:")
println(" Step ", result.bestTerminalState[:step])
println(" Thought: ", result.bestTerminalState[:thought])
println(" Action: ", result.bestTerminalState[:action])
println()
println("Solution complete! ✓")
println("Root node visits: ", result.root.visits)
```
examples/pathfinding.jl
+98
View File
@@ -0,0 +1,98 @@
# Pathfinding Problem - MCTS Example
This example shows how to use MCTS for a pathfinding problem where the goal is to reach a target location.
```julia
using LLMMCTS
# Grid-based pathfinding state
struct Position
x::Int
y::Int
end
# State transition function for pathfinding
function pathfinding_transition(state::Dict, args::NamedTuple)
current_pos = Position(state[:pos_x], state[:pos_y])
target_pos = Position(args.target_x, args.target_y)
# Generate possible moves (up, down, left, right)
moves = [
(0, 1), # up
(0, -1), # down
(1, 0), # right
(-1, 0) # left
]
# In a real scenario, LLM would select which move to try
# For this example, we'll try all moves
move_idx = state[:move_idx] % length(moves) + 1
dx, dy = moves[move_idx]
new_x = current_pos.x + dx
new_y = current_pos.y + dy
# Calculate distance to target
distance = abs(new_x - target_pos.x) + abs(new_y - target_pos.y)
# Reward: negative of distance (closer is better)
reward = -distance
# Progress value: LLM estimate (here we use inverse distance as heuristic)
progressvalue = 10 - distance
newstate = Dict(
:pos_x => new_x,
:pos_y => new_y,
:move_idx => state[:move_idx] + 1,
:reward => reward,
:isterminal => (new_x == target_pos.x && new_y == target_pos.y) ||
(state[:move_idx] >= args.max_moves)
)
return Dict(
:newNodeKey => "pos_$(new_x)_$(new_y)",
:newstate => newstate,
:progressvalue => progressvalue
)
end
# Initial state
initialstate = Dict(
:pos_x => 0,
:pos_y => 0,
:move_idx => 0,
:reward => 0,
:isterminal => false
)
# Target position
target_x, target_y = 3, 2
# Transition arguments
transitionargs = (
target_x = target_x,
target_y = target_y,
max_moves = 10
)
# Run MCTS
result = runMCTS(
initialstate,
pathfinding_transition,
transitionargs;
maxiterations = 20,
explorationweight = 1.5,
maxSimulationDepth = 5,
horizontalSampleExpansionPhase = 4
)
# Display results
println("Target: ($target_x, $target_y)")
println("Best final position: (",
result.bestTerminalState[:pos_x], ", ",
result.bestTerminalState[:pos_y], ")")
println("Final distance: ", abs(result.bestTerminalState[:pos_x] - target_x) +
abs(result.bestTerminalState[:pos_y] - target_y))
println("Root node visits: ", result.root.visits)
```
examples/reasoning.jl
+134
View File
@@ -0,0 +1,134 @@
# Multi-step Reasoning - MCTS with Chain of Thought
This example demonstrates MCTS for multi-step reasoning problems, where the LLM generates chain-of-thought reasoning at each step.
```julia
using LLMMCTS
# State tracks the reasoning process
# thought_history: Dict mapping thought/action keys to their content
function reasoning_transition(state::Dict, args::NamedTuple)
current_step = get(state, :step, 0)
thought_history = get(state, :thought_history, Dict{String, String}())
problem = state[:problem]
if current_step == 0
# Step 1: Understand the problem
thought = "First, I need to understand what the problem is asking. The problem requires me to analyze the given information and determine the solution approach."
action = "Identify the key components of the problem"
new_thought_history = copy(thought_history)
new_thought_history["thought_1"] = thought
new_thought_history["action_1"] = action
newstate = Dict(
:step => 1,
:thought_history => new_thought_history,
:reward => 1.0,
:isterminal => false
)
progressvalue = 3.0
elseif current_step == 1
# Step 2: Break down the problem
thought = "Next, I should break this down into smaller sub-problems. This will make it easier to solve step by step."
action = "Divide the problem into manageable parts"
new_thought_history = copy(thought_history)
new_thought_history["thought_2"] = thought
new_thought_history["action_2"] = action
newstate = Dict(
:step => 2,
:thought_history => new_thought_history,
:reward => 2.0,
:isterminal => false
)
progressvalue = 5.0
elseif current_step == 2
# Step 3: Solve each sub-problem
thought = "Now I'll solve each sub-problem individually, using appropriate methods for each."
action = "Apply solution methods to each sub-problem"
new_thought_history = copy(thought_history)
new_thought_history["thought_3"] = thought
new_thought_history["action_3"] = action
newstate = Dict(
:step => 3,
:thought_history => new_thought_history,
:reward => 3.0,
:isterminal => false
)
progressvalue = 7.0
elseif current_step == 3
# Step 4: Combine solutions
thought = "Finally, I'll combine all the solutions to form the complete answer to the original problem."
action = "Integrate solutions and verify the answer"
new_thought_history = copy(thought_history)
new_thought_history["thought_4"] = thought
new_thought_history["action_4"] = action
newstate = Dict(
:step => 4,
:thought_history => new_thought_history,
:reward => 4.0,
:isterminal => true # Reasoning complete
)
progressvalue = 10.0
else
newstate = Dict(
:step => current_step,
:thought_history => thought_history,
:reward => 10.0,
:isterminal => true
)
progressvalue = 10.0
end
return Dict(
:newNodeKey => "reasoning_step_$current_step",
:newstate => newstate,
:progressvalue => progressvalue
)
end
# Initial state
initialstate = Dict(
:step => 0,
:problem => "Explain how photosynthesis works",
:thought_history => Dict{String, String}(),
:reward => 0,
:isterminal => false
)
# Transition arguments
transitionargs = (max_steps = 4,)
# Run MCTS
result = runMCTS(
initialstate,
reasoning_transition,
transitionargs;
maxiterations = 25,
explorationweight = 1.0,
maxSimulationDepth = 4,
horizontalSampleExpansionPhase = 3
)
# Display results
println("Multi-step Reasoning Example")
println("=============================")
println()
println("Problem: ", initialstate[:problem])
println()
println("Reasoning steps:")
for (key, value) in result.bestTerminalState[:thought_history]
println(" $key: $value")
end
println()
println("Reasoning complete! ✓")
println("Root node visits: ", result.root.visits)
println("Total steps in reasoning chain: ", result.bestTerminalState[:step])
```
examples/simple_example.jl
+59
View File
@@ -0,0 +1,59 @@
# Simple MCTS Example
This example demonstrates basic MCTS usage with a simple state transition function.
```julia
using LLMMCTS
# Define a simple state transition function
function simple_transition(state::Dict, args::NamedTuple)
# In a real scenario, this would call an LLM
# For this example, we'll just generate deterministic next states
current_step = get(state, :step, 0)
new_step = current_step + 1
# Create new state
newstate = Dict(
:step => new_step,
:reward => new_step * 2, # Simple reward function
:isterminal => new_step >= args.max_steps
)
# LLM would provide progressvalue estimate
progressvalue = (new_step / args.max_steps) * 10
return Dict(
:newNodeKey => "step_$(new_step)",
:newstate => newstate,
:progressvalue => progressvalue
)
end
# Initial state
initialstate = Dict(
:step => 0,
:reward => 0,
:isterminal => false
)
# Transition arguments
transitionargs = (max_steps = 5,)
# Run MCTS
result = runMCTS(
initialstate,
simple_transition,
transitionargs;
maxiterations = 10,
explorationweight = 1.0,
maxSimulationDepth = 3,
horizontalSampleExpansionPhase = 3
)
# Access results
println("Root node visits: ", result.root.visits)
println("Best next state: ", result.bestNextState)
println("Best terminal state: ", result.bestTerminalState)
println("High value states: ", result.highValueStateList)
```
examples/tool_use.jl
+109
View File
@@ -0,0 +1,109 @@
# Tool Use Example - MCTS with External Tools
This example shows how MCTS can coordinate with external tools (like APIs, databases, or other services).
```julia
using LLMMCTS
# Simulated tool interface
struct Tool
name::String
description::String
end
const AVAILABLE_TOOLS = [
Tool("calculator", "Perform mathematical calculations"),
Tool("web_search", "Search the web for information"),
Tool("database_query", "Query a database")
]
# State tracks which tools have been used and their results
function tool_use_transition(state::Dict, args::NamedTuple)
current_step = get(state, :step, 0)
tools_used = get(state, :tools_used, String[])
# LLM would decide which tool to use
# For this example, we try tools in order
tool_idx = (current_step - 1) % length(AVAILABLE_TOOLS) + 1
if tool_idx > length(AVAILABLE_TOOLS)
# All tools tried, return terminal state
newstate = Dict(
:step => current_step + 1,
:tools_used => tools_used,
:reward => 8.0,
:isterminal => true
)
return Dict(
:newNodeKey => "all_tools_tried",
:newstate => newstate,
:progressvalue => 8.0
)
end
tool = AVAILABLE_TOOLS[tool_idx]
# Simulate tool execution
tool_result = "Tool '$(tool.name)' executed successfully"
# Calculate reward based on progress
progress = length(tools_used) / length(AVAILABLE_TOOLS)
reward = progress * 5
# Progress value: LLM estimates how close we are to solving
progressvalue = progress * 10
new_tools_used = vcat(tools_used, tool.name)
newstate = Dict(
:step => current_step + 1,
:tools_used => new_tools_used,
:current_tool => tool.name,
:tool_result => tool_result,
:reward => reward,
:isterminal => false
)
return Dict(
:newNodeKey => "tool_$(tool.name)_$current_step",
:newstate => newstate,
:progressvalue => progressvalue
)
end
# Initial state
initialstate = Dict(
:step => 0,
:tools_used => String[],
:reward => 0,
:isterminal => false
)
# Transition arguments
transitionargs = (max_tools = 3,)
# Run MCTS
result = runMCTS(
initialstate,
tool_use_transition,
transitionargs;
maxiterations = 20,
explorationweight = 1.2,
maxSimulationDepth = 4,
horizontalSampleExpansionPhase = 3
)
# Display results
println("Available tools:")
for tool in AVAILABLE_TOOLS
println(" - $(tool.name): $(tool.description)")
end
println()
println("Best tool usage sequence:")
for tool in result.bestTerminalState[:tools_used]
println(" → Used: $tool")
end
println()
println("Root node visits: ", result.root.visits)
println("High value states found: ", length(result.highValueStateList))
```