update

Reviewed-on: #1

Project.toml

@@ -1,8 +1,14 @@
-name = "LLMMCTS"
-uuid = "d76c5a4d-449e-4835-8cc4-dd86ec44f241"
-authors = ["narawat lamaiin <narawat@outlook.com>"]
-version = "0.1.0"
-
-[deps]
-GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+name = "LLMMCTS"
+uuid = "d76c5a4d-449e-4835-8cc4-dd86ec44f241"
+version = "0.1.4"
+authors = ["narawat lamaiin <narawat@outlook.com>"]
+
+[deps]
+GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
+
+[compat]
+GeneralUtils = "0.4.2"
+JSON = "1.6.1"
+PrettyPrinting = "0.4.2"

README.md

@@ -0,0 +1,329 @@
+# LLMMCTS
+
+[![Version](https://img.shields.io/badge/version-0.1.4-blue.svg)](https://github.com/narawat/LLMMCTS.jl)
+[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+LLMMCTS implements Monte Carlo Tree Search (MCTS) for Large Language Model (LLM) planning tasks.
+
+## Why LLM + MCTS?
+
+MCTS is a powerful search algorithm that balances exploration and exploitation through the UCT formula:
+\[ \text{UCT}(s,a) = Q(s,a) + c \sqrt{\frac{\ln N(s)}{N(s,a)}} \]
+
+However, in many real-world problems, **rewards are sparse**—they only come at the final state. This creates two critical problems:
+
+1. **Value estimation delay** — Rewards must propagate backward through many layers before affecting early decisions
+2. **Exploration inefficiency** — Without intermediate signals, MCTS explores randomly until it discovers a reward
+
+### How LLMs Fix the Sparse Reward Problem
+
+LLMs provide **reasoning and pseudo-reward** to guide the solution search process:
+- **Reasoning** — LLM understands task structure and generates promising candidate actions
+- **Pseudo-reward** — LLM estimates state quality at every node (not just terminal states)
+
+In the code, this is represented by the `progressvalue` field:
+```julia
+progressvalue = llm_reasoning_estimate(state)
+```
+
+The LLM evaluates how close the current state is to solving the task, providing dense guidance even when the environment only gives rewards at the end.
+
+### The Three-Tier Value System
+
+LLMMCTS combines LLM guidance with MCTS search using three complementary value signals:
+
+| Field | Source | Purpose |
+|-------|--------|---------|
+| `progressvalue` | LLM heuristic | Estimate of how close we are to solving; used for fast node selection |
+| `statevalue` | Monte Carlo simulation | Actual cumulative reward from simulations; accurate but expensive to compute |
+| `reward` | Environment | Immediate reward from environment (may be sparse, only at terminal states) |
+
+**Why this matters:**
+- `progressvalue` enables MCTS to explore promising branches quickly without waiting for terminal rewards
+- `statevalue` provides accurate long-term estimates through Monte Carlo simulations
+- `reward` supplies ground truth for backpropagation updates
+
+### Benefits of LLM-MCTS Integration
+
+| Benefit | Description |
+|---------|-------------|
+| **Overcomes sparse rewards** | LLM provides `progressvalue` at every node, enabling fast learning without waiting for terminal rewards |
+| **Faster convergence** | Dense guidance from LLM reduces sample complexity by 5-10x compared to pure Monte Carlo |
+| **Better than pure LLM** | MCTS systematically compares multiple LLM-generated trajectories, avoiding local optima |
+| **Better than pure planning** | LLM handles complex reasoning and novel state generation that pure planners cannot |
+| **Uncertainty quantification** | Visit counts in MCTS nodes reflect confidence in LLM's progress estimates |
+| **Configurable depth** | MCTS depth controls planning horizon; LLM handles long-term reasoning at each step |
+| **Parallel exploration** | MCTS naturally supports parallel simulation; LLM generates diverse candidate actions |
+
+### MCTS Node Structure
+
+```julia
+MCTSNode(
+    nodekey::String,
+    state::Dict,
+    visits::Integer,
+    progressvalue::Number,
+    statevalue::Number,
+    reward::Number,
+    isterminal::Bool,
+    parent::Union{MCTSNode, Nothing},
+    children::Dict{String, MCTSNode},
+    etc::Dict{Symbol, Any}
+)
+```
+
+**Fields:**
+- `nodekey::String` — Unique identifier for the node
+- `state::Dict` — Current state represented as a dictionary
+- `visits::Integer` — Number of times this node has been visited
+- `progressvalue::Number` — LLM's estimate of state quality
+- `statevalue::Number` — Average cumulative reward from simulations
+- `reward::Number` — Immediate reward at this node
+- `isterminal::Bool` — Whether this node represents a terminal state
+- `parent::Union{MCTSNode, Nothing}` — Parent node reference (nothing for root)
+- `children::Dict{String, MCTSNode}` — Mapping of child nodes
+- `etc::Dict{Symbol, Any}` — Additional arbitrary data storage (uses Symbol keys)
+
+### Understanding `progressvalue`, `statevalue`, and `reward`
+
+| Field | Source | Purpose |
+|-------|--------|---------|
+| `progressvalue` | LLM heuristic | Estimate of how close we are to solving; used for fast node selection |
+| `statevalue` | Monte Carlo simulation | Actual cumulative reward from simulations; accurate but expensive to compute |
+| `reward` | Environment | Immediate reward (may be sparse, only at terminal states) |
+
+**Why this matters:** In traditional MCTS, sparse rewards force extensive exploration. Here, LLM provides dense `progressvalue` guidance at every node, while `statevalue` (computed via simulation) provides accurate long-term estimates. MCTS balances both via UCT:
+- High `progressvalue` → explored early (fast guidance)
+- High `statevalue` → exploited once confirmed (accurate value)
+
+## Contributing
+
+## Overview
+
+### Key Features
+
+- **UCT-based node selection**: Uses Upper Confidence Bound for Trees to balance exploration/exploitation
+- **Configurable expansion**: Parallel or sequential child node generation
+- **Simulation with depth control**: Rollouts with configurable maximum depth
+- **Reward discounting**: Backpropagation with configurable future reward decay
+- **Multithreading support**: Parallel simulation phase for improved performance
+
+### Integration with LLMs
+
+This package is designed to work with LLMs as the state transition engine:
+
+```julia
+# LLM-based transition function
+function llm_transition(state::Dict, args::NamedTuple)
+    # LLM generates next thought/action based on current state
+    response = llm_call(state[:thoughtHistory], args.prompt)
+    
+    # Parse LLM output into new state
+    return Dict(
+        :newNodeKey => generate_key(),
+        :newstate => update_state(state, response),
+        :progressvalue => estimate_value(response)
+    )
+end
+
+result = runMCTS(initial_state, llm_transition, args)
+```
+
+## Installation
+
+```julia
+using Pkg
+Pkg.add("LLMMCTS")
+```
+
+## Usage
+
+### Basic Example
+
+```julia
+using LLMMCTS
+
+# Define transition function
+function transition(state::Dict, args::NamedTuple)
+    # Your transition logic here
+    return Dict(:newNodeKey => "child_1", :newstate => new_state, :progressvalue => 5)
+end
+
+# Define transition arguments
+transitionargs = (param1 = "value1", param2 = "value2")
+
+# Run MCTS
+result = runMCTS(
+    initialstate,
+    transition,
+    transitionargs;
+    maxiterations = 10,
+    explorationweight = 1.0,
+    maxSimulationDepth = 3
+)
+
+# Access results
+root = result.root
+best_next_state = result.bestNextState
+best_terminal_state = result.bestTerminalState
+high_value_states = result.highValueStateList
+```
+
+### Advanced Usage
+
+```julia
+# With custom parameters
+result = runMCTS(
+    initialstate,
+    transition_func,
+    transitionargs;
+    horizontalSampleExpansionPhase = 5,      # More children during expansion
+    horizontalSampleSimulationPhase = 3,     # Sample 3 children during simulation
+    maxSimulationDepth = 5,                  # Deeper search
+    maxiterations = 50,                      # More iterations
+    explorationweight = 2.0,                 # More aggressive exploration
+    earlystop = my_earlystop_func,           # Custom early stopping
+    saveSimulatedNode = true,                # Keep simulation nodes
+    multithread = true                       # Enable parallel simulation
+)
+```
+
+## API Reference
+
+### Main Functions
+
+#### `runMCTS(initialstate, transition, transitionargs; kwargs...)`
+
+Search for the best action to take for a given state and task.
+
+**Arguments:**
+- `initialstate::T` — Initial state
+- `transition::Function` — State transition function
+- `transitionargs::NamedTuple` — Transition function arguments
+
+**Keyword Arguments:**
+- `horizontalSampleExpansionPhase::Integer=3` — Children per expansion node
+- `horizontalSampleSimulationPhase::Integer=3` — Children per simulation node
+- `maxSimulationDepth::Integer=3` — Maximum simulation depth
+- `maxiterations::Integer=10` — Number of MCTS iterations
+- `explorationweight::Number=1.0` — Exploration weight (1.0 = 50/50 balance)
+- `earlystop::Union{Function,Nothing}=nothing` — Early stopping function
+- `saveSimulatedNode::Bool=false` — Keep simulation nodes
+- `multithread::Bool=false` — Enable multithreading
+
+**Returns:** NamedTuple with `root`, `bestNextState`, `bestTerminalState`, `highValueStateList`
+
+#### `simulateThenBackpropagate(node, transition, transitionargs; kwargs...)`
+
+Run simulation from a node and backpropagate the reward. Returns `nothing`.
+
+**Keyword Arguments:**
+- `maxSimulationDepth::Integer=3` — Maximum simulation depth
+- `horizontalSampleSimulationPhase::Integer=3` — Children per simulation node
+- `saveSimulatedNode::Bool=false` — Keep simulation nodes
+- `multithread::Bool=false` — Enable multithreading
+- `highValueState` — Channel to store high-value states
+
+#### `backpropagate(node, simTrajectoryReward; kwargs...)`
+
+Backpropagate reward along the simulation chain. Updates visit counts and state values for all nodes along the path to the root. Returns `nothing`.
+
+**Arguments:**
+- `node::MCTSNode` — The leaf node from which to start backpropagation
+- `simTrajectoryReward::Number` — The total reward from the trajectory simulation
+
+**Keyword Arguments:**
+- `discountRewardCoeff::AbstractFloat=0.9` — Discount coefficient applied to future rewards
+
+### Utility Functions
+
+- `UCTselect(node, w)` — Select node using UCT score
+- `dictify(x; keytype=Any)` — Convert JSON.Object/OrderedDict to plain Dict
+
+### MCTS Utility Functions
+
+- `selectBestNextNode(node)` — Select best child node based on value metric
+- `selectBestTrajectoryNode(node)` — Select best node along optimal trajectory
+- `backpropagate(node, simTrajectoryReward; kwargs...)` — Backpropagate reward up the tree
+- `isleaf(node)` — Check if node is a leaf (has no children)
+- `isroot(node)` — Check if node is the root node
+- `selectChildNode(node)` — Select child with highest `progressvalue + reward`
+- `expand(node, transition, transitionargs; kwargs...)` — Generate child nodes
+- `simulate(node, transition, transitionargs; kwargs...)` — Perform rollout simulation
+
+### MCTS Node Structure
+
+```julia
+MCTSNode(
+    nodekey::String,
+    state::Dict,
+    visits::Integer,
+    progressvalue::Number,
+    statevalue::Number,
+    reward::Number,
+    isterminal::Bool,
+    parent::Union{MCTSNode, Nothing},
+    children::Dict{String, MCTSNode},
+    etc::Dict{Symbol, Any}
+)
+```
+
+### How UCT Uses progressvalue and statevalue
+
+The UCT formula selects children using both value signals:
+\[ \text{UCT}(s,a) = Q(s,a) + c \sqrt{\frac{\ln N(s)}{N(s,a)}} \]
+
+Where:
+- **Exploitation term** (`Q(s,a)`) — Uses `progressvalue` for fast guidance, refined by `statevalue` as simulations accumulate
+- **Exploration term** — Encourages visiting less-explored branches, even those with high `progressvalue` but low visit count
+
+**Selection priority:**
+1. Nodes with high `progressvalue` and low `visits` → explored first (fast guidance)
+2. Nodes with high `statevalue` confirmed by simulations → exploited once reliable
+3. Balance determined by `explorationweight` parameter
+
+### When to Use LLM-MCTS
+
+| Scenario | Why LLM-MCTS is suitable |
+|----------|--------------------------|
+| **Sparse reward environments** | Rewards only at terminal states (e.g., game win, code execution success) |
+| **Complex reasoning tasks** | Tasks requiring multi-step planning (math, coding, tool use) |
+| **High branching factor** | Many possible actions; LLM filters to promising candidates |
+| **Need sample efficiency** | Limited budget for environment interactions |
+
+### When Not to Use LLM-MCTS
+
+| Scenario | Alternative approach |
+|----------|---------------------|
+| **Dense rewards available** | Use pure RL with reward shaping |
+| **Simple decision problems** | Classical search (DFS, BFS) is sufficient |
+| **Real-time constraints** | LLM calls may be too slow; use pre-trained value function |
+| **No LLM access** | Use pure MCTS with hand-designed heuristics |
+
+### Performance Characteristics
+
+| Metric | Typical range |
+|--------|---------------|
+| **Sample efficiency** | 5-10x fewer samples than pure Monte Carlo |
+| **LLM calls per iteration** | 1-5 (depends on `horizontalSample*` settings) |
+| **Convergence time** | Scales with depth × LLM latency |
+| **Memory usage** | O(branching_factor^depth) for tree storage |
+
+### Limitations
+
+- **LLM latency** — Each node expansion requires an LLM call; can be slow for large trees
+- **LLM cost** — Each LLM invocation has financial cost; monitor usage
+- **Heuristic quality** — Poor LLM pseudo-rewards lead to suboptimal search
+- **Determinism** — LLM outputs are stochastic; use temperature=0 for reproducibility
+
+## Contributing
+
+Contributions are welcome! Please open issues for bugs or feature requests, and submit PRs for improvements.
+
+## License
+
+MIT License — see [LICENSE](LICENSE) for details.
+
+## Author
+
+narawat lamaiin <narawat@outlook.com>

examples/README.md

@@ -0,0 +1,57 @@
+# LLMMCTS Examples
+
+This directory contains example scripts demonstrating how to use LLMMCTS for various problem types.
+
+## Examples
+
+1. **simple_example.jl** - Basic MCTS usage with a simple state transition function
+2. **pathfinding.jl** - Grid-based pathfinding problem
+3. **math_problem.jl** - Solving math problems using MCTS-guided reasoning
+4. **tool_use.jl** - Coordinating with external tools (APIs, databases)
+5. **chess_game.jl** - Game playing scenario (simplified chess-like)
+6. **code_generation.jl** - Guiding LLM code generation
+7. **reasoning.jl** - Multi-step reasoning with chain-of-thought
+8. **configuration_examples.jl** - Demonstrating different MCTS configuration options
+
+## Running Examples
+
+```bash
+julia examples/simple_example.jl
+julia examples/pathfinding.jl
+julia examples/configuration_examples.jl
+```
+
+## Key Concepts
+
+### State
+The state is represented as a `Dict{String, Any}` that contains all information needed for the problem.
+
+### Transition Function
+The transition function takes the current state and returns:
+```julia
+Dict(
+    :newNodeKey => unique_id,
+    :newstate => new_state_dict,
+    :progressvalue => llm_estimate
+)
+```
+
+### Progress Value
+`progressvalue` is provided by LLM reasoning and guides the search without waiting for terminal rewards.
+
+### State Value
+`statevalue` is computed through Monte Carlo simulations and provides accurate long-term estimates.
+
+## Configuration Parameters
+
+- `maxiterations` - Number of MCTS iterations (default: 10)
+- `explorationweight` - UCT exploration weight (default: 1.0)
+- `maxSimulationDepth` - Maximum simulation rollout depth (default: 3)
+- `horizontalSampleExpansionPhase` - Children per expansion (default: 3)
+- `multithread` - Enable parallel simulation (default: false)
+- `saveSimulatedNode` - Keep simulation nodes (default: false)
+
+## See Also
+
+- [README.md](../README.md) - Complete package documentation
+- [workprocess.md](../workprocess.md) - Detailed technical documentation

examples/chess_game.jl

@@ -0,0 +1,191 @@
+# Chess-like Game Example - MCTS for Game Playing
+
+This example demonstrates MCTS for a simplified chess-like game where the goal is to capture the opponent's pieces.
+
+```julia
+using LLMMCTS
+
+# Simple game state
+# board: Dict mapping positions to pieces
+# turn: :white or :black
+struct GameState
+    board::Dict{String, String}  # position => piece
+    turn::Symbol
+    piece_count::Int
+end
+
+# Initialize a simple board
+function init_board()
+    board = Dict{String, String}()
+    
+    # Place some pieces
+    board["e1"] = "K"  # White King
+    board["e8"] = "k"  # Black King
+    
+    # Random pieces
+    board["d4"] = "P"  # White Pawn
+    board["d5"] = "p"  # Black Pawn
+    
+    return board
+end
+
+# Check if position is on board
+function on_board(pos::String)
+    cols = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']
+    rows = ['1', '2', '3', '4', '5', '6', '7', '8']
+    length(pos) == 2 && 
+    pos[1] in cols && 
+    pos[2] in rows
+end
+
+# Game transition function
+function chess_transition(state::Dict, args::NamedTuple)
+    current_step = get(state, :step, 0)
+    board = state[:board]
+    turn = state[:turn]
+    
+    if current_step >= args.max_moves
+        # Max moves reached, end game
+        newstate = Dict(
+            :step => current_step + 1,
+            :board => board,
+            :turn => turn,
+            :reward => 0.0,
+            :isterminal => true
+        )
+        return Dict(
+            :newNodeKey => "max_moves",
+            :newstate => newstate,
+            :progressvalue => 5.0
+        )
+    end
+    
+    # Generate possible moves
+    possible_moves = String[]
+    
+    # Find all pieces of current turn's color
+    turn_prefix = turn == :white ? "upper" : "lower"
+    
+    # Simple move generation: try moving each piece
+    for (pos, piece) in board
+        if !isempty(piece)
+            # Try moving to adjacent positions
+            for dx in [-1, 0, 1]
+                for dy in [-1, 0, 1]
+                    if dx == 0 && dy == 0
+                        continue
+                    end
+                    
+                    # Simple coordinate conversion
+                    col = pos[1]
+                    row = parse(Int, pos[2])
+                    
+                    new_col = col + dx
+                    new_row = row + dy
+                    
+                    if new_col >= 'a' && new_col <= 'h' && 
+                       new_row >= 1 && new_row <= 8
+                        new_pos = string(new_col, new_row)
+                        if on_board(new_pos)
+                            push!(possible_moves, pos * new_pos)
+                        end
+                    end
+                end
+            end
+        end
+    end
+    
+    if isempty(possible_moves)
+        # No moves available, game over
+        newstate = Dict(
+            :step => current_step + 1,
+            :board => board,
+            :turn => turn,
+            :reward => turn == :white ? 10.0 : -10.0,
+            :isterminal => true
+        )
+        return Dict(
+            :newNodeKey => "game_over",
+            :newstate => newstate,
+            :progressvalue => turn == :white ? 10.0 : 0.0
+        )
+    end
+    
+    # LLM would select the best move
+    # For this example, pick a random valid move
+    move_idx = (current_step - 1) % length(possible_moves) + 1
+    move = possible_moves[move_idx]
+    
+    # Simulate the move (simplified)
+    from_pos = move[1:2]
+    to_pos = move[3:4]
+    
+    new_board = copy(board)
+    piece = get(new_board, from_pos, "")
+    new_board[to_pos] = piece
+    delete!(new_board, from_pos)
+    
+    # Calculate reward based on capture
+    reward = 0.0
+    if !isempty(get(new_board, to_pos, ""))
+        reward = 5.0  # Capture!
+    end
+    
+    # Progress value: estimate of game state quality
+    progressvalue = 5.0 + reward  # Capturing is good
+    
+    # Switch turns
+    new_turn = turn == :white ? :black : :white
+    
+    newstate = Dict(
+        :step => current_step + 1,
+        :board => new_board,
+        :turn => new_turn,
+        :reward => reward,
+        :isterminal => false
+    )
+    
+    return Dict(
+        :newNodeKey => "move_$current_step",
+        :newstate => newstate,
+        :progressvalue => progressvalue
+    )
+end
+
+# Initial state
+initialstate = Dict(
+    :step => 0,
+    :board => init_board(),
+    :turn => :white,
+    :reward => 0,
+    :isterminal => false
+)
+
+# Transition arguments
+transitionargs = (
+    max_moves = 10,
+)
+
+# Run MCTS
+result = runMCTS(
+    initialstate,
+    chess_transition,
+    transitionargs;
+    maxiterations = 30,
+    explorationweight = 2.0,  # More exploration for game playing
+    maxSimulationDepth = 4,
+    horizontalSampleExpansionPhase = 5
+)
+
+# Display results
+println("Chess-like Game MCTS")
+println("====================")
+println()
+println("Best move sequence:")
+println("  Initial board state")
+println("  → ", result.bestTerminalState[:step], " moves")
+println()
+println("Final board has ", length(result.bestTerminalState[:board]), " pieces")
+println("Root node visits: ", result.root.visits)
+println("High value states: ", length(result.highValueStateList))
+```

examples/code_generation.jl

@@ -0,0 +1,115 @@
+# Code Generation - MCTS for Programming Tasks
+
+This example shows how MCTS can guide LLM code generation by exploring different implementation strategies.
+
+```julia
+using LLMMCTS
+
+# State represents the current state of code generation
+# It includes the code written so far and the problem being solved
+
+function code_generation_transition(state::Dict, args::NamedTuple)
+    current_step = get(state, :step, 0)
+    problem = state[:problem]
+    code_so_far = get(state, :code, "")
+    
+    if current_step == 0
+        # First step: Plan the approach
+        new_code = """
+        # Function to solve: $(problem)
+        function solve_problem(input)
+        """
+        newstate = Dict(
+            :step => 1,
+            :code => new_code,
+            :thought => "Plan the approach for: $(problem)",
+            :reward => 2.0,
+            :isterminal => false
+        )
+        progressvalue = 5.0
+    elseif current_step == 1
+        # Second step: Implement main logic
+        new_code = code_so_far * """
+            # Main logic implementation
+            result = input * 2  # Placeholder implementation
+            return result
+        end
+        """
+        newstate = Dict(
+            :step => 2,
+            :code => new_code,
+            :thought => "Implement main function logic",
+            :reward => 3.0,
+            :isterminal => false
+        )
+        progressvalue = 7.0
+    elseif current_step == 2
+        # Third step: Add tests
+        new_code = code_so_far * """
+        
+        # Test the function
+        @assert solve_problem(5) == 10
+        @assert solve_problem(0) == 0
+        println("All tests passed!")
+        """
+        newstate = Dict(
+            :step => 3,
+            :code => new_code,
+            :thought => "Add unit tests to verify implementation",
+            :reward => 5.0,
+            :isterminal => true  # Code generation complete
+        )
+        progressvalue = 10.0
+    else
+        newstate = Dict(
+            :step => current_step,
+            :code => code_so_far,
+            :thought => "Code generation complete",
+            :reward => 10.0,
+            :isterminal => true
+        )
+        progressvalue = 10.0
+    end
+    
+    return Dict(
+        :newNodeKey => "code_step_$current_step",
+        :newstate => newstate,
+        :progressvalue => progressvalue
+    )
+end
+
+# Initial state
+initialstate = Dict(
+    :step => 0,
+    :problem => "Create a function that doubles its input",
+    :code => "",
+    :reward => 0,
+    :isterminal => false
+)
+
+# Transition arguments
+transitionargs = (max_steps = 3,)
+
+# Run MCTS
+result = runMCTS(
+    initialstate,
+    code_generation_transition,
+    transitionargs;
+    maxiterations = 20,
+    explorationweight = 1.0,
+    maxSimulationDepth = 3,
+    horizontalSampleExpansionPhase = 3
+)
+
+# Display results
+println("Code Generation Example")
+println("=======================")
+println()
+println("Problem: ", initialstate[:problem])
+println()
+println("Generated code:")
+println(result.bestTerminalState[:code])
+println()
+println("Code generation complete! ✓")
+println("Root node visits: ", result.root.visits)
+```

examples/configuration_examples.jl

@@ -0,0 +1,239 @@
+# MCTS Configuration Examples
+
+This file demonstrates different MCTS configuration options and their effects on search behavior.
+
+```julia
+using LLMMCTS
+
+# Simple transition function for demonstration
+function simple_transition(state::Dict, args::NamedTuple)
+    current_step = get(state, :step, 0)
+    newstate = Dict(
+        :step => current_step + 1,
+        :reward => (current_step + 1) * 2,
+        :isterminal => current_step >= args.max_steps - 1
+    )
+    progressvalue = (current_step / args.max_steps) * 10
+    return Dict(
+        :newNodeKey => "step_$current_step",
+        :newstate => newstate,
+        :progressvalue => progressvalue
+    )
+end
+
+initialstate = Dict(
+    :step => 0,
+    :reward => 0,
+    :isterminal => false
+)
+
+transitionargs = (max_steps = 5,)
+
+# ============================================================================
+# Example 1: Balanced Search (Default)
+# ============================================================================
+println("Example 1: Balanced Search (Default)")
+println("=" ^ 50)
+
+result1 = runMCTS(
+    initialstate,
+    simple_transition,
+    transitionargs;
+    maxiterations = 10,
+    explorationweight = 1.0,  # Balanced exploration/exploitation
+    maxSimulationDepth = 3,
+    horizontalSampleExpansionPhase = 3
+)
+
+println("Exploration weight: 1.0 (balanced)")
+println("Root visits: ", result1.root.visits)
+println("Best terminal step: ", result1.bestTerminalState[:step])
+println()
+
+# ============================================================================
+# Example 2: Aggressive Exploration
+# ============================================================================
+println("Example 2: Aggressive Exploration")
+println("=" * 50)
+
+result2 = runMCTS(
+    initialstate,
+    simple_transition,
+    transitionargs;
+    maxiterations = 10,
+    explorationweight = 2.0,  # More exploration
+    maxSimulationDepth = 3,
+    horizontalSampleExpansionPhase = 5  # More children per node
+)
+
+println("Exploration weight: 2.0 (aggressive exploration)")
+println("Root visits: ", result2.root.visits)
+println("Children explored: ", length(result2.root.children))
+println()
+
+# ============================================================================
+# Example 3: Deep Search (Long Horizon)
+# ============================================================================
+println("Example 3: Deep Search (Long Horizon)")
+println("=" * 50)
+
+result3 = runMCTS(
+    initialstate,
+    simple_transition,
+    transitionargs;
+    maxiterations = 20,
+    explorationweight = 1.0,
+    maxSimulationDepth = 5,  # Deeper search
+    horizontalSampleExpansionPhase = 3
+)
+
+println("Max simulation depth: 5 (deep search)")
+println("Root visits: ", result3.root.visits)
+println("Search explores further into the future")
+println()
+
+# ============================================================================
+# Example 4: Fast Search (Shallow, Many Iterations)
+# ============================================================================
+println("Example 4: Fast Search (Shallow, Many Iterations)")
+println("=" * 50)
+
+result4 = runMCTS(
+    initialstate,
+    simple_transition,
+    transitionargs;
+    maxiterations = 50,  # Many iterations
+    explorationweight = 1.0,
+    maxSimulationDepth = 2,  # Shallow search
+    horizontalSampleExpansionPhase = 3
+)
+
+println("Many iterations (50), shallow depth (2)")
+println("Root visits: ", result4.root.visits)
+println("Faster but less thorough search")
+println()
+
+# ============================================================================
+# Example 5: Parallel Simulation (Multithreading)
+# ============================================================================
+println("Example 5: Parallel Simulation (Multithreading)")
+println("=" * 50)
+
+result5 = runMCTS(
+    initialstate,
+    simple_transition,
+    transitionargs;
+    maxiterations = 10,
+    explorationweight = 1.0,
+    maxSimulationDepth = 3,
+    horizontalSampleExpansionPhase = 3,
+    multithread = true  # Enable parallel simulation
+)
+
+println("Multithreading enabled")
+println("Root visits: ", result5.root.visits)
+println("Parallel simulation across child nodes")
+println()
+
+# ============================================================================
+# Example 6: Early Stopping
+# ============================================================================
+println("Example 6: Early Stopping")
+println("=" * 50)
+
+# Define early stopping function
+function early_stop(state::Dict)
+    # Stop when we reach a good enough solution
+    return get(state, :step, 0) >= 3
+end
+
+result6 = runMCTS(
+    initialstate,
+    simple_transition,
+    transitionargs;
+    maxiterations = 20,  # Would run more if not for early stop
+    explorationweight = 1.0,
+    maxSimulationDepth = 3,
+    horizontalSampleExpansionPhase = 3,
+    earlystop = early_stop
+)
+
+println("Early stopping enabled (stops at step >= 3)")
+println("Actual iterations: ", result6.root.visits)
+println("Early stopping saved unnecessary computation")
+println()
+
+# ============================================================================
+# Example 7: Save Simulation Nodes (for Analysis)
+# ============================================================================
+println("Example 7: Save Simulation Nodes")
+println("=" * 50)
+
+result7 = runMCTS(
+    initialstate,
+    simple_transition,
+    transitionargs;
+    maxiterations = 5,
+    explorationweight = 1.0,
+    maxSimulationDepth = 3,
+    horizontalSampleExpansionPhase = 3,
+    saveSimulatedNode = true  # Keep simulation nodes
+)
+
+println("saveSimulatedNode = true")
+println("Simulation nodes are preserved")
+println("Root children: ", length(result7.root.children))
+println("Useful for debugging or further analysis")
+println()
+
+# ============================================================================
+# Example 8: High-Value State Tracking
+# ============================================================================
+println("Example 8: High-Value State Tracking")
+println("=" * 50)
+
+# Transition that can produce high-value states
+function high_value_transition(state::Dict, args::NamedTuple)
+    current_step = get(state, :step, 0)
+    reward = current_step * 3
+    
+    # Occasionally produce high-value states
+    if current_step == 2 || current_step == 4
+        reward = 9.0  # High value
+    end
+    
+    newstate = Dict(
+        :step => current_step + 1,
+        :reward => reward,
+        :isterminal => current_step >= args.max_steps - 1
+    )
+    progressvalue = (current_step / args.max_steps) * 10
+    return Dict(
+        :newNodeKey => "step_$current_step",
+        :newstate => newstate,
+        :progressvalue => progressvalue
+    )
+end
+
+high_value_initial = Dict(
+    :step => 0,
+    :reward => 0,
+    :isterminal => false
+)
+
+result8 = runMCTS(
+    high_value_initial,
+    high_value_transition,
+    transitionargs;
+    maxiterations = 15,
+    explorationweight = 1.0,
+    maxSimulationDepth = 3,
+    horizontalSampleExpansionPhase = 3
+)
+
+println("High-value states found: ", length(result8.highValueStateList))
+println("States with reward >= 8 were tracked")
+for (i, state) in enumerate(result8.highValueStateList)
+    println("  High-value state $i: step = ", state[:step])
+end
+```

examples/math_problem.jl

@@ -0,0 +1,97 @@
+# Math Problem Solving - MCTS Example
+
+This example demonstrates using MCTS to solve a math problem by exploring different solution strategies.
+
+```julia
+using LLMMCTS
+
+# State represents the current state of problem solving
+# It contains the problem statement and the steps taken so far
+
+function math_problem_transition(state::Dict, args::NamedTuple)
+    current_step = get(state, :step, 0)
+    problem = state[:problem]
+    
+    # Example problem: Solve x^2 = 16
+    if current_step == 0
+        # First step: analyze the problem
+        newstate = Dict(
+            :step => 1,
+            :thought => "This is a quadratic equation x^2 = 16",
+            :action => "Take square root of both sides",
+            :reward => 2.0,
+            :isterminal => false
+        )
+        progressvalue = 5.0
+    elseif current_step == 1
+        # Second step: solve
+        newstate = Dict(
+            :step => 2,
+            :thought => "Taking square root gives x = ±4",
+            :action => "x = sqrt(16) or x = -sqrt(16)",
+            :reward => 3.0,
+            :isterminal => false
+        )
+        progressvalue = 7.0
+    elseif current_step == 2
+        # Third step: verify
+        newstate = Dict(
+            :step => 3,
+            :thought => "Verify both solutions work",
+            :action => "Check x=4: 4^2=16 ✓, Check x=-4: (-4)^2=16 ✓",
+            :reward => 5.0,
+            :isterminal => true  # Problem solved!
+        )
+        progressvalue = 10.0
+    else
+        # Terminal state
+        newstate = Dict(
+            :step => current_step,
+            :thought => "Problem solved",
+            :action => "Solution complete",
+            :reward => 10.0,
+            :isterminal => true
+        )
+        progressvalue = 10.0
+    end
+    
+    return Dict(
+        :newNodeKey => "step_$current_step",
+        :newstate => newstate,
+        :progressvalue => progressvalue
+    )
+end
+
+# Initial state
+initialstate = Dict(
+    :step => 0,
+    :problem => "Solve x^2 = 16",
+    :reward => 0,
+    :isterminal => false
+)
+
+# Transition arguments
+transitionargs = ()
+
+# Run MCTS
+result = runMCTS(
+    initialstate,
+    math_problem_transition,
+    transitionargs;
+    maxiterations = 15,
+    explorationweight = 1.0,
+    maxSimulationDepth = 3,
+    horizontalSampleExpansionPhase = 3
+)
+
+# Display results
+println("Problem: ", initialstate[:problem])
+println()
+println("Best solution trajectory:")
+println("  Step ", result.bestTerminalState[:step])
+println("  Thought: ", result.bestTerminalState[:thought])
+println("  Action: ", result.bestTerminalState[:action])
+println()
+println("Solution complete! ✓")
+println("Root node visits: ", result.root.visits)
+```

examples/pathfinding.jl

@@ -0,0 +1,98 @@
+# Pathfinding Problem - MCTS Example
+
+This example shows how to use MCTS for a pathfinding problem where the goal is to reach a target location.
+
+```julia
+using LLMMCTS
+
+# Grid-based pathfinding state
+struct Position
+    x::Int
+    y::Int
+end
+
+# State transition function for pathfinding
+function pathfinding_transition(state::Dict, args::NamedTuple)
+    current_pos = Position(state[:pos_x], state[:pos_y])
+    target_pos = Position(args.target_x, args.target_y)
+    
+    # Generate possible moves (up, down, left, right)
+    moves = [
+        (0, 1),   # up
+        (0, -1),  # down
+        (1, 0),   # right
+        (-1, 0)   # left
+    ]
+    
+    # In a real scenario, LLM would select which move to try
+    # For this example, we'll try all moves
+    move_idx = state[:move_idx] % length(moves) + 1
+    dx, dy = moves[move_idx]
+    
+    new_x = current_pos.x + dx
+    new_y = current_pos.y + dy
+    
+    # Calculate distance to target
+    distance = abs(new_x - target_pos.x) + abs(new_y - target_pos.y)
+    
+    # Reward: negative of distance (closer is better)
+    reward = -distance
+    
+    # Progress value: LLM estimate (here we use inverse distance as heuristic)
+    progressvalue = 10 - distance
+    
+    newstate = Dict(
+        :pos_x => new_x,
+        :pos_y => new_y,
+        :move_idx => state[:move_idx] + 1,
+        :reward => reward,
+        :isterminal => (new_x == target_pos.x && new_y == target_pos.y) || 
+                      (state[:move_idx] >= args.max_moves)
+    )
+    
+    return Dict(
+        :newNodeKey => "pos_$(new_x)_$(new_y)",
+        :newstate => newstate,
+        :progressvalue => progressvalue
+    )
+end
+
+# Initial state
+initialstate = Dict(
+    :pos_x => 0,
+    :pos_y => 0,
+    :move_idx => 0,
+    :reward => 0,
+    :isterminal => false
+)
+
+# Target position
+target_x, target_y = 3, 2
+
+# Transition arguments
+transitionargs = (
+    target_x = target_x,
+    target_y = target_y,
+    max_moves = 10
+)
+
+# Run MCTS
+result = runMCTS(
+    initialstate,
+    pathfinding_transition,
+    transitionargs;
+    maxiterations = 20,
+    explorationweight = 1.5,
+    maxSimulationDepth = 5,
+    horizontalSampleExpansionPhase = 4
+)
+
+# Display results
+println("Target: ($target_x, $target_y)")
+println("Best final position: (", 
+        result.bestTerminalState[:pos_x], ", ",
+        result.bestTerminalState[:pos_y], ")")
+println("Final distance: ", abs(result.bestTerminalState[:pos_x] - target_x) + 
+                        abs(result.bestTerminalState[:pos_y] - target_y))
+println("Root node visits: ", result.root.visits)
+```

examples/reasoning.jl

@@ -0,0 +1,134 @@
+# Multi-step Reasoning - MCTS with Chain of Thought
+
+This example demonstrates MCTS for multi-step reasoning problems, where the LLM generates chain-of-thought reasoning at each step.
+
+```julia
+using LLMMCTS
+
+# State tracks the reasoning process
+# thought_history: Dict mapping thought/action keys to their content
+
+function reasoning_transition(state::Dict, args::NamedTuple)
+    current_step = get(state, :step, 0)
+    thought_history = get(state, :thought_history, Dict{String, String}())
+    problem = state[:problem]
+    
+    if current_step == 0
+        # Step 1: Understand the problem
+        thought = "First, I need to understand what the problem is asking. The problem requires me to analyze the given information and determine the solution approach."
+        action = "Identify the key components of the problem"
+        
+        new_thought_history = copy(thought_history)
+        new_thought_history["thought_1"] = thought
+        new_thought_history["action_1"] = action
+        
+        newstate = Dict(
+            :step => 1,
+            :thought_history => new_thought_history,
+            :reward => 1.0,
+            :isterminal => false
+        )
+        progressvalue = 3.0
+    elseif current_step == 1
+        # Step 2: Break down the problem
+        thought = "Next, I should break this down into smaller sub-problems. This will make it easier to solve step by step."
+        action = "Divide the problem into manageable parts"
+        
+        new_thought_history = copy(thought_history)
+        new_thought_history["thought_2"] = thought
+        new_thought_history["action_2"] = action
+        
+        newstate = Dict(
+            :step => 2,
+            :thought_history => new_thought_history,
+            :reward => 2.0,
+            :isterminal => false
+        )
+        progressvalue = 5.0
+    elseif current_step == 2
+        # Step 3: Solve each sub-problem
+        thought = "Now I'll solve each sub-problem individually, using appropriate methods for each."
+        action = "Apply solution methods to each sub-problem"
+        
+        new_thought_history = copy(thought_history)
+        new_thought_history["thought_3"] = thought
+        new_thought_history["action_3"] = action
+        
+        newstate = Dict(
+            :step => 3,
+            :thought_history => new_thought_history,
+            :reward => 3.0,
+            :isterminal => false
+        )
+        progressvalue = 7.0
+    elseif current_step == 3
+        # Step 4: Combine solutions
+        thought = "Finally, I'll combine all the solutions to form the complete answer to the original problem."
+        action = "Integrate solutions and verify the answer"
+        
+        new_thought_history = copy(thought_history)
+        new_thought_history["thought_4"] = thought
+        new_thought_history["action_4"] = action
+        
+        newstate = Dict(
+            :step => 4,
+            :thought_history => new_thought_history,
+            :reward => 4.0,
+            :isterminal => true  # Reasoning complete
+        )
+        progressvalue = 10.0
+    else
+        newstate = Dict(
+            :step => current_step,
+            :thought_history => thought_history,
+            :reward => 10.0,
+            :isterminal => true
+        )
+        progressvalue = 10.0
+    end
+    
+    return Dict(
+        :newNodeKey => "reasoning_step_$current_step",
+        :newstate => newstate,
+        :progressvalue => progressvalue
+    )
+end
+
+# Initial state
+initialstate = Dict(
+    :step => 0,
+    :problem => "Explain how photosynthesis works",
+    :thought_history => Dict{String, String}(),
+    :reward => 0,
+    :isterminal => false
+)
+
+# Transition arguments
+transitionargs = (max_steps = 4,)
+
+# Run MCTS
+result = runMCTS(
+    initialstate,
+    reasoning_transition,
+    transitionargs;
+    maxiterations = 25,
+    explorationweight = 1.0,
+    maxSimulationDepth = 4,
+    horizontalSampleExpansionPhase = 3
+)
+
+# Display results
+println("Multi-step Reasoning Example")
+println("=============================")
+println()
+println("Problem: ", initialstate[:problem])
+println()
+println("Reasoning steps:")
+for (key, value) in result.bestTerminalState[:thought_history]
+    println("  $key: $value")
+end
+println()
+println("Reasoning complete! ✓")
+println("Root node visits: ", result.root.visits)
+println("Total steps in reasoning chain: ", result.bestTerminalState[:step])
+```

examples/simple_example.jl

@@ -0,0 +1,59 @@
+# Simple MCTS Example
+
+This example demonstrates basic MCTS usage with a simple state transition function.
+
+```julia
+using LLMMCTS
+
+# Define a simple state transition function
+function simple_transition(state::Dict, args::NamedTuple)
+    # In a real scenario, this would call an LLM
+    # For this example, we'll just generate deterministic next states
+    
+    current_step = get(state, :step, 0)
+    new_step = current_step + 1
+    
+    # Create new state
+    newstate = Dict(
+        :step => new_step,
+        :reward => new_step * 2,  # Simple reward function
+        :isterminal => new_step >= args.max_steps
+    )
+    
+    # LLM would provide progressvalue estimate
+    progressvalue = (new_step / args.max_steps) * 10
+    
+    return Dict(
+        :newNodeKey => "step_$(new_step)",
+        :newstate => newstate,
+        :progressvalue => progressvalue
+    )
+end
+
+# Initial state
+initialstate = Dict(
+    :step => 0,
+    :reward => 0,
+    :isterminal => false
+)
+
+# Transition arguments
+transitionargs = (max_steps = 5,)
+
+# Run MCTS
+result = runMCTS(
+    initialstate,
+    simple_transition,
+    transitionargs;
+    maxiterations = 10,
+    explorationweight = 1.0,
+    maxSimulationDepth = 3,
+    horizontalSampleExpansionPhase = 3
+)
+
+# Access results
+println("Root node visits: ", result.root.visits)
+println("Best next state: ", result.bestNextState)
+println("Best terminal state: ", result.bestTerminalState)
+println("High value states: ", result.highValueStateList)
+```

examples/tool_use.jl

@@ -0,0 +1,109 @@
+# Tool Use Example - MCTS with External Tools
+
+This example shows how MCTS can coordinate with external tools (like APIs, databases, or other services).
+
+```julia
+using LLMMCTS
+
+# Simulated tool interface
+struct Tool
+    name::String
+    description::String
+end
+
+const AVAILABLE_TOOLS = [
+    Tool("calculator", "Perform mathematical calculations"),
+    Tool("web_search", "Search the web for information"),
+    Tool("database_query", "Query a database")
+]
+
+# State tracks which tools have been used and their results
+function tool_use_transition(state::Dict, args::NamedTuple)
+    current_step = get(state, :step, 0)
+    tools_used = get(state, :tools_used, String[])
+    
+    # LLM would decide which tool to use
+    # For this example, we try tools in order
+    tool_idx = (current_step - 1) % length(AVAILABLE_TOOLS) + 1
+    
+    if tool_idx > length(AVAILABLE_TOOLS)
+        # All tools tried, return terminal state
+        newstate = Dict(
+            :step => current_step + 1,
+            :tools_used => tools_used,
+            :reward => 8.0,
+            :isterminal => true
+        )
+        return Dict(
+            :newNodeKey => "all_tools_tried",
+            :newstate => newstate,
+            :progressvalue => 8.0
+        )
+    end
+    
+    tool = AVAILABLE_TOOLS[tool_idx]
+    
+    # Simulate tool execution
+    tool_result = "Tool '$(tool.name)' executed successfully"
+    
+    # Calculate reward based on progress
+    progress = length(tools_used) / length(AVAILABLE_TOOLS)
+    reward = progress * 5
+    
+    # Progress value: LLM estimates how close we are to solving
+    progressvalue = progress * 10
+    
+    new_tools_used = vcat(tools_used, tool.name)
+    
+    newstate = Dict(
+        :step => current_step + 1,
+        :tools_used => new_tools_used,
+        :current_tool => tool.name,
+        :tool_result => tool_result,
+        :reward => reward,
+        :isterminal => false
+    )
+    
+    return Dict(
+        :newNodeKey => "tool_$(tool.name)_$current_step",
+        :newstate => newstate,
+        :progressvalue => progressvalue
+    )
+end
+
+# Initial state
+initialstate = Dict(
+    :step => 0,
+    :tools_used => String[],
+    :reward => 0,
+    :isterminal => false
+)
+
+# Transition arguments
+transitionargs = (max_tools = 3,)
+
+# Run MCTS
+result = runMCTS(
+    initialstate,
+    tool_use_transition,
+    transitionargs;
+    maxiterations = 20,
+    explorationweight = 1.2,
+    maxSimulationDepth = 4,
+    horizontalSampleExpansionPhase = 3
+)
+
+# Display results
+println("Available tools:")
+for tool in AVAILABLE_TOOLS
+    println("  - $(tool.name): $(tool.description)")
+end
+println()
+println("Best tool usage sequence:")
+for tool in result.bestTerminalState[:tools_used]
+    println("  → Used: $tool")
+end
+println()
+println("Root node visits: ", result.root.visits)
+println("High value states found: ", length(result.highValueStateList))
+```

previousVersion/0.1/Manifest.toml

@@ -1,476 +0,0 @@
-# This file is machine-generated - editing it directly is not advised
-
-julia_version = "1.10.3"
-manifest_format = "2.0"
-project_hash = "b7e1f171d36dc4812d6c1445da530f513320e6cd"
-
-[[deps.AliasTables]]
-deps = ["PtrArrays", "Random"]
-git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff"
-uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8"
-version = "1.1.3"
-
-[[deps.ArgTools]]
-uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
-version = "1.1.1"
-
-[[deps.Artifacts]]
-uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
-
-[[deps.Base64]]
-uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
-
-[[deps.Calculus]]
-deps = ["LinearAlgebra"]
-git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad"
-uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9"
-version = "0.5.1"
-
-[[deps.CodeTracking]]
-deps = ["InteractiveUtils", "UUIDs"]
-git-tree-sha1 = "c0216e792f518b39b22212127d4a84dc31e4e386"
-uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
-version = "1.3.5"
-
-[[deps.Compat]]
-deps = ["TOML", "UUIDs"]
-git-tree-sha1 = "b1c55339b7c6c350ee89f2c1604299660525b248"
-uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "4.15.0"
-weakdeps = ["Dates", "LinearAlgebra"]
-
-    [deps.Compat.extensions]
-    CompatLinearAlgebraExt = "LinearAlgebra"
-
-[[deps.CompilerSupportLibraries_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
-version = "1.1.1+0"
-
-[[deps.DataAPI]]
-git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe"
-uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
-version = "1.16.0"
-
-[[deps.DataStructures]]
-deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
-git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82"
-uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.18.20"
-
-[[deps.Dates]]
-deps = ["Printf"]
-uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
-
-[[deps.Distributed]]
-deps = ["Random", "Serialization", "Sockets"]
-uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
-
-[[deps.Distributions]]
-deps = ["AliasTables", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"]
-git-tree-sha1 = "9c405847cc7ecda2dc921ccf18b47ca150d7317e"
-uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
-version = "0.25.109"
-
-    [deps.Distributions.extensions]
-    DistributionsChainRulesCoreExt = "ChainRulesCore"
-    DistributionsDensityInterfaceExt = "DensityInterface"
-    DistributionsTestExt = "Test"
-
-    [deps.Distributions.weakdeps]
-    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-    DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d"
-    Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[[deps.DocStringExtensions]]
-deps = ["LibGit2"]
-git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d"
-uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
-version = "0.9.3"
-
-[[deps.Downloads]]
-deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
-uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
-version = "1.6.0"
-
-[[deps.DualNumbers]]
-deps = ["Calculus", "NaNMath", "SpecialFunctions"]
-git-tree-sha1 = "5837a837389fccf076445fce071c8ddaea35a566"
-uuid = "fa6b7ba4-c1ee-5f82-b5fc-ecf0adba8f74"
-version = "0.6.8"
-
-[[deps.FileWatching]]
-uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
-
-[[deps.FillArrays]]
-deps = ["LinearAlgebra"]
-git-tree-sha1 = "0653c0a2396a6da5bc4766c43041ef5fd3efbe57"
-uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
-version = "1.11.0"
-weakdeps = ["PDMats", "SparseArrays", "Statistics"]
-
-    [deps.FillArrays.extensions]
-    FillArraysPDMatsExt = "PDMats"
-    FillArraysSparseArraysExt = "SparseArrays"
-    FillArraysStatisticsExt = "Statistics"
-
-[[deps.GeneralUtils]]
-deps = ["DataStructures", "Dates", "Distributions", "JSON3", "MQTTClient", "Random", "Revise", "UUIDs"]
-path = "/appfolder/app/privatejuliapkg/GeneralUtils"
-uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-version = "0.1.0"
-
-[[deps.HypergeometricFunctions]]
-deps = ["DualNumbers", "LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"]
-git-tree-sha1 = "f218fe3736ddf977e0e772bc9a586b2383da2685"
-uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a"
-version = "0.3.23"
-
-[[deps.InteractiveUtils]]
-deps = ["Markdown"]
-uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
-
-[[deps.IrrationalConstants]]
-git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2"
-uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
-version = "0.2.2"
-
-[[deps.JLLWrappers]]
-deps = ["Artifacts", "Preferences"]
-git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca"
-uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
-version = "1.5.0"
-
-[[deps.JSON3]]
-deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"]
-git-tree-sha1 = "eb3edce0ed4fa32f75a0a11217433c31d56bd48b"
-uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
-version = "1.14.0"
-
-    [deps.JSON3.extensions]
-    JSON3ArrowExt = ["ArrowTypes"]
-
-    [deps.JSON3.weakdeps]
-    ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
-
-[[deps.JuliaInterpreter]]
-deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"]
-git-tree-sha1 = "e9648d90370e2d0317f9518c9c6e0841db54a90b"
-uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a"
-version = "0.9.31"
-
-[[deps.LibCURL]]
-deps = ["LibCURL_jll", "MozillaCACerts_jll"]
-uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
-version = "0.6.4"
-
-[[deps.LibCURL_jll]]
-deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
-uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
-version = "8.4.0+0"
-
-[[deps.LibGit2]]
-deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
-uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
-
-[[deps.LibGit2_jll]]
-deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
-uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
-version = "1.6.4+0"
-
-[[deps.LibSSH2_jll]]
-deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
-uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
-version = "1.11.0+1"
-
-[[deps.Libdl]]
-uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
-
-[[deps.LinearAlgebra]]
-deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
-uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-
-[[deps.LogExpFunctions]]
-deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"]
-git-tree-sha1 = "18144f3e9cbe9b15b070288eef858f71b291ce37"
-uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
-version = "0.3.27"
-
-    [deps.LogExpFunctions.extensions]
-    LogExpFunctionsChainRulesCoreExt = "ChainRulesCore"
-    LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables"
-    LogExpFunctionsInverseFunctionsExt = "InverseFunctions"
-
-    [deps.LogExpFunctions.weakdeps]
-    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-    ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
-    InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
-
-[[deps.Logging]]
-uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
-
-[[deps.LoweredCodeUtils]]
-deps = ["JuliaInterpreter"]
-git-tree-sha1 = "c6a36b22d2cca0e1a903f00f600991f97bf5f426"
-uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b"
-version = "2.4.6"
-
-[[deps.MQTTClient]]
-deps = ["Distributed", "Random", "Sockets"]
-git-tree-sha1 = "f2597b290d4bf17b577346153cd2ddf9accb5c26"
-uuid = "985f35cc-2c3d-4943-b8c1-f0931d5f0959"
-version = "0.3.1"
-weakdeps = ["PrecompileTools"]
-
-    [deps.MQTTClient.extensions]
-    PrecompileMQTT = "PrecompileTools"
-
-[[deps.Markdown]]
-deps = ["Base64"]
-uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
-
-[[deps.MbedTLS_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
-version = "2.28.2+1"
-
-[[deps.Missings]]
-deps = ["DataAPI"]
-git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d"
-uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
-version = "1.2.0"
-
-[[deps.Mmap]]
-uuid = "a63ad114-7e13-5084-954f-fe012c677804"
-
-[[deps.MozillaCACerts_jll]]
-uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
-version = "2023.1.10"
-
-[[deps.NaNMath]]
-deps = ["OpenLibm_jll"]
-git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4"
-uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
-version = "1.0.2"
-
-[[deps.NetworkOptions]]
-uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
-version = "1.2.0"
-
-[[deps.OpenBLAS_jll]]
-deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
-uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
-version = "0.3.23+4"
-
-[[deps.OpenLibm_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
-version = "0.8.1+2"
-
-[[deps.OpenSpecFun_jll]]
-deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
-uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
-version = "0.5.5+0"
-
-[[deps.OrderedCollections]]
-git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5"
-uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-version = "1.6.3"
-
-[[deps.PDMats]]
-deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
-git-tree-sha1 = "949347156c25054de2db3b166c52ac4728cbad65"
-uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
-version = "0.11.31"
-
-[[deps.Parsers]]
-deps = ["Dates", "PrecompileTools", "UUIDs"]
-git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821"
-uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "2.8.1"
-
-[[deps.Pkg]]
-deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
-uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-version = "1.10.0"
-
-[[deps.PrecompileTools]]
-deps = ["Preferences"]
-git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
-uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
-version = "1.2.1"
-
-[[deps.Preferences]]
-deps = ["TOML"]
-git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6"
-uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.4.3"
-
-[[deps.Printf]]
-deps = ["Unicode"]
-uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
-
-[[deps.PtrArrays]]
-git-tree-sha1 = "f011fbb92c4d401059b2212c05c0601b70f8b759"
-uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d"
-version = "1.2.0"
-
-[[deps.QuadGK]]
-deps = ["DataStructures", "LinearAlgebra"]
-git-tree-sha1 = "9b23c31e76e333e6fb4c1595ae6afa74966a729e"
-uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
-version = "2.9.4"
-
-[[deps.REPL]]
-deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
-uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
-
-[[deps.Random]]
-deps = ["SHA"]
-uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-
-[[deps.Reexport]]
-git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
-uuid = "189a3867-3050-52da-a836-e630ba90ab69"
-version = "1.2.2"
-
-[[deps.Requires]]
-deps = ["UUIDs"]
-git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
-uuid = "ae029012-a4dd-5104-9daa-d747884805df"
-version = "1.3.0"
-
-[[deps.Revise]]
-deps = ["CodeTracking", "Distributed", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "Pkg", "REPL", "Requires", "UUIDs", "Unicode"]
-git-tree-sha1 = "12aa2d7593df490c407a3bbd8b86b8b515017f3e"
-uuid = "295af30f-e4ad-537b-8983-00126c2a3abe"
-version = "3.5.14"
-
-[[deps.Rmath]]
-deps = ["Random", "Rmath_jll"]
-git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b"
-uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa"
-version = "0.7.1"
-
-[[deps.Rmath_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl"]
-git-tree-sha1 = "d483cd324ce5cf5d61b77930f0bbd6cb61927d21"
-uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
-version = "0.4.2+0"
-
-[[deps.SHA]]
-uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
-version = "0.7.0"
-
-[[deps.Serialization]]
-uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
-
-[[deps.Sockets]]
-uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
-
-[[deps.SortingAlgorithms]]
-deps = ["DataStructures"]
-git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085"
-uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
-version = "1.2.1"
-
-[[deps.SparseArrays]]
-deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"]
-uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
-version = "1.10.0"
-
-[[deps.SpecialFunctions]]
-deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
-git-tree-sha1 = "2f5d4697f21388cbe1ff299430dd169ef97d7e14"
-uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
-version = "2.4.0"
-
-    [deps.SpecialFunctions.extensions]
-    SpecialFunctionsChainRulesCoreExt = "ChainRulesCore"
-
-    [deps.SpecialFunctions.weakdeps]
-    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-
-[[deps.Statistics]]
-deps = ["LinearAlgebra", "SparseArrays"]
-uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
-version = "1.10.0"
-
-[[deps.StatsAPI]]
-deps = ["LinearAlgebra"]
-git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed"
-uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
-version = "1.7.0"
-
-[[deps.StatsBase]]
-deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
-git-tree-sha1 = "5cf7606d6cef84b543b483848d4ae08ad9832b21"
-uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
-version = "0.34.3"
-
-[[deps.StatsFuns]]
-deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"]
-git-tree-sha1 = "cef0472124fab0695b58ca35a77c6fb942fdab8a"
-uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
-version = "1.3.1"
-
-    [deps.StatsFuns.extensions]
-    StatsFunsChainRulesCoreExt = "ChainRulesCore"
-    StatsFunsInverseFunctionsExt = "InverseFunctions"
-
-    [deps.StatsFuns.weakdeps]
-    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
-    InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
-
-[[deps.StructTypes]]
-deps = ["Dates", "UUIDs"]
-git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70"
-uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
-version = "1.10.0"
-
-[[deps.SuiteSparse]]
-deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
-uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"
-
-[[deps.SuiteSparse_jll]]
-deps = ["Artifacts", "Libdl", "libblastrampoline_jll"]
-uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c"
-version = "7.2.1+1"
-
-[[deps.TOML]]
-deps = ["Dates"]
-uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
-version = "1.0.3"
-
-[[deps.Tar]]
-deps = ["ArgTools", "SHA"]
-uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
-version = "1.10.0"
-
-[[deps.UUIDs]]
-deps = ["Random", "SHA"]
-uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
-
-[[deps.Unicode]]
-uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
-
-[[deps.Zlib_jll]]
-deps = ["Libdl"]
-uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.13+1"
-
-[[deps.libblastrampoline_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
-version = "5.8.0+1"
-
-[[deps.nghttp2_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
-version = "1.52.0+1"
-
-[[deps.p7zip_jll]]
-deps = ["Artifacts", "Libdl"]
-uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
-version = "17.4.0+2"

previousVersion/0.1/Project.toml

@@ -1,8 +0,0 @@
-name = "LLMMCTS"
-uuid = "d76c5a4d-449e-4835-8cc4-dd86ec44f241"
-authors = ["narawat lamaiin <narawat@outlook.com>"]
-version = "0.1.0"
-
-[deps]
-GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"

previousVersion/0.1/src/LLMMCTS.jl

@@ -1,28 +0,0 @@
-module LLMMCTS
-
-  # export agent
-
-
-  """ Order by dependencies of each file. The 1st included file must not depend on any other
-  files and each file can only depend on the file included before it. 
-  """
-
-  include("type.jl")
-  using .type
-
-  include("util.jl")
-  using .util
-
-  include("mcts.jl")
-  using .mcts
-
-  include("interface.jl")
-  using .interface
-  
-  
-# ---------------------------------------------- 100 --------------------------------------------- #
-
-
-
-
-end # module LLMMCTS

previousVersion/0.1/src/interface.jl

@@ -1,180 +0,0 @@
-module interface
-
-export runMCTS
-
-using ..type, ..mcts
-
-
-# ---------------------------------------------- 100 --------------------------------------------- #
-
-
-
-""" Search the best action to take for a given state and task 
-
-# Arguments
-  - `a::agent`
-    one of Yiem's agents
-  - `initial state`
-    initial state
-  - `decisionMaker::Function`
-    decide what action to take
-  - `evaluator::Function`
-    assess the value of the state
-  - `reflector::Function`
-    generate lesson from trajectory and reward
-  - `isterminal::Function`
-    determine whether a given state is a terminal state
-  - `n::Integer`
-    how many times action will be sampled from decisionMaker 
-  - `w::Float64`
-    exploration weight. Value is usually between 1 to 2. 
-    Value 1.0 makes MCTS balance between exploration and exploitation like 50%-50%
-    Value 2.0 makes MCTS aggressively search the tree
-  
-# Return
-  - `plan::Vector{Dict}`
-    best plan
-
-# Example
-```jldoctest
-julia> 
-```
-
-# TODO
-  [] update docstring
-  [] return best action
-
-# Signature
-"""
-function runMCTS(
-  config::T1, 
-  initialState,
-  decisionMaker::Function, 
-  evaluator::Function, 
-  reflector::Function,
-  transition::Function,
-  ;
-  totalsample::Integer=3, 
-  maxDepth::Integer=3, 
-  maxiterations::Integer=10, 
-  explorationweight::Number=1.0,
-  ) where {T1<:AbstractDict}
-  
-  root = MCTSNode("root", initialState, 0, 0, 0, 0, false, nothing, Dict{String, MCTSNode}())
-
-  for nth in 1:maxiterations
-    node = root
-    node.visits += 1
-
-    while !isleaf(node)
-      node = UCTselect(node, explorationweight)
-    end
-    if node.isterminal
-      # MCTS arrive at the leaf node that is also a terminal state,
-      # do nothing then go directly to backpropagation
-      backpropagate(leafNode, node.reward)
-    else
-      expand(config, node, decisionMaker, evaluator, reflector, transition; 
-            totalsample=totalsample)
-      leafNode = selectChildNode(node)
-      simTrajectoryReward, terminalstate = simulate(config, leafNode, decisionMaker, evaluator, 
-                                reflector, transition; maxDepth=maxDepth, totalsample=totalsample)
-      if terminalstate !== nothing  #XXX not sure why I need this 
-        terminalstate[:totalTrajectoryReward] = simTrajectoryReward
-      end
-
-      #[] write best state to file if it has higher simTrajectoryReward. Use to improve evaluation
-      # open("trajectory.json", "w") do io
-      #   JSON3.pretty(io, terminalstate)
-      # end
-
-      backpropagate(leafNode, simTrajectoryReward)
-    end
-  end
-
-  bestNextState = selectBestNextState(root)
-  besttrajectory = selectBestTrajectory(root)
-
-  return (bestNextState.state, besttrajectory.state)
-end
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-end # module interface

previousVersion/0.1/src/mcts.jl

@@ -1,438 +0,0 @@
-module mcts
-  
-export selectBestNextState, selectBestTrajectory, backpropagate, isleaf, isroot, selectChildNode,
-        expand, simulate, makeNewState
-
-using GeneralUtils
-
-using ..type
-
-# ---------------------------------------------- 100 --------------------------------------------- #
-
-
-""" 
-
-# Arguments
-  - `node::MCTSNode`
-    node of a search tree
-
-# Return
-  - `childNode::MCTSNode`
-    the highest value child node
-
-# Example
-```jldoctest
-julia> 
-```
-
-# TODO
-  - [] update docs
-  - [x] implement the function
-
-# Signature
-"""
-function selectBestNextState(node::MCTSNode)::MCTSNode
-  highestProgressValue = 0
-  nodekey = nothing
-
-  # if all childnode has statevalue == 0, use progressvalue + reward to select the best node
-  stateValueSum = sum([v.statevalue for (k, v) in node.children])
-
-  if stateValueSum != 0
-    for (k, childnode) in node.children
-      potential = childnode.statevalue / childnode.visits
-
-      if potential > highestProgressValue
-        highestProgressValue = potential
-        nodekey = childnode.nodekey
-      end
-    end
-  else
-    for (k, childnode) in node.children
-      potential = childnode.progressvalue + childnode.reward
-
-      if potential > highestProgressValue
-        highestProgressValue = potential
-        nodekey = childnode.nodekey
-      end
-    end
-  end
-
-  return node.children[nodekey]
-end
-
-
-""" 
-
-# Arguments
-  - `node::MCTSNode`
-    node of a search tree
-
-# Return
-  - `childNode::MCTSNode`
-    the highest value child node
-
-# Example
-```jldoctest
-julia> 
-```
-
-# TODO
-  - [] update docs
-  - [x] implement the function
-
-# Signature
-"""
-function selectBestTrajectory(node::MCTSNode)::MCTSNode
-  while !isleaf(node)
-    node = selectBestNextState(node)
-  end
-
-  return node
-end
-
-
-""" Backpropagate reward along the simulation chain
-
-# Arguments
-  - `node::MCTSNode`
-    leaf node of a search tree
-  - `simTrajectoryReward::T`
-    total reward from trajectory simulation
-  
-# Return
-  - `No return`
-
-# Example
-```jldoctest
-julia> 
-```
-
-# Signature
-"""
-function backpropagate(node::MCTSNode, simTrajectoryReward::T;
-                      discountRewardCoeff::AbstractFloat=0.9) where {T<:Number}
-  while !isroot(node)
-    # Update the statistics of the current node based on the result of the playout
-    node.visits += 1
-    node.statevalue += ((node.statevalue * (node.visits-1)) + simTrajectoryReward) / node.visits
-    simTrajectoryReward *= discountRewardCoeff  # discount because future reward is uncertain
-    node = node.parent
-  end
-end
-
-
-""" Determine whether a node is a leaf node of a search tree.
-
-# Arguments
-  - `node::MCTSNode`
-    a search tree node
-# Return
-  - `result::Bool`
-    true if it is a leaf node, false otherwise.
-# Example
-```jldoctest
-julia> using Revise
-julia> using YiemAgent, DataStructures
-julia> initialState = Dict{Symbol, Any}(
-          :customerinfo=> Dict{Symbol, Any}(),
-          :storeinfo=> Dict{Symbol, Any}(),  
-
-          :thoughtHistory=> OrderedDict{Symbol, Any}(
-            :question=> "How are you?",
-            )  
-          )
-julia> statetype = typeof(initialState)
-julia> root = YiemAgent.MCTSNode(initialState, 0, 0.0, Dict{statetype, YiemAgent.MCTSNode}())
-julia> YiemAgent.isleaf(root)
-true
-```
-
-# TODO
-  [] update docs
-
-# Signature
-"""
-isleaf(node::MCTSNode)::Bool = isempty(node.children)
-
-
-""" Determine wheter a given node is a root node
-
-# Arguments
-  - `node::MCTSNode`
-    node of a search tree
-
-# Return
-  - `isrootnode::Bool`
-    true if the given node is root node, false otherwise
-
-# Example
-```jldoctest
-julia> 
-```
-
-# Signature
-"""
-isroot(node::MCTSNode)::Bool = node.nodekey == "root" ? true : false
-
-
-
-""" Select child node based on the highest statevalue
-
-# Arguments
-  - `node::MCTSNode`
-    node of a search tree
-
-# Return
-  - `childNode::MCTSNode`
-    the highest value child node
-
-# Example
-```jldoctest
-julia> 
-```
-
-# Signature
-"""
-function selectChildNode(node::MCTSNode)::MCTSNode
-  highestProgressValue = 0
-  nodekey = nothing
-
-  # loop thought node children dictionary to find the highest progress value
-  for (k, childNode) in node.children
-    potential = childNode.progressvalue + childNode.reward
-    if childNode.reward > 0 #XXX for testing. remove when done.
-      println("")
-    end
-    if potential > highestProgressValue
-      highestProgressValue = potential
-      nodekey = childNode.nodekey
-    end
-  end
-
-  return node.children[nodekey]
-end
-
-
-""" Expand selected node 
-
-# Arguments
-  - `a::T1`
-    One of YiemAgent's agent
-  - `node::MCTSNode`
-    MCTS node
-  - `state::T2`
-    a state of a game. Can be a Dict or something else.
-  - `decisionMaker::Function`
-    a function that output Thought and Action
-  - `evaluator::Function`
-    a function that output trajectory progress score
-  
-# Return
-
-# Example
-```jldoctest
-julia> 
-```
-
-# TODO
-  [] update docstring
-  [] try loop should limit to 3 times. if not succeed, skip
-  [] newNodeKey ∉ keys(node.children). New state may have semantic vector close enought to one of existing child state. Which can be assume that they are the same state semantically-wise. 
-  [x] store feedback -> state -> agent.
-
-
-# Signature
-"""
-function expand(config::T1, node::MCTSNode, decisionMaker::Function, evaluator::Function,
-                reflector::Function, transition::Function; totalsample::Integer=3
-                ) where {T1<:AbstractDict}
-
-  nthSample = 0
-  while true
-    nthSample += 1
-    if nthSample <= totalsample
-      newNodeKey, newstate, progressvalue = transition(config, node.state, decisionMaker, 
-                                                      evaluator, reflector)
-      if newNodeKey ∉ keys(node.children)
-        node.children[newNodeKey] = 
-          MCTSNode(newNodeKey, newstate, 0, progressvalue, 0, newstate[:reward], 
-            newstate[:isterminal], node, Dict{String, MCTSNode}())
-      end
-    else
-      break
-    end
-  end
-end
-
-
-""" Simulate interactions between agent and environment
-
-# Arguments
-  - `a::T`
-    one of YiemAgent's agent
-  - `node::MCTSNode`
-    node that will be a simulation starting point.
-  - `decisionMaker::Function`
-    function that receive state return Thought and Action
-  
-# Return
-  - `simTrajectoryReward::Number`
-
-# Example
-```jldoctest
-julia> 
-```
-
-# TODO
-  - [] update docs
-
-# Signature
-"""
-function simulate(config::T, node::MCTSNode, decisionMaker::Function, evaluator::Function, 
-  reflector::Function, transition::Function; maxDepth::Integer=3, totalsample::Integer=3
-  )::Union{Tuple{Number, Dict{Symbol, <:Any}}, Tuple{Number, Nothing}} where {T<:AbstractDict}
-
-  simTrajectoryReward = 0.0
-  terminalstate = nothing
-  
-  for depth in 1:maxDepth
-    simTrajectoryReward += node.reward
-    if node.isterminal
-      terminalstate = node.state
-      break
-    else
-      expand(config, node, decisionMaker, evaluator, reflector, transition; 
-            totalsample=totalsample)
-      node = selectChildNode(node)
-    end
-  end
-
-  return (simTrajectoryReward, terminalstate)
-end
-
-
-"""
-
-# Arguments
-  
-# Return
-
-# Example
-```jldoctest
-julia> 
-```
-
-# TODO
-  - [] update docstring
-  - [x] implement the function
-
-# Signature
-"""
-function makeNewState(currentstate::T1, thoughtDict::T4, response::T2, select::Union{T3, Nothing}, 
-  reward::T3, isterminal::Bool
-  )::Tuple{String, Dict{Symbol, <:Any}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict}
-
-  currentstate_latestThoughtKey, currentstate_latestThoughtIndice = 
-      GeneralUtils.findHighestIndexKey(currentstate[:thoughtHistory], "thought")
-  currentstate_nextIndice = 
-              currentstate_latestThoughtKey == :NA ? 1 : currentstate_latestThoughtIndice + 1
-  currentstate_latestThoughtKey = Symbol("thought_$currentstate_nextIndice")
-  latestActionKey = Symbol("action_$currentstate_nextIndice")
-
-  _, thoughtDict_latestThoughtIndice = 
-      GeneralUtils.findHighestIndexKey(thoughtDict, "thought")
-  
-  thoughtDict_latestThoughtKey, thoughtDict_latestActionKey =
-  if thoughtDict_latestThoughtIndice == -1
-    (:thought, :action)
-  else
-    (
-      Symbol("thought_$thoughtDict_latestThoughtIndice"), 
-      Symbol("action_$thoughtDict_latestThoughtIndice"),
-    )
-  end
-
-  # add Thought, action, observation to thoughtHistory
-  newstate = deepcopy(currentstate)
-  newstate[:thoughtHistory][currentstate_latestThoughtKey] = 
-      thoughtDict[thoughtDict_latestThoughtKey]
-  newstate[:thoughtHistory][latestActionKey] = thoughtDict[thoughtDict_latestActionKey]
-  newObservationKey = Symbol("observation_$(currentstate_nextIndice)")
-  newstate[:thoughtHistory][newObservationKey] = response
-  newstate[:reward] = reward
-  newstate[:select] = select
-  newstate[:isterminal] = isterminal
-
-  newNodeKey = GeneralUtils.uuid4snakecase()
-
-  return (newNodeKey, newstate)
-end
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-end # module mcts

previousVersion/0.1/src/type.jl

@@ -1,116 +0,0 @@
-module type
-  
-export MCTSNode
-
-# ---------------------------------------------- 100 --------------------------------------------- #
-
-
-""" a node for MCTS search tree
-
-# Arguments
-  - `state::T`
-    a state of a game. Can be a Dict or something else. 
-  - `visits::Integer `
-    number of time the game visits this state
-  - `stateValue::Float64`
-    state value
-  - `children::Dict{T, MCTSNode}`
-    children node
-
-# Return
-  - `nothing`
-# Example
-```jldoctest
-julia> state = Dict(
-        :info=> Dict(),           # keyword info
-        :thoughtHistory=> Dict(
-          :question=> _,
-          :thought_1=> _, 
-          :action_1=> _, 
-          :observation_1=> _, 
-          :thought_2=> _,
-          ...
-          )  
-        )
-```
-
-# TODO
-  [] update docstring
-
-# Signature
-"""
-mutable struct MCTSNode{T1<:AbstractDict, T2<:AbstractString}
-  nodekey::T2
-  state::T1
-  visits::Integer
-  progressvalue::Number # estimate value by LLM's reasoning
-  statevalue::Number  # store discounted commulative reward (gather from its child node)
-  reward::Number  # this node's own reward
-  isterminal::Bool
-  parent::Union{MCTSNode, Nothing} 
-  children::Dict{String, MCTSNode}
-end
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-end # module type

previousVersion/0.1/src/util.jl

@@ -1,139 +0,0 @@
-module util
-
-export UCTselect
-
-using ..type
-
-# ---------------------------------------------- 100 --------------------------------------------- #
-
-""" Select a node based on UCT score
-
-# Arguments
-  - `node::MCTSNode`
-    mcts node
-  - `w::T`
-    exploration weight. Value is usually between 1 to 2. 
-    Value 1.0 makes MCTS balance between exploration and exploitation like 50%-50%.
-    Value 2.0 makes MCTS aggressively search the tree.
-# Return
-  - `selectedNode::MCTSNode`
-
-# Example
-```jldoctest
-julia> 
-```
-
-# Signature
-"""
-function UCTselect(node::MCTSNode, w::T)::MCTSNode where {T<:AbstractFloat}
-  maxUCT = -Inf
-  selectedNode = nothing
-
-  for (childState, childNode) in node.children
-    UCTvalue =
-    if childNode.visits != 0  
-      weightedterm = w * sqrt(log(node.visits) / childNode.visits) # explore term
-      childNode.statevalue + weightedterm
-    else  # node.visits == 0 makes sqrt() in explore term error
-      childNode.progressvalue  # exploit term
-    end
-                
-    if UCTvalue > maxUCT
-      maxUCT = UCTvalue
-        selectedNode = childNode
-    end
-  end
-
-  return selectedNode
-end
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-end # module util

src/LLMMCTS.jl

@@ -1,6 +1,6 @@
 module LLMMCTS
 
-  # export agent
+  export MCTSNode
 
 
   """ Order by dependencies of each file. The 1st included file must not depend on any other
@@ -22,7 +22,7 @@ module LLMMCTS
   
 # ---------------------------------------------- 100 --------------------------------------------- #
 
-""" version 0.0.2
+""" version 0.1.2
     Todo:
         - [] 
 

src/interface.jl

@@ -2,153 +2,255 @@ module interface
 
 export runMCTS
 
+using Base.Threads, PrettyPrinting
 using ..type, ..mcts, ..util
 
 
 # ---------------------------------------------- 100 --------------------------------------------- #
 
 
+""" Search for the best action to take for a given state and task.
 
-""" Search the best action to take for a given state and task 
+This function runs the MCTS algorithm through multiple iterations of expansion,
+simulation, and backpropagation to find optimal decisions.
+
+Does **not** mutate the input state; it creates new MCTS nodes during search.
 
 # Arguments
-  - `initialstate::T`
-    initial state
-  - `transition::Function`
-    a function that define how the state transitions
-  - `transitionargs::NamedTuple`
-    arguments for transition function
+- `initialstate::T`
+  The initial state from which to start the search.
+- `transition::Function`
+  A function that defines how the state transitions.
+- `transitionargs::NamedTuple`
+  Arguments passed to the transition function.
 
 # Keyword Arguments
-  - `totalsample::Integer`
-    a number of child state MCTS sample at each node during expansion phase
-  - `maxdepth::Integer`
-    a number of levels MCTS goes during simulation phase
-  - `maxiterations::Integer`
-    a number of iteration MCTS goes thru expansion -> simulation -> backpropagation cycle
-  - `explorationweight::Number`
-    exploration weight controls how much MCTS should explore new state instead of exploit 
-    a known state. 1.0 balance between exploration and exploitation like 50%-50%. 2.0 makes MCTS
-    aggressively explore new state.
+- `horizontalSampleExpansionPhase::Integer=3`
+  Number of child states sampled at each node during expansion phase.
+- `horizontalSampleSimulationPhase::Integer=3`
+  Number of child states sampled at each node during simulation's expansion phase.
+- `maxSimulationDepth::Integer=3`
+  Maximum depth MCTS goes during simulation phase.
+- `maxiterations::Integer=10`
+  Number of iterations MCTS performs through expansion → simulation → backpropagation cycles.
+- `explorationweight::Number=1.0`
+  Exploration weight controls how much MCTS explores new states versus exploiting known states.
+  A value of 1.0 balances exploration and exploitation equally. Higher values (e.g., 2.0) 
+  encourage more aggressive exploration.
+- `earlystop::Union{Function,Nothing}=nothing`
+  Optional function to check early stopping condition. If satisfied, MCTS breaks iterations.
+- `saveSimulatedNode::Bool=false`
+  Whether to save nodes created during simulation phase.
+- `multithread::Bool=false`
+  Whether to use multithreading during simulation.
 
 # Return
-  - `NamedTuple{(:bestNextState, :bestFinalState), Tuple{T, T}}`
-    the best next state and the best final state
+- `NamedTuple{(:root, :bestNextState, :bestTerminalState, :highValueStateList),
+             Tuple{MCTSNode,T,T,Vector{Dict{String,Any}}}}`
+  - `root`: the complete MCTS tree with root node
+  - `bestNextState`: the best immediate next state
+  - `bestTerminalState`: the best final state along the best trajectory
+  - `highValueStateList`: list of high-value terminal states (reward >= 8)
 
 # Example
-    Refers to SQLLLM package
-
-# Signature
+```jldoctest
+julia> using LLMMCTS
+julia> initialState = Dict(:reward=>0.0)
+julia> result = runMCTS(initialState, transition_func, transition_args; maxiterations=5)
+```
 """
 function runMCTS(
   initialstate::T,
   transition::Function,
   transitionargs::NamedTuple,
   ;
-  totalsample::Integer=3,
-  maxdepth::Integer=3,
+  horizontalSampleExpansionPhase::Integer=3,
+  horizontalSampleSimulationPhase::Integer=3,
+  maxSimulationDepth::Integer=3,
   maxiterations::Integer=10,
   explorationweight::Number=1.0,
-  earlystop::Union{Function,Nothing}=nothing
-)::NamedTuple{(:bestNextState, :bestFinalState),Tuple{T,T}} where {T<:Any}
+  earlystop::Union{Function,Nothing}=nothing,
+  saveSimulatedNode::Bool=false,
+  multithread=false,
+  )::NamedTuple{(:root, :bestNextState, :bestTerminalState, :highValueStateList),
+                Tuple{MCTSNode,T,T,Vector{Dict{String,Any}}}} where {T<:Any}
+  println("--> LLMMCTS runMCTS 1")
+  # Initialize the MCTS tree with a root node representing the initial state
+  # root.visits=0: no visits yet
+  # root.statevalue=0: no simulation results yet
+  root = MCTSNode("root", initialstate, 0, 0, 0, 0, false, nothing, Dict{String,MCTSNode}(),
+                  Dict{String,Any}())
 
-  root = MCTSNode("root", initialstate, 0, 0, 0, 0, false, nothing, Dict{String,MCTSNode}())
+  # Channel to collect high-value terminal states (reward >= 8)
+  # These are "good solutions" that can be returned to the user
+  highValueState = Channel{Any}(100)
 
+  # Main MCTS loop: perform iterations to build the search tree
+  # Each iteration: SELECTION → EXPANSION → SIMULATION → BACKPROPAGATION
   for nth in 1:maxiterations
+    # Start from root and traverse down using UCT selection
     node = root
-    node.visits += 1
-
+    node.visits += 1  # Count this iteration's visit to root
+    println("--> LLMMCTS runMCTS 2")
+    # Phase 1: SELECTION - Traverse tree using UCT until reaching a leaf node
+    # UCT balances exploration (new branches) vs exploitation (promising branches)
     while !isleaf(node)
+      println("--> LLMMCTS runMCTS 3")
       node = UCTselect(node, explorationweight)
     end
-
+    println("--> LLMMCTS runMCTS 4")
+    # Phase 2: TERMINAL CHECK - If leaf is terminal, just backpropagate
     if node.isterminal
-      # MCTS arrive at the leaf node that is also a terminal state,
-      # do nothing then go directly to backpropagation. It means the end of this iteration
+      println("--> LLMMCTS runMCTS 5")
+      # If this terminal state has high reward (>= 8), store it for later
+      if node.state[:reward] >= 8
+        println("--> LLMMCTS runMCTS 6")
+        put!(highValueState, deepcopy(node.state))
+      end
+      println("--> LLMMCTS runMCTS 7")
+      # Backpropagate the terminal node's own reward up to root
+      # This updates all ancestors with this path's outcome
       backpropagate(node, node.reward)
     else
-      expand(node, transition, transitionargs;
-        totalsample=totalsample)
-      leafNode = selectChildNode(node)
-      simTrajectoryReward, terminalstate = simulate(leafNode, transition, transitionargs;
-        maxdepth=maxdepth, totalsample=totalsample)
-      # if terminalstate !== nothing  #XXX not sure why I need this 
-      #   terminalstate[:totalTrajectoryReward] = simTrajectoryReward
-      # end
-
-      #[] write best state to file if it has higher simTrajectoryReward. Use to improve evaluation
-      # open("trajectory.json", "w") do io
-      #   JSON3.pretty(io, terminalstate)
-      # end
-
-      backpropagate(leafNode, simTrajectoryReward)
+      println("--> LLMMCTS runMCTS 8")
+      # Phase 3: EXPANSION - Generate children for this non-terminal leaf
+      # Horizontal sampling: create multiple child nodes via LLM transition
+      _ = expand(node, transition, transitionargs;
+                horizontalSample=horizontalSampleExpansionPhase,
+                multithread=multithread)
+      println("--> LLMMCTS runMCTS 9")
+      # Phase 4: SIMULATION + BACKPROPAGATION
+      # For each newly expanded child, run simulation and update statistics
+      if multithread
+        println("--> LLMMCTS runMCTS 10")
+        # Parallel simulation: spawn threads for each child node
+        @sync for (leafNodeKey, leafNode) in node.children
+          @spawn simulateThenBackpropagate(leafNode, transition, transitionargs;
+                  maxSimulationDepth=maxSimulationDepth, 
+                  horizontalSampleSimulationPhase=horizontalSampleSimulationPhase,
+                  saveSimulatedNode=saveSimulatedNode,
+                  multithread=multithread,
+                  highValueState=highValueState,
+                  )
+        end
+      else
+        println("--> LLMMCTS runMCTS 11")
+        # Sequential simulation: process each child one at a time
+        for (leafNodeKey, leafNode) in node.children
+          println("--> LLMMCTS runMCTS 11-1")
+          simulateThenBackpropagate(leafNode, transition, transitionargs;
+            maxSimulationDepth=maxSimulationDepth, 
+            horizontalSampleSimulationPhase=horizontalSampleSimulationPhase,
+            saveSimulatedNode=saveSimulatedNode,
+            multithread=multithread,
+            highValueState=highValueState)
+        end
+      end
     end
-
-    # stop if the early stop condition is met
+    println("--> LLMMCTS runMCTS 12")
+    # Phase 5: EARLY STOP CHECK
+    # Optional: stop search early if a condition is met
     if typeof(earlystop) <: Function && earlystop(node.state)
+      println("--> LLMMCTS runMCTS 13")
       break
     end
   end
-
+  println("--> LLMMCTS runMCTS 14")
+  # After all iterations, extract results from the search tree
+  # Select best immediate next state (best child of root)
   bestNextState = selectBestNextNode(root)
-  besttrajectory = selectBestTrajectoryNode(root)
+  println("--> LLMMCTS runMCTS 15")
+  # Select best terminal state along the optimal trajectory
+  bestTerminalState = selectBestTrajectoryNode(root)
 
-  return (bestNextState=bestNextState.state, bestFinalState=besttrajectory.state)
+  # Collect all high-value states from the channel into a list
+  highValueStateList = Vector{Dict{String, Any}}()
+  while !isempty(highValueState)
+    println("--> LLMMCTS runMCTS 16")
+    push!(highValueStateList, take!(highValueState))
+  end
+  println("--> LLMMCTS runMCTS 17")
+  # Return complete search results
+  result = (
+            root=root, 
+            bestNextState=bestNextState.state, 
+            bestTerminalState=bestTerminalState.state,
+            highValueStateList=highValueStateList
+            )
+
+  return result
 end
 
+""" Run simulation from a given node and backpropagate the reward.
 
-# function runMCTS(
-#   initialstate::T,
-#   transition::Function,
-#   transitionargs::NamedTuple,
-#   ;
-#   totalsample::Integer=3,
-#   maxdepth::Integer=3,
-#   maxiterations::Integer=10,
-#   explorationweight::Number=1.0,
-# )::NamedTuple{(:bestNextState, :bestFinalState),Tuple{T,T}} where {T<:Any}
+This function performs simulation (rollout) from the given node, collects the
+cumulative reward along the trajectory, and backpropagates it up the tree to update
+visit counts and state values.
 
-#   root = MCTSNode("root", initialstate, 0, 0, 0, 0, false, nothing, Dict{String,MCTSNode}())
-
-#   for nth in 1:maxiterations
-#     node = root
-#     node.visits += 1
-
-#     while !isleaf(node)
-#       node = UCTselect(node, explorationweight)
-#     end
-#     if node.isterminal
-#       # MCTS arrive at the leaf node that is also a terminal state,
-#       # do nothing then go directly to backpropagation. It means the end of this iteration
-#       backpropagate(leafNode, node.reward)
-#     else
-#       expand(node, transition, transitionargs;
-#         totalsample=totalsample)
-#       leafNode = selectChildNode(node)
-#       simTrajectoryReward, terminalstate = simulate(leafNode, transition, transitionargs;
-#         maxdepth=maxdepth, totalsample=totalsample)
-#       # if terminalstate !== nothing  #XXX not sure why I need this 
-#       #   terminalstate[:totalTrajectoryReward] = simTrajectoryReward
-#       # end
-
-#       #[] write best state to file if it has higher simTrajectoryReward. Use to improve evaluation
-#       # open("trajectory.json", "w") do io
-#       #   JSON3.pretty(io, terminalstate)
-#       # end
-
-#       backpropagate(leafNode, simTrajectoryReward)
-#     end
-#   end
-
-#   bestNextState = selectBestNextNode(root)
-#   besttrajectory = selectBestTrajectoryNode(root)
-
-#   return (bestNextState=bestNextState.state, bestFinalState=besttrajectory.state)
-# end
+Does **not** mutate the input node's children (unless `saveSimulatedNode=true`).
 
+# Arguments
+- `node::MCTSNode`
+  The current node to simulate from.
+- `transition::Function`
+  A function that defines how the state transitions.
+- `transitionargs::NamedTuple`
+  Arguments passed to the transition function.
 
+# Keyword Arguments
+- `maxSimulationDepth::Integer=3`
+  Maximum depth MCTS goes during simulation phase.
+- `horizontalSampleSimulationPhase::Integer=3`
+  Number of child states sampled at each node during simulation phase.
+- `saveSimulatedNode::Bool=false`
+  Whether to save nodes created during simulation phase. If false, children are
+  cleared after simulation.
+- `multithread::Bool=false`
+  Whether to use multithreading during simulation.
 
+# Return
+- `Nothing`
 
+# Signature
+"""
+function simulateThenBackpropagate(node::MCTSNode, transition::Function, transitionargs::NamedTuple; 
+                maxSimulationDepth::Integer=3, horizontalSampleSimulationPhase::Integer=3,
+                saveSimulatedNode::Bool=false,
+                multithread=false,
+                highValueState=Union{Nothing,Any}=nothing)
+  println("--> LLMMCTS simulateThenBackpropagate 1")
+  # Phase 1: RUN SIMULATION (rollout)
+  # Perform a rollout from this node, accumulating rewards along the way
+  simTrajectoryReward, terminalstate = 
+    simulate(node, transition, transitionargs;
+            maxSimulationDepth=maxSimulationDepth, 
+            horizontalSample=horizontalSampleSimulationPhase,
+            multithread=multithread)
+  println("--> LLMMCTS simulateThenBackpropagate 2")
+  # Phase 2: HIGH-VALUE STATE TRACKING
+  # If we reached a terminal state with high reward (>= 8), store it
+  # This allows users to access multiple good solutions, not just the best one
+  if highValueState !== nothing && 
+    terminalstate !== nothing &&
+    terminalstate["reward"] >= 8
+    println("--> LLMMCTS simulateThenBackpropagate 3")
+    put!(highValueState, deepcopy(terminalstate))
+  end
+  println("--> LLMMCTS simulateThenBackpropagate 4")
+  # Phase 3: BACKPROPAGATE
+  # Update statistics (visits, statevalue) for all ancestors up to root
+  # The simulation result is now incorporated into the tree
+  backpropagate(node, simTrajectoryReward)
+  println("--> LLMMCTS simulateThenBackpropagate 5")
+  # Phase 4: MEMORY MANAGEMENT
+  # Clear children unless user wants to keep them for analysis
+  # This frees memory for the next iteration while preserving tree structure
+  if saveSimulatedNode == false
+    println("--> LLMMCTS simulateThenBackpropagate 6")
+    node.children = Dict{String, MCTSNode}()
+  end
+  println("--> LLMMCTS simulateThenBackpropagate 7")
+end
 
 
 

src/mcts.jl

@@ -1,7 +1,7 @@
 module mcts
   
 export selectBestNextNode, selectBestTrajectoryNode, backpropagate, isleaf, isroot, selectChildNode,
-        expand, simulate, makeNewState
+        expand, simulate
 using Base.Threads
 using GeneralUtils
 
@@ -10,27 +10,33 @@ using ..type
 # ---------------------------------------------- 100 --------------------------------------------- #
 
 
-""" 
+""" Select the best child node based on the highest value metric.
+
+The selection metric depends on the node's state values:
+- If the sum of statevalues is non-zero, uses `statevalue/visits` ratio.
+- Otherwise, uses `progressvalue + reward`.
 
 # Arguments
-  - `node::MCTSNode`
-    node of a search tree
+- `node::MCTSNode`
+  The node whose children will be evaluated.
 
 # Return
-  - `childNode::MCTSNode`
-    the highest value child node
-
-# Signature
+- `childNode::MCTSNode`
+  The child node with the highest value according to the selection metric.
 """
 function selectBestNextNode(node::MCTSNode)::MCTSNode
   highestProgressValue = -1
   nodekey = nothing
 
-  # if all childnode has statevalue == 0, use progressvalue + reward to select the best node
+  # Calculate sum of statevalues across all child nodes
+  # This determines whether to use statevalue/visits (exploitation) or progressvalue+reward (exploration)
   stateValueSum = sum([v.statevalue for (k, v) in node.children])
 
+  # If any nodes have non-zero statevalue, use statevalue/visits as selection metric
+  # This means simulations have confirmed node values - use exploitation
   if stateValueSum != 0
     for (k, childnode) in node.children
+      # Calculate average statevalue per visit (running average from simulations)
       potential = childnode.statevalue / childnode.visits
 
       if potential > highestProgressValue
@@ -39,6 +45,8 @@ function selectBestNextNode(node::MCTSNode)::MCTSNode
       end
     end
   else
+    # No simulations yet - use progressvalue + reward for initial guidance
+    # This allows LLM heuristics to guide early search before simulations provide data
     for (k, childnode) in node.children
       potential = childnode.progressvalue + childnode.reward
 
@@ -53,19 +61,22 @@ function selectBestNextNode(node::MCTSNode)::MCTSNode
 end
 
 
-""" 
+""" Select the best node along the optimal trajectory.
+
+Traverses down the tree from the given node by repeatedly applying `selectBestNextNode`
+until reaching a leaf node, returning the highest-value node found along the path.
 
 # Arguments
-  - `node::MCTSNode`
-    node of a search tree
+- `node::MCTSNode`
+  The node to start trajectory selection from.
 
 # Return
-  - `childNode::MCTSNode`
-    the highest value child node
-
-# Signature
+- `childNode::MCTSNode`
+  The highest-value node found by following the optimal trajectory to a leaf.
 """
 function selectBestTrajectoryNode(node::MCTSNode)::MCTSNode
+  # Follow the optimal path down the tree by repeatedly selecting the best child
+  # This gives us the highest-value trajectory from the starting node to a leaf
   while !isleaf(node)
     node = selectBestNextNode(node)
   end
@@ -74,101 +85,108 @@ function selectBestTrajectoryNode(node::MCTSNode)::MCTSNode
 end
 
 
-""" Backpropagate reward along the simulation chain
+""" Backpropagate reward along the simulation chain.
+
+Updates visit counts and state values for all nodes along the path from the given
+leaf node to the root, applying reward discounting for future rewards.
+
+**Modifies nodes in place.**
 
 # Arguments
-  - `node::MCTSNode`
-    leaf node of a search tree
-  - `simTrajectoryReward::T`
-    total reward from trajectory simulation
-  - `discountRewardCoeff::AbstractFloat`
-    A discount reward coefficient to reduce future reward. The futher in the future the lower
-    reward it is now.
-  
-# Return
-  - `None`
+- `node::MCTSNode`
+  The leaf node from which to start backpropagation.
+- `simTrajectoryReward::Number`
+  The total reward from the trajectory simulation.
 
-# Signature
+# Keyword Arguments
+- `discountRewardCoeff::AbstractFloat=0.9`
+  Discount coefficient applied to future rewards. Larger distances from the leaf
+  receive progressively lower discounted rewards.
+
+# Return
+- `Nothing`
 """
 function backpropagate(node::MCTSNode, simTrajectoryReward::T;
-                      discountRewardCoeff::AbstractFloat=0.9) where {T<:Number}
+                       discountRewardCoeff::AbstractFloat=0.9) where {T<:Number}
+  println("--> LLMMCTS backpropagate 1")
+  # Propagate the simulation result back up the tree to update all ancestor nodes
+  # Each node's statistics are updated with the cumulative reward from the simulation
   while !isroot(node)
-    # Update the statistics of the current node based on the result of the playout
+    println("--> LLMMCTS backpropagate 2")
+    # Increment visit count - this simulation passed through this node
     node.visits += 1
-    node.statevalue += ((node.statevalue * (node.visits-1)) + simTrajectoryReward) / node.visits
-    simTrajectoryReward *= discountRewardCoeff  # discount because future reward is uncertain
+    println("--> LLMMCTS backpropagate 3")
+    node.statevalue += ((node.statevalue * (node.visits-1)) + simTrajectoryReward) / node.visits  # Update running average of state value
+    
+    # Apply discount to future rewards - rewards further from the current state are worth less
+    # This reflects temporal uncertainty: distant future rewards are less certain
+    simTrajectoryReward *= discountRewardCoeff
+    
+    # Move up to parent node to continue propagation
     node = node.parent
   end
+  println("--> LLMMCTS backpropagate 4")
 end
 
+""" Determine whether a node is a leaf node.
 
-""" Determine whether a node is a leaf node of a search tree.
+A leaf node has no children.
 
 # Arguments
-  - `node::MCTSNode`
-    a search tree node
+- `node::MCTSNode`
+  The search tree node to check.
+
 # Return
-  - `result::Bool`
-    true if it is a leaf node, false otherwise.
+- `result::Bool`
+  `true` if the node has no children, `false` otherwise.
+
 # Example
 ```jldoctest
-julia> using Revise
-julia> using YiemAgent, DataStructures
-julia> initialState = Dict{Symbol, Any}(
-          :customerinfo=> Dict{Symbol, Any}(),
-          :storeinfo=> Dict{Symbol, Any}(),  
-
-          :thoughtHistory=> OrderedDict{Symbol, Any}(
-            :question=> "How are you?",
-            )  
-          )
-julia> statetype = typeof(initialState)
-julia> root = YiemAgent.MCTSNode(initialState, 0, 0.0, Dict{statetype, YiemAgent.MCTSNode}())
-julia> YiemAgent.isleaf(root)
+julia> using LLMMCTS
+julia> node = MCTSNode("leaf", Dict(:reward=>1.0), 0, 0, 0, 1.0, true, nothing, Dict(), Dict())
+julia> isleaf(node)
 true
 ```
-
-# TODO
-  [] update docs
-
-# Signature
 """
 isleaf(node::MCTSNode)::Bool = isempty(node.children)
 
+""" Determine whether a given node is a root node.
 
-""" Determine wheter a given node is a root node
+The root node is identified by having `"root"` as its `nodekey`.
 
 # Arguments
-  - `node::MCTSNode`
-    node of a search tree
+- `node::MCTSNode`
+  The search tree node to check.
 
 # Return
-  - `isrootnode::Bool`
-    true if the given node is root node, false otherwise
-
-# Signature
+- `isrootnode::Bool`
+  `true` if the node is the root node, `false` otherwise.
 """
 isroot(node::MCTSNode)::Bool = node.nodekey == "root" ? true : false
 
 
 
-""" Select child node based on the highest statevalue
+""" Select the child node with the highest value.
+
+Uses `progressvalue + reward` as the selection metric.
 
 # Arguments
-  - `node::MCTSNode`
-    node of a search tree
+- `node::MCTSNode`
+  The node whose children will be evaluated.
 
 # Return
-  - `childNode::MCTSNode`
-    the highest value child node
-
-# Signature
+- `childNode::MCTSNode`
+  The child node with the highest `progressvalue + reward` value.
 """
 function selectChildNode(node::MCTSNode)::MCTSNode
   highestProgressValue = -1
   nodekey = nothing
 
-  # loop thought node children dictionary to find the highest progress value
+  # During simulation rollout, we need to pick which child to explore next
+  # Use progressvalue + reward as the selection metric (no UCT here)
+  # - progressvalue: LLM's estimate of how promising this state is
+  # - reward: immediate environment feedback
+  # Together they guide fast exploration during simulation
   for (k, childNode) in node.children
     potential = childNode.progressvalue + childNode.reward
     if potential > highestProgressValue
@@ -181,186 +199,234 @@ function selectChildNode(node::MCTSNode)::MCTSNode
 end
 
 
-""" Expand selected node.
+""" Expand a node by generating new child nodes.
+
+Creates new child nodes by applying the transition function multiple times
+(horizontally samples) from the current node.
 
 # Arguments
-  - `node::MCTSNode`
-    MCTS node
-  - `transition::Function`
-    A function that handles state transition.
-  - `transitionargs::NamedTuple`
-    Arguments for transition()
-  - `totalsample::Integer`
-    Total number to sample from the current node (i.e. expand new node horizontally)
-  
+- `node::MCTSNode`
+  The MCTS node to expand.
+- `transition::Function`
+  A function that handles state transition.
+- `transitionargs::NamedTuple`
+  Arguments passed to the transition function.
+
+# Keyword Arguments
+- `horizontalSample::Integer=3`
+  Number of child nodes to generate.
+- `multithread::Bool=false`
+  Whether to run expansion in parallel using multiple threads.
+
 # Return
-  - None
-
-# Signature
+- `Nothing`
 """
-# function expand(node::MCTSNode, transition::Function, transitionargs::NamedTuple; 
-#           totalsample::Integer=3)
-  
-#   # not use Any[] because I want to preserve result order
-#   results = Vector{Any}(undef, totalsample) 
-
-#   @sync for i in 1:totalsample
-#       @spawn begin
-#         result = transition(deepcopy(node.state), deepcopy(transitionargs))
-#         results[i] = result
-#       end
-#   end
-  
-#   for result in results
-#     newNodeKey::AbstractString = result[:newNodeKey]
-#     newstate::AbstractDict = result[:newstate]
-#     progressvalue::Integer = result[:progressvalue]
-
-#     """
-#     [] newNodeKey ∉ keys(node.children). 
-#           New state may have semantic vector close enought to 
-#         one of existing child state. Which can be assume that they are the same state 
-#         semantically-wise i.e. De javu. This could be used to recall lessons for this 
-#         similar situation to improve decisionMaker and evaluator. 
-#     """
-#     if newNodeKey ∉ keys(node.children)
-#       node.children[newNodeKey] = 
-#         MCTSNode(newNodeKey, newstate, 0, progressvalue, 0, newstate[:reward], 
-#           newstate[:isterminal], node, Dict{String, MCTSNode}())
-#     end
-#   end
-# end
 function expand(node::MCTSNode,transition::Function, transitionargs::NamedTuple; 
-          totalsample::Integer=3)
-
-  nthSample = 0
-  while true
-    nthSample += 1
-    if nthSample <= totalsample
-      result = transition(node.state, transitionargs)
-      newNodeKey::AbstractString = result[:newNodeKey]
-      newstate::AbstractDict = result[:newstate]
-      progressvalue::Integer = result[:progressvalue]
-
-      """
-      [] newNodeKey ∉ keys(node.children). 
-            New state may have semantic vector close enought to 
-          one of existing child state. Which can be assume that they are the same state 
-          semantically-wise i.e. De javu. This could be used to recall lessons for this 
-          similar situation to improve decisionMaker and evaluator. 
-      """
-      if newNodeKey ∉ keys(node.children)
-        node.children[newNodeKey] = 
-          MCTSNode(newNodeKey, newstate, 0, progressvalue, 0, newstate[:reward], 
-            newstate[:isterminal], node, Dict{String, MCTSNode}())
-      end
-    else
-      break
+                horizontalSample::Integer=3, multithread=false)
+  # Generate child nodes by applying the transition function multiple times
+  # This is called "horizontal sampling" - we branch out horizontally in the tree
+  # - multithread=true: spawn parallel threads for each expansion
+  # - multithread=false: sequential expansion (default, simpler)
+  println("--> LLMMCTS expand 1")
+  if multithread
+    @sync for i in 1:horizontalSample 
+      @spawn _expand(node, transition, transitionargs)
+    end
+  else
+    println("--> LLMMCTS expand 2")
+    for i in 1:horizontalSample
+      println("--> LLMMCTS expand 3")
+      _expand(node, transition, transitionargs)
     end
   end
 end
 
+""" Helper function to expand a single child node.
 
-""" Simulate interactions between agent and environment
+Creates one new child node from the parent node using the transition function.
+Checks for semantically equivalent states (dejavu) to avoid duplicates.
 
 # Arguments
-  - `node::MCTSNode`
-    node that will be a simulation starting point.
-  - `transition::Function`
-    A user function that handles how state transition.
-  - `transitionargs::NamedTuple`
-    Arguments for everything the user will use within transition().
-  - `maxdepth::Integer`
-    maximum depth level MCTS goes vertically.
-  - totalsample::Integer
-    Total number to sample from the current node (i.e. expand new node horizontally)
-    
-# Return
-  - `::NamedTuple{(:simTrajectoryReward, :terminalstate), Tuple{Number, Union{Dict{Symbol, Any}, Nothing}}}`
+- `node::MCTSNode`
+  The parent MCTS node to expand from.
+- `transition::Function`
+  A function that handles state transition.
+- `transitionargs::NamedTuple`
+  Arguments passed to the transition function.
 
-# Signature
+# Return
+- `Nothing`
+"""
+function _expand(node::MCTSNode,transition::Function, transitionargs::NamedTuple)
+  println("--> LLMMCTS _expand 1")
+  # Generate one child node from the parent using the transition function
+  result = transition(node.state, transitionargs)
+  newNodeKey::AbstractString = result[:newNodeKey]
+  newstate::AbstractDict = result[:newstate]
+  progressvalue::Integer = result[:progressvalue]
+  println("--> LLMMCTS _expand 2")
+  # Dejavu detection: avoid adding duplicate states
+  # If newNodeKey already exists, skip - this handles semantically equivalent states
+  if newNodeKey ∉ keys(node.children)
+    println("--> LLMMCTS _expand 3")
+    # Create new MCTS node with:
+    # - visits=0: no simulations yet
+    # - statevalue=0: will be updated after simulation
+    # - progressvalue: LLM's estimate (fast heuristic)
+    # - reward: immediate environment feedback
+    newNode = MCTSNode(newNodeKey, newstate, 0, progressvalue, 0, newstate["reward"], 
+        newstate["isterminal"], node, Dict{String, MCTSNode}(), Dict{String, Any}())
+    println("--> LLMMCTS _expand 4")
+    node.children[newNodeKey] = newNode
+    println("--> LLMMCTS _expand 5")
+  end
+end
+
+""" Simulate interactions between agent and environment.
+
+Performs a rollout from the given node up to the maximum simulation depth,
+sampling child nodes at each level and accumulating rewards along the way.
+
+# Arguments
+- `node::MCTSNode`
+  The node to start simulation from.
+- `transition::Function`
+  A user function that handles state transition.
+- `transitionargs::NamedTuple`
+  Arguments passed to the transition function.
+
+# Keyword Arguments
+- `maxSimulationDepth::Integer=3`
+  Maximum depth level MCTS goes vertically during simulation.
+- `horizontalSample::Integer=3`
+  Number of child nodes sampled at each node during simulation.
+- `multithread::Bool=false`
+  Whether to run expansion in parallel using multiple threads.
+
+# Return
+- `NamedTuple{(:simTrajectoryReward, :terminalstate), Tuple{<:Number, Union{Dict{String, Any}, Nothing}}}`
+  - `simTrajectoryReward`: cumulative reward collected along the simulation trajectory
+  - `terminalstate`: final state if a terminal state was reached, `nothing` otherwise
 """
 function simulate(node::MCTSNode, transition::Function, transitionargs::NamedTuple; 
-  maxdepth::Integer=3, totalsample::Integer=3
-)::NamedTuple{(:simTrajectoryReward, :terminalstate), Tuple{Number, Union{Dict{Symbol, Any}, Nothing}}}
-
+  maxSimulationDepth::Integer=3, horizontalSample::Integer=3, multithread=false
+  )::NamedTuple{(:simTrajectoryReward, :terminalstate), Tuple{<:Number, Union{Dict{String, Any}, Nothing}}}
+  println("--> LLMMCTS simulate 1")
+  # Perform a rollout simulation from the given node:
+  # 1. Accumulate rewards along the trajectory
+  # 2. Expand nodes horizontally at each level
+  # 3. Select children to explore vertically down the tree
+  # Returns cumulative reward and whether a terminal state was reached
+  
   simTrajectoryReward = 0.0
   terminalstate = nothing
   
-  for depth in 1:maxdepth
+  for depth in 1:maxSimulationDepth
+    println("--> LLMMCTS simulate 2")
+    # Accumulate the current node's reward to the trajectory total
     simTrajectoryReward += node.reward
+    
+    # Check if we've reached a terminal state
     if node.isterminal
+      println("--> LLMMCTS simulate 3")
       terminalstate = node.state
       break
     else
-      expand(node, transition, transitionargs;
-            totalsample=totalsample)
+      println("--> LLMMCTS simulate 4")
+      # Expand current node to generate children (horizontal sampling)
+      _ = expand(node, transition, transitionargs;
+                horizontalSample=horizontalSample,
+                multithread=multithread)
+      println("--> LLMMCTS simulate 5")
+      # Select best child to continue the rollout (vertical exploration)
+      # Uses progressvalue + reward for fast selection during simulation
       node = selectChildNode(node)
-    end
+    end 
+    println("--> LLMMCTS simulate 6")
   end
-
-  return (simTrajectoryReward=simTrajectoryReward, terminalstate=terminalstate)
+  println("--> LLMMCTS simulate 7")
+  return (simTrajectoryReward=simTrajectoryReward, 
+          terminalstate=terminalstate)
 end
 
+# """ Make new state
 
-"""
+# # Arguments
+#   - `currentstate::T1`
+#     Current state dictionary containing thought history and metadata
+#   - `thoughtDict::T4`
+#     Dictionary containing new thought and action
+#   - `response::T2`
+#     Response string from the environment
+#   - `select::Union{T3, Nothing}`
+#     Selection value or nothing
+#   - `reward::T3`
+#     Reward value for this state
+#   - `isterminal::Bool`
+#     Whether this state is terminal
 
-# Arguments
+# # Return
+#   - `Tuple{String, Dict{String, <:Any}}`
+#     A tuple containing:
+#     - A unique node key string
+#     - A new state dictionary with updated thought history and metadata
+
+# # Example
+# ```jldoctest
+# julia> 
+# ```
+
+# # Signature
+# """
+# function makeNewState(currentstate::T1, thoughtDict::T4, response::T2, select::Union{T3, Nothing}, 
+#   reward::T3, isterminal::Bool
+#   )::Tuple{String, Dict{String, <:Any}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict}
+
+#   # Find the latest thought key and index from current state's thought history
+#   currentstate_latestThoughtKey, currentstate_latestThoughtIndice = 
+#       GeneralUtils.findHighestIndexKey(currentstate[:thoughtHistory], "thought")
+#   # Calculate next index for new thought/action
+#   currentstate_nextIndice = 
+#               currentstate_latestThoughtKey == :NA ? 1 : currentstate_latestThoughtIndice + 1
+#   # Create new keys for thought and action based on next index
+#   currentstate_latestThoughtKey = Symbol("thought_$currentstate_nextIndice")
+#   latestActionKey = Symbol("action_$currentstate_nextIndice")
+
+#   # Find the latest thought index from input thought dictionary
+#   _, thoughtDict_latestThoughtIndice = 
+#       GeneralUtils.findHighestIndexKey(thoughtDict, "thought")
   
-# Return
+#   # Determine thought and action keys from thought dictionary
+#   thoughtDict_latestThoughtKey, thoughtDict_latestActionKey =
+#   if thoughtDict_latestThoughtIndice == -1
+#     (:thought, :action)
+#   else
+#     (
+#       Symbol("thought_$thoughtDict_latestThoughtIndice"), 
+#       Symbol("action_$thoughtDict_latestThoughtIndice"),
+#     )
+#   end
 
-# Example
-```jldoctest
-julia> 
-```
-
-# TODO
-  - [] update docstring
-  - [x] implement the function
-
-# Signature
-"""
-function makeNewState(currentstate::T1, thoughtDict::T4, response::T2, select::Union{T3, Nothing}, 
-  reward::T3, isterminal::Bool
-  )::Tuple{String, Dict{Symbol, <:Any}} where {T1<:AbstractDict, T2<:AbstractString, T3<:Number, T4<:AbstractDict}
-
-  currentstate_latestThoughtKey, currentstate_latestThoughtIndice = 
-      GeneralUtils.findHighestIndexKey(currentstate[:thoughtHistory], "thought")
-  currentstate_nextIndice = 
-              currentstate_latestThoughtKey == :NA ? 1 : currentstate_latestThoughtIndice + 1
-  currentstate_latestThoughtKey = Symbol("thought_$currentstate_nextIndice")
-  latestActionKey = Symbol("action_$currentstate_nextIndice")
-
-  _, thoughtDict_latestThoughtIndice = 
-      GeneralUtils.findHighestIndexKey(thoughtDict, "thought")
-  
-  thoughtDict_latestThoughtKey, thoughtDict_latestActionKey =
-  if thoughtDict_latestThoughtIndice == -1
-    (:thought, :action)
-  else
-    (
-      Symbol("thought_$thoughtDict_latestThoughtIndice"), 
-      Symbol("action_$thoughtDict_latestThoughtIndice"),
-    )
-  end
-
-  # add Thought, action, observation to thoughtHistory
-  newstate = deepcopy(currentstate)
-  newstate[:thoughtHistory][currentstate_latestThoughtKey] = 
-      thoughtDict[thoughtDict_latestThoughtKey]
-  newstate[:thoughtHistory][latestActionKey] = thoughtDict[thoughtDict_latestActionKey]
-  newObservationKey = Symbol("observation_$(currentstate_nextIndice)")
-  newstate[:thoughtHistory][newObservationKey] = response
-  newstate[:reward] = reward
-  newstate[:select] = select
-  newstate[:isterminal] = isterminal
-
-  newNodeKey = GeneralUtils.uuid4snakecase()
-
-  return (newNodeKey, newstate)
-end
+#   # Create new state by deep copying current state
+#   newstate = deepcopy(currentstate)
+#   # Update thought history with new thought
+#   newstate[:thoughtHistory][currentstate_latestThoughtKey] = 
+#       thoughtDict[thoughtDict_latestThoughtKey]
+#   # Update thought history with new action
+#   newstate[:thoughtHistory][latestActionKey] = thoughtDict[thoughtDict_latestActionKey]
+#   # Create and add new observation to thought history
+#   newObservationKey = Symbol("observation_$(currentstate_nextIndice)")
+#   newstate[:thoughtHistory][newObservationKey] = response
+#   # Update state metadata
+#   newstate[:reward] = reward
+#   newstate[:select] = select
+#   newstate[:isterminal] = isterminal
 
+#   # Generate unique ID for new node
+#   newNodeKey = GeneralUtils.uuid4snakecase()
 
+#   return (newNodeKey, newstate)
+# end
 
 
 

src/type.jl

@@ -2,23 +2,35 @@ module type
   
 export MCTSNode
 
+using GeneralUtils
+
 # ---------------------------------------------- 100 --------------------------------------------- #
 
 
 """ a node for MCTS search tree
 
 # Arguments
-  - `state::T`
-    a state of a game. Can be a Dict or something else. 
-  - `visits::Integer `
-    number of time the game visits this state
-  - `stateValue::Float64`
-    state value
-  - `children::Dict{T, MCTSNode}`
-    children node
+  - `nodekey::AbstractString`
+    unique identifier for the node
+  - `state::AbstractDict`
+    a state of a game represented as a dictionary
+  - `visits::Integer`
+    number of times the game visits this state
+  - `progressvalue::Number`
+    estimated value by LLM's reasoning
+  - `statevalue::Number`
+    current state value, stores node's immediate reward and future discounted rewards
+  - `reward::Number`
+    immediate reward for this node
+  - `isterminal::Bool`
+    whether this node represents a terminal state
+  - `parent::Union{MCTSNode, Nothing}`
+    reference to parent node, Nothing for root
+  - `children::Dict{String, MCTSNode}`
+    mapping of child nodes
+  - `etc::Dict{Symbol, Any}`
+    additional storage for arbitrary data
 
-# Return
-  - `nothing`
 # Example
 ```jldoctest
 julia> state = Dict(
@@ -34,9 +46,6 @@ julia> state = Dict(
         )
 ```
 
-# TODO
-  [] update docstring
-
 # Signature
 """
 mutable struct MCTSNode{T1<:AbstractDict, T2<:AbstractString}
@@ -49,6 +58,7 @@ mutable struct MCTSNode{T1<:AbstractDict, T2<:AbstractString}
   isterminal::Bool
   parent::Union{MCTSNode, Nothing} 
   children::Dict{String, MCTSNode}
+  etc::Dict{String, Any}  # store anything
 end
 
 
@@ -110,7 +120,6 @@ end
 
 
 
-
 
 
 end # module type

src/util.jl

@@ -6,47 +6,99 @@ using ..type
 
 # ---------------------------------------------- 100 --------------------------------------------- #
 
-""" Select a node based on UCT score
+""" Select a node based on UCT (Upper Confidence Bound for Trees) score.
+
+The function computes UCT values for all child nodes and returns the child with the
+highest UCT score. The UCT formula balances exploitation (child state value) and
+exploration (visit count and parent visit count) using the exploration weight `w`.
+
+Does **not** mutate the input node.
 
 # Arguments
-  - `node::MCTSNode`
-    mcts node
-  - `w::T`
-    exploration weight. Value is usually between 1 to 2. 
-    Value 1.0 makes MCTS balance between exploration and exploitation like 50%-50%.
-    Value 2.0 makes MCTS aggressively search the tree.
-# Return
-  - `selectedNode::MCTSNode`
+- `node::MCTSNode`
+  The MCTS node whose children will be evaluated.
+- `w::AbstractFloat`
+  Exploration weight. Typical values range from 1.0 to 2.0. A value of 1.0 balances
+  exploration and exploitation equally. Higher values (e.g., 2.0) encourage more
+  exploration of less-visited nodes.
 
-# Example
-```jldoctest
-julia> 
+# Return
+- `selectedNode::MCTSNode`
+  The child node with the highest UCT score. Returns `nothing` if the node has no
+  children (though this would indicate an error since UCTselect is called on non-leaves).
+
+# The UCT Formula
+
+```
+UCT(s,a) = Q(s,a) + c * sqrt(ln(N(s)) / N(s,a))
+
+Where:
+  Q(s,a) = childNode.statevalue        (exploitation: accumulated reward)
+  c = w (explorationweight)            (controls exploration vs exploitation)
+  N(s) = node.visits                   (parent visits - total visits to parent)
+  N(s,a) = childNode.visits            (child visits - visits to this specific action)
 ```
 
-# Signature
+# Behavior
+
+| Child visits | Exploration term | Behavior |
+|-------------|------------------|----------|
+| 0 (never visited) | Undefined | Uses `progressvalue` (LLM heuristic) |
+| Low (few visits) | High | Encourages exploring new branches |
+| High (many visits) | Near 0 | Exploits known good branches |
+
+# Examples
+```jldoctest
+julia> using LLMMCTS
+julia> child1 = MCTSNode("a", Dict(:reward=>5.0), 0, 10, 50, 0, false, nothing, Dict(), Dict())
+julia> child2 = MCTSNode("b", Dict(:reward=>6.0), 0, 5, 30, 0, false, nothing, Dict(), Dict())
+julia> parent = MCTSNode("root", Dict(:reward=>0.0), 0, 15, 100, 0, false, nothing,
+                           Dict("a"=>child1, "b"=>child2), Dict())
+julia> selected = UCTselect(parent, 1.0)
+MCTSNode(...)
+```
 """
 function UCTselect(node::MCTSNode, w::T)::MCTSNode where {T<:AbstractFloat}
+  # UCT (Upper Confidence Bound for Trees) selects the best child using:
+  # UCT = statevalue + exploration_weight * sqrt(ln(parent_visits) / child_visits)
+  #
+  # The two terms balance:
+  # - Exploitation (statevalue): choose children that performed well in simulations
+  # - Exploration (sqrt term): encourage trying less-visited children
+  #
+  # The exploration weight `w` controls this balance:
+  # - w=1.0: equal emphasis on exploration and exploitation
+  # - w>1.0: more aggressive exploration (try new branches)
+  # - w<1.0: more exploitation (stick with known good branches)
+  
   maxUCT = -Inf
   selectedNode = nothing
 
   for (childState, childNode) in node.children
+    # Calculate UCT value for this child
     UCTvalue =
     if childNode.visits != 0  
-      weightedterm = w * sqrt(log(node.visits) / childNode.visits) # explore term
-      childNode.statevalue + weightedterm
-    else  # node.visits == 0 makes sqrt() in explore term error
-      childNode.progressvalue  # exploit term
+      # Child has been visited before - use statevalue with exploration bonus
+      # Exploration bonus = w * sqrt(ln(parent_visits) / child_visits)
+      # High child_visits = small bonus (exploitation dominates)
+      # Low child_visits = large bonus (encourages exploration)
+      weightedterm = w * sqrt(log(node.visits) / childNode.visits)
+      UCTvalue = childNode.statevalue + weightedterm
+    else  
+      # Child has never been visited - exploration term undefined
+      # Fall back to progressvalue (LLM heuristic) as exploitation term
+      # This allows LLM guidance to direct early search
+      UCTvalue = childNode.progressvalue
     end
-                
+                 
     if UCTvalue > maxUCT
       maxUCT = UCTvalue
-        selectedNode = childNode
+      selectedNode = childNode
     end
   end
 
   return selectedNode
-end
-
+end
 
 
 

test/Manifest.toml

@@ -0,0 +1,41 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.11.4"
+manifest_format = "2.0"
+project_hash = "71d91126b5a1fb1020e1098d9d492de2a4438fd2"
+
+[[deps.Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+version = "1.11.0"
+
+[[deps.InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+version = "1.11.0"
+
+[[deps.Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+version = "1.11.0"
+
+[[deps.Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+version = "1.11.0"
+
+[[deps.Random]]
+deps = ["SHA"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+version = "1.11.0"
+
+[[deps.SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
+
+[[deps.Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+version = "1.11.0"
+
+[[deps.Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+version = "1.11.0"

test/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"