Merge pull request 'v0.5.0-add_llmutils' (#12) from v0.5.0-add_llmutils into main

Reviewed-on: #12

Project.toml

@@ -1,7 +1,7 @@
 name = "GeneralUtils"
 uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
+version = "0.5.1"
 authors = ["tonaerospace <tonaerospace.etc@gmail.com>"]
-version = "0.2.2"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
@@ -9,9 +9,23 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
-MQTTClient = "985f35cc-2c3d-4943-b8c1-f0931d5f0959"
+Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
+HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
+NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
 PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+StringDistances = "88034a9c-02f8-509d-84a9-84ec65e18404"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[compat]
+Graphs = "1.14.0"
+HTTP = "2.5.0 - 2.9.9"
+JSON = "1.3.0 - 1.9.9"
+LibPQ = "1.18.0"
+NATS = "0.1.0"
+Revise = "3.13.2"
+StringDistances = "1.0.0"

README.md

@@ -0,0 +1,5 @@
+Todo:
+  - [WORKING] update with JSON
+
+  Change from previous version:
+  - replace JSON3 with JSON

codesnippet/nats.jl

@@ -0,0 +1,79 @@
+
+using NATS, JSON3
+connection  = NATS.connect("nats.yiem.cc:4222")
+sub1 = NATS.reply(connection, "some_subject"; queue_group="group1") do msg
+                  payload = copy(JSON3.read(msg.payload))
+                  println(payload)
+                  println(msg.reply_to)
+                  # publish(connection, msg.reply_to, "ACK")
+                  return JSON3.write(Dict(:a=>"wassup"))
+              end
+
+
+
+
+
+
+
+
+
+
+using NATS, JSON3, GeneralUtils
+connection  = NATS.connect("nats.yiem.cc:4222")
+
+msgMeta = GeneralUtils.generate_msgMeta(
+  "text2textinstruct_medium.inference.api.v1";
+  msgPurpose= "inference",
+  senderName= "yiemagent",
+  senderId= GeneralUtils.uuid4snakecase(),
+  receiverName= "text2textinstruct",
+)
+
+llmHttpTimeout = 60
+outgoingMsg = Dict(
+    :msgMeta=> msgMeta,
+    :payload=> Dict(
+      :text=> "Wassup buddy!",
+      :kwargs=> Dict(
+        :max_tokens=> 2048,
+        :stop=> ["<|im_end|>"],
+        :temperature=> 0.2,
+      ),
+      :llmHttpTimeout=>llmHttpTimeout,
+    )
+  )
+
+r = NATS.request(String, connection, "text2textinstruct_medium.inference.api.v1", 
+                JSON3.write(outgoingMsg); timer=Timer(llmHttpTimeout))
+
+
+
+
+
+
+
+
+
+
+
+using NATS, JSON3, GeneralUtils
+connection  = NATS.connect("nats.yiem.cc:4222")
+
+msgMeta = GeneralUtils.generate_msgMeta(
+  "tonpc.containerServices",
+  msgPurpose="reset container",
+  senderName= "",
+)
+
+outgoingMsg = Dict(
+  :msgMeta=> msgMeta,
+  :payload=> "docker container restart ollama-instance-2",
+)
+
+# may be I can't use NATS request inside NATS reply??
+r = NATS.request(String, connection, msgMeta[:sendTopic], JSON3.write(outgoingMsg); timer=Timer(10))
+
+
+
+
+

src/GeneralUtils.jl

@@ -2,7 +2,7 @@ module GeneralUtils
     
 
 export  # struct
-        mqttClientInstance,
+        # mqttClientInstance,
         # function
         noNegative!, randomWithProb, randomChoiceWithProb, findIndex, limitvalue 
 
@@ -24,16 +24,7 @@ using .interface
 
 #------------------------------------------------------------------------------------------------100
 
-""" version 0.0.4
-    Todo:
-        - [*1] cartesianAssign for different matrix dimension
 
-    Change from version: 0.0.3
-        - 
-    
-    All features
-        
-"""
 
 
 

src/dbUtil.jl

@@ -2,13 +2,45 @@ module dbUtil
   
 export dictToPostgresKeyValueString, generateInsertSQL, generateUpdateSQL
 
-using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames,
+using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames,
       SHA
 using ..util
-
+#[PENDING] update code to use JSON
 # ---------------------------------------------- 100 --------------------------------------------- #
 
 
+"""
+dictToPostgresKeyValueString - Convert dictionary to PostgreSQL key-value string format
+
+This function takes a dictionary and converts it into a PostgreSQL-compatible key-value string
+format suitable for storage in a TEXT field. The output format uses curly braces with comma-separated
+key-value pairs, where string values are quoted.
+
+# Function Workflow:
+1. Iterates through dictionary key-value pairs
+2. Handles nested dictionaries by recursively converting them
+3. Wraps string values in double quotes
+4. Formats numeric and other values without quotes
+5. Returns a PostgreSQL-compatible key-value string enclosed in curly braces
+
+# Arguments:
+  - `dict::Dict` - Dictionary containing key-value pairs to convert
+
+# Return:
+  - A String in PostgreSQL key-value format: "{key1: value1, key2: \"value2\", ...}"
+
+# Example
+```jldoctest
+julia> data = Dict{String, Any}(
+           "name" => "John",
+           "age" => 30,
+           "city" => "New York"
+       );
+
+julia> dictToPostgresKeyValueString(data)
+"{\"name\": \"John\", \"age\": 30, \"city\": \"New York\"}"
+```
+"""
 function dictToPostgresKeyValueString(dict)
   parts = []
   for (k, v) in dict
@@ -26,61 +58,71 @@ end
 
 
 
-""" Get characters between specified characters.
+""" generateInsertSQL - Generate SQL INSERT statement from dictionary data
 
-# Arguments
-  - `text::T`
-    a text being searched
-  - `startChar::Char`
-    start character
-  - `endChar::Char`
-    end character
-# Keyword Arguments
-  - `endCharLocation::String`
-    end character position after startChar. Can be "next" or "end". "next" means the closed 
-    endChar just after startChar. "end" means the furthest endChar. 
-  - `includeChar::Bool` 
-    whether to include the startChar and endChar. Default is true
-# Return
-  the characters between specified characters.
+This function constructs a SQL INSERT statement by extracting values for specified columns
+from a dictionary and formatting them into a valid PostgreSQL INSERT query.
+
+# Function Workflow:
+1. Iterates through the dictionary key-value pairs
+2. Filters keys to only include those present in `columnToInsert`
+3. Collects column names and their corresponding values
+4. Constructs the final SQL INSERT statement
+
+# Arguments:
+  - `table_name::String` - Name of the database table to insert into
+  - `columnToInsert::Vector{Symbol}` - List of column names to include in the INSERT statement
+  - `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs for the insert
+
+# Return:
+  - A String containing the SQL INSERT statement
 
 # Example
 ```jldoctest
-julia> using Revise
-julia> using GeneralUtils
-julia> insert_data = Dict(
-      :grape => "NA",
-      :acidity => "0",
-      :tannin => "0",
-      :country => "NA",
-      :description => "NA",
-      :region => "NA",
-      :winery => "ccc",
-      :intensity => "0",
-      :sweetness => "0",
-      :tasting_notes => "NA",
-      :wine_name => "new_wine",
-      :wine_id => "9e1deb6a-d57f-4d2c-abbe-da813f4e91ad",
-      :wine_type => "NA",
-      :other_attributes => "{\"attribute3\":{\"attribute5\":666,\"attribute4\":\"text\"},\"attribute1\":\"hello world\",\"attribute2\":555}",
-      :fizziness => "0",
-      :serving_temperature => "0",
-      :additional_search_term => "{NA1,NA2}")
-```
-# TODO
-  - [] update docs
+julia> using UUIDs
 
-# Signature
+# Insert a single record with specific columns
+table_name = "wine"
+columnToInsert = [:acidity, :tannin, :country, :region, :winery]
+data = Dict{Symbol, Any}(
+    :grape => "Cabernet Sauvignon",
+    :acidity => "medium",                 # using descriptive scale (low/medium/full)
+    :tannin => "medium-plus",             # common wine descriptor
+    :country => "France",
+    :description => "A rich and structured red wine with notes of blackcurrant, cedar, and subtle oak.",
+    :region => "Bordeaux",
+    :winery => "Château Margaux",
+    :intensity => "medium",               # intensity is usually low/medium/full
+    :sweetness => "dry",                  # dry/medium-dry/medium/medium-sweet/sweet
+    :tasting_notes => "Blackberry, graphite, tobacco, vanilla, and subtle earth.",
+    :wine_name => "Château Margaux Grand Cru",
+    :wine_id => "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a",  # UUID-like (valid hex)
+    :wine_type => "Red",
+    :other_attributes => Dict{String, Any}(
+        "vintage" => 2018,
+        "alcohol_percent" => 13.5,
+        "ph" => 3.6,
+        " aging_years" => 24,              # years in barrel
+        " producer_code" => "CM-GRAND"
+    ),
+    :fizziness => "still",
+    :serving_temperature => "16–18°C",
+    :additional_search_term => ["Cabernet", "Bordeaux red", "premium wine", "CabSav"]
+)
+
+julia> generateInsertSQL(table_name, columnToInsert, data)
+"INSERT INTO wine (acidity, tannin, country, region, winery) VALUES ('medium', 'medium-plus', 'France', 'Bordeaux', 'Château Margaux');"
+```
 """
-function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, 
-  insert_data::Dict{Symbol, Any})
+function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol}, data::Dict{Symbol, Any})
   columns = String[]
   values = String[]
 
-  for (key, value) in insert_data
+  for (key, value) in data
     if key ∈ columnToInsert
       push!(columns, string(key))
-      push!(values, "'$value'") #[] number should not wrapped in ''
+      value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+      push!(values, value_str)
     end
   end
   
@@ -89,113 +131,101 @@ function generateInsertSQL(table_name::String, columnToInsert::Vector{Symbol},
   
   return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
 end
-# function generateInsertSQL(table_name::String, insert_data::Dict{Symbol, Any})
-#   columns = String[]
-#   values = String[]
-  
-#   for (key, value) in insert_data
-#       push!(columns, string(key))
-#       if key == :other_attributes
-#           push!(values, "'$value'")
-#       else
-#           push!(values, "'$value'")
-#       end
-#   end
-  
-#   columns_str = join(columns, ", ")
-#   values_str = join(values, ", ")
-  
-#   return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
-# end
 
-  
+function generateInsertSQL(table_name::String, data::AbstractDict{String, Any})
+  columns = String[]
+  values = String[]
 
-
-
-"""
-example:
-
-insert_data = Dict(
-    :grape => "NA",
-    :acidity => "0",
-    :tannin => "0",
-    :country => "NA",
-    :description => "NA",
-    :region => "NA",
-    :winery => "ccc",
-    :intensity => "0",
-    :sweetness => "0",
-    :tasting_notes => "NA",
-    :wine_name => "new_wine",
-    :wine_id => "9e1deb6a-d57f-4d2c-abbe-da813f4e91ad",
-    :wine_type => "NA",
-    :other_attributes => "{\"attribute3\":{\"attribute5\":666,\"attribute4\":\"text\"},\"attribute1\":\"hello world\",\"attribute2\":555}",
-    :fizziness => "0",
-    :serving_temperature => "0",
-    :additional_search_term => "{NA1,NA2}")
-
-id_keys is the primary key columns
-"""
-# function generateUpdateSQL(table_name::String, update_data::Dict{Symbol, Any}, id_keys::Vector{Symbol})
-#   set_clauses = String[]
-#   where_clauses = String[]
-  
-#   for (key, value) in update_data
-#       if key in id_keys
-#           push!(where_clauses, "$key = '$value'")
-#       else
-#           if key == :other_attributes
-#               push!(set_clauses, "$key = '$value'")
-#           else
-#               push!(set_clauses, "$key = '$value'")
-#           end
-#       end
-#   end
-  
-#   set_clause = join(set_clauses, ", ")
-#   where_clause = join(where_clauses, " AND ")
-  
-#   return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
-# end
-
-function generateUpdateSQL(table_name::String, columnToUpdate::Vector{Symbol}, 
-  updatedata::Dict{Symbol, Any}, id_keys::Vector{Symbol})
-  
-  set_clauses = String[]
-  where_clauses = String[]
-  
-  for (key, value) in updatedata
-      if key in id_keys
-          push!(where_clauses, "$key = '$value'")
-      else
-        if key ∈ columnToUpdate # update only specified columns
-          push!(set_clauses, "$key = '$value'")
-        end
-      end
+  for (key, value) in data
+    push!(columns, string(key))
+    value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+    push!(values, value_str)
   end
   
-  set_clause = join(set_clauses, ", ")
-  where_clause = join(where_clauses, " AND ")
-  
-  return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
+  columns_str = join(columns, ", ")
+  values_str = join(values, ", ")
+    
+    return "INSERT INTO $table_name ($columns_str) VALUES ($values_str);"
 end
 
 
+# ---------------------------------------------- 100 --------------------------------------------- #
 
+""" generateUpdateSQL - Generate SQL UPDATE statement from dictionary data
 
+This function constructs a SQL UPDATE statement by updating multiple columns
+based on a primary key condition.
 
+# Arguments:
+  - `table_name::String` - Name of the database table to update
+  - `pk_column::Symbol` - The primary key column name
+  - `pk_value` - The primary key value (used in WHERE clause)
+  - `data::Dict{Symbol, Any}` - Dictionary containing column-value pairs to update
 
+# Return:
+  - A String containing the SQL UPDATE statement
 
+# Example
+```jldoctest
+julia> using UUIDs
 
+# Update multiple columns using a dictionary
+table_name = "wine"
+pk_column = :wine_id
+pk_value = "8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a"
+data = Dict{Symbol, Any}(
+    :acidity => "full",
+    :tannin => "medium",
+    :country => "Italy"
+)
 
+julia> generateUpdateSQL(table_name, pk_column, pk_value, data)
+"UPDATE wine SET acidity = 'full', tannin = 'medium', country = 'Italy' WHERE wine_id = '8f3c7a2e-1b4d-4a9f-9c2e-4a8b3d6e5f7a';"
+```
+"""
+function generateUpdateSQL(table_name::String, pk_column::String, pk_value, 
+                          data::AbstractDict{String, Any})
+    # Build SET clause
+    set_parts = String[]
+    for (key, value) in data
+        if key ∉ [pk_column]
+          value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+          push!(set_parts, "$(string(key)) = $value_str")
+        end
+    end
+    
+    set_clause = join(set_parts, ", ")
+    
+    # Handle primary key value
+    pk_val_str = isa(pk_value, AbstractString) ? "'$pk_value'" : "$pk_value"
+    
+    return "UPDATE $table_name SET $set_clause WHERE $pk_column = $pk_val_str;"
+end
 
-
-
-
-
-
-
-
+function generateUpdateSQL(table_name::String, pk_dict::AbstractDict{String, Any}, 
+                          data::AbstractDict{String, Any})
+    # Build SET clause
+    set_parts = String[]
+    for (key, value) in data
+        if key ∉ keys(pk_dict)
+            value_str = isa(value, AbstractString) ? "'$value'" : "$value"
+            push!(set_parts, "$(string(key)) = $value_str")
+        end
+    end
+    
+    set_clause = join(set_parts, ", ")
+    
+    # Build WHERE clause for composite keys
+    where_parts = String[]
+    for (col, val) in pk_dict
+        val_str = isa(val, AbstractString) ? "'$val'" : "$val"
+        push!(where_parts, "$(string(col)) = $val_str")
+    end
+    
+    where_clause = join(where_parts, " AND ")
+    
+    return "UPDATE $table_name SET $set_clause WHERE $where_clause;"
+end
 
 
 end   # module

src/util.jl

@@ -1,47 +1,52 @@
 module util
   
 export timedifference, showstracktrace, findHighestIndexKey, uuid4snakecase, replaceDictKeys,
-        findMatchingDictKey, textToDict, randstring, randstrings, timeout,
+        findMatchingDictKey, randstring, randstrings, timeout,
         dataframeToCSV, dfToVectorDict, disintegrate_vectorDict, getDataFrameValue, dfRowtoString, 
-        dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, 
+        dfToString, dataframe_to_json_list, dictToString, dictToString_noKey, issomething,
         dictToString_numbering, extract_triple_backtick_text, 
-        countGivenWords, remove_french_accents, detect_keyword, extractTextBetweenCharacter,
-        convertCamelSnakeKebabCase
+        countGivenWords, remove_french_accents, removestring,
+        extractTextBetweenCharacter, extractTextBetweenString, 
+        convertCamelSnakeKebabCase, fitrange, recentElementsIndex, nonRecentElementsIndex
 
-using JSON3, DataStructures, Distributions, Random, Dates, UUIDs, MQTTClient, DataFrames
+using JSON, DataStructures, Distributions, Random, Dates, UUIDs, DataFrames
 
 # ---------------------------------------------- 100 --------------------------------------------- #
 
-""" Compute time different between start time and stop time in a given unit. 
- Unit can be "milliseconds", "seconds", "minutes", "hours".
+""" Computes the time difference between two `DateTime` values and returns the
+result in a specified unit: milliseconds, seconds, minutes, or hours.
 
 # Arguments
-  - `starttime::DateTime` 
-    start time
-  - `stoptime::DateTime`  
-    stop time
-  - `unit::String`  
-    unit of time difference
+- `starttime::DateTime`
+  The starting `DateTime` value.
+- `stoptime::DateTime`
+  The ending `DateTime` value.
+- `unit::String`
+  The unit for the result. Must be one of: `"milliseconds"`, `"seconds"`,
+  `"minutes"`, `"hours"`.
 
 # Return
-  - time difference in given unit
+- `Integer`: The time difference converted to the specified unit.
 
-# Example
+# Notes
+- The function computes `stoptime - starttime` and converts the result to the
+  requested unit using integer division.
+- Errors with `ArgumentError` if an invalid unit is specified.
+
+# Examples
 ```jldoctest
-julia> using Revise
 julia> using GeneralUtils, Dates
 julia> a = Dates.now()
-julia> b = a + Dates.Day(5) # add 5 days
-julia> GeneralUtils.timedifference(a, b, "hours")
+julia> b = a + Dates.Day(5)
+julia> timedifference(a, b, "hours")
 120
 ```
-
-# Signature
 """
 function timedifference(starttime::DateTime, stoptime::DateTime, unit::String)::Integer
   diff = stoptime - starttime
   unit = lowercase(unit)
   
+    # Check the unit and calculate the time difference accordingly
   if unit == "milliseconds"
       return diff.value
   elseif unit == "seconds"
@@ -182,21 +187,27 @@ end
 
 
 
-""" Get uuid4 with snake case
-  
-# Return
-  - `uuid4::String`
-    uuid4 with snake case
+""" Generates a UUID4 (version 4) identifier and converts it to snake case by
+replacing hyphens with underscores.
 
-# Example
+# Arguments
+- This function takes no arguments.
+
+# Return
+- `String`: A UUID4 string with underscores instead of hyphens (e.g.,
+  `"0f6e4f_568c_4df4_8c79_1d7a58072f4a"`).
+
+# Notes
+- Uses the `uuid4()` function from the UUIDs standard library to generate a
+  random UUID.
+- The underscore character replaces all hyphens in the UUID string.
+
+# Examples
 ```jldoctest
-julia> using Revise
 julia> using GeneralUtils
-julia> GeneralUtils.uuid4snakecase()
+julia> uuid4snakecase()
 "0f6e4f_568c_4df4_8c79_1d7a58072f4a"
 ```
-
-# Signature
 """
 function uuid4snakecase()::String
   _id = string(uuid4())
@@ -205,32 +216,37 @@ function uuid4snakecase()::String
 end
 
 
-""" Replace a dictionary key with the new key
+""" Replaces keys in a dictionary according to a mapping, returning a new
+dictionary with updated keys while preserving the original values.
 
 # Arguments
-  - `d::Dict` 
-    The input dictionary that you want to modify
-  - `replacementMap::Dict`
-    A dictionary that maps old keys to new keys
+- `d::Dict`
+  The input dictionary to modify.
+- `replacementMap::Dict`
+  A dictionary mapping old keys to new keys. Keys not present in this map are
+  left unchanged.
 
 # Return
-  - `newDict::Dict`
-    new dictionary with the replaced keys
+- `Dict`: A new dictionary with replaced keys. Values are preserved from the
+  original dictionary.
 
-# Example
+# Notes
+- The function creates a new dictionary rather than modifying the input in
+  place.
+- Keys not found in `replacementMap` are copied to the result with their
+  original keys unchanged.
+
+# Examples
 ```jldoctest
-julia> using Revise
 julia> using GeneralUtils
 julia> d = Dict(:a => 1, :b => 2, :c => 3)
 julia> replacement_map = Dict(:a => :x, :b => :y)
-julia> new_dict = GeneralUtils.replaceDictKeys(d, replacement_map)
+julia> replaceDictKeys(d, replacement_map)
 Dict{Any, Any} with 3 entries:
   :y => 2
   :c => 3
   :x => 1
 ```
-
-# Signature
 """
 function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict
   newDict = Dict()
@@ -242,101 +258,6 @@ function replaceDictKeys(d::Dict, replacementMap::Dict)::Dict
 end
 
 
-""" Convert text into a dictionary with a given keywords. This function use keywords to slice
-  a given text into the following format: KW1|kw1_text|KW2|kw2_text|KW3|kw3_text.
-  The left most string which has no keyword will be discarded. WARNING, ordering is important
-
-# Arguments
-  - `text::String`
-    A text to be converted.
-  - `keywords::Vector{String}`
-    A list of keywords to be used to slice the text. 
-    These keywords also be the resulting dict keys.
-# Keyword Arguments
-  - `rightmarker::String`
-    A maker used to make a word to be unique. Ex, A keyword "plan" with rightmarker ":", 
-    the function will search for "plan:" otherwise the function will search for "plan".
-    The marker will not be in the resulting dict keys.
-  - `symbolkey::Bool`
-    If true, resulting dict's key will be Symbols, otherwise string.
-  - `lowercasekey::Bool`
-    set resulting dict's key to be lowercase
-
-# Return
-  - `d::OrderedDict`
-
-# Example
-```jldoctest
-julia> text = "TODAY thought: what to do plan: wake up and going out action: 1. wake up 2. eat 3. sleep"
-julia> sample_keywords = ["thought", "plan", "action"]
-julia> resultdict = GeneralUtils.textToDict(text, sample_keywords; rightmarker=":", symbolkey=true)
-julia> println(resultdict)
-OrderedCollections.OrderedDict{Any, Any}(:thought => "what to do", 
-                                        :plan => "wake up and going out", 
-                                        :action => "1. wake up 2. eat 3. sleep")
-```
-
-# Signature
-"""
-function textToDict(text::String, detectKeywords::Vector{String}; 
-    dictKey::Union{Vector{String}, Nothing}=nothing, 
-    symbolkey::Bool=false, lowercasekey::Bool=false
-    )::OrderedDict
-
-  # make sure this function detect variation of a work e.g. agent, Agent, AGENT
-  kw = []
-  # use for loop and detect_keyword function to get the exact variation of each keyword in the text then push to kw list
-  for keyword in detectKeywords
-    detected = detect_keyword(keyword, text)
-    if detected !== nothing
-      push!(kw, detected)
-    else
-      error("Keyword $keyword not found in text.")
-    end
-  end
-
-  od1, od2 =
-  if symbolkey
-    OrderedDict{Symbol, Any}(), OrderedDict{Symbol, Any}()
-  else
-    OrderedDict{String, Any}(), OrderedDict{String, Any}()
-  end
-
-  remainingtext = text
-  dictKey_ = reverse(dictKey)
-  
-  # process text from back to front
-  for (i,keyword) in enumerate(reverse(kw))
-    # Find the position of the keyword in the text
-    keywordidx = findlast(keyword, remainingtext)
-    dKey = dictKey_[i]
-    
-    if keywordidx !== nothing
-      substr = remainingtext[keywordidx[end]+1:end]
-      str = string(strip(substr))  # Removes both leading and trailing whitespace.
-      _key = lowercasekey == true ? lowercase(dKey) : dKey
-      key = symbolkey == true ? Symbol(_key) : _key
-      od1[key] = str
-      remainingtext = remainingtext[1:keywordidx[1]-1]
-    else
-      error("""keyword "$keyword" not found in the provided text: $text </end of error note>""")
-    end
-  end
-
-  # correct the order
-  ks = reverse([i for i in keys(od1)])
-  for k in ks
-    k = symbolkey == true ? Symbol(k) : k
-    od2[k] = od1[k]
-  end
-  
-  return od2
-end
-
-
-
-
-
 """ Generate a random string
 
 # Arguments
@@ -387,36 +308,43 @@ end
 
 
 
-""" Execute a function with timer.
+""" Executes a function with a timeout mechanism. If the function does not
+complete within the specified time, it is interrupted and a timeout message
+is returned.
 
 # Arguments
-  - `f::Function`
-      a function to run
-  - `timeoutwindow::Integer``
-      timeout in seconds
+- `f::Function`
+  The function to execute.
+- `timeoutwindow::Integer`
+  The timeout duration in seconds.
 
-# Keyword Argument
-  - `fargs`
-    arguments for the function
-  - `timeoutmsg::String`
-    time out message
+# Keyword Arguments
+- `fargs`
+  Arguments to pass to the function `f`. If `nothing`, the function is called
+  without arguments.
+- `timeoutmsg::String`
+  The message to return if the function times out. Defaults to `"task timed out"`.
 
 # Return
-  - task result otherwise timeout message
+- The result of the function if it completes within the timeout, otherwise the
+  `timeoutmsg` string.
 
-# Example
-  ```jldoctest
+# Notes
+- Uses Julia's `@task`, `schedule`, and `Timer` to implement non-blocking
+  execution with interruption via `Base.throwto`.
+- Errors with `InterruptException` if the function exceeds the timeout.
+
+# Examples
+```jldoctest
 julia> function testfunc(x)
-          sleep(x)
-          return "task done"
-        end
+           sleep(x)
+           return "task done"
+       end
 julia> result = timeout(testfunc, 10; fargs=20)
 "task timed out"
 julia> result = timeout(testfunc, 20; fargs=10)
 "task done"
 ```
-
-#  Signature
 """
 function timeout(f::Function, timeoutwindow::Integer; fargs=nothing, timeoutmsg="task timed out")
   tsk = @task f(fargs)
@@ -433,23 +361,26 @@ end
 
 
 
-""" Convert a dataframe into CSV.
+""" Converts a DataFrame to a CSV string representation using the CSV.jl package.
 
 # Arguments
-  - `df::DataFrame`
-    A connection object to Postgres database
-    
-# Return
-  - `result::String`
+- `df::DataFrame`
+  The DataFrame to convert to CSV format.
 
-# Example
+# Return
+- `String`: The DataFrame contents as a CSV-formatted string.
+
+# Notes
+- Uses `CSV.write` with an `IOBuffer` to capture the output as a string.
+- The returned string contains the full CSV representation including headers.
+
+# Examples
 ```jldoctest
 julia> using DataFrames, GeneralUtils
 julia> df = DataFrame(A=1:3, B=5:7, fixed=1)
-julia> result = GeneralUtils.dataframeToCSV(df)
+julia> dataframeToCSV(df)
+"1,5,1\n2,6,1\n3,7,1\n"
 ```
-
-# Signature
 """
 function dataframeToCSV(df::DataFrame)
   # Create an IOBuffer to capture the output
@@ -471,7 +402,7 @@ end
 
 #  Example
   ```jldoctest
-  julia> using DataFrames, JSON3, GeneralUtils
+  julia> using DataFrames, GeneralUtils
   julia> df = DataFrame(A = [1, 2, 3], B = ["apple", "banana", "cherry"])
   julia> vectorDict = GeneralUtils.dfToVectorDict(df)
   [Dict{String, Any}("B" => "apple", "A" => 1), 
@@ -509,7 +440,7 @@ end
 
 # Example
   ```jldoctest
-  julia> using GeneralUtils, Dates, JSON3, UUIDs
+  julia> using GeneralUtils, Dates, UUIDs
   julia> vecDict = [Dict("a" => i) for i in 1:10]
   julia> d = GeneralUtils.disintegrate_vectorDict(vecDict, 3)
   julia> println(d[:data])
@@ -519,8 +450,6 @@ end
   3 => [Dict("a"=>7), Dict("a"=>8), Dict("a"=>9)]
   4 => [Dict("a"=>10)]
   ```
-
-# Signature
 """
 function disintegrate_vectorDict(data::Vector, partsize::Integer
   )
@@ -566,8 +495,6 @@ end
   julia> getDataFrameValue(df[1, :], :name)
   "Alice"
   ```
-
-# Signature
 """
 getDataFrameValue(row::DataFrameRow, key::Symbol) = row.:($key)
 
@@ -636,8 +563,6 @@ end
   julia> dfToString(df)
   "1) name: Alice, age: 25\n2) name: Bob, age: 30"
   ```
-
-# Signature
 """
 function dfToString(df::DataFrame)
   dfstr = ""
@@ -661,7 +586,7 @@ end
 
 # Example
   ```jldoctest
-  julia> using DataFrames
+  julia> using DataFrames, GeneralUtils
 
   julia> df = DataFrame(name=["Alice", "Bob"], age=[25, 30])
   2×2 DataFrame
@@ -671,13 +596,11 @@ end
    │ 1   │ Alice   25    
    │ 2   │ Bob     30    
 
-  julia> dataframe_to_json_list(df)
+  julia> GeneralUtils.dataframe_to_json_list(df)
   2-element Vector{String}:
   "{\"name\":\"Alice\",\"age\":25}"
   "{\"name\":\"Bob\",\"age\":30}"
   ```
-
-# Signature
 """
 function dataframe_to_json_list(df::DataFrame)::Vector{String}
   json_list = []
@@ -711,8 +634,6 @@ end
   julia> dict_to_string(od)
   "1) name: Alice, 2) age: 25"
   ```
-
-# Signature
 """
 function dictToString(od::T) where {T<:AbstractDict}
   items = []
@@ -770,48 +691,19 @@ function extract_triple_backtick_text(input::String)::Vector{String}
 end
 
 
-"""
-    detect_keyword(keyword::String, text::String) -> Union{Nothing, String}
+wordwindow(word::String, startindex::Integer)::UnitRange = startindex:startindex + length(word) -1
 
-Detects if a keyword exists in the text in different case variations (lowercase, uppercase first letter, or all uppercase).
-
-# Arguments:
-- `keyword::String` The keyword to search for
-- `text::String` The text to search in
-
-# Returns:
-- `Union{Nothing, String}` Returns the matched keyword variation if found, otherwise returns nothing
-
-# Examples:
-  ```jldoctest
-  julia> detect_keyword("test", "This is a Test case")
-  "Test"
-
-  julia> detect_keyword("error", "NO ERRORS FOUND")
-  "ERRORS"
-
-  julia> detect_keyword("missing", "complete data")
-  nothing
-  ```
-
-# Signature
-"""
-function detect_keyword(keyword::String, text::String)::Union{Nothing, String}
-  # Define the keyword variations to search for
-  keyword_variations = [keyword, uppercasefirst(keyword), uppercase(keyword), lowercase(keyword)]
-  
-  # Check if any of the keyword variations are in the text
-  for variation in keyword_variations
-      if occursin(variation, text)
-          return variation
-      end
+function cuttext(range, text)
+  # check whether range is outside text boundary
+  if range.start > length(text) || range.stop > length(text)
+    return nothing
+  else
+    return text[range]
   end
-  
-  # Return nothing if no variation is found
-  return nothing
 end
 
 
+
 """
     countGivenWords(text::String, words::Vector{String}) -> Dict{String, Int}
 
@@ -855,27 +747,39 @@ end
 
 
 
-"""
-    remove_french_accents(text::String) -> String
+""" Remove French accents from the given text.
 
-Remove French accents from the given text.
+The function replaces accented French characters with their non-accented
+counterparts using a dictionary mapping. Supported accented characters
+include: à, â, ä, á, é, è, ê, ë, î, ï, í, ñ, ô, ö, ò, ó, ù, û, ü, ÿ, ç,
+and their uppercase variants. The apostrophe character `’` is removed
+completely.
 
 # Arguments
-- `text::String` The input string containing French accents.
+- `text::AbstractString`
+  The input string containing French accented characters.
 
-# Returns
-- `String` The input string with all French accents removed.
+# Return
+- `AbstractString`: A new string with all French accents replaced by their
+  non-accented equivalents.
+
+# Notes
+- The function creates a character list and replaces each accented character
+  according to an internal dictionary mapping.
+- Does **not** mutate the input; it allocates a new string.
 
 # Examples
-  ```jldoctest
-  julia> remove_french_accents("Café")
-  "Cafe"
+```jldoctest
+julia> using GeneralUtils
+julia> remove_french_accents("Café")
+"Cafe"
 
-  julia> remove_french_accents("L'été est beau.")
-  "L'ete est beau."
-  ```
+julia> remove_french_accents("L'été est beau.")
+"L'ete est beau."
 
-# Signature
+julia> remove_french_accents("Noël, naïve, François")
+"Noel, naive, Francois"
+```
 """
 function remove_french_accents(text::AbstractString)::AbstractString
   textcharlist = [i for i in text]
@@ -924,9 +828,11 @@ Extracts and returns the text that is enclosed between two specified characters
 
 # Examples
 ```jldoctest
-julia> text = "Hello [World]!"
+julia> text = "Hello [World]! [Yay]"
 julia> extracted_text = extractTextBetweenCharacter(text, '[', ']')
-println(extracted_text)  # Output: "World"
+2-element Vector{Any}:
+ "World"
+ "Yay"
 ```
 """
 function extractTextBetweenCharacter(text::String, startchar::Char, endchar::Char)
@@ -950,6 +856,29 @@ function extractTextBetweenCharacter(text::String, startchar::Char, endchar::Cha
 end
 
 
+function extractTextBetweenString(text::String, startstr::String, endstr::String)
+  # check whether startstr is in the text or not
+  isStartStr = split(text, startstr)
+  if length(isStartStr) > 2
+    return (success=false, error="There are more than one occurrences of the start string '$startstr' in the text. Text must has only one start string", errorcode=2, result=nothing)
+  elseif length(isStartStr) == 1
+    return (success=false, error="There are no start string '$startstr' in the text. Text must has only one start string", errorcode=1, result=nothing)
+  end
+
+  # check whether endstr is in the text or not
+  isEndStr = split(text, endstr)
+  if length(isEndStr) > 2
+    return (success=false, error="There are more than one occurrences of the end string '$endstr' in the text. Text must has only one end string", errorcode=3, result=nothing)
+  elseif length(isStartStr) == 1
+    return (success=false, error="There are no end string '$endstr' in the text. Text must has only one end string", errorcode=4, result=nothing)
+  end
+
+  s = string(split(isStartStr[2], endstr)[1])
+  
+  return (success=true, error=nothing, errorcode=0, text=s)
+end
+
+
 """
 Determines if the given string follows camel case naming convention.
 
@@ -1075,10 +1004,173 @@ function convertCamelSnakeKebabCase(text::T, tocase::Symbol)::String where {T<:A
 end
 
 
+""" Check if a value is not `nothing`.
+
+# Arguments
+- `x`: The value to check
+
+# Returns
+- `Bool`: `true` if `x` is not `nothing`, `false` otherwise
+
+# Examples
+```jldoctest
+julia> issomething(1)
+true
+julia> issomething(nothing)
+false
+julia> issomething("test")
+true
+````
+"""
+function issomething(x)
+  return x === nothing ? false : true
+end
 
 
+""" Adjust a given range to fit within the bounds of a vector's length.
+
+# Arguments
+  - `v::T1`
+    the input vector to check against
+  - `range::UnitRange`
+    the original range to be adjusted
+
+# Return
+  - `adjusted_range::UnitRange`
+    a range that is constrained to the vector's length, preventing out-of-bounds indexing
+
+# Example
+
+julia> v = [1, 2, 3, 4, 5]
+julia> fitrange(v, 3:10)
+3:5
+
+"""
+function fitrange(v::T1, range::UnitRange) where {T1<:AbstractVector}
+  totalelements = length(v)
+
+  startind =
+    # check if user put start range greater than total event
+    if range.start > totalelements
+      totalelements
+    else
+      range.start
+    end
+
+  stopind = 
+    if range.stop > totalelements
+      totalelements
+    else
+      range.stop
+    end
+
+  return startind:stopind
+end
 
 
+""" Find a unit range for a vector given a number of the most recent elements of interest.
+
+# Arguments
+  - `vectorLength::Integer`
+    the length of the vector to generate range from
+  - `n::Integer`
+    the number of most recent elements to include in range
+
+
+# Return
+  - `UnitRange`
+    a range representing the n most recent elements of a vector with length vectorLength
+
+# Example
+```jldoctest
+julia> a = [1, 2, 3, 4, 5]
+julia> recentElementsIndex(length(a), 3)
+3:5
+julia> recentElementsIndex(length(a), 0)
+5:5
+```
+"""
+function recentElementsIndex(vectorlength::Integer, n::Integer; includelatest::Bool=false)
+  if n == 0
+    error("n must be greater than 0")
+  end
+  
+  if includelatest
+    start = max(1, vectorlength - n + 1)
+    return start:vectorlength
+  else
+    startind = max(1, vectorlength - n)
+    endind = vectorlength -1
+    return startind:endind
+  end
+end
+
+
+""" Find a unit range for a vector excluding the most recent elements.
+
+# Arguments
+    - `vectorlength::Integer`
+      the length of the vector to generate range from
+    - `n::Integer`
+      the number of most recent elements to exclude from range
+
+# Return
+  - `UnitRange`
+    a range representing the elements of the vector excluding the last `n` elements
+
+# Example
+```jldoctest
+julia> a = [1, 2, 3, 4, 5]
+julia> nonRecentElementsIndex(length(a), 3)
+1:2
+julia> nonRecentElementsIndex(length(a), 1)
+1:4
+julia> nonRecentElementsIndex(length(a), 0)
+1:5
+```
+"""
+function nonRecentElementsIndex(vectorlength::Integer, n::Integer)
+  if n < 0
+      error("n must be non-negative")
+  end
+  if n > vectorlength
+      return 1:0  # empty range
+  end
+  return 1:(vectorlength-n)
+end
+
+""" Remove specified substrings from text.
+
+Removes all occurrences of each string in `removelist` from the input text
+by repeatedly replacing them with empty strings until none remain.
+
+# Arguments
+- `text::String`
+  The input string to modify.
+- `removelist::Vector{String}`
+  A vector of substrings to remove from the text.
+
+# Return
+- `String`: The text with all specified substrings removed.
+
+# Examples
+```jldoctest
+julia> using GeneralUtils
+julia> removestring("hello world", ["l", " "])
+"heoword"
+julia> removestring("foo bar baz", ["bar", " "])
+"foobaz"
+```
+"""
+function removestring(text::String, removelist::Vector{String})::String
+  for i in removelist
+    while occursin(i, text)
+      text = replace(text, i => "")
+    end
+  end
+
+  return string(text)
+end
 
 
 

test/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "julia.environmentPath": "/appfolder/app/dev/GeneralUtils/test"
+}

test/Manifest.toml

@@ -0,0 +1,41 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.11.4"
+manifest_format = "2.0"
+project_hash = "71d91126b5a1fb1020e1098d9d492de2a4438fd2"
+
+[[deps.Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+version = "1.11.0"
+
+[[deps.InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+version = "1.11.0"
+
+[[deps.Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+version = "1.11.0"
+
+[[deps.Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+version = "1.11.0"
+
+[[deps.Random]]
+deps = ["SHA"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+version = "1.11.0"
+
+[[deps.SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
+
+[[deps.Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+version = "1.11.0"
+
+[[deps.Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+version = "1.11.0"

test/Project.toml

@@ -0,0 +1,2 @@
+[deps]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/runtest.jl

@@ -1,44 +0,0 @@
-using Revise
-using GeneralUtils, MQTTClient, JSON3
-
-mqttMsgReceiveTopic = ["/receivetopic_1", "/receivetopic_2"]
-mqttMsgReceiveChannel = (ch1=Channel(8), ch2=Channel(32))
-keepaliveChannel = Channel(8)
-function onMsgCallback(topic, payload)
-            jobj = JSON3.read(String(payload))
-            incomingMqttMsg = copy(jobj)  # convert json object into julia dictionary recursively
-            
-            if occursin("topic_1", topic)
-              put!(mqttMsgReceiveChannel[:ch1], incomingMqttMsg)
-            elseif occursin("topic_2", topic)
-              put!(mqttMsgReceiveChannel[:ch2], incomingMqttMsg)
-            elseif occursin("keepalive", topic)
-              put!(keepaliveChannel, incomingMqttMsg)
-            else
-              println("undefined condition ", @__FILE__, " ", @__LINE__)
-            end
-          end
-mqttInstance = GeneralUtils.mqttClientInstance_v2(
-                          "mqtt.yiem.cc",
-                          mqttMsgReceiveTopic,
-                          mqttMsgReceiveChannel,
-                          keepaliveChannel,
-                          onMsgCallback
-                        )
-
-
-_ = GeneralUtils.checkMqttConnection!(mqttInstance)
-
-
-println("GeneralUtils test done")
-
-
-
-
-
-
-
-
-
-
-

test/runtests.jl

@@ -0,0 +1,39 @@
+using Test
+using GeneralUtils: detect_keyword
+
+@testset "detect_keyword tests" begin
+    @test detect_keyword(["test"], "this is a test") == Dict("test" => 1)
+    
+    @test detect_keyword(["hello", "world"], "hello world hello") == Dict("hello" => 2, "world" => 1)
+    
+    @test detect_keyword(["cat"], "category") == Dict("cat" => 1)
+    
+    @test detect_keyword(["cat"], "category"; mode="individual") == Dict("cat" => 0)
+    
+    @test detect_keyword(["dog"], "dogs and cats"; mode="individual", delimiter=[' ']) == Dict("dog" => 0)
+    
+    @test detect_keyword(["test"], "test.case"; mode="individual", delimiter=['.']) == Dict("test" => 1)
+    
+    @test detect_keyword(["word"], "") == Dict("word" => 0)
+    
+    @test detect_keyword(String[], "some text") == Dict{String, Integer}()
+    
+    @test detect_keyword(["a", "b"], "a.b\nc"; delimiter=['.', '\n']) == Dict("a" => 1, "b" => 1)
+    
+    multiline_text = """
+    first line
+    second line
+    first word
+    """
+    @test detect_keyword(["first"], multiline_text) == Dict("first" => 2)
+    
+    @test detect_keyword(["word"], "word"; mode="individual") == Dict("word" => 1)
+    
+    @test detect_keyword(["test"], "testing.test.tester"; mode="individual", delimiter=['.']) == Dict("test" => 1)
+end
+
+
+
+
+
+