1st commit
This commit is contained in:
@@ -0,0 +1,172 @@
|
||||
|
||||
using JSON, Dates, UUIDs, PrettyPrinting, Base64, DataFrames, NATS, HTTP, Base.Threads
|
||||
using GeneralUtils, msghandler
|
||||
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
const config = JSON.parsefile("./appconfig.json")
|
||||
println("Config loaded")
|
||||
|
||||
# ----------------------------------------- healthserver ----------------------------------------- #
|
||||
function healthcheck(req::HTTP.Request)
|
||||
return HTTP.Response(200, JSON.json(Dict("status"=>"ok")))
|
||||
end
|
||||
const ROUTER = HTTP.Router()
|
||||
HTTP.register!(ROUTER, "GET", "/ready", healthcheck)
|
||||
HTTP.register!(ROUTER, "GET", "/health", healthcheck)
|
||||
HTTP.register!(ROUTER, "GET", "/", healthcheck)
|
||||
healthserver = HTTP.serve!(ROUTER, "0.0.0.0", 3000)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ #
|
||||
# registration helper function #
|
||||
# ------------------------------------------------------------------------------------------------ #
|
||||
"""
|
||||
register_service(connection, service_name, topic, config)
|
||||
|
||||
Register a service with the load balancer via NATS request.
|
||||
|
||||
# Arguments
|
||||
- `connection`: NATS connection object
|
||||
- `service_name`: Name of the service being registered
|
||||
- `topic`: NATS topic/channel of the registering service
|
||||
- `config`: Configuration dictionary containing service URLs
|
||||
|
||||
# Returns
|
||||
- Response from the load balancer NATS request
|
||||
|
||||
Registers the service with the servicesloadbalancer topic using msghandler.smartpack
|
||||
to create the registration message envelope.
|
||||
"""
|
||||
function register_service(connection, service_name, topic, config)
|
||||
register_dict = Dict(
|
||||
"register_service_name" => service_name,
|
||||
"topic" => topic,
|
||||
"free_or_paid" => "free"
|
||||
)
|
||||
register_payloads = [("register", register_dict, "dictionary")]
|
||||
_, register_msg_envelope_json_str = msghandler.smartpack(
|
||||
config["externalService"]["servicesloadbalancer"]["url"],
|
||||
register_payloads;
|
||||
msg_purpose="register",
|
||||
broker_url=config["nats_server_info"]["url"],
|
||||
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||
try
|
||||
reply = NATS.request(connection,
|
||||
config["externalService"]["servicesloadbalancer"]["url"],
|
||||
register_msg_envelope_json_str,
|
||||
timeout=60)
|
||||
println("service $service_name is registered")
|
||||
catch
|
||||
println("registering service $service_name failed")
|
||||
end
|
||||
end
|
||||
|
||||
ch = Channel(8)
|
||||
flag = Channel(1)
|
||||
# Initialize NATS client
|
||||
connection = NATS.connect(config["nats_server_info"]["url"])
|
||||
|
||||
# use reply() here so that when this_service scale to >1 instance,
|
||||
# NATS will randomly select only one receiving instance
|
||||
sub1 = NATS.subscribe(connection,
|
||||
config["this_service_input_channel"]["nats"][1];
|
||||
queue_group="group1", spawn=true) do msg
|
||||
put!(ch, msg)
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ #
|
||||
# this service main loop #
|
||||
# ------------------------------------------------------------------------------------------------ #
|
||||
Threads.@spawn begin # Register service periodically
|
||||
while true
|
||||
if isready(flag)
|
||||
register_service(connection,
|
||||
config["this_service_name"],
|
||||
config["this_service_input_channel"]["nats"][1],
|
||||
config)
|
||||
end
|
||||
sleep(60) # this line should make spawned thread sleep while allowing main thread running
|
||||
end
|
||||
end
|
||||
|
||||
function main()
|
||||
|
||||
# ---------------------------------- register service initially ---------------------------------- #
|
||||
register_service(connection,
|
||||
config["this_service_name"],
|
||||
config["this_service_input_channel"]["nats"][1],
|
||||
config)
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
println("NATS to HTTP forwarder service for llamacpp_chat_completion service is ready to accept NATS messages at broker=$(config["nats_server_info"]["url"]), topic=$(config["this_service_input_channel"]["nats"][1])")
|
||||
while true
|
||||
put!(flag, "ready") # ready for new request. do not send register message during llm working
|
||||
incoming_nats_msg = take!(ch) # keep this service running.
|
||||
println("processing new request")
|
||||
|
||||
# start working, the work may take more than a few minutes. pause async loop registeration to
|
||||
# prevent a loadbalancer assigning new work to this service.
|
||||
_ = take!(flag)
|
||||
|
||||
incoming_envelope_json_str = String(incoming_nats_msg.payload)
|
||||
incoming_envelope = msghandler.smartunpack(incoming_envelope_json_str)
|
||||
if incoming_envelope["msg_purpose"] == "ping"
|
||||
outgoing_envelope, outgoing_envelope_json_str =
|
||||
msghandler.smartpack("requester inbox",
|
||||
[("response", "available", "text")];
|
||||
broker_url=config["nats_server_info"]["url"],
|
||||
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||
NATS.publish(connection, incoming_nats_msg.reply_to, outgoing_envelope_json_str)
|
||||
else
|
||||
incoming_text_array = incoming_envelope["payloads"][1][2]
|
||||
body_data = Dict(
|
||||
"model" => "embeddinggemma-300M-BF16",
|
||||
"input" => incoming_text_array
|
||||
)
|
||||
headers = [
|
||||
"Content-Type" => "application/json",
|
||||
"Authorization" => "Bearer no-key"
|
||||
]
|
||||
http_response = HTTP.post(config["externalService"]["llamacpp_chat_completion"]["url"],
|
||||
headers, JSON.json(body_data))
|
||||
|
||||
# HTTP.jl doesn't immediately close connections after post() - it pools them for reuse.
|
||||
# Without explicit cleanup, connections accumulate and the llama.cpp server times out
|
||||
# on subsequent requests. This forces connection pool cleanup between requests.
|
||||
HTTP.close_idle_connections!()
|
||||
http_response_body = JSON.parse(http_response.body)
|
||||
outgoing_envelope, outgoing_envelope_json_str =
|
||||
msghandler.smartpack("requester inbox",
|
||||
[("http_response", http_response_body, "dictionary")];
|
||||
broker_url=config["nats_server_info"]["url"],
|
||||
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||
# re-register this service to let a loadbalancer knows that this service is ready for next request
|
||||
register_service(connection,
|
||||
config["this_service_name"],
|
||||
config["this_service_input_channel"]["nats"][1],
|
||||
config)
|
||||
NATS.publish(connection, incoming_envelope.reply_to, outgoing_envelope_json_str)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
main()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user