1st commit
This commit is contained in:
Executable
+83
@@ -0,0 +1,83 @@
|
|||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# sommpanion/wine-db-api/.drone.yml #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
kind: pipeline
|
||||||
|
type: docker
|
||||||
|
name: build-and-push
|
||||||
|
|
||||||
|
steps:
|
||||||
|
# Step 1: Normal pushes — any branch except tags
|
||||||
|
- name: build-push-dev
|
||||||
|
image: plugins/docker:latest
|
||||||
|
settings:
|
||||||
|
registry: registry.yiem.cc
|
||||||
|
repo: registry.yiem.cc/sommpanion/wine-db-api
|
||||||
|
username:
|
||||||
|
from_secret: registry_Caddy_username # fill in registry.yiem.cc's Caddy basic auth secret in Drone UI
|
||||||
|
password:
|
||||||
|
from_secret: registry_Caddy_password
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
context: .
|
||||||
|
buildkit: true
|
||||||
|
platforms:
|
||||||
|
- linux/amd64
|
||||||
|
- linux/arm64
|
||||||
|
tags:
|
||||||
|
- latest
|
||||||
|
- sha-${DRONE_COMMIT_SHA:0:8}
|
||||||
|
- build-${DRONE_BUILD_NUMBER}
|
||||||
|
- ${DRONE_BRANCH}
|
||||||
|
when:
|
||||||
|
event:
|
||||||
|
- push
|
||||||
|
branch:
|
||||||
|
exclude:
|
||||||
|
- refs/tags/*
|
||||||
|
|
||||||
|
- name: build-push-pr
|
||||||
|
image: plugins/docker:latest
|
||||||
|
settings:
|
||||||
|
registry: registry.yiem.cc
|
||||||
|
repo: registry.yiem.cc/sommpanion/wine-db-api
|
||||||
|
username:
|
||||||
|
from_secret: registry_Caddy_username # fill in registry.yiem.cc's Caddy basic auth secret in Drone UI
|
||||||
|
password:
|
||||||
|
from_secret: registry_Caddy_password
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
context: .
|
||||||
|
buildkit: true
|
||||||
|
platforms:
|
||||||
|
- linux/amd64
|
||||||
|
- linux/arm64
|
||||||
|
tags:
|
||||||
|
- pr-${DRONE_PULL_REQUEST}
|
||||||
|
- sha-${DRONE_COMMIT_SHA:0:8}
|
||||||
|
- build-${DRONE_BUILD_NUMBER}
|
||||||
|
when:
|
||||||
|
event:
|
||||||
|
- pull_request
|
||||||
|
|
||||||
|
- name: build-push-release
|
||||||
|
image: plugins/docker:latest
|
||||||
|
settings:
|
||||||
|
registry: registry.yiem.cc
|
||||||
|
repo: registry.yiem.cc/sommpanion/wine-db-api
|
||||||
|
username:
|
||||||
|
from_secret: registry_Caddy_username # fill in registry.yiem.cc's Caddy basic auth secret in Drone UI
|
||||||
|
password:
|
||||||
|
from_secret: registry_Caddy_password
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
context: .
|
||||||
|
buildkit: true
|
||||||
|
platforms:
|
||||||
|
- linux/amd64
|
||||||
|
- linux/arm64
|
||||||
|
tags:
|
||||||
|
- latest
|
||||||
|
- sha-${DRONE_COMMIT_SHORT}
|
||||||
|
- build-${DRONE_BUILD_NUMBER}
|
||||||
|
- ${DRONE_TAG}
|
||||||
|
when:
|
||||||
|
event:
|
||||||
|
- tag
|
||||||
|
|
||||||
Executable
+107
@@ -0,0 +1,107 @@
|
|||||||
|
# FROM nvidia/cuda:12.2.0-devel-ubuntu20.04
|
||||||
|
FROM julia:1.13
|
||||||
|
# FROM debian:latest
|
||||||
|
|
||||||
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
|
# install required APT packages
|
||||||
|
RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y \
|
||||||
|
# build-essential \
|
||||||
|
# busybox \
|
||||||
|
# g++ \
|
||||||
|
# gcc \
|
||||||
|
# wget \
|
||||||
|
# net-tools \
|
||||||
|
# iputils-ping \
|
||||||
|
# unzip \
|
||||||
|
# unixodbc \
|
||||||
|
# unixodbc-dev \
|
||||||
|
# libicu-dev \
|
||||||
|
# nano \
|
||||||
|
# libmosquitto-dev \
|
||||||
|
# git \
|
||||||
|
# ffmpeg \
|
||||||
|
# libsm6 \
|
||||||
|
# libxext6 \
|
||||||
|
# tar \
|
||||||
|
# zip \
|
||||||
|
# libssl-dev \
|
||||||
|
# # python3 \
|
||||||
|
# # python3-pip \
|
||||||
|
# # python-is-python3 \
|
||||||
|
# postgresql-client \
|
||||||
|
# cargo \
|
||||||
|
# procps \
|
||||||
|
curl
|
||||||
|
|
||||||
|
# # For webapp frontend
|
||||||
|
# RUN apt-get update && apt-get install -y nginx
|
||||||
|
# COPY nginx.conf /etc/nginx/nginx.conf
|
||||||
|
# # Copy your static website files to the Nginx HTML directory
|
||||||
|
# COPY . /usr/share/nginx/html
|
||||||
|
|
||||||
|
# # install nodejs https://deb.nodesource.com/
|
||||||
|
# RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - && \
|
||||||
|
# apt-get update && apt-get install -y nodejs
|
||||||
|
|
||||||
|
# set up the app
|
||||||
|
RUN mkdir /appfolder
|
||||||
|
# RUN mkdir /appfolder/mountvolume
|
||||||
|
RUN mkdir /appfolder/app
|
||||||
|
RUN mkdir /appfolder/app/temp
|
||||||
|
COPY . /appfolder/app/temp
|
||||||
|
COPY ./env_preparation.jl /appfolder/app
|
||||||
|
# RUN mkdir /appfolder
|
||||||
|
# COPY . /appfolder
|
||||||
|
|
||||||
|
# install Conda as primary python environment with specified python version
|
||||||
|
WORKDIR /appfolder
|
||||||
|
# RUN PATH="${HOME}/conda/bin:${PATH}" \
|
||||||
|
# # && wget -O Miniforge3.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh \
|
||||||
|
# && wget -O Miniforge3.sh https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh \
|
||||||
|
# && bash Miniforge3.sh -b -p "${HOME}/conda" \
|
||||||
|
# && rm -f Miniforge3.sh \
|
||||||
|
# && echo "Running $(conda --version)" \
|
||||||
|
# && conda init bash \
|
||||||
|
# && . /root/.bashrc \
|
||||||
|
# && conda update conda -y \
|
||||||
|
# && conda install python=3.10 -y
|
||||||
|
|
||||||
|
# # install pip into conda's base env
|
||||||
|
# RUN PATH="${HOME}/conda/bin:${PATH}" \
|
||||||
|
# && conda install pip \
|
||||||
|
# && pip install --trusted-host pypi.python.org -r required_python_packages.txt \
|
||||||
|
# # && CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS -DLLAMA_AVX2=OFF -DLLAMA_F16C=OFF -DLLAMA_FMA=OFF" FORCE_CMAKE=1 pip install llama-cpp-python --no-cache-dir
|
||||||
|
# && CMAKE_ARGS="-DLLAMA_CUBLAS=on -DLLAMA_AVX2=OFF -DLLAMA_F16C=OFF -DLLAMA_FMA=OFF" FORCE_CMAKE=1 pip install llama-cpp-python==0.2.77 --no-cache-dir
|
||||||
|
# # https://github.com/abetlen/llama-cpp-python/issues/412 old CPU cause CUBLAS compile problem
|
||||||
|
|
||||||
|
# # to be able to use CMD python -m at the last line
|
||||||
|
# ENV PATH=/root/conda/bin:$PATH
|
||||||
|
|
||||||
|
# # install powershell
|
||||||
|
# WORKDIR /appfolder/app/temp
|
||||||
|
# RUN wget https://github.com/PowerShell/PowerShell/releases/download/v7.4.5/powershell_7.4.5-1.deb_amd64.deb
|
||||||
|
# RUN dpkg -i powershell_7.4.5-1.deb_amd64.deb
|
||||||
|
# RUN apt-get install -f
|
||||||
|
|
||||||
|
# using juliaup
|
||||||
|
# RUN curl -fsSL https://install.julialang.org | sh -s -- -y
|
||||||
|
# SHELL ["/bin/bash", "--login" , "-c"]
|
||||||
|
|
||||||
|
# install julia package for my app
|
||||||
|
WORKDIR /appfolder/app
|
||||||
|
RUN julia -t 1 env_preparation.jl
|
||||||
|
|
||||||
|
|
||||||
|
# Make port 80 available to the world outside this container. If I use --publish at docker run command, there is no need to use EXPOSE
|
||||||
|
# EXPOSE 1883
|
||||||
|
|
||||||
|
# Health check server port
|
||||||
|
EXPOSE 3000
|
||||||
|
|
||||||
|
# Run app when the container launches
|
||||||
|
WORKDIR /appfolder/app
|
||||||
|
RUN julia --project -t 1 -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'
|
||||||
|
CMD ["julia", "--project", "-t", "2", "/appfolder/app/main.jl"]
|
||||||
|
# CMD ["nginx", "-g", "daemon off;"]
|
||||||
|
# CMD [ "sleep", "infinity" ]
|
||||||
Executable
+1362
File diff suppressed because it is too large
Load Diff
Executable
+23
@@ -0,0 +1,23 @@
|
|||||||
|
[deps]
|
||||||
|
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
|
||||||
|
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
|
||||||
|
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
|
||||||
|
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
|
||||||
|
FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
|
||||||
|
GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
|
||||||
|
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
||||||
|
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
|
||||||
|
LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1"
|
||||||
|
NATS = "55e73f9c-eeeb-467f-b4cc-a633fde63d2a"
|
||||||
|
PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337"
|
||||||
|
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
|
||||||
|
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
|
||||||
|
Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
|
||||||
|
URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
|
||||||
|
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
||||||
|
ZipArchives = "49080126-0e18-4c2a-b176-c102e4b3760c"
|
||||||
|
msghandler = "f2724d33-f338-4a57-b9f8-1be882570d10"
|
||||||
|
|
||||||
|
[sources]
|
||||||
|
GeneralUtils = {rev = "main", url = "https://git.yiem.cc/ton/GeneralUtils"}
|
||||||
|
msghandler = {rev = "main", url = "https://git.yiem.cc/ton/msghandler"}
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
# llamacpp_embedding_1_nats
|
||||||
|
|
||||||
|
A Julia backend service that forwards messages from NATS to llama.cpp embedding API.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
This service acts as a bridge between a NATS message broker and a llama.cpp HTTP embedding endpoint. It receives messages via NATS queue groups and forwards them to the llama.cpp server for embedding processing.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────┐ NATS ┌──────────────────────┐ HTTP ┌─────────────────┐
|
||||||
|
│ Requester │ ────────────> │ this service │ ────────────> │ llama.cpp │
|
||||||
|
│ │ <─────────── <│ (embedding) │ <─────────── <│ /v1/embeddings │
|
||||||
|
│ │ Queue │ │ │ │
|
||||||
|
│ │ Group │ │ │ │
|
||||||
|
└─────────────┘ (free) └──────────────────────┘ └─────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- NATS queue group subscription for load balancing across multiple service instances
|
||||||
|
- HTTP connection pooling cleanup to prevent llama.cpp server timeouts
|
||||||
|
- Health check endpoint on port 3000
|
||||||
|
- Service registration with load balancer
|
||||||
|
- Message envelope packing/unpacking via `msghandler` package
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
- Julia 1.12+
|
||||||
|
- NATS server
|
||||||
|
- llama.cpp server running with embedding endpoint
|
||||||
|
- Docker (for containerized deployment)
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
Edit `appconfig.json` to configure the service:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"config_version": "0.1",
|
||||||
|
"nats_server_info": {
|
||||||
|
"url": "nats.yiem.cc"
|
||||||
|
},
|
||||||
|
"this_service_name": "embedding",
|
||||||
|
"this_service_input_channel": {
|
||||||
|
"nats": "sommpanion.backend.embedding_nats_1.v1.tonpc.inbox"
|
||||||
|
},
|
||||||
|
"externalService": {
|
||||||
|
"fileserver": {
|
||||||
|
"url": "https://fileserver.yiem.cc"
|
||||||
|
},
|
||||||
|
"servicesloadbalancer": {
|
||||||
|
"url": "sommpanion.backend.servicesloadbalancer.v1.inbox"
|
||||||
|
},
|
||||||
|
"llamacpp_embedding": {
|
||||||
|
"url": "http://192.168.88.50:9010/v1/embeddings"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Local Development
|
||||||
|
|
||||||
|
```bash
|
||||||
|
julia --project -t 2 main.jl
|
||||||
|
```
|
||||||
|
|
||||||
|
The service exposes:
|
||||||
|
- Health check endpoints on port 3000: `/`, `/health`, `/ready`
|
||||||
|
- NATS subscriber on configured topic
|
||||||
|
|
||||||
|
## Docker Deployment
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build -t llamacpp_embedding_1_nats .
|
||||||
|
docker run -d --name llamacpp_embedding_1_nats -p 3000:3000 llamacpp_embedding_1_nats
|
||||||
|
```
|
||||||
|
|
||||||
|
Or use docker-compose:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage Example
|
||||||
|
|
||||||
|
### Using NATS with msghandler
|
||||||
|
|
||||||
|
```julia
|
||||||
|
using JSON, NATS, msghandler
|
||||||
|
|
||||||
|
const config = JSON.parsefile("./appconfig.json")
|
||||||
|
|
||||||
|
# Prepare documents to embed
|
||||||
|
documents = [
|
||||||
|
"System architecture specification for heart monitor logic.",
|
||||||
|
"NATS broker cluster configuration options for low-latency.",
|
||||||
|
"Julia signal processing worker initialization sequence."
|
||||||
|
]
|
||||||
|
|
||||||
|
documents_dict = Dict("documents" => documents)
|
||||||
|
|
||||||
|
# Pack message envelope
|
||||||
|
payloads = [("documents", documents_dict, "dictionary")]
|
||||||
|
message_envelope, forward_msg_envelope_json_str = msghandler.smartpack(
|
||||||
|
config["this_service_input_channel"]["nats"],
|
||||||
|
payloads;
|
||||||
|
msg_purpose="embedding",
|
||||||
|
broker_url=config["nats_server_info"]["url"],
|
||||||
|
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||||
|
|
||||||
|
# Send via NATS
|
||||||
|
connection = NATS.connect(config["nats_server_info"]["url"])
|
||||||
|
reply = NATS.request(connection,
|
||||||
|
config["this_service_input_channel"]["nats"],
|
||||||
|
forward_msg_envelope_json_str,
|
||||||
|
timeout=120)
|
||||||
|
|
||||||
|
# Unpack response
|
||||||
|
incoming_env_json_str = String(reply.payload)
|
||||||
|
incoming_env = msghandler.smartunpack(incoming_env_json_str)
|
||||||
|
embedding_response = incoming_env["payloads"][1][2]
|
||||||
|
|
||||||
|
# Process embeddings
|
||||||
|
for (i, item) in enumerate(embedding_response["data"])
|
||||||
|
vector = item["embedding"]
|
||||||
|
println("Document $i embedding vector size: ", length(vector))
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
### Using Direct HTTP (for testing)
|
||||||
|
|
||||||
|
```julia
|
||||||
|
using HTTP, JSON
|
||||||
|
|
||||||
|
url = "http://localhost:9010/v1/embeddings"
|
||||||
|
headers = [
|
||||||
|
"Content-Type" => "application/json",
|
||||||
|
"Authorization" => "Bearer no-key"
|
||||||
|
]
|
||||||
|
|
||||||
|
documents = [
|
||||||
|
"System architecture specification for heart monitor logic.",
|
||||||
|
"NATS broker cluster configuration options for low-latency."
|
||||||
|
]
|
||||||
|
|
||||||
|
body_data = Dict(
|
||||||
|
"model" => "embeddinggemma-300M-BF16",
|
||||||
|
"input" => documents
|
||||||
|
)
|
||||||
|
|
||||||
|
response = HTTP.post(url, headers, JSON.json(body_data))
|
||||||
|
parsed = JSON.parse(String(response.body))
|
||||||
|
|
||||||
|
for (i, item) in enumerate(parsed["data"])
|
||||||
|
println("Document $i embedding size: ", length(item["embedding"]))
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
## Message Flow
|
||||||
|
|
||||||
|
1. **Registration**: Service registers with load balancer via NATS
|
||||||
|
2. **Subscription**: Subscribes to input topic with queue group "group1"
|
||||||
|
3. **Processing**:
|
||||||
|
- Receives message via NATS
|
||||||
|
- Unpacks message envelope
|
||||||
|
- Forwards to llama.cpp HTTP endpoint
|
||||||
|
- Cleans up HTTP connections
|
||||||
|
- Repacks response and publishes to reply-to topic
|
||||||
|
- Re-registers with load balancer
|
||||||
|
|
||||||
|
## Health Monitoring
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:3000/health
|
||||||
|
curl http://localhost:3000/ready
|
||||||
|
```
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
- `NATS.jl` - NATS client
|
||||||
|
- `HTTP.jl` - HTTP client with connection pooling
|
||||||
|
- `JSON.jl` - JSON serialization
|
||||||
|
- `msghandler` - Message envelope handling
|
||||||
|
- `GeneralUtils` - Utility functions
|
||||||
|
|
||||||
|
## Scaling
|
||||||
|
|
||||||
|
Multiple instances can be deployed. NATS queue group "group1" ensures each message is delivered to only one instance.
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT
|
||||||
Executable
+99
@@ -0,0 +1,99 @@
|
|||||||
|
{
|
||||||
|
"config_version": "0.1",
|
||||||
|
"nats_server_info": {
|
||||||
|
"description": "nats server",
|
||||||
|
"url": "nats.yiem.cc"
|
||||||
|
},
|
||||||
|
"thisServiceName": "embedding",
|
||||||
|
"thisServiceInputChannel": {
|
||||||
|
"inbox": "embedding.janeWhitePC.v1.inbox"
|
||||||
|
},
|
||||||
|
"thisServiceOutputChannel": {
|
||||||
|
"status": "embedding.janeWhitePC.v1.status"
|
||||||
|
},
|
||||||
|
"externalService": {
|
||||||
|
"servicesloadbalancer": {
|
||||||
|
"description": "load balancer",
|
||||||
|
"url": "serviceLoadBalancer.v1.inbox"
|
||||||
|
},
|
||||||
|
"llamacpp_embedding": {
|
||||||
|
"description": "llamacpp_embedding",
|
||||||
|
"url": "https://llmembedding.yiem.cc/v1/embeddings"
|
||||||
|
},
|
||||||
|
"objectStorage": {
|
||||||
|
"description": "shared S3 object storage for file uploads/downloads",
|
||||||
|
"endpoint": "https://s3-api.yiem.cc",
|
||||||
|
"access_key_id": "GKb080154a2e5b19100b1b2c6e",
|
||||||
|
"secret_access_key": "a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||||
|
"bucket": "sommpanion"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Executable
+71
@@ -0,0 +1,71 @@
|
|||||||
|
# A Docker Compose must always start with the version tag.
|
||||||
|
# We use '3' because it's the last version.
|
||||||
|
version: "3.8"
|
||||||
|
|
||||||
|
# You should know that Docker Compose works with services.
|
||||||
|
# 1 service = 1 container.
|
||||||
|
# For example, a service, a server, a client, a database...
|
||||||
|
# We use the keyword 'services' to start to create services.
|
||||||
|
services:
|
||||||
|
wine-db-api-v1: # service name
|
||||||
|
container_name: llamacpp_embedding_1_nats
|
||||||
|
build: .
|
||||||
|
image: llamacpp_embedding_1_nats
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
- TZ=Asia/Bangkok
|
||||||
|
# network_mode: "app-network"
|
||||||
|
# networks:
|
||||||
|
# - backend
|
||||||
|
# ports:
|
||||||
|
# - 3000:3000
|
||||||
|
# volumes:
|
||||||
|
# - ./mountvolume:/appfolder/mountvolume
|
||||||
|
#privileged: true
|
||||||
|
deploy:
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpus: '2'
|
||||||
|
# memory: 7200M
|
||||||
|
# reservations:
|
||||||
|
# devices:
|
||||||
|
# - driver: nvidia
|
||||||
|
# device_ids: ['2', '3'] # choose between device_ids or count
|
||||||
|
# count: 1
|
||||||
|
# capabilities: [gpu]
|
||||||
|
|
||||||
|
# # The name of our service is "database"
|
||||||
|
# # but you can use the name of your choice.
|
||||||
|
# # Note: This may change the commands you are going to use a little bit.
|
||||||
|
# database:
|
||||||
|
# # Official Postgres image from DockerHub (we use the last version)
|
||||||
|
# image: 'postgres:latest'
|
||||||
|
|
||||||
|
# # By default, a Postgres database is running on the 5432 port.
|
||||||
|
# # If we want to access the database from our computer (outside the container),
|
||||||
|
# # we must share the port with our computer's port.
|
||||||
|
# # The syntax is [port we want on our machine]:[port we want to retrieve in the container]
|
||||||
|
# # Note: You are free to change your computer's port,
|
||||||
|
# # but take into consideration that it will change the way
|
||||||
|
# # you are connecting to your database.
|
||||||
|
# networks:
|
||||||
|
# - backend
|
||||||
|
# ports:
|
||||||
|
# - 5432:5432
|
||||||
|
|
||||||
|
# environment:
|
||||||
|
# POSTGRES_USER: username # The PostgreSQL user (useful to connect to the database)
|
||||||
|
# POSTGRES_PASSWORD: password # The PostgreSQL password (useful to connect to the database)
|
||||||
|
|
||||||
|
# volumes:
|
||||||
|
# # In this example, we share the folder `db-data` in our root repository, with the default PostgreSQL data path.
|
||||||
|
# # It means that every time the repository is modifying the data inside
|
||||||
|
# # of `/var/lib/postgresql/data/`, automatically the change will appear in `db-data`.
|
||||||
|
# # You don't need to create the `db-data` folder. Docker Compose will do it for you.
|
||||||
|
# - ./data/db-data/:/var/lib/postgresql/data/
|
||||||
|
|
||||||
|
# networks:
|
||||||
|
# default:
|
||||||
|
# name: https-network
|
||||||
|
# driver: bridge
|
||||||
|
# external: true
|
||||||
Executable
+286
@@ -0,0 +1,286 @@
|
|||||||
|
"""
|
||||||
|
For docker container:
|
||||||
|
- This script must be placed at the root of your project e.g. /appfolder/app
|
||||||
|
- This script expects all other files are in workfolder/temp e.g. /appfolder/app/temp
|
||||||
|
"""
|
||||||
|
|
||||||
|
using Pkg
|
||||||
|
|
||||||
|
programName = "someapp" # use my project name here
|
||||||
|
devPackageName = "somepackage"
|
||||||
|
appfolder = pwd() # use current folder as project folder
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
|
""" list of all recommend package
|
||||||
|
required_packages = [
|
||||||
|
"Arrow"
|
||||||
|
"BenchmarkTools"
|
||||||
|
"CSV"
|
||||||
|
"CUDA"
|
||||||
|
"ClickHouse"
|
||||||
|
"DataFrames"
|
||||||
|
"DataFramesMeta"
|
||||||
|
"DataStructures"
|
||||||
|
"Distributions"
|
||||||
|
"Enzyme"
|
||||||
|
"FileIO"
|
||||||
|
"Flux"
|
||||||
|
"Genie"
|
||||||
|
"HTTP"
|
||||||
|
"ImageTransformations"
|
||||||
|
"Images"
|
||||||
|
"IterTools"
|
||||||
|
"JSON3"
|
||||||
|
"LinearAlgebra"
|
||||||
|
"Logging"
|
||||||
|
"Lux"
|
||||||
|
"MLDataUtils"
|
||||||
|
"MLDatasets"
|
||||||
|
"MLLabelUtils"
|
||||||
|
"Makie"
|
||||||
|
"ODBC"
|
||||||
|
"Optimisers"
|
||||||
|
"ParameterSchedulers"
|
||||||
|
"PkgTemplates"
|
||||||
|
"ProgressMeter"
|
||||||
|
"PyCall"
|
||||||
|
"Random"
|
||||||
|
"Revise"
|
||||||
|
"Serialization"
|
||||||
|
"Statistics"
|
||||||
|
"Stipple"
|
||||||
|
"TensorBoardLogger"
|
||||||
|
"TextAnalysis"
|
||||||
|
"Transformers"
|
||||||
|
"UUIDs"
|
||||||
|
"WordTokenizers"
|
||||||
|
"Zygote"
|
||||||
|
"https://github.com/denglerchr/Mosquitto.jl"
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------- #
|
||||||
|
# add julia packages in container's general registry folder #
|
||||||
|
# ---------------------------------------------------------------------------- #
|
||||||
|
privatejuliapkgpath = "$appfolder/dev"
|
||||||
|
required_julia_private_packages = [ #CHANGE
|
||||||
|
# "https://git.yiem.cc/ton/GeneralUtils",
|
||||||
|
# "https://git.yiem.cc/ton/LLMMCTS",
|
||||||
|
# "https://git.yiem.cc/ton/SQLLLM",
|
||||||
|
# "https://git.yiem.cc/ton/YiemAgent -b v0.1.1",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------- #
|
||||||
|
# add julia package in app folder #
|
||||||
|
# ---------------------------------------------------------------------------- #
|
||||||
|
required_julia_packages = [ #CHANGE
|
||||||
|
# "Revise",
|
||||||
|
# "CondaPkg",
|
||||||
|
# "PythonCall",
|
||||||
|
# "HTTP",
|
||||||
|
# "Oxygen",
|
||||||
|
# "JSON",
|
||||||
|
# "Dates",
|
||||||
|
# "UUIDs",
|
||||||
|
# "Random",
|
||||||
|
# "URIs",
|
||||||
|
# "DataStructures",
|
||||||
|
# "FileIO",
|
||||||
|
# "PrettyPrinting",
|
||||||
|
# "ZipArchives",
|
||||||
|
# "LibPQ",
|
||||||
|
# "ProgressMeter",
|
||||||
|
# "NATS"
|
||||||
|
] # use only for new project # use only for new project
|
||||||
|
|
||||||
|
function copy_folder_contents(src_folder::AbstractString, dst_folder::AbstractString)
|
||||||
|
# Get a list of all files in the source folder
|
||||||
|
files = readdir(src_folder)
|
||||||
|
|
||||||
|
# Iterate over each file and copy it to the destination folder
|
||||||
|
for itemname in files
|
||||||
|
src_path = joinpath(src_folder, itemname)
|
||||||
|
dst_path = joinpath(dst_folder, itemname)
|
||||||
|
if isdir(src_path)
|
||||||
|
run(`cp -r $src_path $dst_path`)
|
||||||
|
else
|
||||||
|
run(`cp $src_path $dst_path`)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
function install_required_julia_packages(required_julia_packages)
|
||||||
|
for i in required_julia_packages
|
||||||
|
println("adding ", i)
|
||||||
|
if startswith(i, "http")
|
||||||
|
Pkg.add(url=i)
|
||||||
|
else
|
||||||
|
Pkg.add(i)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
function install_required_julia_private_packages(required_julia_private_packages::Vector,
|
||||||
|
appfolderpath::String, privatejuliapkgpath::String)
|
||||||
|
|
||||||
|
!isdir(privatejuliapkgpath) ? mkpath(privatejuliapkgpath) : nothing
|
||||||
|
cd(privatejuliapkgpath)
|
||||||
|
|
||||||
|
if length(required_julia_private_packages) > 0
|
||||||
|
# remove private packages already in Project.toml in case private package path is different
|
||||||
|
for i in required_julia_private_packages
|
||||||
|
_pkgname = split(i, "/")[end]
|
||||||
|
_pkgname = split(_pkgname, " ")[1]
|
||||||
|
pkgname = split(_pkgname, ".")[1]
|
||||||
|
println("removing Julia private package: $pkgname if it already in Project.toml")
|
||||||
|
try Pkg.rm(pkgname) catch end
|
||||||
|
end
|
||||||
|
|
||||||
|
# clone all required julia private packages
|
||||||
|
for i in required_julia_private_packages
|
||||||
|
_pkgname = split(i, "/")[end]
|
||||||
|
_pkgname = split(_pkgname, " ")[1]
|
||||||
|
pkgname = split(_pkgname, ".")[1]
|
||||||
|
pkgpath = joinpath(privatejuliapkgpath, pkgname)
|
||||||
|
println("cloning Julia private package: $pkgname")
|
||||||
|
|
||||||
|
# only clone a package if it is not found in local
|
||||||
|
gitcommand = "git clone $i"
|
||||||
|
args = split(gitcommand, " ")
|
||||||
|
!isdir(pkgpath) ? run(`$args`) : nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
# install all required julia private packages
|
||||||
|
for i in required_julia_private_packages
|
||||||
|
_pkgname = split(i, "/")[end]
|
||||||
|
_pkgname = split(_pkgname, " ")[1]
|
||||||
|
pkgname = split(_pkgname, ".")[1] # in case there is a .jl at the end of package name
|
||||||
|
pkgpath = joinpath(privatejuliapkgpath, pkgname)
|
||||||
|
println("installing Julia private package: $pkgname")
|
||||||
|
|
||||||
|
Pkg.develop(path=pkgpath)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
cd(appfolderpath)
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# start script #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
|
||||||
|
# all files are put in ./temp (most likely in a docker container)
|
||||||
|
if isdir(joinpath(appfolder, "temp"))
|
||||||
|
@show "temp folder is available."
|
||||||
|
if isfile(joinpath(appfolder, "temp", "Project.toml")) # use this option if one already have project.toml
|
||||||
|
@show "there is Project.toml. instantiating..."
|
||||||
|
copy_folder_contents(joinpath(appfolder, "temp"), appfolder)
|
||||||
|
Pkg.activate(".")
|
||||||
|
install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath)
|
||||||
|
Pkg.instantiate()
|
||||||
|
else # no Project.toml, create new package
|
||||||
|
@show "there is no Project.toml. creating new package..."
|
||||||
|
copy_folder_contents(joinpath(appfolder, "temp"), appfolder)
|
||||||
|
Pkg.activate(".")
|
||||||
|
length(required_julia_packages) != 0 ? install_required_julia_packages(required_julia_packages) : nothing
|
||||||
|
|
||||||
|
# for app
|
||||||
|
write("main.jl", "using Revise # remove when this package is completed\n", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||||
|
mkdir("testapp")
|
||||||
|
cd("testapp")
|
||||||
|
write("runtests.jl", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||||
|
cd(appfolder)
|
||||||
|
install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath)
|
||||||
|
|
||||||
|
# create new dev package folder
|
||||||
|
!isdir(privatejuliapkgpath) ? mkpath(privatejuliapkgpath) : nothing
|
||||||
|
cd(privatejuliapkgpath)
|
||||||
|
Pkg.generate(devPackageName)
|
||||||
|
newPackagePath = joinpath(privatejuliapkgpath, devPackageName)
|
||||||
|
Pkg.develop(path=newPackagePath)
|
||||||
|
cd(newPackagePath)
|
||||||
|
mkdir("test")
|
||||||
|
cd("./test")
|
||||||
|
write("runtests.jl", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||||
|
cd(appfolder)
|
||||||
|
end
|
||||||
|
else # all files are in the current work folder
|
||||||
|
@show "temp folder is not available."
|
||||||
|
if isfile(joinpath(appfolder, "Project.toml")) # use this option if one already have project.toml
|
||||||
|
@show "there is Project.toml. instantiating..."
|
||||||
|
Pkg.activate(".")
|
||||||
|
install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath)
|
||||||
|
Pkg.instantiate()
|
||||||
|
else # create new package folder
|
||||||
|
@show "there is no Project.toml. creating new package..."
|
||||||
|
Pkg.activate(".")
|
||||||
|
length(required_julia_packages) != 0 ? install_required_julia_packages(required_julia_packages) : nothing
|
||||||
|
|
||||||
|
# for app
|
||||||
|
write("main.jl", "using Revise # remove when this package is completed\n", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||||
|
mkdir("testapp")
|
||||||
|
cd("testapp")
|
||||||
|
write("runtests.jl", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||||
|
cd(appfolder)
|
||||||
|
install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath)
|
||||||
|
|
||||||
|
# create new dev package folder
|
||||||
|
!isdir(privatejuliapkgpath) ? mkpath(privatejuliapkgpath) : nothing
|
||||||
|
cd(privatejuliapkgpath)
|
||||||
|
Pkg.generate(devPackageName)
|
||||||
|
newPackagePath = joinpath(privatejuliapkgpath, devPackageName)
|
||||||
|
Pkg.develop(path=newPackagePath)
|
||||||
|
cd(newPackagePath)
|
||||||
|
mkdir("test")
|
||||||
|
cd("./test")
|
||||||
|
write("runtests.jl", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||||
|
cd(appfolder)
|
||||||
|
|
||||||
|
# preparing typical julia project finished now doing project's specific preparation here
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------- #
|
||||||
|
# python libraries used by Julia's PythonCall #
|
||||||
|
# ---------------------------------------------------------------------------- #
|
||||||
|
pythonlib = []
|
||||||
|
channels = ["anaconda", "conda-forge", "pytorch"]
|
||||||
|
if length(pythonlib) != 0
|
||||||
|
using CondaPkg;
|
||||||
|
for i in channels CondaPkg.add_channel(i) end
|
||||||
|
for i in pythonlib CondaPkg.add_pip(i) end
|
||||||
|
end
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------- #
|
||||||
|
# create symlink #
|
||||||
|
# ---------------------------------------------------------------------------- #
|
||||||
|
# if isfile("/app/run.jl")
|
||||||
|
# symlink("/app/$programName/run.jl", "/app/run.jl")
|
||||||
|
# elseif isfile("/app/$programName/main.py")
|
||||||
|
# symlink("/app/$programName/main.py", "/app/main.py")
|
||||||
|
# else
|
||||||
|
# cd("/app")
|
||||||
|
# write("no run.jl in package folder.txt", "")
|
||||||
|
# end
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# for web app #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# intendedURL = "wine.yiem.cc/hq/agent/sommelier/frontend/dbadmin" # e.g. "wine.yiem.cc/hq/agent/sommelier/frontend/dbadmin"
|
||||||
|
# urlList = split(intendedURL, "/")[2:end] # e.g. ["hq", "agent", "sommelier", "frontend", "dbadmin"]
|
||||||
|
# # use joinpath to compose the path from urlList
|
||||||
|
# _path = joinpath(urlList...)
|
||||||
|
# path = "./$_path"
|
||||||
|
# mkpath(path)
|
||||||
|
# # move all files in the current folder to the path
|
||||||
|
# copy_folder_contents(joinpath(appfolder, "temp"), path)
|
||||||
|
|
||||||
|
|
||||||
|
println("--> Julia env preparation done")
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
|
||||||
|
using JSON, Dates, UUIDs, PrettyPrinting, Base64, DataFrames, NATS, HTTP, Base.Threads
|
||||||
|
using GeneralUtils, msghandler
|
||||||
|
|
||||||
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
const config = JSON.parsefile("./appconfig.json")
|
||||||
|
println("Config loaded")
|
||||||
|
|
||||||
|
# ----------------------------------------- healthserver ----------------------------------------- #
|
||||||
|
function healthcheck(req::HTTP.Request)
|
||||||
|
return HTTP.Response(200, JSON.json(Dict("status"=>"ok")))
|
||||||
|
end
|
||||||
|
const ROUTER = HTTP.Router()
|
||||||
|
HTTP.register!(ROUTER, "GET", "/ready", healthcheck)
|
||||||
|
HTTP.register!(ROUTER, "GET", "/health", healthcheck)
|
||||||
|
HTTP.register!(ROUTER, "GET", "/", healthcheck)
|
||||||
|
healthserver = HTTP.serve!(ROUTER, "0.0.0.0", 3000)
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# registration helper function #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
"""
|
||||||
|
register_service(connection, service_name, topic, config)
|
||||||
|
|
||||||
|
Register a service with the load balancer via NATS request.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `connection`: NATS connection object
|
||||||
|
- `service_name`: Name of the service being registered
|
||||||
|
- `topic`: NATS topic/channel of the registering service
|
||||||
|
- `config`: Configuration dictionary containing service URLs
|
||||||
|
|
||||||
|
# Returns
|
||||||
|
- Response from the load balancer NATS request
|
||||||
|
|
||||||
|
Registers the service with the servicesloadbalancer topic using msghandler.smartpack
|
||||||
|
to create the registration message envelope.
|
||||||
|
"""
|
||||||
|
function register_service(connection, service_name, topic, config)
|
||||||
|
register_dict = Dict(
|
||||||
|
"register_service_name" => service_name,
|
||||||
|
"topic" => topic,
|
||||||
|
"free_or_paid" => "free"
|
||||||
|
)
|
||||||
|
register_payloads = [("register", register_dict, "dictionary")]
|
||||||
|
_, register_msg_envelope_json_str = msghandler.smartpack(
|
||||||
|
config["externalService"]["servicesloadbalancer"]["url"],
|
||||||
|
register_payloads;
|
||||||
|
msg_purpose="register",
|
||||||
|
broker_url=config["nats_server_info"]["url"],
|
||||||
|
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||||
|
try
|
||||||
|
reply = NATS.request(connection,
|
||||||
|
config["externalService"]["servicesloadbalancer"]["url"],
|
||||||
|
register_msg_envelope_json_str,
|
||||||
|
timeout=60)
|
||||||
|
println("service $service_name is registered")
|
||||||
|
catch
|
||||||
|
println("registering service $service_name failed")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
ch = Channel(8)
|
||||||
|
flag = Channel(1)
|
||||||
|
# Initialize NATS client
|
||||||
|
connection = NATS.connect(config["nats_server_info"]["url"])
|
||||||
|
|
||||||
|
# use reply() here so that when this_service scale to >1 instance,
|
||||||
|
# NATS will randomly select only one receiving instance
|
||||||
|
sub1 = NATS.subscribe(connection,
|
||||||
|
config["this_service_input_channel"]["nats"][1];
|
||||||
|
queue_group="group1", spawn=true) do msg
|
||||||
|
put!(ch, msg)
|
||||||
|
end
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# this service main loop #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
Threads.@spawn begin # Register service periodically
|
||||||
|
while true
|
||||||
|
if isready(flag)
|
||||||
|
register_service(connection,
|
||||||
|
config["this_service_name"],
|
||||||
|
config["this_service_input_channel"]["nats"][1],
|
||||||
|
config)
|
||||||
|
end
|
||||||
|
sleep(60) # this line should make spawned thread sleep while allowing main thread running
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
function main()
|
||||||
|
|
||||||
|
# ---------------------------------- register service initially ---------------------------------- #
|
||||||
|
register_service(connection,
|
||||||
|
config["this_service_name"],
|
||||||
|
config["this_service_input_channel"]["nats"][1],
|
||||||
|
config)
|
||||||
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
println("NATS to HTTP forwarder service for llamacpp_chat_completion service is ready to accept NATS messages at broker=$(config["nats_server_info"]["url"]), topic=$(config["this_service_input_channel"]["nats"][1])")
|
||||||
|
while true
|
||||||
|
put!(flag, "ready") # ready for new request. do not send register message during llm working
|
||||||
|
incoming_nats_msg = take!(ch) # keep this service running.
|
||||||
|
println("processing new request")
|
||||||
|
|
||||||
|
# start working, the work may take more than a few minutes. pause async loop registeration to
|
||||||
|
# prevent a loadbalancer assigning new work to this service.
|
||||||
|
_ = take!(flag)
|
||||||
|
|
||||||
|
incoming_envelope_json_str = String(incoming_nats_msg.payload)
|
||||||
|
incoming_envelope = msghandler.smartunpack(incoming_envelope_json_str)
|
||||||
|
if incoming_envelope["msg_purpose"] == "ping"
|
||||||
|
outgoing_envelope, outgoing_envelope_json_str =
|
||||||
|
msghandler.smartpack("requester inbox",
|
||||||
|
[("response", "available", "text")];
|
||||||
|
broker_url=config["nats_server_info"]["url"],
|
||||||
|
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||||
|
NATS.publish(connection, incoming_nats_msg.reply_to, outgoing_envelope_json_str)
|
||||||
|
else
|
||||||
|
incoming_text_array = incoming_envelope["payloads"][1][2]
|
||||||
|
body_data = Dict(
|
||||||
|
"model" => "embeddinggemma-300M-BF16",
|
||||||
|
"input" => incoming_text_array
|
||||||
|
)
|
||||||
|
headers = [
|
||||||
|
"Content-Type" => "application/json",
|
||||||
|
"Authorization" => "Bearer no-key"
|
||||||
|
]
|
||||||
|
http_response = HTTP.post(config["externalService"]["llamacpp_chat_completion"]["url"],
|
||||||
|
headers, JSON.json(body_data))
|
||||||
|
|
||||||
|
# HTTP.jl doesn't immediately close connections after post() - it pools them for reuse.
|
||||||
|
# Without explicit cleanup, connections accumulate and the llama.cpp server times out
|
||||||
|
# on subsequent requests. This forces connection pool cleanup between requests.
|
||||||
|
HTTP.close_idle_connections!()
|
||||||
|
http_response_body = JSON.parse(http_response.body)
|
||||||
|
outgoing_envelope, outgoing_envelope_json_str =
|
||||||
|
msghandler.smartpack("requester inbox",
|
||||||
|
[("http_response", http_response_body, "dictionary")];
|
||||||
|
broker_url=config["nats_server_info"]["url"],
|
||||||
|
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||||
|
# re-register this service to let a loadbalancer knows that this service is ready for next request
|
||||||
|
register_service(connection,
|
||||||
|
config["this_service_name"],
|
||||||
|
config["this_service_input_channel"]["nats"][1],
|
||||||
|
config)
|
||||||
|
NATS.publish(connection, incoming_envelope.reply_to, outgoing_envelope_json_str)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
main()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -0,0 +1,226 @@
|
|||||||
|
|
||||||
|
using JSON, Dates, UUIDs, PrettyPrinting, Base64, DataFrames, NATS, HTTP, Base.Threads
|
||||||
|
using GeneralUtils, msghandler
|
||||||
|
|
||||||
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
const config = JSON.parsefile("./appconfig.json")
|
||||||
|
|
||||||
|
# Initialize NATS client
|
||||||
|
nats_conn = NATS.connect(config["nats_server_info"]["url"])
|
||||||
|
|
||||||
|
es = GeneralUtils.eventSink(natsConn=nats_conn,
|
||||||
|
topic=config["thisServiceOutputChannel"]["status"],
|
||||||
|
service_name=config["thisServiceName"])
|
||||||
|
|
||||||
|
function _log(msg::String)
|
||||||
|
es(msg)
|
||||||
|
end
|
||||||
|
|
||||||
|
# -------------------------------------------- storage --------------------------------------------- #
|
||||||
|
const storage1 = GeneralUtils.GarageStorage(
|
||||||
|
config["externalService"]["objectStorage"]["endpoint"],
|
||||||
|
config["externalService"]["objectStorage"]["access_key_id"],
|
||||||
|
config["externalService"]["objectStorage"]["secret_access_key"],
|
||||||
|
config["externalService"]["objectStorage"]["bucket"]
|
||||||
|
)
|
||||||
|
|
||||||
|
function fileserverUploadHandler(_::String, dataname::String, data::Vector{UInt8})::Dict{String, Any}
|
||||||
|
_, _ = GeneralUtils.s3PutFile(storage1, dataname, data)
|
||||||
|
return Dict("status" => 200, "uploadid" => "", "fileid" => "", "url" => dataname)
|
||||||
|
end
|
||||||
|
|
||||||
|
function fileserverDownloadHandler(url::String, max_retries::Int, base_delay_ms::Int,
|
||||||
|
max_delay_ms::Int, correlation_id::String)::Union{Vector{UInt8}, Nothing}
|
||||||
|
delay = base_delay_ms
|
||||||
|
for attempt in 1:max_retries
|
||||||
|
data = GeneralUtils.s3GetFile(storage1, url)
|
||||||
|
if data !== nothing
|
||||||
|
GeneralUtils.s3DeleteFile(storage1, url)
|
||||||
|
return data
|
||||||
|
else
|
||||||
|
@info "correlation_id: $correlation_id, Attempt $attempt"
|
||||||
|
if attempt < max_retries
|
||||||
|
sleep(delay / 1000.0)
|
||||||
|
delay = min(delay * 2, max_delay_ms)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return nothing
|
||||||
|
end
|
||||||
|
|
||||||
|
# ----------------------------------------- healthserver ----------------------------------------- #
|
||||||
|
# function healthcheck(req::HTTP.Request)
|
||||||
|
# return HTTP.Response(200, JSON.json(Dict("status"=>"ok")))
|
||||||
|
# end
|
||||||
|
# const ROUTER = HTTP.Router()
|
||||||
|
# HTTP.register!(ROUTER, "GET", "/ready", healthcheck)
|
||||||
|
# HTTP.register!(ROUTER, "GET", "/health", healthcheck)
|
||||||
|
# HTTP.register!(ROUTER, "GET", "/", healthcheck)
|
||||||
|
# healthserver = HTTP.serve!(ROUTER, "0.0.0.0", 3000)
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# registration helper function #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
"""
|
||||||
|
register_service(nats_conn, service_name, topic, config)
|
||||||
|
|
||||||
|
Register a service with the load balancer via NATS request.
|
||||||
|
|
||||||
|
# Arguments
|
||||||
|
- `nats_conn`: NATS connection object
|
||||||
|
- `service_name`: Name of the service being registered
|
||||||
|
- `topic`: NATS topic/channel of the registering service
|
||||||
|
- `config`: Configuration dictionary containing service URLs
|
||||||
|
|
||||||
|
# Returns
|
||||||
|
- Response from the load balancer NATS request
|
||||||
|
|
||||||
|
Registers the service with the servicesloadbalancer topic using msghandler.smartpack
|
||||||
|
to create the registration message envelope.
|
||||||
|
"""
|
||||||
|
function register_service(nats_conn, service_name, topic, config)
|
||||||
|
register_dict = Dict(
|
||||||
|
"register_service_name" => service_name,
|
||||||
|
"topic" => topic,
|
||||||
|
"free_or_paid" => "free"
|
||||||
|
)
|
||||||
|
register_payloads = [("register", register_dict, "dictionary")]
|
||||||
|
_, register_msg_envelope_json_str = msghandler.smartpack(
|
||||||
|
config["externalService"]["servicesloadbalancer"]["url"],
|
||||||
|
register_payloads;
|
||||||
|
msg_purpose="register",
|
||||||
|
broker_url=config["nats_server_info"]["url"])
|
||||||
|
try
|
||||||
|
reply = NATS.request(nats_conn,
|
||||||
|
config["externalService"]["servicesloadbalancer"]["url"],
|
||||||
|
register_msg_envelope_json_str,
|
||||||
|
timeout=60)
|
||||||
|
_log("service $service_name is registered")
|
||||||
|
catch e
|
||||||
|
_log("registering service $service_name failed: $(sprint(showerror, e))")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
ch = Channel(1)
|
||||||
|
flag = Channel(1)
|
||||||
|
|
||||||
|
# use reply() here so that when this_service scale to >1 instance,
|
||||||
|
# NATS will randomly select only one receiving instance
|
||||||
|
sub1 = NATS.subscribe(nats_conn,
|
||||||
|
config["thisServiceInputChannel"]["inbox"];
|
||||||
|
queue_group="group1", spawn=true) do msg
|
||||||
|
put!(ch, msg)
|
||||||
|
end
|
||||||
|
|
||||||
|
_log("service started")
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# this service main loop #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
Threads.@spawn begin # Register service periodically
|
||||||
|
while true
|
||||||
|
if isready(flag)
|
||||||
|
register_service(nats_conn,
|
||||||
|
config["thisServiceName"],
|
||||||
|
config["thisServiceInputChannel"]["inbox"],
|
||||||
|
config)
|
||||||
|
# _log("periodic registration sent")
|
||||||
|
end
|
||||||
|
sleep(60) # this line should make spawned thread sleep while allowing main thread running
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
function main()
|
||||||
|
|
||||||
|
# ---------------------------------- register service initially ---------------------------------- #
|
||||||
|
register_service(nats_conn,
|
||||||
|
config["thisServiceName"],
|
||||||
|
config["thisServiceInputChannel"]["inbox"],
|
||||||
|
config)
|
||||||
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
_log("ready to accept NATS messages at broker=$(config["nats_server_info"]["url"]), topic=$(config["thisServiceInputChannel"]["inbox"])")
|
||||||
|
while true
|
||||||
|
try
|
||||||
|
put!(flag, "ready") # ready for new request. do not send register message during llm working
|
||||||
|
incoming_nats_msg = take!(ch) # keep this service running.
|
||||||
|
|
||||||
|
# start working, the work may take more than a few minutes. pause async loop registeration to
|
||||||
|
# prevent a loadbalancer assigning new work to this service.
|
||||||
|
_ = take!(flag)
|
||||||
|
|
||||||
|
incoming_envelope_json_str = String(incoming_nats_msg.payload)
|
||||||
|
incoming_envelope = msghandler.smartunpack(
|
||||||
|
incoming_envelope_json_str;
|
||||||
|
fileserver_download_handler=fileserverDownloadHandler)
|
||||||
|
if incoming_envelope["msg_purpose"] == "ping"
|
||||||
|
_log("ping response")
|
||||||
|
outgoing_envelope, outgoing_envelope_json_str =
|
||||||
|
msghandler.smartpack("requester inbox",
|
||||||
|
[("response", "available", "text")];
|
||||||
|
broker_url=config["nats_server_info"]["url"],
|
||||||
|
fileserver_upload_handler=fileserverUploadHandler)
|
||||||
|
NATS.publish(nats_conn, incoming_nats_msg.reply_to, outgoing_envelope_json_str)
|
||||||
|
else
|
||||||
|
_log("processing new request, msg_purpose=$(incoming_envelope["msg_purpose"])")
|
||||||
|
dataname1 = incoming_envelope["payloads"][1][1]
|
||||||
|
documents_dict = incoming_envelope["payloads"][1][2]
|
||||||
|
documents = documents_dict["documents"]
|
||||||
|
_log("received $(length(documents)) documents")
|
||||||
|
body_data = Dict(
|
||||||
|
"model" => "",
|
||||||
|
"input" => documents
|
||||||
|
)
|
||||||
|
headers = [
|
||||||
|
"Content-Type" => "application/json",
|
||||||
|
"Authorization" => "Bearer no-key"
|
||||||
|
]
|
||||||
|
http_response = HTTP.post(config["externalService"]["llamacpp_embedding"]["url"],
|
||||||
|
headers, JSON.json(body_data))
|
||||||
|
_log("HTTP response status=$(http_response.status)")
|
||||||
|
|
||||||
|
# HTTP.jl doesn't immediately close connections after post() - it pools them for reuse.
|
||||||
|
# Without explicit cleanup, connections accumulate and the llama.cpp server times out
|
||||||
|
# on subsequent requests. This forces connection pool cleanup between requests.
|
||||||
|
HTTP.close_idle_connections!()
|
||||||
|
http_response_body = JSON.parse(http_response.body)
|
||||||
|
dataname2 = "transferjob/$(UUIDs.uuid4())"
|
||||||
|
outgoing_envelope, outgoing_envelope_json_str =
|
||||||
|
msghandler.smartpack("requester inbox",
|
||||||
|
[(dataname2, http_response_body, "dictionary")];
|
||||||
|
msg_purpose=incoming_envelope["msg_purpose"],
|
||||||
|
sender_id=incoming_envelope["sender_id"],
|
||||||
|
broker_url=config["nats_server_info"]["url"],
|
||||||
|
fileserver_upload_handler=fileserverUploadHandler)
|
||||||
|
# re-register this service to let a loadbalancer knows that this service is ready for next request
|
||||||
|
register_service(nats_conn,
|
||||||
|
config["thisServiceName"],
|
||||||
|
config["thisServiceInputChannel"]["inbox"],
|
||||||
|
config)
|
||||||
|
_log("re-registered for next request")
|
||||||
|
reply_topic =
|
||||||
|
if incoming_nats_msg.reply_to !== nothing
|
||||||
|
incoming_nats_msg.reply_to
|
||||||
|
elseif incoming_envelope["reply_to"] != ""
|
||||||
|
incoming_envelope["reply_to"]
|
||||||
|
else
|
||||||
|
_log("no reply_to found, using fallback topic")
|
||||||
|
"sommpanion.unknown_subject"
|
||||||
|
end
|
||||||
|
NATS.publish(nats_conn, reply_topic, outgoing_envelope_json_str)
|
||||||
|
_log("reply published to $reply_topic, dataname2=$dataname2")
|
||||||
|
end
|
||||||
|
catch e
|
||||||
|
bt = catch_backtrace()
|
||||||
|
err_msg = sprint() do io
|
||||||
|
showerror(io, e, bt)
|
||||||
|
println(io)
|
||||||
|
end
|
||||||
|
|
||||||
|
_log(err_msg)
|
||||||
|
# println(err_msg)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
main()
|
||||||
Executable
Executable
+2
@@ -0,0 +1,2 @@
|
|||||||
|
# Base ----------------------------------------
|
||||||
|
python-dev-tools
|
||||||
Executable
BIN
Binary file not shown.
|
After Width: | Height: | Size: 1.2 MiB |
Executable
+243
@@ -0,0 +1,243 @@
|
|||||||
|
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||||
|
|
||||||
|
|
||||||
|
using JSON, Dates, UUIDs, PrettyPrinting, Base64, DataFrames, NATS, HTTP
|
||||||
|
using GeneralUtils, msghandler
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# llama.cpp chat request example (direct HTTP) #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
|
||||||
|
# 1. Read local file and encode to base64 string
|
||||||
|
image_path = "test/large_image.png"
|
||||||
|
image_bytes = read(image_path)
|
||||||
|
base64_string = base64encode(image_bytes)
|
||||||
|
|
||||||
|
# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
|
||||||
|
mime_type = "image/png"
|
||||||
|
data_uri = "data:$(mime_type);base64,$(base64_string)"
|
||||||
|
|
||||||
|
|
||||||
|
headers = [
|
||||||
|
"Content-Type" => "application/json",
|
||||||
|
"Authorization" => "Bearer no-key"
|
||||||
|
]
|
||||||
|
|
||||||
|
# 3. Construct payload with the Data URI
|
||||||
|
messages = Dict(
|
||||||
|
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
|
||||||
|
"messages" => [
|
||||||
|
Dict(
|
||||||
|
"role" => "user",
|
||||||
|
"content" => [
|
||||||
|
Dict("type" => "text", "text" => "Describe this image for me."),
|
||||||
|
Dict(
|
||||||
|
"type" => "image_url",
|
||||||
|
"image_url" => Dict("url" => data_uri)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
],
|
||||||
|
"temperature" => 0.7
|
||||||
|
)
|
||||||
|
url = "http://192.168.1.8:9001/v1/chat/completions"
|
||||||
|
response = HTTP.post(url, headers, JSON.json(messages))
|
||||||
|
r = JSON.parse(response.body)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# Llama.cpp Embedding multiple text example (direct HTTP) #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
using HTTP
|
||||||
|
using JSON
|
||||||
|
|
||||||
|
url = "http://192.168.88.50:9010/v1/embeddings"
|
||||||
|
headers = [
|
||||||
|
"Content-Type" => "application/json",
|
||||||
|
"Authorization" => "Bearer no-key"
|
||||||
|
]
|
||||||
|
|
||||||
|
# Pass an array of strings instead of a single string
|
||||||
|
documents = [
|
||||||
|
"System architecture specification for heart monitor logic.",
|
||||||
|
"NATS broker cluster configuration options for low-latency.",
|
||||||
|
"Julia signal processing worker initialization sequence."
|
||||||
|
]
|
||||||
|
|
||||||
|
body_data = Dict(
|
||||||
|
"model" => "embeddinggemma-300M-BF16",
|
||||||
|
"input" => documents
|
||||||
|
)
|
||||||
|
|
||||||
|
try
|
||||||
|
response = HTTP.post(url, headers, JSON.json(body_data))
|
||||||
|
parsed = JSON.parse(String(response.body))
|
||||||
|
|
||||||
|
# Extract multiple vectors (ordered identically to your input array)
|
||||||
|
for (i, item) in enumerate(parsed["data"])
|
||||||
|
idx = item["index"] + 1 # API indexes are 0-based; Julia is 1-based
|
||||||
|
vector = item["embedding"]
|
||||||
|
println("Document $idx unified vector size: ", length(vector))
|
||||||
|
end
|
||||||
|
|
||||||
|
catch e
|
||||||
|
println("Batch embedding failed: ", e)
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# test openai_message (NATS) #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# Llama.cpp Embedding multiple text with NATS #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
using JSON, Dates, UUIDs, PrettyPrinting, Base64, DataFrames, NATS, HTTP
|
||||||
|
using GeneralUtils, msghandler
|
||||||
|
|
||||||
|
const config = JSON.parsefile("./appconfig.json")
|
||||||
|
|
||||||
|
documents = [
|
||||||
|
"System architecture specification for heart monitor logic.",
|
||||||
|
# "NATS broker cluster configuration options for low-latency.",
|
||||||
|
# "Julia signal processing worker initialization sequence."
|
||||||
|
]
|
||||||
|
|
||||||
|
documents_dict = Dict("documents"=> documents)
|
||||||
|
|
||||||
|
payloads = [("documents", documents_dict, "dictionary")]
|
||||||
|
message_envelope, forward_msg_envelope_json_str = msghandler.smartpack(
|
||||||
|
config["this_service_input_channel"]["nats"],
|
||||||
|
payloads;
|
||||||
|
msg_purpose="embedding",
|
||||||
|
broker_url=config["nats_server_info"]["url"],
|
||||||
|
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||||
|
|
||||||
|
connection = NATS.connect(config["nats_server_info"]["url"])
|
||||||
|
reply = NATS.request(connection,
|
||||||
|
config["this_service_input_channel"]["nats"],
|
||||||
|
forward_msg_envelope_json_str, timeout=120)
|
||||||
|
|
||||||
|
incoming_env_json_str = String(reply.payload)
|
||||||
|
incoming_env = msghandler.smartunpack(incoming_env_json_str)
|
||||||
|
embedding_response = incoming_env["payloads"][1][2]
|
||||||
|
|
||||||
|
for (i, item) in enumerate(embedding_response["data"])
|
||||||
|
vector = item["embedding"]
|
||||||
|
println("Document $i embedding vector size: ", length(vector))
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
# test openai_message (NATS) #
|
||||||
|
# ------------------------------------------------------------------------------------------------ #
|
||||||
|
using JSON, Dates, UUIDs, PrettyPrinting, Base64, DataFrames, NATS, HTTP
|
||||||
|
using GeneralUtils, msghandler
|
||||||
|
|
||||||
|
const config = JSON.parsefile("./appconfig.json")
|
||||||
|
|
||||||
|
# 1. Read local file and encode to base64 string
|
||||||
|
image_path = "test/large_image.png"
|
||||||
|
image_bytes = read(image_path)
|
||||||
|
base64_string = base64encode(image_bytes)
|
||||||
|
|
||||||
|
# 2. Match the MIME type according to your file extension (e.g., png, jpeg)
|
||||||
|
mime_type = "image/png"
|
||||||
|
data_uri = "data:$(mime_type);base64,$(base64_string)"
|
||||||
|
|
||||||
|
|
||||||
|
headers = [
|
||||||
|
"Content-Type" => "application/json",
|
||||||
|
"Authorization" => "Bearer no-key"
|
||||||
|
]
|
||||||
|
|
||||||
|
# 3. Construct payload with the Data URI
|
||||||
|
openai_msg = Dict(
|
||||||
|
"model" => "gemma-4-E4B-it-UD-Q4_K_XL",
|
||||||
|
"messages" => [
|
||||||
|
Dict(
|
||||||
|
"role" => "user",
|
||||||
|
"content" => [
|
||||||
|
Dict("type" => "text", "text" => "Describe this image for me in detail."),
|
||||||
|
Dict(
|
||||||
|
"type" => "image_url",
|
||||||
|
"image_url" => Dict("url" => data_uri)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
],
|
||||||
|
"temperature" => 0.7
|
||||||
|
)
|
||||||
|
|
||||||
|
payloads = [("msg", openai_msg, "dictionary")] # List of tuples
|
||||||
|
message_envelope, forward_msg_envelope_json_str = msghandler.smartpack(
|
||||||
|
config["this_service_input_channel"]["nats"],
|
||||||
|
payloads;
|
||||||
|
msg_purpose="text2text",
|
||||||
|
broker_url=config["nats_server_info"]["url"],
|
||||||
|
fileserver_url=config["externalService"]["fileserver"]["url"])
|
||||||
|
|
||||||
|
|
||||||
|
connection = NATS.connect(config["nats_server_info"]["url"])
|
||||||
|
reply = NATS.request(connection,
|
||||||
|
config["this_service_input_channel"]["nats"],
|
||||||
|
forward_msg_envelope_json_str, timeout=120)
|
||||||
|
|
||||||
|
incoming_env_json_str = String(reply.payload)
|
||||||
|
incoming_env = msghandler.smartunpack(incoming_env_json_str)
|
||||||
|
ai_response = incoming_env["payloads"][1][2]
|
||||||
|
content = ai_response["choices"][1]["message"]["content"]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Reference in New Issue
Block a user