1st push
This commit is contained in:
280
app/env_preparation.jl
Normal file
280
app/env_preparation.jl
Normal file
@@ -0,0 +1,280 @@
|
||||
"""
|
||||
For docker container:
|
||||
- This script must be placed at the root of your project e.g. /appfolder/app
|
||||
- This script expects all other files are in workfolder/temp e.g. /appfolder/app/temp
|
||||
"""
|
||||
|
||||
using Pkg
|
||||
|
||||
programName = "someapp" # use my project name here
|
||||
devPackageName = "somepackage"
|
||||
appfolder = pwd() # use current folder as project folder
|
||||
|
||||
|
||||
# ---------------------------------------------- 100 --------------------------------------------- #
|
||||
|
||||
""" list of all recommend package
|
||||
required_packages = [
|
||||
"Arrow"
|
||||
"BenchmarkTools"
|
||||
"CSV"
|
||||
"CUDA"
|
||||
"ClickHouse"
|
||||
"DataFrames"
|
||||
"DataFramesMeta"
|
||||
"DataStructures"
|
||||
"Distributions"
|
||||
"Enzyme"
|
||||
"FileIO"
|
||||
"Flux"
|
||||
"Genie"
|
||||
"HTTP"
|
||||
"ImageTransformations"
|
||||
"Images"
|
||||
"IterTools"
|
||||
"JSON3"
|
||||
"LinearAlgebra"
|
||||
"Logging"
|
||||
"Lux"
|
||||
"MLDataUtils"
|
||||
"MLDatasets"
|
||||
"MLLabelUtils"
|
||||
"Makie"
|
||||
"ODBC"
|
||||
"Optimisers"
|
||||
"ParameterSchedulers"
|
||||
"PkgTemplates"
|
||||
"ProgressMeter"
|
||||
"PyCall"
|
||||
"Random"
|
||||
"Revise"
|
||||
"Serialization"
|
||||
"Statistics"
|
||||
"Stipple"
|
||||
"TensorBoardLogger"
|
||||
"TextAnalysis"
|
||||
"Transformers"
|
||||
"UUIDs"
|
||||
"WordTokenizers"
|
||||
"Zygote"
|
||||
"https://github.com/denglerchr/Mosquitto.jl"
|
||||
]
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# add julia packages in container's general registry folder #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
privatejuliapkgpath = "$appfolder/dev"
|
||||
required_julia_private_packages = [ #CHANGE
|
||||
# "https://git.yiem.cc/ton/GeneralUtils",
|
||||
# "https://git.yiem.cc/ton/LLMMCTS",
|
||||
# "https://git.yiem.cc/ton/SQLLLM",
|
||||
# "https://git.yiem.cc/ton/YiemAgent -b v0.1.1",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# add julia package in app folder #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
required_julia_packages = [ #CHANGE
|
||||
"Revise",
|
||||
# "CondaPkg",
|
||||
# "PythonCall",
|
||||
"HTTP",
|
||||
"Oxygen",
|
||||
"JSON",
|
||||
"Dates",
|
||||
"UUIDs",
|
||||
"Random",
|
||||
"URIs",
|
||||
"DataStructures",
|
||||
"FileIO",
|
||||
"PrettyPrinting",
|
||||
"ZipArchives",
|
||||
"LibPQ",
|
||||
"ProgressMeter",
|
||||
"NATS"
|
||||
] # use only for new project # use only for new project
|
||||
|
||||
function copy_folder_contents(src_folder::AbstractString, dst_folder::AbstractString)
|
||||
# Get a list of all files in the source folder
|
||||
files = readdir(src_folder)
|
||||
|
||||
# Iterate over each file and copy it to the destination folder
|
||||
for itemname in files
|
||||
src_path = joinpath(src_folder, itemname)
|
||||
dst_path = joinpath(dst_folder, itemname)
|
||||
if isdir(src_path)
|
||||
run(`cp -r $src_path $dst_path`)
|
||||
else
|
||||
run(`cp $src_path $dst_path`)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function install_required_julia_packages(required_julia_packages)
|
||||
for i in required_julia_packages
|
||||
println("adding ", i)
|
||||
if startswith(i, "http")
|
||||
Pkg.add(url=i)
|
||||
else
|
||||
Pkg.add(i)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function install_required_julia_private_packages(required_julia_private_packages::Vector,
|
||||
appfolderpath::String, privatejuliapkgpath::String)
|
||||
|
||||
!isdir(privatejuliapkgpath) ? mkpath(privatejuliapkgpath) : nothing
|
||||
cd(privatejuliapkgpath)
|
||||
|
||||
if length(required_julia_private_packages) > 0
|
||||
# remove private packages already in Project.toml in case private package path is different
|
||||
for i in required_julia_private_packages
|
||||
_pkgname = split(i, "/")[end]
|
||||
_pkgname = split(_pkgname, " ")[1]
|
||||
pkgname = split(_pkgname, ".")[1]
|
||||
println("removing Julia private package: $pkgname if it already in Project.toml")
|
||||
try Pkg.rm(pkgname) catch end
|
||||
end
|
||||
|
||||
# clone all required julia private packages
|
||||
for i in required_julia_private_packages
|
||||
_pkgname = split(i, "/")[end]
|
||||
_pkgname = split(_pkgname, " ")[1]
|
||||
pkgname = split(_pkgname, ".")[1]
|
||||
pkgpath = joinpath(privatejuliapkgpath, pkgname)
|
||||
println("cloning Julia private package: $pkgname")
|
||||
|
||||
# only clone a package if it is not found in local
|
||||
gitcommand = "git clone $i"
|
||||
args = split(gitcommand, " ")
|
||||
!isdir(pkgpath) ? run(`$args`) : nothing
|
||||
end
|
||||
|
||||
# install all required julia private packages
|
||||
for i in required_julia_private_packages
|
||||
_pkgname = split(i, "/")[end]
|
||||
_pkgname = split(_pkgname, " ")[1]
|
||||
pkgname = split(_pkgname, ".")[1] # in case there is a .jl at the end of package name
|
||||
pkgpath = joinpath(privatejuliapkgpath, pkgname)
|
||||
println("installing Julia private package: $pkgname")
|
||||
|
||||
Pkg.develop(path=pkgpath)
|
||||
end
|
||||
end
|
||||
|
||||
cd(appfolderpath)
|
||||
end
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ #
|
||||
# start script #
|
||||
# ------------------------------------------------------------------------------------------------ #
|
||||
|
||||
# all files are put in ./temp (most likely in a docker container)
|
||||
if isdir(joinpath(appfolder, "temp"))
|
||||
if isfile(joinpath(appfolder, "temp", "Project.toml")) # use this option if one already have project.toml
|
||||
copy_folder_contents(joinpath(appfolder, "temp"), appfolder)
|
||||
Pkg.activate(".")
|
||||
install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath)
|
||||
Pkg.instantiate()
|
||||
else # no Project.toml, create new package
|
||||
copy_folder_contents(joinpath(appfolder, "temp"), appfolder)
|
||||
Pkg.activate(".")
|
||||
length(required_julia_packages) != 0 ? install_required_julia_packages(required_julia_packages) : nothing
|
||||
|
||||
# for app
|
||||
write("main.jl", "using Revise # remove when this package is completed\n", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||
mkdir("testapp")
|
||||
cd("testapp")
|
||||
write("runtests.jl", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||
cd(appfolder)
|
||||
install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath)
|
||||
|
||||
# create new dev package folder
|
||||
!isdir(privatejuliapkgpath) ? mkpath(privatejuliapkgpath) : nothing
|
||||
cd(privatejuliapkgpath)
|
||||
Pkg.generate(devPackageName)
|
||||
newPackagePath = joinpath(privatejuliapkgpath, devPackageName)
|
||||
Pkg.develop(path=newPackagePath)
|
||||
cd(newPackagePath)
|
||||
mkdir("test")
|
||||
cd("./test")
|
||||
write("runtests.jl", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||
cd(appfolder)
|
||||
end
|
||||
else # all files are in the current work folder
|
||||
if isfile(joinpath(appfolder, "Project.toml")) # use this option if one already have project.toml
|
||||
Pkg.activate(".")
|
||||
install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath)
|
||||
Pkg.instantiate()
|
||||
else # create new package folder
|
||||
Pkg.activate(".")
|
||||
length(required_julia_packages) != 0 ? install_required_julia_packages(required_julia_packages) : nothing
|
||||
|
||||
# for app
|
||||
write("main.jl", "using Revise # remove when this package is completed\n", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||
mkdir("testapp")
|
||||
cd("testapp")
|
||||
write("runtests.jl", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||
cd(appfolder)
|
||||
install_required_julia_private_packages(required_julia_private_packages, appfolder, privatejuliapkgpath)
|
||||
|
||||
# create new dev package folder
|
||||
!isdir(privatejuliapkgpath) ? mkpath(privatejuliapkgpath) : nothing
|
||||
cd(privatejuliapkgpath)
|
||||
Pkg.generate(devPackageName)
|
||||
newPackagePath = joinpath(privatejuliapkgpath, devPackageName)
|
||||
Pkg.develop(path=newPackagePath)
|
||||
cd(newPackagePath)
|
||||
mkdir("test")
|
||||
cd("./test")
|
||||
write("runtests.jl", "using $devPackageName\n", "# ---------------------------------------------- 100 --------------------------------------------- #\n")
|
||||
cd(appfolder)
|
||||
|
||||
# preparing typical julia project finished now doing project's specific preparation here
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# python libraries used by Julia's PythonCall #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
pythonlib = []
|
||||
channels = ["anaconda", "conda-forge", "pytorch"]
|
||||
if length(pythonlib) != 0
|
||||
using CondaPkg;
|
||||
for i in channels CondaPkg.add_channel(i) end
|
||||
for i in pythonlib CondaPkg.add_pip(i) end
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# create symlink #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# if isfile("/app/run.jl")
|
||||
# symlink("/app/$programName/run.jl", "/app/run.jl")
|
||||
# elseif isfile("/app/$programName/main.py")
|
||||
# symlink("/app/$programName/main.py", "/app/main.py")
|
||||
# else
|
||||
# cd("/app")
|
||||
# write("no run.jl in package folder.txt", "")
|
||||
# end
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ #
|
||||
# for web app #
|
||||
# ------------------------------------------------------------------------------------------------ #
|
||||
# intendedURL = "wine.yiem.cc/hq/agent/sommelier/frontend/dbadmin" # e.g. "wine.yiem.cc/hq/agent/sommelier/frontend/dbadmin"
|
||||
# urlList = split(intendedURL, "/")[2:end] # e.g. ["hq", "agent", "sommelier", "frontend", "dbadmin"]
|
||||
# # use joinpath to compose the path from urlList
|
||||
# _path = joinpath(urlList...)
|
||||
# path = "./$_path"
|
||||
# mkpath(path)
|
||||
# # move all files in the current folder to the path
|
||||
# copy_folder_contents(joinpath(appfolder, "temp"), path)
|
||||
|
||||
|
||||
println("--> Julia env preparation done")
|
||||
Reference in New Issue
Block a user