ton/GeneralUtils
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 9 Wiki Activity

update

Browse Source
This commit is contained in:
narawat
2026-09-05 10:27:31 +07:00
parent 9ad732c436
commit 599765d7ef
2 changed files with 57 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/garageS3.jl
+55 -39
View File
@@ -2,9 +2,9 @@ module garageS3
export export
LifecycleExpiration, build_lifecycle_xml, LifecycleExpiration, build_lifecycle_xml,
GarageStorage, put_file, get_file, list_files, delete_file, GarageStorage, s3PutFile, s3GetFile, s3ListFiles, s3DeleteFile,
set_lifecycle_expiration, get_lifecycle, delete_lifecycle, s3SetLifecycleExpiration, s3GetLifecycle, s3DeleteLifecycle,
s3_sign_url, s3_sign_url_web s3SignUrl, s3SignUrlWeb
using AWS using AWS
@service S3 @service S3
@@ -29,8 +29,8 @@ storage1 = GeneralUtils.GarageStorage(
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl) # Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
# use UUID as filename because GarageS3 requires unique filename for each uploaded object # use UUID as filename because GarageS3 requires unique filename for each uploaded object
urls = GeneralUtils.put_file(storage1, "transferjob-(GeneralUtils.uuid4snakecase())", "{\"status\": \"inactive\"}") urls = GeneralUtils.s3PutFile(storage1, "transferjob-(GeneralUtils.uuid4snakecase())", "{\"status\": \"inactive\"}")
key = GeneralUtils.list_files(storage) key = GeneralUtils.s3ListFiles(storage)
println("Read back: ", data) println("Read back: ", data)
println("Bucket keys: ", key) println("Bucket keys: ", key)
@@ -41,7 +41,7 @@ curl -v \
http://192.168.88.106:3902/users-1002.json http://192.168.88.106:3902/users-1002.json
# test with API # test with API
data = String(GeneralUtils.get_file(storage, key[1])) data = String(GeneralUtils.s3GetFile(storage, key[1]))
# test with a browser (urls.web) # test with a browser (urls.web)
https://s3-web.mydomain.com/my_bucket_name/users-1005.json https://s3-web.mydomain.com/my_bucket_name/users-1005.json
@@ -143,8 +143,8 @@ end
# Examples # Examples
```jldoctest ```jldoctest
julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3") julia> storage1 = GarageStorage("https://s3-api.yiem.cc", "GKb08015", "55114ff94828febca1", "sommpanion-s3")
julia> put_file1 = put_file(storage1) julia> s3PutFile1 = s3PutFile(storage1)
julia> downloadUrl = put_file1("test.json", "{\"status\": \"active\"}") julia> downloadUrl = s3PutFile1("test.json", "{\"status\": \"active\"}")
Successfully uploaded: test.json Successfully uploaded: test.json
julia> downloadUrl.api julia> downloadUrl.api
"https://s3-api.yiem.cc/sommpanion-s3/test.json" "https://s3-api.yiem.cc/sommpanion-s3/test.json"
@@ -152,10 +152,10 @@ julia> downloadUrl.web
"https://s3-web.yiem.cc/sommpanion-s3/test.json" "https://s3-web.yiem.cc/sommpanion-s3/test.json"
``` ```
""" """
mutable struct put_file mutable struct s3PutFile
storage::GarageStorage storage::GarageStorage
end end
function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}} function (pf::s3PutFile)(key::String, data::Union{String, Vector{UInt8}}
)::NamedTuple{(:api, :web), Tuple{String, String}} )::NamedTuple{(:api, :web), Tuple{String, String}}
AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data) AWSS3.s3_put(pf.storage.config, pf.storage.bucket, key, data)
println("Successfully uploaded: ", key) println("Successfully uploaded: ", key)
@@ -184,7 +184,7 @@ end
# Examples # Examples
```julia ```julia
julia> result = put_file(storage, "test.json", "{\"status\": \"active\"}") julia> result = s3PutFile(storage, "test.json", "{\"status\": \"active\"}")
Successfully uploaded: test.json Successfully uploaded: test.json
julia> result.api julia> result.api
"https://s3-api.yiem.cc/sommpanion-s3/test.json" "https://s3-api.yiem.cc/sommpanion-s3/test.json"
@@ -192,7 +192,7 @@ julia> result.web
"https://s3-web.yiem.cc/sommpanion-s3/test.json" "https://s3-web.yiem.cc/sommpanion-s3/test.json"
``` ```
""" """
function put_file(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}} function s3PutFile(storage::GarageStorage, key::String, data::Union{String, Vector{UInt8}}
)::NamedTuple{(:api, :web), Tuple{String, String}} )::NamedTuple{(:api, :web), Tuple{String, String}}
AWSS3.s3_put(storage.config, storage.bucket, key, data) AWSS3.s3_put(storage.config, storage.bucket, key, data)
println("Successfully uploaded: ", key) println("Successfully uploaded: ", key)
@@ -219,11 +219,11 @@ end
# Examples # Examples
```julia ```julia
julia> data = String(get_file(storage, "test.json")) julia> data = String(s3GetFile(storage, "test.json"))
"\"{\\\"status\\\": \\\"active\\\"}\"" "\"{\\\"status\\\": \\\"active\\\"}\""
``` ```
""" """
function get_file(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing} function s3GetFile(storage::GarageStorage, key::String)::Union{Vector{UInt8}, Nothing}
try try
return AWSS3.s3_get(storage.config, storage.bucket, key) return AWSS3.s3_get(storage.config, storage.bucket, key)
catch catch
@@ -242,14 +242,14 @@ end
# Examples # Examples
```julia ```julia
julia> keys = list_files(storage) julia> keys = s3ListFiles(storage)
3-element Vector{String}: 3-element Vector{String}:
"test1.json" "test1.json"
"test2.json" "test2.json"
"users-1002.json" "users-1002.json"
``` ```
""" """
function list_files(storage::GarageStorage) function s3ListFiles(storage::GarageStorage)
# Use delimiter="" to get all individual keys (no grouping by "directory") # Use delimiter="" to get all individual keys (no grouping by "directory")
objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="") objects = AWSS3.s3_list_objects(storage.config, storage.bucket; delimiter="")
return [obj["Key"] for obj in objects] return [obj["Key"] for obj in objects]
@@ -265,15 +265,15 @@ end
# Examples # Examples
```julia ```julia
julia> delete_file(storage, "test.json") julia> s3DeleteFile(storage, "test.json")
``` ```
""" """
function delete_file(storage::GarageStorage, key::String) function s3DeleteFile(storage::GarageStorage, key::String)
AWSS3.s3_delete(storage.config, storage.bucket, key) AWSS3.s3_delete(storage.config, storage.bucket, key)
end end
""" """
s3_sign_url(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String s3SignUrl(storage, key, seconds=3600; verb="GET", content_type="application/octet-stream", signature_version="v4") -> String
Generate a pre-signed URL for an object in the Garage S3 bucket. Generate a pre-signed URL for an object in the Garage S3 bucket.
@@ -297,23 +297,39 @@ using AWS Signature Version 4 (HMAC-SHA256).
# Examples # Examples
```julia ```julia
storage = GarageStorage("https://s3-api.yiem.cc", "key", "secret", "testbucket") using GeneralUtils: GarageStorage, s3SignUrl, s3ListFiles, s3PutFile
# Download — GET (default) storage = GarageStorage(
url = s3_sign_url(storage, "docs/report.pdf", 3600) "https://s3-api.yiem.cc",
"GKb080154a2e5b19100b1b2c6e",
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
"sommpanion"
)
# Upload — PUT # Upload — PUT with curl
url = s3_sign_url(storage, "avatars/user123.png", 300; url = s3SignUrl(storage, "photo.jpg", 3600; verb="PUT", content_type="image/jpeg")
verb="PUT", content_type="image/png") # curl -X PUT -T "/path/to/photo.jpg" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: image/jpeg' <url>
# Download — GET with curl
url = s3SignUrl(storage, "photo.jpg", 3600; verb="GET")
# curl <url> → returns file content
# Upload JSON
url = s3SignUrl(storage, "data/config.json", 3600; verb="PUT", content_type="application/json")
# curl -X PUT -T "/tmp/config.json" -H 'Host: s3-api.yiem.cc' -H 'Content-Type: application/json' <url>
# Delete
url = s3SignUrl(storage, "photo.jpg", 3600; verb="DELETE")
# curl -X DELETE url
# Multiple uploads (chat app) # Multiple uploads (chat app)
urls = [s3_sign_url(storage, "chat/GeneralUtils.uuid4snakecase()", 300; urls = [s3SignUrl(storage, "chat/\$(GeneralUtils.uuid4snakecase())", 300;
verb="PUT", content_type="image/png") verb="PUT", content_type="image/png")
for f in filenames] for _ in 1:10]
# Signature differs by verb and content_type # Signature differs by verb and content_type
url_get = s3_sign_url(storage, "file.txt", 3600; verb="GET") url_get = s3SignUrl(storage, "file.txt", 3600; verb="GET")
url_put = s3_sign_url(storage, "file.txt", 3600; verb="PUT", content_type="text/plain") url_put = s3SignUrl(storage, "file.txt", 3600; verb="PUT", content_type="text/plain")
``` ```
# Notes # Notes
@@ -322,7 +338,7 @@ url_put = s3_sign_url(storage, "file.txt", 3600; verb="PUT", content_type="text/
- PUT uploads require the client's `Content-Type` to match `content_type`. - PUT uploads require the client's `Content-Type` to match `content_type`.
- For browser CORS uploads, configure Garage bucket CORS rules for `Origin` and `Method`. - For browser CORS uploads, configure Garage bucket CORS rules for `Origin` and `Method`.
""" """
function s3_sign_url( function s3SignUrl(
storage::GarageStorage, storage::GarageStorage,
key::String, key::String,
seconds::Int=3600; seconds::Int=3600;
@@ -419,9 +435,9 @@ end
Useful for direct browser access — URLs look like Useful for direct browser access — URLs look like
`https://s3-web.yiem.cc/bucket/key?X-Amz-Signature=...` `https://s3-web.yiem.cc/bucket/key?X-Amz-Signature=...`
Arguments and keyword arguments are the same as `s3_sign_url`. Arguments and keyword arguments are the same as `s3SignUrl`.
""" """
function s3_sign_url_web( function s3SignUrlWeb(
storage::GarageStorage, storage::GarageStorage,
key::String, key::String,
seconds::Int=3600; seconds::Int=3600;
@@ -437,7 +453,7 @@ function s3_sign_url_web(
) )
web_config = GarageConfig(web_endpoint, storage.config.region, web_creds) web_config = GarageConfig(web_endpoint, storage.config.region, web_creds)
web_storage = GarageStorage(web_config, storage.bucket) web_storage = GarageStorage(web_config, storage.bucket)
return s3_sign_url(web_storage, key, seconds; verb, content_type, signature_version) return s3SignUrl(web_storage, key, seconds; verb, content_type, signature_version)
end end
# =================================================================== # ===================================================================
@@ -496,10 +512,10 @@ end
# Example # Example
```julia ```julia
rule = LifecycleExpiration("uploads/", 30) rule = LifecycleExpiration("uploads/", 30)
set_lifecycle_expiration(storage, rule) s3SetLifecycleExpiration(storage, rule)
``` ```
""" """
function set_lifecycle_expiration(storage::GarageStorage, rule::LifecycleExpiration) function s3SetLifecycleExpiration(storage::GarageStorage, rule::LifecycleExpiration)
xml_body = build_lifecycle_xml(rule) xml_body = build_lifecycle_xml(rule)
args = Dict{String,Any}("body" => xml_body) args = Dict{String,Any}("body" => xml_body)
return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config) return S3.put_bucket_lifecycle_configuration(storage.bucket, args; aws_config=storage.config)
@@ -515,10 +531,10 @@ end
# Example # Example
```julia ```julia
config = get_lifecycle(storage) config = s3GetLifecycle(storage)
``` ```
""" """
function get_lifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing} function s3GetLifecycle(storage::GarageStorage)::Union{Dict{String,Any},Nothing}
try try
return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config) return S3.get_bucket_lifecycle_configuration(storage.bucket; aws_config=storage.config)
catch e catch e
@@ -539,10 +555,10 @@ end
# Example # Example
```julia ```julia
delete_lifecycle(storage) s3DeleteLifecycle(storage)
``` ```
""" """
function delete_lifecycle(storage::GarageStorage) function s3DeleteLifecycle(storage::GarageStorage)
return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config) return S3.delete_bucket_lifecycle(storage.bucket; aws_config=storage.config)
end end
test/etc.jl
+1 -1
View File
@@ -13,7 +13,7 @@ curl -X PUT -H "Content-Type: application/json" \
curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \ curl -v -X PUT -T "/home/ton/docker-apps/sommpanion/etc/GeneralUtils/photo.jpg" \
-H 'Host: s3-api.yiem.cc' \ -H 'Host: s3-api.yiem.cc' \
-H 'Content-Type: image/jpeg' \ -H 'Content-Type: image/jpeg' \
"https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Expires=3600&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T025455Z&X-Amz-SignedHeaders=host&X-Amz-Signature=92e86746a92fa07914d8f08169dac66e2c64f36b807f86504bf8efec41fed153" "https://s3-api.yiem.cc/sommpanion/photo.jpg?X-Amz-Expires=300&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=GKb080154a2e5b19100b1b2c6e%2F20260905%2Fgarage%2Fs3%2Faws4_request&X-Amz-Date=20260905T030315Z&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=1107a31f36f8b8ceb9663d69b9a7be7733df14d019e700e750944cbdad50df5d"