V0.6.2 #14
+4
-21
@@ -13,11 +13,11 @@ using AWSS3
|
||||
# this file use local AWS config
|
||||
|
||||
""" Example
|
||||
storage = GeneralUtils.GarageStorage(
|
||||
storage1 = GeneralUtils.GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e", # key ID (create at garage-ui.yiem.cc)
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1", # key ID's secret key
|
||||
"sommpanion-s3"
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
# Safe to run inside concurrent HTTP handlers (e.g., Oxygen.jl, HTTP.jl)
|
||||
@@ -30,7 +30,7 @@ println("Bucket keys: ", key)
|
||||
|
||||
# test with curl
|
||||
curl -v \
|
||||
-H 'Host: sommpanion-s3.s3-web.yiem.cc' \
|
||||
-H 'Host: testbucket.s3-web.yiem.cc' \
|
||||
http://192.168.88.106:3902/users-1002.json
|
||||
|
||||
# test with API
|
||||
@@ -64,23 +64,6 @@ AWS.check_credentials(c::SimpleCredentials) = c
|
||||
AWS.region(aws::GarageConfig) = aws.region
|
||||
AWS.credentials(aws::GarageConfig) = aws.credentials
|
||||
|
||||
""" Generate the base URL for a Garage S3 service request.
|
||||
|
||||
# Arguments
|
||||
- `aws::GarageConfig`
|
||||
The Garage configuration containing the endpoint.
|
||||
- `service::String`
|
||||
The AWS service name (e.g., `"s3"`).
|
||||
- `region::String`
|
||||
The AWS region string.
|
||||
|
||||
# Return
|
||||
- `String`: The stripped endpoint URL (trailing slashes removed).
|
||||
"""
|
||||
function AWS.generate_service_url(aws::GarageConfig, service::String, region::String)
|
||||
return strip(aws.endpoint, '/')
|
||||
end
|
||||
|
||||
""" Generate the full service URL including the resource path for a Garage S3 request.
|
||||
|
||||
# Arguments
|
||||
@@ -183,7 +166,7 @@ function (pf::put_file)(key::String, data::Union{String, Vector{UInt8}}
|
||||
println("Successfully uploaded: ", key)
|
||||
|
||||
# object url
|
||||
api = "$(storage.config.endpoint)/$(storage.bucket)/$key"
|
||||
api = "$(pf.storage.config.endpoint)/$(pf.storage.bucket)/$key"
|
||||
|
||||
# file download URL using browser
|
||||
web = replace(api, "s3-api" => "s3-web")
|
||||
|
||||
+288
-25
@@ -1,39 +1,302 @@
|
||||
using Test
|
||||
using GeneralUtils: detect_keyword
|
||||
using GeneralUtils
|
||||
using GeneralUtils.garageS3
|
||||
using AWS
|
||||
using Random: randstring
|
||||
using UUIDs: uuid4
|
||||
|
||||
@testset "detect_keyword tests" begin
|
||||
@test detect_keyword(["test"], "this is a test") == Dict("test" => 1)
|
||||
@testset "garageS3.jl" begin
|
||||
|
||||
@test detect_keyword(["hello", "world"], "hello world hello") == Dict("hello" => 2, "world" => 1)
|
||||
# --- SimpleCredentials ---
|
||||
|
||||
@test detect_keyword(["cat"], "category") == Dict("cat" => 1)
|
||||
@testset "SimpleCredentials construction" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "sessiontoken")
|
||||
@test creds.access_key_id == "AKIAIOSFODNN7EXAMPLE"
|
||||
@test creds.secret_key == "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
@test creds.token == "sessiontoken"
|
||||
end
|
||||
|
||||
@test detect_keyword(["cat"], "category"; mode="individual") == Dict("cat" => 0)
|
||||
@testset "SimpleCredentials AWS extensions" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key1", "secret1", "token1")
|
||||
@test AWS.credentials(creds) === creds
|
||||
@test AWS.check_credentials(creds) === creds
|
||||
@test AWS.refresh!(creds; force=false) === creds
|
||||
@test AWS.refresh!(creds) === creds
|
||||
end
|
||||
|
||||
@test detect_keyword(["dog"], "dogs and cats"; mode="individual", delimiter=[' ']) == Dict("dog" => 0)
|
||||
# --- GarageConfig ---
|
||||
|
||||
@test detect_keyword(["test"], "test.case"; mode="individual", delimiter=['.']) == Dict("test" => 1)
|
||||
@testset "GarageConfig construction" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "us-east-1", creds)
|
||||
@test config.endpoint == "https://s3-api.example.com"
|
||||
@test config.region == "us-east-1"
|
||||
@test config.credentials === creds
|
||||
@test config isa AWS.AbstractAWSConfig
|
||||
end
|
||||
|
||||
@test detect_keyword(["word"], "") == Dict("word" => 0)
|
||||
@testset "GarageConfig AWS extensions" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
|
||||
@test AWS.region(config) == "garage"
|
||||
@test AWS.credentials(config) === creds
|
||||
end
|
||||
|
||||
@test detect_keyword(String[], "some text") == Dict{String, Integer}()
|
||||
# --- AWS.generate_service_url ---
|
||||
|
||||
@test detect_keyword(["a", "b"], "a.b\nc"; delimiter=['.', '\n']) == Dict("a" => 1, "b" => 1)
|
||||
@testset "generate_service_url with resource starting with /" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
|
||||
url = AWS.generate_service_url(config, "s3", "/mybucket/key")
|
||||
@test url == "https://s3-api.example.com/mybucket/key"
|
||||
end
|
||||
|
||||
multiline_text = """
|
||||
first line
|
||||
second line
|
||||
first word
|
||||
"""
|
||||
@test detect_keyword(["first"], multiline_text) == Dict("first" => 2)
|
||||
@testset "generate_service_url with resource NOT starting with /" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com", "garage", creds)
|
||||
url = AWS.generate_service_url(config, "s3", "mybucket/key")
|
||||
@test url == "https://s3-api.example.com/mybucket/key"
|
||||
end
|
||||
|
||||
@test detect_keyword(["word"], "word"; mode="individual") == Dict("word" => 1)
|
||||
@testset "generate_service_url strips trailing slashes from endpoint" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com/", "garage", creds)
|
||||
url = AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||
@test url == "https://s3-api.example.com/bucket/key"
|
||||
end
|
||||
|
||||
@testset "generate_service_url with multiple trailing slashes" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("https://s3-api.example.com///", "garage", creds)
|
||||
url = AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||
@test url == "https://s3-api.example.com/bucket/key"
|
||||
end
|
||||
|
||||
@testset "generate_service_url preserves http endpoint" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900", "garage", creds)
|
||||
url = AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||
@test url == "http://localhost:3900/bucket/key"
|
||||
end
|
||||
|
||||
@testset "generate_service_url with http endpoint and trailing slash" begin
|
||||
creds = GeneralUtils.garageS3.SimpleCredentials("key", "secret", "")
|
||||
config = GeneralUtils.garageS3.GarageConfig("http://localhost:3900/", "garage", creds)
|
||||
url = AWS.generate_service_url(config, "s3", "/bucket/key")
|
||||
@test url == "http://localhost:3900/bucket/key"
|
||||
end
|
||||
|
||||
# --- GarageStorage ---
|
||||
|
||||
@testset "GarageStorage default region" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket")
|
||||
@test storage.bucket == "mybucket"
|
||||
@test storage.config.region == "garage"
|
||||
@test storage.config.endpoint == "https://s3-api.example.com"
|
||||
end
|
||||
|
||||
@testset "GarageStorage custom region" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key123", "secret123", "mybucket"; region="custom-region")
|
||||
@test storage.config.region == "custom-region"
|
||||
end
|
||||
|
||||
@testset "GarageStorage endpoint without https" begin
|
||||
storage = GarageStorage("http://localhost:3900", "key", "secret", "mybucket")
|
||||
@test storage.config.endpoint == "http://localhost:3900"
|
||||
end
|
||||
|
||||
@testset "GarageStorage credentials are SimpleCredentials" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "mykey", "mysecret", "mybucket")
|
||||
@test storage.config.credentials isa GeneralUtils.garageS3.SimpleCredentials
|
||||
@test storage.config.credentials.access_key_id == "mykey"
|
||||
@test storage.config.credentials.secret_key == "mysecret"
|
||||
@test storage.config.credentials.token == ""
|
||||
end
|
||||
|
||||
@testset "GarageStorage returns correct struct type" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
@test storage isa GarageStorage
|
||||
end
|
||||
|
||||
# --- put_file callable struct (slash detection) ---
|
||||
|
||||
@testset "put_file callable struct returns nothing for keys with slashes" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
uploader = put_file(storage)
|
||||
result = @test_logs (:warn, r"Upload aborted") uploader("a/b.json", "data")
|
||||
@test result === nothing
|
||||
end
|
||||
|
||||
@testset "put_file callable struct type is correct" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
uploader = put_file(storage)
|
||||
@test uploader isa put_file
|
||||
end
|
||||
|
||||
@testset "put_file callable struct construction stores storage" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
uploader = put_file(storage)
|
||||
@test uploader.storage === storage
|
||||
end
|
||||
|
||||
# --- put_file function (slash detection) ---
|
||||
|
||||
@testset "put_file function returns nothing for keys with slashes" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
result = @test_logs (:warn, r"Upload aborted") put_file(storage, "a/b.json", "data")
|
||||
@test result === nothing
|
||||
end
|
||||
|
||||
@testset "put_file function returns nothing for keys with multiple slashes" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
result = @test_logs (:warn, r"Upload aborted") put_file(storage, "a/b/c/d.json", "data")
|
||||
@test result === nothing
|
||||
end
|
||||
|
||||
@testset "put_file function returns nothing for key with single slash in middle" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
result = @test_logs (:warn, r"Upload aborted") put_file(storage, "folder/file.json", "data")
|
||||
@test result === nothing
|
||||
end
|
||||
|
||||
@testset "put_file function returns nothing for key with trailing slash pattern" begin
|
||||
storage = GarageStorage("https://s3-api.example.com", "key", "secret", "bucket")
|
||||
result = @test_logs (:warn, r"Upload aborted") put_file(storage, "a/b", "data")
|
||||
@test result === nothing
|
||||
end
|
||||
|
||||
# --- Integration tests with real Garage server ---
|
||||
|
||||
@testset "integration: full CRUD lifecycle" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-$(randstring(8)).json"
|
||||
test_data = "{\"status\": \"testing\", \"runner\": \"runtests\"}"
|
||||
|
||||
# Test put_file(storage, key, data) - successful upload
|
||||
result = put_file(storage, test_key, test_data)
|
||||
@test result !== nothing
|
||||
@test haskey(result, :api)
|
||||
@test haskey(result, :web)
|
||||
@test occursin("testbucket", result.api)
|
||||
@test occursin("s3-api", result.api)
|
||||
@test occursin("s3-web", result.web)
|
||||
|
||||
# Test get_file - retrieve the uploaded data
|
||||
downloaded = get_file(storage, test_key)
|
||||
@test downloaded !== nothing
|
||||
@test downloaded isa Vector{UInt8}
|
||||
@test String(downloaded) == test_data
|
||||
|
||||
# Test list_files - verify key appears in listing
|
||||
keys = list_files(storage)
|
||||
@test test_key in keys
|
||||
|
||||
# Test put_file callable struct
|
||||
test_key2 = "testrunner-callable-$(randstring(8)).json"
|
||||
callable_result = put_file(storage)(test_key2, "callable-data")
|
||||
@test callable_result !== nothing
|
||||
@test callable_result.api != ""
|
||||
@test callable_result.web != ""
|
||||
|
||||
# Verify callable upload
|
||||
downloaded2 = get_file(storage, test_key2)
|
||||
@test String(downloaded2) == "callable-data"
|
||||
|
||||
# Test delete_file
|
||||
delete_file(storage, test_key)
|
||||
deleted_data = get_file(storage, test_key)
|
||||
@test deleted_data === nothing
|
||||
|
||||
# Cleanup callable test object
|
||||
delete_file(storage, test_key2)
|
||||
deleted_data2 = get_file(storage, test_key2)
|
||||
@test deleted_data2 === nothing
|
||||
end
|
||||
|
||||
@testset "integration: put_file rejects slashes even with real server" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
result = @test_logs (:warn, r"Upload aborted") put_file(storage, "folder/test.json", "data")
|
||||
@test result === nothing
|
||||
end
|
||||
|
||||
@testset "integration: get_file returns nothing for missing key" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
result = get_file(storage, "nonexistent-key-$(randstring(8)).json")
|
||||
@test result === nothing
|
||||
end
|
||||
|
||||
@testset "integration: binary data upload and download" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-binary-$(randstring(8)).bin"
|
||||
binary_data = UInt8[0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x48, 0x65, 0x6c, 0x6c, 0x6f]
|
||||
|
||||
result = put_file(storage, test_key, binary_data)
|
||||
@test result !== nothing
|
||||
|
||||
downloaded = get_file(storage, test_key)
|
||||
@test downloaded == binary_data
|
||||
|
||||
delete_file(storage, test_key)
|
||||
@test get_file(storage, test_key) === nothing
|
||||
end
|
||||
|
||||
@testset "integration: list_files returns vector of strings" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-list-$(randstring(8)).txt"
|
||||
put_file(storage, test_key, "list-test-data")
|
||||
|
||||
keys = list_files(storage)
|
||||
@test keys isa Vector{String}
|
||||
@test test_key in keys
|
||||
|
||||
delete_file(storage, test_key)
|
||||
end
|
||||
|
||||
@testset "integration: url format validation" begin
|
||||
storage = GarageStorage(
|
||||
"https://s3-api.yiem.cc",
|
||||
"GKb080154a2e5b19100b1b2c6e",
|
||||
"a2c6b1379c2f731d3e6e5a408dd4d6cffca7511717675f55114ff94828febca1",
|
||||
"testbucket"
|
||||
)
|
||||
|
||||
test_key = "testrunner-url-$(randstring(8)).json"
|
||||
result = put_file(storage, test_key, "url-test")
|
||||
|
||||
@test startswith(result.api, "https://s3-api.yiem.cc/testbucket/")
|
||||
@test occursin("/$test_key", result.api)
|
||||
@test startswith(result.web, "https://s3-web.yiem.cc/testbucket/")
|
||||
@test occursin("/$test_key", result.web)
|
||||
|
||||
delete_file(storage, test_key)
|
||||
end
|
||||
|
||||
@test detect_keyword(["test"], "testing.test.tester"; mode="individual", delimiter=['.']) == Dict("test" => 1)
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user