From 302f506b5bd087899a4c7e1941942d6479c73308 Mon Sep 17 00:00:00 2001 From: ton Date: Sun, 6 Aug 2023 08:15:44 +0700 Subject: [PATCH] version 0.0.2 --- previousVersion/0.0.2/Manifest.toml | 911 ++++++++++++++++++++++++ previousVersion/0.0.2/Project.toml | 14 + previousVersion/0.0.2/src/IronpenGPU.jl | 85 +++ previousVersion/0.0.2/src/forward.jl | 809 +++++++++++++++++++++ previousVersion/0.0.2/src/interface.jl | 87 +++ previousVersion/0.0.2/src/learn.jl | 312 ++++++++ previousVersion/0.0.2/src/snnUtil.jl | 77 ++ previousVersion/0.0.2/src/type.jl | 427 +++++++++++ src/IronpenGPU.jl | 7 +- src/forward.jl | 20 +- src/learn.jl | 29 +- 11 files changed, 2755 insertions(+), 23 deletions(-) create mode 100644 previousVersion/0.0.2/Manifest.toml create mode 100644 previousVersion/0.0.2/Project.toml create mode 100644 previousVersion/0.0.2/src/IronpenGPU.jl create mode 100644 previousVersion/0.0.2/src/forward.jl create mode 100644 previousVersion/0.0.2/src/interface.jl create mode 100644 previousVersion/0.0.2/src/learn.jl create mode 100644 previousVersion/0.0.2/src/snnUtil.jl create mode 100644 previousVersion/0.0.2/src/type.jl diff --git a/previousVersion/0.0.2/Manifest.toml b/previousVersion/0.0.2/Manifest.toml new file mode 100644 index 0000000..06c61f1 --- /dev/null +++ b/previousVersion/0.0.2/Manifest.toml @@ -0,0 +1,911 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.9.2" +manifest_format = "2.0" +project_hash = "1d38b0278f78d536c218e3a421dfd88a68063099" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "8bc0aaec0ca548eb6cf5f0d7d16351650c1ee956" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.3.2" +weakdeps = ["ChainRulesCore"] + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "76289dc51920fdc6e0013c872ba9551d54961c24" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "3.6.2" +weakdeps = ["StaticArrays"] + + [deps.Adapt.extensions] + AdaptStaticArraysExt = "StaticArrays" + +[[deps.ArgCheck]] +git-tree-sha1 = "a3a402a35a2f7e0b87828ccabbd5ebfbebe356b4" +uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" +version = "2.3.0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Atomix]] +deps = ["UnsafeAtomics"] +git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" +uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" +version = "0.1.0" + +[[deps.BFloat16s]] +deps = ["LinearAlgebra", "Printf", "Random", "Test"] +git-tree-sha1 = "dbf84058d0a8cbbadee18d25cf606934b22d7c66" +uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" +version = "0.4.2" + +[[deps.BangBang]] +deps = ["Compat", "ConstructionBase", "InitialValues", "LinearAlgebra", "Requires", "Setfield", "Tables"] +git-tree-sha1 = "e28912ce94077686443433c2800104b061a827ed" +uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" +version = "0.3.39" + + [deps.BangBang.extensions] + BangBangChainRulesCoreExt = "ChainRulesCore" + BangBangDataFramesExt = "DataFrames" + BangBangStaticArraysExt = "StaticArrays" + BangBangStructArraysExt = "StructArrays" + BangBangTypedTablesExt = "TypedTables" + + [deps.BangBang.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.Baselet]] +git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e" +uuid = "9718e550-a3fa-408a-8086-8db961cd8217" +version = "0.1.1" + +[[deps.CEnum]] +git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.4.2" + +[[deps.CUDA]] +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "CompilerSupportLibraries_jll", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Preferences", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "442d989978ed3ff4e174c928ee879dc09d1ef693" +uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" +version = "4.3.2" + +[[deps.CUDA_Driver_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] +git-tree-sha1 = "498f45593f6ddc0adff64a9310bb6710e851781b" +uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" +version = "0.5.0+1" + +[[deps.CUDA_Runtime_Discovery]] +deps = ["Libdl"] +git-tree-sha1 = "bcc4a23cbbd99c8535a5318455dcf0f2546ec536" +uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" +version = "0.2.2" + +[[deps.CUDA_Runtime_jll]] +deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "5248d9c45712e51e27ba9b30eebec65658c6ce29" +uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" +version = "0.6.0+0" + +[[deps.CUDNN_jll]] +deps = ["Artifacts", "CUDA_Runtime_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "2918fbffb50e3b7a0b9127617587afa76d4276e8" +uuid = "62b44479-cb7b-5706-934f-f13b2eb2e645" +version = "8.8.1+0" + +[[deps.Calculus]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad" +uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9" +version = "0.5.1" + +[[deps.ChainRules]] +deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "Statistics", "StructArrays"] +git-tree-sha1 = "1cdf290d4feec68824bfb84f4bfc9f3aba185647" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "1.51.1" + +[[deps.ChainRulesCore]] +deps = ["Compat", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "e30f2f4e20f7f186dc36529910beaedc60cfa644" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.16.0" + +[[deps.CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + +[[deps.Compat]] +deps = ["UUIDs"] +git-tree-sha1 = "4e88377ae7ebeaf29a047aa1ee40826e0b708a5d" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.7.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.5+0" + +[[deps.CompositionsBase]] +git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" +uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" +version = "0.1.2" + + [deps.CompositionsBase.extensions] + CompositionsBaseInverseFunctionsExt = "InverseFunctions" + + [deps.CompositionsBase.weakdeps] + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.ConstructionBase]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "738fec4d684a9a6ee9598a8bfee305b26831f28c" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.5.2" + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseStaticArraysExt = "StaticArrays" + + [deps.ConstructionBase.weakdeps] + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.ContextVariablesX]] +deps = ["Compat", "Logging", "UUIDs"] +git-tree-sha1 = "25cc3803f1030ab855e383129dcd3dc294e322cc" +uuid = "6add18c4-b38d-439d-96f6-d6bc489c04c5" +version = "0.1.3" + +[[deps.DataAPI]] +git-tree-sha1 = "8da84edb865b0b5b0100c0666a9bc9a0b71c553c" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.15.0" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "cf25ccb972fec4e4817764d01c82386ae94f77b4" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.14" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.DefineSingletons]] +git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c" +uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52" +version = "0.1.2" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.DiffResults]] +deps = ["StaticArraysCore"] +git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.1.0" + +[[deps.DiffRules]] +deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.15.1" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.Distributions]] +deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns", "Test"] +git-tree-sha1 = "e76a3281de2719d7c81ed62c6ea7057380c87b1d" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.98" + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + + [deps.Distributions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.3" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.DualNumbers]] +deps = ["Calculus", "NaNMath", "SpecialFunctions"] +git-tree-sha1 = "5837a837389fccf076445fce071c8ddaea35a566" +uuid = "fa6b7ba4-c1ee-5f82-b5fc-ecf0adba8f74" +version = "0.6.8" + +[[deps.ExprTools]] +git-tree-sha1 = "c1d06d129da9f55715c6c212866f5b1bddc5fa00" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.9" + +[[deps.FLoops]] +deps = ["BangBang", "Compat", "FLoopsBase", "InitialValues", "JuliaVariables", "MLStyle", "Serialization", "Setfield", "Transducers"] +git-tree-sha1 = "ffb97765602e3cbe59a0589d237bf07f245a8576" +uuid = "cc61a311-1640-44b5-9fba-1b764f453329" +version = "0.2.1" + +[[deps.FLoopsBase]] +deps = ["ContextVariablesX"] +git-tree-sha1 = "656f7a6859be8673bf1f35da5670246b923964f7" +uuid = "b9860ae5-e623-471e-878b-f6a53c775ea6" +version = "0.1.1" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] +git-tree-sha1 = "0b3b52afd0f87b0a3f5ada0466352d125c9db458" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.2.1" + +[[deps.Flux]] +deps = ["Adapt", "CUDA", "ChainRulesCore", "Functors", "LinearAlgebra", "MLUtils", "MacroTools", "NNlib", "NNlibCUDA", "OneHotArrays", "Optimisers", "Preferences", "ProgressLogging", "Random", "Reexport", "SparseArrays", "SpecialFunctions", "Statistics", "Zygote", "cuDNN"] +git-tree-sha1 = "3e2c3704c2173ab4b1935362384ca878b53d4c34" +uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" +version = "0.13.17" + + [deps.Flux.extensions] + AMDGPUExt = "AMDGPU" + FluxMetalExt = "Metal" + + [deps.Flux.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + +[[deps.ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] +git-tree-sha1 = "00e252f4d706b3d55a8863432e742bf5717b498d" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.35" +weakdeps = ["StaticArrays"] + + [deps.ForwardDiff.extensions] + ForwardDiffStaticArraysExt = "StaticArrays" + +[[deps.Functors]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "478f8c3145bb91d82c2cf20433e8c1b30df454cc" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.4.4" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[deps.GPUArrays]] +deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] +git-tree-sha1 = "2e57b4a4f9cc15e85a24d603256fe08e527f48d1" +uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" +version = "8.8.1" + +[[deps.GPUArraysCore]] +deps = ["Adapt"] +git-tree-sha1 = "2d6ca471a6c7b536127afccfa7564b5b39227fe0" +uuid = "46192b85-c4d5-4398-a991-12ede77f4527" +version = "0.1.5" + +[[deps.GPUCompiler]] +deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "TimerOutputs", "UUIDs"] +git-tree-sha1 = "cb090aea21c6ca78d59672a7e7d13bd56d09de64" +uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" +version = "0.20.3" + +[[deps.GeneralUtils]] +deps = ["CUDA", "DataStructures", "Distributions", "Flux", "JSON3"] +path = "C:\\Users\\naraw\\.julia\\dev\\GeneralUtils" +uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" +version = "0.1.0" + +[[deps.HypergeometricFunctions]] +deps = ["DualNumbers", "LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "ce7ea9cc5db29563b1fe20196b6d23ab3b111384" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.18" + +[[deps.IRTools]] +deps = ["InteractiveUtils", "MacroTools", "Test"] +git-tree-sha1 = "eac00994ce3229a464c2847e956d77a2c64ad3a5" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.4.10" + +[[deps.InitialValues]] +git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" +uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" +version = "0.3.1" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.2" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Preferences"] +git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.4.1" + +[[deps.JSON3]] +deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] +git-tree-sha1 = "5b62d93f2582b09e469b3099d839c2d2ebf5066d" +uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +version = "1.13.1" + +[[deps.JuliaVariables]] +deps = ["MLStyle", "NameResolution"] +git-tree-sha1 = "49fb3cb53362ddadb4415e9b73926d6b40709e70" +uuid = "b14d175d-62b4-44ba-8fb7-3064adc8c3ec" +version = "0.2.4" + +[[deps.KernelAbstractions]] +deps = ["Adapt", "Atomix", "InteractiveUtils", "LinearAlgebra", "MacroTools", "PrecompileTools", "SparseArrays", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "b48617c5d764908b5fac493cd907cf33cc11eec1" +uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" +version = "0.9.6" + +[[deps.LLVM]] +deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"] +git-tree-sha1 = "5007c1421563108110bbd57f63d8ad4565808818" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "5.2.0" + +[[deps.LLVMExtra_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "1222116d7313cdefecf3d45a2bc1a89c4e7c9217" +uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" +version = "0.0.22+0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "c3ce8e7420b3a6e071e0fe4745f5d4300e37b13f" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.24" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.MLStyle]] +git-tree-sha1 = "bc38dff0548128765760c79eb7388a4b37fae2c8" +uuid = "d8e11817-5142-5d16-987a-aa16d5891078" +version = "0.4.17" + +[[deps.MLUtils]] +deps = ["ChainRulesCore", "Compat", "DataAPI", "DelimitedFiles", "FLoops", "NNlib", "Random", "ShowCases", "SimpleTraits", "Statistics", "StatsBase", "Tables", "Transducers"] +git-tree-sha1 = "3504cdb8c2bc05bde4d4b09a81b01df88fcbbba0" +uuid = "f1d291b0-491e-4a28-83b9-f70985020b54" +version = "0.4.3" + +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "42324d08725e200c23d4dfb549e0d5d89dede2d2" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.10" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" + +[[deps.MicroCollections]] +deps = ["BangBang", "InitialValues", "Setfield"] +git-tree-sha1 = "629afd7d10dbc6935ec59b32daeb33bc4460a42e" +uuid = "128add7d-3638-4c79-886c-908ea0c25c34" +version = "0.1.4" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "f66bdc5de519e8f8ae43bdc598782d35a25b1272" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.1.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" + +[[deps.NNlib]] +deps = ["Adapt", "Atomix", "ChainRulesCore", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "Pkg", "Random", "Requires", "Statistics"] +git-tree-sha1 = "72240e3f5ca031937bd536182cb2c031da5f46dd" +uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +version = "0.8.21" + + [deps.NNlib.extensions] + NNlibAMDGPUExt = "AMDGPU" + + [deps.NNlib.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + +[[deps.NNlibCUDA]] +deps = ["Adapt", "CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics", "cuDNN"] +git-tree-sha1 = "f94a9684394ff0d325cc12b06da7032d8be01aaf" +uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" +version = "0.2.7" + +[[deps.NaNMath]] +deps = ["OpenLibm_jll"] +git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "1.0.2" + +[[deps.NameResolution]] +deps = ["PrettyPrint"] +git-tree-sha1 = "1a0fa0e9613f46c9b8c11eee38ebb4f590013c5e" +uuid = "71a1bf82-56d0-4bbc-8a3c-48b961074391" +version = "0.1.5" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OneHotArrays]] +deps = ["Adapt", "ChainRulesCore", "Compat", "GPUArraysCore", "LinearAlgebra", "NNlib"] +git-tree-sha1 = "5e4029759e8699ec12ebdf8721e51a659443403c" +uuid = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f" +version = "0.2.4" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.21+4" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+0" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.5+0" + +[[deps.Optimisers]] +deps = ["ChainRulesCore", "Functors", "LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "6a01f65dd8583dee82eecc2a19b0ff21521aa749" +uuid = "3bd65402-5787-11e9-1adc-39752487f4e2" +version = "0.2.18" + +[[deps.OrderedCollections]] +git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.0" + +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "67eae2738d63117a196f497d7db789821bce61d1" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.17" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "4b2e829ee66d4218e0cef22c0a64ee37cf258c29" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.7.1" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.2" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.1.2" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.0" + +[[deps.PrettyPrint]] +git-tree-sha1 = "632eb4abab3449ab30c5e1afaa874f0b98b586e4" +uuid = "8162dcfd-2161-5ef2-ae6c-7681170c5f98" +version = "0.2.0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.ProgressLogging]] +deps = ["Logging", "SHA", "UUIDs"] +git-tree-sha1 = "80d919dee55b9c50e8d9e2da5eeafff3fe58b539" +uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" +version = "0.1.4" + +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "6ec7ac8412e83d57e313393220879ede1740f9ee" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.8.2" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA", "Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.Random123]] +deps = ["Random", "RandomNumbers"] +git-tree-sha1 = "552f30e847641591ba3f39fd1bed559b9deb0ef3" +uuid = "74087812-796a-5b5d-8853-05524746bad3" +version = "1.6.1" + +[[deps.RandomNumbers]] +deps = ["Random", "Requires"] +git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.5.3" + +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.7.1" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6ed52fdd3382cf21947b15e8870ac0ddbff736da" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.4.0+0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "30449ee12237627992a99d5e30ae63e4d78cd24a" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.2.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] +git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "1.1.1" + +[[deps.ShowCases]] +git-tree-sha1 = "7f534ad62ab2bd48591bdeac81994ea8c445e4a5" +uuid = "605ecd9f-84a6-4c9e-81e2-4798472b76a3" +version = "0.1.0" + +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.4" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "c60ec5c62180f27efea3ba2908480f8055e17cee" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.1.1" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "7beb031cf8145577fbccacd94b8a8f4ce78428d3" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.3.0" +weakdeps = ["ChainRulesCore"] + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + +[[deps.SplittablesBase]] +deps = ["Setfield", "Test"] +git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5" +uuid = "171d559e-b47b-412a-8079-5efa626c420e" +version = "0.1.15" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "Random", "StaticArraysCore", "Statistics"] +git-tree-sha1 = "832afbae2a45b4ae7e831f86965469a24d1d8a83" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.5.26" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "6b7ba252635a5eff6a0b0664a41ee140a1c9e72a" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.0" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.9.0" + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "45a7769a04a3cf80da1c1c7c60caf932e6f4c9f7" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.6.0" + +[[deps.StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "75ebe04c5bed70b91614d684259b661c9e6274a4" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.0" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "f625d686d5a88bcd2b15cd81f18f98186fdc0c9a" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.3.0" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + +[[deps.StructArrays]] +deps = ["Adapt", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] +git-tree-sha1 = "521a0e828e98bb69042fec1809c1b5a680eb7389" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.6.15" + +[[deps.StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.10.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "5.10.1+6" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] +git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.10.1" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.TimerOutputs]] +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "f548a9e9c490030e545f72074a41edfd0e5bcdd7" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.23" + +[[deps.Transducers]] +deps = ["Adapt", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "Requires", "Setfield", "SplittablesBase", "Tables"] +git-tree-sha1 = "a66fb81baec325cf6ccafa243af573b031e87b00" +uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" +version = "0.4.77" + + [deps.Transducers.extensions] + TransducersBlockArraysExt = "BlockArrays" + TransducersDataFramesExt = "DataFrames" + TransducersLazyArraysExt = "LazyArrays" + TransducersOnlineStatsBaseExt = "OnlineStatsBase" + TransducersReferenceablesExt = "Referenceables" + + [deps.Transducers.weakdeps] + BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" + OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" + Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.UnsafeAtomics]] +git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" +uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" +version = "0.2.1" + +[[deps.UnsafeAtomicsLLVM]] +deps = ["LLVM", "UnsafeAtomics"] +git-tree-sha1 = "ea37e6066bf194ab78f4e747f5245261f17a7175" +uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" +version = "0.1.2" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" + +[[deps.Zygote]] +deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "GPUArrays", "GPUArraysCore", "IRTools", "InteractiveUtils", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NaNMath", "PrecompileTools", "Random", "Requires", "SparseArrays", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "5be3ddb88fc992a7d8ea96c3f10a49a7e98ebc7b" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.6.62" + + [deps.Zygote.extensions] + ZygoteColorsExt = "Colors" + ZygoteDistancesExt = "Distances" + ZygoteTrackerExt = "Tracker" + + [deps.Zygote.weakdeps] + Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" + Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[[deps.ZygoteRules]] +deps = ["ChainRulesCore", "MacroTools"] +git-tree-sha1 = "977aed5d006b840e2e40c0b48984f7463109046d" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.3" + +[[deps.cuDNN]] +deps = ["CEnum", "CUDA", "CUDNN_jll"] +git-tree-sha1 = "f65490d187861d6222cb38bcbbff3fd949a7ec3e" +uuid = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" +version = "1.0.4" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/previousVersion/0.0.2/Project.toml b/previousVersion/0.0.2/Project.toml new file mode 100644 index 0000000..fe5d5b9 --- /dev/null +++ b/previousVersion/0.0.2/Project.toml @@ -0,0 +1,14 @@ +name = "IronpenGPU" +uuid = "3d5396ea-818e-43fc-a9d3-164248e840cd" +authors = ["ton "] +version = "0.1.0" + +[deps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" +JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" diff --git a/previousVersion/0.0.2/src/IronpenGPU.jl b/previousVersion/0.0.2/src/IronpenGPU.jl new file mode 100644 index 0000000..6dfd527 --- /dev/null +++ b/previousVersion/0.0.2/src/IronpenGPU.jl @@ -0,0 +1,85 @@ +module IronpenGPU # this is a parent module + +# export + + +""" Order by dependencies of each file. The 1st included file must not depend on any other +files and each file can only depend on the file included before it. +""" + +include("type.jl") +using .type # bring type into parent module namespace + +include("snnUtil.jl") +using .snnUtil + +include("forward.jl") +using .forward + +include("learn.jl") +using .learn + +include("interface.jl") +using .interface + + +#------------------------------------------------------------------------------------------------100 + +""" version 0.0.2 + Todo: + [] use partial error update for computeNeuron + [] use integrate_neuron_params synapticConnectionPercent LESS THAN 100% + [2] implement dormant connection and pruning machanism. the longer the training the longer + 0 weight stay 0. + [] using RL to control learning signal + [] consider using Dates.now() instead of timestamp because time_stamp may overflow + [] Liquid time constant. training should include adjusting α, neuron membrane potential decay factor + which defined by neuron.tau_m formula in type.jl + + Change from version: 0.0.1 + - knowledgeFn in GPU format + + All features + +""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module IronpenGPU diff --git a/previousVersion/0.0.2/src/forward.jl b/previousVersion/0.0.2/src/forward.jl new file mode 100644 index 0000000..4790036 --- /dev/null +++ b/previousVersion/0.0.2/src/forward.jl @@ -0,0 +1,809 @@ +module forward + +# export + +using Flux, CUDA +using GeneralUtils +using ..type, ..snnUtil + +#------------------------------------------------------------------------------------------------100 + +""" kfn forward + input (row, col, batch) +""" +function (kfn::kfn_1)(input::AbstractArray) + + kfn.timeStep .+= 1 + + #TODO time step forward + if view(kfn.learningStage, 1)[1] == 1 + # reset learning params + kfn.lif_vt .= 0 + kfn.lif_wRecChange .= 0 + kfn.lif_epsilonRec .= 0 + + kfn.alif_vt .= 0 + kfn.alif_epsilonRec .= 0 + kfn.alif_wRecChange .= 0 + + kfn.on_vt .= 0 + kfn.on_epsilonRec .= 0 + kfn.on_wOutChange .= 0 + + kfn.learningStage = [2] + end + + # update activation matrix with "lif_zt1" and "alif_zt1" by concatenating + # (input, lif_zt1, alif_zt1) to form activation matrix + _zit = cat(reshape(input, (size(input, 1), size(input, 2), 1, size(input, 3))), + reshape(kfn.lif_zt, (size(input, 1), :, 1, size(input, 3))), + reshape(kfn.alif_zt, (size(input, 1), :, 1, size(input, 3))), dims=2) + kfn.zit .= reshape(_zit, (size(input, 1), :, size(input, 3))) + + # project 3D kfn zit into 4D lif zit + i1, i2, i3, i4 = size(kfn.lif_zit) + kfn.lif_zit .= reshape(kfn.zit, (i1, i2, 1, i4)) .* kfn.lif_arrayProjection4d + + lifForward( kfn.lif_zit, + kfn.lif_wRec, + kfn.lif_vt, + kfn.lif_vth, + kfn.lif_vRest, + kfn.lif_zt4d, + kfn.lif_alpha, + kfn.lif_phi, + kfn.lif_epsilonRec, + kfn.lif_refractoryCounter, + kfn.lif_refractoryDuration, + kfn.lif_gammaPd, + kfn.lif_firingCounter, + kfn.lif_recSignal,) + + # project 3D kfn zit into 4D alif zit + i1, i2, i3, i4 = size(kfn.alif_zit) + kfn.alif_zit .= reshape(kfn.zit, (i1, i2, 1, i4)) .* kfn.alif_arrayProjection4d + + alifForward(kfn.alif_zit, + kfn.alif_wRec, + kfn.alif_vt, + kfn.alif_vth, + kfn.alif_vRest, + kfn.alif_zt4d, + kfn.alif_alpha, + kfn.alif_phi, + kfn.alif_epsilonRec, + kfn.alif_refractoryCounter, + kfn.alif_refractoryDuration, + kfn.alif_gammaPd, + kfn.alif_firingCounter, + kfn.alif_recSignal, + kfn.alif_epsilonRecA, + kfn.alif_a, + kfn.alif_avth, + kfn.alif_beta, + kfn.alif_rho,) + + # reduce lif_zt4d and alif_zt4d into lif_zt, alif_zt (4d -> 1d) + kfn.lif_zt .= reduce(max, kfn.lif_zt4d, dims=(1,2)) + kfn.alif_zt .= reduce(max, kfn.alif_zt4d, dims=(1,2)) + + # update activation matrix with "lif_zt1" and "alif_zt1" by concatenating + # (input, lif_zt1, alif_zt1) to form activation matrix + _zit = cat(reshape(input, (size(input, 1), size(input, 2), 1, size(input, 3))), + reshape(kfn.lif_zt, (size(input, 1), :, 1, size(input, 3))), + reshape(kfn.alif_zt, (size(input, 1), :, 1, size(input, 3))), dims=2) + kfn.zit .= reshape(_zit, (size(input, 1), :, size(input, 3))) + + # project 3D kfn zit into 4D on zit + i1, i2, i3, i4 = size(kfn.on_zit) + kfn.on_zit .= reshape(kfn.zit, (i1, i2, 1, i4)) .* kfn.on_arrayProjection4d + + # read out + onForward( kfn.on_zit, + kfn.on_wOut, + kfn.on_vt, + kfn.on_vth, + kfn.on_vRest, + kfn.on_zt4d, + kfn.on_alpha, + kfn.on_phi, + kfn.on_epsilonRec, + kfn.on_refractoryCounter, + kfn.on_refractoryDuration, + kfn.on_gammaPd, + kfn.on_firingCounter, + kfn.on_recSignal,) + # error("DEBUG -> kfn forward") + logit = reshape(kfn.on_zt, (size(input, 1), :)) + + return logit, + kfn.zit +end + + +function lifForward(kfn_zit::Array{T}, + zit::Array{T}, + wRec::Array{T}, + vt0::Array{T}, + vt1::Array{T}, + vth::Array{T}, + vRest::Array{T}, + zt1::Array{T}, + alpha::Array{T}, + phi::Array{T}, + epsilonRec::Array{T}, + refractoryCounter::Array{T}, + refractoryDuration::Array{T}, + gammaPd::Array{T}, + firingCounter::Array{T}, + arrayProjection4d::Array{T}, + recSignal::Array{T}, + decayed_vt0::Array{T}, + decayed_epsilonRec::Array{T}, + vt1_diff_vth::Array{T}, + vt1_diff_vth_div_vth::Array{T}, + gammaPd_div_vth::Array{T}, + phiActivation::Array{T}, + ) where T<:Number + + # project 3D kfn zit into 4D lif zit + i1, i2, i3, i4 = size(alif_wRec) + lif_zit .= reshape(kfn_zit, (i1, i2, 1, i4)) .* lif_arrayProjection4d + + for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch + if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active + @. @views refractoryCounter[:,:,i,j] -= 1 + @. @views zt1[:,:,i,j] = 0 + @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] + @. @views phi[:,:,i,j] = 0 + + # compute epsilonRec + @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] + @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + else # refractory period is inactive + @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j] + @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] + @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) + + if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j])) + @. @views zt1[:,:,i,j] = 1 + @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] + @. @views firingCounter[:,:,i,j] += 1 + @. @views vt1[:,:,i,j] = vRest[:,:,i,j] + else + @. @views zt1[:,:,i,j] = 0 + end + + # compute phi, there is a difference from alif formula + @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] + @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] + @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] + @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) + @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] + + # compute epsilonRec + @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] + @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] + end + end +end + +# gpu launcher +function lifForward( lif_zit::CuArray, + lif_wRec::CuArray, + lif_vt::CuArray, + lif_vth::CuArray, + lif_vRest::CuArray, + lif_zt::CuArray, + lif_alpha::CuArray, + lif_phi::CuArray, + lif_epsilonRec::CuArray, + lif_refractoryCounter::CuArray, + lif_refractoryDuration::CuArray, + lif_gammaPd::CuArray, + lif_firingCounter::CuArray, + lif_recSignal::CuArray,) + + kernel = @cuda launch=false lifForward( lif_zit, + lif_wRec, + lif_vt, + lif_vth, + lif_vRest, + lif_zt, + lif_alpha, + lif_phi, + lif_epsilonRec, + lif_refractoryCounter, + lif_refractoryDuration, + lif_gammaPd, + lif_firingCounter, + lif_recSignal, + GeneralUtils.linear_to_cartesian) + config = launch_configuration(kernel.fun) + + + # threads to be launched. Since one can't launch exact thread number the kernel needs, + # one just launch threads more than this kernel needs then use a guard inside the kernel + # to prevent unused threads to access memory. + threads = min(1024, config.threads) # depend on gpu. Most NVIDIA gpu has 1024 threads per block + + # total desired threads to launch to gpu. Usually 1 thread per 1 matrix element + totalThreads = length(lif_wRec) + + blocks = cld(totalThreads, threads) + # println("launching gpu kernel") + CUDA.@sync begin + kernel( lif_zit, + lif_wRec, + lif_vt, + lif_vth, + lif_vRest, + lif_zt, + lif_alpha, + lif_phi, + lif_epsilonRec, + lif_refractoryCounter, + lif_refractoryDuration, + lif_gammaPd, + lif_firingCounter, + lif_recSignal, + GeneralUtils.linear_to_cartesian; threads, blocks) + end +end + +# gpu kernel +function lifForward( zit, + wRec, + vt, + vth, + vRest, + zt, + alpha, + phi, + epsilonRec, + refractoryCounter, + refractoryDuration, + gammaPd, + firingCounter, + recSignal, + linear_to_cartesian) + i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index + + if i <= length(wRec) + # cartesian index + i1, i2, i3, i4 = linear_to_cartesian(i, size(wRec)) + # @cuprintln("gpu thread $i $i1 $i2 $i3 $i4") + + refractoryCounter[i] -= 1 + + if refractoryCounter[i] > 0 # refractory period is active + refractoryCounter[i] -= 1 + zt[i] = 0 + vt[i] = alpha[i] * vt[i] + phi[i] = 0 + + # compute epsilonRec + epsilonRec[i] = (alpha[i] * epsilonRec[i]) + zit[i] + + else # refractory period is inactive + recSignal[i] = zit[i] * wRec[i] + vt[i] = (alpha[i] * vt[i]) + sum(@view(recSignal[:,:,i3,i4])) + + # fires if membrane potential exceed threshold + if vt[i] > vth[i] + zt[i] = 1 + refractoryCounter[i] = refractoryDuration[i] + firingCounter[i] += 1 + vt[i] = vRest[i] + else + zt[i] = 0 + end + + # compute phi, there is a difference from lif formula + phi[i] = (gammaPd[i] / vth[i]) * max(0, 1 - ((vt[i] - vth[i]) / vth[i])) + + # compute epsilonRec + epsilonRec[i] = (alpha[i] * epsilonRec[i]) + zit[i] + end + end + return nothing +end + +function alifForward(zit::Array{T}, + wRec::Array{T}, + vt0::Array{T}, + vt1::Array{T}, + vth::Array{T}, + vRest::Array{T}, + zt1::Array{T}, + alpha::Array{T}, + phi::Array{T}, + epsilonRec::Array{T}, + refractoryCounter::Array{T}, + refractoryDuration::Array{T}, + gammaPd::Array{T}, + firingCounter::Array{T}, + recSignal::Array{T}, + decayed_vt0::Array{T}, + decayed_epsilonRec::Array{T}, + vt1_diff_vth::Array{T}, + vt1_diff_vth_div_vth::Array{T}, + gammaPd_div_vth::Array{T}, + phiActivation::Array{T}, + + epsilonRecA::Array{T}, + avth::Array{T}, + a::Array{T}, + beta::Array{T}, + rho::Array{T}, + phi_x_epsilonRec::Array{T}, + phi_x_beta::Array{T}, + rho_diff_phi_x_beta::Array{T}, + rho_div_phi_x_beta_x_epsilonRecA::Array{T}, + beta_x_a::Array{T}, + ) where T<:Number + + for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch + if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active + @. @views refractoryCounter[:,:,i,j] -= 1 + @. @views zt1[:,:,i,j] = 0 + @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] + @. @views phi[:,:,i,j] = 0 + @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] + + # compute epsilonRec + @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] + @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + + # compute epsilonRecA + @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j] + @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j] + @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j] + @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j] + @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] + + # compute avth + @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j] + @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j] + + else # refractory period is inactive + @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j] + @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] + @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) + + # compute avth + @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j] + @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j] + + if sum(@view(vt1[:,:,i,j])) > sum(@view(avth[:,:,i,j])) + @. @views zt1[:,:,i,j] = 1 + @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] + @. @views firingCounter[:,:,i,j] += 1 + @. @views vt1[:,:,i,j] = vRest[:,:,i,j] + @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] + @. @views a[:,:,i,j] = a[:,:,i,j] += 1 + else + @. @views zt1[:,:,i,j] = 0 + @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] + end + + # compute phi, there is a difference from alif formula + @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] + @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] + @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] + @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) + @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] + + # compute epsilonRec + @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] + @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] + + # compute epsilonRecA + @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j] + @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j] + @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j] + @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j] + @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] + end + end +end + +# gpu launcher +function alifForward( alif_zit::CuArray, + alif_wRec::CuArray, + alif_vt::CuArray, + alif_vth::CuArray, + alif_vRest::CuArray, + alif_zt::CuArray, + alif_alpha::CuArray, + alif_phi::CuArray, + alif_epsilonRec::CuArray, + alif_refractoryCounter::CuArray, + alif_refractoryDuration::CuArray, + alif_gammaPd::CuArray, + alif_firingCounter::CuArray, + alif_recSignal::CuArray, + alif_epsilonRecA::CuArray, + alif_a::CuArray, + alif_avth::CuArray, + alif_beta::CuArray, + alif_rho::CuArray, + ) + + kernel = @cuda launch=false alifForward( alif_zit, + alif_wRec, + alif_vt, + alif_vth, + alif_vRest, + alif_zt, + alif_alpha, + alif_phi, + alif_epsilonRec, + alif_refractoryCounter, + alif_refractoryDuration, + alif_gammaPd, + alif_firingCounter, + alif_recSignal, + alif_epsilonRecA, + alif_a, + alif_avth, + alif_beta, + alif_rho, + GeneralUtils.linear_to_cartesian) + config = launch_configuration(kernel.fun) + + # threads to be launched. Since one can't launch exact thread number the kernel needs, + # one just launch threads more than this kernel needs then use a guard inside the kernel + # to prevent unused threads to access memory. + threads = min(1024, config.threads) # depend on gpu. Most NVIDIA gpu has 1024 threads per block + + # total desired threads to launch to gpu. Usually 1 thread per 1 matrix element + totalThreads = length(alif_wRec) + + blocks = cld(totalThreads, threads) + # println("launching gpu kernel") + CUDA.@sync begin + kernel( alif_zit, + alif_wRec, + alif_vt, + alif_vth, + alif_vRest, + alif_zt, + alif_alpha, + alif_phi, + alif_epsilonRec, + alif_refractoryCounter, + alif_refractoryDuration, + alif_gammaPd, + alif_firingCounter, + alif_recSignal, + alif_epsilonRecA, + alif_a, + alif_avth, + alif_beta, + alif_rho, + GeneralUtils.linear_to_cartesian; threads, blocks) + end +end + +# gpu kernel +function alifForward( zit, + wRec, + vt, + vth, + vRest, + zt, + alpha, + phi, + epsilonRec, + refractoryCounter, + refractoryDuration, + gammaPd, + firingCounter, + recSignal, + epsilonRecA, + a, + avth, + beta, + rho, + linear_to_cartesian) + i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index + + if i <= length(wRec) + # cartesian index + i1, i2, i3, i4 = linear_to_cartesian(i, size(wRec)) + # @cuprintln("gpu thread $i $i1 $i2 $i3 $i4") + + refractoryCounter[i] -= 1 + + if refractoryCounter[i] > 0 # refractory period is active + refractoryCounter[i] -= 1 + zt[i] = 0 + vt[i] = alpha[i] * vt[i] + phi[i] = 0 + a[i] = rho[i] * a[i] + + # compute epsilonRec + epsilonRec[i] = (alpha[i] * epsilonRec[i]) + zit[i] + + # compute epsilonRecA + epsilonRecA[i] = (phi[i] * epsilonRec[i]) + + ((rho[i] - (phi[i] * beta[i])) * epsilonRecA[i]) + + # compute avth + avth[i] = vth[i] + (beta[i] * a[i]) + + else # refractory period is inactive + recSignal[i] = zit[i] * wRec[i] + vt[i] = (alpha[i] * vt[i]) + sum(@view(recSignal[:,:,i3,i4])) + + # compute avth + avth[i] = vth[i] + (beta[i] * a[i]) + + # fires if membrane potential exceed threshold + if vt[i] > avth[i] + zt[i] = 1 + refractoryCounter[i] = refractoryDuration[i] + firingCounter[i] += 1 + vt[i] = vRest[i] + a[i] = (rho[i] * a[i]) + 1 + else + zt[i] = 0 + a[i] = (rho[i] * a[i]) + end + + # compute phi, there is a difference from alif formula + phi[i] = (gammaPd[i] / vth[i]) * max(0, 1 - ((vt[i] - vth[i]) / vth[i])) + + # compute epsilonRec + epsilonRec[i] = (alpha[i] * epsilonRec[i]) + zit[i] + + # compute epsilonRecA + epsilonRecA[i] = (phi[i] * epsilonRec[i]) + + ((rho[i] - (phi[i] * beta[i])) * epsilonRecA[i]) + end + end + return nothing +end + +function onForward(kfn_zit::Array{T}, + zit::Array{T}, + wOut::Array{T}, + vt0::Array{T}, + vt1::Array{T}, + vth::Array{T}, + vRest::Array{T}, + zt1::Array{T}, + alpha::Array{T}, + phi::Array{T}, + epsilonRec::Array{T}, + refractoryCounter::Array{T}, + refractoryDuration::Array{T}, + gammaPd::Array{T}, + firingCounter::Array{T}, + arrayProjection4d::Array{T}, + recSignal::Array{T}, + decayed_vt0::Array{T}, + decayed_epsilonRec::Array{T}, + vt1_diff_vth::Array{T}, + vt1_diff_vth_div_vth::Array{T}, + gammaPd_div_vth::Array{T}, + phiActivation::Array{T}, + ) where T<:Number + + # project 3D kfn zit into 4D lif zit + zit .= reshape(kfn_zit, + (size(wOut, 1), size(wOut, 2), 1, size(wOut, 4))) .* arrayProjection4d + + for j in 1:size(wOut, 4), i in 1:size(wOut, 3) # compute along neurons axis of every batch + if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active + @. @views refractoryCounter[:,:,i,j] -= 1 + @. @views zt1[:,:,i,j] = 0 + @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] + @. @views phi[:,:,i,j] = 0 + + # compute epsilonRec + @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] + @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + else # refractory period is inactive + @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wOut[:,:,i,j] + @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] + @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) + + if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j])) + @. @views zt1[:,:,i,j] = 1 + @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] + @. @views firingCounter[:,:,i,j] += 1 + @. @views vt1[:,:,i,j] = vRest[:,:,i,j] + else + @. @views zt1[:,:,i,j] = 0 + end + + # compute phi, there is a difference from alif formula + @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] + @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] + @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] + @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) + @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] + + # compute epsilonRec + @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] + @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] + end + end +end + +# gpu launcher +function onForward( on_zit::CuArray, + on_wOut::CuArray, + on_vt::CuArray, + on_vth::CuArray, + on_vRest::CuArray, + on_zt::CuArray, + on_alpha::CuArray, + on_phi::CuArray, + on_epsilonRec::CuArray, + on_refractoryCounter::CuArray, + on_refractoryDuration::CuArray, + on_gammaPd::CuArray, + on_firingCounter::CuArray, + on_recSignal::CuArray) + + kernel = @cuda launch=false onForward( on_zit, + on_wOut, + on_vt, + on_vth, + on_vRest, + on_zt, + on_alpha, + on_phi, + on_epsilonRec, + on_refractoryCounter, + on_refractoryDuration, + on_gammaPd, + on_firingCounter, + on_recSignal, + GeneralUtils.linear_to_cartesian) + config = launch_configuration(kernel.fun) + + # threads to be launched. Since one can't launch exact thread number the kernel needs, + # one just launch threads more than this kernel needs then use a guard inside the kernel + # to prevent unused threads to access memory. + threads = min(1024, config.threads) # depend on gpu. Most NVIDIA gpu has 1024 threads per block + + # total desired threads to launch to gpu. Usually 1 thread per 1 matrix element + totalThreads = length(on_wOut) + + blocks = cld(totalThreads, threads) + # println("launching gpu kernel") + CUDA.@sync begin + kernel( on_zit, + on_wOut, + on_vt, + on_vth, + on_vRest, + on_zt, + on_alpha, + on_phi, + on_epsilonRec, + on_refractoryCounter, + on_refractoryDuration, + on_gammaPd, + on_firingCounter, + on_recSignal, + GeneralUtils.linear_to_cartesian; threads, blocks) + end +end + +# gpu kernel +function onForward( zit, + wOut, + vt, + vth, + vRest, + zt, + alpha, + phi, + epsilonRec, + refractoryCounter, + refractoryDuration, + gammaPd, + firingCounter, + recSignal, + linear_to_cartesian) + i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index + + if i <= length(wOut) + # cartesian index + i1, i2, i3, i4 = linear_to_cartesian(i, size(wOut)) + # @cuprintln("gpu thread $i $i1 $i2 $i3 $i4") + + refractoryCounter[i] -= 1 + + if refractoryCounter[i] > 0 # refractory period is active + refractoryCounter[i] -= 1 + zt[i] = 0 + vt[i] = alpha[i] * vt[i] + phi[i] = 0 + + # compute epsilonRec + epsilonRec[i] = (alpha[i] * epsilonRec[i]) + zit[i] + + else # refractory period is inactive + recSignal[i] = zit[i] * wOut[i] + vt[i] = (alpha[i] * vt[i]) + sum(@view(recSignal[:,:,i3,i4])) + + # fires if membrane potential exceed threshold + if vt[i] > vth[i] + zt[i] = 1 + refractoryCounter[i] = refractoryDuration[i] + firingCounter[i] += 1 + vt[i] = vRest[i] + else + zt[i] = 0 + end + + # compute phi, there is a difference from on formula + phi[i] = (gammaPd[i] / vth[i]) * max(0, 1 - ((vt[i] - vth[i]) / vth[i])) + + # compute epsilonRec + epsilonRec[i] = (alpha[i] * epsilonRec[i]) + zit[i] + end + end + return nothing +end + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module \ No newline at end of file diff --git a/previousVersion/0.0.2/src/interface.jl b/previousVersion/0.0.2/src/interface.jl new file mode 100644 index 0000000..705220d --- /dev/null +++ b/previousVersion/0.0.2/src/interface.jl @@ -0,0 +1,87 @@ +module interface + + +# export + +# using Flux, CUDA + +#------------------------------------------------------------------------------------------------100 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module \ No newline at end of file diff --git a/previousVersion/0.0.2/src/learn.jl b/previousVersion/0.0.2/src/learn.jl new file mode 100644 index 0000000..dfbcc85 --- /dev/null +++ b/previousVersion/0.0.2/src/learn.jl @@ -0,0 +1,312 @@ +module learn + +export learn!, compute_paramsChange! + +using Statistics, Random, LinearAlgebra, JSON3, Flux, CUDA, Dates +using GeneralUtils +using ..type, ..snnUtil + +#------------------------------------------------------------------------------------------------100 + +function compute_paramsChange!(kfn::kfn_1, modelError, outputError) + lifComputeParamsChange!(kfn.lif_phi, + kfn.lif_epsilonRec, + kfn.lif_eta, + kfn.lif_eRec, + kfn.lif_wRec, + kfn.lif_wRecChange, + kfn.on_wOut, + kfn.lif_arrayProjection4d, + kfn.lif_error, + modelError) + + alifComputeParamsChange!(kfn.alif_phi, + kfn.alif_epsilonRec, + kfn.alif_eta, + kfn.alif_eRec, + kfn.alif_wRec, + kfn.alif_wRecChange, + kfn.on_wOut, + kfn.alif_arrayProjection4d, + kfn.alif_error, + modelError, + kfn.alif_beta) + + onComputeParamsChange!(kfn.on_phi, + kfn.on_epsilonRec, + kfn.on_eta, + kfn.on_eRec, + kfn.on_wOut, + kfn.on_wOutChange, + outputError) + # error("DEBUG -> kfn compute_paramsChange! $(Dates.now())") +end + +function lifComputeParamsChange!( phi::CuArray, + epsilonRec::CuArray, + eta::CuArray, + eRec::CuArray, + wRec::CuArray, + wRecChange::CuArray, + wOut::CuArray, + arrayProjection4d::CuArray, + nError::CuArray, + modelError::CuArray) + wOutSum = sum(wOut, dims=3) .* arrayProjection4d + + # nError a.k.a. learning signal use dopamine concept, + # this neuron receive summed error signal (modelError) + nError .= (modelError .* arrayProjection4d) .* wOutSum + eRec .= phi .* epsilonRec + # GeneralUtils.isNotEqual(wRec, 0) is a subscribe filter use to filter out non-subscribed wRecChange + wRecChange .+= ((-1 .* eta) .* nError .* eRec) .* GeneralUtils.isNotEqual.(wRec, 0) + # error("DEBUG -> lifComputeParamsChange! $(Dates.now())") +end + +function alifComputeParamsChange!( phi::CuArray, + epsilonRec::CuArray, + eta::CuArray, + eRec::CuArray, + wRec::CuArray, + wRecChange::CuArray, + wOut::CuArray, + arrayProjection4d::CuArray, + nError::CuArray, + modelError::CuArray, + beta::CuArray) + + wOutSum = sum(wOut, dims=3) .* arrayProjection4d + + # nError a.k.a. learning signal use dopamine concept, + # this neuron receive summed error signal (modelError) + nError .= (modelError .* arrayProjection4d) .* wOutSum + eRec .= (phi .* epsilonRec) .+ (phi .* epsilonRec .* beta) + + # GeneralUtils.isNotEqual(wRec, 0) is a subscribe filter use to filter out non-subscribed wRecChange + wRecChange .+= ((-1 .* eta) .* nError .* eRec) .* GeneralUtils.isNotEqual.(wRec, 0) + # error("DEBUG -> alifComputeParamsChange! $(Dates.now())") +end + +function onComputeParamsChange!(phi::CuArray, + epsilonRec::CuArray, + eta::CuArray, + eRec::CuArray, + wOut::CuArray, + wOutChange::CuArray, + outputError::CuArray # outputError is output neuron's error + ) + + # nError a.k.a. learning signal use dopamine concept, + # this neuron receive summed error signal (modelError) + eRec .= (phi .* epsilonRec) .* reshape(outputError, (1, 1, :, size(epsilonRec, 4))) + + # GeneralUtils.isNotEqual(wRec, 0) is a subscribe filter use to filter out non-subscribed wRecChange + wOutChange .+= ((-1 .* eta) .* eRec) .* GeneralUtils.isNotEqual.(wOut, 0) + + # error("DEBUG -> onComputeParamsChange! $(Dates.now())") +end + +function lifComputeParamsChange!( phi::AbstractArray, + epsilonRec::AbstractArray, + eta::AbstractArray, + wRec::AbstractArray, + wRecChange::AbstractArray, + wOut::AbstractArray, + modelError::AbstractArray) + d1, d2, d3, d4 = size(epsilonRec) + error("DEBUG -> lifComputeParamsChange! $(Dates.now())") + # Bₖⱼ in paper, sum() to get each neuron's total wOut weight + wOutSum = reshape(sum(wOut, dims=3), (d1, :, d4)) + + for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch + # how much error of this neuron 1-spike causing each output neuron's error + + view(wRecChange, :, :, i, j) .+= (-1 * view(eta, :, :, i, j)[1]) .* + # eRec + ( + (view(phi, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) .* + # nError a.k.a. learning signal + ( + view(modelError, :, j)[1] * # dopamine concept, this neuron receive summed error signal + # RSNN neuron's total wOut weight (neuron synaptic subscription .* wOutSum) + view(wOutSum, :, :, j)[i] + ) + ) + end +end + +function alifComputeParamsChange!( phi::AbstractArray, + epsilonRec::AbstractArray, + epsilonRecA::AbstractArray, + eta::AbstractArray, + wRec::AbstractArray, + wRecChange::AbstractArray, + beta::AbstractArray, + wOut::AbstractArray, + modelError::AbstractArray) + d1, d2, d3, d4 = size(epsilonRec) + + # Bₖⱼ in paper, sum() to get each neuron's total wOut weight + wOutSum = reshape(sum(wOut, dims=3), (d1, :, d4)) + + for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch + # how much error of this neuron 1-spike causing each output neuron's error + + view(wRecChange, :, :, i, j) .+= (-1 * view(eta, :, :, i, j)[1]) .* + # eRec + ( + # eRec_v + (view(phi, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) .+ + # eRec_a + ((view(phi, :, :, i, j)[1] * view(beta, :, :, i, j)[1]) .* + view(epsilonRecA, :, :, i, j)) + ) .* + # nError a.k.a. learning signal + ( + view(modelError, :, j)[1] * + # RSNN neuron's total wOut weight (neuron synaptic subscription .* wOutSum) + view(wOutSum, :, :, j)[i] + # sum(GeneralUtils.isNotEqual.(view(wRec, :, :, i, j), 0) .* + # view(wOutSum, :, :, j)) + ) + end +end + +function onComputeParamsChange!(phi::AbstractArray, + epsilonRec::AbstractArray, + eta::AbstractArray, + wOutChange::AbstractArray, + outputError::AbstractArray) + d1, d2, d3, d4 = size(epsilonRec) + + for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch + # how much error of this neuron 1-spike causing each output neuron's error + + view(wOutChange, :, :, i, j) .+= (-1 * view(eta, :, :, i, j)[1]) .* + # eRec + ( + (view(phi, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) .* + # nError a.k.a. learning signal, output neuron receives error of its own answer - correct answer. + view(outputError, :, j)[i] + ) + end +end + +function learn!(kfn::kfn_1) + # lif learn + lifLearn!(kfn.lif_wRec, + kfn.lif_wRecChange, + kfn.lif_arrayProjection4d) + + # alif learn + alifLearn!(kfn.alif_wRec, + kfn.alif_wRecChange, + kfn.alif_arrayProjection4d) + + # on learn + onLearn!(kfn.on_wOut, + kfn.on_wOutChange, + kfn.on_arrayProjection4d) + + # wOut decay + kfn.on_wOut .*= 0.0001 + + # wrap up learning session + if kfn.learningStage == [3] + kfn.learningStage = [0] + end + # error("DEBUG -> kfn learn! $(Dates.now())") +end + +function lifLearn!(wRec, + wRecChange, + arrayProjection4d) + + # merge learning weight with average learning weight + wRec .+= (sum(wRecChange) ./ (size(wRec, 4))) .* arrayProjection4d + + #TODO synaptic strength + + #TODO neuroplasticity + +end + +function alifLearn!(wRec, + wRecChange, + arrayProjection4d) + # merge learning weight + wRec .+= (sum(wRecChange) ./ (size(wRec, 4))) .* arrayProjection4d + + #TODO synaptic strength + + #TODO neuroplasticity + +end + +function onLearn!(wOut, + wOutChange, + arrayProjection4d) + # merge learning weight + wOut .+= (sum(wOutChange) ./ (size(wOut, 4))) .* arrayProjection4d + + #TODO synaptic strength + + #TODO neuroplasticity + +end + +#TODO voltage regulator + +#TODO frequency regulator + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module \ No newline at end of file diff --git a/previousVersion/0.0.2/src/snnUtil.jl b/previousVersion/0.0.2/src/snnUtil.jl new file mode 100644 index 0000000..168ba9e --- /dev/null +++ b/previousVersion/0.0.2/src/snnUtil.jl @@ -0,0 +1,77 @@ +module snnUtil + +export refractoryStatus! + +# using + +#------------------------------------------------------------------------------------------------100 + +function refractoryStatus!(refractoryCounter, refractoryActive, refractoryInactive) + d1, d2, d3, d4 = size(refractoryCounter) + for j in 1:d4 + for i in 1:d3 + if refractoryCounter[1, 1, i, j] > 0 # inactive + view(refractoryActive, 1, 1, i, j) .= 0 + view(refractoryInactive, 1, 1, i, j) .= 1 + else # active + view(refractoryActive, 1, 1, i, j) .= 1 + view(refractoryInactive, 1, 1, i, j) .= 0 + end + end + end +end + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module \ No newline at end of file diff --git a/previousVersion/0.0.2/src/type.jl b/previousVersion/0.0.2/src/type.jl new file mode 100644 index 0000000..c56e313 --- /dev/null +++ b/previousVersion/0.0.2/src/type.jl @@ -0,0 +1,427 @@ +module type + +export + # struct + kfn_1 + + # function + +using Random, GeneralUtils + +#------------------------------------------------------------------------------------------------100 +rng = MersenneTwister(1234) + +abstract type Ironpen end +abstract type knowledgeFn <: Ironpen end + +#------------------------------------------------------------------------------------------------100 + +Base.@kwdef mutable struct kfn_1 <: knowledgeFn + params::Union{Dict, Nothing} = nothing # store params of knowledgeFn itself for later use + + timeStep::Union{AbstractArray, Nothing} = nothing + learningStage::Union{AbstractArray, Nothing} = nothing # 0 inference, 1 start, 2 during, 3 end learning + zit::Union{AbstractArray, Nothing} = nothing # 3D activation matrix + modelError::Union{AbstractArray, Nothing} = nothing # store RSNN error + outputError::Union{AbstractArray, Nothing} = nothing # store output neurons error + + # ---------------------------------------------------------------------------- # + # LIF Neurons # + # ---------------------------------------------------------------------------- # + # a projection of kfn.zit into lif dimension for broadcasting later) + lif_zit::Union{AbstractArray, Nothing} = nothing + + # main variables according to papers + lif_wRec::Union{AbstractArray, Nothing} = nothing + lif_vt::Union{AbstractArray, Nothing} = nothing + lif_vth::Union{AbstractArray, Nothing} = nothing + lif_vRest::Union{AbstractArray, Nothing} = nothing + lif_zt::Union{AbstractArray, Nothing} = nothing + lif_zt4d::Union{AbstractArray, Nothing} = nothing + lif_refractoryCounter::Union{AbstractArray, Nothing} = nothing + lif_refractoryDuration::Union{AbstractArray, Nothing} = nothing + lif_alpha::Union{AbstractArray, Nothing} = nothing + lif_delta::Union{AbstractFloat, Nothing} = nothing + lif_tau_m::Union{AbstractFloat, Nothing} = nothing + lif_phi::Union{AbstractArray, Nothing} = nothing + lif_epsilonRec::Union{AbstractArray, Nothing} = nothing + lif_eRec::Union{AbstractArray, Nothing} = nothing + lif_eta::Union{AbstractArray, Nothing} = nothing + lif_gammaPd::Union{AbstractArray, Nothing} = nothing + lif_wRecChange::Union{AbstractArray, Nothing} = nothing + lif_error::Union{AbstractArray, Nothing} = nothing + + lif_firingCounter::Union{AbstractArray, Nothing} = nothing + + # pre-allocation array + lif_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d + lif_recSignal::Union{AbstractArray, Nothing} = nothing + # lif_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing + # lif_vt_diff_vth::Union{AbstractArray, Nothing} = nothing + # lif_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing + # lif_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing + # lif_phiActivation::Union{AbstractArray, Nothing} = nothing + + # ---------------------------------------------------------------------------- # + # ALIF Neurons # + # ---------------------------------------------------------------------------- # + alif_zit::Union{AbstractArray, Nothing} = nothing + + alif_wRec::Union{AbstractArray, Nothing} = nothing + alif_vt::Union{AbstractArray, Nothing} = nothing + alif_vth::Union{AbstractArray, Nothing} = nothing + alif_vRest::Union{AbstractArray, Nothing} = nothing + alif_zt::Union{AbstractArray, Nothing} = nothing + alif_zt4d::Union{AbstractArray, Nothing} = nothing + alif_refractoryCounter::Union{AbstractArray, Nothing} = nothing + alif_refractoryDuration::Union{AbstractArray, Nothing} = nothing + alif_alpha::Union{AbstractArray, Nothing} = nothing + alif_delta::Union{AbstractFloat, Nothing} = nothing + alif_tau_m::Union{AbstractFloat, Nothing} = nothing + alif_phi::Union{AbstractArray, Nothing} = nothing + alif_epsilonRec::Union{AbstractArray, Nothing} = nothing + alif_eRec::Union{AbstractArray, Nothing} = nothing + alif_eta::Union{AbstractArray, Nothing} = nothing + alif_gammaPd::Union{AbstractArray, Nothing} = nothing + alif_wRecChange::Union{AbstractArray, Nothing} = nothing + alif_error::Union{AbstractArray, Nothing} = nothing + + alif_firingCounter::Union{AbstractArray, Nothing} = nothing + + # pre-allocation array + alif_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d + alif_recSignal::Union{AbstractArray, Nothing} = nothing + # alif_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing + # alif_vt_diff_vth::Union{AbstractArray, Nothing} = nothing + # alif_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing + # alif_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing + # alif_phiActivation::Union{AbstractArray, Nothing} = nothing + + # alif specific variables + alif_epsilonRecA::Union{AbstractArray, Nothing} = nothing + alif_avth::Union{AbstractArray, Nothing} = nothing + alif_a::Union{AbstractArray, Nothing} = nothing # threshold adaptation + alif_beta::Union{AbstractArray, Nothing} = nothing # β, constant, value from paper + alif_rho::Union{AbstractArray, Nothing} = nothing # ρ, threshold adaptation decay factor + alif_tau_a::Union{AbstractFloat, Nothing} = nothing # τ_a, adaption time constant in millisecond + + # alif specific pre-allocation array + # alif_phi_x_epsilonRec::Union{AbstractArray, Nothing} = nothing + # alif_phi_x_beta::Union{AbstractArray, Nothing} = nothing + # alif_rho_diff_phi_x_beta::Union{AbstractArray, Nothing} = nothing + # alif_rho_div_phi_x_beta_x_epsilonRecA::Union{AbstractArray, Nothing} = nothing + # alif_beta_x_a::Union{AbstractArray, Nothing} = nothing + + # ---------------------------------------------------------------------------- # + # Output Neurons # + # ---------------------------------------------------------------------------- # + # output neuron is based on LIF + on_zit::Union{AbstractArray, Nothing} = nothing + + # main variables according to papers + on_wOut::Union{AbstractArray, Nothing} = nothing # wOut is wRec, just use the name from paper + on_vt::Union{AbstractArray, Nothing} = nothing + on_vth::Union{AbstractArray, Nothing} = nothing + on_vRest::Union{AbstractArray, Nothing} = nothing + on_zt::Union{AbstractArray, Nothing} = nothing + on_zt4d::Union{AbstractArray, Nothing} = nothing + on_refractoryCounter::Union{AbstractArray, Nothing} = nothing + on_refractoryDuration::Union{AbstractArray, Nothing} = nothing + on_alpha::Union{AbstractArray, Nothing} = nothing + on_delta::Union{AbstractFloat, Nothing} = nothing + on_tau_m::Union{AbstractFloat, Nothing} = nothing + on_phi::Union{AbstractArray, Nothing} = nothing + on_epsilonRec::Union{AbstractArray, Nothing} = nothing + on_eRec::Union{AbstractArray, Nothing} = nothing + on_eta::Union{AbstractArray, Nothing} = nothing + on_gammaPd::Union{AbstractArray, Nothing} = nothing + on_wOutChange::Union{AbstractArray, Nothing} = nothing + on_error::Union{AbstractArray, Nothing} = nothing + + on_firingCounter::Union{AbstractArray, Nothing} = nothing + + # pre-allocation array + on_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d + on_recSignal::Union{AbstractArray, Nothing} = nothing + # on_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing + # on_vt_diff_vth::Union{AbstractArray, Nothing} = nothing + # on_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing + # on_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing + # on_phiActivation::Union{AbstractArray, Nothing} = nothing +end + +# outer constructor +function kfn_1(params::Dict; device=cpu) + kfn = kfn_1() + kfn.params = params + kfn.timeStep = [0] |> device + kfn.learningStage = [0] |> device + + # ---------------------------------------------------------------------------- # + # initialize activation matrix # + # ---------------------------------------------------------------------------- # + # row*col is a 2D matrix represent all RSNN activation + row, col, batch = kfn.params[:inputPort][:signal][:numbers] # z-axis represent signal batch number + # row += kfn.params[:inputPort][:noise][:numbers][1] + col += kfn.params[:inputPort][:noise][:numbers][2] + col += kfn.params[:computeNeuron][:lif][:numbers][2] + col += kfn.params[:computeNeuron][:alif][:numbers][2] + + # activation matrix + kfn.zit = zeros(row, col, batch) |> device + kfn.modelError = zeros(1) |> device + + # ---------------------------------------------------------------------------- # + # LIF config # + # ---------------------------------------------------------------------------- # + # In 3D LIF matrix, z-axis represent each neuron while each 2D slice represent that neuron's + # synaptic subscription to other neurons (via activation matrix) + n = kfn.params[:computeNeuron][:lif][:numbers][1] * kfn.params[:computeNeuron][:lif][:numbers][2] + + # subscription + w = zeros(row, col, n) + synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent] + synapticConnection = Int(floor(row*col * synapticConnectionPercent/100)) + for slice in eachslice(w, dims=3) + pool = shuffle!([1:row*col...])[1:synapticConnection] + for i in pool + slice[i] = randn()/10 # assign weight to synaptic connection + end + end + # project 3D w into 4D kfn.lif_wRec (row, col, n, batch) + kfn.lif_wRec = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device + kfn.lif_zit = (similar(kfn.lif_wRec) .= 0) |> device + kfn.lif_vt = (similar(kfn.lif_wRec) .= 0) |> device + kfn.lif_vth = (similar(kfn.lif_wRec) .= 1) |> device + kfn.lif_vRest = (similar(kfn.lif_wRec) .= 0) |> device + kfn.lif_zt = zeros(1, 1, n, batch) |> device + kfn.lif_zt4d = (similar(kfn.lif_wRec) .= 0) |> device + kfn.lif_refractoryCounter = (similar(kfn.lif_wRec) .= 0) |> device + kfn.lif_refractoryDuration = (similar(kfn.lif_wRec) .= 3) |> device + kfn.lif_delta = 1.0 + kfn.lif_tau_m = 20.0 + kfn.lif_alpha = (similar(kfn.lif_wRec) .= (exp(-kfn.lif_delta / kfn.lif_tau_m))) |> device + kfn.lif_phi = (similar(kfn.lif_wRec) .= 0) |> device + kfn.lif_epsilonRec = (similar(kfn.lif_wRec) .= 0) |> device + kfn.lif_eRec = (similar(kfn.lif_wRec) .= 0) |> device + kfn.lif_eta = (similar(kfn.lif_wRec) .= 0.001) |> device + kfn.lif_gammaPd = (similar(kfn.lif_wRec) .= 0.3) |> device + kfn.lif_wRecChange = (similar(kfn.lif_wRec) .= 0) |> device + kfn.lif_error = (similar(kfn.lif_wRec) .= 0) |> device + + kfn.lif_firingCounter = (similar(kfn.lif_wRec) .= 0) |> device + + kfn.lif_arrayProjection4d = (similar(kfn.lif_wRec) .= 1) |> device + kfn.lif_recSignal = (similar(kfn.lif_wRec) .= 0) |> device + # kfn.lif_decayed_epsilonRec = (similar(kfn.lif_wRec) .= 0) |> device + # kfn.lif_vt_diff_vth = (similar(kfn.lif_wRec) .= 0) |> device + # kfn.lif_vt_diff_vth_div_vth = (similar(kfn.lif_wRec) .= 0) |> device + # kfn.lif_gammaPd_div_vth = (similar(kfn.lif_wRec) .= 0) |> device + # kfn.lif_phiActivation = (similar(kfn.lif_wRec) .= 0) |> device + + # ---------------------------------------------------------------------------- # + # ALIF config # + # ---------------------------------------------------------------------------- # + n = kfn.params[:computeNeuron][:alif][:numbers][1] * kfn.params[:computeNeuron][:alif][:numbers][2] + + # subscription + w = zeros(row, col, n) + synapticConnectionPercent = kfn.params[:computeNeuron][:alif][:params][:synapticConnectionPercent] + synapticConnection = Int(floor(row*col * synapticConnectionPercent/100)) + for slice in eachslice(w, dims=3) + pool = shuffle!([1:row*col...])[1:synapticConnection] + for i in pool + slice[i] = randn()/10 # assign weight to synaptic connection + end + end + # project 3D w into 4D kfn.alif_wRec + kfn.alif_wRec = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device + kfn.alif_zit = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_vt = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_vth = (similar(kfn.alif_wRec) .= 1) |> device + kfn.alif_vRest = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_zt = zeros(1, 1, n, batch) |> device + kfn.alif_zt4d = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_refractoryCounter = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_refractoryDuration = (similar(kfn.alif_wRec) .= 3) |> device + kfn.alif_delta = 1.0 + kfn.alif_tau_m = 20.0 + kfn.alif_alpha = (similar(kfn.alif_wRec) .= (exp(-kfn.alif_delta / kfn.alif_tau_m))) |> device + kfn.alif_phi = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_epsilonRec = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_eRec = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_eta = (similar(kfn.alif_wRec) .= 0.001) |> device + kfn.alif_gammaPd = (similar(kfn.alif_wRec) .= 0.3) |> device + kfn.alif_wRecChange = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_error = (similar(kfn.alif_wRec) .= 0) |> device + + kfn.alif_firingCounter = (similar(kfn.alif_wRec) .= 0) |> device + + kfn.alif_arrayProjection4d = (similar(kfn.alif_wRec) .= 1) |> device + kfn.alif_recSignal = (similar(kfn.alif_wRec) .= 0) |> device + # kfn.alif_decayed_epsilonRec = (similar(kfn.alif_wRec) .= 0) |> device + # kfn.alif_vt_diff_vth = (similar(kfn.alif_wRec) .= 0) |> device + # kfn.alif_vt_diff_vth_div_vth = (similar(kfn.alif_wRec) .= 0) |> device + # kfn.alif_gammaPd_div_vth = (similar(kfn.alif_wRec) .= 0) |> device + # kfn.alif_phiActivation = (similar(kfn.alif_wRec) .= 0) |> device + + # alif specific variables + kfn.alif_epsilonRecA = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_avth = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_a = (similar(kfn.alif_wRec) .= 0) |> device + kfn.alif_beta = (similar(kfn.alif_wRec) .= 0.07) |> device + kfn.alif_tau_a = 100.0 + kfn.alif_rho = (similar(kfn.alif_wRec) .= (exp(-kfn.alif_delta / kfn.alif_tau_a))) |> device + # kfn.alif_phi_x_epsilonRec = (similar(kfn.alif_wRec) .= 0) |> device + # kfn.alif_phi_x_beta = (similar(kfn.alif_wRec) .= 0) |> device + # kfn.alif_rho_diff_phi_x_beta = (similar(kfn.alif_wRec) .= 0) |> device + # kfn.alif_rho_div_phi_x_beta_x_epsilonRecA = (similar(kfn.alif_wRec) .= 0) |> device + # kfn.alif_beta_x_a = (similar(kfn.alif_wRec) .= 0) |> device + + # ---------------------------------------------------------------------------- # + # output config # + # ---------------------------------------------------------------------------- # + n = kfn.params[:outputPort][:numbers][1] * kfn.params[:outputPort][:numbers][2] + + # subscription + w = zeros(row, col, n) + synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent] + synapticConnection = Int(floor(row*col * synapticConnectionPercent/100)) + for slice in eachslice(w, dims=3) + pool = shuffle!([1:row*col...])[1:synapticConnection] + for i in pool + slice[i] = randn()/10 # assign weight to synaptic connection + end + end + # project 3D w into 4D kfn.lif_wOut (row, col, n, batch) + kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device + kfn.on_zit = (similar(kfn.on_wOut) .= 0) |> device + kfn.on_vt = (similar(kfn.on_wOut) .= 0) |> device + kfn.on_vth = (similar(kfn.on_wOut) .= 1) |> device + kfn.on_vRest = (similar(kfn.on_wOut) .= 0) |> device + kfn.on_zt = zeros(1, 1, n, batch) |> device + kfn.on_zt4d = (similar(kfn.on_wOut) .= 0) |> device + kfn.on_refractoryCounter = (similar(kfn.on_wOut) .= 0) |> device + kfn.on_refractoryDuration = (similar(kfn.on_wOut) .= 0) |> device + kfn.on_delta = 1.0 + kfn.on_tau_m = 20.0 + kfn.on_alpha = (similar(kfn.on_wOut) .= (exp(-kfn.on_delta / kfn.on_tau_m))) |> device + kfn.on_phi = (similar(kfn.on_wOut) .= 0) |> device + kfn.on_epsilonRec = (similar(kfn.on_wOut) .= 0) |> device + kfn.on_eRec = (similar(kfn.on_wOut) .= 0) |> device + kfn.on_eta = (similar(kfn.on_wOut) .= 0.001) |> device + kfn.on_gammaPd = (similar(kfn.on_wOut) .= 0.3) |> device + kfn.on_wOutChange = (similar(kfn.on_wOut) .= 0) |> device + kfn.on_error = (similar(kfn.on_wOut) .= 0) |> device + + kfn.on_firingCounter = (similar(kfn.on_wOut) .= 0) |> device + + kfn.on_arrayProjection4d = (similar(kfn.on_wOut) .= 1) |> device + kfn.on_recSignal = (similar(kfn.on_wOut) .= 0) |> device + + + + kfn.outputError = zeros(n, batch) |> device + + + + + # kfn.on_decayed_epsilonRec = (similar(kfn.on_wOut) .= 0 |> device + # kfn.on_vt_diff_vth = (similar(kfn.on_wOut) .= 0 |> device + # kfn.on_vt_diff_vth_div_vth = (similar(kfn.on_wOut) .= 0 |> device + # kfn.on_gammaPd_div_vth = (similar(kfn.on_wOut) .= 0 |> device + # kfn.on_phiActivation = (similar(kfn.on_wOut) .= 0 |> device + + # kfn.on_zit = zeros(row, col, n, batch) |> device + # kfn.on_vt = zeros(1, 1, n, batch) |> device + # kfn.on_vth = ones(1, 1, n, batch) |> device + # kfn.on_vRest = zeros(1, 1, n, batch) |> device + # # kfn.on_zt = zeros(1, 1, n, batch) |> device + # kfn.on_zt4d = zeros(1, 1, n, batch) |> device + # kfn.on_refractoryCounter = zeros(1, 1, n, batch) |> device + # kfn.on_refractoryDuration = ones(1, 1, n, batch) .* 0 |> device + # kfn.on_delta = 1.0 + # kfn.on_tau_m = 20.0 + # kfn.on_alpha = ones(1, 1, n, batch) .* (exp(-kfn.on_delta / kfn.on_tau_m)) |> device + # kfn.on_phi = zeros(1, 1, n, batch) |> device + # kfn.on_epsilonRec = zeros(row, col, n, batch) |> device + # # kfn.on_eRec = zeros(row, col, n, batch) + # kfn.on_eta = zeros(1, 1, n, batch) |> device + # kfn.on_gammaPd = zeros(1, 1, n, batch) .* 0.3 |> device + # kfn.on_wOutChange = zeros(row, col, n, batch) |> device + # # kfn.on_b = randn(1, 1, n, batch) |> device + # # kfn.on_bChange = randn(1, 1, n, batch) |> device + + # kfn.on_firingCounter = zeros(1, 1, n, batch) |> device + # kfn.on_arraySize = [row, col, n, batch] |> device + # kfn.on_arrayProjection4d = ones(row, col, n, batch) |> device + + # # subscription + # w = zeros(row, col, n) + # synapticConnectionPercent = kfn.params[:outputPort][:params][:synapticConnectionPercent] + # synapticConnection = Int(floor(row*col * synapticConnectionPercent/100)) + # for slice in eachslice(w, dims=3) + # pool = shuffle!([1:row*col...])[1:synapticConnection] + # for i in pool + # slice[i] = randn()/10 # assign weight to synaptic connection + # end + # end + # # project 3D w into 4D kfn.on_wOut + # kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device + + + + return kfn +end + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module \ No newline at end of file diff --git a/src/IronpenGPU.jl b/src/IronpenGPU.jl index 1649b84..6dfd527 100644 --- a/src/IronpenGPU.jl +++ b/src/IronpenGPU.jl @@ -25,9 +25,8 @@ using .interface #------------------------------------------------------------------------------------------------100 -""" version 0.0.1 +""" version 0.0.2 Todo: - [*1] knowledgeFn in GPU format [] use partial error update for computeNeuron [] use integrate_neuron_params synapticConnectionPercent LESS THAN 100% [2] implement dormant connection and pruning machanism. the longer the training the longer @@ -37,8 +36,8 @@ using .interface [] Liquid time constant. training should include adjusting α, neuron membrane potential decay factor which defined by neuron.tau_m formula in type.jl - Change from version: - - + Change from version: 0.0.1 + - knowledgeFn in GPU format All features diff --git a/src/forward.jl b/src/forward.jl index 7d1c451..4790036 100644 --- a/src/forward.jl +++ b/src/forward.jl @@ -18,7 +18,19 @@ function (kfn::kfn_1)(input::AbstractArray) #TODO time step forward if view(kfn.learningStage, 1)[1] == 1 # reset learning params - # kfn.learningStage = [2] + kfn.lif_vt .= 0 + kfn.lif_wRecChange .= 0 + kfn.lif_epsilonRec .= 0 + + kfn.alif_vt .= 0 + kfn.alif_epsilonRec .= 0 + kfn.alif_wRecChange .= 0 + + kfn.on_vt .= 0 + kfn.on_epsilonRec .= 0 + kfn.on_wOutChange .= 0 + + kfn.learningStage = [2] end # update activation matrix with "lif_zt1" and "alif_zt1" by concatenating @@ -27,12 +39,6 @@ function (kfn::kfn_1)(input::AbstractArray) reshape(kfn.lif_zt, (size(input, 1), :, 1, size(input, 3))), reshape(kfn.alif_zt, (size(input, 1), :, 1, size(input, 3))), dims=2) kfn.zit .= reshape(_zit, (size(input, 1), :, size(input, 3))) - - # pass input_data into input neuron. -# GeneralUtils.cartesianAssign!(kfn.zit, input) - -# kfn.zit = kfn.zit |> device -# input = input |> device # project 3D kfn zit into 4D lif zit i1, i2, i3, i4 = size(kfn.lif_zit) diff --git a/src/learn.jl b/src/learn.jl index 4b68796..dfbcc85 100644 --- a/src/learn.jl +++ b/src/learn.jl @@ -42,8 +42,6 @@ function compute_paramsChange!(kfn::kfn_1, modelError, outputError) # error("DEBUG -> kfn compute_paramsChange! $(Dates.now())") end - - function lifComputeParamsChange!( phi::CuArray, epsilonRec::CuArray, eta::CuArray, @@ -197,15 +195,18 @@ end function learn!(kfn::kfn_1) # lif learn lifLearn!(kfn.lif_wRec, - kfn.lif_wRecChange) + kfn.lif_wRecChange, + kfn.lif_arrayProjection4d) # alif learn alifLearn!(kfn.alif_wRec, - kfn.alif_wRecChange) + kfn.alif_wRecChange, + kfn.alif_arrayProjection4d) # on learn onLearn!(kfn.on_wOut, - kfn.on_wOutChange) + kfn.on_wOutChange, + kfn.on_arrayProjection4d) # wOut decay kfn.on_wOut .*= 0.0001 @@ -218,9 +219,11 @@ function learn!(kfn::kfn_1) end function lifLearn!(wRec, - wRecChange) - # merge learning weight - wRec .+= wRecChange + wRecChange, + arrayProjection4d) + + # merge learning weight with average learning weight + wRec .+= (sum(wRecChange) ./ (size(wRec, 4))) .* arrayProjection4d #TODO synaptic strength @@ -229,9 +232,10 @@ function lifLearn!(wRec, end function alifLearn!(wRec, - wRecChange) + wRecChange, + arrayProjection4d) # merge learning weight - wRec .+= wRecChange + wRec .+= (sum(wRecChange) ./ (size(wRec, 4))) .* arrayProjection4d #TODO synaptic strength @@ -240,9 +244,10 @@ function alifLearn!(wRec, end function onLearn!(wOut, - wOutChange) + wOutChange, + arrayProjection4d) # merge learning weight - wOut .+= wOutChange + wOut .+= (sum(wOutChange) ./ (size(wOut, 4))) .* arrayProjection4d #TODO synaptic strength