diff --git a/Manifest.toml b/Manifest.toml index 03e715a..208cacb 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.9.2" manifest_format = "2.0" -project_hash = "bf29b9b6c22b1a96a3e10687abec46c3de1b7715" +project_hash = "1a1cddac46fdd2108611b4e2f350497572f0c8d4" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -111,6 +111,12 @@ git-tree-sha1 = "2918fbffb50e3b7a0b9127617587afa76d4276e8" uuid = "62b44479-cb7b-5706-934f-f13b2eb2e645" version = "8.8.1+0" +[[deps.Calculus]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad" +uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9" +version = "0.5.1" + [[deps.ChainRules]] deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "Statistics", "StructArrays"] git-tree-sha1 = "1cdf290d4feec68824bfb84f4bfc9f3aba185647" @@ -222,6 +228,20 @@ version = "1.15.1" deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +[[deps.Distributions]] +deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns", "Test"] +git-tree-sha1 = "e76a3281de2719d7c81ed62c6ea7057380c87b1d" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.98" + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + + [deps.Distributions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d" + [[deps.DocStringExtensions]] deps = ["LibGit2"] git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" @@ -233,6 +253,12 @@ deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" version = "1.6.0" +[[deps.DualNumbers]] +deps = ["Calculus", "NaNMath", "SpecialFunctions"] +git-tree-sha1 = "5837a837389fccf076445fce071c8ddaea35a566" +uuid = "fa6b7ba4-c1ee-5f82-b5fc-ecf0adba8f74" +version = "0.6.8" + [[deps.ExprTools]] git-tree-sha1 = "c1d06d129da9f55715c6c212866f5b1bddc5fa00" uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" @@ -311,6 +337,18 @@ git-tree-sha1 = "cb090aea21c6ca78d59672a7e7d13bd56d09de64" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" version = "0.20.3" +[[deps.GeneralUtils]] +deps = ["CUDA", "DataStructures", "Distributions", "Flux", "JSON3"] +path = "C:\\Users\\naraw\\.julia\\dev\\GeneralUtils" +uuid = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" +version = "0.1.0" + +[[deps.HypergeometricFunctions]] +deps = ["DualNumbers", "LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "ce7ea9cc5db29563b1fe20196b6d23ab3b111384" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.18" + [[deps.IRTools]] deps = ["InteractiveUtils", "MacroTools", "Test"] git-tree-sha1 = "eac00994ce3229a464c2847e956d77a2c64ad3a5" @@ -342,6 +380,12 @@ git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" version = "1.4.1" +[[deps.JSON3]] +deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] +git-tree-sha1 = "5b62d93f2582b09e469b3099d839c2d2ebf5066d" +uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +version = "1.13.1" + [[deps.JuliaVariables]] deps = ["MLStyle", "NameResolution"] git-tree-sha1 = "49fb3cb53362ddadb4415e9b73926d6b40709e70" @@ -527,6 +571,18 @@ git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" version = "1.6.0" +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "67eae2738d63117a196f497d7db789821bce61d1" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.17" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "4b2e829ee66d4218e0cef22c0a64ee37cf258c29" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.7.1" + [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -559,6 +615,12 @@ git-tree-sha1 = "80d919dee55b9c50e8d9e2da5eeafff3fe58b539" uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" version = "0.1.4" +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "6ec7ac8412e83d57e313393220879ede1740f9ee" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.8.2" + [[deps.REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" @@ -596,6 +658,18 @@ git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" uuid = "ae029012-a4dd-5104-9daa-d747884805df" version = "1.3.0" +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.7.1" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6ed52fdd3382cf21947b15e8870ac0ddbff736da" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.4.0+0" + [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" @@ -683,12 +757,36 @@ git-tree-sha1 = "75ebe04c5bed70b91614d684259b661c9e6274a4" uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" version = "0.34.0" +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "f625d686d5a88bcd2b15cd81f18f98186fdc0c9a" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.3.0" + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + + [deps.StatsFuns.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + [[deps.StructArrays]] deps = ["Adapt", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] git-tree-sha1 = "521a0e828e98bb69042fec1809c1b5a680eb7389" uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" version = "0.6.15" +[[deps.StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.10.0" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + [[deps.SuiteSparse_jll]] deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" diff --git a/Project.toml b/Project.toml index 0978590..470b411 100644 --- a/Project.toml +++ b/Project.toml @@ -6,4 +6,5 @@ version = "0.1.0" [deps] CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" diff --git a/src/IronpenGPU.jl b/src/IronpenGPU.jl index ab8242d..360f825 100644 --- a/src/IronpenGPU.jl +++ b/src/IronpenGPU.jl @@ -23,7 +23,7 @@ using .interface Todo: [*1] knowledgeFn in GPU format [] use partial error update for computeNeuron - [] use integrate_neuron_params synapticConnectionPercent = 20% + [] use integrate_neuron_params synapticConnectionPercent LESS THAN 100% [2] implement dormant connection and pruning machanism. the longer the training the longer 0 weight stay 0. [] using RL to control learning signal diff --git a/src/type.jl b/src/type.jl index eeb38ca..361be3d 100644 --- a/src/type.jl +++ b/src/type.jl @@ -2,44 +2,122 @@ module type export # struct - kfn + kfn_1 # function -using Random +using Random, GeneralUtils #------------------------------------------------------------------------------------------------100 +rng = MersenneTwister(1234) abstract type Ironpen end abstract type knowledgeFn <: Ironpen end -rng = MersenneTwister(1234) #------------------------------------------------------------------------------------------------100 - - -Base.@kwdef mutable struct kfn <: knowledgeFn +Base.@kwdef mutable struct kfn_1 <: knowledgeFn params::Dict = Dict() # store params of knowledgeFn itself for later use + timeStep::AbstractArray = [0] refractory::Union{AbstractArray, Nothing} = nothing + learningStage::AbstractArray = [0] # 0 inference, 1 start, 2 during, 3 end learning z_i_t1::Union{AbstractArray, Nothing} = nothing # 2D activation matrix - z_i_t::Union{AbstractArray, Nothing} = nothing - z_t::Union{AbstractArray, Nothing} = nothing - z_t1::Union{AbstractArray, Nothing} = nothing + z_i_t0::Union{AbstractArray, Nothing} = nothing + + lif_w::Union{AbstractArray, Nothing} = nothing + + alif_w::Union{AbstractArray, Nothing} = nothing end -function kfn(kfnParams::Dict) - kfn_1 = kfn() - kfn_1.params = kfnParams +# outer constructor +function kfn_1(params::Dict) + kfn = kfn_1() + kfn.params = params + # initialize activation matrix + row, col = kfn.params[:inputPort][:noise][:numbers] + row += kfn.params[:inputPort][:signal][:numbers][1] + col += kfn.params[:inputPort][:signal][:numbers][2] + col += kfn.params[:computeNeuron][:lif][:numbers][2] + col += kfn.params[:computeNeuron][:alif][:numbers][2] - if kfn_1.params[:computeNeuronNumber] < kfn_1.params[:totalInputPort] - throw(error("number of compute neuron must be greater than input neuron")) + kfn.z_i_t1 = zeros(row, col, 1) + kfn.z_i_t0 = zeros(row, col, 1) + + kfn.lif_w = zeros(row, col, row*col) + kfn.alif_w = zeros(row, col, row*col) + + # lif subscription + row, col, z = size(kfn.lif_w) # row*col is synaptic subscribe weight for each neuron in z + synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent] + synapticConnection = Int(floor(z*synapticConnectionPercent/100)) + for slice in eachslice(kfn.lif_w, dims=3) + pool = shuffle!([1:z...])[1:synapticConnection] + for i in pool + slice[i] = randn()/10 + end end - error("debug end") + # alif subscription + row, col, z = size(kfn.alif_w) # row*col is synaptic subscribe weight for each neuron in z + synapticConnectionPercent = kfn.params[:computeNeuron][:alif][:params][:synapticConnectionPercent] + synapticConnection = Int(floor(z*synapticConnectionPercent/100)) + for slice in eachslice(kfn.alif_w, dims=3) + pool = shuffle!([1:z...])[1:synapticConnection] + for i in pool + slice[i] = randn()/10 + end + end + + + + + + + + + + + + # error("debug end outer constructor") + return kfn end +# kfn forward +function (kfn::kfn_1)(input::AbstractArray) + kfn.timeStep .+= 1 + # row, col = size(input) # if input is a 2D matrix + println(">>> 5 ", size(input)) + println(">>> 6 ", size(kfn.z_i_t1)) + + #WORKING multiply input with kfn.z_i_t1 may be using cartesian coordinates + println(">>> 7 ", view(kfn.z_i_t1, :, 1, :)) + view(kfn.z_i_t1, :, 1) .= input + println(">>> 8 ", kfn.z_i_t1[:, 1]) + + # multiply kfn.z_i_t1 with kfn.lif_w + r = GeneralUtils.batchMatEleMul(kfn.z_i_t1, kfn.lif_w) + + println(size(r)) + + + + + + + + + + + + + + + + + error("debug end kfn forward") +end