ton/IronpenGPU
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add learn()

Browse Source
This commit is contained in:
ton
2023-07-26 15:21:34 +07:00
parent ff9909fd8d
commit a94354efb3
6 changed files with 224 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Manifest.toml
View File

@@ -2,7 +2,7 @@
julia_version = "1.9.2"
manifest_format = "2.0"
project_hash = "1a1cddac46fdd2108611b4e2f350497572f0c8d4"
project_hash = "1d38b0278f78d536c218e3a421dfd88a68063099"
[[deps.AbstractFFTs]]
deps = ["LinearAlgebra"]

4
Project.toml
View File

@@ -5,6 +5,10 @@ version = "0.1.0"
[deps]
CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
GeneralUtils = "c6c72f09-b708-4ac8-ac7c-2084d70108fe"
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

4
src/IronpenGPU.jl
View File

@@ -1,4 +1,4 @@
module IronpenGPU
module IronpenGPU # this is a parent module
# export
@@ -8,7 +8,7 @@ files and each file can only depend on the file included before it.
"""
include("type.jl")
using .type # bring model into this module namespace (this module is a parent module)
using .type # bring type into parent module namespace
include("snnUtil.jl")
using .snnUtil

47
src/forward.jl
View File

@@ -18,16 +18,16 @@ function (kfn::kfn_1)(input::AbstractArray)
# reset learning params
end
println(">>> input ", size(input))
d1, d2, d3 = size(input)
println(">>> zit ", size(kfn.zit))
println(">>> lif_zit ", size(kfn.lif_zit))
# println(">>> input ", size(input))
# println(">>> zit ", size(kfn.zit))
# println(">>> lif_zit ", size(kfn.lif_zit))
# println(">>> lif_recSignal ", size(kfn.lif_recSignal))
println(">>> lif_wRec ", size(kfn.lif_wRec))
println(">>> lif_refractoryCounter ", size(kfn.lif_refractoryCounter))
println(">>> lif_alpha ", size(kfn.lif_alpha))
println(">>> lif_vt0 ", size(kfn.lif_vt0))
println(">>> lif_vt0 sum ", sum(kfn.lif_vt0))
# println(">>> lif_wRec ", size(kfn.lif_wRec))
# println(">>> lif_refractoryCounter ", size(kfn.lif_refractoryCounter))
# println(">>> lif_alpha ", size(kfn.lif_alpha))
# println(">>> lif_vt0 ", size(kfn.lif_vt0))
# println(">>> lif_vt0 sum ", sum(kfn.lif_vt0))
# pass input_data into input neuron.
GeneralUtils.cartesianAssign!(kfn.zit, input)
@@ -45,7 +45,8 @@ function (kfn::kfn_1)(input::AbstractArray)
kfn.lif_epsilonRec,
kfn.lif_refractoryCounter,
kfn.lif_refractoryDuration,
kfn.lif_gammaPd)
kfn.lif_gammaPd,
kfn.lif_firingCounter)
alifForward( kfn.zit,
kfn.alif_zit,
@@ -65,7 +66,8 @@ function (kfn::kfn_1)(input::AbstractArray)
kfn.alif_a,
kfn.alif_beta,
kfn.alif_rho,
kfn.alif_gammaPd)
kfn.alif_gammaPd,
kfn.alif_firingCounter)
# update activation matrix by concatenate (input, lif_zt1, alif_zt1) to form activation matrix
_zit = cat(reshape(input, (d1, d2, 1, d3)),
@@ -76,7 +78,7 @@ function (kfn::kfn_1)(input::AbstractArray)
# read out
onForward( kfn.zit,
kfn.on_zit,
kfn.on_wRec,
kfn.on_wOut,
kfn.on_vt0,
kfn.on_vt1,
kfn.on_vth,
@@ -87,9 +89,11 @@ function (kfn::kfn_1)(input::AbstractArray)
kfn.on_epsilonRec,
kfn.on_refractoryCounter,
kfn.on_refractoryDuration,
kfn.on_gammaPd)
kfn.on_gammaPd,
kfn.on_firingCounter)
return kfn.on_zt1
return reshape(kfn.on_zt1, (d1, :)),
kfn.zit
end
function lifForward(kfn_zit,
@@ -105,7 +109,8 @@ function lifForward(kfn_zit,
epsilonRec,
refractoryCounter,
refractoryDuration,
gammaPd)
gammaPd,
firingCounter)
d1, d2, d3, d4 = size(wRec)
zit .= reshape(kfn_zit, (d1, d2, 1, d4)) .* ones(size(wRec)...) # project zit into zit
@@ -161,7 +166,8 @@ function alifForward(kfn_zit,
a,
beta,
rho,
gammaPd)
gammaPd,
firingCounter)
d1, d2, d3, d4 = size(wRec)
zit .= reshape(kfn_zit, (d1, d2, 1, d4)) .* ones(size(wRec)...) # project zit into zit
@@ -215,7 +221,7 @@ end
function onForward(kfn_zit,
zit,
wRec,
wOut,
vt0,
vt1,
vth,
@@ -226,9 +232,10 @@ function onForward(kfn_zit,
epsilonRec,
refractoryCounter,
refractoryDuration,
gammaPd)
d1, d2, d3, d4 = size(wRec)
zit .= reshape(kfn_zit, (d1, d2, 1, d4)) .* ones(size(wRec)...) # project zit into zit
gammaPd,
firingCounter)
d1, d2, d3, d4 = size(wOut)
zit .= reshape(kfn_zit, (d1, d2, 1, d4)) .* ones(size(wOut)...) # project zit into zit
for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch
if view(refractoryCounter, :, :, i, j)[1] > 0 # neuron is inactive (in refractory period)
@@ -242,7 +249,7 @@ function onForward(kfn_zit,
else # neuron is active
view(vt1, :, :, i, j)[1] =
(view(alpha, :, :, i, j)[1] * view(vt0,:, :, i, j)[1]) +
sum(view(zit, :, :, i, j) .* view(wRec, :, :, i, j))
sum(view(zit, :, :, i, j) .* view(wOut, :, :, i, j))
if view(vt1, :, :, i, j)[1] > view(vth, :, :, i, j)[1]
view(zt1, :, :, i, j)[1] = 1
view(refractoryCounter, :, :, i, j)[1] =

119
src/learn.jl
View File

@@ -1,16 +1,131 @@
module learn
# export
export learn!, compute_paramsChange!
# using
using Statistics, Random, LinearAlgebra, JSON3, Flux, Dates
using GeneralUtils
using ..type, ..snnUtil
#------------------------------------------------------------------------------------------------100
function compute_paramsChange!(kfn::kfn_1, modelError, outputError)
#WORKING
lifComputeParamsChange!(kfn.lif_phi,
kfn.lif_epsilonRec,
kfn.lif_eta,
kfn.lif_wRec,
kfn.lif_wRecChange,
kfn.on_wOut,
modelError)
alifComputeParamsChange!(kfn.alif_phi,
kfn.alif_epsilonRec,
kfn.alif_epsilonRecA,
kfn.alif_eta,
kfn.alif_wRec,
kfn.alif_wRecChange,
kfn.alif_beta,
kfn.on_wOut,
modelError)
error("debug end -> kfn compute_paramsChange! $(Dates.now())")
# Threads.@threads for n in kfn.neuronsArray
# # for n in kfn.neuronsArray
# if typeof(n) <: computeNeuron
# wOut = Int64[]
# for oN in kfn.outputNeuronsArray
# wIndex = findall(isequal.(oN.subscriptionList, n.id))
# if length(wIndex) != 0
# push!(wOut, wIndex[1])
# end
# end
# if length(wOut) != 0
# compute_wRecChange!(n, wOut, modelError)
# # compute_alphaChange!(n, modelError)
# compute_firingRateError!(n, kfn.kfnParams[:neuronFiringRateTarget],
# kfn.kfnParams[:totalComputeNeuron])
# end
# end
# end
# for oN in kfn.outputNeuronsArray
# compute_wRecChange!(oN, outputError[oN.id])
# # compute_alphaChaZnge!(oN, outputError[oN.id])
# end
end
function lifComputeParamsChange!( phi,
epsilonRec,
eta,
wRec,
wRecChange,
wOut,
modelError)
d1, d2, d3, d4 = size(epsilonRec)
# Bₖⱼ in paper, sum() to get each neuron's total wOut weight
wOutSum = reshape(sum(wOut, dims=3), (d1, :, d4))
for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch
# how much error of this neuron 1-spike causing each output neuron's error
view(wRecChange, :, :, i, j) .+= (-1 * view(eta, :, :, i, j)[1]) .*
# eRec
( (view(phi, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) .*
# nError a.k.a. learning signal
(view(modelError, :, j)[1] .*
# RSNN neuron's total wOut weight (neuron synaptic subscription .* wOutSum)
sum(GeneralUtils.isNotEqual.(view(wRec, :, :, i, j), 0) .*
view(wOutSum, :, :, j))
)
)
end
end
function alifComputeParamsChange!( phi,
epsilonRec,
epsilonRecA,
eta,
wRec,
wRecChange,
beta,
wOut,
modelError)
d1, d2, d3, d4 = size(epsilonRec)
# Bₖⱼ in paper, sum() to get each neuron's total wOut weight
wOutSum = reshape(sum(wOut, dims=3), (d1, :, d4))
for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch
# how much error of this neuron 1-spike causing each output neuron's error
view(wRecChange, :, :, i, j) .+= (-1 * view(eta, :, :, i, j)[1]) .*
# eRec
(
# eRec_v
(view(phi, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) .+
# eRec_a
((view(phi, :, :, i, j)[1] * view(beta, :, :, i, j)[1]) .*
view(epsilonRecA, :, :, i, j))
) .*
# nError a.k.a. learning signal
(
view(modelError, :, j)[1] .*
# RSNN neuron's total wOut weight (neuron synaptic subscription .* wOutSum)
sum(GeneralUtils.isNotEqual.(view(wRec, :, :, i, j), 0) .*
view(wOutSum, :, :, j))
)
end
end

135
src/type.jl
View File

@@ -43,8 +43,10 @@ Base.@kwdef mutable struct kfn_1 <: knowledgeFn
lif_tau_m::AbstractFloat = 20.0
lif_phi::Union{AbstractArray, Nothing} = nothing
lif_epsilonRec::Union{AbstractArray, Nothing} = nothing
# lif_eRec::Union{AbstractArray, Nothing} = nothing
lif_eta::Union{AbstractArray, Nothing} = nothing
lif_gammaPd::Union{AbstractArray, Nothing} = nothing
lif_wRecChange::Union{AbstractArray, Nothing} = nothing
lif_firingCounter::Union{AbstractArray, Nothing} = nothing
@@ -69,8 +71,10 @@ Base.@kwdef mutable struct kfn_1 <: knowledgeFn
alif_phi::Union{AbstractArray, Nothing} = nothing
alif_epsilonRec::Union{AbstractArray, Nothing} = nothing
alif_epsilonRecA::Union{AbstractArray, Nothing} = nothing
# alif_eRec::Union{AbstractArray, Nothing} = nothing
alif_eta::Union{AbstractArray, Nothing} = nothing
alif_gammaPd::Union{AbstractArray, Nothing} = nothing
alif_wRecChange::Union{AbstractArray, Nothing} = nothing
alif_firingCounter::Union{AbstractArray, Nothing} = nothing
@@ -85,7 +89,7 @@ Base.@kwdef mutable struct kfn_1 <: knowledgeFn
# output neuron is based on LIF
on_zit::Union{AbstractArray, Nothing} = nothing
on_wRec::Union{AbstractArray, Nothing} = nothing
on_wOut::Union{AbstractArray, Nothing} = nothing # same as lif_wRec
on_vt0::Union{AbstractArray, Nothing} = nothing
on_vt1::Union{AbstractArray, Nothing} = nothing
on_vth::Union{AbstractArray, Nothing} = nothing
@@ -99,8 +103,10 @@ Base.@kwdef mutable struct kfn_1 <: knowledgeFn
on_tau_m::AbstractFloat = 20.0
on_phi::Union{AbstractArray, Nothing} = nothing
on_epsilonRec::Union{AbstractArray, Nothing} = nothing
on_eRec::Union{AbstractArray, Nothing} = nothing
on_eta::Union{AbstractArray, Nothing} = nothing
on_gammaPd::Union{AbstractArray, Nothing} = nothing
on_wOutChange::Union{AbstractArray, Nothing} = nothing
on_firingCounter::Union{AbstractArray, Nothing} = nothing
end
@@ -128,24 +134,26 @@ function kfn_1(params::Dict)
# ---------------------------------------------------------------------------- #
# In 3D LIF matrix, z-axis represent each neuron while each 2D slice represent that neuron's
# synaptic subscription to other neurons (via activation matrix)
z = kfn.params[:computeNeuron][:lif][:numbers][1] * kfn.params[:computeNeuron][:lif][:numbers][2]
kfn.lif_zit = zeros(row, col, z, batch)
kfn.lif_vt0 = zeros(1, 1, z, batch)
kfn.lif_vt1 = zeros(1, 1, z, batch)
kfn.lif_vth = ones(1, 1, z, batch)
kfn.lif_vRest = zeros(1, 1, z, batch)
kfn.lif_zt0 = zeros(1, 1, z, batch)
kfn.lif_zt1 = zeros(1, 1, z, batch)
kfn.lif_refractoryCounter = zeros(1, 1, z, batch)
kfn.lif_refractoryDuration = ones(1, 1, z, batch) .* 3
kfn.lif_alpha = ones(1, 1, z, batch) .* (exp(-kfn.lif_delta / kfn.lif_tau_m))
kfn.lif_phi = zeros(1, 1, z, batch)
kfn.lif_epsilonRec = zeros(row, col, z, batch)
kfn.lif_eta = zeros(1, 1, z, batch)
kfn.lif_gammaPd = zeros(1, 1, z, batch) .* 0.3
n = kfn.params[:computeNeuron][:lif][:numbers][1] * kfn.params[:computeNeuron][:lif][:numbers][2]
kfn.lif_zit = zeros(row, col, n, batch)
kfn.lif_vt0 = zeros(1, 1, n, batch)
kfn.lif_vt1 = zeros(1, 1, n, batch)
kfn.lif_vth = ones(1, 1, n, batch)
kfn.lif_vRest = zeros(1, 1, n, batch)
kfn.lif_zt0 = zeros(1, 1, n, batch)
kfn.lif_zt1 = zeros(1, 1, n, batch)
kfn.lif_refractoryCounter = zeros(1, 1, n, batch)
kfn.lif_refractoryDuration = ones(1, 1, n, batch) .* 3
kfn.lif_alpha = ones(1, 1, n, batch) .* (exp(-kfn.lif_delta / kfn.lif_tau_m))
kfn.lif_phi = zeros(1, 1, n, batch)
kfn.lif_epsilonRec = zeros(row, col, n, batch)
# kfn.lif_eRec = zeros(row, col, n, batch)
kfn.lif_eta = zeros(1, 1, n, batch)
kfn.lif_gammaPd = zeros(1, 1, n, batch) .* 0.3
kfn.lif_wRecChange = zeros(row, col, n, batch)
# subscription
w = zeros(row, col, z)
w = zeros(row, col, n)
synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent]
synapticConnection = Int(floor(row*col * synapticConnectionPercent/100))
for slice in eachslice(w, dims=3)
@@ -155,36 +163,38 @@ function kfn_1(params::Dict)
end
end
# project 3D w into 4D kfn.lif_wRec
kfn.lif_wRec = reshape(w, (row, col, z, 1)) .* ones(row, col, z, batch)
kfn.lif_firingCounter = zeros(1, 1, z, batch)
kfn.lif_wRec = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch)
kfn.lif_firingCounter = zeros(1, 1, n, batch)
# ---------------------------------------------------------------------------- #
# ALIF config #
# ---------------------------------------------------------------------------- #
z = kfn.params[:computeNeuron][:alif][:numbers][1] * kfn.params[:computeNeuron][:alif][:numbers][2]
kfn.alif_zit = zeros(row, col, z, batch)
kfn.alif_vt0 = zeros(1, 1, z, batch)
kfn.alif_vt1 = zeros(1, 1, z, batch)
kfn.alif_vth = ones(1, 1, z, batch)
kfn.alif_avth = ones(1, 1, z, batch)
kfn.alif_vRest = zeros(1, 1, z, batch)
kfn.alif_zt0 = zeros(1, 1, z, batch)
kfn.alif_zt1 = zeros(1, 1, z, batch)
kfn.alif_refractoryCounter = zeros(1, 1, z, batch)
kfn.alif_refractoryDuration = ones(1, 1, z, batch) .* 3
kfn.alif_alpha = ones(1, 1, z, batch) .* (exp(-kfn.alif_delta / kfn.alif_tau_m))
kfn.alif_phi = zeros(1, 1, z, batch)
kfn.alif_epsilonRec = zeros(row, col, z, batch)
kfn.alif_epsilonRecA = zeros(row, col, z, batch)
kfn.alif_eta = zeros(1, 1, z, batch)
kfn.alif_gammaPd = zeros(1, 1, z, batch) .* 0.3
n = kfn.params[:computeNeuron][:alif][:numbers][1] * kfn.params[:computeNeuron][:alif][:numbers][2]
kfn.alif_zit = zeros(row, col, n, batch)
kfn.alif_vt0 = zeros(1, 1, n, batch)
kfn.alif_vt1 = zeros(1, 1, n, batch)
kfn.alif_vth = ones(1, 1, n, batch)
kfn.alif_avth = ones(1, 1, n, batch)
kfn.alif_vRest = zeros(1, 1, n, batch)
kfn.alif_zt0 = zeros(1, 1, n, batch)
kfn.alif_zt1 = zeros(1, 1, n, batch)
kfn.alif_refractoryCounter = zeros(1, 1, n, batch)
kfn.alif_refractoryDuration = ones(1, 1, n, batch) .* 3
kfn.alif_alpha = ones(1, 1, n, batch) .* (exp(-kfn.alif_delta / kfn.alif_tau_m))
kfn.alif_phi = zeros(1, 1, n, batch)
kfn.alif_epsilonRec = zeros(row, col, n, batch)
kfn.alif_epsilonRecA = zeros(row, col, n, batch)
# kfn.alif_eRec = zeros(row, col, n, batch)
kfn.alif_eta = zeros(1, 1, n, batch)
kfn.alif_gammaPd = zeros(1, 1, n, batch) .* 0.3
kfn.alif_wRecChange = zeros(row, col, n, batch)
kfn.alif_a = zeros(1, 1, z, batch)
kfn.alif_beta = zeros(1, 1, z, batch) .* 0.15
kfn.alif_rho = zeros(1, 1, z, batch) .* (exp(-kfn.alif_delta / kfn.alif_tau_a))
kfn.alif_a = zeros(1, 1, n, batch)
kfn.alif_beta = zeros(1, 1, n, batch) .* 0.15
kfn.alif_rho = zeros(1, 1, n, batch) .* (exp(-kfn.alif_delta / kfn.alif_tau_a))
# subscription
w = zeros(row, col, z)
w = zeros(row, col, n)
synapticConnectionPercent = kfn.params[:computeNeuron][:alif][:params][:synapticConnectionPercent]
synapticConnection = Int(floor(row*col * synapticConnectionPercent/100))
for slice in eachslice(w, dims=3)
@@ -194,31 +204,32 @@ function kfn_1(params::Dict)
end
end
# project 3D w into 4D kfn.alif_wRec
kfn.alif_wRec = reshape(w, (row, col, z, 1)) .* ones(row, col, z, batch)
kfn.alif_firingCounter = zeros(1, 1, z, batch)
kfn.alif_wRec = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch)
kfn.alif_firingCounter = zeros(1, 1, n, batch)
# ---------------------------------------------------------------------------- #
# output config #
# ---------------------------------------------------------------------------- #
#WORKING
z = kfn.params[:outputPort][:numbers][1] * kfn.params[:outputPort][:numbers][2]
kfn.on_zit = zeros(row, col, z, batch)
kfn.on_vt0 = zeros(1, 1, z, batch)
kfn.on_vt1 = zeros(1, 1, z, batch)
kfn.on_vth = ones(1, 1, z, batch)
kfn.on_vRest = zeros(1, 1, z, batch)
kfn.on_zt0 = zeros(1, 1, z, batch)
kfn.on_zt1 = zeros(1, 1, z, batch)
kfn.on_refractoryCounter = zeros(1, 1, z, batch)
kfn.on_refractoryDuration = ones(1, 1, z, batch) .* 1
kfn.on_alpha = ones(1, 1, z, batch) .* (exp(-kfn.on_delta / kfn.on_tau_m))
kfn.on_phi = zeros(1, 1, z, batch)
kfn.on_epsilonRec = zeros(row, col, z, batch)
kfn.on_eta = zeros(1, 1, z, batch)
kfn.on_gammaPd = zeros(1, 1, z, batch) .* 0.3
n = kfn.params[:outputPort][:numbers][1] * kfn.params[:outputPort][:numbers][2]
kfn.on_zit = zeros(row, col, n, batch)
kfn.on_vt0 = zeros(1, 1, n, batch)
kfn.on_vt1 = zeros(1, 1, n, batch)
kfn.on_vth = ones(1, 1, n, batch)
kfn.on_vRest = zeros(1, 1, n, batch)
kfn.on_zt0 = zeros(1, 1, n, batch)
kfn.on_zt1 = zeros(1, 1, n, batch)
kfn.on_refractoryCounter = zeros(1, 1, n, batch)
kfn.on_refractoryDuration = ones(1, 1, n, batch) .* 1
kfn.on_alpha = ones(1, 1, n, batch) .* (exp(-kfn.on_delta / kfn.on_tau_m))
kfn.on_phi = zeros(1, 1, n, batch)
kfn.on_epsilonRec = zeros(row, col, n, batch)
# kfn.on_eRec = zeros(row, col, n, batch)
kfn.on_eta = zeros(1, 1, n, batch)
kfn.on_gammaPd = zeros(1, 1, n, batch) .* 0.3
kfn.on_wOutChange = zeros(row, col, n, batch)
# subscription
w = zeros(row, col, z)
w = zeros(row, col, n)
synapticConnectionPercent = kfn.params[:outputPort][:params][:synapticConnectionPercent]
synapticConnection = Int(floor(row*col * synapticConnectionPercent/100))
for slice in eachslice(w, dims=3)
@@ -227,9 +238,9 @@ function kfn_1(params::Dict)
slice[i] = randn()/10 # assign weight to synaptic connection
end
end
# project 3D w into 4D kfn.on_wRec
kfn.on_wRec = reshape(w, (row, col, z, 1)) .* ones(row, col, z, batch)
kfn.on_firingCounter = zeros(1, 1, z, batch)
# project 3D w into 4D kfn.on_wOut
kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch)
kfn.on_firingCounter = zeros(1, 1, n, batch)