add user folder
This commit is contained in:
89
previousVersion/0.0.8/src/IronpenGPU.jl
Normal file
89
previousVersion/0.0.8/src/IronpenGPU.jl
Normal file
@@ -0,0 +1,89 @@
|
||||
module IronpenGPU # this is a parent module
|
||||
|
||||
# export
|
||||
|
||||
|
||||
""" Order by dependencies of each file. The 1st included file must not depend on any other
|
||||
files and each file can only depend on the file included before it.
|
||||
"""
|
||||
|
||||
include("type.jl")
|
||||
using .type # bring type into parent module namespace
|
||||
|
||||
include("snnUtil.jl")
|
||||
using .snnUtil
|
||||
|
||||
include("forward.jl")
|
||||
using .forward
|
||||
|
||||
include("learn.jl")
|
||||
using .learn
|
||||
|
||||
include("interface.jl")
|
||||
using .interface
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------------------------100
|
||||
|
||||
""" version 0.0.7
|
||||
Todo:
|
||||
[] synaptic liquidity range 0 to 100,000 -> 1.0 to 0.99
|
||||
[] add weight liquidity
|
||||
[-] add temporal summation in addition to already used spatial summation.
|
||||
CANCELLED, spatial summation every second until membrane potential reach a threshold
|
||||
is in itself a temporal summation.
|
||||
[4] implement dormant connection and pruning machanism. the longer the training the longer
|
||||
0 weight stay 0.
|
||||
[] using RL to control learning signal
|
||||
[] consider using Dates.now() instead of timestamp because time_stamp may overflow
|
||||
[] Liquid time constant. training should include adjusting α, neuron membrane potential decay factor
|
||||
which defined by neuron.tau_m formula in type.jl
|
||||
|
||||
Change from version: 0.0.6
|
||||
-
|
||||
|
||||
All features
|
||||
- excitatory/inhabitory matrix
|
||||
- neuroplasticity
|
||||
"""
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
end # module IronpenGPU
|
||||
897
previousVersion/0.0.8/src/forward.jl
Normal file
897
previousVersion/0.0.8/src/forward.jl
Normal file
@@ -0,0 +1,897 @@
|
||||
module forward
|
||||
|
||||
# export
|
||||
|
||||
using Flux, CUDA
|
||||
using GeneralUtils
|
||||
using ..type, ..snnUtil
|
||||
|
||||
#------------------------------------------------------------------------------------------------100
|
||||
|
||||
""" kfn forward
|
||||
input (row, col, batch)
|
||||
"""
|
||||
function (kfn::kfn_1)(input::AbstractArray)
|
||||
|
||||
kfn.timeStep .+= 1
|
||||
|
||||
# what to do at the start of learning round
|
||||
if view(kfn.learningStage, 1)[1] == 1
|
||||
# reset learning params
|
||||
kfn.zitCumulative .= 0
|
||||
|
||||
kfn.lif_vt .= 0
|
||||
kfn.lif_wRecChange .= 0
|
||||
kfn.lif_epsilonRec .= 0
|
||||
kfn.lif_firingCounter .= 0
|
||||
kfn.lif_refractoryCounter .= 0
|
||||
kfn.lif_zt .= 0
|
||||
|
||||
kfn.alif_vt .= 0
|
||||
kfn.alif_a .= 0
|
||||
kfn.alif_epsilonRec .= 0
|
||||
kfn.alif_epsilonRecA .= 0
|
||||
kfn.alif_wRecChange .= 0
|
||||
kfn.alif_firingCounter .= 0
|
||||
kfn.alif_refractoryCounter .= 0
|
||||
kfn.alif_zt .= 0
|
||||
|
||||
kfn.on_vt .= 0
|
||||
kfn.on_epsilonRec .= 0
|
||||
kfn.on_wOutChange .= 0
|
||||
kfn.on_refractoryCounter .= 0
|
||||
|
||||
kfn.learningStage = [2]
|
||||
end
|
||||
|
||||
# update activation matrix with "lif_zt1" and "alif_zt1" by concatenating
|
||||
# (input, lif_zt1, alif_zt1) to form activation matrix
|
||||
_zit = cat(reshape(input, (size(input, 1), size(input, 2), 1, size(input, 3))),
|
||||
reshape(kfn.lif_zt, (size(input, 1), :, 1, size(input, 3))),
|
||||
reshape(kfn.alif_zt, (size(input, 1), :, 1, size(input, 3))), dims=2)
|
||||
kfn.zit .= reshape(_zit, (size(input, 1), :, size(input, 3)))
|
||||
|
||||
@sync begin
|
||||
@async begin
|
||||
# project 3D kfn zit into 4D lif zit
|
||||
i1, i2, i3, i4 = size(kfn.lif_zit)
|
||||
kfn.lif_zit .= reshape(kfn.zit, (i1, i2, 1, i4)) .* kfn.lif_arrayProjection4d
|
||||
kfn.lif_exInType .= kfn.exInType .* kfn.lif_arrayProjection4d
|
||||
|
||||
lifForward( kfn.lif_zit,
|
||||
kfn.lif_wRec,
|
||||
kfn.lif_vt,
|
||||
kfn.lif_vth,
|
||||
kfn.lif_vRest,
|
||||
kfn.lif_zt4d,
|
||||
kfn.lif_alpha,
|
||||
kfn.lif_phi,
|
||||
kfn.lif_epsilonRec,
|
||||
kfn.lif_refractoryCounter,
|
||||
kfn.lif_refractoryDuration,
|
||||
kfn.lif_gammaPd,
|
||||
kfn.lif_firingCounter,
|
||||
kfn.lif_recSignal,
|
||||
kfn.lif_exInType,
|
||||
kfn.lif_neuronInactivityCounter,
|
||||
kfn.lif_synapticInactivityCounter,
|
||||
)
|
||||
end
|
||||
@async begin
|
||||
# project 3D kfn zit into 4D alif zit
|
||||
i1, i2, i3, i4 = size(kfn.alif_zit)
|
||||
kfn.alif_zit .= reshape(kfn.zit, (i1, i2, 1, i4)) .* kfn.alif_arrayProjection4d
|
||||
kfn.alif_exInType .= kfn.exInType .* kfn.alif_arrayProjection4d
|
||||
|
||||
alifForward(kfn.alif_zit,
|
||||
kfn.alif_wRec,
|
||||
kfn.alif_vt,
|
||||
kfn.alif_vth,
|
||||
kfn.alif_vRest,
|
||||
kfn.alif_zt4d,
|
||||
kfn.alif_alpha,
|
||||
kfn.alif_phi,
|
||||
kfn.alif_epsilonRec,
|
||||
kfn.alif_refractoryCounter,
|
||||
kfn.alif_refractoryDuration,
|
||||
kfn.alif_gammaPd,
|
||||
kfn.alif_firingCounter,
|
||||
kfn.alif_recSignal,
|
||||
kfn.alif_exInType,
|
||||
kfn.alif_neuronInactivityCounter,
|
||||
kfn.alif_synapticInactivityCounter,
|
||||
kfn.alif_epsilonRecA,
|
||||
kfn.alif_a,
|
||||
kfn.alif_avth,
|
||||
kfn.alif_beta,
|
||||
kfn.alif_rho,
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
# reduce lif_zt4d and alif_zt4d into lif_zt, alif_zt (4d -> 1d)
|
||||
kfn.lif_zt .= reduce(max, kfn.lif_zt4d, dims=(1,2))
|
||||
kfn.alif_zt .= reduce(max, kfn.alif_zt4d, dims=(1,2))
|
||||
|
||||
# update activation matrix with "lif_zt1" and "alif_zt1" by concatenating
|
||||
# (input, lif_zt1, alif_zt1) to form activation matrix
|
||||
_zit = cat(reshape(input, (size(input, 1), size(input, 2), 1, size(input, 3))),
|
||||
reshape(kfn.lif_zt, (size(input, 1), :, 1, size(input, 3))),
|
||||
reshape(kfn.alif_zt, (size(input, 1), :, 1, size(input, 3))), dims=2)
|
||||
kfn.zit .= reshape(_zit, (size(input, 1), :, size(input, 3)))
|
||||
kfn.zitCumulative .+= kfn.zit
|
||||
|
||||
# project 3D kfn zit into 4D on zit
|
||||
i1, i2, i3, i4 = size(kfn.on_zit)
|
||||
kfn.on_zit .= reshape(kfn.zit, (i1, i2, 1, i4)) .* kfn.on_arrayProjection4d
|
||||
|
||||
# read out
|
||||
onForward( kfn.on_zit,
|
||||
kfn.on_wOut,
|
||||
kfn.on_vt,
|
||||
kfn.on_vth,
|
||||
kfn.on_vRest,
|
||||
kfn.on_zt4d,
|
||||
kfn.on_alpha,
|
||||
kfn.on_phi,
|
||||
kfn.on_epsilonRec,
|
||||
kfn.on_refractoryCounter,
|
||||
kfn.on_refractoryDuration,
|
||||
kfn.on_gammaPd,
|
||||
kfn.on_firingCounter,
|
||||
kfn.on_recSignal,
|
||||
)
|
||||
# get on_zt4d to on_zt
|
||||
kfn.on_zt .= reduce(max, kfn.on_zt4d, dims=(1,2))
|
||||
logit = reshape(kfn.on_zt, (size(input, 1), :))
|
||||
|
||||
return logit,
|
||||
kfn.zit
|
||||
end
|
||||
|
||||
# gpu launcher
|
||||
function lifForward( zit::CuArray,
|
||||
wRec::CuArray,
|
||||
vt::CuArray,
|
||||
vth::CuArray,
|
||||
vRest::CuArray,
|
||||
zt::CuArray,
|
||||
alpha::CuArray,
|
||||
phi::CuArray,
|
||||
epsilonRec::CuArray,
|
||||
refractoryCounter::CuArray,
|
||||
refractoryDuration::CuArray,
|
||||
gammaPd::CuArray,
|
||||
firingCounter::CuArray,
|
||||
recSignal::CuArray,
|
||||
exInType::CuArray,
|
||||
neuronInactivityCounter::CuArray,
|
||||
synapticInactivityCounter::CuArray,
|
||||
)
|
||||
|
||||
kernel = @cuda launch=false lifForward( zit,
|
||||
wRec,
|
||||
vt,
|
||||
vth,
|
||||
vRest,
|
||||
zt,
|
||||
alpha,
|
||||
phi,
|
||||
epsilonRec,
|
||||
refractoryCounter,
|
||||
refractoryDuration,
|
||||
gammaPd,
|
||||
firingCounter,
|
||||
recSignal,
|
||||
exInType,
|
||||
neuronInactivityCounter,
|
||||
synapticInactivityCounter,
|
||||
GeneralUtils.linear_to_cartesian,
|
||||
)
|
||||
config = launch_configuration(kernel.fun)
|
||||
|
||||
|
||||
# threads to be launched. Since one can't launch exact thread number the kernel needs,
|
||||
# one just launch threads more than this kernel needs then use a guard inside the kernel
|
||||
# to prevent unused threads to access memory.
|
||||
threads = min(1024, config.threads) # depend on gpu. Most NVIDIA gpu has 1024 threads per block
|
||||
|
||||
# total desired threads to launch to gpu. Usually 1 thread per 1 matrix element
|
||||
totalThreads = length(wRec)
|
||||
|
||||
blocks = cld(totalThreads, threads)
|
||||
# println("launching gpu kernel")
|
||||
CUDA.@sync begin
|
||||
kernel( zit,
|
||||
wRec,
|
||||
vt,
|
||||
vth,
|
||||
vRest,
|
||||
zt,
|
||||
alpha,
|
||||
phi,
|
||||
epsilonRec,
|
||||
refractoryCounter,
|
||||
refractoryDuration,
|
||||
gammaPd,
|
||||
firingCounter,
|
||||
recSignal,
|
||||
exInType,
|
||||
neuronInactivityCounter,
|
||||
synapticInactivityCounter,
|
||||
GeneralUtils.linear_to_cartesian; threads, blocks)
|
||||
end
|
||||
end
|
||||
|
||||
# gpu kernel
|
||||
function lifForward( zit,
|
||||
wRec,
|
||||
vt,
|
||||
vth,
|
||||
vRest,
|
||||
zt,
|
||||
alpha,
|
||||
phi,
|
||||
epsilonRec,
|
||||
refractoryCounter,
|
||||
refractoryDuration,
|
||||
gammaPd,
|
||||
firingCounter,
|
||||
recSignal,
|
||||
exInType,
|
||||
neuronInactivityCounter,
|
||||
synapticInactivityCounter,
|
||||
linear_to_cartesian,
|
||||
)
|
||||
i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index
|
||||
|
||||
if i <= length(wRec)
|
||||
# cartesian index
|
||||
i1, i2, i3, i4 = linear_to_cartesian(i, size(wRec))
|
||||
# @cuprintln("gpu thread $i $i1 $i2 $i3 $i4")
|
||||
|
||||
if refractoryCounter[i1,i2,i3,i4] > 0 # refractory period is active
|
||||
refractoryCounter[i1,i2,i3,i4] -= 1
|
||||
recSignal[i1,i2,i3,i4] = 0
|
||||
zt[i1,i2,i3,i4] = 0
|
||||
vt[i1,i2,i3,i4] = alpha[i1,i2,i3,i4] * vt[i1,i2,i3,i4]
|
||||
phi[i1,i2,i3,i4] = 0
|
||||
|
||||
# compute epsilonRec
|
||||
epsilonRec[i1,i2,i3,i4] = (alpha[i1,i2,i3,i4] * epsilonRec[i1,i2,i3,i4])
|
||||
|
||||
else # refractory period is inactive
|
||||
recSignal[i1,i2,i3,i4] = wRec[i1,i2,i3,i4] * zit[i1,i2,i3,i4] *
|
||||
exInType[i1,i2,i3,i4]
|
||||
vt[i1,i2,i3,i4] = (alpha[i1,i2,i3,i4] * vt[i1,i2,i3,i4]) +
|
||||
sum(@view(recSignal[:,:,i3,i4]))
|
||||
|
||||
# fires if membrane potential exceed threshold
|
||||
if vt[i1,i2,i3,i4] > vth[i1,i2,i3,i4]
|
||||
zt[i1,i2,i3,i4] = 1
|
||||
refractoryCounter[i1,i2,i3,i4] = refractoryDuration[i1,i2,i3,i4]
|
||||
firingCounter[i1,i2,i3,i4] += 1
|
||||
vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4]
|
||||
|
||||
# reset counter if neuron fires
|
||||
neuronInactivityCounter[i1,i2,i3,i4] = 0
|
||||
else
|
||||
zt[i1,i2,i3,i4] = 0
|
||||
neuronInactivityCounter[i1,i2,i3,i4] -= 1
|
||||
end
|
||||
|
||||
# compute phi, there is a difference from lif formula
|
||||
phi[i1,i2,i3,i4] = (gammaPd[i1,i2,i3,i4] / vth[i1,i2,i3,i4]) *
|
||||
max(0, 1 - ((vt[i1,i2,i3,i4] - vth[i1,i2,i3,i4]) / vth[i1,i2,i3,i4]))
|
||||
|
||||
# compute epsilonRec
|
||||
epsilonRec[i1,i2,i3,i4] = (alpha[i1,i2,i3,i4] * epsilonRec[i1,i2,i3,i4]) +
|
||||
(zit[i1,i2,i3,i4] * !iszero(wRec[i1,i2,i3,i4]))
|
||||
# !iszero indicates synaptic subscription
|
||||
|
||||
# count synaptic inactivity
|
||||
if !iszero(wRec[i1,i2,i3,i4]) # check if this is wRec subscription
|
||||
if !iszero(zit[i1,i2,i3,i4]) # synapse is active, reset counter
|
||||
#WORKING should be function based. range +1.0 to +0.1
|
||||
synapticInactivityCounter[i1,i2,i3,i4] += 1
|
||||
else # synapse is inactive, counting
|
||||
#WORKING should be function based. range +1.0 to +0.01
|
||||
synapticInactivityCounter[i1,i2,i3,i4] -= 1
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
# gpu launcher
|
||||
function alifForward( zit::CuArray,
|
||||
wRec::CuArray,
|
||||
vt::CuArray,
|
||||
vth::CuArray,
|
||||
vRest::CuArray,
|
||||
zt::CuArray,
|
||||
alpha::CuArray,
|
||||
phi::CuArray,
|
||||
epsilonRec::CuArray,
|
||||
refractoryCounter::CuArray,
|
||||
refractoryDuration::CuArray,
|
||||
gammaPd::CuArray,
|
||||
firingCounter::CuArray,
|
||||
recSignal::CuArray,
|
||||
exInType::CuArray,
|
||||
neuronInactivityCounter::CuArray,
|
||||
synapticInactivityCounter::CuArray,
|
||||
epsilonRecA::CuArray,
|
||||
a::CuArray,
|
||||
avth::CuArray,
|
||||
beta::CuArray,
|
||||
rho::CuArray,
|
||||
)
|
||||
|
||||
kernel = @cuda launch=false alifForward( zit,
|
||||
wRec,
|
||||
vt,
|
||||
vth,
|
||||
vRest,
|
||||
zt,
|
||||
alpha,
|
||||
phi,
|
||||
epsilonRec,
|
||||
refractoryCounter,
|
||||
refractoryDuration,
|
||||
gammaPd,
|
||||
firingCounter,
|
||||
recSignal,
|
||||
exInType,
|
||||
neuronInactivityCounter,
|
||||
synapticInactivityCounter,
|
||||
epsilonRecA,
|
||||
a,
|
||||
avth,
|
||||
beta,
|
||||
rho,
|
||||
GeneralUtils.linear_to_cartesian,
|
||||
)
|
||||
config = launch_configuration(kernel.fun)
|
||||
|
||||
# threads to be launched. Since one can't launch exact thread number the kernel needs,
|
||||
# one just launch threads more than this kernel needs then use a guard inside the kernel
|
||||
# to prevent unused threads to access memory.
|
||||
threads = min(1024, config.threads) # depend on gpu. Most NVIDIA gpu has 1024 threads per block
|
||||
|
||||
# total desired threads to launch to gpu. Usually 1 thread per 1 matrix element
|
||||
totalThreads = length(wRec)
|
||||
|
||||
blocks = cld(totalThreads, threads)
|
||||
# println("launching gpu kernel")
|
||||
CUDA.@sync begin
|
||||
kernel( zit,
|
||||
wRec,
|
||||
vt,
|
||||
vth,
|
||||
vRest,
|
||||
zt,
|
||||
alpha,
|
||||
phi,
|
||||
epsilonRec,
|
||||
refractoryCounter,
|
||||
refractoryDuration,
|
||||
gammaPd,
|
||||
firingCounter,
|
||||
recSignal,
|
||||
exInType,
|
||||
neuronInactivityCounter,
|
||||
synapticInactivityCounter,
|
||||
epsilonRecA,
|
||||
a,
|
||||
avth,
|
||||
beta,
|
||||
rho,
|
||||
GeneralUtils.linear_to_cartesian; threads, blocks)
|
||||
end
|
||||
end
|
||||
|
||||
# gpu kernel
|
||||
function alifForward( zit,
|
||||
wRec,
|
||||
vt,
|
||||
vth,
|
||||
vRest,
|
||||
zt,
|
||||
alpha,
|
||||
phi,
|
||||
epsilonRec,
|
||||
refractoryCounter,
|
||||
refractoryDuration,
|
||||
gammaPd,
|
||||
firingCounter,
|
||||
recSignal,
|
||||
exInType,
|
||||
neuronInactivityCounter,
|
||||
synapticInactivityCounter,
|
||||
epsilonRecA,
|
||||
a,
|
||||
avth,
|
||||
beta,
|
||||
rho,
|
||||
linear_to_cartesian,
|
||||
)
|
||||
i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index
|
||||
|
||||
if i <= length(wRec)
|
||||
# cartesian index
|
||||
i1, i2, i3, i4 = linear_to_cartesian(i, size(wRec))
|
||||
# @cuprintln("gpu thread $i $i1 $i2 $i3 $i4")
|
||||
|
||||
if refractoryCounter[i1,i2,i3,i4] > 0 # refractory period is active
|
||||
refractoryCounter[i1,i2,i3,i4] -= 1
|
||||
recSignal[i1,i2,i3,i4] = 0
|
||||
zt[i1,i2,i3,i4] = 0
|
||||
vt[i1,i2,i3,i4] = alpha[i1,i2,i3,i4] * vt[i1,i2,i3,i4]
|
||||
phi[i1,i2,i3,i4] = 0
|
||||
a[i1,i2,i3,i4] = rho[i1,i2,i3,i4] * a[i1,i2,i3,i4]
|
||||
|
||||
# compute epsilonRec
|
||||
epsilonRec[i1,i2,i3,i4] = (alpha[i1,i2,i3,i4] * epsilonRec[i1,i2,i3,i4])
|
||||
|
||||
# compute epsilonRecA use eq.26
|
||||
epsilonRecA[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] *
|
||||
(phi[i1,i2,i3,i4] * epsilonRec[i1,i2,i3,i4]))
|
||||
|
||||
# compute avth
|
||||
avth[i1,i2,i3,i4] = vth[i1,i2,i3,i4] + (beta[i1,i2,i3,i4] * a[i1,i2,i3,i4])
|
||||
|
||||
else # refractory period is inactive
|
||||
recSignal[i1,i2,i3,i4] = wRec[i1,i2,i3,i4] * zit[i1,i2,i3,i4] *
|
||||
exInType[i1,i2,i3,i4]
|
||||
vt[i1,i2,i3,i4] = (alpha[i1,i2,i3,i4] * vt[i1,i2,i3,i4]) +
|
||||
sum(@view(recSignal[:,:,i3,i4]))
|
||||
|
||||
# compute avth
|
||||
avth[i1,i2,i3,i4] = vth[i1,i2,i3,i4] + (beta[i1,i2,i3,i4] * a[i1,i2,i3,i4])
|
||||
|
||||
# fires if membrane potential exceed threshold
|
||||
if vt[i1,i2,i3,i4] > avth[i1,i2,i3,i4]
|
||||
zt[i1,i2,i3,i4] = 1
|
||||
refractoryCounter[i1,i2,i3,i4] = refractoryDuration[i1,i2,i3,i4]
|
||||
firingCounter[i1,i2,i3,i4] += 1
|
||||
vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4]
|
||||
a[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] * a[i1,i2,i3,i4]) + 1
|
||||
neuronInactivityCounter[i1,i2,i3,i4] = 0
|
||||
else
|
||||
zt[i1,i2,i3,i4] = 0
|
||||
a[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] * a[i1,i2,i3,i4])
|
||||
neuronInactivityCounter[i1,i2,i3,i4] -= 1
|
||||
end
|
||||
|
||||
# compute phi, there is a difference from alif formula
|
||||
phi[i1,i2,i3,i4] = (gammaPd[i1,i2,i3,i4] / vth[i1,i2,i3,i4]) *
|
||||
max(0, 1 - ((vt[i1,i2,i3,i4] - vth[i1,i2,i3,i4]) / vth[i1,i2,i3,i4]))
|
||||
|
||||
# compute epsilonRec
|
||||
epsilonRec[i1,i2,i3,i4] = (alpha[i1,i2,i3,i4] * epsilonRec[i1,i2,i3,i4]) +
|
||||
(zit[i1,i2,i3,i4] * !iszero(wRec[i1,i2,i3,i4]))
|
||||
# compute epsilonRecA use eq.26
|
||||
epsilonRecA[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] *
|
||||
(phi[i1,i2,i3,i4] * epsilonRec[i1,i2,i3,i4])) +
|
||||
(zit[i1,i2,i3,i4] * !iszero(wRec[i1,i2,i3,i4]))
|
||||
|
||||
# count synaptic inactivity
|
||||
if !iszero(wRec[i1,i2,i3,i4]) # check if this is wRec subscription
|
||||
if !iszero(zit[i1,i2,i3,i4]) # synapse is active, reset counter
|
||||
synapticInactivityCounter[i1,i2,i3,i4] += 1
|
||||
else # synapse is inactive, counting
|
||||
synapticInactivityCounter[i1,i2,i3,i4] -= 1
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
# gpu launcher
|
||||
function onForward( zit::CuArray,
|
||||
wOut::CuArray,
|
||||
vt::CuArray,
|
||||
vth::CuArray,
|
||||
vRest::CuArray,
|
||||
zt::CuArray,
|
||||
alpha::CuArray,
|
||||
phi::CuArray,
|
||||
epsilonRec::CuArray,
|
||||
refractoryCounter::CuArray,
|
||||
refractoryDuration::CuArray,
|
||||
gammaPd::CuArray,
|
||||
firingCounter::CuArray,
|
||||
recSignal::CuArray,
|
||||
)
|
||||
|
||||
kernel = @cuda launch=false onForward( zit,
|
||||
wOut,
|
||||
vt,
|
||||
vth,
|
||||
vRest,
|
||||
zt,
|
||||
alpha,
|
||||
phi,
|
||||
epsilonRec,
|
||||
refractoryCounter,
|
||||
refractoryDuration,
|
||||
gammaPd,
|
||||
firingCounter,
|
||||
recSignal,
|
||||
GeneralUtils.linear_to_cartesian,
|
||||
)
|
||||
config = launch_configuration(kernel.fun)
|
||||
|
||||
# threads to be launched. Since one can't launch exact thread number the kernel needs,
|
||||
# one just launch threads more than this kernel needs then use a guard inside the kernel
|
||||
# to prevent unused threads to access memory.
|
||||
threads = min(1024, config.threads) # depend on gpu. Most NVIDIA gpu has 1024 threads per block
|
||||
|
||||
# total desired threads to launch to gpu. Usually 1 thread per 1 matrix element
|
||||
totalThreads = length(wOut)
|
||||
|
||||
blocks = cld(totalThreads, threads)
|
||||
# println("launching gpu kernel")
|
||||
CUDA.@sync begin
|
||||
kernel( zit,
|
||||
wOut,
|
||||
vt,
|
||||
vth,
|
||||
vRest,
|
||||
zt,
|
||||
alpha,
|
||||
phi,
|
||||
epsilonRec,
|
||||
refractoryCounter,
|
||||
refractoryDuration,
|
||||
gammaPd,
|
||||
firingCounter,
|
||||
recSignal,
|
||||
GeneralUtils.linear_to_cartesian; threads, blocks)
|
||||
end
|
||||
end
|
||||
|
||||
# gpu kernel
|
||||
function onForward( zit,
|
||||
wOut,
|
||||
vt,
|
||||
vth,
|
||||
vRest,
|
||||
zt,
|
||||
alpha,
|
||||
phi,
|
||||
epsilonRec,
|
||||
refractoryCounter,
|
||||
refractoryDuration,
|
||||
gammaPd,
|
||||
firingCounter,
|
||||
recSignal,
|
||||
linear_to_cartesian,
|
||||
)
|
||||
i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index
|
||||
|
||||
if i <= length(wOut)
|
||||
# cartesian index
|
||||
i1, i2, i3, i4 = linear_to_cartesian(i, size(wOut))
|
||||
# @cuprintln("gpu thread $i $i1 $i2 $i3 $i4")
|
||||
|
||||
if refractoryCounter[i1,i2,i3,i4] > 0 # refractory period is active
|
||||
refractoryCounter[i1,i2,i3,i4] -= 1
|
||||
recSignal[i1,i2,i3,i4] = 0
|
||||
zt[i1,i2,i3,i4] = 0
|
||||
vt[i1,i2,i3,i4] = alpha[i1,i2,i3,i4] * vt[i1,i2,i3,i4]
|
||||
phi[i1,i2,i3,i4] = 0
|
||||
|
||||
# compute epsilonRec
|
||||
epsilonRec[i1,i2,i3,i4] = (alpha[i1,i2,i3,i4] * epsilonRec[i1,i2,i3,i4])
|
||||
|
||||
else # refractory period is inactive
|
||||
recSignal[i1,i2,i3,i4] = zit[i1,i2,i3,i4] * wOut[i1,i2,i3,i4]
|
||||
vt[i1,i2,i3,i4] = (alpha[i1,i2,i3,i4] * vt[i1,i2,i3,i4]) + sum(@view(recSignal[:,:,i3,i4]))
|
||||
|
||||
# fires if membrane potential exceed threshold
|
||||
if vt[i1,i2,i3,i4] > vth[i1,i2,i3,i4]
|
||||
zt[i1,i2,i3,i4] = 1
|
||||
refractoryCounter[i1,i2,i3,i4] = refractoryDuration[i1,i2,i3,i4]
|
||||
firingCounter[i1,i2,i3,i4] += 1
|
||||
vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4]
|
||||
else
|
||||
zt[i1,i2,i3,i4] = 0
|
||||
end
|
||||
|
||||
# compute phi, there is a difference from on formula
|
||||
phi[i1,i2,i3,i4] = (gammaPd[i1,i2,i3,i4] / vth[i1,i2,i3,i4]) *
|
||||
max(0, 1 - ((vt[i1,i2,i3,i4] - vth[i1,i2,i3,i4]) / vth[i1,i2,i3,i4]))
|
||||
|
||||
# compute epsilonRec
|
||||
epsilonRec[i1,i2,i3,i4] = (alpha[i1,i2,i3,i4] * epsilonRec[i1,i2,i3,i4]) +
|
||||
(zit[i1,i2,i3,i4] * !iszero(wOut[i1,i2,i3,i4]))
|
||||
end
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
# function lifForward(kfn_zit::Array{T},
|
||||
# zit::Array{T},
|
||||
# wRec::Array{T},
|
||||
# vt0::Array{T},
|
||||
# vt1::Array{T},
|
||||
# vth::Array{T},
|
||||
# vRest::Array{T},
|
||||
# zt1::Array{T},
|
||||
# alpha::Array{T},
|
||||
# phi::Array{T},
|
||||
# epsilonRec::Array{T},
|
||||
# refractoryCounter::Array{T},
|
||||
# refractoryDuration::Array{T},
|
||||
# gammaPd::Array{T},
|
||||
# firingCounter::Array{T},
|
||||
# arrayProjection4d::Array{T},
|
||||
# recSignal::Array{T},
|
||||
# decayed_vt0::Array{T},
|
||||
# decayed_epsilonRec::Array{T},
|
||||
# vt1_diff_vth::Array{T},
|
||||
# vt1_diff_vth_div_vth::Array{T},
|
||||
# gammaPd_div_vth::Array{T},
|
||||
# phiActivation::Array{T},
|
||||
# ) where T<:Number
|
||||
|
||||
# # project 3D kfn zit into 4D lif zit
|
||||
# i1, i2, i3, i4 = size(alif_wRec)
|
||||
# lif_zit .= reshape(kfn_zit, (i1, i2, 1, i4)) .* lif_arrayProjection4d
|
||||
|
||||
# for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
|
||||
# if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
|
||||
# @. @views refractoryCounter[:,:,i,j] -= 1
|
||||
# @. @views zt1[:,:,i,j] = 0
|
||||
# @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
|
||||
# @. @views phi[:,:,i,j] = 0
|
||||
|
||||
# # compute epsilonRec
|
||||
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
|
||||
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
|
||||
# else # refractory period is inactive
|
||||
# @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j]
|
||||
# @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
|
||||
# @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
|
||||
|
||||
# if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j]))
|
||||
# @. @views zt1[:,:,i,j] = 1
|
||||
# @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
|
||||
# @. @views firingCounter[:,:,i,j] += 1
|
||||
# @. @views vt1[:,:,i,j] = vRest[:,:,i,j]
|
||||
# else
|
||||
# @. @views zt1[:,:,i,j] = 0
|
||||
# end
|
||||
|
||||
# # compute phi, there is a difference from alif formula
|
||||
# @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
|
||||
# @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
|
||||
# @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
|
||||
# @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
|
||||
# @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
|
||||
|
||||
# # compute epsilonRec
|
||||
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
|
||||
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
|
||||
# end
|
||||
# end
|
||||
# end
|
||||
|
||||
# function alifForward(zit::Array{T},
|
||||
# wRec::Array{T},
|
||||
# vt0::Array{T},
|
||||
# vt1::Array{T},
|
||||
# vth::Array{T},
|
||||
# vRest::Array{T},
|
||||
# zt1::Array{T},
|
||||
# alpha::Array{T},
|
||||
# phi::Array{T},
|
||||
# epsilonRec::Array{T},
|
||||
# refractoryCounter::Array{T},
|
||||
# refractoryDuration::Array{T},
|
||||
# gammaPd::Array{T},
|
||||
# firingCounter::Array{T},
|
||||
# recSignal::Array{T},
|
||||
# decayed_vt0::Array{T},
|
||||
# decayed_epsilonRec::Array{T},
|
||||
# vt1_diff_vth::Array{T},
|
||||
# vt1_diff_vth_div_vth::Array{T},
|
||||
# gammaPd_div_vth::Array{T},
|
||||
# phiActivation::Array{T},
|
||||
|
||||
# epsilonRecA::Array{T},
|
||||
# avth::Array{T},
|
||||
# a::Array{T},
|
||||
# beta::Array{T},
|
||||
# rho::Array{T},
|
||||
# phi_x_epsilonRec::Array{T},
|
||||
# phi_x_beta::Array{T},
|
||||
# rho_diff_phi_x_beta::Array{T},
|
||||
# rho_div_phi_x_beta_x_epsilonRecA::Array{T},
|
||||
# beta_x_a::Array{T},
|
||||
# ) where T<:Number
|
||||
|
||||
# for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
|
||||
# if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
|
||||
# @. @views refractoryCounter[:,:,i,j] -= 1
|
||||
# @. @views zt1[:,:,i,j] = 0
|
||||
# @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
|
||||
# @. @views phi[:,:,i,j] = 0
|
||||
# @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
|
||||
|
||||
# # compute epsilonRec
|
||||
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
|
||||
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
|
||||
|
||||
# # compute epsilonRecA
|
||||
# @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j]
|
||||
# @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j]
|
||||
# @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j]
|
||||
# @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j]
|
||||
# @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j]
|
||||
|
||||
# # compute avth
|
||||
# @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j]
|
||||
# @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j]
|
||||
|
||||
# else # refractory period is inactive
|
||||
# @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j]
|
||||
# @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
|
||||
# @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
|
||||
|
||||
# # compute avth
|
||||
# @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j]
|
||||
# @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j]
|
||||
|
||||
# if sum(@view(vt1[:,:,i,j])) > sum(@view(avth[:,:,i,j]))
|
||||
# @. @views zt1[:,:,i,j] = 1
|
||||
# @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
|
||||
# @. @views firingCounter[:,:,i,j] += 1
|
||||
# @. @views vt1[:,:,i,j] = vRest[:,:,i,j]
|
||||
# @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
|
||||
# @. @views a[:,:,i,j] = a[:,:,i,j] += 1
|
||||
# else
|
||||
# @. @views zt1[:,:,i,j] = 0
|
||||
# @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
|
||||
# end
|
||||
|
||||
# # compute phi, there is a difference from alif formula
|
||||
# @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
|
||||
# @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
|
||||
# @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
|
||||
# @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
|
||||
# @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
|
||||
|
||||
# # compute epsilonRec
|
||||
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
|
||||
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
|
||||
|
||||
# # compute epsilonRecA
|
||||
# @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j]
|
||||
# @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j]
|
||||
# @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j]
|
||||
# @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j]
|
||||
# @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j]
|
||||
# end
|
||||
# end
|
||||
# end
|
||||
|
||||
# function onForward(kfn_zit::Array{T},
|
||||
# zit::Array{T},
|
||||
# wOut::Array{T},
|
||||
# vt0::Array{T},
|
||||
# vt1::Array{T},
|
||||
# vth::Array{T},
|
||||
# vRest::Array{T},
|
||||
# zt1::Array{T},
|
||||
# alpha::Array{T},
|
||||
# phi::Array{T},
|
||||
# epsilonRec::Array{T},
|
||||
# refractoryCounter::Array{T},
|
||||
# refractoryDuration::Array{T},
|
||||
# gammaPd::Array{T},
|
||||
# firingCounter::Array{T},
|
||||
# arrayProjection4d::Array{T},
|
||||
# recSignal::Array{T},
|
||||
# decayed_vt0::Array{T},
|
||||
# decayed_epsilonRec::Array{T},
|
||||
# vt1_diff_vth::Array{T},
|
||||
# vt1_diff_vth_div_vth::Array{T},
|
||||
# gammaPd_div_vth::Array{T},
|
||||
# phiActivation::Array{T},
|
||||
# ) where T<:Number
|
||||
|
||||
# # project 3D kfn zit into 4D lif zit
|
||||
# zit .= reshape(kfn_zit,
|
||||
# (size(wOut, 1), size(wOut, 2), 1, size(wOut, 4))) .* arrayProjection4d
|
||||
|
||||
# for j in 1:size(wOut, 4), i in 1:size(wOut, 3) # compute along neurons axis of every batch
|
||||
# if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
|
||||
# @. @views refractoryCounter[:,:,i,j] -= 1
|
||||
# @. @views zt1[:,:,i,j] = 0
|
||||
# @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
|
||||
# @. @views phi[:,:,i,j] = 0
|
||||
|
||||
# # compute epsilonRec
|
||||
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
|
||||
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
|
||||
# else # refractory period is inactive
|
||||
# @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wOut[:,:,i,j]
|
||||
# @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
|
||||
# @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
|
||||
|
||||
# if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j]))
|
||||
# @. @views zt1[:,:,i,j] = 1
|
||||
# @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
|
||||
# @. @views firingCounter[:,:,i,j] += 1
|
||||
# @. @views vt1[:,:,i,j] = vRest[:,:,i,j]
|
||||
# else
|
||||
# @. @views zt1[:,:,i,j] = 0
|
||||
# end
|
||||
|
||||
# # compute phi, there is a difference from alif formula
|
||||
# @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
|
||||
# @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
|
||||
# @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
|
||||
# @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
|
||||
# @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
|
||||
|
||||
# # compute epsilonRec
|
||||
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
|
||||
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
|
||||
# end
|
||||
# end
|
||||
# end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
end # module
|
||||
87
previousVersion/0.0.8/src/interface.jl
Normal file
87
previousVersion/0.0.8/src/interface.jl
Normal file
@@ -0,0 +1,87 @@
|
||||
module interface
|
||||
|
||||
|
||||
# export
|
||||
|
||||
# using Flux, CUDA
|
||||
|
||||
#------------------------------------------------------------------------------------------------100
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
end # module
|
||||
549
previousVersion/0.0.8/src/learn.jl
Normal file
549
previousVersion/0.0.8/src/learn.jl
Normal file
@@ -0,0 +1,549 @@
|
||||
module learn
|
||||
|
||||
export learn!, compute_paramsChange!
|
||||
|
||||
using Statistics, Random, LinearAlgebra, JSON3, Flux, CUDA, Dates
|
||||
using GeneralUtils
|
||||
using ..type, ..snnUtil
|
||||
|
||||
#------------------------------------------------------------------------------------------------100
|
||||
|
||||
function compute_paramsChange!(kfn::kfn_1, modelError, outputError)
|
||||
|
||||
lifComputeParamsChange!(kfn.timeStep,
|
||||
kfn.lif_phi,
|
||||
kfn.lif_epsilonRec,
|
||||
kfn.lif_eta,
|
||||
kfn.lif_eRec,
|
||||
kfn.lif_wRec,
|
||||
kfn.lif_exInType,
|
||||
kfn.lif_wRecChange,
|
||||
kfn.on_wOut,
|
||||
kfn.lif_firingCounter,
|
||||
kfn.lif_firingTargetFrequency,
|
||||
kfn.lif_arrayProjection4d,
|
||||
kfn.lif_error,
|
||||
modelError,
|
||||
|
||||
kfn.inputSize,
|
||||
)
|
||||
|
||||
alifComputeParamsChange!(kfn.timeStep,
|
||||
kfn.alif_phi,
|
||||
kfn.alif_epsilonRec,
|
||||
kfn.alif_eta,
|
||||
kfn.alif_eRec,
|
||||
kfn.alif_wRec,
|
||||
kfn.alif_exInType,
|
||||
kfn.alif_wRecChange,
|
||||
kfn.on_wOut,
|
||||
kfn.alif_firingCounter,
|
||||
kfn.alif_firingTargetFrequency,
|
||||
kfn.alif_arrayProjection4d,
|
||||
kfn.alif_error,
|
||||
modelError,
|
||||
|
||||
kfn.alif_epsilonRecA,
|
||||
kfn.alif_beta,
|
||||
)
|
||||
|
||||
onComputeParamsChange!(kfn.on_phi,
|
||||
kfn.on_epsilonRec,
|
||||
kfn.on_eta,
|
||||
kfn.on_eRec,
|
||||
kfn.on_wOutChange,
|
||||
kfn.on_arrayProjection4d,
|
||||
kfn.on_error,
|
||||
outputError,
|
||||
)
|
||||
# error("DEBUG -> kfn compute_paramsChange! $(Dates.now())")
|
||||
end
|
||||
|
||||
function lifComputeParamsChange!( timeStep::CuArray,
|
||||
phi::CuArray,
|
||||
epsilonRec::CuArray,
|
||||
eta::CuArray,
|
||||
eRec::CuArray,
|
||||
wRec::CuArray,
|
||||
exInType::CuArray,
|
||||
wRecChange::CuArray,
|
||||
wOut::CuArray,
|
||||
firingCounter::CuArray,
|
||||
firingTargetFrequency::CuArray,
|
||||
arrayProjection4d::CuArray,
|
||||
nError::CuArray,
|
||||
modelError::CuArray,
|
||||
|
||||
inputSize::CuArray,
|
||||
)
|
||||
# Bₖⱼ in paper, sum() to get each neuron's total wOut weight,
|
||||
# use absolute because only magnitude is needed
|
||||
wOutSum_all = reshape( abs.(sum(wOut, dims=3)), (1,1,:, size(wOut, 4)) ) # (1,1,allNeuron,batch)
|
||||
|
||||
# get only each lif neuron's wOut, leaving out other neuron's wOut
|
||||
startIndex = prod(inputSize) +1
|
||||
stopIndex = startIndex + size(wRec, 3) -1
|
||||
wOutSum = @view(wOutSum_all[1,1, startIndex:stopIndex, :])
|
||||
wOutSum = reshape(wOutSum, (1, 1, size(wOutSum, 1), size(wOutSum, 2))) # (1,1,n,batch)
|
||||
|
||||
# nError a.k.a. learning signal use dopamine concept,
|
||||
# this neuron receive summed error signal (modelError)
|
||||
nError .= (modelError .* wOutSum) .* arrayProjection4d
|
||||
eRec .= phi .* epsilonRec
|
||||
wRecChange .+= (-eta .* nError .* eRec)
|
||||
|
||||
# frequency regulator
|
||||
wRecChange .+= 0.001 .* ((firingTargetFrequency - (firingCounter./timeStep)) ./ timeStep) .*
|
||||
eta .* eRec
|
||||
|
||||
# if sum(timeStep) == 785
|
||||
# epsilonRec_cpu = epsilonRec |> cpu
|
||||
# println("modelError ", modelError)
|
||||
# println("")
|
||||
# wchange = (-eta .* nError .* eRec) |> cpu
|
||||
# println("wchange 5 1 ", wchange[:,:,5,1])
|
||||
# println("")
|
||||
# println("wchange 5 2 ", wchange[:,:,5,2])
|
||||
# println("")
|
||||
# println("epsilonRec 5 1 ", epsilonRec_cpu[:,:,5,1])
|
||||
# println("")
|
||||
# println("epsilonRec 5 2 ", epsilonRec_cpu[:,:,5,2])
|
||||
# println("")
|
||||
# error("DEBUG lifComputeParamsChange!")
|
||||
# end
|
||||
|
||||
# reset epsilonRec
|
||||
epsilonRec .= 0
|
||||
end
|
||||
|
||||
function alifComputeParamsChange!( timeStep::CuArray,
|
||||
phi::CuArray,
|
||||
epsilonRec::CuArray,
|
||||
eta::CuArray,
|
||||
eRec::CuArray,
|
||||
wRec::CuArray,
|
||||
exInType::CuArray,
|
||||
wRecChange::CuArray,
|
||||
wOut::CuArray,
|
||||
firingCounter::CuArray,
|
||||
firingTargetFrequency::CuArray,
|
||||
arrayProjection4d::CuArray,
|
||||
nError::CuArray,
|
||||
modelError::CuArray,
|
||||
|
||||
epsilonRecA::CuArray,
|
||||
beta::CuArray
|
||||
)
|
||||
|
||||
# Bₖⱼ in paper, sum() to get each neuron's total wOut weight,
|
||||
# use absolute because only magnitude is needed
|
||||
wOutSum_all = reshape( abs.(sum(wOut, dims=3)), (1,1,:, size(wOut, 4)) ) # (1,1,allNeuron,batch)
|
||||
|
||||
# get only each lif neuron's wOut, leaving out other neuron's wOut
|
||||
wOutSum = @view(wOutSum_all[1,1, end-size(wRec, 3)+1:end, :])
|
||||
wOutSum = reshape(wOutSum, (1, 1, size(wOutSum, 1), size(wOutSum, 2))) # (1,1,n,batch)
|
||||
|
||||
# nError a.k.a. learning signal use dopamine concept,
|
||||
# this neuron receive summed error signal (modelError)
|
||||
nError .= (modelError .* wOutSum) .* arrayProjection4d
|
||||
eRec .= phi .* (epsilonRec .- (beta .* epsilonRecA)) # use eq. 25
|
||||
wRecChange .+= (-eta .* nError .* eRec)
|
||||
|
||||
# frequency regulator
|
||||
wRecChange .+= 0.001 .* ((firingTargetFrequency - (firingCounter./timeStep)) ./ timeStep) .*
|
||||
eta .* eRec
|
||||
|
||||
# reset epsilonRec
|
||||
epsilonRec .= 0
|
||||
epsilonRecA .= 0
|
||||
|
||||
# error("DEBUG -> alifComputeParamsChange! $(Dates.now())")
|
||||
end
|
||||
|
||||
function onComputeParamsChange!(phi::CuArray,
|
||||
epsilonRec::CuArray,
|
||||
eta::CuArray,
|
||||
eRec::CuArray,
|
||||
wOutChange::CuArray,
|
||||
arrayProjection4d::CuArray,
|
||||
nError::CuArray,
|
||||
outputError::CuArray # outputError is output neuron's error
|
||||
)
|
||||
|
||||
eRec .= phi .* epsilonRec
|
||||
nError .= reshape(outputError, (1, 1, :, size(outputError, 2))) .* arrayProjection4d
|
||||
wOutChange .+= (-eta .* nError .* eRec)
|
||||
|
||||
# reset epsilonRec
|
||||
epsilonRec .= 0
|
||||
|
||||
# error("DEBUG -> onComputeParamsChange! $(Dates.now())")
|
||||
end
|
||||
|
||||
function lifComputeParamsChange!( phi::AbstractArray,
|
||||
epsilonRec::AbstractArray,
|
||||
eta::AbstractArray,
|
||||
wRec::AbstractArray,
|
||||
wRecChange::AbstractArray,
|
||||
wOut::AbstractArray,
|
||||
modelError::AbstractArray)
|
||||
d1, d2, d3, d4 = size(epsilonRec)
|
||||
# Bₖⱼ in paper, sum() to get each neuron's total wOut weight
|
||||
wOutSum = reshape(sum(wOut, dims=3), (d1, :, d4))
|
||||
|
||||
for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch
|
||||
# how much error of this neuron 1-spike causing each output neuron's error
|
||||
|
||||
view(wRecChange, :, :, i, j) .+= (-1 * view(eta, :, :, i, j)[1]) .*
|
||||
# eRec
|
||||
(
|
||||
(view(phi, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) .*
|
||||
# nError a.k.a. learning signal
|
||||
(
|
||||
view(modelError, :, j)[1] * # dopamine concept, this neuron receive summed error signal
|
||||
# RSNN neuron's total wOut weight (neuron synaptic subscription .* wOutSum)
|
||||
view(wOutSum, :, :, j)[i]
|
||||
)
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
function alifComputeParamsChange!( phi::AbstractArray,
|
||||
epsilonRec::AbstractArray,
|
||||
epsilonRecA::AbstractArray,
|
||||
eta::AbstractArray,
|
||||
wRec::AbstractArray,
|
||||
wRecChange::AbstractArray,
|
||||
beta::AbstractArray,
|
||||
wOut::AbstractArray,
|
||||
modelError::AbstractArray)
|
||||
d1, d2, d3, d4 = size(epsilonRec)
|
||||
|
||||
# Bₖⱼ in paper, sum() to get each neuron's total wOut weight
|
||||
wOutSum = reshape(sum(wOut, dims=3), (d1, :, d4))
|
||||
|
||||
for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch
|
||||
# how much error of this neuron 1-spike causing each output neuron's error
|
||||
|
||||
view(wRecChange, :, :, i, j) .+= (-1 * view(eta, :, :, i, j)[1]) .*
|
||||
# eRec
|
||||
(
|
||||
# eRec_v
|
||||
(view(phi, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) .+
|
||||
# eRec_a
|
||||
((view(phi, :, :, i, j)[1] * view(beta, :, :, i, j)[1]) .*
|
||||
view(epsilonRecA, :, :, i, j))
|
||||
) .*
|
||||
# nError a.k.a. learning signal
|
||||
(
|
||||
view(modelError, :, j)[1] *
|
||||
# RSNN neuron's total wOut weight (neuron synaptic subscription .* wOutSum)
|
||||
view(wOutSum, :, :, j)[i]
|
||||
# sum(GeneralUtils.isNotEqual.(view(wRec, :, :, i, j), 0) .*
|
||||
# view(wOutSum, :, :, j))
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
function onComputeParamsChange!(phi::AbstractArray,
|
||||
epsilonRec::AbstractArray,
|
||||
eta::AbstractArray,
|
||||
wOutChange::AbstractArray,
|
||||
outputError::AbstractArray)
|
||||
d1, d2, d3, d4 = size(epsilonRec)
|
||||
|
||||
for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch
|
||||
# how much error of this neuron 1-spike causing each output neuron's error
|
||||
|
||||
view(wOutChange, :, :, i, j) .+= (-1 * view(eta, :, :, i, j)[1]) .*
|
||||
# eRec
|
||||
(
|
||||
(view(phi, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) .*
|
||||
# nError a.k.a. learning signal, output neuron receives error of its own answer - correct answer.
|
||||
view(outputError, :, j)[i]
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
function learn!(kfn::kfn_1, device=cpu)
|
||||
# lif learn
|
||||
kfn.lif_wRec, kfn.lif_neuronInactivityCounter, kfn.lif_synapticInactivityCounter =
|
||||
lifLearn(kfn.lif_wRec,
|
||||
kfn.lif_exInType,
|
||||
kfn.lif_wRecChange,
|
||||
kfn.lif_arrayProjection4d,
|
||||
kfn.lif_neuronInactivityCounter,
|
||||
kfn.lif_synapticInactivityCounter,
|
||||
kfn.lif_synapticConnectionNumber,
|
||||
kfn.lif_synapticWChangeCounter,
|
||||
kfn.zitCumulative,
|
||||
device)
|
||||
|
||||
# alif learn
|
||||
kfn.alif_wRec, kfn.alif_neuronInactivityCounter, kfn.alif_synapticInactivityCounter =
|
||||
alifLearn(kfn.alif_wRec,
|
||||
kfn.alif_exInType,
|
||||
kfn.alif_wRecChange,
|
||||
kfn.alif_arrayProjection4d,
|
||||
kfn.alif_neuronInactivityCounter,
|
||||
kfn.alif_synapticInactivityCounter,
|
||||
kfn.alif_synapticConnectionNumber,
|
||||
kfn.alif_synapticWChangeCounter,
|
||||
kfn.zitCumulative,
|
||||
device)
|
||||
|
||||
# on learn
|
||||
onLearn!(kfn.on_wOut,
|
||||
kfn.on_wOutChange,
|
||||
kfn.on_arrayProjection4d)
|
||||
|
||||
# wrap up learning session
|
||||
if kfn.learningStage == [3]
|
||||
kfn.learningStage = [0]
|
||||
end
|
||||
# error("DEBUG -> kfn learn! $(Dates.now())")
|
||||
end
|
||||
|
||||
function lifLearn(wRec,
|
||||
exInType,
|
||||
wRecChange,
|
||||
arrayProjection4d,
|
||||
neuronInactivityCounter,
|
||||
synapticInactivityCounter,
|
||||
synapticConnectionNumber,
|
||||
synapticWChangeCounter, #WORKING
|
||||
zitCumulative,
|
||||
device)
|
||||
|
||||
# merge learning weight with average learning weight of all batch
|
||||
wch = sum(wRecChange, dims=4) ./ (size(wRec, 4)) .* arrayProjection4d
|
||||
wRec .= (exInType .* wRec) .+ wch
|
||||
|
||||
arrayProjection4d_cpu = arrayProjection4d |> cpu
|
||||
wRec_cpu = wRec |> cpu
|
||||
wRec_cpu = wRec_cpu[:,:,:,1] # since every batch has the same neuron wRec, (row, col, n)
|
||||
neuronInactivityCounter_cpu = neuronInactivityCounter |> cpu
|
||||
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu[:,:,:,1] # (row, col, n)
|
||||
synapticInactivityCounter_cpu = synapticInactivityCounter |> cpu
|
||||
synapticInactivityCounter_cpu = synapticInactivityCounter_cpu[:,:,:,1]
|
||||
zitCumulative_cpu = zitCumulative |> cpu
|
||||
zitCumulative_cpu = zitCumulative_cpu[:,:,1] # (row, col)
|
||||
|
||||
# weak / negative synaptic connection will get randomed in neuroplasticity()
|
||||
wRec_cpu = GeneralUtils.replaceBetween.(wRec_cpu, 0.0, 0.01, -1.0) # mark with -1.0
|
||||
|
||||
# synaptic connection that has no activity will get randomed in neuroplasticity()
|
||||
mask = isless.(synapticInactivityCounter_cpu, -100000)
|
||||
GeneralUtils.replace_elements!(mask, 1, wRec_cpu, -1.0)
|
||||
# reset lif_inactivity elements to base value
|
||||
GeneralUtils.replace_elements!(mask, 1, synapticInactivityCounter_cpu, 0.0)
|
||||
|
||||
# neuroplasticity, work on CPU side
|
||||
wRec_cpu = neuroplasticity(synapticConnectionNumber,
|
||||
zitCumulative_cpu,
|
||||
wRec_cpu,
|
||||
neuronInactivityCounter_cpu,
|
||||
synapticInactivityCounter_cpu)
|
||||
|
||||
wRec_cpu = wRec_cpu .* arrayProjection4d_cpu
|
||||
wRec = wRec_cpu |> device
|
||||
|
||||
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu .* arrayProjection4d_cpu
|
||||
neuronInactivityCounter = neuronInactivityCounter_cpu |> device
|
||||
|
||||
synapticInactivityCounter_cpu = synapticInactivityCounter_cpu .* arrayProjection4d_cpu
|
||||
synapticInactivityCounter = synapticInactivityCounter_cpu |> device
|
||||
|
||||
# error("DEBUG -> lifLearn! $(Dates.now())")
|
||||
return wRec, neuronInactivityCounter, synapticInactivityCounter
|
||||
end
|
||||
|
||||
function alifLearn(wRec,
|
||||
exInType,
|
||||
wRecChange,
|
||||
arrayProjection4d,
|
||||
neuronInactivityCounter,
|
||||
synapticInactivityCounter,
|
||||
synapticConnectionNumber,
|
||||
synapticWChangeCounter,
|
||||
zitCumulative,
|
||||
device)
|
||||
|
||||
# merge learning weight with average learning weight of all batch
|
||||
wch = sum(wRecChange, dims=4) ./ (size(wRec, 4)) .* arrayProjection4d
|
||||
wRec .= (exInType .* wRec) .+ wch
|
||||
|
||||
arrayProjection4d_cpu = arrayProjection4d |> cpu
|
||||
wRec_cpu = wRec |> cpu
|
||||
wRec_cpu = wRec_cpu[:,:,:,1] # since every batch has the same neuron wRec, (row, col, n)
|
||||
neuronInactivityCounter_cpu = neuronInactivityCounter |> cpu
|
||||
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu[:,:,:,1] # (row, col, n)
|
||||
synapticInactivityCounter_cpu = synapticInactivityCounter |> cpu
|
||||
synapticInactivityCounter_cpu = synapticInactivityCounter_cpu[:,:,:,1]
|
||||
zitCumulative_cpu = zitCumulative |> cpu
|
||||
zitCumulative_cpu = zitCumulative_cpu[:,:,1] # (row, col)
|
||||
|
||||
# weak / negative synaptic connection will get randomed in neuroplasticity()
|
||||
wRec_cpu = GeneralUtils.replaceBetween.(wRec_cpu, 0.0, 0.01, -1.0) # mark with -1.0
|
||||
|
||||
# synaptic connection that has no activity will get randomed in neuroplasticity()
|
||||
mask = isless.(synapticInactivityCounter_cpu, -100000)
|
||||
GeneralUtils.replace_elements!(mask, 1, wRec_cpu, -1.0)
|
||||
# reset alif_inactivity elements to base value
|
||||
GeneralUtils.replace_elements!(mask, 1, synapticInactivityCounter_cpu, 0.0)
|
||||
|
||||
# neuroplasticity, work on CPU side
|
||||
wRec_cpu = neuroplasticity(synapticConnectionNumber,
|
||||
zitCumulative_cpu,
|
||||
wRec_cpu,
|
||||
neuronInactivityCounter_cpu,
|
||||
synapticInactivityCounter_cpu)
|
||||
|
||||
wRec_cpu = wRec_cpu .* arrayProjection4d_cpu
|
||||
wRec = wRec_cpu |> device
|
||||
|
||||
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu .* arrayProjection4d_cpu
|
||||
neuronInactivityCounter = neuronInactivityCounter_cpu |> device
|
||||
|
||||
synapticInactivityCounter_cpu = synapticInactivityCounter_cpu .* arrayProjection4d_cpu
|
||||
synapticInactivityCounter = synapticInactivityCounter_cpu |> device
|
||||
|
||||
# error("DEBUG -> alifLearn! $(Dates.now())")
|
||||
return wRec, neuronInactivityCounter, synapticInactivityCounter
|
||||
end
|
||||
|
||||
function onLearn!(wOut,
|
||||
wOutChange,
|
||||
arrayProjection4d)
|
||||
# merge learning weight with average learning weight
|
||||
wOut .+= (sum(wOutChange, dims=4) ./ (size(wOut, 4))) .* arrayProjection4d
|
||||
|
||||
# adaptive wOut to help convergence using c_decay
|
||||
wOut .-= 0.001 .* wOut
|
||||
|
||||
#TODO synaptic strength
|
||||
|
||||
#TODO neuroplasticity
|
||||
|
||||
end
|
||||
|
||||
function neuroplasticity(synapticConnectionNumber,
|
||||
zitCumulative, # (row, col)
|
||||
wRec, # (row, col, n)
|
||||
neuronInactivityCounter,
|
||||
synapticInactivityCounter) # (row, col, n)
|
||||
|
||||
i1,i2,i3 = size(wRec)
|
||||
|
||||
# for each neuron, find total number of synaptic conn that should draw
|
||||
# new connection to firing and non-firing neurons pool
|
||||
subToFireNeuron_toBe = Int(floor(0.7 * synapticConnectionNumber))
|
||||
|
||||
# for each neuron, count how many synap already subscribed to firing-neurons
|
||||
zw = zitCumulative .* wRec
|
||||
subToFireNeuron_current = sum(GeneralUtils.isBetween.(zw, 0.0, 100.0), dims=(1,2)) # (1, 1, n)
|
||||
zitMask = (!iszero).(zitCumulative) # zitMask of firing neurons = 1, non-firing = 0
|
||||
projection = ones(i1,i2,i3)
|
||||
zitMask = zitMask .* projection # (row, col, n)
|
||||
totalNewConn = sum(isequal.(wRec, -1.0), dims=(1,2)) # count new conn mark (-1.0), (1, 1, n)
|
||||
# println("neuroplasticity, from $synapticConnectionNumber, $totalNewConn are replaced")
|
||||
|
||||
# clear -1.0 marker
|
||||
GeneralUtils.replace_elements!(wRec, -1.0, synapticInactivityCounter, -0.99)
|
||||
GeneralUtils.replace_elements!(wRec, -1.0, 0.0) # -1.0 marker is no longer required
|
||||
|
||||
for i in 1:i3
|
||||
if neuronInactivityCounter[1:1:i][1] < -10000 # neuron die i.e. reset all weight
|
||||
println("neuron die")
|
||||
neuronInactivityCounter[:,:,i] .= 0 # reset
|
||||
w = random_wRec(i1,i2,1,synapticConnectionNumber)
|
||||
wRec[:,:,i] .= w
|
||||
|
||||
a = similar(w) .= -0.99 # synapticConnectionNumber of this neuron
|
||||
mask = (!iszero).(w)
|
||||
GeneralUtils.replace_elements!(mask, 1, a, 0)
|
||||
synapticInactivityCounter[:,:,i] = a
|
||||
else
|
||||
remaining = 0
|
||||
if subToFireNeuron_current[1,1,i] < subToFireNeuron_toBe
|
||||
toAddConn = subToFireNeuron_toBe - subToFireNeuron_current[1,1,i]
|
||||
totalNewConn[1,1,i] = totalNewConn[1,1,i] - toAddConn
|
||||
# add new conn to firing neurons pool
|
||||
remaining = addNewSynapticConn!(zitMask[:,:,i], 1,
|
||||
@view(wRec[:,:,i]),
|
||||
@view(synapticInactivityCounter[:,:,i]),
|
||||
toAddConn)
|
||||
totalNewConn[1,1,i] += remaining
|
||||
end
|
||||
|
||||
# add new conn to non-firing neurons pool
|
||||
remaining = addNewSynapticConn!(zitMask[:,:,i], 0,
|
||||
@view(wRec[:,:,i]),
|
||||
@view(synapticInactivityCounter[:,:,i]),
|
||||
totalNewConn[1,1,i])
|
||||
if remaining > 0 # final get-all round if somehow non-firing pool has not enough slot
|
||||
remaining = addNewSynapticConn!(zitMask[:,:,i], 1,
|
||||
@view(wRec[:,:,i]),
|
||||
@view(synapticInactivityCounter[:,:,i]),
|
||||
remaining)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# error("DEBUG -> neuroplasticity $(Dates.now())")
|
||||
return wRec
|
||||
end
|
||||
|
||||
# learningLiquidity(x) = -0.0001x + 1 # -10000 to +10000; f(x) = -5e-05x+0.5
|
||||
|
||||
function learningLiquidity(x)
|
||||
if x > 10000
|
||||
y = 0.0
|
||||
elseif x < -10000
|
||||
y = 1.0
|
||||
else
|
||||
y = -5e-05x+0.5 # range -10000 to +10000
|
||||
end
|
||||
return y
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
end # module
|
||||
120
previousVersion/0.0.8/src/snnUtil.jl
Normal file
120
previousVersion/0.0.8/src/snnUtil.jl
Normal file
@@ -0,0 +1,120 @@
|
||||
module snnUtil
|
||||
|
||||
export refractoryStatus!, addNewSynapticConn!
|
||||
|
||||
using Random
|
||||
|
||||
#------------------------------------------------------------------------------------------------100
|
||||
|
||||
function refractoryStatus!(refractoryCounter, refractoryActive, refractoryInactive)
|
||||
d1, d2, d3, d4 = size(refractoryCounter)
|
||||
for j in 1:d4
|
||||
for i in 1:d3
|
||||
if refractoryCounter[1, 1, i, j] > 0 # inactive
|
||||
view(refractoryActive, 1, 1, i, j) .= 0
|
||||
view(refractoryInactive, 1, 1, i, j) .= 1
|
||||
else # active
|
||||
view(refractoryActive, 1, 1, i, j) .= 1
|
||||
view(refractoryInactive, 1, 1, i, j) .= 0
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# function frobenius_distance(A, B)
|
||||
# # Check if the matrices have the same size
|
||||
# if size(A) != size(B)
|
||||
# error("The matrices must have the same size")
|
||||
# end
|
||||
# # Initialize the distance to zero
|
||||
# distance = 0.0
|
||||
# # Loop over the elements of the matrices and add the squared differences
|
||||
# for i in 1:size(A, 1)
|
||||
# for j in 1:size(A, 2)
|
||||
# distance += (A[i, j] - B[i, j])^2
|
||||
# end
|
||||
# end
|
||||
# # Return the square root of the distance
|
||||
# return sqrt(distance)
|
||||
# end
|
||||
|
||||
function addNewSynapticConn!(mask::AbstractArray{<:Any}, x::Number, wRec::AbstractArray{<:Any},
|
||||
counter::AbstractArray{<:Any}, n=0;
|
||||
rng::AbstractRNG=MersenneTwister(1234))
|
||||
# println("mask ", mask, size(mask))
|
||||
# println("")
|
||||
# println("x ", x, size(x))
|
||||
# println("")
|
||||
# println("wRec ", wRec, size(wRec))
|
||||
# println("")
|
||||
# println("counter ", counter, size(counter))
|
||||
# println("")
|
||||
# println("n ", n, size(n))
|
||||
# println("")
|
||||
|
||||
# check if mask and wRec have the same size
|
||||
if size(mask) != size(wRec)
|
||||
error("mask and wRec must have the same size")
|
||||
end
|
||||
# get the indices of elements in mask that equal x
|
||||
indices = findall(x -> x == x, mask)
|
||||
alreadySub = findall(x -> x != 0, wRec) # get already subscribe
|
||||
setdiff!(indices, alreadySub) # remove already sub conn from pool
|
||||
|
||||
remaining = 0
|
||||
if n == 0 || n > length(indices)
|
||||
remaining = n - length(indices)
|
||||
n = length(indices)
|
||||
end
|
||||
|
||||
# shuffle the indices using the rng function
|
||||
shuffle!(rng, indices)
|
||||
# select the first n indices
|
||||
n > length(indices) ? println(">>> ", total_x_tobeReplced) : nothing
|
||||
selected = indices[1:n]
|
||||
# replace the elements in wRec at the selected positions with a
|
||||
for i in selected
|
||||
wRec[i] = rand(0.01:0.01:0.1)
|
||||
if counter !== nothing
|
||||
counter[i] = 0 # counting start from 0
|
||||
end
|
||||
end
|
||||
# error("DEBUG addNewSynapticConn!")
|
||||
return remaining
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
end # module
|
||||
443
previousVersion/0.0.8/src/type.jl
Normal file
443
previousVersion/0.0.8/src/type.jl
Normal file
@@ -0,0 +1,443 @@
|
||||
module type
|
||||
|
||||
export
|
||||
# struct
|
||||
kfn_1,
|
||||
|
||||
# function
|
||||
random_wRec
|
||||
|
||||
using Random, GeneralUtils
|
||||
|
||||
#------------------------------------------------------------------------------------------------100
|
||||
rng = MersenneTwister(1234)
|
||||
|
||||
abstract type Ironpen end
|
||||
abstract type knowledgeFn <: Ironpen end
|
||||
|
||||
#------------------------------------------------------------------------------------------------100
|
||||
|
||||
Base.@kwdef mutable struct kfn_1 <: knowledgeFn
|
||||
params::Union{Dict, Nothing} = nothing # store params of knowledgeFn itself for later use
|
||||
|
||||
timeStep::Union{AbstractArray, Nothing} = nothing
|
||||
learningStage::Union{AbstractArray, Nothing} = nothing # 0 inference, 1 start, 2 during, 3 end learning
|
||||
inputSize::Union{AbstractArray, Nothing} = nothing
|
||||
zit::Union{AbstractArray, Nothing} = nothing # 3D activation matrix
|
||||
zitCumulative::Union{AbstractArray, Nothing} = nothing
|
||||
exInType::Union{AbstractArray, Nothing} = nothing
|
||||
modelError::Union{AbstractArray, Nothing} = nothing # store RSNN error
|
||||
outputError::Union{AbstractArray, Nothing} = nothing # store output neurons error
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# LIF Neurons #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# a projection of kfn.zit into lif dimension for broadcasting later)
|
||||
lif_zit::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
# main variables according to papers
|
||||
lif_wRec::Union{AbstractArray, Nothing} = nothing
|
||||
lif_vt::Union{AbstractArray, Nothing} = nothing
|
||||
lif_vth::Union{AbstractArray, Nothing} = nothing
|
||||
lif_vRest::Union{AbstractArray, Nothing} = nothing
|
||||
lif_zt::Union{AbstractArray, Nothing} = nothing
|
||||
lif_zt4d::Union{AbstractArray, Nothing} = nothing
|
||||
lif_refractoryCounter::Union{AbstractArray, Nothing} = nothing
|
||||
lif_refractoryDuration::Union{AbstractArray, Nothing} = nothing
|
||||
lif_alpha::Union{AbstractArray, Nothing} = nothing
|
||||
lif_delta::Union{AbstractFloat, Nothing} = nothing
|
||||
lif_tau_m::Union{AbstractFloat, Nothing} = nothing
|
||||
lif_phi::Union{AbstractArray, Nothing} = nothing
|
||||
lif_epsilonRec::Union{AbstractArray, Nothing} = nothing
|
||||
lif_eRec::Union{AbstractArray, Nothing} = nothing
|
||||
lif_eta::Union{AbstractArray, Nothing} = nothing
|
||||
lif_gammaPd::Union{AbstractArray, Nothing} = nothing
|
||||
lif_wRecChange::Union{AbstractArray, Nothing} = nothing
|
||||
lif_error::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
lif_firingCounter::Union{AbstractArray, Nothing} = nothing
|
||||
lif_firingTargetFrequency::Union{AbstractArray, Nothing} = nothing
|
||||
lif_neuronInactivityCounter::Union{AbstractArray, Nothing} = nothing
|
||||
lif_synapticInactivityCounter::Union{AbstractArray, Nothing} = nothing
|
||||
lif_synapticConnectionNumber::Union{Int, Nothing} = nothing
|
||||
lif_synapticWChangeCounter::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
# pre-allocation array
|
||||
lif_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d
|
||||
lif_recSignal::Union{AbstractArray, Nothing} = nothing
|
||||
lif_exInType::Union{AbstractArray, Nothing} = nothing
|
||||
# lif_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing
|
||||
# lif_vt_diff_vth::Union{AbstractArray, Nothing} = nothing
|
||||
# lif_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing
|
||||
# lif_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing
|
||||
# lif_phiActivation::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# ALIF Neurons #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
alif_zit::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
alif_wRec::Union{AbstractArray, Nothing} = nothing
|
||||
alif_vt::Union{AbstractArray, Nothing} = nothing
|
||||
alif_vth::Union{AbstractArray, Nothing} = nothing
|
||||
alif_vRest::Union{AbstractArray, Nothing} = nothing
|
||||
alif_zt::Union{AbstractArray, Nothing} = nothing
|
||||
alif_zt4d::Union{AbstractArray, Nothing} = nothing
|
||||
alif_refractoryCounter::Union{AbstractArray, Nothing} = nothing
|
||||
alif_refractoryDuration::Union{AbstractArray, Nothing} = nothing
|
||||
alif_alpha::Union{AbstractArray, Nothing} = nothing
|
||||
alif_delta::Union{AbstractFloat, Nothing} = nothing
|
||||
alif_tau_m::Union{AbstractFloat, Nothing} = nothing
|
||||
alif_phi::Union{AbstractArray, Nothing} = nothing
|
||||
alif_epsilonRec::Union{AbstractArray, Nothing} = nothing
|
||||
alif_eRec::Union{AbstractArray, Nothing} = nothing
|
||||
alif_eta::Union{AbstractArray, Nothing} = nothing
|
||||
alif_gammaPd::Union{AbstractArray, Nothing} = nothing
|
||||
alif_wRecChange::Union{AbstractArray, Nothing} = nothing
|
||||
alif_error::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
alif_firingCounter::Union{AbstractArray, Nothing} = nothing
|
||||
alif_firingTargetFrequency::Union{AbstractArray, Nothing} = nothing
|
||||
alif_neuronInactivityCounter::Union{AbstractArray, Nothing} = nothing
|
||||
alif_synapticInactivityCounter::Union{AbstractArray, Nothing} = nothing
|
||||
alif_synapticConnectionNumber::Union{Int, Nothing} = nothing
|
||||
alif_synapticWChangeCounter::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
# pre-allocation array
|
||||
alif_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d
|
||||
alif_recSignal::Union{AbstractArray, Nothing} = nothing
|
||||
alif_exInType::Union{AbstractArray, Nothing} = nothing
|
||||
# alif_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing
|
||||
# alif_vt_diff_vth::Union{AbstractArray, Nothing} = nothing
|
||||
# alif_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing
|
||||
# alif_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing
|
||||
# alif_phiActivation::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
# alif specific variables
|
||||
alif_epsilonRecA::Union{AbstractArray, Nothing} = nothing
|
||||
alif_avth::Union{AbstractArray, Nothing} = nothing
|
||||
alif_a::Union{AbstractArray, Nothing} = nothing # threshold adaptation
|
||||
alif_beta::Union{AbstractArray, Nothing} = nothing # β, constant, value from paper
|
||||
alif_rho::Union{AbstractArray, Nothing} = nothing # ρ, threshold adaptation decay factor
|
||||
alif_tau_a::Union{AbstractFloat, Nothing} = nothing # τ_a, adaption time constant in millisecond
|
||||
|
||||
# alif specific pre-allocation array
|
||||
# alif_phi_x_epsilonRec::Union{AbstractArray, Nothing} = nothing
|
||||
# alif_phi_x_beta::Union{AbstractArray, Nothing} = nothing
|
||||
# alif_rho_diff_phi_x_beta::Union{AbstractArray, Nothing} = nothing
|
||||
# alif_rho_div_phi_x_beta_x_epsilonRecA::Union{AbstractArray, Nothing} = nothing
|
||||
# alif_beta_x_a::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# Output Neurons #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# output neuron is based on LIF
|
||||
on_zit::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
# main variables according to papers
|
||||
on_wOut::Union{AbstractArray, Nothing} = nothing # wOut is wRec, just use the name from paper
|
||||
on_vt::Union{AbstractArray, Nothing} = nothing
|
||||
on_vth::Union{AbstractArray, Nothing} = nothing
|
||||
on_vRest::Union{AbstractArray, Nothing} = nothing
|
||||
on_zt::Union{AbstractArray, Nothing} = nothing
|
||||
on_zt4d::Union{AbstractArray, Nothing} = nothing
|
||||
on_refractoryCounter::Union{AbstractArray, Nothing} = nothing
|
||||
on_refractoryDuration::Union{AbstractArray, Nothing} = nothing
|
||||
on_alpha::Union{AbstractArray, Nothing} = nothing
|
||||
on_delta::Union{AbstractFloat, Nothing} = nothing
|
||||
on_tau_m::Union{AbstractFloat, Nothing} = nothing
|
||||
on_phi::Union{AbstractArray, Nothing} = nothing
|
||||
on_epsilonRec::Union{AbstractArray, Nothing} = nothing
|
||||
on_eRec::Union{AbstractArray, Nothing} = nothing
|
||||
on_eta::Union{AbstractArray, Nothing} = nothing
|
||||
on_gammaPd::Union{AbstractArray, Nothing} = nothing
|
||||
on_wOutChange::Union{AbstractArray, Nothing} = nothing
|
||||
on_error::Union{AbstractArray, Nothing} = nothing
|
||||
on_subscription::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
on_firingCounter::Union{AbstractArray, Nothing} = nothing
|
||||
|
||||
# pre-allocation array
|
||||
on_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d
|
||||
on_recSignal::Union{AbstractArray, Nothing} = nothing
|
||||
# on_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing
|
||||
# on_vt_diff_vth::Union{AbstractArray, Nothing} = nothing
|
||||
# on_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing
|
||||
# on_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing
|
||||
# on_phiActivation::Union{AbstractArray, Nothing} = nothing
|
||||
end
|
||||
|
||||
# outer constructor
|
||||
function kfn_1(params::Dict; device=cpu)
|
||||
kfn = kfn_1()
|
||||
kfn.params = params
|
||||
kfn.timeStep = [0] |> device
|
||||
kfn.learningStage = [0] |> device
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# initialize activation matrix #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# row*col is a 2D matrix represent all RSNN activation
|
||||
row, signal_col, batch = kfn.params[:inputPort][:signal][:numbers] # z-axis represent signal batch number
|
||||
|
||||
kfn.inputSize = [row, signal_col] |> device
|
||||
lif_col = kfn.params[:computeNeuron][:lif][:numbers][2]
|
||||
alif_col = kfn.params[:computeNeuron][:alif][:numbers][2]
|
||||
|
||||
col = signal_col + lif_col + alif_col
|
||||
|
||||
# activation matrix
|
||||
kfn.zit = zeros(row, col, batch) |> device
|
||||
kfn.zitCumulative = (similar(kfn.zit) .= 0)
|
||||
kfn.modelError = zeros(1) |> device
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# LIF config #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# In 3D LIF matrix, z-axis represent each neuron while each 2D slice represent that neuron's
|
||||
# synaptic subscription to other neurons (via activation matrix)
|
||||
lif_n = kfn.params[:computeNeuron][:lif][:numbers][1] * kfn.params[:computeNeuron][:lif][:numbers][2]
|
||||
|
||||
# subscription
|
||||
synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent]
|
||||
kfn.lif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100))
|
||||
w = random_wRec(row, col, lif_n, kfn.lif_synapticConnectionNumber)
|
||||
|
||||
# project 3D w into 4D kfn.lif_wRec (row, col, n, batch)
|
||||
kfn.lif_wRec = reshape(w, (row, col, lif_n, 1)) .* ones(row, col, lif_n, batch) |> device
|
||||
kfn.lif_zit = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_vt = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_vth = (similar(kfn.lif_wRec) .= 1)
|
||||
kfn.lif_vRest = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_zt = zeros(1, 1, lif_n, batch) |> device
|
||||
kfn.lif_zt4d = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_refractoryCounter = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_refractoryDuration = (similar(kfn.lif_wRec) .= 3)
|
||||
kfn.lif_delta = 1.0
|
||||
kfn.lif_tau_m = 20.0
|
||||
kfn.lif_alpha = (similar(kfn.lif_wRec) .= (exp(-kfn.lif_delta / kfn.lif_tau_m)))
|
||||
kfn.lif_phi = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_epsilonRec = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_eRec = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_eta = (similar(kfn.lif_wRec) .= 0.01)
|
||||
kfn.lif_gammaPd = (similar(kfn.lif_wRec) .= 0.3)
|
||||
kfn.lif_wRecChange = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_error = (similar(kfn.lif_wRec) .= 0)
|
||||
|
||||
kfn.lif_firingCounter = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_firingTargetFrequency = (similar(kfn.lif_wRec) .= 0.1)
|
||||
kfn.lif_neuronInactivityCounter = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_synapticInactivityCounter = Array(similar(kfn.lif_wRec) .= -0.99) # -9 for non-sub conn
|
||||
mask = Array((!iszero).(kfn.lif_wRec))
|
||||
# initial value subscribed conn, synapticInactivityCounter range -10000 to +10000
|
||||
GeneralUtils.replace_elements!(mask, 1, kfn.lif_synapticInactivityCounter, 0)
|
||||
kfn.lif_synapticInactivityCounter = kfn.lif_synapticInactivityCounter |> device
|
||||
kfn.lif_synapticWChangeCounter = Array(similar(kfn.lif_wRec) .= -0.99) # -9 for non-sub conn
|
||||
mask = Array((!iszero).(kfn.lif_wRec))
|
||||
# initial value subscribed conn, synapticInactivityCounter range -10000 to +10000
|
||||
GeneralUtils.replace_elements!(mask, 1, kfn.lif_synapticWChangeCounter, 1.0)
|
||||
kfn.lif_synapticWChangeCounter = kfn.lif_synapticWChangeCounter |> device
|
||||
|
||||
kfn.lif_arrayProjection4d = (similar(kfn.lif_wRec) .= 1)
|
||||
kfn.lif_recSignal = (similar(kfn.lif_wRec) .= 0)
|
||||
kfn.lif_exInType = (similar(kfn.lif_wRec) .= 0)
|
||||
# kfn.lif_decayed_epsilonRec = (similar(kfn.lif_wRec) .= 0)
|
||||
# kfn.lif_vt_diff_vth = (similar(kfn.lif_wRec) .= 0)
|
||||
# kfn.lif_vt_diff_vth_div_vth = (similar(kfn.lif_wRec) .= 0)
|
||||
# kfn.lif_gammaPd_div_vth = (similar(kfn.lif_wRec) .= 0)
|
||||
# kfn.lif_phiActivation = (similar(kfn.lif_wRec) .= 0)
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# ALIF config #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
alif_n = kfn.params[:computeNeuron][:alif][:numbers][1] * kfn.params[:computeNeuron][:alif][:numbers][2]
|
||||
|
||||
# subscription
|
||||
synapticConnectionPercent = kfn.params[:computeNeuron][:alif][:params][:synapticConnectionPercent]
|
||||
kfn.alif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100))
|
||||
w = random_wRec(row, col, alif_n, kfn.alif_synapticConnectionNumber)
|
||||
|
||||
# project 3D w into 4D kfn.alif_wRec
|
||||
kfn.alif_wRec = reshape(w, (row, col, alif_n, 1)) .* ones(row, col, alif_n, batch) |> device
|
||||
kfn.alif_zit = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_vt = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_vth = (similar(kfn.alif_wRec) .= 1)
|
||||
kfn.alif_vRest = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_zt = zeros(1, 1, alif_n, batch) |> device
|
||||
kfn.alif_zt4d = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_refractoryCounter = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_refractoryDuration = (similar(kfn.alif_wRec) .= 3)
|
||||
kfn.alif_delta = 1.0
|
||||
kfn.alif_tau_m = 20.0
|
||||
kfn.alif_alpha = (similar(kfn.alif_wRec) .= (exp(-kfn.alif_delta / kfn.alif_tau_m)))
|
||||
kfn.alif_phi = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_epsilonRec = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_eRec = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_eta = (similar(kfn.alif_wRec) .= 0.01)
|
||||
kfn.alif_gammaPd = (similar(kfn.alif_wRec) .= 0.3)
|
||||
kfn.alif_wRecChange = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_error = (similar(kfn.alif_wRec) .= 0)
|
||||
|
||||
kfn.alif_firingCounter = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_firingTargetFrequency = (similar(kfn.alif_wRec) .= 0.1)
|
||||
kfn.alif_neuronInactivityCounter = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_synapticInactivityCounter = Array(similar(kfn.alif_wRec) .= -0.99) # -9 for non-sub conn
|
||||
mask = Array((!iszero).(kfn.alif_wRec))
|
||||
# initial value subscribed conn, synapticInactivityCounter range -10000 to +10000
|
||||
GeneralUtils.replace_elements!(mask, 1, kfn.alif_synapticInactivityCounter, 0)
|
||||
kfn.alif_synapticInactivityCounter = kfn.alif_synapticInactivityCounter |> device
|
||||
kfn.alif_synapticWChangeCounter = Array(similar(kfn.alif_wRec) .= -0.99) # -9 for non-sub conn
|
||||
mask = Array((!iszero).(kfn.alif_wRec))
|
||||
# initial value subscribed conn, synapticInactivityCounter range -10000 to +10000
|
||||
GeneralUtils.replace_elements!(mask, 1, kfn.alif_synapticWChangeCounter, 1.0)
|
||||
kfn.alif_synapticWChangeCounter = kfn.alif_synapticWChangeCounter |> device
|
||||
|
||||
kfn.alif_arrayProjection4d = (similar(kfn.alif_wRec) .= 1)
|
||||
kfn.alif_recSignal = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_exInType = (similar(kfn.alif_wRec) .= 0)
|
||||
# kfn.alif_decayed_epsilonRec = (similar(kfn.alif_wRec) .= 0)
|
||||
# kfn.alif_vt_diff_vth = (similar(kfn.alif_wRec) .= 0)
|
||||
# kfn.alif_vt_diff_vth_div_vth = (similar(kfn.alif_wRec) .= 0)
|
||||
# kfn.alif_gammaPd_div_vth = (similar(kfn.alif_wRec) .= 0)
|
||||
# kfn.alif_phiActivation = (similar(kfn.alif_wRec) .= 0)
|
||||
|
||||
# alif specific variables
|
||||
kfn.alif_epsilonRecA = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_avth = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_a = (similar(kfn.alif_wRec) .= 0)
|
||||
kfn.alif_beta = (similar(kfn.alif_wRec) .= 0.07)
|
||||
kfn.alif_tau_a = 800.0
|
||||
kfn.alif_rho = (similar(kfn.alif_wRec) .= (exp(-kfn.alif_delta / kfn.alif_tau_a))) |> device
|
||||
# kfn.alif_phi_x_epsilonRec = (similar(kfn.alif_wRec) .= 0)
|
||||
# kfn.alif_phi_x_beta = (similar(kfn.alif_wRec) .= 0)
|
||||
# kfn.alif_rho_diff_phi_x_beta = (similar(kfn.alif_wRec) .= 0)
|
||||
# kfn.alif_rho_div_phi_x_beta_x_epsilonRecA = (similar(kfn.alif_wRec) .= 0)
|
||||
# kfn.alif_beta_x_a = (similar(kfn.alif_wRec) .= 0)
|
||||
|
||||
# ---------------------------------------------------------------------------- #
|
||||
# output config #
|
||||
# ---------------------------------------------------------------------------- #
|
||||
n = kfn.params[:outputPort][:numbers][1] * kfn.params[:outputPort][:numbers][2]
|
||||
|
||||
# subscription
|
||||
w = zeros(row, col, n)
|
||||
synapticConnectionPercent = kfn.params[:outputPort][:params][:synapticConnectionPercent]
|
||||
subable = size(kfn.lif_wRec, 3) + size(kfn.alif_wRec, 3) # sub to lif, alif only
|
||||
synapticConnection = Int(floor(subable * synapticConnectionPercent/100))
|
||||
for slice in eachslice(w, dims=3) # each slice is a neuron
|
||||
startInd = row*col - subable + 1 # e.g. 100(row*col) - 50(subable) = 50 -> startInd = 51
|
||||
# pool must contain only lif, alif neurons
|
||||
pool = shuffle!([startInd:row*col...])[1:synapticConnection]
|
||||
for i in pool
|
||||
slice[i] = rand() # assign weight to synaptic connection. /10 to start small,
|
||||
# otherwise RSNN's vt Usually stay negative (-)
|
||||
end
|
||||
end
|
||||
|
||||
# 10% of neuron connection should be enough to start to make neuron fires
|
||||
should_be_avg_weight = 1 / (0.1 * n)
|
||||
w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight
|
||||
|
||||
# project 3D w into 4D kfn.lif_wOut (row, col, n, batch)
|
||||
kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device
|
||||
kfn.on_zit = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_vt = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_vth = (similar(kfn.on_wOut) .= 1)
|
||||
kfn.on_vRest = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_zt = zeros(1, 1, n, batch) |> device
|
||||
kfn.on_zt4d = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_refractoryCounter = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_refractoryDuration = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_delta = 1.0
|
||||
kfn.on_tau_m = 20.0
|
||||
kfn.on_alpha = (similar(kfn.on_wOut) .= (exp(-kfn.on_delta / kfn.on_tau_m)))
|
||||
kfn.on_phi = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_epsilonRec = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_eRec = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_eta = (similar(kfn.on_wOut) .= 0.01)
|
||||
kfn.on_gammaPd = (similar(kfn.on_wOut) .= 0.3)
|
||||
kfn.on_wOutChange = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_error = (similar(kfn.on_wOut) .= 0)
|
||||
kfn.on_subscription = (GeneralUtils.isNotEqual.(kfn.on_wOut, 0)) |> device
|
||||
|
||||
kfn.on_firingCounter = (similar(kfn.on_wOut) .= 0)
|
||||
|
||||
kfn.on_arrayProjection4d = (similar(kfn.on_wOut) .= 1)
|
||||
kfn.on_recSignal = (similar(kfn.on_wOut) .= 0)
|
||||
|
||||
kfn.outputError = zeros(n, batch) |> device
|
||||
totalComputeNeurons = lif_n + alif_n
|
||||
inhabitoryNeurons = Int(floor(totalComputeNeurons * 30/100))
|
||||
mask1 = ones(row, signal_col)
|
||||
mask2 = GeneralUtils.multiply_random_elements(ones(row, lif_col + alif_col),
|
||||
-1, inhabitoryNeurons, MersenneTwister(1234))
|
||||
kfn.exInType = cat(mask1, mask2, dims=2) |> device
|
||||
|
||||
return kfn
|
||||
end
|
||||
|
||||
function random_wRec(row, col, n, synapticConnectionNumber)
|
||||
# subscription
|
||||
w = zeros(row, col, n)
|
||||
|
||||
for slice in eachslice(w, dims=3)
|
||||
pool = shuffle!([1:row*col...])[1:synapticConnectionNumber]
|
||||
for i in pool
|
||||
slice[i] = rand(0.01:0.01:0.1) # assign weight to synaptic connection. /10 to start small,
|
||||
# otherwise RSNN's vt Usually stay negative (-)
|
||||
end
|
||||
end
|
||||
|
||||
# adjust weight so that RSNN fires small amount of neurons at the beginning to avoid overwhelming
|
||||
# all-fire situation. it also better than not-fire-at-all situation.
|
||||
avgWeight = sum(w)/length(w)
|
||||
w = w .* (0.01 / avgWeight) # adjust overall weight
|
||||
|
||||
return w #(row, col, n)
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
end # module
|
||||
Reference in New Issue
Block a user