ton/IronpenGPU
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

version 0.0.5

Browse Source
This commit is contained in:
ton
2023-08-26 16:16:59 +07:00
parent d4a075ae88
commit 3ca52b03f5
11 changed files with 599 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
previousVersion/0.0.05-alpha/src/snnUtil.jl
View File

@@ -1,87 +0,0 @@
module snnUtil
export refractoryStatus!
# using
#------------------------------------------------------------------------------------------------100
function refractoryStatus!(refractoryCounter, refractoryActive, refractoryInactive)
d1, d2, d3, d4 = size(refractoryCounter)
for j in 1:d4
for i in 1:d3
if refractoryCounter[1, 1, i, j] > 0 # inactive
view(refractoryActive, 1, 1, i, j) .= 0
view(refractoryInactive, 1, 1, i, j) .= 1
else # active
view(refractoryActive, 1, 1, i, j) .= 1
view(refractoryInactive, 1, 1, i, j) .= 0
end
end
end
end
function frobenius_distance(A, B)
# Check if the matrices have the same size
if size(A) != size(B)
error("The matrices must have the same size")
end
# Initialize the distance to zero
distance = 0.0
# Loop over the elements of the matrices and add the squared differences
for i in 1:size(A, 1)
for j in 1:size(A, 2)
distance += (A[i, j] - B[i, j])^2
end
end
# Return the square root of the distance
return sqrt(distance)
end
end # module

0
previousVersion/0.0.05-alpha/Manifest.toml → previousVersion/0.0.5/Manifest.toml
View File

0
previousVersion/0.0.05-alpha/Project.toml → previousVersion/0.0.5/Project.toml
View File

32
previousVersion/0.0.05-alpha/main_gpu_1.jl → previousVersion/0.0.5/main_gpu_1.jl
View File

@@ -67,8 +67,8 @@ function generate_snn(filename::String, location::String)
output_portnumbers = (10, 1)
# 5000 neurons are maximum for 64GB memory i.e. 300 LIF : 200 ALIF
lif_neuron_number = (signalInput_portnumbers[1], 3) # CHANGE
alif_neuron_number = (signalInput_portnumbers[1], 2) # CHANGE from Allen Institute, ALIF is 20-40% of LIF
lif_neuron_number = (signalInput_portnumbers[1], 60) # CHANGE
alif_neuron_number = (signalInput_portnumbers[1], 40) # CHANGE from Allen Institute, ALIF is 20-40% of LIF
# totalNeurons = computeNeuronNumber + noise_portnumbers + signalInput_portnumbers
# totalInputPort = noise_portnumbers + signalInput_portnumbers
@@ -91,7 +91,7 @@ function generate_snn(filename::String, location::String)
# exert more force (larger w_out_change) to move neuron into direction that reduce error
# For example, model error from 7 to 2e6.
:synapticConnectionPercent => 50, # % coverage of total neurons in kfn
:synapticConnectionPercent => 20, # % coverage of total neurons in kfn
)
alif_neuron_params = Dict{Symbol, Any}(
@@ -114,14 +114,14 @@ function generate_snn(filename::String, location::String)
# From "Spike frequency adaptation supports network computations on temporally dispersed
# information"
:synapticConnectionPercent => 50, # % coverage of total neurons in kfn
:synapticConnectionPercent => 20, # % coverage of total neurons in kfn
)
linear_neuron_params = Dict{Symbol, Any}(
:type => "linearNeuron",
:v_th => 1.0, # neuron firing threshold (this value is treated as maximum bound if I use auto generate)
:tau_out => 100.0, # output time constant in millisecond.
:synapticConnectionPercent => 50, # % coverage of total neurons in kfn
:synapticConnectionPercent => 20, # % coverage of total neurons in kfn
# Good starting value is 1/50th of tau_a
# This is problem specific parameter.
# It controls how leaky the neuron is.
@@ -405,7 +405,7 @@ function train_snn(model, trainData, validateData, labelDict::Vector)
for (imgBatch, labels) in trainData # imgBatch (28, 28, 4) i.e. (row, col, batch)
stop == 3 ? break : false
# signal (10, 2, 784, 4) i.e. (row, col, timestep, batch)
signal = spikeGenerator(imgBatch, [0.05, 0.1, 0.2, 0.3, 0.5], noise=(true, 1, 1.0), copies=18)
signal = spikeGenerator(imgBatch, [0.05, 0.1, 0.2, 0.3, 0.5], noise=(true, 1, 0.1), copies=18)
if length(size(signal)) == 3
row, col, sequence = size(signal)
batch = 1
@@ -462,7 +462,6 @@ function train_snn(model, trainData, validateData, labelDict::Vector)
lif_wRecChange_cpu = model.lif_wRecChange |> cpu
# if sum(lif_wRecChange_cpu) != 0
# println("")
# lif_vt_cpu = model.lif_vt |> cpu
@@ -535,22 +534,23 @@ function train_snn(model, trainData, validateData, labelDict::Vector)
# commit learned weight only if the model answer incorrectly
finalAnswer_cpu = finalAnswer |> cpu
println("label $(labels[1]) finalAnswer $finalAnswer_cpu")
# println("label $(labels[1]) finalAnswer $finalAnswer_cpu")
max = isequal.(finalAnswer_cpu[:,1], maximum(finalAnswer_cpu[:,1]))
if sum(max) == 1 && findall(max)[1] -1 == labels[1]
finalAnswer_cpu = findall(max)[1] - 1
println("OK")
# println("label $(labels[1]) finalAnswer $finalAnswer_cpu")
println("label $(labels[1]) finalAnswer $finalAnswer_cpu CORRECT")
elseif sum(max) == 1 && findall(max)[1] -1 != labels[1]
finalAnswer = findall(max)[1] - 1
IronpenGPU.learn!(model)
println("LEARNING")
# println("label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING")
IronpenGPU.learn!(model, device)
println("label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING")
else
IronpenGPU.learn!(model)
println("LEARNING")
# println("epoch $epoch label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING")
IronpenGPU.learn!(model, device)
if sum(finalAnswer_cpu) > 1
println("epoch $epoch label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING")
else
println("epoch $epoch label $(labels[1]) finalAnswer ZERO answer LEARNING")
end
end
# error("DEBUG -> main $(Dates.now())")

0
previousVersion/0.0.05-alpha/src/IronpenGPU.jl → previousVersion/0.0.5/src/IronpenGPU.jl
View File

426
previousVersion/0.0.05-alpha/src/forward.jl → previousVersion/0.0.5/src/forward.jl
View File

@@ -18,7 +18,7 @@ function (kfn::kfn_1)(input::AbstractArray)
# what to do at the start of learning round
if view(kfn.learningStage, 1)[1] == 1
# reset learning params
kfn.zit_cumulative .= 0
kfn.zitCumulative .= 0
kfn.lif_vt .= 0
kfn.lif_wRecChange .= 0
@@ -118,7 +118,7 @@ function (kfn::kfn_1)(input::AbstractArray)
reshape(kfn.lif_zt, (size(input, 1), :, 1, size(input, 3))),
reshape(kfn.alif_zt, (size(input, 1), :, 1, size(input, 3))), dims=2)
kfn.zit .= reshape(_zit, (size(input, 1), :, size(input, 3)))
kfn.zit_cumulative .+= kfn.zit
kfn.zitCumulative .+= kfn.zit
# project 3D kfn zit into 4D on zit
i1, i2, i3, i4 = size(kfn.on_zit)
@@ -273,7 +273,7 @@ function lifForward( zit,
vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4]
# reset counter if neuron fires
neuronInactivityCounter[i1,i2,i3,i4] = 10000
neuronInactivityCounter[i1,i2,i3,i4] = 0
else
zt[i1,i2,i3,i4] = 0
neuronInactivityCounter[i1,i2,i3,i4] -= 1
@@ -291,7 +291,7 @@ function lifForward( zit,
# count synaptic inactivity
if !iszero(wRec[i1,i2,i3,i4]) # check if this is wRec subscription
if !iszero(zit[i1,i2,i3,i4]) # synapse is active, reset counter
synapticInactivityCounter[i1,i2,i3,i4] = 10000
synapticInactivityCounter[i1,i2,i3,i4] += 1
else # synapse is inactive, counting
synapticInactivityCounter[i1,i2,i3,i4] -= 1
end
@@ -456,7 +456,7 @@ function alifForward( zit,
firingCounter[i1,i2,i3,i4] += 1
vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4]
a[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] * a[i1,i2,i3,i4]) + 1
neuronInactivityCounter[i1,i2,i3,i4] = 10000
neuronInactivityCounter[i1,i2,i3,i4] = 0
else
zt[i1,i2,i3,i4] = 0
a[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] * a[i1,i2,i3,i4])
@@ -478,7 +478,7 @@ function alifForward( zit,
# count synaptic inactivity
if !iszero(wRec[i1,i2,i3,i4]) # check if this is wRec subscription
if !iszero(zit[i1,i2,i3,i4]) # synapse is active, reset counter
synapticInactivityCounter[i1,i2,i3,i4] = 10000
synapticInactivityCounter[i1,i2,i3,i4] += 1
else # synapse is inactive, counting
synapticInactivityCounter[i1,i2,i3,i4] -= 1
end
@@ -612,238 +612,238 @@ function onForward( zit,
return nothing
end
function lifForward(kfn_zit::Array{T},
zit::Array{T},
wRec::Array{T},
vt0::Array{T},
vt1::Array{T},
vth::Array{T},
vRest::Array{T},
zt1::Array{T},
alpha::Array{T},
phi::Array{T},
epsilonRec::Array{T},
refractoryCounter::Array{T},
refractoryDuration::Array{T},
gammaPd::Array{T},
firingCounter::Array{T},
arrayProjection4d::Array{T},
recSignal::Array{T},
decayed_vt0::Array{T},
decayed_epsilonRec::Array{T},
vt1_diff_vth::Array{T},
vt1_diff_vth_div_vth::Array{T},
gammaPd_div_vth::Array{T},
phiActivation::Array{T},
) where T<:Number
# function lifForward(kfn_zit::Array{T},
# zit::Array{T},
# wRec::Array{T},
# vt0::Array{T},
# vt1::Array{T},
# vth::Array{T},
# vRest::Array{T},
# zt1::Array{T},
# alpha::Array{T},
# phi::Array{T},
# epsilonRec::Array{T},
# refractoryCounter::Array{T},
# refractoryDuration::Array{T},
# gammaPd::Array{T},
# firingCounter::Array{T},
# arrayProjection4d::Array{T},
# recSignal::Array{T},
# decayed_vt0::Array{T},
# decayed_epsilonRec::Array{T},
# vt1_diff_vth::Array{T},
# vt1_diff_vth_div_vth::Array{T},
# gammaPd_div_vth::Array{T},
# phiActivation::Array{T},
# ) where T<:Number
# project 3D kfn zit into 4D lif zit
i1, i2, i3, i4 = size(alif_wRec)
lif_zit .= reshape(kfn_zit, (i1, i2, 1, i4)) .* lif_arrayProjection4d
# # project 3D kfn zit into 4D lif zit
# i1, i2, i3, i4 = size(alif_wRec)
# lif_zit .= reshape(kfn_zit, (i1, i2, 1, i4)) .* lif_arrayProjection4d
for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
@. @views refractoryCounter[:,:,i,j] -= 1
@. @views zt1[:,:,i,j] = 0
@. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@. @views phi[:,:,i,j] = 0
# for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
# if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
# @. @views refractoryCounter[:,:,i,j] -= 1
# @. @views zt1[:,:,i,j] = 0
# @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
# @. @views phi[:,:,i,j] = 0
# compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
else # refractory period is inactive
@. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j]
@. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
# # compute epsilonRec
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
# else # refractory period is inactive
# @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j]
# @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
# @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j]))
@. @views zt1[:,:,i,j] = 1
@. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
@. @views firingCounter[:,:,i,j] += 1
@. @views vt1[:,:,i,j] = vRest[:,:,i,j]
else
@. @views zt1[:,:,i,j] = 0
end
# if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j]))
# @. @views zt1[:,:,i,j] = 1
# @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
# @. @views firingCounter[:,:,i,j] += 1
# @. @views vt1[:,:,i,j] = vRest[:,:,i,j]
# else
# @. @views zt1[:,:,i,j] = 0
# end
# compute phi, there is a difference from alif formula
@. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
@. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
@. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
@view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
@. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# # compute phi, there is a difference from alif formula
# @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
# @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
# @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
# @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
# @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
end
end
end
# # compute epsilonRec
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
# end
# end
# end
function alifForward(zit::Array{T},
wRec::Array{T},
vt0::Array{T},
vt1::Array{T},
vth::Array{T},
vRest::Array{T},
zt1::Array{T},
alpha::Array{T},
phi::Array{T},
epsilonRec::Array{T},
refractoryCounter::Array{T},
refractoryDuration::Array{T},
gammaPd::Array{T},
firingCounter::Array{T},
recSignal::Array{T},
decayed_vt0::Array{T},
decayed_epsilonRec::Array{T},
vt1_diff_vth::Array{T},
vt1_diff_vth_div_vth::Array{T},
gammaPd_div_vth::Array{T},
phiActivation::Array{T},
# function alifForward(zit::Array{T},
# wRec::Array{T},
# vt0::Array{T},
# vt1::Array{T},
# vth::Array{T},
# vRest::Array{T},
# zt1::Array{T},
# alpha::Array{T},
# phi::Array{T},
# epsilonRec::Array{T},
# refractoryCounter::Array{T},
# refractoryDuration::Array{T},
# gammaPd::Array{T},
# firingCounter::Array{T},
# recSignal::Array{T},
# decayed_vt0::Array{T},
# decayed_epsilonRec::Array{T},
# vt1_diff_vth::Array{T},
# vt1_diff_vth_div_vth::Array{T},
# gammaPd_div_vth::Array{T},
# phiActivation::Array{T},
epsilonRecA::Array{T},
avth::Array{T},
a::Array{T},
beta::Array{T},
rho::Array{T},
phi_x_epsilonRec::Array{T},
phi_x_beta::Array{T},
rho_diff_phi_x_beta::Array{T},
rho_div_phi_x_beta_x_epsilonRecA::Array{T},
beta_x_a::Array{T},
) where T<:Number
# epsilonRecA::Array{T},
# avth::Array{T},
# a::Array{T},
# beta::Array{T},
# rho::Array{T},
# phi_x_epsilonRec::Array{T},
# phi_x_beta::Array{T},
# rho_diff_phi_x_beta::Array{T},
# rho_div_phi_x_beta_x_epsilonRecA::Array{T},
# beta_x_a::Array{T},
# ) where T<:Number
for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
@. @views refractoryCounter[:,:,i,j] -= 1
@. @views zt1[:,:,i,j] = 0
@. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@. @views phi[:,:,i,j] = 0
@. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
# for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
# if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
# @. @views refractoryCounter[:,:,i,j] -= 1
# @. @views zt1[:,:,i,j] = 0
# @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
# @. @views phi[:,:,i,j] = 0
# @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
# compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
# # compute epsilonRec
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
# compute epsilonRecA
@. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j]
@. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j]
@. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j]
@. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j]
# # compute epsilonRecA
# @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j]
# @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j]
# @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j]
# @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j]
# @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j]
# compute avth
@. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j]
@. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j]
# # compute avth
# @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j]
# @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j]
else # refractory period is inactive
@. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j]
@. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
# else # refractory period is inactive
# @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j]
# @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
# @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
# compute avth
@. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j]
@. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j]
# # compute avth
# @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j]
# @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j]
if sum(@view(vt1[:,:,i,j])) > sum(@view(avth[:,:,i,j]))
@. @views zt1[:,:,i,j] = 1
@. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
@. @views firingCounter[:,:,i,j] += 1
@. @views vt1[:,:,i,j] = vRest[:,:,i,j]
@. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
@. @views a[:,:,i,j] = a[:,:,i,j] += 1
else
@. @views zt1[:,:,i,j] = 0
@. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
end
# if sum(@view(vt1[:,:,i,j])) > sum(@view(avth[:,:,i,j]))
# @. @views zt1[:,:,i,j] = 1
# @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
# @. @views firingCounter[:,:,i,j] += 1
# @. @views vt1[:,:,i,j] = vRest[:,:,i,j]
# @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
# @. @views a[:,:,i,j] = a[:,:,i,j] += 1
# else
# @. @views zt1[:,:,i,j] = 0
# @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
# end
# compute phi, there is a difference from alif formula
@. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
@. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
@. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
@view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
@. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# # compute phi, there is a difference from alif formula
# @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
# @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
# @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
# @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
# @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
# # compute epsilonRec
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
# compute epsilonRecA
@. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j]
@. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j]
@. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j]
@. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j]
end
end
end
# # compute epsilonRecA
# @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j]
# @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j]
# @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j]
# @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j]
# @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j]
# end
# end
# end
function onForward(kfn_zit::Array{T},
zit::Array{T},
wOut::Array{T},
vt0::Array{T},
vt1::Array{T},
vth::Array{T},
vRest::Array{T},
zt1::Array{T},
alpha::Array{T},
phi::Array{T},
epsilonRec::Array{T},
refractoryCounter::Array{T},
refractoryDuration::Array{T},
gammaPd::Array{T},
firingCounter::Array{T},
arrayProjection4d::Array{T},
recSignal::Array{T},
decayed_vt0::Array{T},
decayed_epsilonRec::Array{T},
vt1_diff_vth::Array{T},
vt1_diff_vth_div_vth::Array{T},
gammaPd_div_vth::Array{T},
phiActivation::Array{T},
) where T<:Number
# function onForward(kfn_zit::Array{T},
# zit::Array{T},
# wOut::Array{T},
# vt0::Array{T},
# vt1::Array{T},
# vth::Array{T},
# vRest::Array{T},
# zt1::Array{T},
# alpha::Array{T},
# phi::Array{T},
# epsilonRec::Array{T},
# refractoryCounter::Array{T},
# refractoryDuration::Array{T},
# gammaPd::Array{T},
# firingCounter::Array{T},
# arrayProjection4d::Array{T},
# recSignal::Array{T},
# decayed_vt0::Array{T},
# decayed_epsilonRec::Array{T},
# vt1_diff_vth::Array{T},
# vt1_diff_vth_div_vth::Array{T},
# gammaPd_div_vth::Array{T},
# phiActivation::Array{T},
# ) where T<:Number
# project 3D kfn zit into 4D lif zit
zit .= reshape(kfn_zit,
(size(wOut, 1), size(wOut, 2), 1, size(wOut, 4))) .* arrayProjection4d
# # project 3D kfn zit into 4D lif zit
# zit .= reshape(kfn_zit,
# (size(wOut, 1), size(wOut, 2), 1, size(wOut, 4))) .* arrayProjection4d
for j in 1:size(wOut, 4), i in 1:size(wOut, 3) # compute along neurons axis of every batch
if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
@. @views refractoryCounter[:,:,i,j] -= 1
@. @views zt1[:,:,i,j] = 0
@. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@. @views phi[:,:,i,j] = 0
# for j in 1:size(wOut, 4), i in 1:size(wOut, 3) # compute along neurons axis of every batch
# if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
# @. @views refractoryCounter[:,:,i,j] -= 1
# @. @views zt1[:,:,i,j] = 0
# @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
# @. @views phi[:,:,i,j] = 0
# compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
else # refractory period is inactive
@. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wOut[:,:,i,j]
@. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
# # compute epsilonRec
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
# else # refractory period is inactive
# @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wOut[:,:,i,j]
# @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
# @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j]))
@. @views zt1[:,:,i,j] = 1
@. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
@. @views firingCounter[:,:,i,j] += 1
@. @views vt1[:,:,i,j] = vRest[:,:,i,j]
else
@. @views zt1[:,:,i,j] = 0
end
# if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j]))
# @. @views zt1[:,:,i,j] = 1
# @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
# @. @views firingCounter[:,:,i,j] += 1
# @. @views vt1[:,:,i,j] = vRest[:,:,i,j]
# else
# @. @views zt1[:,:,i,j] = 0
# end
# compute phi, there is a difference from alif formula
@. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
@. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
@. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
@view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
@. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# # compute phi, there is a difference from alif formula
# @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
# @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
# @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
# @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
# @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
end
end
end
# # compute epsilonRec
# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
# end
# end
# end

0
previousVersion/0.0.05-alpha/src/interface.jl → previousVersion/0.0.5/src/interface.jl
View File

192
previousVersion/0.0.05-alpha/src/learn.jl → previousVersion/0.0.5/src/learn.jl
View File

@@ -266,21 +266,25 @@ end
function learn!(kfn::kfn_1, device=cpu)
# lif learn
lifLearn!(kfn.lif_wRec,
kfn.lif_wRec, kfn.lif_neuronInactivityCounter, kfn.lif_synapticInactivityCounter =
lifLearn(kfn.lif_wRec,
kfn.lif_wRecChange,
kfn.lif_arrayProjection4d,
kfn.lif_neuronInactivityCounter,
kfn.lif_synapticInactivityCounter,
kfn.lif_synapticConnectionNumber,
kfn.zit_cumulative,
kfn.zitCumulative,
device)
# alif learn
alifLearn!(kfn.alif_wRec,
kfn.alif_wRec, kfn.alif_neuronInactivityCounter, kfn.alif_synapticInactivityCounter =
alifLearn(kfn.alif_wRec,
kfn.alif_wRecChange,
kfn.alif_arrayProjection4d,
kfn.alif_neuronInactivityCounter,
kfn.alif_synapticInactivityCounter,
kfn.alif_synapticConnectionNumber,
kfn.zit_cumulative,
kfn.zitCumulative,
device)
# on learn
@@ -295,58 +299,108 @@ function learn!(kfn::kfn_1, device=cpu)
# error("DEBUG -> kfn learn! $(Dates.now())")
end
function lifLearn!(wRec,
function lifLearn(wRec,
wRecChange,
arrayProjection4d,
inactivityCounter,
neuronInactivityCounter,
synapticInactivityCounter,
synapticConnectionNumber,
zit_cumulative,
zitCumulative,
device)
#WORKING - synapticInactivityCounter -10000 to 10000, weight change liquidity range from 1.0 to 0.1 respectively
# merge learning weight with average learning weight of all batch
wRec .+= (sum(wRecChange, dims=4) ./ (size(wRec, 4))) .* arrayProjection4d
arrayProjection4d_cpu = arrayProjection4d |> cpu
wRec_cpu = wRec |> cpu
wRec_cpu = wRec_cpu[:,:,:,1] # since every batch has the same neuron wRec, (row, col, n)
inactivityCounter_cpu = inactivityCounter |> cpu
inactivityCounter_cpu = inactivityCounter_cpu[:,:,:,1] # (row, col, n)
zit_cumulative_cpu = zit_cumulative |> cpu
zit_cumulative_cpu = zit_cumulative_cpu[:,:,1] # (row, col)
neuronInactivityCounter_cpu = neuronInactivityCounter |> cpu
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu[:,:,:,1] # (row, col, n)
synapticInactivityCounter_cpu = synapticInactivityCounter |> cpu
synapticInactivityCounter_cpu = synapticInactivityCounter_cpu[:,:,:,1]
zitCumulative_cpu = zitCumulative |> cpu
zitCumulative_cpu = zitCumulative_cpu[:,:,1] # (row, col)
# weak / negative synaptic connection will get randomed in neuroplasticity()
wRec_cpu = GeneralUtils.replaceBetween.(wRec_cpu, 0.0, 0.1, -1.0) # mark with -1.0
# synaptic connection that has no inactivity will get randomed in neuroplasticity()
GeneralUtils.replace_elements!(inactivityCounter_cpu, 0.0, wRec_cpu, -1.0)
# reset lif_inactivity elements to 10000
GeneralUtils.replace_elements!(inactivityCounter_cpu, 0.0, -9.0) # -9.0 is base value
# synaptic connection that has no activity will get randomed in neuroplasticity()
mask = isless.(synapticInactivityCounter_cpu, -10_000)
GeneralUtils.replace_elements!(mask, 1, wRec_cpu, -1.0)
# reset lif_inactivity elements to base value
GeneralUtils.replace_elements!(mask, 1, synapticInactivityCounter_cpu, 0.0)
#WORKING neuroplasticity
wRec_cpu = neuroplasticity(synapticConnectionNumber, zit_cumulative_cpu, wRec_cpu,
inactivityCounter_cpu)
error("DEBUG -> lifLearn! $(Dates.now())")
# #TODO send to device with correct dimension
# wRec = wRec |> device
# inactivityCounter = inactivityCounter_cpu |> device
# neuroplasticity, work on CPU side
wRec_cpu = neuroplasticity(synapticConnectionNumber,
zitCumulative_cpu,
wRec_cpu,
neuronInactivityCounter_cpu,
synapticInactivityCounter_cpu)
wRec_cpu = wRec_cpu .* arrayProjection4d_cpu
wRec = wRec_cpu |> device
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu .* arrayProjection4d_cpu
neuronInactivityCounter = neuronInactivityCounter_cpu |> device
synapticInactivityCounter_cpu = synapticInactivityCounter_cpu .* arrayProjection4d_cpu
synapticInactivityCounter = synapticInactivityCounter_cpu |> device
# error("DEBUG -> lifLearn! $(Dates.now())")
return wRec, neuronInactivityCounter, synapticInactivityCounter
end
function alifLearn!(wRec,
function alifLearn(wRec,
wRecChange,
arrayProjection4d,
inactivityCounter,
neuronInactivityCounter,
synapticInactivityCounter,
synapticConnectionNumber,
zit_cumulative,
zitCumulative,
device)
# merge learning weight with average learning weight
#WORKING - synapticInactivityCounter -10000 to 10000, weight change liquidity range from 1.0 to 0.1 respectively
# merge learning weight with average learning weight of all batch
wRec .+= (sum(wRecChange, dims=4) ./ (size(wRec, 4))) .* arrayProjection4d
arrayProjection4d_cpu = arrayProjection4d |> cpu
wRec_cpu = wRec |> cpu
wRec_cpu = wRec_cpu[:,:,:,1] # since every batch has the same neuron wRec, (row, col, n)
neuronInactivityCounter_cpu = neuronInactivityCounter |> cpu
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu[:,:,:,1] # (row, col, n)
synapticInactivityCounter_cpu = synapticInactivityCounter |> cpu
synapticInactivityCounter_cpu = synapticInactivityCounter_cpu[:,:,:,1]
zitCumulative_cpu = zitCumulative |> cpu
zitCumulative_cpu = zitCumulative_cpu[:,:,1] # (row, col)
# weak / negative synaptic connection will get randomed in neuroplasticity()
wRec .= GeneralUtils.replaceLessThan.(wRec, 0.01, 0.0)
wRec_cpu = GeneralUtils.replaceBetween.(wRec_cpu, 0.0, 0.1, -1.0) # mark with -1.0
#TODO synaptic strength
# synaptic connection that has no activity will get randomed in neuroplasticity()
mask = isless.(synapticInactivityCounter_cpu, -10_000)
GeneralUtils.replace_elements!(mask, 1, wRec_cpu, -1.0)
# reset alif_inactivity elements to base value
GeneralUtils.replace_elements!(mask, 1, synapticInactivityCounter_cpu, 0.0)
#TODO neuroplasticity
# neuroplasticity, work on CPU side
wRec_cpu = neuroplasticity(synapticConnectionNumber,
zitCumulative_cpu,
wRec_cpu,
neuronInactivityCounter_cpu,
synapticInactivityCounter_cpu)
wRec_cpu = wRec_cpu .* arrayProjection4d_cpu
wRec = wRec_cpu |> device
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu .* arrayProjection4d_cpu
neuronInactivityCounter = neuronInactivityCounter_cpu |> device
synapticInactivityCounter_cpu = synapticInactivityCounter_cpu .* arrayProjection4d_cpu
synapticInactivityCounter = synapticInactivityCounter_cpu |> device
# error("DEBUG -> alifLearn! $(Dates.now())")
return wRec, neuronInactivityCounter, synapticInactivityCounter
end
function onLearn!(wOut,
@@ -365,53 +419,67 @@ function onLearn!(wOut,
end
function neuroplasticity(synapticConnectionNumber,
zit_cumulative, # (row, col)
zitCumulative, # (row, col)
wRec, # (row, col, n)
inactivityCounter_cpu) # (row, col, n)
neuronInactivityCounter,
synapticInactivityCounter) # (row, col, n)
i1,i2,i3 = size(wRec)
# for each neuron, find total number of synaptic conn that should draw
# new connection to firing and non-firing neurons pool
subToFireNeuron_toBe = Int(floor(0.7 * synapticConnectionNumber))
subToNonFiringNeuron_toBe = synapticConnectionNumber - subToFireNeuron_toBe
#WORKING for each neuron, count how many synap already subscribed to firing-neurons
subToFireNeuron_current = sum((!iszero).(zit_cumulative .* wRec), dims=(1,2)) # (1, 1, n)
subToNonFiringNeuron_current = synapticConnectionNumber .- subToFireNeuron_current # (1, 1, n)
mask = (!iszero).(zit_cumulative) # mask of firing neurons = 1, non-firing = 0
# for each neuron, count how many synap already subscribed to firing-neurons
zw = zitCumulative .* wRec
subToFireNeuron_current = sum(GeneralUtils.isBetween.(zw, 0.0, 100.0), dims=(1,2)) # (1, 1, n)
zitMask = (!iszero).(zitCumulative) # zitMask of firing neurons = 1, non-firing = 0
projection = ones(i1,i2,i3)
mask = mask .* projection # (row, col, n)
zitMask = zitMask .* projection # (row, col, n)
totalNewConn = sum(isequal.(wRec, -1.0), dims=(1,2)) # count new conn mark (-1.0), (1, 1, n)
println("mask ", size(mask))
println("wRec ", size(wRec))
println("inactivityCounter_cpu ", size(inactivityCounter_cpu))
println("totalNeurons ", totalNewConn, size(totalNewConn))
error("DEBUG -> neuroplasticity $(Dates.now())")
# clear -1.0 marker
GeneralUtils.replace_elements!(wRec, -1.0, synapticInactivityCounter, -0.99)
GeneralUtils.replace_elements!(wRec, -1.0, 0.0) # -1.0 marker is no longer required
for i in 1:i3
if neuronInactivityCounter[1:1:i][1] < -10_000 # neuron die i.e. reset all weight
neuronInactivityCounter[:,:,i] .= 0 # reset
w = wRec(i1,i2,1,synapticConnectionNumber)
wRec[:,:,i] = w
a = similar(w) .= -0.99 # synapticConnectionNumber of this neuron
mask = (!iszero).(w)
GeneralUtils.replace_elements!(mask, 1, a, 0)
synapticInactivityCounter[:,:,i] = a
else
remaining = 0
if subToFireNeuron_current[1,1,i] < subToFireNeuron_toBe
toAddConn = subToFireNeuron_toBe - subToFireNeuron_current[1,1,i]
totalNewConn[1,1,i] = totalNewConn[1,1,i] - toAddConn
# add new conn to firing neurons pool
remaining = GeneralUtils.replace_elements(mask[:,:,i],
1,
wRecmask[:,:,i],
inactivityCounter_cpumask[:,:,i],
totalNewConn[:,:,i])
#TODO add new conn to non-firing neurons pool
remaining = addNewSynapticConn!(zitMask[:,:,i], 1,
@view(wRec[:,:,i]),
@view(synapticInactivityCounter[:,:,i]),
toAddConn)
totalNewConn[1,1,i] += remaining
end
# add new conn to non-firing neurons pool
remaining = addNewSynapticConn!(zitMask[:,:,i], 0,
@view(wRec[:,:,i]),
@view(synapticInactivityCounter[:,:,i]),
totalNewConn[1,1,i])
if remaining > 0 # final get-all round if somehow non-firing pool has not enough slot
remaining = addNewSynapticConn!(zitMask[:,:,i], 1,
@view(wRec[:,:,i]),
@view(synapticInactivityCounter[:,:,i]),
remaining)
end
end
end
newFiringConn = subToFireNeuron_toBe - subToFireNeuron_current
newFiringConn = newFiringConn > 0 ? newFiringConn : 0
newNonFiringConn = subToNonFiringNeuron_toBe - subToNonFiringNeuron_current
# error("DEBUG -> neuroplasticity $(Dates.now())")
return wRec
end
@@ -454,10 +522,6 @@ end

191
previousVersion/0.0.5/src/snnUtil.jl Normal file
View File

@@ -0,0 +1,191 @@
module snnUtil
export refractoryStatus!, addNewSynapticConn!
using Random
#------------------------------------------------------------------------------------------------100
function refractoryStatus!(refractoryCounter, refractoryActive, refractoryInactive)
d1, d2, d3, d4 = size(refractoryCounter)
for j in 1:d4
for i in 1:d3
if refractoryCounter[1, 1, i, j] > 0 # inactive
view(refractoryActive, 1, 1, i, j) .= 0
view(refractoryInactive, 1, 1, i, j) .= 1
else # active
view(refractoryActive, 1, 1, i, j) .= 1
view(refractoryInactive, 1, 1, i, j) .= 0
end
end
end
end
# function frobenius_distance(A, B)
# # Check if the matrices have the same size
# if size(A) != size(B)
# error("The matrices must have the same size")
# end
# # Initialize the distance to zero
# distance = 0.0
# # Loop over the elements of the matrices and add the squared differences
# for i in 1:size(A, 1)
# for j in 1:size(A, 2)
# distance += (A[i, j] - B[i, j])^2
# end
# end
# # Return the square root of the distance
# return sqrt(distance)
# end
function addNewSynapticConn!(mask::AbstractArray{<:Any}, x::Number, wRec::AbstractArray{<:Any},
counter::AbstractArray{<:Any}, n=0;
rng::AbstractRNG=MersenneTwister(1234))
# println("mask ", mask, size(mask))
# println("")
# println("x ", x, size(x))
# println("")
# println("wRec ", wRec, size(wRec))
# println("")
# println("counter ", counter, size(counter))
# println("")
# println("n ", n, size(n))
# println("")
total_x_tobeReplced = sum(isequal.(mask, x))
remaining = 0
if n == 0 || n > total_x_tobeReplced
remaining = n - total_x_tobeReplced
n = total_x_tobeReplced
end
# check if mask and wRec have the same size
if size(mask) != size(wRec)
error("mask and wRec must have the same size")
end
# get the indices of elements in mask that equal x
indices = findall(x -> x == x, mask)
alreadySub = findall(x -> x != 0, wRec) # get already subscribe
setdiff!(indices, alreadySub) # remove already sub conn from pool
# shuffle the indices using the rng function
shuffle!(rng, indices)
# select the first n indices
selected = indices[1:n]
# replace the elements in wRec at the selected positions with a
for i in selected
wRec[i] = 0.1 #rand(0.1:0.01:0.3)
if counter !== nothing
counter[i] = 0 # reset
end
end
# println("==================")
# println("mask ", mask, size(mask))
# println("")
# println("x ", x, size(x))
# println("")
# println("wRec ", wRec, size(wRec))
# println("")
# println("counter ", counter, size(counter))
# println("")
# println("n ", n, size(n))
# println("")
# error("DEBUG addNewSynapticConn!")
return remaining
end
# function addNewSynapticConn!(mask::AbstractArray{<:Any}, x::Number, A::AbstractArray{<:Any},
# A2::AbstractArray{<:Any}, n=0;
# rng::AbstractRNG=MersenneTwister(1234))
# # println("mask ", mask, size(mask))
# # println("")
# # println("x ", x, size(x))
# # println("")
# # println("A ", A, size(A))
# # println("")
# # println("A2 ", A2, size(A2))
# # println("")
# # println("n ", n, size(n))
# # println("")
# total_x_tobeReplced = sum(isequal.(mask, x))
# remaining = 0
# if n == 0 || n > total_x_tobeReplced
# remaining = n - total_x_tobeReplced
# n = total_x_tobeReplced
# end
# # check if mask and A have the same size
# if size(mask) != size(A)
# error("mask and A must have the same size")
# end
# # get the indices of elements in mask that equal x
# indices = findall(x -> x == x, mask)
# # shuffle the indices using the rng function
# shuffle!(rng, indices)
# # select the first n indices
# selected = indices[1:n]
# # replace the elements in A at the selected positions with a
# for i in selected
# A[i] = rand(0.1:0.01:0.3)
# if A2 !== nothing
# A2[i] = 10000
# end
# end
# # println("==================")
# # println("mask ", mask, size(mask))
# # println("")
# # println("x ", x, size(x))
# # println("")
# # println("A ", A, size(A))
# # println("")
# # println("A2 ", A2, size(A2))
# # println("")
# # println("n ", n, size(n))
# # println("")
# # error("DEBUG addNewSynapticConn!")
# return remaining
# end
end # module

74
previousVersion/0.0.05-alpha/src/type.jl → previousVersion/0.0.5/src/type.jl
View File

@@ -23,7 +23,7 @@ Base.@kwdef mutable struct kfn_1 <: knowledgeFn
learningStage::Union{AbstractArray, Nothing} = nothing # 0 inference, 1 start, 2 during, 3 end learning
inputSize::Union{AbstractArray, Nothing} = nothing
zit::Union{AbstractArray, Nothing} = nothing # 3D activation matrix
zit_cumulative::Union{AbstractArray, Nothing} = nothing
zitCumulative::Union{AbstractArray, Nothing} = nothing
exInType::Union{AbstractArray, Nothing} = nothing
modelError::Union{AbstractArray, Nothing} = nothing # store RSNN error
outputError::Union{AbstractArray, Nothing} = nothing # store output neurons error
@@ -185,7 +185,7 @@ function kfn_1(params::Dict; device=cpu)
# activation matrix
kfn.zit = zeros(row, col, batch) |> device
kfn.zit_cumulative = (similar(kfn.zit) .= 0)
kfn.zitCumulative = (similar(kfn.zit) .= 0)
kfn.modelError = zeros(1) |> device
# ---------------------------------------------------------------------------- #
@@ -196,20 +196,9 @@ function kfn_1(params::Dict; device=cpu)
lif_n = kfn.params[:computeNeuron][:lif][:numbers][1] * kfn.params[:computeNeuron][:lif][:numbers][2]
# subscription
w = zeros(row, col, lif_n)
synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent]
kfn.lif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100))
for slice in eachslice(w, dims=3)
pool = shuffle!([1:row*col...])[1:kfn.lif_synapticConnectionNumber]
for i in pool
slice[i] = rand() # assign weight to synaptic connection. /10 to start small,
# otherwise RSNN's vt Usually stay negative (-)
end
end
# 10% of neuron connection should be enough to start to make neuron fires
should_be_avg_weight = 1 / (0.1 * lif_n)
w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight
w = wRec(row, col, lif_n, kfn.lif_synapticConnectionNumber)
# project 3D w into 4D kfn.lif_wRec (row, col, n, batch)
kfn.lif_wRec = reshape(w, (row, col, lif_n, 1)) .* ones(row, col, lif_n, batch) |> device
@@ -234,10 +223,11 @@ function kfn_1(params::Dict; device=cpu)
kfn.lif_firingCounter = (similar(kfn.lif_wRec) .= 0)
kfn.lif_firingTargetFrequency = (similar(kfn.lif_wRec) .= 0.1)
kfn.lif_neuronInactivityCounter = (similar(kfn.lif_wRec) .= 10000)
kfn.lif_synapticInactivityCounter = Array(similar(kfn.lif_wRec) .= -9) # -9 for non-sub conn
kfn.lif_neuronInactivityCounter = (similar(kfn.lif_wRec) .= 0)
kfn.lif_synapticInactivityCounter = Array(similar(kfn.lif_wRec) .= -0.99) # -9 for non-sub conn
mask = Array((!iszero).(kfn.lif_wRec))
GeneralUtils.replace_elements!(mask, 1, kfn.lif_synapticInactivityCounter, 10000)
# initial value subscribed conn, synapticInactivityCounter range -10000 to +10000
GeneralUtils.replace_elements!(mask, 1, kfn.lif_synapticInactivityCounter, 0)
kfn.lif_synapticInactivityCounter = kfn.lif_synapticInactivityCounter |> device
kfn.lif_arrayProjection4d = (similar(kfn.lif_wRec) .= 1)
@@ -255,20 +245,9 @@ function kfn_1(params::Dict; device=cpu)
alif_n = kfn.params[:computeNeuron][:alif][:numbers][1] * kfn.params[:computeNeuron][:alif][:numbers][2]
# subscription
w = zeros(row, col, alif_n)
synapticConnectionPercent = kfn.params[:computeNeuron][:alif][:params][:synapticConnectionPercent]
kfn.alif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100))
for slice in eachslice(w, dims=3)
pool = shuffle!([1:row*col...])[1:kfn.alif_synapticConnectionNumber]
for i in pool
slice[i] = rand() # assign weight to synaptic connection. /10 to start small,
# otherwise RSNN's vt Usually stay negative (-)
end
end
# 10% of neuron connection should be enough to start to make neuron fires
should_be_avg_weight = 1 / (0.1 * alif_n)
w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight
w = wRec(row, col, alif_n, kfn.alif_synapticConnectionNumber)
# project 3D w into 4D kfn.alif_wRec
kfn.alif_wRec = reshape(w, (row, col, alif_n, 1)) .* ones(row, col, alif_n, batch) |> device
@@ -293,10 +272,11 @@ function kfn_1(params::Dict; device=cpu)
kfn.alif_firingCounter = (similar(kfn.alif_wRec) .= 0)
kfn.alif_firingTargetFrequency = (similar(kfn.alif_wRec) .= 0.1)
kfn.alif_neuronInactivityCounter = (similar(kfn.alif_wRec) .= 10000)
kfn.alif_synapticInactivityCounter = Array(similar(kfn.alif_wRec) .= -9) # -9 for non-sub conn
kfn.alif_neuronInactivityCounter = (similar(kfn.alif_wRec) .= 0)
kfn.alif_synapticInactivityCounter = Array(similar(kfn.alif_wRec) .= -0.99) # -9 for non-sub conn
mask = Array((!iszero).(kfn.alif_wRec))
GeneralUtils.replace_elements!(mask, 1, kfn.alif_synapticInactivityCounter, 10000)
# initial value subscribed conn, synapticInactivityCounter range -10000 to +10000
GeneralUtils.replace_elements!(mask, 1, kfn.alif_synapticInactivityCounter, 0)
kfn.alif_synapticInactivityCounter = kfn.alif_synapticInactivityCounter |> device
kfn.alif_arrayProjection4d = (similar(kfn.alif_wRec) .= 1)
@@ -333,7 +313,6 @@ function kfn_1(params::Dict; device=cpu)
synapticConnection = Int(floor(subable * synapticConnectionPercent/100))
for slice in eachslice(w, dims=3) # each slice is a neuron
startInd = row*col - subable + 1 # e.g. 100(row*col) - 50(subable) = 50 -> startInd = 51
# pool must contain only lif, alif neurons
pool = shuffle!([startInd:row*col...])[1:synapticConnection]
for i in pool
@@ -342,9 +321,9 @@ function kfn_1(params::Dict; device=cpu)
end
end
# # 10% of neuron connection should be enough to start to make neuron fires
# should_be_avg_weight = 1 / (0.2 * n)
# w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight
# 10% of neuron connection should be enough to start to make neuron fires
should_be_avg_weight = 1 / (0.1 * n)
w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight
# project 3D w into 4D kfn.lif_wOut (row, col, n, batch)
kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device
@@ -384,6 +363,25 @@ function kfn_1(params::Dict; device=cpu)
return kfn
end
function wRec(row, col, n, synapticConnectionNumber)
# subscription
w = zeros(row, col, n)
for slice in eachslice(w, dims=3)
pool = shuffle!([1:row*col...])[1:synapticConnectionNumber]
for i in pool
slice[i] = rand() # assign weight to synaptic connection. /10 to start small,
# otherwise RSNN's vt Usually stay negative (-)
end
end
# 10% of neuron connection should be enough to start to make neuron fires
should_be_avg_weight = 1 / (0.1 * synapticConnectionNumber)
w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight
return w #(row, col, n)
end
@@ -425,10 +423,6 @@ end

3
src/IronpenGPU.jl
View File

@@ -27,11 +27,12 @@ using .interface
""" version 0.0.5
Todo:
[] add weight liquidity
[DONE] add excitatory/inhabitory matrix
[-] add temporal summation in addition to already used spatial summation.
CANCELLED, spatial summation every second until membrane potential reach a threshold
is in itself a temporal summation.
[x] add neuroplasticity
[DONE] add neuroplasticity
[4] implement dormant connection and pruning machanism. the longer the training the longer
0 weight stay 0.
[] using RL to control learning signal