ton/IronpenGPU
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

version 0.0.5

Browse Source
This commit is contained in:
ton
2023-08-26 16:16:59 +07:00
parent d4a075ae88
commit 3ca52b03f5
11 changed files with 599 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
previousVersion/0.0.05-alpha/src/snnUtil.jl
View File

@@ -1,87 +0,0 @@
module snnUtil
export refractoryStatus!
# using
#------------------------------------------------------------------------------------------------100
function refractoryStatus!(refractoryCounter, refractoryActive, refractoryInactive)
d1, d2, d3, d4 = size(refractoryCounter)
for j in 1:d4
for i in 1:d3
if refractoryCounter[1, 1, i, j] > 0 # inactive
view(refractoryActive, 1, 1, i, j) .= 0
view(refractoryInactive, 1, 1, i, j) .= 1
else # active
view(refractoryActive, 1, 1, i, j) .= 1
view(refractoryInactive, 1, 1, i, j) .= 0
end
end
end
end
function frobenius_distance(A, B)
# Check if the matrices have the same size
if size(A) != size(B)
error("The matrices must have the same size")
end
# Initialize the distance to zero
distance = 0.0
# Loop over the elements of the matrices and add the squared differences
for i in 1:size(A, 1)
for j in 1:size(A, 2)
distance += (A[i, j] - B[i, j])^2
end
end
# Return the square root of the distance
return sqrt(distance)
end
end # module

0
previousVersion/0.0.05-alpha/Manifest.toml → previousVersion/0.0.5/Manifest.toml
View File

0
previousVersion/0.0.05-alpha/Project.toml → previousVersion/0.0.5/Project.toml
View File

32
previousVersion/0.0.05-alpha/main_gpu_1.jl → previousVersion/0.0.5/main_gpu_1.jl
View File

@@ -67,8 +67,8 @@ function generate_snn(filename::String, location::String)
output_portnumbers = (10, 1) output_portnumbers = (10, 1)
# 5000 neurons are maximum for 64GB memory i.e. 300 LIF : 200 ALIF # 5000 neurons are maximum for 64GB memory i.e. 300 LIF : 200 ALIF
lif_neuron_number = (signalInput_portnumbers[1], 3) # CHANGE lif_neuron_number = (signalInput_portnumbers[1], 60) # CHANGE
alif_neuron_number = (signalInput_portnumbers[1], 2) # CHANGE from Allen Institute, ALIF is 20-40% of LIF alif_neuron_number = (signalInput_portnumbers[1], 40) # CHANGE from Allen Institute, ALIF is 20-40% of LIF
# totalNeurons = computeNeuronNumber + noise_portnumbers + signalInput_portnumbers # totalNeurons = computeNeuronNumber + noise_portnumbers + signalInput_portnumbers
# totalInputPort = noise_portnumbers + signalInput_portnumbers # totalInputPort = noise_portnumbers + signalInput_portnumbers
@@ -91,7 +91,7 @@ function generate_snn(filename::String, location::String)
# exert more force (larger w_out_change) to move neuron into direction that reduce error # exert more force (larger w_out_change) to move neuron into direction that reduce error
# For example, model error from 7 to 2e6. # For example, model error from 7 to 2e6.
:synapticConnectionPercent => 50, # % coverage of total neurons in kfn :synapticConnectionPercent => 20, # % coverage of total neurons in kfn
) )
alif_neuron_params = Dict{Symbol, Any}( alif_neuron_params = Dict{Symbol, Any}(
@@ -114,14 +114,14 @@ function generate_snn(filename::String, location::String)
# From "Spike frequency adaptation supports network computations on temporally dispersed # From "Spike frequency adaptation supports network computations on temporally dispersed
# information" # information"
:synapticConnectionPercent => 50, # % coverage of total neurons in kfn :synapticConnectionPercent => 20, # % coverage of total neurons in kfn
) )
linear_neuron_params = Dict{Symbol, Any}( linear_neuron_params = Dict{Symbol, Any}(
:type => "linearNeuron", :type => "linearNeuron",
:v_th => 1.0, # neuron firing threshold (this value is treated as maximum bound if I use auto generate) :v_th => 1.0, # neuron firing threshold (this value is treated as maximum bound if I use auto generate)
:tau_out => 100.0, # output time constant in millisecond. :tau_out => 100.0, # output time constant in millisecond.
:synapticConnectionPercent => 50, # % coverage of total neurons in kfn :synapticConnectionPercent => 20, # % coverage of total neurons in kfn
# Good starting value is 1/50th of tau_a # Good starting value is 1/50th of tau_a
# This is problem specific parameter. # This is problem specific parameter.
# It controls how leaky the neuron is. # It controls how leaky the neuron is.
@@ -405,7 +405,7 @@ function train_snn(model, trainData, validateData, labelDict::Vector)
for (imgBatch, labels) in trainData # imgBatch (28, 28, 4) i.e. (row, col, batch) for (imgBatch, labels) in trainData # imgBatch (28, 28, 4) i.e. (row, col, batch)
stop == 3 ? break : false stop == 3 ? break : false
# signal (10, 2, 784, 4) i.e. (row, col, timestep, batch) # signal (10, 2, 784, 4) i.e. (row, col, timestep, batch)
signal = spikeGenerator(imgBatch, [0.05, 0.1, 0.2, 0.3, 0.5], noise=(true, 1, 1.0), copies=18) signal = spikeGenerator(imgBatch, [0.05, 0.1, 0.2, 0.3, 0.5], noise=(true, 1, 0.1), copies=18)
if length(size(signal)) == 3 if length(size(signal)) == 3
row, col, sequence = size(signal) row, col, sequence = size(signal)
batch = 1 batch = 1
@@ -462,7 +462,6 @@ function train_snn(model, trainData, validateData, labelDict::Vector)
lif_wRecChange_cpu = model.lif_wRecChange |> cpu lif_wRecChange_cpu = model.lif_wRecChange |> cpu
# if sum(lif_wRecChange_cpu) != 0 # if sum(lif_wRecChange_cpu) != 0
# println("") # println("")
# lif_vt_cpu = model.lif_vt |> cpu # lif_vt_cpu = model.lif_vt |> cpu
@@ -535,22 +534,23 @@ function train_snn(model, trainData, validateData, labelDict::Vector)
# commit learned weight only if the model answer incorrectly # commit learned weight only if the model answer incorrectly
finalAnswer_cpu = finalAnswer |> cpu finalAnswer_cpu = finalAnswer |> cpu
println("label $(labels[1]) finalAnswer $finalAnswer_cpu") # println("label $(labels[1]) finalAnswer $finalAnswer_cpu")
max = isequal.(finalAnswer_cpu[:,1], maximum(finalAnswer_cpu[:,1])) max = isequal.(finalAnswer_cpu[:,1], maximum(finalAnswer_cpu[:,1]))
if sum(max) == 1 && findall(max)[1] -1 == labels[1] if sum(max) == 1 && findall(max)[1] -1 == labels[1]
finalAnswer_cpu = findall(max)[1] - 1 finalAnswer_cpu = findall(max)[1] - 1
println("OK") println("label $(labels[1]) finalAnswer $finalAnswer_cpu CORRECT")
# println("label $(labels[1]) finalAnswer $finalAnswer_cpu")
elseif sum(max) == 1 && findall(max)[1] -1 != labels[1] elseif sum(max) == 1 && findall(max)[1] -1 != labels[1]
finalAnswer = findall(max)[1] - 1 finalAnswer = findall(max)[1] - 1
IronpenGPU.learn!(model) IronpenGPU.learn!(model, device)
println("LEARNING") println("label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING")
# println("label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING")
else else
IronpenGPU.learn!(model) IronpenGPU.learn!(model, device)
println("LEARNING") if sum(finalAnswer_cpu) > 1
# println("epoch $epoch label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING") println("epoch $epoch label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING")
else
println("epoch $epoch label $(labels[1]) finalAnswer ZERO answer LEARNING")
end
end end
# error("DEBUG -> main $(Dates.now())") # error("DEBUG -> main $(Dates.now())")

0
previousVersion/0.0.05-alpha/src/IronpenGPU.jl → previousVersion/0.0.5/src/IronpenGPU.jl
View File

430
previousVersion/0.0.05-alpha/src/forward.jl → previousVersion/0.0.5/src/forward.jl
View File

@@ -18,7 +18,7 @@ function (kfn::kfn_1)(input::AbstractArray)
# what to do at the start of learning round # what to do at the start of learning round
if view(kfn.learningStage, 1)[1] == 1 if view(kfn.learningStage, 1)[1] == 1
# reset learning params # reset learning params
kfn.zit_cumulative .= 0 kfn.zitCumulative .= 0
kfn.lif_vt .= 0 kfn.lif_vt .= 0
kfn.lif_wRecChange .= 0 kfn.lif_wRecChange .= 0
@@ -118,7 +118,7 @@ function (kfn::kfn_1)(input::AbstractArray)
reshape(kfn.lif_zt, (size(input, 1), :, 1, size(input, 3))), reshape(kfn.lif_zt, (size(input, 1), :, 1, size(input, 3))),
reshape(kfn.alif_zt, (size(input, 1), :, 1, size(input, 3))), dims=2) reshape(kfn.alif_zt, (size(input, 1), :, 1, size(input, 3))), dims=2)
kfn.zit .= reshape(_zit, (size(input, 1), :, size(input, 3))) kfn.zit .= reshape(_zit, (size(input, 1), :, size(input, 3)))
kfn.zit_cumulative .+= kfn.zit kfn.zitCumulative .+= kfn.zit
# project 3D kfn zit into 4D on zit # project 3D kfn zit into 4D on zit
i1, i2, i3, i4 = size(kfn.on_zit) i1, i2, i3, i4 = size(kfn.on_zit)
@@ -243,7 +243,7 @@ function lifForward( zit,
linear_to_cartesian, linear_to_cartesian,
) )
i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index
if i <= length(wRec) if i <= length(wRec)
# cartesian index # cartesian index
i1, i2, i3, i4 = linear_to_cartesian(i, size(wRec)) i1, i2, i3, i4 = linear_to_cartesian(i, size(wRec))
@@ -273,7 +273,7 @@ function lifForward( zit,
vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4] vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4]
# reset counter if neuron fires # reset counter if neuron fires
neuronInactivityCounter[i1,i2,i3,i4] = 10000 neuronInactivityCounter[i1,i2,i3,i4] = 0
else else
zt[i1,i2,i3,i4] = 0 zt[i1,i2,i3,i4] = 0
neuronInactivityCounter[i1,i2,i3,i4] -= 1 neuronInactivityCounter[i1,i2,i3,i4] -= 1
@@ -291,7 +291,7 @@ function lifForward( zit,
# count synaptic inactivity # count synaptic inactivity
if !iszero(wRec[i1,i2,i3,i4]) # check if this is wRec subscription if !iszero(wRec[i1,i2,i3,i4]) # check if this is wRec subscription
if !iszero(zit[i1,i2,i3,i4]) # synapse is active, reset counter if !iszero(zit[i1,i2,i3,i4]) # synapse is active, reset counter
synapticInactivityCounter[i1,i2,i3,i4] = 10000 synapticInactivityCounter[i1,i2,i3,i4] += 1
else # synapse is inactive, counting else # synapse is inactive, counting
synapticInactivityCounter[i1,i2,i3,i4] -= 1 synapticInactivityCounter[i1,i2,i3,i4] -= 1
end end
@@ -416,7 +416,7 @@ function alifForward( zit,
linear_to_cartesian, linear_to_cartesian,
) )
i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index
if i <= length(wRec) if i <= length(wRec)
# cartesian index # cartesian index
i1, i2, i3, i4 = linear_to_cartesian(i, size(wRec)) i1, i2, i3, i4 = linear_to_cartesian(i, size(wRec))
@@ -456,7 +456,7 @@ function alifForward( zit,
firingCounter[i1,i2,i3,i4] += 1 firingCounter[i1,i2,i3,i4] += 1
vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4] vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4]
a[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] * a[i1,i2,i3,i4]) + 1 a[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] * a[i1,i2,i3,i4]) + 1
neuronInactivityCounter[i1,i2,i3,i4] = 10000 neuronInactivityCounter[i1,i2,i3,i4] = 0
else else
zt[i1,i2,i3,i4] = 0 zt[i1,i2,i3,i4] = 0
a[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] * a[i1,i2,i3,i4]) a[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] * a[i1,i2,i3,i4])
@@ -478,7 +478,7 @@ function alifForward( zit,
# count synaptic inactivity # count synaptic inactivity
if !iszero(wRec[i1,i2,i3,i4]) # check if this is wRec subscription if !iszero(wRec[i1,i2,i3,i4]) # check if this is wRec subscription
if !iszero(zit[i1,i2,i3,i4]) # synapse is active, reset counter if !iszero(zit[i1,i2,i3,i4]) # synapse is active, reset counter
synapticInactivityCounter[i1,i2,i3,i4] = 10000 synapticInactivityCounter[i1,i2,i3,i4] += 1
else # synapse is inactive, counting else # synapse is inactive, counting
synapticInactivityCounter[i1,i2,i3,i4] -= 1 synapticInactivityCounter[i1,i2,i3,i4] -= 1
end end
@@ -612,238 +612,238 @@ function onForward( zit,
return nothing return nothing
end end
function lifForward(kfn_zit::Array{T}, # function lifForward(kfn_zit::Array{T},
zit::Array{T}, # zit::Array{T},
wRec::Array{T}, # wRec::Array{T},
vt0::Array{T}, # vt0::Array{T},
vt1::Array{T}, # vt1::Array{T},
vth::Array{T}, # vth::Array{T},
vRest::Array{T}, # vRest::Array{T},
zt1::Array{T}, # zt1::Array{T},
alpha::Array{T}, # alpha::Array{T},
phi::Array{T}, # phi::Array{T},
epsilonRec::Array{T}, # epsilonRec::Array{T},
refractoryCounter::Array{T}, # refractoryCounter::Array{T},
refractoryDuration::Array{T}, # refractoryDuration::Array{T},
gammaPd::Array{T}, # gammaPd::Array{T},
firingCounter::Array{T}, # firingCounter::Array{T},
arrayProjection4d::Array{T}, # arrayProjection4d::Array{T},
recSignal::Array{T}, # recSignal::Array{T},
decayed_vt0::Array{T}, # decayed_vt0::Array{T},
decayed_epsilonRec::Array{T}, # decayed_epsilonRec::Array{T},
vt1_diff_vth::Array{T}, # vt1_diff_vth::Array{T},
vt1_diff_vth_div_vth::Array{T}, # vt1_diff_vth_div_vth::Array{T},
gammaPd_div_vth::Array{T}, # gammaPd_div_vth::Array{T},
phiActivation::Array{T}, # phiActivation::Array{T},
) where T<:Number # ) where T<:Number
# project 3D kfn zit into 4D lif zit # # project 3D kfn zit into 4D lif zit
i1, i2, i3, i4 = size(alif_wRec) # i1, i2, i3, i4 = size(alif_wRec)
lif_zit .= reshape(kfn_zit, (i1, i2, 1, i4)) .* lif_arrayProjection4d # lif_zit .= reshape(kfn_zit, (i1, i2, 1, i4)) .* lif_arrayProjection4d
for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch # for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active # if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
@. @views refractoryCounter[:,:,i,j] -= 1 # @. @views refractoryCounter[:,:,i,j] -= 1
@. @views zt1[:,:,i,j] = 0 # @. @views zt1[:,:,i,j] = 0
@. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] # @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@. @views phi[:,:,i,j] = 0 # @. @views phi[:,:,i,j] = 0
# compute epsilonRec # # compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] # @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] # @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
else # refractory period is inactive # else # refractory period is inactive
@. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j] # @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j]
@. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] # @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) # @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j])) # if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j]))
@. @views zt1[:,:,i,j] = 1 # @. @views zt1[:,:,i,j] = 1
@. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] # @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
@. @views firingCounter[:,:,i,j] += 1 # @. @views firingCounter[:,:,i,j] += 1
@. @views vt1[:,:,i,j] = vRest[:,:,i,j] # @. @views vt1[:,:,i,j] = vRest[:,:,i,j]
else # else
@. @views zt1[:,:,i,j] = 0 # @. @views zt1[:,:,i,j] = 0
end # end
# compute phi, there is a difference from alif formula # # compute phi, there is a difference from alif formula
@. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] # @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
@. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] # @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
@. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] # @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
@view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) # @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
@. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] # @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# compute epsilonRec # # compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] # @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] # @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
end # end
end # end
end # end
function alifForward(zit::Array{T}, # function alifForward(zit::Array{T},
wRec::Array{T}, # wRec::Array{T},
vt0::Array{T}, # vt0::Array{T},
vt1::Array{T}, # vt1::Array{T},
vth::Array{T}, # vth::Array{T},
vRest::Array{T}, # vRest::Array{T},
zt1::Array{T}, # zt1::Array{T},
alpha::Array{T}, # alpha::Array{T},
phi::Array{T}, # phi::Array{T},
epsilonRec::Array{T}, # epsilonRec::Array{T},
refractoryCounter::Array{T}, # refractoryCounter::Array{T},
refractoryDuration::Array{T}, # refractoryDuration::Array{T},
gammaPd::Array{T}, # gammaPd::Array{T},
firingCounter::Array{T}, # firingCounter::Array{T},
recSignal::Array{T}, # recSignal::Array{T},
decayed_vt0::Array{T}, # decayed_vt0::Array{T},
decayed_epsilonRec::Array{T}, # decayed_epsilonRec::Array{T},
vt1_diff_vth::Array{T}, # vt1_diff_vth::Array{T},
vt1_diff_vth_div_vth::Array{T}, # vt1_diff_vth_div_vth::Array{T},
gammaPd_div_vth::Array{T}, # gammaPd_div_vth::Array{T},
phiActivation::Array{T}, # phiActivation::Array{T},
epsilonRecA::Array{T}, # epsilonRecA::Array{T},
avth::Array{T}, # avth::Array{T},
a::Array{T}, # a::Array{T},
beta::Array{T}, # beta::Array{T},
rho::Array{T}, # rho::Array{T},
phi_x_epsilonRec::Array{T}, # phi_x_epsilonRec::Array{T},
phi_x_beta::Array{T}, # phi_x_beta::Array{T},
rho_diff_phi_x_beta::Array{T}, # rho_diff_phi_x_beta::Array{T},
rho_div_phi_x_beta_x_epsilonRecA::Array{T}, # rho_div_phi_x_beta_x_epsilonRecA::Array{T},
beta_x_a::Array{T}, # beta_x_a::Array{T},
) where T<:Number # ) where T<:Number
for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch # for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active # if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
@. @views refractoryCounter[:,:,i,j] -= 1 # @. @views refractoryCounter[:,:,i,j] -= 1
@. @views zt1[:,:,i,j] = 0 # @. @views zt1[:,:,i,j] = 0
@. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] # @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@. @views phi[:,:,i,j] = 0 # @. @views phi[:,:,i,j] = 0
@. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] # @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
# compute epsilonRec # # compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] # @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] # @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
# compute epsilonRecA # # compute epsilonRecA
@. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j] # @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j] # @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j]
@. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j] # @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j]
@. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j] # @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j]
@. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] # @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j]
# compute avth # # compute avth
@. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j] # @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j]
@. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j] # @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j]
else # refractory period is inactive # else # refractory period is inactive
@. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j] # @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j]
@. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] # @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) # @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
# compute avth # # compute avth
@. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j] # @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j]
@. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j] # @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j]
if sum(@view(vt1[:,:,i,j])) > sum(@view(avth[:,:,i,j])) # if sum(@view(vt1[:,:,i,j])) > sum(@view(avth[:,:,i,j]))
@. @views zt1[:,:,i,j] = 1 # @. @views zt1[:,:,i,j] = 1
@. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] # @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
@. @views firingCounter[:,:,i,j] += 1 # @. @views firingCounter[:,:,i,j] += 1
@. @views vt1[:,:,i,j] = vRest[:,:,i,j] # @. @views vt1[:,:,i,j] = vRest[:,:,i,j]
@. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] # @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
@. @views a[:,:,i,j] = a[:,:,i,j] += 1 # @. @views a[:,:,i,j] = a[:,:,i,j] += 1
else # else
@. @views zt1[:,:,i,j] = 0 # @. @views zt1[:,:,i,j] = 0
@. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] # @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
end # end
# compute phi, there is a difference from alif formula # # compute phi, there is a difference from alif formula
@. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] # @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
@. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] # @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
@. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] # @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
@view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) # @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
@. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] # @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# compute epsilonRec # # compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] # @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] # @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
# compute epsilonRecA # # compute epsilonRecA
@. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j] # @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j] # @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j]
@. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j] # @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j]
@. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j] # @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j]
@. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] # @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j]
end # end
end # end
end # end
function onForward(kfn_zit::Array{T}, # function onForward(kfn_zit::Array{T},
zit::Array{T}, # zit::Array{T},
wOut::Array{T}, # wOut::Array{T},
vt0::Array{T}, # vt0::Array{T},
vt1::Array{T}, # vt1::Array{T},
vth::Array{T}, # vth::Array{T},
vRest::Array{T}, # vRest::Array{T},
zt1::Array{T}, # zt1::Array{T},
alpha::Array{T}, # alpha::Array{T},
phi::Array{T}, # phi::Array{T},
epsilonRec::Array{T}, # epsilonRec::Array{T},
refractoryCounter::Array{T}, # refractoryCounter::Array{T},
refractoryDuration::Array{T}, # refractoryDuration::Array{T},
gammaPd::Array{T}, # gammaPd::Array{T},
firingCounter::Array{T}, # firingCounter::Array{T},
arrayProjection4d::Array{T}, # arrayProjection4d::Array{T},
recSignal::Array{T}, # recSignal::Array{T},
decayed_vt0::Array{T}, # decayed_vt0::Array{T},
decayed_epsilonRec::Array{T}, # decayed_epsilonRec::Array{T},
vt1_diff_vth::Array{T}, # vt1_diff_vth::Array{T},
vt1_diff_vth_div_vth::Array{T}, # vt1_diff_vth_div_vth::Array{T},
gammaPd_div_vth::Array{T}, # gammaPd_div_vth::Array{T},
phiActivation::Array{T}, # phiActivation::Array{T},
) where T<:Number # ) where T<:Number
# project 3D kfn zit into 4D lif zit # # project 3D kfn zit into 4D lif zit
zit .= reshape(kfn_zit, # zit .= reshape(kfn_zit,
(size(wOut, 1), size(wOut, 2), 1, size(wOut, 4))) .* arrayProjection4d # (size(wOut, 1), size(wOut, 2), 1, size(wOut, 4))) .* arrayProjection4d
for j in 1:size(wOut, 4), i in 1:size(wOut, 3) # compute along neurons axis of every batch # for j in 1:size(wOut, 4), i in 1:size(wOut, 3) # compute along neurons axis of every batch
if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active # if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
@. @views refractoryCounter[:,:,i,j] -= 1 # @. @views refractoryCounter[:,:,i,j] -= 1
@. @views zt1[:,:,i,j] = 0 # @. @views zt1[:,:,i,j] = 0
@. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] # @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@. @views phi[:,:,i,j] = 0 # @. @views phi[:,:,i,j] = 0
# compute epsilonRec # # compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] # @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] # @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
else # refractory period is inactive # else # refractory period is inactive
@. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wOut[:,:,i,j] # @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wOut[:,:,i,j]
@. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] # @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) # @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j])) # if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j]))
@. @views zt1[:,:,i,j] = 1 # @. @views zt1[:,:,i,j] = 1
@. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] # @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
@. @views firingCounter[:,:,i,j] += 1 # @. @views firingCounter[:,:,i,j] += 1
@. @views vt1[:,:,i,j] = vRest[:,:,i,j] # @. @views vt1[:,:,i,j] = vRest[:,:,i,j]
else # else
@. @views zt1[:,:,i,j] = 0 # @. @views zt1[:,:,i,j] = 0
end # end
# compute phi, there is a difference from alif formula # # compute phi, there is a difference from alif formula
@. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] # @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
@. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] # @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
@. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] # @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
@view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) # @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
@. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] # @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# compute epsilonRec # # compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] # @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] # @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
end # end
end # end
end # end

0
previousVersion/0.0.05-alpha/src/interface.jl → previousVersion/0.0.5/src/interface.jl
View File

218
previousVersion/0.0.05-alpha/src/learn.jl → previousVersion/0.0.5/src/learn.jl
View File

@@ -266,22 +266,26 @@ end
function learn!(kfn::kfn_1, device=cpu) function learn!(kfn::kfn_1, device=cpu)
# lif learn # lif learn
lifLearn!(kfn.lif_wRec, kfn.lif_wRec, kfn.lif_neuronInactivityCounter, kfn.lif_synapticInactivityCounter =
kfn.lif_wRecChange, lifLearn(kfn.lif_wRec,
kfn.lif_arrayProjection4d, kfn.lif_wRecChange,
kfn.lif_neuronInactivityCounter, kfn.lif_arrayProjection4d,
kfn.lif_synapticConnectionNumber, kfn.lif_neuronInactivityCounter,
kfn.zit_cumulative, kfn.lif_synapticInactivityCounter,
device) kfn.lif_synapticConnectionNumber,
kfn.zitCumulative,
device)
# alif learn # alif learn
alifLearn!(kfn.alif_wRec, kfn.alif_wRec, kfn.alif_neuronInactivityCounter, kfn.alif_synapticInactivityCounter =
kfn.alif_wRecChange, alifLearn(kfn.alif_wRec,
kfn.alif_arrayProjection4d, kfn.alif_wRecChange,
kfn.alif_neuronInactivityCounter, kfn.alif_arrayProjection4d,
kfn.alif_synapticConnectionNumber, kfn.alif_neuronInactivityCounter,
kfn.zit_cumulative, kfn.alif_synapticInactivityCounter,
device) kfn.alif_synapticConnectionNumber,
kfn.zitCumulative,
device)
# on learn # on learn
onLearn!(kfn.on_wOut, onLearn!(kfn.on_wOut,
@@ -295,58 +299,108 @@ function learn!(kfn::kfn_1, device=cpu)
# error("DEBUG -> kfn learn! $(Dates.now())") # error("DEBUG -> kfn learn! $(Dates.now())")
end end
function lifLearn!(wRec, function lifLearn(wRec,
wRecChange, wRecChange,
arrayProjection4d, arrayProjection4d,
inactivityCounter, neuronInactivityCounter,
synapticInactivityCounter,
synapticConnectionNumber, synapticConnectionNumber,
zit_cumulative, zitCumulative,
device) device)
#WORKING - synapticInactivityCounter -10000 to 10000, weight change liquidity range from 1.0 to 0.1 respectively
# merge learning weight with average learning weight of all batch # merge learning weight with average learning weight of all batch
wRec .+= (sum(wRecChange, dims=4) ./ (size(wRec, 4))) .* arrayProjection4d wRec .+= (sum(wRecChange, dims=4) ./ (size(wRec, 4))) .* arrayProjection4d
arrayProjection4d_cpu = arrayProjection4d |> cpu
wRec_cpu = wRec |> cpu wRec_cpu = wRec |> cpu
wRec_cpu = wRec_cpu[:,:,:,1] # since every batch has the same neuron wRec, (row, col, n) wRec_cpu = wRec_cpu[:,:,:,1] # since every batch has the same neuron wRec, (row, col, n)
inactivityCounter_cpu = inactivityCounter |> cpu neuronInactivityCounter_cpu = neuronInactivityCounter |> cpu
inactivityCounter_cpu = inactivityCounter_cpu[:,:,:,1] # (row, col, n) neuronInactivityCounter_cpu = neuronInactivityCounter_cpu[:,:,:,1] # (row, col, n)
zit_cumulative_cpu = zit_cumulative |> cpu synapticInactivityCounter_cpu = synapticInactivityCounter |> cpu
zit_cumulative_cpu = zit_cumulative_cpu[:,:,1] # (row, col) synapticInactivityCounter_cpu = synapticInactivityCounter_cpu[:,:,:,1]
zitCumulative_cpu = zitCumulative |> cpu
zitCumulative_cpu = zitCumulative_cpu[:,:,1] # (row, col)
# weak / negative synaptic connection will get randomed in neuroplasticity() # weak / negative synaptic connection will get randomed in neuroplasticity()
wRec_cpu = GeneralUtils.replaceBetween.(wRec_cpu, 0.0, 0.1, -1.0) # mark with -1.0 wRec_cpu = GeneralUtils.replaceBetween.(wRec_cpu, 0.0, 0.1, -1.0) # mark with -1.0
# synaptic connection that has no inactivity will get randomed in neuroplasticity() # synaptic connection that has no activity will get randomed in neuroplasticity()
GeneralUtils.replace_elements!(inactivityCounter_cpu, 0.0, wRec_cpu, -1.0) mask = isless.(synapticInactivityCounter_cpu, -10_000)
# reset lif_inactivity elements to 10000 GeneralUtils.replace_elements!(mask, 1, wRec_cpu, -1.0)
GeneralUtils.replace_elements!(inactivityCounter_cpu, 0.0, -9.0) # -9.0 is base value # reset lif_inactivity elements to base value
GeneralUtils.replace_elements!(mask, 1, synapticInactivityCounter_cpu, 0.0)
# neuroplasticity, work on CPU side
wRec_cpu = neuroplasticity(synapticConnectionNumber,
zitCumulative_cpu,
wRec_cpu,
neuronInactivityCounter_cpu,
synapticInactivityCounter_cpu)
#WORKING neuroplasticity wRec_cpu = wRec_cpu .* arrayProjection4d_cpu
wRec_cpu = neuroplasticity(synapticConnectionNumber, zit_cumulative_cpu, wRec_cpu, wRec = wRec_cpu |> device
inactivityCounter_cpu)
error("DEBUG -> lifLearn! $(Dates.now())")
# #TODO send to device with correct dimension
# wRec = wRec |> device
# inactivityCounter = inactivityCounter_cpu |> device
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu .* arrayProjection4d_cpu
neuronInactivityCounter = neuronInactivityCounter_cpu |> device
synapticInactivityCounter_cpu = synapticInactivityCounter_cpu .* arrayProjection4d_cpu
synapticInactivityCounter = synapticInactivityCounter_cpu |> device
# error("DEBUG -> lifLearn! $(Dates.now())")
return wRec, neuronInactivityCounter, synapticInactivityCounter
end end
function alifLearn!(wRec, function alifLearn(wRec,
wRecChange, wRecChange,
arrayProjection4d, arrayProjection4d,
inactivityCounter, neuronInactivityCounter,
synapticInactivityCounter,
synapticConnectionNumber, synapticConnectionNumber,
zit_cumulative, zitCumulative,
device) device)
# merge learning weight with average learning weight #WORKING - synapticInactivityCounter -10000 to 10000, weight change liquidity range from 1.0 to 0.1 respectively
# merge learning weight with average learning weight of all batch
wRec .+= (sum(wRecChange, dims=4) ./ (size(wRec, 4))) .* arrayProjection4d wRec .+= (sum(wRecChange, dims=4) ./ (size(wRec, 4))) .* arrayProjection4d
arrayProjection4d_cpu = arrayProjection4d |> cpu
wRec_cpu = wRec |> cpu
wRec_cpu = wRec_cpu[:,:,:,1] # since every batch has the same neuron wRec, (row, col, n)
neuronInactivityCounter_cpu = neuronInactivityCounter |> cpu
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu[:,:,:,1] # (row, col, n)
synapticInactivityCounter_cpu = synapticInactivityCounter |> cpu
synapticInactivityCounter_cpu = synapticInactivityCounter_cpu[:,:,:,1]
zitCumulative_cpu = zitCumulative |> cpu
zitCumulative_cpu = zitCumulative_cpu[:,:,1] # (row, col)
# weak / negative synaptic connection will get randomed in neuroplasticity() # weak / negative synaptic connection will get randomed in neuroplasticity()
wRec .= GeneralUtils.replaceLessThan.(wRec, 0.01, 0.0) wRec_cpu = GeneralUtils.replaceBetween.(wRec_cpu, 0.0, 0.1, -1.0) # mark with -1.0
# synaptic connection that has no activity will get randomed in neuroplasticity()
mask = isless.(synapticInactivityCounter_cpu, -10_000)
GeneralUtils.replace_elements!(mask, 1, wRec_cpu, -1.0)
# reset alif_inactivity elements to base value
GeneralUtils.replace_elements!(mask, 1, synapticInactivityCounter_cpu, 0.0)
#TODO synaptic strength # neuroplasticity, work on CPU side
wRec_cpu = neuroplasticity(synapticConnectionNumber,
zitCumulative_cpu,
wRec_cpu,
neuronInactivityCounter_cpu,
synapticInactivityCounter_cpu)
wRec_cpu = wRec_cpu .* arrayProjection4d_cpu
wRec = wRec_cpu |> device
neuronInactivityCounter_cpu = neuronInactivityCounter_cpu .* arrayProjection4d_cpu
neuronInactivityCounter = neuronInactivityCounter_cpu |> device
#TODO neuroplasticity synapticInactivityCounter_cpu = synapticInactivityCounter_cpu .* arrayProjection4d_cpu
synapticInactivityCounter = synapticInactivityCounter_cpu |> device
# error("DEBUG -> alifLearn! $(Dates.now())")
return wRec, neuronInactivityCounter, synapticInactivityCounter
end end
function onLearn!(wOut, function onLearn!(wOut,
@@ -365,53 +419,67 @@ function onLearn!(wOut,
end end
function neuroplasticity(synapticConnectionNumber, function neuroplasticity(synapticConnectionNumber,
zit_cumulative, # (row, col) zitCumulative, # (row, col)
wRec, # (row, col, n) wRec, # (row, col, n)
inactivityCounter_cpu) # (row, col, n) neuronInactivityCounter,
synapticInactivityCounter) # (row, col, n)
i1,i2,i3 = size(wRec) i1,i2,i3 = size(wRec)
# for each neuron, find total number of synaptic conn that should draw # for each neuron, find total number of synaptic conn that should draw
# new connection to firing and non-firing neurons pool # new connection to firing and non-firing neurons pool
subToFireNeuron_toBe = Int(floor(0.7 * synapticConnectionNumber)) subToFireNeuron_toBe = Int(floor(0.7 * synapticConnectionNumber))
subToNonFiringNeuron_toBe = synapticConnectionNumber - subToFireNeuron_toBe
#WORKING for each neuron, count how many synap already subscribed to firing-neurons # for each neuron, count how many synap already subscribed to firing-neurons
subToFireNeuron_current = sum((!iszero).(zit_cumulative .* wRec), dims=(1,2)) # (1, 1, n) zw = zitCumulative .* wRec
subToNonFiringNeuron_current = synapticConnectionNumber .- subToFireNeuron_current # (1, 1, n) subToFireNeuron_current = sum(GeneralUtils.isBetween.(zw, 0.0, 100.0), dims=(1,2)) # (1, 1, n)
mask = (!iszero).(zit_cumulative) # mask of firing neurons = 1, non-firing = 0 zitMask = (!iszero).(zitCumulative) # zitMask of firing neurons = 1, non-firing = 0
projection = ones(i1,i2,i3) projection = ones(i1,i2,i3)
mask = mask .* projection # (row, col, n) zitMask = zitMask .* projection # (row, col, n)
totalNewConn = sum(isequal.(wRec, -1.0), dims=(1,2)) # count new conn mark (-1.0), (1, 1, n) totalNewConn = sum(isequal.(wRec, -1.0), dims=(1,2)) # count new conn mark (-1.0), (1, 1, n)
println("mask ", size(mask))
println("wRec ", size(wRec)) # clear -1.0 marker
println("inactivityCounter_cpu ", size(inactivityCounter_cpu)) GeneralUtils.replace_elements!(wRec, -1.0, synapticInactivityCounter, -0.99)
println("totalNeurons ", totalNewConn, size(totalNewConn)) GeneralUtils.replace_elements!(wRec, -1.0, 0.0) # -1.0 marker is no longer required
error("DEBUG -> neuroplasticity $(Dates.now())")
for i in 1:i3 for i in 1:i3
if neuronInactivityCounter[1:1:i][1] < -10_000 # neuron die i.e. reset all weight
neuronInactivityCounter[:,:,i] .= 0 # reset
w = wRec(i1,i2,1,synapticConnectionNumber)
wRec[:,:,i] = w
a = similar(w) .= -0.99 # synapticConnectionNumber of this neuron
# add new conn to firing neurons pool mask = (!iszero).(w)
remaining = GeneralUtils.replace_elements(mask[:,:,i], GeneralUtils.replace_elements!(mask, 1, a, 0)
1, synapticInactivityCounter[:,:,i] = a
wRecmask[:,:,i], else
inactivityCounter_cpumask[:,:,i], remaining = 0
totalNewConn[:,:,i]) if subToFireNeuron_current[1,1,i] < subToFireNeuron_toBe
toAddConn = subToFireNeuron_toBe - subToFireNeuron_current[1,1,i]
#TODO add new conn to non-firing neurons pool totalNewConn[1,1,i] = totalNewConn[1,1,i] - toAddConn
# add new conn to firing neurons pool
remaining = addNewSynapticConn!(zitMask[:,:,i], 1,
@view(wRec[:,:,i]),
@view(synapticInactivityCounter[:,:,i]),
toAddConn)
totalNewConn[1,1,i] += remaining
end
# add new conn to non-firing neurons pool
remaining = addNewSynapticConn!(zitMask[:,:,i], 0,
@view(wRec[:,:,i]),
@view(synapticInactivityCounter[:,:,i]),
totalNewConn[1,1,i])
if remaining > 0 # final get-all round if somehow non-firing pool has not enough slot
remaining = addNewSynapticConn!(zitMask[:,:,i], 1,
@view(wRec[:,:,i]),
@view(synapticInactivityCounter[:,:,i]),
remaining)
end
end
end end
# error("DEBUG -> neuroplasticity $(Dates.now())")
newFiringConn = subToFireNeuron_toBe - subToFireNeuron_current
newFiringConn = newFiringConn > 0 ? newFiringConn : 0
newNonFiringConn = subToNonFiringNeuron_toBe - subToNonFiringNeuron_current
return wRec return wRec
end end
@@ -454,10 +522,6 @@ end

191
previousVersion/0.0.5/src/snnUtil.jl Normal file
View File

@@ -0,0 +1,191 @@
module snnUtil
export refractoryStatus!, addNewSynapticConn!
using Random
#------------------------------------------------------------------------------------------------100
function refractoryStatus!(refractoryCounter, refractoryActive, refractoryInactive)
d1, d2, d3, d4 = size(refractoryCounter)
for j in 1:d4
for i in 1:d3
if refractoryCounter[1, 1, i, j] > 0 # inactive
view(refractoryActive, 1, 1, i, j) .= 0
view(refractoryInactive, 1, 1, i, j) .= 1
else # active
view(refractoryActive, 1, 1, i, j) .= 1
view(refractoryInactive, 1, 1, i, j) .= 0
end
end
end
end
# function frobenius_distance(A, B)
# # Check if the matrices have the same size
# if size(A) != size(B)
# error("The matrices must have the same size")
# end
# # Initialize the distance to zero
# distance = 0.0
# # Loop over the elements of the matrices and add the squared differences
# for i in 1:size(A, 1)
# for j in 1:size(A, 2)
# distance += (A[i, j] - B[i, j])^2
# end
# end
# # Return the square root of the distance
# return sqrt(distance)
# end
function addNewSynapticConn!(mask::AbstractArray{<:Any}, x::Number, wRec::AbstractArray{<:Any},
counter::AbstractArray{<:Any}, n=0;
rng::AbstractRNG=MersenneTwister(1234))
# println("mask ", mask, size(mask))
# println("")
# println("x ", x, size(x))
# println("")
# println("wRec ", wRec, size(wRec))
# println("")
# println("counter ", counter, size(counter))
# println("")
# println("n ", n, size(n))
# println("")
total_x_tobeReplced = sum(isequal.(mask, x))
remaining = 0
if n == 0 || n > total_x_tobeReplced
remaining = n - total_x_tobeReplced
n = total_x_tobeReplced
end
# check if mask and wRec have the same size
if size(mask) != size(wRec)
error("mask and wRec must have the same size")
end
# get the indices of elements in mask that equal x
indices = findall(x -> x == x, mask)
alreadySub = findall(x -> x != 0, wRec) # get already subscribe
setdiff!(indices, alreadySub) # remove already sub conn from pool
# shuffle the indices using the rng function
shuffle!(rng, indices)
# select the first n indices
selected = indices[1:n]
# replace the elements in wRec at the selected positions with a
for i in selected
wRec[i] = 0.1 #rand(0.1:0.01:0.3)
if counter !== nothing
counter[i] = 0 # reset
end
end
# println("==================")
# println("mask ", mask, size(mask))
# println("")
# println("x ", x, size(x))
# println("")
# println("wRec ", wRec, size(wRec))
# println("")
# println("counter ", counter, size(counter))
# println("")
# println("n ", n, size(n))
# println("")
# error("DEBUG addNewSynapticConn!")
return remaining
end
# function addNewSynapticConn!(mask::AbstractArray{<:Any}, x::Number, A::AbstractArray{<:Any},
# A2::AbstractArray{<:Any}, n=0;
# rng::AbstractRNG=MersenneTwister(1234))
# # println("mask ", mask, size(mask))
# # println("")
# # println("x ", x, size(x))
# # println("")
# # println("A ", A, size(A))
# # println("")
# # println("A2 ", A2, size(A2))
# # println("")
# # println("n ", n, size(n))
# # println("")
# total_x_tobeReplced = sum(isequal.(mask, x))
# remaining = 0
# if n == 0 || n > total_x_tobeReplced
# remaining = n - total_x_tobeReplced
# n = total_x_tobeReplced
# end
# # check if mask and A have the same size
# if size(mask) != size(A)
# error("mask and A must have the same size")
# end
# # get the indices of elements in mask that equal x
# indices = findall(x -> x == x, mask)
# # shuffle the indices using the rng function
# shuffle!(rng, indices)
# # select the first n indices
# selected = indices[1:n]
# # replace the elements in A at the selected positions with a
# for i in selected
# A[i] = rand(0.1:0.01:0.3)
# if A2 !== nothing
# A2[i] = 10000
# end
# end
# # println("==================")
# # println("mask ", mask, size(mask))
# # println("")
# # println("x ", x, size(x))
# # println("")
# # println("A ", A, size(A))
# # println("")
# # println("A2 ", A2, size(A2))
# # println("")
# # println("n ", n, size(n))
# # println("")
# # error("DEBUG addNewSynapticConn!")
# return remaining
# end
end # module

74
previousVersion/0.0.05-alpha/src/type.jl → previousVersion/0.0.5/src/type.jl
View File

@@ -23,7 +23,7 @@ Base.@kwdef mutable struct kfn_1 <: knowledgeFn
learningStage::Union{AbstractArray, Nothing} = nothing # 0 inference, 1 start, 2 during, 3 end learning learningStage::Union{AbstractArray, Nothing} = nothing # 0 inference, 1 start, 2 during, 3 end learning
inputSize::Union{AbstractArray, Nothing} = nothing inputSize::Union{AbstractArray, Nothing} = nothing
zit::Union{AbstractArray, Nothing} = nothing # 3D activation matrix zit::Union{AbstractArray, Nothing} = nothing # 3D activation matrix
zit_cumulative::Union{AbstractArray, Nothing} = nothing zitCumulative::Union{AbstractArray, Nothing} = nothing
exInType::Union{AbstractArray, Nothing} = nothing exInType::Union{AbstractArray, Nothing} = nothing
modelError::Union{AbstractArray, Nothing} = nothing # store RSNN error modelError::Union{AbstractArray, Nothing} = nothing # store RSNN error
outputError::Union{AbstractArray, Nothing} = nothing # store output neurons error outputError::Union{AbstractArray, Nothing} = nothing # store output neurons error
@@ -185,7 +185,7 @@ function kfn_1(params::Dict; device=cpu)
# activation matrix # activation matrix
kfn.zit = zeros(row, col, batch) |> device kfn.zit = zeros(row, col, batch) |> device
kfn.zit_cumulative = (similar(kfn.zit) .= 0) kfn.zitCumulative = (similar(kfn.zit) .= 0)
kfn.modelError = zeros(1) |> device kfn.modelError = zeros(1) |> device
# ---------------------------------------------------------------------------- # # ---------------------------------------------------------------------------- #
@@ -196,20 +196,9 @@ function kfn_1(params::Dict; device=cpu)
lif_n = kfn.params[:computeNeuron][:lif][:numbers][1] * kfn.params[:computeNeuron][:lif][:numbers][2] lif_n = kfn.params[:computeNeuron][:lif][:numbers][1] * kfn.params[:computeNeuron][:lif][:numbers][2]
# subscription # subscription
w = zeros(row, col, lif_n)
synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent] synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent]
kfn.lif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100)) kfn.lif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100))
for slice in eachslice(w, dims=3) w = wRec(row, col, lif_n, kfn.lif_synapticConnectionNumber)
pool = shuffle!([1:row*col...])[1:kfn.lif_synapticConnectionNumber]
for i in pool
slice[i] = rand() # assign weight to synaptic connection. /10 to start small,
# otherwise RSNN's vt Usually stay negative (-)
end
end
# 10% of neuron connection should be enough to start to make neuron fires
should_be_avg_weight = 1 / (0.1 * lif_n)
w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight
# project 3D w into 4D kfn.lif_wRec (row, col, n, batch) # project 3D w into 4D kfn.lif_wRec (row, col, n, batch)
kfn.lif_wRec = reshape(w, (row, col, lif_n, 1)) .* ones(row, col, lif_n, batch) |> device kfn.lif_wRec = reshape(w, (row, col, lif_n, 1)) .* ones(row, col, lif_n, batch) |> device
@@ -234,10 +223,11 @@ function kfn_1(params::Dict; device=cpu)
kfn.lif_firingCounter = (similar(kfn.lif_wRec) .= 0) kfn.lif_firingCounter = (similar(kfn.lif_wRec) .= 0)
kfn.lif_firingTargetFrequency = (similar(kfn.lif_wRec) .= 0.1) kfn.lif_firingTargetFrequency = (similar(kfn.lif_wRec) .= 0.1)
kfn.lif_neuronInactivityCounter = (similar(kfn.lif_wRec) .= 10000) kfn.lif_neuronInactivityCounter = (similar(kfn.lif_wRec) .= 0)
kfn.lif_synapticInactivityCounter = Array(similar(kfn.lif_wRec) .= -9) # -9 for non-sub conn kfn.lif_synapticInactivityCounter = Array(similar(kfn.lif_wRec) .= -0.99) # -9 for non-sub conn
mask = Array((!iszero).(kfn.lif_wRec)) mask = Array((!iszero).(kfn.lif_wRec))
GeneralUtils.replace_elements!(mask, 1, kfn.lif_synapticInactivityCounter, 10000) # initial value subscribed conn, synapticInactivityCounter range -10000 to +10000
GeneralUtils.replace_elements!(mask, 1, kfn.lif_synapticInactivityCounter, 0)
kfn.lif_synapticInactivityCounter = kfn.lif_synapticInactivityCounter |> device kfn.lif_synapticInactivityCounter = kfn.lif_synapticInactivityCounter |> device
kfn.lif_arrayProjection4d = (similar(kfn.lif_wRec) .= 1) kfn.lif_arrayProjection4d = (similar(kfn.lif_wRec) .= 1)
@@ -255,20 +245,9 @@ function kfn_1(params::Dict; device=cpu)
alif_n = kfn.params[:computeNeuron][:alif][:numbers][1] * kfn.params[:computeNeuron][:alif][:numbers][2] alif_n = kfn.params[:computeNeuron][:alif][:numbers][1] * kfn.params[:computeNeuron][:alif][:numbers][2]
# subscription # subscription
w = zeros(row, col, alif_n)
synapticConnectionPercent = kfn.params[:computeNeuron][:alif][:params][:synapticConnectionPercent] synapticConnectionPercent = kfn.params[:computeNeuron][:alif][:params][:synapticConnectionPercent]
kfn.alif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100)) kfn.alif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100))
for slice in eachslice(w, dims=3) w = wRec(row, col, alif_n, kfn.alif_synapticConnectionNumber)
pool = shuffle!([1:row*col...])[1:kfn.alif_synapticConnectionNumber]
for i in pool
slice[i] = rand() # assign weight to synaptic connection. /10 to start small,
# otherwise RSNN's vt Usually stay negative (-)
end
end
# 10% of neuron connection should be enough to start to make neuron fires
should_be_avg_weight = 1 / (0.1 * alif_n)
w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight
# project 3D w into 4D kfn.alif_wRec # project 3D w into 4D kfn.alif_wRec
kfn.alif_wRec = reshape(w, (row, col, alif_n, 1)) .* ones(row, col, alif_n, batch) |> device kfn.alif_wRec = reshape(w, (row, col, alif_n, 1)) .* ones(row, col, alif_n, batch) |> device
@@ -293,10 +272,11 @@ function kfn_1(params::Dict; device=cpu)
kfn.alif_firingCounter = (similar(kfn.alif_wRec) .= 0) kfn.alif_firingCounter = (similar(kfn.alif_wRec) .= 0)
kfn.alif_firingTargetFrequency = (similar(kfn.alif_wRec) .= 0.1) kfn.alif_firingTargetFrequency = (similar(kfn.alif_wRec) .= 0.1)
kfn.alif_neuronInactivityCounter = (similar(kfn.alif_wRec) .= 10000) kfn.alif_neuronInactivityCounter = (similar(kfn.alif_wRec) .= 0)
kfn.alif_synapticInactivityCounter = Array(similar(kfn.alif_wRec) .= -9) # -9 for non-sub conn kfn.alif_synapticInactivityCounter = Array(similar(kfn.alif_wRec) .= -0.99) # -9 for non-sub conn
mask = Array((!iszero).(kfn.alif_wRec)) mask = Array((!iszero).(kfn.alif_wRec))
GeneralUtils.replace_elements!(mask, 1, kfn.alif_synapticInactivityCounter, 10000) # initial value subscribed conn, synapticInactivityCounter range -10000 to +10000
GeneralUtils.replace_elements!(mask, 1, kfn.alif_synapticInactivityCounter, 0)
kfn.alif_synapticInactivityCounter = kfn.alif_synapticInactivityCounter |> device kfn.alif_synapticInactivityCounter = kfn.alif_synapticInactivityCounter |> device
kfn.alif_arrayProjection4d = (similar(kfn.alif_wRec) .= 1) kfn.alif_arrayProjection4d = (similar(kfn.alif_wRec) .= 1)
@@ -333,7 +313,6 @@ function kfn_1(params::Dict; device=cpu)
synapticConnection = Int(floor(subable * synapticConnectionPercent/100)) synapticConnection = Int(floor(subable * synapticConnectionPercent/100))
for slice in eachslice(w, dims=3) # each slice is a neuron for slice in eachslice(w, dims=3) # each slice is a neuron
startInd = row*col - subable + 1 # e.g. 100(row*col) - 50(subable) = 50 -> startInd = 51 startInd = row*col - subable + 1 # e.g. 100(row*col) - 50(subable) = 50 -> startInd = 51
# pool must contain only lif, alif neurons # pool must contain only lif, alif neurons
pool = shuffle!([startInd:row*col...])[1:synapticConnection] pool = shuffle!([startInd:row*col...])[1:synapticConnection]
for i in pool for i in pool
@@ -342,9 +321,9 @@ function kfn_1(params::Dict; device=cpu)
end end
end end
# # 10% of neuron connection should be enough to start to make neuron fires # 10% of neuron connection should be enough to start to make neuron fires
# should_be_avg_weight = 1 / (0.2 * n) should_be_avg_weight = 1 / (0.1 * n)
# w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight
# project 3D w into 4D kfn.lif_wOut (row, col, n, batch) # project 3D w into 4D kfn.lif_wOut (row, col, n, batch)
kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device
@@ -384,6 +363,25 @@ function kfn_1(params::Dict; device=cpu)
return kfn return kfn
end end
function wRec(row, col, n, synapticConnectionNumber)
# subscription
w = zeros(row, col, n)
for slice in eachslice(w, dims=3)
pool = shuffle!([1:row*col...])[1:synapticConnectionNumber]
for i in pool
slice[i] = rand() # assign weight to synaptic connection. /10 to start small,
# otherwise RSNN's vt Usually stay negative (-)
end
end
# 10% of neuron connection should be enough to start to make neuron fires
should_be_avg_weight = 1 / (0.1 * synapticConnectionNumber)
w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight
return w #(row, col, n)
end
@@ -425,10 +423,6 @@ end

3
src/IronpenGPU.jl
View File

@@ -27,11 +27,12 @@ using .interface
""" version 0.0.5 """ version 0.0.5
Todo: Todo:
[] add weight liquidity
[DONE] add excitatory/inhabitory matrix [DONE] add excitatory/inhabitory matrix
[-] add temporal summation in addition to already used spatial summation. [-] add temporal summation in addition to already used spatial summation.
CANCELLED, spatial summation every second until membrane potential reach a threshold CANCELLED, spatial summation every second until membrane potential reach a threshold
is in itself a temporal summation. is in itself a temporal summation.
[x] add neuroplasticity [DONE] add neuroplasticity
[4] implement dormant connection and pruning machanism. the longer the training the longer [4] implement dormant connection and pruning machanism. the longer the training the longer
0 weight stay 0. 0 weight stay 0.
[] using RL to control learning signal [] using RL to control learning signal