From 3ca52b03f53b7124d493793f5c68eb442912afaa Mon Sep 17 00:00:00 2001 From: ton Date: Sat, 26 Aug 2023 16:16:59 +0700 Subject: [PATCH] version 0.0.5 --- previousVersion/0.0.05-alpha/src/snnUtil.jl | 87 ---- .../{0.0.05-alpha => 0.0.5}/Manifest.toml | 0 .../{0.0.05-alpha => 0.0.5}/Project.toml | 0 .../{0.0.05-alpha => 0.0.5}/main_gpu_1.jl | 32 +- .../{0.0.05-alpha => 0.0.5}/src/IronpenGPU.jl | 0 .../{0.0.05-alpha => 0.0.5}/src/forward.jl | 430 +++++++++--------- .../{0.0.05-alpha => 0.0.5}/src/interface.jl | 0 .../{0.0.05-alpha => 0.0.5}/src/learn.jl | 218 +++++---- previousVersion/0.0.5/src/snnUtil.jl | 191 ++++++++ .../{0.0.05-alpha => 0.0.5}/src/type.jl | 74 ++- src/IronpenGPU.jl | 3 +- 11 files changed, 599 insertions(+), 436 deletions(-) delete mode 100644 previousVersion/0.0.05-alpha/src/snnUtil.jl rename previousVersion/{0.0.05-alpha => 0.0.5}/Manifest.toml (100%) rename previousVersion/{0.0.05-alpha => 0.0.5}/Project.toml (100%) rename previousVersion/{0.0.05-alpha => 0.0.5}/main_gpu_1.jl (96%) rename previousVersion/{0.0.05-alpha => 0.0.5}/src/IronpenGPU.jl (100%) rename previousVersion/{0.0.05-alpha => 0.0.5}/src/forward.jl (68%) rename previousVersion/{0.0.05-alpha => 0.0.5}/src/interface.jl (100%) rename previousVersion/{0.0.05-alpha => 0.0.5}/src/learn.jl (64%) create mode 100644 previousVersion/0.0.5/src/snnUtil.jl rename previousVersion/{0.0.05-alpha => 0.0.5}/src/type.jl (95%) diff --git a/previousVersion/0.0.05-alpha/src/snnUtil.jl b/previousVersion/0.0.05-alpha/src/snnUtil.jl deleted file mode 100644 index deabccb..0000000 --- a/previousVersion/0.0.05-alpha/src/snnUtil.jl +++ /dev/null @@ -1,87 +0,0 @@ -module snnUtil - -export refractoryStatus! - -# using - -#------------------------------------------------------------------------------------------------100 - -function refractoryStatus!(refractoryCounter, refractoryActive, refractoryInactive) - d1, d2, d3, d4 = size(refractoryCounter) - for j in 1:d4 - for i in 1:d3 - if refractoryCounter[1, 1, i, j] > 0 # inactive - view(refractoryActive, 1, 1, i, j) .= 0 - view(refractoryInactive, 1, 1, i, j) .= 1 - else # active - view(refractoryActive, 1, 1, i, j) .= 1 - view(refractoryInactive, 1, 1, i, j) .= 0 - end - end - end -end - -function frobenius_distance(A, B) - # Check if the matrices have the same size - if size(A) != size(B) - error("The matrices must have the same size") - end - # Initialize the distance to zero - distance = 0.0 - # Loop over the elements of the matrices and add the squared differences - for i in 1:size(A, 1) - for j in 1:size(A, 2) - distance += (A[i, j] - B[i, j])^2 - end - end - # Return the square root of the distance - return sqrt(distance) -end - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -end # module \ No newline at end of file diff --git a/previousVersion/0.0.05-alpha/Manifest.toml b/previousVersion/0.0.5/Manifest.toml similarity index 100% rename from previousVersion/0.0.05-alpha/Manifest.toml rename to previousVersion/0.0.5/Manifest.toml diff --git a/previousVersion/0.0.05-alpha/Project.toml b/previousVersion/0.0.5/Project.toml similarity index 100% rename from previousVersion/0.0.05-alpha/Project.toml rename to previousVersion/0.0.5/Project.toml diff --git a/previousVersion/0.0.05-alpha/main_gpu_1.jl b/previousVersion/0.0.5/main_gpu_1.jl similarity index 96% rename from previousVersion/0.0.05-alpha/main_gpu_1.jl rename to previousVersion/0.0.5/main_gpu_1.jl index 7f63ef9..58ac73d 100644 --- a/previousVersion/0.0.05-alpha/main_gpu_1.jl +++ b/previousVersion/0.0.5/main_gpu_1.jl @@ -67,8 +67,8 @@ function generate_snn(filename::String, location::String) output_portnumbers = (10, 1) # 5000 neurons are maximum for 64GB memory i.e. 300 LIF : 200 ALIF - lif_neuron_number = (signalInput_portnumbers[1], 3) # CHANGE - alif_neuron_number = (signalInput_portnumbers[1], 2) # CHANGE from Allen Institute, ALIF is 20-40% of LIF + lif_neuron_number = (signalInput_portnumbers[1], 60) # CHANGE + alif_neuron_number = (signalInput_portnumbers[1], 40) # CHANGE from Allen Institute, ALIF is 20-40% of LIF # totalNeurons = computeNeuronNumber + noise_portnumbers + signalInput_portnumbers # totalInputPort = noise_portnumbers + signalInput_portnumbers @@ -91,7 +91,7 @@ function generate_snn(filename::String, location::String) # exert more force (larger w_out_change) to move neuron into direction that reduce error # For example, model error from 7 to 2e6. - :synapticConnectionPercent => 50, # % coverage of total neurons in kfn + :synapticConnectionPercent => 20, # % coverage of total neurons in kfn ) alif_neuron_params = Dict{Symbol, Any}( @@ -114,14 +114,14 @@ function generate_snn(filename::String, location::String) # From "Spike frequency adaptation supports network computations on temporally dispersed # information" - :synapticConnectionPercent => 50, # % coverage of total neurons in kfn + :synapticConnectionPercent => 20, # % coverage of total neurons in kfn ) linear_neuron_params = Dict{Symbol, Any}( :type => "linearNeuron", :v_th => 1.0, # neuron firing threshold (this value is treated as maximum bound if I use auto generate) :tau_out => 100.0, # output time constant in millisecond. - :synapticConnectionPercent => 50, # % coverage of total neurons in kfn + :synapticConnectionPercent => 20, # % coverage of total neurons in kfn # Good starting value is 1/50th of tau_a # This is problem specific parameter. # It controls how leaky the neuron is. @@ -405,7 +405,7 @@ function train_snn(model, trainData, validateData, labelDict::Vector) for (imgBatch, labels) in trainData # imgBatch (28, 28, 4) i.e. (row, col, batch) stop == 3 ? break : false # signal (10, 2, 784, 4) i.e. (row, col, timestep, batch) - signal = spikeGenerator(imgBatch, [0.05, 0.1, 0.2, 0.3, 0.5], noise=(true, 1, 1.0), copies=18) + signal = spikeGenerator(imgBatch, [0.05, 0.1, 0.2, 0.3, 0.5], noise=(true, 1, 0.1), copies=18) if length(size(signal)) == 3 row, col, sequence = size(signal) batch = 1 @@ -462,7 +462,6 @@ function train_snn(model, trainData, validateData, labelDict::Vector) lif_wRecChange_cpu = model.lif_wRecChange |> cpu - # if sum(lif_wRecChange_cpu) != 0 # println("") # lif_vt_cpu = model.lif_vt |> cpu @@ -535,22 +534,23 @@ function train_snn(model, trainData, validateData, labelDict::Vector) # commit learned weight only if the model answer incorrectly finalAnswer_cpu = finalAnswer |> cpu - println("label $(labels[1]) finalAnswer $finalAnswer_cpu") + # println("label $(labels[1]) finalAnswer $finalAnswer_cpu") max = isequal.(finalAnswer_cpu[:,1], maximum(finalAnswer_cpu[:,1])) if sum(max) == 1 && findall(max)[1] -1 == labels[1] finalAnswer_cpu = findall(max)[1] - 1 - println("OK") - # println("label $(labels[1]) finalAnswer $finalAnswer_cpu") + println("label $(labels[1]) finalAnswer $finalAnswer_cpu CORRECT") elseif sum(max) == 1 && findall(max)[1] -1 != labels[1] finalAnswer = findall(max)[1] - 1 - IronpenGPU.learn!(model) - println("LEARNING") - # println("label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING") + IronpenGPU.learn!(model, device) + println("label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING") else - IronpenGPU.learn!(model) - println("LEARNING") - # println("epoch $epoch label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING") + IronpenGPU.learn!(model, device) + if sum(finalAnswer_cpu) > 1 + println("epoch $epoch label $(labels[1]) finalAnswer $finalAnswer_cpu LEARNING") + else + println("epoch $epoch label $(labels[1]) finalAnswer ZERO answer LEARNING") + end end # error("DEBUG -> main $(Dates.now())") diff --git a/previousVersion/0.0.05-alpha/src/IronpenGPU.jl b/previousVersion/0.0.5/src/IronpenGPU.jl similarity index 100% rename from previousVersion/0.0.05-alpha/src/IronpenGPU.jl rename to previousVersion/0.0.5/src/IronpenGPU.jl diff --git a/previousVersion/0.0.05-alpha/src/forward.jl b/previousVersion/0.0.5/src/forward.jl similarity index 68% rename from previousVersion/0.0.05-alpha/src/forward.jl rename to previousVersion/0.0.5/src/forward.jl index ef7f79d..a62c636 100644 --- a/previousVersion/0.0.05-alpha/src/forward.jl +++ b/previousVersion/0.0.5/src/forward.jl @@ -18,7 +18,7 @@ function (kfn::kfn_1)(input::AbstractArray) # what to do at the start of learning round if view(kfn.learningStage, 1)[1] == 1 # reset learning params - kfn.zit_cumulative .= 0 + kfn.zitCumulative .= 0 kfn.lif_vt .= 0 kfn.lif_wRecChange .= 0 @@ -118,7 +118,7 @@ function (kfn::kfn_1)(input::AbstractArray) reshape(kfn.lif_zt, (size(input, 1), :, 1, size(input, 3))), reshape(kfn.alif_zt, (size(input, 1), :, 1, size(input, 3))), dims=2) kfn.zit .= reshape(_zit, (size(input, 1), :, size(input, 3))) - kfn.zit_cumulative .+= kfn.zit + kfn.zitCumulative .+= kfn.zit # project 3D kfn zit into 4D on zit i1, i2, i3, i4 = size(kfn.on_zit) @@ -243,7 +243,7 @@ function lifForward( zit, linear_to_cartesian, ) i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index - + if i <= length(wRec) # cartesian index i1, i2, i3, i4 = linear_to_cartesian(i, size(wRec)) @@ -273,7 +273,7 @@ function lifForward( zit, vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4] # reset counter if neuron fires - neuronInactivityCounter[i1,i2,i3,i4] = 10000 + neuronInactivityCounter[i1,i2,i3,i4] = 0 else zt[i1,i2,i3,i4] = 0 neuronInactivityCounter[i1,i2,i3,i4] -= 1 @@ -291,7 +291,7 @@ function lifForward( zit, # count synaptic inactivity if !iszero(wRec[i1,i2,i3,i4]) # check if this is wRec subscription if !iszero(zit[i1,i2,i3,i4]) # synapse is active, reset counter - synapticInactivityCounter[i1,i2,i3,i4] = 10000 + synapticInactivityCounter[i1,i2,i3,i4] += 1 else # synapse is inactive, counting synapticInactivityCounter[i1,i2,i3,i4] -= 1 end @@ -416,7 +416,7 @@ function alifForward( zit, linear_to_cartesian, ) i = (blockIdx().x - 1) * blockDim().x + threadIdx().x # gpu threads index - + if i <= length(wRec) # cartesian index i1, i2, i3, i4 = linear_to_cartesian(i, size(wRec)) @@ -456,7 +456,7 @@ function alifForward( zit, firingCounter[i1,i2,i3,i4] += 1 vt[i1,i2,i3,i4] = vRest[i1,i2,i3,i4] a[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] * a[i1,i2,i3,i4]) + 1 - neuronInactivityCounter[i1,i2,i3,i4] = 10000 + neuronInactivityCounter[i1,i2,i3,i4] = 0 else zt[i1,i2,i3,i4] = 0 a[i1,i2,i3,i4] = (rho[i1,i2,i3,i4] * a[i1,i2,i3,i4]) @@ -478,7 +478,7 @@ function alifForward( zit, # count synaptic inactivity if !iszero(wRec[i1,i2,i3,i4]) # check if this is wRec subscription if !iszero(zit[i1,i2,i3,i4]) # synapse is active, reset counter - synapticInactivityCounter[i1,i2,i3,i4] = 10000 + synapticInactivityCounter[i1,i2,i3,i4] += 1 else # synapse is inactive, counting synapticInactivityCounter[i1,i2,i3,i4] -= 1 end @@ -612,238 +612,238 @@ function onForward( zit, return nothing end -function lifForward(kfn_zit::Array{T}, - zit::Array{T}, - wRec::Array{T}, - vt0::Array{T}, - vt1::Array{T}, - vth::Array{T}, - vRest::Array{T}, - zt1::Array{T}, - alpha::Array{T}, - phi::Array{T}, - epsilonRec::Array{T}, - refractoryCounter::Array{T}, - refractoryDuration::Array{T}, - gammaPd::Array{T}, - firingCounter::Array{T}, - arrayProjection4d::Array{T}, - recSignal::Array{T}, - decayed_vt0::Array{T}, - decayed_epsilonRec::Array{T}, - vt1_diff_vth::Array{T}, - vt1_diff_vth_div_vth::Array{T}, - gammaPd_div_vth::Array{T}, - phiActivation::Array{T}, - ) where T<:Number +# function lifForward(kfn_zit::Array{T}, +# zit::Array{T}, +# wRec::Array{T}, +# vt0::Array{T}, +# vt1::Array{T}, +# vth::Array{T}, +# vRest::Array{T}, +# zt1::Array{T}, +# alpha::Array{T}, +# phi::Array{T}, +# epsilonRec::Array{T}, +# refractoryCounter::Array{T}, +# refractoryDuration::Array{T}, +# gammaPd::Array{T}, +# firingCounter::Array{T}, +# arrayProjection4d::Array{T}, +# recSignal::Array{T}, +# decayed_vt0::Array{T}, +# decayed_epsilonRec::Array{T}, +# vt1_diff_vth::Array{T}, +# vt1_diff_vth_div_vth::Array{T}, +# gammaPd_div_vth::Array{T}, +# phiActivation::Array{T}, +# ) where T<:Number - # project 3D kfn zit into 4D lif zit - i1, i2, i3, i4 = size(alif_wRec) - lif_zit .= reshape(kfn_zit, (i1, i2, 1, i4)) .* lif_arrayProjection4d +# # project 3D kfn zit into 4D lif zit +# i1, i2, i3, i4 = size(alif_wRec) +# lif_zit .= reshape(kfn_zit, (i1, i2, 1, i4)) .* lif_arrayProjection4d - for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch - if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active - @. @views refractoryCounter[:,:,i,j] -= 1 - @. @views zt1[:,:,i,j] = 0 - @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] - @. @views phi[:,:,i,j] = 0 +# for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch +# if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active +# @. @views refractoryCounter[:,:,i,j] -= 1 +# @. @views zt1[:,:,i,j] = 0 +# @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] +# @. @views phi[:,:,i,j] = 0 - # compute epsilonRec - @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] - @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] - else # refractory period is inactive - @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j] - @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] - @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) +# # compute epsilonRec +# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] +# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] +# else # refractory period is inactive +# @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j] +# @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] +# @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) - if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j])) - @. @views zt1[:,:,i,j] = 1 - @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] - @. @views firingCounter[:,:,i,j] += 1 - @. @views vt1[:,:,i,j] = vRest[:,:,i,j] - else - @. @views zt1[:,:,i,j] = 0 - end +# if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j])) +# @. @views zt1[:,:,i,j] = 1 +# @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] +# @. @views firingCounter[:,:,i,j] += 1 +# @. @views vt1[:,:,i,j] = vRest[:,:,i,j] +# else +# @. @views zt1[:,:,i,j] = 0 +# end - # compute phi, there is a difference from alif formula - @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] - @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] - @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] - @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) - @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] +# # compute phi, there is a difference from alif formula +# @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] +# @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] +# @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] +# @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) +# @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] - # compute epsilonRec - @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] - @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] - end - end -end +# # compute epsilonRec +# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] +# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] +# end +# end +# end -function alifForward(zit::Array{T}, - wRec::Array{T}, - vt0::Array{T}, - vt1::Array{T}, - vth::Array{T}, - vRest::Array{T}, - zt1::Array{T}, - alpha::Array{T}, - phi::Array{T}, - epsilonRec::Array{T}, - refractoryCounter::Array{T}, - refractoryDuration::Array{T}, - gammaPd::Array{T}, - firingCounter::Array{T}, - recSignal::Array{T}, - decayed_vt0::Array{T}, - decayed_epsilonRec::Array{T}, - vt1_diff_vth::Array{T}, - vt1_diff_vth_div_vth::Array{T}, - gammaPd_div_vth::Array{T}, - phiActivation::Array{T}, +# function alifForward(zit::Array{T}, +# wRec::Array{T}, +# vt0::Array{T}, +# vt1::Array{T}, +# vth::Array{T}, +# vRest::Array{T}, +# zt1::Array{T}, +# alpha::Array{T}, +# phi::Array{T}, +# epsilonRec::Array{T}, +# refractoryCounter::Array{T}, +# refractoryDuration::Array{T}, +# gammaPd::Array{T}, +# firingCounter::Array{T}, +# recSignal::Array{T}, +# decayed_vt0::Array{T}, +# decayed_epsilonRec::Array{T}, +# vt1_diff_vth::Array{T}, +# vt1_diff_vth_div_vth::Array{T}, +# gammaPd_div_vth::Array{T}, +# phiActivation::Array{T}, - epsilonRecA::Array{T}, - avth::Array{T}, - a::Array{T}, - beta::Array{T}, - rho::Array{T}, - phi_x_epsilonRec::Array{T}, - phi_x_beta::Array{T}, - rho_diff_phi_x_beta::Array{T}, - rho_div_phi_x_beta_x_epsilonRecA::Array{T}, - beta_x_a::Array{T}, - ) where T<:Number +# epsilonRecA::Array{T}, +# avth::Array{T}, +# a::Array{T}, +# beta::Array{T}, +# rho::Array{T}, +# phi_x_epsilonRec::Array{T}, +# phi_x_beta::Array{T}, +# rho_diff_phi_x_beta::Array{T}, +# rho_div_phi_x_beta_x_epsilonRecA::Array{T}, +# beta_x_a::Array{T}, +# ) where T<:Number - for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch - if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active - @. @views refractoryCounter[:,:,i,j] -= 1 - @. @views zt1[:,:,i,j] = 0 - @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] - @. @views phi[:,:,i,j] = 0 - @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] +# for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch +# if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active +# @. @views refractoryCounter[:,:,i,j] -= 1 +# @. @views zt1[:,:,i,j] = 0 +# @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] +# @. @views phi[:,:,i,j] = 0 +# @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] - # compute epsilonRec - @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] - @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] +# # compute epsilonRec +# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] +# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] - # compute epsilonRecA - @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j] - @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j] - @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j] - @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j] - @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] +# # compute epsilonRecA +# @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j] +# @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j] +# @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j] +# @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j] +# @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] - # compute avth - @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j] - @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j] +# # compute avth +# @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j] +# @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j] - else # refractory period is inactive - @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j] - @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] - @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) +# else # refractory period is inactive +# @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j] +# @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] +# @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) - # compute avth - @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j] - @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j] +# # compute avth +# @. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j] +# @. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j] - if sum(@view(vt1[:,:,i,j])) > sum(@view(avth[:,:,i,j])) - @. @views zt1[:,:,i,j] = 1 - @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] - @. @views firingCounter[:,:,i,j] += 1 - @. @views vt1[:,:,i,j] = vRest[:,:,i,j] - @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] - @. @views a[:,:,i,j] = a[:,:,i,j] += 1 - else - @. @views zt1[:,:,i,j] = 0 - @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] - end +# if sum(@view(vt1[:,:,i,j])) > sum(@view(avth[:,:,i,j])) +# @. @views zt1[:,:,i,j] = 1 +# @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] +# @. @views firingCounter[:,:,i,j] += 1 +# @. @views vt1[:,:,i,j] = vRest[:,:,i,j] +# @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] +# @. @views a[:,:,i,j] = a[:,:,i,j] += 1 +# else +# @. @views zt1[:,:,i,j] = 0 +# @. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j] +# end - # compute phi, there is a difference from alif formula - @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] - @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] - @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] - @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) - @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] +# # compute phi, there is a difference from alif formula +# @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] +# @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] +# @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] +# @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) +# @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] - # compute epsilonRec - @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] - @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] +# # compute epsilonRec +# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] +# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] - # compute epsilonRecA - @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j] - @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j] - @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j] - @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j] - @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] - end - end -end +# # compute epsilonRecA +# @. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j] +# @. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j] +# @. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j] +# @. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j] +# @. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] +# end +# end +# end -function onForward(kfn_zit::Array{T}, - zit::Array{T}, - wOut::Array{T}, - vt0::Array{T}, - vt1::Array{T}, - vth::Array{T}, - vRest::Array{T}, - zt1::Array{T}, - alpha::Array{T}, - phi::Array{T}, - epsilonRec::Array{T}, - refractoryCounter::Array{T}, - refractoryDuration::Array{T}, - gammaPd::Array{T}, - firingCounter::Array{T}, - arrayProjection4d::Array{T}, - recSignal::Array{T}, - decayed_vt0::Array{T}, - decayed_epsilonRec::Array{T}, - vt1_diff_vth::Array{T}, - vt1_diff_vth_div_vth::Array{T}, - gammaPd_div_vth::Array{T}, - phiActivation::Array{T}, - ) where T<:Number +# function onForward(kfn_zit::Array{T}, +# zit::Array{T}, +# wOut::Array{T}, +# vt0::Array{T}, +# vt1::Array{T}, +# vth::Array{T}, +# vRest::Array{T}, +# zt1::Array{T}, +# alpha::Array{T}, +# phi::Array{T}, +# epsilonRec::Array{T}, +# refractoryCounter::Array{T}, +# refractoryDuration::Array{T}, +# gammaPd::Array{T}, +# firingCounter::Array{T}, +# arrayProjection4d::Array{T}, +# recSignal::Array{T}, +# decayed_vt0::Array{T}, +# decayed_epsilonRec::Array{T}, +# vt1_diff_vth::Array{T}, +# vt1_diff_vth_div_vth::Array{T}, +# gammaPd_div_vth::Array{T}, +# phiActivation::Array{T}, +# ) where T<:Number - # project 3D kfn zit into 4D lif zit - zit .= reshape(kfn_zit, - (size(wOut, 1), size(wOut, 2), 1, size(wOut, 4))) .* arrayProjection4d +# # project 3D kfn zit into 4D lif zit +# zit .= reshape(kfn_zit, +# (size(wOut, 1), size(wOut, 2), 1, size(wOut, 4))) .* arrayProjection4d - for j in 1:size(wOut, 4), i in 1:size(wOut, 3) # compute along neurons axis of every batch - if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active - @. @views refractoryCounter[:,:,i,j] -= 1 - @. @views zt1[:,:,i,j] = 0 - @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] - @. @views phi[:,:,i,j] = 0 +# for j in 1:size(wOut, 4), i in 1:size(wOut, 3) # compute along neurons axis of every batch +# if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active +# @. @views refractoryCounter[:,:,i,j] -= 1 +# @. @views zt1[:,:,i,j] = 0 +# @. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] +# @. @views phi[:,:,i,j] = 0 - # compute epsilonRec - @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] - @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] - else # refractory period is inactive - @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wOut[:,:,i,j] - @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] - @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) +# # compute epsilonRec +# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] +# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] +# else # refractory period is inactive +# @. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wOut[:,:,i,j] +# @. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j] +# @view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j])) - if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j])) - @. @views zt1[:,:,i,j] = 1 - @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] - @. @views firingCounter[:,:,i,j] += 1 - @. @views vt1[:,:,i,j] = vRest[:,:,i,j] - else - @. @views zt1[:,:,i,j] = 0 - end +# if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j])) +# @. @views zt1[:,:,i,j] = 1 +# @. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j] +# @. @views firingCounter[:,:,i,j] += 1 +# @. @views vt1[:,:,i,j] = vRest[:,:,i,j] +# else +# @. @views zt1[:,:,i,j] = 0 +# end - # compute phi, there is a difference from alif formula - @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] - @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] - @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] - @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) - @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] +# # compute phi, there is a difference from alif formula +# @. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j] +# @. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j] +# @. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j] +# @view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j]))) +# @. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j] - # compute epsilonRec - @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] - @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] - end - end -end +# # compute epsilonRec +# @. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j] +# @. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j] +# end +# end +# end diff --git a/previousVersion/0.0.05-alpha/src/interface.jl b/previousVersion/0.0.5/src/interface.jl similarity index 100% rename from previousVersion/0.0.05-alpha/src/interface.jl rename to previousVersion/0.0.5/src/interface.jl diff --git a/previousVersion/0.0.05-alpha/src/learn.jl b/previousVersion/0.0.5/src/learn.jl similarity index 64% rename from previousVersion/0.0.05-alpha/src/learn.jl rename to previousVersion/0.0.5/src/learn.jl index ad34b6e..4d9db2c 100644 --- a/previousVersion/0.0.05-alpha/src/learn.jl +++ b/previousVersion/0.0.5/src/learn.jl @@ -266,22 +266,26 @@ end function learn!(kfn::kfn_1, device=cpu) # lif learn - lifLearn!(kfn.lif_wRec, - kfn.lif_wRecChange, - kfn.lif_arrayProjection4d, - kfn.lif_neuronInactivityCounter, - kfn.lif_synapticConnectionNumber, - kfn.zit_cumulative, - device) - + kfn.lif_wRec, kfn.lif_neuronInactivityCounter, kfn.lif_synapticInactivityCounter = + lifLearn(kfn.lif_wRec, + kfn.lif_wRecChange, + kfn.lif_arrayProjection4d, + kfn.lif_neuronInactivityCounter, + kfn.lif_synapticInactivityCounter, + kfn.lif_synapticConnectionNumber, + kfn.zitCumulative, + device) + # alif learn - alifLearn!(kfn.alif_wRec, - kfn.alif_wRecChange, - kfn.alif_arrayProjection4d, - kfn.alif_neuronInactivityCounter, - kfn.alif_synapticConnectionNumber, - kfn.zit_cumulative, - device) + kfn.alif_wRec, kfn.alif_neuronInactivityCounter, kfn.alif_synapticInactivityCounter = + alifLearn(kfn.alif_wRec, + kfn.alif_wRecChange, + kfn.alif_arrayProjection4d, + kfn.alif_neuronInactivityCounter, + kfn.alif_synapticInactivityCounter, + kfn.alif_synapticConnectionNumber, + kfn.zitCumulative, + device) # on learn onLearn!(kfn.on_wOut, @@ -295,58 +299,108 @@ function learn!(kfn::kfn_1, device=cpu) # error("DEBUG -> kfn learn! $(Dates.now())") end -function lifLearn!(wRec, +function lifLearn(wRec, wRecChange, arrayProjection4d, - inactivityCounter, + neuronInactivityCounter, + synapticInactivityCounter, synapticConnectionNumber, - zit_cumulative, + zitCumulative, device) + #WORKING - synapticInactivityCounter -10000 to 10000, weight change liquidity range from 1.0 to 0.1 respectively + # merge learning weight with average learning weight of all batch wRec .+= (sum(wRecChange, dims=4) ./ (size(wRec, 4))) .* arrayProjection4d + arrayProjection4d_cpu = arrayProjection4d |> cpu wRec_cpu = wRec |> cpu wRec_cpu = wRec_cpu[:,:,:,1] # since every batch has the same neuron wRec, (row, col, n) - inactivityCounter_cpu = inactivityCounter |> cpu - inactivityCounter_cpu = inactivityCounter_cpu[:,:,:,1] # (row, col, n) - zit_cumulative_cpu = zit_cumulative |> cpu - zit_cumulative_cpu = zit_cumulative_cpu[:,:,1] # (row, col) + neuronInactivityCounter_cpu = neuronInactivityCounter |> cpu + neuronInactivityCounter_cpu = neuronInactivityCounter_cpu[:,:,:,1] # (row, col, n) + synapticInactivityCounter_cpu = synapticInactivityCounter |> cpu + synapticInactivityCounter_cpu = synapticInactivityCounter_cpu[:,:,:,1] + zitCumulative_cpu = zitCumulative |> cpu + zitCumulative_cpu = zitCumulative_cpu[:,:,1] # (row, col) # weak / negative synaptic connection will get randomed in neuroplasticity() wRec_cpu = GeneralUtils.replaceBetween.(wRec_cpu, 0.0, 0.1, -1.0) # mark with -1.0 - # synaptic connection that has no inactivity will get randomed in neuroplasticity() - GeneralUtils.replace_elements!(inactivityCounter_cpu, 0.0, wRec_cpu, -1.0) - # reset lif_inactivity elements to 10000 - GeneralUtils.replace_elements!(inactivityCounter_cpu, 0.0, -9.0) # -9.0 is base value + # synaptic connection that has no activity will get randomed in neuroplasticity() + mask = isless.(synapticInactivityCounter_cpu, -10_000) + GeneralUtils.replace_elements!(mask, 1, wRec_cpu, -1.0) + # reset lif_inactivity elements to base value + GeneralUtils.replace_elements!(mask, 1, synapticInactivityCounter_cpu, 0.0) + + # neuroplasticity, work on CPU side + wRec_cpu = neuroplasticity(synapticConnectionNumber, + zitCumulative_cpu, + wRec_cpu, + neuronInactivityCounter_cpu, + synapticInactivityCounter_cpu) - #WORKING neuroplasticity - wRec_cpu = neuroplasticity(synapticConnectionNumber, zit_cumulative_cpu, wRec_cpu, - inactivityCounter_cpu) - error("DEBUG -> lifLearn! $(Dates.now())") - # #TODO send to device with correct dimension - # wRec = wRec |> device - # inactivityCounter = inactivityCounter_cpu |> device + wRec_cpu = wRec_cpu .* arrayProjection4d_cpu + wRec = wRec_cpu |> device + neuronInactivityCounter_cpu = neuronInactivityCounter_cpu .* arrayProjection4d_cpu + neuronInactivityCounter = neuronInactivityCounter_cpu |> device + + synapticInactivityCounter_cpu = synapticInactivityCounter_cpu .* arrayProjection4d_cpu + synapticInactivityCounter = synapticInactivityCounter_cpu |> device + + # error("DEBUG -> lifLearn! $(Dates.now())") + return wRec, neuronInactivityCounter, synapticInactivityCounter end -function alifLearn!(wRec, +function alifLearn(wRec, wRecChange, arrayProjection4d, - inactivityCounter, + neuronInactivityCounter, + synapticInactivityCounter, synapticConnectionNumber, - zit_cumulative, + zitCumulative, device) - # merge learning weight with average learning weight + #WORKING - synapticInactivityCounter -10000 to 10000, weight change liquidity range from 1.0 to 0.1 respectively + + # merge learning weight with average learning weight of all batch wRec .+= (sum(wRecChange, dims=4) ./ (size(wRec, 4))) .* arrayProjection4d + arrayProjection4d_cpu = arrayProjection4d |> cpu + wRec_cpu = wRec |> cpu + wRec_cpu = wRec_cpu[:,:,:,1] # since every batch has the same neuron wRec, (row, col, n) + neuronInactivityCounter_cpu = neuronInactivityCounter |> cpu + neuronInactivityCounter_cpu = neuronInactivityCounter_cpu[:,:,:,1] # (row, col, n) + synapticInactivityCounter_cpu = synapticInactivityCounter |> cpu + synapticInactivityCounter_cpu = synapticInactivityCounter_cpu[:,:,:,1] + zitCumulative_cpu = zitCumulative |> cpu + zitCumulative_cpu = zitCumulative_cpu[:,:,1] # (row, col) + # weak / negative synaptic connection will get randomed in neuroplasticity() - wRec .= GeneralUtils.replaceLessThan.(wRec, 0.01, 0.0) + wRec_cpu = GeneralUtils.replaceBetween.(wRec_cpu, 0.0, 0.1, -1.0) # mark with -1.0 + + # synaptic connection that has no activity will get randomed in neuroplasticity() + mask = isless.(synapticInactivityCounter_cpu, -10_000) + GeneralUtils.replace_elements!(mask, 1, wRec_cpu, -1.0) + # reset alif_inactivity elements to base value + GeneralUtils.replace_elements!(mask, 1, synapticInactivityCounter_cpu, 0.0) - #TODO synaptic strength + # neuroplasticity, work on CPU side + wRec_cpu = neuroplasticity(synapticConnectionNumber, + zitCumulative_cpu, + wRec_cpu, + neuronInactivityCounter_cpu, + synapticInactivityCounter_cpu) + + wRec_cpu = wRec_cpu .* arrayProjection4d_cpu + wRec = wRec_cpu |> device + + neuronInactivityCounter_cpu = neuronInactivityCounter_cpu .* arrayProjection4d_cpu + neuronInactivityCounter = neuronInactivityCounter_cpu |> device - #TODO neuroplasticity + synapticInactivityCounter_cpu = synapticInactivityCounter_cpu .* arrayProjection4d_cpu + synapticInactivityCounter = synapticInactivityCounter_cpu |> device + # error("DEBUG -> alifLearn! $(Dates.now())") + return wRec, neuronInactivityCounter, synapticInactivityCounter end function onLearn!(wOut, @@ -365,53 +419,67 @@ function onLearn!(wOut, end function neuroplasticity(synapticConnectionNumber, - zit_cumulative, # (row, col) + zitCumulative, # (row, col) wRec, # (row, col, n) - inactivityCounter_cpu) # (row, col, n) + neuronInactivityCounter, + synapticInactivityCounter) # (row, col, n) i1,i2,i3 = size(wRec) # for each neuron, find total number of synaptic conn that should draw # new connection to firing and non-firing neurons pool subToFireNeuron_toBe = Int(floor(0.7 * synapticConnectionNumber)) - subToNonFiringNeuron_toBe = synapticConnectionNumber - subToFireNeuron_toBe - #WORKING for each neuron, count how many synap already subscribed to firing-neurons - subToFireNeuron_current = sum((!iszero).(zit_cumulative .* wRec), dims=(1,2)) # (1, 1, n) - subToNonFiringNeuron_current = synapticConnectionNumber .- subToFireNeuron_current # (1, 1, n) - mask = (!iszero).(zit_cumulative) # mask of firing neurons = 1, non-firing = 0 + # for each neuron, count how many synap already subscribed to firing-neurons + zw = zitCumulative .* wRec + subToFireNeuron_current = sum(GeneralUtils.isBetween.(zw, 0.0, 100.0), dims=(1,2)) # (1, 1, n) + zitMask = (!iszero).(zitCumulative) # zitMask of firing neurons = 1, non-firing = 0 projection = ones(i1,i2,i3) - mask = mask .* projection # (row, col, n) + zitMask = zitMask .* projection # (row, col, n) totalNewConn = sum(isequal.(wRec, -1.0), dims=(1,2)) # count new conn mark (-1.0), (1, 1, n) - println("mask ", size(mask)) - println("wRec ", size(wRec)) - println("inactivityCounter_cpu ", size(inactivityCounter_cpu)) - println("totalNeurons ", totalNewConn, size(totalNewConn)) - error("DEBUG -> neuroplasticity $(Dates.now())") + + # clear -1.0 marker + GeneralUtils.replace_elements!(wRec, -1.0, synapticInactivityCounter, -0.99) + GeneralUtils.replace_elements!(wRec, -1.0, 0.0) # -1.0 marker is no longer required + for i in 1:i3 + if neuronInactivityCounter[1:1:i][1] < -10_000 # neuron die i.e. reset all weight + neuronInactivityCounter[:,:,i] .= 0 # reset + w = wRec(i1,i2,1,synapticConnectionNumber) + wRec[:,:,i] = w - - # add new conn to firing neurons pool - remaining = GeneralUtils.replace_elements(mask[:,:,i], - 1, - wRecmask[:,:,i], - inactivityCounter_cpumask[:,:,i], - totalNewConn[:,:,i]) - - #TODO add new conn to non-firing neurons pool - + a = similar(w) .= -0.99 # synapticConnectionNumber of this neuron + mask = (!iszero).(w) + GeneralUtils.replace_elements!(mask, 1, a, 0) + synapticInactivityCounter[:,:,i] = a + else + remaining = 0 + if subToFireNeuron_current[1,1,i] < subToFireNeuron_toBe + toAddConn = subToFireNeuron_toBe - subToFireNeuron_current[1,1,i] + totalNewConn[1,1,i] = totalNewConn[1,1,i] - toAddConn + # add new conn to firing neurons pool + remaining = addNewSynapticConn!(zitMask[:,:,i], 1, + @view(wRec[:,:,i]), + @view(synapticInactivityCounter[:,:,i]), + toAddConn) + totalNewConn[1,1,i] += remaining + end + + # add new conn to non-firing neurons pool + remaining = addNewSynapticConn!(zitMask[:,:,i], 0, + @view(wRec[:,:,i]), + @view(synapticInactivityCounter[:,:,i]), + totalNewConn[1,1,i]) + if remaining > 0 # final get-all round if somehow non-firing pool has not enough slot + remaining = addNewSynapticConn!(zitMask[:,:,i], 1, + @view(wRec[:,:,i]), + @view(synapticInactivityCounter[:,:,i]), + remaining) + end + end end - - - - - - newFiringConn = subToFireNeuron_toBe - subToFireNeuron_current - newFiringConn = newFiringConn > 0 ? newFiringConn : 0 - - newNonFiringConn = subToNonFiringNeuron_toBe - subToNonFiringNeuron_current - + # error("DEBUG -> neuroplasticity $(Dates.now())") return wRec end @@ -454,10 +522,6 @@ end - - - - diff --git a/previousVersion/0.0.5/src/snnUtil.jl b/previousVersion/0.0.5/src/snnUtil.jl new file mode 100644 index 0000000..ca7c024 --- /dev/null +++ b/previousVersion/0.0.5/src/snnUtil.jl @@ -0,0 +1,191 @@ +module snnUtil + +export refractoryStatus!, addNewSynapticConn! + +using Random + +#------------------------------------------------------------------------------------------------100 + +function refractoryStatus!(refractoryCounter, refractoryActive, refractoryInactive) + d1, d2, d3, d4 = size(refractoryCounter) + for j in 1:d4 + for i in 1:d3 + if refractoryCounter[1, 1, i, j] > 0 # inactive + view(refractoryActive, 1, 1, i, j) .= 0 + view(refractoryInactive, 1, 1, i, j) .= 1 + else # active + view(refractoryActive, 1, 1, i, j) .= 1 + view(refractoryInactive, 1, 1, i, j) .= 0 + end + end + end +end + +# function frobenius_distance(A, B) +# # Check if the matrices have the same size +# if size(A) != size(B) +# error("The matrices must have the same size") +# end +# # Initialize the distance to zero +# distance = 0.0 +# # Loop over the elements of the matrices and add the squared differences +# for i in 1:size(A, 1) +# for j in 1:size(A, 2) +# distance += (A[i, j] - B[i, j])^2 +# end +# end +# # Return the square root of the distance +# return sqrt(distance) +# end + +function addNewSynapticConn!(mask::AbstractArray{<:Any}, x::Number, wRec::AbstractArray{<:Any}, + counter::AbstractArray{<:Any}, n=0; + rng::AbstractRNG=MersenneTwister(1234)) + # println("mask ", mask, size(mask)) + # println("") + # println("x ", x, size(x)) + # println("") + # println("wRec ", wRec, size(wRec)) + # println("") + # println("counter ", counter, size(counter)) + # println("") + # println("n ", n, size(n)) + # println("") + + total_x_tobeReplced = sum(isequal.(mask, x)) + remaining = 0 + if n == 0 || n > total_x_tobeReplced + remaining = n - total_x_tobeReplced + n = total_x_tobeReplced + end + + # check if mask and wRec have the same size + if size(mask) != size(wRec) + error("mask and wRec must have the same size") + end + # get the indices of elements in mask that equal x + indices = findall(x -> x == x, mask) + alreadySub = findall(x -> x != 0, wRec) # get already subscribe + setdiff!(indices, alreadySub) # remove already sub conn from pool + + # shuffle the indices using the rng function + shuffle!(rng, indices) + # select the first n indices + selected = indices[1:n] + # replace the elements in wRec at the selected positions with a + for i in selected + wRec[i] = 0.1 #rand(0.1:0.01:0.3) + if counter !== nothing + counter[i] = 0 # reset + end + end + # println("==================") + # println("mask ", mask, size(mask)) + # println("") + # println("x ", x, size(x)) + # println("") + # println("wRec ", wRec, size(wRec)) + # println("") + # println("counter ", counter, size(counter)) + # println("") + # println("n ", n, size(n)) + # println("") + # error("DEBUG addNewSynapticConn!") + return remaining +end + + +# function addNewSynapticConn!(mask::AbstractArray{<:Any}, x::Number, A::AbstractArray{<:Any}, +# A2::AbstractArray{<:Any}, n=0; +# rng::AbstractRNG=MersenneTwister(1234)) +# # println("mask ", mask, size(mask)) +# # println("") +# # println("x ", x, size(x)) +# # println("") +# # println("A ", A, size(A)) +# # println("") +# # println("A2 ", A2, size(A2)) +# # println("") +# # println("n ", n, size(n)) +# # println("") + +# total_x_tobeReplced = sum(isequal.(mask, x)) +# remaining = 0 +# if n == 0 || n > total_x_tobeReplced +# remaining = n - total_x_tobeReplced +# n = total_x_tobeReplced +# end + +# # check if mask and A have the same size +# if size(mask) != size(A) +# error("mask and A must have the same size") +# end +# # get the indices of elements in mask that equal x +# indices = findall(x -> x == x, mask) +# # shuffle the indices using the rng function +# shuffle!(rng, indices) +# # select the first n indices +# selected = indices[1:n] +# # replace the elements in A at the selected positions with a +# for i in selected +# A[i] = rand(0.1:0.01:0.3) +# if A2 !== nothing +# A2[i] = 10000 +# end +# end +# # println("==================") +# # println("mask ", mask, size(mask)) +# # println("") +# # println("x ", x, size(x)) +# # println("") +# # println("A ", A, size(A)) +# # println("") +# # println("A2 ", A2, size(A2)) +# # println("") +# # println("n ", n, size(n)) +# # println("") +# # error("DEBUG addNewSynapticConn!") +# return remaining +# end + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +end # module \ No newline at end of file diff --git a/previousVersion/0.0.05-alpha/src/type.jl b/previousVersion/0.0.5/src/type.jl similarity index 95% rename from previousVersion/0.0.05-alpha/src/type.jl rename to previousVersion/0.0.5/src/type.jl index 7b203ea..84fb6ac 100644 --- a/previousVersion/0.0.05-alpha/src/type.jl +++ b/previousVersion/0.0.5/src/type.jl @@ -23,7 +23,7 @@ Base.@kwdef mutable struct kfn_1 <: knowledgeFn learningStage::Union{AbstractArray, Nothing} = nothing # 0 inference, 1 start, 2 during, 3 end learning inputSize::Union{AbstractArray, Nothing} = nothing zit::Union{AbstractArray, Nothing} = nothing # 3D activation matrix - zit_cumulative::Union{AbstractArray, Nothing} = nothing + zitCumulative::Union{AbstractArray, Nothing} = nothing exInType::Union{AbstractArray, Nothing} = nothing modelError::Union{AbstractArray, Nothing} = nothing # store RSNN error outputError::Union{AbstractArray, Nothing} = nothing # store output neurons error @@ -185,7 +185,7 @@ function kfn_1(params::Dict; device=cpu) # activation matrix kfn.zit = zeros(row, col, batch) |> device - kfn.zit_cumulative = (similar(kfn.zit) .= 0) + kfn.zitCumulative = (similar(kfn.zit) .= 0) kfn.modelError = zeros(1) |> device # ---------------------------------------------------------------------------- # @@ -196,20 +196,9 @@ function kfn_1(params::Dict; device=cpu) lif_n = kfn.params[:computeNeuron][:lif][:numbers][1] * kfn.params[:computeNeuron][:lif][:numbers][2] # subscription - w = zeros(row, col, lif_n) synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent] kfn.lif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100)) - for slice in eachslice(w, dims=3) - pool = shuffle!([1:row*col...])[1:kfn.lif_synapticConnectionNumber] - for i in pool - slice[i] = rand() # assign weight to synaptic connection. /10 to start small, - # otherwise RSNN's vt Usually stay negative (-) - end - end - - # 10% of neuron connection should be enough to start to make neuron fires - should_be_avg_weight = 1 / (0.1 * lif_n) - w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight + w = wRec(row, col, lif_n, kfn.lif_synapticConnectionNumber) # project 3D w into 4D kfn.lif_wRec (row, col, n, batch) kfn.lif_wRec = reshape(w, (row, col, lif_n, 1)) .* ones(row, col, lif_n, batch) |> device @@ -234,10 +223,11 @@ function kfn_1(params::Dict; device=cpu) kfn.lif_firingCounter = (similar(kfn.lif_wRec) .= 0) kfn.lif_firingTargetFrequency = (similar(kfn.lif_wRec) .= 0.1) - kfn.lif_neuronInactivityCounter = (similar(kfn.lif_wRec) .= 10000) - kfn.lif_synapticInactivityCounter = Array(similar(kfn.lif_wRec) .= -9) # -9 for non-sub conn + kfn.lif_neuronInactivityCounter = (similar(kfn.lif_wRec) .= 0) + kfn.lif_synapticInactivityCounter = Array(similar(kfn.lif_wRec) .= -0.99) # -9 for non-sub conn mask = Array((!iszero).(kfn.lif_wRec)) - GeneralUtils.replace_elements!(mask, 1, kfn.lif_synapticInactivityCounter, 10000) + # initial value subscribed conn, synapticInactivityCounter range -10000 to +10000 + GeneralUtils.replace_elements!(mask, 1, kfn.lif_synapticInactivityCounter, 0) kfn.lif_synapticInactivityCounter = kfn.lif_synapticInactivityCounter |> device kfn.lif_arrayProjection4d = (similar(kfn.lif_wRec) .= 1) @@ -255,20 +245,9 @@ function kfn_1(params::Dict; device=cpu) alif_n = kfn.params[:computeNeuron][:alif][:numbers][1] * kfn.params[:computeNeuron][:alif][:numbers][2] # subscription - w = zeros(row, col, alif_n) synapticConnectionPercent = kfn.params[:computeNeuron][:alif][:params][:synapticConnectionPercent] kfn.alif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100)) - for slice in eachslice(w, dims=3) - pool = shuffle!([1:row*col...])[1:kfn.alif_synapticConnectionNumber] - for i in pool - slice[i] = rand() # assign weight to synaptic connection. /10 to start small, - # otherwise RSNN's vt Usually stay negative (-) - end - end - - # 10% of neuron connection should be enough to start to make neuron fires - should_be_avg_weight = 1 / (0.1 * alif_n) - w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight + w = wRec(row, col, alif_n, kfn.alif_synapticConnectionNumber) # project 3D w into 4D kfn.alif_wRec kfn.alif_wRec = reshape(w, (row, col, alif_n, 1)) .* ones(row, col, alif_n, batch) |> device @@ -293,10 +272,11 @@ function kfn_1(params::Dict; device=cpu) kfn.alif_firingCounter = (similar(kfn.alif_wRec) .= 0) kfn.alif_firingTargetFrequency = (similar(kfn.alif_wRec) .= 0.1) - kfn.alif_neuronInactivityCounter = (similar(kfn.alif_wRec) .= 10000) - kfn.alif_synapticInactivityCounter = Array(similar(kfn.alif_wRec) .= -9) # -9 for non-sub conn + kfn.alif_neuronInactivityCounter = (similar(kfn.alif_wRec) .= 0) + kfn.alif_synapticInactivityCounter = Array(similar(kfn.alif_wRec) .= -0.99) # -9 for non-sub conn mask = Array((!iszero).(kfn.alif_wRec)) - GeneralUtils.replace_elements!(mask, 1, kfn.alif_synapticInactivityCounter, 10000) + # initial value subscribed conn, synapticInactivityCounter range -10000 to +10000 + GeneralUtils.replace_elements!(mask, 1, kfn.alif_synapticInactivityCounter, 0) kfn.alif_synapticInactivityCounter = kfn.alif_synapticInactivityCounter |> device kfn.alif_arrayProjection4d = (similar(kfn.alif_wRec) .= 1) @@ -333,7 +313,6 @@ function kfn_1(params::Dict; device=cpu) synapticConnection = Int(floor(subable * synapticConnectionPercent/100)) for slice in eachslice(w, dims=3) # each slice is a neuron startInd = row*col - subable + 1 # e.g. 100(row*col) - 50(subable) = 50 -> startInd = 51 - # pool must contain only lif, alif neurons pool = shuffle!([startInd:row*col...])[1:synapticConnection] for i in pool @@ -342,9 +321,9 @@ function kfn_1(params::Dict; device=cpu) end end - # # 10% of neuron connection should be enough to start to make neuron fires - # should_be_avg_weight = 1 / (0.2 * n) - # w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight + # 10% of neuron connection should be enough to start to make neuron fires + should_be_avg_weight = 1 / (0.1 * n) + w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight # project 3D w into 4D kfn.lif_wOut (row, col, n, batch) kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device @@ -384,6 +363,25 @@ function kfn_1(params::Dict; device=cpu) return kfn end +function wRec(row, col, n, synapticConnectionNumber) + # subscription + w = zeros(row, col, n) + + for slice in eachslice(w, dims=3) + pool = shuffle!([1:row*col...])[1:synapticConnectionNumber] + for i in pool + slice[i] = rand() # assign weight to synaptic connection. /10 to start small, + # otherwise RSNN's vt Usually stay negative (-) + end + end + + # 10% of neuron connection should be enough to start to make neuron fires + should_be_avg_weight = 1 / (0.1 * synapticConnectionNumber) + w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight + + return w #(row, col, n) +end + @@ -425,10 +423,6 @@ end - - - - diff --git a/src/IronpenGPU.jl b/src/IronpenGPU.jl index 391ce27..df1fc93 100644 --- a/src/IronpenGPU.jl +++ b/src/IronpenGPU.jl @@ -27,11 +27,12 @@ using .interface """ version 0.0.5 Todo: + [] add weight liquidity [DONE] add excitatory/inhabitory matrix [-] add temporal summation in addition to already used spatial summation. CANCELLED, spatial summation every second until membrane potential reach a threshold is in itself a temporal summation. - [x] add neuroplasticity + [DONE] add neuroplasticity [4] implement dormant connection and pruning machanism. the longer the training the longer 0 weight stay 0. [] using RL to control learning signal