module type export # struct kfn_1 # function using Random, GeneralUtils #------------------------------------------------------------------------------------------------100 rng = MersenneTwister(1234) abstract type Ironpen end abstract type knowledgeFn <: Ironpen end #------------------------------------------------------------------------------------------------100 Base.@kwdef mutable struct kfn_1 <: knowledgeFn params::Union{Dict, Nothing} = nothing # store params of knowledgeFn itself for later use timeStep::Union{AbstractArray, Nothing} = nothing learningStage::Union{AbstractArray, Nothing} = nothing # 0 inference, 1 start, 2 during, 3 end learning inputSize::Union{AbstractArray, Nothing} = nothing zit::Union{AbstractArray, Nothing} = nothing # 3D activation matrix zitCumulative::Union{AbstractArray, Nothing} = nothing exInType::Union{AbstractArray, Nothing} = nothing modelError::Union{AbstractArray, Nothing} = nothing # store RSNN error outputError::Union{AbstractArray, Nothing} = nothing # store output neurons error # ---------------------------------------------------------------------------- # # LIF Neurons # # ---------------------------------------------------------------------------- # # a projection of kfn.zit into lif dimension for broadcasting later) lif_zit::Union{AbstractArray, Nothing} = nothing # main variables according to papers lif_wRec::Union{AbstractArray, Nothing} = nothing lif_vt::Union{AbstractArray, Nothing} = nothing lif_vth::Union{AbstractArray, Nothing} = nothing lif_vRest::Union{AbstractArray, Nothing} = nothing lif_zt::Union{AbstractArray, Nothing} = nothing lif_zt4d::Union{AbstractArray, Nothing} = nothing lif_refractoryCounter::Union{AbstractArray, Nothing} = nothing lif_refractoryDuration::Union{AbstractArray, Nothing} = nothing lif_alpha::Union{AbstractArray, Nothing} = nothing lif_delta::Union{AbstractFloat, Nothing} = nothing lif_tau_m::Union{AbstractFloat, Nothing} = nothing lif_phi::Union{AbstractArray, Nothing} = nothing lif_epsilonRec::Union{AbstractArray, Nothing} = nothing lif_eRec::Union{AbstractArray, Nothing} = nothing lif_eta::Union{AbstractArray, Nothing} = nothing lif_gammaPd::Union{AbstractArray, Nothing} = nothing lif_wRecChange::Union{AbstractArray, Nothing} = nothing lif_error::Union{AbstractArray, Nothing} = nothing lif_firingCounter::Union{AbstractArray, Nothing} = nothing lif_firingTargetFrequency::Union{AbstractArray, Nothing} = nothing lif_neuronInactivityCounter::Union{AbstractArray, Nothing} = nothing lif_synapticInactivityCounter::Union{AbstractArray, Nothing} = nothing lif_synapticConnectionNumber::Union{Int, Nothing} = nothing # pre-allocation array lif_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d lif_recSignal::Union{AbstractArray, Nothing} = nothing lif_exInType::Union{AbstractArray, Nothing} = nothing # lif_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing # lif_vt_diff_vth::Union{AbstractArray, Nothing} = nothing # lif_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing # lif_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing # lif_phiActivation::Union{AbstractArray, Nothing} = nothing # ---------------------------------------------------------------------------- # # ALIF Neurons # # ---------------------------------------------------------------------------- # alif_zit::Union{AbstractArray, Nothing} = nothing alif_wRec::Union{AbstractArray, Nothing} = nothing alif_vt::Union{AbstractArray, Nothing} = nothing alif_vth::Union{AbstractArray, Nothing} = nothing alif_vRest::Union{AbstractArray, Nothing} = nothing alif_zt::Union{AbstractArray, Nothing} = nothing alif_zt4d::Union{AbstractArray, Nothing} = nothing alif_refractoryCounter::Union{AbstractArray, Nothing} = nothing alif_refractoryDuration::Union{AbstractArray, Nothing} = nothing alif_alpha::Union{AbstractArray, Nothing} = nothing alif_delta::Union{AbstractFloat, Nothing} = nothing alif_tau_m::Union{AbstractFloat, Nothing} = nothing alif_phi::Union{AbstractArray, Nothing} = nothing alif_epsilonRec::Union{AbstractArray, Nothing} = nothing alif_eRec::Union{AbstractArray, Nothing} = nothing alif_eta::Union{AbstractArray, Nothing} = nothing alif_gammaPd::Union{AbstractArray, Nothing} = nothing alif_wRecChange::Union{AbstractArray, Nothing} = nothing alif_error::Union{AbstractArray, Nothing} = nothing alif_firingCounter::Union{AbstractArray, Nothing} = nothing alif_firingTargetFrequency::Union{AbstractArray, Nothing} = nothing alif_neuronInactivityCounter::Union{AbstractArray, Nothing} = nothing alif_synapticInactivityCounter::Union{AbstractArray, Nothing} = nothing alif_synapticConnectionNumber::Union{Int, Nothing} = nothing # pre-allocation array alif_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d alif_recSignal::Union{AbstractArray, Nothing} = nothing alif_exInType::Union{AbstractArray, Nothing} = nothing # alif_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing # alif_vt_diff_vth::Union{AbstractArray, Nothing} = nothing # alif_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing # alif_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing # alif_phiActivation::Union{AbstractArray, Nothing} = nothing # alif specific variables alif_epsilonRecA::Union{AbstractArray, Nothing} = nothing alif_avth::Union{AbstractArray, Nothing} = nothing alif_a::Union{AbstractArray, Nothing} = nothing # threshold adaptation alif_beta::Union{AbstractArray, Nothing} = nothing # β, constant, value from paper alif_rho::Union{AbstractArray, Nothing} = nothing # ρ, threshold adaptation decay factor alif_tau_a::Union{AbstractFloat, Nothing} = nothing # τ_a, adaption time constant in millisecond # alif specific pre-allocation array # alif_phi_x_epsilonRec::Union{AbstractArray, Nothing} = nothing # alif_phi_x_beta::Union{AbstractArray, Nothing} = nothing # alif_rho_diff_phi_x_beta::Union{AbstractArray, Nothing} = nothing # alif_rho_div_phi_x_beta_x_epsilonRecA::Union{AbstractArray, Nothing} = nothing # alif_beta_x_a::Union{AbstractArray, Nothing} = nothing # ---------------------------------------------------------------------------- # # Output Neurons # # ---------------------------------------------------------------------------- # # output neuron is based on LIF on_zit::Union{AbstractArray, Nothing} = nothing # main variables according to papers on_wOut::Union{AbstractArray, Nothing} = nothing # wOut is wRec, just use the name from paper on_vt::Union{AbstractArray, Nothing} = nothing on_vth::Union{AbstractArray, Nothing} = nothing on_vRest::Union{AbstractArray, Nothing} = nothing on_zt::Union{AbstractArray, Nothing} = nothing on_zt4d::Union{AbstractArray, Nothing} = nothing on_refractoryCounter::Union{AbstractArray, Nothing} = nothing on_refractoryDuration::Union{AbstractArray, Nothing} = nothing on_alpha::Union{AbstractArray, Nothing} = nothing on_delta::Union{AbstractFloat, Nothing} = nothing on_tau_m::Union{AbstractFloat, Nothing} = nothing on_phi::Union{AbstractArray, Nothing} = nothing on_epsilonRec::Union{AbstractArray, Nothing} = nothing on_eRec::Union{AbstractArray, Nothing} = nothing on_eta::Union{AbstractArray, Nothing} = nothing on_gammaPd::Union{AbstractArray, Nothing} = nothing on_wOutChange::Union{AbstractArray, Nothing} = nothing on_error::Union{AbstractArray, Nothing} = nothing on_subscription::Union{AbstractArray, Nothing} = nothing on_firingCounter::Union{AbstractArray, Nothing} = nothing # pre-allocation array on_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d on_recSignal::Union{AbstractArray, Nothing} = nothing # on_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing # on_vt_diff_vth::Union{AbstractArray, Nothing} = nothing # on_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing # on_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing # on_phiActivation::Union{AbstractArray, Nothing} = nothing end # outer constructor function kfn_1(params::Dict; device=cpu) kfn = kfn_1() kfn.params = params kfn.timeStep = [0] |> device kfn.learningStage = [0] |> device # ---------------------------------------------------------------------------- # # initialize activation matrix # # ---------------------------------------------------------------------------- # # row*col is a 2D matrix represent all RSNN activation row, signal_col, batch = kfn.params[:inputPort][:signal][:numbers] # z-axis represent signal batch number kfn.inputSize = [row, signal_col] |> device lif_col = kfn.params[:computeNeuron][:lif][:numbers][2] alif_col = kfn.params[:computeNeuron][:alif][:numbers][2] col = signal_col + lif_col + alif_col # activation matrix kfn.zit = zeros(row, col, batch) |> device kfn.zitCumulative = (similar(kfn.zit) .= 0) kfn.modelError = zeros(1) |> device # ---------------------------------------------------------------------------- # # LIF config # # ---------------------------------------------------------------------------- # # In 3D LIF matrix, z-axis represent each neuron while each 2D slice represent that neuron's # synaptic subscription to other neurons (via activation matrix) lif_n = kfn.params[:computeNeuron][:lif][:numbers][1] * kfn.params[:computeNeuron][:lif][:numbers][2] # subscription synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent] kfn.lif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100)) w = wRec(row, col, lif_n, kfn.lif_synapticConnectionNumber) # project 3D w into 4D kfn.lif_wRec (row, col, n, batch) kfn.lif_wRec = reshape(w, (row, col, lif_n, 1)) .* ones(row, col, lif_n, batch) |> device kfn.lif_zit = (similar(kfn.lif_wRec) .= 0) kfn.lif_vt = (similar(kfn.lif_wRec) .= 0) kfn.lif_vth = (similar(kfn.lif_wRec) .= 1) kfn.lif_vRest = (similar(kfn.lif_wRec) .= 0) kfn.lif_zt = zeros(1, 1, lif_n, batch) |> device kfn.lif_zt4d = (similar(kfn.lif_wRec) .= 0) kfn.lif_refractoryCounter = (similar(kfn.lif_wRec) .= 0) kfn.lif_refractoryDuration = (similar(kfn.lif_wRec) .= 3) kfn.lif_delta = 1.0 kfn.lif_tau_m = 20.0 kfn.lif_alpha = (similar(kfn.lif_wRec) .= (exp(-kfn.lif_delta / kfn.lif_tau_m))) kfn.lif_phi = (similar(kfn.lif_wRec) .= 0) kfn.lif_epsilonRec = (similar(kfn.lif_wRec) .= 0) kfn.lif_eRec = (similar(kfn.lif_wRec) .= 0) kfn.lif_eta = (similar(kfn.lif_wRec) .= 0.001) kfn.lif_gammaPd = (similar(kfn.lif_wRec) .= 0.3) kfn.lif_wRecChange = (similar(kfn.lif_wRec) .= 0) kfn.lif_error = (similar(kfn.lif_wRec) .= 0) kfn.lif_firingCounter = (similar(kfn.lif_wRec) .= 0) kfn.lif_firingTargetFrequency = (similar(kfn.lif_wRec) .= 0.1) kfn.lif_neuronInactivityCounter = (similar(kfn.lif_wRec) .= 0) kfn.lif_synapticInactivityCounter = Array(similar(kfn.lif_wRec) .= -0.99) # -9 for non-sub conn mask = Array((!iszero).(kfn.lif_wRec)) # initial value subscribed conn, synapticInactivityCounter range -10000 to +10000 GeneralUtils.replace_elements!(mask, 1, kfn.lif_synapticInactivityCounter, 0) kfn.lif_synapticInactivityCounter = kfn.lif_synapticInactivityCounter |> device kfn.lif_arrayProjection4d = (similar(kfn.lif_wRec) .= 1) kfn.lif_recSignal = (similar(kfn.lif_wRec) .= 0) kfn.lif_exInType = (similar(kfn.lif_wRec) .= 0) # kfn.lif_decayed_epsilonRec = (similar(kfn.lif_wRec) .= 0) # kfn.lif_vt_diff_vth = (similar(kfn.lif_wRec) .= 0) # kfn.lif_vt_diff_vth_div_vth = (similar(kfn.lif_wRec) .= 0) # kfn.lif_gammaPd_div_vth = (similar(kfn.lif_wRec) .= 0) # kfn.lif_phiActivation = (similar(kfn.lif_wRec) .= 0) # ---------------------------------------------------------------------------- # # ALIF config # # ---------------------------------------------------------------------------- # alif_n = kfn.params[:computeNeuron][:alif][:numbers][1] * kfn.params[:computeNeuron][:alif][:numbers][2] # subscription synapticConnectionPercent = kfn.params[:computeNeuron][:alif][:params][:synapticConnectionPercent] kfn.alif_synapticConnectionNumber = Int(floor(row*col * synapticConnectionPercent/100)) w = wRec(row, col, alif_n, kfn.alif_synapticConnectionNumber) # project 3D w into 4D kfn.alif_wRec kfn.alif_wRec = reshape(w, (row, col, alif_n, 1)) .* ones(row, col, alif_n, batch) |> device kfn.alif_zit = (similar(kfn.alif_wRec) .= 0) kfn.alif_vt = (similar(kfn.alif_wRec) .= 0) kfn.alif_vth = (similar(kfn.alif_wRec) .= 1) kfn.alif_vRest = (similar(kfn.alif_wRec) .= 0) kfn.alif_zt = zeros(1, 1, alif_n, batch) |> device kfn.alif_zt4d = (similar(kfn.alif_wRec) .= 0) kfn.alif_refractoryCounter = (similar(kfn.alif_wRec) .= 0) kfn.alif_refractoryDuration = (similar(kfn.alif_wRec) .= 3) kfn.alif_delta = 1.0 kfn.alif_tau_m = 20.0 kfn.alif_alpha = (similar(kfn.alif_wRec) .= (exp(-kfn.alif_delta / kfn.alif_tau_m))) kfn.alif_phi = (similar(kfn.alif_wRec) .= 0) kfn.alif_epsilonRec = (similar(kfn.alif_wRec) .= 0) kfn.alif_eRec = (similar(kfn.alif_wRec) .= 0) kfn.alif_eta = (similar(kfn.alif_wRec) .= 0.001) kfn.alif_gammaPd = (similar(kfn.alif_wRec) .= 0.3) kfn.alif_wRecChange = (similar(kfn.alif_wRec) .= 0) kfn.alif_error = (similar(kfn.alif_wRec) .= 0) kfn.alif_firingCounter = (similar(kfn.alif_wRec) .= 0) kfn.alif_firingTargetFrequency = (similar(kfn.alif_wRec) .= 0.1) kfn.alif_neuronInactivityCounter = (similar(kfn.alif_wRec) .= 0) kfn.alif_synapticInactivityCounter = Array(similar(kfn.alif_wRec) .= -0.99) # -9 for non-sub conn mask = Array((!iszero).(kfn.alif_wRec)) # initial value subscribed conn, synapticInactivityCounter range -10000 to +10000 GeneralUtils.replace_elements!(mask, 1, kfn.alif_synapticInactivityCounter, 0) kfn.alif_synapticInactivityCounter = kfn.alif_synapticInactivityCounter |> device kfn.alif_arrayProjection4d = (similar(kfn.alif_wRec) .= 1) kfn.alif_recSignal = (similar(kfn.alif_wRec) .= 0) kfn.alif_exInType = (similar(kfn.alif_wRec) .= 0) # kfn.alif_decayed_epsilonRec = (similar(kfn.alif_wRec) .= 0) # kfn.alif_vt_diff_vth = (similar(kfn.alif_wRec) .= 0) # kfn.alif_vt_diff_vth_div_vth = (similar(kfn.alif_wRec) .= 0) # kfn.alif_gammaPd_div_vth = (similar(kfn.alif_wRec) .= 0) # kfn.alif_phiActivation = (similar(kfn.alif_wRec) .= 0) # alif specific variables kfn.alif_epsilonRecA = (similar(kfn.alif_wRec) .= 0) kfn.alif_avth = (similar(kfn.alif_wRec) .= 0) kfn.alif_a = (similar(kfn.alif_wRec) .= 0) kfn.alif_beta = (similar(kfn.alif_wRec) .= 0.07) kfn.alif_tau_a = 800.0 kfn.alif_rho = (similar(kfn.alif_wRec) .= (exp(-kfn.alif_delta / kfn.alif_tau_a))) |> device # kfn.alif_phi_x_epsilonRec = (similar(kfn.alif_wRec) .= 0) # kfn.alif_phi_x_beta = (similar(kfn.alif_wRec) .= 0) # kfn.alif_rho_diff_phi_x_beta = (similar(kfn.alif_wRec) .= 0) # kfn.alif_rho_div_phi_x_beta_x_epsilonRecA = (similar(kfn.alif_wRec) .= 0) # kfn.alif_beta_x_a = (similar(kfn.alif_wRec) .= 0) # ---------------------------------------------------------------------------- # # output config # # ---------------------------------------------------------------------------- # n = kfn.params[:outputPort][:numbers][1] * kfn.params[:outputPort][:numbers][2] # subscription w = zeros(row, col, n) synapticConnectionPercent = kfn.params[:outputPort][:params][:synapticConnectionPercent] subable = size(kfn.lif_wRec, 3) + size(kfn.alif_wRec, 3) # sub to lif, alif only synapticConnection = Int(floor(subable * synapticConnectionPercent/100)) for slice in eachslice(w, dims=3) # each slice is a neuron startInd = row*col - subable + 1 # e.g. 100(row*col) - 50(subable) = 50 -> startInd = 51 # pool must contain only lif, alif neurons pool = shuffle!([startInd:row*col...])[1:synapticConnection] for i in pool slice[i] = rand() # assign weight to synaptic connection. /10 to start small, # otherwise RSNN's vt Usually stay negative (-) end end # 10% of neuron connection should be enough to start to make neuron fires should_be_avg_weight = 1 / (0.1 * n) w = w .* (should_be_avg_weight / maximum(w)) # adjust overall weight # project 3D w into 4D kfn.lif_wOut (row, col, n, batch) kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device kfn.on_zit = (similar(kfn.on_wOut) .= 0) kfn.on_vt = (similar(kfn.on_wOut) .= 0) kfn.on_vth = (similar(kfn.on_wOut) .= 1) kfn.on_vRest = (similar(kfn.on_wOut) .= 0) kfn.on_zt = zeros(1, 1, n, batch) |> device kfn.on_zt4d = (similar(kfn.on_wOut) .= 0) kfn.on_refractoryCounter = (similar(kfn.on_wOut) .= 0) kfn.on_refractoryDuration = (similar(kfn.on_wOut) .= 0) kfn.on_delta = 1.0 kfn.on_tau_m = 20.0 kfn.on_alpha = (similar(kfn.on_wOut) .= (exp(-kfn.on_delta / kfn.on_tau_m))) kfn.on_phi = (similar(kfn.on_wOut) .= 0) kfn.on_epsilonRec = (similar(kfn.on_wOut) .= 0) kfn.on_eRec = (similar(kfn.on_wOut) .= 0) kfn.on_eta = (similar(kfn.on_wOut) .= 0.001) kfn.on_gammaPd = (similar(kfn.on_wOut) .= 0.3) kfn.on_wOutChange = (similar(kfn.on_wOut) .= 0) kfn.on_error = (similar(kfn.on_wOut) .= 0) kfn.on_subscription = (GeneralUtils.isNotEqual.(kfn.on_wOut, 0)) |> device kfn.on_firingCounter = (similar(kfn.on_wOut) .= 0) kfn.on_arrayProjection4d = (similar(kfn.on_wOut) .= 1) kfn.on_recSignal = (similar(kfn.on_wOut) .= 0) kfn.outputError = zeros(n, batch) |> device totalComputeNeurons = lif_n + alif_n inhabitoryNeurons = Int(floor(totalComputeNeurons * 30/100)) mask1 = ones(row, signal_col) mask2 = GeneralUtils.multiply_random_elements(ones(row, lif_col + alif_col), -1, inhabitoryNeurons, MersenneTwister(1234)) kfn.exInType = cat(mask1, mask2, dims=2) |> device return kfn end function wRec(row, col, n, synapticConnectionNumber) # subscription w = zeros(row, col, n) for slice in eachslice(w, dims=3) pool = shuffle!([1:row*col...])[1:synapticConnectionNumber] for i in pool slice[i] = rand(0.01:0.01:0.1) # assign weight to synaptic connection. /10 to start small, # otherwise RSNN's vt Usually stay negative (-) end end # 10% of neuron connection should be enough to start to make neuron fires avgWeight = sum(w)/length(w) w = w .* (0.01 / avgWeight) # adjust overall weight return w #(row, col, n) end end # module