module type export # struct kfn_1 # function using Random, GeneralUtils #------------------------------------------------------------------------------------------------100 rng = MersenneTwister(1234) abstract type Ironpen end abstract type knowledgeFn <: Ironpen end #------------------------------------------------------------------------------------------------100 Base.@kwdef mutable struct kfn_1 <: knowledgeFn params::Union{Dict, Nothing} = nothing # store params of knowledgeFn itself for later use timeStep::Union{AbstractArray, Nothing} = nothing learningStage::Union{AbstractArray, Nothing} = nothing # 0 inference, 1 start, 2 during, 3 end learning zit::Union{AbstractArray, Nothing} = nothing # 3D activation matrix modelError::Union{AbstractArray, Nothing} = nothing # store RSNN error outputError::Union{AbstractArray, Nothing} = nothing # store output neurons error # ---------------------------------------------------------------------------- # # LIF Neurons # # ---------------------------------------------------------------------------- # # a projection of kfn.zit into lif dimension for broadcasting later) lif_zit::Union{AbstractArray, Nothing} = nothing # main variables according to papers lif_wRec::Union{AbstractArray, Nothing} = nothing lif_vt::Union{AbstractArray, Nothing} = nothing lif_vth::Union{AbstractArray, Nothing} = nothing lif_vRest::Union{AbstractArray, Nothing} = nothing lif_zt::Union{AbstractArray, Nothing} = nothing lif_zt4d::Union{AbstractArray, Nothing} = nothing lif_refractoryCounter::Union{AbstractArray, Nothing} = nothing lif_refractoryDuration::Union{AbstractArray, Nothing} = nothing lif_alpha::Union{AbstractArray, Nothing} = nothing lif_delta::Union{AbstractFloat, Nothing} = nothing lif_tau_m::Union{AbstractFloat, Nothing} = nothing lif_phi::Union{AbstractArray, Nothing} = nothing lif_epsilonRec::Union{AbstractArray, Nothing} = nothing lif_eRec::Union{AbstractArray, Nothing} = nothing lif_eta::Union{AbstractArray, Nothing} = nothing lif_gammaPd::Union{AbstractArray, Nothing} = nothing lif_wRecChange::Union{AbstractArray, Nothing} = nothing lif_error::Union{AbstractArray, Nothing} = nothing lif_firingCounter::Union{AbstractArray, Nothing} = nothing # pre-allocation array lif_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d lif_recSignal::Union{AbstractArray, Nothing} = nothing # lif_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing # lif_vt_diff_vth::Union{AbstractArray, Nothing} = nothing # lif_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing # lif_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing # lif_phiActivation::Union{AbstractArray, Nothing} = nothing # ---------------------------------------------------------------------------- # # ALIF Neurons # # ---------------------------------------------------------------------------- # alif_zit::Union{AbstractArray, Nothing} = nothing alif_wRec::Union{AbstractArray, Nothing} = nothing alif_vt::Union{AbstractArray, Nothing} = nothing alif_vth::Union{AbstractArray, Nothing} = nothing alif_vRest::Union{AbstractArray, Nothing} = nothing alif_zt::Union{AbstractArray, Nothing} = nothing alif_zt4d::Union{AbstractArray, Nothing} = nothing alif_refractoryCounter::Union{AbstractArray, Nothing} = nothing alif_refractoryDuration::Union{AbstractArray, Nothing} = nothing alif_alpha::Union{AbstractArray, Nothing} = nothing alif_delta::Union{AbstractFloat, Nothing} = nothing alif_tau_m::Union{AbstractFloat, Nothing} = nothing alif_phi::Union{AbstractArray, Nothing} = nothing alif_epsilonRec::Union{AbstractArray, Nothing} = nothing alif_eRec::Union{AbstractArray, Nothing} = nothing alif_eta::Union{AbstractArray, Nothing} = nothing alif_gammaPd::Union{AbstractArray, Nothing} = nothing alif_wRecChange::Union{AbstractArray, Nothing} = nothing alif_error::Union{AbstractArray, Nothing} = nothing alif_firingCounter::Union{AbstractArray, Nothing} = nothing # pre-allocation array alif_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d alif_recSignal::Union{AbstractArray, Nothing} = nothing # alif_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing # alif_vt_diff_vth::Union{AbstractArray, Nothing} = nothing # alif_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing # alif_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing # alif_phiActivation::Union{AbstractArray, Nothing} = nothing # alif specific variables alif_epsilonRecA::Union{AbstractArray, Nothing} = nothing alif_avth::Union{AbstractArray, Nothing} = nothing alif_a::Union{AbstractArray, Nothing} = nothing # threshold adaptation alif_beta::Union{AbstractArray, Nothing} = nothing # β, constant, value from paper alif_rho::Union{AbstractArray, Nothing} = nothing # ρ, threshold adaptation decay factor alif_tau_a::Union{AbstractFloat, Nothing} = nothing # τ_a, adaption time constant in millisecond # alif specific pre-allocation array # alif_phi_x_epsilonRec::Union{AbstractArray, Nothing} = nothing # alif_phi_x_beta::Union{AbstractArray, Nothing} = nothing # alif_rho_diff_phi_x_beta::Union{AbstractArray, Nothing} = nothing # alif_rho_div_phi_x_beta_x_epsilonRecA::Union{AbstractArray, Nothing} = nothing # alif_beta_x_a::Union{AbstractArray, Nothing} = nothing # ---------------------------------------------------------------------------- # # Output Neurons # # ---------------------------------------------------------------------------- # # output neuron is based on LIF on_zit::Union{AbstractArray, Nothing} = nothing # main variables according to papers on_wOut::Union{AbstractArray, Nothing} = nothing # wOut is wRec, just use the name from paper on_vt::Union{AbstractArray, Nothing} = nothing on_vth::Union{AbstractArray, Nothing} = nothing on_vRest::Union{AbstractArray, Nothing} = nothing on_zt::Union{AbstractArray, Nothing} = nothing on_zt4d::Union{AbstractArray, Nothing} = nothing on_refractoryCounter::Union{AbstractArray, Nothing} = nothing on_refractoryDuration::Union{AbstractArray, Nothing} = nothing on_alpha::Union{AbstractArray, Nothing} = nothing on_delta::Union{AbstractFloat, Nothing} = nothing on_tau_m::Union{AbstractFloat, Nothing} = nothing on_phi::Union{AbstractArray, Nothing} = nothing on_epsilonRec::Union{AbstractArray, Nothing} = nothing on_eRec::Union{AbstractArray, Nothing} = nothing on_eta::Union{AbstractArray, Nothing} = nothing on_gammaPd::Union{AbstractArray, Nothing} = nothing on_wOutChange::Union{AbstractArray, Nothing} = nothing on_error::Union{AbstractArray, Nothing} = nothing on_firingCounter::Union{AbstractArray, Nothing} = nothing # pre-allocation array on_arrayProjection4d::Union{AbstractArray, Nothing} = nothing # use to project 3d array to 4d on_recSignal::Union{AbstractArray, Nothing} = nothing # on_decayed_epsilonRec::Union{AbstractArray, Nothing} = nothing # on_vt_diff_vth::Union{AbstractArray, Nothing} = nothing # on_vt_diff_vth_div_vth::Union{AbstractArray, Nothing} = nothing # on_gammaPd_div_vth::Union{AbstractArray, Nothing} = nothing # on_phiActivation::Union{AbstractArray, Nothing} = nothing end # outer constructor function kfn_1(params::Dict; device=cpu) kfn = kfn_1() kfn.params = params kfn.timeStep = [0] |> device kfn.learningStage = [0] |> device # ---------------------------------------------------------------------------- # # initialize activation matrix # # ---------------------------------------------------------------------------- # # row*col is a 2D matrix represent all RSNN activation row, col, batch = kfn.params[:inputPort][:signal][:numbers] # z-axis represent signal batch number # row += kfn.params[:inputPort][:noise][:numbers][1] col += kfn.params[:inputPort][:noise][:numbers][2] col += kfn.params[:computeNeuron][:lif][:numbers][2] col += kfn.params[:computeNeuron][:alif][:numbers][2] # activation matrix kfn.zit = zeros(row, col, batch) |> device kfn.modelError = zeros(1) |> device # ---------------------------------------------------------------------------- # # LIF config # # ---------------------------------------------------------------------------- # # In 3D LIF matrix, z-axis represent each neuron while each 2D slice represent that neuron's # synaptic subscription to other neurons (via activation matrix) n = kfn.params[:computeNeuron][:lif][:numbers][1] * kfn.params[:computeNeuron][:lif][:numbers][2] # subscription w = zeros(row, col, n) synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent] synapticConnection = Int(floor(row*col * synapticConnectionPercent/100)) for slice in eachslice(w, dims=3) pool = shuffle!([1:row*col...])[1:synapticConnection] for i in pool slice[i] = randn()/10 # assign weight to synaptic connection end end # project 3D w into 4D kfn.lif_wRec (row, col, n, batch) kfn.lif_wRec = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device kfn.lif_zit = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_vt = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_vth = (similar(kfn.lif_wRec) .= 1) |> device kfn.lif_vRest = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_zt = zeros(1, 1, n, batch) |> device kfn.lif_zt4d = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_refractoryCounter = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_refractoryDuration = (similar(kfn.lif_wRec) .= 3) |> device kfn.lif_delta = 1.0 kfn.lif_tau_m = 20.0 kfn.lif_alpha = (similar(kfn.lif_wRec) .= (exp(-kfn.lif_delta / kfn.lif_tau_m))) |> device kfn.lif_phi = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_epsilonRec = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_eRec = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_eta = (similar(kfn.lif_wRec) .= 0.001) |> device kfn.lif_gammaPd = (similar(kfn.lif_wRec) .= 0.3) |> device kfn.lif_wRecChange = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_error = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_firingCounter = (similar(kfn.lif_wRec) .= 0) |> device kfn.lif_arrayProjection4d = (similar(kfn.lif_wRec) .= 1) |> device kfn.lif_recSignal = (similar(kfn.lif_wRec) .= 0) |> device # kfn.lif_decayed_epsilonRec = (similar(kfn.lif_wRec) .= 0) |> device # kfn.lif_vt_diff_vth = (similar(kfn.lif_wRec) .= 0) |> device # kfn.lif_vt_diff_vth_div_vth = (similar(kfn.lif_wRec) .= 0) |> device # kfn.lif_gammaPd_div_vth = (similar(kfn.lif_wRec) .= 0) |> device # kfn.lif_phiActivation = (similar(kfn.lif_wRec) .= 0) |> device # ---------------------------------------------------------------------------- # # ALIF config # # ---------------------------------------------------------------------------- # n = kfn.params[:computeNeuron][:alif][:numbers][1] * kfn.params[:computeNeuron][:alif][:numbers][2] # subscription w = zeros(row, col, n) synapticConnectionPercent = kfn.params[:computeNeuron][:alif][:params][:synapticConnectionPercent] synapticConnection = Int(floor(row*col * synapticConnectionPercent/100)) for slice in eachslice(w, dims=3) pool = shuffle!([1:row*col...])[1:synapticConnection] for i in pool slice[i] = randn()/10 # assign weight to synaptic connection end end # project 3D w into 4D kfn.alif_wRec kfn.alif_wRec = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device kfn.alif_zit = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_vt = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_vth = (similar(kfn.alif_wRec) .= 1) |> device kfn.alif_vRest = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_zt = zeros(1, 1, n, batch) |> device kfn.alif_zt4d = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_refractoryCounter = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_refractoryDuration = (similar(kfn.alif_wRec) .= 3) |> device kfn.alif_delta = 1.0 kfn.alif_tau_m = 20.0 kfn.alif_alpha = (similar(kfn.alif_wRec) .= (exp(-kfn.alif_delta / kfn.alif_tau_m))) |> device kfn.alif_phi = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_epsilonRec = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_eRec = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_eta = (similar(kfn.alif_wRec) .= 0.001) |> device kfn.alif_gammaPd = (similar(kfn.alif_wRec) .= 0.3) |> device kfn.alif_wRecChange = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_error = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_firingCounter = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_arrayProjection4d = (similar(kfn.alif_wRec) .= 1) |> device kfn.alif_recSignal = (similar(kfn.alif_wRec) .= 0) |> device # kfn.alif_decayed_epsilonRec = (similar(kfn.alif_wRec) .= 0) |> device # kfn.alif_vt_diff_vth = (similar(kfn.alif_wRec) .= 0) |> device # kfn.alif_vt_diff_vth_div_vth = (similar(kfn.alif_wRec) .= 0) |> device # kfn.alif_gammaPd_div_vth = (similar(kfn.alif_wRec) .= 0) |> device # kfn.alif_phiActivation = (similar(kfn.alif_wRec) .= 0) |> device # alif specific variables kfn.alif_epsilonRecA = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_avth = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_a = (similar(kfn.alif_wRec) .= 0) |> device kfn.alif_beta = (similar(kfn.alif_wRec) .= 0.07) |> device kfn.alif_tau_a = 100.0 kfn.alif_rho = (similar(kfn.alif_wRec) .= (exp(-kfn.alif_delta / kfn.alif_tau_a))) |> device # kfn.alif_phi_x_epsilonRec = (similar(kfn.alif_wRec) .= 0) |> device # kfn.alif_phi_x_beta = (similar(kfn.alif_wRec) .= 0) |> device # kfn.alif_rho_diff_phi_x_beta = (similar(kfn.alif_wRec) .= 0) |> device # kfn.alif_rho_div_phi_x_beta_x_epsilonRecA = (similar(kfn.alif_wRec) .= 0) |> device # kfn.alif_beta_x_a = (similar(kfn.alif_wRec) .= 0) |> device # ---------------------------------------------------------------------------- # # output config # # ---------------------------------------------------------------------------- # n = kfn.params[:outputPort][:numbers][1] * kfn.params[:outputPort][:numbers][2] # subscription w = zeros(row, col, n) synapticConnectionPercent = kfn.params[:computeNeuron][:lif][:params][:synapticConnectionPercent] synapticConnection = Int(floor(row*col * synapticConnectionPercent/100)) for slice in eachslice(w, dims=3) pool = shuffle!([1:row*col...])[1:synapticConnection] for i in pool slice[i] = randn()/10 # assign weight to synaptic connection end end # project 3D w into 4D kfn.lif_wOut (row, col, n, batch) kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device kfn.on_zit = (similar(kfn.on_wOut) .= 0) |> device kfn.on_vt = (similar(kfn.on_wOut) .= 0) |> device kfn.on_vth = (similar(kfn.on_wOut) .= 1) |> device kfn.on_vRest = (similar(kfn.on_wOut) .= 0) |> device kfn.on_zt = zeros(1, 1, n, batch) |> device kfn.on_zt4d = (similar(kfn.on_wOut) .= 0) |> device kfn.on_refractoryCounter = (similar(kfn.on_wOut) .= 0) |> device kfn.on_refractoryDuration = (similar(kfn.on_wOut) .= 0) |> device kfn.on_delta = 1.0 kfn.on_tau_m = 20.0 kfn.on_alpha = (similar(kfn.on_wOut) .= (exp(-kfn.on_delta / kfn.on_tau_m))) |> device kfn.on_phi = (similar(kfn.on_wOut) .= 0) |> device kfn.on_epsilonRec = (similar(kfn.on_wOut) .= 0) |> device kfn.on_eRec = (similar(kfn.on_wOut) .= 0) |> device kfn.on_eta = (similar(kfn.on_wOut) .= 0.001) |> device kfn.on_gammaPd = (similar(kfn.on_wOut) .= 0.3) |> device kfn.on_wOutChange = (similar(kfn.on_wOut) .= 0) |> device kfn.on_error = (similar(kfn.on_wOut) .= 0) |> device kfn.on_firingCounter = (similar(kfn.on_wOut) .= 0) |> device kfn.on_arrayProjection4d = (similar(kfn.on_wOut) .= 1) |> device kfn.on_recSignal = (similar(kfn.on_wOut) .= 0) |> device kfn.outputError = zeros(n, batch) |> device # kfn.on_decayed_epsilonRec = (similar(kfn.on_wOut) .= 0 |> device # kfn.on_vt_diff_vth = (similar(kfn.on_wOut) .= 0 |> device # kfn.on_vt_diff_vth_div_vth = (similar(kfn.on_wOut) .= 0 |> device # kfn.on_gammaPd_div_vth = (similar(kfn.on_wOut) .= 0 |> device # kfn.on_phiActivation = (similar(kfn.on_wOut) .= 0 |> device # kfn.on_zit = zeros(row, col, n, batch) |> device # kfn.on_vt = zeros(1, 1, n, batch) |> device # kfn.on_vth = ones(1, 1, n, batch) |> device # kfn.on_vRest = zeros(1, 1, n, batch) |> device # # kfn.on_zt = zeros(1, 1, n, batch) |> device # kfn.on_zt4d = zeros(1, 1, n, batch) |> device # kfn.on_refractoryCounter = zeros(1, 1, n, batch) |> device # kfn.on_refractoryDuration = ones(1, 1, n, batch) .* 0 |> device # kfn.on_delta = 1.0 # kfn.on_tau_m = 20.0 # kfn.on_alpha = ones(1, 1, n, batch) .* (exp(-kfn.on_delta / kfn.on_tau_m)) |> device # kfn.on_phi = zeros(1, 1, n, batch) |> device # kfn.on_epsilonRec = zeros(row, col, n, batch) |> device # # kfn.on_eRec = zeros(row, col, n, batch) # kfn.on_eta = zeros(1, 1, n, batch) |> device # kfn.on_gammaPd = zeros(1, 1, n, batch) .* 0.3 |> device # kfn.on_wOutChange = zeros(row, col, n, batch) |> device # # kfn.on_b = randn(1, 1, n, batch) |> device # # kfn.on_bChange = randn(1, 1, n, batch) |> device # kfn.on_firingCounter = zeros(1, 1, n, batch) |> device # kfn.on_arraySize = [row, col, n, batch] |> device # kfn.on_arrayProjection4d = ones(row, col, n, batch) |> device # # subscription # w = zeros(row, col, n) # synapticConnectionPercent = kfn.params[:outputPort][:params][:synapticConnectionPercent] # synapticConnection = Int(floor(row*col * synapticConnectionPercent/100)) # for slice in eachslice(w, dims=3) # pool = shuffle!([1:row*col...])[1:synapticConnection] # for i in pool # slice[i] = randn()/10 # assign weight to synaptic connection # end # end # # project 3D w into 4D kfn.on_wOut # kfn.on_wOut = reshape(w, (row, col, n, 1)) .* ones(row, col, n, batch) |> device return kfn end end # module