ton/IronpenGPU
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

alif forward()

Browse Source
This commit is contained in:
ton
2023-07-30 06:17:13 +07:00
parent 22b063effc
commit 50df913106
2 changed files with 352 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

387
src/forward.jl
View File

@@ -36,8 +36,8 @@ function (kfn::kfn_1)(input::AbstractArray)
lifForward( kfn.zit,
kfn.lif_zit,
kfn.lif_wRec,
kfn.lif_vt0,
kfn.lif_wRec,
kfn.lif_vt0,
kfn.lif_vt1,
kfn.lif_vth,
kfn.lif_vRest,
@@ -49,29 +49,49 @@ function (kfn::kfn_1)(input::AbstractArray)
kfn.lif_refractoryDuration,
kfn.lif_gammaPd,
kfn.lif_firingCounter,
kfn.lif_arraySize,
kfn.lif_arrayProjection3DTo4D)
kfn.lif_arrayProjection3DTo4D,
kfn.lif_recSignal,
kfn.lif_decayed_vt0,
kfn.lif_decayed_epsilonRec,
kfn.lif_vt1_diff_vth,
kfn.lif_vt1_diff_vth_div_vth,
kfn.lif_gammaPd_div_vth,
kfn.lif_phiActivation)
alifForward( kfn.zit,
kfn.alif_zit,
kfn.alif_wRec,
kfn.alif_vt0,
kfn.alif_vt1,
kfn.alif_vth,
kfn.alif_avth,
kfn.alif_vRest,
kfn.alif_zt1,
kfn.alif_alpha,
kfn.alif_phi,
kfn.alif_epsilonRec,
kfn.alif_epsilonRecA,
kfn.alif_refractoryCounter,
kfn.alif_refractoryDuration,
kfn.alif_a,
kfn.alif_beta,
kfn.alif_rho,
kfn.alif_gammaPd,
kfn.alif_firingCounter)
kfn.alif_zit,
kfn.alif_wRec,
kfn.alif_vt0,
kfn.alif_vt1,
kfn.alif_vth,
kfn.alif_vRest,
kfn.alif_zt1,
kfn.alif_alpha,
kfn.alif_phi,
kfn.alif_epsilonRec,
kfn.alif_refractoryCounter,
kfn.alif_refractoryDuration,
kfn.alif_gammaPd,
kfn.alif_firingCounter,
kfn.alif_arrayProjection3DTo4D,
kfn.alif_recSignal,
kfn.alif_decayed_vt0,
kfn.alif_decayed_epsilonRec,
kfn.alif_vt1_diff_vth,
kfn.alif_vt1_diff_vth_div_vth,
kfn.alif_gammaPd_div_vth,
kfn.alif_phiActivation,
kfn.alif_epsilonRecA,
kfn.alif_avth,
kfn.alif_a,
kfn.alif_beta,
kfn.alif_rho,
kfn.alif_phi_x_epsilonRec,
kfn.alif_phi_x_beta,
kfn.alif_rho_diff_phi_x_beta,
kfn.alif_rho_div_phi_x_beta_x_epsilonRecA,
kfn.alif_beta_x_a)
# error("DEBUG -> kfn forward")
@@ -103,7 +123,74 @@ function (kfn::kfn_1)(input::AbstractArray)
kfn.zit
end
function lifForward(kfn_zit,
zit,
wRec,
vt0,
vt1,
vth,
vRest,
zt1,
alpha,
phi,
epsilonRec,
refractoryCounter,
refractoryDuration,
gammaPd,
firingCounter,
arrayProjection3DTo4D,
recSignal,
decayed_vt0,
decayed_epsilonRec,
vt1_diff_vth,
vt1_diff_vth_div_vth,
gammaPd_div_vth,
phiActivation)
# project 3D kfn zit into 4D lif zit
zit .= reshape(kfn_zit,
(size(wRec, 1), size(wRec, 2), 1, size(wRec, 4))) .* arrayProjection3DTo4D
for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
@. @views refractoryCounter[:,:,i,j] -= 1
@. @views zt1[:,:,i,j] = 0
@. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@. @views phi[:,:,i,j] = 0
# compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
else # refractory period is inactive
@. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j]
@. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
if sum(@view(vt1[:,:,i,j])) > sum(@view(vth[:,:,i,j]))
@. @views zt1[:,:,i,j] = 1
@. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
@. @views firingCounter[:,:,i,j] += 1
@. @views vt1[:,:,i,j] = vRest[:,:,i,j]
else
@. @views zt1[:,:,i,j] = 0
end
# compute phi, there is a difference from alif formula
@. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
@. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
@. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
@view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
@. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
end
end
end
function alifForward(kfn_zit,
zit,
wRec,
vt0,
@@ -118,119 +205,167 @@ function lifForward(kfn_zit,
refractoryDuration,
gammaPd,
firingCounter,
arraySize,
arrayProjection3DTo4D)
# project 3D kfn zit into 4D lif zit
zit .= reshape(kfn_zit,
(view(arraySize, 1)[1], view(arraySize, 2)[1], 1, view(arraySize, 4)[1])) .*
arrayProjection3DTo4D
# error("DEBUG -> lif forward") #WORKING
for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
if view(refractoryCounter, :, :, i, j)[1] > 0 # refractory period is active
view(refractoryCounter, :, :, i, j)[1] -= 1
view(zt1, :, :, i, j)[1] = 0
view(vt1, :, :, i, j)[1] =
view(alpha, :, :, i, j)[1] * view(vt0, :, :, i, j)[1]
view(phi, :, :, i, j)[1] = 0.0
view(epsilonRec, :, :, i, j) .= view(alpha, :, :, i, j)[1] .*
view(epsilonRec, :, :, i, j)
else # refractory period is inactive
view(vt1, :, :, i, j)[1] =
(view(alpha, :, :, i, j)[1] * view(vt0,:, :, i, j)[1]) +
sum(view(zit, :, :, i, j) .* view(wRec, :, :, i, j))
if view(vt1, :, :, i, j)[1] > view(vth, :, :, i, j)[1]
view(zt1, :, :, i, j)[1] = 1
view(refractoryCounter, :, :, i, j)[1] =
view(refractoryDuration, :, :, i, j)[1]
view(firingCounter, :, :, i, j)[1] += 1
view(vt1, :, :, i, j)[1] = view(vRest, :, :, i, j)[1]
else
view(zt1, :, :, i, j)[1] = 0
end
# there is a difference from alif formula
view(phi, :, :, i, j)[1] =
(view(gammaPd, :, :, i, j)[1] / view(vth, :, :, i, j)[1]) *
max(0, 1 - ((view(vt1, :, :, i, j)[1] - view(vth, :, :, i, j)[1]) /
view(vth, :, :, i, j)[1]))
view(epsilonRec, :, :, i, j) .=
(view(alpha, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) +
view(zit, :, :, i, j)
end
end
end
arrayProjection3DTo4D,
recSignal,
decayed_vt0,
decayed_epsilonRec,
vt1_diff_vth,
vt1_diff_vth_div_vth,
gammaPd_div_vth,
phiActivation,
function alifForward(kfn_zit,
zit,
wRec,
vt0,
vt1,
vth,
avth,
vRest,
zt1,
alpha,
phi,
epsilonRec,
epsilonRecA,
refractoryCounter,
refractoryDuration,
avth,
a,
beta,
rho,
gammaPd,
firingCounter)
d1, d2, d3, d4 = size(wRec)
zit .= reshape(kfn_zit, (d1, d2, 1, d4)) .* ones(size(wRec)...) # project zit into zit
phi_x_epsilonRec,
phi_x_beta,
rho_diff_phi_x_beta,
rho_div_phi_x_beta_x_epsilonRecA,
beta_x_a)
for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch
if view(refractoryCounter, :, :, i, j)[1] > 0 # refractory period is active
view(refractoryCounter, :, :, i, j)[1] -= 1
view(zt1, :, :, i, j)[1] = 0
view(vt1, :, :, i, j)[1] = view(alpha, :, :, i, j)[1] *
view(vt0, :, :, i, j)[1]
view(phi, :, :, i, j)[1] = 0.0
view(epsilonRec, :, :, i, j) .= view(alpha, :, :, i, j)[1] .*
view(epsilonRec, :, :, i, j)
view(a, :, :, i, j)[1] =
(view(rho, :, :, i, j)[1] * view(a, :, :, i, j)[1]) + 0
else # refractory period is inactive
view(vt1, :, :, i, j)[1] =
(view(alpha, :, :, i, j)[1] * view(vt0,:, :, i, j)[1]) +
sum(view(zit, :, :, i, j) .* view(wRec, :, :, i, j))
view(avth, :, :, i, j)[1] = view(vth, :, :, i, j)[1] +
(view(beta, :, :, i, j)[1] * view(a, :, :, i, j)[1])
if view(vt1, :, :, i, j)[1] > view(avth, :, :, i, j)[1]
view(zt1, :, :, i, j)[1] = 1
view(refractoryCounter, :, :, i, j)[1] =
view(refractoryDuration, :, :, i, j)[1]
view(firingCounter, :, :, i, j)[1] += 1
view(vt1, :, :, i, j)[1] = view(vRest, :, :, i, j)[1]
view(a, :, :, i, j)[1] = (view(rho, :, :, i, j)[1] *
view(a, :, :, i, j)[1]) + 1
else
view(zt1, :, :, i, j)[1] = 0
view(a, :, :, i, j)[1] =
(view(rho, :, :, i, j)[1] * view(a, :, :, i, j)[1]) + 0
end
# there is a difference from alif formula
view(phi, :, :, i, j)[1] =
(view(gammaPd, :, :, i, j)[1] / view(vth, :, :, i, j)[1]) *
max(0, 1 - ((view(vt1, :, :, i, j)[1] - view(avth, :, :, i, j)[1]) /
view(vth, :, :, i, j)[1]))
view(epsilonRec, :, :, i, j) .=
(view(alpha, :, :, i, j) .* view(epsilonRec, :, :, i, j)) +
view(zit, :, :, i, j)
view(epsilonRecA, :, :, i, j) .=
(view(phi, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) +
((view(rho, :, :, i, j)[1] -
(view(phi, :, :, i, j)[1] * view(beta, :, :, i, j)[1])) .*
view(epsilonRecA, :, :, i, j))
# project 3D kfn zit into 4D lif zit
zit .= reshape(kfn_zit,
(size(wRec, 1), size(wRec, 2), 1, size(wRec, 4))) .* arrayProjection3DTo4D
for j in 1:size(wRec, 4), i in 1:size(wRec, 3) # compute along neurons axis of every batch
if sum(@view(refractoryCounter[:,:,i,j])) > 0 # refractory period is active
@. @views refractoryCounter[:,:,i,j] -= 1
@. @views zt1[:,:,i,j] = 0
@. @views vt1[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@. @views phi[:,:,i,j] = 0
@. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
# compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j]
# compute epsilonRecA
@. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j]
@. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j]
@. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j]
@. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j]
# compute avth
@. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j]
@. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j]
else # refractory period is inactive
@. @views recSignal[:,:,i,j] = zit[:,:,i,j] * wRec[:,:,i,j]
@. @views decayed_vt0[:,:,i,j] = alpha[:,:,i,j] * vt0[:,:,i,j]
@view(vt1[:,:,i,j]) .= @view(decayed_vt0[:,:,i,j]) .+ sum(@view(recSignal[:,:,i,j]))
# compute avth
@. @views beta_x_a[:,:,i,j] = beta[:,:,i,j] * a[:,:,i,j]
@. @views avth[:,:,i,j] = vth[:,:,i,j] + beta_x_a[:,:,i,j]
if sum(@view(vt1[:,:,i,j])) > sum(@view(avth[:,:,i,j]))
@. @views zt1[:,:,i,j] = 1
@. @views refractoryCounter[:,:,i,j] = refractoryDuration[:,:,i,j]
@. @views firingCounter[:,:,i,j] += 1
@. @views vt1[:,:,i,j] = vRest[:,:,i,j]
@. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
@. @views a[:,:,i,j] = a[:,:,i,j] += 1
else
@. @views zt1[:,:,i,j] = 0
@. @views a[:,:,i,j] = rho[:,:,i,j] * a[:,:,i,j]
end
# compute phi, there is a difference from alif formula
@. @views gammaPd_div_vth[:,:,i,j] = gammaPd[:,:,i,j] / vth[:,:,i,j]
@. @views vt1_diff_vth[:,:,i,j] = vt1[:,:,i,j] - vth[:,:,i,j]
@. @views vt1_diff_vth_div_vth[:,:,i,j] = vt1_diff_vth[:,:,i,j] / vth[:,:,i,j]
@view(phiActivation[:,:,i,j]) .= max(0, 1 - sum(@view(vt1_diff_vth_div_vth[:,:,i,j])))
@. @views phi[:,:,i,j] = gammaPd_div_vth[:,:,i,j] * phiActivation[:,:,i,j]
# compute epsilonRec
@. @views decayed_epsilonRec[:,:,i,j] = alpha[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views epsilonRec[:,:,i,j] = decayed_epsilonRec[:,:,i,j] + zit[:,:,i,j]
# compute epsilonRecA
@. @views phi_x_epsilonRec[:,:,i,j] = phi[:,:,i,j] * epsilonRec[:,:,i,j]
@. @views phi_x_beta[:,:,i,j] = phi[:,:,i,j] * beta[:,:,i,j]
@. @views rho_diff_phi_x_beta[:,:,i,j] = rho[:,:,i,j] - phi_x_beta[:,:,i,j]
@. @views rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j] = rho_diff_phi_x_beta[:,:,i,j] * epsilonRecA[:,:,i,j]
@. @views epsilonRecA[:,:,i,j] = phi_x_epsilonRec[:,:,i,j] + rho_div_phi_x_beta_x_epsilonRecA[:,:,i,j]
end
end
end
end
# function alifForward(kfn_zit,
# zit,
# wRec,
# vt0,
# vt1,
# vth,
# avth,
# vRest,
# zt1,
# alpha,
# phi,
# epsilonRec,
# epsilonRecA,
# refractoryCounter,
# refractoryDuration,
# a,
# beta,
# rho,
# gammaPd,
# firingCounter)
# d1, d2, d3, d4 = size(wRec)
# zit .= reshape(kfn_zit, (d1, d2, 1, d4)) .* ones(size(wRec)...) # project zit into zit
# for j in 1:d4, i in 1:d3 # compute along neurons axis of every batch
# if view(refractoryCounter, :, :, i, j)[1] > 0 # refractory period is active
# view(refractoryCounter, :, :, i, j)[1] -= 1
# view(zt1, :, :, i, j)[1] = 0
# view(vt1, :, :, i, j)[1] = view(alpha, :, :, i, j)[1] *
# view(vt0, :, :, i, j)[1]
# view(phi, :, :, i, j)[1] = 0.0
# view(epsilonRec, :, :, i, j) .= view(alpha, :, :, i, j)[1] .*
# view(epsilonRec, :, :, i, j)
# view(a, :, :, i, j)[1] =
# (view(rho, :, :, i, j)[1] * view(a, :, :, i, j)[1]) + 0
# else # refractory period is inactive
# view(vt1, :, :, i, j)[1] =
# (view(alpha, :, :, i, j)[1] * view(vt0,:, :, i, j)[1]) +
# sum(view(zit, :, :, i, j) .* view(wRec, :, :, i, j))
# view(avth, :, :, i, j)[1] = view(vth, :, :, i, j)[1] +
# (view(beta, :, :, i, j)[1] * view(a, :, :, i, j)[1])
# if view(vt1, :, :, i, j)[1] > view(avth, :, :, i, j)[1]
# view(zt1, :, :, i, j)[1] = 1
# view(refractoryCounter, :, :, i, j)[1] =
# view(refractoryDuration, :, :, i, j)[1]
# view(firingCounter, :, :, i, j)[1] += 1
# view(vt1, :, :, i, j)[1] = view(vRest, :, :, i, j)[1]
# view(a, :, :, i, j)[1] = (view(rho, :, :, i, j)[1] *
# view(a, :, :, i, j)[1]) + 1
# else
# view(zt1, :, :, i, j)[1] = 0
# view(a, :, :, i, j)[1] =
# (view(rho, :, :, i, j)[1] * view(a, :, :, i, j)[1]) + 0
# end
# # there is a difference from alif formula
# view(phi, :, :, i, j)[1] =
# (view(gammaPd, :, :, i, j)[1] / view(vth, :, :, i, j)[1]) *
# max(0, 1 - ((view(vt1, :, :, i, j)[1] - view(avth, :, :, i, j)[1]) /
# view(vth, :, :, i, j)[1]))
# view(epsilonRec, :, :, i, j) .=
# (view(alpha, :, :, i, j) .* view(epsilonRec, :, :, i, j)) +
# view(zit, :, :, i, j)
# view(epsilonRecA, :, :, i, j) .=
# (view(phi, :, :, i, j)[1] .* view(epsilonRec, :, :, i, j)) +
# ((view(rho, :, :, i, j)[1] -
# (view(phi, :, :, i, j)[1] * view(beta, :, :, i, j)[1])) .*
# view(epsilonRecA, :, :, i, j))
# end
# end
# end
function onForward(kfn_zit,
zit,
wOut,