x, target = 2.0, 1.0
params = {'w': 0.5, 'b': 0.1, 'v': 2.0, 'c': -0.2}
def forward(p):
    z = p['w']*x+p['b']
    h = max(0.0, z)
    output = p['v']*h+p['c']
    return z, h, output

def loss(p):
    _, _, output = forward(p)
    return 0.5*(output-target)**2

z, h, output = forward(params)
gout = output-target
gz = gout*params['v']*(1.0 if z > 0 else 0.0)
grad = {'w': gz*x, 'b': gz, 'v': gout*h, 'c': gout}
eps = 1e-6
errors = {}
for name in params:
    plus, minus = params.copy(), params.copy()
    plus[name] += eps
    minus[name] -= eps
    numeric = (loss(plus)-loss(minus))/(2*eps)
    errors[name] = abs(numeric-grad[name])
updated = {k: value-0.01*grad[k] for k, value in params.items()}
print('forward', z, h, output, loss(params))
print('gradients', grad)
print('finite difference max error', max(errors.values()))
print('updated', updated, 'new loss', loss(updated))
assert abs(grad['w']-4) < 1e-12 and abs(grad['v']-1.1) < 1e-12
assert max(errors.values()) < 1e-7
assert abs(loss(updated)-0.3034205) < 1e-12