{
  "simulation_only": true,
  "epsilon": "1/5",
  "policy_seed": 7,
  "environment_seed": 11,
  "warmup_steps": 4,
  "environment_rates": {
    "red": {
      "A": "4/5",
      "B": "1/5"
    },
    "blue": {
      "A": "1/5",
      "B": "4/5"
    }
  },
  "trace": [
    {
      "step": 1,
      "context": "red",
      "action": "A",
      "reason": "forced_warmup",
      "greedy_action_before_selection": null,
      "selection_propensity_conditional_on_history": "1",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 0,
          "successes": 0
        },
        "B": {
          "count": 0,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 1,
          "successes": 1
        },
        "B": {
          "count": 0,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 2,
      "context": "red",
      "action": "B",
      "reason": "forced_warmup",
      "greedy_action_before_selection": null,
      "selection_propensity_conditional_on_history": "1",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 1,
          "successes": 1
        },
        "B": {
          "count": 0,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 1,
          "successes": 1
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 3,
      "context": "blue",
      "action": "A",
      "reason": "forced_warmup",
      "greedy_action_before_selection": null,
      "selection_propensity_conditional_on_history": "1",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 0,
          "successes": 0
        },
        "B": {
          "count": 0,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 1,
          "successes": 0
        },
        "B": {
          "count": 0,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 4,
      "context": "blue",
      "action": "B",
      "reason": "forced_warmup",
      "greedy_action_before_selection": null,
      "selection_propensity_conditional_on_history": "1",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 1,
          "successes": 0
        },
        "B": {
          "count": 0,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 1,
          "successes": 0
        },
        "B": {
          "count": 1,
          "successes": 1
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 5,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 1,
          "successes": 1
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 2,
          "successes": 2
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 6,
      "context": "blue",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "1/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 1,
          "successes": 0
        },
        "B": {
          "count": 1,
          "successes": 1
        }
      },
      "after": {
        "A": {
          "count": 2,
          "successes": 0
        },
        "B": {
          "count": 1,
          "successes": 1
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 7,
      "context": "red",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 2,
          "successes": 2
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 3,
          "successes": 3
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 8,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 2,
          "successes": 0
        },
        "B": {
          "count": 1,
          "successes": 1
        }
      },
      "after": {
        "A": {
          "count": 2,
          "successes": 0
        },
        "B": {
          "count": 2,
          "successes": 2
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 9,
      "context": "red",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 3,
          "successes": 3
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 4,
          "successes": 4
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 10,
      "context": "blue",
      "action": "B",
      "reason": "exploration",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 2,
          "successes": 0
        },
        "B": {
          "count": 2,
          "successes": 2
        }
      },
      "after": {
        "A": {
          "count": 2,
          "successes": 0
        },
        "B": {
          "count": 3,
          "successes": 3
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 11,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 4,
          "successes": 4
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 5,
          "successes": 5
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 12,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 2,
          "successes": 0
        },
        "B": {
          "count": 3,
          "successes": 3
        }
      },
      "after": {
        "A": {
          "count": 2,
          "successes": 0
        },
        "B": {
          "count": 4,
          "successes": 4
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 13,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 5,
          "successes": 5
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 6,
          "successes": 6
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 14,
      "context": "blue",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "1/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 2,
          "successes": 0
        },
        "B": {
          "count": 4,
          "successes": 4
        }
      },
      "after": {
        "A": {
          "count": 3,
          "successes": 0
        },
        "B": {
          "count": 4,
          "successes": 4
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 15,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 6,
          "successes": 6
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 7
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 16,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 3,
          "successes": 0
        },
        "B": {
          "count": 4,
          "successes": 4
        }
      },
      "after": {
        "A": {
          "count": 3,
          "successes": 0
        },
        "B": {
          "count": 5,
          "successes": 5
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 17,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 7,
          "successes": 7
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 8,
          "successes": 7
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 18,
      "context": "blue",
      "action": "B",
      "reason": "exploration",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 3,
          "successes": 0
        },
        "B": {
          "count": 5,
          "successes": 5
        }
      },
      "after": {
        "A": {
          "count": 3,
          "successes": 0
        },
        "B": {
          "count": 6,
          "successes": 5
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 19,
      "context": "red",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 8,
          "successes": 7
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 9,
          "successes": 8
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 20,
      "context": "blue",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "1/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 3,
          "successes": 0
        },
        "B": {
          "count": 6,
          "successes": 5
        }
      },
      "after": {
        "A": {
          "count": 4,
          "successes": 0
        },
        "B": {
          "count": 6,
          "successes": 5
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 21,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 9,
          "successes": 8
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 10,
          "successes": 9
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 22,
      "context": "blue",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "1/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 4,
          "successes": 0
        },
        "B": {
          "count": 6,
          "successes": 5
        }
      },
      "after": {
        "A": {
          "count": 5,
          "successes": 1
        },
        "B": {
          "count": 6,
          "successes": 5
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 23,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 10,
          "successes": 9
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 11,
          "successes": 10
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 24,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 5,
          "successes": 1
        },
        "B": {
          "count": 6,
          "successes": 5
        }
      },
      "after": {
        "A": {
          "count": 5,
          "successes": 1
        },
        "B": {
          "count": 7,
          "successes": 6
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 25,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 11,
          "successes": 10
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 12,
          "successes": 11
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 26,
      "context": "blue",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "1/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 5,
          "successes": 1
        },
        "B": {
          "count": 7,
          "successes": 6
        }
      },
      "after": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 7,
          "successes": 6
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 27,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 12,
          "successes": 11
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 13,
          "successes": 12
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 28,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 7,
          "successes": 6
        }
      },
      "after": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 8,
          "successes": 7
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 29,
      "context": "red",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 13,
          "successes": 12
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 14,
          "successes": 13
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 30,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 8,
          "successes": 7
        }
      },
      "after": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 9,
          "successes": 7
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 31,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 14,
          "successes": 13
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 15,
          "successes": 14
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 32,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 9,
          "successes": 7
        }
      },
      "after": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 10,
          "successes": 8
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 33,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 15,
          "successes": 14
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 16,
          "successes": 15
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 34,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 10,
          "successes": 8
        }
      },
      "after": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 11,
          "successes": 9
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 35,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 16,
          "successes": 15
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 17,
          "successes": 16
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 36,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 11,
          "successes": 9
        }
      },
      "after": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 12,
          "successes": 10
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 37,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 17,
          "successes": 16
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 18,
          "successes": 16
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 38,
      "context": "blue",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "1/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 6,
          "successes": 1
        },
        "B": {
          "count": 12,
          "successes": 10
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 12,
          "successes": 10
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 39,
      "context": "red",
      "action": "B",
      "reason": "exploration",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "1/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 18,
          "successes": 16
        },
        "B": {
          "count": 1,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 18,
          "successes": 16
        },
        "B": {
          "count": 2,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 40,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 12,
          "successes": 10
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 13,
          "successes": 11
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 41,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 18,
          "successes": 16
        },
        "B": {
          "count": 2,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 19,
          "successes": 17
        },
        "B": {
          "count": 2,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 42,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 13,
          "successes": 11
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 14,
          "successes": 12
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 43,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 19,
          "successes": 17
        },
        "B": {
          "count": 2,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 20,
          "successes": 18
        },
        "B": {
          "count": 2,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 44,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 14,
          "successes": 12
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 15,
          "successes": 13
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 45,
      "context": "red",
      "action": "B",
      "reason": "exploration",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "1/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 20,
          "successes": 18
        },
        "B": {
          "count": 2,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 20,
          "successes": 18
        },
        "B": {
          "count": 3,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 46,
      "context": "blue",
      "action": "B",
      "reason": "exploration",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 15,
          "successes": 13
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 16,
          "successes": 14
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 47,
      "context": "red",
      "action": "B",
      "reason": "exploration",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "1/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 20,
          "successes": 18
        },
        "B": {
          "count": 3,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 20,
          "successes": 18
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 48,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 16,
          "successes": 14
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 17,
          "successes": 15
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 49,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 20,
          "successes": 18
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 21,
          "successes": 18
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 50,
      "context": "blue",
      "action": "B",
      "reason": "exploration",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 17,
          "successes": 15
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 18,
          "successes": 15
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 51,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 21,
          "successes": 18
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 22,
          "successes": 19
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 52,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 18,
          "successes": 15
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 19,
          "successes": 16
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 53,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 22,
          "successes": 19
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 23,
          "successes": 19
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 54,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 19,
          "successes": 16
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 20,
          "successes": 17
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 55,
      "context": "red",
      "action": "A",
      "reason": "exploration",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 23,
          "successes": 19
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 24,
          "successes": 19
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 56,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 20,
          "successes": 17
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 21,
          "successes": 18
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 57,
      "context": "red",
      "action": "A",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 24,
          "successes": 19
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 25,
          "successes": 20
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 58,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 21,
          "successes": 18
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 22,
          "successes": 19
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    },
    {
      "step": 59,
      "context": "red",
      "action": "B",
      "reason": "exploration",
      "greedy_action_before_selection": "A",
      "selection_propensity_conditional_on_history": "1/10",
      "observed_selected_reward": 0,
      "before": {
        "A": {
          "count": 25,
          "successes": 20
        },
        "B": {
          "count": 4,
          "successes": 0
        }
      },
      "after": {
        "A": {
          "count": 25,
          "successes": 20
        },
        "B": {
          "count": 5,
          "successes": 0
        }
      },
      "expected_pseudo_regret_from_known_simulator": "3/5",
      "unchosen_reward_observed": false
    },
    {
      "step": 60,
      "context": "blue",
      "action": "B",
      "reason": "greedy_branch",
      "greedy_action_before_selection": "B",
      "selection_propensity_conditional_on_history": "9/10",
      "observed_selected_reward": 1,
      "before": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 22,
          "successes": 19
        }
      },
      "after": {
        "A": {
          "count": 7,
          "successes": 1
        },
        "B": {
          "count": 23,
          "successes": 20
        }
      },
      "expected_pseudo_regret_from_known_simulator": "0",
      "unchosen_reward_observed": false
    }
  ],
  "final_counts": {
    "red": {
      "A": 25,
      "B": 5
    },
    "blue": {
      "A": 7,
      "B": 23
    }
  },
  "final_successes": {
    "red": {
      "A": 20,
      "B": 0
    },
    "blue": {
      "A": 1,
      "B": 20
    }
  },
  "total_observed_reward": 41,
  "expected_pseudo_regret": "36/5",
  "real_users_clicks_sales_or_actions": false,
  "VW_executed": false
}