{
  "model_config": {
    "vocab_size": 14,
    "d_model": 64,
    "n_heads": 4,
    "n_layers": 1,
    "max_seq_len": 12
  },
  "qk_formula": "score = q_ans @ k_source / sqrt(d_head)",
  "shapes": {
    "head_0": {
      "W_Q": [
        16,
        64
      ],
      "W_K": [
        16,
        64
      ],
      "q_ans": [
        16
      ],
      "digit_scores": [
        5,
        10
      ],
      "special_scores": [
        3
      ]
    },
    "head_1": {
      "W_Q": [
        16,
        64
      ],
      "W_K": [
        16,
        64
      ],
      "q_ans": [
        16
      ],
      "digit_scores": [
        5,
        10
      ],
      "special_scores": [
        3
      ]
    },
    "head_2": {
      "W_Q": [
        16,
        64
      ],
      "W_K": [
        16,
        64
      ],
      "q_ans": [
        16
      ],
      "digit_scores": [
        5,
        10
      ],
      "special_scores": [
        3
      ]
    },
    "head_3": {
      "W_Q": [
        16,
        64
      ],
      "W_K": [
        16,
        64
      ],
      "q_ans": [
        16
      ],
      "digit_scores": [
        5,
        10
      ],
      "special_scores": [
        3
      ]
    }
  },
  "metrics": [
    {
      "head": 0,
      "digit_score_correlation": 0.7576097249984741,
      "score_range": 27.277576446533203,
      "argmax_digit": 9,
      "argmin_digit": 0,
      "position_dependence_max_abs": 0.020326614379882812,
      "mean_special_score": -9.954524040222168,
      "max_special_score": 7.8903021812438965,
      "max_special_token": "ANS"
    },
    {
      "head": 1,
      "digit_score_correlation": -0.49038273096084595,
      "score_range": 36.23717498779297,
      "argmax_digit": 1,
      "argmin_digit": 6,
      "position_dependence_max_abs": 0.12836933135986328,
      "mean_special_score": 0.509637176990509,
      "max_special_score": 24.950904846191406,
      "max_special_token": "ANS"
    },
    {
      "head": 2,
      "digit_score_correlation": 0.8846294283866882,
      "score_range": 30.124908447265625,
      "argmax_digit": 9,
      "argmin_digit": 0,
      "position_dependence_max_abs": 0.012168407440185547,
      "mean_special_score": -11.944351196289062,
      "max_special_score": 2.7943782806396484,
      "max_special_token": "ANS"
    },
    {
      "head": 3,
      "digit_score_correlation": 0.9946726560592651,
      "score_range": 45.76317596435547,
      "argmax_digit": 9,
      "argmin_digit": 0,
      "position_dependence_max_abs": 0.008373260498046875,
      "mean_special_score": -20.010902404785156,
      "max_special_score": -12.100517272949219,
      "max_special_token": "ANS"
    }
  ],
  "headline": "Head 3 has the cleanest monotonic digit-ranking QK circuit; Head 2 and Head 0 also prefer higher digits less cleanly, while Head 1 does not."
}