1 write to _numAttentionHeads
Microsoft.ML.GenAI.Phi (1)
Module\Phi2Attention.cs (1)
62
this.
_numAttentionHeads
= config.NumAttentionHeads;
8 references to _numAttentionHeads
Microsoft.ML.GenAI.Phi (8)
Module\Phi2Attention.cs (8)
63
this._headDim = this._hiddenSize / this.
_numAttentionHeads
;
65
this._numKeyValueGroups = this.
_numAttentionHeads
/ this._numKeyValueHeads;
70
Contract.Assert(this._hiddenSize % (this._headDim * this.
_numAttentionHeads
) == 0, "hidden_size must be divisible by num_attention_heads");
71
this.q_proj = new GenAILinear(this._hiddenSize, this.
_numAttentionHeads
* this._headDim, hasBias: true, dtype: config.Dtype);
74
this.dense = new GenAILinear(this.
_numAttentionHeads
* this._headDim, this._hiddenSize, hasBias: true, dtype: config.Dtype);
79
this.q_layernorm = nn.LayerNorm(this._hiddenSize / this.
_numAttentionHeads
, eps: config.LayerNormEps, elementwise_affine: true, dtype: config.Dtype);
80
this.k_layernorm = nn.LayerNorm(this._hiddenSize / this.
_numAttentionHeads
, eps: config.LayerNormEps, elementwise_affine: true, dtype: config.Dtype);
121
queryStates = queryStates.view(batchSize, seqLen, this.
_numAttentionHeads
, this._headDim).transpose_(1, 2);