1 write to _numHeads
Microsoft.ML.GenAI.Core (1)
Module\Attention.cs (1)
105
this.
_numHeads
= numHeads;
6 references to _numHeads
Microsoft.ML.GenAI.Core (6)
Module\Attention.cs (6)
112
Contract.Assert(this._hiddenSize % (this._headDim * this.
_numHeads
) == 0, "hidden_size must be divisible by num_heads");
117
var opSize = this.
_numHeads
* this._headDim + 2 * (this._numKeyValueHeads * this._headDim);
122
this.q_proj = new QuantizedLinear(this._hiddenSize, this.
_numHeads
* this._headDim, hasBias: attentionBias, dtype: dtype);
147
var queryPos = this.
_numHeads
* this._headDim;
163
queryStates = queryStates.view(bsz, qLen, this.
_numHeads
, this._headDim).transpose(1, 2);
190
Contract.Assert(attnWeights.shape[1] == this.
_numHeads
);