1 write to _hiddenSize
Microsoft.ML.GenAI.Phi (1)
Module\Phi2Attention.cs (1)
61
this.
_hiddenSize
= config.HiddenSize;
9 references to _hiddenSize
Microsoft.ML.GenAI.Phi (9)
Module\Phi2Attention.cs (9)
63
this._headDim = this.
_hiddenSize
/ this._numAttentionHeads;
70
Contract.Assert(this.
_hiddenSize
% (this._headDim * this._numAttentionHeads) == 0, "hidden_size must be divisible by num_attention_heads");
71
this.q_proj = new GenAILinear(this.
_hiddenSize
, this._numAttentionHeads * this._headDim, hasBias: true, dtype: config.Dtype);
72
this.k_proj = new GenAILinear(this.
_hiddenSize
, this._numKeyValueHeads * this._headDim, hasBias: true, dtype: config.Dtype);
73
this.v_proj = new GenAILinear(this.
_hiddenSize
, this._numKeyValueHeads * this._headDim, hasBias: true, dtype: config.Dtype);
74
this.dense = new GenAILinear(this._numAttentionHeads * this._headDim, this.
_hiddenSize
, hasBias: true, dtype: config.Dtype);
79
this.q_layernorm = nn.LayerNorm(this.
_hiddenSize
/ this._numAttentionHeads, eps: config.LayerNormEps, elementwise_affine: true, dtype: config.Dtype);
80
this.k_layernorm = nn.LayerNorm(this.
_hiddenSize
/ this._numAttentionHeads, eps: config.LayerNormEps, elementwise_affine: true, dtype: config.Dtype);
156
attnOutput = attnOutput.reshape(batchSize, seqLen, this.
_hiddenSize
);