System\Numerics\Tensors\netcore\Common\TensorPrimitives.IAggregationOperator.cs (139)
112if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TTransformOperator.Vectorizable)
152Vector128<T> vresult = Vector128.Create(TAggregationOperator.IdentityValue);
156Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
157Vector128<T> end = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
216vector1 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)));
217vector2 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)));
218vector3 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)));
219vector4 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)));
228vector1 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)));
229vector2 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)));
230vector3 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)));
231vector4 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)));
255beg = Vector128.ConditionalSelect(CreateAlignmentMaskVector128<T>((int)misalignment), beg, Vector128.Create(TAggregationOperator.IdentityValue));
272Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)));
279Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)));
286Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)));
293Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)));
300Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)));
307Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)));
314Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 1)));
322end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)trailing), end, Vector128.Create(TAggregationOperator.IdentityValue));
797Debug.Assert(Vector128.IsHardwareAccelerated);
799Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
800Vector128<T> end = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
802end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
811Debug.Assert(Vector128.IsHardwareAccelerated);
813Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
944Debug.Assert(Vector128.IsHardwareAccelerated);
946Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
947Vector128<T> end = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
949end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
958Debug.Assert(Vector128.IsHardwareAccelerated);
960Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
1044Debug.Assert(Vector128.IsHardwareAccelerated);
1046Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
1047Vector128<T> end = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
1049end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
1057Debug.Assert(Vector128.IsHardwareAccelerated);
1059Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
1127Debug.Assert(Vector128.IsHardwareAccelerated);
1129Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
1130Vector128<T> end = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
1132end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
1140Debug.Assert(Vector128.IsHardwareAccelerated);
1142Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
1231if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TBinaryOperator.Vectorizable)
1272Vector128<T> vresult = Vector128.Create(TAggregationOperator.IdentityValue);
1276Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1277Vector128.LoadUnsafe(ref yRef));
1278Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1279Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
1345vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
1346Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)));
1347vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
1348Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)));
1349vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
1350Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)));
1351vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
1352Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)));
1361vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
1362Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)));
1363vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
1364Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)));
1365vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
1366Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)));
1367vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
1368Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)));
1394beg = Vector128.ConditionalSelect(CreateAlignmentMaskVector128<T>((int)misalignment), beg, Vector128.Create(TAggregationOperator.IdentityValue));
1411Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)),
1412Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 7)));
1419Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)),
1420Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 6)));
1427Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)),
1428Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 5)));
1435Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)),
1436Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 4)));
1443Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)),
1444Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 3)));
1451Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)),
1452Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 2)));
1459Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 1)),
1460Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 1)));
1468end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)trailing), end, Vector128.Create(TAggregationOperator.IdentityValue));
1996Debug.Assert(Vector128.IsHardwareAccelerated);
1998Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1999Vector128.LoadUnsafe(ref yRef));
2000Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2001Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
2003end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
2012Debug.Assert(Vector128.IsHardwareAccelerated);
2014Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2015Vector128.LoadUnsafe(ref yRef));
2149Debug.Assert(Vector128.IsHardwareAccelerated);
2151Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2152Vector128.LoadUnsafe(ref yRef));
2153Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2154Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
2156end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
2165Debug.Assert(Vector128.IsHardwareAccelerated);
2167Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2168Vector128.LoadUnsafe(ref yRef));
2255Debug.Assert(Vector128.IsHardwareAccelerated);
2257Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2258Vector128.LoadUnsafe(ref yRef));
2259Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2260Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
2262end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
2270Debug.Assert(Vector128.IsHardwareAccelerated);
2272Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2273Vector128.LoadUnsafe(ref yRef));
2346Debug.Assert(Vector128.IsHardwareAccelerated);
2348Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2349Vector128.LoadUnsafe(ref yRef));
2350Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2351Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
2353end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
2361Debug.Assert(Vector128.IsHardwareAccelerated);
2363Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2364Vector128.LoadUnsafe(ref yRef));
2395return Vector128.LoadUnsafe(
2402return Vector128.LoadUnsafe(
2409return Vector128.LoadUnsafe(
2416return Vector128.LoadUnsafe(
2503return Vector128.LoadUnsafe(
2510return Vector128.LoadUnsafe(
2517return Vector128.LoadUnsafe(
2524return Vector128.LoadUnsafe(
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IBinaryOperator.cs (158)
121if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TBinaryOperator.Vectorizable)
160Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
161Vector128.LoadUnsafe(ref yRef));
162Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
163Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
217vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
218Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)));
219vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
220Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)));
221vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
222Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)));
223vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
224Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)));
233vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
234Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)));
235vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
236Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)));
237vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
238Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)));
239vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
240Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)));
263vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
264Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)));
265vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
266Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)));
267vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
268Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)));
269vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
270Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)));
279vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
280Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)));
281vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
282Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)));
283vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
284Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)));
285vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
286Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)));
326Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8)),
327Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 8)));
334Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)),
335Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 7)));
342Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)),
343Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 6)));
350Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)),
351Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 5)));
358Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)),
359Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 4)));
366Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)),
367Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 3)));
374Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)),
375Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 2)));
984Debug.Assert(Vector128.IsHardwareAccelerated);
986Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
987Vector128.LoadUnsafe(ref yRef));
988Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
989Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
1000Debug.Assert(Vector128.IsHardwareAccelerated);
1002Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1003Vector128.LoadUnsafe(ref yRef));
1148Debug.Assert(Vector128.IsHardwareAccelerated);
1150Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1151Vector128.LoadUnsafe(ref yRef));
1152Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1153Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
1164Debug.Assert(Vector128.IsHardwareAccelerated);
1166Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1167Vector128.LoadUnsafe(ref yRef));
1257Debug.Assert(Vector128.IsHardwareAccelerated);
1259Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1260Vector128.LoadUnsafe(ref yRef));
1261Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1262Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
1272Debug.Assert(Vector128.IsHardwareAccelerated);
1274Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1275Vector128.LoadUnsafe(ref yRef));
1345Debug.Assert(Vector128.IsHardwareAccelerated);
1347Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1348Vector128.LoadUnsafe(ref yRef));
1349Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1350Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
1360Debug.Assert(Vector128.IsHardwareAccelerated);
1362Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1363Vector128.LoadUnsafe(ref yRef));
1453if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TTransformOperator.Vectorizable && TBinaryOperator.Vectorizable)
1492Vector128<T> yVec = Vector128.Create(y);
1494Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
1496Vector128<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count)),
1548vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0))),
1550vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1))),
1552vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2))),
1554vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3))),
1564vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4))),
1566vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5))),
1568vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6))),
1570vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7))),
1593vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0))),
1595vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1))),
1597vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2))),
1599vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3))),
1609vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4))),
1611vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5))),
1613vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6))),
1615vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7))),
1654Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8))),
1662Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7))),
1670Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6))),
1678Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5))),
1686Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4))),
1694Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3))),
1702Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2))),
2306Debug.Assert(Vector128.IsHardwareAccelerated);
2308Vector128<T> yVec = Vector128.Create(y);
2310Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2312Vector128<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count)),
2324Debug.Assert(Vector128.IsHardwareAccelerated);
2326Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2327Vector128.Create(y));
2474Debug.Assert(Vector128.IsHardwareAccelerated);
2476Vector128<T> yVec = Vector128.Create(y);
2478Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2480Vector128<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count)),
2492Debug.Assert(Vector128.IsHardwareAccelerated);
2494Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2495Vector128.Create(y));
2587Debug.Assert(Vector128.IsHardwareAccelerated);
2589Vector128<T> yVec = Vector128.Create(y);
2591Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2593Vector128<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count)),
2604Debug.Assert(Vector128.IsHardwareAccelerated);
2606Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2607Vector128.Create(y));
2679Debug.Assert(Vector128.IsHardwareAccelerated);
2681Vector128<T> yVec = Vector128.Create(y);
2683Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2685Vector128<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count)),
2696Debug.Assert(Vector128.IsHardwareAccelerated);
2698Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2699Vector128.Create(y));
2743x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As<byte, T>());
2744x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>());
2745x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>());
2746x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>());
2750x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As<short, T>());
2751x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As<short, T>());
2752x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As<short, T>());
2756x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt32(), Vector128.Create(2, 3, 0, 1)).As<int, T>());
2757x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt32(), Vector128.Create(1, 0, 3, 2)).As<int, T>());
2762x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt64(), Vector128.Create(1, 0)).As<long, T>());
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IBooleanUnaryOperator.cs (37)
41public static bool ShouldEarlyExit(Vector128<T> result) => Vector128.AnyWhereAllBitsSet(result);
69typeof(T) == typeof(float) ? Vector128.EqualsAny(result.AsUInt32(), Vector128<uint>.Zero) :
70typeof(T) == typeof(double) ? Vector128.EqualsAny(result.AsUInt64(), Vector128<ulong>.Zero) :
71Vector128.EqualsAny(result, Vector128<T>.Zero);
157if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128<T>.IsSupported)
165if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i))))
176TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128<T>.Count)))))
297if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128<T>.IsSupported)
321Vector128<byte> v = TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i)).AsByte();
411if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128<T>.IsSupported)
436Vector128.Narrow(
437TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i)).AsUInt16(),
438TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128<T>.Count))).AsUInt16());
536if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128<T>.IsSupported)
561Vector128.Narrow(
562Vector128.Narrow(
563TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i)).AsUInt32(),
564TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128<T>.Count))).AsUInt32()),
565Vector128.Narrow(
566TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (2 * Vector128<T>.Count)))).AsUInt32(),
567TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (3 * Vector128<T>.Count)))).AsUInt32()));
683if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128<T>.IsSupported)
710Vector128.Narrow(
711Vector128.Narrow(
712Vector128.Narrow(
713TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i)).AsUInt64(),
714TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128<T>.Count))).AsUInt64()),
715Vector128.Narrow(
716TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (2 * Vector128<T>.Count)))).AsUInt64(),
717TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (3 * Vector128<T>.Count)))).AsUInt64())),
718Vector128.Narrow(
719Vector128.Narrow(
720TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (4 * Vector128<T>.Count)))).AsUInt64(),
721TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (5 * Vector128<T>.Count)))).AsUInt64()),
722Vector128.Narrow(
723TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (6 * Vector128<T>.Count)))).AsUInt64(),
724TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (7 * Vector128<T>.Count)))).AsUInt64())));
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IIndexOfOperator.cs (44)
30tmpResult = Vector128.Shuffle(result.AsInt64(), Vector128.Create(1, 0)).As<long, T>();
31tmpIndex = Vector128.Shuffle(resultIndex.AsInt64(), Vector128.Create(1, 0)).As<long, T>();
41tmpResult = Vector128.Shuffle(result.AsInt32(), Vector128.Create(2, 3, 0, 1)).As<int, T>();
42tmpIndex = Vector128.Shuffle(resultIndex.AsInt32(), Vector128.Create(2, 3, 0, 1)).As<int, T>();
46tmpResult = Vector128.Shuffle(result.AsInt32(), Vector128.Create(1, 0, 3, 2)).As<int, T>();
47tmpIndex = Vector128.Shuffle(resultIndex.AsInt32(), Vector128.Create(1, 0, 3, 2)).As<int, T>();
57tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As<short, T>();
58tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As<short, T>();
62tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As<short, T>();
63tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As<short, T>();
67tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As<short, T>();
68tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As<short, T>();
78tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As<byte, T>();
79tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As<byte, T>();
83tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
84tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
88tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
89tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
93tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
94tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
127sizeof(T) == sizeof(long) ? Vector128.LessThan(indices1.AsInt64(), indices2.AsInt64()).As<long, T>() :
128sizeof(T) == sizeof(int) ? Vector128.LessThan(indices1.AsInt32(), indices2.AsInt32()).As<int, T>() :
129sizeof(T) == sizeof(short) ? Vector128.LessThan(indices1.AsInt16(), indices2.AsInt16()).As<short, T>() :
130Vector128.LessThan(indices1.AsByte(), indices2.AsByte()).As<byte, T>();
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IStatefulUnaryOperator.cs (46)
87if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TStatefulUnaryOperator.Vectorizable)
125Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
126Vector128<T> end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
177vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)));
178vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)));
179vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)));
180vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)));
189vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)));
190vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)));
191vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)));
192vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)));
214vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)));
215vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)));
216vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)));
217vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)));
226vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)));
227vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)));
228vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)));
229vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)));
267Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8)));
274Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)));
281Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)));
288Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)));
295Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)));
302Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)));
309Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)));
853Debug.Assert(Vector128.IsHardwareAccelerated);
855Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
856Vector128<T> end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
867Debug.Assert(Vector128.IsHardwareAccelerated);
869Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
997Debug.Assert(Vector128.IsHardwareAccelerated);
999Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
1000Vector128<T> end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
1011Debug.Assert(Vector128.IsHardwareAccelerated);
1013Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
1094Debug.Assert(Vector128.IsHardwareAccelerated);
1096Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
1097Vector128<T> end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
1107Debug.Assert(Vector128.IsHardwareAccelerated);
1109Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
1174Debug.Assert(Vector128.IsHardwareAccelerated);
1176Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
1177Vector128<T> end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
1187Debug.Assert(Vector128.IsHardwareAccelerated);
1189Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
System\Numerics\Tensors\netcore\Common\TensorPrimitives.ITernaryOperator.cs (276)
106if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported)
146Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
147Vector128.LoadUnsafe(ref yRef),
148Vector128.LoadUnsafe(ref zRef));
149Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
150Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
151Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128<T>.Count));
208vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
209Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)),
210Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 0)));
211vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
212Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)),
213Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 1)));
214vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
215Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)),
216Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 2)));
217vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
218Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)),
219Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 3)));
228vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
229Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)),
230Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 4)));
231vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
232Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)),
233Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 5)));
234vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
235Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)),
236Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 6)));
237vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
238Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)),
239Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 7)));
263vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
264Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)),
265Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 0)));
266vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
267Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)),
268Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 1)));
269vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
270Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)),
271Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 2)));
272vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
273Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)),
274Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 3)));
283vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
284Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)),
285Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 4)));
286vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
287Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)),
288Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 5)));
289vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
290Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)),
291Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 6)));
292vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
293Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)),
294Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 7)));
336Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8)),
337Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 8)),
338Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 8)));
345Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)),
346Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 7)),
347Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 7)));
354Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)),
355Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 6)),
356Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 6)));
363Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)),
364Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 5)),
365Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 5)));
372Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)),
373Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 4)),
374Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 4)));
381Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)),
382Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 3)),
383Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 3)));
390Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)),
391Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 2)),
392Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 2)));
1066Debug.Assert(Vector128.IsHardwareAccelerated);
1068Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1069Vector128.LoadUnsafe(ref yRef),
1070Vector128.LoadUnsafe(ref zRef));
1071Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1072Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
1073Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128<T>.Count));
1084Debug.Assert(Vector128.IsHardwareAccelerated);
1086Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1087Vector128.LoadUnsafe(ref yRef),
1088Vector128.LoadUnsafe(ref zRef));
1250Debug.Assert(Vector128.IsHardwareAccelerated);
1252Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1253Vector128.LoadUnsafe(ref yRef),
1254Vector128.LoadUnsafe(ref zRef));
1255Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1256Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
1257Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128<T>.Count));
1268Debug.Assert(Vector128.IsHardwareAccelerated);
1270Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1271Vector128.LoadUnsafe(ref yRef),
1272Vector128.LoadUnsafe(ref zRef));
1371Debug.Assert(Vector128.IsHardwareAccelerated);
1373Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1374Vector128.LoadUnsafe(ref yRef),
1375Vector128.LoadUnsafe(ref zRef));
1376Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1377Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
1378Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128<T>.Count));
1388Debug.Assert(Vector128.IsHardwareAccelerated);
1390Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1391Vector128.LoadUnsafe(ref yRef),
1392Vector128.LoadUnsafe(ref zRef));
1467Debug.Assert(Vector128.IsHardwareAccelerated);
1469Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1470Vector128.LoadUnsafe(ref yRef),
1471Vector128.LoadUnsafe(ref zRef));
1472Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1473Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
1474Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128<T>.Count));
1484Debug.Assert(Vector128.IsHardwareAccelerated);
1486Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1487Vector128.LoadUnsafe(ref yRef),
1488Vector128.LoadUnsafe(ref zRef));
1581if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported)
1621Vector128<T> zVec = Vector128.Create(z);
1623Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1624Vector128.LoadUnsafe(ref yRef),
1626Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1627Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
1682vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
1683Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)),
1685vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
1686Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)),
1688vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
1689Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)),
1691vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
1692Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)),
1702vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
1703Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)),
1705vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
1706Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)),
1708vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
1709Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)),
1711vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
1712Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)),
1736vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
1737Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)),
1739vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
1740Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)),
1742vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
1743Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)),
1745vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
1746Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)),
1756vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
1757Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)),
1759vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
1760Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)),
1762vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
1763Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)),
1765vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
1766Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)),
1807Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8)),
1808Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 8)),
1816Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)),
1817Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 7)),
1825Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)),
1826Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 6)),
1834Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)),
1835Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 5)),
1843Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)),
1844Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 4)),
1852Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)),
1853Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 3)),
1861Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)),
1862Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 2)),
2531Debug.Assert(Vector128.IsHardwareAccelerated);
2533Vector128<T> zVec = Vector128.Create(z);
2535Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2536Vector128.LoadUnsafe(ref yRef),
2538Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2539Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
2551Debug.Assert(Vector128.IsHardwareAccelerated);
2553Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2554Vector128.LoadUnsafe(ref yRef),
2555Vector128.Create(z));
2719Debug.Assert(Vector128.IsHardwareAccelerated);
2721Vector128<T> zVec = Vector128.Create(z);
2723Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2724Vector128.LoadUnsafe(ref yRef),
2726Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2727Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
2739Debug.Assert(Vector128.IsHardwareAccelerated);
2741Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2742Vector128.LoadUnsafe(ref yRef),
2743Vector128.Create(z));
2844Debug.Assert(Vector128.IsHardwareAccelerated);
2846Vector128<T> zVec = Vector128.Create(z);
2848Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2849Vector128.LoadUnsafe(ref yRef),
2851Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2852Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
2863Debug.Assert(Vector128.IsHardwareAccelerated);
2865Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2866Vector128.LoadUnsafe(ref yRef),
2867Vector128.Create(z));
2944Debug.Assert(Vector128.IsHardwareAccelerated);
2946Vector128<T> zVec = Vector128.Create(z);
2948Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2949Vector128.LoadUnsafe(ref yRef),
2951Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2952Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
2963Debug.Assert(Vector128.IsHardwareAccelerated);
2965Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2966Vector128.LoadUnsafe(ref yRef),
2967Vector128.Create(z));
3067if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported)
3107Vector128<T> yVec = Vector128.Create(y);
3108Vector128<T> zVec = Vector128.Create(z);
3110Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
3113Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
3166vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
3169vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
3172vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
3175vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
3186vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
3189vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
3192vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
3195vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
3219vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
3222vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
3225vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
3228vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
3239vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
3242vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
3245vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
3248vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
3288Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8)),
3297Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)),
3306Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)),
3315Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)),
3324Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)),
3333Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)),
3342Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)),
4003Debug.Assert(Vector128.IsHardwareAccelerated);
4005Vector128<T> yVec = Vector128.Create(y);
4006Vector128<T> zVec = Vector128.Create(z);
4008Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4011Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
4024Debug.Assert(Vector128.IsHardwareAccelerated);
4026Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4027Vector128.Create(y),
4028Vector128.Create(z));
4193Debug.Assert(Vector128.IsHardwareAccelerated);
4195Vector128<T> yVec = Vector128.Create(y);
4196Vector128<T> zVec = Vector128.Create(z);
4198Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4201Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
4214Debug.Assert(Vector128.IsHardwareAccelerated);
4216Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4217Vector128.Create(y),
4218Vector128.Create(z));
4320Debug.Assert(Vector128.IsHardwareAccelerated);
4322Vector128<T> yVec = Vector128.Create(y);
4323Vector128<T> zVec = Vector128.Create(z);
4325Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4328Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
4340Debug.Assert(Vector128.IsHardwareAccelerated);
4342Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4343Vector128.Create(y),
4344Vector128.Create(z));
4422Debug.Assert(Vector128.IsHardwareAccelerated);
4424Vector128<T> yVec = Vector128.Create(y);
4425Vector128<T> zVec = Vector128.Create(z);
4427Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4430Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
4442Debug.Assert(Vector128.IsHardwareAccelerated);
4444Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4445Vector128.Create(y),
4446Vector128.Create(z));
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IUnaryInputBinaryOutput.cs (30)
131if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TUnaryOperator.Vectorizable)
139(Vector128<T> first, Vector128<T> second) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i));
152(Vector128<T> first, Vector128<T> second) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i));
302if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TOperator.Vectorizable)
310(Vector128<T> first, Vector128<T> second) = TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i), Vector128.LoadUnsafe(ref yRef, (uint)i));
321Vector128<T> mask = Vector128.Equals(CreateRemainderMaskVector128<T>(x.Length - i), Vector128<T>.Zero);
325Vector128<T> first = Vector128.ConditionalSelect(mask,
326Vector128.Create(TOperator.RemainderMaskValue),
327Vector128.LoadUnsafe(ref xRef, (uint)i));
329Vector128<T> second = Vector128.ConditionalSelect(mask,
330Vector128.Create(TOperator.RemainderMaskValue),
331Vector128.LoadUnsafe(ref yRef, (uint)i));
335Vector128.ConditionalSelect(mask,
336Vector128.LoadUnsafe(ref destination1Ref, (uint)i),
340Vector128.ConditionalSelect(mask,
341Vector128.LoadUnsafe(ref destination2Ref, (uint)i),
487if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TOperator.Vectorizable)
492Vector128<T> yVec = Vector128.Create(y);
497(Vector128<T> first, Vector128<T> second) = TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i), yVec);
508Vector128<T> mask = Vector128.Equals(CreateRemainderMaskVector128<T>(x.Length - i), Vector128<T>.Zero);
512Vector128<T> first = Vector128.ConditionalSelect(mask,
513Vector128.Create(TOperator.RemainderMaskValue),
514Vector128.LoadUnsafe(ref xRef, (uint)i));
516Vector128<T> second = Vector128.ConditionalSelect(mask,
517Vector128.Create(TOperator.RemainderMaskValue),
522Vector128.ConditionalSelect(mask,
523Vector128.LoadUnsafe(ref destination1Ref, (uint)i),
527Vector128.ConditionalSelect(mask,
528Vector128.LoadUnsafe(ref destination2Ref, (uint)i),
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IUnaryOperator.cs (46)
127if (Vector128.IsHardwareAccelerated && Vector128<TInput>.IsSupported && Vector128<TOutput>.IsSupported && TUnaryOperator.Vectorizable && sizeof(TInput) == sizeof(TOutput))
165Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
166Vector128<TOutput> end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<TInput>.Count));
217vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 0)));
218vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 1)));
219vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 2)));
220vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 3)));
229vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 4)));
230vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 5)));
231vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 6)));
232vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 7)));
254vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 0)));
255vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 1)));
256vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 2)));
257vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 3)));
266vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 4)));
267vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 5)));
268vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 6)));
269vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 7)));
307Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 8)));
314Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 7)));
321Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 6)));
328Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 5)));
335Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 4)));
342Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 3)));
349Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 2)));
893Debug.Assert(Vector128.IsHardwareAccelerated);
895Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
896Vector128<TOutput> end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<TInput>.Count));
907Debug.Assert(Vector128.IsHardwareAccelerated);
909Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1037Debug.Assert(Vector128.IsHardwareAccelerated);
1039Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1040Vector128<TOutput> end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<TInput>.Count));
1051Debug.Assert(Vector128.IsHardwareAccelerated);
1053Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1134Debug.Assert(Vector128.IsHardwareAccelerated);
1136Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1137Vector128<TOutput> end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<TInput>.Count));
1147Debug.Assert(Vector128.IsHardwareAccelerated);
1149Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1214Debug.Assert(Vector128.IsHardwareAccelerated);
1216Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1217Vector128<TOutput> end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<TInput>.Count));
1227Debug.Assert(Vector128.IsHardwareAccelerated);
1229Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
System\Numerics\Tensors\netcore\TensorPrimitives.Clamp.cs (3)
178public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> min, Vector128<T> max) => Vector128.Clamp(x, min, max);
226public static Vector128<T> Invoke(Vector128<T> min, Vector128<T> max, Vector128<T> x) => Vector128.Clamp(x, min, max);
275public static Vector128<T> Invoke(Vector128<T> max, Vector128<T> x, Vector128<T> min) => Vector128.Clamp(x, min, max);
System\Numerics\Tensors\netcore\TensorPrimitives.ConvertHelpers.cs (46)
170public static Vector128<float> Invoke(Vector128<int> x) => Vector128.ConvertToSingle(x);
181public static Vector128<float> Invoke(Vector128<uint> x) => Vector128.ConvertToSingle(x);
192public static Vector128<double> Invoke(Vector128<ulong> x) => Vector128.ConvertToDouble(x);
203public static Vector128<double> Invoke(Vector128<long> x) => Vector128.ConvertToDouble(x);
214public static (Vector128<double> Lower, Vector128<double> Upper) Invoke(Vector128<float> x) => Vector128.Widen(x);
225public static Vector128<float> Invoke(Vector128<double> lower, Vector128<double> upper) => Vector128.Narrow(lower, upper);
236public static (Vector128<ushort> Lower, Vector128<ushort> Upper) Invoke(Vector128<byte> x) => Vector128.Widen(x);
251(Vector128<ushort> Lower, Vector128<ushort> Upper) ushorts = Vector128.Widen(x);
252(Vector128<uint> Lower, Vector128<uint> Upper) uintsLower = Vector128.Widen(ushorts.Lower);
253(Vector128<uint> Lower, Vector128<uint> Upper) uintsUpper = Vector128.Widen(ushorts.Upper);
288Vector128.ConvertToSingle(results.Item1),
289Vector128.ConvertToSingle(results.Item2),
290Vector128.ConvertToSingle(results.Item3),
291Vector128.ConvertToSingle(results.Item4)
326public static (Vector128<short> Lower, Vector128<short> Upper) Invoke(Vector128<sbyte> x) => Vector128.Widen(x);
337public static (Vector128<uint> Lower, Vector128<uint> Upper) Invoke(Vector128<ushort> x) => Vector128.Widen(x);
348public static (Vector128<int> Lower, Vector128<int> Upper) Invoke(Vector128<short> x) => Vector128.Widen(x);
359public static (Vector128<ulong> Lower, Vector128<ulong> Upper) Invoke(Vector128<uint> x) => Vector128.Widen(x);
370public static (Vector128<long> Lower, Vector128<long> Upper) Invoke(Vector128<int> x) => Vector128.Widen(x);
396(Vector128<int> lowerInt32, Vector128<int> upperInt32) = Vector128.Widen(x);
404Vector128<uint> sign = value & Vector128.Create(SingleSignMask);
410Vector128<uint> offsetExponent = bitValueInProcess & Vector128.Create(HalfExponentMask);
413Vector128<uint> subnormalMask = Vector128.Equals(offsetExponent, Vector128<uint>.Zero);
416Vector128<uint> infinityOrNaNMask = Vector128.Equals(offsetExponent, Vector128.Create(HalfExponentMask));
419Vector128<uint> maskedExponentLowerBound = subnormalMask & Vector128.Create(ExponentLowerBound);
422Vector128<uint> offsetMaskedExponentLowerBound = Vector128.Create(ExponentOffset) | maskedExponentLowerBound;
425bitValueInProcess = Vector128.ShiftLeft(bitValueInProcess, 13);
428offsetMaskedExponentLowerBound = Vector128.ConditionalSelect(Vector128.Equals(infinityOrNaNMask, Vector128<uint>.Zero),
430Vector128.ShiftLeft(offsetMaskedExponentLowerBound, 1));
433bitValueInProcess &= Vector128.Create(HalfToSingleBitsMask);
577return Vector128.Narrow(
586Vector128<uint> sign = Vector128.ShiftRightLogical(bitValue & Vector128.Create(SingleSignMask), 16);
589Vector128<uint> realMask = Vector128.Equals(value, value).AsUInt32();
592value = Vector128.Abs(value);
595value = Vector128.Min(Vector128.Create(MaxHalfValueBelowInfinity), value);
598Vector128<uint> exponentOffset0 = Vector128.Max(value, Vector128.Create(MinExp).AsSingle()).AsUInt32();
601exponentOffset0 &= Vector128.Create(SingleBiasedExponentMask);
604exponentOffset0 += Vector128.Create(Exponent13);
611Vector128<uint> maskedHalfExponentForNaN = ~realMask & Vector128.Create(ExponentMask);
614bitValue -= Vector128.Create(Exponent126);
617Vector128<uint> newExponent = Vector128.ShiftRightLogical(bitValue, 13);
System\Numerics\Tensors\netcore\TensorPrimitives.ConvertTruncating.cs (10)
109public static Vector128<int> Invoke(Vector128<float> x) => Vector128.ConvertToInt32(x);
120public static Vector128<uint> Invoke(Vector128<float> x) => Vector128.ConvertToUInt32(x);
131public static Vector128<ulong> Invoke(Vector128<double> x) => Vector128.ConvertToUInt64(x);
142public static Vector128<long> Invoke(Vector128<double> x) => Vector128.ConvertToInt64(x);
153public static Vector128<byte> Invoke(Vector128<ushort> lower, Vector128<ushort> upper) => Vector128.Narrow(lower, upper);
164public static Vector128<sbyte> Invoke(Vector128<short> lower, Vector128<short> upper) => Vector128.Narrow(lower, upper);
175public static Vector128<ushort> Invoke(Vector128<uint> lower, Vector128<uint> upper) => Vector128.Narrow(lower, upper);
186public static Vector128<short> Invoke(Vector128<int> lower, Vector128<int> upper) => Vector128.Narrow(lower, upper);
197public static Vector128<uint> Invoke(Vector128<ulong> lower, Vector128<ulong> upper) => Vector128.Narrow(lower, upper);
208public static Vector128<int> Invoke(Vector128<long> lower, Vector128<long> upper) => Vector128.Narrow(lower, upper);
System\Numerics\Tensors\netcore\TensorPrimitives.FloatHelpers.cs (6)
11Vector128.Create(TOperator.Invoke(floats[0]), TOperator.Invoke(floats[1]), TOperator.Invoke(floats[2]), TOperator.Invoke(floats[3]));
20Vector128.Create(TOperator.Invoke(doubles[0]), TOperator.Invoke(doubles[1]));
37Vector128.Create(firstRes0, firstRes1, firstRes2, firstRes3),
38Vector128.Create(secondRes0, secondRes1, secondRes2, secondRes3)
73Vector128.Create(firstRes0, firstRes1),
74Vector128.Create(secondRes0, secondRes1)
System\Numerics\Tensors\netcore\TensorPrimitives.IndexOfMax.cs (15)
37Vector128<T> useResult = Vector128.GreaterThan(result, current);
38Vector128<T> equalMask = Vector128.Equals(result, current);
47Vector128<T> sameSign = Vector128.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As<int, T>();
330if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && x.Length >= Vector128<T>.Count)
336sizeof(T) == sizeof(long) ? Vector128.Create((long)i).As<long, T>() :
337sizeof(T) == sizeof(int) ? Vector128.Create(i).As<int, T>() :
338sizeof(T) == sizeof(short) ? Vector128.Create((short)i).As<short, T>() :
339Vector128.Create((byte)i).As<byte, T>();
359Vector128<T> result = Vector128.LoadUnsafe(ref xRef);
365nanMask = ~Vector128.Equals(result, result);
379current = Vector128.LoadUnsafe(ref xRef, (uint)i);
384nanMask = ~Vector128.Equals(current, current);
399current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128<T>.Count));
404nanMask = ~Vector128.Equals(current, current);
483return Vector128.ConditionalSelect(mask, left, right);
System\Numerics\Tensors\netcore\TensorPrimitives.LeadingZeroCount.cs (5)
43Vector128<byte> lookupVectorLow = Vector128.Create((byte)8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4);
44Vector128<byte> lookupVectorHigh = Vector128.Create((byte)3, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0);
45Vector128<byte> nibbleMask = Vector128.Create<byte>(0xF);
46Vector128<byte> permuteMask = Vector128.Create<byte>(0x80);
60Vector128<uint> lowHalf = Vector128.Create((uint)0x0000FFFF);
System\Numerics\Tensors\netcore\TensorPrimitives.Tan.cs (47)
123Vector128<float> uxMasked = Vector128.Abs(x);
124if (Vector128.GreaterThanAny(uxMasked.AsUInt32(), Vector128.Create(MaxVectorizedValue)))
129Vector128<float> dn = MultiplyAddEstimateOperator<float>.Invoke(uxMasked, Vector128.Create(2 / float.Pi), Vector128.Create(AlmHuge));
131dn -= Vector128.Create(AlmHuge);
134f = MultiplyAddEstimateOperator<float>.Invoke(dn, Vector128.Create(-float.Pi / 2), f);
135f = MultiplyAddEstimateOperator<float>.Invoke(dn, Vector128.Create(Pi_Tail2), f);
136f = MultiplyAddEstimateOperator<float>.Invoke(dn, Vector128.Create(Pi_Tail3), f);
143Vector128<float> a1 = MultiplyAddEstimateOperator<float>.Invoke(Vector128.Create(C2), f2, Vector128.Create(C1));
144Vector128<float> a2 = MultiplyAddEstimateOperator<float>.Invoke(Vector128.Create(C4), f2, Vector128.Create(C3));
145Vector128<float> a3 = MultiplyAddEstimateOperator<float>.Invoke(Vector128.Create(C6), f2, Vector128.Create(C5));
147Vector128<float> b2 = MultiplyAddEstimateOperator<float>.Invoke(f8, a3, f12 * Vector128.Create(C7));
150Vector128<float> result = (poly.AsUInt32() ^ (x.AsUInt32() & Vector128.Create(~SignMask))).AsSingle();
151return Vector128.ConditionalSelect(Vector128.Equals(odd, Vector128<uint>.Zero).AsSingle(),
153Vector128.Create(-1f) / result);
256Vector128<double> uxMasked = Vector128.Abs(x);
257if (Vector128.GreaterThanAny(uxMasked.AsUInt64(), Vector128.Create(MaxVectorizedValue)))
263Vector128<double> dn = MultiplyAddEstimateOperator<double>.Invoke(uxMasked, Vector128.Create(2 / double.Pi), Vector128.Create(AlmHuge));
265dn -= Vector128.Create(AlmHuge);
269f = MultiplyAddEstimateOperator<double>.Invoke(dn, Vector128.Create(-double.Pi / 2), f);
270f = MultiplyAddEstimateOperator<double>.Invoke(dn, Vector128.Create(-HalfPi2), f);
271f = MultiplyAddEstimateOperator<double>.Invoke(dn, Vector128.Create(-HalfPi3), f);
282Vector128<double> a1 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C3), g, Vector128.Create(C1));
283Vector128<double> a2 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C7), g, Vector128.Create(C5));
284Vector128<double> a3 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C11), g, Vector128.Create(C9));
285Vector128<double> a4 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C15), g, Vector128.Create(C13));
286Vector128<double> a5 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C19), g, Vector128.Create(C17));
287Vector128<double> a6 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C23), g, Vector128.Create(C21));
288Vector128<double> a7 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C27), g, Vector128.Create(C25));
295Vector128<double> result = (poly.AsUInt64() ^ (x.AsUInt64() & Vector128.Create(~SignMask))).AsDouble();
296return Vector128.ConditionalSelect(Vector128.Equals(odd, Vector128<ulong>.Zero).AsDouble(),
298Vector128.Create(-1.0) / result);