System\Numerics\Tensors\netcore\Common\TensorPrimitives.IAggregationOperator.cs (139)
92if (Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && TTransformOperator.Vectorizable)
333Vector256<T> vresult = Vector256.Create(TAggregationOperator.IdentityValue);
337Vector256<T> beg = transform.Invoke(Vector256.LoadUnsafe(ref xRef));
338Vector256<T> end = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count));
398vector1 = transform.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)));
399vector2 = transform.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)));
400vector3 = transform.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)));
401vector4 = transform.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)));
410vector1 = transform.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)));
411vector2 = transform.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)));
412vector3 = transform.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)));
413vector4 = transform.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)));
437beg = Vector256.ConditionalSelect(CreateAlignmentMaskVector256<T>((int)misalignment), beg, Vector256.Create(TAggregationOperator.IdentityValue));
454Vector256<T> vector = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 7)));
461Vector256<T> vector = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 6)));
468Vector256<T> vector = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 5)));
475Vector256<T> vector = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 4)));
482Vector256<T> vector = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 3)));
489Vector256<T> vector = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 2)));
496Vector256<T> vector = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 1)));
504end = Vector256.ConditionalSelect(CreateRemainderMaskVector256<T>((int)trailing), end, Vector256.Create(TAggregationOperator.IdentityValue));
758Debug.Assert(Vector256.IsHardwareAccelerated);
760Vector256<T> beg = transform.Invoke(Vector256.LoadUnsafe(ref xRef));
761Vector256<T> end = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count));
763end = Vector256.ConditionalSelect(CreateRemainderMaskVector256<T>((int)(remainder % (uint)Vector256<T>.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue));
772Debug.Assert(Vector256.IsHardwareAccelerated);
774Vector256<T> beg = transform.Invoke(Vector256.LoadUnsafe(ref xRef));
913Debug.Assert(Vector256.IsHardwareAccelerated);
915Vector256<T> beg = transform.Invoke(Vector256.LoadUnsafe(ref xRef));
916Vector256<T> end = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count));
918end = Vector256.ConditionalSelect(CreateRemainderMaskVector256<T>((int)(remainder % (uint)Vector256<T>.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue));
927Debug.Assert(Vector256.IsHardwareAccelerated);
929Vector256<T> beg = transform.Invoke(Vector256.LoadUnsafe(ref xRef));
1019Debug.Assert(Vector256.IsHardwareAccelerated);
1021Vector256<T> beg = transform.Invoke(Vector256.LoadUnsafe(ref xRef));
1022Vector256<T> end = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count));
1024end = Vector256.ConditionalSelect(CreateRemainderMaskVector256<T>((int)(remainder % (uint)Vector256<T>.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue));
1032Debug.Assert(Vector256.IsHardwareAccelerated);
1034Vector256<T> beg = transform.Invoke(Vector256.LoadUnsafe(ref xRef));
1104Debug.Assert(Vector256.IsHardwareAccelerated);
1106Vector256<T> beg = transform.Invoke(Vector256.LoadUnsafe(ref xRef));
1107Vector256<T> end = transform.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count));
1109end = Vector256.ConditionalSelect(CreateRemainderMaskVector256<T>((int)(remainder % (uint)Vector256<T>.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue));
1117Debug.Assert(Vector256.IsHardwareAccelerated);
1119Vector256<T> beg = transform.Invoke(Vector256.LoadUnsafe(ref xRef));
1211if (Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && TBinaryOperator.Vectorizable)
1479Vector256<T> vresult = Vector256.Create(TAggregationOperator.IdentityValue);
1483Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1484Vector256.LoadUnsafe(ref yRef));
1485Vector256<T> end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
1486Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count));
1552vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)),
1553Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 0)));
1554vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)),
1555Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 1)));
1556vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)),
1557Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 2)));
1558vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)),
1559Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 3)));
1568vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)),
1569Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 4)));
1570vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)),
1571Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 5)));
1572vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)),
1573Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 6)));
1574vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)),
1575Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 7)));
1601beg = Vector256.ConditionalSelect(CreateAlignmentMaskVector256<T>((int)misalignment), beg, Vector256.Create(TAggregationOperator.IdentityValue));
1618Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 7)),
1619Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 7)));
1626Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 6)),
1627Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 6)));
1634Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 5)),
1635Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 5)));
1642Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 4)),
1643Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 4)));
1650Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 3)),
1651Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 3)));
1658Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 2)),
1659Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 2)));
1666Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 1)),
1667Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 1)));
1675end = Vector256.ConditionalSelect(CreateRemainderMaskVector256<T>((int)trailing), end, Vector256.Create(TAggregationOperator.IdentityValue));
1954Debug.Assert(Vector256.IsHardwareAccelerated);
1956Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1957Vector256.LoadUnsafe(ref yRef));
1958Vector256<T> end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
1959Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count));
1961end = Vector256.ConditionalSelect(CreateRemainderMaskVector256<T>((int)(remainder % (uint)Vector256<T>.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue));
1970Debug.Assert(Vector256.IsHardwareAccelerated);
1972Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1973Vector256.LoadUnsafe(ref yRef));
2115Debug.Assert(Vector256.IsHardwareAccelerated);
2117Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2118Vector256.LoadUnsafe(ref yRef));
2119Vector256<T> end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
2120Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count));
2122end = Vector256.ConditionalSelect(CreateRemainderMaskVector256<T>((int)(remainder % (uint)Vector256<T>.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue));
2131Debug.Assert(Vector256.IsHardwareAccelerated);
2133Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2134Vector256.LoadUnsafe(ref yRef));
2227Debug.Assert(Vector256.IsHardwareAccelerated);
2229Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2230Vector256.LoadUnsafe(ref yRef));
2231Vector256<T> end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
2232Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count));
2234end = Vector256.ConditionalSelect(CreateRemainderMaskVector256<T>((int)(remainder % (uint)Vector256<T>.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue));
2242Debug.Assert(Vector256.IsHardwareAccelerated);
2244Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2245Vector256.LoadUnsafe(ref yRef));
2320Debug.Assert(Vector256.IsHardwareAccelerated);
2322Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2323Vector256.LoadUnsafe(ref yRef));
2324Vector256<T> end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
2325Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count));
2327end = Vector256.ConditionalSelect(CreateRemainderMaskVector256<T>((int)(remainder % (uint)Vector256<T>.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue));
2335Debug.Assert(Vector256.IsHardwareAccelerated);
2337Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2338Vector256.LoadUnsafe(ref yRef));
2431return Vector256.LoadUnsafe(
2438return Vector256.LoadUnsafe(
2445return Vector256.LoadUnsafe(
2452return Vector256.LoadUnsafe(
2539return Vector256.LoadUnsafe(
2546return Vector256.LoadUnsafe(
2553return Vector256.LoadUnsafe(
2560return Vector256.LoadUnsafe(
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IBinaryOperator.cs (138)
103if (Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && TBinaryOperator.Vectorizable)
402Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
403Vector256.LoadUnsafe(ref yRef));
404Vector256<T> end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
405Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count));
459vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)),
460Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 0)));
461vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)),
462Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 1)));
463vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)),
464Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 2)));
465vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)),
466Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 3)));
475vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)),
476Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 4)));
477vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)),
478Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 5)));
479vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)),
480Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 6)));
481vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)),
482Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 7)));
505vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)),
506Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 0)));
507vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)),
508Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 1)));
509vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)),
510Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 2)));
511vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)),
512Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 3)));
521vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)),
522Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 4)));
523vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)),
524Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 5)));
525vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)),
526Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 6)));
527vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)),
528Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 7)));
568Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 8)),
569Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 8)));
576Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 7)),
577Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 7)));
584Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 6)),
585Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 6)));
592Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 5)),
593Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 5)));
600Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 4)),
601Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 4)));
608Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 3)),
609Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 3)));
616Vector256<T> vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 2)),
617Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 2)));
942Debug.Assert(Vector256.IsHardwareAccelerated);
944Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
945Vector256.LoadUnsafe(ref yRef));
946Vector256<T> end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
947Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count));
958Debug.Assert(Vector256.IsHardwareAccelerated);
960Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
961Vector256.LoadUnsafe(ref yRef));
1114Debug.Assert(Vector256.IsHardwareAccelerated);
1116Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1117Vector256.LoadUnsafe(ref yRef));
1118Vector256<T> end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
1119Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count));
1130Debug.Assert(Vector256.IsHardwareAccelerated);
1132Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1133Vector256.LoadUnsafe(ref yRef));
1229Debug.Assert(Vector256.IsHardwareAccelerated);
1231Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1232Vector256.LoadUnsafe(ref yRef));
1233Vector256<T> end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
1234Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count));
1244Debug.Assert(Vector256.IsHardwareAccelerated);
1246Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1247Vector256.LoadUnsafe(ref yRef));
1319Debug.Assert(Vector256.IsHardwareAccelerated);
1321Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1322Vector256.LoadUnsafe(ref yRef));
1323Vector256<T> end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
1324Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count));
1334Debug.Assert(Vector256.IsHardwareAccelerated);
1336Vector256<T> beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1337Vector256.LoadUnsafe(ref yRef));
1435if (Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && TTransformOperator.Vectorizable && TBinaryOperator.Vectorizable)
1730Vector256<T> yVec = Vector256.Create(y);
1732Vector256<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)),
1734Vector256<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count)),
1786vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0))),
1788vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1))),
1790vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2))),
1792vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3))),
1802vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4))),
1804vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5))),
1806vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6))),
1808vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7))),
1831vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0))),
1833vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1))),
1835vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2))),
1837vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3))),
1847vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4))),
1849vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5))),
1851vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6))),
1853vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7))),
1892Vector256<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 8))),
1900Vector256<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 7))),
1908Vector256<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 6))),
1916Vector256<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 5))),
1924Vector256<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 4))),
1932Vector256<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 3))),
1940Vector256<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 2))),
2262Debug.Assert(Vector256.IsHardwareAccelerated);
2264Vector256<T> yVec = Vector256.Create(y);
2266Vector256<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)),
2268Vector256<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count)),
2280Debug.Assert(Vector256.IsHardwareAccelerated);
2282Vector256<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)),
2283Vector256.Create(y));
2438Debug.Assert(Vector256.IsHardwareAccelerated);
2440Vector256<T> yVec = Vector256.Create(y);
2442Vector256<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)),
2444Vector256<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count)),
2456Debug.Assert(Vector256.IsHardwareAccelerated);
2458Vector256<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)),
2459Vector256.Create(y));
2557Debug.Assert(Vector256.IsHardwareAccelerated);
2559Vector256<T> yVec = Vector256.Create(y);
2561Vector256<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)),
2563Vector256<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count)),
2574Debug.Assert(Vector256.IsHardwareAccelerated);
2576Vector256<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)),
2577Vector256.Create(y));
2651Debug.Assert(Vector256.IsHardwareAccelerated);
2653Vector256<T> yVec = Vector256.Create(y);
2655Vector256<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)),
2657Vector256<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count)),
2668Debug.Assert(Vector256.IsHardwareAccelerated);
2670Vector256<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)),
2671Vector256.Create(y));
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IBooleanUnaryOperator.cs (37)
42public static bool ShouldEarlyExit(Vector256<T> result) => Vector256.AnyWhereAllBitsSet(result);
74typeof(T) == typeof(float) ? Vector256.EqualsAny(result.AsUInt32(), Vector256<uint>.Zero) :
75typeof(T) == typeof(double) ? Vector256.EqualsAny(result.AsUInt64(), Vector256<ulong>.Zero) :
76Vector256.EqualsAny(result, Vector256<T>.Zero);
129if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256<T>.IsSupported)
137if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)i))))
148TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256<T>.Count)))))
266if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256<T>.IsSupported)
290Vector256<byte> v = TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)i)).AsByte();
377if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256<T>.IsSupported)
402Vector256.Narrow(
403TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)i)).AsUInt16(),
404TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256<T>.Count))).AsUInt16());
498if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256<T>.IsSupported)
523Vector256.Narrow(
524Vector256.Narrow(
525TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)i)).AsUInt32(),
526TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256<T>.Count))).AsUInt32()),
527Vector256.Narrow(
528TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + (2 * Vector256<T>.Count)))).AsUInt32(),
529TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + (3 * Vector256<T>.Count)))).AsUInt32()));
635if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256<T>.IsSupported)
662Vector256.Narrow(
663Vector256.Narrow(
664Vector256.Narrow(
665TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)i)).AsUInt64(),
666TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256<T>.Count))).AsUInt64()),
667Vector256.Narrow(
668TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + (2 * Vector256<T>.Count)))).AsUInt64(),
669TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + (3 * Vector256<T>.Count)))).AsUInt64())),
670Vector256.Narrow(
671Vector256.Narrow(
672TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + (4 * Vector256<T>.Count)))).AsUInt64(),
673TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + (5 * Vector256<T>.Count)))).AsUInt64()),
674Vector256.Narrow(
675TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + (6 * Vector256<T>.Count)))).AsUInt64(),
676TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(i + (7 * Vector256<T>.Count)))).AsUInt64())));
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IStatefulUnaryOperator.cs (46)
69if (Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && TStatefulUnaryOperator.Vectorizable)
336Vector256<T> beg = op.Invoke(Vector256.LoadUnsafe(ref xRef));
337Vector256<T> end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count));
388vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)));
389vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)));
390vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)));
391vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)));
400vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)));
401vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)));
402vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)));
403vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)));
425vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)));
426vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)));
427vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)));
428vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)));
437vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)));
438vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)));
439vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)));
440vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)));
478Vector256<T> vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 8)));
485Vector256<T> vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 7)));
492Vector256<T> vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 6)));
499Vector256<T> vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 5)));
506Vector256<T> vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 4)));
513Vector256<T> vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 3)));
520Vector256<T> vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 2)));
814Debug.Assert(Vector256.IsHardwareAccelerated);
816Vector256<T> beg = op.Invoke(Vector256.LoadUnsafe(ref xRef));
817Vector256<T> end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count));
828Debug.Assert(Vector256.IsHardwareAccelerated);
830Vector256<T> beg = op.Invoke(Vector256.LoadUnsafe(ref xRef));
966Debug.Assert(Vector256.IsHardwareAccelerated);
968Vector256<T> beg = op.Invoke(Vector256.LoadUnsafe(ref xRef));
969Vector256<T> end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count));
980Debug.Assert(Vector256.IsHardwareAccelerated);
982Vector256<T> beg = op.Invoke(Vector256.LoadUnsafe(ref xRef));
1069Debug.Assert(Vector256.IsHardwareAccelerated);
1071Vector256<T> beg = op.Invoke(Vector256.LoadUnsafe(ref xRef));
1072Vector256<T> end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count));
1082Debug.Assert(Vector256.IsHardwareAccelerated);
1084Vector256<T> beg = op.Invoke(Vector256.LoadUnsafe(ref xRef));
1151Debug.Assert(Vector256.IsHardwareAccelerated);
1153Vector256<T> beg = op.Invoke(Vector256.LoadUnsafe(ref xRef));
1154Vector256<T> end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count));
1164Debug.Assert(Vector256.IsHardwareAccelerated);
1166Vector256<T> beg = op.Invoke(Vector256.LoadUnsafe(ref xRef));
System\Numerics\Tensors\netcore\Common\TensorPrimitives.ITernaryOperator.cs (276)
90if (TTernaryOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported)
421Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
422Vector256.LoadUnsafe(ref yRef),
423Vector256.LoadUnsafe(ref zRef));
424Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
425Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count),
426Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256<T>.Count));
483vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)),
484Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 0)),
485Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 0)));
486vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)),
487Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 1)),
488Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 1)));
489vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)),
490Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 2)),
491Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 2)));
492vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)),
493Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 3)),
494Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 3)));
503vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)),
504Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 4)),
505Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 4)));
506vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)),
507Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 5)),
508Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 5)));
509vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)),
510Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 6)),
511Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 6)));
512vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)),
513Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 7)),
514Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 7)));
538vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)),
539Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 0)),
540Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 0)));
541vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)),
542Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 1)),
543Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 1)));
544vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)),
545Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 2)),
546Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 2)));
547vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)),
548Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 3)),
549Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 3)));
558vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)),
559Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 4)),
560Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 4)));
561vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)),
562Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 5)),
563Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 5)));
564vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)),
565Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 6)),
566Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 6)));
567vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)),
568Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 7)),
569Vector256.Load(zPtr + (uint)(Vector256<T>.Count * 7)));
611Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 8)),
612Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 8)),
613Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256<T>.Count * 8)));
620Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 7)),
621Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 7)),
622Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256<T>.Count * 7)));
629Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 6)),
630Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 6)),
631Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256<T>.Count * 6)));
638Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 5)),
639Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 5)),
640Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256<T>.Count * 5)));
647Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 4)),
648Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 4)),
649Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256<T>.Count * 4)));
656Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 3)),
657Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 3)),
658Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256<T>.Count * 3)));
665Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 2)),
666Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 2)),
667Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256<T>.Count * 2)));
1023Debug.Assert(Vector256.IsHardwareAccelerated);
1025Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1026Vector256.LoadUnsafe(ref yRef),
1027Vector256.LoadUnsafe(ref zRef));
1028Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
1029Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count),
1030Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256<T>.Count));
1041Debug.Assert(Vector256.IsHardwareAccelerated);
1043Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1044Vector256.LoadUnsafe(ref yRef),
1045Vector256.LoadUnsafe(ref zRef));
1215Debug.Assert(Vector256.IsHardwareAccelerated);
1217Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1218Vector256.LoadUnsafe(ref yRef),
1219Vector256.LoadUnsafe(ref zRef));
1220Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
1221Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count),
1222Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256<T>.Count));
1233Debug.Assert(Vector256.IsHardwareAccelerated);
1235Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1236Vector256.LoadUnsafe(ref yRef),
1237Vector256.LoadUnsafe(ref zRef));
1342Debug.Assert(Vector256.IsHardwareAccelerated);
1344Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1345Vector256.LoadUnsafe(ref yRef),
1346Vector256.LoadUnsafe(ref zRef));
1347Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
1348Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count),
1349Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256<T>.Count));
1359Debug.Assert(Vector256.IsHardwareAccelerated);
1361Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1362Vector256.LoadUnsafe(ref yRef),
1363Vector256.LoadUnsafe(ref zRef));
1440Debug.Assert(Vector256.IsHardwareAccelerated);
1442Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1443Vector256.LoadUnsafe(ref yRef),
1444Vector256.LoadUnsafe(ref zRef));
1445Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
1446Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count),
1447Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256<T>.Count));
1457Debug.Assert(Vector256.IsHardwareAccelerated);
1459Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1460Vector256.LoadUnsafe(ref yRef),
1461Vector256.LoadUnsafe(ref zRef));
1565if (TTernaryOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported)
1892Vector256<T> zVec = Vector256.Create(z);
1894Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
1895Vector256.LoadUnsafe(ref yRef),
1897Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
1898Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count),
1953vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)),
1954Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 0)),
1956vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)),
1957Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 1)),
1959vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)),
1960Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 2)),
1962vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)),
1963Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 3)),
1973vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)),
1974Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 4)),
1976vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)),
1977Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 5)),
1979vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)),
1980Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 6)),
1982vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)),
1983Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 7)),
2007vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)),
2008Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 0)),
2010vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)),
2011Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 1)),
2013vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)),
2014Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 2)),
2016vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)),
2017Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 3)),
2027vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)),
2028Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 4)),
2030vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)),
2031Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 5)),
2033vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)),
2034Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 6)),
2036vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)),
2037Vector256.Load(yPtr + (uint)(Vector256<T>.Count * 7)),
2078Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 8)),
2079Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 8)),
2087Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 7)),
2088Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 7)),
2096Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 6)),
2097Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 6)),
2105Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 5)),
2106Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 5)),
2114Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 4)),
2115Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 4)),
2123Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 3)),
2124Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 3)),
2132Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 2)),
2133Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256<T>.Count * 2)),
2486Debug.Assert(Vector256.IsHardwareAccelerated);
2488Vector256<T> zVec = Vector256.Create(z);
2490Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2491Vector256.LoadUnsafe(ref yRef),
2493Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
2494Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count),
2506Debug.Assert(Vector256.IsHardwareAccelerated);
2508Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2509Vector256.LoadUnsafe(ref yRef),
2510Vector256.Create(z));
2682Debug.Assert(Vector256.IsHardwareAccelerated);
2684Vector256<T> zVec = Vector256.Create(z);
2686Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2687Vector256.LoadUnsafe(ref yRef),
2689Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
2690Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count),
2702Debug.Assert(Vector256.IsHardwareAccelerated);
2704Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2705Vector256.LoadUnsafe(ref yRef),
2706Vector256.Create(z));
2813Debug.Assert(Vector256.IsHardwareAccelerated);
2815Vector256<T> zVec = Vector256.Create(z);
2817Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2818Vector256.LoadUnsafe(ref yRef),
2820Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
2821Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count),
2832Debug.Assert(Vector256.IsHardwareAccelerated);
2834Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2835Vector256.LoadUnsafe(ref yRef),
2836Vector256.Create(z));
2915Debug.Assert(Vector256.IsHardwareAccelerated);
2917Vector256<T> zVec = Vector256.Create(z);
2919Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2920Vector256.LoadUnsafe(ref yRef),
2922Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
2923Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256<T>.Count),
2934Debug.Assert(Vector256.IsHardwareAccelerated);
2936Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
2937Vector256.LoadUnsafe(ref yRef),
2938Vector256.Create(z));
3051if (TTernaryOperator.Vectorizable && Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported)
3373Vector256<T> yVec = Vector256.Create(y);
3374Vector256<T> zVec = Vector256.Create(z);
3376Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
3379Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
3432vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)),
3435vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)),
3438vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)),
3441vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)),
3452vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)),
3455vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)),
3458vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)),
3461vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)),
3485vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 0)),
3488vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 1)),
3491vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 2)),
3494vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 3)),
3505vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 4)),
3508vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 5)),
3511vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 6)),
3514vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<T>.Count * 7)),
3554Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 8)),
3563Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 7)),
3572Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 6)),
3581Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 5)),
3590Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 4)),
3599Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 3)),
3608Vector256<T> vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<T>.Count * 2)),
3957Debug.Assert(Vector256.IsHardwareAccelerated);
3959Vector256<T> yVec = Vector256.Create(y);
3960Vector256<T> zVec = Vector256.Create(z);
3962Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
3965Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
3978Debug.Assert(Vector256.IsHardwareAccelerated);
3980Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
3981Vector256.Create(y),
3982Vector256.Create(z));
4155Debug.Assert(Vector256.IsHardwareAccelerated);
4157Vector256<T> yVec = Vector256.Create(y);
4158Vector256<T> zVec = Vector256.Create(z);
4160Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
4163Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
4176Debug.Assert(Vector256.IsHardwareAccelerated);
4178Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
4179Vector256.Create(y),
4180Vector256.Create(z));
4288Debug.Assert(Vector256.IsHardwareAccelerated);
4290Vector256<T> yVec = Vector256.Create(y);
4291Vector256<T> zVec = Vector256.Create(z);
4293Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
4296Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
4308Debug.Assert(Vector256.IsHardwareAccelerated);
4310Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
4311Vector256.Create(y),
4312Vector256.Create(z));
4392Debug.Assert(Vector256.IsHardwareAccelerated);
4394Vector256<T> yVec = Vector256.Create(y);
4395Vector256<T> zVec = Vector256.Create(z);
4397Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
4400Vector256<T> end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<T>.Count),
4412Debug.Assert(Vector256.IsHardwareAccelerated);
4414Vector256<T> beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef),
4415Vector256.Create(y),
4416Vector256.Create(z));
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IUnaryInputBinaryOutput.cs (30)
101if (Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && TUnaryOperator.Vectorizable)
109(Vector256<T> first, Vector256<T> second) = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref sourceRef, (uint)i));
122(Vector256<T> first, Vector256<T> second) = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref sourceRef, (uint)i));
254if (Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && TOperator.Vectorizable)
262(Vector256<T> first, Vector256<T> second) = TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)i), Vector256.LoadUnsafe(ref yRef, (uint)i));
273Vector256<T> mask = Vector256.Equals(CreateRemainderMaskVector256<T>(x.Length - i), Vector256<T>.Zero);
277Vector256<T> first = Vector256.ConditionalSelect(mask,
278Vector256.Create(TOperator.RemainderMaskValue),
279Vector256.LoadUnsafe(ref xRef, (uint)i));
281Vector256<T> second = Vector256.ConditionalSelect(mask,
282Vector256.Create(TOperator.RemainderMaskValue),
283Vector256.LoadUnsafe(ref yRef, (uint)i));
287Vector256.ConditionalSelect(mask,
288Vector256.LoadUnsafe(ref destination1Ref, (uint)i),
292Vector256.ConditionalSelect(mask,
293Vector256.LoadUnsafe(ref destination2Ref, (uint)i),
437if (Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && TOperator.Vectorizable)
442Vector256<T> yVec = Vector256.Create(y);
447(Vector256<T> first, Vector256<T> second) = TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)i), yVec);
458Vector256<T> mask = Vector256.Equals(CreateRemainderMaskVector256<T>(x.Length - i), Vector256<T>.Zero);
462Vector256<T> first = Vector256.ConditionalSelect(mask,
463Vector256.Create(TOperator.RemainderMaskValue),
464Vector256.LoadUnsafe(ref xRef, (uint)i));
466Vector256<T> second = Vector256.ConditionalSelect(mask,
467Vector256.Create(TOperator.RemainderMaskValue),
472Vector256.ConditionalSelect(mask,
473Vector256.LoadUnsafe(ref destination1Ref, (uint)i),
477Vector256.ConditionalSelect(mask,
478Vector256.LoadUnsafe(ref destination2Ref, (uint)i),
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IUnaryOperator.cs (46)
109if (Vector256.IsHardwareAccelerated && Vector256<TInput>.IsSupported && Vector256<TOutput>.IsSupported && TUnaryOperator.Vectorizable && sizeof(TInput) == sizeof(TOutput))
376Vector256<TOutput> beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef));
377Vector256<TOutput> end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<TInput>.Count));
428vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 0)));
429vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 1)));
430vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 2)));
431vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 3)));
440vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 4)));
441vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 5)));
442vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 6)));
443vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 7)));
465vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 0)));
466vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 1)));
467vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 2)));
468vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 3)));
477vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 4)));
478vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 5)));
479vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 6)));
480vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256<TInput>.Count * 7)));
518Vector256<TOutput> vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<TInput>.Count * 8)));
525Vector256<TOutput> vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<TInput>.Count * 7)));
532Vector256<TOutput> vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<TInput>.Count * 6)));
539Vector256<TOutput> vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<TInput>.Count * 5)));
546Vector256<TOutput> vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<TInput>.Count * 4)));
553Vector256<TOutput> vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<TInput>.Count * 3)));
560Vector256<TOutput> vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256<TInput>.Count * 2)));
854Debug.Assert(Vector256.IsHardwareAccelerated);
856Vector256<TOutput> beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef));
857Vector256<TOutput> end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<TInput>.Count));
868Debug.Assert(Vector256.IsHardwareAccelerated);
870Vector256<TOutput> beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef));
1006Debug.Assert(Vector256.IsHardwareAccelerated);
1008Vector256<TOutput> beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef));
1009Vector256<TOutput> end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<TInput>.Count));
1020Debug.Assert(Vector256.IsHardwareAccelerated);
1022Vector256<TOutput> beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef));
1109Debug.Assert(Vector256.IsHardwareAccelerated);
1111Vector256<TOutput> beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef));
1112Vector256<TOutput> end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<TInput>.Count));
1122Debug.Assert(Vector256.IsHardwareAccelerated);
1124Vector256<TOutput> beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef));
1191Debug.Assert(Vector256.IsHardwareAccelerated);
1193Vector256<TOutput> beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef));
1194Vector256<TOutput> end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256<TInput>.Count));
1204Debug.Assert(Vector256.IsHardwareAccelerated);
1206Vector256<TOutput> beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef));
System\Numerics\Tensors\netcore\TensorPrimitives.Clamp.cs (3)
226public static Vector256<T> Invoke(Vector256<T> x, Vector256<T> min, Vector256<T> max) => Vector256.Clamp(x, min, max);
274public static Vector256<T> Invoke(Vector256<T> min, Vector256<T> max, Vector256<T> x) => Vector256.Clamp(x, min, max);
323public static Vector256<T> Invoke(Vector256<T> max, Vector256<T> x, Vector256<T> min) => Vector256.Clamp(x, min, max);
System\Numerics\Tensors\netcore\TensorPrimitives.ConvertHelpers.cs (46)
171public static Vector256<float> Invoke(Vector256<int> x) => Vector256.ConvertToSingle(x);
182public static Vector256<float> Invoke(Vector256<uint> x) => Vector256.ConvertToSingle(x);
193public static Vector256<double> Invoke(Vector256<ulong> x) => Vector256.ConvertToDouble(x);
204public static Vector256<double> Invoke(Vector256<long> x) => Vector256.ConvertToDouble(x);
215public static (Vector256<double> Lower, Vector256<double> Upper) Invoke(Vector256<float> x) => Vector256.Widen(x);
226public static Vector256<float> Invoke(Vector256<double> lower, Vector256<double> upper) => Vector256.Narrow(lower, upper);
237public static (Vector256<ushort> Lower, Vector256<ushort> Upper) Invoke(Vector256<byte> x) => Vector256.Widen(x);
260(Vector256<ushort> Lower, Vector256<ushort> Upper) ushorts = Vector256.Widen(x);
261(Vector256<uint> Lower, Vector256<uint> Upper) uintsLower = Vector256.Widen(ushorts.Lower);
262(Vector256<uint> Lower, Vector256<uint> Upper) uintsUpper = Vector256.Widen(ushorts.Upper);
300Vector256.ConvertToSingle(results.Item1),
301Vector256.ConvertToSingle(results.Item2),
302Vector256.ConvertToSingle(results.Item3),
303Vector256.ConvertToSingle(results.Item4)
327public static (Vector256<short> Lower, Vector256<short> Upper) Invoke(Vector256<sbyte> x) => Vector256.Widen(x);
338public static (Vector256<uint> Lower, Vector256<uint> Upper) Invoke(Vector256<ushort> x) => Vector256.Widen(x);
349public static (Vector256<int> Lower, Vector256<int> Upper) Invoke(Vector256<short> x) => Vector256.Widen(x);
360public static (Vector256<ulong> Lower, Vector256<ulong> Upper) Invoke(Vector256<uint> x) => Vector256.Widen(x);
371public static (Vector256<long> Lower, Vector256<long> Upper) Invoke(Vector256<int> x) => Vector256.Widen(x);
448(Vector256<int> lowerInt32, Vector256<int> upperInt32) = Vector256.Widen(x);
456Vector256<uint> sign = value & Vector256.Create(SingleSignMask);
462Vector256<uint> offsetExponent = bitValueInProcess & Vector256.Create(HalfExponentMask);
465Vector256<uint> subnormalMask = Vector256.Equals(offsetExponent, Vector256<uint>.Zero);
468Vector256<uint> infinityOrNaNMask = Vector256.Equals(offsetExponent, Vector256.Create(HalfExponentMask));
471Vector256<uint> maskedExponentLowerBound = subnormalMask & Vector256.Create(ExponentLowerBound);
474Vector256<uint> offsetMaskedExponentLowerBound = Vector256.Create(ExponentOffset) | maskedExponentLowerBound;
477bitValueInProcess = Vector256.ShiftLeft(bitValueInProcess, 13);
480offsetMaskedExponentLowerBound = Vector256.ConditionalSelect(Vector256.Equals(infinityOrNaNMask, Vector256<uint>.Zero),
482Vector256.ShiftLeft(offsetMaskedExponentLowerBound, 1));
485bitValueInProcess &= Vector256.Create(HalfToSingleBitsMask);
641return Vector256.Narrow(
650Vector256<uint> sign = Vector256.ShiftRightLogical(bitValue & Vector256.Create(SingleSignMask), 16);
653Vector256<uint> realMask = Vector256.Equals(value, value).AsUInt32();
656value = Vector256.Abs(value);
659value = Vector256.Min(Vector256.Create(MaxHalfValueBelowInfinity), value);
662Vector256<uint> exponentOffset0 = Vector256.Max(value, Vector256.Create(MinExp).AsSingle()).AsUInt32();
665exponentOffset0 &= Vector256.Create(SingleBiasedExponentMask);
668exponentOffset0 += Vector256.Create(Exponent13);
675Vector256<uint> maskedHalfExponentForNaN = ~realMask & Vector256.Create(ExponentMask);
678bitValue -= Vector256.Create(Exponent126);
681Vector256<uint> newExponent = Vector256.ShiftRightLogical(bitValue, 13);
System\Numerics\Tensors\netcore\TensorPrimitives.ConvertTruncating.cs (10)
115public static Vector256<int> Invoke(Vector256<float> x) => Vector256.ConvertToInt32(x);
131public static Vector256<uint> Invoke(Vector256<float> x) => Vector256.ConvertToUInt32(x);
147public static Vector256<ulong> Invoke(Vector256<double> x) => Vector256.ConvertToUInt64(x);
163public static Vector256<long> Invoke(Vector256<double> x) => Vector256.ConvertToInt64(x);
174public static Vector256<byte> Invoke(Vector256<ushort> lower, Vector256<ushort> upper) => Vector256.Narrow(lower, upper);
185public static Vector256<sbyte> Invoke(Vector256<short> lower, Vector256<short> upper) => Vector256.Narrow(lower, upper);
196public static Vector256<ushort> Invoke(Vector256<uint> lower, Vector256<uint> upper) => Vector256.Narrow(lower, upper);
207public static Vector256<short> Invoke(Vector256<int> lower, Vector256<int> upper) => Vector256.Narrow(lower, upper);
218public static Vector256<uint> Invoke(Vector256<ulong> lower, Vector256<ulong> upper) => Vector256.Narrow(lower, upper);
229public static Vector256<int> Invoke(Vector256<long> lower, Vector256<long> upper) => Vector256.Narrow(lower, upper);
System\Numerics\Tensors\netcore\TensorPrimitives.CosineSimilarity.cs (13)
98if (Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && x.Length >= Vector256<T>.Count)
113Vector256.LoadUnsafe(ref xRef, (uint)i),
114Vector256.LoadUnsafe(ref yRef, (uint)i),
127Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256<T>.Count)) & remainderMask,
128Vector256.LoadUnsafe(ref yRef, (uint)(x.Length - Vector256<T>.Count)) & remainderMask,
243if (Vector256.IsHardwareAccelerated && x.Length >= Vector256<short>.Count)
258(Vector256<float> xVecLower, Vector256<float> xVecUpper) = WidenHalfAsInt16ToSingleOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)i));
259(Vector256<float> yVecLower, Vector256<float> yVecUpper) = WidenHalfAsInt16ToSingleOperator.Invoke(Vector256.LoadUnsafe(ref yRef, (uint)i));
274Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256<short>.Count)) & remainderMask);
276Vector256.LoadUnsafe(ref yRef, (uint)(x.Length - Vector256<short>.Count)) & remainderMask);
380Vector256.Sum(dotProductVector) / (T.Sqrt(Vector256.Sum(xSumOfSquaresVector)) * T.Sqrt(Vector256.Sum(ySumOfSquaresVector)));
System\Numerics\Tensors\netcore\TensorPrimitives.IndexOfMax.cs (19)
63Vector256<T> useResult = Vector256.GreaterThan(result, current);
64Vector256<T> equalMask = Vector256.Equals(result, current);
73Vector256<T> sameSign = Vector256.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As<int, T>();
239if (Vector256.IsHardwareAccelerated && Vector256<T>.IsSupported && x.Length >= Vector256<T>.Count)
245sizeof(T) == sizeof(long) ? Vector256.Create((long)i).As<long, T>() :
246sizeof(T) == sizeof(int) ? Vector256.Create(i).As<int, T>() :
247sizeof(T) == sizeof(short) ? Vector256.Create((short)i).As<short, T>() :
248Vector256.Create((byte)i).As<byte, T>();
268Vector256<T> result = Vector256.LoadUnsafe(ref xRef);
274nanMask = ~Vector256.Equals(result, result);
288current = Vector256.LoadUnsafe(ref xRef, (uint)i);
293nanMask = ~Vector256.Equals(current, current);
308current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256<T>.Count));
313nanMask = ~Vector256.Equals(current, current);
454sizeof(T) == sizeof(long) ? Vector256.LessThan(indices1.AsInt64(), indices2.AsInt64()).As<long, T>() :
455sizeof(T) == sizeof(int) ? Vector256.LessThan(indices1.AsInt32(), indices2.AsInt32()).As<int, T>() :
456sizeof(T) == sizeof(short) ? Vector256.LessThan(indices1.AsInt16(), indices2.AsInt16()).As<short, T>() :
457Vector256.LessThan(indices1.AsByte(), indices2.AsByte()).As<byte, T>();
500return Vector256.ConditionalSelect(mask, left, right);
System\Numerics\Tensors\netcore\TensorPrimitives.LeadingZeroCount.cs (4)
96Vector256.Create((byte)8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4,
98Vector256<byte> nibbleMask = Vector256.Create<byte>(0xF);
111Vector256<uint> lowHalf = Vector256.Create((uint)0x0000FFFF);
131return Vector256.Create(Invoke(x.GetLower()), Invoke(x.GetUpper()));
System\Numerics\Tensors\netcore\TensorPrimitives.Tan.cs (47)
165Vector256<float> uxMasked = Vector256.Abs(x);
166if (Vector256.GreaterThanAny(uxMasked.AsUInt32(), Vector256.Create(MaxVectorizedValue)))
171Vector256<float> dn = MultiplyAddEstimateOperator<float>.Invoke(uxMasked, Vector256.Create(2 / float.Pi), Vector256.Create(AlmHuge));
173dn -= Vector256.Create(AlmHuge);
176f = MultiplyAddEstimateOperator<float>.Invoke(dn, Vector256.Create(-float.Pi / 2), f);
177f = MultiplyAddEstimateOperator<float>.Invoke(dn, Vector256.Create(Pi_Tail2), f);
178f = MultiplyAddEstimateOperator<float>.Invoke(dn, Vector256.Create(Pi_Tail3), f);
185Vector256<float> a1 = MultiplyAddEstimateOperator<float>.Invoke(Vector256.Create(C2), f2, Vector256.Create(C1));
186Vector256<float> a2 = MultiplyAddEstimateOperator<float>.Invoke(Vector256.Create(C4), f2, Vector256.Create(C3));
187Vector256<float> a3 = MultiplyAddEstimateOperator<float>.Invoke(Vector256.Create(C6), f2, Vector256.Create(C5));
189Vector256<float> b2 = MultiplyAddEstimateOperator<float>.Invoke(f8, a3, f12 * Vector256.Create(C7));
192Vector256<float> result = (poly.AsUInt32() ^ (x.AsUInt32() & Vector256.Create(~SignMask))).AsSingle();
193return Vector256.ConditionalSelect(Vector256.Equals(odd, Vector256<uint>.Zero).AsSingle(),
195Vector256.Create(-1f) / result);
310Vector256<double> uxMasked = Vector256.Abs(x);
311if (Vector256.GreaterThanAny(uxMasked.AsUInt64(), Vector256.Create(MaxVectorizedValue)))
317Vector256<double> dn = MultiplyAddEstimateOperator<double>.Invoke(uxMasked, Vector256.Create(2 / double.Pi), Vector256.Create(AlmHuge));
319dn -= Vector256.Create(AlmHuge);
323f = MultiplyAddEstimateOperator<double>.Invoke(dn, Vector256.Create(-double.Pi / 2), f);
324f = MultiplyAddEstimateOperator<double>.Invoke(dn, Vector256.Create(-HalfPi2), f);
325f = MultiplyAddEstimateOperator<double>.Invoke(dn, Vector256.Create(-HalfPi3), f);
336Vector256<double> a1 = MultiplyAddEstimateOperator<double>.Invoke(Vector256.Create(C3), g, Vector256.Create(C1));
337Vector256<double> a2 = MultiplyAddEstimateOperator<double>.Invoke(Vector256.Create(C7), g, Vector256.Create(C5));
338Vector256<double> a3 = MultiplyAddEstimateOperator<double>.Invoke(Vector256.Create(C11), g, Vector256.Create(C9));
339Vector256<double> a4 = MultiplyAddEstimateOperator<double>.Invoke(Vector256.Create(C15), g, Vector256.Create(C13));
340Vector256<double> a5 = MultiplyAddEstimateOperator<double>.Invoke(Vector256.Create(C19), g, Vector256.Create(C17));
341Vector256<double> a6 = MultiplyAddEstimateOperator<double>.Invoke(Vector256.Create(C23), g, Vector256.Create(C21));
342Vector256<double> a7 = MultiplyAddEstimateOperator<double>.Invoke(Vector256.Create(C27), g, Vector256.Create(C25));
349Vector256<double> result = (poly.AsUInt64() ^ (x.AsUInt64() & Vector256.Create(~SignMask))).AsDouble();
350return Vector256.ConditionalSelect(Vector256.Equals(odd, Vector256<ulong>.Zero).AsDouble(),
352Vector256.Create(-1.0) / result);