System\Numerics\Tensors\netcore\Common\TensorPrimitives.IAggregationOperator.cs (139)
112if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TTransformOperator.Vectorizable)
152Vector128<T> vresult = Vector128.Create(TAggregationOperator.IdentityValue);
156Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
157Vector128<T> end = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
216vector1 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)));
217vector2 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)));
218vector3 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)));
219vector4 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)));
228vector1 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)));
229vector2 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)));
230vector3 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)));
231vector4 = transform.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)));
255beg = Vector128.ConditionalSelect(CreateAlignmentMaskVector128<T>((int)misalignment), beg, Vector128.Create(TAggregationOperator.IdentityValue));
272Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)));
279Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)));
286Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)));
293Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)));
300Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)));
307Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)));
314Vector128<T> vector = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 1)));
322end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)trailing), end, Vector128.Create(TAggregationOperator.IdentityValue));
797Debug.Assert(Vector128.IsHardwareAccelerated);
799Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
800Vector128<T> end = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
802end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
811Debug.Assert(Vector128.IsHardwareAccelerated);
813Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
944Debug.Assert(Vector128.IsHardwareAccelerated);
946Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
947Vector128<T> end = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
949end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
958Debug.Assert(Vector128.IsHardwareAccelerated);
960Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
1044Debug.Assert(Vector128.IsHardwareAccelerated);
1046Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
1047Vector128<T> end = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
1049end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
1057Debug.Assert(Vector128.IsHardwareAccelerated);
1059Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
1127Debug.Assert(Vector128.IsHardwareAccelerated);
1129Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
1130Vector128<T> end = transform.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
1132end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
1140Debug.Assert(Vector128.IsHardwareAccelerated);
1142Vector128<T> beg = transform.Invoke(Vector128.LoadUnsafe(ref xRef));
1231if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TBinaryOperator.Vectorizable)
1272Vector128<T> vresult = Vector128.Create(TAggregationOperator.IdentityValue);
1276Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1277Vector128.LoadUnsafe(ref yRef));
1278Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1279Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
1345vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
1346Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)));
1347vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
1348Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)));
1349vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
1350Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)));
1351vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
1352Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)));
1361vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
1362Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)));
1363vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
1364Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)));
1365vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
1366Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)));
1367vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
1368Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)));
1394beg = Vector128.ConditionalSelect(CreateAlignmentMaskVector128<T>((int)misalignment), beg, Vector128.Create(TAggregationOperator.IdentityValue));
1411Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)),
1412Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 7)));
1419Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)),
1420Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 6)));
1427Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)),
1428Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 5)));
1435Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)),
1436Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 4)));
1443Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)),
1444Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 3)));
1451Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)),
1452Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 2)));
1459Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 1)),
1460Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 1)));
1468end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)trailing), end, Vector128.Create(TAggregationOperator.IdentityValue));
1996Debug.Assert(Vector128.IsHardwareAccelerated);
1998Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1999Vector128.LoadUnsafe(ref yRef));
2000Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2001Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
2003end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
2012Debug.Assert(Vector128.IsHardwareAccelerated);
2014Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2015Vector128.LoadUnsafe(ref yRef));
2149Debug.Assert(Vector128.IsHardwareAccelerated);
2151Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2152Vector128.LoadUnsafe(ref yRef));
2153Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2154Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
2156end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
2165Debug.Assert(Vector128.IsHardwareAccelerated);
2167Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2168Vector128.LoadUnsafe(ref yRef));
2255Debug.Assert(Vector128.IsHardwareAccelerated);
2257Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2258Vector128.LoadUnsafe(ref yRef));
2259Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2260Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
2262end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
2270Debug.Assert(Vector128.IsHardwareAccelerated);
2272Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2273Vector128.LoadUnsafe(ref yRef));
2346Debug.Assert(Vector128.IsHardwareAccelerated);
2348Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2349Vector128.LoadUnsafe(ref yRef));
2350Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2351Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
2353end = Vector128.ConditionalSelect(CreateRemainderMaskVector128<T>((int)(remainder % (uint)Vector128<T>.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue));
2361Debug.Assert(Vector128.IsHardwareAccelerated);
2363Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2364Vector128.LoadUnsafe(ref yRef));
2395return Vector128.LoadUnsafe(
2402return Vector128.LoadUnsafe(
2409return Vector128.LoadUnsafe(
2416return Vector128.LoadUnsafe(
2503return Vector128.LoadUnsafe(
2510return Vector128.LoadUnsafe(
2517return Vector128.LoadUnsafe(
2524return Vector128.LoadUnsafe(
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IBinaryOperator.cs (158)
121if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TBinaryOperator.Vectorizable)
160Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
161Vector128.LoadUnsafe(ref yRef));
162Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
163Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
217vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
218Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)));
219vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
220Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)));
221vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
222Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)));
223vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
224Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)));
233vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
234Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)));
235vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
236Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)));
237vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
238Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)));
239vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
240Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)));
263vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
264Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)));
265vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
266Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)));
267vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
268Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)));
269vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
270Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)));
279vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
280Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)));
281vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
282Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)));
283vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
284Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)));
285vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
286Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)));
326Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8)),
327Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 8)));
334Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)),
335Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 7)));
342Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)),
343Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 6)));
350Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)),
351Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 5)));
358Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)),
359Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 4)));
366Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)),
367Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 3)));
374Vector128<T> vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)),
375Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 2)));
984Debug.Assert(Vector128.IsHardwareAccelerated);
986Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
987Vector128.LoadUnsafe(ref yRef));
988Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
989Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
1000Debug.Assert(Vector128.IsHardwareAccelerated);
1002Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1003Vector128.LoadUnsafe(ref yRef));
1148Debug.Assert(Vector128.IsHardwareAccelerated);
1150Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1151Vector128.LoadUnsafe(ref yRef));
1152Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1153Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
1164Debug.Assert(Vector128.IsHardwareAccelerated);
1166Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1167Vector128.LoadUnsafe(ref yRef));
1257Debug.Assert(Vector128.IsHardwareAccelerated);
1259Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1260Vector128.LoadUnsafe(ref yRef));
1261Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1262Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
1272Debug.Assert(Vector128.IsHardwareAccelerated);
1274Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1275Vector128.LoadUnsafe(ref yRef));
1345Debug.Assert(Vector128.IsHardwareAccelerated);
1347Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1348Vector128.LoadUnsafe(ref yRef));
1349Vector128<T> end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1350Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count));
1360Debug.Assert(Vector128.IsHardwareAccelerated);
1362Vector128<T> beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1363Vector128.LoadUnsafe(ref yRef));
1453if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TTransformOperator.Vectorizable && TBinaryOperator.Vectorizable)
1492Vector128<T> yVec = Vector128.Create(y);
1494Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
1496Vector128<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count)),
1548vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0))),
1550vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1))),
1552vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2))),
1554vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3))),
1564vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4))),
1566vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5))),
1568vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6))),
1570vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7))),
1593vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0))),
1595vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1))),
1597vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2))),
1599vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3))),
1609vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4))),
1611vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5))),
1613vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6))),
1615vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7))),
1654Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8))),
1662Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7))),
1670Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6))),
1678Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5))),
1686Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4))),
1694Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3))),
1702Vector128<T> vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2))),
2306Debug.Assert(Vector128.IsHardwareAccelerated);
2308Vector128<T> yVec = Vector128.Create(y);
2310Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2312Vector128<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count)),
2324Debug.Assert(Vector128.IsHardwareAccelerated);
2326Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2327Vector128.Create(y));
2474Debug.Assert(Vector128.IsHardwareAccelerated);
2476Vector128<T> yVec = Vector128.Create(y);
2478Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2480Vector128<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count)),
2492Debug.Assert(Vector128.IsHardwareAccelerated);
2494Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2495Vector128.Create(y));
2587Debug.Assert(Vector128.IsHardwareAccelerated);
2589Vector128<T> yVec = Vector128.Create(y);
2591Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2593Vector128<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count)),
2604Debug.Assert(Vector128.IsHardwareAccelerated);
2606Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2607Vector128.Create(y));
2679Debug.Assert(Vector128.IsHardwareAccelerated);
2681Vector128<T> yVec = Vector128.Create(y);
2683Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2685Vector128<T> end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count)),
2696Debug.Assert(Vector128.IsHardwareAccelerated);
2698Vector128<T> beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)),
2699Vector128.Create(y));
2743x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As<byte, T>());
2744x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>());
2745x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>());
2746x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>());
2750x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As<short, T>());
2751x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As<short, T>());
2752x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As<short, T>());
2756x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt32(), Vector128.Create(2, 3, 0, 1)).As<int, T>());
2757x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt32(), Vector128.Create(1, 0, 3, 2)).As<int, T>());
2762x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt64(), Vector128.Create(1, 0)).As<long, T>());
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IBooleanUnaryOperator.cs (37)
41public static bool ShouldEarlyExit(Vector128<T> result) => Vector128.AnyWhereAllBitsSet(result);
69typeof(T) == typeof(float) ? Vector128.EqualsAny(result.AsUInt32(), Vector128<uint>.Zero) :
70typeof(T) == typeof(double) ? Vector128.EqualsAny(result.AsUInt64(), Vector128<ulong>.Zero) :
71Vector128.EqualsAny(result, Vector128<T>.Zero);
157if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128<T>.IsSupported)
165if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i))))
176TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128<T>.Count)))))
297if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128<T>.IsSupported)
321Vector128<byte> v = TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i)).AsByte();
411if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128<T>.IsSupported)
436Vector128.Narrow(
437TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i)).AsUInt16(),
438TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128<T>.Count))).AsUInt16());
536if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128<T>.IsSupported)
561Vector128.Narrow(
562Vector128.Narrow(
563TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i)).AsUInt32(),
564TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128<T>.Count))).AsUInt32()),
565Vector128.Narrow(
566TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (2 * Vector128<T>.Count)))).AsUInt32(),
567TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (3 * Vector128<T>.Count)))).AsUInt32()));
683if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128<T>.IsSupported)
710Vector128.Narrow(
711Vector128.Narrow(
712Vector128.Narrow(
713TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i)).AsUInt64(),
714TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128<T>.Count))).AsUInt64()),
715Vector128.Narrow(
716TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (2 * Vector128<T>.Count)))).AsUInt64(),
717TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (3 * Vector128<T>.Count)))).AsUInt64())),
718Vector128.Narrow(
719Vector128.Narrow(
720TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (4 * Vector128<T>.Count)))).AsUInt64(),
721TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (5 * Vector128<T>.Count)))).AsUInt64()),
722Vector128.Narrow(
723TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (6 * Vector128<T>.Count)))).AsUInt64(),
724TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(i + (7 * Vector128<T>.Count)))).AsUInt64())));
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IIndexOfOperator.cs (44)
30tmpResult = Vector128.Shuffle(result.AsInt64(), Vector128.Create(1, 0)).As<long, T>();
31tmpIndex = Vector128.Shuffle(resultIndex.AsInt64(), Vector128.Create(1, 0)).As<long, T>();
41tmpResult = Vector128.Shuffle(result.AsInt32(), Vector128.Create(2, 3, 0, 1)).As<int, T>();
42tmpIndex = Vector128.Shuffle(resultIndex.AsInt32(), Vector128.Create(2, 3, 0, 1)).As<int, T>();
46tmpResult = Vector128.Shuffle(result.AsInt32(), Vector128.Create(1, 0, 3, 2)).As<int, T>();
47tmpIndex = Vector128.Shuffle(resultIndex.AsInt32(), Vector128.Create(1, 0, 3, 2)).As<int, T>();
57tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As<short, T>();
58tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As<short, T>();
62tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As<short, T>();
63tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As<short, T>();
67tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As<short, T>();
68tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As<short, T>();
78tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As<byte, T>();
79tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As<byte, T>();
83tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
84tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
88tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
89tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
93tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
94tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As<byte, T>();
127sizeof(T) == sizeof(long) ? Vector128.LessThan(indices1.AsInt64(), indices2.AsInt64()).As<long, T>() :
128sizeof(T) == sizeof(int) ? Vector128.LessThan(indices1.AsInt32(), indices2.AsInt32()).As<int, T>() :
129sizeof(T) == sizeof(short) ? Vector128.LessThan(indices1.AsInt16(), indices2.AsInt16()).As<short, T>() :
130Vector128.LessThan(indices1.AsByte(), indices2.AsByte()).As<byte, T>();
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IStatefulUnaryOperator.cs (46)
87if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TStatefulUnaryOperator.Vectorizable)
125Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
126Vector128<T> end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
177vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)));
178vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)));
179vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)));
180vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)));
189vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)));
190vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)));
191vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)));
192vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)));
214vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)));
215vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)));
216vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)));
217vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)));
226vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)));
227vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)));
228vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)));
229vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)));
267Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8)));
274Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)));
281Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)));
288Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)));
295Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)));
302Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)));
309Vector128<T> vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)));
853Debug.Assert(Vector128.IsHardwareAccelerated);
855Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
856Vector128<T> end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
867Debug.Assert(Vector128.IsHardwareAccelerated);
869Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
997Debug.Assert(Vector128.IsHardwareAccelerated);
999Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
1000Vector128<T> end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
1011Debug.Assert(Vector128.IsHardwareAccelerated);
1013Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
1094Debug.Assert(Vector128.IsHardwareAccelerated);
1096Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
1097Vector128<T> end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
1107Debug.Assert(Vector128.IsHardwareAccelerated);
1109Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
1174Debug.Assert(Vector128.IsHardwareAccelerated);
1176Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
1177Vector128<T> end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count));
1187Debug.Assert(Vector128.IsHardwareAccelerated);
1189Vector128<T> beg = op.Invoke(Vector128.LoadUnsafe(ref xRef));
System\Numerics\Tensors\netcore\Common\TensorPrimitives.ITernaryOperator.cs (276)
108if (TTernaryOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported)
148Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
149Vector128.LoadUnsafe(ref yRef),
150Vector128.LoadUnsafe(ref zRef));
151Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
152Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
153Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128<T>.Count));
210vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
211Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)),
212Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 0)));
213vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
214Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)),
215Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 1)));
216vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
217Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)),
218Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 2)));
219vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
220Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)),
221Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 3)));
230vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
231Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)),
232Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 4)));
233vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
234Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)),
235Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 5)));
236vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
237Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)),
238Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 6)));
239vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
240Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)),
241Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 7)));
265vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
266Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)),
267Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 0)));
268vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
269Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)),
270Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 1)));
271vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
272Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)),
273Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 2)));
274vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
275Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)),
276Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 3)));
285vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
286Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)),
287Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 4)));
288vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
289Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)),
290Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 5)));
291vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
292Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)),
293Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 6)));
294vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
295Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)),
296Vector128.Load(zPtr + (uint)(Vector128<T>.Count * 7)));
338Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8)),
339Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 8)),
340Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 8)));
347Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)),
348Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 7)),
349Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 7)));
356Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)),
357Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 6)),
358Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 6)));
365Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)),
366Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 5)),
367Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 5)));
374Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)),
375Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 4)),
376Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 4)));
383Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)),
384Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 3)),
385Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 3)));
392Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)),
393Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 2)),
394Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128<T>.Count * 2)));
1068Debug.Assert(Vector128.IsHardwareAccelerated);
1070Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1071Vector128.LoadUnsafe(ref yRef),
1072Vector128.LoadUnsafe(ref zRef));
1073Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1074Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
1075Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128<T>.Count));
1086Debug.Assert(Vector128.IsHardwareAccelerated);
1088Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1089Vector128.LoadUnsafe(ref yRef),
1090Vector128.LoadUnsafe(ref zRef));
1252Debug.Assert(Vector128.IsHardwareAccelerated);
1254Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1255Vector128.LoadUnsafe(ref yRef),
1256Vector128.LoadUnsafe(ref zRef));
1257Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1258Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
1259Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128<T>.Count));
1270Debug.Assert(Vector128.IsHardwareAccelerated);
1272Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1273Vector128.LoadUnsafe(ref yRef),
1274Vector128.LoadUnsafe(ref zRef));
1373Debug.Assert(Vector128.IsHardwareAccelerated);
1375Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1376Vector128.LoadUnsafe(ref yRef),
1377Vector128.LoadUnsafe(ref zRef));
1378Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1379Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
1380Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128<T>.Count));
1390Debug.Assert(Vector128.IsHardwareAccelerated);
1392Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1393Vector128.LoadUnsafe(ref yRef),
1394Vector128.LoadUnsafe(ref zRef));
1469Debug.Assert(Vector128.IsHardwareAccelerated);
1471Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1472Vector128.LoadUnsafe(ref yRef),
1473Vector128.LoadUnsafe(ref zRef));
1474Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1475Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
1476Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128<T>.Count));
1486Debug.Assert(Vector128.IsHardwareAccelerated);
1488Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1489Vector128.LoadUnsafe(ref yRef),
1490Vector128.LoadUnsafe(ref zRef));
1583if (TTernaryOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported)
1623Vector128<T> zVec = Vector128.Create(z);
1625Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
1626Vector128.LoadUnsafe(ref yRef),
1628Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
1629Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
1684vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
1685Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)),
1687vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
1688Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)),
1690vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
1691Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)),
1693vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
1694Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)),
1704vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
1705Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)),
1707vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
1708Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)),
1710vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
1711Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)),
1713vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
1714Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)),
1738vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
1739Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 0)),
1741vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
1742Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 1)),
1744vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
1745Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 2)),
1747vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
1748Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 3)),
1758vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
1759Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 4)),
1761vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
1762Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 5)),
1764vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
1765Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 6)),
1767vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
1768Vector128.Load(yPtr + (uint)(Vector128<T>.Count * 7)),
1809Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8)),
1810Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 8)),
1818Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)),
1819Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 7)),
1827Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)),
1828Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 6)),
1836Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)),
1837Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 5)),
1845Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)),
1846Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 4)),
1854Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)),
1855Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 3)),
1863Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)),
1864Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128<T>.Count * 2)),
2533Debug.Assert(Vector128.IsHardwareAccelerated);
2535Vector128<T> zVec = Vector128.Create(z);
2537Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2538Vector128.LoadUnsafe(ref yRef),
2540Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2541Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
2553Debug.Assert(Vector128.IsHardwareAccelerated);
2555Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2556Vector128.LoadUnsafe(ref yRef),
2557Vector128.Create(z));
2721Debug.Assert(Vector128.IsHardwareAccelerated);
2723Vector128<T> zVec = Vector128.Create(z);
2725Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2726Vector128.LoadUnsafe(ref yRef),
2728Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2729Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
2741Debug.Assert(Vector128.IsHardwareAccelerated);
2743Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2744Vector128.LoadUnsafe(ref yRef),
2745Vector128.Create(z));
2846Debug.Assert(Vector128.IsHardwareAccelerated);
2848Vector128<T> zVec = Vector128.Create(z);
2850Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2851Vector128.LoadUnsafe(ref yRef),
2853Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2854Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
2865Debug.Assert(Vector128.IsHardwareAccelerated);
2867Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2868Vector128.LoadUnsafe(ref yRef),
2869Vector128.Create(z));
2946Debug.Assert(Vector128.IsHardwareAccelerated);
2948Vector128<T> zVec = Vector128.Create(z);
2950Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2951Vector128.LoadUnsafe(ref yRef),
2953Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
2954Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128<T>.Count),
2965Debug.Assert(Vector128.IsHardwareAccelerated);
2967Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
2968Vector128.LoadUnsafe(ref yRef),
2969Vector128.Create(z));
3069if (TTernaryOperator.Vectorizable && Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported)
3109Vector128<T> yVec = Vector128.Create(y);
3110Vector128<T> zVec = Vector128.Create(z);
3112Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
3115Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
3168vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
3171vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
3174vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
3177vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
3188vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
3191vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
3194vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
3197vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
3221vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 0)),
3224vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 1)),
3227vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 2)),
3230vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 3)),
3241vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 4)),
3244vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 5)),
3247vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 6)),
3250vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<T>.Count * 7)),
3290Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 8)),
3299Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 7)),
3308Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 6)),
3317Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 5)),
3326Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 4)),
3335Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 3)),
3344Vector128<T> vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<T>.Count * 2)),
4005Debug.Assert(Vector128.IsHardwareAccelerated);
4007Vector128<T> yVec = Vector128.Create(y);
4008Vector128<T> zVec = Vector128.Create(z);
4010Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4013Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
4026Debug.Assert(Vector128.IsHardwareAccelerated);
4028Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4029Vector128.Create(y),
4030Vector128.Create(z));
4195Debug.Assert(Vector128.IsHardwareAccelerated);
4197Vector128<T> yVec = Vector128.Create(y);
4198Vector128<T> zVec = Vector128.Create(z);
4200Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4203Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
4216Debug.Assert(Vector128.IsHardwareAccelerated);
4218Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4219Vector128.Create(y),
4220Vector128.Create(z));
4322Debug.Assert(Vector128.IsHardwareAccelerated);
4324Vector128<T> yVec = Vector128.Create(y);
4325Vector128<T> zVec = Vector128.Create(z);
4327Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4330Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
4342Debug.Assert(Vector128.IsHardwareAccelerated);
4344Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4345Vector128.Create(y),
4346Vector128.Create(z));
4424Debug.Assert(Vector128.IsHardwareAccelerated);
4426Vector128<T> yVec = Vector128.Create(y);
4427Vector128<T> zVec = Vector128.Create(z);
4429Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4432Vector128<T> end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<T>.Count),
4444Debug.Assert(Vector128.IsHardwareAccelerated);
4446Vector128<T> beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef),
4447Vector128.Create(y),
4448Vector128.Create(z));
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IUnaryInputBinaryOutput.cs (30)
131if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TUnaryOperator.Vectorizable)
139(Vector128<T> first, Vector128<T> second) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i));
152(Vector128<T> first, Vector128<T> second) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i));
302if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TOperator.Vectorizable)
310(Vector128<T> first, Vector128<T> second) = TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i), Vector128.LoadUnsafe(ref yRef, (uint)i));
321Vector128<T> mask = Vector128.Equals(CreateRemainderMaskVector128<T>(x.Length - i), Vector128<T>.Zero);
325Vector128<T> first = Vector128.ConditionalSelect(mask,
326Vector128.Create(TOperator.RemainderMaskValue),
327Vector128.LoadUnsafe(ref xRef, (uint)i));
329Vector128<T> second = Vector128.ConditionalSelect(mask,
330Vector128.Create(TOperator.RemainderMaskValue),
331Vector128.LoadUnsafe(ref yRef, (uint)i));
335Vector128.ConditionalSelect(mask,
336Vector128.LoadUnsafe(ref destination1Ref, (uint)i),
340Vector128.ConditionalSelect(mask,
341Vector128.LoadUnsafe(ref destination2Ref, (uint)i),
487if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && TOperator.Vectorizable)
492Vector128<T> yVec = Vector128.Create(y);
497(Vector128<T> first, Vector128<T> second) = TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i), yVec);
508Vector128<T> mask = Vector128.Equals(CreateRemainderMaskVector128<T>(x.Length - i), Vector128<T>.Zero);
512Vector128<T> first = Vector128.ConditionalSelect(mask,
513Vector128.Create(TOperator.RemainderMaskValue),
514Vector128.LoadUnsafe(ref xRef, (uint)i));
516Vector128<T> second = Vector128.ConditionalSelect(mask,
517Vector128.Create(TOperator.RemainderMaskValue),
522Vector128.ConditionalSelect(mask,
523Vector128.LoadUnsafe(ref destination1Ref, (uint)i),
527Vector128.ConditionalSelect(mask,
528Vector128.LoadUnsafe(ref destination2Ref, (uint)i),
System\Numerics\Tensors\netcore\Common\TensorPrimitives.IUnaryOperator.cs (46)
127if (Vector128.IsHardwareAccelerated && Vector128<TInput>.IsSupported && Vector128<TOutput>.IsSupported && TUnaryOperator.Vectorizable && sizeof(TInput) == sizeof(TOutput))
165Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
166Vector128<TOutput> end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<TInput>.Count));
217vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 0)));
218vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 1)));
219vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 2)));
220vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 3)));
229vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 4)));
230vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 5)));
231vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 6)));
232vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 7)));
254vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 0)));
255vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 1)));
256vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 2)));
257vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 3)));
266vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 4)));
267vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 5)));
268vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 6)));
269vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128<TInput>.Count * 7)));
307Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 8)));
314Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 7)));
321Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 6)));
328Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 5)));
335Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 4)));
342Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 3)));
349Vector128<TOutput> vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128<TInput>.Count * 2)));
893Debug.Assert(Vector128.IsHardwareAccelerated);
895Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
896Vector128<TOutput> end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<TInput>.Count));
907Debug.Assert(Vector128.IsHardwareAccelerated);
909Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1037Debug.Assert(Vector128.IsHardwareAccelerated);
1039Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1040Vector128<TOutput> end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<TInput>.Count));
1051Debug.Assert(Vector128.IsHardwareAccelerated);
1053Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1134Debug.Assert(Vector128.IsHardwareAccelerated);
1136Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1137Vector128<TOutput> end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<TInput>.Count));
1147Debug.Assert(Vector128.IsHardwareAccelerated);
1149Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1214Debug.Assert(Vector128.IsHardwareAccelerated);
1216Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
1217Vector128<TOutput> end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128<TInput>.Count));
1227Debug.Assert(Vector128.IsHardwareAccelerated);
1229Vector128<TOutput> beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef));
System\Numerics\Tensors\netcore\TensorPrimitives.Clamp.cs (3)
225public static Vector128<T> Invoke(Vector128<T> x, Vector128<T> min, Vector128<T> max) => Vector128.Clamp(x, min, max);
273public static Vector128<T> Invoke(Vector128<T> min, Vector128<T> max, Vector128<T> x) => Vector128.Clamp(x, min, max);
322public static Vector128<T> Invoke(Vector128<T> max, Vector128<T> x, Vector128<T> min) => Vector128.Clamp(x, min, max);
System\Numerics\Tensors\netcore\TensorPrimitives.ConvertHelpers.cs (46)
170public static Vector128<float> Invoke(Vector128<int> x) => Vector128.ConvertToSingle(x);
181public static Vector128<float> Invoke(Vector128<uint> x) => Vector128.ConvertToSingle(x);
192public static Vector128<double> Invoke(Vector128<ulong> x) => Vector128.ConvertToDouble(x);
203public static Vector128<double> Invoke(Vector128<long> x) => Vector128.ConvertToDouble(x);
214public static (Vector128<double> Lower, Vector128<double> Upper) Invoke(Vector128<float> x) => Vector128.Widen(x);
225public static Vector128<float> Invoke(Vector128<double> lower, Vector128<double> upper) => Vector128.Narrow(lower, upper);
236public static (Vector128<ushort> Lower, Vector128<ushort> Upper) Invoke(Vector128<byte> x) => Vector128.Widen(x);
251(Vector128<ushort> Lower, Vector128<ushort> Upper) ushorts = Vector128.Widen(x);
252(Vector128<uint> Lower, Vector128<uint> Upper) uintsLower = Vector128.Widen(ushorts.Lower);
253(Vector128<uint> Lower, Vector128<uint> Upper) uintsUpper = Vector128.Widen(ushorts.Upper);
288Vector128.ConvertToSingle(results.Item1),
289Vector128.ConvertToSingle(results.Item2),
290Vector128.ConvertToSingle(results.Item3),
291Vector128.ConvertToSingle(results.Item4)
326public static (Vector128<short> Lower, Vector128<short> Upper) Invoke(Vector128<sbyte> x) => Vector128.Widen(x);
337public static (Vector128<uint> Lower, Vector128<uint> Upper) Invoke(Vector128<ushort> x) => Vector128.Widen(x);
348public static (Vector128<int> Lower, Vector128<int> Upper) Invoke(Vector128<short> x) => Vector128.Widen(x);
359public static (Vector128<ulong> Lower, Vector128<ulong> Upper) Invoke(Vector128<uint> x) => Vector128.Widen(x);
370public static (Vector128<long> Lower, Vector128<long> Upper) Invoke(Vector128<int> x) => Vector128.Widen(x);
396(Vector128<int> lowerInt32, Vector128<int> upperInt32) = Vector128.Widen(x);
404Vector128<uint> sign = value & Vector128.Create(SingleSignMask);
410Vector128<uint> offsetExponent = bitValueInProcess & Vector128.Create(HalfExponentMask);
413Vector128<uint> subnormalMask = Vector128.Equals(offsetExponent, Vector128<uint>.Zero);
416Vector128<uint> infinityOrNaNMask = Vector128.Equals(offsetExponent, Vector128.Create(HalfExponentMask));
419Vector128<uint> maskedExponentLowerBound = subnormalMask & Vector128.Create(ExponentLowerBound);
422Vector128<uint> offsetMaskedExponentLowerBound = Vector128.Create(ExponentOffset) | maskedExponentLowerBound;
425bitValueInProcess = Vector128.ShiftLeft(bitValueInProcess, 13);
428offsetMaskedExponentLowerBound = Vector128.ConditionalSelect(Vector128.Equals(infinityOrNaNMask, Vector128<uint>.Zero),
430Vector128.ShiftLeft(offsetMaskedExponentLowerBound, 1));
433bitValueInProcess &= Vector128.Create(HalfToSingleBitsMask);
577return Vector128.Narrow(
586Vector128<uint> sign = Vector128.ShiftRightLogical(bitValue & Vector128.Create(SingleSignMask), 16);
589Vector128<uint> realMask = Vector128.Equals(value, value).AsUInt32();
592value = Vector128.Abs(value);
595value = Vector128.Min(Vector128.Create(MaxHalfValueBelowInfinity), value);
598Vector128<uint> exponentOffset0 = Vector128.Max(value, Vector128.Create(MinExp).AsSingle()).AsUInt32();
601exponentOffset0 &= Vector128.Create(SingleBiasedExponentMask);
604exponentOffset0 += Vector128.Create(Exponent13);
611Vector128<uint> maskedHalfExponentForNaN = ~realMask & Vector128.Create(ExponentMask);
614bitValue -= Vector128.Create(Exponent126);
617Vector128<uint> newExponent = Vector128.ShiftRightLogical(bitValue, 13);
System\Numerics\Tensors\netcore\TensorPrimitives.ConvertTruncating.cs (10)
114public static Vector128<int> Invoke(Vector128<float> x) => Vector128.ConvertToInt32(x);
130public static Vector128<uint> Invoke(Vector128<float> x) => Vector128.ConvertToUInt32(x);
146public static Vector128<ulong> Invoke(Vector128<double> x) => Vector128.ConvertToUInt64(x);
162public static Vector128<long> Invoke(Vector128<double> x) => Vector128.ConvertToInt64(x);
173public static Vector128<byte> Invoke(Vector128<ushort> lower, Vector128<ushort> upper) => Vector128.Narrow(lower, upper);
184public static Vector128<sbyte> Invoke(Vector128<short> lower, Vector128<short> upper) => Vector128.Narrow(lower, upper);
195public static Vector128<ushort> Invoke(Vector128<uint> lower, Vector128<uint> upper) => Vector128.Narrow(lower, upper);
206public static Vector128<short> Invoke(Vector128<int> lower, Vector128<int> upper) => Vector128.Narrow(lower, upper);
217public static Vector128<uint> Invoke(Vector128<ulong> lower, Vector128<ulong> upper) => Vector128.Narrow(lower, upper);
228public static Vector128<int> Invoke(Vector128<long> lower, Vector128<long> upper) => Vector128.Narrow(lower, upper);
System\Numerics\Tensors\netcore\TensorPrimitives.CosineSimilarity.cs (13)
135if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && x.Length >= Vector128<T>.Count)
150Vector128.LoadUnsafe(ref xRef, (uint)i),
151Vector128.LoadUnsafe(ref yRef, (uint)i),
164Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128<T>.Count)) & remainderMask,
165Vector128.LoadUnsafe(ref yRef, (uint)(x.Length - Vector128<T>.Count)) & remainderMask,
285if (Vector128.IsHardwareAccelerated && x.Length >= Vector128<short>.Count)
300(Vector128<float> xVecLower, Vector128<float> xVecUpper) = WidenHalfAsInt16ToSingleOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i));
301(Vector128<float> yVecLower, Vector128<float> yVecUpper) = WidenHalfAsInt16ToSingleOperator.Invoke(Vector128.LoadUnsafe(ref yRef, (uint)i));
316Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128<short>.Count)) & remainderMask);
318Vector128.LoadUnsafe(ref yRef, (uint)(x.Length - Vector128<short>.Count)) & remainderMask);
376Vector128.Sum(dotProductVector) / (T.Sqrt(Vector128.Sum(xSumOfSquaresVector)) * T.Sqrt(Vector128.Sum(ySumOfSquaresVector)));
System\Numerics\Tensors\netcore\TensorPrimitives.FloatHelpers.cs (6)
11Vector128.Create(TOperator.Invoke(floats[0]), TOperator.Invoke(floats[1]), TOperator.Invoke(floats[2]), TOperator.Invoke(floats[3]));
20Vector128.Create(TOperator.Invoke(doubles[0]), TOperator.Invoke(doubles[1]));
37Vector128.Create(firstRes0, firstRes1, firstRes2, firstRes3),
38Vector128.Create(secondRes0, secondRes1, secondRes2, secondRes3)
73Vector128.Create(firstRes0, firstRes1),
74Vector128.Create(secondRes0, secondRes1)
System\Numerics\Tensors\netcore\TensorPrimitives.IndexOfMax.cs (15)
37Vector128<T> useResult = Vector128.GreaterThan(result, current);
38Vector128<T> equalMask = Vector128.Equals(result, current);
47Vector128<T> sameSign = Vector128.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As<int, T>();
330if (Vector128.IsHardwareAccelerated && Vector128<T>.IsSupported && x.Length >= Vector128<T>.Count)
336sizeof(T) == sizeof(long) ? Vector128.Create((long)i).As<long, T>() :
337sizeof(T) == sizeof(int) ? Vector128.Create(i).As<int, T>() :
338sizeof(T) == sizeof(short) ? Vector128.Create((short)i).As<short, T>() :
339Vector128.Create((byte)i).As<byte, T>();
359Vector128<T> result = Vector128.LoadUnsafe(ref xRef);
365nanMask = ~Vector128.Equals(result, result);
379current = Vector128.LoadUnsafe(ref xRef, (uint)i);
384nanMask = ~Vector128.Equals(current, current);
399current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128<T>.Count));
404nanMask = ~Vector128.Equals(current, current);
483return Vector128.ConditionalSelect(mask, left, right);
System\Numerics\Tensors\netcore\TensorPrimitives.LeadingZeroCount.cs (5)
43Vector128<byte> lookupVectorLow = Vector128.Create((byte)8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4);
44Vector128<byte> lookupVectorHigh = Vector128.Create((byte)3, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0);
45Vector128<byte> nibbleMask = Vector128.Create<byte>(0xF);
46Vector128<byte> permuteMask = Vector128.Create<byte>(0x80);
60Vector128<uint> lowHalf = Vector128.Create((uint)0x0000FFFF);
System\Numerics\Tensors\netcore\TensorPrimitives.Tan.cs (47)
130Vector128<float> uxMasked = Vector128.Abs(x);
131if (Vector128.GreaterThanAny(uxMasked.AsUInt32(), Vector128.Create(MaxVectorizedValue)))
136Vector128<float> dn = MultiplyAddEstimateOperator<float>.Invoke(uxMasked, Vector128.Create(2 / float.Pi), Vector128.Create(AlmHuge));
138dn -= Vector128.Create(AlmHuge);
141f = MultiplyAddEstimateOperator<float>.Invoke(dn, Vector128.Create(-float.Pi / 2), f);
142f = MultiplyAddEstimateOperator<float>.Invoke(dn, Vector128.Create(Pi_Tail2), f);
143f = MultiplyAddEstimateOperator<float>.Invoke(dn, Vector128.Create(Pi_Tail3), f);
150Vector128<float> a1 = MultiplyAddEstimateOperator<float>.Invoke(Vector128.Create(C2), f2, Vector128.Create(C1));
151Vector128<float> a2 = MultiplyAddEstimateOperator<float>.Invoke(Vector128.Create(C4), f2, Vector128.Create(C3));
152Vector128<float> a3 = MultiplyAddEstimateOperator<float>.Invoke(Vector128.Create(C6), f2, Vector128.Create(C5));
154Vector128<float> b2 = MultiplyAddEstimateOperator<float>.Invoke(f8, a3, f12 * Vector128.Create(C7));
157Vector128<float> result = (poly.AsUInt32() ^ (x.AsUInt32() & Vector128.Create(~SignMask))).AsSingle();
158return Vector128.ConditionalSelect(Vector128.Equals(odd, Vector128<uint>.Zero).AsSingle(),
160Vector128.Create(-1f) / result);
263Vector128<double> uxMasked = Vector128.Abs(x);
264if (Vector128.GreaterThanAny(uxMasked.AsUInt64(), Vector128.Create(MaxVectorizedValue)))
270Vector128<double> dn = MultiplyAddEstimateOperator<double>.Invoke(uxMasked, Vector128.Create(2 / double.Pi), Vector128.Create(AlmHuge));
272dn -= Vector128.Create(AlmHuge);
276f = MultiplyAddEstimateOperator<double>.Invoke(dn, Vector128.Create(-double.Pi / 2), f);
277f = MultiplyAddEstimateOperator<double>.Invoke(dn, Vector128.Create(-HalfPi2), f);
278f = MultiplyAddEstimateOperator<double>.Invoke(dn, Vector128.Create(-HalfPi3), f);
289Vector128<double> a1 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C3), g, Vector128.Create(C1));
290Vector128<double> a2 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C7), g, Vector128.Create(C5));
291Vector128<double> a3 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C11), g, Vector128.Create(C9));
292Vector128<double> a4 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C15), g, Vector128.Create(C13));
293Vector128<double> a5 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C19), g, Vector128.Create(C17));
294Vector128<double> a6 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C23), g, Vector128.Create(C21));
295Vector128<double> a7 = MultiplyAddEstimateOperator<double>.Invoke(Vector128.Create(C27), g, Vector128.Create(C25));
302Vector128<double> result = (poly.AsUInt64() ^ (x.AsUInt64() & Vector128.Create(~SignMask))).AsDouble();
303return Vector128.ConditionalSelect(Vector128.Equals(odd, Vector128<ulong>.Zero).AsDouble(),
305Vector128.Create(-1.0) / result);